It looks like this is normal opengl behavior and not configurable. Since I'm technically mixing Direct3D and OpenGL, full screen prevents the child VLC window from rendering. Interesting that it's limited to the primary display. Driver setting maybe? Not really a graphics guy.
Easy enough to work around by using a dedicated full screen video window or by offsetting 1pt.
Toggle full screen example for wxFrame:
void MainFrame::AppToggleFullScreen()
{
#ifdef _WIN32
if (this->GetAppFullScreen())
{
SetWindowLong(this->GetHandle(), GWL_STYLE, this->GetNormalStyle());
if (!(this->GetNormalStyle() & WS_MAXIMIZE))
{
this->SetSize(this->GetNormalRect());
}
this->SetAppFullScreen(false);
}
else
{
this->SetNormalRect(this->GetRect());
this->SetNormalStyle(GetWindowLong(this->GetHandle(), GWL_STYLE));
SetWindowLong(this->GetHandle(), GWL_STYLE, WS_POPUP | WS_VISIBLE);
wxDisplay d(wxDisplay::GetFromWindow(this));
wxRect monitorRect = d.GetGeometry();
int x = monitorRect.x;
int y = monitorRect.y - 1;
int width = monitorRect.width;
int height = monitorRect.height + 2;
MoveWindow(this->GetHandle(), x, y, width, height, FALSE);
this->SetAppFullScreen(true);
}
#endif
}