Qt: Don't create wasteful display container on KDE

Still needed on GNOME because it's fucking stupid and they refuse to
support server-side decorations, with no signs this will ever change.

src/duckstation-qt/displaywidget.cpp

@@ -413,19 +413,6 @@ DisplayContainer::DisplayContainer() : QStackedWidget(nullptr)
 
 DisplayContainer::~DisplayContainer() = default;
 
-bool DisplayContainer::isNeeded(bool fullscreen, bool render_to_main)
-{
-#if defined(_WIN32) || defined(__APPLE__)
-  return false;
-#else
-  if (!QtHost::IsRunningOnWayland())
-    return false;
-
-  // We only need this on Wayland because of client-side decorations...
-  return (fullscreen || !render_to_main);
-#endif
-}
-
 void DisplayContainer::setDisplayWidget(DisplayWidget* widget)
 {
   Assert(!m_display_widget);
@@ -620,7 +607,7 @@ AuxiliaryDisplayWidget* AuxiliaryDisplayWidget::create(s32 pos_x, s32 pos_y, u32
                                                        const QString& title, const QString& icon_name, void* userdata)
 {
   QStackedWidget* parent = nullptr;
-  if (DisplayContainer::isNeeded(false, false))
+  if (QtHost::IsDisplayWidgetContainerNeeded())
   {
     parent = new QStackedWidget(nullptr);
     parent->resize(width, height);

src/duckstation-qt/displaywidget.h

@@ -78,8 +78,6 @@ public:
   DisplayContainer();
   ~DisplayContainer();
 
-  static bool isNeeded(bool fullscreen, bool render_to_main);
-
   void setDisplayWidget(DisplayWidget* widget);
   DisplayWidget* removeDisplayWidget();
 

src/duckstation-qt/mainwindow.cpp

@@ -292,11 +292,9 @@ std::optional<WindowInfo> MainWindow::acquireRenderWindow(RenderAPI render_api,
   m_exclusive_fullscreen_requested = exclusive_fullscreen;
 
   // Skip recreating the surface if we're just transitioning between fullscreen and windowed with render-to-main off.
-  // .. except on Wayland, where everything tends to break if you don't recreate.
-  // Container can also be null if we're messing with settings while surfaceless.
-  const bool has_container = (m_display_container != nullptr);
-  const bool needs_container = DisplayContainer::isNeeded(fullscreen, render_to_main);
-  if (container && !is_rendering_to_main && !render_to_main && !has_container && !needs_container)
+  // We also need to unparent the display widget from the container when switching to fullscreen on Wayland.
+  const bool needs_container = (QtHost::IsDisplayWidgetContainerNeeded() && !fullscreen && !is_rendering_to_main);
+  if (container && !is_rendering_to_main && !render_to_main && (m_display_container != nullptr) == needs_container)
   {
     DEV_LOG("Toggling to {} without recreating surface", (fullscreen ? "fullscreen" : "windowed"));
     m_exclusive_fullscreen_requested = exclusive_fullscreen;
@@ -364,7 +362,7 @@ void MainWindow::createDisplayWidget(bool fullscreen, bool render_to_main)
     setVisible(true);
 
   QWidget* container;
-  if (DisplayContainer::isNeeded(fullscreen, render_to_main))
+  if (!fullscreen && !render_to_main && QtHost::IsDisplayWidgetContainerNeeded())
   {
     m_display_container = new DisplayContainer();
     m_display_widget = new DisplayWidget(m_display_container);

src/duckstation-qt/qthost.cpp

@@ -137,6 +137,10 @@ static void PrintCommandLineVersion();
 static void PrintCommandLineHelp(const char* progname);
 static bool ParseCommandLineParametersAndInitializeConfig(QApplication& app,
                                                           std::shared_ptr<SystemBootParameters>& boot_params);
+
+#ifdef __linux__
+static void ApplyWaylandWorkarounds();
+#endif
 } // namespace QtHost
 
 namespace {
@@ -158,6 +162,10 @@ struct State
   bool start_fullscreen_ui_fullscreen = false;
   bool run_setup_wizard = false;
   bool cleanup_after_update = false;
+
+#ifdef __linux__
+  bool display_container_needed = false;
+#endif
 };
 } // namespace
 
@@ -233,12 +241,8 @@ bool QtHost::VeryEarlyProcessStartup()
 
 bool QtHost::EarlyProcessStartup()
 {
-#if !defined(_WIN32) && !defined(__APPLE__)
-  // On Wayland, turning any window into a native window causes DPI scaling to break, as well as window
-  // updates, creating a complete mess of a window. Setting this attribute isn't ideal, since you'd think
-  // that setting WA_DontCreateNativeAncestors on the widget would be sufficient, but apparently not.
-  if (QtHost::IsRunningOnWayland())
-    QGuiApplication::setAttribute(Qt::AA_DontCreateNativeWidgetSiblings, true);
+#ifdef __linux__
+  ApplyWaylandWorkarounds();
 #endif
 
   // redirect qt errors
@@ -305,16 +309,51 @@ bool QtHost::InNoGUIMode()
   return s_state.nogui_mode;
 }
 
-bool QtHost::IsRunningOnWayland()
+bool QtHost::IsDisplayWidgetContainerNeeded()
 {
-#if defined(_WIN32) || defined(__APPLE__)
-  return false;
+#ifdef __linux__
+  return s_state.display_container_needed;
 #else
-  const QString platform_name = QGuiApplication::platformName();
-  return (platform_name == QStringLiteral("wayland"));
+  return false;
 #endif
 }
 
+#ifdef __linux__
+
+void QtHost::ApplyWaylandWorkarounds()
+{
+  const QString platform_name = QGuiApplication::platformName();
+  if (platform_name != QStringLiteral("wayland"))
+  {
+    std::fputs("Wayland not detected, not applying workarounds.\n", stderr);
+    return;
+  }
+
+  std::fputs("Wayland platform detected, applying workarounds.\n", stderr);
+
+  // On Wayland, turning any window into a native window causes DPI scaling to break, as well as window
+  // updates, creating a complete mess of a window. Setting this attribute isn't ideal, since you'd think
+  // that setting WA_DontCreateNativeAncestors on the widget would be sufficient, but apparently not.
+  QGuiApplication::setAttribute(Qt::AA_DontCreateNativeWidgetSiblings, true);
+
+  // When rendering fullscreen or to a separate window on Wayland, because we take over the surface we need
+  // to wrap the widget in a container because GNOME is stupid and refuses to ever support server-side
+  // decorations. There's no sign of this ever changing. Fuck Wayland.
+  if (const char* desktop = std::getenv("XDG_SESSION_DESKTOP"); desktop && std::strcmp(desktop, "KDE") == 0)
+  {
+    std::fputs("Wayland with KDE detected, not creating display widget containers.\n", stderr);
+  }
+  else
+  {
+    std::fputs("Wayland with non-KDE desktop detected, creating display widget containers.\n"
+               "Don't complain when things break.\n",
+               stderr);
+    s_state.display_container_needed = true;
+  }
+}
+
+#endif
+
 QString QtHost::GetAppNameAndVersion()
 {
   return QStringLiteral("DuckStation %1").arg(QLatin1StringView(g_scm_version_str));

src/duckstation-qt/qthost.h

@@ -321,8 +321,8 @@ bool InBatchMode();
 /// Sets NoGUI mode (implys batch mode, does not display main window, exits on shutdown).
 bool InNoGUIMode();
 
-/// Returns true if the application is running under Wayland.
-bool IsRunningOnWayland();
+/// Returns true if display widgets need to wrapped in a container, thanks to Wayland stupidity.
+bool IsDisplayWidgetContainerNeeded();
 
 /// Returns true if rendering to the main window should be allowed.
 bool CanRenderToMainWindow();