EGL/Wayland (Wayland, pt 3)

.github/workflows/main.yml

@@ -419,6 +419,7 @@ jobs:
            pkg-config \
            python3-dev \
            qtbase5-dev \
+           qtbase5-private-dev \
            libqt5svg5-dev \
            swig \
            libcmocka-dev \

CI/install-dependencies-linux.sh

@@ -42,6 +42,7 @@ sudo apt-get install -y \
         pkg-config \
         python3-dev \
         qtbase5-dev \
+        qtbase5-private-dev \
         libqt5svg5-dev \
         swig \
         linux-generic \

CMakeLists.txt

@@ -202,6 +202,8 @@ if(APPLE)
 	list(APPEND CMAKE_INSTALL_RPATH "@loader_path/" "@executable_path/")
 elseif(UNIX)
 	option(USE_XDG "Utilize XDG Base Directory Specification" ON)
+	option(ENABLE_WAYLAND "Build support for Wayland" ON)
+
 	if(USE_XDG)
 		add_definitions(-DUSE_XDG)
 	endif()

UI/CMakeLists.txt

@@ -396,6 +396,16 @@ if(WIN32)
 			OUTPUT_NAME "obs${_output_suffix}")
 endif()
 
+if (ENABLE_WAYLAND)
+	find_package(Qt5Gui REQUIRED)
+	include_directories(${Qt5Gui_PRIVATE_INCLUDE_DIRS})
+
+	set(obs_PLATFORM_LIBRARIES
+		${obs_PLATFORM_LIBRARIES}
+		Qt5::Gui
+		Qt5::GuiPrivate)
+endif()
+
 target_link_libraries(obs
 	libobs
 	Qt5::Widgets

UI/obs-app.cpp

@@ -58,6 +58,16 @@
 #include <pthread.h>
 #endif
 
+#if !defined(_WIN32) && !defined(__APPLE__)
+#include <obs-nix-platform.h>
+#include <QX11Info>
+
+#ifdef ENABLE_WAYLAND
+#include <qpa/qplatformnativeinterface.h>
+#endif
+
+#endif
+
 #include <iostream>
 
 #include "ui-config.h"
@@ -1384,6 +1394,29 @@ bool OBSApp::OBSInit()
 
 	qRegisterMetaType<VoidFunc>();
 
+#if !defined(_WIN32) && !defined(__APPLE__)
+	obs_set_nix_platform(OBS_NIX_PLATFORM_X11_GLX);
+	if (QApplication::platformName() == "xcb") {
+		if (getenv("OBS_USE_EGL")) {
+			blog(LOG_INFO, "Using EGL/X11");
+			obs_set_nix_platform(OBS_NIX_PLATFORM_X11_EGL);
+		}
+		obs_set_nix_platform_display(QX11Info::display());
+	}
+
+#ifdef ENABLE_WAYLAND
+	if (QApplication::platformName().contains("wayland")) {
+		obs_set_nix_platform(OBS_NIX_PLATFORM_WAYLAND);
+		QPlatformNativeInterface *native =
+			QGuiApplication::platformNativeInterface();
+		obs_set_nix_platform_display(
+			native->nativeResourceForIntegration("display"));
+
+		blog(LOG_INFO, "Platform: Wayland");
+	}
+#endif
+#endif
+
 	if (!StartupOBS(locale.c_str(), GetProfilerNameStore()))
 		return false;
 

UI/qt-display.cpp

@@ -6,6 +6,62 @@
 #include <QResizeEvent>
 #include <QShowEvent>
 
+#include <obs-config.h>
+
+#ifdef ENABLE_WAYLAND
+#include <obs-nix-platform.h>
+
+class SurfaceEventFilter : public QObject {
+	OBSQTDisplay *display;
+	int mTimerId;
+
+public:
+	SurfaceEventFilter(OBSQTDisplay *src) : display(src), mTimerId(0) {}
+
+protected:
+	bool eventFilter(QObject *obj, QEvent *event) override
+	{
+		bool result = QObject::eventFilter(obj, event);
+		QPlatformSurfaceEvent *surfaceEvent;
+
+		switch (event->type()) {
+		case QEvent::PlatformSurface:
+			surfaceEvent =
+				static_cast<QPlatformSurfaceEvent *>(event);
+			if (surfaceEvent->surfaceEventType() !=
+			    QPlatformSurfaceEvent::SurfaceCreated)
+				return result;
+
+			if (display->windowHandle()->isExposed())
+				createOBSDisplay();
+			else
+				mTimerId = startTimer(67); // Arbitrary
+			break;
+		case QEvent::Expose:
+			createOBSDisplay();
+			break;
+		default:
+			break;
+		}
+
+		return result;
+	}
+
+	void timerEvent(QTimerEvent *) { createOBSDisplay(true); }
+
+private:
+	void createOBSDisplay(bool force = false)
+	{
+		display->CreateDisplay(force);
+		if (mTimerId > 0) {
+			killTimer(mTimerId);
+			mTimerId = 0;
+		}
+	}
+};
+
+#endif
+
 static inline long long color_to_int(const QColor &color)
 {
 	auto shift = [&](unsigned val, int shift) {
@@ -33,8 +89,13 @@ OBSQTDisplay::OBSQTDisplay(QWidget *parent, Qt::WindowFlags flags)
 	setAttribute(Qt::WA_NativeWindow);
 
 	auto windowVisible = [this](bool visible) {
-		if (!visible)
+		if (!visible) {
+#ifdef ENABLE_WAYLAND
+			if (obs_get_nix_platform() == OBS_NIX_PLATFORM_WAYLAND)
+				display = nullptr;
+#endif
 			return;
+		}
 
 		if (!display) {
 			CreateDisplay();
@@ -45,15 +106,21 @@ OBSQTDisplay::OBSQTDisplay(QWidget *parent, Qt::WindowFlags flags)
 		}
 	};
 
-	auto sizeChanged = [this](QScreen *) {
+	auto screenChanged = [this](QScreen *) {
 		CreateDisplay();
 
 		QSize size = GetPixelSize(this);
 		obs_display_resize(display, size.width(), size.height());
 	};
 
 	connect(windowHandle(), &QWindow::visibleChanged, windowVisible);
-	connect(windowHandle(), &QWindow::screenChanged, sizeChanged);
+	connect(windowHandle(), &QWindow::screenChanged, screenChanged);
+
+#ifdef ENABLE_WAYLAND
+	if (obs_get_nix_platform() == OBS_NIX_PLATFORM_WAYLAND)
+		windowHandle()->installEventFilter(
+			new SurfaceEventFilter(this));
+#endif
 }
 
 QColor OBSQTDisplay::GetDisplayBackgroundColor() const
@@ -76,9 +143,12 @@ void OBSQTDisplay::UpdateDisplayBackgroundColor()
 	obs_display_set_background_color(display, backgroundColor);
 }
 
-void OBSQTDisplay::CreateDisplay()
+void OBSQTDisplay::CreateDisplay(bool force)
 {
-	if (display || !windowHandle()->isExposed())
+	if (display)
+		return;
+
+	if (!windowHandle()->isExposed() && !force)
 		return;
 
 	QSize size = GetPixelSize(this);
@@ -89,7 +159,8 @@ void OBSQTDisplay::CreateDisplay()
 	info.format = GS_BGRA;
 	info.zsformat = GS_ZS_NONE;
 
-	QTToGSWindow(winId(), info.window);
+	if (!QTToGSWindow(windowHandle(), info.window))
+		return;
 
 	display = obs_display_create(&info, backgroundColor);
 

UI/qt-display.hpp

@@ -13,8 +13,6 @@ class OBSQTDisplay : public QWidget {
 
 	OBSDisplay display;
 
-	void CreateDisplay();
-
 	void resizeEvent(QResizeEvent *event) override;
 	void paintEvent(QPaintEvent *event) override;
 
@@ -25,6 +23,7 @@ class OBSQTDisplay : public QWidget {
 public:
 	OBSQTDisplay(QWidget *parent = nullptr,
 		     Qt::WindowFlags flags = Qt::WindowFlags());
+	~OBSQTDisplay() { display = nullptr; }
 
 	virtual QPaintEngine *paintEngine() const override;
 
@@ -35,4 +34,5 @@ class OBSQTDisplay : public QWidget {
 	QColor GetDisplayBackgroundColor() const;
 	void SetDisplayBackgroundColor(const QColor &color);
 	void UpdateDisplayBackgroundColor();
+	void CreateDisplay(bool force = false);
 };

UI/qt-wrappers.cpp

@@ -30,9 +30,14 @@
 #include <QStandardItemModel>
 
 #if !defined(_WIN32) && !defined(__APPLE__)
+#include <obs-nix-platform.h>
 #include <QX11Info>
 #endif
 
+#ifdef ENABLE_WAYLAND
+#include <qpa/qplatformnativeinterface.h>
+#endif
+
 static inline void OBSErrorBoxva(QWidget *parent, const char *msg, va_list args)
 {
 	char full_message[4096];
@@ -108,16 +113,33 @@ void OBSMessageBox::critical(QWidget *parent, const QString &title,
 	mb.exec();
 }
 
-void QTToGSWindow(WId windowId, gs_window &gswindow)
+bool QTToGSWindow(QWindow *window, gs_window &gswindow)
 {
+	bool success = true;
+
 #ifdef _WIN32
-	gswindow.hwnd = (HWND)windowId;
+	gswindow.hwnd = (HWND)window->winId();
 #elif __APPLE__
-	gswindow.view = (id)windowId;
+	gswindow.view = (id)window->winId();
 #else
-	gswindow.id = windowId;
-	gswindow.display = QX11Info::display();
+	switch (obs_get_nix_platform()) {
+	case OBS_NIX_PLATFORM_X11_GLX:
+	case OBS_NIX_PLATFORM_X11_EGL:
+		gswindow.id = window->winId();
+		gswindow.display = obs_get_nix_platform_display();
+		break;
+#ifdef ENABLE_WAYLAND
+	case OBS_NIX_PLATFORM_WAYLAND:
+		QPlatformNativeInterface *native =
+			QGuiApplication::platformNativeInterface();
+		gswindow.display =
+			native->nativeResourceForWindow("surface", window);
+		success = gswindow.display != nullptr;
+		break;
+#endif
+	}
 #endif
+	return success;
 }
 
 uint32_t TranslateQtKeyboardEventModifiers(Qt::KeyboardModifiers mods)

UI/qt-wrappers.hpp

@@ -20,6 +20,7 @@
 #include <QApplication>
 #include <QMessageBox>
 #include <QWidget>
+#include <QWindow>
 #include <QThread>
 #include <obs.hpp>
 
@@ -56,7 +57,7 @@ class OBSMessageBox {
 
 void OBSErrorBox(QWidget *parent, const char *msg, ...);
 
-void QTToGSWindow(WId windowId, gs_window &gswindow);
+bool QTToGSWindow(QWindow *window, gs_window &gswindow);
 
 uint32_t TranslateQtKeyboardEventModifiers(Qt::KeyboardModifiers mods);
 

UI/window-basic-main.cpp

@@ -79,6 +79,10 @@
 
 #include <json11.hpp>
 
+#ifdef ENABLE_WAYLAND
+#include <obs-nix-platform.h>
+#endif
+
 using namespace json11;
 using namespace std;
 
@@ -1865,9 +1869,22 @@ void OBSBasic::OBSInit()
 
 	bool alwaysOnTop = config_get_bool(App()->GlobalConfig(), "BasicWindow",
 					   "AlwaysOnTop");
-	if (alwaysOnTop || opt_always_on_top) {
+
+#ifdef ENABLE_WAYLAND
+	bool isWayland = obs_get_nix_platform() == OBS_NIX_PLATFORM_WAYLAND;
+#else
+	bool isWayland = false;
+#endif
+
+	if (!isWayland && (alwaysOnTop || opt_always_on_top)) {
 		SetAlwaysOnTop(this, true);
 		ui->actionAlwaysOnTop->setChecked(true);
+	} else if (isWayland) {
+		if (opt_always_on_top)
+			blog(LOG_INFO,
+			     "Always On Top not available on Wayland, ignoring…");
+		ui->actionAlwaysOnTop->setEnabled(false);
+		ui->actionAlwaysOnTop->setVisible(false);
 	}
 
 #ifndef _WIN32

cmake/Modules/FindEGL.cmake

@@ -0,0 +1,53 @@
+# - Try to Find EGL
+# Once done, this will define
+#
+#  EGL_FOUND - system has EGL installed.
+#  EGL_INCLUDE_DIRS - directories which contain the EGL headers.
+#  EGL_LIBRARIES - libraries required to link against EGL.
+#  EGL_DEFINITIONS - Compiler switches required for using EGL.
+#
+# Copyright (C) 2012 Intel Corporation. All rights reserved.
+#               2020 Georges Basile Stavracas Neto
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1.  Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+# 2.  Redistributions in binary form must reproduce the above copyright
+#     notice, this list of conditions and the following disclaimer in the
+#     documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND ITS CONTRIBUTORS ``AS
+# IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR ITS
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+find_package(PkgConfig)
+
+pkg_check_modules(PC_EGL egl)
+
+if (PC_EGL_FOUND)
+    set(EGL_DEFINITIONS ${PC_EGL_CFLAGS_OTHER})
+endif ()
+
+find_path(EGL_INCLUDE_DIRS NAMES EGL/egl.h
+    HINTS ${PC_EGL_INCLUDE_DIR} ${PC_EGL_INCLUDE_DIRS}
+)
+
+find_library(EGL_LIBRARIES NAMES egl EGL
+    HINTS ${PC_EGL_LIBRARY_DIRS}
+)
+
+include(FindPackageHandleStandardArgs)
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(EGL DEFAULT_MSG EGL_INCLUDE_DIRS EGL_LIBRARIES)
+
+mark_as_advanced(EGL_INCLUDE_DIRS EGL_LIBRARIES)