[Linux/BSD] Offload RenderingDevice creation test to subprocess.

(cherry picked from commit 6ed12bfc5d)

core/os/os.h

@@ -362,9 +362,9 @@ public:
 	// This is invoked by the GDExtensionManager after loading GDExtensions specified by the project.
 	virtual void load_platform_gdextensions() const {}
 
-	// Windows only. Tests OpenGL context and Rendering Device simultaneous creation. This function is expected to crash on some NVIDIA drivers.
+	// Tests OpenGL context and Rendering Device simultaneous creation. This function is expected to crash on some NVIDIA drivers.
-	virtual bool _test_create_rendering_device_and_gl() const { return true; }
+	virtual bool _test_create_rendering_device_and_gl(const String &p_display_driver) const { return true; }
-	virtual bool _test_create_rendering_device() const { return true; }
+	virtual bool _test_create_rendering_device(const String &p_display_driver) const { return true; }
 
 	OS();
 	virtual ~OS();

main/main.cpp

@@ -979,7 +979,7 @@ Error Main::setup(const char *execpath, int argc, char *argv[], bool p_second_ph
 	String project_path = ".";
 	bool upwards = false;
 	String debug_uri = "";
-#if defined(TOOLS_ENABLED) && defined(WINDOWS_ENABLED)
+#if defined(TOOLS_ENABLED) && (defined(WINDOWS_ENABLED) || defined(LINUXBSD_ENABLED))
 	bool test_rd_creation = false;
 	bool test_rd_support = false;
 #endif
@@ -1671,7 +1671,7 @@ Error Main::setup(const char *execpath, int argc, char *argv[], bool p_second_ph
 		} else if (arg == "--debug-stringnames") {
 			StringName::set_debug_stringnames(true);
 #endif
-#if defined(TOOLS_ENABLED) && defined(WINDOWS_ENABLED)
+#if defined(TOOLS_ENABLED) && (defined(WINDOWS_ENABLED) || defined(LINUXBSD_ENABLED))
 		} else if (arg == "--test-rd-support") {
 			test_rd_support = true;
 		} else if (arg == "--test-rd-creation") {
@@ -1880,12 +1880,12 @@ Error Main::setup(const char *execpath, int argc, char *argv[], bool p_second_ph
 #endif
 	}
 
-#if defined(TOOLS_ENABLED) && defined(WINDOWS_ENABLED)
+#if defined(TOOLS_ENABLED) && (defined(WINDOWS_ENABLED) || defined(LINUXBSD_ENABLED))
 	if (test_rd_support) {
 		// Test Rendering Device creation and exit.
 
 		OS::get_singleton()->set_crash_handler_silent();
-		if (OS::get_singleton()->_test_create_rendering_device()) {
+		if (OS::get_singleton()->_test_create_rendering_device(display_driver)) {
 			exit_err = ERR_HELP;
 		} else {
 			exit_err = ERR_UNAVAILABLE;
@@ -1895,7 +1895,7 @@ Error Main::setup(const char *execpath, int argc, char *argv[], bool p_second_ph
 		// Test OpenGL context and Rendering Device simultaneous creation and exit.
 
 		OS::get_singleton()->set_crash_handler_silent();
-		if (OS::get_singleton()->_test_create_rendering_device_and_gl()) {
+		if (OS::get_singleton()->_test_create_rendering_device_and_gl(display_driver)) {
 			exit_err = ERR_HELP;
 		} else {
 			exit_err = ERR_UNAVAILABLE;

platform/linuxbsd/os_linuxbsd.cpp

@@ -37,10 +37,12 @@
 #include "servers/rendering_server.h"
 
 #ifdef X11_ENABLED
+#include "x11/detect_prime_x11.h"
 #include "x11/display_server_x11.h"
 #endif
 
 #ifdef WAYLAND_ENABLED
+#include "wayland/detect_prime_egl.h"
 #include "wayland/display_server_wayland.h"
 #endif
 
@@ -49,6 +51,22 @@
 #include "modules/regex/regex.h"
 #endif
 
+#if defined(RD_ENABLED)
+#include "servers/rendering/rendering_device.h"
+#endif
+
+#if defined(VULKAN_ENABLED)
+#ifdef X11_ENABLED
+#include "x11/rendering_context_driver_vulkan_x11.h"
+#endif
+#ifdef WAYLAND_ENABLED
+#include "wayland/rendering_context_driver_vulkan_wayland.h"
+#endif
+#endif
+#if defined(GLES3_ENABLED)
+#include "drivers/gles3/rasterizer_gles3.h"
+#endif
+
 #include <dlfcn.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -1180,6 +1198,73 @@ String OS_LinuxBSD::get_system_ca_certificates() {
 	return f->get_as_text();
 }
 
+bool OS_LinuxBSD::_test_create_rendering_device(const String &p_display_driver) const {
+	// Tests Rendering Device creation.
+
+	bool ok = false;
+#if defined(RD_ENABLED)
+	Error err;
+	RenderingContextDriver *rcd = nullptr;
+
+#if defined(VULKAN_ENABLED)
+#ifdef X11_ENABLED
+	if (p_display_driver == "x11" || p_display_driver.is_empty()) {
+		rcd = memnew(RenderingContextDriverVulkanX11);
+	}
+#endif
+#ifdef WAYLAND_ENABLED
+	if (p_display_driver == "wayland") {
+		rcd = memnew(RenderingContextDriverVulkanWayland);
+	}
+#endif
+#endif
+	if (rcd != nullptr) {
+		err = rcd->initialize();
+		if (err == OK) {
+			RenderingDevice *rd = memnew(RenderingDevice);
+			err = rd->initialize(rcd);
+			memdelete(rd);
+			rd = nullptr;
+			if (err == OK) {
+				ok = true;
+			}
+		}
+		memdelete(rcd);
+		rcd = nullptr;
+	}
+#endif
+	return ok;
+}
+
+bool OS_LinuxBSD::_test_create_rendering_device_and_gl(const String &p_display_driver) const {
+	// Tests OpenGL context and Rendering Device simultaneous creation. This function is expected to crash on some drivers.
+
+#ifdef GLES3_ENABLED
+#ifdef X11_ENABLED
+	if (p_display_driver == "x11" || p_display_driver.is_empty()) {
+#ifdef SOWRAP_ENABLED
+		if (initialize_xlib(0) != 0) {
+			return false;
+		}
+#endif
+		DetectPrimeX11::create_context();
+	}
+#endif
+#ifdef WAYLAND_ENABLED
+	if (p_display_driver == "wayland") {
+#ifdef SOWRAP_ENABLED
+		if (initialize_wayland_egl(0) != 0) {
+			return false;
+		}
+#endif
+		DetectPrimeEGL::create_context(EGL_PLATFORM_WAYLAND_KHR);
+	}
+#endif
+	RasterizerGLES3::make_current(true);
+#endif
+	return _test_create_rendering_device(p_display_driver);
+}
+
 OS_LinuxBSD::OS_LinuxBSD() {
 	main_loop = nullptr;
 

platform/linuxbsd/os_linuxbsd.h

@@ -138,6 +138,9 @@ public:
 
 	virtual String get_system_ca_certificates() override;
 
+	virtual bool _test_create_rendering_device_and_gl(const String &p_display_driver) const override;
+	virtual bool _test_create_rendering_device(const String &p_display_driver) const override;
+
 	OS_LinuxBSD();
 	~OS_LinuxBSD();
 };

platform/linuxbsd/wayland/detect_prime_egl.h

@@ -77,9 +77,9 @@ private:
 		{ nullptr, 0 }
 	};
 
+public:
 	static void create_context(EGLenum p_platform_enum);
 
-public:
 	static int detect_prime(EGLenum p_platform_enum);
 };
 

platform/linuxbsd/x11/detect_prime_x11.cpp

@@ -60,25 +60,23 @@ typedef GLXContext (*GLXCREATECONTEXTATTRIBSARBPROC)(Display *, GLXFBConfig, GLX
 // To prevent shadowing warnings
 #undef glGetString
 
-struct vendor {
+int silent_error_handler(Display *display, XErrorEvent *error) {
-	const char *glxvendor = nullptr;
+	static char message[1024];
-	int priority = 0;
+	XGetErrorText(display, error->error_code, message, sizeof(message));
-};
+	print_verbose(vformat("XServer error: %s"
+						  "\n   Major opcode of failed request: %d"
+						  "\n   Serial number of failed request: %d"
+						  "\n   Current serial number in output stream: %d",
+			String::utf8(message), (uint64_t)error->request_code, (uint64_t)error->minor_code, (uint64_t)error->serial));
 
-vendor vendormap[] = {
+	quick_exit(1);
-	{ "Advanced Micro Devices, Inc.", 30 },
+	return 0;
-	{ "AMD", 30 },
+}
-	{ "NVIDIA Corporation", 30 },
-	{ "X.Org", 30 },
-	{ "Intel Open Source Technology Center", 20 },
-	{ "Intel", 20 },
-	{ "nouveau", 10 },
-	{ "Mesa Project", 0 },
-	{ nullptr, 0 }
-};
 
 // Runs inside a child. Exiting will not quit the engine.
-void create_context() {
+void DetectPrimeX11::create_context() {
+	XSetErrorHandler(&silent_error_handler);
+
 	Display *x11_display = XOpenDisplay(nullptr);
 	Window x11_window;
 	GLXContext glx_context;
@@ -137,20 +135,7 @@ void create_context() {
 	XFree(vi);
 }
 
-int silent_error_handler(Display *display, XErrorEvent *error) {
+int DetectPrimeX11::detect_prime() {
-	static char message[1024];
-	XGetErrorText(display, error->error_code, message, sizeof(message));
-	print_verbose(vformat("XServer error: %s"
-						  "\n   Major opcode of failed request: %d"
-						  "\n   Serial number of failed request: %d"
-						  "\n   Current serial number in output stream: %d",
-			String::utf8(message), (uint64_t)error->request_code, (uint64_t)error->minor_code, (uint64_t)error->serial));
-
-	quick_exit(1);
-	return 0;
-}
-
-int detect_prime() {
 	pid_t p;
 	int priorities[2] = {};
 	String vendors[2];
@@ -202,7 +187,6 @@ int detect_prime() {
 			// cleaning up these processes, and fork() makes a copy
 			// of all globals.
 			CoreGlobals::leak_reporting_enabled = false;
-			XSetErrorHandler(&silent_error_handler);
 
 			char string[201];
 
@@ -253,7 +237,7 @@ int detect_prime() {
 	}
 
 	for (int i = 1; i >= 0; --i) {
-		vendor *v = vendormap;
+		const Vendor *v = vendor_map;
 		while (v->glxvendor) {
 			if (v->glxvendor == vendors[i]) {
 				priorities[i] = v->priority;

platform/linuxbsd/x11/detect_prime_x11.h

@@ -33,7 +33,30 @@
 
 #if defined(X11_ENABLED) && defined(GLES3_ENABLED)
 
-int detect_prime();
+class DetectPrimeX11 {
+private:
+	struct Vendor {
+		const char *glxvendor = nullptr;
+		int priority = 0;
+	};
+
+	static constexpr Vendor vendor_map[] = {
+		{ "Advanced Micro Devices, Inc.", 30 },
+		{ "AMD", 30 },
+		{ "NVIDIA Corporation", 30 },
+		{ "X.Org", 30 },
+		{ "Intel Open Source Technology Center", 20 },
+		{ "Intel", 20 },
+		{ "nouveau", 10 },
+		{ "Mesa Project", 0 },
+		{ nullptr, 0 }
+	};
+
+public:
+	static void create_context();
+
+	static int detect_prime();
+};
 
 #endif // X11_ENABLED && GLES3_ENABLED
 

platform/linuxbsd/x11/display_server_x11.cpp

@@ -6852,7 +6852,7 @@ DisplayServerX11::DisplayServerX11(const String &p_rendering_driver, WindowMode
 
 			if (use_prime == -1) {
 				print_verbose("Detecting GPUs, set DRI_PRIME in the environment to override GPU detection logic.");
-				use_prime = detect_prime();
+				use_prime = DetectPrimeX11::detect_prime();
 			}
 
 			if (use_prime) {

platform/windows/os_windows.cpp

@@ -2367,7 +2367,7 @@ void OS_Windows::add_frame_delay(bool p_can_draw) {
 	}
 }
 
-bool OS_Windows::_test_create_rendering_device() const {
+bool OS_Windows::_test_create_rendering_device(const String &p_display_driver) const {
 	// Tests Rendering Device creation.
 
 	bool ok = false;
@@ -2402,7 +2402,7 @@ bool OS_Windows::_test_create_rendering_device() const {
 	return ok;
 }
 
-bool OS_Windows::_test_create_rendering_device_and_gl() const {
+bool OS_Windows::_test_create_rendering_device_and_gl(const String &p_display_driver) const {
 	// Tests OpenGL context and Rendering Device simultaneous creation. This function is expected to crash on some NVIDIA drivers.
 
 	WNDCLASSEXW wc_probe;
@@ -2446,7 +2446,7 @@ bool OS_Windows::_test_create_rendering_device_and_gl() const {
 	}
 
 	if (ok) {
-		ok = _test_create_rendering_device();
+		ok = _test_create_rendering_device(p_display_driver);
 	}
 
 #ifdef GLES3_ENABLED

platform/windows/os_windows.h

@@ -252,8 +252,8 @@ public:
 
 	void set_main_window(HWND p_main_window) { main_window = p_main_window; }
 
-	virtual bool _test_create_rendering_device_and_gl() const override;
+	virtual bool _test_create_rendering_device_and_gl(const String &p_display_driver) const override;
-	virtual bool _test_create_rendering_device() const override;
+	virtual bool _test_create_rendering_device(const String &p_display_driver) const override;
 
 	HINSTANCE get_hinstance() { return hInstance; }
 	OS_Windows(HINSTANCE _hInstance);

servers/display_server.cpp

@@ -1331,10 +1331,14 @@ bool DisplayServer::is_rendering_device_supported() {
 
 	Error err;
 
-#ifdef WINDOWS_ENABLED
+#if defined(WINDOWS_ENABLED) || defined(LINUXBSD_ENABLED)
-	// On some NVIDIA drivers combining OpenGL and RenderingDevice can result in crash, offload the check to the subprocess.
+	// On some drivers combining OpenGL and RenderingDevice can result in crash, offload the check to the subprocess.
 	List<String> arguments;
 	arguments.push_back("--test-rd-support");
+	if (get_singleton()) {
+		arguments.push_back("--display-driver");
+		arguments.push_back(get_singleton()->get_name().to_lower());
+	}
 
 	String pipe;
 	int exitcode = 0;