win-capture: Fix potential crash due to unhandled exceptions

libobs-winrt/winrt-capture.cpp

@@ -311,22 +311,27 @@ static void winrt_capture_device_loss_rebuild(void *device_void, void *data)
 thread_local bool initialized_tls;
 
 extern "C" EXPORT struct winrt_capture *
-winrt_capture_init(BOOL cursor, HWND window, BOOL client_area)
-{
+winrt_capture_init(BOOL cursor, HWND window, BOOL client_area, char **error,
+		   HRESULT *hr_out)
+try {
 	ID3D11Device *const d3d_device = (ID3D11Device *)gs_get_device_obj();
 	ComPtr<IDXGIDevice> dxgi_device;
-	if (FAILED(d3d_device->QueryInterface(&dxgi_device))) {
-		blog(LOG_WARNING, "[winrt_capture_init] Failed to "
-				  "get DXGI device");
+
+	*error = nullptr;
+
+	HRESULT hr = d3d_device->QueryInterface(&dxgi_device);
+	if (FAILED(hr)) {
+		*error = bstrdup("Failed to get DXGI device");
+		*hr_out = hr;
 		return nullptr;
 	}
 
 	winrt::com_ptr<IInspectable> inspectable;
-	HRESULT hr = CreateDirect3D11DeviceFromDXGIDevice(dxgi_device.Get(),
-							  inspectable.put());
+	hr = CreateDirect3D11DeviceFromDXGIDevice(dxgi_device.Get(),
+						  inspectable.put());
 	if (FAILED(hr)) {
-		blog(LOG_WARNING, "[winrt_capture_init] Failed to "
-				  "get WinRT device");
+		*error = bstrdup("Failed to get WinRT device");
+		*hr_out = hr;
 		return nullptr;
 	}
 
@@ -341,10 +346,9 @@ winrt_capture_init(BOOL cursor, HWND window, BOOL client_area)
 			winrt::guid_of<ABI::Windows::Graphics::Capture::
 					       IGraphicsCaptureItem>(),
 			reinterpret_cast<void **>(winrt::put_abi(item)));
-	} catch (winrt::hresult_invalid_argument &) {
-		/* too spammy */
-		//blog(LOG_WARNING, "[winrt_capture_init] Failed to "
-		//		  "create GraphicsCaptureItem");
+	} catch (winrt::hresult_error &err) {
+		*error = bstrdup("CreateForWindow failed");
+		*hr_out = err.code();
 		return nullptr;
 	}
 
@@ -398,6 +402,11 @@ winrt_capture_init(BOOL cursor, HWND window, BOOL client_area)
 	gs_register_loss_callbacks(&callbacks);
 
 	return capture;
+
+} catch (winrt::hresult_error &err) {
+	*error = bstrdup("oh wow something else in winrt_capture_init failed");
+	*hr_out = err.code();
+	return nullptr;
 }
 
 extern "C" EXPORT void winrt_capture_free(struct winrt_capture *capture)

libobs-winrt/winrt-capture.h

@@ -12,7 +12,8 @@ extern "C" {
 EXPORT BOOL winrt_capture_supported();
 EXPORT BOOL winrt_capture_cursor_toggle_supported();
 EXPORT struct winrt_capture *winrt_capture_init(BOOL cursor, HWND window,
-						BOOL client_area);
+						BOOL client_area, char **error,
+						HRESULT *hr);
 EXPORT void winrt_capture_free(struct winrt_capture *capture);
 
 EXPORT void winrt_capture_show_cursor(struct winrt_capture *capture,

plugins/win-capture/window-capture.c

@@ -29,7 +29,8 @@ struct winrt_exports {
 	BOOL *(*winrt_capture_supported)();
 	BOOL *(*winrt_capture_cursor_toggle_supported)();
 	struct winrt_capture *(*winrt_capture_init)(BOOL cursor, HWND window,
-						    BOOL client_area);
+						    BOOL client_area,
+						    char **error, HRESULT *hr);
 	void (*winrt_capture_free)(struct winrt_capture *capture);
 	void (*winrt_capture_show_cursor)(struct winrt_capture *capture,
 					  BOOL visible);
@@ -61,6 +62,7 @@ struct window_capture {
 	struct dc_capture capture;
 
 	bool wgc_supported;
+	bool previously_failed;
 	void *winrt_module;
 	struct winrt_exports exports;
 	struct winrt_capture *capture_winrt;
@@ -478,8 +480,24 @@ static void wc_tick(void *data, float seconds)
 		dc_capture_capture(&wc->capture, wc->window);
 	} else if (wc->method == METHOD_WGC) {
 		if (wc->window && (wc->capture_winrt == NULL)) {
+			char *error = NULL;
+			HRESULT hr;
+
 			wc->capture_winrt = wc->exports.winrt_capture_init(
-				wc->cursor, wc->window, wc->client_area);
+				wc->cursor, wc->window, wc->client_area, &error,
+				&hr);
+
+			if (!wc->capture_winrt && !wc->previously_failed) {
+				blog(LOG_WARNING,
+				     "%s: winrt_capture_init failed: %s: %lX",
+				     obs_source_get_name(wc->source), error,
+				     hr);
+				wc->previously_failed = true;
+			} else if (wc->capture_winrt) {
+				wc->previously_failed = false;
+			}
+
+			bfree(error);
 		}
 	}