Screen capture api

README.md

@@ -59,6 +59,7 @@ Now you can start building. Open a command window or terminal in the root direct
 
 ### Linux
 
+If building with GCC, you need version 13.1 or later. If building with Clang, you need version 14 or later.
 - `cmake -S . -B build -D CMAKE_BUILD_TYPE=Release`
   - note: you'll need to specify either `-D BOLT_LUAJIT_INCLUDE_DIR=/usr/include/luajit-2.1` OR `-D BOLT_SKIP_LIBRARIES=1` depending on whether you want to build the plugin library
   - note: build types "Debug" and "Release" are supported

src/browser/app.cxx

@@ -6,6 +6,18 @@
 
 #include <fmt/core.h>
 
+#if defined(_WIN32)
+#include <Windows.h>
+static HANDLE shm_handle;
+#else
+#include <sys/mman.h>
+#include <fcntl.h>
+static int shm_fd;
+#endif
+static void* shm_file;
+static size_t shm_length;
+static bool shm_inited = false;
+
 class ArrayBufferReleaseCallbackFree: public CefV8ArrayBufferReleaseCallback {
 	void ReleaseBuffer(void* buffer) override {
 		::free(buffer);
@@ -172,6 +184,74 @@ bool Browser::App::OnProcessMessageReceived(CefRefPtr<CefBrowser> browser, CefRe
 		return true;
 	}
 
+	if (name == "__bolt_plugin_capture") {
+		CefRefPtr<CefListValue> list = message->GetArgumentList();
+		if (list->GetSize() >= 2) {
+			const int width = list->GetInt(0);
+			const int height = list->GetInt(1);
+			const size_t size = (size_t)width * (size_t)height * 3;
+
+			if (list->GetSize() != 2) {
+#if defined(_WIN32)
+				const std::wstring path = list->GetString(2).ToWString();
+				if (shm_inited) {
+					UnmapViewOfFile(shm_file);
+					CloseHandle(shm_handle);
+				}
+				shm_handle = CreateFileMappingW(INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE, 0, (DWORD)size, path.c_str());
+				shm_file = MapViewOfFile(shm_handle, FILE_MAP_READ, 0, 0, size);
+#else
+				if (list->GetType(2) == VTYPE_STRING) {
+					const std::string path = list->GetString(2).ToString();
+					if (shm_inited) {
+						munmap(shm_file, shm_length);
+						close(shm_fd);
+					}
+					shm_fd = shm_open(path.c_str(), O_RDWR, 0644);
+					shm_file = mmap(NULL, size, PROT_READ, MAP_SHARED, shm_fd, 0);
+					fmt::print("[R] shm named-remap '{}' -> {}, {}\n", path, shm_fd, (unsigned long long)shm_file);
+				} else if (shm_inited) {
+					shm_file = mremap(shm_file, shm_length, size, MREMAP_MAYMOVE);
+					fmt::print("[R] shm unnamed-remap -> {} ({})\n", (unsigned long long)shm_file, errno);
+				} else {
+					fmt::print("[R] warning: shm not set up and wasn't provided with enough information to do setup; {} will be ignored\n", name.ToString());
+					return true;
+				}
+#endif
+				shm_length = size;
+				shm_inited = true;
+			}
+
+			CefRefPtr<CefV8ArrayBufferReleaseCallback> cb = new ArrayBufferReleaseCallbackFree();
+			void* buffer = malloc(size);
+			memcpy(buffer, shm_file, size);
+
+			CefRefPtr<CefProcessMessage> response_message = CefProcessMessage::Create("__bolt_plugin_capture_done");
+			frame->SendProcessMessage(PID_BROWSER, response_message);
+
+			CefRefPtr<CefV8Context> context = frame->GetV8Context();
+			context->Enter();
+			CefRefPtr<CefV8Value> content = CefV8Value::CreateArrayBuffer(buffer, size, cb);
+			CefRefPtr<CefV8Value> dict = CefV8Value::CreateObject(nullptr, nullptr);
+			dict->SetValue("type", CefV8Value::CreateString("screenCapture"), V8_PROPERTY_ATTRIBUTE_READONLY);
+			dict->SetValue("content", content, V8_PROPERTY_ATTRIBUTE_READONLY);
+			dict->SetValue("width", CefV8Value::CreateInt(width), V8_PROPERTY_ATTRIBUTE_READONLY);
+			dict->SetValue("height", CefV8Value::CreateInt(height), V8_PROPERTY_ATTRIBUTE_READONLY);
+			CefV8ValueList value_list = {dict, CefV8Value::CreateString("*")};
+			CefRefPtr<CefV8Value> post_message = context->GetGlobal()->GetValue("postMessage");
+			if (post_message->IsFunction()) {
+				post_message->ExecuteFunctionWithContext(context, nullptr, value_list);
+			} else {
+				fmt::print("[R] warning: window.postMessage is not a function, {} will be ignored\n", name.ToString());
+			}
+			context->Exit();
+		} else {
+			fmt::print("[R] warning: too few arguments, {} will be ignored\n", name.ToString());
+		}
+
+		return true;
+	}
+
 	return false;
 }
 

src/browser/client.cxx

@@ -393,7 +393,7 @@ bool Browser::Client::IPCHandleMessage(int fd) {
 				.resizeable = true,
 				.frame = true,
 			};
-			plugin->windows.push_back(new Browser::PluginWindow(this, details, url, plugin, fd, &this->send_lock, header.window_id, header.plugin_id, false));
+			plugin->windows.push_back(new Browser::PluginWindow(this, details, url, plugin, fd, &this->send_lock, this->runtime_dir, header.window_id, header.plugin_id, false));
 			delete[] url;
 			break;
 		}
@@ -406,7 +406,7 @@ bool Browser::Client::IPCHandleMessage(int fd) {
 
 			CefRefPtr<ActivePlugin> plugin = this->GetPluginFromFDAndID(client, header.plugin_id);
 			if (plugin) {
-				CefRefPtr<Browser::WindowOSR> window = new Browser::WindowOSR(CefString((char*)url), header.w, header.h, fd, this, &this->send_lock, header.pid, header.window_id, header.plugin_id, plugin);
+				CefRefPtr<Browser::WindowOSR> window = new Browser::WindowOSR(CefString((char*)url), header.w, header.h, fd, this, &this->send_lock, this->runtime_dir, header.pid, header.window_id, header.plugin_id, plugin);
 				plugin->windows_osr.push_back(window);
 			}
 			delete[] url;
@@ -433,6 +433,20 @@ bool Browser::Client::IPCHandleMessage(int fd) {
 			if (window) window->HandleAck();
 			break;
 		}
+		case IPC_MSG_CAPTURENOTIFY_EXTERNAL: {
+			BoltIPCCaptureNotifyHeader header;
+			_bolt_ipc_receive(fd, &header, sizeof(header));
+			CefRefPtr<Browser::PluginWindow> window = this->GetExternalWindowFromFDAndIDs(client, header.plugin_id, header.window_id);
+			if (window && !window->IsDeleted()) window->HandleCaptureNotify(header.pid, header.capture_id, header.width, header.height, header.needs_remap != 0);
+			break;
+		}
+		case IPC_MSG_CAPTURENOTIFY_OSR: {
+			BoltIPCCaptureNotifyHeader header;
+			_bolt_ipc_receive(fd, &header, sizeof(header));
+			CefRefPtr<Browser::WindowOSR> window = this->GetOsrWindowFromFDAndIDs(client, header.plugin_id, header.window_id);
+			if (window && !window->IsDeleted()) window->HandleCaptureNotify(header.pid, header.capture_id, header.width, header.height, header.needs_remap != 0);
+			break;
+		}
 
 #define DEF_OSR_EVENT(EVNAME, HANDLER, EVTYPE) case IPC_MSG_EV##EVNAME: { \
 	BoltIPCEvHeader header; \

src/browser/window_osr.cxx

@@ -29,8 +29,8 @@ static void SendUpdateMsg(BoltSocketType fd, std::mutex* send_lock, uint64_t id,
 	send_lock->unlock();
 }
 
-Browser::WindowOSR::WindowOSR(CefString url, int width, int height, BoltSocketType client_fd, Browser::Client* main_client, std::mutex* send_lock, int pid, uint64_t window_id, uint64_t plugin_id, CefRefPtr<FileManager::Directory> file_manager):
-	PluginRequestHandler(IPC_MSG_OSRBROWSERMESSAGE, send_lock),
+Browser::WindowOSR::WindowOSR(CefString url, int width, int height, BoltSocketType client_fd, Browser::Client* main_client, std::mutex* send_lock, std::filesystem::path runtime_dir, int pid, uint64_t window_id, uint64_t plugin_id, CefRefPtr<FileManager::Directory> file_manager):
+	PluginRequestHandler(IPC_MSG_OSRBROWSERMESSAGE, send_lock, runtime_dir),
 	deleted(false), pending_delete(false), client_fd(client_fd), width(width), height(height), browser(nullptr), window_id(window_id),
 	plugin_id(plugin_id), main_client(main_client), stored(nullptr), remote_has_remapped(false), remote_is_idle(true), file_manager(file_manager)
 {
@@ -217,6 +217,16 @@ bool Browser::WindowOSR::OnProcessMessageReceived(CefRefPtr<CefBrowser> browser,
 	const CefString name = message->GetName();
 	if (name == "__bolt_pluginbrowser_close") {
 		this->browser->GetHost()->TryCloseBrowser();
+		return true;
+	}
+	if (name == "__bolt_plugin_capture_done") {
+		const BoltIPCMessageTypeToClient msg_type = IPC_MSG_OSRCAPTUREDONE;
+		const BoltIPCOsrCaptureDoneHeader header = { .window_id = this->WindowID() };
+		this->send_lock->lock();
+		_bolt_ipc_send(this->client_fd, &msg_type, sizeof(msg_type));
+		_bolt_ipc_send(this->client_fd, &header, sizeof(header));
+		this->send_lock->unlock();
+		return true;
 	}
 	return false;
 }

src/browser/window_osr.hxx

@@ -21,7 +21,7 @@ namespace Browser {
 	struct Client;
 
 	struct WindowOSR: public CefClient, CefLifeSpanHandler, CefRenderHandler, PluginRequestHandler {
-		WindowOSR(CefString url, int width, int height, BoltSocketType client_fd, Client* main_client, std::mutex* send_lock, int pid, uint64_t window_id, uint64_t plugin_id, CefRefPtr<FileManager::Directory>);
+		WindowOSR(CefString url, int width, int height, BoltSocketType client_fd, Client* main_client, std::mutex* send_lock, std::filesystem::path runtime_dir, int pid, uint64_t window_id, uint64_t plugin_id, CefRefPtr<FileManager::Directory>);
 
 		bool IsDeleted();
 

src/browser/window_plugin.cxx

@@ -17,8 +17,8 @@ struct InitTask: public CefTask {
 		DISALLOW_COPY_AND_ASSIGN(InitTask);
 };
 
-Browser::PluginWindow::PluginWindow(CefRefPtr<Client> main_client, Details details, const char* url, CefRefPtr<FileManager::Directory> file_manager, BoltSocketType fd, std::mutex* send_lock, uint64_t id, uint64_t plugin_id, bool show_devtools):
-	PluginRequestHandler(IPC_MSG_EXTERNALBROWSERMESSAGE, send_lock),
+Browser::PluginWindow::PluginWindow(CefRefPtr<Client> main_client, Details details, const char* url, CefRefPtr<FileManager::Directory> file_manager, BoltSocketType fd, std::mutex* send_lock, std::filesystem::path runtime_dir, uint64_t id, uint64_t plugin_id, bool show_devtools):
+	PluginRequestHandler(IPC_MSG_EXTERNALBROWSERMESSAGE, send_lock, runtime_dir),
 	Window(main_client, details, show_devtools),
 	file_manager(file_manager), client_fd(fd), window_id(id), plugin_id(plugin_id), closing(false), deleted(false)
 {
@@ -80,6 +80,21 @@ bool Browser::PluginWindow::OnBeforePopup(
 	return true;
 }
 
+bool Browser::PluginWindow::OnProcessMessageReceived(CefRefPtr<CefBrowser> browser, CefRefPtr<CefFrame> frame, CefProcessId process, CefRefPtr<CefProcessMessage> message) {
+	if (!(this->browser && this->browser->IsSame(browser))) return false;
+	const CefString name = message->GetName();
+	if (name == "__bolt_plugin_capture_done") {
+		const BoltIPCMessageTypeToClient msg_type = IPC_MSG_EXTERNALCAPTUREDONE;
+		const BoltIPCExternalCaptureDoneHeader header = { .window_id = this->window_id, .plugin_id = this->plugin_id };
+		this->send_lock->lock();
+		_bolt_ipc_send(this->client_fd, &msg_type, sizeof(msg_type));
+		_bolt_ipc_send(this->client_fd, &header, sizeof(header));
+		this->send_lock->unlock();
+		return true;
+	}
+	return false;
+}
+
 CefRefPtr<CefRequestHandler> Browser::PluginWindow::GetRequestHandler() {
 	return this;
 }

src/browser/window_plugin.hxx

@@ -8,7 +8,7 @@
 
 namespace Browser {
 	struct PluginWindow: public Window, PluginRequestHandler {
-		PluginWindow(CefRefPtr<Client> main_client, Details details, const char* url, CefRefPtr<FileManager::Directory> file_manager, BoltSocketType fd, std::mutex* send_lock, uint64_t id, uint64_t plugin_id, bool show_devtools);
+		PluginWindow(CefRefPtr<Client> main_client, Details details, const char* url, CefRefPtr<FileManager::Directory> file_manager, BoltSocketType fd, std::mutex* send_lock, std::filesystem::path runtime_dir, uint64_t id, uint64_t plugin_id, bool show_devtools);
 		bool IsDeleted() const;
 
 		uint64_t WindowID() const override;
@@ -33,6 +33,8 @@ namespace Browser {
 			bool*
 		) override;
 
+		bool OnProcessMessageReceived(CefRefPtr<CefBrowser>, CefRefPtr<CefFrame>, CefProcessId, CefRefPtr<CefProcessMessage>) override;
+
 		CefRefPtr<CefRequestHandler> GetRequestHandler() override;
 		void Close() override;
 		bool CanClose(CefRefPtr<CefWindow>) override;

src/browser/window_plugin_requests.cxx

@@ -3,6 +3,8 @@
 #include "resource_handler.hxx"
 #include "request.hxx"
 
+#include <format>
+
 void Browser::PluginRequestHandler::HandlePluginMessage(const uint8_t* data, size_t len) {
 	CefRefPtr<CefProcessMessage> message = CefProcessMessage::Create("__bolt_plugin_message");
 	CefRefPtr<CefListValue> list = message->GetArgumentList();
@@ -12,6 +14,33 @@ void Browser::PluginRequestHandler::HandlePluginMessage(const uint8_t* data, siz
 	if (browser) browser->GetMainFrame()->SendProcessMessage(PID_RENDERER, message);
 }
 
+void Browser::PluginRequestHandler::HandleCaptureNotify(uint64_t pid, uint64_t capture_id, int width, int height, bool needs_remap) {
+	CefRefPtr<CefProcessMessage> message = CefProcessMessage::Create("__bolt_plugin_capture");
+	CefRefPtr<CefListValue> list = message->GetArgumentList();
+	if (needs_remap) {
+		list->SetSize(3);
+#if defined(_WIN32)
+		std::filesystem::path shm_path = this->runtime_dir;
+		shm_path.append(std::format("bolt-{}-sc-{}", pid, capture_id));
+		list->SetString(2, shm_path.string());
+#else
+		if (capture_id != this->current_capture_id) {
+			const CefString str = std::format("/bolt-{}-sc-{}", pid, capture_id);
+			list->SetString(2, str);
+		} else {
+			list->SetNull(2);
+		}
+#endif
+	} else {
+		list->SetSize(2);
+	}
+	list->SetInt(0, width);
+	list->SetInt(1, height);
+	CefRefPtr<CefBrowser> browser = this->Browser();
+	if (browser) browser->GetMainFrame()->SendProcessMessage(PID_RENDERER, message);
+	this->current_capture_id = capture_id;
+}
+
 CefRefPtr<CefResourceRequestHandler> Browser::PluginRequestHandler::GetResourceRequestHandler(
 	CefRefPtr<CefBrowser> browser,
 	CefRefPtr<CefFrame> frame,

src/browser/window_plugin_requests.hxx

@@ -1,5 +1,6 @@
 #ifndef _BOLT_WINDOW_PLUGIN_REQUESTS_HXX
 #define _BOLT_WINDOW_PLUGIN_REQUESTS_HXX
+#include <filesystem>
 #if defined(BOLT_PLUGINS)
 #include "../library/ipc.h"
 #include "../file_manager.hxx"
@@ -10,9 +11,11 @@
 namespace Browser {
 	/// Abstract class handling requests from plugin-managed browsers
 	struct PluginRequestHandler: public CefRequestHandler {
-		PluginRequestHandler(BoltIPCMessageTypeToClient message_type, std::mutex* send_lock): message_type(message_type), send_lock(send_lock) {}
+		PluginRequestHandler(BoltIPCMessageTypeToClient message_type, std::mutex* send_lock, std::filesystem::path runtime_dir):
+			message_type(message_type), send_lock(send_lock), runtime_dir(runtime_dir), current_capture_id(-1) {}
 
 		void HandlePluginMessage(const uint8_t*, size_t);
+		void HandleCaptureNotify(uint64_t, uint64_t, int, int, bool);
 
 		CefRefPtr<CefResourceRequestHandler> GetResourceRequestHandler(
 			CefRefPtr<CefBrowser>,
@@ -33,6 +36,8 @@ namespace Browser {
 
 		protected:
 			std::mutex* send_lock;
+			std::filesystem::path runtime_dir;
+			uint64_t current_capture_id;
 
 		private:
 			BoltIPCMessageTypeToClient message_type;

src/library/dll/main.c

@@ -102,6 +102,7 @@ DWORD __stdcall BOLT_STUB_ENTRYNAME(struct PluginInjectParams* data) {
     libgl.Flush = (void(*)(void))data->pGetProcAddress(libgl_module, "glFlush");
     libgl.GenTextures = (void(*)(GLsizei, GLuint*))data->pGetProcAddress(libgl_module, "glGenTextures");
     libgl.GetError = (GLenum(*)(void))data->pGetProcAddress(libgl_module, "glGetError");
+    libgl.ReadPixels = (void(*)(GLint, GLint, GLsizei, GLsizei, GLenum, GLenum, void*))data->pGetProcAddress(libgl_module, "glReadPixels");
     libgl.TexParameteri = (void(*)(GLenum, GLenum, GLfloat))data->pGetProcAddress(libgl_module, "glTexParameteri");
     libgl.TexSubImage2D = (void(*)(GLenum, GLint, GLint, GLint, GLsizei, GLsizei, GLenum, GLenum, const void*))data->pGetProcAddress(libgl_module, "glTexSubImage2D");
     libgl.Viewport = (void(*)(GLint, GLint, GLsizei, GLsizei))data->pGetProcAddress(libgl_module, "glViewport");

src/library/gl.c

@@ -128,6 +128,7 @@ static void _bolt_gl_plugin_surface_subimage(void* userdata, int x, int y, int w
 static void _bolt_gl_plugin_surface_drawtoscreen(void* userdata, int sx, int sy, int sw, int sh, int dx, int dy, int dw, int dh);
 static void _bolt_gl_plugin_surface_drawtosurface(void* userdata, void* target, int sx, int sy, int sw, int sh, int dx, int dy, int dw, int dh);
 static void _bolt_gl_plugin_draw_region_outline(void* userdata, int16_t x, int16_t y, uint16_t width, uint16_t height);
+static void _bolt_gl_plugin_read_screen_pixels(uint32_t width, uint32_t height, void* data);
 
 #define MAX_TEXTURE_UNITS 4096 // would be nice if there was a way to query this at runtime, but it would be awkward to set up
 #define BUFFER_LIST_CAPACITY 256 * 256
@@ -1246,6 +1247,7 @@ void _bolt_gl_onMakeCurrent(void* context) {
             .surface_destroy = _bolt_gl_plugin_surface_destroy,
             .surface_resize_and_clear = _bolt_gl_plugin_surface_resize,
             .draw_region_outline = _bolt_gl_plugin_draw_region_outline,
+            .read_screen_pixels = _bolt_gl_plugin_read_screen_pixels,
         };
         _bolt_plugin_init(&functions);
     }
@@ -1909,3 +1911,7 @@ static void _bolt_gl_plugin_draw_region_outline(void* userdata, int16_t x, int16
     gl.BindVertexArray(c->bound_vao->id);
     gl.UseProgram(c->bound_program ? c->bound_program->id : 0);
 }
+
+static void _bolt_gl_plugin_read_screen_pixels(uint32_t width, uint32_t height, void* data) {
+    lgl->ReadPixels(0, 0, width, height, GL_RGB, GL_UNSIGNED_BYTE, data);
+}

src/library/gl.h

@@ -96,6 +96,7 @@ struct GLLibFunctions {
     void (*Flush)(void);
     void (*GenTextures)(GLsizei, GLuint*);
     GLenum (*GetError)(void);
+    void (*ReadPixels)(GLint, GLint, GLsizei, GLsizei, GLenum, GLenum, void*);
     void (*TexParameteri)(GLenum, GLenum, GLfloat);
     void (*TexSubImage2D)(GLenum, GLint, GLint, GLint, GLsizei, GLsizei, GLenum, GLenum, const void*);
     void (*Viewport)(GLint, GLint, GLsizei, GLsizei);
@@ -104,6 +105,7 @@ struct GLLibFunctions {
 /* consts used from libgl */
 #define GL_TEXTURE 5890
 #define GL_TEXTURE_2D 3553
+#define GL_RGB 6407
 #define GL_RGBA 6408
 #define GL_BGRA 32993
 #define GL_UNSIGNED_BYTE 5121