Merge pull request #81 from oligamiq/main

implement thread-spawn
bjorn3 · Nov 10, 2024 · 3f705d1 · 3f705d1
2 parents 9ca07f8 + 622e4c0
commit 3f705d1
Show file tree

Hide file tree

Showing 90 changed files with 14,508 additions and 0 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,4 @@
 node_modules/
 dist/
 typings/
+.vscode/
diff --git a/.gitmodules b/.gitmodules
@@ -5,3 +5,7 @@
 	path = test/wasi-testsuite
 	url = https://github.com/WebAssembly/wasi-testsuite
 	branch = prod/testsuite-base
+[submodule "threads/examples/wasi_multi_threads_rustc/rust_wasm"]
+	path = threads/examples/wasi_multi_threads_rustc/rust_wasm
+	url = https://github.com/oligamiq/rust_wasm
+	sharrow = true
diff --git a/threads/.gitignore b/threads/.gitignore
@@ -0,0 +1,4 @@
+/.vscode
+/dist
+/node_modules
+/types
diff --git a/threads/.swcrc b/threads/.swcrc
@@ -0,0 +1,18 @@
+{
+  "$schema": "https://json.schemastore.org/swcrc",
+  "jsc": {
+    "parser": {
+      "syntax": "typescript",
+      "tsx": false,
+      "dynamicImport": false,
+      "decorators": false,
+      "dts": true
+    },
+    "transform": {},
+    "target": "esnext",
+    "loose": false,
+    "externalHelpers": false,
+    "keepClassNames": true
+  },
+  "minify": true
+}
diff --git a/threads/README.md b/threads/README.md
@@ -0,0 +1,24 @@
+# A pure javascript shim for WASI Preview 1 threads
+
+> [!WARNING]
+> The code in this directory is less production ready than the main browser_wasi_shim code.
+
+This project is implement threads on browser_wasi_shim
+
+# Features
+- [x] thread creation
+- [x] Filesystem wrapper accessible by multiple workers
+- [ ] thread pool
+
+# Building
+```sh
+$ npm install
+$ npm run build
+```
+
+# Running the demo
+```sh
+$ git submodule update --init
+$ cd examples && npm install && npm run dev
+```
+And visit http://localhost
diff --git a/threads/biome.json b/threads/biome.json
@@ -0,0 +1,30 @@
+{
+  "$schema": "https://biomejs.dev/schemas/1.9.1/schema.json",
+  "vcs": {
+    "enabled": false,
+    "clientKind": "git",
+    "useIgnoreFile": false
+  },
+  "files": {
+    "ignoreUnknown": false,
+    "ignore": []
+  },
+  "formatter": {
+    "enabled": true,
+    "indentStyle": "space"
+  },
+  "organizeImports": {
+    "enabled": true
+  },
+  "linter": {
+    "enabled": true,
+    "rules": {
+      "recommended": true
+    }
+  },
+  "javascript": {
+    "formatter": {
+      "quoteStyle": "double"
+    }
+  }
+}
diff --git a/threads/bun.lockb b/threads/bun.lockb
diff --git a/threads/examples/bun.lockb b/threads/examples/bun.lockb
diff --git a/threads/examples/package.json b/threads/examples/package.json
@@ -0,0 +1,8 @@
+{
+  "dependencies": {
+    "@oligami/shared-object": "^0.1.1",
+    "@xterm/xterm": "^5.5",
+    "xterm-addon-fit": "^0.8.0",
+    "@bjorn3/browser_wasi_shim": "^0.3.0"
+  }
+}
diff --git a/threads/examples/wasi_multi_threads/index.html b/threads/examples/wasi_multi_threads/index.html
@@ -0,0 +1,12 @@
+<!DOCTYPE html>
+<html>
+<head>
+<meta charset="utf-8">
+
+<p id="note">
+  ####
+</p>
+
+<script type="module" src="./index.ts"></script>
+</body>
+</html>
diff --git a/threads/examples/wasi_multi_threads/index.ts b/threads/examples/wasi_multi_threads/index.ts
@@ -0,0 +1,27 @@
+import { WASIFarm, WASIFarmAnimal } from "../../src";
+import {
+  File,
+  OpenFile,
+  ConsoleStdout,
+  PreopenDirectory,
+} from "@bjorn3/browser_wasi_shim";
+
+const farm = new WASIFarm(
+  new OpenFile(new File([])), // stdin
+  ConsoleStdout.lineBuffered((msg) => console.log(`[WASI stdout] ${msg}`)),
+  ConsoleStdout.lineBuffered((msg) => console.warn(`[WASI stderr] ${msg}`)),
+  [],
+);
+
+console.log(farm);
+
+const worker = new Worker("worker.ts", { type: "module" });
+// const worker = new Worker(new URL("./worker.js", import.meta.url).href, { type: "module" });
+
+console.log(worker);
+
+worker.postMessage({
+  wasi_ref: farm.get_ref(),
+});
+
+console.log("Sent WASI ref to worker");
diff --git a/threads/examples/wasi_multi_threads/main.rs b/threads/examples/wasi_multi_threads/main.rs
@@ -0,0 +1,13 @@
+fn main() {
+    println!("Hello, world!");
+
+    let _: std::thread::JoinHandle<()> = std::thread::spawn(|| {
+        for i in 1..1000 {
+            println!("hi number {} from the spawned thread!", i);
+        }
+    });
+
+    for i in 1..1000 {
+        println!("hi number {} from the main thread!", i);
+    }
+}
diff --git a/threads/examples/wasi_multi_threads/multi_thread_echo.wasm b/threads/examples/wasi_multi_threads/multi_thread_echo.wasm
diff --git a/threads/examples/wasi_multi_threads/thread_spawn.ts b/threads/examples/wasi_multi_threads/thread_spawn.ts
@@ -0,0 +1,5 @@
+import { thread_spawn_on_worker } from "../../src";
+
+self.onmessage = (event) => {
+  thread_spawn_on_worker(event.data);
+};
diff --git a/threads/examples/wasi_multi_threads/worker.ts b/threads/examples/wasi_multi_threads/worker.ts
@@ -0,0 +1,37 @@
+import { WASIFarmAnimal } from "../../src";
+
+self.onmessage = async (e) => {
+  const { wasi_ref } = e.data;
+
+  const wasm = await WebAssembly.compileStreaming(
+    fetch("./multi_thread_echo.wasm"),
+  );
+
+  const wasi = new WASIFarmAnimal(
+    wasi_ref,
+    [], // args
+    [], // env
+    {
+      can_thread_spawn: true,
+      thread_spawn_worker_url: new URL("./thread_spawn.ts", import.meta.url)
+        .href,
+      thread_spawn_wasm: wasm,
+    },
+  );
+
+  await wasi.wait_worker_background_worker();
+
+  const inst = await WebAssembly.instantiate(wasm, {
+    env: {
+      memory: wasi.get_share_memory(),
+    },
+    wasi: wasi.wasiThreadImport,
+    wasi_snapshot_preview1: wasi.wasiImport,
+  });
+
+  wasi.start(
+    inst as unknown as {
+      exports: { memory: WebAssembly.Memory; _start: () => unknown };
+    },
+  );
+};
diff --git a/threads/examples/wasi_multi_threads_channel/channel.wasm b/threads/examples/wasi_multi_threads_channel/channel.wasm
diff --git a/threads/examples/wasi_multi_threads_channel/index.html b/threads/examples/wasi_multi_threads_channel/index.html
@@ -0,0 +1,12 @@
+<!DOCTYPE html>
+<html>
+<head>
+<meta charset="utf-8">
+
+<p id="note">
+  ####
+</p>
+
+<script type="module" src="./index.ts"></script>
+</body>
+</html>
diff --git a/threads/examples/wasi_multi_threads_channel/index.ts b/threads/examples/wasi_multi_threads_channel/index.ts
@@ -0,0 +1,15 @@
+import { OpenFile, File, ConsoleStdout } from "@bjorn3/browser_wasi_shim";
+import { WASIFarm } from "../../src";
+
+const farm = new WASIFarm(
+  new OpenFile(new File([])), // stdin
+  ConsoleStdout.lineBuffered((msg) => console.log(`[WASI stdout] ${msg}`)),
+  ConsoleStdout.lineBuffered((msg) => console.warn(`[WASI stderr] ${msg}`)),
+  [],
+);
+
+const worker = new Worker("./worker.ts", { type: "module" });
+
+worker.postMessage({
+  wasi_ref: farm.get_ref(),
+});
diff --git a/threads/examples/wasi_multi_threads_channel/main.rs b/threads/examples/wasi_multi_threads_channel/main.rs
@@ -0,0 +1,16 @@
+// https://doc.rust-lang.org/book/ch16-02-message-passing.html
+
+use std::sync::mpsc;
+use std::thread;
+
+fn main() {
+    let (tx, rx) = mpsc::channel();
+
+    thread::spawn(move || {
+        let val = String::from("hi");
+        tx.send(val).unwrap();
+    });
+
+    let received = rx.recv().unwrap();
+    println!("Got: {received}");
+}
diff --git a/threads/examples/wasi_multi_threads_channel/thread_spawn.ts b/threads/examples/wasi_multi_threads_channel/thread_spawn.ts
@@ -0,0 +1,5 @@
+import { thread_spawn_on_worker } from "../../src";
+
+self.onmessage = (event) => {
+  thread_spawn_on_worker(event.data);
+};
diff --git a/threads/examples/wasi_multi_threads_channel/worker.ts b/threads/examples/wasi_multi_threads_channel/worker.ts
@@ -0,0 +1,36 @@
+import { WASIFarmAnimal } from "../../src";
+
+self.onmessage = async (e) => {
+  const { wasi_ref } = e.data;
+
+  const wasm = await WebAssembly.compileStreaming(fetch("./channel.wasm"));
+
+  const wasi = new WASIFarmAnimal(
+    wasi_ref,
+    [], // args
+    [], // env
+    {
+      can_thread_spawn: true,
+      thread_spawn_worker_url: new URL("./thread_spawn.ts", import.meta.url)
+        .href,
+      // thread_spawn_worker_url: "./thread_spawn.ts",
+      thread_spawn_wasm: wasm,
+    },
+  );
+
+  await wasi.wait_worker_background_worker();
+
+  const inst = await WebAssembly.instantiate(wasm, {
+    env: {
+      memory: wasi.get_share_memory(),
+    },
+    wasi: wasi.wasiThreadImport,
+    wasi_snapshot_preview1: wasi.wasiImport,
+  });
+
+  wasi.start(
+    inst as unknown as {
+      exports: { memory: WebAssembly.Memory; _start: () => unknown };
+    },
+  );
+};
diff --git a/threads/examples/wasi_multi_threads_rustc/clang.ts b/threads/examples/wasi_multi_threads_rustc/clang.ts
@@ -0,0 +1,76 @@
+import { strace } from "@bjorn3/browser_wasi_shim";
+import { WASIFarmAnimal } from "../../src";
+import { SharedObject } from "@oligami/shared-object";
+
+let wasi: WASIFarmAnimal;
+let inst: {
+  exports: { memory: WebAssembly.Memory; _start: () => unknown };
+};
+let wasm: WebAssembly.Module;
+
+let shared_clang: SharedObject;
+let shared_tools: SharedObject;
+let shared_wasm_ld: SharedObject;
+
+globalThis.onmessage = async (e) => {
+  const { wasi_refs } = e.data;
+
+  if (wasi_refs) {
+    wasm = await WebAssembly.compileStreaming(
+      fetch("./rust_wasm/llvm-tools/llvm-opt.wasm"),
+    );
+
+    wasi = new WASIFarmAnimal(
+      wasi_refs,
+      ["llvm"], // args
+      [], // env
+    );
+
+    // Memory is rewritten at this time.
+    // inst = await WebAssembly.instantiate(wasm, {
+    // 	wasi_snapshot_preview1: wasi.wasiImport,
+    // });
+    inst = (await WebAssembly.instantiate(wasm, {
+      wasi_snapshot_preview1: strace(wasi.wasiImport, []),
+    })) as unknown as {
+      exports: { memory: WebAssembly.Memory; _start: () => unknown };
+    };
+
+    console.log("wasi.start, inst", inst);
+
+    const memory_reset = inst.exports.memory.buffer;
+    const memory_reset_view = new Uint8Array(memory_reset).slice();
+
+    shared_clang = new SharedObject((...args) => {
+      console.log("clang args", args);
+      // If I don't reset memory, I get some kind of error.
+      wasi.args = ["llvm", "clang", ...args];
+      const memory_view = new Uint8Array(inst.exports.memory.buffer);
+      memory_view.set(memory_reset_view);
+      wasi.start(inst);
+      console.log("clang wasi.start done");
+    }, "clang");
+
+    shared_tools = new SharedObject((...args) => {
+      console.log("tools args", args);
+      // If I don't reset memory, I get some kind of error.
+      wasi.args = ["llvm-tools", ...args];
+      const memory_view = new Uint8Array(inst.exports.memory.buffer);
+      memory_view.set(memory_reset_view);
+      wasi.start(inst);
+      console.log("tools wasi.start done");
+    }, "llvm-tools");
+
+    shared_wasm_ld = new SharedObject((...args) => {
+      console.log("wasm-ld args", args);
+      // If I don't reset memory, I get some kind of error.
+      wasi.args = ["llvm-tools", "wasm-ld", ...args];
+      const memory_view = new Uint8Array(inst.exports.memory.buffer);
+      memory_view.set(memory_reset_view);
+      wasi.start(inst);
+      console.log("wasm-ld wasi.start done");
+    }, "wasm-ld");
+
+    postMessage({ ready: true });
+  }
+};