add filesync proto

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+  "deno.enable": true
+}

crates/proto/build.rs

@@ -1,29 +1,41 @@
 use std::{io::Result, path::PathBuf};
 
-const VENDOR_DIR: &str = "vendor";
 const BUILDKIT_DIR: &str = "vendor/github.com/moby/buildkit";
 
 fn main() -> Result<()> {
     println!("cargo:rerun-if-changed=build.rs");
+    let out_dir = PathBuf::from(std::env::var("OUT_DIR").unwrap());
+
+    let includes = ["vendor", "vendor/github.com/tonistiigi/fsutil/types"];
 
     let protos = [
         format!("{BUILDKIT_DIR}/frontend/gateway/pb/gateway.proto"),
         format!("{BUILDKIT_DIR}/api/services/control/control.proto"),
     ];
-    let includes = [VENDOR_DIR.into(), format!("{BUILDKIT_DIR}/vendor")];
 
     tonic_build::configure()
         .build_server(false)
         .compile(&protos, &includes)?;
 
-    let protos = [
-        format!("{BUILDKIT_DIR}/session/secrets/secrets.proto"),
-        format!("{BUILDKIT_DIR}/session/auth/auth.proto"),
-    ];
+    for (session_type, pkg_name) in [
+        ("auth", "moby.filesync.v1"),
+        ("filesync", "moby.filesync.v1"),
+        ("secrets", "moby.buildkit.secrets.v1"),
+    ] {
+        let protos = [format!(
+            "{BUILDKIT_DIR}/session/{session_type}/{session_type}.proto"
+        )];
 
-    tonic_build::configure()
-        .build_client(false)
-        .compile(&protos, &includes)?;
+        tonic_build::configure()
+            .build_client(false)
+            .compile(&protos, &includes)?;
+
+        // Move the generated files to a new location
+        let src = out_dir.join(format!("{pkg_name}.rs"));
+        let dest = out_dir.join(format!("{session_type}.rs"));
+
+        std::fs::rename(src, dest).unwrap();
+    }
 
     Ok(())
 }

crates/proto/src/lib.rs

@@ -40,14 +40,15 @@ pub mod moby {
 
         pub mod secrets {
             pub mod v1 {
-                include!(concat!(env!("OUT_DIR"), "/moby.buildkit.secrets.v1.rs"));
+                include!(concat!(env!("OUT_DIR"), "/secrets.rs"));
             }
         }
     }
 
     pub mod filesync {
         pub mod v1 {
-            include!(concat!(env!("OUT_DIR"), "/moby.filesync.v1.rs"));
+            include!(concat!(env!("OUT_DIR"), "/auth.rs"));
+            include!(concat!(env!("OUT_DIR"), "/filesync.rs"));
         }
     }
 }

crates/proto/update.ts

@@ -0,0 +1,95 @@
+import { join } from "https://deno.land/[email protected]/path/mod.ts";
+
+async function getLatestVersion(url: string) {
+  const res = await fetch(url);
+  const data = await res.json();
+  const version = data[0].tag_name;
+  return version;
+}
+
+const latestVersion = await getLatestVersion(
+  `https://api.github.com/repos/moby/buildkit/releases`,
+);
+
+const githubDir = join("vendor", "github.com");
+const buildkitDir = join(githubDir, "moby", "buildkit");
+
+console.log(`Updating buildkit to ${latestVersion}`);
+
+const recursiveUpdateBuildkit = async (dir: string) => {
+  for await (const dirEntry of Deno.readDir(dir)) {
+    if (dirEntry.isDirectory) {
+      await recursiveUpdateBuildkit(join(dir, dirEntry.name));
+    } else if (dirEntry.isFile) {
+      const filePath = join(dir, dirEntry.name);
+      const buildkitPath = filePath.replace(buildkitDir, "");
+
+      const url =
+        `https://raw.githubusercontent.com/moby/buildkit/${latestVersion}/${buildkitPath}`;
+
+      const fetchRes = await fetch(url);
+
+      if (!fetchRes.ok) {
+        console.log(
+          `Failed to update moby/buildkit ${filePath} ${fetchRes.status} ${fetchRes.statusText}`,
+        );
+        Deno.exit(1);
+      }
+
+      console.log(
+        `Updating moby/buildkit ${filePath}`,
+      );
+
+      await Deno.writeTextFile(filePath, await fetchRes.text());
+    }
+  }
+};
+
+await recursiveUpdateBuildkit(buildkitDir);
+
+const recursiveUpdateOther = async (repo: string, dir: string) => {
+  for await (const dirEntry of Deno.readDir(dir)) {
+    if (dirEntry.isDirectory) {
+      await recursiveUpdateOther(repo, join(dir, dirEntry.name));
+    } else if (dirEntry.isFile) {
+      const filePath = join(dir, dirEntry.name);
+      const path = filePath.replace(join(githubDir, repo), "");
+
+      const url =
+        `https://raw.githubusercontent.com/moby/buildkit/${latestVersion}/vendor/github.com/${repo}/${path}`;
+
+      const fetchRes = await fetch(url);
+
+      if (!fetchRes.ok) {
+        console.log(
+          `Failed to update ${repo} ${filePath} ${fetchRes.status} ${fetchRes.statusText}`,
+        );
+        Deno.exit(1);
+      }
+
+      console.log(
+        `Updating ${repo} ${filePath}`,
+      );
+
+      await Deno.writeTextFile(filePath, await fetchRes.text());
+    }
+  }
+};
+
+const repos = Array.from(Deno.readDirSync(githubDir)).flatMap((orgEntry) => {
+  if (orgEntry.isDirectory) {
+    return Array.from(Deno.readDirSync(join(githubDir, orgEntry.name))).map(
+      (repoEntry) => {
+        if (repoEntry.isDirectory) {
+          return join(orgEntry.name, repoEntry.name);
+        }
+      },
+    );
+  }
+}).filter((repo) => repo !== undefined && repo !== "moby/buildkit");
+
+for (const repo of repos) {
+  if (repo) {
+    await recursiveUpdateOther(repo, join(githubDir, repo));
+  }
+}