From 7bd9c758e7d000b44609b1794e5e66f31ca93bb0 Mon Sep 17 00:00:00 2001 From: Grant Gurvis Date: Wed, 19 Apr 2023 20:18:23 -0700 Subject: [PATCH] add filesync proto --- .vscode/settings.json | 3 + crates/proto/build.rs | 30 ++++-- crates/proto/src/lib.rs | 5 +- crates/proto/update.ts | 95 +++++++++++++++++++ .../buildkit/session/filesync/filesync.proto | 22 +++++ 5 files changed, 144 insertions(+), 11 deletions(-) create mode 100644 .vscode/settings.json create mode 100644 crates/proto/update.ts create mode 100644 crates/proto/vendor/github.com/moby/buildkit/session/filesync/filesync.proto diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..cbac569 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "deno.enable": true +} diff --git a/crates/proto/build.rs b/crates/proto/build.rs index aea0aba..4f83f7b 100644 --- a/crates/proto/build.rs +++ b/crates/proto/build.rs @@ -1,29 +1,41 @@ use std::{io::Result, path::PathBuf}; -const VENDOR_DIR: &str = "vendor"; const BUILDKIT_DIR: &str = "vendor/github.com/moby/buildkit"; fn main() -> Result<()> { println!("cargo:rerun-if-changed=build.rs"); + let out_dir = PathBuf::from(std::env::var("OUT_DIR").unwrap()); + + let includes = ["vendor", "vendor/github.com/tonistiigi/fsutil/types"]; let protos = [ format!("{BUILDKIT_DIR}/frontend/gateway/pb/gateway.proto"), format!("{BUILDKIT_DIR}/api/services/control/control.proto"), ]; - let includes = [VENDOR_DIR.into(), format!("{BUILDKIT_DIR}/vendor")]; tonic_build::configure() .build_server(false) .compile(&protos, &includes)?; - let protos = [ - format!("{BUILDKIT_DIR}/session/secrets/secrets.proto"), - format!("{BUILDKIT_DIR}/session/auth/auth.proto"), - ]; + for (session_type, pkg_name) in [ + ("auth", "moby.filesync.v1"), + ("filesync", "moby.filesync.v1"), + ("secrets", "moby.buildkit.secrets.v1"), + ] { + let protos = [format!( + "{BUILDKIT_DIR}/session/{session_type}/{session_type}.proto" + )]; - tonic_build::configure() - .build_client(false) - .compile(&protos, &includes)?; + tonic_build::configure() + .build_client(false) + .compile(&protos, &includes)?; + + // Move the generated files to a new location + let src = out_dir.join(format!("{pkg_name}.rs")); + let dest = out_dir.join(format!("{session_type}.rs")); + + std::fs::rename(src, dest).unwrap(); + } Ok(()) } diff --git a/crates/proto/src/lib.rs b/crates/proto/src/lib.rs index cfc8d78..ba82276 100644 --- a/crates/proto/src/lib.rs +++ b/crates/proto/src/lib.rs @@ -40,14 +40,15 @@ pub mod moby { pub mod secrets { pub mod v1 { - include!(concat!(env!("OUT_DIR"), "/moby.buildkit.secrets.v1.rs")); + include!(concat!(env!("OUT_DIR"), "/secrets.rs")); } } } pub mod filesync { pub mod v1 { - include!(concat!(env!("OUT_DIR"), "/moby.filesync.v1.rs")); + include!(concat!(env!("OUT_DIR"), "/auth.rs")); + include!(concat!(env!("OUT_DIR"), "/filesync.rs")); } } } diff --git a/crates/proto/update.ts b/crates/proto/update.ts new file mode 100644 index 0000000..6676abf --- /dev/null +++ b/crates/proto/update.ts @@ -0,0 +1,95 @@ +import { join } from "https://deno.land/std@0.184.0/path/mod.ts"; + +async function getLatestVersion(url: string) { + const res = await fetch(url); + const data = await res.json(); + const version = data[0].tag_name; + return version; +} + +const latestVersion = await getLatestVersion( + `https://api.github.com/repos/moby/buildkit/releases`, +); + +const githubDir = join("vendor", "github.com"); +const buildkitDir = join(githubDir, "moby", "buildkit"); + +console.log(`Updating buildkit to ${latestVersion}`); + +const recursiveUpdateBuildkit = async (dir: string) => { + for await (const dirEntry of Deno.readDir(dir)) { + if (dirEntry.isDirectory) { + await recursiveUpdateBuildkit(join(dir, dirEntry.name)); + } else if (dirEntry.isFile) { + const filePath = join(dir, dirEntry.name); + const buildkitPath = filePath.replace(buildkitDir, ""); + + const url = + `https://raw.githubusercontent.com/moby/buildkit/${latestVersion}/${buildkitPath}`; + + const fetchRes = await fetch(url); + + if (!fetchRes.ok) { + console.log( + `Failed to update moby/buildkit ${filePath} ${fetchRes.status} ${fetchRes.statusText}`, + ); + Deno.exit(1); + } + + console.log( + `Updating moby/buildkit ${filePath}`, + ); + + await Deno.writeTextFile(filePath, await fetchRes.text()); + } + } +}; + +await recursiveUpdateBuildkit(buildkitDir); + +const recursiveUpdateOther = async (repo: string, dir: string) => { + for await (const dirEntry of Deno.readDir(dir)) { + if (dirEntry.isDirectory) { + await recursiveUpdateOther(repo, join(dir, dirEntry.name)); + } else if (dirEntry.isFile) { + const filePath = join(dir, dirEntry.name); + const path = filePath.replace(join(githubDir, repo), ""); + + const url = + `https://raw.githubusercontent.com/moby/buildkit/${latestVersion}/vendor/github.com/${repo}/${path}`; + + const fetchRes = await fetch(url); + + if (!fetchRes.ok) { + console.log( + `Failed to update ${repo} ${filePath} ${fetchRes.status} ${fetchRes.statusText}`, + ); + Deno.exit(1); + } + + console.log( + `Updating ${repo} ${filePath}`, + ); + + await Deno.writeTextFile(filePath, await fetchRes.text()); + } + } +}; + +const repos = Array.from(Deno.readDirSync(githubDir)).flatMap((orgEntry) => { + if (orgEntry.isDirectory) { + return Array.from(Deno.readDirSync(join(githubDir, orgEntry.name))).map( + (repoEntry) => { + if (repoEntry.isDirectory) { + return join(orgEntry.name, repoEntry.name); + } + }, + ); + } +}).filter((repo) => repo !== undefined && repo !== "moby/buildkit"); + +for (const repo of repos) { + if (repo) { + await recursiveUpdateOther(repo, join(githubDir, repo)); + } +} diff --git a/crates/proto/vendor/github.com/moby/buildkit/session/filesync/filesync.proto b/crates/proto/vendor/github.com/moby/buildkit/session/filesync/filesync.proto new file mode 100644 index 0000000..9e39179 --- /dev/null +++ b/crates/proto/vendor/github.com/moby/buildkit/session/filesync/filesync.proto @@ -0,0 +1,22 @@ +syntax = "proto3"; + +package moby.filesync.v1; + +option go_package = "filesync"; + +import "github.com/tonistiigi/fsutil/types/wire.proto"; + +service FileSync{ + rpc DiffCopy(stream fsutil.types.Packet) returns (stream fsutil.types.Packet); + rpc TarStream(stream fsutil.types.Packet) returns (stream fsutil.types.Packet); +} + +service FileSend{ + rpc DiffCopy(stream BytesMessage) returns (stream BytesMessage); +} + + +// BytesMessage contains a chunk of byte data +message BytesMessage{ + bytes data = 1; +}