Linting_tests

.github/workflows/golangci-lint.yml

@@ -0,0 +1,30 @@
+name: golangci-lint
+on:
+  push:
+    branches:
+      - main
+      - master
+  pull_request:
+
+permissions:
+  contents: read
+  # Optional: allow read access to pull request. Use with `only-new-issues` option.
+  pull-requests: read
+
+jobs:
+  golangci:
+    strategy:
+      matrix:
+        go: [stable]
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    name: lint
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version: ${{ matrix.go }}
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v6
+        with:
+          version: v1.60

.github/workflows/test.yml → .github/workflows/langTest.yml

@@ -34,4 +34,4 @@ jobs:
         run: ./test/test.sh rootless
 
       - name: Test Daemon
-        run: ./test/test.sh client
+        run: ./test/test.sh client

.github/workflows/startuptest.yml

@@ -0,0 +1,38 @@
+name: Build and Test MetaCall Builder for startup flow
+# Seperate because the space is limited in the github actions
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - "v*.*.*"
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      # TODO:
+      # - name: Lint
+      #   run: docker run --rm -v $(pwd):/app -w /app golangci/golangci-lint:v1.59.1 golangci-lint run -v --enable-all
+
+      - name: Build
+        run: docker compose build
+
+      - name: Test Binary
+        env:
+          BUILDER_ARGS: "runtime rb"
+        run: docker compose up --exit-code-from binary binary
+
+      - name: Test Startup-Rootless
+        run: ./test/test.sh rootless startup
+
+      - name: Test Startup-Daemon
+        run: ./test/test.sh client startup

Dockerfile

@@ -32,9 +32,13 @@ FROM moby/buildkit:master-rootless AS builder_rootless
 
 COPY --from=builder_binary --chown=user:user /builder /home/user/builder
 
+# TO-DO : Replace with copying from scripts locally
 RUN printf '#!/bin/sh\n\
 export BUILDKITD_FLAGS=--oci-worker-no-process-sandbox\n\
-/home/user/builder $@ | buildctl-daemonless.sh build --output type=image,name=registry:5000/metacall/builder_output,push=true,registry.insecure=true\n'\
+/home/user/builder $@ | buildctl-daemonless.sh build \
+--export-cache type=registry,ref=${EXPORT_REGISTRY},registry.insecure=true \
+--import-cache type=registry,ref=${IMPORT_REGISTRY},registry.insecure=true \
+--output type=image,name=registry:5000/metacall/builder_output_$1,push=true,registry.insecure=true\n'\
 >> /home/user/builder.sh \
     && chmod 700 /home/user/builder.sh \
     && chmod 700 /home/user/builder
@@ -46,8 +50,12 @@ COPY --from=builder_binary --chown=root:root /builder /home/builder
 
 RUN apk add --no-cache docker
 
+# TO-DO : Replace with copying from scripts locally
 RUN printf '#!/bin/sh\n\
-/home/builder $@ | buildctl --addr="docker-container://metacall_builder_buildkit" build --output type=image,name=registry:5000/metacall/builder_output,push=true,registry.insecure=true\n'\
+/home/builder $@ | buildctl --addr="docker-container://metacall_builder_buildkit" build \
+--export-cache type=registry,ref=${EXPORT_REGISTRY},registry.insecure=true \
+--import-cache type=registry,ref=${IMPORT_REGISTRY},registry.insecure=true \
+--output type=image,name=registry:5000/metacall/builder_output_$1,push=true,registry.insecure=true\n'\
 >> /home/builder.sh \
     && chmod 700 /home/builder.sh \
     && chmod 700 /home/builder

cmd/builder/flags.go

@@ -6,8 +6,13 @@ import (
 
 type RuntimeImageFlags struct {
 	MetacallCli bool
+	Startup bool
 }
 
 func (i *RuntimeImageFlags) Set(cmd *cobra.Command) {
 	cmd.Flags().BoolVar(&i.MetacallCli, "cli", false, "set to also get metacall cli in the runtime image")
 }
+
+func (i *RuntimeImageFlags) SetStartup(cmd *cobra.Command) {
+	cmd.Flags().BoolVar(&i.Startup, "startup", false, "startup flag to be used for building image with all languages")
+}

cmd/builder/runtime.go

@@ -21,15 +21,21 @@ func NewRuntimeCmd(o *RuntimeOptions) *cobra.Command {
 		Use:   "runtime",
 		Short: "Build runtime image for MetaCall",
 		RunE: func(cmd *cobra.Command, args []string) error {
+
 			if o.RuntimeImageFlags.MetacallCli {
 				args = append(args, "node")
 			}
+
+			if o.RuntimeImageFlags.Startup {
+				args = staging.GetAllLanguages()
+			}
+
 			base := cmd.Context().Value(baseKey{}).(llb.State)
 
 			devBaseLang := staging.DevBase(base, branch, args)
 			devImage := staging.MergeStates(devBaseLang)
 
-			runtimeLang:= staging.RuntimeBase(base, branch, args)
+			runtimeLang := staging.RuntimeBase(base, branch, args)
 			runtimeImage := staging.MergeStates(runtimeLang)
 
 			if o.RuntimeImageFlags.MetacallCli {
@@ -40,14 +46,14 @@ func NewRuntimeCmd(o *RuntimeOptions) *cobra.Command {
 			if err != nil {
 				return err
 			}
-
 			cmd.SetContext(context.WithValue(cmd.Context(), finalKey{}, runtime))
 			return nil
 
 		},
 		Example: `"builder runtime -b develop --cli nodejs typescript go rust wasm java c cobol"`,
 	}
 	o.RuntimeImageFlags.Set(cmd)
+	o.RuntimeImageFlags.SetStartup(cmd)
 
 	return cmd
 }

docker-compose.yml

@@ -1,8 +1,5 @@
 version: "3.7"
 
-# volumes:
-#   buildkit:
-
 services:
   binary:
     image: metacall/builder_binary
@@ -21,6 +18,10 @@ services:
       dockerfile: Dockerfile
       target: builder_rootless
     entrypoint: /home/user/builder.sh ${BUILDER_ARGS:-}
+    # entrypoint:  /bin/sh -c "sleep infinity"
+    environment:
+      IMPORT_REGISTRY: ${IMPORT_REGISTRY:-registry:5000/metacall/builder_startup} # Default won't be needed once we copy the script
+      EXPORT_REGISTRY: ${EXPORT_REGISTRY:-registry:5000/metacall/builder_dump}
     security_opt:
       - "seccomp=unconfined"
       - "apparmor=unconfined"
@@ -37,6 +38,9 @@ services:
       dockerfile: Dockerfile
       target: builder_client
     entrypoint: /home/builder.sh ${BUILDER_ARGS:-}
+    environment:
+      IMPORT_REGISTRY: ${IMPORT_REGISTRY:-testing}
+      EXPORT_REGISTRY: ${EXPORT_REGISTRY:-testing}
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
       # - buildkit:/run/user/1000/buildkit/buildkitd.sock
@@ -51,10 +55,6 @@ services:
     container_name: metacall_builder_buildkit
     restart: always
     privileged: true
-    # volumes:
-    #   - /var/run/docker.sock:/var/run/docker.sock
-    #   - /run/containerd/containerd.sock:/run/containerd/containerd.sock
-    #   - buildkit:/run/buildkit/buildkitd.sock
     healthcheck:
       test: buildctl debug workers
       interval: 10s

pkg/env/env.go

@@ -137,8 +137,8 @@ func (e Env) MetaCallBuild(arg string) Env {
 }
 
 func (e Env) MetacallRuntime(arg string) Env {
-	e.state = e.state.Run(llb.Shlexf("bash /usr/local/metacall/tools/metacall-runtime.sh base ports clean %v", arg)).
-		Root() // TODO: Add backtrace
+	e.state = e.state.Run(llb.Shlexf("bash /usr/local/metacall/tools/metacall-runtime.sh base backtrace ports clean %v", arg)).
+		Root()
 
 	return e
 }

pkg/staging/helper.go

@@ -41,22 +41,22 @@ func copy(src llb.State, srcPath string, dest llb.State, destPath string) llb.St
 	}))
 }
 
-func copyForStates(src llb.State, dst llb.State, srcpaths []string, dstpath string) llb.State {
-	return dst.With(
-		copyMultiple(src, srcpaths, dstpath),
-	)
-}
-
-func copyMultiple(src llb.State, srcPaths []string, destPath string) llb.StateOption {
-	var stateOptions []llb.StateOption
-	for _, srcPath := range srcPaths {
-		stateOptions = append(stateOptions, copyFrom(src, srcPath, destPath))
-	}
-
-	return func(s llb.State) llb.State {
-		for _, stateOption := range stateOptions {
-			s = stateOption(s)
-		}
-		return s
-	}
-}
+// Functions which might be useful later, left as comments here : 
+// func copyForStates(src llb.State, dst llb.State, srcpaths []string, dstpath string) llb.State {
+// 	return dst.With(
+// 		copyMultiple(src, srcpaths, dstpath),
+// 	)
+// }
+
+// func copyMultiple(src llb.State, srcPaths []string, destPath string) llb.StateOption {
+// 	var stateOptions []llb.StateOption
+// 	for _, srcPath := range srcPaths {
+// 		stateOptions = append(stateOptions, copyFrom(src, srcPath, destPath))
+// 	}
+// 	return func(s llb.State) llb.State {
+// 		for _, stateOption := range stateOptions {
+// 			s = stateOption(s)
+// 		}
+// 		return s
+// 	}
+// }

pkg/staging/stages.go

@@ -6,9 +6,8 @@ import (
 )
 
 var languageMap = map[string]string{
-	"base": "base",
-	"py":   "python",
-	"rb":   "ruby",
+	"py": "python",
+	"rb": "ruby",
 	// "netcore":   "netcore",
 	// "netcore2":  "netcore2",
 	// "netcore5":  "netcore5",
@@ -22,15 +21,15 @@ var languageMap = map[string]string{
 	// "v8rep52":   "v8rep52",
 	// "v8rep51":   "v8rep51",
 	"node": "nodejs",
-	"ts":   "typescript",
+	// "ts":   "typescript",
 	// "file":       "file",
 	// "rpc":        "rpc",
-	"wasm": "wasm",
-	"java": "java",
-	"c":    "c",
-	"cob":  "cobol",
-	"go":   "go",
-	"rs":   "rust",
+	// "wasm": "wasm",
+	// "java": "java",
+	// // "c":    "c",
+	// "cob":  "cobol",
+	// "go":   "go",
+	// "rs":   "rust",
 	// "backtrace"	: "backtrace",
 	// "sandbox"	: "sandbox",
 }
@@ -135,3 +134,11 @@ func MergeStates(individualLangStates map[string]llb.State) llb.State {
 	}
 	return llb.Merge(states)
 }
+
+func GetAllLanguages() []string {
+	langs := []string{}
+	for lang := range languageMap {
+		langs = append(langs, lang)
+	}
+	return langs
+}

scripts/builder.sh

@@ -0,0 +1,20 @@
+#!/bin/sh\n\
+export BUILDKITD_FLAGS=--oci-worker-no-process-sandbox\n\
+
+if [ -z "$EXPORT_REGISTRY" ]; then
+    EXPORT_REGISTRY="registry:5000/metacall/builder_cache"
+fi
+
+if [ -z "$IMPORT_REGISTRY" ]; then
+    IMPORT_REGISTRY="registry:5000/metacall/builder_startup"
+fi
+
+if [ -z "$BUILDER_BINARY" ]; then
+    BUILDER_BINARY="/home/user/builder"
+fi
+
+
+${BUILDER_BINARY} $@ | buildctl-daemonless.sh build \
+--export-cache type=registry,ref=${EXPORT_REGISTRY},registry.insecure=true \
+--import-cache type=registry,ref=${IMPORT_REGISTRY},registry.insecure=true \
+--output type=image,name=registry:5000/metacall/builder_output,push=true,registry.insecure=true\n'\