Feat: beam pod (#970)

This adds the ability to run generic (not just python based) services in beta9. For example, you can run a golang webserver. For now, it's restricted to HTTP based applications (or anything that runs over HTTP, like websockets). The model for deployment is mostly the same as the rest of the platform, however deployments are by default unable to scale to zero - for now. There are two modes of operation: 1. Create a `Pod` programmatically: app.py ``` from beta9 import Pod pod = Pod( ports=[8080], entrypoint=["sh", "-c", "python -m http.server 8080"], secrets=["MY_SECRET"], ) pod.create() ``` 2. Via CLI: app.py ``` from beta9 import Pod pod = Pod( ports=[8080], entrypoint=["sh", "-c", "python -m http.server 8080"], secrets=["MY_SECRET"], ) ``` - To run a one-off container: `beta9 run app.py:pod` - To deploy a stateful, scalable set of containers (load balanced behind a common name): `beta9 deploy app.py:pod --name some-pod` --------- Co-authored-by: jsun-m <[email protected]> Co-authored-by: John <[email protected]> Co-authored-by: Luke Lombardi <[email protected]> Co-authored-by: Luke Lombardi <[email protected]> Co-authored-by: Luke Lombardi <[email protected]>
beam-cloud · Feb 21, 2025 · 846607c · 846607c
1 parent e116c2b
commit 846607c
Show file tree

Hide file tree

Showing 72 changed files with 4,798 additions and 1,658 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -112,5 +112,6 @@ jobs:
       - name: Verify Protobuf Codegen
         env:
           GO_PATH: ~/go
+          PROTOC_INCLUDE_PATH: /usr/local/include
         run: |
           make verify-protocol
diff --git a/.gitignore b/.gitignore
@@ -29,3 +29,4 @@ oci*
 okteto.yaml
 .vscode/launch.json
 pyrightconfig.json
+.go-version
diff --git a/Makefile b/Makefile
@@ -56,7 +56,7 @@ stop:
 
 protocol:
 	poetry install --directory ./sdk
-	poetry run --no-interaction --directory ./sdk bin/gen_proto.sh
+	poetry run --no-interaction --directory ./sdk bash -c "cd .. && ./bin/gen_proto.sh"
 
 verify-protocol:
 	./bin/verify_proto.sh

diff --git a/bin/gen_proto.sh b/bin/gen_proto.sh
@@ -12,45 +12,48 @@ protoc -I ./pkg/types -I ./pkg/gateway/ --go_out=./proto --go_opt=paths=source_r
 protoc -I ./pkg/types -I ./pkg/gateway/ --python_betterproto_beta9_out=./sdk/src/beta9/clients/ ./pkg/gateway/gateway.proto
 
 # Repository services
-protoc -I ./pkg/types -I ./pkg/gateway/services/repository/ --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative ./pkg/gateway/services/repository/container_repo.proto
-protoc -I ./pkg/types -I ./pkg/gateway/services/repository/ --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative ./pkg/gateway/services/repository/worker_repo.proto
+protoc -I $PROTOC_INCLUDE_PATH -I ./pkg/types -I ./pkg/gateway/services/repository/ --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative ./pkg/gateway/services/repository/container_repo.proto
+protoc -I $PROTOC_INCLUDE_PATH -I ./pkg/types -I ./pkg/gateway/services/repository/ --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative ./pkg/gateway/services/repository/worker_repo.proto
 
 # Generate code for abstractions
-protoc -I ./pkg/abstractions/image/ --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative ./pkg/abstractions/image/image.proto
-protoc -I ./pkg/abstractions/image/ --python_betterproto_beta9_out=./sdk/src/beta9/clients/ ./pkg/abstractions/image/image.proto
+protoc -I $PROTOC_INCLUDE_PATH -I ./pkg/abstractions/image/ --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative ./pkg/abstractions/image/image.proto
+protoc -I $PROTOC_INCLUDE_PATH -I ./pkg/abstractions/image/ --python_betterproto_beta9_out=./sdk/src/beta9/clients/ ./pkg/abstractions/image/image.proto
 
-protoc -I ./pkg/abstractions/map/ --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative ./pkg/abstractions/map/map.proto
-protoc -I ./pkg/abstractions/map/ --python_betterproto_beta9_out=./sdk/src/beta9/clients/ ./pkg/abstractions/map/map.proto
+protoc -I $PROTOC_INCLUDE_PATH -I ./pkg/abstractions/map/ --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative ./pkg/abstractions/map/map.proto
+protoc -I $PROTOC_INCLUDE_PATH -I ./pkg/abstractions/map/ --python_betterproto_beta9_out=./sdk/src/beta9/clients/ ./pkg/abstractions/map/map.proto
 
-protoc -I ./pkg/abstractions/function/ --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative ./pkg/abstractions/function/function.proto
-protoc -I ./pkg/abstractions/function/ --python_betterproto_beta9_out=./sdk/src/beta9/clients/ ./pkg/abstractions/function/function.proto
+protoc -I $PROTOC_INCLUDE_PATH -I ./pkg/abstractions/function/ --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative ./pkg/abstractions/function/function.proto
+protoc -I $PROTOC_INCLUDE_PATH -I ./pkg/abstractions/function/ --python_betterproto_beta9_out=./sdk/src/beta9/clients/ ./pkg/abstractions/function/function.proto
 
-protoc -I ./pkg/abstractions/queue/ --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative ./pkg/abstractions/queue/queue.proto
-protoc -I ./pkg/abstractions/queue/ --python_betterproto_beta9_out=./sdk/src/beta9/clients/ ./pkg/abstractions/queue/queue.proto
+protoc -I $PROTOC_INCLUDE_PATH -I ./pkg/abstractions/queue/ --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative ./pkg/abstractions/queue/queue.proto
+protoc -I $PROTOC_INCLUDE_PATH -I ./pkg/abstractions/queue/ --python_betterproto_beta9_out=./sdk/src/beta9/clients/ ./pkg/abstractions/queue/queue.proto
 
-protoc -I ./pkg/abstractions/volume/ --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative ./pkg/abstractions/volume/volume.proto
-protoc -I ./pkg/abstractions/volume/ --python_betterproto_beta9_out=./sdk/src/beta9/clients/ ./pkg/abstractions/volume/volume.proto
+protoc -I $PROTOC_INCLUDE_PATH -I ./pkg/abstractions/volume/ --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative ./pkg/abstractions/volume/volume.proto
+protoc -I $PROTOC_INCLUDE_PATH -I ./pkg/abstractions/volume/ --python_betterproto_beta9_out=./sdk/src/beta9/clients/ ./pkg/abstractions/volume/volume.proto
 
-protoc -I ./pkg/abstractions/taskqueue/ --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative ./pkg/abstractions/taskqueue/taskqueue.proto
-protoc -I ./pkg/abstractions/taskqueue/ --python_betterproto_beta9_out=./sdk/src/beta9/clients/ ./pkg/abstractions/taskqueue/taskqueue.proto
+protoc -I $PROTOC_INCLUDE_PATH -I ./pkg/abstractions/taskqueue/ --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative ./pkg/abstractions/taskqueue/taskqueue.proto
+protoc -I $PROTOC_INCLUDE_PATH -I ./pkg/abstractions/taskqueue/ --python_betterproto_beta9_out=./sdk/src/beta9/clients/ ./pkg/abstractions/taskqueue/taskqueue.proto
 
-protoc -I ./pkg/abstractions/endpoint/ --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative ./pkg/abstractions/endpoint/endpoint.proto
-protoc -I ./pkg/abstractions/endpoint/ --python_betterproto_beta9_out=./sdk/src/beta9/clients/ ./pkg/abstractions/endpoint/endpoint.proto
+protoc -I $PROTOC_INCLUDE_PATH -I ./pkg/abstractions/endpoint/ --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative ./pkg/abstractions/endpoint/endpoint.proto
+protoc -I $PROTOC_INCLUDE_PATH -I ./pkg/abstractions/endpoint/ --python_betterproto_beta9_out=./sdk/src/beta9/clients/ ./pkg/abstractions/endpoint/endpoint.proto
 
-protoc -I ./pkg/abstractions/container/ --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative ./pkg/abstractions/container/container.proto
-protoc -I ./pkg/abstractions/container/ --python_betterproto_beta9_out=./sdk/src/beta9/clients/ ./pkg/abstractions/container/container.proto
+protoc -I $PROTOC_INCLUDE_PATH -I ./pkg/abstractions/container/ --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative ./pkg/abstractions/container/container.proto
+protoc -I $PROTOC_INCLUDE_PATH -I ./pkg/abstractions/container/ --python_betterproto_beta9_out=./sdk/src/beta9/clients/ ./pkg/abstractions/container/container.proto
 
-protoc -I ./pkg/abstractions/output/ --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative ./pkg/abstractions/output/output.proto
-protoc -I ./pkg/abstractions/output/ --python_betterproto_beta9_out=./sdk/src/beta9/clients/ ./pkg/abstractions/output/output.proto
+protoc -I $PROTOC_INCLUDE_PATH -I ./pkg/abstractions/pod/ --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative ./pkg/abstractions/pod/pod.proto
+protoc -I $PROTOC_INCLUDE_PATH -I ./pkg/abstractions/pod/ --python_betterproto_beta9_out=./sdk/src/beta9/clients/ ./pkg/abstractions/pod/pod.proto
 
-protoc -I ./pkg/abstractions/secret/ --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative ./pkg/abstractions/secret/secret.proto
-protoc -I ./pkg/abstractions/secret/ --python_betterproto_beta9_out=./sdk/src/beta9/clients/ ./pkg/abstractions/secret/secret.proto
+protoc -I $PROTOC_INCLUDE_PATH -I ./pkg/abstractions/output/ --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative ./pkg/abstractions/output/output.proto
+protoc -I $PROTOC_INCLUDE_PATH -I ./pkg/abstractions/output/ --python_betterproto_beta9_out=./sdk/src/beta9/clients/ ./pkg/abstractions/output/output.proto
 
-protoc -I ./pkg/abstractions/experimental/signal/ --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative ./pkg/abstractions/experimental/signal/signal.proto
-protoc -I ./pkg/abstractions/experimental/signal/ --python_betterproto_beta9_out=./sdk/src/beta9/clients/ ./pkg/abstractions/experimental/signal/signal.proto
+protoc -I $PROTOC_INCLUDE_PATH -I ./pkg/abstractions/secret/ --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative ./pkg/abstractions/secret/secret.proto
+protoc -I $PROTOC_INCLUDE_PATH -I ./pkg/abstractions/secret/ --python_betterproto_beta9_out=./sdk/src/beta9/clients/ ./pkg/abstractions/secret/secret.proto
 
-protoc -I ./pkg/abstractions/experimental/bot/ --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative ./pkg/abstractions/experimental/bot/bot.proto
-protoc -I ./pkg/abstractions/experimental/bot/ --python_betterproto_beta9_out=./sdk/src/beta9/clients/ ./pkg/abstractions/experimental/bot/bot.proto
+protoc -I $PROTOC_INCLUDE_PATH -I ./pkg/abstractions/experimental/signal/ --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative ./pkg/abstractions/experimental/signal/signal.proto
+protoc -I $PROTOC_INCLUDE_PATH -I ./pkg/abstractions/experimental/signal/ --python_betterproto_beta9_out=./sdk/src/beta9/clients/ ./pkg/abstractions/experimental/signal/signal.proto
 
-protoc -I ./pkg/abstractions/shell/ --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative ./pkg/abstractions/shell/shell.proto
-protoc -I ./pkg/abstractions/shell/ --python_betterproto_beta9_out=./sdk/src/beta9/clients/ ./pkg/abstractions/shell/shell.proto
+protoc -I $PROTOC_INCLUDE_PATH -I ./pkg/abstractions/experimental/bot/ --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative ./pkg/abstractions/experimental/bot/bot.proto
+protoc -I $PROTOC_INCLUDE_PATH -I ./pkg/abstractions/experimental/bot/ --python_betterproto_beta9_out=./sdk/src/beta9/clients/ ./pkg/abstractions/experimental/bot/bot.proto
+
+protoc -I $PROTOC_INCLUDE_PATH -I ./pkg/abstractions/shell/ --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative ./pkg/abstractions/shell/shell.proto
+protoc -I $PROTOC_INCLUDE_PATH -I ./pkg/abstractions/shell/ --python_betterproto_beta9_out=./sdk/src/beta9/clients/ ./pkg/abstractions/shell/shell.proto