Artf/cleanup post shell (#4646)

Clean up. * Remove the inputs/outputs logic from the cloud publisher on the admin side and move it to the event handler on the artifact side * Remove the initialization sql bit and made hstore extension creation just a migration * Refactor the handler to be outside of the receiver in artifact events * Add adminclient into the servicecallhandler object * Set gizmo sqs config to false for consuming base64 and update the sqs processor to be roughly functional * Fixing lint issues * Add artifact configuration to the sandbox values file * Infra related updates * Remove the old python artifact service helm templates * update local sandbox configuration to support routing back out to local development endpoint Signed-off-by: Yee Hing Tong <[email protected]> Signed-off-by: Eduardo Apolinario <[email protected]> Co-authored-by: Eduardo Apolinario <[email protected]>
flyteorg · Dec 29, 2023 · c06527e · c06527e
1 parent f60ae81
commit c06527e
Show file tree

Hide file tree

Showing 80 changed files with 3,113 additions and 496 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -12,6 +12,7 @@ WORKDIR /flyteorg/build
 
 COPY datacatalog datacatalog
 COPY flyteadmin flyteadmin
+COPY flyteartifacts flyteartifacts
 COPY flytecopilot flytecopilot
 COPY flyteidl flyteidl
 COPY flyteplugins flyteplugins

diff --git a/Dockerfile.flyteartifacts b/Dockerfile.flyteartifacts
@@ -1,4 +1,4 @@
-FROM --platform=${BUILDPLATFORM} golang:1.19-alpine3.16 as builder
+FROM --platform=${BUILDPLATFORM} golang:1.21.5-alpine3.18 as builder
 
 ARG TARGETARCH
 ENV GOARCH "${TARGETARCH}"
@@ -21,7 +21,7 @@ COPY flytepropeller ../flytepropeller
 COPY flytestdlib ../flytestdlib
 
 # This 'linux_compile' target should compile binaries to the /artifacts directory
-# The main entrypoint should be compiled to /artifacts/flyteadmin
+# The main entrypoint should be compiled to /artifacts/flyteartifacts
 RUN make linux_compile
 
 # update the PATH to include the /artifacts directory

diff --git a/charts/flyte-sandbox/README.md b/charts/flyte-sandbox/README.md
@@ -11,7 +11,6 @@ A Helm chart for the Flyte local sandbox
 | file://../flyte-binary | flyte-binary | v0.1.10 |
 | https://charts.bitnami.com/bitnami | minio | 12.1.1 |
 | https://charts.bitnami.com/bitnami | postgresql | 12.1.9 |
-| https://charts.bitnami.com/bitnami | redis | 18.0.1 |
 | https://helm.twun.io/ | docker-registry | 2.2.2 |
 | https://kubernetes.github.io/dashboard/ | kubernetes-dashboard | 6.0.0 |
 
@@ -28,6 +27,12 @@ A Helm chart for the Flyte local sandbox
 | flyte-binary.clusterResourceTemplates.inlineConfigMap | string | `"{{ include \"flyte-sandbox.clusterResourceTemplates.inlineConfigMap\" . }}"` |  |
 | flyte-binary.configuration.database.host | string | `"{{ printf \"%s-postgresql\" .Release.Name | trunc 63 | trimSuffix \"-\" }}"` |  |
 | flyte-binary.configuration.database.password | string | `"postgres"` |  |
+| flyte-binary.configuration.inline.artifacts.host | string | `"localhost"` |  |
+| flyte-binary.configuration.inline.artifacts.insecure | bool | `true` |  |
+| flyte-binary.configuration.inline.artifacts.port | int | `50051` |  |
+| flyte-binary.configuration.inline.cloudEvents.cloudEventVersion | string | `"v2"` |  |
+| flyte-binary.configuration.inline.cloudEvents.enable | bool | `true` |  |
+| flyte-binary.configuration.inline.cloudEvents.type | string | `"sandbox"` |  |
 | flyte-binary.configuration.inline.plugins.k8s.default-env-vars[0].FLYTE_AWS_ENDPOINT | string | `"http://{{ printf \"%s-minio\" .Release.Name | trunc 63 | trimSuffix \"-\" }}.{{ .Release.Namespace }}:9000"` |  |
 | flyte-binary.configuration.inline.plugins.k8s.default-env-vars[1].FLYTE_AWS_ACCESS_KEY_ID | string | `"minio"` |  |
 | flyte-binary.configuration.inline.plugins.k8s.default-env-vars[2].FLYTE_AWS_SECRET_ACCESS_KEY | string | `"miniostorage"` |  |
@@ -101,13 +106,6 @@ A Helm chart for the Flyte local sandbox
 | postgresql.volumePermissions.enabled | bool | `true` |  |
 | postgresql.volumePermissions.image.pullPolicy | string | `"Never"` |  |
 | postgresql.volumePermissions.image.tag | string | `"sandbox"` |  |
-| redis.auth.enabled | bool | `false` |  |
-| redis.enabled | bool | `true` |  |
-| redis.image.pullPolicy | string | `"Never"` |  |
-| redis.image.tag | string | `"sandbox"` |  |
-| redis.master.service.nodePorts.redis | int | `30004` |  |
-| redis.master.service.type | string | `"NodePort"` |  |
-| redis.replica.replicaCount | int | `0` |  |
 | sandbox.buildkit.enabled | bool | `true` |  |
 | sandbox.buildkit.image.pullPolicy | string | `"Never"` |  |
 | sandbox.buildkit.image.repository | string | `"moby/buildkit"` |  |

diff --git a/charts/flyte-sandbox/templates/artifacts/deployment.yaml b/charts/flyte-sandbox/templates/artifacts/deployment.yaml
diff --git a/charts/flyte-sandbox/templates/artifacts/service.yaml b/charts/flyte-sandbox/templates/artifacts/service.yaml
diff --git a/charts/flyte-sandbox/templates/local/endpoint.yaml b/charts/flyte-sandbox/templates/local/endpoint.yaml
@@ -19,4 +19,10 @@ subsets:
       - name: webhook
         port: 9443
         protocol: TCP
+      - name: artifact-http
+        port: 50050
+        protocol: TCP
+      - name: artifact-grpc
+        port: 50051
+        protocol: TCP
 {{- end }}
diff --git a/charts/flyte-sandbox/templates/local/service.yaml b/charts/flyte-sandbox/templates/local/service.yaml
@@ -18,4 +18,10 @@ spec:
     - name: webhook
       port: 9443
       protocol: TCP
+    - name: artifact-http
+      port: 50050
+      protocol: TCP
+    - name: artifact-grpc
+      port: 50051
+      protocol: TCP
 {{- end }}
diff --git a/charts/flyte-sandbox/templates/proxy/configmap.yaml b/charts/flyte-sandbox/templates/proxy/configmap.yaml
@@ -84,6 +84,10 @@ data:
                                                   prefix: "/v1"
                                               route:
                                                   cluster: flyte
+                                            - match:
+                                                  prefix: "/artifacts/api"
+                                              route:
+                                                  cluster: artifact_http
                                             - match:
                                                   prefix: "/flyteidl.service.AdminService"
                                               route:
@@ -111,7 +115,7 @@ data:
                                             - match:
                                                   prefix: "/flyteidl.artifact.ArtifactRegistry"
                                               route:
-                                                  cluster: artifact
+                                                  cluster: artifact_grpc
                                             {{- end }}
                                             {{- if index .Values "kubernetes-dashboard" "enabled" }}
                                             - match:
@@ -176,6 +180,34 @@ data:
                                           address: {{ include "flyte-sandbox.localHeadlessService" . }}
                                           {{- end }}
                                           port_value: 8089
+            - name: artifact_http
+              connect_timeout: 0.25s
+              type: STRICT_DNS
+              lb_policy: ROUND_ROBIN
+              http2_protocol_options: {}
+              load_assignment:
+                  cluster_name: artifact_http
+                  endpoints:
+                      - lb_endpoints:
+                            - endpoint:
+                                  address:
+                                      socket_address:
+                                          address: flyte-sandbox-local
+                                          port_value: 50050
+            - name: artifact_grpc
+              connect_timeout: 0.25s
+              type: STRICT_DNS
+              lb_policy: ROUND_ROBIN
+              http2_protocol_options: {}
+              load_assignment:
+                  cluster_name: artifact_grpc
+                  endpoints:
+                      - lb_endpoints:
+                            - endpoint:
+                                  address:
+                                      socket_address:
+                                          address: flyte-sandbox-local
+                                          port_value: 50051
             {{- end }}
             {{- if index .Values "kubernetes-dashboard" "enabled" }}
             - name: kubernetes-dashboard
@@ -207,19 +239,4 @@ data:
                                           address: {{ .Release.Name }}-minio
                                           port_value: 9001
             {{- end }}
-            - name: artifact
-              connect_timeout: 0.25s
-              type: STRICT_DNS
-              lb_policy: ROUND_ROBIN
-              http2_protocol_options: {}
-              load_assignment:
-                  cluster_name: artifact
-                  endpoints:
-                      - lb_endpoints:
-                            - endpoint:
-                                  address:
-                                      socket_address:
-                                          address: artifact-service
-                                          port_value: 50051
-
 {{- end }}
diff --git a/charts/flyte-sandbox/values.yaml b/charts/flyte-sandbox/values.yaml
@@ -51,10 +51,28 @@ flyte-binary:
         enable: true
         cloudEventVersion: v2
         type: sandbox
-      artifacts:
-        host: localhost
-        port: 50051
-        insecure: true
+      artifactsServer:
+        artifactBlobStoreConfig:
+          type: stow
+          stow:
+            kind: s3
+            config:
+              disable_ssl: true
+              v2_signing: true
+              endpoint: http://localhost:30002
+              auth_type: accesskey
+              access_key_id: minio
+              secret_key: miniostorage
+        artifactDatabaseConfig:
+          postgres:
+            username: postgres
+            password: postgres
+            host: '{{ printf "%s-postgresql" .Release.Name | trunc 63 | trimSuffix "-" }}'
+            port: 5432
+            dbname: artifacts
+            options: "sslmode=disable"
+      artifactsProcessor:
+        cloudProvider: Sandbox
       storage:
         signedURL:
           stowConfigOverride:

diff --git a/cmd/single/start.go b/cmd/single/start.go
@@ -75,9 +75,8 @@ func startArtifact(ctx context.Context, cfg Artifacts) error {
 	// Roughly copies main/NewMigrateCmd
 	logger.Infof(ctx, "Artifacts: running database migrations if any...")
 	migs := artifactsServer.GetMigrations(ctx)
-	initializationSql := "create extension if not exists hstore;"
 	dbConfig := artifactsServer.GetDbConfig()
-	err := database.Migrate(context.Background(), dbConfig, migs, initializationSql)
+	err := database.Migrate(context.Background(), dbConfig, migs)
 	if err != nil {
 		logger.Errorf(ctx, "Failed to run Artifacts database migrations. Error: %v", err)
 		return err

diff --git a/datacatalog/go.mod b/datacatalog/go.mod
@@ -132,7 +132,7 @@ require (
 	k8s.io/klog/v2 v2.100.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect
 	k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect
-	sigs.k8s.io/controller-runtime v0.0.0-00010101000000-000000000000 // indirect
+	sigs.k8s.io/controller-runtime v0.16.2 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect

diff --git a/docker/sandbox-bundled/manifests/complete-agent.yaml b/docker/sandbox-bundled/manifests/complete-agent.yaml
@@ -508,6 +508,32 @@ data:
           auth_type: accesskey
       container: my-s3-bucket
   100-inline-config.yaml: |
+    artifactsProcessor:
+      cloudProvider: Sandbox
+    artifactsServer:
+      artifactBlobStoreConfig:
+        stow:
+          config:
+            access_key_id: minio
+            auth_type: accesskey
+            disable_ssl: true
+            endpoint: http://localhost:30002
+            secret_key: miniostorage
+            v2_signing: true
+          kind: s3
+        type: stow
+      artifactDatabaseConfig:
+        postgres:
+          dbname: artifacts
+          host: 'flyte-sandbox-postgresql'
+          options: sslmode=disable
+          password: postgres
+          port: 5432
+          username: postgres
+    cloudEvents:
+      cloudEventVersion: v2
+      enable: true
+      type: sandbox
     plugins:
       k8s:
         default-env-vars:
@@ -666,6 +692,10 @@ data:
                                                   prefix: "/v1"
                                               route:
                                                   cluster: flyte
+                                            - match:
+                                                  prefix: "/artifacts/api"
+                                              route:
+                                                  cluster: artifact_http
                                             - match:
                                                   prefix: "/flyteidl.service.AdminService"
                                               route:
@@ -690,6 +720,10 @@ data:
                                                   prefix: "/flyteidl.service.SignalService"
                                               route:
                                                   cluster: flyte_grpc
+                                            - match:
+                                                  prefix: "/flyteidl.artifact.ArtifactRegistry"
+                                              route:
+                                                  cluster: artifact_grpc
                                             - match:
                                                   path: "/kubernetes-dashboard"
                                               redirect:
@@ -740,6 +774,34 @@ data:
                                       socket_address:
                                           address: flyte-sandbox-grpc
                                           port_value: 8089
+            - name: artifact_http
+              connect_timeout: 0.25s
+              type: STRICT_DNS
+              lb_policy: ROUND_ROBIN
+              http2_protocol_options: {}
+              load_assignment:
+                  cluster_name: artifact_http
+                  endpoints:
+                      - lb_endpoints:
+                            - endpoint:
+                                  address:
+                                      socket_address:
+                                          address: flyte-sandbox-local
+                                          port_value: 50050
+            - name: artifact_grpc
+              connect_timeout: 0.25s
+              type: STRICT_DNS
+              lb_policy: ROUND_ROBIN
+              http2_protocol_options: {}
+              load_assignment:
+                  cluster_name: artifact_grpc
+                  endpoints:
+                      - lb_endpoints:
+                            - endpoint:
+                                  address:
+                                      socket_address:
+                                          address: flyte-sandbox-local
+                                          port_value: 50051
             - name: kubernetes-dashboard
               connect_timeout: 0.25s
               type: STRICT_DNS
@@ -816,7 +878,7 @@ type: Opaque
 ---
 apiVersion: v1
 data:
-  haSharedSecret: SlF2M0tXNXdGTUpHZzdvNg==
+  haSharedSecret: YnZWQm1tMm9ETTY3enZUcQ==
   proxyPassword: ""
   proxyUsername: ""
 kind: Secret
@@ -1246,7 +1308,7 @@ spec:
     metadata:
       annotations:
         checksum/cluster-resource-templates: 6fd9b172465e3089fcc59f738b92b8dc4d8939360c19de8ee65f68b0e7422035
-        checksum/configuration: 9cb50a36963e1d23a3b500b148de3407c1fcc1b65da0f7da8b038cfd35c97fca
+        checksum/configuration: c3f03f3b03d7909e7fac95f3490d43d9e50cf4759a67ac38d4c1992f08f64ce3
         checksum/configuration-secret: 09216ffaa3d29e14f88b1f30af580d02a2a5e014de4d750b7f275cc07ed4e914
       labels:
         app.kubernetes.io/component: flyte-binary
@@ -1411,7 +1473,7 @@ spec:
     metadata:
       annotations:
         checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81
-        checksum/secret: b76914839daf95717da7a8ef31769c3b97f6c3c46a09776e5d0f8cc3fe9b1e9a
+        checksum/secret: db210d75941d85245039373f167036040860dde6eb2db965c9a482e496483240
       labels:
         app: docker-registry
         release: flyte-sandbox