Merged master

Signed-off-by: Kevin Su <[email protected]>

.github/workflows/flyteidl-buf-publish.yml

@@ -3,6 +3,7 @@ name: Publish flyteidl Buf Package
 on:
   push:
     branches:
+      - artifacts
       - master
     paths:
       - 'flyteidl/**'

charts/flyte-core/README.md

@@ -176,6 +176,11 @@ helm install gateway bitnami/contour -n flyte
 | flyteadmin.serviceAccount.create | bool | `true` | Should a service account be created for flyteadmin |
 | flyteadmin.serviceAccount.createClusterRole | bool | `true` | Should a ClusterRole be created for Flyteadmin |
 | flyteadmin.serviceAccount.imagePullSecrets | list | `[]` | ImagePullSecrets to automatically assign to the service account |
+| flyteadmin.serviceMonitor | object | `{"enabled":false,"interval":"60s","labels":{},"scrapeTimeout":"30s"}` | Settings for flyteadmin service monitor |
+| flyteadmin.serviceMonitor.enabled | bool | `false` | If enabled create the flyteadmin service monitor |
+| flyteadmin.serviceMonitor.interval | string | `"60s"` | Sets the interval at which metrics will be scraped by prometheus |
+| flyteadmin.serviceMonitor.labels | object | `{}` | Sets the labels for the service monitor which are required by the prometheus to auto-detect the service monitor and start scrapping the metrics |
+| flyteadmin.serviceMonitor.scrapeTimeout | string | `"30s"` | Sets the timeout after which request to scrape metrics will time out |
 | flyteadmin.tolerations | list | `[]` | tolerations for Flyteadmin deployment |
 | flyteagent.enabled | bool | `false` |  |
 | flyteconsole.affinity | object | `{}` | affinity for Flyteconsole deployment |
@@ -211,10 +216,17 @@ helm install gateway bitnami/contour -n flyte
 | flytepropeller.priorityClassName | string | `""` | Sets priorityClassName for propeller pod(s). |
 | flytepropeller.replicaCount | int | `1` | Replicas count for Flytepropeller deployment |
 | flytepropeller.resources | object | `{"limits":{"cpu":"200m","ephemeral-storage":"100Mi","memory":"200Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"100Mi"}}` | Default resources requests and limits for Flytepropeller deployment |
+| flytepropeller.service | object | `{"enabled":false}` | Settings for flytepropeller service |
+| flytepropeller.service.enabled | bool | `false` | If enabled create the flytepropeller service |
 | flytepropeller.serviceAccount | object | `{"annotations":{},"create":true,"imagePullSecrets":[]}` | Configuration for service accounts for FlytePropeller |
 | flytepropeller.serviceAccount.annotations | object | `{}` | Annotations for ServiceAccount attached to FlytePropeller pods |
 | flytepropeller.serviceAccount.create | bool | `true` | Should a service account be created for FlytePropeller |
 | flytepropeller.serviceAccount.imagePullSecrets | list | `[]` | ImagePullSecrets to automatically assign to the service account |
+| flytepropeller.serviceMonitor | object | `{"enabled":false,"interval":"60s","labels":{},"scrapeTimeout":"30s"}` | Settings for flytepropeller service monitor |
+| flytepropeller.serviceMonitor.enabled | bool | `false` | If enabled create the flyetepropeller service monitor |
+| flytepropeller.serviceMonitor.interval | string | `"60s"` | Sets the interval at which metrics will be scraped by prometheus |
+| flytepropeller.serviceMonitor.labels | object | `{}` | Sets the labels for the service monitor which are required by the prometheus to auto-detect the service monitor and start scrapping the metrics |
+| flytepropeller.serviceMonitor.scrapeTimeout | string | `"30s"` | Sets the timeout after which request to scrape metrics will time out |
 | flytepropeller.terminationMessagePolicy | string | `"FallbackToLogsOnError"` | Error reporting |
 | flytepropeller.tolerations | list | `[]` | tolerations for Flytepropeller deployment |
 | flytescheduler.additionalContainers | list | `[]` | Appends additional containers to the deployment spec. May include template values. |

charts/flyte-core/templates/admin/service-monitor.yaml

@@ -0,0 +1,19 @@
+{{- if and .Values.flyteadmin.serviceMonitor.enabled .Values.flyteadmin.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ template "flyteadmin.name" . }}
+  namespace: {{ template "flyte.namespace" . }}
+  labels:
+  {{- with .Values.flyteadmin.serviceMonitor.labels }}
+  {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  endpoints:
+    - interval: {{ .Values.flyteadmin.serviceMonitor.interval }}
+      port: http-metrics
+      path: /metrics
+      scrapeTimeout: {{ .Values.flyteadmin.serviceMonitor.scrapeTimeout }}
+  selector:
+    matchLabels: {{ include "flyteadmin.selectorLabels" . | nindent 6 }}
+{{- end }}

charts/flyte-core/templates/propeller/service-monitor.yaml

@@ -0,0 +1,19 @@
+{{- if and .Values.flytepropeller.service.enabled .Values.flytepropeller.serviceMonitor.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  namespace: {{ template "flyte.namespace" . }}
+  name: {{ template "flytepropeller.name" . }}
+  labels:
+  {{- with .Values.flytepropeller.serviceMonitor.labels }}
+  {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  endpoints:
+    - interval: {{ .Values.flytepropeller.serviceMonitor.interval }}
+      port: http-metrics
+      path: /metrics
+      scrapeTimeout: {{ .Values.flytepropeller.serviceMonitor.scrapeTimeout }}
+  selector:
+    matchLabels: {{ include "flytepropeller.selectorLabels" . | nindent 6 }}
+{{- end }}

charts/flyte-core/templates/propeller/service.yaml

@@ -0,0 +1,15 @@
+{{- if and .Values.flytepropeller.enabled .Values.flytepropeller.service.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  namespace: {{ template "flyte.namespace" . }}
+  name: {{ template "flytepropeller.name" . }}
+  labels: {{ include "flytepropeller.labels" . | nindent 4 }}
+spec:
+  type: ClusterIP
+  ports:
+    - name: http-metrics
+      protocol: TCP
+      port: 10254
+  selector: {{ include "flytepropeller.selectorLabels" . | nindent 4 }}
+{{- end }}

charts/flyte-core/values-eks.yaml

@@ -23,6 +23,9 @@ flyteadmin:
     annotations:
       eks.amazonaws.com/role-arn: arn:aws:iam::{{ .Values.userSettings.accountNumber }}:role/iam-role-flyte
 
+  serviceMonitor:
+    enabled: false
+
   resources:
     limits:
       ephemeral-storage: 200Mi
@@ -78,6 +81,10 @@ datacatalog:
 flytepropeller:
   replicaCount: 2
   manager: false
+  serviceMonitor:
+    enabled: false
+  service:
+    enabled: false
   serviceAccount:
     # -- If the service account is created by you, make this false
     create: true

charts/flyte-core/values-gcp.yaml

@@ -14,6 +14,8 @@ userSettings:
 
 flyteadmin:
   replicaCount: 1
+  serviceMonitor:
+    enabled: false
   serviceAccount:
     # -- If the service account is created by you, make this false, else a new service account will be created and the flyteadmin role will be added
     # you can change the name of this role
@@ -82,6 +84,10 @@ datacatalog:
 flytepropeller:
   replicaCount: 1
   manager: false
+  serviceMonitor:
+    enabled: false
+  service:
+    enabled: false
   serviceAccount:
     # -- If the service account is created by you, make this false, else a new service account will be created and the iam-role-flyte will be added
     # you can change the name of this role

charts/flyte-core/values-keycloak-idp-flyteclients-without-browser.yaml

@@ -11,6 +11,8 @@
 
 flyteadmin:
   enabled: true
+  serviceMonitor:
+    enabled: false
   # -- Replicas count for Flyteadmin deployment
   replicaCount: 1
   image:

charts/flyte-core/values-sandbox.yaml

@@ -1,5 +1,7 @@
 
 flyteadmin:
+  serviceMonitor:
+    enabled: false
   service:
     annotations:
       projectcontour.io/upstream-protocol.h2c: grpc

charts/flyte-core/values.yaml

@@ -110,6 +110,17 @@ flyteadmin:
   # -- Sets priorityClassName for flyteadmin pod(s).
   priorityClassName: ""
 
+  # -- Settings for flyteadmin service monitor
+  serviceMonitor:
+    # -- If enabled create the flyteadmin service monitor
+    enabled: false
+    # -- Sets the interval at which metrics will be scraped by prometheus
+    interval: 60s
+    # -- Sets the timeout after which request to scrape metrics will time out
+    scrapeTimeout: 30s
+    # -- Sets the labels for the service monitor which are required by the
+    # prometheus to auto-detect the service monitor and start scrapping the metrics
+    labels: {}
 #
 # FLYTESCHEDULER SETTINGS
 #
@@ -290,6 +301,22 @@ flytepropeller:
   # -- Sets priorityClassName for propeller pod(s).
   priorityClassName: ""
 
+  # -- Settings for flytepropeller service
+  service:
+    # -- If enabled create the flytepropeller service
+    enabled: false
+
+  # -- Settings for flytepropeller service monitor
+  serviceMonitor:
+    # -- If enabled create the flyetepropeller service monitor
+    enabled: false
+    # -- Sets the labels for the service monitor which are required by the
+    # prometheus to auto-detect the service monitor and start scrapping the metrics
+    labels: {}
+    # -- Sets the interval at which metrics will be scraped by prometheus
+    interval: 60s
+    # -- Sets the timeout after which request to scrape metrics will time out
+    scrapeTimeout: 30s
 #
 # FLYTECONSOLE SETTINGS
 #
@@ -895,4 +922,4 @@ databricks:
       databricks:
         entrypointFile: dbfs:///FileStore/tables/entrypoint.py
         # Databricks account
-        databricksInstance: dbc-a53b7a3c-614c
+        databricksInstance: dbc-a53b7a3c-614c

cmd/single/start.go

@@ -17,12 +17,12 @@ import (
 	adminScheduler "github.com/flyteorg/flyte/flyteadmin/scheduler"
 	propellerEntrypoint "github.com/flyteorg/flyte/flytepropeller/pkg/controller"
 	propellerConfig "github.com/flyteorg/flyte/flytepropeller/pkg/controller/config"
-	"github.com/flyteorg/flyte/flytepropeller/pkg/controller/executors"
 	"github.com/flyteorg/flyte/flytepropeller/pkg/signals"
 	webhookEntrypoint "github.com/flyteorg/flyte/flytepropeller/pkg/webhook"
 	webhookConfig "github.com/flyteorg/flyte/flytepropeller/pkg/webhook/config"
 	"github.com/flyteorg/flyte/flytestdlib/contextutils"
 	"github.com/flyteorg/flyte/flytestdlib/logger"
+	"github.com/flyteorg/flyte/flytestdlib/otelutils"
 	"github.com/flyteorg/flyte/flytestdlib/profutils"
 	"github.com/flyteorg/flyte/flytestdlib/promutils"
 	"github.com/flyteorg/flyte/flytestdlib/promutils/labeled"
@@ -120,8 +120,21 @@ func startPropeller(ctx context.Context, cfg Propeller) error {
 			SyncPeriod:        &propellerCfg.DownstreamEval.Duration,
 			DefaultNamespaces: namespaceConfigs,
 		},
+		NewCache: func (config *rest.Config, options cache.Options) (cache.Cache, error) {
+			k8sCache, err := cache.New(config, options)
+			if err != nil {
+				return k8sCache, err
+			}
+
+			return otelutils.WrapK8sCache(k8sCache), nil
+		},
 		NewClient: func(config *rest.Config, options client.Options) (client.Client, error) {
-			return executors.NewFallbackClientBuilder(propellerScope.NewSubScope("kube")).Build(nil, config, options)
+			k8sClient, err := client.New(config, options)
+			if err != nil {
+				return k8sClient, err
+			}
+
+			return otelutils.WrapK8sClient(k8sClient), nil
 		},
 		Metrics: metricsserver.Options{
 			// Disable metrics serving
@@ -191,6 +204,15 @@ var startCmd = &cobra.Command{
 		g, childCtx := errgroup.WithContext(ctx)
 		cfg := GetConfig()
 
+		for _, serviceName := range []string{otelutils.AdminClientTracer, otelutils.AdminGormTracer, otelutils.AdminServerTracer,
+			otelutils.BlobstoreClientTracer, otelutils.DataCatalogClientTracer, otelutils.DataCatalogGormTracer,
+			otelutils.DataCatalogServerTracer, otelutils.FlytePropellerTracer, otelutils.K8sClientTracer} {
+			if err := otelutils.RegisterTracerProvider(serviceName, otelutils.GetConfig()) ; err != nil {
+				logger.Errorf(ctx, "Failed to create otel tracer provider. %v", err)
+				return err
+			}
+		}
+
 		if !cfg.Admin.Disabled {
 			g.Go(func() error {
 				err := startAdmin(childCtx, cfg.Admin)

datacatalog/cmd/entrypoints/serve.go

@@ -10,6 +10,7 @@ import (
 	"github.com/flyteorg/flyte/datacatalog/pkg/runtime"
 	"github.com/flyteorg/flyte/flytestdlib/contextutils"
 	"github.com/flyteorg/flyte/flytestdlib/logger"
+	"github.com/flyteorg/flyte/flytestdlib/otelutils"
 	"github.com/flyteorg/flyte/flytestdlib/profutils"
 	"github.com/flyteorg/flyte/flytestdlib/promutils/labeled"
 )
@@ -42,6 +43,14 @@ var serveCmd = &cobra.Command{
 		// Set Keys
 		labeled.SetMetricKeys(contextutils.AppNameKey, contextutils.ProjectKey, contextutils.DomainKey)
 
+		// register otel tracer providers
+		for _, serviceName := range []string{otelutils.DataCatalogGormTracer, otelutils.DataCatalogServerTracer} {
+			if err := otelutils.RegisterTracerProvider(serviceName, otelutils.GetConfig()); err != nil {
+				logger.Errorf(ctx, "Failed to create otel tracer provider. %v", err)
+				return err
+			}
+		}
+
 		return datacatalogservice.ServeInsecure(ctx, cfg)
 	},
 }