Skip to content

Commit

Permalink
Merge branch 'master' into 4105-spark-driver-executor-podtemplate
Browse files Browse the repository at this point in the history
Signed-off-by: machichima <[email protected]>
  • Loading branch information
machichima committed Jan 19, 2025
2 parents d29181d + ab463c3 commit 65f81fa
Show file tree
Hide file tree
Showing 71 changed files with 2,319 additions and 1,526 deletions.
3 changes: 3 additions & 0 deletions charts/flyte-binary/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ Chart for basic single Flyte executable deployment
| deployment.extraVolumes | list | `[]` | |
| deployment.genAdminAuthSecret.args | list | `[]` | |
| deployment.genAdminAuthSecret.command | list | `[]` | |
| deployment.genAdminAuthSecret.securityContext | object | `{}` | |
| deployment.image.pullPolicy | string | `"IfNotPresent"` | |
| deployment.image.repository | string | `"cr.flyte.org/flyteorg/flyte-binary"` | |
| deployment.image.tag | string | `"latest"` | |
Expand All @@ -105,13 +106,15 @@ Chart for basic single Flyte executable deployment
| deployment.podSecurityContext.runAsGroup | int | `65534` | |
| deployment.podSecurityContext.runAsUser | int | `65534` | |
| deployment.readinessProbe | object | `{}` | |
| deployment.securityContext | object | `{}` | |
| deployment.sidecars | list | `[]` | |
| deployment.startupProbe | object | `{}` | |
| deployment.waitForDB.args | list | `[]` | |
| deployment.waitForDB.command | list | `[]` | |
| deployment.waitForDB.image.pullPolicy | string | `"IfNotPresent"` | |
| deployment.waitForDB.image.repository | string | `"postgres"` | |
| deployment.waitForDB.image.tag | string | `"15-alpine"` | |
| deployment.waitForDB.securityContext | object | `{}` | |
| enabled_plugins.tasks | object | `{"task-plugins":{"default-for-task-types":{"container":"container","container_array":"k8s-array","sidecar":"sidecar"},"enabled-plugins":["container","sidecar","k8s-array","agent-service","echo"]}}` | Tasks specific configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#GetConfig) |
| enabled_plugins.tasks.task-plugins | object | `{"default-for-task-types":{"container":"container","container_array":"k8s-array","sidecar":"sidecar"},"enabled-plugins":["container","sidecar","k8s-array","agent-service","echo"]}` | Plugins configuration, [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#TaskPluginConfig) |
| enabled_plugins.tasks.task-plugins.enabled-plugins | list | `["container","sidecar","k8s-array","agent-service","echo"]` | [Enabled Plugins](https://pkg.go.dev/github.com/lyft/flyteplugins/go/tasks/config#Config). Enable sagemaker*, athena if you install the backend plugins |
Expand Down
9 changes: 9 additions & 0 deletions charts/flyte-binary/templates/deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,9 @@ spec:
{{- if .Values.deployment.resources }}
resources: {{- toYaml .Values.deployment.resources | nindent 12 }}
{{- end }}
{{- if .Values.deployment.waitForDB.securityContext }}
securityContext: {{- toYaml .Values.deployment.waitForDB.securityContext | nindent 12 }}
{{- end }}
{{- end }}
{{- if .Values.configuration.auth.enabled }}
- name: gen-admin-auth-secret
Expand Down Expand Up @@ -131,6 +134,9 @@ spec:
{{- if .Values.deployment.resources }}
resources: {{- toYaml .Values.deployment.resources | nindent 12 }}
{{- end }}
{{- if .Values.deployment.genAdminAuthSecret.securityContext }}
securityContext: {{- toYaml .Values.deployment.genAdminAuthSecret.securityContext | nindent 12 }}
{{- end }}
{{- end }}
{{- if .Values.deployment.initContainers }}
{{- tpl ( .Values.deployment.initContainers | toYaml ) . | nindent 8 }}
Expand Down Expand Up @@ -224,6 +230,9 @@ spec:
{{- if .Values.deployment.extraVolumeMounts }}
{{- tpl ( .Values.deployment.extraVolumeMounts | toYaml ) . | nindent 12 }}
{{- end }}
{{- if .Values.deployment.securityContext }}
securityContext: {{- toYaml .Values.deployment.securityContext | nindent 12 }}
{{- end }}
{{- if .Values.deployment.sidecars }}
{{- tpl ( .Values.deployment.sidecars | toYaml ) . | nindent 8 }}
{{- end }}
Expand Down
6 changes: 6 additions & 0 deletions charts/flyte-binary/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -275,12 +275,16 @@ deployment:
command: []
# args Override default init container args
args: []
# securityContext Specify security context for wait-for-db init container
securityContext: {}
# genAdminAuthSecret Configure init container to generate secrets for internal use
genAdminAuthSecret:
# command Override default init container command
command: []
# args Override default init container args
args: []
# securityContext Specify security context for gen-admin-auth-secret init container
securityContext: {}
# labels Add labels to Flyte deployment
labels: {}
# annotations Add annotations to Flyte deployment
Expand All @@ -300,6 +304,8 @@ deployment:
# extraPodSpec Specify additional configuration for Flyte pod
# This can be used for adding affinity, tolerations, hostNetwork, etc.
extraPodSpec: {}
# securityContext Specify security context for Flyte container
securityContext: {}

# service Configure service for Flyte
service:
Expand Down
2 changes: 1 addition & 1 deletion datacatalog/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ require (
github.com/evanphx/json-patch/v5 v5.6.0 // indirect
github.com/fatih/color v1.13.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/flyteorg/stow v0.3.10 // indirect
github.com/flyteorg/stow v0.3.11 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/ghodss/yaml v1.0.0 // indirect
github.com/go-gormigrate/gormigrate/v2 v2.1.1 // indirect
Expand Down
4 changes: 2 additions & 2 deletions datacatalog/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -109,8 +109,8 @@ github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/flyteorg/stow v0.3.10 h1:uEe+tI+CGKn21H93uXp9z05hqynEki2BO9KkW/GweY8=
github.com/flyteorg/stow v0.3.10/go.mod h1:fArjMpsYJNWkp/hyDKKdbcv07gxbuLmKFcb7YT1aSOM=
github.com/flyteorg/stow v0.3.11 h1:Uf4fzVbghCqMNvx50XvYzwdNeQDBSKQJ7zddWu7p3eI=
github.com/flyteorg/stow v0.3.11/go.mod h1:nyaBf8ZWkpHWkKIl4rqKI2uXfPx+VbL0PmEtvq4Pxkc=
github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
Expand Down
4 changes: 2 additions & 2 deletions docker/sandbox-bundled/manifests/complete-agent.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -823,7 +823,7 @@ type: Opaque
---
apiVersion: v1
data:
haSharedSecret: VFRKYXBKM2xsTERQb1ZDaQ==
haSharedSecret: UnZJZHEzUExzbkJsOW1wYw==
proxyPassword: ""
proxyUsername: ""
kind: Secret
Expand Down Expand Up @@ -1420,7 +1420,7 @@ spec:
metadata:
annotations:
checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81
checksum/secret: 4c7bc4fbd20227034bda9ead9cd941c94879028d6920b98113f9a08ef3c6f9d9
checksum/secret: ce172103045f4215e361b4c109776a78fe06660a4ade01c7351ea07212e7cfb9
labels:
app: docker-registry
release: flyte-sandbox
Expand Down
4 changes: 2 additions & 2 deletions docker/sandbox-bundled/manifests/complete.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -805,7 +805,7 @@ type: Opaque
---
apiVersion: v1
data:
haSharedSecret: Zm4xWHVaRlNsb2EyVFFIVg==
haSharedSecret: dDFiem04NjFzb29ZWHFtNA==
proxyPassword: ""
proxyUsername: ""
kind: Secret
Expand Down Expand Up @@ -1369,7 +1369,7 @@ spec:
metadata:
annotations:
checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81
checksum/secret: fdbda8db15a09ca5943cc534a824402836c98722de81d1631c45099a8e04b043
checksum/secret: 529d34a9c4d3c82b9eec5028fcc30f26e923fa77a57eb29c4705d28c85355963
labels:
app: docker-registry
release: flyte-sandbox
Expand Down
4 changes: 2 additions & 2 deletions docker/sandbox-bundled/manifests/dev.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -499,7 +499,7 @@ metadata:
---
apiVersion: v1
data:
haSharedSecret: RjY3OEtOeFVDMjFabkI0SA==
haSharedSecret: Y1V1RU03eGVhUDFFc1pSdQ==
proxyPassword: ""
proxyUsername: ""
kind: Secret
Expand Down Expand Up @@ -934,7 +934,7 @@ spec:
metadata:
annotations:
checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81
checksum/secret: 6ab17d37fe9743709a5aa2a140c06a7a25fe33cfe4d6c0cbcb2ddf0a2dcf1675
checksum/secret: 66507f448be8010226a1ad2c741fb2866ef4372b68e61287c7500b47fae05572
labels:
app: docker-registry
release: flyte-sandbox
Expand Down
2 changes: 1 addition & 1 deletion flyteadmin/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ require (
github.com/flyteorg/flyte/flyteplugins v0.0.0-00010101000000-000000000000
github.com/flyteorg/flyte/flytepropeller v0.0.0-00010101000000-000000000000
github.com/flyteorg/flyte/flytestdlib v0.0.0-00010101000000-000000000000
github.com/flyteorg/stow v0.3.10
github.com/flyteorg/stow v0.3.11
github.com/ghodss/yaml v1.0.0
github.com/go-gormigrate/gormigrate/v2 v2.1.1
github.com/golang-jwt/jwt v3.2.2+incompatible
Expand Down
4 changes: 2 additions & 2 deletions flyteadmin/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -239,8 +239,8 @@ github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga
github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/flyteorg/stow v0.3.10 h1:uEe+tI+CGKn21H93uXp9z05hqynEki2BO9KkW/GweY8=
github.com/flyteorg/stow v0.3.10/go.mod h1:fArjMpsYJNWkp/hyDKKdbcv07gxbuLmKFcb7YT1aSOM=
github.com/flyteorg/stow v0.3.11 h1:Uf4fzVbghCqMNvx50XvYzwdNeQDBSKQJ7zddWu7p3eI=
github.com/flyteorg/stow v0.3.11/go.mod h1:nyaBf8ZWkpHWkKIl4rqKI2uXfPx+VbL0PmEtvq4Pxkc=
github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,20 +5,30 @@ import (
"io/ioutil"
"os"
"strings"
"time"

"github.com/sendgrid/rest"
"github.com/sendgrid/sendgrid-go"
"github.com/sendgrid/sendgrid-go/helpers/mail"

"github.com/flyteorg/flyte/flyteadmin/pkg/async"
"github.com/flyteorg/flyte/flyteadmin/pkg/async/notifications/interfaces"
runtimeInterfaces "github.com/flyteorg/flyte/flyteadmin/pkg/runtime/interfaces"
"github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/admin"
"github.com/flyteorg/flyte/flytestdlib/logger"
"github.com/flyteorg/flyte/flytestdlib/promutils"
)

//go:generate mockery -all -case=underscore -output=../mocks -case=underscore

type SendgridClient interface {
Send(email *mail.SGMailV3) (*rest.Response, error)
}

type SendgridEmailer struct {
client *sendgrid.Client
client SendgridClient
systemMetrics emailMetrics
cfg *runtimeInterfaces.NotificationsConfig
}

func getEmailAddresses(addresses []string) []*mail.Email {
Expand Down Expand Up @@ -63,9 +73,18 @@ func getAPIKey(config runtimeInterfaces.EmailServerConfig) string {
func (s SendgridEmailer) SendEmail(ctx context.Context, email *admin.EmailMessage) error {
m := getSendgridEmail(email)
s.systemMetrics.SendTotal.Inc()
response, err := s.client.Send(m)
var response *rest.Response
var err error
err = async.Retry(s.cfg.ReconnectAttempts, time.Duration(s.cfg.ReconnectDelaySeconds)*time.Second, func() error {
response, err = s.client.Send(m)
if err != nil {
logger.Errorf(ctx, "Sendgrid error sending email: %+v with: %+v", email, err)
return err
}
return nil
})
if err != nil {
logger.Errorf(ctx, "Sendgrid error sending %s", err)
logger.Errorf(ctx, "all attempts to send email %+v via sendgrid failed: %+v", email, err)
s.systemMetrics.SendError.Inc()
return err
}
Expand All @@ -79,5 +98,6 @@ func NewSendGridEmailer(config runtimeInterfaces.NotificationsConfig, scope prom
return &SendgridEmailer{
client: sendgrid.NewSendClient(getAPIKey(config.NotificationsEmailerConfig.EmailerConfig)),
systemMetrics: newEmailMetrics(scope.NewSubScope("sendgrid")),
cfg: &config,
}
}
Original file line number Diff line number Diff line change
@@ -1,27 +1,26 @@
package implementations

import (
"context"
"errors"
"io/ioutil"
"os"
"path"
"strings"
"testing"

"github.com/prometheus/client_golang/prometheus/testutil"
"github.com/sendgrid/rest"
"github.com/stretchr/testify/assert"

"github.com/flyteorg/flyte/flyteadmin/pkg/async/notifications/mocks"
runtimeInterfaces "github.com/flyteorg/flyte/flyteadmin/pkg/runtime/interfaces"
"github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/admin"
"github.com/flyteorg/flyte/flytestdlib/promutils"
)

func TestAddresses(t *testing.T) {
addresses := []string{"[email protected]", "[email protected]"}
sgAddresses := getEmailAddresses(addresses)
assert.Equal(t, sgAddresses[0].Address, "[email protected]")
assert.Equal(t, sgAddresses[1].Address, "[email protected]")
}

func TestGetEmail(t *testing.T) {
emailNotification := &admin.EmailMessage{
var (
emailNotification = &admin.EmailMessage{
SubjectLine: "Notice: Execution \"name\" has succeeded in \"domain\".",
SenderEmail: "[email protected]",
RecipientsEmail: []string{
Expand All @@ -32,7 +31,16 @@ func TestGetEmail(t *testing.T) {
"<a href=\"https://example.com/executions/T/B/D\">" +
"https://example.com/executions/T/B/D</a>.",
}
)

func TestAddresses(t *testing.T) {
addresses := []string{"[email protected]", "[email protected]"}
sgAddresses := getEmailAddresses(addresses)
assert.Equal(t, sgAddresses[0].Address, "[email protected]")
assert.Equal(t, sgAddresses[1].Address, "[email protected]")
}

func TestGetEmail(t *testing.T) {
sgEmail := getSendgridEmail(emailNotification)
assert.Equal(t, `Notice: Execution "name" has succeeded in "domain".`, sgEmail.Personalizations[0].Subject)
assert.Equal(t, "[email protected]", sgEmail.Personalizations[0].To[1].Address)
Expand Down Expand Up @@ -98,3 +106,63 @@ func TestNoFile(t *testing.T) {
// shouldn't reach here
t.Errorf("did not panic")
}

func TestSendEmail(t *testing.T) {
ctx := context.TODO()
expectedErr := errors.New("expected")
t.Run("exhaust all retry attempts", func(t *testing.T) {
sendgridClient := &mocks.SendgridClient{}
expectedEmail := getSendgridEmail(emailNotification)
sendgridClient.OnSendMatch(expectedEmail).
Return(nil, expectedErr).Times(3)
sendgridClient.OnSendMatch(expectedEmail).
Return(&rest.Response{Body: "email body"}, nil).Once()
scope := promutils.NewScope("bademailer")
emailerMetrics := newEmailMetrics(scope)

emailer := SendgridEmailer{
client: sendgridClient,
systemMetrics: emailerMetrics,
cfg: &runtimeInterfaces.NotificationsConfig{
ReconnectAttempts: 1,
},
}

err := emailer.SendEmail(ctx, emailNotification)
assert.EqualError(t, err, expectedErr.Error())

assert.NoError(t, testutil.CollectAndCompare(emailerMetrics.SendError, strings.NewReader(`
# HELP bademailer:send_error Number of errors when sending email via Emailer
# TYPE bademailer:send_error counter
bademailer:send_error 1
`)))
})
t.Run("exhaust all retry attempts", func(t *testing.T) {
ctx := context.TODO()
sendgridClient := &mocks.SendgridClient{}
expectedEmail := getSendgridEmail(emailNotification)
sendgridClient.OnSendMatch(expectedEmail).
Return(nil, expectedErr).Once()
sendgridClient.OnSendMatch(expectedEmail).
Return(&rest.Response{Body: "email body"}, nil).Once()
scope := promutils.NewScope("goodemailer")
emailerMetrics := newEmailMetrics(scope)

emailer := SendgridEmailer{
client: sendgridClient,
systemMetrics: emailerMetrics,
cfg: &runtimeInterfaces.NotificationsConfig{
ReconnectAttempts: 1,
},
}

err := emailer.SendEmail(ctx, emailNotification)
assert.NoError(t, err)

assert.NoError(t, testutil.CollectAndCompare(emailerMetrics.SendError, strings.NewReader(`
# HELP goodemailer:send_error Number of errors when sending email via Emailer
# TYPE goodemailer:send_error counter
goodemailer:send_error 0
`)))
})
}
Loading

0 comments on commit 65f81fa

Please sign in to comment.