From 4b45194a726cc3878c64de346f5850dcbf432a1f Mon Sep 17 00:00:00 2001 From: Chris Grass Date: Tue, 19 Dec 2023 11:56:56 -0600 Subject: [PATCH 01/10] Add azure_remote_url and corresponding test. Signed-off-by: Chris Grass --- flyteadmin/pkg/common/cloud.go | 1 + flyteadmin/pkg/data/factory.go | 6 ++- .../data/implementations/azure_remote_url.go | 46 +++++++++++++++++++ .../implementations/azure_remote_url_test.go | 38 +++++++++++++++ 4 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 flyteadmin/pkg/data/implementations/azure_remote_url.go create mode 100644 flyteadmin/pkg/data/implementations/azure_remote_url_test.go diff --git a/flyteadmin/pkg/common/cloud.go b/flyteadmin/pkg/common/cloud.go index 93f3669a55..f982438594 100644 --- a/flyteadmin/pkg/common/cloud.go +++ b/flyteadmin/pkg/common/cloud.go @@ -7,6 +7,7 @@ type CloudProvider = string const ( AWS CloudProvider = "aws" GCP CloudProvider = "gcp" + Azure CloudProvider = "azure" Sandbox CloudProvider = "sandbox" Local CloudProvider = "local" None CloudProvider = "none" diff --git a/flyteadmin/pkg/data/factory.go b/flyteadmin/pkg/data/factory.go index b746d972fc..7d25e4e1fe 100644 --- a/flyteadmin/pkg/data/factory.go +++ b/flyteadmin/pkg/data/factory.go @@ -50,7 +50,11 @@ func GetRemoteDataHandler(cfg RemoteDataHandlerConfig) RemoteDataHandler { return &remoteDataHandler{ remoteURL: implementations.NewGCPRemoteURL(cfg.SigningPrincipal, signedURLDuration), } - + case common.Azure: + signedURLDuration := time.Minute * time.Duration(cfg.SignedURLDurationMinutes) + return &remoteDataHandler{ + remoteURL: implementations.NewAzureRemoteURL(*cfg.RemoteDataStoreClient, signedURLDuration), + } case common.Local: logger.Infof(context.TODO(), "setting up local signer ----- ") // Since minio = aws s3, we are creating the same client but using the config primitives from aws diff --git a/flyteadmin/pkg/data/implementations/azure_remote_url.go b/flyteadmin/pkg/data/implementations/azure_remote_url.go new file mode 100644 index 0000000000..a9649ef943 --- /dev/null +++ b/flyteadmin/pkg/data/implementations/azure_remote_url.go @@ -0,0 +1,46 @@ +package implementations + +import ( + "context" + "github.com/flyteorg/flyte/flyteadmin/pkg/data/interfaces" + "github.com/flyteorg/flyte/flyteadmin/pkg/errors" + "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/admin" + "github.com/flyteorg/flyte/flytestdlib/storage" + "github.com/flyteorg/stow" + "google.golang.org/grpc/codes" + "time" +) + +type AzureRemoteURL struct { + remoteDataStoreClient storage.DataStore + presignDuration time.Duration +} + +func (n *AzureRemoteURL) Get(ctx context.Context, uri string) (admin.UrlBlob, error) { + metadata, err := n.remoteDataStoreClient.Head(ctx, storage.DataReference(uri)) + if err != nil { + return admin.UrlBlob{}, errors.NewFlyteAdminErrorf(codes.Internal, + "failed to get metadata for uri: %s with err: %v", uri, err) + } + + signedUri, err := n.remoteDataStoreClient.CreateSignedURL(ctx, storage.DataReference(uri), storage.SignedURLProperties{ + Scope: stow.ClientMethodGet, + ExpiresIn: n.presignDuration, + }) + if err != nil { + return admin.UrlBlob{}, errors.NewFlyteAdminErrorf(codes.Internal, + "failed to get metadata for uri: %s with err: %v", uri, err) + } + + return admin.UrlBlob{ + Url: signedUri.URL.String(), + Bytes: metadata.Size(), + }, nil +} + +func NewAzureRemoteURL(remoteDataStoreClient storage.DataStore, presignDuration time.Duration) interfaces.RemoteURLInterface { + return &AzureRemoteURL{ + remoteDataStoreClient: remoteDataStoreClient, + presignDuration: presignDuration, + } +} diff --git a/flyteadmin/pkg/data/implementations/azure_remote_url_test.go b/flyteadmin/pkg/data/implementations/azure_remote_url_test.go new file mode 100644 index 0000000000..f48adc52e9 --- /dev/null +++ b/flyteadmin/pkg/data/implementations/azure_remote_url_test.go @@ -0,0 +1,38 @@ +package implementations + +import ( + "context" + commonMocks "github.com/flyteorg/flyte/flyteadmin/pkg/common/mocks" + "github.com/flyteorg/flyte/flytestdlib/storage" + "github.com/stretchr/testify/assert" + "testing" +) + +type mockMetadata struct{} + +func (m mockMetadata) Exists() bool { + return true +} + +func (m mockMetadata) Size() int64 { + return 1 +} + +func (m mockMetadata) Etag() string { + return "etag" +} + +func TestAzureGet(t *testing.T) { + inputUri := "abfs//test/data" + mockStorage := commonMocks.GetMockStorageClient() + mockStorage.ComposedProtobufStore.(*commonMocks.TestDataStore).HeadCb = + func(ctx context.Context, reference storage.DataReference) (storage.Metadata, error) { + return mockMetadata{}, nil + } + remoteUrl := AzureRemoteURL{ + remoteDataStoreClient: *mockStorage, presignDuration: 1, + } + + result, _ := remoteUrl.Get(context.TODO(), inputUri) + assert.Contains(t, inputUri, result.Url) +} From cf6528ee145afbfe51816592125128d4386f6d69 Mon Sep 17 00:00:00 2001 From: Chris Grass Date: Wed, 10 Jan 2024 11:07:30 -0600 Subject: [PATCH 02/10] lint Signed-off-by: Chris Grass --- flyteadmin/pkg/data/implementations/azure_remote_url_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flyteadmin/pkg/data/implementations/azure_remote_url_test.go b/flyteadmin/pkg/data/implementations/azure_remote_url_test.go index f48adc52e9..67da9551f7 100644 --- a/flyteadmin/pkg/data/implementations/azure_remote_url_test.go +++ b/flyteadmin/pkg/data/implementations/azure_remote_url_test.go @@ -29,10 +29,10 @@ func TestAzureGet(t *testing.T) { func(ctx context.Context, reference storage.DataReference) (storage.Metadata, error) { return mockMetadata{}, nil } - remoteUrl := AzureRemoteURL{ + remoteURL := AzureRemoteURL{ remoteDataStoreClient: *mockStorage, presignDuration: 1, } - result, _ := remoteUrl.Get(context.TODO(), inputUri) + result, _ := remoteURL.Get(context.TODO(), inputUri) assert.Contains(t, inputUri, result.Url) } From f0cb2674a619cdd7365de42ee36443f387cda103 Mon Sep 17 00:00:00 2001 From: Chris Grass Date: Fri, 13 Dec 2024 14:25:16 -0800 Subject: [PATCH 03/10] trigger cli From bc24fb8af6760af2ac67a9bba93447d0919ca5ed Mon Sep 17 00:00:00 2001 From: Chris Grass Date: Fri, 13 Dec 2024 14:55:27 -0800 Subject: [PATCH 04/10] update after rebase --- flyteadmin/pkg/data/implementations/azure_remote_url.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flyteadmin/pkg/data/implementations/azure_remote_url.go b/flyteadmin/pkg/data/implementations/azure_remote_url.go index a9649ef943..0508b8b6f9 100644 --- a/flyteadmin/pkg/data/implementations/azure_remote_url.go +++ b/flyteadmin/pkg/data/implementations/azure_remote_url.go @@ -16,7 +16,7 @@ type AzureRemoteURL struct { presignDuration time.Duration } -func (n *AzureRemoteURL) Get(ctx context.Context, uri string) (admin.UrlBlob, error) { +func (n *AzureRemoteURL) Get(ctx context.Context, uri string) (*admin.UrlBlob, error) { metadata, err := n.remoteDataStoreClient.Head(ctx, storage.DataReference(uri)) if err != nil { return admin.UrlBlob{}, errors.NewFlyteAdminErrorf(codes.Internal, From e93b251d8e7187e54d2e53f717b37edeb6efb9da Mon Sep 17 00:00:00 2001 From: Chris Grass Date: Fri, 13 Dec 2024 15:01:21 -0800 Subject: [PATCH 05/10] update after rebase2 --- flyteadmin/pkg/data/implementations/azure_remote_url.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flyteadmin/pkg/data/implementations/azure_remote_url.go b/flyteadmin/pkg/data/implementations/azure_remote_url.go index 0508b8b6f9..4f4f6e7c50 100644 --- a/flyteadmin/pkg/data/implementations/azure_remote_url.go +++ b/flyteadmin/pkg/data/implementations/azure_remote_url.go @@ -19,7 +19,7 @@ type AzureRemoteURL struct { func (n *AzureRemoteURL) Get(ctx context.Context, uri string) (*admin.UrlBlob, error) { metadata, err := n.remoteDataStoreClient.Head(ctx, storage.DataReference(uri)) if err != nil { - return admin.UrlBlob{}, errors.NewFlyteAdminErrorf(codes.Internal, + return *admin.UrlBlob{}, errors.NewFlyteAdminErrorf(codes.Internal, "failed to get metadata for uri: %s with err: %v", uri, err) } @@ -28,11 +28,11 @@ func (n *AzureRemoteURL) Get(ctx context.Context, uri string) (*admin.UrlBlob, e ExpiresIn: n.presignDuration, }) if err != nil { - return admin.UrlBlob{}, errors.NewFlyteAdminErrorf(codes.Internal, + return *admin.UrlBlob{}, errors.NewFlyteAdminErrorf(codes.Internal, "failed to get metadata for uri: %s with err: %v", uri, err) } - return admin.UrlBlob{ + return *admin.UrlBlob{ Url: signedUri.URL.String(), Bytes: metadata.Size(), }, nil From b979084a67812098597d9bb2544a6d977f3f9d0a Mon Sep 17 00:00:00 2001 From: Chris Grass Date: Fri, 13 Dec 2024 14:25:16 -0800 Subject: [PATCH 06/10] trigger cli Signed-off-by: Chris Grass From 31e0bfaabe5511188f8941d6e6b02536690fda58 Mon Sep 17 00:00:00 2001 From: Flyte Bot Date: Thu, 24 Jun 2021 06:12:47 -0700 Subject: [PATCH 07/10] Update Boilerplate (#98) Signed-off-by: Flyte-Bot Co-authored-by: flyte-bot Signed-off-by: Chris Grass From 141d6bdb190491e9b0e47af51c7c0446872c4ba1 Mon Sep 17 00:00:00 2001 From: Flyte Bot Date: Thu, 17 Feb 2022 21:15:13 -0800 Subject: [PATCH 08/10] Update documentation (#284) Signed-off-by: Flyte-Bot Co-authored-by: evalsocket Signed-off-by: Chris Grass From d0361a068ee27146654b1f2abcca751419054e5d Mon Sep 17 00:00:00 2001 From: Chris Grass Date: Fri, 13 Dec 2024 15:08:42 -0800 Subject: [PATCH 09/10] i forgot how golang works. Signed-off-by: Chris Grass --- flyteadmin/pkg/data/implementations/azure_remote_url.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flyteadmin/pkg/data/implementations/azure_remote_url.go b/flyteadmin/pkg/data/implementations/azure_remote_url.go index 4f4f6e7c50..991fa1e1c0 100644 --- a/flyteadmin/pkg/data/implementations/azure_remote_url.go +++ b/flyteadmin/pkg/data/implementations/azure_remote_url.go @@ -19,7 +19,7 @@ type AzureRemoteURL struct { func (n *AzureRemoteURL) Get(ctx context.Context, uri string) (*admin.UrlBlob, error) { metadata, err := n.remoteDataStoreClient.Head(ctx, storage.DataReference(uri)) if err != nil { - return *admin.UrlBlob{}, errors.NewFlyteAdminErrorf(codes.Internal, + return &admin.UrlBlob{}, errors.NewFlyteAdminErrorf(codes.Internal, "failed to get metadata for uri: %s with err: %v", uri, err) } @@ -28,11 +28,11 @@ func (n *AzureRemoteURL) Get(ctx context.Context, uri string) (*admin.UrlBlob, e ExpiresIn: n.presignDuration, }) if err != nil { - return *admin.UrlBlob{}, errors.NewFlyteAdminErrorf(codes.Internal, + return &admin.UrlBlob{}, errors.NewFlyteAdminErrorf(codes.Internal, "failed to get metadata for uri: %s with err: %v", uri, err) } - return *admin.UrlBlob{ + return &admin.UrlBlob{ Url: signedUri.URL.String(), Bytes: metadata.Size(), }, nil From 5ef1ae7349eff2bd7b881b8730886f7c41022c34 Mon Sep 17 00:00:00 2001 From: Chris Grass Date: Fri, 13 Dec 2024 17:30:38 -0800 Subject: [PATCH 10/10] fix test --- flyteadmin/pkg/data/implementations/azure_remote_url_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/flyteadmin/pkg/data/implementations/azure_remote_url_test.go b/flyteadmin/pkg/data/implementations/azure_remote_url_test.go index 67da9551f7..cb422d1169 100644 --- a/flyteadmin/pkg/data/implementations/azure_remote_url_test.go +++ b/flyteadmin/pkg/data/implementations/azure_remote_url_test.go @@ -10,6 +10,10 @@ import ( type mockMetadata struct{} +func (m mockMetadata) ContentMD5() string { + return "" +} + func (m mockMetadata) Exists() bool { return true }