From d7573493c5b200bb4cb9664acf05eabda930526d Mon Sep 17 00:00:00 2001 From: Ashwin Kumar Date: Fri, 4 Oct 2024 15:30:04 -0700 Subject: [PATCH 1/4] feat(internal-remove): add new internal downloadData api --- .../internals/apis/downloadData.test.ts | 76 ++++++++++++++++ .../src/internals/apis/downloadData.ts | 41 +++++++++ packages/storage/src/internals/index.ts | 3 + .../storage/src/internals/types/inputs.ts | 11 +++ .../storage/src/internals/types/outputs.ts | 6 ++ .../src/providers/s3/apis/downloadData.ts | 87 +----------------- .../s3/apis/internal/downloadData.ts | 90 +++++++++++++++++++ 7 files changed, 230 insertions(+), 84 deletions(-) create mode 100644 packages/storage/__tests__/internals/apis/downloadData.test.ts create mode 100644 packages/storage/src/internals/apis/downloadData.ts create mode 100644 packages/storage/src/providers/s3/apis/internal/downloadData.ts diff --git a/packages/storage/__tests__/internals/apis/downloadData.test.ts b/packages/storage/__tests__/internals/apis/downloadData.test.ts new file mode 100644 index 00000000000..60ac8c8054e --- /dev/null +++ b/packages/storage/__tests__/internals/apis/downloadData.test.ts @@ -0,0 +1,76 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { downloadData as advancedDownloadData } from '../../../src/internals'; +import { downloadData as downloadDataInternal } from '../../../src/providers/s3/apis/internal/downloadData'; + +jest.mock('../../../src/providers/s3/apis/internal/downloadData'); +const mockedDownloadDataInternal = jest.mocked(downloadDataInternal); + +describe('remove (internal)', () => { + beforeEach(() => { + mockedDownloadDataInternal.mockReturnValue({ + result: Promise.resolve({ + path: 'output/path/to/mock/object', + body: { + blob: () => Promise.resolve(new Blob()), + json: () => Promise.resolve(''), + text: () => Promise.resolve(''), + }, + }), + cancel: jest.fn(), + state: 'SUCCESS', + }); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + it('should pass advanced option locationCredentialsProvider to internal downloadData', async () => { + const useAccelerateEndpoint = true; + const bucket = { bucketName: 'bucket', region: 'us-east-1' }; + const locationCredentialsProvider = async () => ({ + credentials: { + accessKeyId: 'akid', + secretAccessKey: 'secret', + sessionToken: 'token', + expiration: new Date(), + }, + }); + const onProgress = jest.fn(); + const bytesRange = { start: 1024, end: 2048 }; + + const output = await advancedDownloadData({ + path: 'input/path/to/mock/object', + options: { + useAccelerateEndpoint, + bucket, + locationCredentialsProvider, + onProgress, + bytesRange, + }, + }); + + expect(mockedDownloadDataInternal).toHaveBeenCalledTimes(1); + expect(mockedDownloadDataInternal).toHaveBeenCalledWith({ + path: 'input/path/to/mock/object', + options: { + useAccelerateEndpoint, + bucket, + locationCredentialsProvider, + onProgress, + bytesRange, + }, + }); + + expect(await output.result).toEqual({ + path: 'output/path/to/mock/object', + body: { + blob: expect.any(Function), + json: expect.any(Function), + text: expect.any(Function), + }, + }); + }); +}); diff --git a/packages/storage/src/internals/apis/downloadData.ts b/packages/storage/src/internals/apis/downloadData.ts new file mode 100644 index 00000000000..aeb8fae76a4 --- /dev/null +++ b/packages/storage/src/internals/apis/downloadData.ts @@ -0,0 +1,41 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { downloadData as downloadDataInternal } from '../../providers/s3/apis/internal/downloadData'; +import { DownloadDataInput } from '../types/inputs'; +import { DownloadDataOutput } from '../types/outputs'; + +/** + * Download S3 object data to memory + * + * @param input - The `DownloadDataWithPathInput` object. + * @returns A cancelable task exposing result promise from `result` property. + * @throws service: `S3Exception` - thrown when checking for existence of the object + * @throws validation: `StorageValidationErrorCode` - Validation errors + * + * @example + * ```ts + * // Download a file from s3 bucket + * const { body, eTag } = await downloadData({ path, options: { + * onProgress, // Optional progress callback. + * } }).result; + * ``` + * @example + * ```ts + * // Cancel a task + * const downloadTask = downloadData({ path }); + * //... + * downloadTask.cancel(); + * try { + * await downloadTask.result; + * } catch (error) { + * if(isCancelError(error)) { + * // Handle error thrown by task cancelation. + * } + * } + *``` + * + * @internal + */ +export const downloadData = (input: DownloadDataInput): DownloadDataOutput => + downloadDataInternal(input) as DownloadDataOutput; diff --git a/packages/storage/src/internals/index.ts b/packages/storage/src/internals/index.ts index a514ad88547..64dd2e92840 100644 --- a/packages/storage/src/internals/index.ts +++ b/packages/storage/src/internals/index.ts @@ -14,18 +14,21 @@ export { GetPropertiesInput, CopyInput, RemoveInput, + DownloadDataInput, } from './types/inputs'; export { GetDataAccessOutput, ListCallerAccessGrantsOutput, GetPropertiesOutput, RemoveOutput, + DownloadDataOutput, } from './types/outputs'; export { getDataAccess } from './apis/getDataAccess'; export { listCallerAccessGrants } from './apis/listCallerAccessGrants'; export { getProperties } from './apis/getProperties'; export { remove } from './apis/remove'; +export { downloadData } from './apis/downloadData'; /* CredentialsStore exports diff --git a/packages/storage/src/internals/types/inputs.ts b/packages/storage/src/internals/types/inputs.ts index f8a36943066..52ced799e55 100644 --- a/packages/storage/src/internals/types/inputs.ts +++ b/packages/storage/src/internals/types/inputs.ts @@ -8,6 +8,7 @@ import { } from '../../types/inputs'; import { CopyWithPathInput, + DownloadDataWithPathInput, GetPropertiesWithPathInput, RemoveWithPathInput, } from '../../providers/s3'; @@ -68,6 +69,16 @@ export type CopyInput = ExtendCopyInputWithAdvancedOptions< } >; +/** + * @internal + */ +export type DownloadDataInput = ExtendInputWithAdvancedOptions< + DownloadDataWithPathInput, + { + locationCredentialsProvider?: CredentialsProvider; + } +>; + /** * Generic types that extend the public non-copy API input types with extended * options. This is a temporary solution to support advanced options from internal APIs. diff --git a/packages/storage/src/internals/types/outputs.ts b/packages/storage/src/internals/types/outputs.ts index cf1e96acfe2..0cd4dfcdd46 100644 --- a/packages/storage/src/internals/types/outputs.ts +++ b/packages/storage/src/internals/types/outputs.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { + DownloadDataWithPathOutput, GetPropertiesWithPathOutput, RemoveWithPathOutput, } from '../../providers/s3/types'; @@ -27,3 +28,8 @@ export type GetPropertiesOutput = GetPropertiesWithPathOutput; * @internal */ export type RemoveOutput = RemoveWithPathOutput; + +/** + * @internal + */ +export type DownloadDataOutput = DownloadDataWithPathOutput; diff --git a/packages/storage/src/providers/s3/apis/downloadData.ts b/packages/storage/src/providers/s3/apis/downloadData.ts index 32f0c558642..0eeca69899f 100644 --- a/packages/storage/src/providers/s3/apis/downloadData.ts +++ b/packages/storage/src/providers/s3/apis/downloadData.ts @@ -1,26 +1,14 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { Amplify } from '@aws-amplify/core'; -import { StorageAction } from '@aws-amplify/core/internals/utils'; - import { DownloadDataInput, DownloadDataOutput, DownloadDataWithPathInput, DownloadDataWithPathOutput, } from '../types'; -import { resolveS3ConfigAndInput } from '../utils/resolveS3ConfigAndInput'; -import { createDownloadTask, validateStorageOperationInput } from '../utils'; -import { getObject } from '../utils/client/s3data'; -import { getStorageUserAgentValue } from '../utils/userAgent'; -import { logger } from '../../../utils'; -import { - StorageDownloadDataOutput, - StorageItemWithKey, - StorageItemWithPath, -} from '../../../types'; -import { STORAGE_INPUT_KEY } from '../utils/constants'; + +import { downloadData as downloadDataInternal } from './internal/downloadData'; /** * Download S3 object data to memory @@ -89,77 +77,8 @@ export function downloadData( *``` */ export function downloadData(input: DownloadDataInput): DownloadDataOutput; - export function downloadData( input: DownloadDataInput | DownloadDataWithPathInput, ) { - const abortController = new AbortController(); - - const downloadTask = createDownloadTask({ - job: downloadDataJob(input, abortController.signal), - onCancel: (message?: string) => { - abortController.abort(message); - }, - }); - - return downloadTask; + return downloadDataInternal(input); } - -const downloadDataJob = - ( - downloadDataInput: DownloadDataInput | DownloadDataWithPathInput, - abortSignal: AbortSignal, - ) => - async (): Promise< - StorageDownloadDataOutput - > => { - const { options: downloadDataOptions } = downloadDataInput; - const { bucket, keyPrefix, s3Config, identityId } = - await resolveS3ConfigAndInput(Amplify, downloadDataInput); - const { inputType, objectKey } = validateStorageOperationInput( - downloadDataInput, - identityId, - ); - const finalKey = - inputType === STORAGE_INPUT_KEY ? keyPrefix + objectKey : objectKey; - - logger.debug(`download ${objectKey} from ${finalKey}.`); - - const { - Body: body, - LastModified: lastModified, - ContentLength: size, - ETag: eTag, - Metadata: metadata, - VersionId: versionId, - ContentType: contentType, - } = await getObject( - { - ...s3Config, - abortSignal, - onDownloadProgress: downloadDataOptions?.onProgress, - userAgentValue: getStorageUserAgentValue(StorageAction.DownloadData), - }, - { - Bucket: bucket, - Key: finalKey, - ...(downloadDataOptions?.bytesRange && { - Range: `bytes=${downloadDataOptions.bytesRange.start}-${downloadDataOptions.bytesRange.end}`, - }), - }, - ); - - const result = { - body, - lastModified, - size, - contentType, - eTag, - metadata, - versionId, - }; - - return inputType === STORAGE_INPUT_KEY - ? { key: objectKey, ...result } - : { path: objectKey, ...result }; - }; diff --git a/packages/storage/src/providers/s3/apis/internal/downloadData.ts b/packages/storage/src/providers/s3/apis/internal/downloadData.ts new file mode 100644 index 00000000000..fe9b187231c --- /dev/null +++ b/packages/storage/src/providers/s3/apis/internal/downloadData.ts @@ -0,0 +1,90 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Amplify } from '@aws-amplify/core'; +import { StorageAction } from '@aws-amplify/core/internals/utils'; + +import { resolveS3ConfigAndInput } from '../../utils/resolveS3ConfigAndInput'; +import { createDownloadTask, validateStorageOperationInput } from '../../utils'; +import { getObject } from '../../utils/client/s3data'; +import { getStorageUserAgentValue } from '../../utils/userAgent'; +import { logger } from '../../../../utils'; +import { DownloadDataInput, DownloadDataWithPathInput } from '../../types'; +import { STORAGE_INPUT_KEY } from '../../utils/constants'; +import { + StorageDownloadDataOutput, + StorageItemWithKey, + StorageItemWithPath, +} from '../../../../types'; +// TODO: Remove this interface when we move to public advanced APIs. +import { DownloadDataInput as DownloadDataWithPathInputWithAdvancedOptions } from '../../../../internals/types/inputs'; + +export const downloadData = ( + input: DownloadDataInput | DownloadDataWithPathInputWithAdvancedOptions, +) => { + const abortController = new AbortController(); + const downloadTask = createDownloadTask({ + job: downloadDataJob(input, abortController.signal), + onCancel: (message?: string) => { + abortController.abort(message); + }, + }); + + return downloadTask; +}; + +const downloadDataJob = + ( + downloadDataInput: DownloadDataInput | DownloadDataWithPathInput, + abortSignal: AbortSignal, + ) => + async (): Promise< + StorageDownloadDataOutput + > => { + const { options: downloadDataOptions } = downloadDataInput; + const { bucket, keyPrefix, s3Config, identityId } = + await resolveS3ConfigAndInput(Amplify, downloadDataInput); + const { inputType, objectKey } = validateStorageOperationInput( + downloadDataInput, + identityId, + ); + const finalKey = + inputType === STORAGE_INPUT_KEY ? keyPrefix + objectKey : objectKey; + logger.debug(`download ${objectKey} from ${finalKey}.`); + const { + Body: body, + LastModified: lastModified, + ContentLength: size, + ETag: eTag, + Metadata: metadata, + VersionId: versionId, + ContentType: contentType, + } = await getObject( + { + ...s3Config, + abortSignal, + onDownloadProgress: downloadDataOptions?.onProgress, + userAgentValue: getStorageUserAgentValue(StorageAction.DownloadData), + }, + { + Bucket: bucket, + Key: finalKey, + ...(downloadDataOptions?.bytesRange && { + Range: `bytes=${downloadDataOptions.bytesRange.start}-${downloadDataOptions.bytesRange.end}`, + }), + }, + ); + const result = { + body, + lastModified, + size, + contentType, + eTag, + metadata, + versionId, + }; + + return inputType === STORAGE_INPUT_KEY + ? { key: objectKey, ...result } + : { path: objectKey, ...result }; + }; From 0f9f51ae960691bde8814a3013dd94353fb4a5ac Mon Sep 17 00:00:00 2001 From: Ashwin Kumar Date: Fri, 4 Oct 2024 15:45:20 -0700 Subject: [PATCH 2/4] code cleanup --- packages/storage/src/internals/apis/downloadData.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/storage/src/internals/apis/downloadData.ts b/packages/storage/src/internals/apis/downloadData.ts index aeb8fae76a4..e0e59b91f28 100644 --- a/packages/storage/src/internals/apis/downloadData.ts +++ b/packages/storage/src/internals/apis/downloadData.ts @@ -38,4 +38,15 @@ import { DownloadDataOutput } from '../types/outputs'; * @internal */ export const downloadData = (input: DownloadDataInput): DownloadDataOutput => - downloadDataInternal(input) as DownloadDataOutput; + downloadDataInternal({ + path: input.path, + options: { + useAccelerateEndpoint: input?.options?.useAccelerateEndpoint, + bucket: input?.options?.bucket, + locationCredentialsProvider: input?.options?.locationCredentialsProvider, + bytesRange: input?.options?.bytesRange, + onProgress: input?.options?.onProgress, + }, + // Type casting is necessary because `removeInternal` supports both Gen1 and Gen2 signatures, but here + // given in input can only be Gen2 signature, the return can only ben Gen2 signature. + }) as DownloadDataOutput; From 09b72040e2f8838a142e8eb5b62f66d0055928a2 Mon Sep 17 00:00:00 2001 From: Ashwin Kumar Date: Fri, 4 Oct 2024 16:50:14 -0700 Subject: [PATCH 3/4] code cleanup --- packages/storage/__tests__/internals/apis/downloadData.test.ts | 2 +- packages/storage/src/internals/apis/downloadData.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/storage/__tests__/internals/apis/downloadData.test.ts b/packages/storage/__tests__/internals/apis/downloadData.test.ts index 60ac8c8054e..4e27e68a1a4 100644 --- a/packages/storage/__tests__/internals/apis/downloadData.test.ts +++ b/packages/storage/__tests__/internals/apis/downloadData.test.ts @@ -7,7 +7,7 @@ import { downloadData as downloadDataInternal } from '../../../src/providers/s3/ jest.mock('../../../src/providers/s3/apis/internal/downloadData'); const mockedDownloadDataInternal = jest.mocked(downloadDataInternal); -describe('remove (internal)', () => { +describe('downloadData (internal)', () => { beforeEach(() => { mockedDownloadDataInternal.mockReturnValue({ result: Promise.resolve({ diff --git a/packages/storage/src/internals/apis/downloadData.ts b/packages/storage/src/internals/apis/downloadData.ts index e0e59b91f28..db2fb448ee7 100644 --- a/packages/storage/src/internals/apis/downloadData.ts +++ b/packages/storage/src/internals/apis/downloadData.ts @@ -47,6 +47,6 @@ export const downloadData = (input: DownloadDataInput): DownloadDataOutput => bytesRange: input?.options?.bytesRange, onProgress: input?.options?.onProgress, }, - // Type casting is necessary because `removeInternal` supports both Gen1 and Gen2 signatures, but here + // Type casting is necessary because `downloadDataInternal` supports both Gen1 and Gen2 signatures, but here // given in input can only be Gen2 signature, the return can only ben Gen2 signature. }) as DownloadDataOutput; From fc66f4da1902ebf4cdab0083e1ff8eb0fbf0222c Mon Sep 17 00:00:00 2001 From: Ashwin Kumar Date: Fri, 4 Oct 2024 17:01:19 -0700 Subject: [PATCH 4/4] chore: fix ts doc --- packages/storage/src/internals/apis/downloadData.ts | 2 +- packages/storage/src/internals/apis/getProperties.ts | 2 +- packages/storage/src/internals/apis/remove.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/storage/src/internals/apis/downloadData.ts b/packages/storage/src/internals/apis/downloadData.ts index db2fb448ee7..9c05dd525b4 100644 --- a/packages/storage/src/internals/apis/downloadData.ts +++ b/packages/storage/src/internals/apis/downloadData.ts @@ -8,7 +8,7 @@ import { DownloadDataOutput } from '../types/outputs'; /** * Download S3 object data to memory * - * @param input - The `DownloadDataWithPathInput` object. + * @param input - The `DownloadDataInput` object. * @returns A cancelable task exposing result promise from `result` property. * @throws service: `S3Exception` - thrown when checking for existence of the object * @throws validation: `StorageValidationErrorCode` - Validation errors diff --git a/packages/storage/src/internals/apis/getProperties.ts b/packages/storage/src/internals/apis/getProperties.ts index d1050c4c1a2..37cba3bed52 100644 --- a/packages/storage/src/internals/apis/getProperties.ts +++ b/packages/storage/src/internals/apis/getProperties.ts @@ -10,7 +10,7 @@ import { GetPropertiesOutput } from '../types/outputs'; /** * Gets the properties of a file. The properties include S3 system metadata and * the user metadata that was provided when uploading the file. - * @param input - The `GetPropertiesWithPathInput` object. + * @param input - The `GetPropertiesInput` object. * @returns Requested object properties. * @throws An `S3Exception` when the underlying S3 service returned error. * @throws A `StorageValidationErrorCode` when API call parameters are invalid. diff --git a/packages/storage/src/internals/apis/remove.ts b/packages/storage/src/internals/apis/remove.ts index c80ea97bba3..8ac90197f7d 100644 --- a/packages/storage/src/internals/apis/remove.ts +++ b/packages/storage/src/internals/apis/remove.ts @@ -9,7 +9,7 @@ import { RemoveOutput } from '../types/outputs'; /** * Remove a file from your S3 bucket. - * @param input - The `RemoveWithPathInput` object. + * @param input - The `RemoveInput` object. * @return Output containing the removed object path. * @throws service: `S3Exception` - S3 service errors thrown while while removing the object. * @throws validation: `StorageValidationErrorCode` - Validation errors thrown