diff --git a/packages/storage/__tests__/providers/s3/apis/internal/list.test.ts b/packages/storage/__tests__/providers/s3/apis/internal/list.test.ts index 4165926ad6a..e861652a90e 100644 --- a/packages/storage/__tests__/providers/s3/apis/internal/list.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/internal/list.test.ts @@ -184,11 +184,11 @@ describe('list API', () => { expect(listObjectsV2).toHaveBeenCalledTimes(1); await expect(listObjectsV2).toBeLastCalledWithConfigAndInput( listObjectClientConfig, - { + expect.objectContaining({ Bucket: bucket, MaxKeys: 1000, Prefix: expectedKey, - }, + }), ); }); }); @@ -226,12 +226,12 @@ describe('list API', () => { expect(listObjectsV2).toHaveBeenCalledTimes(1); await expect(listObjectsV2).toBeLastCalledWithConfigAndInput( listObjectClientConfig, - { + expect.objectContaining({ Bucket: bucket, Prefix: expectedKey, ContinuationToken: nextToken, MaxKeys: customPageSize, - }, + }), ); }); }); @@ -260,11 +260,11 @@ describe('list API', () => { expect(listObjectsV2).toHaveBeenCalledTimes(1); await expect(listObjectsV2).toBeLastCalledWithConfigAndInput( listObjectClientConfig, - { + expect.objectContaining({ Bucket: bucket, MaxKeys: 1000, Prefix: expectedKey, - }, + }), ); }); }); @@ -297,23 +297,23 @@ describe('list API', () => { await expect(listObjectsV2).toHaveBeenNthCalledWithConfigAndInput( 1, listObjectClientConfig, - { + expect.objectContaining({ Bucket: bucket, Prefix: expectedKey, MaxKeys: 1000, ContinuationToken: undefined, - }, + }), ); // last input receives TEST_TOKEN as the Continuation Token await expect(listObjectsV2).toHaveBeenNthCalledWithConfigAndInput( 3, listObjectClientConfig, - { + expect.objectContaining({ Bucket: bucket, Prefix: expectedKey, MaxKeys: 1000, ContinuationToken: nextToken, - }, + }), ); }); }, @@ -348,11 +348,11 @@ describe('list API', () => { region: mockRegion, userAgentValue: expect.any(String), }, - { + expect.objectContaining({ Bucket: mockBucketName, MaxKeys: 1000, Prefix: `public/${inputKey}`, - }, + }), ); }); @@ -382,11 +382,11 @@ describe('list API', () => { region, userAgentValue: expect.any(String), }, - { + expect.objectContaining({ Bucket: bucket, MaxKeys: 1000, Prefix: `public/${inputKey}`, - }, + }), ); }); }); @@ -444,11 +444,11 @@ describe('list API', () => { expect(listObjectsV2).toHaveBeenCalledTimes(1); await expect(listObjectsV2).toBeLastCalledWithConfigAndInput( listObjectClientConfig, - { + expect.objectContaining({ Bucket: bucket, MaxKeys: 1000, Prefix: resolvePath(inputPath), - }, + }), ); }, ); @@ -487,12 +487,12 @@ describe('list API', () => { expect(listObjectsV2).toHaveBeenCalledTimes(1); await expect(listObjectsV2).toBeLastCalledWithConfigAndInput( listObjectClientConfig, - { + expect.objectContaining({ Bucket: bucket, Prefix: resolvePath(inputPath), ContinuationToken: nextToken, MaxKeys: customPageSize, - }, + }), ); }, ); @@ -516,11 +516,11 @@ describe('list API', () => { expect(listObjectsV2).toHaveBeenCalledTimes(1); await expect(listObjectsV2).toBeLastCalledWithConfigAndInput( listObjectClientConfig, - { + expect.objectContaining({ Bucket: bucket, MaxKeys: 1000, Prefix: resolvePath(path), - }, + }), ); }, ); @@ -552,23 +552,23 @@ describe('list API', () => { await expect(listObjectsV2).toHaveBeenNthCalledWithConfigAndInput( 1, listObjectClientConfig, - { + expect.objectContaining({ Bucket: bucket, Prefix: resolvedPath, MaxKeys: 1000, ContinuationToken: undefined, - }, + }), ); // last input receives TEST_TOKEN as the Continuation Token await expect(listObjectsV2).toHaveBeenNthCalledWithConfigAndInput( 3, listObjectClientConfig, - { + expect.objectContaining({ Bucket: bucket, Prefix: resolvedPath, MaxKeys: 1000, ContinuationToken: nextToken, - }, + }), ); }, ); @@ -602,11 +602,11 @@ describe('list API', () => { region: mockRegion, userAgentValue: expect.any(String), }, - { + expect.objectContaining({ Bucket: mockBucketName, MaxKeys: 1000, Prefix: 'path/', - }, + }), ); }); @@ -636,11 +636,11 @@ describe('list API', () => { region, userAgentValue: expect.any(String), }, - { + expect.objectContaining({ Bucket: bucket, MaxKeys: 1000, Prefix: 'path/', - }, + }), ); }); }); @@ -664,11 +664,11 @@ describe('list API', () => { expect(listObjectsV2).toHaveBeenCalledTimes(1); await expect(listObjectsV2).toBeLastCalledWithConfigAndInput( listObjectClientConfig, - { + expect.objectContaining({ Bucket: bucket, MaxKeys: 1000, Prefix: 'public/', - }, + }), ); expect(error.$metadata.httpStatusCode).toBe(404); } @@ -772,12 +772,12 @@ describe('list API', () => { expect(listObjectsV2).toHaveBeenCalledTimes(1); await expect(listObjectsV2).toBeLastCalledWithConfigAndInput( listObjectClientConfig, - { + expect.objectContaining({ Bucket: bucket, MaxKeys: 1000, Prefix: mockedPath, Delimiter: '/', - }, + }), ); }); @@ -806,12 +806,12 @@ describe('list API', () => { expect(listObjectsV2).toHaveBeenCalledTimes(1); await expect(listObjectsV2).toBeLastCalledWithConfigAndInput( listObjectClientConfig, - { + expect.objectContaining({ Bucket: bucket, MaxKeys: 1000, Prefix: mockedPath, Delimiter: '/', - }, + }), ); }); @@ -828,12 +828,12 @@ describe('list API', () => { expect(listObjectsV2).toHaveBeenCalledTimes(1); await expect(listObjectsV2).toBeLastCalledWithConfigAndInput( listObjectClientConfig, - { + expect.objectContaining({ Bucket: bucket, MaxKeys: 3, Prefix: mockedPath, Delimiter: '/', - }, + }), ); }); @@ -850,12 +850,12 @@ describe('list API', () => { expect(listObjectsV2).toHaveBeenCalledTimes(1); await expect(listObjectsV2).toBeLastCalledWithConfigAndInput( listObjectClientConfig, - { + expect.objectContaining({ Bucket: bucket, MaxKeys: 1000, Prefix: mockedPath, Delimiter: '-', - }, + }), ); }); @@ -871,12 +871,12 @@ describe('list API', () => { expect(listObjectsV2).toHaveBeenCalledTimes(1); await expect(listObjectsV2).toBeLastCalledWithConfigAndInput( listObjectClientConfig, - { + expect.objectContaining({ Bucket: bucket, MaxKeys: 1000, Prefix: mockedPath, Delimiter: undefined, - }, + }), ); }); @@ -887,12 +887,12 @@ describe('list API', () => { expect(listObjectsV2).toHaveBeenCalledTimes(1); await expect(listObjectsV2).toBeLastCalledWithConfigAndInput( listObjectClientConfig, - { + expect.objectContaining({ Bucket: bucket, MaxKeys: 1000, Prefix: mockedPath, Delimiter: undefined, - }, + }), ); }); }); @@ -1024,4 +1024,84 @@ describe('list API', () => { }); }); }); + + describe.each([ + { + type: 'Prefix', + listFunction: (options?: any) => + list(Amplify, { + prefix: 'some folder with unprintable unicode/', + options, + }), + key: 'key', + }, + { + type: 'Path', + listFunction: (options?: any) => + list(Amplify, { + path: 'public/some folder with unprintable unicode/', + options, + }), + key: 'path', + }, + ])('Encoding for List with $type', ({ listFunction, key }) => { + afterEach(() => { + mockListObject.mockClear(); + }); + it('should decode encoded list output', async () => { + const encodedBadKeys = [ + 'some+folder+with+spaces/', + 'real%0A%0A%0A%0A%0A%0A%0A%0A%0Afunny%0A%0A%0A%0A%0A%0A%0A%0A%0Abiz', + 'some+folder+with+%E3%81%8A%E3%81%AF%E3%82%88%E3%81%86+multibyte+unicode/', + 'bad%3Cdiv%3Ekey', + 'bad%00key', + 'bad%01key', + ]; + + mockListObject.mockReturnValueOnce({ + Name: bucket, + Prefix: 'public/some+folder+with++unprintable+unicode/', + Delimiter: 'bad%08key', + MaxKeys: 1000, + StartAfter: 'bad%7Fbiz/', + EncodingType: 'url', + Contents: encodedBadKeys.map(badKey => ({ + ...listObjectClientBaseResultItem, + Key: key === 'key' ? `public/${badKey}` : badKey, + })), + }); + + const result = await listFunction({ + subpathStrategy: { strategy: 'exclude', delimiter: 'bad\x08key' }, + }); + + expect(listObjectsV2).toBeLastCalledWithConfigAndInput( + expect.any(Object), + expect.objectContaining({ + Bucket: bucket, + EncodingType: 'url', + }), + ); + + const decodedKeys = [ + 'some folder with spaces/', + 'real\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x0afunny\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x0abiz', + 'some folder with おはよう multibyte unicode/', + 'bad
key', + 'bad\x00key', + 'bad\x01key', + ]; + + const expectedResult = { + items: decodedKeys.map(decodedKey => ({ + [key]: decodedKey, + eTag: 'eTag', + lastModified: 'lastModified', + size: 'size', + })), + nextToken: undefined, + }; + expect(result).toEqual(expectedResult); + }); + }); }); diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listObjectsV2.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listObjectsV2.ts index aa60b74f906..1a7ff38e323 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listObjectsV2.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listObjectsV2.ts @@ -217,7 +217,7 @@ const listObjectsV2ErrorCase403: ApiFunctionalTestCase = [ NoSuchKey The resource you requested does not exist - /mybucket/myfoto.jpg + /mybucket/myfoto.jpg 4442587FB7D0A2F9 `, }, @@ -420,6 +420,175 @@ const listObjectsV2HappyCaseCustomEndpoint: ApiFunctionalTestCase< }) as any, ]; +const listObjectsV2HappyCaseWithEncoding: ApiFunctionalTestCase< + typeof listObjectsV2 +> = [ + 'happy case', + 'listObjectsV2 unicode values with encoding', + listObjectsV2, + { + ...defaultConfig, + }, + { + Bucket: 'bucket', + Prefix: 'Prefix', + EncodingType: 'url', + }, + expect.any(Object), + { + status: 200, + headers: DEFAULT_RESPONSE_HEADERS, + body: ` + + bucket + some%20folder%20with%20%00%20unprintable%20unicode%2F + bad%08key + bad%01key + 6 + 101 + url + false + + public/bad%3Cdiv%3Ekey + 2024-11-05T18:13:11.000Z + "c0e066cc5238dd7937e464fe7572b71a" + 5455 + STANDARD + + + bad%00key + 2024-11-05T18:13:11.000Z + "c0e066cc5238dd7937e464fe7572b71a" + 5455 + STANDARD + + + public/bad%7Fkey + 2024-11-05T18:13:11.000Z + "c0e066cc5238dd7937e464fe7572b71a" + 5455 + STANDARD + + + public/some%20folder%20with%20spaces%2F + + + public/real%0A%0A%0A%0A%0A%0A%0A%0A%0Afunny%0A%0A%0A%0A%0A%0A%0A%0A%0Abiz%2F + + + public/some%20folder%20with%20%E3%81%8A%E3%81%AF%E3%82%88%E3%81%86%20multibyte%20unicode%2F + +`, + }, + expect.objectContaining({ + CommonPrefixes: [ + { + Prefix: 'public/some%20folder%20with%20spaces%2F', + }, + { + Prefix: + 'public/real%0A%0A%0A%0A%0A%0A%0A%0A%0Afunny%0A%0A%0A%0A%0A%0A%0A%0A%0Abiz%2F', + }, + { + Prefix: + 'public/some%20folder%20with%20%E3%81%8A%E3%81%AF%E3%82%88%E3%81%86%20multibyte%20unicode%2F', + }, + ], + Contents: [ + { + Key: 'public/bad%3Cdiv%3Ekey', + LastModified: new Date('2024-11-05T18:13:11.000Z'), + ETag: '"c0e066cc5238dd7937e464fe7572b71a"', + Size: 5455, + StorageClass: 'STANDARD', + }, + { + Key: 'bad%00key', + LastModified: new Date('2024-11-05T18:13:11.000Z'), + ETag: '"c0e066cc5238dd7937e464fe7572b71a"', + Size: 5455, + StorageClass: 'STANDARD', + }, + { + Key: 'public/bad%7Fkey', + LastModified: new Date('2024-11-05T18:13:11.000Z'), + ETag: '"c0e066cc5238dd7937e464fe7572b71a"', + Size: 5455, + StorageClass: 'STANDARD', + }, + ], + Prefix: 'some%20folder%20with%20%00%20unprintable%20unicode%2F', + Delimiter: 'bad%08key', + StartAfter: 'bad%01key', + EncodingType: 'url', + Name: 'bucket', + }) as any, +]; + +const listObjectsV2ErrorCaseNoEncoding: ApiFunctionalTestCase< + typeof listObjectsV2 +> = [ + 'error case', + 'listObjectsV2 unicode values without encoding', + listObjectsV2, + { + ...defaultConfig, + }, + { + Bucket: 'bucket', + Prefix: 'Prefix', + EncodingType: undefined, + }, + expect.any(Object), + { + status: 200, + headers: DEFAULT_RESPONSE_HEADERS, + body: ` + + badname + bad\x01key + 5 + 101 + bad\x08key + false + おはよう multibyte unicode + + public/bad
key + 2024-11-05T18:13:11.000Z + "c0e066cc5238dd7937e464fe7572b71a" + 5455 + STANDARD + + + bad\x00key + 2024-11-05T18:13:11.000Z + "c0e066cc5238dd7937e464fe7572b71a" + 5455 + STANDARD + + + public/bad\x7fkey + 2024-11-05T18:13:11.000Z + "c0e066cc5238dd7937e464fe7572b71a" + 5455 + STANDARD + + + public/some folder with spaces/ + + + public/some folder with \x00 unprintable unicode/ + +`, + }, + { + message: 'An unknown error has occurred.', + name: 'Unknown', + }, +]; + export default [ listObjectsV2HappyCaseTruncated, listObjectsV2HappyCaseComplete, @@ -428,4 +597,6 @@ export default [ listObjectsV2ErrorCaseMissingTruncated, listObjectsV2ErrorCaseMissingToken, listObjectsV2ErrorCase403, + listObjectsV2HappyCaseWithEncoding, + listObjectsV2ErrorCaseNoEncoding, ]; diff --git a/packages/storage/src/providers/s3/apis/internal/list.ts b/packages/storage/src/providers/s3/apis/internal/list.ts index 19fbf050653..ef8d277b48d 100644 --- a/packages/storage/src/providers/s3/apis/internal/list.ts +++ b/packages/storage/src/providers/s3/apis/internal/list.ts @@ -14,6 +14,7 @@ import { } from '../../types'; import { resolveS3ConfigAndInput, + urlDecode, validateBucketOwnerID, validateStorageOperationInputWithPrefix, } from '../../utils'; @@ -85,6 +86,7 @@ export const list = async ( ContinuationToken: options?.listAll ? undefined : options?.nextToken, Delimiter: getDelimiter(options), ExpectedBucketOwner: options?.expectedBucketOwner, + EncodingType: 'url', }; logger.debug(`listing items from "${listParams.Prefix}"`); @@ -159,16 +161,18 @@ const _listWithPrefix = async ({ listParamsClone, ); - validateEchoedElements(listParamsClone, response); + const listOutput = decodeEncodedElements(response); - if (!response?.Contents) { + validateEchoedElements(listParamsClone, listOutput); + + if (!listOutput?.Contents) { return { items: [], }; } return { - items: response.Contents.map(item => ({ + items: listOutput.Contents.map(item => ({ key: generatedPrefix ? item.Key!.substring(generatedPrefix.length) : item.Key!, @@ -176,7 +180,7 @@ const _listWithPrefix = async ({ lastModified: item.LastModified, size: item.Size, })), - nextToken: response.NextContinuationToken, + nextToken: listOutput.NextContinuationToken, }; }; @@ -221,7 +225,7 @@ const _listWithPath = async ({ listParamsClone.MaxKeys = MAX_PAGE_SIZE; } - const listOutput = await listObjectsV2( + const response = await listObjectsV2( { ...s3Config, userAgentValue: getStorageUserAgentValue(StorageAction.List), @@ -229,6 +233,8 @@ const _listWithPath = async ({ listParamsClone, ); + const listOutput = decodeEncodedElements(response); + validateEchoedElements(listParamsClone, listOutput); const { @@ -295,3 +301,45 @@ const validateEchoedElements = ( throw new IntegrityError(); } }; + +/** + * Decodes URL-encoded elements in the S3 `ListObjectsV2Output` response when `EncodingType` is `'url'`. + * Applies to values for 'Delimiter', 'Prefix', 'StartAfter' and 'Key' in the response. + */ +const decodeEncodedElements = ( + listOutput: ListObjectsV2Output, +): ListObjectsV2Output => { + if (listOutput.EncodingType !== 'url') { + return listOutput; + } + + const decodedListOutput = { ...listOutput }; + + // Decode top-level properties + (['Delimiter', 'Prefix', 'StartAfter'] as const).forEach(prop => { + const value = listOutput[prop]; + if (typeof value === 'string') { + decodedListOutput[prop] = urlDecode(value); + } + }); + + // Decode 'Key' in each item of 'Contents', if it exists + if (listOutput.Contents) { + decodedListOutput.Contents = listOutput.Contents.map(content => ({ + ...content, + Key: content.Key ? urlDecode(content.Key) : content.Key, + })); + } + + // Decode 'Prefix' in each item of 'CommonPrefixes', if it exists + if (listOutput.CommonPrefixes) { + decodedListOutput.CommonPrefixes = listOutput.CommonPrefixes.map( + content => ({ + ...content, + Prefix: content.Prefix ? urlDecode(content.Prefix) : content.Prefix, + }), + ); + } + + return decodedListOutput; +}; diff --git a/packages/storage/src/providers/s3/utils/client/s3data/listObjectsV2.ts b/packages/storage/src/providers/s3/utils/client/s3data/listObjectsV2.ts index 664a8c7e273..6caa8a46a8e 100644 --- a/packages/storage/src/providers/s3/utils/client/s3data/listObjectsV2.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/listObjectsV2.ts @@ -98,6 +98,7 @@ const listObjectsV2Deserializer = async ( $metadata: parseMetadata(response), ...contents, }; + validateCorroboratingElements(output); return output; diff --git a/packages/storage/src/providers/s3/utils/index.ts b/packages/storage/src/providers/s3/utils/index.ts index 3a1a05c6a8d..a709e025988 100644 --- a/packages/storage/src/providers/s3/utils/index.ts +++ b/packages/storage/src/providers/s3/utils/index.ts @@ -8,3 +8,4 @@ export { validateBucketOwnerID } from './validateBucketOwnerID'; export { validateStorageOperationInput } from './validateStorageOperationInput'; export { validateStorageOperationInputWithPrefix } from './validateStorageOperationInputWithPrefix'; export { isInputWithPath } from './isInputWithPath'; +export { urlDecode } from './urlDecoder'; diff --git a/packages/storage/src/providers/s3/utils/urlDecoder.ts b/packages/storage/src/providers/s3/utils/urlDecoder.ts new file mode 100644 index 00000000000..e812c8a23f4 --- /dev/null +++ b/packages/storage/src/providers/s3/utils/urlDecoder.ts @@ -0,0 +1,13 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Decodes a URL-encoded string by replacing '+' characters with spaces and applying `decodeURIComponent`. + * Reference: + * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/decodeURIComponent#decoding_query_parameters_from_a_url + * @param {string} value - The URL-encoded string to decode. + * @returns {string} The decoded string. + */ +export const urlDecode = (value: string): string => { + return decodeURIComponent(value.replace(/\+/g, ' ')); +};