From 4094c82f0b62f5a72b7b3801c5b5f406f7ebb893 Mon Sep 17 00:00:00 2001 From: flakey5 <73616808+flakey5@users.noreply.github.com> Date: Sat, 7 Oct 2023 11:32:59 -0700 Subject: [PATCH] fix e2e tests --- src/env.ts | 4 +- src/handlers/strategies/directoryListing.ts | 4 +- tests/e2e/directory.test.ts | 51 +++++++++++++------ tests/e2e/test-data/expected-html/dist.txt | 2 +- ...2.xml => ListObjectsV2-does-not-exist.xml} | 2 +- .../expected-s3/ListObjectsV2-exists.xml | 16 ++++++ wrangler.toml | 38 +++++++------- 7 files changed, 77 insertions(+), 40 deletions(-) rename tests/e2e/test-data/expected-s3/{ListObjectsV2.xml => ListObjectsV2-does-not-exist.xml} (89%) create mode 100644 tests/e2e/test-data/expected-s3/ListObjectsV2-exists.xml diff --git a/src/env.ts b/src/env.ts index 2bc9926..7f9865a 100644 --- a/src/env.ts +++ b/src/env.ts @@ -8,9 +8,9 @@ export interface Env { */ R2_BUCKET: R2Bucket; /** - * Account tag/public id of the account that the worker is deployed on + * Endpoint to hit when using the S3 api. */ - CF_ACCOUNT_ID: string; + S3_ENDPOINT: string; /** * Id of the api token used for the S3 api. * The token needs >=Object Read only permissions diff --git a/src/handlers/strategies/directoryListing.ts b/src/handlers/strategies/directoryListing.ts index 7a923a3..8cfec11 100644 --- a/src/handlers/strategies/directoryListing.ts +++ b/src/handlers/strategies/directoryListing.ts @@ -138,7 +138,7 @@ async function fetchR2Result( break; } catch (err) { // Got an error, let's log it and retry - console.log(`R2 ListObjectsV2 error: ${err}`); + console.error(`R2 ListObjectsV2 error: ${err}`); retriesRemaining--; } @@ -177,7 +177,7 @@ export async function listDirectory( // using it for now. const client = new S3Client({ region: 'auto', - endpoint: `https://${env.CF_ACCOUNT_ID}.r2.cloudflarestorage.com`, + endpoint: env.S3_ENDPOINT, credentials: { accessKeyId: env.S3_ACCESS_KEY_ID, secretAccessKey: env.S3_ACCESS_KEY_SECRET, diff --git a/tests/e2e/directory.test.ts b/tests/e2e/directory.test.ts index 2d642a3..cc64212 100644 --- a/tests/e2e/directory.test.ts +++ b/tests/e2e/directory.test.ts @@ -1,27 +1,43 @@ import { after, before, describe, it } from 'node:test'; import assert from 'node:assert'; +import { readFileSync } from 'node:fs'; import { readFile } from 'node:fs/promises'; +import http from 'http'; import { Miniflare } from 'miniflare'; -import { MockAgent, setGlobalDispatcher } from 'undici'; -async function getFetchMockAgent(cfAccountId: string): Promise { - const agent = new MockAgent(); - setGlobalDispatcher(agent); +async function startS3Mock(): Promise { + const server = http.createServer((req, res) => { + const url = new URL(req.url!, `http://${req.headers.host}`); + + let xmlFilePath = './tests/e2e/test-data/expected-s3/'; + if ( + ['nodejs/release/', 'nodejs/', 'metrics/'].includes( + url.searchParams.get('prefix')! + ) + ) { + xmlFilePath += 'ListObjectsV2-exists.xml'; + } else { + xmlFilePath += 'ListObjectsV2-does-not-exist.xml'; + } + + const listObjectsResponse = readFileSync(xmlFilePath, { + encoding: 'utf-8', + }); - const pool = agent.get(`https://dist-prod.${cfAccountId}.r2.cloudflarestorage.com`); - - const listObjectsResponse = await readFile('./tests/e2e/test-data/expected-s3/ListObjectsV2.xml', { encoding: 'utf-8' }); - pool.intercept({ path: '/' }).reply(200, listObjectsResponse); + res.write(listObjectsResponse); + res.end(); + }); + server.listen(8080); - return agent; + return server; } describe('Directory Tests (Restricted Directory Listing)', () => { + let s3Mock: http.Server; let mf: Miniflare; let url: URL; before(async () => { - const CF_ACCOUNT_ID = 'testing'; - const fetchMock = await getFetchMockAgent(CF_ACCOUNT_ID); + s3Mock = await startS3Mock(); // Setup miniflare mf = new Miniflare({ @@ -29,16 +45,18 @@ describe('Directory Tests (Restricted Directory Listing)', () => { modules: true, bindings: { BUCKET_NAME: 'dist-prod', - CF_ACCOUNT_ID, + // S3_ENDPOINT needs to be an ip here otherwise s3 sdk will try to hit + // the bucket's subdomain (e.g. http://dist-prod.localhost) + S3_ENDPOINT: 'http://127.0.0.1:8080', S3_ACCESS_KEY_ID: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', - S3_ACCESS_KEY_SECRET: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + S3_ACCESS_KEY_SECRET: + 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', DIRECTORY_LISTING: 'restricted', FILE_CACHE_CONTROL: 'no-store', DIRECTORY_CACHE_CONTROL: 'no-store', }, r2Persist: './tests/e2e/test-data', r2Buckets: ['R2_BUCKET'], - fetchMock, }); // Wait for it Miniflare to start @@ -118,5 +136,8 @@ describe('Directory Tests (Restricted Directory Listing)', () => { }); // Cleanup Miniflare - after(async () => mf.dispose()); + after(async () => { + await mf.dispose(); + s3Mock.close(); + }); }); diff --git a/tests/e2e/test-data/expected-html/dist.txt b/tests/e2e/test-data/expected-html/dist.txt index c7cff4d..0909a6b 100644 --- a/tests/e2e/test-data/expected-html/dist.txt +++ b/tests/e2e/test-data/expected-html/dist.txt @@ -1 +1 @@ -Index of /dist/

Index of /dist/

FilenameModifiedSize
../--
latest/--
index.json2023-09-12 05:43Z18 B
+Index of /dist/

Index of /dist/

FilenameModifiedSize
../--
latest/--
index.json2023-09-12 05:43Z18 B
\ No newline at end of file diff --git a/tests/e2e/test-data/expected-s3/ListObjectsV2.xml b/tests/e2e/test-data/expected-s3/ListObjectsV2-does-not-exist.xml similarity index 89% rename from tests/e2e/test-data/expected-s3/ListObjectsV2.xml rename to tests/e2e/test-data/expected-s3/ListObjectsV2-does-not-exist.xml index df9fef9..d040ff3 100644 --- a/tests/e2e/test-data/expected-s3/ListObjectsV2.xml +++ b/tests/e2e/test-data/expected-s3/ListObjectsV2-does-not-exist.xml @@ -4,4 +4,4 @@ 1000 false - \ No newline at end of file + diff --git a/tests/e2e/test-data/expected-s3/ListObjectsV2-exists.xml b/tests/e2e/test-data/expected-s3/ListObjectsV2-exists.xml new file mode 100644 index 0000000..5bed3e2 --- /dev/null +++ b/tests/e2e/test-data/expected-s3/ListObjectsV2-exists.xml @@ -0,0 +1,16 @@ + + dist-prod + + + 1000 + false + + nodejs/release/latest/ + + + "asd123" + nodejs/release/index.json + 2023-09-12T05:43:00.000Z + 18 + + diff --git a/wrangler.toml b/wrangler.toml index aa602a7..45ba4e8 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -1,52 +1,52 @@ -name = "dist-worker" -main = "src/worker.ts" -compatibility_date = "2023-08-07" -account_id = "07be8d2fbc940503ca1be344714cb0d1" +name = 'dist-worker' +main = 'src/worker.ts' +compatibility_date = '2023-08-07' +account_id = '07be8d2fbc940503ca1be344714cb0d1' # Dev (default) [vars] workers_dev = true ENVIRONMENT = 'dev' -CF_ACCOUNT_ID = '07be8d2fbc940503ca1be344714cb0d1' +S3_ENDPOINT = 'https://07be8d2fbc940503ca1be344714cb0d1.r2.cloudflarestorage.com' DIRECTORY_LISTING = 'on' FILE_CACHE_CONTROL = 'public, max-age=3600, s-maxage=14400' DIRECTORY_CACHE_CONTROL = 'public, max-age=3600, s-maxage=14400' -BUCKET_NAME="dist-prod" +BUCKET_NAME='dist-prod' [[r2_buckets]] -binding = "R2_BUCKET" -preview_bucket_name = "dist-prod" -bucket_name = "dist-prod" +binding = 'R2_BUCKET' +preview_bucket_name = 'dist-prod' +bucket_name = 'dist-prod' # Staging [env.staging] [env.staging.vars] workers_dev = true ENVIRONMENT = 'staging' -CF_ACCOUNT_ID = '07be8d2fbc940503ca1be344714cb0d1' +S3_ENDPOINT = 'https://07be8d2fbc940503ca1be344714cb0d1.r2.cloudflarestorage.com' DIRECTORY_LISTING = 'restricted' FILE_CACHE_CONTROL = 'public, max-age=3600, s-maxage=14400' DIRECTORY_CACHE_CONTROL = 'public, max-age=3600, s-maxage=14400' -BUCKET_NAME="dist-prod" +BUCKET_NAME='dist-prod' [[env.staging.r2_buckets]] -binding = "R2_BUCKET" -preview_bucket_name = "dist-prod" -bucket_name = "dist-prod" +binding = 'R2_BUCKET' +preview_bucket_name = 'dist-prod' +bucket_name = 'dist-prod' # Prod [env.prod] [env.prod.vars] workers_dev = false ENVIRONMENT = 'prod' -CF_ACCOUNT_ID = '07be8d2fbc940503ca1be344714cb0d1' +S3_ENDPOINT = 'https://07be8d2fbc940503ca1be344714cb0d1.r2.cloudflarestorage.com' DIRECTORY_LISTING = 'restricted' FILE_CACHE_CONTROL = 'public, max-age=3600, s-maxage=14400' DIRECTORY_CACHE_CONTROL = 'public, max-age=3600, s-maxage=14400' -BUCKET_NAME="dist-prod" +BUCKET_NAME='dist-prod' [[env.prod.r2_buckets]] -binding = "R2_BUCKET" -preview_bucket_name = "dist-prod" -bucket_name = "dist-prod" +binding = 'R2_BUCKET' +preview_bucket_name = 'dist-prod' +bucket_name = 'dist-prod'