fix e2e tests

src/env.ts

@@ -8,9 +8,9 @@ export interface Env {
    */
   R2_BUCKET: R2Bucket;
   /**
-   * Account tag/public id of the account that the worker is deployed on
+   * Endpoint to hit when using the S3 api.
    */
-  CF_ACCOUNT_ID: string;
+  S3_ENDPOINT: string;
   /**
    * Id of the api token used for the S3 api.
    * The token needs >=Object Read only permissions

src/handlers/strategies/directoryListing.ts

@@ -138,7 +138,7 @@ async function fetchR2Result(
       break;
     } catch (err) {
       // Got an error, let's log it and retry
-      console.log(`R2 ListObjectsV2 error: ${err}`);
+      console.error(`R2 ListObjectsV2 error: ${err}`);
 
       retriesRemaining--;
     }
@@ -177,7 +177,7 @@ export async function listDirectory(
   //  using it for now.
   const client = new S3Client({
     region: 'auto',
-    endpoint: `https://${env.CF_ACCOUNT_ID}.r2.cloudflarestorage.com`,
+    endpoint: env.S3_ENDPOINT,
     credentials: {
       accessKeyId: env.S3_ACCESS_KEY_ID,
       secretAccessKey: env.S3_ACCESS_KEY_SECRET,

tests/e2e/directory.test.ts

@@ -1,44 +1,62 @@
 import { after, before, describe, it } from 'node:test';
 import assert from 'node:assert';
+import { readFileSync } from 'node:fs';
 import { readFile } from 'node:fs/promises';
+import http from 'http';
 import { Miniflare } from 'miniflare';
-import { MockAgent, setGlobalDispatcher } from 'undici';
 
-async function getFetchMockAgent(cfAccountId: string): Promise<MockAgent> {
-  const agent = new MockAgent();
-  setGlobalDispatcher(agent);
+async function startS3Mock(): Promise<http.Server> {
+  const server = http.createServer((req, res) => {
+    const url = new URL(req.url!, `http://${req.headers.host}`);
+
+    let xmlFilePath = './tests/e2e/test-data/expected-s3/';
+    if (
+      ['nodejs/release/', 'nodejs/', 'metrics/'].includes(
+        url.searchParams.get('prefix')!
+      )
+    ) {
+      xmlFilePath += 'ListObjectsV2-exists.xml';
+    } else {
+      xmlFilePath += 'ListObjectsV2-does-not-exist.xml';
+    }
+
+    const listObjectsResponse = readFileSync(xmlFilePath, {
+      encoding: 'utf-8',
+    });
 
-  const pool = agent.get(`https://dist-prod.${cfAccountId}.r2.cloudflarestorage.com`);
-
-  const listObjectsResponse = await readFile('./tests/e2e/test-data/expected-s3/ListObjectsV2.xml', { encoding: 'utf-8' });
-  pool.intercept({ path: '/' }).reply(200, listObjectsResponse);
+    res.write(listObjectsResponse);
+    res.end();
+  });
+  server.listen(8080);
 
-  return agent;
+  return server;
 }
 
 describe('Directory Tests (Restricted Directory Listing)', () => {
+  let s3Mock: http.Server;
   let mf: Miniflare;
   let url: URL;
   before(async () => {
-    const CF_ACCOUNT_ID = 'testing';
-    const fetchMock = await getFetchMockAgent(CF_ACCOUNT_ID);
+    s3Mock = await startS3Mock();
 
     // Setup miniflare
     mf = new Miniflare({
       scriptPath: './dist/worker.js',
       modules: true,
       bindings: {
         BUCKET_NAME: 'dist-prod',
-        CF_ACCOUNT_ID,
+        // S3_ENDPOINT needs to be an ip here otherwise s3 sdk will try to hit
+        //  the bucket's subdomain (e.g. http://dist-prod.localhost)
+        S3_ENDPOINT: 'http://127.0.0.1:8080',
         S3_ACCESS_KEY_ID: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
-        S3_ACCESS_KEY_SECRET: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
+        S3_ACCESS_KEY_SECRET:
+          'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
         DIRECTORY_LISTING: 'restricted',
         FILE_CACHE_CONTROL: 'no-store',
         DIRECTORY_CACHE_CONTROL: 'no-store',
       },
       r2Persist: './tests/e2e/test-data',
       r2Buckets: ['R2_BUCKET'],
-      fetchMock,
     });
 
     // Wait for it Miniflare to start
@@ -118,5 +136,8 @@ describe('Directory Tests (Restricted Directory Listing)', () => {
   });
 
   // Cleanup Miniflare
-  after(async () => mf.dispose());
+  after(async () => {
+    await mf.dispose();
+    s3Mock.close();
+  });
 });

tests/e2e/test-data/expected-html/dist.txt

@@ -1 +1 @@
-<!DOCTYPE html><html><head><title>Index of /dist/</title><meta name='viewport' content='width=device-width, initial-scale=1.0' /><meta charset='utf-8' /><style type='text/css'>td { padding-right: 16px; text-align: right; font-family: monospace }td:nth-of-type(1) { text-align: left; overflow-wrap: anywhere }td:nth-of-type(3) { white-space: nowrap } th { text-align: left; } @media(prefers-color-scheme: dark) { body { color: white; background-color:#1c1b22; } a { color: #3391ff; } a:visited { color: #C63B65; } }</style></head><body><h1>Index of /dist/</h1><table><tr><th>Filename</th><th>Modified</th><th>Size</th></tr><tr><td><a href='../'>../</a></td><td>-</td><td>-</td></tr><tr><td><a href='/dist/latest/'>latest/</a></td><td>-</td><td>-</td></tr><tr><td><a href='/dist/index.json'>index.json</a></td><td>2023-09-12 05:43Z</td><td>18 B</td></tr></table></body></html>
+<!DOCTYPE html><html><head><title>Index of /dist/</title><meta name='viewport' content='width=device-width, initial-scale=1.0' /><meta charset='utf-8' /><style type='text/css'>td { padding-right: 16px; text-align: right; font-family: monospace }td:nth-of-type(1) { text-align: left; overflow-wrap: anywhere }td:nth-of-type(3) { white-space: nowrap } th { text-align: left; } @media(prefers-color-scheme: dark) { body { color: white; background-color:#1c1b22; } a { color: #3391ff; } a:visited { color: #C63B65; } }</style></head><body><h1>Index of /dist/</h1><table><tr><th>Filename</th><th>Modified</th><th>Size</th></tr><tr><td><a href='../'>../</a></td><td>-</td><td>-</td></tr><tr><td><a href='/dist/latest/'>latest/</a></td><td>-</td><td>-</td></tr><tr><td><a href='/dist/index.json'>index.json</a></td><td>2023-09-12 05:43Z</td><td>18 B</td></tr></table></body></html>

tests/e2e/test-data/expected-s3/ListObjectsV2.xml → tests/e2e/test-data/expected-s3/ListObjectsV2-does-not-exist.xml

@@ -4,4 +4,4 @@
   <Marker />
   <MaxKeys>1000</MaxKeys>
   <IsTruncated>false</IsTruncated>
-</ListBucketResult>
+</ListBucketResult>

tests/e2e/test-data/expected-s3/ListObjectsV2-exists.xml

@@ -0,0 +1,16 @@
+<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01">
+  <Name>dist-prod</Name>
+  <Prefix />
+  <Marker />
+  <MaxKeys>1000</MaxKeys>
+  <IsTruncated>false</IsTruncated>
+  <CommonPrefixes>
+    <Prefix>nodejs/release/latest/</Prefix>
+  </CommonPrefixes>
+  <Contents>
+    <ETag>"asd123"</ETag>
+    <Key>nodejs/release/index.json</Key>
+    <LastModified>2023-09-12T05:43:00.000Z</LastModified>
+    <Size>18</Size>
+  </Contents>
+</ListBucketResult>

wrangler.toml

@@ -1,52 +1,52 @@
-name = "dist-worker"
-main = "src/worker.ts"
-compatibility_date = "2023-08-07"
-account_id = "07be8d2fbc940503ca1be344714cb0d1"
+name = 'dist-worker'
+main = 'src/worker.ts'
+compatibility_date = '2023-08-07'
+account_id = '07be8d2fbc940503ca1be344714cb0d1'
 
 
 # Dev (default)
 [vars]
 workers_dev = true
 ENVIRONMENT = 'dev'
-CF_ACCOUNT_ID = '07be8d2fbc940503ca1be344714cb0d1'
+S3_ENDPOINT = 'https://07be8d2fbc940503ca1be344714cb0d1.r2.cloudflarestorage.com'
 DIRECTORY_LISTING = 'on'
 FILE_CACHE_CONTROL = 'public, max-age=3600, s-maxage=14400'
 DIRECTORY_CACHE_CONTROL = 'public, max-age=3600, s-maxage=14400'
-BUCKET_NAME="dist-prod"
+BUCKET_NAME='dist-prod'
 
 [[r2_buckets]]
-binding = "R2_BUCKET"
-preview_bucket_name = "dist-prod"
-bucket_name = "dist-prod"
+binding = 'R2_BUCKET'
+preview_bucket_name = 'dist-prod'
+bucket_name = 'dist-prod'
 
 # Staging
 [env.staging]
 [env.staging.vars]
 workers_dev = true
 ENVIRONMENT = 'staging'
-CF_ACCOUNT_ID = '07be8d2fbc940503ca1be344714cb0d1'
+S3_ENDPOINT = 'https://07be8d2fbc940503ca1be344714cb0d1.r2.cloudflarestorage.com'
 DIRECTORY_LISTING = 'restricted'
 FILE_CACHE_CONTROL = 'public, max-age=3600, s-maxage=14400'
 DIRECTORY_CACHE_CONTROL = 'public, max-age=3600, s-maxage=14400'
-BUCKET_NAME="dist-prod"
+BUCKET_NAME='dist-prod'
 
 [[env.staging.r2_buckets]]
-binding = "R2_BUCKET"
-preview_bucket_name = "dist-prod"
-bucket_name = "dist-prod"
+binding = 'R2_BUCKET'
+preview_bucket_name = 'dist-prod'
+bucket_name = 'dist-prod'
 
 # Prod
 [env.prod]
 [env.prod.vars]
 workers_dev = false
 ENVIRONMENT = 'prod'
-CF_ACCOUNT_ID = '07be8d2fbc940503ca1be344714cb0d1'
+S3_ENDPOINT = 'https://07be8d2fbc940503ca1be344714cb0d1.r2.cloudflarestorage.com'
 DIRECTORY_LISTING = 'restricted'
 FILE_CACHE_CONTROL = 'public, max-age=3600, s-maxage=14400'
 DIRECTORY_CACHE_CONTROL = 'public, max-age=3600, s-maxage=14400'
-BUCKET_NAME="dist-prod"
+BUCKET_NAME='dist-prod'
 
 [[env.prod.r2_buckets]]
-binding = "R2_BUCKET"
-preview_bucket_name = "dist-prod"
-bucket_name = "dist-prod"
+binding = 'R2_BUCKET'
+preview_bucket_name = 'dist-prod'
+bucket_name = 'dist-prod'