switch RedShift Driver to use extractUnloadedFilesFromS3() from BaseD…

…vier impl
cube-js · Sep 25, 2024 · f09ae8e · f09ae8e
1 parent 232fca5
commit f09ae8e
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 33 deletions.
diff --git a/packages/cubejs-redshift-driver/package.json b/packages/cubejs-redshift-driver/package.json
@@ -25,8 +25,6 @@
     "lint:fix": "eslint --fix src/* --ext .ts"
   },
   "dependencies": {
-    "@aws-sdk/client-s3": "^3.17.0",
-    "@aws-sdk/s3-request-presigner": "^3.17.0",
     "@cubejs-backend/base-driver": "^0.36.0",
     "@cubejs-backend/postgres-driver": "^0.36.2",
     "@cubejs-backend/shared": "^0.36.0"

diff --git a/packages/cubejs-redshift-driver/src/RedshiftDriver.ts b/packages/cubejs-redshift-driver/src/RedshiftDriver.ts
@@ -8,8 +8,6 @@ import { getEnv } from '@cubejs-backend/shared';
 import { PostgresDriver, PostgresDriverConfiguration } from '@cubejs-backend/postgres-driver';
 import { DownloadTableCSVData, DriverCapabilities, UnloadOptions } from '@cubejs-backend/base-driver';
 import crypto from 'crypto';
-import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
-import { S3, GetObjectCommand } from '@aws-sdk/client-s3';
 
 interface RedshiftDriverExportRequiredAWS {
   bucketType: 's3',
@@ -213,7 +211,7 @@ export class RedshiftDriver extends PostgresDriver<RedshiftDriverConfiguration>
         UNLOAD ('SELECT ${columns} FROM ${tableName}')
         TO '${bucketType}://${bucketName}/${exportPathName}/'
       `;
-      
+
       // Prefer the unloadArn if it is present
       const credentialQuery = unloadArn
         ? `iam_role '${unloadArn}'`
@@ -234,36 +232,27 @@ export class RedshiftDriver extends PostgresDriver<RedshiftDriverConfiguration>
         };
       }
 
-      const client = new S3({
-        credentials: (keyId && secretKey) ? {
-          accessKeyId: keyId,
-          secretAccessKey: secretKey,
-        } : undefined,
-        region,
-      });
-      const list = await client.listObjectsV2({
-        Bucket: bucketName,
-        Prefix: exportPathName,
-      });
-      if (list && list.Contents) {
-        const csvFile = await Promise.all(
-          list.Contents.map(async (file) => {
-            const command = new GetObjectCommand({
-              Bucket: bucketName,
-              Key: file.Key,
-            });
-            return getSignedUrl(client, command, { expiresIn: 3600 });
-          })
-        );
-
-        return {
-          exportBucketCsvEscapeSymbol: this.config.exportBucketCsvEscapeSymbol,
-          csvFile,
-          types
-        };
+      const csvFile = await this.extractUnloadedFilesFromS3(
+        {
+          credentials: (keyId && secretKey) ? {
+            accessKeyId: keyId,
+            secretAccessKey: secretKey,
+          } : undefined,
+          region,
+        },
+        bucketName,
+        exportPathName,
+      );
+
+      if (csvFile.length === 0) {
+        throw new Error('Unable to UNLOAD table, there are no files in S3 storage');
       }
 
-      throw new Error('Unable to UNLOAD table, there are no files in S3 storage');
+      return {
+        exportBucketCsvEscapeSymbol: this.config.exportBucketCsvEscapeSymbol,
+        csvFile,
+        types
+      };
     } finally {
       conn.removeAllListeners('notice');