fix databricks export to azure flow + align test config

cube-js · Sep 27, 2024 · 1361d5d · 1361d5d
1 parent 27e0d8c
commit 1361d5d
Show file tree

Hide file tree

Showing 3 changed files with 21 additions and 3 deletions.
diff --git a/packages/cubejs-databricks-jdbc-driver/src/DatabricksDriver.ts b/packages/cubejs-databricks-jdbc-driver/src/DatabricksDriver.ts
@@ -672,15 +672,17 @@ export class DatabricksDriver extends JDBCDriver {
     // this.config.exportBucket includes schema
     // so it looks like:
     // s3://real-bucket-name
-    // wasbs://account.blob.core.windows.net/real-bucket-name
+    // wasbs://real-container-name@account.blob.core.windows.net
     // The extractors in BaseDriver expect just clean bucket name
     const url = new URL(this.config.exportBucket || '');
 
     switch (this.config.bucketType) {
       case 'azure':
         return this.extractFilesFromAzure(
           { azureKey: this.config.azureKey || '' },
-          url.host,
+          // Databricks uses different bucket address form, so we need to transform it
+          // to the one understandable by extractFilesFromAzure implementation
+          `${url.host}/${url.username}`,
           tableName,
         );
       case 's3':

diff --git a/packages/cubejs-testing-drivers/fixtures/databricks-jdbc.json b/packages/cubejs-testing-drivers/fixtures/databricks-jdbc.json
@@ -15,7 +15,7 @@
       "cube": {
         "environment": {
           "CUBEJS_DB_EXPORT_BUCKET_TYPE": "azure",
-          "CUBEJS_DB_EXPORT_BUCKET": "wasbs://coreteamdevtest.blob.core.windows.net/databricks-drivers-tests-preaggs",
+          "CUBEJS_DB_EXPORT_BUCKET": "wasbs://databricks-drivers-tests-preaggs@coreteamdevtest.blob.core.windows.net",
           "CUBEJS_DB_EXPORT_BUCKET_AZURE_KEY": "${DRIVERS_TESTS_CUBEJS_DB_EXPORT_BUCKET_AZURE_KEY}"
         }
       }

diff --git a/...-testing-drivers/test/__snapshots__/databricks-jdbc-export-bucket-azure-full.test.ts.snap b/...-testing-drivers/test/__snapshots__/databricks-jdbc-export-bucket-azure-full.test.ts.snap
@@ -6003,6 +6003,14 @@ Array [
 ]
 `;
 
+exports[`Queries with the @cubejs-backend/databricks-jdbc-driver export-bucket-azure SQL API: metabase count cast to float32 from push down: metabase_count_cast_to_float32_from_push_down 1`] = `
+Array [
+  Object {
+    "a0": 41,
+  },
+]
+`;
+
 exports[`Queries with the @cubejs-backend/databricks-jdbc-driver export-bucket-azure SQL API: post-aggregate percentage of total: post_aggregate_percentage_of_total 1`] = `
 Array [
   Object {
@@ -8296,6 +8304,14 @@ Array [
 
 exports[`Queries with the @cubejs-backend/databricks-jdbc-driver export-bucket-azure filtering Products: startsWith filter + dimensions + order, third 1`] = `Array []`;
 
+exports[`Queries with the @cubejs-backend/databricks-jdbc-driver export-bucket-azure pre-aggregations Customers: running total without time dimension 1`] = `
+Array [
+  Object {
+    "Customers.runningTotal": "41",
+  },
+]
+`;
+
 exports[`Queries with the @cubejs-backend/databricks-jdbc-driver export-bucket-azure querying BigECommerce: null boolean 1`] = `
 Array [
   Object {