From f7c07a7572c95de26db81308194577b32e289e53 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Fri, 13 Sep 2024 12:28:09 +0300 Subject: [PATCH] feat(schema-compiler): Support date_bin function in Cube Store queries (#8684) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(schema-compiler): implement dateBin() in CubeStoreQuery * Add tests for dateBin in CubeStoreQuery * add e2e tests for custom granularities with preaggregations * Remove console.log --- .../src/adapter/CubeStoreQuery.ts | 72 +++++++++++++++++-- .../test/unit/base-query.test.ts | 29 +++++++- .../fixtures/_schemas.json | 23 +++++- .../src/tests/testQueries.ts | 33 +++++++++ 4 files changed, 147 insertions(+), 10 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/CubeStoreQuery.ts b/packages/cubejs-schema-compiler/src/adapter/CubeStoreQuery.ts index f714ded821ab8..99c318412a8cf 100644 --- a/packages/cubejs-schema-compiler/src/adapter/CubeStoreQuery.ts +++ b/packages/cubejs-schema-compiler/src/adapter/CubeStoreQuery.ts @@ -1,4 +1,5 @@ import moment from 'moment-timezone'; +import { parseSqlInterval } from '@cubejs-backend/shared'; import { BaseQuery } from './BaseQuery'; import { BaseFilter } from './BaseFilter'; import { BaseMeasure } from './BaseMeasure'; @@ -42,7 +43,7 @@ export class CubeStoreQuery extends BaseQuery { } public timeStampCast(value) { - return `CAST(${value} as TIMESTAMP)`; // TODO + return `CAST(${value} as TIMESTAMP)`; } public timestampFormat() { @@ -53,18 +54,77 @@ export class CubeStoreQuery extends BaseQuery { return `to_timestamp(${value})`; } - public subtractInterval(date, interval) { - return `DATE_SUB(${date}, INTERVAL '${interval}')`; + public subtractInterval(date: string, interval: string) { + return `DATE_SUB(${date}, INTERVAL ${this.formatInterval(interval)})`; } - public addInterval(date, interval) { - return `DATE_ADD(${date}, INTERVAL '${interval}')`; + public addInterval(date: string, interval: string) { + return `DATE_ADD(${date}, INTERVAL ${this.formatInterval(interval)})`; } - public timeGroupedColumn(granularity, dimension) { + public timeGroupedColumn(granularity: string, dimension: string) { return `date_trunc('${GRANULARITY_TO_INTERVAL[granularity]}', ${dimension})`; } + /** + * Returns sql for source expression floored to timestamps aligned with + * intervals relative to origin timestamp point. + */ + public dateBin(interval: string, source: string, origin: string): string { + return `DATE_BIN(INTERVAL ${this.formatInterval(interval)}, ${this.timeStampCast(source)}, ${this.timeStampCast(`'${origin}'`)})`; + } + + /** + * The input interval with (possible) plural units, like "2 years", "3 months", "4 weeks", "5 days"... + * will be converted to CubeStore (DataFusion) dialect. + */ + private formatInterval(interval: string): string { + const intervalParsed = parseSqlInterval(interval); + const intKeys = Object.keys(intervalParsed).length; + + if (intervalParsed.year && intKeys === 1) { + return `'${intervalParsed.year} YEAR'`; + } else if (intervalParsed.year && intervalParsed.month && intKeys === 2) { + return `'${intervalParsed.year} YEAR ${intervalParsed.month} MONTH'`; + } else if (intervalParsed.year && intervalParsed.month && intervalParsed.quarter && intKeys === 3) { + return `'${intervalParsed.year} YEAR ${intervalParsed.quarter} QUARTER ${intervalParsed.month} MONTH'`; + } else if (intervalParsed.quarter && intKeys === 1) { + return `'${intervalParsed.quarter} QUARTER'`; + } else if (intervalParsed.quarter && intervalParsed.month && intKeys === 2) { + return `'${intervalParsed.quarter} QUARTER ${intervalParsed.month} MONTH'`; + } else if (intervalParsed.month && intKeys === 1) { + return `'${intervalParsed.month} MONTH'`; + } else if (intervalParsed.week && intKeys === 1) { + return `'${intervalParsed.week} WEEK'`; + } else if (intervalParsed.week && intervalParsed.day && intKeys === 2) { + return `'${intervalParsed.week} WEEK ${intervalParsed.day} DAY'`; + } else if (intervalParsed.week && intervalParsed.day && intervalParsed.hour && intKeys === 3) { + return `'${intervalParsed.week} WEEK ${intervalParsed.day} DAY ${intervalParsed.hour} HOUR'`; + } else if (intervalParsed.week && intervalParsed.day && intervalParsed.hour && intervalParsed.minute && intKeys === 4) { + return `'${intervalParsed.week} WEEK ${intervalParsed.day} DAY ${intervalParsed.hour} HOUR ${intervalParsed.minute} MINUTE'`; + } else if (intervalParsed.week && intervalParsed.day && intervalParsed.hour && intervalParsed.minute && intervalParsed.second && intKeys === 5) { + return `'${intervalParsed.week} WEEK ${intervalParsed.day} DAY ${intervalParsed.hour} HOUR ${intervalParsed.minute} MINUTE ${intervalParsed.second} SECOND'`; + } else if (intervalParsed.day && intKeys === 1) { + return `'${intervalParsed.day} DAY'`; + } else if (intervalParsed.day && intervalParsed.hour && intKeys === 2) { + return `'${intervalParsed.day} DAY ${intervalParsed.hour} HOUR'`; + } else if (intervalParsed.day && intervalParsed.hour && intervalParsed.minute && intKeys === 3) { + return `'${intervalParsed.day} DAY ${intervalParsed.hour} HOUR ${intervalParsed.minute} MINUTE'`; + } else if (intervalParsed.day && intervalParsed.hour && intervalParsed.minute && intervalParsed.second && intKeys === 4) { + return `'${intervalParsed.day} DAY ${intervalParsed.hour} HOUR ${intervalParsed.minute} MINUTE ${intervalParsed.second} SECOND'`; + } else if (intervalParsed.hour && intervalParsed.minute && intKeys === 2) { + return `'${intervalParsed.hour} HOUR ${intervalParsed.minute} MINUTE'`; + } else if (intervalParsed.hour && intervalParsed.minute && intervalParsed.second && intKeys === 3) { + return `'${intervalParsed.hour} HOUR ${intervalParsed.minute} MINUTE ${intervalParsed.second} SECOND'`; + } else if (intervalParsed.minute && intervalParsed.second && intKeys === 2) { + return `'${intervalParsed.minute} MINUTE ${intervalParsed.second} SECOND'`; + } + + // No need to support microseconds. + + throw new Error(`Cannot transform interval expression "${interval}" to CubeStore dialect`); + } + public escapeColumnName(name) { return `\`${name}\``; } diff --git a/packages/cubejs-schema-compiler/test/unit/base-query.test.ts b/packages/cubejs-schema-compiler/test/unit/base-query.test.ts index 0f28474a518a7..5a8d9c2dc6fae 100644 --- a/packages/cubejs-schema-compiler/test/unit/base-query.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/base-query.test.ts @@ -1,5 +1,5 @@ import moment from 'moment-timezone'; -import { BaseQuery, PostgresQuery, MssqlQuery, UserError } from '../../src'; +import { BaseQuery, PostgresQuery, MssqlQuery, UserError, CubeStoreQuery } from '../../src'; import { prepareCompiler, prepareYamlCompiler } from './PrepareCompiler'; import { createCubeSchema, @@ -295,7 +295,6 @@ describe('SQL Generation', () => { const query = new PostgresQuery(compilers, q); const queryAndParams = query.buildSqlAndParams(); const queryString = queryAndParams[0]; - console.log('Generated query: ', queryString); if (q.measures[0].includes('count')) { expect(queryString.includes('INTERVAL \'6 months\'')).toBeTruthy(); @@ -307,6 +306,31 @@ describe('SQL Generation', () => { }); }); }); + + describe('via CubeStoreQuery', () => { + beforeAll(async () => { + await compilers.compiler.compile(); + }); + + queries.forEach(q => { + it(`measure "${q.measures[0]}" + granularity "${q.timeDimensions[0].granularity}"`, () => { + const query = new CubeStoreQuery(compilers, q); + const queryAndParams = query.buildSqlAndParams(); + const queryString = queryAndParams[0]; + + if (q.measures[0].includes('count')) { + expect(queryString.includes('DATE_BIN(INTERVAL')).toBeTruthy(); + expect(queryString.includes('INTERVAL \'6 MONTH\'')).toBeTruthy(); + } else if (q.measures[0].includes('rollingCountByTrailing2Day')) { + expect(queryString.includes('date_trunc(\'day\'')).toBeTruthy(); + expect(queryString.includes('INTERVAL \'2 DAY\'')).toBeTruthy(); + } else if (q.measures[0].includes('rollingCountByLeading2Day')) { + expect(queryString.includes('date_trunc(\'day\'')).toBeTruthy(); + expect(queryString.includes('INTERVAL \'3 DAY\'')).toBeTruthy(); + } + }); + }); + }); }); describe('Common - JS', () => { @@ -1141,7 +1165,6 @@ describe('SQL Generation', () => { ], }); const cubeSQL = query.cubeSql('Order'); - console.log('TEST: ', cubeSQL); expect(cubeSQL).toContain('select * from order where ((type = $0$))'); }); }); diff --git a/packages/cubejs-testing-drivers/fixtures/_schemas.json b/packages/cubejs-testing-drivers/fixtures/_schemas.json index 74f31cc4ce9c9..5dab6f5668caf 100644 --- a/packages/cubejs-testing-drivers/fixtures/_schemas.json +++ b/packages/cubejs-testing-drivers/fixtures/_schemas.json @@ -88,7 +88,28 @@ { "name": "orderDate", "sql": "order_date", - "type": "time" + "type": "time", + "granularities": [ + { + "name": "half_year", + "interval": "6 months" + }, + { + "name": "half_year_by_1st_april", + "interval": "6 months", + "offset": "3 months" + }, + { + "name": "two_mo_by_feb", + "interval": "2 months", + "origin": "2020-02-01 00:00:00" + }, + { + "name": "three_months_by_march", + "interval": "3 month 3 days 3 hours", + "origin": "2020-03-15" + } + ] }, { "name": "customOrderDateNoPreAgg", diff --git a/packages/cubejs-testing-drivers/src/tests/testQueries.ts b/packages/cubejs-testing-drivers/src/tests/testQueries.ts index acec3d7afc698..c4a2f9f417d7c 100644 --- a/packages/cubejs-testing-drivers/src/tests/testQueries.ts +++ b/packages/cubejs-testing-drivers/src/tests/testQueries.ts @@ -1750,6 +1750,39 @@ export function testQueries(type: string, { includeIncrementalSchemaSuite, exten expect(response.rawData()).toMatchSnapshot(); }); + execute('querying custom granularities (with preaggregation) ECommerce: totalQuantity by half_year + no dimension', async () => { + const response = await client.load({ + measures: [ + 'ECommerce.totalQuantity', + ], + timeDimensions: [{ + dimension: 'ECommerce.orderDate', + granularity: 'half_year', + dateRange: ['2020-01-01', '2020-12-31'], + }], + }); + expect(response.rawData()).toMatchSnapshot(); + }); + + execute('querying custom granularities (with preaggregation) ECommerce: totalQuantity by half_year + dimension', async () => { + const response = await client.load({ + measures: [ + 'ECommerce.totalQuantity', + ], + timeDimensions: [{ + dimension: 'ECommerce.orderDate', + granularity: 'half_year', + dateRange: ['2020-01-01', '2020-12-31'], + }], + dimensions: ['ECommerce.productName'], + order: [ + ['ECommerce.orderDate', 'asc'], + ['ECommerce.productName', 'asc'] + ], + }); + expect(response.rawData()).toMatchSnapshot(); + }); + if (includeIncrementalSchemaSuite) { incrementalSchemaLoadingSuite(execute, () => driver, tables); }