Skip to content

Commit

Permalink
feat(schema-compiler): Support date_bin function in Cube Store queries (
Browse files Browse the repository at this point in the history 
#8684)

* feat(schema-compiler): implement dateBin() in CubeStoreQuery

* Add tests for dateBin in CubeStoreQuery

* add e2e tests for custom granularities with preaggregations

* Remove console.log
  • Loading branch information
@KSDaemon
KSDaemon authored Sep 13, 2024
1 parent 678a13a commit f7c07a7
Show file tree
Hide file tree
Showing 4 changed files with 147 additions and 10 deletions.
72 changes: 66 additions & 6 deletions packages/cubejs-schema-compiler/src/adapter/CubeStoreQuery.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import moment from 'moment-timezone';
import { parseSqlInterval } from '@cubejs-backend/shared';
import { BaseQuery } from './BaseQuery';
import { BaseFilter } from './BaseFilter';
import { BaseMeasure } from './BaseMeasure';
Expand Down Expand Up @@ -42,7 +43,7 @@ export class CubeStoreQuery extends BaseQuery {
}

public timeStampCast(value) {
return `CAST(${value} as TIMESTAMP)`; // TODO
return `CAST(${value} as TIMESTAMP)`;
}

public timestampFormat() {
Expand All @@ -53,18 +54,77 @@ export class CubeStoreQuery extends BaseQuery {
return `to_timestamp(${value})`;
}

public subtractInterval(date, interval) {
return `DATE_SUB(${date}, INTERVAL '${interval}')`;
public subtractInterval(date: string, interval: string) {
return `DATE_SUB(${date}, INTERVAL ${this.formatInterval(interval)})`;
}

public addInterval(date, interval) {
return `DATE_ADD(${date}, INTERVAL '${interval}')`;
public addInterval(date: string, interval: string) {
return `DATE_ADD(${date}, INTERVAL ${this.formatInterval(interval)})`;
}

public timeGroupedColumn(granularity, dimension) {
public timeGroupedColumn(granularity: string, dimension: string) {
return `date_trunc('${GRANULARITY_TO_INTERVAL[granularity]}', ${dimension})`;
}

/**
* Returns sql for source expression floored to timestamps aligned with
* intervals relative to origin timestamp point.
*/
public dateBin(interval: string, source: string, origin: string): string {
return `DATE_BIN(INTERVAL ${this.formatInterval(interval)}, ${this.timeStampCast(source)}, ${this.timeStampCast(`'${origin}'`)})`;
}

/**
* The input interval with (possible) plural units, like "2 years", "3 months", "4 weeks", "5 days"...
* will be converted to CubeStore (DataFusion) dialect.
*/
private formatInterval(interval: string): string {
const intervalParsed = parseSqlInterval(interval);
const intKeys = Object.keys(intervalParsed).length;

if (intervalParsed.year && intKeys === 1) {
return `'${intervalParsed.year} YEAR'`;
} else if (intervalParsed.year && intervalParsed.month && intKeys === 2) {
return `'${intervalParsed.year} YEAR ${intervalParsed.month} MONTH'`;
} else if (intervalParsed.year && intervalParsed.month && intervalParsed.quarter && intKeys === 3) {
return `'${intervalParsed.year} YEAR ${intervalParsed.quarter} QUARTER ${intervalParsed.month} MONTH'`;
} else if (intervalParsed.quarter && intKeys === 1) {
return `'${intervalParsed.quarter} QUARTER'`;
} else if (intervalParsed.quarter && intervalParsed.month && intKeys === 2) {
return `'${intervalParsed.quarter} QUARTER ${intervalParsed.month} MONTH'`;
} else if (intervalParsed.month && intKeys === 1) {
return `'${intervalParsed.month} MONTH'`;
} else if (intervalParsed.week && intKeys === 1) {
return `'${intervalParsed.week} WEEK'`;
} else if (intervalParsed.week && intervalParsed.day && intKeys === 2) {
return `'${intervalParsed.week} WEEK ${intervalParsed.day} DAY'`;
} else if (intervalParsed.week && intervalParsed.day && intervalParsed.hour && intKeys === 3) {
return `'${intervalParsed.week} WEEK ${intervalParsed.day} DAY ${intervalParsed.hour} HOUR'`;
} else if (intervalParsed.week && intervalParsed.day && intervalParsed.hour && intervalParsed.minute && intKeys === 4) {
return `'${intervalParsed.week} WEEK ${intervalParsed.day} DAY ${intervalParsed.hour} HOUR ${intervalParsed.minute} MINUTE'`;
} else if (intervalParsed.week && intervalParsed.day && intervalParsed.hour && intervalParsed.minute && intervalParsed.second && intKeys === 5) {
return `'${intervalParsed.week} WEEK ${intervalParsed.day} DAY ${intervalParsed.hour} HOUR ${intervalParsed.minute} MINUTE ${intervalParsed.second} SECOND'`;
} else if (intervalParsed.day && intKeys === 1) {
return `'${intervalParsed.day} DAY'`;
} else if (intervalParsed.day && intervalParsed.hour && intKeys === 2) {
return `'${intervalParsed.day} DAY ${intervalParsed.hour} HOUR'`;
} else if (intervalParsed.day && intervalParsed.hour && intervalParsed.minute && intKeys === 3) {
return `'${intervalParsed.day} DAY ${intervalParsed.hour} HOUR ${intervalParsed.minute} MINUTE'`;
} else if (intervalParsed.day && intervalParsed.hour && intervalParsed.minute && intervalParsed.second && intKeys === 4) {
return `'${intervalParsed.day} DAY ${intervalParsed.hour} HOUR ${intervalParsed.minute} MINUTE ${intervalParsed.second} SECOND'`;
} else if (intervalParsed.hour && intervalParsed.minute && intKeys === 2) {
return `'${intervalParsed.hour} HOUR ${intervalParsed.minute} MINUTE'`;
} else if (intervalParsed.hour && intervalParsed.minute && intervalParsed.second && intKeys === 3) {
return `'${intervalParsed.hour} HOUR ${intervalParsed.minute} MINUTE ${intervalParsed.second} SECOND'`;
} else if (intervalParsed.minute && intervalParsed.second && intKeys === 2) {
return `'${intervalParsed.minute} MINUTE ${intervalParsed.second} SECOND'`;
}

// No need to support microseconds.

throw new Error(`Cannot transform interval expression "${interval}" to CubeStore dialect`);
}

public escapeColumnName(name) {
return `\`${name}\``;
}
Expand Down
29 changes: 26 additions & 3 deletions packages/cubejs-schema-compiler/test/unit/base-query.test.ts
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import moment from 'moment-timezone';
import { BaseQuery, PostgresQuery, MssqlQuery, UserError } from '../../src';
import { BaseQuery, PostgresQuery, MssqlQuery, UserError, CubeStoreQuery } from '../../src';
import { prepareCompiler, prepareYamlCompiler } from './PrepareCompiler';
import {
createCubeSchema,
Expand Down Expand Up @@ -295,7 +295,6 @@ describe('SQL Generation', () => {
const query = new PostgresQuery(compilers, q);
const queryAndParams = query.buildSqlAndParams();
const queryString = queryAndParams[0];
console.log('Generated query: ', queryString);

if (q.measures[0].includes('count')) {
expect(queryString.includes('INTERVAL \'6 months\'')).toBeTruthy();
Expand All @@ -307,6 +306,31 @@ describe('SQL Generation', () => {
});
});
});

describe('via CubeStoreQuery', () => {
beforeAll(async () => {
await compilers.compiler.compile();
});

queries.forEach(q => {
it(`measure "${q.measures[0]}" + granularity "${q.timeDimensions[0].granularity}"`, () => {
const query = new CubeStoreQuery(compilers, q);
const queryAndParams = query.buildSqlAndParams();
const queryString = queryAndParams[0];

if (q.measures[0].includes('count')) {
expect(queryString.includes('DATE_BIN(INTERVAL')).toBeTruthy();
expect(queryString.includes('INTERVAL \'6 MONTH\'')).toBeTruthy();
} else if (q.measures[0].includes('rollingCountByTrailing2Day')) {
expect(queryString.includes('date_trunc(\'day\'')).toBeTruthy();
expect(queryString.includes('INTERVAL \'2 DAY\'')).toBeTruthy();
} else if (q.measures[0].includes('rollingCountByLeading2Day')) {
expect(queryString.includes('date_trunc(\'day\'')).toBeTruthy();
expect(queryString.includes('INTERVAL \'3 DAY\'')).toBeTruthy();
}
});
});
});
});

describe('Common - JS', () => {
Expand Down Expand Up @@ -1141,7 +1165,6 @@ describe('SQL Generation', () => {
],
});
const cubeSQL = query.cubeSql('Order');
console.log('TEST: ', cubeSQL);
expect(cubeSQL).toContain('select * from order where ((type = $0$))');
});
});
Expand Down
23 changes: 22 additions & 1 deletion packages/cubejs-testing-drivers/fixtures/_schemas.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,28 @@
{
"name": "orderDate",
"sql": "order_date",
"type": "time"
"type": "time",
"granularities": [
{
"name": "half_year",
"interval": "6 months"
},
{
"name": "half_year_by_1st_april",
"interval": "6 months",
"offset": "3 months"
},
{
"name": "two_mo_by_feb",
"interval": "2 months",
"origin": "2020-02-01 00:00:00"
},
{
"name": "three_months_by_march",
"interval": "3 month 3 days 3 hours",
"origin": "2020-03-15"
}
]
},
{
"name": "customOrderDateNoPreAgg",
Expand Down
33 changes: 33 additions & 0 deletions packages/cubejs-testing-drivers/src/tests/testQueries.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1750,6 +1750,39 @@ export function testQueries(type: string, { includeIncrementalSchemaSuite, exten
expect(response.rawData()).toMatchSnapshot();
});

execute('querying custom granularities (with preaggregation) ECommerce: totalQuantity by half_year + no dimension', async () => {
const response = await client.load({
measures: [
'ECommerce.totalQuantity',
],
timeDimensions: [{
dimension: 'ECommerce.orderDate',
granularity: 'half_year',
dateRange: ['2020-01-01', '2020-12-31'],
}],
});
expect(response.rawData()).toMatchSnapshot();
});

execute('querying custom granularities (with preaggregation) ECommerce: totalQuantity by half_year + dimension', async () => {
const response = await client.load({
measures: [
'ECommerce.totalQuantity',
],
timeDimensions: [{
dimension: 'ECommerce.orderDate',
granularity: 'half_year',
dateRange: ['2020-01-01', '2020-12-31'],
}],
dimensions: ['ECommerce.productName'],
order: [
['ECommerce.orderDate', 'asc'],
['ECommerce.productName', 'asc']
],
});
expect(response.rawData()).toMatchSnapshot();
});

if (includeIncrementalSchemaSuite) {
incrementalSchemaLoadingSuite(execute, () => driver, tables);
}
Expand Down

0 comments on commit f7c07a7

Please sign in to comment.