From e6669b1f35dd45079df5ce477661ea75de17fe36 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Tue, 24 Sep 2024 23:10:23 +0300 Subject: [PATCH] feat(client-core): support custom intervals time series generation --- packages/cubejs-client-core/index.d.ts | 6 + packages/cubejs-client-core/src/ResultSet.js | 80 ++--- packages/cubejs-client-core/src/index.js | 11 +- .../src/tests/utils.test.js | 2 +- packages/cubejs-client-core/src/time.js | 296 ++++++++++++++++++ packages/cubejs-client-core/src/utils.js | 23 +- 6 files changed, 342 insertions(+), 76 deletions(-) create mode 100644 packages/cubejs-client-core/src/time.js diff --git a/packages/cubejs-client-core/index.d.ts b/packages/cubejs-client-core/index.d.ts index ea4c875466392..76af406b6bb93 100644 --- a/packages/cubejs-client-core/index.d.ts +++ b/packages/cubejs-client-core/index.d.ts @@ -1303,4 +1303,10 @@ declare module '@cubejs-client/core' { stage: string; timeElapsed: number; }; + + export function granularityFor(dateStr: string): string; + + export function minGranularityForIntervals(i1: string, i2: string): string; + + export function isPredefinedGranularity(granularity: TimeDimensionGranularity): boolean; } diff --git a/packages/cubejs-client-core/src/ResultSet.js b/packages/cubejs-client-core/src/ResultSet.js index 4d304ebe37cdc..3ec2e184ae40d 100644 --- a/packages/cubejs-client-core/src/ResultSet.js +++ b/packages/cubejs-client-core/src/ResultSet.js @@ -1,33 +1,19 @@ import dayjs from 'dayjs'; -import quarterOfYear from 'dayjs/plugin/quarterOfYear'; - -import en from 'dayjs/locale/en'; import { groupBy, pipe, fromPairs, uniq, filter, map, dropLast, equals, reduce, minBy, maxBy, clone, mergeDeepLeft, pluck, mergeAll, flatten, } from 'ramda'; import { aliasSeries } from './utils'; - -dayjs.extend(quarterOfYear); - -// When granularity is week, weekStart Value must be 1. However, since the client can change it globally (https://day.js.org/docs/en/i18n/changing-locale) -// So the function below has been added. -const internalDayjs = (...args) => dayjs(...args).locale({ ...en, weekStart: 1 }); - -export const TIME_SERIES = { - day: (range) => range.by('d').map(d => d.format('YYYY-MM-DDT00:00:00.000')), - month: (range) => range.snapTo('month').by('M').map(d => d.format('YYYY-MM-01T00:00:00.000')), - year: (range) => range.snapTo('year').by('y').map(d => d.format('YYYY-01-01T00:00:00.000')), - hour: (range) => range.by('h').map(d => d.format('YYYY-MM-DDTHH:00:00.000')), - minute: (range) => range.by('m').map(d => d.format('YYYY-MM-DDTHH:mm:00.000')), - second: (range) => range.by('s').map(d => d.format('YYYY-MM-DDTHH:mm:ss.000')), - week: (range) => range.snapTo('week').by('w').map(d => d.startOf('week').format('YYYY-MM-DDT00:00:00.000')), - quarter: (range) => range.snapTo('quarter').by('quarter').map(d => d.startOf('quarter').format('YYYY-MM-DDT00:00:00.000')), -}; - -const DateRegex = /^\d\d\d\d-\d\d-\d\d$/; -const LocalDateRegex = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z?$/; +import { + DateRegex, + dayRange, + internalDayjs, + isPredefinedGranularity, + LocalDateRegex, + TIME_SERIES, + timeSeriesFromCustomInterval +} from './time'; const groupByToPairs = (keyFn) => { const acc = new Map(); @@ -56,25 +42,6 @@ const unnest = (arr) => { return res; }; -export const dayRange = (from, to) => ({ - by: (value) => { - const results = []; - - let start = internalDayjs(from); - const end = internalDayjs(to); - - while (start.isBefore(end) || start.isSame(end)) { - results.push(start); - start = start.add(1, value); - } - - return results; - }, - snapTo: (value) => dayRange(internalDayjs(from).startOf(value), internalDayjs(to).endOf(value)), - start: internalDayjs(from), - end: internalDayjs(to), -}); - export const QUERY_TYPE = { REGULAR_QUERY: 'regularQuery', COMPARE_DATE_RANGE_QUERY: 'compareDateRangeQuery', @@ -163,15 +130,15 @@ class ResultSet { if (granularity !== undefined) { const range = dayRange(value, value).snapTo(granularity); const originalTimeDimension = query.timeDimensions.find((td) => td.dimension); - + let dateRange = [ range.start, range.end ]; - + if (originalTimeDimension?.dateRange) { const [originalStart, originalEnd] = originalTimeDimension.dateRange; - + dateRange = [ dayjs(originalStart) > range.start ? dayjs(originalStart) : range.start, dayjs(originalEnd) < range.end ? dayjs(originalEnd) : range.end, @@ -195,7 +162,7 @@ class ResultSet { }); } }); - + if ( timeDimensions.length === 0 && query.timeDimensions.length > 0 && @@ -321,7 +288,7 @@ class ResultSet { return ResultSet.getNormalizedPivotConfig(this.loadResponse.pivotQuery, pivotConfig); } - timeSeries(timeDimension, resultIndex) { + timeSeries(timeDimension, resultIndex, annotations) { if (!timeDimension.granularity) { return null; } @@ -352,12 +319,18 @@ class ResultSet { const [start, end] = dateRange; const range = dayRange(start, end); - if (!TIME_SERIES[timeDimension.granularity]) { - throw new Error(`Unsupported time granularity: ${timeDimension.granularity}`); + if (isPredefinedGranularity(timeDimension.granularity)) { + return TIME_SERIES[timeDimension.granularity]( + padToDay ? range.snapTo('d') : range + ); + } + + if (!annotations[`${timeDimension.dimension}.${timeDimension.granularity}`]) { + throw new Error(`Granularity "${timeDimension.granularity}" not found in time dimension "${timeDimension.dimension}"`); } - return TIME_SERIES[timeDimension.granularity]( - padToDay ? range.snapTo('d') : range + return timeSeriesFromCustomInterval( + start, end, annotations[`${timeDimension.dimension}.${timeDimension.granularity}`].granularity ); } @@ -381,7 +354,10 @@ class ResultSet { )) ) { const series = this.loadResponses.map( - (loadResponse) => this.timeSeries(loadResponse.query.timeDimensions[0], resultIndex) + (loadResponse) => this.timeSeries( + loadResponse.query.timeDimensions[0], + resultIndex, loadResponse.annotation.timeDimensions + ) ); if (series[0]) { diff --git a/packages/cubejs-client-core/src/index.js b/packages/cubejs-client-core/src/index.js index fac5642c3d4ae..92d12b5c8d564 100644 --- a/packages/cubejs-client-core/src/index.js +++ b/packages/cubejs-client-core/src/index.js @@ -271,22 +271,22 @@ class CubeApi { if (v.type === 'number') { return k; } - + return undefined; }).filter(Boolean); - + result.data = result.data.map((row) => { numericMembers.forEach((key) => { if (row[key] != null) { row[key] = Number(row[key]); } }); - + return row; }); }); } - + if (response.results[0].query.responseFormat && response.results[0].query.responseFormat === ResultType.COMPACT) { response.results.forEach((result, j) => { @@ -302,7 +302,7 @@ class CubeApi { }); } } - + return new ResultSet(response, { parseDateMeasures: this.parseDateMeasures }); @@ -388,3 +388,4 @@ export default (apiToken, options) => new CubeApi(apiToken, options); export { CubeApi, HttpTransport, ResultSet, RequestError, Meta }; export * from './utils'; +export * from './time'; diff --git a/packages/cubejs-client-core/src/tests/utils.test.js b/packages/cubejs-client-core/src/tests/utils.test.js index b3b690adee561..59c7bf0fb4acb 100644 --- a/packages/cubejs-client-core/src/tests/utils.test.js +++ b/packages/cubejs-client-core/src/tests/utils.test.js @@ -1,7 +1,7 @@ import 'jest'; -import { TIME_SERIES, dayRange } from '../ResultSet'; import { defaultOrder } from '../utils'; +import { dayRange, TIME_SERIES } from '../time'; describe('utils', () => { test('default order', () => { diff --git a/packages/cubejs-client-core/src/time.js b/packages/cubejs-client-core/src/time.js new file mode 100644 index 0000000000000..4a9b4806820fe --- /dev/null +++ b/packages/cubejs-client-core/src/time.js @@ -0,0 +1,296 @@ +import dayjs from 'dayjs'; +import quarterOfYear from 'dayjs/plugin/quarterOfYear'; +import duration from 'dayjs/plugin/duration'; +import isoWeek from 'dayjs/plugin/isoWeek'; +import en from 'dayjs/locale/en'; + +dayjs.extend(quarterOfYear); +dayjs.extend(duration); +dayjs.extend(isoWeek); + +export const GRANULARITIES = [ + { name: undefined, title: 'w/o grouping' }, + { name: 'second', title: 'Second' }, + { name: 'minute', title: 'Minute' }, + { name: 'hour', title: 'Hour' }, + { name: 'day', title: 'Day' }, + { name: 'week', title: 'Week' }, + { name: 'month', title: 'Month' }, + { name: 'quarter', title: 'Quarter' }, + { name: 'year', title: 'Year' }, +]; + +export const DEFAULT_GRANULARITY = 'day'; + +// When granularity is week, weekStart Value must be 1. However, since the client can change it globally +// (https://day.js.org/docs/en/i18n/changing-locale) So the function below has been added. +export const internalDayjs = (...args) => dayjs(...args).locale({ ...en, weekStart: 1 }); + +export const TIME_SERIES = { + day: (range) => range.by('d').map(d => d.format('YYYY-MM-DDT00:00:00.000')), + month: (range) => range.snapTo('month').by('M').map(d => d.format('YYYY-MM-01T00:00:00.000')), + year: (range) => range.snapTo('year').by('y').map(d => d.format('YYYY-01-01T00:00:00.000')), + hour: (range) => range.by('h').map(d => d.format('YYYY-MM-DDTHH:00:00.000')), + minute: (range) => range.by('m').map(d => d.format('YYYY-MM-DDTHH:mm:00.000')), + second: (range) => range.by('s').map(d => d.format('YYYY-MM-DDTHH:mm:ss.000')), + week: (range) => range.snapTo('week').by('w').map(d => d.startOf('week').format('YYYY-MM-DDT00:00:00.000')), + quarter: (range) => range.snapTo('quarter').by('quarter').map(d => d.startOf('quarter').format( + 'YYYY-MM-DDT00:00:00.000' + )), +}; + +export const isPredefinedGranularity = (granularity) => !!TIME_SERIES[granularity]; + +export const DateRegex = /^\d\d\d\d-\d\d-\d\d$/; +export const LocalDateRegex = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z?$/; + +export const dayRange = (from, to) => ({ + by: (value) => { + const results = []; + + let start = internalDayjs(from); + const end = internalDayjs(to); + + while (start.isBefore(end) || start.isSame(end)) { + results.push(start); + start = start.add(1, value); + } + + return results; + }, + snapTo: (value) => dayRange(internalDayjs(from).startOf(value), internalDayjs(to).endOf(value)), + start: internalDayjs(from), + end: internalDayjs(to), +}); + +/** + * Parse PostgreSQL-like interval string into object + * E.g. '2 years 15 months 100 weeks 99 hours 15 seconds' + * Negative units are also supported + * E.g. '-2 months 5 days -10 hours' + * + * TODO: It's copy/paste of parseSqlInterval from @cubejs-backend/shared [time.ts] + * It's not referenced to omit imports of moment.js staff. + * Probably one day we should choose one implementation and reuse it in other places. + */ +export function parseSqlInterval(intervalStr) { + const interval = {}; + const parts = intervalStr.split(/\s+/); + + for (let i = 0; i < parts.length; i += 2) { + const value = parseInt(parts[i], 10); + const unit = parts[i + 1]; + + // Remove ending 's' (e.g., 'days' -> 'day') + const singularUnit = unit.endsWith('s') ? unit.slice(0, -1) : unit; + interval[singularUnit] = value; + } + + return interval; +} + +/** + * Adds interval to provided date. + * TODO: It's copy/paste of addInterval from @cubejs-backend/shared [time.ts] + * but operates with dayjs instead of moment.js + * @param {dayjs} date + * @param interval + * @returns {dayjs} + */ +export function addInterval(date, interval) { + let res = date.clone(); + + Object.entries(interval).forEach(([key, value]) => { + res = res.add(value, key); + }); + + return res; +} + +/** + * Adds interval to provided date. + * TODO: It's copy/paste of subtractInterval from @cubejs-backend/shared [time.ts] + * but operates with dayjs instead of moment.js + * @param {dayjs} date + * @param interval + * @returns {dayjs} + */ +export function subtractInterval(date, interval) { + let res = date.clone(); + + Object.entries(interval).forEach(([key, value]) => { + res = res.subtract(value, key); + }); + + return res; +} + +/** + * Returns the closest date prior to date parameter aligned with the origin point + * TODO: It's copy/paste of alignToOrigin from @cubejs-backend/shared [time.ts] + * but operates with dayjs instead of moment.js + */ +function alignToOrigin(startDate, interval, origin) { + let alignedDate = startDate.clone(); + let intervalOp; + let isIntervalNegative = false; + + let offsetDate = addInterval(origin, interval); + + // The easiest way to check the interval sign + if (offsetDate.isBefore(origin)) { + isIntervalNegative = true; + } + + offsetDate = origin.clone(); + + if (startDate.isBefore(origin)) { + intervalOp = isIntervalNegative ? addInterval : subtractInterval; + + while (offsetDate.isAfter(startDate)) { + offsetDate = intervalOp(offsetDate, interval); + } + alignedDate = offsetDate; + } else { + intervalOp = isIntervalNegative ? subtractInterval : addInterval; + + while (offsetDate.isBefore(startDate)) { + alignedDate = offsetDate.clone(); + offsetDate = intervalOp(offsetDate, interval); + } + + if (offsetDate.isSame(startDate)) { + alignedDate = offsetDate; + } + } + + return alignedDate; +} + +/** + * Returns the time series points for the custom interval + * TODO: It's almost a copy/paste of timeSeriesFromCustomInterval from + * @cubejs-backend/shared [time.ts] but operates with dayjs instead of moment.js + */ +export const timeSeriesFromCustomInterval = (from, to, granularity) => { + const intervalParsed = parseSqlInterval(granularity.interval); + const start = internalDayjs(from); + const end = internalDayjs(to); + let origin = granularity.origin ? internalDayjs(granularity.origin) : internalDayjs().startOf('year'); + if (granularity.offset) { + origin = addInterval(origin, parseSqlInterval(granularity.offset)); + } + let alignedStart = alignToOrigin(start, intervalParsed, origin); + + const dates = []; + + while (alignedStart.isBefore(end) || alignedStart.isSame(end)) { + dates.push(alignedStart.format('YYYY-MM-DDTHH:00:00.000')); + alignedStart = addInterval(alignedStart, intervalParsed); + } + + return dates; +}; + +/** + * Returns the lowest time unit for the interval + * @protected + * @param {string} interval + * @returns {string} + */ +export const diffTimeUnitForInterval = (interval) => { + if (/second/i.test(interval)) { + return 'second'; + } else if (/minute/i.test(interval)) { + return 'minute'; + } else if (/hour/i.test(interval)) { + return 'hour'; + } else if (/day/i.test(interval)) { + return 'day'; + } else if (/week/i.test(interval)) { + return 'day'; + } else if (/month/i.test(interval)) { + return 'month'; + } else if (/quarter/i.test(interval)) { + return 'month'; + } else /* if (/year/i.test(interval)) */ { + return 'year'; + } +}; + +const granularityOrder = ['year', 'quarter', 'month', 'week', 'day', 'hour', 'minute', 'second']; + +export const minGranularityForIntervals = (i1, i2) => { + const g1 = diffTimeUnitForInterval(i1); + const g2 = diffTimeUnitForInterval(i2); + const g1pos = granularityOrder.indexOf(g1); + const g2pos = granularityOrder.indexOf(g2); + + if (g1pos > g2pos) { + return g1; + } + + return g2; +}; + +export const granularityFor = (dateStr) => { + const dayjsDate = internalDayjs(dateStr); + const month = dayjsDate.month(); + const date = dayjsDate.date(); + const hours = dayjsDate.hour(); + const minutes = dayjsDate.minute(); + const seconds = dayjsDate.second(); + const milliseconds = dayjsDate.millisecond(); + const weekDay = dayjsDate.isoWeekday(); + + if ( + month === 0 && + date === 1 && + hours === 0 && + minutes === 0 && + seconds === 0 && + milliseconds === 0 + ) { + return 'year'; + } else if ( + date === 1 && + hours === 0 && + minutes === 0 && + seconds === 0 && + milliseconds === 0 + ) { + return 'month'; + } else if ( + weekDay === 1 && + hours === 0 && + minutes === 0 && + seconds === 0 && + milliseconds === 0 + ) { + return 'week'; + } else if ( + hours === 0 && + minutes === 0 && + seconds === 0 && + milliseconds === 0 + ) { + return 'day'; + } else if ( + minutes === 0 && + seconds === 0 && + milliseconds === 0 + ) { + return 'hour'; + } else if ( + seconds === 0 && + milliseconds === 0 + ) { + return 'minute'; + } else if ( + milliseconds === 0 + ) { + return 'second'; + } + + return 'second'; // TODO return 'millisecond'; +}; diff --git a/packages/cubejs-client-core/src/utils.js b/packages/cubejs-client-core/src/utils.js index b8f6ea25774b7..48ef2e274a056 100644 --- a/packages/cubejs-client-core/src/utils.js +++ b/packages/cubejs-client-core/src/utils.js @@ -1,22 +1,9 @@ -import { indexBy, prop, clone, equals, fromPairs, toPairs } from 'ramda'; - -export const DEFAULT_GRANULARITY = 'day'; - -export const GRANULARITIES = [ - { name: undefined, title: 'w/o grouping' }, - { name: 'second', title: 'Second' }, - { name: 'minute', title: 'Minute' }, - { name: 'hour', title: 'Hour' }, - { name: 'day', title: 'Day' }, - { name: 'week', title: 'Week' }, - { name: 'month', title: 'Month' }, - { name: 'quarter', title: 'Quarter' }, - { name: 'year', title: 'Year' }, -]; +import { clone, equals, fromPairs, indexBy, prop, toPairs } from 'ramda'; +import { DEFAULT_GRANULARITY } from './time'; export function removeEmptyQueryFields(_query) { const query = _query || {}; - + return fromPairs( toPairs(query) .map(([key, value]) => { @@ -27,7 +14,7 @@ export function removeEmptyQueryFields(_query) { return null; } } - + if (key === 'order' && value) { if (Array.isArray(value) && !value.length) { return null; @@ -44,7 +31,7 @@ export function removeEmptyQueryFields(_query) { export function validateQuery(_query) { const query = _query || {}; - + return removeEmptyQueryFields({ ...query, filters: (query.filters || []).filter((f) => f.operator),