Add custom granularity time intervals generation (used in rollups)

packages/cubejs-backend-shared/src/time.ts

@@ -6,6 +6,11 @@ const Moment = require('moment-timezone');
 const moment = extendMoment(Moment);
 
 type QueryDateRange = [string, string];
+type SqlInterval = string;
+type TimeSeriesOptions = {
+  timestampPrecision: number
+};
+type ParsedInterval = Partial<Record<unitOfTime.DurationConstructor, number>>;
 
 export const TIME_SERIES: Record<string, (range: DateRange, timestampPrecision: number) => QueryDateRange[]> = {
   day: (range: DateRange, digits) => Array.from(range.snapTo('day').by('day'))
@@ -26,8 +31,103 @@ export const TIME_SERIES: Record<string, (range: DateRange, timestampPrecision:
     .map(d => [d.format(`YYYY-MM-DDT00:00:00.${'0'.repeat(digits)}`), d.endOf('quarter').format(`YYYY-MM-DDT23:59:59.${'9'.repeat(digits)}`)]),
 };
 
-type TimeSeriesOptions = {
-  timestampPrecision: number
+/**
+ * Parse PostgreSQL-like interval string into object
+ * E.g. '2 years 15 months 100 weeks 99 hours 15 seconds'
+ * Negative units are also supported
+ * E.g. '-2 months 5 days -10 hours'
+ */
+function parseSqlInterval(intervalStr: SqlInterval): ParsedInterval {
+  const regex = /(-?\d+)\s*(year|quarter|month|week|day|hour|minute|second)s?/g;
+  const interval: ParsedInterval = {};
+
+  for (const match of intervalStr.matchAll(regex)) {
+    const value = parseInt(match[1], 10);
+    const unit = match[2] as unitOfTime.DurationConstructor;
+    interval[unit] = value;
+  }
+
+  return interval;
+}
+
+function addInterval(date: moment.Moment, interval: ParsedInterval): moment.Moment {
+  const res = date.clone();
+
+  Object.entries(interval).forEach(([key, value]) => {
+    res.add(value, key as unitOfTime.DurationConstructor);
+  });
+
+  return res;
+}
+
+function subtractInterval(date: moment.Moment, interval: ParsedInterval): moment.Moment {
+  const res = date.clone();
+
+  Object.entries(interval).forEach(([key, value]) => {
+    res.subtract(value, key as unitOfTime.DurationConstructor);
+  });
+
+  return res;
+}
+
+/**
+ * Returns the closest date prior to date parameter aligned with the offset and interval
+ * If no offset provided, the beginning of the year will be taken as pivot point
+ */
+function alignToOffset(date: moment.Moment, interval: ParsedInterval, offset?: ParsedInterval): moment.Moment {
+  let alignedDate = date.clone();
+  let intervalOp;
+
+  const startOfYear = moment().year(date.year()).startOf('year');
+  let offsetDate = offset ? addInterval(startOfYear, offset) : startOfYear;
+
+  if (date.isBefore(offsetDate)) {
+    intervalOp = offsetDate.isBefore(startOfYear) ? addInterval : subtractInterval;
+
+    while (date.isBefore(offsetDate)) {
+      offsetDate = intervalOp(offsetDate, interval);
+    }
+    alignedDate = offsetDate;
+  } else if (offsetDate.isBefore(startOfYear)) {
+    intervalOp = offsetDate.isBefore(startOfYear) ? addInterval : subtractInterval;
+
+    while (date.isAfter(offsetDate)) {
+      alignedDate = offsetDate.clone();
+      offsetDate = intervalOp(offsetDate, interval);
+    }
+  } else {
+    intervalOp = offsetDate.isBefore(startOfYear) ? subtractInterval : addInterval;
+
+    while (date.isAfter(offsetDate)) {
+      alignedDate = offsetDate.clone();
+      offsetDate = intervalOp(offsetDate, interval);
+    }
+  }
+
+  return alignedDate;
+}
+
+export const timeSeriesFromCustomInterval = (intervalStr: string, [startStr, endStr]: QueryDateRange, offsetStr: string | undefined, options: TimeSeriesOptions = {timestampPrecision: 3}): QueryDateRange[] => {
+  const intervalParsed = parseSqlInterval(intervalStr);
+  const offsetParsed = offsetStr ? parseSqlInterval(offsetStr) : undefined;
+  const start = moment(startStr);
+  const end = moment(endStr);
+  let alignedStart = alignToOffset(start, intervalParsed, offsetParsed);
+
+  const dates: QueryDateRange[] = [];
+
+  while (alignedStart.isBefore(end)) {
+    const s = alignedStart.clone();
+    alignedStart = addInterval(alignedStart, intervalParsed);
+    dates.push([
+      s.format(`YYYY-MM-DDTHH:mm:ss.${'0'.repeat(options.timestampPrecision)}`),
+      alignedStart.clone()
+        .subtract(1, 'second')
+        .format(`YYYY-MM-DDTHH:mm:ss.${'9'.repeat(options.timestampPrecision)}`)
+    ]);
+  }
+
+  return dates;
 };
 
 export const timeSeries = (granularity: string, dateRange: QueryDateRange, options: TimeSeriesOptions = { timestampPrecision: 3 }): QueryDateRange[] => {

packages/cubejs-schema-compiler/src/adapter/BaseTimeDimension.ts

@@ -1,5 +1,13 @@
 import moment from 'moment-timezone';
-import { timeSeries, isPredefinedGranularity, FROM_PARTITION_RANGE, TO_PARTITION_RANGE, BUILD_RANGE_START_LOCAL, BUILD_RANGE_END_LOCAL } from '@cubejs-backend/shared';
+import {
+  timeSeries,
+  isPredefinedGranularity,
+  timeSeriesFromCustomInterval,
+  FROM_PARTITION_RANGE,
+  TO_PARTITION_RANGE,
+  BUILD_RANGE_START_LOCAL,
+  BUILD_RANGE_END_LOCAL
+} from '@cubejs-backend/shared';
 
 import { BaseFilter } from './BaseFilter';
 import { UserError } from '../compiler/UserError';
@@ -278,9 +286,13 @@ export class BaseTimeDimension extends BaseFilter {
       ];
     }
 
-    return timeSeries(this.granularity, [this.dateFromFormatted(), this.dateToFormatted()], {
-      timestampPrecision: this.query.timestampPrecision(),
-    });
+    if (this.isPredefined) {
+      return timeSeries(this.granularity, [this.dateFromFormatted(), this.dateToFormatted()], {
+        timestampPrecision: this.query.timestampPrecision(),
+      });
+    }
+
+    return timeSeriesFromCustomInterval(this.granularityInterval, [this.dateFromFormatted(), this.dateToFormatted()], this.granularityOffset, {timestampPrecision: this.query.timestampPrecision()});
   }
 
   public wildcardRange() {