From 7700da71d9305e367bb7cb88f04ae78ad1ce6e81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Nordstr=C3=B6m?= Date: Fri, 21 Jun 2024 13:38:09 +0200 Subject: [PATCH] Push down orderby scankeys to Hypercore TAM Quals on orderby columns can be pushed down to Hypercore TAM and be transformed to the corresponding min/max scankeys on the compressed relation. Previously, only quals on non-compressed segmentby columns were pushed down as scankeys. Pushing down orderby scan keys seem to give a good performance boost for columnar scans when no index exists. The scankey push down can be disabled with a new GUC: `timescaledb.enable_hypercore_scankey_pushdown=false` --- src/guc.c | 15 + src/guc.h | 1 + tsl/src/hypercore/hypercore_handler.c | 139 +++++++- tsl/src/hypercore/hypercore_handler.h | 18 +- tsl/src/nodes/columnar_scan/columnar_scan.c | 323 +++++++++++-------- tsl/test/expected/hypercore_dump_restore.out | 8 +- tsl/test/expected/hypercore_scans.out | 276 ++++++++++++++++ tsl/test/sql/hypercore_scans.sql | 130 ++++++++ 8 files changed, 762 insertions(+), 148 deletions(-) diff --git a/src/guc.c b/src/guc.c index 2f3c488421d..2d9c66ec7a4 100644 --- a/src/guc.c +++ b/src/guc.c @@ -164,6 +164,7 @@ TSDLLEXPORT bool ts_guc_enable_merge_on_cagg_refresh = false; TSDLLEXPORT char *ts_guc_hypercore_indexam_whitelist; TSDLLEXPORT HypercoreCopyToBehavior ts_guc_hypercore_copy_to_behavior = HYPERCORE_COPY_NO_COMPRESSED_DATA; +TSDLLEXPORT bool ts_guc_enable_hypercore_scankey_pushdown = true; /* default value of ts_guc_max_open_chunks_per_insert and * ts_guc_max_cached_chunks_per_hypertable will be set as their respective boot-value when the @@ -1069,6 +1070,20 @@ _guc_init(void) NULL, NULL); + DefineCustomBoolVariable(/* name= */ MAKE_EXTOPTION("enable_hypercore_scankey_pushdown"), + /* short_desc= */ + "Push down qualifiers as scankeys when using Hypercore TAM", + /* long_desc= */ + "Enabling this setting might lead to faster scans when " + "query qualifiers match Hypercore segmentby and orderby columns.", + /* valueAddr= */ &ts_guc_enable_hypercore_scankey_pushdown, + /* bootValue= */ true, + /* context= */ PGC_USERSET, + /* flags= */ 0, + /* check_hook= */ NULL, + /* assign_hook= */ NULL, + /* show_hook= */ NULL); + DefineCustomIntVariable(/* name= */ MAKE_EXTOPTION("debug_bgw_scheduler_exit_status"), /* short_desc= */ "exit status to use when shutting down the scheduler", /* long_desc= */ "this is for debugging purposes", diff --git a/src/guc.h b/src/guc.h index 7691be19850..80d405b79c7 100644 --- a/src/guc.h +++ b/src/guc.h @@ -148,6 +148,7 @@ typedef enum HypercoreCopyToBehavior } HypercoreCopyToBehavior; extern TSDLLEXPORT HypercoreCopyToBehavior ts_guc_hypercore_copy_to_behavior; +extern TSDLLEXPORT bool ts_guc_enable_hypercore_scankey_pushdown; void _guc_init(void); diff --git a/tsl/src/hypercore/hypercore_handler.c b/tsl/src/hypercore/hypercore_handler.c index bbd53fef7bc..ed075675f45 100644 --- a/tsl/src/hypercore/hypercore_handler.c +++ b/tsl/src/hypercore/hypercore_handler.c @@ -8,9 +8,11 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -307,6 +309,19 @@ lazy_build_hypercore_info_cache(Relation rel, bool create_chunk_constraints, colsettings->cattnum = get_attnum(hsinfo->compressed_relid, attname); else colsettings->cattnum = InvalidAttrNumber; + + if (colsettings->is_orderby) + { + const char *min_attname = column_segment_min_name(orderby_pos); + const char *max_attname = column_segment_max_name(orderby_pos); + colsettings->cattnum_min = get_attnum(hsinfo->compressed_relid, min_attname); + colsettings->cattnum_max = get_attnum(hsinfo->compressed_relid, max_attname); + } + else + { + colsettings->cattnum_min = InvalidAttrNumber; + colsettings->cattnum_max = InvalidAttrNumber; + } } Ensure(hsinfo->relation_id > 0, "invalid chunk ID"); @@ -476,15 +491,17 @@ initscan(HypercoreScanDesc scan, ScanKey keys, int nkeys) * * It is only possible to use scankeys in the following two cases: * - * 1. The scankey is for a segment_by column - * 2. The scankey is for a column that has min/max metadata (i.e., order_by column). + * 1. The scankey is for a segmentby column + * 2. The scankey is for a column that has min/max metadata (e.g., orderby column). * - * TODO: Implement support for (2) above, which involves transforming a - * scankey to the corresponding min/max scankeys. + * For case (2), it is necessary to translate the scankey on the + * non-compressed relation to two min and max scankeys on the compressed + * relation. */ if (NULL != keys && nkeys > 0) { const HypercoreInfo *hsinfo = RelationGetHypercoreInfo(scan->rs_base.rs_rd); + CompressionSettings *settings = NULL; for (int i = 0; i < nkeys; i++) { @@ -503,6 +520,106 @@ initscan(HypercoreScanDesc scan, ScanKey keys, int nkeys) nvalidkeys++; break; } + + /* Transform equality to min/max on metadata columns */ + else if (key->sk_attno == column->attnum && hypercore_column_has_minmax(column)) + { + if (settings == NULL) + settings = ts_compression_settings_get(hsinfo->compressed_relid); + + TypeCacheEntry *tce = + lookup_type_cache(column->typid, TYPECACHE_BTREE_OPFAMILY); + + switch (key->sk_strategy) + { + case BTLessStrategyNumber: + case BTLessEqualStrategyNumber: + { + /* + * The operators '<' and '<=' translate to the + * same operators on the min metadata column + */ + Oid opno = get_opfamily_member(tce->btree_opf, + tce->type_id, + key->sk_subtype, + key->sk_strategy); + ScanKeyEntryInitialize(&scan->rs_base.rs_key[nvalidkeys++], + 0, + column->cattnum_min, + key->sk_strategy, + key->sk_subtype, + key->sk_collation, + get_opcode(opno), + key->sk_argument); + break; + } + case BTEqualStrategyNumber: + { + /* + * Equality ('=') translates to: + * + * x <= min_col AND x >= max_col + */ + StrategyNumber strategy_le = BTLessEqualStrategyNumber; + Oid opno_le = get_opfamily_member(tce->btree_opf, + tce->type_id, + key->sk_subtype, + strategy_le); + + ScanKeyEntryInitialize(&scan->rs_base.rs_key[nvalidkeys++], + 0, + column->cattnum_min, + strategy_le, + key->sk_subtype, + key->sk_collation, + get_opcode(opno_le), + key->sk_argument); + + StrategyNumber strategy_ge = BTGreaterEqualStrategyNumber; + Oid opno_ge = get_opfamily_member(tce->btree_opf, + tce->type_id, + key->sk_subtype, + strategy_ge); + + ScanKeyEntryInitialize(&scan->rs_base.rs_key[nvalidkeys++], + 0, + column->cattnum_max, + strategy_ge, + key->sk_subtype, + key->sk_collation, + get_opcode(opno_ge), + key->sk_argument); + break; + } + case BTGreaterEqualStrategyNumber: + case BTGreaterStrategyNumber: + { + /* + * The operators '>' and '>=' translate to the + * same operators on the max metadata column + */ + Oid opno = get_opfamily_member(tce->btree_opf, + tce->type_id, + key->sk_subtype, + key->sk_strategy); + ScanKeyEntryInitialize(&scan->rs_base.rs_key[nvalidkeys++], + 0, + column->cattnum_max, + key->sk_strategy, + key->sk_subtype, + key->sk_collation, + get_opcode(opno), + key->sk_argument); + break; + } + default: + pg_unreachable(); + Assert(false); + break; + } + + break; + } } } } @@ -566,16 +683,19 @@ hypercore_beginscan(Relation relation, Snapshot snapshot, int nkeys, ScanKey key RelationIncrementReferenceCount(relation); - TS_DEBUG_LOG("starting %s scan of relation %s parallel_scan=%p", + TS_DEBUG_LOG("starting %s scan of relation %s parallel_scan=%p nkeys=%d", get_scan_type(flags), RelationGetRelationName(relation), - parallel_scan); + parallel_scan, + nkeys); scan = palloc0(sizeof(HypercoreScanDescData)); scan->rs_base.rs_rd = relation; scan->rs_base.rs_snapshot = snapshot; scan->rs_base.rs_nkeys = nkeys; - scan->rs_base.rs_key = nkeys > 0 ? palloc0(sizeof(ScanKeyData) * nkeys) : NULL; + /* Allocate double the scan keys to account for some being transformed to + * two min/max keys */ + scan->rs_base.rs_key = nkeys > 0 ? palloc0(sizeof(ScanKeyData) * nkeys * 2) : NULL; scan->rs_base.rs_flags = flags; scan->rs_base.rs_parallel = parallel_scan; scan->returned_noncompressed_count = 0; @@ -590,8 +710,8 @@ hypercore_beginscan(Relation relation, Snapshot snapshot, int nkeys, ScanKey key return &scan->rs_base; } - HypercoreInfo *hsinfo = RelationGetHypercoreInfo(relation); - scan->compressed_rel = table_open(hsinfo->compressed_relid, AccessShareLock); + HypercoreInfo *hcinfo = RelationGetHypercoreInfo(relation); + scan->compressed_rel = table_open(hcinfo->compressed_relid, AccessShareLock); if (should_skip_compressed_data(&scan->rs_base)) { @@ -668,6 +788,7 @@ hypercore_endscan(TableScanDesc sscan) HypercoreScanDesc scan = (HypercoreScanDesc) sscan; RelationDecrementReferenceCount(sscan->rs_rd); + if (scan->cscan_desc) table_endscan(scan->cscan_desc); if (scan->compressed_rel) diff --git a/tsl/src/hypercore/hypercore_handler.h b/tsl/src/hypercore/hypercore_handler.h index 8d26b5b69b1..db5206340a6 100644 --- a/tsl/src/hypercore/hypercore_handler.h +++ b/tsl/src/hypercore/hypercore_handler.h @@ -6,6 +6,7 @@ #pragma once #include +#include #include #include #include @@ -36,9 +37,17 @@ extern int hypercore_decompress_update_segment(Relation relation, const ItemPoin typedef struct ColumnCompressionSettings { + /* Attribute name in the non-compressed relation */ NameData attname; + /* Attribute number in non-compressed relation */ AttrNumber attnum; - AttrNumber cattnum; /* Attribute number in the compressed relation */ + /* Attribute number in the compressed relation */ + AttrNumber cattnum; + /* For orderby columns, these are the attribute numbers of the the min/max + * metadata columns. */ + AttrNumber cattnum_min; + AttrNumber cattnum_max; + /* Attribute type */ Oid typid; bool is_orderby; bool is_segmentby; @@ -67,3 +76,10 @@ typedef struct HypercoreInfo #define REL_IS_HYPERCORE(rel) ((rel)->rd_tableam == hypercore_routine()) extern HypercoreInfo *RelationGetHypercoreInfo(Relation rel); + +static inline bool +hypercore_column_has_minmax(const ColumnCompressionSettings *column) +{ + return AttributeNumberIsValid(column->cattnum_min) && + AttributeNumberIsValid(column->cattnum_max); +} diff --git a/tsl/src/nodes/columnar_scan/columnar_scan.c b/tsl/src/nodes/columnar_scan/columnar_scan.c index db48f3ea492..75ea320a61d 100644 --- a/tsl/src/nodes/columnar_scan/columnar_scan.c +++ b/tsl/src/nodes/columnar_scan/columnar_scan.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -28,6 +29,7 @@ #include "columnar_scan.h" #include "compression/compression.h" +#include "guc.h" #include "hypercore/arrow_tts.h" #include "hypercore/hypercore_handler.h" #include "hypercore/vector_quals.h" @@ -49,6 +51,7 @@ typedef struct ColumnarScanState ScanKey scankeys; int nscankeys; List *scankey_quals; + List *quals_orig; List *vectorized_quals_orig; SimpleProjInfo sprojinfo; } ColumnarScanState; @@ -66,167 +69,219 @@ match_relvar(Expr *expr, Index relid) return false; } +typedef struct QualProcessInfo +{ + const HypercoreInfo *hcinfo; + Index relid; + List *quals; + List *remaining_quals; + List *scankey_quals; + ScanKey scankeys; + unsigned scankeys_capacity; + unsigned nscankeys; + bool collect_remaining_quals; +} QualProcessInfo; + /* - * Utility function to extract quals that can be used as scankeys. The - * remaining "normal" quals are optionally collected in the corresponding - * argument. - * - * Returns: - * - * 1. A list of scankey quals if scankeys is NULL, or + * Process OP-like expression. * - * 2. NIL if scankeys is non-NULL. + * Returns true if the qual should be kept as a regular qual filter in the + * scan, or false if it should not be kept. * + * Note that only scankeys on segmentby columns can completely replace a + * qual. Scankeys on other (compressed) columns will be transformed into + * scankeys on metadata columns, and those filters are not 100% accurate so + * the original qual must be kept. + */ +static bool +process_opexpr(QualProcessInfo *qpi, Expr *qual, List *args, Oid opno, Oid inputcollid, + Oid opfuncid) +{ + Expr *leftop, *rightop, *expr = NULL; + Var *relvar = NULL; + Datum scanvalue = 0; + bool argfound = false; + + if (list_length(args) != 2) + return true; + + leftop = linitial(args); + rightop = lsecond(args); + + /* Strip any relabeling */ + if (IsA(leftop, RelabelType)) + leftop = ((RelabelType *) leftop)->arg; + if (IsA(rightop, RelabelType)) + rightop = ((RelabelType *) rightop)->arg; + + if (match_relvar(leftop, qpi->relid)) + { + relvar = castNode(Var, leftop); + expr = rightop; + } + else if (match_relvar(rightop, qpi->relid)) + { + relvar = castNode(Var, rightop); + expr = leftop; + opno = get_commutator(opno); + } + else + { + /* If neither right nor left argument is a variable, we + * don't use it as scan key */ + return true; + } + + if (!OidIsValid(opno) || !op_strict(opno)) + { + return true; + } + + Assert(expr != NULL); + + if (IsA(expr, Const)) + { + Const *c = castNode(Const, expr); + scanvalue = c->constvalue; + argfound = true; + } + + const ColumnCompressionSettings *ccs = + &qpi->hcinfo->columns[AttrNumberGetAttrOffset(relvar->varattno)]; + + /* Add a scankey if this is a segmentby column or the column + * has min/max metadata */ + if (argfound && (ccs->is_segmentby || hypercore_column_has_minmax(ccs))) + { + TypeCacheEntry *tce = lookup_type_cache(relvar->vartype, TYPECACHE_BTREE_OPFAMILY); + int op_strategy = get_op_opfamily_strategy(opno, tce->btree_opf); + + if (op_strategy != InvalidStrategy) + { + Oid op_lefttype; + Oid op_righttype; + + get_op_opfamily_properties(opno, + tce->btree_opf, + false, + &op_strategy, + &op_lefttype, + &op_righttype); + + Assert(relvar != NULL); + + if (qpi->scankeys != NULL) + { + int flags = IsA(qual, ScalarArrayOpExpr) ? SK_SEARCHARRAY : 0; + ScanKeyEntryInitialize(&qpi->scankeys[qpi->nscankeys++], + flags, + relvar->varattno, + op_strategy, + op_righttype, + inputcollid, + opfuncid, + scanvalue); + } + + qpi->scankey_quals = lappend(qpi->scankey_quals, qual); + + return !ccs->is_segmentby; + } + } + + return true; +} + +/* + * Utility function to extract quals that can be used as scankeys. + * Thus, this function is designed to be called over two passes: one at plan * time to split the quals into scankey quals and remaining quals, and one at * execution time to populate a scankey array with the scankey quals found in * the first pass. * - * The scankey quals returned in pass 1 is used for EXPLAIN. + * The scankey quals collected in pass 1 is used for EXPLAIN. */ -static List * -process_scan_key_quals(const HypercoreInfo *hsinfo, Index relid, const List *quals, - List **remaining_quals, ScanKey scankeys, unsigned scankeys_capacity) +static void +process_scan_key_quals(QualProcessInfo *qpi, const List *quals) { - List *scankey_quals = NIL; - unsigned nkeys = 0; ListCell *lc; - Assert(scankeys == NULL || (scankeys_capacity >= (unsigned) list_length(quals))); - foreach (lc, quals) { Expr *qual = lfirst(lc); - bool scankey_found = false; - - /* ignore volatile expressions */ - if (contain_volatile_functions((Node *) qual)) - { - if (remaining_quals != NULL) - *remaining_quals = lappend(*remaining_quals, qual); - continue; - } + bool keep_qual = true; - switch (nodeTag(qual)) + if (!contain_volatile_functions((Node *) qual)) { - case T_OpExpr: + switch (nodeTag(qual)) { - OpExpr *opexpr = castNode(OpExpr, qual); - Oid opno = opexpr->opno; - Expr *leftop, *rightop, *expr = NULL; - Var *relvar = NULL; - Datum scanvalue = 0; - bool argfound = false; - - if (list_length(opexpr->args) != 2) - break; - - leftop = linitial(opexpr->args); - rightop = lsecond(opexpr->args); - - /* Strip any relabeling */ - if (IsA(leftop, RelabelType)) - leftop = ((RelabelType *) leftop)->arg; - if (IsA(rightop, RelabelType)) - rightop = ((RelabelType *) rightop)->arg; - - if (match_relvar(leftop, relid)) + case T_OpExpr: { - relvar = castNode(Var, leftop); - expr = rightop; - } - else if (match_relvar(rightop, relid)) - { - relvar = castNode(Var, rightop); - expr = leftop; - opno = get_commutator(opno); - } - else - { - /* If neither right nor left argument is a variable, we - * don't use it as scan key */ + OpExpr *opexpr = castNode(OpExpr, qual); + + if (opexpr->opresulttype == BOOLOID) + keep_qual = process_opexpr(qpi, + qual, + opexpr->args, + opexpr->opno, + opexpr->inputcollid, + opexpr->opfuncid); break; } - if (!OidIsValid(opno) || !op_strict(opno)) - break; - - Assert(expr != NULL); - - if (IsA(expr, Const)) - { - Const *c = castNode(Const, expr); - scanvalue = c->constvalue; - argfound = true; - } - - bool is_segmentby = - hsinfo->columns[AttrNumberGetAttrOffset(relvar->varattno)].is_segmentby; - if (argfound && is_segmentby) + case T_ScalarArrayOpExpr: { - TypeCacheEntry *tce = - lookup_type_cache(relvar->vartype, TYPECACHE_BTREE_OPFAMILY); - int op_strategy = get_op_opfamily_strategy(opno, tce->btree_opf); - - if (op_strategy != InvalidStrategy) - { - Oid op_lefttype; - Oid op_righttype; - - scankey_found = true; - get_op_opfamily_properties(opno, - tce->btree_opf, - false, - &op_strategy, - &op_lefttype, - &op_righttype); - - Assert(relvar != NULL); - - if (scankeys != NULL) - { - ScanKeyEntryInitialize(&scankeys[nkeys++], - 0 /* flags */, - relvar->varattno, /* attribute number to scan */ - op_strategy, /* op's strategy */ - op_righttype, /* strategy subtype */ - opexpr->inputcollid, /* collation */ - opexpr->opfuncid, /* reg proc to use */ - scanvalue); /* constant */ - } - else - { - scankey_quals = lappend(scankey_quals, qual); - } - } + /* + * Currently cannot push down "foo IN (1, 3)". The expression + * can be transformed into a scankey, but it only works for + * index scans. + */ + break; } - break; + default: + break; } - default: - break; } - if (!scankey_found && remaining_quals != NULL) - *remaining_quals = lappend(*remaining_quals, qual); + if (keep_qual && qpi->collect_remaining_quals) + qpi->remaining_quals = lappend(qpi->remaining_quals, qual); } - - return scankey_quals; } static List * -extract_scankey_quals(const HypercoreInfo *hsinfo, Index relid, const List *quals, +extract_scankey_quals(const HypercoreInfo *hcinfo, Index relid, const List *quals, List **remaining_quals) { - return process_scan_key_quals(hsinfo, relid, quals, remaining_quals, NULL, 0); + QualProcessInfo qpi = { + .hcinfo = hcinfo, + .relid = relid, + .collect_remaining_quals = remaining_quals != NULL, + }; + + process_scan_key_quals(&qpi, quals); + + if (remaining_quals) + *remaining_quals = qpi.remaining_quals; + + return qpi.scankey_quals; } static ScanKey -create_scankeys_from_quals(const HypercoreInfo *hsinfo, Index relid, const List *quals) +create_scankeys_from_quals(const HypercoreInfo *hcinfo, Index relid, const List *quals) { unsigned capacity = list_length(quals); - ScanKey scankeys = palloc0(sizeof(ScanKeyData) * capacity); - process_scan_key_quals(hsinfo, relid, quals, NULL, scankeys, capacity); - return scankeys; + QualProcessInfo qpi = { + .hcinfo = hcinfo, + .relid = relid, + .scankeys = palloc0(sizeof(ScanKeyData) * capacity), + .scankeys_capacity = capacity, + .collect_remaining_quals = false, + }; + + process_scan_key_quals(&qpi, quals); + + return qpi.scankeys; } static pg_attribute_always_inline TupleTableSlot * @@ -746,6 +801,7 @@ columnar_scan_state_create(CustomScan *cscan) #if PG16_GE cstate->css.slotOps = &TTSOpsArrowTuple; #endif + cstate->quals_orig = list_concat_copy(cstate->vectorized_quals_orig, cscan->scan.plan.qual); return (Node *) cstate; } @@ -819,7 +875,13 @@ columnar_scan_plan_create(PlannerInfo *root, RelOptInfo *rel, CustomPath *best_p /* Reduce RestrictInfo list to bare expressions; ignore pseudoconstants */ scan_clauses = extract_actual_clauses(scan_clauses, false); - foreach (lc, scan_clauses) + if (ts_guc_enable_hypercore_scankey_pushdown) + scankey_quals = + extract_scankey_quals(vqih.hsinfo, rel->relid, scan_clauses, &remaining_quals); + else + remaining_quals = scan_clauses; + + foreach (lc, remaining_quals) { Node *source_qual = lfirst(lc); Node *vectorized_qual = vector_qual_make(source_qual, &vqih.vqinfo); @@ -835,18 +897,7 @@ columnar_scan_plan_create(PlannerInfo *root, RelOptInfo *rel, CustomPath *best_p nonvectorized_quals = lappend(nonvectorized_quals, source_qual); } } - - /* Need to split the nonvectorized quals into scankey quals and remaining - * quals before ExecInitQual() in CustomScanState::begin() since the qual - * execution state is created from the remaining quals. Note that it is - * not possible to create the scankeys themselves here because the only - * way to pass those on to the scan state is via custom_private. And - * anything that goes into custom_private needs to be a Node that is - * printable with nodeToString(). */ - scankey_quals = - extract_scankey_quals(vqih.hsinfo, rel->relid, nonvectorized_quals, &remaining_quals); - - columnar_scan_plan->scan.plan.qual = remaining_quals; + columnar_scan_plan->scan.plan.qual = nonvectorized_quals; columnar_scan_plan->custom_exprs = list_make2(vectorized_quals, scankey_quals); RelationClose(relation); diff --git a/tsl/test/expected/hypercore_dump_restore.out b/tsl/test/expected/hypercore_dump_restore.out index df39397aada..5a322f1e39c 100644 --- a/tsl/test/expected/hypercore_dump_restore.out +++ b/tsl/test/expected/hypercore_dump_restore.out @@ -176,10 +176,12 @@ select * from hyperdump where time < '2022-06-03'; ------------------------------------------------------------------------------------------------ Append -> Custom Scan (ColumnarScan) on _hyper_1_1_chunk + Scankey: ("time" < 'Fri Jun 03 00:00:00 2022 PDT'::timestamp with time zone) Vectorized Filter: ("time" < 'Fri Jun 03 00:00:00 2022 PDT'::timestamp with time zone) -> Custom Scan (ColumnarScan) on _hyper_1_2_chunk + Scankey: ("time" < 'Fri Jun 03 00:00:00 2022 PDT'::timestamp with time zone) Vectorized Filter: ("time" < 'Fri Jun 03 00:00:00 2022 PDT'::timestamp with time zone) -(5 rows) +(7 rows) reindex table hyperdump; explain (costs off) @@ -188,10 +190,12 @@ select * from hyperdump where time < '2022-06-03'; ------------------------------------------------------------------------------------------------ Append -> Custom Scan (ColumnarScan) on _hyper_1_1_chunk + Scankey: ("time" < 'Fri Jun 03 00:00:00 2022 PDT'::timestamp with time zone) Vectorized Filter: ("time" < 'Fri Jun 03 00:00:00 2022 PDT'::timestamp with time zone) -> Custom Scan (ColumnarScan) on _hyper_1_2_chunk + Scankey: ("time" < 'Fri Jun 03 00:00:00 2022 PDT'::timestamp with time zone) Vectorized Filter: ("time" < 'Fri Jun 03 00:00:00 2022 PDT'::timestamp with time zone) -(5 rows) +(7 rows) select format('\! diff -u --label "hypercore original" --label "hypercore restored" %s %s', :'TEST_RESULTS_ORIGINAL', :'TEST_RESULTS_RESTORED') as "DIFF_CMD" \gset -- Original output and restored output should be the same, i.e., no diff --git a/tsl/test/expected/hypercore_scans.out b/tsl/test/expected/hypercore_scans.out index 67bb6dbfa84..3e3166d9fd4 100644 --- a/tsl/test/expected/hypercore_scans.out +++ b/tsl/test/expected/hypercore_scans.out @@ -602,3 +602,279 @@ select compress_chunk(ch, hypercore_use_access_method=>true) from show_chunks('r -- table. create temp table test_bump as select * from readings order by time, device; +---------------------------------------------------------------------- +-- +-- Test scankey push-downs on orderby column. Vectorized filters (or +-- normal qual filters) should remain on all orderby columns, but not +-- segmentby columns. +-- +---------------------------------------------------------------------- +-- +-- Test BETWEEN on time and equality on segmentby +-- +explain (costs off) +select * from readings +where time between '2022-06-03' and '2022-06-05' and device = 1; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Custom Scan (ColumnarScan) on _hyper_1_2_chunk + Scankey: (("time" >= 'Fri Jun 03 00:00:00 2022 PDT'::timestamp with time zone) AND ("time" <= 'Sun Jun 05 00:00:00 2022 PDT'::timestamp with time zone) AND (device = 1)) + Vectorized Filter: (("time" >= 'Fri Jun 03 00:00:00 2022 PDT'::timestamp with time zone) AND ("time" <= 'Sun Jun 05 00:00:00 2022 PDT'::timestamp with time zone)) +(3 rows) + +select sum(humidity) from readings +where time between '2022-06-03' and '2022-06-05' and device = 1; + sum +------------------ + 1979.06637167468 +(1 row) + +set timescaledb.enable_hypercore_scankey_pushdown=false; +explain (costs off) +select * from readings +where time between '2022-06-03' and '2022-06-05' and device = 1; + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Custom Scan (ColumnarScan) on _hyper_1_2_chunk + Filter: (device = 1) + Vectorized Filter: (("time" >= 'Fri Jun 03 00:00:00 2022 PDT'::timestamp with time zone) AND ("time" <= 'Sun Jun 05 00:00:00 2022 PDT'::timestamp with time zone)) +(3 rows) + +select sum(humidity) from readings +where time between '2022-06-03' and '2022-06-05' and device = 1; + sum +------------------ + 1979.06637167468 +(1 row) + +set timescaledb.enable_hypercore_scankey_pushdown=true; +-- +-- Test < (LT) on time and > (GT) on segmentby +-- +explain (costs off) +select * from readings +where time < '2022-06-05' and device > 5; + QUERY PLAN +--------------------------------------------------------------------------------------------------------- + Append + -> Custom Scan (ColumnarScan) on _hyper_1_1_chunk + Scankey: (("time" < 'Sun Jun 05 00:00:00 2022 PDT'::timestamp with time zone) AND (device > 5)) + Vectorized Filter: ("time" < 'Sun Jun 05 00:00:00 2022 PDT'::timestamp with time zone) + -> Custom Scan (ColumnarScan) on _hyper_1_2_chunk + Scankey: (("time" < 'Sun Jun 05 00:00:00 2022 PDT'::timestamp with time zone) AND (device > 5)) + Vectorized Filter: ("time" < 'Sun Jun 05 00:00:00 2022 PDT'::timestamp with time zone) +(7 rows) + +select sum(humidity) from readings +where time < '2022-06-05' and device > 5; + sum +------------------ + 97096.6439132039 +(1 row) + +set timescaledb.enable_hypercore_scankey_pushdown=false; +explain (costs off) +select * from readings +where time < '2022-06-05' and device > 5; + QUERY PLAN +------------------------------------------------------------------------------------------------ + Append + -> Custom Scan (ColumnarScan) on _hyper_1_1_chunk + Filter: (device > 5) + Vectorized Filter: ("time" < 'Sun Jun 05 00:00:00 2022 PDT'::timestamp with time zone) + -> Custom Scan (ColumnarScan) on _hyper_1_2_chunk + Filter: (device > 5) + Vectorized Filter: ("time" < 'Sun Jun 05 00:00:00 2022 PDT'::timestamp with time zone) +(7 rows) + +select sum(humidity) from readings +where time < '2022-06-05' and device > 5; + sum +------------------ + 97096.6439132039 +(1 row) + +set timescaledb.enable_hypercore_scankey_pushdown=true; +-- +-- Test >= (GE) on time +-- +explain (costs off) +select * from readings +where time >= '2022-06-05' and device > 5; + QUERY PLAN +---------------------------------------------------------------------------------------------------------- + Append + -> Custom Scan (ColumnarScan) on _hyper_1_2_chunk + Scankey: (("time" >= 'Sun Jun 05 00:00:00 2022 PDT'::timestamp with time zone) AND (device > 5)) + Vectorized Filter: ("time" >= 'Sun Jun 05 00:00:00 2022 PDT'::timestamp with time zone) + -> Custom Scan (ColumnarScan) on _hyper_1_3_chunk + Scankey: (("time" >= 'Sun Jun 05 00:00:00 2022 PDT'::timestamp with time zone) AND (device > 5)) + Vectorized Filter: ("time" >= 'Sun Jun 05 00:00:00 2022 PDT'::timestamp with time zone) + -> Custom Scan (ColumnarScan) on _hyper_1_4_chunk + Scankey: (("time" >= 'Sun Jun 05 00:00:00 2022 PDT'::timestamp with time zone) AND (device > 5)) + Vectorized Filter: ("time" >= 'Sun Jun 05 00:00:00 2022 PDT'::timestamp with time zone) + -> Custom Scan (ColumnarScan) on _hyper_1_5_chunk + Scankey: (("time" >= 'Sun Jun 05 00:00:00 2022 PDT'::timestamp with time zone) AND (device > 5)) + Vectorized Filter: ("time" >= 'Sun Jun 05 00:00:00 2022 PDT'::timestamp with time zone) + -> Custom Scan (ColumnarScan) on _hyper_1_6_chunk + Scankey: (("time" >= 'Sun Jun 05 00:00:00 2022 PDT'::timestamp with time zone) AND (device > 5)) + Vectorized Filter: ("time" >= 'Sun Jun 05 00:00:00 2022 PDT'::timestamp with time zone) +(16 rows) + +select sum(humidity) from readings +where time >= '2022-06-05' and device > 5; + sum +----------------- + 619887.78450012 +(1 row) + +set timescaledb.enable_hypercore_scankey_pushdown=false; +explain (costs off) +select * from readings +where time >= '2022-06-05' and device > 5; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Append + -> Custom Scan (ColumnarScan) on _hyper_1_2_chunk + Filter: (device > 5) + Vectorized Filter: ("time" >= 'Sun Jun 05 00:00:00 2022 PDT'::timestamp with time zone) + -> Custom Scan (ColumnarScan) on _hyper_1_3_chunk + Filter: (device > 5) + Vectorized Filter: ("time" >= 'Sun Jun 05 00:00:00 2022 PDT'::timestamp with time zone) + -> Custom Scan (ColumnarScan) on _hyper_1_4_chunk + Filter: (device > 5) + Vectorized Filter: ("time" >= 'Sun Jun 05 00:00:00 2022 PDT'::timestamp with time zone) + -> Custom Scan (ColumnarScan) on _hyper_1_5_chunk + Filter: (device > 5) + Vectorized Filter: ("time" >= 'Sun Jun 05 00:00:00 2022 PDT'::timestamp with time zone) + -> Custom Scan (ColumnarScan) on _hyper_1_6_chunk + Filter: (device > 5) + Vectorized Filter: ("time" >= 'Sun Jun 05 00:00:00 2022 PDT'::timestamp with time zone) +(16 rows) + +select sum(humidity) from readings +where time >= '2022-06-05' and device > 5; + sum +----------------- + 619887.78450012 +(1 row) + +set timescaledb.enable_hypercore_scankey_pushdown=true; +-- +-- Test = (equality) on time +-- +explain (costs off) +select * from readings +where time = '2022-06-01' and device > 5; + QUERY PLAN +--------------------------------------------------------------------------------------------------- + Custom Scan (ColumnarScan) on _hyper_1_1_chunk + Scankey: ((device > 5) AND ("time" = 'Wed Jun 01 00:00:00 2022 PDT'::timestamp with time zone)) + Vectorized Filter: ("time" = 'Wed Jun 01 00:00:00 2022 PDT'::timestamp with time zone) +(3 rows) + +select sum(humidity) from readings +where time = '2022-06-01' and device > 5; + sum +------------------ + 115.397092269175 +(1 row) + +set timescaledb.enable_hypercore_scankey_pushdown=false; +explain (costs off) +select * from readings +where time = '2022-06-01' and device > 5; + QUERY PLAN +------------------------------------------------------------------------------------------ + Custom Scan (ColumnarScan) on _hyper_1_1_chunk + Filter: (device > 5) + Vectorized Filter: ("time" = 'Wed Jun 01 00:00:00 2022 PDT'::timestamp with time zone) +(3 rows) + +select sum(humidity) from readings +where time = '2022-06-01' and device > 5; + sum +------------------ + 115.397092269175 +(1 row) + +set timescaledb.enable_hypercore_scankey_pushdown=true; +-- +-- Test "foo IN (1, 2)" (ScalarArrayOpExpr) +-- +-- This is currently not pushed down as scan key +-- +explain (costs off) +select * from readings +where time <= '2022-06-02' and device in (1, 4); + QUERY PLAN +------------------------------------------------------------------------------------------------- + Append + -> Custom Scan (ColumnarScan) on _hyper_1_1_chunk + Filter: (device = ANY ('{1,4}'::integer[])) + Scankey: ("time" <= 'Thu Jun 02 00:00:00 2022 PDT'::timestamp with time zone) + Vectorized Filter: ("time" <= 'Thu Jun 02 00:00:00 2022 PDT'::timestamp with time zone) + -> Custom Scan (ColumnarScan) on _hyper_1_2_chunk + Filter: (device = ANY ('{1,4}'::integer[])) + Scankey: ("time" <= 'Thu Jun 02 00:00:00 2022 PDT'::timestamp with time zone) + Vectorized Filter: ("time" <= 'Thu Jun 02 00:00:00 2022 PDT'::timestamp with time zone) +(9 rows) + +select sum(humidity) from readings +where time <= '2022-06-02' and device in (1, 4); + sum +------------------ + 2113.95150227923 +(1 row) + +set timescaledb.enable_hypercore_scankey_pushdown=false; +explain (costs off) +select * from readings +where time <= '2022-06-02' and device in (1, 4); + QUERY PLAN +------------------------------------------------------------------------------------------------- + Append + -> Custom Scan (ColumnarScan) on _hyper_1_1_chunk + Filter: (device = ANY ('{1,4}'::integer[])) + Vectorized Filter: ("time" <= 'Thu Jun 02 00:00:00 2022 PDT'::timestamp with time zone) + -> Custom Scan (ColumnarScan) on _hyper_1_2_chunk + Filter: (device = ANY ('{1,4}'::integer[])) + Vectorized Filter: ("time" <= 'Thu Jun 02 00:00:00 2022 PDT'::timestamp with time zone) +(7 rows) + +select sum(humidity) from readings +where time <= '2022-06-02' and device in (1, 4); + sum +------------------ + 2113.95150227923 +(1 row) + +set timescaledb.enable_hypercore_scankey_pushdown=true; +-- Compare push down of ScalarArrayOpExpr "foo IN (1, 2)" with +-- transparent decompression. Currently, such expressions are not +-- pushed down as scan keys. +set timescaledb.enable_transparent_decompression='hypercore'; +explain (costs off) +select * from readings +where time <= '2022-06-02' and device in (1, 4); + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------- + Append + -> Custom Scan (DecompressChunk) on _hyper_1_1_chunk + Vectorized Filter: ("time" <= 'Thu Jun 02 00:00:00 2022 PDT'::timestamp with time zone) + -> Seq Scan on compress_hyper_2_7_chunk + Filter: ((_ts_meta_min_1 <= 'Thu Jun 02 00:00:00 2022 PDT'::timestamp with time zone) AND (device = ANY ('{1,4}'::integer[]))) + -> Custom Scan (DecompressChunk) on _hyper_1_2_chunk + Vectorized Filter: ("time" <= 'Thu Jun 02 00:00:00 2022 PDT'::timestamp with time zone) + -> Seq Scan on compress_hyper_2_8_chunk + Filter: ((_ts_meta_min_1 <= 'Thu Jun 02 00:00:00 2022 PDT'::timestamp with time zone) AND (device = ANY ('{1,4}'::integer[]))) +(9 rows) + +select sum(humidity) from readings +where time <= '2022-06-02' and device in (1, 4); + sum +------------------ + 2113.95150227923 +(1 row) + +set timescaledb.enable_transparent_decompression=true; diff --git a/tsl/test/sql/hypercore_scans.sql b/tsl/test/sql/hypercore_scans.sql index b7cc7f276c7..2b105a179bc 100644 --- a/tsl/test/sql/hypercore_scans.sql +++ b/tsl/test/sql/hypercore_scans.sql @@ -208,3 +208,133 @@ select compress_chunk(ch, hypercore_use_access_method=>true) from show_chunks('r -- table. create temp table test_bump as select * from readings order by time, device; + +---------------------------------------------------------------------- +-- +-- Test scankey push-downs on orderby column. Vectorized filters (or +-- normal qual filters) should remain on all orderby columns, but not +-- segmentby columns. +-- +---------------------------------------------------------------------- + + +-- +-- Test BETWEEN on time and equality on segmentby +-- +explain (costs off) +select * from readings +where time between '2022-06-03' and '2022-06-05' and device = 1; + +select sum(humidity) from readings +where time between '2022-06-03' and '2022-06-05' and device = 1; + +set timescaledb.enable_hypercore_scankey_pushdown=false; + +explain (costs off) +select * from readings +where time between '2022-06-03' and '2022-06-05' and device = 1; + +select sum(humidity) from readings +where time between '2022-06-03' and '2022-06-05' and device = 1; + +set timescaledb.enable_hypercore_scankey_pushdown=true; + +-- +-- Test < (LT) on time and > (GT) on segmentby +-- +explain (costs off) +select * from readings +where time < '2022-06-05' and device > 5; + +select sum(humidity) from readings +where time < '2022-06-05' and device > 5; + +set timescaledb.enable_hypercore_scankey_pushdown=false; + +explain (costs off) +select * from readings +where time < '2022-06-05' and device > 5; + +select sum(humidity) from readings +where time < '2022-06-05' and device > 5; + +set timescaledb.enable_hypercore_scankey_pushdown=true; + +-- +-- Test >= (GE) on time +-- +explain (costs off) +select * from readings +where time >= '2022-06-05' and device > 5; + +select sum(humidity) from readings +where time >= '2022-06-05' and device > 5; + +set timescaledb.enable_hypercore_scankey_pushdown=false; + +explain (costs off) +select * from readings +where time >= '2022-06-05' and device > 5; + +select sum(humidity) from readings +where time >= '2022-06-05' and device > 5; + +set timescaledb.enable_hypercore_scankey_pushdown=true; + +-- +-- Test = (equality) on time +-- +explain (costs off) +select * from readings +where time = '2022-06-01' and device > 5; + +select sum(humidity) from readings +where time = '2022-06-01' and device > 5; + +set timescaledb.enable_hypercore_scankey_pushdown=false; + +explain (costs off) +select * from readings +where time = '2022-06-01' and device > 5; + +select sum(humidity) from readings +where time = '2022-06-01' and device > 5; + +set timescaledb.enable_hypercore_scankey_pushdown=true; + +-- +-- Test "foo IN (1, 2)" (ScalarArrayOpExpr) +-- +-- This is currently not pushed down as scan key +-- +explain (costs off) +select * from readings +where time <= '2022-06-02' and device in (1, 4); + +select sum(humidity) from readings +where time <= '2022-06-02' and device in (1, 4); + +set timescaledb.enable_hypercore_scankey_pushdown=false; + +explain (costs off) +select * from readings +where time <= '2022-06-02' and device in (1, 4); + +select sum(humidity) from readings +where time <= '2022-06-02' and device in (1, 4); + +set timescaledb.enable_hypercore_scankey_pushdown=true; + +-- Compare push down of ScalarArrayOpExpr "foo IN (1, 2)" with +-- transparent decompression. Currently, such expressions are not +-- pushed down as scan keys. +set timescaledb.enable_transparent_decompression='hypercore'; + +explain (costs off) +select * from readings +where time <= '2022-06-02' and device in (1, 4); + +select sum(humidity) from readings +where time <= '2022-06-02' and device in (1, 4); + +set timescaledb.enable_transparent_decompression=true;