From ac00ac423d51083c33d77fd6c2cbe20ef44863c6 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Mon, 6 Jan 2025 14:06:04 +0100 Subject: [PATCH 1/3] Vector agg: skip entire null words of filter bitmap This improves the performance for highly selective filters. --- .../nodes/decompress_chunk/compressed_batch.h | 2 +- .../decompress_chunk/vector_predicates.h | 2 +- .../nodes/vector_agg/grouping_policy_hash.c | 69 ++++++++++++++++++- .../nodes/vector_agg/grouping_policy_hash.h | 1 + 4 files changed, 70 insertions(+), 4 deletions(-) diff --git a/tsl/src/nodes/decompress_chunk/compressed_batch.h b/tsl/src/nodes/decompress_chunk/compressed_batch.h index 9871503e37c..dcacc40a8a0 100644 --- a/tsl/src/nodes/decompress_chunk/compressed_batch.h +++ b/tsl/src/nodes/decompress_chunk/compressed_batch.h @@ -103,7 +103,7 @@ typedef struct DecompressBatchState * row. Indexed same as arrow arrays, w/o accounting for the reverse scan * direction. Initialized to all ones, i.e. all rows pass. */ - uint64 *restrict vector_qual_result; + const uint64 *restrict vector_qual_result; /* * This follows DecompressContext.compressed_chunk_columns, but does not diff --git a/tsl/src/nodes/decompress_chunk/vector_predicates.h b/tsl/src/nodes/decompress_chunk/vector_predicates.h index cace90dbc44..204177c07a7 100644 --- a/tsl/src/nodes/decompress_chunk/vector_predicates.h +++ b/tsl/src/nodes/decompress_chunk/vector_predicates.h @@ -26,7 +26,7 @@ typedef enum VectorQualSummary } VectorQualSummary; static pg_attribute_always_inline VectorQualSummary -get_vector_qual_summary(uint64 *restrict qual_result, size_t n_rows) +get_vector_qual_summary(const uint64 *qual_result, size_t n_rows) { bool any_rows_pass = false; bool all_rows_pass = true; diff --git a/tsl/src/nodes/vector_agg/grouping_policy_hash.c b/tsl/src/nodes/vector_agg/grouping_policy_hash.c index 05a8e13d474..886489ab1b3 100644 --- a/tsl/src/nodes/vector_agg/grouping_policy_hash.c +++ b/tsl/src/nodes/vector_agg/grouping_policy_hash.c @@ -102,6 +102,7 @@ gp_hash_reset(GroupingPolicy *obj) policy->stat_input_valid_rows = 0; policy->stat_input_total_rows = 0; + policy->stat_bulk_filtered_rows = 0; policy->stat_consecutive_keys = 0; } @@ -331,7 +332,71 @@ gp_hash_add_batch(GroupingPolicy *gp, DecompressBatchState *batch_state) * Add the batch rows to aggregate function states. */ const uint64_t *restrict filter = batch_state->vector_qual_result; - add_one_range(policy, batch_state, 0, n); + if (filter == NULL) + { + /* + * We don't have a filter on this batch, so aggregate it entirely in one + * go. + */ + add_one_range(policy, batch_state, 0, n); + } + else + { + /* + * If we have a filter, skip the rows for which the entire words of the + * filter bitmap are zero. This improves performance for highly + * selective filters. + */ + int statistics_range_row = 0; + int start_word = 0; + int end_word = 0; + int past_the_end_word = (n - 1) / 64 + 1; + for (;;) + { + /* + * Skip the bitmap words which are zero. + */ + for (start_word = end_word; start_word < past_the_end_word && filter[start_word] == 0; + start_word++) + ; + + if (start_word >= past_the_end_word) + { + break; + } + + /* + * Collect the consecutive bitmap words which are nonzero. + */ + for (end_word = start_word + 1; end_word < past_the_end_word && filter[end_word] != 0; + end_word++) + ; + + /* + * Now we have the [start, end] range of bitmap words that are + * nonzero. + * + * Determine starting and ending rows, also skipping the starting + * and trailing zero bits at the ends of the range. + */ + const int start_row = start_word * 64 + pg_rightmost_one_pos64(filter[start_word]); + Assert(start_row <= n); + + /* + * The bits for past-the-end rows must be set to zero, so this + * calculation should yield no more than n. + */ + Assert(end_word > start_word); + const int end_row = + (end_word - 1) * 64 + pg_leftmost_one_pos64(filter[end_word - 1]) + 1; + Assert(end_row <= n); + + statistics_range_row += end_row - start_row; + + add_one_range(policy, batch_state, start_row, end_row); + } + policy->stat_bulk_filtered_rows += batch_state->total_batch_rows - statistics_range_row; + } policy->stat_input_total_rows += batch_state->total_batch_rows; policy->stat_input_valid_rows += arrow_num_valid(filter, batch_state->total_batch_rows); @@ -378,7 +443,7 @@ gp_hash_do_emit(GroupingPolicy *gp, TupleTableSlot *aggregated_slot) "%f ratio, %ld curctx bytes, %ld aggstate bytes", policy->stat_input_total_rows, policy->stat_input_valid_rows, - 0UL, + policy->stat_bulk_filtered_rows, policy->stat_consecutive_keys, keys, policy->stat_input_valid_rows / keys, diff --git a/tsl/src/nodes/vector_agg/grouping_policy_hash.h b/tsl/src/nodes/vector_agg/grouping_policy_hash.h index 3fc0016b444..2d8c3538131 100644 --- a/tsl/src/nodes/vector_agg/grouping_policy_hash.h +++ b/tsl/src/nodes/vector_agg/grouping_policy_hash.h @@ -129,6 +129,7 @@ typedef struct GroupingPolicyHash */ uint64 stat_input_total_rows; uint64 stat_input_valid_rows; + uint64 stat_bulk_filtered_rows; uint64 stat_consecutive_keys; } GroupingPolicyHash; From 1a8d0ca99f8ff5e4ffed7157d44ed91605c1976e Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Mon, 3 Feb 2025 15:28:55 +0100 Subject: [PATCH 2/3] fix --- tsl/src/nodes/vector_agg/grouping_policy_hash.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tsl/src/nodes/vector_agg/grouping_policy_hash.c b/tsl/src/nodes/vector_agg/grouping_policy_hash.c index 35a876b7b47..dc99d976aad 100644 --- a/tsl/src/nodes/vector_agg/grouping_policy_hash.c +++ b/tsl/src/nodes/vector_agg/grouping_policy_hash.c @@ -398,7 +398,8 @@ gp_hash_add_batch(GroupingPolicy *gp, TupleTableSlot *vector_slot) add_one_range(policy, vector_slot, start_row, end_row); } - policy->stat_bulk_filtered_rows += batch_state->total_batch_rows - statistics_range_row; + + policy->stat_bulk_filtered_rows += n - statistics_range_row; } policy->stat_input_total_rows += n; From a3a9f8ef0af7087baf90d1d8f014cd74ddf2d050 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Mon, 3 Feb 2025 16:19:01 +0100 Subject: [PATCH 3/3] format --- tsl/src/nodes/vector_agg/grouping_policy_hash.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tsl/src/nodes/vector_agg/grouping_policy_hash.c b/tsl/src/nodes/vector_agg/grouping_policy_hash.c index dc99d976aad..3170f48bb05 100644 --- a/tsl/src/nodes/vector_agg/grouping_policy_hash.c +++ b/tsl/src/nodes/vector_agg/grouping_policy_hash.c @@ -326,7 +326,7 @@ gp_hash_add_batch(GroupingPolicy *gp, TupleTableSlot *vector_slot) *vector_slot_get_compressed_column_values(vector_slot, AttrOffsetGetAttrNumber(def->input_offset)); } - + /* * Call the per-batch initialization function of the hashing strategy. */