diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/DataSketchesHllBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/DataSketchesHllBenchmark.java index 28dc5db99054..810829cd032a 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/DataSketchesHllBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/DataSketchesHllBenchmark.java @@ -23,6 +23,7 @@ import org.apache.druid.common.config.NullHandling; import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.BufferAggregator; +import org.apache.druid.query.aggregation.datasketches.SketchConfig; import org.apache.druid.query.aggregation.datasketches.hll.HllSketchMergeAggregatorFactory; import org.apache.druid.query.dimension.DimensionSpec; import org.apache.druid.segment.ColumnSelectorFactory; @@ -66,7 +67,8 @@ public class DataSketchesHllBenchmark null, null, null, - false + false, + new SketchConfig() ); private final ByteBuffer buf = ByteBuffer.allocateDirect(aggregatorFactory.getMaxIntermediateSize()); diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java index f153a4ff28b0..fe8496e60c1c 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java @@ -31,6 +31,7 @@ import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.query.QueryContexts; import org.apache.druid.query.QueryRunnerFactoryConglomerate; +import org.apache.druid.query.aggregation.datasketches.SketchConfig; import org.apache.druid.query.aggregation.datasketches.hll.sql.HllSketchApproxCountDistinctSqlAggregator; import org.apache.druid.query.aggregation.datasketches.hll.sql.HllSketchApproxCountDistinctUtf8SqlAggregator; import org.apache.druid.query.aggregation.datasketches.quantiles.sql.DoublesSketchApproxQuantileSqlAggregator; @@ -546,16 +547,17 @@ private void addSegmentToWalker( private static DruidOperatorTable createOperatorTable() { try { + final SketchConfig sketchConfig = new SketchConfig(); final Set extractionOperators = new HashSet<>(); extractionOperators.add(CalciteTests.INJECTOR.getInstance(QueryLookupOperatorConversion.class)); final ApproxCountDistinctSqlAggregator countDistinctSqlAggregator = - new ApproxCountDistinctSqlAggregator(new HllSketchApproxCountDistinctSqlAggregator()); + new ApproxCountDistinctSqlAggregator(new HllSketchApproxCountDistinctSqlAggregator(sketchConfig)); final Set aggregators = new HashSet<>( ImmutableList.of( new DoublesSketchApproxQuantileSqlAggregator(), new DoublesSketchObjectSqlAggregator(), - new HllSketchApproxCountDistinctSqlAggregator(), - new HllSketchApproxCountDistinctUtf8SqlAggregator(), + new HllSketchApproxCountDistinctSqlAggregator(sketchConfig), + new HllSketchApproxCountDistinctUtf8SqlAggregator(sketchConfig), new ThetaSketchApproxCountDistinctSqlAggregator(), new CountSqlAggregator(countDistinctSqlAggregator), countDistinctSqlAggregator diff --git a/docs/development/extensions-core/datasketches-hll.md b/docs/development/extensions-core/datasketches-hll.md index 3312dcc340d7..78f2e482e14b 100644 --- a/docs/development/extensions-core/datasketches-hll.md +++ b/docs/development/extensions-core/datasketches-hll.md @@ -51,6 +51,14 @@ For additional sketch types supported in Druid, see [DataSketches extension](dat The default `lgK` value has proven to be sufficient for most use cases; expect only very negligible improvements in accuracy with `lgK` values over `16` in normal circumstances. ::: +### Runtime properties + +The following runtime properties apply: + +|Property| Description| Default | +|--------|------------|------| +|`druid.sketch.config.hllMaxLgK`| The maximum possible value of lgK that HLL sketches can be created with. Useful to limit the maximum lgK in sketches, to avoid the significant usage of resources used by sketches at higher values of lgK. An exception will be thrown if a query configures a lgK value higher than this. This property needs to be set on the broker and middle-manager/indexer. | 20 | + ### HLLSketchBuild aggregator ``` diff --git a/docs/release-info/release-notes.md b/docs/release-info/release-notes.md index 768ceef697bc..1cd932dd0aff 100644 --- a/docs/release-info/release-notes.md +++ b/docs/release-info/release-notes.md @@ -91,6 +91,8 @@ This section contains detailed release notes separated by areas. ### Extensions +- Datasketches extension has a new runtime parameter `druid.sketch.config.hllMaxLgK`, which configures the maximum allowed value of lgK for HLL sketches. + ### Documentation improvements ## Upgrade notes and incompatible changes diff --git a/extensions-core/datasketches/pom.xml b/extensions-core/datasketches/pom.xml index c99cd793e8fd..442920a25890 100644 --- a/extensions-core/datasketches/pom.xml +++ b/extensions-core/datasketches/pom.xml @@ -92,6 +92,11 @@ guice provided + + jakarta.validation + jakarta.validation-api + provided + commons-codec commons-codec diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/SketchConfig.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/SketchConfig.java new file mode 100644 index 000000000000..f1ddc4da51fd --- /dev/null +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/SketchConfig.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.datasketches; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import javax.annotation.Nullable; +import javax.validation.constraints.Max; +import javax.validation.constraints.Min; + +/** + * Contains runtime configurations specific to datasketches extension. + */ +public class SketchConfig +{ + public static final int DEFAULT_HLL_MAX_LG_K = 20; + + @JsonProperty + @Min(1) + @Max(21) + private int hllMaxLgK = DEFAULT_HLL_MAX_LG_K; + + public SketchConfig(@Nullable Integer maxLgK) + { + this.hllMaxLgK = maxLgK == null ? DEFAULT_HLL_MAX_LG_K : maxLgK; + } + + public SketchConfig() + { + } + + public int getHllMaxLgK() + { + return hllMaxLgK; + } +} diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/SketchModule.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/SketchModule.java new file mode 100644 index 000000000000..163ee17f3561 --- /dev/null +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/SketchModule.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.datasketches; + +import com.google.inject.Binder; +import org.apache.druid.guice.JsonConfigProvider; +import org.apache.druid.initialization.DruidModule; + +public class SketchModule implements DruidModule +{ + @Override + public void configure(Binder binder) + { + JsonConfigProvider.bind(binder, "druid.sketch.config", SketchConfig.class); + } +} diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchAggregatorFactory.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchAggregatorFactory.java index eb874dbea27d..a9239b6905bf 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchAggregatorFactory.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchAggregatorFactory.java @@ -24,6 +24,7 @@ import org.apache.datasketches.hll.HllSketch; import org.apache.datasketches.hll.TgtHllType; import org.apache.datasketches.hll.Union; +import org.apache.druid.error.DruidException; import org.apache.druid.jackson.DefaultTrueJsonIncludeFilter; import org.apache.druid.java.util.common.StringEncoding; import org.apache.druid.java.util.common.StringEncodingDefaultUTF16LEJsonIncludeFilter; @@ -62,6 +63,31 @@ public abstract class HllSketchAggregatorFactory extends AggregatorFactory private final boolean shouldFinalize; private final boolean round; + HllSketchAggregatorFactory( + final String name, + final String fieldName, + @Nullable final Integer lgK, + @Nullable final String tgtHllType, + @Nullable final StringEncoding stringEncoding, + final Boolean shouldFinalize, + final boolean round, + final int maxLgK + ) + { + this(name, fieldName, lgK, tgtHllType, stringEncoding, shouldFinalize, round); + + if (this.lgK > maxLgK) { + throw DruidException.forPersona(DruidException.Persona.USER) + .ofCategory(DruidException.Category.INVALID_INPUT) + .build( + "LgK value [%s] for HLL sketch cannot be greater than [%s]. Reduce the lgK value" + + " or increase the runtime property druid.sketch.config.hllMaxLgK", + this.lgK, + maxLgK + ); + } + } + HllSketchAggregatorFactory( final String name, final String fieldName, diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchBuildAggregatorFactory.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchBuildAggregatorFactory.java index 6d42c678bbb1..fb9b28d355a9 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchBuildAggregatorFactory.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchBuildAggregatorFactory.java @@ -19,6 +19,7 @@ package org.apache.druid.query.aggregation.datasketches.hll; +import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.datasketches.hll.HllSketch; @@ -31,6 +32,7 @@ import org.apache.druid.query.aggregation.AggregatorUtil; import org.apache.druid.query.aggregation.BufferAggregator; import org.apache.druid.query.aggregation.VectorAggregator; +import org.apache.druid.query.aggregation.datasketches.SketchConfig; import org.apache.druid.query.dimension.DefaultDimensionSpec; import org.apache.druid.segment.ColumnInspector; import org.apache.druid.segment.ColumnSelectorFactory; @@ -62,12 +64,25 @@ public HllSketchBuildAggregatorFactory( @JsonProperty("tgtHllType") @Nullable final String tgtHllType, @JsonProperty("stringEncoding") @Nullable final StringEncoding stringEncoding, @JsonProperty("shouldFinalize") final Boolean shouldFinalize, - @JsonProperty("round") final boolean round + @JsonProperty("round") final boolean round, + @JacksonInject SketchConfig serverConfig ) { - super(name, fieldName, lgK, tgtHllType, stringEncoding, shouldFinalize, round); + super(name, fieldName, lgK, tgtHllType, stringEncoding, shouldFinalize, round, serverConfig.getHllMaxLgK()); } + HllSketchBuildAggregatorFactory( + String name, + String fieldName, + Integer lgK, + String tgtHllType, + StringEncoding stringEncoding, + Boolean shouldFinalize, + boolean round + ) + { + super(name, fieldName, lgK, tgtHllType, stringEncoding, shouldFinalize, round); + } @Override public ColumnType getIntermediateType() diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchMergeAggregatorFactory.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchMergeAggregatorFactory.java index 833df8ab1a55..049475effaff 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchMergeAggregatorFactory.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchMergeAggregatorFactory.java @@ -19,6 +19,7 @@ package org.apache.druid.query.aggregation.datasketches.hll; +import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.datasketches.hll.HllSketch; @@ -31,6 +32,7 @@ import org.apache.druid.query.aggregation.AggregatorUtil; import org.apache.druid.query.aggregation.BufferAggregator; import org.apache.druid.query.aggregation.VectorAggregator; +import org.apache.druid.query.aggregation.datasketches.SketchConfig; import org.apache.druid.segment.ColumnInspector; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; @@ -61,7 +63,21 @@ public HllSketchMergeAggregatorFactory( @JsonProperty("tgtHllType") @Nullable final String tgtHllType, @JsonProperty("stringEncoding") @Nullable final StringEncoding stringEncoding, @JsonProperty("shouldFinalize") final Boolean shouldFinalize, - @JsonProperty("round") final boolean round + @JsonProperty("round") final boolean round, + @JacksonInject SketchConfig serverConfig + ) + { + super(name, fieldName, lgK, tgtHllType, stringEncoding, shouldFinalize, round, serverConfig.getHllMaxLgK()); + } + + HllSketchMergeAggregatorFactory( + String name, + String fieldName, + Integer lgK, + String tgtHllType, + StringEncoding stringEncoding, + Boolean shouldFinalize, + boolean round ) { super(name, fieldName, lgK, tgtHllType, stringEncoding, shouldFinalize, round); diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchApproxCountDistinctSqlAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchApproxCountDistinctSqlAggregator.java index 757674d6aa68..8508425043a5 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchApproxCountDistinctSqlAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchApproxCountDistinctSqlAggregator.java @@ -19,6 +19,7 @@ package org.apache.druid.query.aggregation.datasketches.hll.sql; +import com.google.inject.Inject; import org.apache.calcite.sql.SqlAggFunction; import org.apache.calcite.sql.SqlFunctionCategory; import org.apache.calcite.sql.type.InferTypes; @@ -26,6 +27,7 @@ import org.apache.calcite.sql.type.SqlTypeName; import org.apache.druid.java.util.common.StringEncoding; import org.apache.druid.query.aggregation.AggregatorFactory; +import org.apache.druid.query.aggregation.datasketches.SketchConfig; import org.apache.druid.query.aggregation.post.FinalizingFieldAccessPostAggregator; import org.apache.druid.sql.calcite.aggregation.Aggregation; import org.apache.druid.sql.calcite.aggregation.SqlAggregator; @@ -47,9 +49,10 @@ public class HllSketchApproxCountDistinctSqlAggregator extends HllSketchBaseSqlA .functionCategory(SqlFunctionCategory.NUMERIC) .build(); - public HllSketchApproxCountDistinctSqlAggregator() + @Inject + public HllSketchApproxCountDistinctSqlAggregator(SketchConfig sketchConfig) { - super(true, StringEncoding.UTF16LE); + super(true, StringEncoding.UTF16LE, sketchConfig); } @Override diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchApproxCountDistinctUtf8SqlAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchApproxCountDistinctUtf8SqlAggregator.java index 070fbd9f7337..4757bd36ed44 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchApproxCountDistinctUtf8SqlAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchApproxCountDistinctUtf8SqlAggregator.java @@ -19,6 +19,7 @@ package org.apache.druid.query.aggregation.datasketches.hll.sql; +import com.google.inject.Inject; import org.apache.calcite.sql.SqlAggFunction; import org.apache.calcite.sql.SqlFunctionCategory; import org.apache.calcite.sql.type.InferTypes; @@ -26,6 +27,7 @@ import org.apache.calcite.sql.type.SqlTypeName; import org.apache.druid.java.util.common.StringEncoding; import org.apache.druid.query.aggregation.AggregatorFactory; +import org.apache.druid.query.aggregation.datasketches.SketchConfig; import org.apache.druid.query.aggregation.post.FinalizingFieldAccessPostAggregator; import org.apache.druid.sql.calcite.aggregation.Aggregation; import org.apache.druid.sql.calcite.aggregation.SqlAggregator; @@ -58,9 +60,10 @@ public class HllSketchApproxCountDistinctUtf8SqlAggregator .functionCategory(SqlFunctionCategory.NUMERIC) .build(); - public HllSketchApproxCountDistinctUtf8SqlAggregator() + @Inject + public HllSketchApproxCountDistinctUtf8SqlAggregator(SketchConfig sketchConfig) { - super(true, StringEncoding.UTF8); + super(true, StringEncoding.UTF8, sketchConfig); } @Override diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchBaseSqlAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchBaseSqlAggregator.java index d221b72ac1c6..76889b354780 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchBaseSqlAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchBaseSqlAggregator.java @@ -27,6 +27,7 @@ import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.StringEncoding; import org.apache.druid.query.aggregation.AggregatorFactory; +import org.apache.druid.query.aggregation.datasketches.SketchConfig; import org.apache.druid.query.aggregation.datasketches.SketchQueryContext; import org.apache.druid.query.aggregation.datasketches.hll.HllSketchAggregatorFactory; import org.apache.druid.query.aggregation.datasketches.hll.HllSketchBuildAggregatorFactory; @@ -53,11 +54,13 @@ public abstract class HllSketchBaseSqlAggregator implements SqlAggregator private final boolean finalizeSketch; private final StringEncoding stringEncoding; + private final SketchConfig serverConfig; - protected HllSketchBaseSqlAggregator(boolean finalizeSketch, StringEncoding stringEncoding) + protected HllSketchBaseSqlAggregator(boolean finalizeSketch, StringEncoding stringEncoding, SketchConfig serverConfig) { this.finalizeSketch = finalizeSketch; this.stringEncoding = stringEncoding; + this.serverConfig = serverConfig; } @Nullable @@ -128,7 +131,8 @@ public Aggregation toDruidAggregation( // the input encoding of the original sketches was, so we set it to the default. HllSketchAggregatorFactory.DEFAULT_STRING_ENCODING, finalizeSketch || SketchQueryContext.isFinalizeOuterSketches(plannerContext), - ROUND + ROUND, + serverConfig ); } else { final RelDataType dataType = columnRexNode.getType(); @@ -165,7 +169,8 @@ public Aggregation toDruidAggregation( // the input encoding of the original sketches was, so we set it to the default. HllSketchAggregatorFactory.DEFAULT_STRING_ENCODING, finalizeSketch || SketchQueryContext.isFinalizeOuterSketches(plannerContext), - ROUND + ROUND, + serverConfig ); } else { aggregatorFactory = new HllSketchBuildAggregatorFactory( @@ -175,7 +180,8 @@ public Aggregation toDruidAggregation( tgtHllType, stringEncoding, finalizeSketch || SketchQueryContext.isFinalizeOuterSketches(plannerContext), - ROUND + ROUND, + serverConfig ); } } diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchObjectSqlAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchObjectSqlAggregator.java index 9d8ade636f1b..46f1130eced8 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchObjectSqlAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchObjectSqlAggregator.java @@ -19,12 +19,14 @@ package org.apache.druid.query.aggregation.datasketches.hll.sql; +import com.google.inject.Inject; import org.apache.calcite.sql.SqlAggFunction; import org.apache.calcite.sql.SqlFunctionCategory; import org.apache.calcite.sql.type.InferTypes; import org.apache.calcite.sql.type.SqlTypeFamily; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.druid.query.aggregation.AggregatorFactory; +import org.apache.druid.query.aggregation.datasketches.SketchConfig; import org.apache.druid.query.aggregation.datasketches.hll.HllSketchAggregatorFactory; import org.apache.druid.sql.calcite.aggregation.Aggregation; import org.apache.druid.sql.calcite.aggregation.SqlAggregator; @@ -46,9 +48,10 @@ public class HllSketchObjectSqlAggregator extends HllSketchBaseSqlAggregator imp .functionCategory(SqlFunctionCategory.USER_DEFINED_FUNCTION) .build(); - public HllSketchObjectSqlAggregator() + @Inject + public HllSketchObjectSqlAggregator(SketchConfig sketchConfig) { - super(false, HllSketchAggregatorFactory.DEFAULT_STRING_ENCODING); + super(false, HllSketchAggregatorFactory.DEFAULT_STRING_ENCODING, sketchConfig); } @Override diff --git a/extensions-core/datasketches/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule b/extensions-core/datasketches/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule index d5d6dafb9e75..c30d946bda84 100644 --- a/extensions-core/datasketches/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule +++ b/extensions-core/datasketches/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule @@ -19,3 +19,4 @@ org.apache.druid.query.aggregation.datasketches.quantiles.DoublesSketchModule org.apache.druid.query.aggregation.datasketches.tuple.ArrayOfDoublesSketchModule org.apache.druid.query.aggregation.datasketches.hll.HllSketchModule org.apache.druid.query.aggregation.datasketches.kll.KllSketchModule +org.apache.druid.query.aggregation.datasketches.SketchModule \ No newline at end of file diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchAggregatorTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchAggregatorTest.java index 7c218934034b..b16dae24262e 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchAggregatorTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchAggregatorTest.java @@ -20,7 +20,9 @@ package org.apache.druid.query.aggregation.datasketches.hll; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.InjectableValues; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.exc.ValueInstantiationException; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.apache.datasketches.hll.HllSketch; @@ -33,6 +35,7 @@ import org.apache.druid.query.QueryContexts; import org.apache.druid.query.Result; import org.apache.druid.query.aggregation.AggregationTestHelper; +import org.apache.druid.query.aggregation.datasketches.SketchConfig; import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator; import org.apache.druid.query.groupby.GroupByQuery; import org.apache.druid.query.groupby.GroupByQueryConfig; @@ -79,9 +82,13 @@ public HllSketchAggregatorTest(GroupByQueryConfig config, String vectorize, Stri groupByHelper = AggregationTestHelper.createGroupByQueryAggregationTestHelper( new HllSketchModule().getJacksonModules(), config, groupByFolder ); + groupByHelper.getObjectMapper() + .setInjectableValues(new InjectableValues.Std().addValue(SketchConfig.class, new SketchConfig(21))); timeseriesHelper = AggregationTestHelper.createTimeseriesQueryAggregationTestHelper( new HllSketchModule().getJacksonModules(), timeseriesFolder ); + timeseriesHelper.getObjectMapper() + .setInjectableValues(new InjectableValues.Std().addValue(SketchConfig.class, new SketchConfig(21))); this.vectorize = QueryContexts.Vectorize.fromString(vectorize); this.stringEncoding = stringEncoding; } @@ -102,6 +109,33 @@ public static Collection constructorFeeder() return constructors; } + @Test + public void testHllSketchLimit() + { + AggregationTestHelper ingestHelper = AggregationTestHelper.createGroupByQueryAggregationTestHelper( + new HllSketchModule().getJacksonModules(), new GroupByQueryConfig(), groupByFolder + ); + ingestHelper.getObjectMapper() + .setInjectableValues(new InjectableValues.Std().addValue(SketchConfig.class, new SketchConfig(2))); + + Assert.assertThrows( + "LgK value [%s] for HLL sketch cannot be greater than [%s]. Reduce the lgK value or increase the runtime property druid.sketch.config.hllMaxLgK", + ValueInstantiationException.class, + () -> ingestHelper.createIndexAndRunQueryOnSegment( + new File(this.getClass().getClassLoader().getResource("hll/hll_sketches.tsv").getFile()), + buildParserJson( + Arrays.asList("dim", "multiDim"), + Arrays.asList("timestamp", "dim", "multiDim", "sketch") + ), + buildAggregatorJson("HLLSketchMerge", "sketch", !ROUND, stringEncoding), + 0, // minTimestamp + Granularities.NONE, + 200, // maxRowCount + buildGroupByQueryJson("HLLSketchMerge", "sketch", !ROUND, stringEncoding) + ) + ); + } + @Test public void ingestSketches() throws Exception { diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchBuildAggregatorFactoryTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchBuildAggregatorFactoryTest.java index 51ca671cd0d7..fd7b3b0ed842 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchBuildAggregatorFactoryTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchBuildAggregatorFactoryTest.java @@ -19,11 +19,13 @@ package org.apache.druid.query.aggregation.datasketches.hll; +import com.fasterxml.jackson.databind.InjectableValues; import com.fasterxml.jackson.databind.ObjectMapper; import nl.jqno.equalsverifier.EqualsVerifier; import org.apache.datasketches.hll.TgtHllType; import org.apache.druid.java.util.common.StringEncoding; import org.apache.druid.query.aggregation.AggregatorFactory; +import org.apache.druid.query.aggregation.datasketches.SketchConfig; import org.apache.druid.segment.TestHelper; import org.junit.Assert; import org.junit.Test; @@ -38,6 +40,7 @@ public HllSketchBuildAggregatorFactoryTest() { this.jsonMapper = TestHelper.makeJsonMapper().copy(); jsonMapper.registerModules(new HllSketchModule().getJacksonModules()); + jsonMapper.setInjectableValues(new InjectableValues.Std().addValue(SketchConfig.class, new SketchConfig())); } @Test diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchMergeAggregatorFactoryTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchMergeAggregatorFactoryTest.java index 101b25b99be0..a39a440612ff 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchMergeAggregatorFactoryTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchMergeAggregatorFactoryTest.java @@ -19,12 +19,14 @@ package org.apache.druid.query.aggregation.datasketches.hll; +import com.fasterxml.jackson.databind.InjectableValues; import com.fasterxml.jackson.databind.ObjectMapper; import nl.jqno.equalsverifier.EqualsVerifier; import org.apache.datasketches.hll.TgtHllType; import org.apache.druid.java.util.common.StringEncoding; import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.AggregatorFactoryNotMergeableException; +import org.apache.druid.query.aggregation.datasketches.SketchConfig; import org.apache.druid.segment.TestHelper; import org.junit.Assert; import org.junit.Before; @@ -215,6 +217,7 @@ public void testSerde() throws IOException { final ObjectMapper jsonMapper = TestHelper.makeJsonMapper().copy(); jsonMapper.registerModules(new HllSketchModule().getJacksonModules()); + jsonMapper.setInjectableValues(new InjectableValues.Std().addValue(SketchConfig.class, new SketchConfig())); final HllSketchMergeAggregatorFactory factory = new HllSketchMergeAggregatorFactory( "foo", @@ -247,6 +250,7 @@ public void testSerdeWithDefaults() throws IOException { final ObjectMapper jsonMapper = TestHelper.makeJsonMapper().copy(); jsonMapper.registerModules(new HllSketchModule().getJacksonModules()); + jsonMapper.setInjectableValues(new InjectableValues.Std().addValue(SketchConfig.class, new SketchConfig())); final HllSketchMergeAggregatorFactory factory = new HllSketchMergeAggregatorFactory( "foo", diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchSqlAggregatorTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchSqlAggregatorTest.java index 8701abbe486b..b121cb41ac6b 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchSqlAggregatorTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchSqlAggregatorTest.java @@ -40,6 +40,7 @@ import org.apache.druid.query.aggregation.FilteredAggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.query.aggregation.PostAggregator; +import org.apache.druid.query.aggregation.datasketches.SketchConfig; import org.apache.druid.query.aggregation.datasketches.SketchQueryContext; import org.apache.druid.query.aggregation.datasketches.hll.HllSketchBuildAggregatorFactory; import org.apache.druid.query.aggregation.datasketches.hll.HllSketchMergeAggregatorFactory; @@ -144,6 +145,8 @@ public class HllSketchSqlAggregatorTest extends BaseCalciteQueryTest 2L }; + private static final SketchConfig SKETCH_CONFIG = new SketchConfig(); + /** * Expected virtual columns for {@link #testHllSketchPostAggsTimeseries()}, * {@link #testHllSketchPostAggsGroupBy()}, {@link #testHllSketchFilteredAggregatorsTimeseries()}, and @@ -170,13 +173,13 @@ public class HllSketchSqlAggregatorTest extends BaseCalciteQueryTest */ private static final List EXPECTED_PA_AGGREGATORS = ImmutableList.of( - new HllSketchBuildAggregatorFactory("a0", "dim2", null, null, null, false, true), - new HllSketchBuildAggregatorFactory("a1", "m1", null, null, null, false, true), - new HllSketchBuildAggregatorFactory("a2", "cnt", null, null, null, false, true), - new HllSketchBuildAggregatorFactory("a3", "v0", null, null, null, false, true), - new HllSketchBuildAggregatorFactory("a4", "v1", null, null, null, false, true), - new HllSketchBuildAggregatorFactory("a5", "dim2", null, null, null, true, true), - new HllSketchBuildAggregatorFactory("a6", "dim2", null, null, StringEncoding.UTF8, true, true) + new HllSketchBuildAggregatorFactory("a0", "dim2", null, null, null, false, true, SKETCH_CONFIG), + new HllSketchBuildAggregatorFactory("a1", "m1", null, null, null, false, true, SKETCH_CONFIG), + new HllSketchBuildAggregatorFactory("a2", "cnt", null, null, null, false, true, SKETCH_CONFIG), + new HllSketchBuildAggregatorFactory("a3", "v0", null, null, null, false, true, SKETCH_CONFIG), + new HllSketchBuildAggregatorFactory("a4", "v1", null, null, null, false, true, SKETCH_CONFIG), + new HllSketchBuildAggregatorFactory("a5", "dim2", null, null, null, true, true, SKETCH_CONFIG), + new HllSketchBuildAggregatorFactory("a6", "dim2", null, null, StringEncoding.UTF8, true, true, SKETCH_CONFIG) ); /** @@ -265,12 +268,12 @@ public SpecificSegmentsQuerySegmentWalker createQuerySegmentWalker( .withMetrics( new CountAggregatorFactory("cnt"), new DoubleSumAggregatorFactory("m1", "m1"), - new HllSketchBuildAggregatorFactory("hllsketch_dim1", "dim1", null, null, null, false, ROUND), - new HllSketchBuildAggregatorFactory("hllsketch_dim3", "dim3", null, null, null, false, false), - new HllSketchBuildAggregatorFactory("hllsketch_m1", "m1", null, null, null, false, ROUND), - new HllSketchBuildAggregatorFactory("hllsketch_f1", "f1", null, null, null, false, ROUND), - new HllSketchBuildAggregatorFactory("hllsketch_l1", "l1", null, null, null, false, ROUND), - new HllSketchBuildAggregatorFactory("hllsketch_d1", "d1", null, null, null, false, ROUND) + new HllSketchBuildAggregatorFactory("hllsketch_dim1", "dim1", null, null, null, false, ROUND, new SketchConfig()), + new HllSketchBuildAggregatorFactory("hllsketch_dim3", "dim3", null, null, null, false, false, new SketchConfig()), + new HllSketchBuildAggregatorFactory("hllsketch_m1", "m1", null, null, null, false, ROUND, new SketchConfig()), + new HllSketchBuildAggregatorFactory("hllsketch_f1", "f1", null, null, null, false, ROUND, new SketchConfig()), + new HllSketchBuildAggregatorFactory("hllsketch_l1", "l1", null, null, null, false, ROUND, new SketchConfig()), + new HllSketchBuildAggregatorFactory("hllsketch_d1", "d1", null, null, null, false, ROUND, new SketchConfig()) ) .withRollup(false) .build() @@ -302,9 +305,9 @@ public void testApproxCountDistinctHllSketch() + " APPROX_COUNT_DISTINCT_DS_HLL(dim2) FILTER(WHERE dim2 <> ''),\n" // lowercase; also, filtered + " APPROX_COUNT_DISTINCT(SUBSTRING(dim2, 1, 1)),\n" // on extractionFn, using generic A.C.D. + " COUNT(DISTINCT SUBSTRING(dim2, 1, 1) || 'x'),\n" // on expression, using COUNT DISTINCT - + " APPROX_COUNT_DISTINCT_DS_HLL(hllsketch_dim1, 21, 'HLL_8'),\n" // on native HllSketch column + + " APPROX_COUNT_DISTINCT_DS_HLL(hllsketch_dim1, 20, 'HLL_8'),\n" // on native HllSketch column + " APPROX_COUNT_DISTINCT_DS_HLL(hllsketch_dim1),\n" // on native HllSketch column - + " APPROX_COUNT_DISTINCT_DS_HLL(hllsketch_dim1, CAST(21 AS BIGINT))\n" // also native column + + " APPROX_COUNT_DISTINCT_DS_HLL(hllsketch_dim1, CAST(20 AS BIGINT))\n" // also native column + "FROM druid.foo"; final List expectedResults; @@ -343,16 +346,16 @@ public void testApproxCountDistinctHllSketch() .aggregators( ImmutableList.of( new LongSumAggregatorFactory("a0", "cnt"), - new HllSketchBuildAggregatorFactory("a1", "dim2", null, null, null, null, ROUND), + new HllSketchBuildAggregatorFactory("a1", "dim2", null, null, null, null, ROUND, SKETCH_CONFIG), new FilteredAggregatorFactory( - new HllSketchBuildAggregatorFactory("a2", "dim2", null, null, null, null, ROUND), + new HllSketchBuildAggregatorFactory("a2", "dim2", null, null, null, null, ROUND, SKETCH_CONFIG), not(equality("dim2", "", ColumnType.STRING)) ), - new HllSketchBuildAggregatorFactory("a3", "v0", null, null, null, null, ROUND), - new HllSketchBuildAggregatorFactory("a4", "v1", null, null, null, null, ROUND), - new HllSketchMergeAggregatorFactory("a5", "hllsketch_dim1", 21, "HLL_8", null, null, ROUND), - new HllSketchMergeAggregatorFactory("a6", "hllsketch_dim1", null, null, null, null, ROUND), - new HllSketchMergeAggregatorFactory("a7", "hllsketch_dim1", 21, "HLL_4", null, null, ROUND) + new HllSketchBuildAggregatorFactory("a3", "v0", null, null, null, null, ROUND, SKETCH_CONFIG), + new HllSketchBuildAggregatorFactory("a4", "v1", null, null, null, null, ROUND, SKETCH_CONFIG), + new HllSketchMergeAggregatorFactory("a5", "hllsketch_dim1", 20, "HLL_8", null, null, ROUND, SKETCH_CONFIG), + new HllSketchMergeAggregatorFactory("a6", "hllsketch_dim1", null, null, null, null, ROUND, SKETCH_CONFIG), + new HllSketchMergeAggregatorFactory("a7", "hllsketch_dim1", 20, "HLL_4", null, null, ROUND, SKETCH_CONFIG) ) ) .context(QUERY_CONTEXT_DEFAULT) @@ -402,7 +405,8 @@ public void testAvgDailyCountDistinctHllSketch() null, null, null, - ROUND + ROUND, + SKETCH_CONFIG ) ) ) @@ -479,7 +483,7 @@ public void testApproxCountDistinctHllSketchIsRounded() .setGranularity(Granularities.ALL) .setAggregatorSpecs( aggregators( - new HllSketchBuildAggregatorFactory("a0", "m1", null, null, null, true, true) + new HllSketchBuildAggregatorFactory("a0", "m1", null, null, null, true, true, SKETCH_CONFIG) ) ) .setHavingSpec(having(equality("a0", 2L, ColumnType.LONG))) @@ -728,11 +732,11 @@ public void testHllSketchPostAggsFinalizeOuterSketches() ) ) .aggregators( - new HllSketchBuildAggregatorFactory("a0", "dim2", null, null, null, true, true), - new HllSketchBuildAggregatorFactory("a1", "m1", null, null, null, true, true), - new HllSketchBuildAggregatorFactory("a2", "v0", null, null, null, true, true), - new HllSketchBuildAggregatorFactory("a3", "v1", null, null, null, true, true), - new HllSketchBuildAggregatorFactory("a4", "dim2", null, null, null, true, true) + new HllSketchBuildAggregatorFactory("a0", "dim2", null, null, null, true, true, SKETCH_CONFIG), + new HllSketchBuildAggregatorFactory("a1", "m1", null, null, null, true, true, SKETCH_CONFIG), + new HllSketchBuildAggregatorFactory("a2", "v0", null, null, null, true, true, SKETCH_CONFIG), + new HllSketchBuildAggregatorFactory("a3", "v1", null, null, null, true, true, SKETCH_CONFIG), + new HllSketchBuildAggregatorFactory("a4", "dim2", null, null, null, true, true, SKETCH_CONFIG) ) .postAggregators( new HllSketchToEstimatePostAggregator("p1", new FieldAccessPostAggregator("p0", "a0"), false), @@ -815,7 +819,8 @@ public void testtHllSketchPostAggsPostSort() null, null, false, - true + true, + SKETCH_CONFIG ) ) ) @@ -861,7 +866,8 @@ public void testEmptyTimeseriesResults() null, null, null, - true + true, + SKETCH_CONFIG ), new HllSketchBuildAggregatorFactory( "a1", @@ -870,7 +876,8 @@ public void testEmptyTimeseriesResults() null, null, false, - true + true, + SKETCH_CONFIG ) ) ) @@ -907,7 +914,8 @@ public void testGroupByAggregatorDefaultValues() null, null, null, - true + true, + SKETCH_CONFIG ), equality("dim1", "nonexistent", ColumnType.STRING) ), @@ -919,7 +927,8 @@ public void testGroupByAggregatorDefaultValues() null, null, false, - true + true, + SKETCH_CONFIG ), equality("dim1", "nonexistent", ColumnType.STRING) ) @@ -959,11 +968,11 @@ public void testGroupByAggregatorDefaultValuesFinalizeOuterSketches() .setAggregatorSpecs( aggregators( new FilteredAggregatorFactory( - new HllSketchBuildAggregatorFactory("a0", "v0", null, null, null, null, true), + new HllSketchBuildAggregatorFactory("a0", "v0", null, null, null, null, true, SKETCH_CONFIG), equality("dim1", "nonexistent", ColumnType.STRING) ), new FilteredAggregatorFactory( - new HllSketchBuildAggregatorFactory("a1", "v0", null, null, null, null, true), + new HllSketchBuildAggregatorFactory("a1", "v0", null, null, null, null, true, SKETCH_CONFIG), equality("dim1", "nonexistent", ColumnType.STRING) ) ) @@ -1181,8 +1190,8 @@ public void testEstimateStringAndDoubleAreDifferent() .intervals(querySegmentSpec(Filtration.eternity())) .granularity(Granularities.ALL) .aggregators( - new HllSketchMergeAggregatorFactory("a0", "hllsketch_d1", null, null, null, false, true), - new HllSketchMergeAggregatorFactory("a1", "hllsketch_m1", null, null, null, false, true) + new HllSketchMergeAggregatorFactory("a0", "hllsketch_d1", null, null, null, false, true, SKETCH_CONFIG), + new HllSketchMergeAggregatorFactory("a1", "hllsketch_m1", null, null, null, false, true, SKETCH_CONFIG) ) .postAggregators( new HllSketchToEstimatePostAggregator( @@ -1231,8 +1240,8 @@ public void testFloatAndDoubleAreConsideredTheSame() .intervals(querySegmentSpec(Filtration.eternity())) .granularity(Granularities.ALL) .aggregators( - new HllSketchMergeAggregatorFactory("a0", "hllsketch_d1", null, null, null, false, true), - new HllSketchMergeAggregatorFactory("a1", "hllsketch_f1", null, null, null, false, true) + new HllSketchMergeAggregatorFactory("a0", "hllsketch_d1", null, null, null, false, true, SKETCH_CONFIG), + new HllSketchMergeAggregatorFactory("a1", "hllsketch_f1", null, null, null, false, true, SKETCH_CONFIG) ) .postAggregators( new HllSketchToEstimatePostAggregator( diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQDataSketchesTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQDataSketchesTest.java index 1f856027de30..1e6b79018148 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQDataSketchesTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQDataSketchesTest.java @@ -27,6 +27,7 @@ import org.apache.druid.msq.indexing.destination.TaskReportMSQDestination; import org.apache.druid.msq.test.MSQTestBase; import org.apache.druid.query.aggregation.FilteredAggregatorFactory; +import org.apache.druid.query.aggregation.datasketches.SketchConfig; import org.apache.druid.query.aggregation.datasketches.hll.HllSketchBuildAggregatorFactory; import org.apache.druid.query.dimension.DefaultDimensionSpec; import org.apache.druid.query.groupby.GroupByQuery; @@ -43,6 +44,9 @@ */ public class MSQDataSketchesTest extends MSQTestBase { + + private static final SketchConfig SKETCH_CONFIG = new SketchConfig(); + @Test public void testHavingOnDsHll() { @@ -60,7 +64,7 @@ public void testHavingOnDsHll() .setDimensions(dimensions(new DefaultDimensionSpec("dim2", "d0"))) .setAggregatorSpecs( aggregators( - new HllSketchBuildAggregatorFactory("a0", "m1", 12, "HLL_4", null, false, true) + new HllSketchBuildAggregatorFactory("a0", "m1", 12, "HLL_4", null, false, true, SKETCH_CONFIG) ) ) .setHavingSpec(having(expressionFilter(("(hll_sketch_estimate(\"a0\") > 1)")))) @@ -113,7 +117,7 @@ public void testEmptyHllSketch() .setAggregatorSpecs( aggregators( new FilteredAggregatorFactory( - new HllSketchBuildAggregatorFactory("a0", "dim2", 12, "HLL_4", null, true, true), + new HllSketchBuildAggregatorFactory("a0", "dim2", 12, "HLL_4", null, true, true, SKETCH_CONFIG), equality("dim1", "nonexistent", ColumnType.STRING), "a0" ) diff --git a/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionTest.java b/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionTest.java index 26df03e0d81f..cad8e37f61d2 100644 --- a/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionTest.java +++ b/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionTest.java @@ -47,6 +47,7 @@ import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory; import org.apache.druid.query.aggregation.FloatSumAggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; +import org.apache.druid.query.aggregation.datasketches.SketchConfig; import org.apache.druid.query.aggregation.datasketches.hll.HllSketchBuildAggregatorFactory; import org.apache.druid.query.aggregation.datasketches.quantiles.DoublesSketchAggregatorFactory; import org.apache.druid.query.aggregation.datasketches.theta.SketchMergeAggregatorFactory; @@ -172,7 +173,7 @@ public void testAutoCompactionRowWithMetricAndRowWithoutMetricShouldPreserveExis // FloatSumAggregator combine method takes in two Float but return Double new FloatSumAggregatorFactory("sum_added", "added"), new SketchMergeAggregatorFactory("thetaSketch", "user", 16384, true, false, null), - new HllSketchBuildAggregatorFactory("HLLSketchBuild", "user", 12, TgtHllType.HLL_4.name(), null, false, false), + new HllSketchBuildAggregatorFactory("HLLSketchBuild", "user", 12, TgtHllType.HLL_4.name(), null, false, false, new SketchConfig()), new DoublesSketchAggregatorFactory("quantilesDoublesSketch", "delta", 128, 1000000000L, null) }, false @@ -273,7 +274,8 @@ public void testAutoCompactionRowWithMetricAndRowWithoutMetricShouldPreserveExis TgtHllType.HLL_4.name(), null, false, - false + false, + new SketchConfig(15) ), new DoublesSketchAggregatorFactory("quantilesDoublesSketch", "delta", 128, 1000000000L, null) },