From db957f272b809f8f507201cebc75a010f6da34a0 Mon Sep 17 00:00:00 2001 From: Aleksandar Stanchev Date: Thu, 20 Jun 2024 22:17:31 +0300 Subject: [PATCH 01/13] Extended search based custom operator metrics Signed-off-by: Aleksandar Stanchev --- .../metrics/instruments/gauge/KamonGauge.java | 2 +- thingsearch/service/pom.xml | 4 + .../config/CustomSearchMetricConfig.java | 151 +++++++ .../DefaultCustomSearchMetricConfig.java | 181 +++++++++ .../config/DefaultOperatorMetricsConfig.java | 51 +++ .../common/config/OperatorMetricsConfig.java | 14 +- .../OperatorSearchMetricsProviderActor.java | 379 ++++++++++++++++++ .../service/starter/actors/SearchActor.java | 7 +- .../actors/SearchUpdaterRootActor.java | 4 + .../src/main/resources/search-dev.conf | 38 ++ 10 files changed, 827 insertions(+), 4 deletions(-) create mode 100644 thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/CustomSearchMetricConfig.java create mode 100644 thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultCustomSearchMetricConfig.java create mode 100644 thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/OperatorSearchMetricsProviderActor.java diff --git a/internal/utils/metrics/src/main/java/org/eclipse/ditto/internal/utils/metrics/instruments/gauge/KamonGauge.java b/internal/utils/metrics/src/main/java/org/eclipse/ditto/internal/utils/metrics/instruments/gauge/KamonGauge.java index 9a4b49dd01..dfb62ba9cf 100644 --- a/internal/utils/metrics/src/main/java/org/eclipse/ditto/internal/utils/metrics/instruments/gauge/KamonGauge.java +++ b/internal/utils/metrics/src/main/java/org/eclipse/ditto/internal/utils/metrics/instruments/gauge/KamonGauge.java @@ -102,7 +102,7 @@ public TagSet getTagSet() { @Override public boolean reset() { getKamonInternalGauge().update(0); - LOGGER.trace("Reset histogram with name <{}>.", name); + LOGGER.trace("Reset gauge with name <{}>.", name); return true; } diff --git a/thingsearch/service/pom.xml b/thingsearch/service/pom.xml index 7cd9c28d52..9ddbcc2a86 100644 --- a/thingsearch/service/pom.xml +++ b/thingsearch/service/pom.xml @@ -154,6 +154,10 @@ test test-jar + + org.eclipse.ditto + ditto-edge-service + diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/CustomSearchMetricConfig.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/CustomSearchMetricConfig.java new file mode 100644 index 0000000000..a133263367 --- /dev/null +++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/CustomSearchMetricConfig.java @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2024 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.thingsearch.service.common.config; + +import org.eclipse.ditto.internal.utils.config.KnownConfigValue; + +import java.time.Duration; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * Provides the configuration settings for a single custom search metric. + */ +public interface CustomSearchMetricConfig { + + + /** + * Returns the name of the custom metric. + * + * @return the name of the custom metric. + */ + String getCustomMetricName(); + + /** + * Returns whether this specific search operator metric gathering is turned on. + * + * @return true or false. + */ + boolean isEnabled(); + + /** + * Returns the optional scrape interval override for this specific custom metric, how often the metrics should be + * gathered. + * + * @return the optional scrape interval override. + */ + Optional getScrapeInterval(); + + /** + * Returns the namespaces the custom metric should be executed in or an empty list for gathering metrics in all + * namespaces. + * + * @return a list of namespaces. + */ + List getNamespaces(); + + /** + * Return optional tags to report to the custom Gauge metric. + * + * @return optional tags to report. + */ + Map getTags(); + + /** + * Returns the filter configurations for this custom metric. + * + * @return the filter configurations. + */ + List getFilterConfigs(); + + enum CustomSearchMetricConfigValue implements KnownConfigValue { + /** + * Whether the metrics should be gathered. + */ + ENABLED("enabled", true), + + /** + * The optional custom scrape interval, how often the metrics should be gathered. + * If this is {@code Duration.ZERO}, then there no overwriting for the "global" scrape-interval to be applied. + */ + SCRAPE_INTERVAL("scrape-interval", Duration.ZERO), + + /** + * The namespaces the custom metric should be executed in or an empty list for gathering metrics in all + * namespaces. + */ + NAMESPACES("namespaces", List.of()), + + /** + * The optional tags to report to the custom Gauge metric. + */ + TAGS("tags", Map.of()), + + FILTERS("filters", List.of()); + + private final String path; + private final Object defaultValue; + + CustomSearchMetricConfigValue(final String thePath, final Object theDefaultValue) { + path = thePath; + defaultValue = theDefaultValue; + } + + @Override + public Object getDefaultValue() { + return defaultValue; + } + + @Override + public String getConfigPath() { + return path; + } + } + + interface FilterConfig { + + String getFilterName(); + + String getFilter(); + + List getFields(); + + Map getInlinePlaceholderValues(); + + enum FilterConfigValues implements KnownConfigValue { + FILTER("filter", ""), + FIELDS("fields", Collections.emptyList()), + INLINE_PLACEHOLDER_VALUES("inline-placeholder-values", Map.of()); + + private final String path; + private final Object defaultValue; + + FilterConfigValues(final String thePath, final Object theDefaultValue) { + path = thePath; + defaultValue = theDefaultValue; + } + + @Override + public Object getDefaultValue() { + return defaultValue; + } + + @Override + public String getConfigPath() { + return path; + } + } + } +} diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultCustomSearchMetricConfig.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultCustomSearchMetricConfig.java new file mode 100644 index 0000000000..26b7809584 --- /dev/null +++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultCustomSearchMetricConfig.java @@ -0,0 +1,181 @@ +/* + * Copyright (c) 2024 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ + +package org.eclipse.ditto.thingsearch.service.common.config; + +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; + +import java.time.Duration; +import java.util.*; +import java.util.stream.Collectors; + +import org.eclipse.ditto.internal.utils.config.ConfigWithFallback; + +public final class DefaultCustomSearchMetricConfig implements CustomSearchMetricConfig { + + private final String customMetricName; + private final boolean enabled; + private final Duration scrapeInterval; + private final List namespaces; + private final Map tags; + private final List filterConfigs; + + private DefaultCustomSearchMetricConfig(final String key, final ConfigWithFallback configWithFallback) { + this.customMetricName = key; + enabled = configWithFallback.getBoolean(CustomSearchMetricConfigValue.ENABLED.getConfigPath()); + scrapeInterval = configWithFallback.getDuration(CustomSearchMetricConfigValue.SCRAPE_INTERVAL.getConfigPath()); + namespaces = configWithFallback.getStringList(CustomSearchMetricConfigValue.NAMESPACES.getConfigPath()); + tags = configWithFallback.getObject(CustomSearchMetricConfig.CustomSearchMetricConfigValue.TAGS.getConfigPath()).unwrapped() + .entrySet() + .stream() + .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, e -> String.valueOf(e.getValue()))); + filterConfigs = configWithFallback.getObject(CustomSearchMetricConfigValue.FILTERS.getConfigPath()).entrySet().stream() + .map(entry -> DefaultFilterConfig.of(entry.getKey(), ConfigFactory.empty().withFallback(entry.getValue()))) + .collect(Collectors.toList()); + } + + @Override + public String getCustomMetricName() { + return customMetricName; + } + + @Override + public boolean isEnabled() { + return enabled; + } + + @Override + public Optional getScrapeInterval() { + return scrapeInterval.isZero() ? Optional.empty() : Optional.of(scrapeInterval); + } + + @Override + public List getNamespaces() { + return namespaces; + } + + @Override + public Map getTags() { + return tags; + } + + @Override + public List getFilterConfigs() { + return filterConfigs; + } + + public static DefaultCustomSearchMetricConfig of(final String key, final Config config){ + return new DefaultCustomSearchMetricConfig(key , ConfigWithFallback.newInstance(config, CustomSearchMetricConfigValue.values())); + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final DefaultCustomSearchMetricConfig that = (DefaultCustomSearchMetricConfig) o; + return enabled == that.enabled && + Objects.equals(scrapeInterval, that.scrapeInterval) && + Objects.equals(namespaces, that.namespaces) && + Objects.equals(tags, that.tags) && + Objects.equals(filterConfigs, that.filterConfigs); + } + + @Override + public int hashCode() { + return Objects.hash(enabled, scrapeInterval, namespaces, tags, filterConfigs); + } + + @Override + public String toString() { + return getClass().getSimpleName() + " [" + + "enabled=" + enabled + + ", scrapeInterval=" + scrapeInterval + + ", namespaces=" + namespaces + + ", tags=" + tags + + ", filterConfig=" + filterConfigs + + "]"; + } + + public static final class DefaultFilterConfig implements FilterConfig { + + private final String filterName; + private final String filter; + private final List fields; + private final Map inlinePlaceholderValues; + + private DefaultFilterConfig(final String name, final ConfigWithFallback configWithFallback) { + this.filterName = name; + this.filter = configWithFallback.getString(FilterConfigValues.FILTER.getConfigPath()); + this.fields = configWithFallback.getStringList(FilterConfigValues.FIELDS.getConfigPath()); + this.inlinePlaceholderValues = configWithFallback.getObject(FilterConfigValues.INLINE_PLACEHOLDER_VALUES.getConfigPath()).unwrapped() + .entrySet() + .stream() + .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, e -> String.valueOf(e.getValue()))); + } + + @Override + public String getFilterName() { + return filterName; + } + + @Override + public String getFilter() { + return filter; + } + + @Override + public List getFields() { + return fields; + } + + @Override + public Map getInlinePlaceholderValues() { + return inlinePlaceholderValues; + } + + public static FilterConfig of(final String name, final Config config) { + return new DefaultFilterConfig( + name, ConfigWithFallback.newInstance(config, CustomSearchMetricConfigValue.values())); + } + + @Override + public boolean equals(final Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + final DefaultFilterConfig that = (DefaultFilterConfig) o; + return Objects.equals(filterName, that.filterName) && Objects.equals(filter, that.filter) && + Objects.equals(fields, that.fields) && + Objects.equals(inlinePlaceholderValues, that.inlinePlaceholderValues); + } + + @Override + public int hashCode() { + return Objects.hash(filterName, filter, fields, inlinePlaceholderValues); + } + + @Override + public String toString() { + return getClass().getSimpleName() + " [" + + "filterName='" + filterName + '\'' + + ", filter='" + filter + '\'' + + ", fields=" + fields + + ", inlinePlaceholderValues=" + inlinePlaceholderValues + + ']'; + } + } +} diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultOperatorMetricsConfig.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultOperatorMetricsConfig.java index c582f69b7b..fea4391d92 100644 --- a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultOperatorMetricsConfig.java +++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultOperatorMetricsConfig.java @@ -50,12 +50,15 @@ public final class DefaultOperatorMetricsConfig implements OperatorMetricsConfig private final boolean enabled; private final Duration scrapeInterval; private final Map customMetricConfigurations; + private final Map customSearchMetricConfigurations; private DefaultOperatorMetricsConfig(final ConfigWithFallback updaterScopedConfig) { enabled = updaterScopedConfig.getBoolean(OperatorMetricsConfigValue.ENABLED.getConfigPath()); scrapeInterval = updaterScopedConfig.getNonNegativeDurationOrThrow(OperatorMetricsConfigValue.SCRAPE_INTERVAL); customMetricConfigurations = loadCustomMetricConfigurations(updaterScopedConfig, OperatorMetricsConfigValue.CUSTOM_METRICS); + customSearchMetricConfigurations = loadCustomSearchMetricConfigurations(updaterScopedConfig, + OperatorMetricsConfigValue.CUSTOM_SEARCH_METRICS); } /** @@ -78,6 +81,14 @@ private static Map loadCustomMetricConfigurations(fi return customMetricsConfig.entrySet().stream().collect(CustomMetricConfigCollector.toMap()); } + private Map loadCustomSearchMetricConfigurations( + final ConfigWithFallback config, final KnownConfigValue configValue) { + + final ConfigObject customSearchMetricsConfig = config.getObject(configValue.getConfigPath()); + + return customSearchMetricsConfig.entrySet().stream().collect(CustomSearchMetricConfigCollector.toMap()); + } + @Override public boolean equals(final Object o) { if (this == o) { @@ -120,6 +131,11 @@ public Map getCustomMetricConfigurations() { return customMetricConfigurations; } + @Override + public Map getCustomSearchMetricConfigurations() { + return customSearchMetricConfigurations; + } + private static class CustomMetricConfigCollector implements Collector, Map, Map> { @@ -155,4 +171,39 @@ public Set characteristics() { return Collections.singleton(Characteristics.UNORDERED); } } + + private static class CustomSearchMetricConfigCollector implements + Collector, Map, Map> { + + private static DefaultOperatorMetricsConfig.CustomSearchMetricConfigCollector toMap() { + return new DefaultOperatorMetricsConfig.CustomSearchMetricConfigCollector(); + } + + @Override + public Supplier> supplier() { + return LinkedHashMap::new; + } + + @Override + public BiConsumer, Map.Entry> accumulator() { + return (map, entry) -> map.put(entry.getKey(), + DefaultCustomSearchMetricConfig.of(entry.getKey(), ConfigFactory.empty().withFallback(entry.getValue()))); + } + + @Override + public BinaryOperator> combiner() { + return (left, right) -> Stream.concat(left.entrySet().stream(), right.entrySet().stream()) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + @Override + public Function, Map> finisher() { + return map -> Collections.unmodifiableMap(new LinkedHashMap<>(map)); + } + + @Override + public Set characteristics() { + return Collections.singleton(Characteristics.UNORDERED); + } + } } diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/OperatorMetricsConfig.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/OperatorMetricsConfig.java index f36b429d85..e443624631 100644 --- a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/OperatorMetricsConfig.java +++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/OperatorMetricsConfig.java @@ -47,6 +47,13 @@ public interface OperatorMetricsConfig { */ Map getCustomMetricConfigurations(); + /** + * Returns all registered custom search metrics with the key being the metric name to use. + * + * @return the registered custom search metrics. + */ + Map getCustomSearchMetricConfigurations(); + /** * An enumeration of the known config path expressions and their associated default values for * OperatorMetricsConfig. @@ -66,7 +73,12 @@ enum OperatorMetricsConfigValue implements KnownConfigValue { /** * All registered custom metrics with the key being the metric name to use. */ - CUSTOM_METRICS("custom-metrics", Collections.emptyMap()); + CUSTOM_METRICS("custom-metrics", Collections.emptyMap()), + + /** + * All registered custom search metrics with the key being the metric name to use. + */ + CUSTOM_SEARCH_METRICS("custom-search-metrics", Collections.emptyMap()); private final String path; private final Object defaultValue; diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/OperatorSearchMetricsProviderActor.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/OperatorSearchMetricsProviderActor.java new file mode 100644 index 0000000000..505fe01179 --- /dev/null +++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/OperatorSearchMetricsProviderActor.java @@ -0,0 +1,379 @@ +/* + * Copyright (c) 2024 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.thingsearch.service.starter.actors; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ThreadLocalRandom; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.apache.pekko.actor.AbstractActorWithTimers; +import org.apache.pekko.actor.ActorRef; +import org.apache.pekko.actor.Props; +import org.apache.pekko.actor.Status; +import org.apache.pekko.japi.pf.ReceiveBuilder; +import org.apache.pekko.pattern.Patterns; +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.edge.service.dispatching.ThingsAggregatorProxyActor; +import org.eclipse.ditto.edge.service.placeholders.ThingJsonPlaceholder; +import org.eclipse.ditto.internal.utils.metrics.instruments.gauge.Gauge; +import org.eclipse.ditto.internal.utils.metrics.instruments.gauge.KamonGauge; +import org.eclipse.ditto.internal.utils.metrics.instruments.tag.KamonTagSetConverter; +import org.eclipse.ditto.internal.utils.metrics.instruments.tag.Tag; +import org.eclipse.ditto.internal.utils.metrics.instruments.tag.TagSet; +import org.eclipse.ditto.internal.utils.pekko.logging.DittoDiagnosticLoggingAdapter; +import org.eclipse.ditto.internal.utils.pekko.logging.DittoLoggerFactory; +import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonObject; +import org.eclipse.ditto.json.JsonParseOptions; +import org.eclipse.ditto.things.api.commands.sudo.SudoRetrieveThings; +import org.eclipse.ditto.things.api.commands.sudo.SudoRetrieveThingsResponse; +import org.eclipse.ditto.things.model.Thing; +import org.eclipse.ditto.things.model.ThingId; +import org.eclipse.ditto.things.model.ThingsModelFactory; +import org.eclipse.ditto.thingsearch.model.signals.commands.query.QueryThings; +import org.eclipse.ditto.thingsearch.model.signals.commands.query.QueryThingsResponse; +import org.eclipse.ditto.thingsearch.service.common.config.CustomSearchMetricConfig; +import org.eclipse.ditto.thingsearch.service.common.config.OperatorMetricsConfig; + +import kamon.Kamon; +import scala.Tuple2; + +/** + * Actor which is started as singleton for "search" role and is responsible for querying for extended operator defined + * "custom metrics" (configured via Ditto search service configuration) to expose as {@link Gauge} via Prometheus. + */ +public final class OperatorSearchMetricsProviderActor extends AbstractActorWithTimers { + + /** + * This Actor's actor name. + */ + public static final String ACTOR_NAME = "operatorSearchMetricsProvider"; + + private static final int MIN_INITIAL_DELAY_SECONDS = 30; + private static final int MAX_INITIAL_DELAY_SECONDS = 90; + private static final int DEFAULT_SEARCH_TIMEOUT_SECONDS = 60; + + private final DittoDiagnosticLoggingAdapter log = DittoLoggerFactory.getDiagnosticLoggingAdapter(this); + private final ThingJsonPlaceholder thingJsonPlaceholder = ThingJsonPlaceholder.getInstance(); + private final Map aggregatedByTagsValues = new HashMap<>(); + + private final ActorRef searchActor; + private final Map> metricsGauges; + private final Gauge customSearchMetricsGauge; + private final ActorRef thingsAggregatorProxyActor; + + @SuppressWarnings("unused") + private OperatorSearchMetricsProviderActor(final OperatorMetricsConfig operatorMetricsConfig, + final ActorRef searchActor, final ActorRef pubSubMediator) { + customSearchMetricsGauge = KamonGauge.newGauge("custom-search-metrics"); + this.searchActor = searchActor; + thingsAggregatorProxyActor = getContext().actorOf(ThingsAggregatorProxyActor.props(pubSubMediator), + ThingsAggregatorProxyActor.ACTOR_NAME); + metricsGauges = new HashMap<>(); + operatorMetricsConfig.getCustomSearchMetricConfigurations().forEach((metricName, config) -> { + if (config.isEnabled()) { + initializeCustomMetricTimer(operatorMetricsConfig, metricName, config); + } else { + log.info("Initializing custom search metric Gauge for metric <{}> is DISABLED", metricName); + } + }); + initializeCustomMetricsCleanupTimer(operatorMetricsConfig); + } + + /** + * Create Props for this actor. + * + * @param operatorMetricsConfig the config to use + * @param searchActor the SearchActor Actor reference + * @return the Props object. + */ + public static Props props(final OperatorMetricsConfig operatorMetricsConfig, final ActorRef searchActor, + final ActorRef pubSubMediator) { + return Props.create(OperatorSearchMetricsProviderActor.class, operatorMetricsConfig, searchActor, + pubSubMediator); + } + + @Override + public Receive createReceive() { + return ReceiveBuilder.create() + .match(GatherMetrics.class, this::handleGatheringMetrics) + .match(CleanupUnusedMetrics.class, this::handleCleanupUnusedMetrics) + .match(Status.Failure.class, f -> log.error(f.cause(), "Got failure: {}", f)) + .matchAny(m -> { + log.warning("Unknown message: {}", m); + unhandled(m); + }) + .build(); + } + + private void initializeCustomMetricTimer(final OperatorMetricsConfig operatorMetricsConfig, final String metricName, + final CustomSearchMetricConfig config) { + // start each custom metric provider with a random initialDelay + final Duration initialDelay = Duration.ofSeconds( + ThreadLocalRandom.current().nextInt(MIN_INITIAL_DELAY_SECONDS, MAX_INITIAL_DELAY_SECONDS) + ); + final Duration scrapeInterval = config.getScrapeInterval() + .orElse(operatorMetricsConfig.getScrapeInterval()); + log.info("Initializing custom metric timer for metric <{}> with initialDelay <{}> and scrapeInterval <{}>", + metricName, + initialDelay, scrapeInterval); + getTimers().startTimerAtFixedRate( + metricName, createGatherCustomMetric(metricName, config), initialDelay, scrapeInterval); + } + + private void initializeCustomMetricsCleanupTimer(final OperatorMetricsConfig operatorMetricsConfig) { + final Duration interval = getMaxConfiguredScrapeInterval(operatorMetricsConfig).multipliedBy(3); + log.info("Initializing custom metric cleanup timer Interval <{}>", interval); + getTimers().startTimerAtFixedRate("cleanup-unused-metrics", new CleanupUnusedMetrics(operatorMetricsConfig), + interval); + } + + private void handleCleanupUnusedMetrics(CleanupUnusedMetrics cleanupUnusedMetrics) { + // remove metrics who were not used for longer than three times the max configured scrape interval + final long currentTime = System.currentTimeMillis(); + metricsGauges.entrySet().stream() + .filter(entry -> { + final long time = entry.getValue()._2(); + return currentTime - time > getMaxConfiguredScrapeInterval(cleanupUnusedMetrics.config()) + .multipliedBy(3).toMillis(); + }) + .forEach(entry -> { + final String realName = realName(entry.getKey()); + if (Kamon.gauge(realName) + .withTags(KamonTagSetConverter.getKamonTagSet(entry.getValue()._1().getTagSet())) + .remove()) { + log.debug("Removed custom search metric instrument: {} {}", realName, + entry.getValue()._1().getTagSet()); + customSearchMetricsGauge.decrement(); + } else { + log.info("Could not remove unused custom search metric instrument: {}", entry.getKey()); + } + }); + } + + private void handleGatheringMetrics(final GatherMetrics gatherMetrics) { + final String metricName = gatherMetrics.metricName(); + final CustomSearchMetricConfig config = gatherMetrics.config(); + final DittoHeaders dittoHeaders = DittoHeaders.newBuilder() + .correlationId("gather-search-metrics_" + metricName + "_" + UUID.randomUUID()) + .putHeader("ditto-sudo", "true") + .build(); + + final long startTs = System.nanoTime(); + config.getFilterConfigs().forEach(filterConfig -> { + QueryThings searchThings = QueryThings.of(filterConfig.getFilter(), null, null, + new HashSet<>(config.getNamespaces()), dittoHeaders); + askSearchActor(searchThings, metricName, startTs, filterConfig, config, dittoHeaders); + }); + } + + private static GatherMetrics createGatherCustomMetric(final String metricName, + final CustomSearchMetricConfig config) { + return new GatherMetrics(metricName, config); + } + + private void askSearchActor(final QueryThings searchThings, final String metricName, final long startTs, + final CustomSearchMetricConfig.FilterConfig filterConfig, final CustomSearchMetricConfig config, + final DittoHeaders dittoHeaders) { + log.withCorrelationId(dittoHeaders).debug("Asking for things for custom metric <{}>..", metricName); + Patterns.ask(searchActor, searchThings, Duration.ofSeconds(DEFAULT_SEARCH_TIMEOUT_SECONDS)) + .whenComplete((response, throwable) -> { + if (response instanceof QueryThingsResponse queryThingsResponse) { + log.withCorrelationId(queryThingsResponse) + .debug("Received QueryThingsResponse for custom search metric <{}>: {} - " + + "duration: <{}ms>", + metricName, queryThingsResponse.getSearchResult().getItems().getSize(), + Duration.ofNanos(System.nanoTime() - startTs).toMillis() + ); + aggregateResponse(queryThingsResponse, filterConfig, metricName, config, dittoHeaders); + if (queryThingsResponse.getSearchResult().hasNextPage() && + queryThingsResponse.getSearchResult().getCursor().isPresent()) { + + QueryThings nextPageSearch = QueryThings.of(searchThings.getFilter().orElse(null), + List.of("cursor(" + queryThingsResponse.getSearchResult().getCursor().get() + ")"), + null, + new HashSet<>(config.getNamespaces()), dittoHeaders); + log.withCorrelationId(queryThingsResponse) + .debug("Asking for next page {} for custom search metric <{}>..", + queryThingsResponse.getSearchResult().getNextPageOffset().orElse(-1L), + metricName); + askSearchActor(nextPageSearch, metricName, startTs, filterConfig, config, dittoHeaders); + } + recordMetric(metricName); + + } else if (response instanceof DittoRuntimeException dre) { + log.withCorrelationId(dittoHeaders).warning( + "Received DittoRuntimeException when gathering things for " + + "custom search metric <{}> with queryThings {}: {}", metricName, searchThings, + dre.getMessage(), dre + ); + } else { + log.withCorrelationId(dittoHeaders).warning( + "Received unexpected result or throwable when gathering things for " + + "custom search metric <{}> with queryThings {}: {}", metricName, searchThings, + response, throwable + ); + } + }); + } + + private void aggregateResponse(QueryThingsResponse queryThingsResponse, + CustomSearchMetricConfig.FilterConfig filterConfig, String metricName, + CustomSearchMetricConfig config, DittoHeaders dittoHeaders) { + if (isOnlyThingIdField(filterConfig.getFields())) { + final List things = queryThingsResponse.getSearchResult() + .getItems() + .stream() + .map(jsonValue -> ThingsModelFactory.newThing( + jsonValue.asObject())) + .toList(); + aggregateByTags(things, filterConfig, metricName, config, dittoHeaders); + } else { + final List thingIds = queryThingsResponse.getSearchResult() + .getItems() + .stream() + .map(jsonValue -> ThingsModelFactory.newThing(jsonValue.asObject()).getEntityId()) + .filter(Optional::isPresent) + .map(Optional::get) + .toList(); + log.withCorrelationId(dittoHeaders) + .debug("Retrieved Things for custom search metric {}: {}", metricName, thingIds); + final SudoRetrieveThings sudoRetrieveThings = SudoRetrieveThings.of(thingIds, + JsonFactory.newFieldSelector(ensureThingId(filterConfig.getFields()), JsonParseOptions.newBuilder() + .withoutUrlDecoding().build()), dittoHeaders); + Patterns.ask(thingsAggregatorProxyActor, sudoRetrieveThings, + Duration.ofSeconds(DEFAULT_SEARCH_TIMEOUT_SECONDS)) + .handle((response, throwable) -> { + if (response instanceof SudoRetrieveThingsResponse retrieveThingsResponse) { + // aggregate response by tags and record after all pages are received + aggregateByTags(retrieveThingsResponse.getThings(), filterConfig, metricName, config, + dittoHeaders); + } else { + log.withCorrelationId(dittoHeaders).warning( + "Received unexpected result or throwable when gathering things for " + + "custom search metric <{}> with queryThings {}: {}", metricName, + queryThingsResponse, + response, throwable + ); + } + return null; + }); + } + } + + private List ensureThingId(final List fields) { + return fields.contains("thingId") ? fields : Stream.concat(fields.stream(), Stream.of("thingId")) + .collect(Collectors.toList()); + } + + private void aggregateByTags(final List things, + final CustomSearchMetricConfig.FilterConfig filterConfig, final String metricName, + final CustomSearchMetricConfig config, final DittoHeaders dittoHeaders) { + final List tagSets = things.stream() + .map(thing -> TagSet.ofTagCollection(config.getTags().entrySet().stream() + .map(e -> Tag.of(e.getKey(), + resolvePlaceHolder(thing.toJson().asObject(), e.getValue(), filterConfig, dittoHeaders, + metricName))) + .sorted(Comparator.comparing(Tag::getValue)) + .toList())).toList(); + tagSets.forEach(tagSet -> aggregatedByTagsValues.merge(tagSet, 1.0, Double::sum)); + } + + private void recordMetric(final String metricName) { + aggregatedByTagsValues.forEach( + (tags, value) -> metricsGauges.computeIfAbsent(uniqueName(metricName, tags), name -> { + log.info("Initializing custom search metric Gauge for metric <{}> with tags <{}>", + metricName, tags); + customSearchMetricsGauge.increment(); + return Tuple2.apply(KamonGauge.newGauge(metricName) + .tags(tags), System.currentTimeMillis()); + }) + ._1().set(value)); + aggregatedByTagsValues.clear(); + } + + private String resolvePlaceHolder(final JsonObject thingJson, final String value, + final CustomSearchMetricConfig.FilterConfig filterConfig, final DittoHeaders dittoHeaders, + final String metricName) { + if (!isPlaceHolder(value)) { + return value; + } + String placeholder = value.substring(2, value.length() - 2).trim(); + final List resolvedValues = + new ArrayList<>( + thingJsonPlaceholder.resolveValues(ThingsModelFactory.newThing(thingJson), placeholder)); + + if (resolvedValues.isEmpty()) { + filterConfig.getInlinePlaceholderValues().forEach((k, v) -> { + if (placeholder.equals(k)) { + resolvedValues.add(v); + } + }); + } + if (resolvedValues.isEmpty()) { + log.withCorrelationId(dittoHeaders) + .warning("Custom search metric {}. Could not resolve placeholder <{}> in thing <{}>. " + + "Check that you have your fields configured correctly.", metricName, + placeholder, thingJson); + return value; + } + + + return resolvedValues.stream().findFirst() + .orElse(value); + } + + private String uniqueName(final String metricName, final TagSet tags) { + final ArrayList list = new ArrayList<>(); + tags.iterator().forEachRemaining(list::add); + return list.stream().sorted(Comparator.comparing(Tag::getKey)).map(t -> t.getKey() + "=" + t.getValue()) + .collect(Collectors.joining("_", metricName + "#", "")); + } + + private String realName(final String uniqueName) { + return uniqueName.substring(0, uniqueName.indexOf("#")); + } + + private boolean isPlaceHolder(final String value) { + return value.startsWith("{{") && value.endsWith("}}"); + } + + private boolean isOnlyThingIdField(final List filterConfig) { + return filterConfig.isEmpty() || + (filterConfig.size() == 1 && filterConfig.get(0).equals(Thing.JsonFields.ID.getPointer().toString())); + } + + private Duration getMaxConfiguredScrapeInterval(final OperatorMetricsConfig operatorMetricsConfig) { + return Stream.concat(Stream.of(operatorMetricsConfig.getScrapeInterval()), + operatorMetricsConfig.getCustomSearchMetricConfigurations().values().stream() + .map(CustomSearchMetricConfig::getScrapeInterval) + .filter(Optional::isPresent) + .map(Optional::get)) + .max(Comparator.naturalOrder()) + .orElse(operatorMetricsConfig.getScrapeInterval()); + } + + private record GatherMetrics(String metricName, CustomSearchMetricConfig config) {} + + private record CleanupUnusedMetrics(OperatorMetricsConfig config) {} +} diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/SearchActor.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/SearchActor.java index 53986a3dac..5ad7c4108d 100644 --- a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/SearchActor.java +++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/SearchActor.java @@ -477,8 +477,11 @@ private CompletionStage performQuery(final QueryThings queryThings, fina final StartedTimer databaseAccessTimer = searchTimer.startNewSegment(DATABASE_ACCESS_SEGMENT_NAME); - final List subjectIds = - command.getDittoHeaders() + final boolean isSudo = queryThings.getDittoHeaders() + .isSudo(); + + final List subjectIds = isSudo ? null + : command.getDittoHeaders() .getAuthorizationContext() .getAuthorizationSubjectIds(); final Source, NotUsed> findAllResult = diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/updater/actors/SearchUpdaterRootActor.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/updater/actors/SearchUpdaterRootActor.java index c9f20244ec..732cd9991a 100644 --- a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/updater/actors/SearchUpdaterRootActor.java +++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/updater/actors/SearchUpdaterRootActor.java @@ -38,6 +38,7 @@ import org.eclipse.ditto.thingsearch.service.persistence.write.streaming.SearchUpdaterStream; import org.eclipse.ditto.thingsearch.service.starter.actors.MongoClientExtension; import org.eclipse.ditto.thingsearch.service.starter.actors.OperatorMetricsProviderActor; +import org.eclipse.ditto.thingsearch.service.starter.actors.OperatorSearchMetricsProviderActor; /** * Our "Parent" Actor which takes care of supervision of all other Actors in our system. @@ -132,6 +133,9 @@ private SearchUpdaterRootActor(final SearchConfig searchConfig, startClusterSingletonActor(OperatorMetricsProviderActor.ACTOR_NAME, OperatorMetricsProviderActor.props(searchConfig.getOperatorMetricsConfig(), searchActor) ); + startClusterSingletonActor(OperatorSearchMetricsProviderActor.ACTOR_NAME, + OperatorSearchMetricsProviderActor.props(searchConfig.getOperatorMetricsConfig(), searchActor, pubSubMediator) + ); } startChildActor(ThingsSearchPersistenceOperationsActor.ACTOR_NAME, diff --git a/thingsearch/service/src/main/resources/search-dev.conf b/thingsearch/service/src/main/resources/search-dev.conf index 1f98578de3..f41a9c9dfd 100755 --- a/thingsearch/service/src/main/resources/search-dev.conf +++ b/thingsearch/service/src/main/resources/search-dev.conf @@ -8,8 +8,11 @@ ditto { search { operator-metrics { + enabled = true + scrape-interval = 1m custom-metrics { my_awesome_things { + enabled = false scrape-interval = 1m # overwrite scrape interval, run each minute namespaces = [ "org.eclipse.ditto.foo" @@ -22,6 +25,41 @@ ditto { } } } + custom-search-metrics { + online_status { + enabled = true + scrape-interval = 1m # override scrape interval, run each minute + namespaces = [ + "org.eclipse.ditto" + ] + tags: { + "online" = "{{online_placeholder}}" + "location" = "{{attributes/Info/location}}" + } + filters = { + online-filter = { + filter = "gt(features/ConnectionStatus/properties/status/readyUntil/,time:now)" + inline-placeholder-values = { + // inline-placeholder-values are used to define hardcoded values to be used in the tags values if the placeholders are not json paths to an actual field in the thing + // this is used to define different tags values based on the filter that matched the thing + "online_placeholder" = true + } + // in order to do placeholder resolving from thing fields, the fields should be defined in the fields array + // by default only the thingId is available for placeholder resolving + fields = ["attributes/Info/location"] + // The metric-value is used to define the value of the metric for the thing that matched the filter. + // It does not support placeholders and should be a numeric value + } + offline-filter = { + filter = "lt(features/ConnectionStatus/properties/status/readyUntil/,time:now)" + inline-placeholder-values = { + "online_placeholder" = false + } + fields = ["attributes/Info/location"] + } + } + } + } } } } From 284d711b9b51e7a876fb8b56f133f4ca2de5fb4f Mon Sep 17 00:00:00 2001 From: Aleksandar Stanchev Date: Mon, 24 Jun 2024 08:03:07 +0300 Subject: [PATCH 02/13] Documentation for usage of extended search based metrics Signed-off-by: Aleksandar Stanchev --- .../pages/ditto/installation-operating.md | 86 +++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/documentation/src/main/resources/pages/ditto/installation-operating.md b/documentation/src/main/resources/pages/ditto/installation-operating.md index 9608cd75e1..271df0f30c 100644 --- a/documentation/src/main/resources/pages/ditto/installation-operating.md +++ b/documentation/src/main/resources/pages/ditto/installation-operating.md @@ -585,6 +585,92 @@ In Prometheus format this would look like: ``` all_produced_and_not_installed_devices{company="acme-corp"} 42.0 ``` +### Operator defined custom search based metrics +Starting with Ditto 3.6.0, the "custom metrics" functionality is extended to support search based metrics. +This is configured via the [search](architecture-services-things-search.html) service configuration and builds on the +[search things](basic-search.html#search-queries) functionality. + +Now you can augment the statistic about "Things" managed in Ditto +fulfilling a certain condition with tags with either predefined values or values retrieved from the things. + +This would be an example search service configuration snippet for e.g. providing a metric named +`online_devices` defining a query on the values of a `ConnectionStatus` feature: +```hocon +ditto { + search { + operator-metrics { + enabled = true + scrape-interval = 30m + custom-metrics { + ... + } + custom-search-metrics { + online_status { + enabled = true + scrape-interval = 20m # override scrape interval, run every 20 minute + namespaces = [ + "org.eclipse.ditto" + ] + tags: { + "online" = "{{online_placeholder}}" + "location" = "{{attributes/Info/location}}" + } + filters = { + online-filter = { + filter = "gt(features/ConnectionStatus/properties/status/readyUntil/,time:now)" + inline-placeholder-values = { + // inline-placeholder-values are used to define hardcoded values to be used in the tags values if the placeholders are not json paths to an actual field in the thing + // this is used to define different tags values based on the filter that matched the thing + "online_placeholder" = true + } + // in order to do placeholder resolving from thing fields, the fields should be defined in the fields array + // by default only the thingId is available for placeholder resolving + fields = ["attributes/Info/location"] + // The metric-value is used to define the value of the metric for the thing that matched the filter. + // It does not support placeholders and should be a numeric value + } + offline-filter = { + filter = "lt(features/ConnectionStatus/properties/status/readyUntil/,time:now)" + inline-placeholder-values = { + "online_placeholder" = false + } + fields = ["attributes/Info/location"] + } + } + } + } + } + } +} +``` + +To add custom metrics via System properties, the following example shows how the above metric can be configured: +``` +-Dditto.search.operator-metrics.custom-search-metrics.online_status.enabled=true +-Dditto.search.operator-metrics.custom-search-metrics.online_status.scrape-interval=20m +-Dditto.search.operator-metrics.custom-search-metrics.online_status.namespaces.0=org.eclipse.ditto +-Dditto.search.operator-metrics.custom-search-metrics.online_status.tags.online="{{online_placeholder}}" +-Dditto.search.operator-metrics.custom-search-metrics.online_status.tags.location="{{attributes/Info/location}}" + +-Dditto.search.operator-metrics.custom-search-metrics.online_status.filters.online-filter.filter=gt(features/ConnectionStatus/properties/status/readyUntil/,time:now) +-Dditto.search.operator-metrics.custom-search-metrics.online_status.filters.online-filter.inline-placeholder-values.online_placeholder=true +-Dditto.search.operator-metrics.custom-search-metrics.online_status.filters.online-filter.fields.0=attributes/Info/location + +-Dditto.search.operator-metrics.custom-search-metrics.online_status.filters.offline-filter.filter=lt(features/ConnectionStatus/properties/status/readyUntil/,time:now) +-Dditto.search.operator-metrics.custom-search-metrics.online_status.filters.offline-filter.inline-placeholder-values.online_placeholder=false +-Dditto.search.operator-metrics.custom-search-metrics.online_status.filters.offline-filter.fields.0=attributes/Info/location + +``` + +Ditto will perform a [search things operation](basic-search.html#search-queries) every `20m` (20 minutes), providing +a gauge named `online_devices` with the value of devices that match the filter. +The tags `online` and `location` will be added. +Their values will be resolved from the placeholders `{{online_placeholder}}` and `{{attributes/Info/location}}` respectively. + +In Prometheus format this would look like: +``` +online_status{location="Berlin",online="false"} 6.0 +online_status{location="Immenstaad",online="true"} 8.0 ## Tracing From 8d389ad6631288e4714c931df30b45900c740ece Mon Sep 17 00:00:00 2001 From: Aleksandar Stanchev Date: Mon, 24 Jun 2024 08:14:18 +0300 Subject: [PATCH 03/13] Disable by default Signed-off-by: Aleksandar Stanchev --- thingsearch/service/src/main/resources/search-dev.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/thingsearch/service/src/main/resources/search-dev.conf b/thingsearch/service/src/main/resources/search-dev.conf index f41a9c9dfd..72e679189b 100755 --- a/thingsearch/service/src/main/resources/search-dev.conf +++ b/thingsearch/service/src/main/resources/search-dev.conf @@ -27,7 +27,7 @@ ditto { } custom-search-metrics { online_status { - enabled = true + enabled = false scrape-interval = 1m # override scrape interval, run each minute namespaces = [ "org.eclipse.ditto" From e850bd3631cc151e1ffb16ad51d313f183cb8de9 Mon Sep 17 00:00:00 2001 From: Aleksandar Stanchev Date: Mon, 24 Jun 2024 08:32:12 +0300 Subject: [PATCH 04/13] Update usage timestamp for accurate cleanup Signed-off-by: Aleksandar Stanchev --- .../OperatorSearchMetricsProviderActor.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/OperatorSearchMetricsProviderActor.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/OperatorSearchMetricsProviderActor.java index 505fe01179..4598543f8f 100644 --- a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/OperatorSearchMetricsProviderActor.java +++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/OperatorSearchMetricsProviderActor.java @@ -309,6 +309,25 @@ private void recordMetric(final String metricName) { .tags(tags), System.currentTimeMillis()); }) ._1().set(value)); + + aggregatedByTagsValues.forEach( + (tags, value) -> metricsGauges.compute(uniqueName(metricName, tags), (key, tupleValue) -> { + if (tupleValue == null) { + log.info("Initializing custom search metric Gauge for metric <{}> with tags <{}>", + metricName, tags); + customSearchMetricsGauge.increment(); + return Tuple2.apply(KamonGauge.newGauge(metricName) + .tags(tags), System.currentTimeMillis()); + + } else { + // update the timestamp + log.debug("Updating custom search metric Gauge for metric <{}> with tags <{}>", + metricName, tags); + return Tuple2.apply(tupleValue._1(), System.currentTimeMillis()); + } + }) + ._1().set(value)); + aggregatedByTagsValues.clear(); } From e93c6fab444ed492410c5e59847fa21e81022939 Mon Sep 17 00:00:00 2001 From: Aleksandar Stanchev Date: Tue, 9 Jul 2024 15:19:01 +0300 Subject: [PATCH 05/13] fix global registries Signed-off-by: Aleksandar Stanchev --- .../starter/ThingSearchServiceGlobalErrorRegistryTest.java | 4 +++- .../ThingsSearchServiceGlobalCommandRegistryTest.java | 4 +++- ...ThingsSearchServiceGlobalCommandResponseRegistryTest.java | 5 ++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/ThingSearchServiceGlobalErrorRegistryTest.java b/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/ThingSearchServiceGlobalErrorRegistryTest.java index b532295d95..a963746b26 100644 --- a/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/ThingSearchServiceGlobalErrorRegistryTest.java +++ b/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/ThingSearchServiceGlobalErrorRegistryTest.java @@ -27,6 +27,7 @@ import org.eclipse.ditto.base.model.signals.commands.exceptions.PathUnknownException; import org.eclipse.ditto.connectivity.model.ConnectionIdInvalidException; import org.eclipse.ditto.connectivity.model.signals.commands.exceptions.ConnectionConflictException; +import org.eclipse.ditto.edge.service.EdgeServiceTimeoutException; import org.eclipse.ditto.internal.utils.test.GlobalErrorRegistryTestCases; import org.eclipse.ditto.messages.model.AuthorizationSubjectBlockedException; import org.eclipse.ditto.placeholders.PlaceholderFunctionSignatureInvalidException; @@ -70,7 +71,8 @@ public ThingSearchServiceGlobalErrorRegistryTest() { ConnectionConflictException.class, UnknownTopicPathException.class, UnknownSignalException.class, - IllegalAdaptableException.class); + IllegalAdaptableException.class, + EdgeServiceTimeoutException.class); } } diff --git a/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/ThingsSearchServiceGlobalCommandRegistryTest.java b/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/ThingsSearchServiceGlobalCommandRegistryTest.java index 512235581b..8282a27db6 100644 --- a/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/ThingsSearchServiceGlobalCommandRegistryTest.java +++ b/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/ThingsSearchServiceGlobalCommandRegistryTest.java @@ -19,6 +19,7 @@ import org.eclipse.ditto.base.model.namespaces.signals.commands.PurgeNamespace; import org.eclipse.ditto.base.model.signals.commands.streaming.SubscribeForPersistedEvents; import org.eclipse.ditto.base.service.cluster.ModifySplitBrainResolver; +import org.eclipse.ditto.connectivity.api.commands.sudo.SudoRetrieveConnectionTags; import org.eclipse.ditto.connectivity.model.signals.commands.modify.ModifyConnection; import org.eclipse.ditto.connectivity.model.signals.commands.query.RetrieveConnection; import org.eclipse.ditto.internal.models.streaming.SudoStreamPids; @@ -62,7 +63,8 @@ public ThingsSearchServiceGlobalCommandRegistryTest() { ModifyConnection.class, ModifySplitBrainResolver.class, RetrieveConnection.class, - SubscribeForPersistedEvents.class + SubscribeForPersistedEvents.class, + SudoRetrieveConnectionTags.class ); } diff --git a/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/ThingsSearchServiceGlobalCommandResponseRegistryTest.java b/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/ThingsSearchServiceGlobalCommandResponseRegistryTest.java index 80d39bf860..cdd4c20ab5 100644 --- a/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/ThingsSearchServiceGlobalCommandResponseRegistryTest.java +++ b/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/ThingsSearchServiceGlobalCommandResponseRegistryTest.java @@ -19,6 +19,8 @@ import org.eclipse.ditto.base.model.namespaces.signals.commands.PurgeNamespaceResponse; import org.eclipse.ditto.base.model.signals.acks.Acknowledgement; import org.eclipse.ditto.base.service.cluster.ModifySplitBrainResolverResponse; +import org.eclipse.ditto.connectivity.api.commands.sudo.ConnectivitySudoQueryCommandResponse; +import org.eclipse.ditto.connectivity.api.commands.sudo.SudoRetrieveConnectionTagsResponse; import org.eclipse.ditto.connectivity.model.signals.commands.ConnectivityErrorResponse; import org.eclipse.ditto.connectivity.model.signals.commands.modify.ModifyConnectionResponse; import org.eclipse.ditto.connectivity.model.signals.commands.query.RetrieveConnectionResponse; @@ -66,7 +68,8 @@ public ThingsSearchServiceGlobalCommandResponseRegistryTest() { ModifyConnectionResponse.class, RetrieveConnectionResponse.class, ModifySplitBrainResolverResponse.class, - ConnectivityErrorResponse.class + ConnectivityErrorResponse.class, + SudoRetrieveConnectionTagsResponse.class ); } From 15b57ff53fc9b966d81e7d49200ad34c905b84d3 Mon Sep 17 00:00:00 2001 From: Aleksandar Stanchev Date: Mon, 26 Aug 2024 11:39:17 +0300 Subject: [PATCH 06/13] refactor aggregation to be done in mongodb Signed-off-by: Aleksandar Stanchev --- .../query/AggregateThingsMetrics.java | 94 ++++ .../query/AggregateThingsMetricsResponse.java | 121 +++++ .../config/CustomSearchMetricConfig.java | 42 +- .../DefaultCustomSearchMetricConfig.java | 197 +++++-- .../config/DefaultOperatorMetricsConfig.java | 8 +- .../common/config/OperatorMetricsConfig.java | 2 +- .../MongoThingsAggregationPersistence.java | 123 +++++ .../read/ThingsAggregationPersistence.java | 37 ++ .../read/ThingsSearchPersistence.java | 2 +- ...CreateBsonAggregationPredicateVisitor.java | 205 ++++++++ .../CreateBsonAggregationVisitor.java | 61 +++ .../criteria/visitors/CreateBsonVisitor.java | 2 +- .../GroupByPlaceholderResolver.java | 59 +++ .../InlinePlaceholderResolver.java | 58 +++ .../actors/AggregationThingsMetricsActor.java | 155 ++++++ .../OperatorSearchMetricsProviderActor.java | 482 ++++++++---------- .../actors/ThingsAggregationConstants.java | 46 ++ .../actors/SearchUpdaterRootActor.java | 2 +- .../src/main/resources/search-dev.conf | 31 +- .../DefaultCustomSearchMetricConfigTest.java | 104 ++++ ...gSearchServiceGlobalErrorRegistryTest.java | 4 +- .../AggregationThingsMetricsActorTest.java | 128 +++++ .../resources/custom-search-metric-test.conf | 80 +++ 23 files changed, 1687 insertions(+), 356 deletions(-) create mode 100644 thingsearch/model/src/main/java/org/eclipse/ditto/thingsearch/model/signals/commands/query/AggregateThingsMetrics.java create mode 100644 thingsearch/model/src/main/java/org/eclipse/ditto/thingsearch/model/signals/commands/query/AggregateThingsMetricsResponse.java create mode 100644 thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/MongoThingsAggregationPersistence.java create mode 100644 thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/ThingsAggregationPersistence.java create mode 100644 thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/criteria/visitors/CreateBsonAggregationPredicateVisitor.java create mode 100644 thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/criteria/visitors/CreateBsonAggregationVisitor.java create mode 100644 thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/placeholders/GroupByPlaceholderResolver.java create mode 100644 thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/placeholders/InlinePlaceholderResolver.java create mode 100644 thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/AggregationThingsMetricsActor.java create mode 100644 thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/ThingsAggregationConstants.java create mode 100644 thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultCustomSearchMetricConfigTest.java create mode 100644 thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/actors/AggregationThingsMetricsActorTest.java create mode 100644 thingsearch/service/src/test/resources/custom-search-metric-test.conf diff --git a/thingsearch/model/src/main/java/org/eclipse/ditto/thingsearch/model/signals/commands/query/AggregateThingsMetrics.java b/thingsearch/model/src/main/java/org/eclipse/ditto/thingsearch/model/signals/commands/query/AggregateThingsMetrics.java new file mode 100644 index 0000000000..c66d96a383 --- /dev/null +++ b/thingsearch/model/src/main/java/org/eclipse/ditto/thingsearch/model/signals/commands/query/AggregateThingsMetrics.java @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2024 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + * + */ + +package org.eclipse.ditto.thingsearch.model.signals.commands.query; + +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.headers.WithDittoHeaders; + +public class AggregateThingsMetrics implements WithDittoHeaders { + + private final String metricName; + private final Map groupingBy; + private final Map namedFilters; + private final DittoHeaders dittoHeaders; + private final Set namespaces; + + private AggregateThingsMetrics(final String metricName, final Map groupingBy, final Map namedFilters, final Set namespaces, + final DittoHeaders dittoHeaders) { + this.metricName = metricName; + this.groupingBy = groupingBy; + this.namedFilters = namedFilters; + this.namespaces = namespaces; + this.dittoHeaders = dittoHeaders; + } + + public static AggregateThingsMetrics of(final String metricName, final Map groupingBy, final Map namedFilters, final Set namespaces, + final DittoHeaders dittoHeaders) { + return new AggregateThingsMetrics(metricName, groupingBy, namedFilters, namespaces, dittoHeaders); + } + + public String getMetricName() { + return metricName; + } + + public Map getGroupingBy() { + return groupingBy; + } + + public Map getNamedFilters() { + return namedFilters; + } + + @Override + public DittoHeaders getDittoHeaders() { + return dittoHeaders; + } + + public Set getNamespaces() { + return namespaces; + } + + @Override + public boolean equals(final Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + final AggregateThingsMetrics that = (AggregateThingsMetrics) o; + return Objects.equals(metricName, that.metricName) && + Objects.equals(groupingBy, that.groupingBy) && + Objects.equals(namedFilters, that.namedFilters) && + Objects.equals(dittoHeaders, that.dittoHeaders) && + Objects.equals(namespaces, that.namespaces); + } + + @Override + public int hashCode() { + return Objects.hash(metricName, groupingBy, namedFilters, dittoHeaders, namespaces); + } + + @Override + public String toString() { + return "AggregateThingsMetrics{" + + "metricName='" + metricName + '\'' + + ", groupingBy=" + groupingBy + + ", namedFilters=" + namedFilters + + ", dittoHeaders=" + dittoHeaders + + ", namespaces=" + namespaces + + '}'; + } +} diff --git a/thingsearch/model/src/main/java/org/eclipse/ditto/thingsearch/model/signals/commands/query/AggregateThingsMetricsResponse.java b/thingsearch/model/src/main/java/org/eclipse/ditto/thingsearch/model/signals/commands/query/AggregateThingsMetricsResponse.java new file mode 100644 index 0000000000..32b4e321e0 --- /dev/null +++ b/thingsearch/model/src/main/java/org/eclipse/ditto/thingsearch/model/signals/commands/query/AggregateThingsMetricsResponse.java @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2024 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + * + */ + +package org.eclipse.ditto.thingsearch.model.signals.commands.query; + +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.headers.WithDittoHeaders; +import org.eclipse.ditto.json.JsonMissingFieldException; +import org.eclipse.ditto.json.JsonObject; +import org.eclipse.ditto.json.JsonPointer; + +public class AggregateThingsMetricsResponse implements WithDittoHeaders { + + private final Map groupedBy; + + private final Map result; + + private final DittoHeaders dittoHeaders; + private final JsonObject aggregation; + private final String metricName; + private AggregateThingsMetricsResponse(final JsonObject aggregation, final DittoHeaders dittoHeaders, + final String metricName, final Set filterNames) { + this.aggregation = aggregation; + this.dittoHeaders = dittoHeaders; + this.metricName = metricName; + groupedBy = aggregation.getValue("_id") + .map(json -> { + if (json.isObject()) { + return json.asObject().stream() + .collect(Collectors.toMap(o -> o.getKey().toString(), + o1 -> o1.getValue().formatAsString())); + } else { + return new HashMap(); + } + } + ) + .orElse(new HashMap<>()); + result = filterNames.stream().filter(aggregation::contains).collect(Collectors.toMap(key -> + key, key -> aggregation.getValue(JsonPointer.of(key)) + .orElseThrow(getJsonMissingFieldExceptionSupplier(key)) + .asLong())); + // value should always be a number as it will be used for the gauge value in the metrics + } + + public static AggregateThingsMetricsResponse of(final JsonObject aggregation, + final AggregateThingsMetrics aggregateThingsMetrics) { + return of(aggregation, aggregateThingsMetrics.getDittoHeaders(), aggregateThingsMetrics.getMetricName(), aggregateThingsMetrics.getNamedFilters().keySet()); + } + + public static AggregateThingsMetricsResponse of(final JsonObject aggregation, final DittoHeaders dittoHeaders, + final String metricName, final Set filterNames) { + return new AggregateThingsMetricsResponse(aggregation, dittoHeaders, metricName, filterNames); + } + + @Override + public DittoHeaders getDittoHeaders() { + return dittoHeaders; + } + + public Map getGroupedBy() { + return groupedBy; + } + + public Map getResult() { + return result; + } + + public String getMetricName() { + return metricName; + } + + private Supplier getJsonMissingFieldExceptionSupplier(String field) { + return () -> JsonMissingFieldException.newBuilder().fieldName(field).build(); + } + + @Override + public boolean equals(final Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + final AggregateThingsMetricsResponse response = (AggregateThingsMetricsResponse) o; + return Objects.equals(groupedBy, response.groupedBy) && + Objects.equals(result, response.result) && + Objects.equals(dittoHeaders, response.dittoHeaders) && + Objects.equals(aggregation, response.aggregation) && + Objects.equals(metricName, response.metricName); + } + + @Override + public int hashCode() { + return Objects.hash(groupedBy, result, dittoHeaders, aggregation, metricName); + } + + @Override + public String toString() { + return "AggregateThingsMetricsResponse{" + + "groupedBy=" + groupedBy + + ", result=" + result + + ", dittoHeaders=" + dittoHeaders + + ", aggregation=" + aggregation + + ", metricName='" + metricName + '\'' + + '}'; + } +} diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/CustomSearchMetricConfig.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/CustomSearchMetricConfig.java index a133263367..0e6f2c6df9 100644 --- a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/CustomSearchMetricConfig.java +++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/CustomSearchMetricConfig.java @@ -15,7 +15,6 @@ import org.eclipse.ditto.internal.utils.config.KnownConfigValue; import java.time.Duration; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; @@ -31,7 +30,7 @@ public interface CustomSearchMetricConfig { * * @return the name of the custom metric. */ - String getCustomMetricName(); + String getMetricName(); /** * Returns whether this specific search operator metric gathering is turned on. @@ -56,6 +55,14 @@ public interface CustomSearchMetricConfig { */ List getNamespaces(); + /** + * Returns the fields we want our metric aggregation to be grouped by. + * Field name and thing json pointer + * + * @return the fields we want our metric aggregation to be grouped by. + */ + Map getGroupBy(); + /** * Return optional tags to report to the custom Gauge metric. * @@ -88,11 +95,19 @@ enum CustomSearchMetricConfigValue implements KnownConfigValue { */ NAMESPACES("namespaces", List.of()), + /** + * The fields we want our metric aggregation to be grouped by. + */ + GROUP_BY("group-by", Map.of()), + /** * The optional tags to report to the custom Gauge metric. */ TAGS("tags", Map.of()), + /** + * The filter configurations for this custom metric. + */ FILTERS("filters", List.of()); private final String path; @@ -114,19 +129,36 @@ public String getConfigPath() { } } + /** + * Provides the configuration settings for a single filter configuration. + */ interface FilterConfig { String getFilterName(); + /** + * Returns the filter to be used. + * @return the filter. + */ String getFilter(); - List getFields(); - + /** + * Returns the inline placeholder values to be used for resolving. + * @return the inline placeholder values. + */ Map getInlinePlaceholderValues(); + /** + * The known configuration values for a filter configuration. + */ enum FilterConfigValues implements KnownConfigValue { + /** + * The filter to be used. + */ FILTER("filter", ""), - FIELDS("fields", Collections.emptyList()), + /** + * The inline placeholder values to be used for resolving. + */ INLINE_PLACEHOLDER_VALUES("inline-placeholder-values", Map.of()); private final String path; diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultCustomSearchMetricConfig.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultCustomSearchMetricConfig.java index 26b7809584..b6c95ab211 100644 --- a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultCustomSearchMetricConfig.java +++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultCustomSearchMetricConfig.java @@ -13,41 +13,73 @@ package org.eclipse.ditto.thingsearch.service.common.config; -import com.typesafe.config.Config; -import com.typesafe.config.ConfigFactory; - import java.time.Duration; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; +import javax.annotation.concurrent.Immutable; + import org.eclipse.ditto.internal.utils.config.ConfigWithFallback; +import org.eclipse.ditto.thingsearch.service.placeholders.GroupByPlaceholderResolver; +import org.eclipse.ditto.thingsearch.service.placeholders.InlinePlaceholderResolver; + +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; +@Immutable public final class DefaultCustomSearchMetricConfig implements CustomSearchMetricConfig { - private final String customMetricName; + private final String metricName; private final boolean enabled; private final Duration scrapeInterval; private final List namespaces; + private final Map groupBy; private final Map tags; private final List filterConfigs; private DefaultCustomSearchMetricConfig(final String key, final ConfigWithFallback configWithFallback) { - this.customMetricName = key; + this.metricName = key; enabled = configWithFallback.getBoolean(CustomSearchMetricConfigValue.ENABLED.getConfigPath()); scrapeInterval = configWithFallback.getDuration(CustomSearchMetricConfigValue.SCRAPE_INTERVAL.getConfigPath()); - namespaces = configWithFallback.getStringList(CustomSearchMetricConfigValue.NAMESPACES.getConfigPath()); - tags = configWithFallback.getObject(CustomSearchMetricConfig.CustomSearchMetricConfigValue.TAGS.getConfigPath()).unwrapped() - .entrySet() - .stream() - .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, e -> String.valueOf(e.getValue()))); - filterConfigs = configWithFallback.getObject(CustomSearchMetricConfigValue.FILTERS.getConfigPath()).entrySet().stream() - .map(entry -> DefaultFilterConfig.of(entry.getKey(), ConfigFactory.empty().withFallback(entry.getValue()))) - .collect(Collectors.toList()); + namespaces = Collections.unmodifiableList(new ArrayList<>( + configWithFallback.getStringList(CustomSearchMetricConfigValue.NAMESPACES.getConfigPath()))); + groupBy = Collections.unmodifiableMap(new HashMap<>( + configWithFallback.getObject(CustomSearchMetricConfigValue.GROUP_BY.getConfigPath()).unwrapped() + .entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> String.valueOf(e.getValue()))))); + tags = Collections.unmodifiableMap(new HashMap<>( + configWithFallback.getObject(CustomSearchMetricConfigValue.TAGS.getConfigPath()).unwrapped() + .entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> String.valueOf(e.getValue()))))); + filterConfigs = + Collections.unmodifiableList(new ArrayList<>( + configWithFallback.getObject(CustomSearchMetricConfigValue.FILTERS.getConfigPath()) + .entrySet() + .stream() + .map(entry -> DefaultFilterConfig.of(entry.getKey(), + ConfigFactory.empty().withFallback(entry.getValue()))) + .toList())); + validateConfig(); + } + + public static DefaultCustomSearchMetricConfig of(final String key, final Config config) { + return new DefaultCustomSearchMetricConfig(key, + ConfigWithFallback.newInstance(config, CustomSearchMetricConfigValue.values())); } @Override - public String getCustomMetricName() { - return customMetricName; + public String getMetricName() { + return metricName; } @Override @@ -65,6 +97,11 @@ public List getNamespaces() { return namespaces; } + @Override + public Map getGroupBy() { + return groupBy; + } + @Override public Map getTags() { return tags; @@ -75,8 +112,73 @@ public List getFilterConfigs() { return filterConfigs; } - public static DefaultCustomSearchMetricConfig of(final String key, final Config config){ - return new DefaultCustomSearchMetricConfig(key , ConfigWithFallback.newInstance(config, CustomSearchMetricConfigValue.values())); + + private void validateConfig() { + if (getGroupBy().isEmpty()) { + throw new IllegalArgumentException("Custom search metric Gauge for metric <" + metricName + + "> must have at least one groupBy tag configured or else disable."); + } + getFilterConfigs().forEach(filterConfig -> { + if (filterConfig.getFilter().isEmpty()) { + throw new IllegalArgumentException("Custom search metric Gauge for metric <" + metricName + + "> must have at least one filter configured or else disable."); + } + if (filterConfig.getFilterName().contains("-")) { + throw new IllegalArgumentException("Custom search metric Gauge for metric <" + metricName + + "> filter name <" + filterConfig.getFilterName() + + "> must not contain the character '-'. Not supported in Mongo aggregations."); + } + }); + getTags().values().stream() + .filter(this::isPlaceHolder) + .map(value -> value.substring(2, value.length() - 2).trim()) + .forEach(placeholder -> { + if (!placeholder.contains("inline:") && !placeholder.contains("group-by:")) { + throw new IllegalArgumentException("Custom search metric Gauge for metric <" + metricName + + "> tag placeholder <" + placeholder + + "> is not supported. Supported placeholder types are 'inline' and 'group-by'."); + } + }); + + final Set requiredInlinePlaceholders = getDeclaredInlinePlaceholderExpressions(getTags()); + getFilterConfigs().forEach(filterConfig -> { + final Set definedInlinePlaceholderValues = filterConfig.getInlinePlaceholderValues().keySet(); + if (!requiredInlinePlaceholders.equals(definedInlinePlaceholderValues)) { + throw new IllegalArgumentException("Custom search metric Gauge for metric <" + metricName + + "> filter <" + filterConfig.getFilterName() + + "> must have the same inline-placeholder-values keys as the configured placeholders in tags."); + } + }); + + final Set requiredGroupByPlaceholders = getDeclaredGroupByPlaceholdersExpressions(getTags()); + if (!requiredGroupByPlaceholders.equals(getGroupBy().keySet())) { + throw new IllegalArgumentException("Custom search metric Gauge for metric <" + + metricName + "> must have the same groupBy fields as the configured placeholder expressions in tags. Required: " + requiredGroupByPlaceholders + " Configured: " + getGroupBy().keySet()); + } + } + + private Set getDeclaredInlinePlaceholderExpressions(final Map tags) { + return tags.values().stream() + .filter(this::isPlaceHolder) + .map(value -> value.substring(2, value.length() - 2).trim()) + .filter(value -> value.startsWith(InlinePlaceholderResolver.PREFIX + ":")) + .map(value -> value.substring((InlinePlaceholderResolver.PREFIX + ":").length())) + .collect(Collectors.toSet()); + } + + private Set getDeclaredGroupByPlaceholdersExpressions(final Map tags) { + return tags.values().stream() + .filter(this::isPlaceHolder) + .map(value -> value.substring(2, value.length() - 2).trim()) + .filter(value -> value.startsWith(GroupByPlaceholderResolver.PREFIX + ":")) + .map(value -> value.substring((GroupByPlaceholderResolver.PREFIX + ":").length())) + .map(value -> Arrays.stream(value.split("\\|")).findFirst().map(String::trim).orElse("")) + .filter(value -> !value.isEmpty()) + .collect(Collectors.toSet()); + } + + private boolean isPlaceHolder(final String value) { + return value.startsWith("{{") && value.endsWith("}}"); } @Override @@ -89,6 +191,7 @@ public boolean equals(final Object o) { } final DefaultCustomSearchMetricConfig that = (DefaultCustomSearchMetricConfig) o; return enabled == that.enabled && + Objects.equals(metricName, that.metricName) && Objects.equals(scrapeInterval, that.scrapeInterval) && Objects.equals(namespaces, that.namespaces) && Objects.equals(tags, that.tags) && @@ -97,35 +200,53 @@ public boolean equals(final Object o) { @Override public int hashCode() { - return Objects.hash(enabled, scrapeInterval, namespaces, tags, filterConfigs); + return Objects.hash(metricName, enabled, scrapeInterval, namespaces, tags, filterConfigs); } @Override public String toString() { return getClass().getSimpleName() + " [" + - "enabled=" + enabled + + "customMetricName=" + metricName + + ", enabled=" + enabled + ", scrapeInterval=" + scrapeInterval + ", namespaces=" + namespaces + + ", groupBy=" + groupBy + ", tags=" + tags + ", filterConfig=" + filterConfigs + "]"; } + @Immutable public static final class DefaultFilterConfig implements FilterConfig { private final String filterName; private final String filter; - private final List fields; private final Map inlinePlaceholderValues; private DefaultFilterConfig(final String name, final ConfigWithFallback configWithFallback) { - this.filterName = name; - this.filter = configWithFallback.getString(FilterConfigValues.FILTER.getConfigPath()); - this.fields = configWithFallback.getStringList(FilterConfigValues.FIELDS.getConfigPath()); - this.inlinePlaceholderValues = configWithFallback.getObject(FilterConfigValues.INLINE_PLACEHOLDER_VALUES.getConfigPath()).unwrapped() - .entrySet() - .stream() - .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, e -> String.valueOf(e.getValue()))); + this(name, configWithFallback.getString(FilterConfigValues.FILTER.getConfigPath()), + configWithFallback.getObject(FilterConfigValues.INLINE_PLACEHOLDER_VALUES.getConfigPath()) + .unwrapped() + .entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, + e -> String.valueOf(e.getValue())))); + } + + private DefaultFilterConfig(final String filterName, final String filter, final Map inlinePlaceholderValues) { + this.filterName = filterName; + this.filter = filter; + this.inlinePlaceholderValues = Collections.unmodifiableMap(new HashMap<>(inlinePlaceholderValues)); + } + + public static FilterConfig of(final String name, final Config config) { + return new DefaultFilterConfig( + name, ConfigWithFallback.newInstance(config, CustomSearchMetricConfigValue.values())); + } + + public static FilterConfig of(final FilterConfig filterConfig) { + return new DefaultFilterConfig(filterConfig.getFilterName(), filterConfig.getFilter(), + filterConfig.getInlinePlaceholderValues()); } @Override @@ -138,42 +259,30 @@ public String getFilter() { return filter; } - @Override - public List getFields() { - return fields; - } - @Override public Map getInlinePlaceholderValues() { return inlinePlaceholderValues; } - public static FilterConfig of(final String name, final Config config) { - return new DefaultFilterConfig( - name, ConfigWithFallback.newInstance(config, CustomSearchMetricConfigValue.values())); - } - @Override public boolean equals(final Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; final DefaultFilterConfig that = (DefaultFilterConfig) o; return Objects.equals(filterName, that.filterName) && Objects.equals(filter, that.filter) && - Objects.equals(fields, that.fields) && Objects.equals(inlinePlaceholderValues, that.inlinePlaceholderValues); } @Override public int hashCode() { - return Objects.hash(filterName, filter, fields, inlinePlaceholderValues); + return Objects.hash(filterName, filter, inlinePlaceholderValues); } @Override public String toString() { - return getClass().getSimpleName() + " [" + - "filterName='" + filterName + '\'' + - ", filter='" + filter + '\'' + - ", fields=" + fields + + return getClass().getSimpleName() + " [" + + "filterName=" + filterName + + ", filter=" + filter + ", inlinePlaceholderValues=" + inlinePlaceholderValues + ']'; } diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultOperatorMetricsConfig.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultOperatorMetricsConfig.java index fea4391d92..a5389dc635 100644 --- a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultOperatorMetricsConfig.java +++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultOperatorMetricsConfig.java @@ -50,14 +50,14 @@ public final class DefaultOperatorMetricsConfig implements OperatorMetricsConfig private final boolean enabled; private final Duration scrapeInterval; private final Map customMetricConfigurations; - private final Map customSearchMetricConfigurations; + private final Map customSearchMetricConfigs; private DefaultOperatorMetricsConfig(final ConfigWithFallback updaterScopedConfig) { enabled = updaterScopedConfig.getBoolean(OperatorMetricsConfigValue.ENABLED.getConfigPath()); scrapeInterval = updaterScopedConfig.getNonNegativeDurationOrThrow(OperatorMetricsConfigValue.SCRAPE_INTERVAL); customMetricConfigurations = loadCustomMetricConfigurations(updaterScopedConfig, OperatorMetricsConfigValue.CUSTOM_METRICS); - customSearchMetricConfigurations = loadCustomSearchMetricConfigurations(updaterScopedConfig, + customSearchMetricConfigs = loadCustomSearchMetricConfigurations(updaterScopedConfig, OperatorMetricsConfigValue.CUSTOM_SEARCH_METRICS); } @@ -132,8 +132,8 @@ public Map getCustomMetricConfigurations() { } @Override - public Map getCustomSearchMetricConfigurations() { - return customSearchMetricConfigurations; + public Map getCustomSearchMetricConfigs() { + return customSearchMetricConfigs; } private static class CustomMetricConfigCollector diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/OperatorMetricsConfig.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/OperatorMetricsConfig.java index e443624631..99386ee00c 100644 --- a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/OperatorMetricsConfig.java +++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/OperatorMetricsConfig.java @@ -52,7 +52,7 @@ public interface OperatorMetricsConfig { * * @return the registered custom search metrics. */ - Map getCustomSearchMetricConfigurations(); + Map getCustomSearchMetricConfigs(); /** * An enumeration of the known config path expressions and their associated default values for diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/MongoThingsAggregationPersistence.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/MongoThingsAggregationPersistence.java new file mode 100644 index 0000000000..f31bd13faf --- /dev/null +++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/MongoThingsAggregationPersistence.java @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2024 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + * + */ + +package org.eclipse.ditto.thingsearch.service.persistence.read; + +import static com.mongodb.client.model.Aggregates.group; +import static com.mongodb.client.model.Aggregates.match; +import static com.mongodb.client.model.Filters.in; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import org.apache.pekko.NotUsed; +import org.apache.pekko.event.LoggingAdapter; +import org.apache.pekko.stream.javadsl.Source; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.eclipse.ditto.internal.utils.persistence.mongo.DittoMongoClient; +import org.eclipse.ditto.rql.parser.RqlPredicateParser; +import org.eclipse.ditto.rql.query.expression.ThingsFieldExpressionFactory; +import org.eclipse.ditto.rql.query.filter.QueryFilterCriteriaFactory; +import org.eclipse.ditto.thingsearch.model.signals.commands.query.AggregateThingsMetrics; +import org.eclipse.ditto.thingsearch.service.common.config.SearchConfig; +import org.eclipse.ditto.thingsearch.service.common.config.SearchPersistenceConfig; +import org.eclipse.ditto.thingsearch.service.persistence.PersistenceConstants; +import org.eclipse.ditto.thingsearch.service.persistence.read.criteria.visitors.CreateBsonAggregationVisitor; + +import com.mongodb.client.model.BsonField; +import com.mongodb.reactivestreams.client.MongoCollection; +import com.mongodb.reactivestreams.client.MongoDatabase; + +public class MongoThingsAggregationPersistence implements ThingsAggregationPersistence { + + + private final MongoCollection collection; + private final LoggingAdapter log; + private final Duration maxQueryTime; + private final MongoHints hints; + private final QueryFilterCriteriaFactory queryFilterCriteriaFactory; + + /** + * Initializes the things search persistence with a passed in {@code persistence}. + * + * @param mongoClient the mongoDB persistence wrapper. + * @param persistenceConfig the search persistence configuration. + * @param log the logger. + */ + private MongoThingsAggregationPersistence(final DittoMongoClient mongoClient, + final Optional mongoHintsByNamespace, + final Map simpleFieldMappings, final SearchPersistenceConfig persistenceConfig, + final LoggingAdapter log) { + this.queryFilterCriteriaFactory = + QueryFilterCriteriaFactory.of(ThingsFieldExpressionFactory.of(simpleFieldMappings), + RqlPredicateParser.getInstance()); + final MongoDatabase database = mongoClient.getDefaultDatabase(); + final var readConcern = persistenceConfig.readConcern(); + final var readPreference = persistenceConfig.readPreference().getMongoReadPreference(); + collection = database.getCollection(PersistenceConstants.THINGS_COLLECTION_NAME) + .withReadConcern(readConcern.getMongoReadConcern()) + .withReadPreference(readPreference); + this.log = log; + maxQueryTime = mongoClient.getDittoSettings().getMaxQueryTime(); + hints = mongoHintsByNamespace.map(MongoHints::byNamespace).orElse(MongoHints.empty()); + log.info("Aggregation readConcern=<{}> readPreference=<{}>", readConcern, readPreference); + } + + public static ThingsAggregationPersistence of(final DittoMongoClient mongoClient, + final SearchConfig searchConfig, final LoggingAdapter log) { + return new MongoThingsAggregationPersistence(mongoClient, searchConfig.getMongoHintsByNamespace(), + searchConfig.getSimpleFieldMappings(), searchConfig.getQueryPersistenceConfig(), log); + } + + @Override + public Source aggregateThings(final AggregateThingsMetrics aggregateCommand) { + final List aggregatePipeline = new ArrayList<>(); + + // Add $match stage if namespaces are present + if (!aggregateCommand.getNamespaces().isEmpty()) { + aggregatePipeline.add(match(in(PersistenceConstants.FIELD_NAMESPACE, aggregateCommand.getNamespaces()))); + } + + // Construct the $group stage + final Map groupingBy = + aggregateCommand.getGroupingBy().entrySet().stream().collect(Collectors.toMap( + Map.Entry::getKey, entry -> "$t." + entry.getValue().replace("/", "."))); + final List accumulatorFields = aggregateCommand.getNamedFilters() + .entrySet() + .stream() + .map(entry -> new BsonField(entry.getKey(), new Document("$sum", + new Document("$cond", Arrays.asList(CreateBsonAggregationVisitor.sudoApply( + queryFilterCriteriaFactory.filterCriteria(entry.getValue(), + aggregateCommand.getDittoHeaders())), 1, 0))))) + .collect(Collectors.toList()); + final Bson group = group(new Document(groupingBy), accumulatorFields); + aggregatePipeline.add(group); + log.info("aggregatePipeline: {}", // TODO debug + aggregatePipeline.stream().map(bson -> bson.toBsonDocument().toJson()).collect( + Collectors.toList())); + // Execute the aggregation pipeline + return Source.fromPublisher(collection.aggregate(aggregatePipeline) + .hint(hints.getHint(aggregateCommand.getNamespaces()) + .orElse(null)) + .allowDiskUse(true) + .maxTime(maxQueryTime.toMillis(), TimeUnit.MILLISECONDS)); + } +} diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/ThingsAggregationPersistence.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/ThingsAggregationPersistence.java new file mode 100644 index 0000000000..91981b9b7d --- /dev/null +++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/ThingsAggregationPersistence.java @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2024 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + * + */ +package org.eclipse.ditto.thingsearch.service.persistence.read; + + +import org.apache.pekko.NotUsed; +import org.apache.pekko.stream.javadsl.Source; +import org.bson.Document; +import org.eclipse.ditto.thingsearch.model.signals.commands.query.AggregateThingsMetrics; + +/** + * Interface for thing aggregations on the search collection. + * + * @since 3.6.0 + */ +public interface ThingsAggregationPersistence { + + /** + * Aggregate things based on the given aggregateCommand. + * + * @param aggregateCommand the aggregateCommand to aggregate things + * @return the aggregated things + */ + Source aggregateThings(AggregateThingsMetrics aggregateCommand); + +} diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/ThingsSearchPersistence.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/ThingsSearchPersistence.java index f059443f5e..2ad03b6f2f 100644 --- a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/ThingsSearchPersistence.java +++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/ThingsSearchPersistence.java @@ -79,7 +79,7 @@ public interface ThingsSearchPersistence { * @return an {@link Source} which emits the IDs. * @throws NullPointerException if {@code query} is {@code null}. */ - Source, NotUsed> findAll(Query query, List authorizationSubjectIds, + Source, NotUsed> findAll(Query query, @Nullable List authorizationSubjectIds, @Nullable Set namespaces); /** diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/criteria/visitors/CreateBsonAggregationPredicateVisitor.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/criteria/visitors/CreateBsonAggregationPredicateVisitor.java new file mode 100644 index 0000000000..96ab0daadb --- /dev/null +++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/criteria/visitors/CreateBsonAggregationPredicateVisitor.java @@ -0,0 +1,205 @@ +/* + * Copyright (c) 2017 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.thingsearch.service.persistence.read.criteria.visitors; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.function.Function; +import java.util.regex.Pattern; + +import javax.annotation.Nullable; + +import org.bson.BsonArray; +import org.bson.BsonDocument; +import org.bson.BsonInt64; +import org.bson.BsonString; +import org.bson.BsonValue; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.eclipse.ditto.placeholders.PlaceholderResolver; +import org.eclipse.ditto.rql.model.ParsedPlaceholder; +import org.eclipse.ditto.rql.query.criteria.Predicate; +import org.eclipse.ditto.rql.query.criteria.visitors.PredicateVisitor; + +/** + * Creates Bson of a predicate. + */ +public class CreateBsonAggregationPredicateVisitor implements PredicateVisitor> { + + private static CreateBsonAggregationPredicateVisitor instance; + + private static final String LEADING_WILDCARD = "^\\Q\\E.*"; + private static final String TRAILING_WILDCARD = ".*\\Q\\E$"; + + private final List> additionalPlaceholderResolvers; + + private CreateBsonAggregationPredicateVisitor(final Collection> additionalPlaceholderResolvers) { + this.additionalPlaceholderResolvers = + Collections.unmodifiableList(new ArrayList<>(additionalPlaceholderResolvers)); + } + + /** + * Gets the singleton instance of this {@link org.eclipse.ditto.thingsearch.service.persistence.read.criteria.visitors.CreateBsonAggregationPredicateVisitor}. + * + * @return the singleton instance. + */ + public static CreateBsonAggregationPredicateVisitor getInstance() { + if (null == instance) { + instance = new CreateBsonAggregationPredicateVisitor(Collections.emptyList()); + } + return instance; + } + + /** + * Creates a new instance of {@code CreateBsonPredicateVisitor} with additional custom placeholder resolvers. + * + * @param additionalPlaceholderResolvers the additional {@code PlaceholderResolver} to use for resolving + * placeholders in RQL predicates. + * @return the created instance. + * @since 2.3.0 + */ + public static CreateBsonAggregationPredicateVisitor createInstance( + final PlaceholderResolver... additionalPlaceholderResolvers) { + return createInstance(Arrays.asList(additionalPlaceholderResolvers)); + } + + /** + * Creates a new instance of {@code CreateBsonPredicateVisitor} with additional custom placeholder resolvers. + * + * @param additionalPlaceholderResolvers the additional {@code PlaceholderResolver} to use for resolving + * placeholders in RQL predicates. + * @return the created instance. + * @since 2.3.0 + */ + public static CreateBsonAggregationPredicateVisitor createInstance( + final Collection> additionalPlaceholderResolvers) { + return new CreateBsonAggregationPredicateVisitor(additionalPlaceholderResolvers); + } + + /** + * Creates a Bson from a predicate and its field name. + * + * @param predicate The predicate to generate the Bson from. + * @param fieldName Name of the field where the predicate is applied to. + * @return The created Bson. + */ + public static Bson apply(final Predicate predicate, final String fieldName) { + return predicate.accept(getInstance()).apply(fieldName); + } + + @Override + public Function visitEq(@Nullable final Object value) { + return fieldName -> new BsonDocument("$eq", new BsonArray(List.of(new BsonString("$" + fieldName), resolveValue(value)))); + } + + @Override + public Function visitGe(@Nullable final Object value) { + return fieldName -> new BsonDocument("$gte", new BsonArray(List.of(new BsonString("$" + fieldName), resolveValue(value)))); + } + + @Override + public Function visitGt(@Nullable final Object value) { + return fieldName -> new BsonDocument("$gt", new BsonArray(List.of(new BsonString("$" + fieldName), resolveValue(value)))); + } + + @Override + public Function visitIn(final List values) { + final List collect = values.stream().map(this::resolveValue).toList(); + return fieldName -> new Document("$in", Arrays.asList("$" + fieldName, collect)); + } + + @Override + public Function visitLe(@Nullable final Object value) { + return fieldName -> new BsonDocument("$lte", new BsonArray(List.of(new BsonString("$" + fieldName), resolveValue(value)))); + } + + @Override + public Function visitLike(final String value) { + // remove leading or trailing wildcard because queries like /^a/ are much faster than /^a.*$/ or /^a.*/ + // from mongodb docs: + // "Additionally, while /^a/, /^a.*/, and /^a.*$/ match equivalent strings, they have different performance + // characteristics. All of these expressions use an index if an appropriate index exists; + // however, /^a.*/, and /^a.*$/ are slower. /^a/ can stop scanning after matching the prefix." + final String valueWithoutLeadingOrTrailingWildcard = removeLeadingOrTrailingWildcard(value); + return fieldName -> new Document("$regexMatch", + new Document("input", "$" + fieldName) + .append("regex", valueWithoutLeadingOrTrailingWildcard)); + } + + @Override + public Function visitILike(final String value) { + // remove leading or trailing wildcard because queries like /^a/ are much faster than /^a.*$/ or /^a.*/ + // from mongodb docs: + // "Additionally, while /^a/, /^a.*/, and /^a.*$/ match equivalent strings, they have different performance + // characteristics. All of these expressions use an index if an appropriate index exists; + // however, /^a.*/, and /^a.*$/ are slower. /^a/ can stop scanning after matching the prefix." + final String valueWithoutLeadingOrTrailingWildcard = removeLeadingOrTrailingWildcard(value); + Pattern pattern = Pattern.compile(valueWithoutLeadingOrTrailingWildcard, Pattern.CASE_INSENSITIVE); + return fieldName -> new Document("$regexMatch", + new Document("input", "$" + fieldName) + .append("regex", pattern)); + } + + private static String removeLeadingOrTrailingWildcard(final String valueString) { + String valueWithoutLeadingOrTrailingWildcard = valueString; + if (valueString.startsWith(LEADING_WILDCARD)) { + valueWithoutLeadingOrTrailingWildcard = valueWithoutLeadingOrTrailingWildcard + .substring(LEADING_WILDCARD.length()); + } + if (valueString.endsWith(TRAILING_WILDCARD)) { + final int endIndex = valueWithoutLeadingOrTrailingWildcard.length() - TRAILING_WILDCARD.length(); + if (endIndex > 0) { + valueWithoutLeadingOrTrailingWildcard = valueWithoutLeadingOrTrailingWildcard.substring(0, endIndex); + } + } + return valueWithoutLeadingOrTrailingWildcard; + } + + @Override + public Function visitLt(final Object value) { + return fieldName -> new BsonDocument("$lt", new BsonArray(List.of(new BsonString("$" + fieldName), resolveValue(value)))); + } + + @Override + public Function visitNe(final Object value) { + return fieldName -> new BsonDocument("$ne", new BsonArray(List.of(new BsonString("$" + fieldName), resolveValue(value)))); + } + + private BsonValue resolveValue(final Object value) { + if (value instanceof ParsedPlaceholder) { + final String prefix = ((ParsedPlaceholder) value).getPrefix(); + final String name = ((ParsedPlaceholder) value).getName(); + return additionalPlaceholderResolvers.stream() + .filter(pr -> prefix.equals(pr.getPrefix())) + .filter(pr -> pr.supports(name)) + .flatMap(pr -> pr.resolveValues(name).stream()) + .map(BsonString::new) + .findFirst() + .orElse(null); + } + if (value instanceof Long) { + return new BsonInt64((Long) value); + } else if (value instanceof Integer) { + return new BsonInt64((Integer) value); + } else if (value instanceof String) { + return new BsonString((String) value); + } else if (value instanceof ArrayList) { + return new BsonArray((ArrayList) value); + } else { + throw new IllegalArgumentException("Unsupported value type: " + value.getClass()); + } + } +} diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/criteria/visitors/CreateBsonAggregationVisitor.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/criteria/visitors/CreateBsonAggregationVisitor.java new file mode 100644 index 0000000000..4bb091ed96 --- /dev/null +++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/criteria/visitors/CreateBsonAggregationVisitor.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2017 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.thingsearch.service.persistence.read.criteria.visitors; + +import java.util.List; +import java.util.function.Function; + +import javax.annotation.Nullable; + +import org.bson.conversions.Bson; +import org.eclipse.ditto.placeholders.PlaceholderFactory; +import org.eclipse.ditto.placeholders.TimePlaceholder; +import org.eclipse.ditto.rql.query.criteria.Criteria; +import org.eclipse.ditto.rql.query.criteria.Predicate; +import org.eclipse.ditto.rql.query.expression.FilterFieldExpression; +import org.eclipse.ditto.thingsearch.service.persistence.read.expression.visitors.GetFilterBsonVisitor; + +/** + * Creates the Bson object used for querying. + */ +public class CreateBsonAggregationVisitor extends CreateBsonVisitor { + + private static final TimePlaceholder TIME_PLACEHOLDER = TimePlaceholder.getInstance(); + + + private CreateBsonAggregationVisitor(@Nullable final List authorizationSubjectIds) { + super(authorizationSubjectIds); + } + + /** + * Creates the Bson object used for querying with no restriction of visibility. + * + * @param criteria the criteria to create Bson for. + * @return the Bson object + */ + public static Bson sudoApply(final Criteria criteria) { + return criteria.accept(new CreateBsonAggregationVisitor(null)); + } + + @Override + public Bson visitField(final FilterFieldExpression fieldExpression, final Predicate predicate) { + final Function predicateCreator = predicate.accept( + CreateBsonAggregationPredicateVisitor.createInstance( + PlaceholderFactory.newPlaceholderResolver(TIME_PLACEHOLDER, new Object()) + ) + ); + return GetFilterBsonVisitor.apply(fieldExpression, predicateCreator, null); + } + + +} diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/criteria/visitors/CreateBsonVisitor.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/criteria/visitors/CreateBsonVisitor.java index 2ba21bf4ef..7933197866 100644 --- a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/criteria/visitors/CreateBsonVisitor.java +++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/criteria/visitors/CreateBsonVisitor.java @@ -44,7 +44,7 @@ public class CreateBsonVisitor implements CriteriaVisitor { @Nullable private final List authorizationSubjectIds; - private CreateBsonVisitor(@Nullable final List authorizationSubjectIds) { + CreateBsonVisitor(@Nullable final List authorizationSubjectIds) { this.authorizationSubjectIds = authorizationSubjectIds; } diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/placeholders/GroupByPlaceholderResolver.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/placeholders/GroupByPlaceholderResolver.java new file mode 100644 index 0000000000..a631a365b7 --- /dev/null +++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/placeholders/GroupByPlaceholderResolver.java @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2024 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + * + */ + +package org.eclipse.ditto.thingsearch.service.placeholders; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import org.eclipse.ditto.placeholders.PlaceholderResolver; + +public class GroupByPlaceholderResolver implements PlaceholderResolver> { + + public static final String PREFIX = "group-by"; + private final List supportedNames; + private final Map source; + + public GroupByPlaceholderResolver(final Set supportedNames, final Map source) { + this.supportedNames = List.of(supportedNames.toArray(new String[0])); + this.source = source; + } + + @Override + public List resolveValues(final Map placeholderSource, final String name) { + return Optional.ofNullable(placeholderSource.get(name)).map(e1 -> List.of(e1)).orElse(List.of()); + } + + @Override + public List> getPlaceholderSources() { + return List.of(source); + } + + @Override + public String getPrefix() { + return PREFIX; + } + + @Override + public List getSupportedNames() { + return supportedNames; + } + + @Override + public boolean supports(final String name) { + return supportedNames.contains(name); + } +} diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/placeholders/InlinePlaceholderResolver.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/placeholders/InlinePlaceholderResolver.java new file mode 100644 index 0000000000..4376a1aaa6 --- /dev/null +++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/placeholders/InlinePlaceholderResolver.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2024 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + * + */ + +package org.eclipse.ditto.thingsearch.service.placeholders; + +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import org.eclipse.ditto.placeholders.PlaceholderResolver; + +public class InlinePlaceholderResolver implements PlaceholderResolver> { + + public static final String PREFIX = "inline"; + private final Map source; + private final List supportedNames; + + public InlinePlaceholderResolver(final Map source) { + this.source = source; + this.supportedNames = source.keySet().stream().toList(); + } + + @Override + public List> getPlaceholderSources() { + return List.of(source); + } + + @Override + public List resolveValues(final Map placeholderSource, final String name) { + return Optional.ofNullable(placeholderSource.get(name)).map(List::of).orElse(List.of()); + } + + @Override + public String getPrefix() { + return PREFIX; + } + + @Override + public List getSupportedNames() { + return supportedNames; + } + + @Override + public boolean supports(final String name) { + return supportedNames.contains(name); + } +} diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/AggregationThingsMetricsActor.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/AggregationThingsMetricsActor.java new file mode 100644 index 0000000000..9c3249c3b2 --- /dev/null +++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/AggregationThingsMetricsActor.java @@ -0,0 +1,155 @@ +/* + * Copyright (c) 2024 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + * + */ + +package org.eclipse.ditto.thingsearch.service.starter.actors; + +import java.util.concurrent.CompletableFuture; + +import org.apache.pekko.NotUsed; +import org.apache.pekko.actor.AbstractActor; +import org.apache.pekko.actor.ActorRef; +import org.apache.pekko.actor.Props; +import org.apache.pekko.japi.pf.PFBuilder; +import org.apache.pekko.japi.pf.ReceiveBuilder; +import org.apache.pekko.pattern.Patterns; +import org.apache.pekko.stream.Graph; +import org.apache.pekko.stream.SourceShape; +import org.apache.pekko.stream.SystemMaterializer; +import org.apache.pekko.stream.javadsl.Flow; +import org.apache.pekko.stream.javadsl.Sink; +import org.apache.pekko.stream.javadsl.Source; +import org.bson.Document; +import org.eclipse.ditto.base.model.exceptions.DittoInternalErrorException; +import org.eclipse.ditto.base.model.exceptions.DittoJsonException; +import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.base.model.headers.WithDittoHeaders; +import org.eclipse.ditto.internal.utils.metrics.DittoMetrics; +import org.eclipse.ditto.internal.utils.metrics.instruments.timer.StartedTimer; +import org.eclipse.ditto.internal.utils.pekko.logging.DittoLoggerFactory; +import org.eclipse.ditto.internal.utils.pekko.logging.ThreadSafeDittoLoggingAdapter; +import org.eclipse.ditto.internal.utils.tracing.DittoTracing; +import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.thingsearch.model.signals.commands.query.AggregateThingsMetrics; +import org.eclipse.ditto.thingsearch.model.signals.commands.query.AggregateThingsMetricsResponse; +import org.eclipse.ditto.thingsearch.service.persistence.read.ThingsAggregationPersistence; + +/** + * Actor handling custom metrics aggregations {@link org.eclipse.ditto.thingsearch.model.signals.commands.query.AggregateThingsMetrics}. + */ +public class AggregationThingsMetricsActor + extends AbstractActor { + + /** + * The name of this actor in the system. + */ + static final String ACTOR_NAME = ThingsAggregationConstants.AGGREGATE_ACTOR_NAME; + private static final String TRACING_THINGS_AGGREGATION = "aggregate_things_metrics"; + + private final ThreadSafeDittoLoggingAdapter log; + private final ThingsAggregationPersistence thingsAggregationPersistence; + + private AggregationThingsMetricsActor(final ThingsAggregationPersistence aggregationPersistence) { + log = DittoLoggerFactory.getThreadSafeDittoLoggingAdapter(this); + this.thingsAggregationPersistence = aggregationPersistence; + } + + public static Props props(final ThingsAggregationPersistence aggregationPersistence) { + return Props.create(AggregationThingsMetricsActor.class,aggregationPersistence); + } + + @Override + public Receive createReceive() { + return ReceiveBuilder.create() + .match(AggregateThingsMetrics.class, this::aggregate) + .matchAny(any -> { + log.warning("Got unknown message '{}'", any); + }) + .build(); + } + + private void aggregate(AggregateThingsMetrics aggregateThingsMetrics) { + log.debug("Received aggregate command for {}", aggregateThingsMetrics); + final StartedTimer aggregationTimer = startNewTimer(aggregateThingsMetrics); + final Source source = + DittoJsonException.wrapJsonRuntimeException(aggregateThingsMetrics, aggregateThingsMetrics.getDittoHeaders(), + (command, headers) -> thingsAggregationPersistence.aggregateThings(command)); + final Source aggregationResult = + processAggregationPersistenceResult(source, aggregateThingsMetrics.getDittoHeaders()) + .map(aggregation -> JsonFactory.newObject(aggregation.toJson())) + .map(aggregation -> AggregateThingsMetricsResponse.of(aggregation, aggregateThingsMetrics)); + final ActorRef sender = getSender(); // Save sender as it is not available after the first element is processed + final Source replySourceWithErrorHandling = + aggregationResult.via(stopTimerAndHandleError(aggregationTimer, aggregateThingsMetrics)); + + replySourceWithErrorHandling.runWith(Sink.foreach(elem -> { + Patterns.pipe(CompletableFuture.completedFuture(elem), getContext().dispatcher()).to(sender); + + }), SystemMaterializer.get(getContext().getSystem()).materializer()); + } + +private Source processAggregationPersistenceResult(final Source source, + final DittoHeaders dittoHeaders) { + + final Flow logAndFinishPersistenceSegmentFlow = + Flow.fromFunction(result -> { + log.withCorrelationId(dittoHeaders) + .debug("aggregation element: {}", result); + return result; + }); +return source.via(logAndFinishPersistenceSegmentFlow); +} + + private static StartedTimer startNewTimer(final WithDittoHeaders withDittoHeaders) { + final StartedTimer startedTimer = DittoMetrics.timer(TRACING_THINGS_AGGREGATION) + .start(); + DittoTracing.newStartedSpanByTimer(withDittoHeaders.getDittoHeaders(), startedTimer); + + return startedTimer; + } + + private static void stopTimer(final StartedTimer timer) { + try { + timer.stop(); + } catch (final IllegalStateException e) { + // it is okay if the timer was stopped. + } + } + private Flow stopTimerAndHandleError(final StartedTimer searchTimer, + final WithDittoHeaders command) { + return Flow.fromFunction( + element -> { + stopTimer(searchTimer); + return element; + }) + .recoverWithRetries(1, new PFBuilder, NotUsed>>() + .matchAny(error -> { + stopTimer(searchTimer); + return Source.single(asDittoRuntimeException(error, command)); + }) + .build() + ); + } + private DittoRuntimeException asDittoRuntimeException(final Throwable error, final WithDittoHeaders trigger) { + return DittoRuntimeException.asDittoRuntimeException(error, t -> { + log.error(error, "AggregateThingsMetricsActor failed to execute <{}>", trigger); + + return DittoInternalErrorException.newBuilder() + .dittoHeaders(trigger.getDittoHeaders()) + .message(error.getClass() + ": " + error.getMessage()) + .cause(t) + .build(); + }); + } +} diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/OperatorSearchMetricsProviderActor.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/OperatorSearchMetricsProviderActor.java index 4598543f8f..10ff90fb32 100644 --- a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/OperatorSearchMetricsProviderActor.java +++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/OperatorSearchMetricsProviderActor.java @@ -12,14 +12,17 @@ */ package org.eclipse.ditto.thingsearch.service.starter.actors; +import static org.eclipse.ditto.thingsearch.service.starter.actors.ThingsAggregationConstants.CLUSTER_ROLE; + import java.time.Duration; -import java.util.ArrayList; import java.util.Comparator; import java.util.HashMap; -import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.UUID; import java.util.concurrent.ThreadLocalRandom; import java.util.stream.Collectors; @@ -30,11 +33,8 @@ import org.apache.pekko.actor.Props; import org.apache.pekko.actor.Status; import org.apache.pekko.japi.pf.ReceiveBuilder; -import org.apache.pekko.pattern.Patterns; -import org.eclipse.ditto.base.model.exceptions.DittoRuntimeException; import org.eclipse.ditto.base.model.headers.DittoHeaders; -import org.eclipse.ditto.edge.service.dispatching.ThingsAggregatorProxyActor; -import org.eclipse.ditto.edge.service.placeholders.ThingJsonPlaceholder; +import org.eclipse.ditto.internal.utils.cluster.ClusterUtil; import org.eclipse.ditto.internal.utils.metrics.instruments.gauge.Gauge; import org.eclipse.ditto.internal.utils.metrics.instruments.gauge.KamonGauge; import org.eclipse.ditto.internal.utils.metrics.instruments.tag.KamonTagSetConverter; @@ -42,21 +42,20 @@ import org.eclipse.ditto.internal.utils.metrics.instruments.tag.TagSet; import org.eclipse.ditto.internal.utils.pekko.logging.DittoDiagnosticLoggingAdapter; import org.eclipse.ditto.internal.utils.pekko.logging.DittoLoggerFactory; -import org.eclipse.ditto.json.JsonFactory; -import org.eclipse.ditto.json.JsonObject; -import org.eclipse.ditto.json.JsonParseOptions; -import org.eclipse.ditto.things.api.commands.sudo.SudoRetrieveThings; -import org.eclipse.ditto.things.api.commands.sudo.SudoRetrieveThingsResponse; -import org.eclipse.ditto.things.model.Thing; -import org.eclipse.ditto.things.model.ThingId; -import org.eclipse.ditto.things.model.ThingsModelFactory; -import org.eclipse.ditto.thingsearch.model.signals.commands.query.QueryThings; -import org.eclipse.ditto.thingsearch.model.signals.commands.query.QueryThingsResponse; +import org.eclipse.ditto.internal.utils.persistence.mongo.DittoMongoClient; +import org.eclipse.ditto.placeholders.ExpressionResolver; +import org.eclipse.ditto.placeholders.PlaceholderFactory; +import org.eclipse.ditto.placeholders.PlaceholderResolver; +import org.eclipse.ditto.thingsearch.model.signals.commands.query.AggregateThingsMetrics; +import org.eclipse.ditto.thingsearch.model.signals.commands.query.AggregateThingsMetricsResponse; import org.eclipse.ditto.thingsearch.service.common.config.CustomSearchMetricConfig; import org.eclipse.ditto.thingsearch.service.common.config.OperatorMetricsConfig; +import org.eclipse.ditto.thingsearch.service.common.config.SearchConfig; +import org.eclipse.ditto.thingsearch.service.persistence.read.MongoThingsAggregationPersistence; +import org.eclipse.ditto.thingsearch.service.placeholders.GroupByPlaceholderResolver; +import org.eclipse.ditto.thingsearch.service.placeholders.InlinePlaceholderResolver; import kamon.Kamon; -import scala.Tuple2; /** * Actor which is started as singleton for "search" role and is responsible for querying for extended operator defined @@ -71,53 +70,49 @@ public final class OperatorSearchMetricsProviderActor extends AbstractActorWithT private static final int MIN_INITIAL_DELAY_SECONDS = 30; private static final int MAX_INITIAL_DELAY_SECONDS = 90; - private static final int DEFAULT_SEARCH_TIMEOUT_SECONDS = 60; + private static final String METRIC_NAME = "metric-name"; private final DittoDiagnosticLoggingAdapter log = DittoLoggerFactory.getDiagnosticLoggingAdapter(this); - private final ThingJsonPlaceholder thingJsonPlaceholder = ThingJsonPlaceholder.getInstance(); - private final Map aggregatedByTagsValues = new HashMap<>(); - - private final ActorRef searchActor; - private final Map> metricsGauges; + private final ActorRef thingsAggregatorActorSingletonProxy; + private final Map customSearchMetricConfigMap; + private final Map metricsGauges; private final Gauge customSearchMetricsGauge; - private final ActorRef thingsAggregatorProxyActor; + private final Map>> inlinePlaceholderResolvers; @SuppressWarnings("unused") - private OperatorSearchMetricsProviderActor(final OperatorMetricsConfig operatorMetricsConfig, - final ActorRef searchActor, final ActorRef pubSubMediator) { - customSearchMetricsGauge = KamonGauge.newGauge("custom-search-metrics"); - this.searchActor = searchActor; - thingsAggregatorProxyActor = getContext().actorOf(ThingsAggregatorProxyActor.props(pubSubMediator), - ThingsAggregatorProxyActor.ACTOR_NAME); - metricsGauges = new HashMap<>(); - operatorMetricsConfig.getCustomSearchMetricConfigurations().forEach((metricName, config) -> { - if (config.isEnabled()) { - initializeCustomMetricTimer(operatorMetricsConfig, metricName, config); - } else { - log.info("Initializing custom search metric Gauge for metric <{}> is DISABLED", metricName); - } + private OperatorSearchMetricsProviderActor(final SearchConfig searchConfig) { + this.thingsAggregatorActorSingletonProxy = initializeAggregationThingsMetricsActor(searchConfig); + this.customSearchMetricConfigMap = searchConfig.getOperatorMetricsConfig().getCustomSearchMetricConfigs(); + this.metricsGauges = new HashMap<>(); + this.inlinePlaceholderResolvers = new HashMap<>(); + this.customSearchMetricsGauge = KamonGauge.newGauge("custom-search-metrics-count-of-instruments"); + this.customSearchMetricConfigMap.forEach((metricName, customSearchMetricConfig) -> { + initializeCustomMetricTimer(metricName, customSearchMetricConfig, getMaxConfiguredScrapeInterval(searchConfig.getOperatorMetricsConfig())); + + // Initialize the inline resolvers here as they use a static source from config + customSearchMetricConfig.getFilterConfigs() + .forEach(fc -> inlinePlaceholderResolvers.put(new FilterIdentifier(metricName, fc.getFilterName()), + new InlinePlaceholderResolver(fc.getInlinePlaceholderValues()))); }); - initializeCustomMetricsCleanupTimer(operatorMetricsConfig); + initializeCustomMetricsCleanupTimer(searchConfig.getOperatorMetricsConfig()); } /** * Create Props for this actor. * - * @param operatorMetricsConfig the config to use - * @param searchActor the SearchActor Actor reference + * @param searchConfig the searchConfig to use * @return the Props object. */ - public static Props props(final OperatorMetricsConfig operatorMetricsConfig, final ActorRef searchActor, - final ActorRef pubSubMediator) { - return Props.create(OperatorSearchMetricsProviderActor.class, operatorMetricsConfig, searchActor, - pubSubMediator); + public static Props props(final SearchConfig searchConfig) { + return Props.create(OperatorSearchMetricsProviderActor.class, searchConfig); } @Override public Receive createReceive() { return ReceiveBuilder.create() - .match(GatherMetrics.class, this::handleGatheringMetrics) - .match(CleanupUnusedMetrics.class, this::handleCleanupUnusedMetrics) + .match(GatherMetricsCommand.class, this::handleGatheringMetrics) + .match(AggregateThingsMetricsResponse.class, this::handleAggregateThingsResponse) + .match(CleanupUnusedMetricsCommand.class, this::handleCleanupUnusedMetrics) .match(Status.Failure.class, f -> log.error(f.cause(), "Got failure: {}", f)) .matchAny(m -> { log.warning("Unknown message: {}", m); @@ -126,273 +121,198 @@ public Receive createReceive() { .build(); } - private void initializeCustomMetricTimer(final OperatorMetricsConfig operatorMetricsConfig, final String metricName, - final CustomSearchMetricConfig config) { - // start each custom metric provider with a random initialDelay - final Duration initialDelay = Duration.ofSeconds( - ThreadLocalRandom.current().nextInt(MIN_INITIAL_DELAY_SECONDS, MAX_INITIAL_DELAY_SECONDS) - ); - final Duration scrapeInterval = config.getScrapeInterval() - .orElse(operatorMetricsConfig.getScrapeInterval()); - log.info("Initializing custom metric timer for metric <{}> with initialDelay <{}> and scrapeInterval <{}>", - metricName, - initialDelay, scrapeInterval); - getTimers().startTimerAtFixedRate( - metricName, createGatherCustomMetric(metricName, config), initialDelay, scrapeInterval); - } - - private void initializeCustomMetricsCleanupTimer(final OperatorMetricsConfig operatorMetricsConfig) { - final Duration interval = getMaxConfiguredScrapeInterval(operatorMetricsConfig).multipliedBy(3); - log.info("Initializing custom metric cleanup timer Interval <{}>", interval); - getTimers().startTimerAtFixedRate("cleanup-unused-metrics", new CleanupUnusedMetrics(operatorMetricsConfig), - interval); + private ActorRef initializeAggregationThingsMetricsActor(final SearchConfig searchConfig) { + final DittoMongoClient mongoDbClient = MongoClientExtension.get(getContext().system()).getSearchClient(); + final var props = AggregationThingsMetricsActor.props(MongoThingsAggregationPersistence.of(mongoDbClient, searchConfig, log)); + final ActorRef aggregationThingsMetricsActorProxy = ClusterUtil + .startSingletonProxy(getContext(), CLUSTER_ROLE, + ClusterUtil.startSingleton(getContext(), CLUSTER_ROLE, AggregationThingsMetricsActor.ACTOR_NAME, + props)); + log.info("Started child actor <{}> with path <{}>.", AggregationThingsMetricsActor.ACTOR_NAME, + aggregationThingsMetricsActorProxy); + return aggregationThingsMetricsActorProxy; } - private void handleCleanupUnusedMetrics(CleanupUnusedMetrics cleanupUnusedMetrics) { - // remove metrics who were not used for longer than three times the max configured scrape interval - final long currentTime = System.currentTimeMillis(); - metricsGauges.entrySet().stream() - .filter(entry -> { - final long time = entry.getValue()._2(); - return currentTime - time > getMaxConfiguredScrapeInterval(cleanupUnusedMetrics.config()) - .multipliedBy(3).toMillis(); - }) - .forEach(entry -> { - final String realName = realName(entry.getKey()); - if (Kamon.gauge(realName) - .withTags(KamonTagSetConverter.getKamonTagSet(entry.getValue()._1().getTagSet())) - .remove()) { - log.debug("Removed custom search metric instrument: {} {}", realName, - entry.getValue()._1().getTagSet()); - customSearchMetricsGauge.decrement(); - } else { - log.info("Could not remove unused custom search metric instrument: {}", entry.getKey()); - } - }); - } - - private void handleGatheringMetrics(final GatherMetrics gatherMetrics) { - final String metricName = gatherMetrics.metricName(); - final CustomSearchMetricConfig config = gatherMetrics.config(); + private void handleGatheringMetrics(final GatherMetricsCommand gatherMetricsCommand) { + final String metricName = gatherMetricsCommand.metricName(); + final CustomSearchMetricConfig config = gatherMetricsCommand.config(); final DittoHeaders dittoHeaders = DittoHeaders.newBuilder() .correlationId("gather-search-metrics_" + metricName + "_" + UUID.randomUUID()) - .putHeader("ditto-sudo", "true") .build(); - final long startTs = System.nanoTime(); - config.getFilterConfigs().forEach(filterConfig -> { - QueryThings searchThings = QueryThings.of(filterConfig.getFilter(), null, null, - new HashSet<>(config.getNamespaces()), dittoHeaders); - askSearchActor(searchThings, metricName, startTs, filterConfig, config, dittoHeaders); + final Map namedFilters = config.getFilterConfigs().stream() + .collect(Collectors.toMap(CustomSearchMetricConfig.FilterConfig::getFilterName, + CustomSearchMetricConfig.FilterConfig::getFilter)); + AggregateThingsMetrics + aggregateThingsMetrics = AggregateThingsMetrics.of(metricName, config.getGroupBy(), namedFilters, + Set.of(config.getNamespaces().toArray(new String[0])), dittoHeaders); + thingsAggregatorActorSingletonProxy.tell(aggregateThingsMetrics, getSelf()); + } + + + private void handleAggregateThingsResponse(AggregateThingsMetricsResponse response) { + log.withCorrelationId(response).info("Received aggregate things response: {} thread: {}", //TODO debug + response, Thread.currentThread().getName()); + final String metricName = response.getMetricName(); + // record by filter name and tags + response.getResult().forEach((filterName, value) -> { + resolveTags(filterName, customSearchMetricConfigMap.get(metricName), response); + final CustomSearchMetricConfig customSearchMetricConfig = customSearchMetricConfigMap.get(metricName); + final TagSet tagSet = resolveTags(filterName, customSearchMetricConfig, response) + .putTag(Tag.of("filter", filterName)); + recordMetric(metricName, tagSet, value); + customSearchMetricsGauge.tag(Tag.of(METRIC_NAME, metricName)).set(Long.valueOf(metricsGauges.size()));; }); } - private static GatherMetrics createGatherCustomMetric(final String metricName, - final CustomSearchMetricConfig config) { - return new GatherMetrics(metricName, config); + private void recordMetric(final String metricName, final TagSet tagSet, final Long value) { + metricsGauges.compute(new GageIdentifier(metricName, tagSet), (gageIdentifier, timestampedGauge) -> { + if (timestampedGauge == null) { + final Gauge gauge = KamonGauge.newGauge(metricName) + .tags(tagSet); + gauge.set(value); + return new TimestampedGauge(gauge); + } else { + return timestampedGauge.set(value); + } + }); } - private void askSearchActor(final QueryThings searchThings, final String metricName, final long startTs, - final CustomSearchMetricConfig.FilterConfig filterConfig, final CustomSearchMetricConfig config, - final DittoHeaders dittoHeaders) { - log.withCorrelationId(dittoHeaders).debug("Asking for things for custom metric <{}>..", metricName); - Patterns.ask(searchActor, searchThings, Duration.ofSeconds(DEFAULT_SEARCH_TIMEOUT_SECONDS)) - .whenComplete((response, throwable) -> { - if (response instanceof QueryThingsResponse queryThingsResponse) { - log.withCorrelationId(queryThingsResponse) - .debug("Received QueryThingsResponse for custom search metric <{}>: {} - " + - "duration: <{}ms>", - metricName, queryThingsResponse.getSearchResult().getItems().getSize(), - Duration.ofNanos(System.nanoTime() - startTs).toMillis() - ); - aggregateResponse(queryThingsResponse, filterConfig, metricName, config, dittoHeaders); - if (queryThingsResponse.getSearchResult().hasNextPage() && - queryThingsResponse.getSearchResult().getCursor().isPresent()) { - - QueryThings nextPageSearch = QueryThings.of(searchThings.getFilter().orElse(null), - List.of("cursor(" + queryThingsResponse.getSearchResult().getCursor().get() + ")"), - null, - new HashSet<>(config.getNamespaces()), dittoHeaders); - log.withCorrelationId(queryThingsResponse) - .debug("Asking for next page {} for custom search metric <{}>..", - queryThingsResponse.getSearchResult().getNextPageOffset().orElse(-1L), - metricName); - askSearchActor(nextPageSearch, metricName, startTs, filterConfig, config, dittoHeaders); - } - recordMetric(metricName); - - } else if (response instanceof DittoRuntimeException dre) { - log.withCorrelationId(dittoHeaders).warning( - "Received DittoRuntimeException when gathering things for " + - "custom search metric <{}> with queryThings {}: {}", metricName, searchThings, - dre.getMessage(), dre - ); - } else { - log.withCorrelationId(dittoHeaders).warning( - "Received unexpected result or throwable when gathering things for " + - "custom search metric <{}> with queryThings {}: {}", metricName, searchThings, - response, throwable - ); - } - }); + private TagSet resolveTags(final String filterName, final CustomSearchMetricConfig customSearchMetricConfig, + final AggregateThingsMetricsResponse response) { + return TagSet.ofTagCollection(customSearchMetricConfig.getTags().entrySet().stream().map(tagEntry-> { + if (!isPlaceHolder(tagEntry.getValue())) { + return Tag.of(tagEntry.getKey(), tagEntry.getValue()); + } else { + + final ExpressionResolver expressionResolver = + PlaceholderFactory.newExpressionResolver(List.of( + new GroupByPlaceholderResolver(customSearchMetricConfig.getGroupBy().keySet(), response.getGroupedBy()) + , inlinePlaceholderResolvers.get(new FilterIdentifier(customSearchMetricConfig.getMetricName(), filterName)))); + return expressionResolver.resolve(tagEntry.getValue()) + .findFirst() + .map(resolvedValue -> Tag.of(tagEntry.getKey(), resolvedValue)) + .orElse(Tag.of(tagEntry.getKey(), tagEntry.getValue())); + } + }).collect(Collectors.toSet())); } - private void aggregateResponse(QueryThingsResponse queryThingsResponse, - CustomSearchMetricConfig.FilterConfig filterConfig, String metricName, - CustomSearchMetricConfig config, DittoHeaders dittoHeaders) { - if (isOnlyThingIdField(filterConfig.getFields())) { - final List things = queryThingsResponse.getSearchResult() - .getItems() - .stream() - .map(jsonValue -> ThingsModelFactory.newThing( - jsonValue.asObject())) - .toList(); - aggregateByTags(things, filterConfig, metricName, config, dittoHeaders); - } else { - final List thingIds = queryThingsResponse.getSearchResult() - .getItems() - .stream() - .map(jsonValue -> ThingsModelFactory.newThing(jsonValue.asObject()).getEntityId()) - .filter(Optional::isPresent) - .map(Optional::get) - .toList(); - log.withCorrelationId(dittoHeaders) - .debug("Retrieved Things for custom search metric {}: {}", metricName, thingIds); - final SudoRetrieveThings sudoRetrieveThings = SudoRetrieveThings.of(thingIds, - JsonFactory.newFieldSelector(ensureThingId(filterConfig.getFields()), JsonParseOptions.newBuilder() - .withoutUrlDecoding().build()), dittoHeaders); - Patterns.ask(thingsAggregatorProxyActor, sudoRetrieveThings, - Duration.ofSeconds(DEFAULT_SEARCH_TIMEOUT_SECONDS)) - .handle((response, throwable) -> { - if (response instanceof SudoRetrieveThingsResponse retrieveThingsResponse) { - // aggregate response by tags and record after all pages are received - aggregateByTags(retrieveThingsResponse.getThings(), filterConfig, metricName, config, - dittoHeaders); - } else { - log.withCorrelationId(dittoHeaders).warning( - "Received unexpected result or throwable when gathering things for " + - "custom search metric <{}> with queryThings {}: {}", metricName, - queryThingsResponse, - response, throwable - ); - } - return null; - }); + private void handleCleanupUnusedMetrics(final CleanupUnusedMetricsCommand cleanupCommand) { + // remove metrics who were not used for longer than three times the max configured scrape interval + final long currentTime = System.currentTimeMillis(); + final Iterator> iterator = metricsGauges.entrySet().iterator(); + while (iterator.hasNext()) { + final Map.Entry next = iterator.next(); + final long lastUpdated = next.getValue().getLastUpdated(); + final long unusedPeriod = getMaxConfiguredScrapeInterval(cleanupCommand.config()).multipliedBy(2).toMillis(); + final long expire = lastUpdated + unusedPeriod; + log.debug("cleanup metrics: expired: {}, time left: {} lastUpdated: {} expire: {} currentTime: {}", + currentTime > expire, expire - currentTime, lastUpdated, expire, currentTime); + if (currentTime > expire) { + final String metricName = next.getKey().metricName(); + // setting to zero as there is a bug in Kamon where the gauge is not removed and is still reported + // https://github.com/kamon-io/Kamon/issues/566 + next.getValue().set(0L); + if ( Kamon.gauge(metricName).remove(KamonTagSetConverter.getKamonTagSet(next.getValue().getTagSet()))) { + log.debug("Removed custom search metric instrument: {} {}", metricName, + next.getValue().getTagSet()); + iterator.remove(); + customSearchMetricsGauge.tag(Tag.of(METRIC_NAME, metricName)).set( + Long.valueOf(metricsGauges.size())); + } else { + log.warning("Could not remove unused custom search metric instrument: {}", next.getKey()); + } + } } } - private List ensureThingId(final List fields) { - return fields.contains("thingId") ? fields : Stream.concat(fields.stream(), Stream.of("thingId")) - .collect(Collectors.toList()); + private Duration getMaxConfiguredScrapeInterval(final OperatorMetricsConfig operatorMetricsConfig) { + return Stream.concat(Stream.of(operatorMetricsConfig.getScrapeInterval()), + operatorMetricsConfig.getCustomSearchMetricConfigs().values().stream() + .map(CustomSearchMetricConfig::getScrapeInterval) + .filter(Optional::isPresent) + .map(Optional::get)) + .max(Comparator.naturalOrder()) + .orElse(operatorMetricsConfig.getScrapeInterval()); + } + + private void initializeCustomMetricTimer(final String metricName, final CustomSearchMetricConfig config, final Duration scrapeInterval ) { + if (!config.isEnabled()) { + log.info("Custom search metric Gauge for metric <{}> is DISABLED. Skipping init.", metricName); + return; + } + // start each custom metric provider with a random initialDelay + final Duration initialDelay = Duration.ofSeconds( + ThreadLocalRandom.current().nextInt(MIN_INITIAL_DELAY_SECONDS, MAX_INITIAL_DELAY_SECONDS) + ); + log.info("Initializing custom metric timer for metric <{}> with initialDelay <{}> and scrapeInterval <{}>", + metricName, + initialDelay, scrapeInterval); + getTimers().startTimerAtFixedRate(metricName, new GatherMetricsCommand(metricName, config), initialDelay, + scrapeInterval); } - private void aggregateByTags(final List things, - final CustomSearchMetricConfig.FilterConfig filterConfig, final String metricName, - final CustomSearchMetricConfig config, final DittoHeaders dittoHeaders) { - final List tagSets = things.stream() - .map(thing -> TagSet.ofTagCollection(config.getTags().entrySet().stream() - .map(e -> Tag.of(e.getKey(), - resolvePlaceHolder(thing.toJson().asObject(), e.getValue(), filterConfig, dittoHeaders, - metricName))) - .sorted(Comparator.comparing(Tag::getValue)) - .toList())).toList(); - tagSets.forEach(tagSet -> aggregatedByTagsValues.merge(tagSet, 1.0, Double::sum)); + private void initializeCustomMetricsCleanupTimer(final OperatorMetricsConfig operatorMetricsConfig) { + final Duration interval = getMaxConfiguredScrapeInterval(operatorMetricsConfig); + log.info("Initializing custom metric cleanup timer Interval <{}>", interval); + getTimers().startTimerAtFixedRate("cleanup-unused-metrics", new CleanupUnusedMetricsCommand(operatorMetricsConfig), + interval); } - private void recordMetric(final String metricName) { - aggregatedByTagsValues.forEach( - (tags, value) -> metricsGauges.computeIfAbsent(uniqueName(metricName, tags), name -> { - log.info("Initializing custom search metric Gauge for metric <{}> with tags <{}>", - metricName, tags); - customSearchMetricsGauge.increment(); - return Tuple2.apply(KamonGauge.newGauge(metricName) - .tags(tags), System.currentTimeMillis()); - }) - ._1().set(value)); - - aggregatedByTagsValues.forEach( - (tags, value) -> metricsGauges.compute(uniqueName(metricName, tags), (key, tupleValue) -> { - if (tupleValue == null) { - log.info("Initializing custom search metric Gauge for metric <{}> with tags <{}>", - metricName, tags); - customSearchMetricsGauge.increment(); - return Tuple2.apply(KamonGauge.newGauge(metricName) - .tags(tags), System.currentTimeMillis()); - - } else { - // update the timestamp - log.debug("Updating custom search metric Gauge for metric <{}> with tags <{}>", - metricName, tags); - return Tuple2.apply(tupleValue._1(), System.currentTimeMillis()); - } - }) - ._1().set(value)); - - aggregatedByTagsValues.clear(); + private boolean isPlaceHolder(final String value) { + return value.startsWith("{{") && value.endsWith("}}"); } - private String resolvePlaceHolder(final JsonObject thingJson, final String value, - final CustomSearchMetricConfig.FilterConfig filterConfig, final DittoHeaders dittoHeaders, - final String metricName) { - if (!isPlaceHolder(value)) { - return value; - } - String placeholder = value.substring(2, value.length() - 2).trim(); - final List resolvedValues = - new ArrayList<>( - thingJsonPlaceholder.resolveValues(ThingsModelFactory.newThing(thingJson), placeholder)); - - if (resolvedValues.isEmpty()) { - filterConfig.getInlinePlaceholderValues().forEach((k, v) -> { - if (placeholder.equals(k)) { - resolvedValues.add(v); - } - }); + private static class TimestampedGauge { + + private final Gauge gauge; + private final Long timestamp; + + private TimestampedGauge(Gauge gauge) { + this.gauge = gauge; + this.timestamp = System.currentTimeMillis(); } - if (resolvedValues.isEmpty()) { - log.withCorrelationId(dittoHeaders) - .warning("Custom search metric {}. Could not resolve placeholder <{}> in thing <{}>. " + - "Check that you have your fields configured correctly.", metricName, - placeholder, thingJson); - return value; + + public TimestampedGauge set(final Long value) { + gauge.set(value); + return new TimestampedGauge(gauge); } + private TagSet getTagSet() { + return gauge.getTagSet(); + } - return resolvedValues.stream().findFirst() - .orElse(value); - } + private long getLastUpdated() { + return timestamp; + } - private String uniqueName(final String metricName, final TagSet tags) { - final ArrayList list = new ArrayList<>(); - tags.iterator().forEachRemaining(list::add); - return list.stream().sorted(Comparator.comparing(Tag::getKey)).map(t -> t.getKey() + "=" + t.getValue()) - .collect(Collectors.joining("_", metricName + "#", "")); - } + @Override + public boolean equals(final Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + final TimestampedGauge that = (TimestampedGauge) o; + return Objects.equals(gauge, that.gauge) && Objects.equals(timestamp, that.timestamp); + } - private String realName(final String uniqueName) { - return uniqueName.substring(0, uniqueName.indexOf("#")); - } + @Override + public int hashCode() { + return Objects.hash(gauge, timestamp); + } - private boolean isPlaceHolder(final String value) { - return value.startsWith("{{") && value.endsWith("}}"); + @Override + public String toString() { + return "GageWithTimestamp{" + + "gauge=" + gauge + + ", timestamp=" + timestamp + + '}'; + } } - private boolean isOnlyThingIdField(final List filterConfig) { - return filterConfig.isEmpty() || - (filterConfig.size() == 1 && filterConfig.get(0).equals(Thing.JsonFields.ID.getPointer().toString())); - } + private record GatherMetricsCommand(String metricName, CustomSearchMetricConfig config) {} - private Duration getMaxConfiguredScrapeInterval(final OperatorMetricsConfig operatorMetricsConfig) { - return Stream.concat(Stream.of(operatorMetricsConfig.getScrapeInterval()), - operatorMetricsConfig.getCustomSearchMetricConfigurations().values().stream() - .map(CustomSearchMetricConfig::getScrapeInterval) - .filter(Optional::isPresent) - .map(Optional::get)) - .max(Comparator.naturalOrder()) - .orElse(operatorMetricsConfig.getScrapeInterval()); - } + private record CleanupUnusedMetricsCommand(OperatorMetricsConfig config) {} - private record GatherMetrics(String metricName, CustomSearchMetricConfig config) {} + private record FilterIdentifier(String metricName, String filterName) {} - private record CleanupUnusedMetrics(OperatorMetricsConfig config) {} + private record GageIdentifier(String metricName, TagSet tags) {} } diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/ThingsAggregationConstants.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/ThingsAggregationConstants.java new file mode 100644 index 0000000000..66853ed044 --- /dev/null +++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/ThingsAggregationConstants.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2024 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + * + */ +package org.eclipse.ditto.thingsearch.service.starter.actors; + +import javax.annotation.concurrent.Immutable; + +/** + * Constants for the Things Aggregate. + */ +@Immutable +public final class ThingsAggregationConstants { + + + /** + * Name of the pekko cluster role. + */ + public static final String CLUSTER_ROLE = "search"; + + private static final String PATH_DELIMITER = "/"; + + @SuppressWarnings("squid:S1075") + private static final String USER_PATH = "/user"; + + /** + * Name of the Aggregate actor + */ + public static final String AGGREGATE_ACTOR_NAME = "aggregateThingsMetrics"; + + /* + * Inhibit instantiation of this utility class. + */ + private ThingsAggregationConstants() { + // no-op + } +} diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/updater/actors/SearchUpdaterRootActor.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/updater/actors/SearchUpdaterRootActor.java index 732cd9991a..aa6005a713 100644 --- a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/updater/actors/SearchUpdaterRootActor.java +++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/updater/actors/SearchUpdaterRootActor.java @@ -134,7 +134,7 @@ private SearchUpdaterRootActor(final SearchConfig searchConfig, OperatorMetricsProviderActor.props(searchConfig.getOperatorMetricsConfig(), searchActor) ); startClusterSingletonActor(OperatorSearchMetricsProviderActor.ACTOR_NAME, - OperatorSearchMetricsProviderActor.props(searchConfig.getOperatorMetricsConfig(), searchActor, pubSubMediator) + OperatorSearchMetricsProviderActor.props(searchConfig) ); } diff --git a/thingsearch/service/src/main/resources/search-dev.conf b/thingsearch/service/src/main/resources/search-dev.conf index 72e679189b..8af313a0df 100755 --- a/thingsearch/service/src/main/resources/search-dev.conf +++ b/thingsearch/service/src/main/resources/search-dev.conf @@ -28,34 +28,35 @@ ditto { custom-search-metrics { online_status { enabled = false - scrape-interval = 1m # override scrape interval, run each minute + scrape-interval = 1m # override scrape interval, run every 20 minute namespaces = [ "org.eclipse.ditto" ] + group-by:{ + "location" = "attributes/Info/location" + "isGateway" = "attributes/Info/gateway" + } tags: { - "online" = "{{online_placeholder}}" - "location" = "{{attributes/Info/location}}" + "online" = "{{ inline:online_placeholder }}" + "health" = "{{ inline:health }}" + "hardcoded-tag" = "value" + "location" = "{{ group-by:location | fn:default('missing location') }}" + "altitude" = "{{ group-by:isGateway }}" } filters = { - online-filter = { - filter = "gt(features/ConnectionStatus/properties/status/readyUntil/,time:now)" + online_filter = { + filter = "and(gt(features/ConnectionStatus/properties/status/readyUntil/,time:now),in(features/coffee-brewer/properties/brewed-coffees,0,1,2))" inline-placeholder-values = { - // inline-placeholder-values are used to define hardcoded values to be used in the tags values if the placeholders are not json paths to an actual field in the thing - // this is used to define different tags values based on the filter that matched the thing "online_placeholder" = true + "health" = "good" } - // in order to do placeholder resolving from thing fields, the fields should be defined in the fields array - // by default only the thingId is available for placeholder resolving - fields = ["attributes/Info/location"] - // The metric-value is used to define the value of the metric for the thing that matched the filter. - // It does not support placeholders and should be a numeric value } - offline-filter = { - filter = "lt(features/ConnectionStatus/properties/status/readyUntil/,time:now)" + offline_filter = { + filter = "and(lt(features/ConnectionStatus/properties/status/readyUntil/,time:now),ilike(policyId,'*Eclipse*'))" inline-placeholder-values = { "online_placeholder" = false + "health" = "bad" } - fields = ["attributes/Info/location"] } } } diff --git a/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultCustomSearchMetricConfigTest.java b/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultCustomSearchMetricConfigTest.java new file mode 100644 index 0000000000..9579551ee7 --- /dev/null +++ b/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultCustomSearchMetricConfigTest.java @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2024 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + * + */ + +package org.eclipse.ditto.thingsearch.service.common.config; + +import static org.mutabilitydetector.unittesting.AllowedReason.provided; +import static org.mutabilitydetector.unittesting.MutabilityAssert.assertInstancesOf; +import static org.mutabilitydetector.unittesting.MutabilityMatchers.areImmutable; + +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +import org.assertj.core.api.JUnitSoftAssertions; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; + +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; + +import nl.jqno.equalsverifier.EqualsVerifier; + +public class DefaultCustomSearchMetricConfigTest { + + private static Config config; + private static Config customSearchMetricTestConfig; + + @Rule + public final JUnitSoftAssertions softly = new JUnitSoftAssertions(); + + @BeforeClass + public static void initTestFixture() { + config = ConfigFactory.load("custom-search-metric-test"); + customSearchMetricTestConfig = config.getConfig("ditto.search.operator-metrics.custom-search-metrics"); + } + + @Test + public void assertImmutabilityFilterConfig() { + assertInstancesOf(DefaultCustomSearchMetricConfig.DefaultFilterConfig.class, areImmutable(), provided(Config.class).isAlsoImmutable()); + } + @Test + public void assertImmutabilityCustomSearchMetricConfig() { + assertInstancesOf(DefaultCustomSearchMetricConfig.class, areImmutable(), provided(Config.class).isAlsoImmutable(), + provided(DefaultCustomSearchMetricConfig.FilterConfig.class).isAlsoImmutable()); + } + + @Test + public void testHashCodeAndEquals() { + EqualsVerifier.forClass(DefaultCustomSearchMetricConfig.class) + .usingGetClass() + .verify(); + } + + @Test + public void gettersReturnConfiguredValues() { + final DefaultCustomSearchMetricConfig underTest = + DefaultCustomSearchMetricConfig.of("online_status", + customSearchMetricTestConfig.getConfig("online_status")); + + softly.assertThat(underTest.isEnabled()) + .as(CustomSearchMetricConfig.CustomSearchMetricConfigValue.ENABLED.getConfigPath()) + .isEqualTo(true); + softly.assertThat(underTest.getScrapeInterval()) + .as(CustomSearchMetricConfig.CustomSearchMetricConfigValue.SCRAPE_INTERVAL.getConfigPath()) + .isEqualTo(Optional.ofNullable(customSearchMetricTestConfig.getDuration( + "online_status.scrape-interval"))); + softly.assertThat(underTest.getNamespaces()) + .as(CustomSearchMetricConfig.CustomSearchMetricConfigValue.NAMESPACES.getConfigPath()) + .containsExactlyInAnyOrder("org.eclipse.ditto.test.1", "org.eclipse.ditto.test.2"); + softly.assertThat(underTest.getTags()) + .as(CustomSearchMetricConfig.CustomSearchMetricConfigValue.TAGS.getConfigPath()) + .containsExactlyInAnyOrderEntriesOf( + customSearchMetricTestConfig.getObject("online_status.tags") + .unwrapped().entrySet().stream().collect( + Collectors.toMap(Map.Entry::getKey, o -> o.getValue().toString()))); + softly.assertThat(underTest.getFilterConfigs()) + .as(CustomSearchMetricConfig.CustomSearchMetricConfigValue.FILTERS.getConfigPath()) + .hasSize(2); + softly.assertThat(underTest.getFilterConfigs().get(0).getFilterName()) + .as("filter name") + .isEqualTo("online-filter"); + softly.assertThat(underTest.getFilterConfigs().get(1).getFilterName()) + .as("filter name") + .isEqualTo("offline-filter"); + softly.assertThat(underTest.getTags()) + .as("tags") + .containsExactlyInAnyOrderEntriesOf( + customSearchMetricTestConfig.getObject("online_status.tags") + .unwrapped().entrySet().stream().collect( + Collectors.toMap(Map.Entry::getKey, o -> o.getValue().toString()))); + } +} \ No newline at end of file diff --git a/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/ThingSearchServiceGlobalErrorRegistryTest.java b/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/ThingSearchServiceGlobalErrorRegistryTest.java index a963746b26..b532295d95 100644 --- a/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/ThingSearchServiceGlobalErrorRegistryTest.java +++ b/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/ThingSearchServiceGlobalErrorRegistryTest.java @@ -27,7 +27,6 @@ import org.eclipse.ditto.base.model.signals.commands.exceptions.PathUnknownException; import org.eclipse.ditto.connectivity.model.ConnectionIdInvalidException; import org.eclipse.ditto.connectivity.model.signals.commands.exceptions.ConnectionConflictException; -import org.eclipse.ditto.edge.service.EdgeServiceTimeoutException; import org.eclipse.ditto.internal.utils.test.GlobalErrorRegistryTestCases; import org.eclipse.ditto.messages.model.AuthorizationSubjectBlockedException; import org.eclipse.ditto.placeholders.PlaceholderFunctionSignatureInvalidException; @@ -71,8 +70,7 @@ public ThingSearchServiceGlobalErrorRegistryTest() { ConnectionConflictException.class, UnknownTopicPathException.class, UnknownSignalException.class, - IllegalAdaptableException.class, - EdgeServiceTimeoutException.class); + IllegalAdaptableException.class); } } diff --git a/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/actors/AggregationThingsMetricsActorTest.java b/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/actors/AggregationThingsMetricsActorTest.java new file mode 100644 index 0000000000..063b810532 --- /dev/null +++ b/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/actors/AggregationThingsMetricsActorTest.java @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2024 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + * + */ + +package org.eclipse.ditto.thingsearch.service.starter.actors; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.pekko.actor.ActorSystem; +import org.apache.pekko.actor.Props; +import org.apache.pekko.stream.javadsl.Source; +import org.apache.pekko.testkit.javadsl.TestKit; +import org.bson.Document; +import org.eclipse.ditto.base.model.headers.DittoHeaders; +import org.eclipse.ditto.internal.utils.tracing.DittoTracing; +import org.eclipse.ditto.internal.utils.tracing.config.DefaultTracingConfig; +import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonObject; +import org.eclipse.ditto.thingsearch.model.signals.commands.query.AggregateThingsMetrics; +import org.eclipse.ditto.thingsearch.model.signals.commands.query.AggregateThingsMetricsResponse; +import org.eclipse.ditto.thingsearch.service.persistence.read.ThingsAggregationPersistence; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +public class AggregationThingsMetricsActorTest { + + static ActorSystem system = ActorSystem.create(); + + @BeforeClass + public static void setup() { + DittoTracing.init(DefaultTracingConfig.of(system.settings().config())); + } + + @AfterClass + public static void teardown() { + TestKit.shutdownActorSystem(system); + system = null; + } + + @Test + public void testHandleAggregateThingsMetrics() { + new TestKit(system) {{ + + // Create a mock persistence object + ThingsAggregationPersistence mockPersistence = mock(ThingsAggregationPersistence.class); + doReturn(Source.from(List.of( + new Document("_id", new Document(Map.of("_revision", 1L, "location", "Berlin"))) + .append("online", 6) + .append("offline", 0), + new Document("_id", new Document(Map.of("_revision", 1L, "location", "Immenstaad"))) + .append("online", 5) + .append("offline", 0), + new Document("_id", new Document(Map.of("_revision", 1L, "location", "Sofia"))) + .append("online", 5) + .append("offline", 3))) + ).when(mockPersistence) + .aggregateThings(any()); + + // Create the actor + Props props = AggregationThingsMetricsActor.props(mockPersistence); + final var actorRef = system.actorOf(props); + + // Prepare the test message + Map groupingBy = Map.of("_revision", "$_revision", "location", "$t.attributes.Info.location"); + Map namedFilters = Map.of( + "online", "gt(features/ConnectionStatus/properties/status/readyUntil/,time:now)", + "offline","lt(features/ConnectionStatus/properties/status/readyUntil/,time:now)"); + Set namespaces = Collections.singleton("namespace"); + DittoHeaders headers = DittoHeaders.newBuilder().build(); + AggregateThingsMetrics metrics = AggregateThingsMetrics.of("metricName", groupingBy, namedFilters, namespaces, headers); + + // Send the message to the actor + actorRef.tell(metrics, getRef()); + + final JsonObject mongoAggregationResult = JsonFactory.newObjectBuilder() + .set("_id", JsonFactory.newObjectBuilder() + .set("_revision", 1) + .set("location", "Berlin") + .build()) + .set("online", 6) + .set("offline", 0) + .build(); + AggregateThingsMetricsResponse + expectedResponse = AggregateThingsMetricsResponse.of(mongoAggregationResult, metrics); + expectMsg(expectedResponse); + + // Verify interactions with the mock (this depends on your actual implementation) + verify(mockPersistence, times(1)).aggregateThings(metrics); + }}; + } + + @Test + public void testUnknownMessage() { + new TestKit(system) {{ + // Create a mock persistence object + + // Create the actor + Props props = AggregationThingsMetricsActor.props(mock(ThingsAggregationPersistence.class)); + final var actorRef = system.actorOf(props); + + // Send an unknown message to the actor + actorRef.tell("unknown message", getRef()); + + // Verify that the actor does not crash and handles the unknown message gracefully + expectNoMessage(); + }}; + } +} diff --git a/thingsearch/service/src/test/resources/custom-search-metric-test.conf b/thingsearch/service/src/test/resources/custom-search-metric-test.conf new file mode 100644 index 0000000000..cfbba3b900 --- /dev/null +++ b/thingsearch/service/src/test/resources/custom-search-metric-test.conf @@ -0,0 +1,80 @@ +ditto { + mapping-strategy.implementation = "org.eclipse.ditto.thingsearch.api.ThingSearchMappingStrategies" + limits { + # limiations for the "search" service + search { + default-page-size = 25 + # the allowed maximum page size limit - e.g. specified when doing a search via HTTP: + # /api/1/search/things?filter=...&option=limit(0,200) + max-page-size = 200 + } + } + mongodb { + uri = "mongodb://localhost:27017/test" + pool { + max-size = 100 + max-wait-time = 30s + max-wait-queue-size = 500000 + } + } + search { + query { + persistence { + readPreference = "nearest" + readConcern = "linearizable" + } + } + query-criteria-validator = "org.eclipse.ditto.thingsearch.service.persistence.query.validation.DefaultQueryCriteriaValidator" + search-update-mapper.implementation = "org.eclipse.ditto.thingsearch.service.persistence.write.streaming.DefaultSearchUpdateMapper" + search-update-observer.implementation = "org.eclipse.ditto.thingsearch.service.updater.actors.DefaultSearchUpdateObserver" + + operator-metrics { + enabled = true + enabled = ${?THINGS_SEARCH_OPERATOR_METRICS_ENABLED} + # by default, execute "count" metrics once every 15 minutes: + scrape-interval = 15m + scrape-interval = ${?THINGS_SEARCH_OPERATOR_METRICS_SCRAPE_INTERVAL} + custom-metrics { + } + custom-search-metrics { + online-status { + enabled = true + scrape-interval = 20m # override scrape interval, run every 20 minute + namespaces = [ + "org.eclipse.ditto.test.1" + "org.eclipse.ditto.test.2" + ] + group-by:{ + "location" = "features/Location/properties/location" + "altitude" = "features/Location/properties/altitude" + } + tags: { + "online" = "{{online_placeholder}}" + "health" = "{{health}}" + "hardcoded-tag" = "value" + "location" = "{{location}}" + "altitude" = "{{altitude}}" + } + filters = { + online-filter = { + filter = "gt(features/ConnectionStatus/properties/status/readyUntil/,{{ time:now }})" + inline-placeholder-values = { + // inline-placeholder-values are used to define hardcoded values to be used in the tags values if the placeholders are not json paths to an actual field in the thing + // this is used to define different tags values based on the filter that matched the thing + "online_placeholder" = true + "health" = "good" + } + } + offline-filter = { + filter = "lt(features/ConnectionStatus/properties/status/readyUntil/,{{ time:now }})" + inline-placeholder-values = { + "online_placeholder" = false + "health" = "bad" + } + } + } + } + } + } + } +} From f21d69681a726492cf5ceff48183cfa706c58fc5 Mon Sep 17 00:00:00 2001 From: Aleksandar Stanchev Date: Mon, 26 Aug 2024 11:39:28 +0300 Subject: [PATCH 07/13] update documentation Signed-off-by: Aleksandar Stanchev --- .../pages/ditto/installation-operating.md | 39 ++++++++++++------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/documentation/src/main/resources/pages/ditto/installation-operating.md b/documentation/src/main/resources/pages/ditto/installation-operating.md index 271df0f30c..fdb351b84e 100644 --- a/documentation/src/main/resources/pages/ditto/installation-operating.md +++ b/documentation/src/main/resources/pages/ditto/installation-operating.md @@ -586,12 +586,20 @@ In Prometheus format this would look like: all_produced_and_not_installed_devices{company="acme-corp"} 42.0 ``` ### Operator defined custom search based metrics -Starting with Ditto 3.6.0, the "custom metrics" functionality is extended to support search based metrics. +Starting with Ditto 3.6.0, the "custom metrics" functionality is extended to support search-based metrics. This is configured via the [search](architecture-services-things-search.html) service configuration and builds on the [search things](basic-search.html#search-queries) functionality. +> :warning: **Abstain of defining grouping by fields that have a high cardinality, as this will lead to a high number of metrics and +may overload the Prometheus server!** + Now you can augment the statistic about "Things" managed in Ditto -fulfilling a certain condition with tags with either predefined values or values retrieved from the things. +fulfilling a certain condition with tags with either predefined values, +values retrieved from the things or values which are defined based on the matching filter. +This is fulfill by using hardcoded values or placeholders in the tags configuration. +The supported placeholder types are inline and group-by placeholders. +[Function expressions](basic-placeholders.html#function-expressions) are also supported +to manipulate the values of the placeholders before they are used in the tags. This would be an example search service configuration snippet for e.g. providing a metric named `online_devices` defining a query on the values of a `ConnectionStatus` feature: @@ -607,34 +615,35 @@ ditto { custom-search-metrics { online_status { enabled = true - scrape-interval = 20m # override scrape interval, run every 20 minute + scrape-interval = 1m # override scrape interval, run every 20 minute namespaces = [ "org.eclipse.ditto" ] + group-by:{ + "location" = "attributes/Info/location" + "isGateway" = "attributes/Info/gateway" + } tags: { - "online" = "{{online_placeholder}}" - "location" = "{{attributes/Info/location}}" + "online" = "{{ inline:online_placeholder }}" + "health" = "{{ inline:health }}" + "hardcoded-tag" = "hardcoded_value" + "location" = "{{ group-by:location | fn:default('missing location') }}" + "altitude" = "{{ group-by:isGateway }}" } filters = { - online-filter = { + online_filter = { filter = "gt(features/ConnectionStatus/properties/status/readyUntil/,time:now)" inline-placeholder-values = { - // inline-placeholder-values are used to define hardcoded values to be used in the tags values if the placeholders are not json paths to an actual field in the thing - // this is used to define different tags values based on the filter that matched the thing "online_placeholder" = true + "health" = "good" } - // in order to do placeholder resolving from thing fields, the fields should be defined in the fields array - // by default only the thingId is available for placeholder resolving - fields = ["attributes/Info/location"] - // The metric-value is used to define the value of the metric for the thing that matched the filter. - // It does not support placeholders and should be a numeric value } - offline-filter = { + offline_filter = { filter = "lt(features/ConnectionStatus/properties/status/readyUntil/,time:now)" inline-placeholder-values = { "online_placeholder" = false + "health" = "bad" } - fields = ["attributes/Info/location"] } } } From 7bba97ec63b91b4a3e659f9a479b83ecd9a269df Mon Sep 17 00:00:00 2001 From: Aleksandar Stanchev Date: Mon, 26 Aug 2024 11:44:20 +0300 Subject: [PATCH 08/13] Fix build and tests Signed-off-by: Aleksandar Stanchev --- thingsearch/service/pom.xml | 4 --- .../DefaultCustomSearchMetricConfig.java | 26 +++++++------------ ...CreateBsonAggregationPredicateVisitor.java | 2 +- .../CreateBsonAggregationVisitor.java | 2 +- .../src/main/resources/search-dev.conf | 4 +-- .../DefaultCustomSearchMetricConfigTest.java | 4 +-- ...earchServiceGlobalCommandRegistryTest.java | 4 +-- ...viceGlobalCommandResponseRegistryTest.java | 5 +--- .../resources/custom-search-metric-test.conf | 26 +++++++++---------- 9 files changed, 30 insertions(+), 47 deletions(-) diff --git a/thingsearch/service/pom.xml b/thingsearch/service/pom.xml index 9ddbcc2a86..7cd9c28d52 100644 --- a/thingsearch/service/pom.xml +++ b/thingsearch/service/pom.xml @@ -154,10 +154,6 @@ test test-jar - - org.eclipse.ditto - ditto-edge-service - diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultCustomSearchMetricConfig.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultCustomSearchMetricConfig.java index b6c95ab211..5033fea6f6 100644 --- a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultCustomSearchMetricConfig.java +++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultCustomSearchMetricConfig.java @@ -183,37 +183,31 @@ private boolean isPlaceHolder(final String value) { @Override public boolean equals(final Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; final DefaultCustomSearchMetricConfig that = (DefaultCustomSearchMetricConfig) o; - return enabled == that.enabled && - Objects.equals(metricName, that.metricName) && + return enabled == that.enabled && Objects.equals(metricName, that.metricName) && Objects.equals(scrapeInterval, that.scrapeInterval) && - Objects.equals(namespaces, that.namespaces) && - Objects.equals(tags, that.tags) && - Objects.equals(filterConfigs, that.filterConfigs); + Objects.equals(namespaces, that.namespaces) && Objects.equals(groupBy, that.groupBy) && + Objects.equals(tags, that.tags) && Objects.equals(filterConfigs, that.filterConfigs); } @Override public int hashCode() { - return Objects.hash(metricName, enabled, scrapeInterval, namespaces, tags, filterConfigs); + return Objects.hash(metricName, enabled, scrapeInterval, namespaces, groupBy, tags, filterConfigs); } @Override public String toString() { - return getClass().getSimpleName() + " [" + - "customMetricName=" + metricName + + return "DefaultCustomSearchMetricConfig{" + + "metricName='" + metricName + '\'' + ", enabled=" + enabled + ", scrapeInterval=" + scrapeInterval + ", namespaces=" + namespaces + ", groupBy=" + groupBy + ", tags=" + tags + - ", filterConfig=" + filterConfigs + - "]"; + ", filterConfigs=" + filterConfigs + + '}'; } @Immutable diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/criteria/visitors/CreateBsonAggregationPredicateVisitor.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/criteria/visitors/CreateBsonAggregationPredicateVisitor.java index 96ab0daadb..55198aae36 100644 --- a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/criteria/visitors/CreateBsonAggregationPredicateVisitor.java +++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/criteria/visitors/CreateBsonAggregationPredicateVisitor.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 Contributors to the Eclipse Foundation + * Copyright (c) 2024 Contributors to the Eclipse Foundation * * See the NOTICE file(s) distributed with this work for additional * information regarding copyright ownership. diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/criteria/visitors/CreateBsonAggregationVisitor.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/criteria/visitors/CreateBsonAggregationVisitor.java index 4bb091ed96..040903d49b 100644 --- a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/criteria/visitors/CreateBsonAggregationVisitor.java +++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/criteria/visitors/CreateBsonAggregationVisitor.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 Contributors to the Eclipse Foundation + * Copyright (c) 2024 Contributors to the Eclipse Foundation * * See the NOTICE file(s) distributed with this work for additional * information regarding copyright ownership. diff --git a/thingsearch/service/src/main/resources/search-dev.conf b/thingsearch/service/src/main/resources/search-dev.conf index 8af313a0df..9d927ae1a6 100755 --- a/thingsearch/service/src/main/resources/search-dev.conf +++ b/thingsearch/service/src/main/resources/search-dev.conf @@ -45,14 +45,14 @@ ditto { } filters = { online_filter = { - filter = "and(gt(features/ConnectionStatus/properties/status/readyUntil/,time:now),in(features/coffee-brewer/properties/brewed-coffees,0,1,2))" + filter = "gt(features/ConnectionStatus/properties/status/readyUntil/,time:now)" inline-placeholder-values = { "online_placeholder" = true "health" = "good" } } offline_filter = { - filter = "and(lt(features/ConnectionStatus/properties/status/readyUntil/,time:now),ilike(policyId,'*Eclipse*'))" + filter = "lt(features/ConnectionStatus/properties/status/readyUntil/,time:now)" inline-placeholder-values = { "online_placeholder" = false "health" = "bad" diff --git a/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultCustomSearchMetricConfigTest.java b/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultCustomSearchMetricConfigTest.java index 9579551ee7..67a6533f2a 100644 --- a/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultCustomSearchMetricConfigTest.java +++ b/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultCustomSearchMetricConfigTest.java @@ -90,10 +90,10 @@ public void gettersReturnConfiguredValues() { .hasSize(2); softly.assertThat(underTest.getFilterConfigs().get(0).getFilterName()) .as("filter name") - .isEqualTo("online-filter"); + .isEqualTo("online_filter"); softly.assertThat(underTest.getFilterConfigs().get(1).getFilterName()) .as("filter name") - .isEqualTo("offline-filter"); + .isEqualTo("offline_filter"); softly.assertThat(underTest.getTags()) .as("tags") .containsExactlyInAnyOrderEntriesOf( diff --git a/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/ThingsSearchServiceGlobalCommandRegistryTest.java b/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/ThingsSearchServiceGlobalCommandRegistryTest.java index 8282a27db6..512235581b 100644 --- a/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/ThingsSearchServiceGlobalCommandRegistryTest.java +++ b/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/ThingsSearchServiceGlobalCommandRegistryTest.java @@ -19,7 +19,6 @@ import org.eclipse.ditto.base.model.namespaces.signals.commands.PurgeNamespace; import org.eclipse.ditto.base.model.signals.commands.streaming.SubscribeForPersistedEvents; import org.eclipse.ditto.base.service.cluster.ModifySplitBrainResolver; -import org.eclipse.ditto.connectivity.api.commands.sudo.SudoRetrieveConnectionTags; import org.eclipse.ditto.connectivity.model.signals.commands.modify.ModifyConnection; import org.eclipse.ditto.connectivity.model.signals.commands.query.RetrieveConnection; import org.eclipse.ditto.internal.models.streaming.SudoStreamPids; @@ -63,8 +62,7 @@ public ThingsSearchServiceGlobalCommandRegistryTest() { ModifyConnection.class, ModifySplitBrainResolver.class, RetrieveConnection.class, - SubscribeForPersistedEvents.class, - SudoRetrieveConnectionTags.class + SubscribeForPersistedEvents.class ); } diff --git a/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/ThingsSearchServiceGlobalCommandResponseRegistryTest.java b/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/ThingsSearchServiceGlobalCommandResponseRegistryTest.java index cdd4c20ab5..80d39bf860 100644 --- a/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/ThingsSearchServiceGlobalCommandResponseRegistryTest.java +++ b/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/ThingsSearchServiceGlobalCommandResponseRegistryTest.java @@ -19,8 +19,6 @@ import org.eclipse.ditto.base.model.namespaces.signals.commands.PurgeNamespaceResponse; import org.eclipse.ditto.base.model.signals.acks.Acknowledgement; import org.eclipse.ditto.base.service.cluster.ModifySplitBrainResolverResponse; -import org.eclipse.ditto.connectivity.api.commands.sudo.ConnectivitySudoQueryCommandResponse; -import org.eclipse.ditto.connectivity.api.commands.sudo.SudoRetrieveConnectionTagsResponse; import org.eclipse.ditto.connectivity.model.signals.commands.ConnectivityErrorResponse; import org.eclipse.ditto.connectivity.model.signals.commands.modify.ModifyConnectionResponse; import org.eclipse.ditto.connectivity.model.signals.commands.query.RetrieveConnectionResponse; @@ -68,8 +66,7 @@ public ThingsSearchServiceGlobalCommandResponseRegistryTest() { ModifyConnectionResponse.class, RetrieveConnectionResponse.class, ModifySplitBrainResolverResponse.class, - ConnectivityErrorResponse.class, - SudoRetrieveConnectionTagsResponse.class + ConnectivityErrorResponse.class ); } diff --git a/thingsearch/service/src/test/resources/custom-search-metric-test.conf b/thingsearch/service/src/test/resources/custom-search-metric-test.conf index cfbba3b900..78e16c5a96 100644 --- a/thingsearch/service/src/test/resources/custom-search-metric-test.conf +++ b/thingsearch/service/src/test/resources/custom-search-metric-test.conf @@ -37,36 +37,34 @@ ditto { custom-metrics { } custom-search-metrics { - online-status { + online_status { enabled = true - scrape-interval = 20m # override scrape interval, run every 20 minute + scrape-interval = 1m # override scrape interval, run every 20 minute namespaces = [ "org.eclipse.ditto.test.1" "org.eclipse.ditto.test.2" ] group-by:{ - "location" = "features/Location/properties/location" - "altitude" = "features/Location/properties/altitude" + "location" = "attributes/Info/location" + "isGateway" = "attributes/Info/gateway" } tags: { - "online" = "{{online_placeholder}}" - "health" = "{{health}}" + "online" = "{{ inline:online_placeholder }}" + "health" = "{{ inline:health }}" "hardcoded-tag" = "value" - "location" = "{{location}}" - "altitude" = "{{altitude}}" + "location" = "{{ group-by:location | fn:default('missing location') }}" + "altitude" = "{{ group-by:isGateway }}" } filters = { - online-filter = { - filter = "gt(features/ConnectionStatus/properties/status/readyUntil/,{{ time:now }})" + online_filter = { + filter = "gt(features/ConnectionStatus/properties/status/readyUntil/,time:now)" inline-placeholder-values = { - // inline-placeholder-values are used to define hardcoded values to be used in the tags values if the placeholders are not json paths to an actual field in the thing - // this is used to define different tags values based on the filter that matched the thing "online_placeholder" = true "health" = "good" } } - offline-filter = { - filter = "lt(features/ConnectionStatus/properties/status/readyUntil/,{{ time:now }})" + offline_filter = { + filter = "lt(features/ConnectionStatus/properties/status/readyUntil/,time:now)" inline-placeholder-values = { "online_placeholder" = false "health" = "bad" From fed6a5c7e74bc7a489dab6e4f2fefbb8fb07626a Mon Sep 17 00:00:00 2001 From: Aleksandar Stanchev Date: Thu, 29 Aug 2024 12:31:52 +0300 Subject: [PATCH 09/13] Disable tracing Signed-off-by: Aleksandar Stanchev --- .../starter/actors/AggregationThingsMetricsActorTest.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/actors/AggregationThingsMetricsActorTest.java b/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/actors/AggregationThingsMetricsActorTest.java index 063b810532..5daf1296d4 100644 --- a/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/actors/AggregationThingsMetricsActorTest.java +++ b/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/actors/AggregationThingsMetricsActorTest.java @@ -32,6 +32,7 @@ import org.bson.Document; import org.eclipse.ditto.base.model.headers.DittoHeaders; import org.eclipse.ditto.internal.utils.tracing.DittoTracing; +import org.eclipse.ditto.internal.utils.tracing.DittoTracingInitResource; import org.eclipse.ditto.internal.utils.tracing.config.DefaultTracingConfig; import org.eclipse.ditto.json.JsonFactory; import org.eclipse.ditto.json.JsonObject; @@ -40,11 +41,15 @@ import org.eclipse.ditto.thingsearch.service.persistence.read.ThingsAggregationPersistence; import org.junit.AfterClass; import org.junit.BeforeClass; +import org.junit.ClassRule; import org.junit.Test; public class AggregationThingsMetricsActorTest { + @ClassRule + public static final DittoTracingInitResource DITTO_TRACING_INIT_RESOURCE = + DittoTracingInitResource.disableDittoTracing(); - static ActorSystem system = ActorSystem.create(); + private static ActorSystem system = ActorSystem.create(); @BeforeClass public static void setup() { From 1e2ca4c625d2aaec8f84a5d5da03b103db88a786 Mon Sep 17 00:00:00 2001 From: Aleksandar Stanchev Date: Fri, 30 Aug 2024 11:03:28 +0300 Subject: [PATCH 10/13] remove mutability usage Signed-off-by: Aleksandar Stanchev --- .../DefaultCustomSearchMetricConfigTest.java | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultCustomSearchMetricConfigTest.java b/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultCustomSearchMetricConfigTest.java index 67a6533f2a..6c69a5d132 100644 --- a/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultCustomSearchMetricConfigTest.java +++ b/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultCustomSearchMetricConfigTest.java @@ -14,10 +14,6 @@ package org.eclipse.ditto.thingsearch.service.common.config; -import static org.mutabilitydetector.unittesting.AllowedReason.provided; -import static org.mutabilitydetector.unittesting.MutabilityAssert.assertInstancesOf; -import static org.mutabilitydetector.unittesting.MutabilityMatchers.areImmutable; - import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; @@ -46,16 +42,6 @@ public static void initTestFixture() { customSearchMetricTestConfig = config.getConfig("ditto.search.operator-metrics.custom-search-metrics"); } - @Test - public void assertImmutabilityFilterConfig() { - assertInstancesOf(DefaultCustomSearchMetricConfig.DefaultFilterConfig.class, areImmutable(), provided(Config.class).isAlsoImmutable()); - } - @Test - public void assertImmutabilityCustomSearchMetricConfig() { - assertInstancesOf(DefaultCustomSearchMetricConfig.class, areImmutable(), provided(Config.class).isAlsoImmutable(), - provided(DefaultCustomSearchMetricConfig.FilterConfig.class).isAlsoImmutable()); - } - @Test public void testHashCodeAndEquals() { EqualsVerifier.forClass(DefaultCustomSearchMetricConfig.class) From 81f185b90080c24abddb27be4d5edf81b8adf6b8 Mon Sep 17 00:00:00 2001 From: Aleksandar Stanchev Date: Mon, 2 Sep 2024 11:16:15 +0300 Subject: [PATCH 11/13] Disable traceing (2) Signed-off-by: Aleksandar Stanchev --- .../starter/actors/AggregationThingsMetricsActorTest.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/actors/AggregationThingsMetricsActorTest.java b/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/actors/AggregationThingsMetricsActorTest.java index 5daf1296d4..ecc475a64d 100644 --- a/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/actors/AggregationThingsMetricsActorTest.java +++ b/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/actors/AggregationThingsMetricsActorTest.java @@ -45,17 +45,13 @@ import org.junit.Test; public class AggregationThingsMetricsActorTest { + @ClassRule public static final DittoTracingInitResource DITTO_TRACING_INIT_RESOURCE = DittoTracingInitResource.disableDittoTracing(); private static ActorSystem system = ActorSystem.create(); - @BeforeClass - public static void setup() { - DittoTracing.init(DefaultTracingConfig.of(system.settings().config())); - } - @AfterClass public static void teardown() { TestKit.shutdownActorSystem(system); From cde6c87a6ac7e2804d8c9efde8570e2c5114a3da Mon Sep 17 00:00:00 2001 From: Aleksandar Stanchev Date: Mon, 9 Sep 2024 15:18:44 +0300 Subject: [PATCH 12/13] fix review findings Signed-off-by: Aleksandar Stanchev --- .../pages/ditto/installation-operating.md | 37 ++- .../query/AggregateThingsMetrics.java | 172 ++++++++++++- .../query/AggregateThingsMetricsResponse.java | 227 ++++++++++++++---- ...ava => CustomAggregationMetricConfig.java} | 2 +- ...DefaultCustomAggregationMetricConfig.java} | 10 +- .../config/DefaultOperatorMetricsConfig.java | 34 +-- .../common/config/OperatorMetricsConfig.java | 10 +- .../MongoThingsAggregationPersistence.java | 12 +- ...CreateBsonAggregationPredicateVisitor.java | 10 +- .../CreateBsonAggregationVisitor.java | 4 +- .../criteria/visitors/CreateBsonVisitor.java | 2 +- .../GroupByPlaceholderResolver.java | 6 +- .../InlinePlaceholderResolver.java | 6 +- ....java => AggregateThingsMetricsActor.java} | 52 ++-- .../OperatorSearchMetricsProviderActor.java | 73 +++--- .../actors/ThingsAggregationConstants.java | 46 ---- .../src/main/resources/search-dev.conf | 4 +- ...ultCustomAggregationMetricConfigTest.java} | 18 +- ...a => AggregateThingsMetricsActorTest.java} | 43 ++-- 19 files changed, 519 insertions(+), 249 deletions(-) rename thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/{CustomSearchMetricConfig.java => CustomAggregationMetricConfig.java} (99%) rename thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/{DefaultCustomSearchMetricConfig.java => DefaultCustomAggregationMetricConfig.java} (96%) rename thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/{AggregationThingsMetricsActor.java => AggregateThingsMetricsActor.java} (78%) delete mode 100644 thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/ThingsAggregationConstants.java rename thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/common/config/{DefaultCustomSearchMetricConfigTest.java => DefaultCustomAggregationMetricConfigTest.java} (79%) rename thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/actors/{AggregationThingsMetricsActorTest.java => AggregateThingsMetricsActorTest.java} (70%) diff --git a/documentation/src/main/resources/pages/ditto/installation-operating.md b/documentation/src/main/resources/pages/ditto/installation-operating.md index fdb351b84e..6c2f47eebc 100644 --- a/documentation/src/main/resources/pages/ditto/installation-operating.md +++ b/documentation/src/main/resources/pages/ditto/installation-operating.md @@ -581,14 +581,14 @@ In order to add custom metrics via System properties, the following example show Ditto will perform a [count things operation](basic-search.html#search-count-queries) each `5m` (5 minutes), providing a gauge named `all_produced_and_not_installed_devices` with the count of the query, adding the tag `company="acme-corp"`. -In Prometheus format this would look like: +In Prometheus format, this would look like: ``` all_produced_and_not_installed_devices{company="acme-corp"} 42.0 ``` -### Operator defined custom search based metrics -Starting with Ditto 3.6.0, the "custom metrics" functionality is extended to support search-based metrics. -This is configured via the [search](architecture-services-things-search.html) service configuration and builds on the -[search things](basic-search.html#search-queries) functionality. + +### Operator defined custom aggregation based metrics +Starting with Ditto 3.6.0, the "custom metrics" functionality is extended to support custom aggregation metrics. +This is configured via the [search](architecture-services-things-search.html) service configuration. > :warning: **Abstain of defining grouping by fields that have a high cardinality, as this will lead to a high number of metrics and may overload the Prometheus server!** @@ -596,8 +596,8 @@ may overload the Prometheus server!** Now you can augment the statistic about "Things" managed in Ditto fulfilling a certain condition with tags with either predefined values, values retrieved from the things or values which are defined based on the matching filter. -This is fulfill by using hardcoded values or placeholders in the tags configuration. -The supported placeholder types are inline and group-by placeholders. +This is fulfilled by using hardcoded values or placeholders in the tags configuration. +The supported placeholder types are `inline` and `group-by` placeholders. [Function expressions](basic-placeholders.html#function-expressions) are also supported to manipulate the values of the placeholders before they are used in the tags. @@ -612,34 +612,33 @@ ditto { custom-metrics { ... } - custom-search-metrics { + custom-aggregate-metrics { online_status { enabled = true scrape-interval = 1m # override scrape interval, run every 20 minute namespaces = [ "org.eclipse.ditto" ] - group-by:{ + group-by { "location" = "attributes/Info/location" "isGateway" = "attributes/Info/gateway" } - tags: { + tags { "online" = "{{ inline:online_placeholder }}" "health" = "{{ inline:health }}" "hardcoded-tag" = "hardcoded_value" "location" = "{{ group-by:location | fn:default('missing location') }}" - "altitude" = "{{ group-by:isGateway }}" } - filters = { - online_filter = { - filter = "gt(features/ConnectionStatus/properties/status/readyUntil/,time:now)" - inline-placeholder-values = { + filters { + online_filter { + filter = "gt(features/ConnectionStatus/properties/status/readyUntil,time:now)" + inline-placeholder-values { "online_placeholder" = true "health" = "good" } } - offline_filter = { - filter = "lt(features/ConnectionStatus/properties/status/readyUntil/,time:now)" + offline_filter { + filter = "lt(features/ConnectionStatus/properties/status/readyUntil,time:now)" inline-placeholder-values = { "online_placeholder" = false "health" = "bad" @@ -671,12 +670,12 @@ To add custom metrics via System properties, the following example shows how the ``` -Ditto will perform a [search things operation](basic-search.html#search-queries) every `20m` (20 minutes), providing +Ditto will perform an [aggregation operation](https://www.mongodb.com/docs/manual/aggregation/) over the search db collection every `20m` (20 minutes), providing a gauge named `online_devices` with the value of devices that match the filter. The tags `online` and `location` will be added. Their values will be resolved from the placeholders `{{online_placeholder}}` and `{{attributes/Info/location}}` respectively. -In Prometheus format this would look like: +In Prometheus format, this would look like: ``` online_status{location="Berlin",online="false"} 6.0 online_status{location="Immenstaad",online="true"} 8.0 diff --git a/thingsearch/model/src/main/java/org/eclipse/ditto/thingsearch/model/signals/commands/query/AggregateThingsMetrics.java b/thingsearch/model/src/main/java/org/eclipse/ditto/thingsearch/model/signals/commands/query/AggregateThingsMetrics.java index c66d96a383..6ba2d7be74 100644 --- a/thingsearch/model/src/main/java/org/eclipse/ditto/thingsearch/model/signals/commands/query/AggregateThingsMetrics.java +++ b/thingsearch/model/src/main/java/org/eclipse/ditto/thingsearch/model/signals/commands/query/AggregateThingsMetrics.java @@ -14,14 +14,66 @@ package org.eclipse.ditto.thingsearch.model.signals.commands.query; +import java.util.Collections; +import java.util.HashMap; import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.function.Predicate; +import java.util.function.Supplier; +import java.util.stream.Collectors; import org.eclipse.ditto.base.model.headers.DittoHeaders; -import org.eclipse.ditto.base.model.headers.WithDittoHeaders; +import org.eclipse.ditto.base.model.json.FieldType; +import org.eclipse.ditto.base.model.json.JsonParsableCommand; +import org.eclipse.ditto.base.model.json.JsonSchemaVersion; +import org.eclipse.ditto.base.model.signals.commands.AbstractCommand; +import org.eclipse.ditto.base.model.signals.commands.CommandJsonDeserializer; +import org.eclipse.ditto.json.JsonArray; +import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonField; +import org.eclipse.ditto.json.JsonFieldDefinition; +import org.eclipse.ditto.json.JsonMissingFieldException; +import org.eclipse.ditto.json.JsonObject; +import org.eclipse.ditto.json.JsonObjectBuilder; +import org.eclipse.ditto.json.JsonPointer; +import org.eclipse.ditto.json.JsonValue; + +/** + * A command to aggregate metrics for things. + */ +@JsonParsableCommand(typePrefix = AggregateThingsMetrics.TYPE_PREFIX, name = AggregateThingsMetrics.NAME) +public final class AggregateThingsMetrics extends AbstractCommand { + + public static final String NAME = "things-metrics"; + /** + * Aggregation resource type. + */ + static final String RESOURCE_TYPE = "aggregation"; + /** + * Type prefix of aggregation command. + */ + public static final String TYPE_PREFIX = RESOURCE_TYPE + "." + TYPE_QUALIFIER + ":"; + /** + * The name of this command. + */ + public static final String TYPE = TYPE_PREFIX + NAME; + + static final JsonFieldDefinition JSON_FILTER = + JsonFactory.newJsonObjectFieldDefinition("filter", FieldType.REGULAR, + JsonSchemaVersion.V_2); + + private static final JsonFieldDefinition METRIC_NAME = + JsonFactory.newStringFieldDefinition("metric-name", FieldType.REGULAR, JsonSchemaVersion.V_2); + private static final JsonFieldDefinition GROUPING_BY = + JsonFactory.newJsonObjectFieldDefinition("grouping-by", FieldType.REGULAR, JsonSchemaVersion.V_2); + private static final JsonFieldDefinition NAMED_FILTERS = + JsonFactory.newJsonObjectFieldDefinition("named-filters", FieldType.REGULAR, JsonSchemaVersion.V_2); + + private static final JsonFieldDefinition NAMESPACES = + JsonFactory.newJsonArrayFieldDefinition("namespaces", FieldType.REGULAR, + JsonSchemaVersion.V_2); -public class AggregateThingsMetrics implements WithDittoHeaders { private final String metricName; private final Map groupingBy; @@ -29,20 +81,80 @@ public class AggregateThingsMetrics implements WithDittoHeaders { private final DittoHeaders dittoHeaders; private final Set namespaces; - private AggregateThingsMetrics(final String metricName, final Map groupingBy, final Map namedFilters, final Set namespaces, + private AggregateThingsMetrics(final String metricName, final Map groupingBy, + final Map namedFilters, final Set namespaces, final DittoHeaders dittoHeaders) { + super(TYPE, dittoHeaders); this.metricName = metricName; - this.groupingBy = groupingBy; - this.namedFilters = namedFilters; - this.namespaces = namespaces; + this.groupingBy = Collections.unmodifiableMap(groupingBy); + this.namedFilters = Collections.unmodifiableMap(namedFilters); + this.namespaces = Collections.unmodifiableSet(namespaces); this.dittoHeaders = dittoHeaders; } - public static AggregateThingsMetrics of(final String metricName, final Map groupingBy, final Map namedFilters, final Set namespaces, + /** + * Creates a new {@link AggregateThingsMetrics} instance. + * + * @param metricName the name of the metric to aggregate. + * @param groupingBy the fields we want our metric aggregation to be grouped by. + * @param namedFilters the named filters to use for the aggregation. + * @param namespaces the namespaces the metric should be executed for. + * @param dittoHeaders the headers to use for the command. + * @return a new {@link AggregateThingsMetrics} instance. + */ + public static AggregateThingsMetrics of(final String metricName, final Map groupingBy, + final Map namedFilters, final Set namespaces, final DittoHeaders dittoHeaders) { return new AggregateThingsMetrics(metricName, groupingBy, namedFilters, namespaces, dittoHeaders); } + /** + * Creates a new {@code AggregateThingsMetrics} from a JSON string. + * + * @param jsonString the JSON string of which the command is to be created. + * @param dittoHeaders the headers of the command. + * @return the command. + * @throws NullPointerException if {@code jsonString} is {@code null}. + * @throws IllegalArgumentException if {@code jsonString} is empty. + * @throws org.eclipse.ditto.json.JsonParseException if the passed in {@code jsonString} was not in the expected + * format. + */ + public static AggregateThingsMetrics fromJson(final String jsonString, final DittoHeaders dittoHeaders) { + return fromJson(JsonFactory.newObject(jsonString), dittoHeaders); + } + + /** + * Creates a new {@code AggregateThingsMetrics} from a JSON object. + * + * @param jsonObject the JSON object of which the command is to be created. + * @param dittoHeaders the headers of the command. + * @return the command. + * @throws NullPointerException if {@code jsonObject} is {@code null}. + * @throws org.eclipse.ditto.json.JsonParseException if the passed in {@code jsonObject} was not in the expected + * format. + */ + public static AggregateThingsMetrics fromJson(final JsonObject jsonObject, final DittoHeaders dittoHeaders) { + return new CommandJsonDeserializer(TYPE, jsonObject).deserialize(() -> { + final String metricName = jsonObject.getValue(METRIC_NAME).orElseThrow(getJsonMissingFieldExceptionSupplier(METRIC_NAME.getPointer().toString(), jsonObject)); + final JsonObject extractedGroupingBy = jsonObject.getValue(GROUPING_BY).orElseThrow(getJsonMissingFieldExceptionSupplier(GROUPING_BY.getPointer().toString(), jsonObject)); + final HashMap groupingBy = new HashMap<>(); + extractedGroupingBy.forEach(jf -> groupingBy.put(jf.getKey().toString(), jf.getValue().asString())); + + final JsonObject extractedFilter = jsonObject.getValue(JSON_FILTER).orElseThrow(getJsonMissingFieldExceptionSupplier(JSON_FILTER.getPointer().toString(), jsonObject)); + final HashMap namedFiltersMap = new HashMap<>(); + extractedFilter.forEach(jf -> namedFiltersMap.put(jf.getKey().toString(), jf.getValue().asString())); + + final Set extractedNamespaces = jsonObject.getValue(NAMESPACES) + .map(jsonValues -> jsonValues.stream() + .filter(JsonValue::isString) + .map(JsonValue::asString) + .collect(Collectors.toSet())) + .orElse(Collections.emptySet()); + + return new AggregateThingsMetrics(metricName, groupingBy, namedFiltersMap, extractedNamespaces, dittoHeaders); + }); + } + public String getMetricName() { return metricName; } @@ -56,14 +168,53 @@ public Map getNamedFilters() { } @Override - public DittoHeaders getDittoHeaders() { - return dittoHeaders; + protected void appendPayload(final JsonObjectBuilder jsonObjectBuilder, final JsonSchemaVersion schemaVersion, + final Predicate thePredicate) { + + final Predicate predicate = schemaVersion.and(thePredicate); + jsonObjectBuilder.set(METRIC_NAME, metricName, predicate); + final JsonObjectBuilder groupingBy = JsonFactory.newObjectBuilder(); + this.groupingBy.forEach(groupingBy::set); + jsonObjectBuilder.set(GROUPING_BY, groupingBy.build(), predicate); + final JsonObjectBuilder jsonFields = JsonFactory.newObjectBuilder(); + namedFilters.forEach(jsonFields::set); + jsonObjectBuilder.set(NAMED_FILTERS, jsonFields.build(), predicate); + final JsonArray array = + JsonFactory.newArrayBuilder(namespaces.stream().map(JsonFactory::newValue).collect( + Collectors.toSet())).build(); + jsonObjectBuilder.set(NAMESPACES, array, predicate); + } public Set getNamespaces() { return namespaces; } + @Override + public String getTypePrefix() { + return TYPE_PREFIX; + } + + @Override + public Category getCategory() { + return Category.STREAM; + } + + @Override + public AggregateThingsMetrics setDittoHeaders(final DittoHeaders dittoHeaders) { + return of(getMetricName(), getGroupingBy(), getNamedFilters(), getNamespaces(), dittoHeaders); + } + + @Override + public JsonPointer getResourcePath() { + return JsonPointer.empty(); + } + + @Override + public String getResourceType() { + return RESOURCE_TYPE; + } + @Override public boolean equals(final Object o) { if (this == o) return true; @@ -91,4 +242,7 @@ public String toString() { ", namespaces=" + namespaces + '}'; } + private static Supplier getJsonMissingFieldExceptionSupplier(final String field, final JsonObject jsonObject) { + return () -> JsonMissingFieldException.newBuilder().fieldName(field).description(jsonObject.asString()).build(); + } } diff --git a/thingsearch/model/src/main/java/org/eclipse/ditto/thingsearch/model/signals/commands/query/AggregateThingsMetricsResponse.java b/thingsearch/model/src/main/java/org/eclipse/ditto/thingsearch/model/signals/commands/query/AggregateThingsMetricsResponse.java index 32b4e321e0..a5c8987f48 100644 --- a/thingsearch/model/src/main/java/org/eclipse/ditto/thingsearch/model/signals/commands/query/AggregateThingsMetricsResponse.java +++ b/thingsearch/model/src/main/java/org/eclipse/ditto/thingsearch/model/signals/commands/query/AggregateThingsMetricsResponse.java @@ -18,104 +18,237 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.function.Predicate; import java.util.function.Supplier; import java.util.stream.Collectors; +import org.eclipse.ditto.base.model.common.HttpStatus; import org.eclipse.ditto.base.model.headers.DittoHeaders; -import org.eclipse.ditto.base.model.headers.WithDittoHeaders; +import org.eclipse.ditto.base.model.json.FieldType; +import org.eclipse.ditto.base.model.json.JsonParsableCommandResponse; +import org.eclipse.ditto.base.model.json.JsonSchemaVersion; +import org.eclipse.ditto.base.model.signals.commands.AbstractCommandResponse; +import org.eclipse.ditto.json.JsonArray; +import org.eclipse.ditto.json.JsonArrayBuilder; +import org.eclipse.ditto.json.JsonFactory; +import org.eclipse.ditto.json.JsonField; +import org.eclipse.ditto.json.JsonFieldDefinition; import org.eclipse.ditto.json.JsonMissingFieldException; import org.eclipse.ditto.json.JsonObject; +import org.eclipse.ditto.json.JsonObjectBuilder; import org.eclipse.ditto.json.JsonPointer; +import org.eclipse.ditto.json.JsonValue; + +/** + * A response to an {@link AggregateThingsMetrics} command. + * Contains the original aggregation as returned by the database as well as fields initialized by that aggregation + * (grouped by and result) + * The result contains the returned values for each filter defined in a metric. + * The groupedBy contains the values that the result was grouped by. + */ +@JsonParsableCommandResponse(type = AggregateThingsMetricsResponse.RESOURCE_TYPE) +public final class AggregateThingsMetricsResponse extends AbstractCommandResponse { + + public static final String NAME = "things-metrics"; + public static final String RESOURCE_TYPE = "aggregation." + TYPE_QUALIFIER; + private static final String TYPE_PREFIX = RESOURCE_TYPE + ":"; + private static final String TYPE = TYPE_PREFIX + NAME; + + static final JsonFieldDefinition METRIC_NAME = + JsonFactory.newStringFieldDefinition("metric-name", FieldType.REGULAR, + JsonSchemaVersion.V_2); + static final JsonFieldDefinition DITTO_HEADERS = + JsonFactory.newJsonObjectFieldDefinition("ditto-headers", FieldType.REGULAR, + JsonSchemaVersion.V_2); + static final JsonFieldDefinition FILTERS_NAMES = + JsonFactory.newJsonArrayFieldDefinition("filters-names", FieldType.REGULAR, + JsonSchemaVersion.V_2); + static final JsonFieldDefinition AGGREGATION = + JsonFactory.newJsonObjectFieldDefinition("aggregation", FieldType.REGULAR, + JsonSchemaVersion.V_2); -public class AggregateThingsMetricsResponse implements WithDittoHeaders { - - private final Map groupedBy; - - private final Map result; - + private final String metricName; private final DittoHeaders dittoHeaders; + private final Set filterNames; private final JsonObject aggregation; - private final String metricName; - private AggregateThingsMetricsResponse(final JsonObject aggregation, final DittoHeaders dittoHeaders, - final String metricName, final Set filterNames) { - this.aggregation = aggregation; - this.dittoHeaders = dittoHeaders; + + private AggregateThingsMetricsResponse(final String metricName, final DittoHeaders dittoHeaders, + final Set filterNames, final JsonObject aggregation) { + super(TYPE, HttpStatus.OK, dittoHeaders); this.metricName = metricName; - groupedBy = aggregation.getValue("_id") - .map(json -> { - if (json.isObject()) { - return json.asObject().stream() - .collect(Collectors.toMap(o -> o.getKey().toString(), - o1 -> o1.getValue().formatAsString())); - } else { - return new HashMap(); - } - } - ) - .orElse(new HashMap<>()); - result = filterNames.stream().filter(aggregation::contains).collect(Collectors.toMap(key -> - key, key -> aggregation.getValue(JsonPointer.of(key)) - .orElseThrow(getJsonMissingFieldExceptionSupplier(key)) - .asLong())); - // value should always be a number as it will be used for the gauge value in the metrics + this.dittoHeaders = DittoHeaders.of(dittoHeaders); + this.filterNames = filterNames; + this.aggregation = aggregation; } + /** + * Creates a new {@link AggregateThingsMetricsResponse} instance. + * + * @param aggregation the aggregation result. + * @param aggregateThingsMetrics the command that was executed. + * @return the AggregateThingsMetricsResponse instance. + */ public static AggregateThingsMetricsResponse of(final JsonObject aggregation, final AggregateThingsMetrics aggregateThingsMetrics) { - return of(aggregation, aggregateThingsMetrics.getDittoHeaders(), aggregateThingsMetrics.getMetricName(), aggregateThingsMetrics.getNamedFilters().keySet()); + return of(aggregation, aggregateThingsMetrics.getDittoHeaders(), aggregateThingsMetrics.getMetricName(), + aggregateThingsMetrics.getNamedFilters().keySet()); } + /** + * Creates a new {@link AggregateThingsMetricsResponse} instance. + * + * @param aggregation the aggregation result. + * @param dittoHeaders the headers to use for the response. + * @param metricName the name of the metric. + * @param filterNames the names of the filters. + * @return the AggregateThingsMetricsResponse instance. + */ public static AggregateThingsMetricsResponse of(final JsonObject aggregation, final DittoHeaders dittoHeaders, final String metricName, final Set filterNames) { - return new AggregateThingsMetricsResponse(aggregation, dittoHeaders, metricName, filterNames); + return new AggregateThingsMetricsResponse(metricName, dittoHeaders, filterNames, aggregation); + } + + /** + * Creates a new {@code AggregateThingsMetricsResponse} from a JSON string. + * + * @param jsonString the JSON string of which the command is to be created. + * @param dittoHeaders the headers of the command. + * @return the command. + * @throws NullPointerException if {@code jsonString} is {@code null}. + * @throws IllegalArgumentException if {@code jsonString} is empty. + * @throws org.eclipse.ditto.json.JsonParseException if the passed in {@code jsonString} was not in the expected + * format. + */ + public static AggregateThingsMetricsResponse fromJson(final String jsonString, final DittoHeaders dittoHeaders) { + return fromJson(JsonFactory.newObject(jsonString), dittoHeaders); + } + + /** + * Creates a new {@code AggregateThingsMetricsResponse} from a JSON object. + * + * @param jsonObject the JSON object of which the command is to be created. + * @param dittoHeaders the headers of the command. + * @return the command. + * @throws NullPointerException if {@code jsonObject} is {@code null}. + * @throws org.eclipse.ditto.json.JsonParseException if the passed in {@code jsonObject} was not in the expected + * format. + */ + public static AggregateThingsMetricsResponse fromJson(final JsonObject jsonObject, + final DittoHeaders dittoHeaders) { + + final JsonObject aggregation = jsonObject.getValue(AGGREGATION) + .orElseThrow(getJsonMissingFieldExceptionSupplier(AGGREGATION.getPointer().toString())); + final String metricName = jsonObject.getValue(METRIC_NAME) + .orElseThrow(getJsonMissingFieldExceptionSupplier(METRIC_NAME.getPointer().toString())); + final JsonArray filterNames = jsonObject.getValue(FILTERS_NAMES) + .orElseThrow(getJsonMissingFieldExceptionSupplier(FILTERS_NAMES.getPointer().toString())); + Set filters = + filterNames.stream().map(JsonValue::formatAsString).collect(Collectors.toSet()); + return AggregateThingsMetricsResponse.of(aggregation, dittoHeaders, metricName, filters); + } + + @Override + public AggregateThingsMetricsResponse setDittoHeaders(final DittoHeaders dittoHeaders) { + return AggregateThingsMetricsResponse.of(aggregation, dittoHeaders, metricName, filterNames); } @Override - public DittoHeaders getDittoHeaders() { - return dittoHeaders; + public JsonPointer getResourcePath() { + return JsonPointer.empty(); } + @Override + public String getResourceType() { + return RESOURCE_TYPE; + } + + + @Override + protected void appendPayload(final JsonObjectBuilder jsonObjectBuilder, final JsonSchemaVersion schemaVersion, + final Predicate thePredicate) { + + final Predicate predicate = schemaVersion.and(thePredicate); + jsonObjectBuilder.set(METRIC_NAME, metricName, predicate); + jsonObjectBuilder.set(DITTO_HEADERS, dittoHeaders.toJson(), predicate); + final JsonArrayBuilder filterNamesBuilder = JsonFactory.newArrayBuilder(); + filterNames.forEach(filterNamesBuilder::add); + jsonObjectBuilder.set(FILTERS_NAMES, filterNamesBuilder.build(), predicate); + jsonObjectBuilder.set(AGGREGATION, aggregation, predicate); + } + + /** + * Returns the grouping by values by witch the result was grouped. + * + * @return the groupedBy of the response. + */ public Map getGroupedBy() { - return groupedBy; + return aggregation.getValue("_id") + .map(json -> { + if (json.isObject()) { + return json.asObject().stream() + .collect(Collectors.toMap(o -> o.getKey().toString(), + o1 -> o1.getValue().formatAsString())); + } else { + return new HashMap(); + } + } + ) + .orElse(new HashMap<>()); } + /** + * Returns the values for each filter defined in the metric + * + * @return the result of the aggregation. + */ public Map getResult() { - return result; + return extractFiltersResults(aggregation, filterNames); } + /** + * Returns the metric name. + * + * @return the metric name. + */ public String getMetricName() { return metricName; } - private Supplier getJsonMissingFieldExceptionSupplier(String field) { - return () -> JsonMissingFieldException.newBuilder().fieldName(field).build(); - } - @Override public boolean equals(final Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; final AggregateThingsMetricsResponse response = (AggregateThingsMetricsResponse) o; - return Objects.equals(groupedBy, response.groupedBy) && - Objects.equals(result, response.result) && + return Objects.equals(metricName, response.metricName) && Objects.equals(dittoHeaders, response.dittoHeaders) && - Objects.equals(aggregation, response.aggregation) && - Objects.equals(metricName, response.metricName); + Objects.equals(filterNames, response.filterNames) && + Objects.equals(aggregation, response.aggregation); + } @Override public int hashCode() { - return Objects.hash(groupedBy, result, dittoHeaders, aggregation, metricName); + return Objects.hash(super.hashCode(), metricName, dittoHeaders, filterNames, aggregation); } @Override public String toString() { return "AggregateThingsMetricsResponse{" + - "groupedBy=" + groupedBy + - ", result=" + result + + "metricName='" + metricName + '\'' + ", dittoHeaders=" + dittoHeaders + + ", filterNames=" + filterNames + ", aggregation=" + aggregation + - ", metricName='" + metricName + '\'' + '}'; } + + private Map extractFiltersResults(final JsonObject aggregation, final Set filterNames) { + return filterNames.stream().filter(aggregation::contains).collect(Collectors.toMap(key -> + key, key -> aggregation.getValue(JsonPointer.of(key)) + .orElseThrow(getJsonMissingFieldExceptionSupplier(key)) + .asLong())); + } + + private static Supplier getJsonMissingFieldExceptionSupplier(final String field) { + return () -> JsonMissingFieldException.newBuilder().fieldName(field).build(); + } } diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/CustomSearchMetricConfig.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/CustomAggregationMetricConfig.java similarity index 99% rename from thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/CustomSearchMetricConfig.java rename to thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/CustomAggregationMetricConfig.java index 0e6f2c6df9..77966f8c23 100644 --- a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/CustomSearchMetricConfig.java +++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/CustomAggregationMetricConfig.java @@ -22,7 +22,7 @@ /** * Provides the configuration settings for a single custom search metric. */ -public interface CustomSearchMetricConfig { +public interface CustomAggregationMetricConfig { /** diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultCustomSearchMetricConfig.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultCustomAggregationMetricConfig.java similarity index 96% rename from thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultCustomSearchMetricConfig.java rename to thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultCustomAggregationMetricConfig.java index 5033fea6f6..eb09aa61ab 100644 --- a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultCustomSearchMetricConfig.java +++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultCustomAggregationMetricConfig.java @@ -35,7 +35,7 @@ import com.typesafe.config.ConfigFactory; @Immutable -public final class DefaultCustomSearchMetricConfig implements CustomSearchMetricConfig { +public final class DefaultCustomAggregationMetricConfig implements CustomAggregationMetricConfig { private final String metricName; private final boolean enabled; @@ -45,7 +45,7 @@ public final class DefaultCustomSearchMetricConfig implements CustomSearchMetric private final Map tags; private final List filterConfigs; - private DefaultCustomSearchMetricConfig(final String key, final ConfigWithFallback configWithFallback) { + private DefaultCustomAggregationMetricConfig(final String key, final ConfigWithFallback configWithFallback) { this.metricName = key; enabled = configWithFallback.getBoolean(CustomSearchMetricConfigValue.ENABLED.getConfigPath()); scrapeInterval = configWithFallback.getDuration(CustomSearchMetricConfigValue.SCRAPE_INTERVAL.getConfigPath()); @@ -72,8 +72,8 @@ private DefaultCustomSearchMetricConfig(final String key, final ConfigWithFallba validateConfig(); } - public static DefaultCustomSearchMetricConfig of(final String key, final Config config) { - return new DefaultCustomSearchMetricConfig(key, + public static DefaultCustomAggregationMetricConfig of(final String key, final Config config) { + return new DefaultCustomAggregationMetricConfig(key, ConfigWithFallback.newInstance(config, CustomSearchMetricConfigValue.values())); } @@ -185,7 +185,7 @@ private boolean isPlaceHolder(final String value) { public boolean equals(final Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - final DefaultCustomSearchMetricConfig that = (DefaultCustomSearchMetricConfig) o; + final DefaultCustomAggregationMetricConfig that = (DefaultCustomAggregationMetricConfig) o; return enabled == that.enabled && Objects.equals(metricName, that.metricName) && Objects.equals(scrapeInterval, that.scrapeInterval) && Objects.equals(namespaces, that.namespaces) && Objects.equals(groupBy, that.groupBy) && diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultOperatorMetricsConfig.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultOperatorMetricsConfig.java index a5389dc635..1f0c9a5f5c 100644 --- a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultOperatorMetricsConfig.java +++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultOperatorMetricsConfig.java @@ -50,15 +50,15 @@ public final class DefaultOperatorMetricsConfig implements OperatorMetricsConfig private final boolean enabled; private final Duration scrapeInterval; private final Map customMetricConfigurations; - private final Map customSearchMetricConfigs; + private final Map customAggregationMetricConfigs; private DefaultOperatorMetricsConfig(final ConfigWithFallback updaterScopedConfig) { enabled = updaterScopedConfig.getBoolean(OperatorMetricsConfigValue.ENABLED.getConfigPath()); scrapeInterval = updaterScopedConfig.getNonNegativeDurationOrThrow(OperatorMetricsConfigValue.SCRAPE_INTERVAL); customMetricConfigurations = loadCustomMetricConfigurations(updaterScopedConfig, OperatorMetricsConfigValue.CUSTOM_METRICS); - customSearchMetricConfigs = loadCustomSearchMetricConfigurations(updaterScopedConfig, - OperatorMetricsConfigValue.CUSTOM_SEARCH_METRICS); + customAggregationMetricConfigs = loadCustomAggregatedMetricConfigurations(updaterScopedConfig, + OperatorMetricsConfigValue.CUSTOM_AGGREGATION_METRIC); } /** @@ -81,12 +81,12 @@ private static Map loadCustomMetricConfigurations(fi return customMetricsConfig.entrySet().stream().collect(CustomMetricConfigCollector.toMap()); } - private Map loadCustomSearchMetricConfigurations( + private Map loadCustomAggregatedMetricConfigurations( final ConfigWithFallback config, final KnownConfigValue configValue) { - final ConfigObject customSearchMetricsConfig = config.getObject(configValue.getConfigPath()); + final ConfigObject customAggregatedMetricsConfig = config.getObject(configValue.getConfigPath()); - return customSearchMetricsConfig.entrySet().stream().collect(CustomSearchMetricConfigCollector.toMap()); + return customAggregatedMetricsConfig.entrySet().stream().collect(CustomAggregatedMetricConfigCollector.toMap()); } @Override @@ -132,8 +132,8 @@ public Map getCustomMetricConfigurations() { } @Override - public Map getCustomSearchMetricConfigs() { - return customSearchMetricConfigs; + public Map getCustomAggregationMetricConfigs() { + return customAggregationMetricConfigs; } private static class CustomMetricConfigCollector @@ -172,32 +172,32 @@ public Set characteristics() { } } - private static class CustomSearchMetricConfigCollector implements - Collector, Map, Map> { + private static class CustomAggregatedMetricConfigCollector implements + Collector, Map, Map> { - private static DefaultOperatorMetricsConfig.CustomSearchMetricConfigCollector toMap() { - return new DefaultOperatorMetricsConfig.CustomSearchMetricConfigCollector(); + private static CustomAggregatedMetricConfigCollector toMap() { + return new CustomAggregatedMetricConfigCollector(); } @Override - public Supplier> supplier() { + public Supplier> supplier() { return LinkedHashMap::new; } @Override - public BiConsumer, Map.Entry> accumulator() { + public BiConsumer, Map.Entry> accumulator() { return (map, entry) -> map.put(entry.getKey(), - DefaultCustomSearchMetricConfig.of(entry.getKey(), ConfigFactory.empty().withFallback(entry.getValue()))); + DefaultCustomAggregationMetricConfig.of(entry.getKey(), ConfigFactory.empty().withFallback(entry.getValue()))); } @Override - public BinaryOperator> combiner() { + public BinaryOperator> combiner() { return (left, right) -> Stream.concat(left.entrySet().stream(), right.entrySet().stream()) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } @Override - public Function, Map> finisher() { + public Function, Map> finisher() { return map -> Collections.unmodifiableMap(new LinkedHashMap<>(map)); } diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/OperatorMetricsConfig.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/OperatorMetricsConfig.java index 99386ee00c..49b1d6f726 100644 --- a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/OperatorMetricsConfig.java +++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/OperatorMetricsConfig.java @@ -48,11 +48,11 @@ public interface OperatorMetricsConfig { Map getCustomMetricConfigurations(); /** - * Returns all registered custom search metrics with the key being the metric name to use. + * Returns all registered custom aggregation metrics with the key being the metric name to use. * - * @return the registered custom search metrics. + * @return the registered custom aggregation metrics. */ - Map getCustomSearchMetricConfigs(); + Map getCustomAggregationMetricConfigs(); /** * An enumeration of the known config path expressions and their associated default values for @@ -76,9 +76,9 @@ enum OperatorMetricsConfigValue implements KnownConfigValue { CUSTOM_METRICS("custom-metrics", Collections.emptyMap()), /** - * All registered custom search metrics with the key being the metric name to use. + * All registered custom aggregation metrics with the key being the metric name to use. */ - CUSTOM_SEARCH_METRICS("custom-search-metrics", Collections.emptyMap()); + CUSTOM_AGGREGATION_METRIC("custom-aggregation-metrics", Collections.emptyMap()); private final String path; private final Object defaultValue; diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/MongoThingsAggregationPersistence.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/MongoThingsAggregationPersistence.java index f31bd13faf..2fd22ae577 100644 --- a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/MongoThingsAggregationPersistence.java +++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/MongoThingsAggregationPersistence.java @@ -46,7 +46,10 @@ import com.mongodb.reactivestreams.client.MongoCollection; import com.mongodb.reactivestreams.client.MongoDatabase; -public class MongoThingsAggregationPersistence implements ThingsAggregationPersistence { +/** + * Persistence implementation for aggregating things. + */ +public final class MongoThingsAggregationPersistence implements ThingsAggregationPersistence { private final MongoCollection collection; @@ -59,6 +62,8 @@ public class MongoThingsAggregationPersistence implements ThingsAggregationPersi * Initializes the things search persistence with a passed in {@code persistence}. * * @param mongoClient the mongoDB persistence wrapper. + * @param mongoHintsByNamespace the mongo hints by namespace. + * @param simpleFieldMappings the simple field mappings. * @param persistenceConfig the search persistence configuration. * @param log the logger. */ @@ -78,7 +83,6 @@ private MongoThingsAggregationPersistence(final DittoMongoClient mongoClient, this.log = log; maxQueryTime = mongoClient.getDittoSettings().getMaxQueryTime(); hints = mongoHintsByNamespace.map(MongoHints::byNamespace).orElse(MongoHints.empty()); - log.info("Aggregation readConcern=<{}> readPreference=<{}>", readConcern, readPreference); } public static ThingsAggregationPersistence of(final DittoMongoClient mongoClient, @@ -110,7 +114,7 @@ public Source aggregateThings(final AggregateThingsMetrics ag .collect(Collectors.toList()); final Bson group = group(new Document(groupingBy), accumulatorFields); aggregatePipeline.add(group); - log.info("aggregatePipeline: {}", // TODO debug + log.debug("aggregation Pipeline: {}", aggregatePipeline.stream().map(bson -> bson.toBsonDocument().toJson()).collect( Collectors.toList())); // Execute the aggregation pipeline @@ -118,6 +122,6 @@ public Source aggregateThings(final AggregateThingsMetrics ag .hint(hints.getHint(aggregateCommand.getNamespaces()) .orElse(null)) .allowDiskUse(true) - .maxTime(maxQueryTime.toMillis(), TimeUnit.MILLISECONDS)); + .maxTime(maxQueryTime.toMillis(), TimeUnit.MILLISECONDS)).log("aggregateThings"); } } diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/criteria/visitors/CreateBsonAggregationPredicateVisitor.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/criteria/visitors/CreateBsonAggregationPredicateVisitor.java index 55198aae36..44705378cc 100644 --- a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/criteria/visitors/CreateBsonAggregationPredicateVisitor.java +++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/criteria/visitors/CreateBsonAggregationPredicateVisitor.java @@ -35,7 +35,7 @@ import org.eclipse.ditto.rql.query.criteria.visitors.PredicateVisitor; /** - * Creates Bson of a predicate. + * Creates Aggregation Bson of a predicate. */ public class CreateBsonAggregationPredicateVisitor implements PredicateVisitor> { @@ -64,12 +64,12 @@ public static CreateBsonAggregationPredicateVisitor getInstance() { } /** - * Creates a new instance of {@code CreateBsonPredicateVisitor} with additional custom placeholder resolvers. + * Creates a new instance of {@code CreateBsonAggregationPredicateVisitor} with additional custom placeholder resolvers. * * @param additionalPlaceholderResolvers the additional {@code PlaceholderResolver} to use for resolving * placeholders in RQL predicates. * @return the created instance. - * @since 2.3.0 + * @since 3.6.0 */ public static CreateBsonAggregationPredicateVisitor createInstance( final PlaceholderResolver... additionalPlaceholderResolvers) { @@ -77,12 +77,12 @@ public static CreateBsonAggregationPredicateVisitor createInstance( } /** - * Creates a new instance of {@code CreateBsonPredicateVisitor} with additional custom placeholder resolvers. + * Creates a new instance of {@code CreateBsonAggregationPredicateVisitor} with additional custom placeholder resolvers. * * @param additionalPlaceholderResolvers the additional {@code PlaceholderResolver} to use for resolving * placeholders in RQL predicates. * @return the created instance. - * @since 2.3.0 + * @since 3.6.0 */ public static CreateBsonAggregationPredicateVisitor createInstance( final Collection> additionalPlaceholderResolvers) { diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/criteria/visitors/CreateBsonAggregationVisitor.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/criteria/visitors/CreateBsonAggregationVisitor.java index 040903d49b..6348060414 100644 --- a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/criteria/visitors/CreateBsonAggregationVisitor.java +++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/criteria/visitors/CreateBsonAggregationVisitor.java @@ -26,7 +26,7 @@ import org.eclipse.ditto.thingsearch.service.persistence.read.expression.visitors.GetFilterBsonVisitor; /** - * Creates the Bson object used for querying. + * Creates the Aggregation Bson object used for the aggregation. */ public class CreateBsonAggregationVisitor extends CreateBsonVisitor { @@ -54,7 +54,7 @@ public Bson visitField(final FilterFieldExpression fieldExpression, final Predic PlaceholderFactory.newPlaceholderResolver(TIME_PLACEHOLDER, new Object()) ) ); - return GetFilterBsonVisitor.apply(fieldExpression, predicateCreator, null); + return GetFilterBsonVisitor.apply(fieldExpression, predicateCreator, authorizationSubjectIds); } diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/criteria/visitors/CreateBsonVisitor.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/criteria/visitors/CreateBsonVisitor.java index 7933197866..e5ecdede62 100644 --- a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/criteria/visitors/CreateBsonVisitor.java +++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/read/criteria/visitors/CreateBsonVisitor.java @@ -42,7 +42,7 @@ public class CreateBsonVisitor implements CriteriaVisitor { private static final TimePlaceholder TIME_PLACEHOLDER = TimePlaceholder.getInstance(); @Nullable - private final List authorizationSubjectIds; + protected final List authorizationSubjectIds; CreateBsonVisitor(@Nullable final List authorizationSubjectIds) { this.authorizationSubjectIds = authorizationSubjectIds; diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/placeholders/GroupByPlaceholderResolver.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/placeholders/GroupByPlaceholderResolver.java index a631a365b7..ac8c61be37 100644 --- a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/placeholders/GroupByPlaceholderResolver.java +++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/placeholders/GroupByPlaceholderResolver.java @@ -21,7 +21,11 @@ import org.eclipse.ditto.placeholders.PlaceholderResolver; -public class GroupByPlaceholderResolver implements PlaceholderResolver> { +/** + * Placeholder resolver for group-by. + * Resolves the group-by placeholders from the given source. + */ +public final class GroupByPlaceholderResolver implements PlaceholderResolver> { public static final String PREFIX = "group-by"; private final List supportedNames; diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/placeholders/InlinePlaceholderResolver.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/placeholders/InlinePlaceholderResolver.java index 4376a1aaa6..d5643133c6 100644 --- a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/placeholders/InlinePlaceholderResolver.java +++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/placeholders/InlinePlaceholderResolver.java @@ -20,7 +20,11 @@ import org.eclipse.ditto.placeholders.PlaceholderResolver; -public class InlinePlaceholderResolver implements PlaceholderResolver> { +/** + * Placeholder resolver for inline. + * Resolves the inline placeholders from the given source. + */ +public final class InlinePlaceholderResolver implements PlaceholderResolver> { public static final String PREFIX = "inline"; private final Map source; diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/AggregationThingsMetricsActor.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/AggregateThingsMetricsActor.java similarity index 78% rename from thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/AggregationThingsMetricsActor.java rename to thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/AggregateThingsMetricsActor.java index 9c3249c3b2..2db9544518 100644 --- a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/AggregationThingsMetricsActor.java +++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/AggregateThingsMetricsActor.java @@ -24,6 +24,7 @@ import org.apache.pekko.japi.pf.ReceiveBuilder; import org.apache.pekko.pattern.Patterns; import org.apache.pekko.stream.Graph; +import org.apache.pekko.stream.Materializer; import org.apache.pekko.stream.SourceShape; import org.apache.pekko.stream.SystemMaterializer; import org.apache.pekko.stream.javadsl.Flow; @@ -48,25 +49,30 @@ /** * Actor handling custom metrics aggregations {@link org.eclipse.ditto.thingsearch.model.signals.commands.query.AggregateThingsMetrics}. */ -public class AggregationThingsMetricsActor - extends AbstractActor { +public final class AggregateThingsMetricsActor extends AbstractActor { /** * The name of this actor in the system. */ - static final String ACTOR_NAME = ThingsAggregationConstants.AGGREGATE_ACTOR_NAME; + public static final String ACTOR_NAME = "aggregateThingsMetrics"; + /** + * Name of the pekko cluster role. + */ + public static final String CLUSTER_ROLE = "search"; private static final String TRACING_THINGS_AGGREGATION = "aggregate_things_metrics"; private final ThreadSafeDittoLoggingAdapter log; private final ThingsAggregationPersistence thingsAggregationPersistence; + private final Materializer materializer; - private AggregationThingsMetricsActor(final ThingsAggregationPersistence aggregationPersistence) { + private AggregateThingsMetricsActor(final ThingsAggregationPersistence aggregationPersistence) { log = DittoLoggerFactory.getThreadSafeDittoLoggingAdapter(this); - this.thingsAggregationPersistence = aggregationPersistence; + thingsAggregationPersistence = aggregationPersistence; + materializer = SystemMaterializer.get(getContext().getSystem()).materializer(); } public static Props props(final ThingsAggregationPersistence aggregationPersistence) { - return Props.create(AggregationThingsMetricsActor.class,aggregationPersistence); + return Props.create(AggregateThingsMetricsActor.class, aggregationPersistence); } @Override @@ -79,11 +85,12 @@ public Receive createReceive() { .build(); } - private void aggregate(AggregateThingsMetrics aggregateThingsMetrics) { + private void aggregate(final AggregateThingsMetrics aggregateThingsMetrics) { log.debug("Received aggregate command for {}", aggregateThingsMetrics); final StartedTimer aggregationTimer = startNewTimer(aggregateThingsMetrics); final Source source = - DittoJsonException.wrapJsonRuntimeException(aggregateThingsMetrics, aggregateThingsMetrics.getDittoHeaders(), + DittoJsonException.wrapJsonRuntimeException(aggregateThingsMetrics, + aggregateThingsMetrics.getDittoHeaders(), (command, headers) -> thingsAggregationPersistence.aggregateThings(command)); final Source aggregationResult = processAggregationPersistenceResult(source, aggregateThingsMetrics.getDittoHeaders()) @@ -93,23 +100,22 @@ private void aggregate(AggregateThingsMetrics aggregateThingsMetrics) { final Source replySourceWithErrorHandling = aggregationResult.via(stopTimerAndHandleError(aggregationTimer, aggregateThingsMetrics)); - replySourceWithErrorHandling.runWith(Sink.foreach(elem -> { - Patterns.pipe(CompletableFuture.completedFuture(elem), getContext().dispatcher()).to(sender); - - }), SystemMaterializer.get(getContext().getSystem()).materializer()); + replySourceWithErrorHandling.runWith(Sink.foreach( + elem -> Patterns.pipe(CompletableFuture.completedFuture(elem), getContext().dispatcher()).to(sender)), + materializer); } -private Source processAggregationPersistenceResult(final Source source, - final DittoHeaders dittoHeaders) { + private Source processAggregationPersistenceResult(final Source source, + final DittoHeaders dittoHeaders) { - final Flow logAndFinishPersistenceSegmentFlow = - Flow.fromFunction(result -> { - log.withCorrelationId(dittoHeaders) - .debug("aggregation element: {}", result); - return result; - }); -return source.via(logAndFinishPersistenceSegmentFlow); -} + final Flow logAndFinishPersistenceSegmentFlow = + Flow.fromFunction(result -> { + log.withCorrelationId(dittoHeaders) + .debug("aggregation element: {}", result); + return result; + }); + return source.via(logAndFinishPersistenceSegmentFlow); + } private static StartedTimer startNewTimer(final WithDittoHeaders withDittoHeaders) { final StartedTimer startedTimer = DittoMetrics.timer(TRACING_THINGS_AGGREGATION) @@ -126,6 +132,7 @@ private static void stopTimer(final StartedTimer timer) { // it is okay if the timer was stopped. } } + private Flow stopTimerAndHandleError(final StartedTimer searchTimer, final WithDittoHeaders command) { return Flow.fromFunction( @@ -141,6 +148,7 @@ private Flow stopTimerAndHandleError(final StartedTimer .build() ); } + private DittoRuntimeException asDittoRuntimeException(final Throwable error, final WithDittoHeaders trigger) { return DittoRuntimeException.asDittoRuntimeException(error, t -> { log.error(error, "AggregateThingsMetricsActor failed to execute <{}>", trigger); diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/OperatorSearchMetricsProviderActor.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/OperatorSearchMetricsProviderActor.java index 10ff90fb32..826fc4af8b 100644 --- a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/OperatorSearchMetricsProviderActor.java +++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/OperatorSearchMetricsProviderActor.java @@ -12,7 +12,7 @@ */ package org.eclipse.ditto.thingsearch.service.starter.actors; -import static org.eclipse.ditto.thingsearch.service.starter.actors.ThingsAggregationConstants.CLUSTER_ROLE; +import static org.eclipse.ditto.thingsearch.service.starter.actors.AggregateThingsMetricsActor.CLUSTER_ROLE; import java.time.Duration; import java.util.Comparator; @@ -48,7 +48,7 @@ import org.eclipse.ditto.placeholders.PlaceholderResolver; import org.eclipse.ditto.thingsearch.model.signals.commands.query.AggregateThingsMetrics; import org.eclipse.ditto.thingsearch.model.signals.commands.query.AggregateThingsMetricsResponse; -import org.eclipse.ditto.thingsearch.service.common.config.CustomSearchMetricConfig; +import org.eclipse.ditto.thingsearch.service.common.config.CustomAggregationMetricConfig; import org.eclipse.ditto.thingsearch.service.common.config.OperatorMetricsConfig; import org.eclipse.ditto.thingsearch.service.common.config.SearchConfig; import org.eclipse.ditto.thingsearch.service.persistence.read.MongoThingsAggregationPersistence; @@ -74,7 +74,7 @@ public final class OperatorSearchMetricsProviderActor extends AbstractActorWithT private final DittoDiagnosticLoggingAdapter log = DittoLoggerFactory.getDiagnosticLoggingAdapter(this); private final ActorRef thingsAggregatorActorSingletonProxy; - private final Map customSearchMetricConfigMap; + private final Map customSearchMetricConfigMap; private final Map metricsGauges; private final Gauge customSearchMetricsGauge; private final Map>> inlinePlaceholderResolvers; @@ -82,12 +82,13 @@ public final class OperatorSearchMetricsProviderActor extends AbstractActorWithT @SuppressWarnings("unused") private OperatorSearchMetricsProviderActor(final SearchConfig searchConfig) { this.thingsAggregatorActorSingletonProxy = initializeAggregationThingsMetricsActor(searchConfig); - this.customSearchMetricConfigMap = searchConfig.getOperatorMetricsConfig().getCustomSearchMetricConfigs(); + this.customSearchMetricConfigMap = searchConfig.getOperatorMetricsConfig().getCustomAggregationMetricConfigs(); this.metricsGauges = new HashMap<>(); this.inlinePlaceholderResolvers = new HashMap<>(); this.customSearchMetricsGauge = KamonGauge.newGauge("custom-search-metrics-count-of-instruments"); this.customSearchMetricConfigMap.forEach((metricName, customSearchMetricConfig) -> { - initializeCustomMetricTimer(metricName, customSearchMetricConfig, getMaxConfiguredScrapeInterval(searchConfig.getOperatorMetricsConfig())); + initializeCustomMetricTimer(metricName, customSearchMetricConfig, + getMaxConfiguredScrapeInterval(searchConfig.getOperatorMetricsConfig())); // Initialize the inline resolvers here as they use a static source from config customSearchMetricConfig.getFilterConfigs() @@ -123,45 +124,48 @@ public Receive createReceive() { private ActorRef initializeAggregationThingsMetricsActor(final SearchConfig searchConfig) { final DittoMongoClient mongoDbClient = MongoClientExtension.get(getContext().system()).getSearchClient(); - final var props = AggregationThingsMetricsActor.props(MongoThingsAggregationPersistence.of(mongoDbClient, searchConfig, log)); + final var props = AggregateThingsMetricsActor.props( + MongoThingsAggregationPersistence.of(mongoDbClient, searchConfig, log)); final ActorRef aggregationThingsMetricsActorProxy = ClusterUtil .startSingletonProxy(getContext(), CLUSTER_ROLE, - ClusterUtil.startSingleton(getContext(), CLUSTER_ROLE, AggregationThingsMetricsActor.ACTOR_NAME, + ClusterUtil.startSingleton(getContext(), CLUSTER_ROLE, AggregateThingsMetricsActor.ACTOR_NAME, props)); - log.info("Started child actor <{}> with path <{}>.", AggregationThingsMetricsActor.ACTOR_NAME, + log.info("Started child actor <{}> with path <{}>.", AggregateThingsMetricsActor.ACTOR_NAME, aggregationThingsMetricsActorProxy); return aggregationThingsMetricsActorProxy; } private void handleGatheringMetrics(final GatherMetricsCommand gatherMetricsCommand) { final String metricName = gatherMetricsCommand.metricName(); - final CustomSearchMetricConfig config = gatherMetricsCommand.config(); + final CustomAggregationMetricConfig config = gatherMetricsCommand.config(); final DittoHeaders dittoHeaders = DittoHeaders.newBuilder() .correlationId("gather-search-metrics_" + metricName + "_" + UUID.randomUUID()) .build(); final Map namedFilters = config.getFilterConfigs().stream() - .collect(Collectors.toMap(CustomSearchMetricConfig.FilterConfig::getFilterName, - CustomSearchMetricConfig.FilterConfig::getFilter)); - AggregateThingsMetrics + .collect(Collectors.toMap(CustomAggregationMetricConfig.FilterConfig::getFilterName, + CustomAggregationMetricConfig.FilterConfig::getFilter)); + final AggregateThingsMetrics aggregateThingsMetrics = AggregateThingsMetrics.of(metricName, config.getGroupBy(), namedFilters, Set.of(config.getNamespaces().toArray(new String[0])), dittoHeaders); thingsAggregatorActorSingletonProxy.tell(aggregateThingsMetrics, getSelf()); } - private void handleAggregateThingsResponse(AggregateThingsMetricsResponse response) { - log.withCorrelationId(response).info("Received aggregate things response: {} thread: {}", //TODO debug + private void handleAggregateThingsResponse(final AggregateThingsMetricsResponse response) { + log.withCorrelationId(response).debug("Received aggregate things response: {} thread: {}", response, Thread.currentThread().getName()); final String metricName = response.getMetricName(); // record by filter name and tags response.getResult().forEach((filterName, value) -> { resolveTags(filterName, customSearchMetricConfigMap.get(metricName), response); - final CustomSearchMetricConfig customSearchMetricConfig = customSearchMetricConfigMap.get(metricName); - final TagSet tagSet = resolveTags(filterName, customSearchMetricConfig, response) + final CustomAggregationMetricConfig customAggregationMetricConfig = + customSearchMetricConfigMap.get(metricName); + final TagSet tagSet = resolveTags(filterName, customAggregationMetricConfig, response) .putTag(Tag.of("filter", filterName)); recordMetric(metricName, tagSet, value); - customSearchMetricsGauge.tag(Tag.of(METRIC_NAME, metricName)).set(Long.valueOf(metricsGauges.size()));; + customSearchMetricsGauge.tag(Tag.of(METRIC_NAME, metricName)).set(Long.valueOf(metricsGauges.size())); + ; }); } @@ -170,7 +174,7 @@ private void recordMetric(final String metricName, final TagSet tagSet, final Lo if (timestampedGauge == null) { final Gauge gauge = KamonGauge.newGauge(metricName) .tags(tagSet); - gauge.set(value); + gauge.set(value); return new TimestampedGauge(gauge); } else { return timestampedGauge.set(value); @@ -178,17 +182,21 @@ private void recordMetric(final String metricName, final TagSet tagSet, final Lo }); } - private TagSet resolveTags(final String filterName, final CustomSearchMetricConfig customSearchMetricConfig, + private TagSet resolveTags(final String filterName, + final CustomAggregationMetricConfig customAggregationMetricConfig, final AggregateThingsMetricsResponse response) { - return TagSet.ofTagCollection(customSearchMetricConfig.getTags().entrySet().stream().map(tagEntry-> { + return TagSet.ofTagCollection(customAggregationMetricConfig.getTags().entrySet().stream().map(tagEntry -> { if (!isPlaceHolder(tagEntry.getValue())) { return Tag.of(tagEntry.getKey(), tagEntry.getValue()); } else { final ExpressionResolver expressionResolver = PlaceholderFactory.newExpressionResolver(List.of( - new GroupByPlaceholderResolver(customSearchMetricConfig.getGroupBy().keySet(), response.getGroupedBy()) - , inlinePlaceholderResolvers.get(new FilterIdentifier(customSearchMetricConfig.getMetricName(), filterName)))); + new GroupByPlaceholderResolver(customAggregationMetricConfig.getGroupBy().keySet(), + response.getGroupedBy()) + , inlinePlaceholderResolvers.get( + new FilterIdentifier(customAggregationMetricConfig.getMetricName(), + filterName)))); return expressionResolver.resolve(tagEntry.getValue()) .findFirst() .map(resolvedValue -> Tag.of(tagEntry.getKey(), resolvedValue)) @@ -204,7 +212,8 @@ private void handleCleanupUnusedMetrics(final CleanupUnusedMetricsCommand cleanu while (iterator.hasNext()) { final Map.Entry next = iterator.next(); final long lastUpdated = next.getValue().getLastUpdated(); - final long unusedPeriod = getMaxConfiguredScrapeInterval(cleanupCommand.config()).multipliedBy(2).toMillis(); + final long unusedPeriod = + getMaxConfiguredScrapeInterval(cleanupCommand.config()).multipliedBy(2).toMillis(); final long expire = lastUpdated + unusedPeriod; log.debug("cleanup metrics: expired: {}, time left: {} lastUpdated: {} expire: {} currentTime: {}", currentTime > expire, expire - currentTime, lastUpdated, expire, currentTime); @@ -213,7 +222,7 @@ private void handleCleanupUnusedMetrics(final CleanupUnusedMetricsCommand cleanu // setting to zero as there is a bug in Kamon where the gauge is not removed and is still reported // https://github.com/kamon-io/Kamon/issues/566 next.getValue().set(0L); - if ( Kamon.gauge(metricName).remove(KamonTagSetConverter.getKamonTagSet(next.getValue().getTagSet()))) { + if (Kamon.gauge(metricName).remove(KamonTagSetConverter.getKamonTagSet(next.getValue().getTagSet()))) { log.debug("Removed custom search metric instrument: {} {}", metricName, next.getValue().getTagSet()); iterator.remove(); @@ -228,15 +237,16 @@ private void handleCleanupUnusedMetrics(final CleanupUnusedMetricsCommand cleanu private Duration getMaxConfiguredScrapeInterval(final OperatorMetricsConfig operatorMetricsConfig) { return Stream.concat(Stream.of(operatorMetricsConfig.getScrapeInterval()), - operatorMetricsConfig.getCustomSearchMetricConfigs().values().stream() - .map(CustomSearchMetricConfig::getScrapeInterval) + operatorMetricsConfig.getCustomAggregationMetricConfigs().values().stream() + .map(CustomAggregationMetricConfig::getScrapeInterval) .filter(Optional::isPresent) .map(Optional::get)) .max(Comparator.naturalOrder()) .orElse(operatorMetricsConfig.getScrapeInterval()); } - private void initializeCustomMetricTimer(final String metricName, final CustomSearchMetricConfig config, final Duration scrapeInterval ) { + private void initializeCustomMetricTimer(final String metricName, final CustomAggregationMetricConfig config, + final Duration scrapeInterval) { if (!config.isEnabled()) { log.info("Custom search metric Gauge for metric <{}> is DISABLED. Skipping init.", metricName); return; @@ -255,7 +265,8 @@ private void initializeCustomMetricTimer(final String metricName, final CustomSe private void initializeCustomMetricsCleanupTimer(final OperatorMetricsConfig operatorMetricsConfig) { final Duration interval = getMaxConfiguredScrapeInterval(operatorMetricsConfig); log.info("Initializing custom metric cleanup timer Interval <{}>", interval); - getTimers().startTimerAtFixedRate("cleanup-unused-metrics", new CleanupUnusedMetricsCommand(operatorMetricsConfig), + getTimers().startTimerAtFixedRate("cleanup-unused-metrics", + new CleanupUnusedMetricsCommand(operatorMetricsConfig), interval); } @@ -263,7 +274,7 @@ private boolean isPlaceHolder(final String value) { return value.startsWith("{{") && value.endsWith("}}"); } - private static class TimestampedGauge { + private final static class TimestampedGauge { private final Gauge gauge; private final Long timestamp; @@ -301,14 +312,14 @@ public int hashCode() { @Override public String toString() { - return "GageWithTimestamp{" + + return "TimestampedGauge{" + "gauge=" + gauge + ", timestamp=" + timestamp + '}'; } } - private record GatherMetricsCommand(String metricName, CustomSearchMetricConfig config) {} + private record GatherMetricsCommand(String metricName, CustomAggregationMetricConfig config) {} private record CleanupUnusedMetricsCommand(OperatorMetricsConfig config) {} diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/ThingsAggregationConstants.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/ThingsAggregationConstants.java deleted file mode 100644 index 66853ed044..0000000000 --- a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/ThingsAggregationConstants.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (c) 2024 Contributors to the Eclipse Foundation - * - * See the NOTICE file(s) distributed with this work for additional - * information regarding copyright ownership. - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Public License 2.0 which is available at - * http://www.eclipse.org/legal/epl-2.0 - * - * SPDX-License-Identifier: EPL-2.0 - * - */ -package org.eclipse.ditto.thingsearch.service.starter.actors; - -import javax.annotation.concurrent.Immutable; - -/** - * Constants for the Things Aggregate. - */ -@Immutable -public final class ThingsAggregationConstants { - - - /** - * Name of the pekko cluster role. - */ - public static final String CLUSTER_ROLE = "search"; - - private static final String PATH_DELIMITER = "/"; - - @SuppressWarnings("squid:S1075") - private static final String USER_PATH = "/user"; - - /** - * Name of the Aggregate actor - */ - public static final String AGGREGATE_ACTOR_NAME = "aggregateThingsMetrics"; - - /* - * Inhibit instantiation of this utility class. - */ - private ThingsAggregationConstants() { - // no-op - } -} diff --git a/thingsearch/service/src/main/resources/search-dev.conf b/thingsearch/service/src/main/resources/search-dev.conf index 9d927ae1a6..405632d405 100755 --- a/thingsearch/service/src/main/resources/search-dev.conf +++ b/thingsearch/service/src/main/resources/search-dev.conf @@ -25,7 +25,7 @@ ditto { } } } - custom-search-metrics { + custom-aggregation-metrics { online_status { enabled = false scrape-interval = 1m # override scrape interval, run every 20 minute @@ -41,7 +41,7 @@ ditto { "health" = "{{ inline:health }}" "hardcoded-tag" = "value" "location" = "{{ group-by:location | fn:default('missing location') }}" - "altitude" = "{{ group-by:isGateway }}" + "isGateway" = "{{ group-by:isGateway }}" } filters = { online_filter = { diff --git a/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultCustomSearchMetricConfigTest.java b/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultCustomAggregationMetricConfigTest.java similarity index 79% rename from thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultCustomSearchMetricConfigTest.java rename to thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultCustomAggregationMetricConfigTest.java index 6c69a5d132..dc39d9ab61 100644 --- a/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultCustomSearchMetricConfigTest.java +++ b/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultCustomAggregationMetricConfigTest.java @@ -28,7 +28,7 @@ import nl.jqno.equalsverifier.EqualsVerifier; -public class DefaultCustomSearchMetricConfigTest { +public class DefaultCustomAggregationMetricConfigTest { private static Config config; private static Config customSearchMetricTestConfig; @@ -44,35 +44,35 @@ public static void initTestFixture() { @Test public void testHashCodeAndEquals() { - EqualsVerifier.forClass(DefaultCustomSearchMetricConfig.class) + EqualsVerifier.forClass(DefaultCustomAggregationMetricConfig.class) .usingGetClass() .verify(); } @Test public void gettersReturnConfiguredValues() { - final DefaultCustomSearchMetricConfig underTest = - DefaultCustomSearchMetricConfig.of("online_status", + final DefaultCustomAggregationMetricConfig underTest = + DefaultCustomAggregationMetricConfig.of("online_status", customSearchMetricTestConfig.getConfig("online_status")); softly.assertThat(underTest.isEnabled()) - .as(CustomSearchMetricConfig.CustomSearchMetricConfigValue.ENABLED.getConfigPath()) + .as(CustomAggregationMetricConfig.CustomSearchMetricConfigValue.ENABLED.getConfigPath()) .isEqualTo(true); softly.assertThat(underTest.getScrapeInterval()) - .as(CustomSearchMetricConfig.CustomSearchMetricConfigValue.SCRAPE_INTERVAL.getConfigPath()) + .as(CustomAggregationMetricConfig.CustomSearchMetricConfigValue.SCRAPE_INTERVAL.getConfigPath()) .isEqualTo(Optional.ofNullable(customSearchMetricTestConfig.getDuration( "online_status.scrape-interval"))); softly.assertThat(underTest.getNamespaces()) - .as(CustomSearchMetricConfig.CustomSearchMetricConfigValue.NAMESPACES.getConfigPath()) + .as(CustomAggregationMetricConfig.CustomSearchMetricConfigValue.NAMESPACES.getConfigPath()) .containsExactlyInAnyOrder("org.eclipse.ditto.test.1", "org.eclipse.ditto.test.2"); softly.assertThat(underTest.getTags()) - .as(CustomSearchMetricConfig.CustomSearchMetricConfigValue.TAGS.getConfigPath()) + .as(CustomAggregationMetricConfig.CustomSearchMetricConfigValue.TAGS.getConfigPath()) .containsExactlyInAnyOrderEntriesOf( customSearchMetricTestConfig.getObject("online_status.tags") .unwrapped().entrySet().stream().collect( Collectors.toMap(Map.Entry::getKey, o -> o.getValue().toString()))); softly.assertThat(underTest.getFilterConfigs()) - .as(CustomSearchMetricConfig.CustomSearchMetricConfigValue.FILTERS.getConfigPath()) + .as(CustomAggregationMetricConfig.CustomSearchMetricConfigValue.FILTERS.getConfigPath()) .hasSize(2); softly.assertThat(underTest.getFilterConfigs().get(0).getFilterName()) .as("filter name") diff --git a/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/actors/AggregationThingsMetricsActorTest.java b/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/actors/AggregateThingsMetricsActorTest.java similarity index 70% rename from thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/actors/AggregationThingsMetricsActorTest.java rename to thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/actors/AggregateThingsMetricsActorTest.java index ecc475a64d..687f146768 100644 --- a/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/actors/AggregationThingsMetricsActorTest.java +++ b/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/starter/actors/AggregateThingsMetricsActorTest.java @@ -31,26 +31,23 @@ import org.apache.pekko.testkit.javadsl.TestKit; import org.bson.Document; import org.eclipse.ditto.base.model.headers.DittoHeaders; -import org.eclipse.ditto.internal.utils.tracing.DittoTracing; import org.eclipse.ditto.internal.utils.tracing.DittoTracingInitResource; -import org.eclipse.ditto.internal.utils.tracing.config.DefaultTracingConfig; import org.eclipse.ditto.json.JsonFactory; import org.eclipse.ditto.json.JsonObject; import org.eclipse.ditto.thingsearch.model.signals.commands.query.AggregateThingsMetrics; import org.eclipse.ditto.thingsearch.model.signals.commands.query.AggregateThingsMetricsResponse; import org.eclipse.ditto.thingsearch.service.persistence.read.ThingsAggregationPersistence; import org.junit.AfterClass; -import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; -public class AggregationThingsMetricsActorTest { +public class AggregateThingsMetricsActorTest { @ClassRule public static final DittoTracingInitResource DITTO_TRACING_INIT_RESOURCE = DittoTracingInitResource.disableDittoTracing(); - private static ActorSystem system = ActorSystem.create(); + private static ActorSystem system = ActorSystem.create(); @AfterClass public static void teardown() { @@ -65,30 +62,32 @@ public void testHandleAggregateThingsMetrics() { // Create a mock persistence object ThingsAggregationPersistence mockPersistence = mock(ThingsAggregationPersistence.class); doReturn(Source.from(List.of( - new Document("_id", new Document(Map.of("_revision", 1L, "location", "Berlin"))) - .append("online", 6) - .append("offline", 0), - new Document("_id", new Document(Map.of("_revision", 1L, "location", "Immenstaad"))) - .append("online", 5) - .append("offline", 0), - new Document("_id", new Document(Map.of("_revision", 1L, "location", "Sofia"))) - .append("online", 5) - .append("offline", 3))) - ).when(mockPersistence) + new Document("_id", new Document(Map.of("_revision", 1L, "location", "Berlin"))) + .append("online", 6) + .append("offline", 0), + new Document("_id", new Document(Map.of("_revision", 1L, "location", "Immenstaad"))) + .append("online", 5) + .append("offline", 0), + new Document("_id", new Document(Map.of("_revision", 1L, "location", "Sofia"))) + .append("online", 5) + .append("offline", 3))) + ).when(mockPersistence) .aggregateThings(any()); // Create the actor - Props props = AggregationThingsMetricsActor.props(mockPersistence); + Props props = AggregateThingsMetricsActor.props(mockPersistence); final var actorRef = system.actorOf(props); // Prepare the test message - Map groupingBy = Map.of("_revision", "$_revision", "location", "$t.attributes.Info.location"); - Map namedFilters = Map.of( + Map groupingBy = + Map.of("_revision", "$_revision", "location", "$t.attributes.Info.location"); + Map namedFilters = Map.of( "online", "gt(features/ConnectionStatus/properties/status/readyUntil/,time:now)", - "offline","lt(features/ConnectionStatus/properties/status/readyUntil/,time:now)"); + "offline", "lt(features/ConnectionStatus/properties/status/readyUntil/,time:now)"); Set namespaces = Collections.singleton("namespace"); DittoHeaders headers = DittoHeaders.newBuilder().build(); - AggregateThingsMetrics metrics = AggregateThingsMetrics.of("metricName", groupingBy, namedFilters, namespaces, headers); + AggregateThingsMetrics metrics = + AggregateThingsMetrics.of("metricName", groupingBy, namedFilters, namespaces, headers); // Send the message to the actor actorRef.tell(metrics, getRef()); @@ -101,7 +100,7 @@ public void testHandleAggregateThingsMetrics() { .set("online", 6) .set("offline", 0) .build(); - AggregateThingsMetricsResponse + final AggregateThingsMetricsResponse expectedResponse = AggregateThingsMetricsResponse.of(mongoAggregationResult, metrics); expectMsg(expectedResponse); @@ -116,7 +115,7 @@ public void testUnknownMessage() { // Create a mock persistence object // Create the actor - Props props = AggregationThingsMetricsActor.props(mock(ThingsAggregationPersistence.class)); + Props props = AggregateThingsMetricsActor.props(mock(ThingsAggregationPersistence.class)); final var actorRef = system.actorOf(props); // Send an unknown message to the actor From 83ccdf9a07cdff419eda3b0d83c3163398791829 Mon Sep 17 00:00:00 2001 From: Aleksandar Stanchev Date: Thu, 19 Sep 2024 14:45:00 +0300 Subject: [PATCH 13/13] applied patch from review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - removed leftover hardcoded filter name tag. - renamed the Operator name from Search to Aggregate - timer is now stopped он termination of the source not at each element Signed-off-by: Aleksandar Stanchev --- .../pages/ditto/installation-operating.md | 28 ++++++------ .../actors/AggregateThingsMetricsActor.java | 43 ++++++++++--------- ...peratorAggregateMetricsProviderActor.java} | 12 +++--- .../actors/SearchUpdaterRootActor.java | 9 ++-- .../src/main/resources/search-dev.conf | 19 +++----- ...aultCustomAggregationMetricConfigTest.java | 2 +- .../resources/custom-search-metric-test.conf | 4 +- 7 files changed, 56 insertions(+), 61 deletions(-) rename thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/{OperatorSearchMetricsProviderActor.java => OperatorAggregateMetricsProviderActor.java} (97%) diff --git a/documentation/src/main/resources/pages/ditto/installation-operating.md b/documentation/src/main/resources/pages/ditto/installation-operating.md index 6c2f47eebc..26fcc86a47 100644 --- a/documentation/src/main/resources/pages/ditto/installation-operating.md +++ b/documentation/src/main/resources/pages/ditto/installation-operating.md @@ -612,10 +612,10 @@ ditto { custom-metrics { ... } - custom-aggregate-metrics { + custom-aggregation-metrics { online_status { enabled = true - scrape-interval = 1m # override scrape interval, run every 20 minute + scrape-interval = 20m # override scrape interval, run every 20 minutes namespaces = [ "org.eclipse.ditto" ] @@ -628,6 +628,7 @@ ditto { "health" = "{{ inline:health }}" "hardcoded-tag" = "hardcoded_value" "location" = "{{ group-by:location | fn:default('missing location') }}" + "isGateway" = "{{ group-by:isGateway }}" } filters { online_filter { @@ -654,19 +655,19 @@ ditto { To add custom metrics via System properties, the following example shows how the above metric can be configured: ``` --Dditto.search.operator-metrics.custom-search-metrics.online_status.enabled=true --Dditto.search.operator-metrics.custom-search-metrics.online_status.scrape-interval=20m --Dditto.search.operator-metrics.custom-search-metrics.online_status.namespaces.0=org.eclipse.ditto --Dditto.search.operator-metrics.custom-search-metrics.online_status.tags.online="{{online_placeholder}}" --Dditto.search.operator-metrics.custom-search-metrics.online_status.tags.location="{{attributes/Info/location}}" +-Dditto.search.operator-metrics.custom-aggregation-metrics.online_status.enabled=true +-Dditto.search.operator-metrics.custom-aggregation-metrics.online_status.scrape-interval=20m +-Dditto.search.operator-metrics.custom-aggregation-metrics.online_status.namespaces.0=org.eclipse.ditto +-Dditto.search.operator-metrics.custom-aggregation-metrics.online_status.tags.online="{{online_placeholder}}" +-Dditto.search.operator-metrics.custom-aggregation-metrics.online_status.tags.location="{{attributes/Info/location}}" --Dditto.search.operator-metrics.custom-search-metrics.online_status.filters.online-filter.filter=gt(features/ConnectionStatus/properties/status/readyUntil/,time:now) --Dditto.search.operator-metrics.custom-search-metrics.online_status.filters.online-filter.inline-placeholder-values.online_placeholder=true --Dditto.search.operator-metrics.custom-search-metrics.online_status.filters.online-filter.fields.0=attributes/Info/location +-Dditto.search.operator-metrics.custom-aggregation-metrics.online_status.filters.online-filter.filter=gt(features/ConnectionStatus/properties/status/readyUntil/,time:now) +-Dditto.search.operator-metrics.custom-aggregation-metrics.online_status.filters.online-filter.inline-placeholder-values.online_placeholder=true +-Dditto.search.operator-metrics.custom-aggregation-metrics.online_status.filters.online-filter.fields.0=attributes/Info/location --Dditto.search.operator-metrics.custom-search-metrics.online_status.filters.offline-filter.filter=lt(features/ConnectionStatus/properties/status/readyUntil/,time:now) --Dditto.search.operator-metrics.custom-search-metrics.online_status.filters.offline-filter.inline-placeholder-values.online_placeholder=false --Dditto.search.operator-metrics.custom-search-metrics.online_status.filters.offline-filter.fields.0=attributes/Info/location +-Dditto.search.operator-metrics.custom-aggregation-metrics.online_status.filters.offline-filter.filter=lt(features/ConnectionStatus/properties/status/readyUntil/,time:now) +-Dditto.search.operator-metrics.custom-aggregation-metrics.online_status.filters.offline-filter.inline-placeholder-values.online_placeholder=false +-Dditto.search.operator-metrics.custom-aggregation-metrics.online_status.filters.offline-filter.fields.0=attributes/Info/location ``` @@ -679,6 +680,7 @@ In Prometheus format, this would look like: ``` online_status{location="Berlin",online="false"} 6.0 online_status{location="Immenstaad",online="true"} 8.0 +``` ## Tracing diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/AggregateThingsMetricsActor.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/AggregateThingsMetricsActor.java index 2db9544518..07f6872657 100644 --- a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/AggregateThingsMetricsActor.java +++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/AggregateThingsMetricsActor.java @@ -14,6 +14,7 @@ package org.eclipse.ditto.thingsearch.service.starter.actors; +import java.time.Duration; import java.util.concurrent.CompletableFuture; import org.apache.pekko.NotUsed; @@ -79,9 +80,7 @@ public static Props props(final ThingsAggregationPersistence aggregationPersiste public Receive createReceive() { return ReceiveBuilder.create() .match(AggregateThingsMetrics.class, this::aggregate) - .matchAny(any -> { - log.warning("Got unknown message '{}'", any); - }) + .matchAny(any -> log.warning("Got unknown message '{}'", any)) .build(); } @@ -111,12 +110,28 @@ private Source processAggregationPersistenceResult(final Source< final Flow logAndFinishPersistenceSegmentFlow = Flow.fromFunction(result -> { log.withCorrelationId(dittoHeaders) - .debug("aggregation element: {}", result); + .info("aggregation element: {}", result); return result; }); return source.via(logAndFinishPersistenceSegmentFlow); } + private Flow stopTimerAndHandleError(final StartedTimer searchTimer, + final AggregateThingsMetrics command) { + return Flow.fromFunction(element -> element) + .recoverWithRetries(1, new PFBuilder, NotUsed>>() + .matchAny(error -> Source.single(asDittoRuntimeException(error, command))) + .build() + ).watchTermination((notUsed, done) ->{ + final long now = System.nanoTime(); + stopTimer(searchTimer); + final long duration = + Duration.ofNanos(now- searchTimer.getStartInstant().toNanos()).toMillis(); + log.withCorrelationId(command).info("Db aggregation for metric <{}> - took: <{}ms>", command.getMetricName(), duration); + return NotUsed.getInstance(); + }); + } + private static StartedTimer startNewTimer(final WithDittoHeaders withDittoHeaders) { final StartedTimer startedTimer = DittoMetrics.timer(TRACING_THINGS_AGGREGATION) .start(); @@ -127,28 +142,14 @@ private static StartedTimer startNewTimer(final WithDittoHeaders withDittoHeader private static void stopTimer(final StartedTimer timer) { try { - timer.stop(); + if (timer.isRunning()) { + timer.stop(); + } } catch (final IllegalStateException e) { // it is okay if the timer was stopped. } } - private Flow stopTimerAndHandleError(final StartedTimer searchTimer, - final WithDittoHeaders command) { - return Flow.fromFunction( - element -> { - stopTimer(searchTimer); - return element; - }) - .recoverWithRetries(1, new PFBuilder, NotUsed>>() - .matchAny(error -> { - stopTimer(searchTimer); - return Source.single(asDittoRuntimeException(error, command)); - }) - .build() - ); - } - private DittoRuntimeException asDittoRuntimeException(final Throwable error, final WithDittoHeaders trigger) { return DittoRuntimeException.asDittoRuntimeException(error, t -> { log.error(error, "AggregateThingsMetricsActor failed to execute <{}>", trigger); diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/OperatorSearchMetricsProviderActor.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/OperatorAggregateMetricsProviderActor.java similarity index 97% rename from thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/OperatorSearchMetricsProviderActor.java rename to thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/OperatorAggregateMetricsProviderActor.java index 826fc4af8b..e1b93bc5f3 100644 --- a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/OperatorSearchMetricsProviderActor.java +++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/starter/actors/OperatorAggregateMetricsProviderActor.java @@ -61,7 +61,7 @@ * Actor which is started as singleton for "search" role and is responsible for querying for extended operator defined * "custom metrics" (configured via Ditto search service configuration) to expose as {@link Gauge} via Prometheus. */ -public final class OperatorSearchMetricsProviderActor extends AbstractActorWithTimers { +public final class OperatorAggregateMetricsProviderActor extends AbstractActorWithTimers { /** * This Actor's actor name. @@ -80,12 +80,12 @@ public final class OperatorSearchMetricsProviderActor extends AbstractActorWithT private final Map>> inlinePlaceholderResolvers; @SuppressWarnings("unused") - private OperatorSearchMetricsProviderActor(final SearchConfig searchConfig) { + private OperatorAggregateMetricsProviderActor(final SearchConfig searchConfig) { this.thingsAggregatorActorSingletonProxy = initializeAggregationThingsMetricsActor(searchConfig); this.customSearchMetricConfigMap = searchConfig.getOperatorMetricsConfig().getCustomAggregationMetricConfigs(); this.metricsGauges = new HashMap<>(); this.inlinePlaceholderResolvers = new HashMap<>(); - this.customSearchMetricsGauge = KamonGauge.newGauge("custom-search-metrics-count-of-instruments"); + this.customSearchMetricsGauge = KamonGauge.newGauge("custom-aggregation-metrics-count-of-instruments"); this.customSearchMetricConfigMap.forEach((metricName, customSearchMetricConfig) -> { initializeCustomMetricTimer(metricName, customSearchMetricConfig, getMaxConfiguredScrapeInterval(searchConfig.getOperatorMetricsConfig())); @@ -105,7 +105,7 @@ private OperatorSearchMetricsProviderActor(final SearchConfig searchConfig) { * @return the Props object. */ public static Props props(final SearchConfig searchConfig) { - return Props.create(OperatorSearchMetricsProviderActor.class, searchConfig); + return Props.create(OperatorAggregateMetricsProviderActor.class, searchConfig); } @Override @@ -161,11 +161,9 @@ private void handleAggregateThingsResponse(final AggregateThingsMetricsResponse resolveTags(filterName, customSearchMetricConfigMap.get(metricName), response); final CustomAggregationMetricConfig customAggregationMetricConfig = customSearchMetricConfigMap.get(metricName); - final TagSet tagSet = resolveTags(filterName, customAggregationMetricConfig, response) - .putTag(Tag.of("filter", filterName)); + final TagSet tagSet = resolveTags(filterName, customAggregationMetricConfig, response); recordMetric(metricName, tagSet, value); customSearchMetricsGauge.tag(Tag.of(METRIC_NAME, metricName)).set(Long.valueOf(metricsGauges.size())); - ; }); } diff --git a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/updater/actors/SearchUpdaterRootActor.java b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/updater/actors/SearchUpdaterRootActor.java index aa6005a713..70061d775d 100644 --- a/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/updater/actors/SearchUpdaterRootActor.java +++ b/thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/updater/actors/SearchUpdaterRootActor.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 Contributors to the Eclipse Foundation + * Copyright (c) 2024 Contributors to the Eclipse Foundation * * See the NOTICE file(s) distributed with this work for additional * information regarding copyright ownership. @@ -9,6 +9,7 @@ * http://www.eclipse.org/legal/epl-2.0 * * SPDX-License-Identifier: EPL-2.0 + * */ package org.eclipse.ditto.thingsearch.service.updater.actors; @@ -38,7 +39,7 @@ import org.eclipse.ditto.thingsearch.service.persistence.write.streaming.SearchUpdaterStream; import org.eclipse.ditto.thingsearch.service.starter.actors.MongoClientExtension; import org.eclipse.ditto.thingsearch.service.starter.actors.OperatorMetricsProviderActor; -import org.eclipse.ditto.thingsearch.service.starter.actors.OperatorSearchMetricsProviderActor; +import org.eclipse.ditto.thingsearch.service.starter.actors.OperatorAggregateMetricsProviderActor; /** * Our "Parent" Actor which takes care of supervision of all other Actors in our system. @@ -133,8 +134,8 @@ private SearchUpdaterRootActor(final SearchConfig searchConfig, startClusterSingletonActor(OperatorMetricsProviderActor.ACTOR_NAME, OperatorMetricsProviderActor.props(searchConfig.getOperatorMetricsConfig(), searchActor) ); - startClusterSingletonActor(OperatorSearchMetricsProviderActor.ACTOR_NAME, - OperatorSearchMetricsProviderActor.props(searchConfig) + startClusterSingletonActor(OperatorAggregateMetricsProviderActor.ACTOR_NAME, + OperatorAggregateMetricsProviderActor.props(searchConfig) ); } diff --git a/thingsearch/service/src/main/resources/search-dev.conf b/thingsearch/service/src/main/resources/search-dev.conf index 405632d405..8dcb94db35 100755 --- a/thingsearch/service/src/main/resources/search-dev.conf +++ b/thingsearch/service/src/main/resources/search-dev.conf @@ -27,37 +27,30 @@ ditto { } custom-aggregation-metrics { online_status { - enabled = false + enabled = true scrape-interval = 1m # override scrape interval, run every 20 minute namespaces = [ "org.eclipse.ditto" ] - group-by:{ + group-by { "location" = "attributes/Info/location" "isGateway" = "attributes/Info/gateway" } - tags: { + tags { "online" = "{{ inline:online_placeholder }}" "health" = "{{ inline:health }}" "hardcoded-tag" = "value" "location" = "{{ group-by:location | fn:default('missing location') }}" "isGateway" = "{{ group-by:isGateway }}" } - filters = { - online_filter = { + filters { + online_filter { filter = "gt(features/ConnectionStatus/properties/status/readyUntil/,time:now)" - inline-placeholder-values = { + inline-placeholder-values { "online_placeholder" = true "health" = "good" } } - offline_filter = { - filter = "lt(features/ConnectionStatus/properties/status/readyUntil/,time:now)" - inline-placeholder-values = { - "online_placeholder" = false - "health" = "bad" - } - } } } } diff --git a/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultCustomAggregationMetricConfigTest.java b/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultCustomAggregationMetricConfigTest.java index dc39d9ab61..48d9ac3f28 100644 --- a/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultCustomAggregationMetricConfigTest.java +++ b/thingsearch/service/src/test/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultCustomAggregationMetricConfigTest.java @@ -39,7 +39,7 @@ public class DefaultCustomAggregationMetricConfigTest { @BeforeClass public static void initTestFixture() { config = ConfigFactory.load("custom-search-metric-test"); - customSearchMetricTestConfig = config.getConfig("ditto.search.operator-metrics.custom-search-metrics"); + customSearchMetricTestConfig = config.getConfig("ditto.search.operator-metrics.custom-aggregation-metrics"); } @Test diff --git a/thingsearch/service/src/test/resources/custom-search-metric-test.conf b/thingsearch/service/src/test/resources/custom-search-metric-test.conf index 78e16c5a96..b75aa48000 100644 --- a/thingsearch/service/src/test/resources/custom-search-metric-test.conf +++ b/thingsearch/service/src/test/resources/custom-search-metric-test.conf @@ -36,7 +36,7 @@ ditto { scrape-interval = ${?THINGS_SEARCH_OPERATOR_METRICS_SCRAPE_INTERVAL} custom-metrics { } - custom-search-metrics { + custom-aggregation-metrics { online_status { enabled = true scrape-interval = 1m # override scrape interval, run every 20 minute @@ -53,7 +53,7 @@ ditto { "health" = "{{ inline:health }}" "hardcoded-tag" = "value" "location" = "{{ group-by:location | fn:default('missing location') }}" - "altitude" = "{{ group-by:isGateway }}" + "isGateway" = "{{ group-by:isGateway }}" } filters = { online_filter = {