Skip to content

Commit

Permalink
Change cardinality overflow warning from once to every five minutes
Browse files Browse the repository at this point in the history
Signed-off-by: Jonah Calvo <[email protected]>
  • Loading branch information
JonahCalvo committed Aug 16, 2023
1 parent cca9e28 commit 06bcfae
Showing 1 changed file with 5 additions and 4 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.Collection;
import java.util.HashMap;
import java.util.LinkedList;
Expand All @@ -45,8 +47,7 @@ public class AnomalyDetectorProcessor extends AbstractProcessor<Record<Event>, R
private final AnomalyDetectorProcessorConfig anomalyDetectorProcessorConfig;
private static final Logger LOG = LoggerFactory.getLogger(AnomalyDetectorProcessor.class);
private final Counter cardinalityOverflowCounter;
private boolean overflowWarned = false;

Instant nextWarnTime = Instant.MIN;
@DataPrepperPluginConstructor
public AnomalyDetectorProcessor(final AnomalyDetectorProcessorConfig anomalyDetectorProcessorConfig, final PluginMetrics pluginMetrics, final PluginFactory pluginFactory) {
super(pluginMetrics);
Expand Down Expand Up @@ -86,9 +87,9 @@ public Collection<Record<Event>> doExecute(Collection<Record<Event>> records) {
forestMap.put(identificationKeysMap.hashCode(), forest);
recordsOut.addAll(forest.handleEvents(List.of(record)));
} else {
if (!overflowWarned) {
if (Instant.now().isAfter(nextWarnTime)) {
LOG.warn("Cardinality limit reached, see cardinalityOverflow metric for count of skipped records");
overflowWarned = true;
nextWarnTime = Instant.now().plus(5, ChronoUnit.MINUTES);
}
cardinalityOverflowCounter.increment();
}
Expand Down

0 comments on commit 06bcfae

Please sign in to comment.