diff --git a/.circleci/config.yml b/.circleci/config.yml index 9fc6e226..65c38f79 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,7 +15,7 @@ - run: echo "deb http://ftp.debian.org/debian stable main contrib non-free" >> /etc/apt/sources.list - run: apt update - run: apt install -y unzip - # Install JAVA 11 + # Install JAVA 17 - run: mkdir -p /usr/share/man/man1 # FIX https://github.com/geerlingguy/ansible-role-java/issues/64 - run: apt install -y openjdk-17-jdk # BEGIN Dependencies for RocksDB @@ -24,7 +24,7 @@ # END Dependencies for RocksDB - run: set JAVA_HOME /usr/lib/jvm/java-17-openjdk-amd64/ - run: export JAVA_HOME - - run: dotnet sonarscanner begin /k:LGouellec_kafka-streams-dotnet /o:kafka-streams-dotnet /d:sonar.login=${SONAR_TOKEN} /d:sonar.host.url=https://sonarcloud.io /d:sonar.cs.opencover.reportsPaths="**\coverage*.opencover.xml" /d:sonar.coverage.exclusions="**sample*/*.cs,**test*/*.cs,**Tests*.cs,**Mock*.cs" + - run: dotnet sonarscanner begin /k:LGouellec_kafka-streams-dotnet /o:kafka-streams-dotnet /d:sonar.login=${SONAR_TOKEN} /d:sonar.host.url=https://sonarcloud.io /d:sonar.cs.opencover.reportsPaths="**\coverage*.opencover.xml" /d:sonar.coverage.exclusions="**sample*/*.cs,**test*/*.cs,**Tests*.cs,**Mock*.cs,**State/Cache/Internal/*.cs" - run: dotnet build - run: dotnet test --no-restore --no-build --verbosity normal -f net6.0 --collect:"XPlat Code Coverage" /p:CollectCoverage=true /p:CoverletOutputFormat=opencover test/Streamiz.Kafka.Net.Tests/Streamiz.Kafka.Net.Tests.csproj - run: dotnet sonarscanner end /d:sonar.login=${SONAR_TOKEN} diff --git a/README.md b/README.md index 22a65cac..a5130497 100644 --- a/README.md +++ b/README.md @@ -104,22 +104,22 @@ static async System.Threading.Tasks.Task Main(string[] args) |:------------------------------------------------------------:|:----------------------------------:|:----------------------:|:------------------------------------------:| | Stateless processors | X | X | | | RocksDb store | X | X | | -| Standby replicas | X | | No plan for now | +| Standby replicas | X | | No plan for now | | InMemory store | X | X | | | Transformer, Processor API | X | X | | -| Punctuate | X | X | | +| Punctuate | X | X | | | KStream-KStream Join | X | X | | | KTable-KTable Join | X | X | | -| KTable-KTable FK Join | X | | Plan for 1.6.0 | +| KTable-KTable FK Join | X | | Plan for future | | KStream-KTable Join | X | X | | | KStream-GlobalKTable Join | X | X | | -| KStream Async Processing (external call inside the topology) | | X | Not supported in Kafka Streams JAVA | +| KStream Async Processing (external call inside the topology) | | X | Not supported in Kafka Streams JAVA | | Hopping window | X | X | | | Tumbling window | X | X | | | Sliding window | X | | No plan for now | | Session window | X | | No plan for now | -| Cache | X | | Plan for 1.6.0 | -| Suppress(..) | X | | No plan for now | +| Cache | X | X | EA 1.6.0 | +| Suppress(..) | X | | No plan for now | | Interactive Queries | X | | No plan for now | | State store batch restoring | X | | No plan for now | | Exactly Once (v1 and v2) | X | X | EOS V1 supported, EOS V2 not supported yet | @@ -137,3 +137,13 @@ When adding or changing a service please add tests and documentations. # Support You can found support [here](https://discord.gg/J7Jtxum) + +# Star History + + + + + + Star History Chart + + diff --git a/core/Crosscutting/DictionaryExtensions.cs b/core/Crosscutting/DictionaryExtensions.cs index f28d3b5e..6a3cd1f1 100644 --- a/core/Crosscutting/DictionaryExtensions.cs +++ b/core/Crosscutting/DictionaryExtensions.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using Confluent.Kafka; namespace Streamiz.Kafka.Net.Crosscutting @@ -26,11 +27,9 @@ public static bool AddOrUpdate(this IDictionary map, K key, V value) map[key] = value; return false; } - else - { - map.Add(key, value); - return true; - } + + map.Add(key, value); + return true; } /// @@ -134,5 +133,32 @@ public static void CreateListOrAdd(this IDictionary> source, K source.Add(key, new List{value}); } + #if NETSTANDARD2_0 + public static bool TryAdd( this IDictionary dictionary, + K key, + V value){ + if (dictionary == null) + throw new ArgumentNullException(nameof (dictionary)); + if (dictionary.ContainsKey(key)) + return false; + dictionary.Add(key, value); + return true; + } + + public static bool Remove( this IDictionary dictionary, + K key, + out V value){ + bool result = dictionary.TryGetValue(key, out V valueTmp); + if (result) + { + value = valueTmp; + dictionary.Remove(key); + return true; + } + value = default(V); + return false; + } + #endif + } } diff --git a/core/Crosscutting/KafkaExtensions.cs b/core/Crosscutting/KafkaExtensions.cs index 553b4132..758471f3 100644 --- a/core/Crosscutting/KafkaExtensions.cs +++ b/core/Crosscutting/KafkaExtensions.cs @@ -86,6 +86,9 @@ internal static Headers Clone(this Headers headers) originHeader.ForEach(h => copyHeaders.Add(h.Key, h.Item2)); return copyHeaders; } + + internal static long GetEstimatedSize(this Headers headers) + => headers.Sum(header => header.Key.Length + header.GetValueBytes().LongLength); internal static Headers AddOrUpdate(this Headers headers, string key, byte[] value) { diff --git a/core/Crosscutting/SortedDictionaryExtensions.cs b/core/Crosscutting/SortedDictionaryExtensions.cs index 4d2b05be..d946f324 100644 --- a/core/Crosscutting/SortedDictionaryExtensions.cs +++ b/core/Crosscutting/SortedDictionaryExtensions.cs @@ -33,5 +33,19 @@ internal static IEnumerable> SubMap(this SortedDictiona } } } + + internal static IEnumerable> TailMap(this SortedDictionary sortedDic, K keyFrom, + bool inclusive) + { + foreach (K k in sortedDic.Keys) + { + int rT = sortedDic.Comparer.Compare(keyFrom, k); + + if ((inclusive && rT <= 0) || (!inclusive && rT < 0)) + { + yield return new KeyValuePair(k, sortedDic[k]); + } + } + } } } diff --git a/core/Crosscutting/Utils.cs b/core/Crosscutting/Utils.cs index 357ea00d..4996d7fc 100644 --- a/core/Crosscutting/Utils.cs +++ b/core/Crosscutting/Utils.cs @@ -42,5 +42,11 @@ public static bool IsNumeric(object expression, out Double number) , NumberFormatInfo.InvariantInfo , out number); } + + public static void CheckIfNotNull(object parameter, string nameAccessor) + { + if(parameter == null) + throw new ArgumentException($"{nameAccessor} must not be null"); + } } } diff --git a/core/KafkaStream.cs b/core/KafkaStream.cs index 662bba83..babef907 100644 --- a/core/KafkaStream.cs +++ b/core/KafkaStream.cs @@ -324,7 +324,7 @@ string Protect(string str) () => StreamState != null && StreamState.IsRunning() ? 1 : 0, () => threads.Count(t => t.State != ThreadState.DEAD && t.State != ThreadState.PENDING_SHUTDOWN), metricsRegistry); - + threads = new IThread[numStreamThreads]; var threadState = new Dictionary(); diff --git a/core/Metrics/Internal/CachingMetrics.cs b/core/Metrics/Internal/CachingMetrics.cs new file mode 100644 index 00000000..3683c887 --- /dev/null +++ b/core/Metrics/Internal/CachingMetrics.cs @@ -0,0 +1,69 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using Streamiz.Kafka.Net.Processors.Internal; + +namespace Streamiz.Kafka.Net.Metrics.Internal +{ + internal class CachingMetrics + { + internal static string CACHE_SIZE_BYTES_TOTAL = "cache-size-bytes-total"; + private static string CACHE_SIZE_BYTES_TOTAL_DESCRIPTION = "The total size in bytes of this cache state store."; + + internal static string HIT_RATIO = "hit-ratio"; + private static string HIT_RATIO_DESCRIPTION = "The hit ratio defined as the ratio of cache read hits over the total cache read requests."; + private static string HIT_RATIO_AVG_DESCRIPTION = "The average cache hit ratio"; + private static string HIT_RATIO_MIN_DESCRIPTION = "The minimum cache hit ratio"; + private static string HIT_RATIO_MAX_DESCRIPTION = "The maximum cache hit ratio"; + + public static Sensor HitRatioSensor( + TaskId taskId, + string storeType, + string storeName, + StreamMetricsRegistry streamsMetrics) { + + Sensor sensor; + string hitMetricName = HIT_RATIO; + IDictionary tags = + streamsMetrics.StoreLevelTags(GetThreadId(), taskId.ToString(), storeName, storeType); + + sensor = streamsMetrics.StoreLevelSensor(GetThreadId(), taskId, storeName, hitMetricName, HIT_RATIO_DESCRIPTION, MetricsRecordingLevel.DEBUG); + + SensorHelper.AddAvgAndMinAndMaxToSensor( + sensor, + StreamMetricsRegistry.STATE_STORE_LEVEL_GROUP, + tags, + hitMetricName, + HIT_RATIO_AVG_DESCRIPTION, + HIT_RATIO_MAX_DESCRIPTION, + HIT_RATIO_MIN_DESCRIPTION); + + return sensor; + } + + public static Sensor TotalCacheSizeBytesSensor( + TaskId taskId, + string storeType, + string storeName, + StreamMetricsRegistry streamsMetrics) { + + Sensor sensor; + string totalCacheMetricName = CACHE_SIZE_BYTES_TOTAL; + IDictionary tags = + streamsMetrics.StoreLevelTags(GetThreadId(), taskId.ToString(), storeName, storeType); + + sensor = streamsMetrics.StoreLevelSensor(GetThreadId(), taskId, storeName, totalCacheMetricName, CACHE_SIZE_BYTES_TOTAL_DESCRIPTION, MetricsRecordingLevel.DEBUG); + + SensorHelper.AddValueMetricToSensor( + sensor, + StreamMetricsRegistry.STATE_STORE_LEVEL_GROUP, + tags, + totalCacheMetricName, + CACHE_SIZE_BYTES_TOTAL_DESCRIPTION); + + return sensor; + } + + private static string GetThreadId() => Thread.CurrentThread.Name; + } +} \ No newline at end of file diff --git a/core/Metrics/Internal/SensorHelper.cs b/core/Metrics/Internal/SensorHelper.cs index a104e032..13c532ac 100644 --- a/core/Metrics/Internal/SensorHelper.cs +++ b/core/Metrics/Internal/SensorHelper.cs @@ -125,6 +125,40 @@ internal static void AddAvgAndMaxToSensor(Sensor sensor, ); } + internal static void AddAvgAndMinAndMaxToSensor(Sensor sensor, + string group, + IDictionary tags, + string operation, + string descriptionOfAvg, + string descriptionOfMax, + string descriptionOfMin) { + + sensor.AddStatMetric( + new MetricName( + operation + StreamMetricsRegistry.AVG_SUFFIX, + group, + descriptionOfAvg, + tags), + new Avg() + ); + sensor.AddStatMetric( + new MetricName( + operation + StreamMetricsRegistry.MAX_SUFFIX, + group, + descriptionOfMax, + tags), + new Max() + ); + sensor.AddStatMetric( + new MetricName( + operation + StreamMetricsRegistry.MIN_SUFFIX, + group, + descriptionOfMin, + tags), + new Min() + ); + } + internal static void AddRateOfSumAndSumMetricsToSensor(Sensor sensor, string group, IDictionary tags, diff --git a/core/Metrics/Sensor.cs b/core/Metrics/Sensor.cs index d5c2c00d..5b1a4d0b 100644 --- a/core/Metrics/Sensor.cs +++ b/core/Metrics/Sensor.cs @@ -154,6 +154,9 @@ internal void Record() internal void Record(long value) => Record(value, DateTime.Now.GetMilliseconds()); + internal void Record(double value) + => Record(value, DateTime.Now.GetMilliseconds()); + internal virtual void Record(double value, long timeMs) => RecordInternal(value, timeMs); diff --git a/core/Mock/ClusterInMemoryTopologyDriver.cs b/core/Mock/ClusterInMemoryTopologyDriver.cs index 61f4b63d..e47fcff7 100644 --- a/core/Mock/ClusterInMemoryTopologyDriver.cs +++ b/core/Mock/ClusterInMemoryTopologyDriver.cs @@ -226,6 +226,12 @@ void stateChangedHandeler(IThread thread, ThreadStateTransitionValidator old, } } } + + public void TriggerCommit() + { + throw new NotImplementedException(); + //((StreamThread)threadTopology)?.Manager.CommitAll(); + } #endregion } diff --git a/core/Mock/IBehaviorTopologyTestDriver.cs b/core/Mock/IBehaviorTopologyTestDriver.cs index 40cd5b92..68958831 100644 --- a/core/Mock/IBehaviorTopologyTestDriver.cs +++ b/core/Mock/IBehaviorTopologyTestDriver.cs @@ -14,5 +14,6 @@ internal interface IBehaviorTopologyTestDriver : IDisposable TestOutputTopic CreateOutputTopic(string topicName, TimeSpan consumeTimeout, ISerDes keySerdes = null, ISerDes valueSerdes = null); TestMultiInputTopic CreateMultiInputTopic(string[] topics, ISerDes keySerdes = null, ISerDes valueSerdes = null); IStateStore GetStateStore(string name); + void TriggerCommit(); } } diff --git a/core/Mock/TaskSynchronousTopologyDriver.cs b/core/Mock/TaskSynchronousTopologyDriver.cs index 75d9828c..df9b06a4 100644 --- a/core/Mock/TaskSynchronousTopologyDriver.cs +++ b/core/Mock/TaskSynchronousTopologyDriver.cs @@ -237,7 +237,10 @@ public TestInputTopic CreateInputTopic(string topicName, ISerDes null); foreach (var topic in topicsLink) + { + GetTask(topic); pipeInput.Flushed += () => ForwardRepartitionTopic(consumer, topic); + } return new TestInputTopic(pipeInput, configuration, keySerdes, valueSerdes); } @@ -296,6 +299,21 @@ public IStateStore GetStateStore(string name) return hasGlobalTopology ? globalProcessorContext.GetStateStore(name) : null; } + public void TriggerCommit() + { + foreach(var extProcessor in externalProcessorTopologies.Values) + extProcessor.Flush(); + + foreach (var task in tasks.Values) + { + var consumer = supplier.GetConsumer(topicConfiguration.ToConsumerConfig("consumer-repartition-forwarder"), + null); + task.Commit(); + } + + globalTask?.FlushState(); + } + #endregion } } \ No newline at end of file diff --git a/core/Mock/TopologyTestDriver.cs b/core/Mock/TopologyTestDriver.cs index 168c6bf7..44c16e08 100644 --- a/core/Mock/TopologyTestDriver.cs +++ b/core/Mock/TopologyTestDriver.cs @@ -165,6 +165,14 @@ public void Dispose() behavior.Dispose(); } + /// + /// Trigger the driver to commit, especially needed if you use caching + /// + public void Commit() + { + behavior.TriggerCommit(); + } + #region Create Input Topic /// diff --git a/core/ProcessorContext.cs b/core/ProcessorContext.cs index 64b2d766..5d1033d5 100644 --- a/core/ProcessorContext.cs +++ b/core/ProcessorContext.cs @@ -68,6 +68,8 @@ public class ProcessorContext /// public virtual string StateDir => $"{Path.Combine(Configuration.StateDir, Configuration.ApplicationId, Id.ToString())}"; + internal bool ConfigEnableCache => Configuration.StateStoreCacheMaxBytes > 0; + // FOR TESTING internal ProcessorContext() { @@ -91,6 +93,11 @@ internal ProcessorContext UseRecordCollector(IRecordCollector collector) return this; } + internal void SetRecordMetaData(IRecordContext context) + { + RecordContext = context; + } + internal void SetRecordMetaData(ConsumeResult result) { RecordContext = new RecordContext(result); diff --git a/core/Processors/AbstractKTableProcessor.cs b/core/Processors/AbstractKTableProcessor.cs index f19641b2..c1eb653e 100644 --- a/core/Processors/AbstractKTableProcessor.cs +++ b/core/Processors/AbstractKTableProcessor.cs @@ -1,4 +1,5 @@ -using Streamiz.Kafka.Net.Errors; +using Confluent.Kafka; +using Streamiz.Kafka.Net.Errors; using Streamiz.Kafka.Net.Processors.Internal; using Streamiz.Kafka.Net.State; using Streamiz.Kafka.Net.Table.Internal; @@ -11,7 +12,7 @@ internal abstract class AbstractKTableProcessor : AbstractProcesso protected readonly bool sendOldValues; private readonly bool throwException = false; protected ITimestampedKeyValueStore store; - protected TimestampedTupleForwarder tupleForwarder; + protected TimestampedTupleForwarder tupleForwarder; protected AbstractKTableProcessor(string queryableStoreName, bool sendOldValues, bool throwExceptionStateNull = false) { @@ -27,7 +28,16 @@ public override void Init(ProcessorContext context) if (queryableStoreName != null) { store = (ITimestampedKeyValueStore)context.GetStateStore(queryableStoreName); - tupleForwarder = new TimestampedTupleForwarder(this, sendOldValues); + tupleForwarder = new TimestampedTupleForwarder( + store, + this,kv => { + context.CurrentProcessor = this; + Forward(kv.Key, + new Change(sendOldValues ? kv.Value.OldValue.Value : default, kv.Value.NewValue.Value), + kv.Value.NewValue.Timestamp); + }, + sendOldValues, + context.ConfigEnableCache); } if (throwException && (queryableStoreName == null || store == null || tupleForwarder == null)) diff --git a/core/Processors/Internal/RecordContext.cs b/core/Processors/Internal/RecordContext.cs index ca8bc278..4aa76a05 100644 --- a/core/Processors/Internal/RecordContext.cs +++ b/core/Processors/Internal/RecordContext.cs @@ -5,21 +5,22 @@ namespace Streamiz.Kafka.Net.Processors.Internal internal class RecordContext : IRecordContext { public RecordContext() + : this(new Headers(), -1, -1, -1, "") { - Offset = -1; - Timestamp = -1; - Topic = ""; - Partition = -1; - Headers = new Headers(); + } + + public RecordContext(Headers headers, long offset, long timestamp, int partition, string topic) + { + Offset = offset; + Timestamp = timestamp; + Topic = topic; + Partition = partition; + Headers = headers; } public RecordContext(ConsumeResult result) + : this(result.Message.Headers, result.Offset, result.Message.Timestamp.UnixTimestampMs, result.Partition, result.Topic) { - Offset = result.Offset; - Timestamp = result.Message.Timestamp.UnixTimestampMs; - Topic = result.Topic; - Partition = result.Partition; - Headers = result.Message.Headers; } public long Offset { get; } diff --git a/core/Processors/Internal/StoreChangelogReader.cs b/core/Processors/Internal/StoreChangelogReader.cs index 249593b5..d8ca3925 100644 --- a/core/Processors/Internal/StoreChangelogReader.cs +++ b/core/Processors/Internal/StoreChangelogReader.cs @@ -203,7 +203,7 @@ private void RestoreChangelog(ChangelogMetadata changelogMetadata) log.LogDebug($"Restored {numRecords} records from " + $"changelog {changelogMetadata.StoreMetadata.Store.Name} " + $"to store {changelogMetadata.StoreMetadata.ChangelogTopicPartition}, " + - $"end offset is {(changelogMetadata.RestoreEndOffset.HasValue ? changelogMetadata.RestoreEndOffset.Value : "unknown")}, " + + $"end offset is {(changelogMetadata.RestoreEndOffset.HasValue ? changelogMetadata.RestoreEndOffset.Value.ToString() : "unknown")}, " + $"current offset is {currentOffset}"); changelogMetadata.BufferedLimit = 0; diff --git a/core/Processors/Internal/TimestampedTupleForwarder.cs b/core/Processors/Internal/TimestampedTupleForwarder.cs index a3bc488a..51958ada 100644 --- a/core/Processors/Internal/TimestampedTupleForwarder.cs +++ b/core/Processors/Internal/TimestampedTupleForwarder.cs @@ -1,19 +1,30 @@ -using Streamiz.Kafka.Net.Table.Internal; +using System; +using System.Collections.Generic; +using Streamiz.Kafka.Net.State; +using Streamiz.Kafka.Net.State.Cache; +using Streamiz.Kafka.Net.State.Internal; +using Streamiz.Kafka.Net.Table.Internal; namespace Streamiz.Kafka.Net.Processors.Internal { - // TODO REFACTOR internal class TimestampedTupleForwarder { private readonly IProcessor processor; private readonly bool sendOldValues; private readonly bool cachingEnabled; - public TimestampedTupleForwarder(IProcessor processor, bool sendOldValues) + public TimestampedTupleForwarder( + IStateStore store, + IProcessor processor, + Action>>> listener, + bool sendOldValues, + bool configCachingEnabled) { this.processor = processor; this.sendOldValues = sendOldValues; - cachingEnabled = false; + cachingEnabled = configCachingEnabled && + ((IWrappedStateStore)store).IsCachedStore && + ((ICachedStateStore>)store).SetFlushListener(listener, sendOldValues); } public void MaybeForward(K key, V newValue, V oldValue) @@ -30,7 +41,8 @@ public void MaybeForward(K key, VR newValue, VR oldValue) public void MaybeForward(K key, V newValue, V oldValue, long timestamp) { - processor?.Forward(key, new Change(sendOldValues ? oldValue : default, newValue), timestamp); + if (!cachingEnabled) + processor?.Forward(key, new Change(sendOldValues ? oldValue : default, newValue), timestamp); } public void MaybeForward(K key, VR newValue, VR oldValue, long timestamp) diff --git a/core/Processors/KStreamWindowAggregateProcessor.cs b/core/Processors/KStreamWindowAggregateProcessor.cs index 1470041f..2afce3da 100644 --- a/core/Processors/KStreamWindowAggregateProcessor.cs +++ b/core/Processors/KStreamWindowAggregateProcessor.cs @@ -3,6 +3,7 @@ using Streamiz.Kafka.Net.Stream; using System; using Microsoft.Extensions.Logging; +using Streamiz.Kafka.Net.Table.Internal; namespace Streamiz.Kafka.Net.Processors { @@ -36,7 +37,18 @@ public override void Init(ProcessorContext context) { base.Init(context); windowStore = (ITimestampedWindowStore)context.GetStateStore(storeName); - tupleForwarder = new TimestampedTupleForwarder, Agg>(this, sendOldValues); + tupleForwarder = new TimestampedTupleForwarder, Agg>( + windowStore, + this, + kv => + { + context.CurrentProcessor = this; + Forward(kv.Key, + new Change(sendOldValues ? kv.Value.OldValue.Value : default, kv.Value.NewValue.Value), + kv.Value.NewValue.Timestamp); + }, + sendOldValues, + context.ConfigEnableCache); } public override void Process(K key, V value) diff --git a/core/Processors/KTableMapProcessor.cs b/core/Processors/KTableMapProcessor.cs index 8b83b396..b73ea082 100644 --- a/core/Processors/KTableMapProcessor.cs +++ b/core/Processors/KTableMapProcessor.cs @@ -27,14 +27,23 @@ public override void Process(K key, Change value) KeyValuePair oldPair = value.OldValue == null ? default : mapper.Apply(key, value.OldValue); KeyValuePair newPair = value.NewValue == null ? default : mapper.Apply(key, value.NewValue); - // if the value is null, we do not need to forward its selected key-value further + bool oldPairNotNull = value.OldValue != null; + bool newPairNotNull = value.NewValue != null; + // if the selected repartition key or value is null, skip // forward oldPair first, to be consistent with reduce and aggregate - if (oldPair.Key != null && oldPair.Value != null) - Forward(oldPair.Key, new Change(oldPair.Value, default)); - - if (newPair.Key != null && newPair.Value != null) - Forward(newPair.Key, new Change(default, newPair.Value)); + if (oldPairNotNull && newPairNotNull && oldPair.Key.Equals(newPair.Key)) + { + Forward(oldPair.Key, new Change(oldPair.Value, newPair.Value)); + } + else + { + if(oldPairNotNull) + Forward(oldPair.Key, new Change(oldPair.Value, default)); + + if(newPairNotNull) + Forward(newPair.Key, new Change(default, newPair.Value)); + } } } } diff --git a/core/Processors/KTableSourceProcessor.cs b/core/Processors/KTableSourceProcessor.cs index c75bfc72..d7002e0c 100644 --- a/core/Processors/KTableSourceProcessor.cs +++ b/core/Processors/KTableSourceProcessor.cs @@ -28,7 +28,17 @@ public override void Init(ProcessorContext context) if (this.queryableName != null) { store = (ITimestampedKeyValueStore)context.GetStateStore(queryableName); - tupleForwarder = new TimestampedTupleForwarder(this, sendOldValues); + tupleForwarder = new TimestampedTupleForwarder( + store, + this, + kv => { + context.CurrentProcessor = this; + context.CurrentProcessor.Forward(kv.Key, + new Change(sendOldValues ? (kv.Value.OldValue != null ? kv.Value.OldValue.Value : default) : default, kv.Value.NewValue.Value), + kv.Value.NewValue.Timestamp); + }, + sendOldValues, + context.ConfigEnableCache); } } diff --git a/core/Processors/StatefullProcessor.cs b/core/Processors/StatefullProcessor.cs index ae972237..1ffde514 100644 --- a/core/Processors/StatefullProcessor.cs +++ b/core/Processors/StatefullProcessor.cs @@ -1,5 +1,7 @@ -using Streamiz.Kafka.Net.Processors.Internal; +using System.Collections.Generic; +using Streamiz.Kafka.Net.Processors.Internal; using Streamiz.Kafka.Net.State; +using Streamiz.Kafka.Net.Table.Internal; namespace Streamiz.Kafka.Net.Processors { @@ -8,7 +10,7 @@ internal abstract class StatefullProcessor : AbstractProcessor store; - protected TimestampedTupleForwarder tupleForwarder; + protected TimestampedTupleForwarder tupleForwarder; protected StatefullProcessor(string storeName, bool sendOldValues) { @@ -20,7 +22,18 @@ public override void Init(ProcessorContext context) { base.Init(context); store = (ITimestampedKeyValueStore)context.GetStateStore(storeName); - tupleForwarder = new TimestampedTupleForwarder(this, sendOldValues); + tupleForwarder = new TimestampedTupleForwarder( + store, + this, + kv => + { + context.CurrentProcessor = this; + Forward(kv.Key, + new Change(sendOldValues ? kv.Value.OldValue.Value : default, kv.Value.NewValue.Value), + kv.Value.NewValue.Timestamp); + }, + sendOldValues, + context.ConfigEnableCache); } } } diff --git a/core/Processors/StreamThread.cs b/core/Processors/StreamThread.cs index 9bb6e1c0..8574e6c9 100644 --- a/core/Processors/StreamThread.cs +++ b/core/Processors/StreamThread.cs @@ -106,7 +106,7 @@ internal static IThread Create(string threadId, string clientId, InternalTopolog private readonly ILogger log = Logger.GetLogger(typeof(StreamThread)); private readonly Thread thread; private readonly IConsumer consumer; - private readonly TaskManager manager; + private TaskManager Manager { get; } private readonly InternalTopologyBuilder builder; private readonly TimeSpan consumeTimeout; private readonly string threadId; @@ -155,7 +155,7 @@ private StreamThread(string threadId, string clientId, TaskManager manager, ICon InternalTopologyBuilder builder, IChangelogReader storeChangelogReader, StreamMetricsRegistry streamMetricsRegistry, TimeSpan timeSpan, long commitInterval) { - this.manager = manager; + this.Manager = manager; this.consumer = consumer; this.builder = builder; consumeTimeout = timeSpan; @@ -234,7 +234,7 @@ public void Run() try { - if (!manager.RebalanceInProgress) + if (!Manager.RebalanceInProgress) { RestorePhase(); @@ -268,10 +268,10 @@ public void Run() { long processLatency = 0; - if (!manager.RebalanceInProgress) + if (!Manager.RebalanceInProgress) processLatency = ActionHelper.MeasureLatency(() => { - processed = manager.Process(now); + processed = Manager.Process(now); }); else processed = 0; @@ -296,7 +296,7 @@ public void Run() int punctuated = 0; var punctuateLatency = ActionHelper.MeasureLatency(() => { - punctuated = manager.Punctuate(); + punctuated = Manager.Punctuate(); }); totalPunctuateLatency += punctuateLatency; summaryPunctuated += punctuated; @@ -399,16 +399,16 @@ public void Run() private void RestorePhase() { - if (State == ThreadState.PARTITIONS_ASSIGNED || State == ThreadState.RUNNING && manager.NeedRestoration()) + if (State == ThreadState.PARTITIONS_ASSIGNED || State == ThreadState.RUNNING && Manager.NeedRestoration()) { log.LogDebug($"{logPrefix} State is {State}, initializing and restoring tasks if necessary"); restorationInProgress = true; - if (manager.TryToCompleteRestoration()) + if (Manager.TryToCompleteRestoration()) { restorationInProgress = false; log.LogInformation( - $"Restoration took {DateTime.Now.GetMilliseconds() - LastPartitionAssignedTime}ms for all tasks {string.Join(",", manager.ActiveTaskIds)}"); + $"Restoration took {DateTime.Now.GetMilliseconds() - LastPartitionAssignedTime}ms for all tasks {string.Join(",", Manager.ActiveTaskIds)}"); if (State == ThreadState.PARTITIONS_ASSIGNED) SetState(ThreadState.RUNNING); } @@ -435,7 +435,7 @@ private int AddToTasks(IEnumerable> records) foreach (var record in records) { count++; - var task = manager.ActiveTaskFor(record.TopicPartition); + var task = Manager.ActiveTaskFor(record.TopicPartition); if (task != null) { if (task.IsClosed) @@ -482,7 +482,7 @@ public void Start(CancellationToken token) ThreadMetrics.CreateStartThreadSensor(threadId, DateTime.Now.GetMilliseconds(), streamMetricsRegistry); } - public IEnumerable ActiveTasks => manager.ActiveTasks; + public IEnumerable ActiveTasks => Manager.ActiveTasks; public long LastPartitionAssignedTime { get; internal set; } @@ -501,7 +501,7 @@ private void HandleTaskMigrated(TaskMigratedException e) "{LogPrefix}Detected that the thread is being fenced. This implies that this thread missed a rebalance and dropped out of the consumer group. Will close out all assigned tasks and rejoin the consumer group", logPrefix); - manager.HandleLostAll(); + Manager.HandleLostAll(); consumer.Unsubscribe(); consumer.Subscribe(builder.GetSourceTopics()); } @@ -512,12 +512,12 @@ private void HandleInnerException() "{LogPrefix}Detected that the thread throw an inner exception. Your configuration manager has decided to continue running stream processing. So will close out all assigned tasks and rejoin the consumer group", logPrefix); - manager.HandleLostAll(); + Manager.HandleLostAll(); consumer.Unsubscribe(); consumer.Subscribe(builder.GetSourceTopics()); } - private int Commit() + internal int Commit() // for testing { int committed = 0; if (DateTime.Now - lastCommit > TimeSpan.FromMilliseconds(commitTimeMs)) @@ -525,12 +525,12 @@ private int Commit() DateTime beginCommit = DateTime.Now; log.LogDebug( "Committing all active tasks {TaskIDs} since {DateTime}ms has elapsed (commit interval is {CommitTime}ms)", - string.Join(",", manager.ActiveTaskIds), (DateTime.Now - lastCommit).TotalMilliseconds, + string.Join(",", Manager.ActiveTaskIds), (DateTime.Now - lastCommit).TotalMilliseconds, commitTimeMs); - committed = manager.CommitAll(); + committed = Manager.CommitAll(); if (committed > 0) log.LogDebug("Committed all active tasks {TaskIDs} in {TimeElapsed}ms", - string.Join(",", manager.ActiveTaskIds), (DateTime.Now - beginCommit).TotalMilliseconds); + string.Join(",", Manager.ActiveTaskIds), (DateTime.Now - beginCommit).TotalMilliseconds); if (committed == -1) { @@ -571,7 +571,7 @@ private void CompleteShutdown() IsRunning = false; - manager.Close(); + Manager.Close(); consumer.Unsubscribe(); consumer.Close(); @@ -595,7 +595,7 @@ private void CompleteShutdown() private IEnumerable> PollRequest(TimeSpan ts) { - if (!restorationInProgress && !manager.RebalanceInProgress) + if (!restorationInProgress && !Manager.RebalanceInProgress) { lastPollMs = DateTime.Now.GetMilliseconds(); return consumer.ConsumeRecords(ts, streamConfig.MaxPollRecords); diff --git a/core/SerDes/Internal/BytesSerDes.cs b/core/SerDes/Internal/BytesSerDes.cs index 0917b884..de9884e7 100644 --- a/core/SerDes/Internal/BytesSerDes.cs +++ b/core/SerDes/Internal/BytesSerDes.cs @@ -9,6 +9,6 @@ public override Bytes Deserialize(byte[] data, SerializationContext context) => Bytes.Wrap(data); public override byte[] Serialize(Bytes data, SerializationContext context) - => data != null ? data.Get : null; + => data?.Get; } } diff --git a/core/State/AbstractStoreBuilder.cs b/core/State/AbstractStoreBuilder.cs index 1a0b619a..72169347 100644 --- a/core/State/AbstractStoreBuilder.cs +++ b/core/State/AbstractStoreBuilder.cs @@ -30,8 +30,8 @@ public abstract class AbstractStoreBuilder : IStoreBuilder /// Value serdes /// protected readonly ISerDes valueSerdes; - - // private bool enableCaching; + + private bool enableCaching = false; private bool enableLogging = true; /// @@ -59,6 +59,11 @@ public abstract class AbstractStoreBuilder : IStoreBuilder /// public bool LoggingEnabled => enableLogging; + /// + /// Caching enabled or not + /// + public bool CachingEnabled => enableCaching; + /// /// /// @@ -78,7 +83,7 @@ protected AbstractStoreBuilder(String name, ISerDes keySerde, ISerDes valu /// public IStoreBuilder WithCachingEnabled() { - //enableCaching = true; + enableCaching = true; return this; } @@ -88,7 +93,7 @@ public IStoreBuilder WithCachingEnabled() /// public IStoreBuilder WithCachingDisabled() { - //enableCaching = false; + enableCaching = false; return this; } diff --git a/core/State/Cache/CacheEntryValue.cs b/core/State/Cache/CacheEntryValue.cs new file mode 100644 index 00000000..e5d32313 --- /dev/null +++ b/core/State/Cache/CacheEntryValue.cs @@ -0,0 +1,33 @@ +using Confluent.Kafka; +using Streamiz.Kafka.Net.Crosscutting; +using Streamiz.Kafka.Net.Processors; +using Streamiz.Kafka.Net.Processors.Internal; + +namespace Streamiz.Kafka.Net.State.Cache +{ + internal class CacheEntryValue + { + public byte[] Value { get; } + public IRecordContext Context { get; } + + internal CacheEntryValue(byte[] value) + { + Context = new RecordContext(); + Value = value; + } + + internal CacheEntryValue(byte[] value, Headers headers, long offset, long timestamp, int partition, string topic) + { + Context = new RecordContext(headers.Clone(), offset, timestamp, partition, topic); + Value = value; + } + + public long Size => + (Value != null ? Value.LongLength : 0) + + sizeof(int) + // partition + sizeof(long) + //offset + sizeof(long) + // timestamp + Context.Topic.Length + // topic length + Context.Headers.GetEstimatedSize(); // headers size + } +} \ No newline at end of file diff --git a/core/State/Cache/CachingKeyValueStore.cs b/core/State/Cache/CachingKeyValueStore.cs new file mode 100644 index 00000000..7ea8e479 --- /dev/null +++ b/core/State/Cache/CachingKeyValueStore.cs @@ -0,0 +1,287 @@ +using System; +using System.Collections.Generic; +using Streamiz.Kafka.Net.Crosscutting; +using Streamiz.Kafka.Net.Metrics; +using Streamiz.Kafka.Net.Metrics.Internal; +using Streamiz.Kafka.Net.Processors; +using Streamiz.Kafka.Net.State.Cache.Enumerator; +using Streamiz.Kafka.Net.State.Cache.Internal; +using Streamiz.Kafka.Net.State.Enumerator; +using Streamiz.Kafka.Net.State.Internal; +using Streamiz.Kafka.Net.Table.Internal; + +namespace Streamiz.Kafka.Net.State.Cache +{ + internal class CachingKeyValueStore : + WrappedStateStore>, + IKeyValueStore, + ICachedStateStore + { + private MemoryCache cache; + private Action>> flushListener; + private bool sendOldValue; + private bool cachingEnabled; + + private Sensor hitRatioSensor = NoRunnableSensor.Empty; + private Sensor totalCacheSizeSensor = NoRunnableSensor.Empty; + + public CachingKeyValueStore(IKeyValueStore wrapped) + : base(wrapped) + { } + + protected virtual void RegisterMetrics() + { + if (cachingEnabled) + { + hitRatioSensor = CachingMetrics.HitRatioSensor(context.Id, "cache-store", Name, context.Metrics); + totalCacheSizeSensor = + CachingMetrics.TotalCacheSizeBytesSensor(context.Id, "cache-store", Name, context.Metrics); + } + } + + public override bool IsCachedStore => true; + + public bool SetFlushListener(Action>> listener, bool sendOldChanges) + { + flushListener = listener; + sendOldValue = sendOldChanges; + return true; + } + + // Only for testing + internal void CreateCache(ProcessorContext context) + { + cachingEnabled = context.Configuration.StateStoreCacheMaxBytes > 0; + if(cachingEnabled) + cache = new MemoryCache(new MemoryCacheOptions { + SizeLimit = context.Configuration.StateStoreCacheMaxBytes, + CompactionPercentage = .20 + }, new BytesComparer()); + } + + private byte[] GetInternal(Bytes key) + { + if (cachingEnabled) + { + byte[] value; + + if (cache.TryGetValue(key, out CacheEntryValue priorEntry)) + value = priorEntry.Value; + else + { + value = wrapped.Get(key); + if(value != null) + PutInternal(key, new CacheEntryValue(value), true); + } + + var currentStat = cache.GetCurrentStatistics(); + hitRatioSensor.Record((double)currentStat.TotalHits / (currentStat.TotalMisses + currentStat.TotalHits)); + + return value; + } + + return wrapped.Get(key); + } + + public override void Init(ProcessorContext context, IStateStore root) + { + base.Init(context, root); + CreateCache(context); + RegisterMetrics(); + } + + private void UpdateRatioSensor() + { + var currentStat = cache.GetCurrentStatistics(); + hitRatioSensor.Record((double)currentStat.TotalHits / (currentStat.TotalMisses + currentStat.TotalHits)); + } + + private void CacheEntryEviction(Bytes key, CacheEntryValue value, EvictionReason reason, MemoryCache state) + { + if (reason is EvictionReason.Replaced or EvictionReason.None) return; + + if (flushListener != null) + { + byte[] rawNewValue = value.Value; + byte[] rawOldValue = rawNewValue == null || sendOldValue ? wrapped.Get(key) : null; + + // this is an optimization: if this key did not exist in underlying store and also not in the cache, + // we can skip flushing to downstream as well as writing to underlying store + if (rawNewValue != null || rawOldValue != null) + { + var currentContext = context.RecordContext; + context.SetRecordMetaData(value.Context); + wrapped.Put(key, rawNewValue); + flushListener(new KeyValuePair>( + key.Get, + new Change(sendOldValue ? rawOldValue : null, rawNewValue))); + context.SetRecordMetaData(currentContext); + } + } + else + { + var currentContext = context.RecordContext; + context.SetRecordMetaData(value.Context); + wrapped.Put(key, value.Value); + context.SetRecordMetaData(currentContext); + } + + totalCacheSizeSensor.Record(cache.Size); + } + + public override void Flush() + { + if (cachingEnabled) + { + cache.Compact(1); // Compact 100% of the cache + base.Flush(); + } + else + wrapped.Flush(); + } + + public byte[] Get(Bytes key) + => GetInternal(key); + + public IKeyValueEnumerator Range(Bytes from, Bytes to) + { + if (cachingEnabled) + { + var storeEnumerator = wrapped.Range(from, to); + var cacheEnumerator = + new CacheEnumerator( + cache.KeyRange(from, to, true, true), + cache, + UpdateRatioSensor); + + return new MergedStoredCacheKeyValueEnumerator(cacheEnumerator, storeEnumerator, true); + } + return wrapped.Range(from, to); + } + + public IKeyValueEnumerator ReverseRange(Bytes from, Bytes to) + { + if (cachingEnabled) + { + var storeEnumerator = wrapped.ReverseRange(from, to); + var cacheEnumerator = + new CacheEnumerator( + cache.KeyRange(from, to, true, false), + cache, + UpdateRatioSensor); + + return new MergedStoredCacheKeyValueEnumerator(cacheEnumerator, storeEnumerator, false); + } + + return wrapped.ReverseRange(from, to); + } + + private IEnumerable> InternalAll(bool reverse) + { + var storeEnumerator = new WrapEnumerableKeyValueEnumerator(wrapped.All()); + var cacheEnumerator = new CacheEnumerator( + cache.KeySetEnumerable(reverse), + cache, + UpdateRatioSensor); + + var mergedEnumerator = new MergedStoredCacheKeyValueEnumerator(cacheEnumerator, storeEnumerator, reverse); + while (mergedEnumerator.MoveNext()) + if (mergedEnumerator.Current != null) + yield return mergedEnumerator.Current.Value; + } + + public IEnumerable> All() + { + if (cachingEnabled) + return InternalAll(true); + return wrapped.All(); + } + + public IEnumerable> ReverseAll() + { + if (cachingEnabled) + return InternalAll(false); + + return wrapped.ReverseAll(); + } + + public long ApproximateNumEntries() => cachingEnabled ? cache.Count : wrapped.ApproximateNumEntries(); + + public void Put(Bytes key, byte[] value) + { + if (cachingEnabled) + { + var cacheEntry = new CacheEntryValue( + value, + context.RecordContext.Headers, + context.Offset, + context.Timestamp, + context.Partition, + context.Topic); + + PutInternal(key, cacheEntry); + } + else + wrapped.Put(key, value); + } + + private void PutInternal(Bytes key, CacheEntryValue entry, bool fromWrappedCache = false) + { + long totalSize = key.Get.LongLength + entry.Size; + + var memoryCacheEntryOptions = new MemoryCacheEntryOptions() + .SetSize(totalSize) + .RegisterPostEvictionCallback(CacheEntryEviction, cache); + + cache.Set(key, entry, memoryCacheEntryOptions, fromWrappedCache ? EvictionReason.None : EvictionReason.Setted); + totalCacheSizeSensor.Record(cache.Size); + } + + public byte[] PutIfAbsent(Bytes key, byte[] value) + { + if (cachingEnabled) + { + var v = GetInternal(key); + if (v == null) + Put(key, value); + return v; + } + + return wrapped.PutIfAbsent(key, value); + } + + public void PutAll(IEnumerable> entries) + { + if (cachingEnabled) + { + foreach (var entry in entries) + Put(entry.Key, entry.Value); + } + else + wrapped.PutAll(entries); + } + + public byte[] Delete(Bytes key) + { + if (cachingEnabled) + { + var rawValue = Get(key); + Put(key, null); + return rawValue; + } + + return wrapped.Delete(key); + } + + public new void Close() + { + if (cachingEnabled) + { + cache.Dispose(); + base.Close(); + } + else + wrapped.Close(); + } + } +} \ No newline at end of file diff --git a/core/State/Cache/CachingWindowStore.cs b/core/State/Cache/CachingWindowStore.cs new file mode 100644 index 00000000..05997ea4 --- /dev/null +++ b/core/State/Cache/CachingWindowStore.cs @@ -0,0 +1,289 @@ +using System; +using System.Collections.Generic; +using Streamiz.Kafka.Net.Crosscutting; +using Streamiz.Kafka.Net.Metrics; +using Streamiz.Kafka.Net.Metrics.Internal; +using Streamiz.Kafka.Net.Processors; +using Streamiz.Kafka.Net.SerDes.Internal; +using Streamiz.Kafka.Net.State.Cache.Enumerator; +using Streamiz.Kafka.Net.State.Cache.Internal; +using Streamiz.Kafka.Net.State.Enumerator; +using Streamiz.Kafka.Net.State.Helper; +using Streamiz.Kafka.Net.State.Internal; +using Streamiz.Kafka.Net.Table.Internal; + +namespace Streamiz.Kafka.Net.State.Cache +{ + // TODO : involve hit Ratio + internal class CachingWindowStore : + WrappedStateStore>, + IWindowStore, + ICachedStateStore + { + private readonly long _windowSize; + + private MemoryCache cache; + private bool cachingEnabled; + private bool sendOldValue; + + private Action>> flushListener; + + private Sensor hitRatioSensor; + private Sensor totalCacheSizeSensor; + + public CachingWindowStore( + IWindowStore wrapped, + long windowSize, + long segmentInterval, + IKeySchema keySchema) + : base(wrapped) + { + _windowSize = windowSize; + SegmentInterval = segmentInterval; + KeySchema = keySchema; + SegmentCacheFunction = new SegmentedCacheFunction(KeySchema, segmentInterval); + MaxObservedTimestamp = -1; + } + + internal MemoryCache Cache => cache; + internal long MaxObservedTimestamp { get; set; } + internal long SegmentInterval { get; } + internal ICacheFunction SegmentCacheFunction { get; } + internal IKeySchema KeySchema { get; } + + public override bool IsCachedStore => true; + + public override void Init(ProcessorContext context, IStateStore root) + { + base.Init(context, root); + CreateCache(context); + RegisterMetrics(); + } + + protected virtual void RegisterMetrics() + { + if (cachingEnabled) + { + hitRatioSensor = CachingMetrics.HitRatioSensor(context.Id, "cache-window-store", Name, context.Metrics); + totalCacheSizeSensor = + CachingMetrics.TotalCacheSizeBytesSensor(context.Id, "cache-window-store", Name, context.Metrics); + } + } + + // Only for testing + internal void CreateCache(ProcessorContext context) + { + cachingEnabled = context.Configuration.StateStoreCacheMaxBytes > 0; + if(cachingEnabled) + cache = new MemoryCache(new MemoryCacheOptions { + SizeLimit = context.Configuration.StateStoreCacheMaxBytes, + CompactionPercentage = .20 + }, new BytesComparer()); + } + + public byte[] Fetch(Bytes key, long time) + { + if (cachingEnabled) + { + var bytesKey = WindowKeyHelper.ToStoreKeyBinary(key, time, 0); + var cacheKey = SegmentCacheFunction.CacheKey(bytesKey); + + byte[] value = null; + + if (cache.TryGetValue(cacheKey, out CacheEntryValue priorEntry)) + value = priorEntry.Value; + + var currentStat = cache.GetCurrentStatistics(); + hitRatioSensor.Record((double)currentStat.TotalHits / (currentStat.TotalMisses + currentStat.TotalHits)); + + return value ?? wrapped.Fetch(key, time); + } + + return wrapped.Fetch(key, time); + } + + public IWindowStoreEnumerator Fetch(Bytes key, DateTime from, DateTime to) + => Fetch(key, from.GetMilliseconds(), to.GetMilliseconds()); + + public IWindowStoreEnumerator Fetch(Bytes key, long from, long to) + { + if (cachingEnabled) + { + var wrappedEnumerator = wrapped.Fetch(key, from, to); + IKeyValueEnumerator cacheEnumerator = wrapped.Persistent + ? new CacheEnumeratorWrapper(this, key, from, to, true) + : FetchInternal( + SegmentCacheFunction.CacheKey(KeySchema.LowerRangeFixedSize(key, from)), + SegmentCacheFunction.CacheKey(KeySchema.UpperRangeFixedSize(key, to)), true); + + var hasNextCondition = KeySchema.HasNextCondition(key, key, from, to); + var filteredCacheEnumerator = + new FilteredCacheEnumerator(cacheEnumerator, hasNextCondition, SegmentCacheFunction); + + return new MergedSortedCacheWindowStoreEnumerator( + SegmentCacheFunction, + filteredCacheEnumerator, + wrappedEnumerator, + true); + } + return wrapped.Fetch(key, from, to); + } + + public IKeyValueEnumerator, byte[]> All() + { + if (cachingEnabled) + { + var wrappedEnumerator = wrapped.All(); + var cacheEnumerator = new CacheEnumerator( + cache.KeySetEnumerable(true), + cache, + UpdateRatioSensor); + + return new MergedSortedCacheWindowStoreKeyValueEnumerator( + cacheEnumerator, + wrappedEnumerator, + _windowSize, + SegmentCacheFunction, + new BytesSerDes(), + changelogTopic, + WindowKeyHelper.FromStoreKey, + WindowKeyHelper.ToStoreKeyBinary, + true); + } + return wrapped.All(); + } + + public IKeyValueEnumerator, byte[]> FetchAll(DateTime from, DateTime to) + { + if (cachingEnabled) + { + var wrappedEnumerator = wrapped.FetchAll(from, to); + var cacheEnumerator = new CacheEnumerator( + cache.KeySetEnumerable(true), + cache, + UpdateRatioSensor + ); + + var hasNextCondition = + KeySchema.HasNextCondition(null, null, from.GetMilliseconds(), to.GetMilliseconds()); + var filteredCacheEnumerator = + new FilteredCacheEnumerator(cacheEnumerator, hasNextCondition, SegmentCacheFunction); + + return new MergedSortedCacheWindowStoreKeyValueEnumerator( + filteredCacheEnumerator, + wrappedEnumerator, + _windowSize, + SegmentCacheFunction, + new BytesSerDes(), + changelogTopic, + WindowKeyHelper.FromStoreKey, + WindowKeyHelper.ToStoreKeyBinary, + true); + } + + return wrapped.FetchAll(from, to); + } + + public void Put(Bytes key, byte[] value, long windowStartTimestamp) + { + if (cachingEnabled) + { + var keyBytes = WindowKeyHelper.ToStoreKeyBinary(key, windowStartTimestamp, 0); + var cacheEntry = new CacheEntryValue( + value, + context.RecordContext.Headers, + context.Offset, + context.Timestamp, + context.Partition, + context.Topic); + + PutInternal(SegmentCacheFunction.CacheKey(keyBytes), cacheEntry); + MaxObservedTimestamp = Math.Max(KeySchema.SegmentTimestamp(keyBytes), MaxObservedTimestamp); + } + else + wrapped.Put(key, value, windowStartTimestamp); + } + + public override void Flush() + { + if (cachingEnabled) + { + cache.Compact(1); // Compact 100% of the cache + base.Flush(); + } + else + wrapped.Flush(); + } + + public bool SetFlushListener(Action>> listener, bool sendOldChanges) + { + flushListener = listener; + sendOldValue = sendOldChanges; + return true; + } + + private void PutInternal(Bytes key, CacheEntryValue entry, bool fromWrappedCache = false) + { + long totalSize = key.Get.LongLength + entry.Size; + + var memoryCacheEntryOptions = new MemoryCacheEntryOptions() + .SetSize(totalSize) + .RegisterPostEvictionCallback(CacheEntryEviction, cache); + + cache.Set(key, entry, memoryCacheEntryOptions, fromWrappedCache ? EvictionReason.None : EvictionReason.Setted); + totalCacheSizeSensor.Record(cache.Size); + } + + internal CacheEnumerator FetchInternal(Bytes keyFrom, Bytes keyTo, bool forward) + { + return new CacheEnumerator( + cache.KeyRange(keyFrom, keyTo, true, forward), + cache, + UpdateRatioSensor); + } + + private void UpdateRatioSensor() + { + var currentStat = cache.GetCurrentStatistics(); + hitRatioSensor.Record((double)currentStat.TotalHits / (currentStat.TotalMisses + currentStat.TotalHits)); + } + + private void CacheEntryEviction(Bytes key, CacheEntryValue value, EvictionReason reason, MemoryCache state) + { + if (reason is EvictionReason.Replaced or EvictionReason.None) return; + + var binaryWindowKey = SegmentCacheFunction.Key(key).Get; + var windowedKeyBytes = WindowKeyHelper.FromStoreBytesKey(binaryWindowKey, _windowSize); + long windowStartTs = windowedKeyBytes.Window.StartMs; + var binaryKey = windowedKeyBytes.Key; + + if (flushListener != null) + { + byte[] rawNewValue = value.Value; + byte[] rawOldValue = rawNewValue == null || sendOldValue ? wrapped.Fetch(binaryKey, windowStartTs) : null; + + // this is an optimization: if this key did not exist in underlying store and also not in the cache, + // we can skip flushing to downstream as well as writing to underlying store + if (rawNewValue != null || rawOldValue != null) + { + var currentContext = context.RecordContext; + context.SetRecordMetaData(value.Context); + wrapped.Put(binaryKey, rawNewValue, windowStartTs); + flushListener(new KeyValuePair>( + binaryWindowKey, + new Change(sendOldValue ? rawOldValue : null, rawNewValue))); + context.SetRecordMetaData(currentContext); + } + } + else + { + var currentContext = context.RecordContext; + context.SetRecordMetaData(value.Context); + wrapped.Put(binaryKey, value.Value, windowStartTs); + context.SetRecordMetaData(currentContext); + } + + totalCacheSizeSensor.Record(cache.Size); + } + } +} \ No newline at end of file diff --git a/core/State/Cache/Enumerator/AbstractMergedEnumerator.cs b/core/State/Cache/Enumerator/AbstractMergedEnumerator.cs new file mode 100644 index 00000000..bc08a472 --- /dev/null +++ b/core/State/Cache/Enumerator/AbstractMergedEnumerator.cs @@ -0,0 +1,154 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using Streamiz.Kafka.Net.Crosscutting; +using Streamiz.Kafka.Net.State.Enumerator; + +namespace Streamiz.Kafka.Net.State.Cache.Enumerator +{ + internal abstract class AbstractMergedEnumerator : IKeyValueEnumerator + { + private enum LastChoice + { + NONE, + STORE, + CACHE, + BOTH + }; + + private readonly bool forward; + private readonly IKeyValueEnumerator storeEnumerator; + private readonly IKeyValueEnumerator cacheEnumerator; + private LastChoice _lastChoice = LastChoice.NONE; + + protected AbstractMergedEnumerator( + IKeyValueEnumerator cacheEnumerator, + IKeyValueEnumerator storeEnumerator, + bool forward) + { + this.storeEnumerator = storeEnumerator; + this.cacheEnumerator = cacheEnumerator; + this.forward = forward; + } + + private bool IsDeletedCacheEntry(KeyValuePair? nextFromCache) + => nextFromCache?.Value.Value == null; + + public K PeekNextKey() => Current.Value.Key; + + public bool MoveNext() + { + // advance the store enumerator if choice is NONE ou STORE + if(_lastChoice is LastChoice.NONE or LastChoice.STORE or LastChoice.BOTH) + storeEnumerator.MoveNext(); + + if (_lastChoice is LastChoice.NONE or LastChoice.CACHE or LastChoice.BOTH) + { + // skip over items deleted from cache, and corresponding store items if they have the same key + while (cacheEnumerator.MoveNext() && IsDeletedCacheEntry(cacheEnumerator.Current)) + { + var currentKeyStore = storeEnumerator.Current; + // advance the store enumerator if the key is the same as the deleted cache key + if (currentKeyStore != null && + cacheEnumerator.Current != null && + Compare(cacheEnumerator.Current.Value.Key, currentKeyStore.Value.Key) == 0) + storeEnumerator.MoveNext(); + } + } + + Bytes nextCacheKey = cacheEnumerator.Current?.Key; + bool nullStoreKey = !storeEnumerator.Current.HasValue; + + if (nextCacheKey == null) { + Current = CurrentStoreValue(); + } + else if (nullStoreKey) { + Current = CurrentCacheValue(); + } + else + { + int comparison = Compare(nextCacheKey, storeEnumerator.Current.Value.Key); + Current = ChooseCurrentValue(comparison); + } + + return Current != null; + } + + public void Reset() + { + Current = null; + cacheEnumerator.Reset(); + storeEnumerator.Reset(); + } + + public KeyValuePair? Current { get; private set; } + + object IEnumerator.Current => Current; + + public void Dispose() + { + cacheEnumerator.Dispose(); + storeEnumerator.Dispose(); + GC.SuppressFinalize(this); + } + + #region Abstract + + protected abstract int Compare(Bytes cacheKey, KS storeKey); + protected abstract K DeserializeStoreKey(KS storeKey); + protected abstract KeyValuePair DeserializeStorePair(KeyValuePair pair); + protected abstract K DeserializeCacheKey(Bytes cacheKey); + protected abstract V DeserializeCacheValue(CacheEntryValue cacheEntry); + + #endregion + + private KeyValuePair? ChooseCurrentValue(int comparison) + { + if (forward) + { + if (comparison > 0) { + return CurrentStoreValue(); + } + + if (comparison < 0) { + return CurrentCacheValue(); + } + + return CurrentCacheValue(true); + } + + if (comparison < 0) { + return CurrentStoreValue(); + } + + if (comparison > 0) { + return CurrentCacheValue(); + } + + return CurrentCacheValue(true); + } + + private KeyValuePair? CurrentStoreValue() + { + if (storeEnumerator.Current is not null) + { + _lastChoice = LastChoice.STORE; + return DeserializeStorePair(storeEnumerator.Current.Value); + } + + return null; + } + + private KeyValuePair? CurrentCacheValue(bool storeEqual = false) + { + if (cacheEnumerator.Current is not null) + { + KeyValuePair next = cacheEnumerator.Current.Value; + _lastChoice = !storeEqual ? LastChoice.CACHE : LastChoice.BOTH; + return new KeyValuePair(DeserializeCacheKey(next.Key), DeserializeCacheValue(next.Value)); + } + + return null; + } + } +} \ No newline at end of file diff --git a/core/State/Cache/Enumerator/CacheEnumerator.cs b/core/State/Cache/Enumerator/CacheEnumerator.cs new file mode 100644 index 00000000..f916de4f --- /dev/null +++ b/core/State/Cache/Enumerator/CacheEnumerator.cs @@ -0,0 +1,57 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using Streamiz.Kafka.Net.State.Cache.Internal; +using Streamiz.Kafka.Net.State.Enumerator; + +namespace Streamiz.Kafka.Net.State.Cache.Enumerator +{ + internal class CacheEnumerator : IKeyValueEnumerator + where K : class + where V : class + { + private IEnumerator keys; + private readonly MemoryCache cache; + private readonly Action _beforeClosing; + private KeyValuePair? current; + + public CacheEnumerator( + IEnumerable keys, + MemoryCache cache, + Action beforeClosing) + { + this.keys = keys.GetEnumerator(); + this.cache = cache; + _beforeClosing = beforeClosing; + } + + public K PeekNextKey() + => current?.Key; + + public bool MoveNext() + { + var result = keys.MoveNext(); + if (result) + current = new KeyValuePair(keys.Current, cache.Get(keys.Current)); + else + current = null; + return result; + } + + public void Reset() + { + keys.Reset(); + } + + public KeyValuePair? Current => current; + + object IEnumerator.Current => Current; + + public void Dispose() + { + current = null; + keys.Dispose(); + _beforeClosing?.Invoke(); + } + } +} \ No newline at end of file diff --git a/core/State/Cache/Enumerator/CacheEnumeratorWrapper.cs b/core/State/Cache/Enumerator/CacheEnumeratorWrapper.cs new file mode 100644 index 00000000..2ddce03c --- /dev/null +++ b/core/State/Cache/Enumerator/CacheEnumeratorWrapper.cs @@ -0,0 +1,163 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using Streamiz.Kafka.Net.Crosscutting; +using Streamiz.Kafka.Net.State.Enumerator; +using Streamiz.Kafka.Net.State.Helper; + +namespace Streamiz.Kafka.Net.State.Cache.Enumerator +{ + internal class CacheEnumeratorWrapper : IKeyValueEnumerator + { + private readonly bool forward; + private readonly long timeTo; + private readonly Bytes keyTo; + private readonly CachingWindowStore _windowStore; + private readonly Bytes keyFrom; + private readonly long segmentInterval; + + private long lastSegmentId; + private long currentSegmentId; + private Bytes cacheKeyFrom; + private Bytes cacheKeyTo; + + private IKeyValueEnumerator current; + + public CacheEnumeratorWrapper(CachingWindowStore windowStore, Bytes key, long timeFrom, long timeTo, bool forward) + : this(windowStore, key, key, timeFrom, timeTo, forward) + { + } + + public CacheEnumeratorWrapper(CachingWindowStore windowStore, Bytes keyFrom, Bytes keyTo, long timeFrom, long timeTo, bool forward) + { + _windowStore = windowStore; + this.keyFrom = keyFrom; + this.keyTo = keyTo; + this.timeTo = timeTo; + this.forward = forward; + + segmentInterval = windowStore.SegmentCacheFunction.SegmentInterval; + + if (forward) + { + lastSegmentId = windowStore.SegmentCacheFunction.SegmentId(Math.Min(timeTo, windowStore.MaxObservedTimestamp)); + currentSegmentId = windowStore.SegmentCacheFunction.SegmentId(timeFrom); + + SetCacheKeyRange(timeFrom, CurrentSegmentLastTime()); + current = windowStore.FetchInternal(cacheKeyFrom, cacheKeyTo, true); + } + else + { + currentSegmentId = windowStore.SegmentCacheFunction.SegmentId(Math.Min(timeTo, windowStore.MaxObservedTimestamp)); + lastSegmentId = windowStore.SegmentCacheFunction.SegmentId(timeFrom); + + SetCacheKeyRange(CurrentSegmentBeginTime(), Math.Min(timeTo, windowStore.MaxObservedTimestamp)); + current = windowStore.FetchInternal(cacheKeyFrom, cacheKeyTo, false); + } + } + + #region Private + + private void SetCacheKeyRange(long lowerRangeEndTime, long upperRangeEndTime) { + + if (_windowStore.SegmentCacheFunction.SegmentId(lowerRangeEndTime) != _windowStore.SegmentCacheFunction.SegmentId(upperRangeEndTime)) { + throw new ArgumentException("Error iterating over segments: segment interval has changed"); + } + + if (keyFrom != null && keyTo != null && keyFrom.Equals(keyTo)) { + cacheKeyFrom = _windowStore.SegmentCacheFunction.CacheKey(SegmentLowerRangeFixedSize(keyFrom, lowerRangeEndTime)); + cacheKeyTo = _windowStore.SegmentCacheFunction.CacheKey(SegmentUpperRangeFixedSize(keyTo, upperRangeEndTime)); + } else { + cacheKeyFrom = keyFrom == null ? null : + _windowStore.SegmentCacheFunction.CacheKey(_windowStore.KeySchema.LowerRange(keyFrom, lowerRangeEndTime), currentSegmentId); + cacheKeyTo = keyTo == null ? null : + _windowStore.SegmentCacheFunction.CacheKey(_windowStore.KeySchema.UpperRange(keyTo, timeTo), currentSegmentId); + } + } + + private Bytes SegmentLowerRangeFixedSize(Bytes key, long segmentBeginTime) { + return WindowKeyHelper.ToStoreKeyBinary(key, Math.Max(0, segmentBeginTime), 0); + } + + private Bytes SegmentUpperRangeFixedSize(Bytes key, long segmentEndTime) { + return WindowKeyHelper.ToStoreKeyBinary(key, segmentEndTime, Int32.MaxValue); + } + + private long CurrentSegmentBeginTime() { + return currentSegmentId * segmentInterval; + } + + private long CurrentSegmentLastTime() { + return Math.Min(timeTo, CurrentSegmentBeginTime() + segmentInterval - 1); + } + + private void GetNextSegmentIterator() { + if (forward) { + ++currentSegmentId; + // updating as maxObservedTimestamp can change while iterating + lastSegmentId = + _windowStore.SegmentCacheFunction.SegmentId(Math.Min(timeTo, _windowStore.MaxObservedTimestamp)); + + if (currentSegmentId > lastSegmentId) { + current = null; + return; + } + + SetCacheKeyRange(CurrentSegmentBeginTime(), CurrentSegmentLastTime()); + + current.Dispose(); + + current = _windowStore.FetchInternal(cacheKeyFrom, cacheKeyTo, true); + } else { + --currentSegmentId; + + // last segment id is stable when iterating backward, therefore no need to update + if (currentSegmentId < lastSegmentId) { + current = null; + return; + } + + SetCacheKeyRange(CurrentSegmentBeginTime(), CurrentSegmentLastTime()); + + current.Dispose(); + + current = _windowStore.FetchInternal(cacheKeyFrom, cacheKeyTo, false); + } + } + + #endregion + + #region IKeyValueEnumerator impl + + public Bytes PeekNextKey() + => current.PeekNextKey(); + + public bool MoveNext() + { + if (current == null) return false; + + if (current.MoveNext()) + return true; + + while (!current.MoveNext()) + { + GetNextSegmentIterator(); + if (current == null) + return false; + } + + return true; + } + + public void Reset() => throw new NotImplementedException(); + + public KeyValuePair? Current => current?.Current; + + object IEnumerator.Current => Current; + + public void Dispose() + => current?.Dispose(); + + #endregion + } +} \ No newline at end of file diff --git a/core/State/Cache/Enumerator/FilteredCacheEnumerator.cs b/core/State/Cache/Enumerator/FilteredCacheEnumerator.cs new file mode 100644 index 00000000..905b44a2 --- /dev/null +++ b/core/State/Cache/Enumerator/FilteredCacheEnumerator.cs @@ -0,0 +1,72 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using Streamiz.Kafka.Net.Crosscutting; +using Streamiz.Kafka.Net.State.Enumerator; + +namespace Streamiz.Kafka.Net.State.Cache.Enumerator +{ + internal class FilteredCacheEnumerator : IKeyValueEnumerator + { + private class WrappredFilteredCacheEnumerator : IKeyValueEnumerator + { + private readonly ICacheFunction _cacheFunction; + private readonly IKeyValueEnumerator _cacheEnumerator; + + public WrappredFilteredCacheEnumerator( + ICacheFunction cacheFunction, + IKeyValueEnumerator cacheEnumerator) + { + _cacheFunction = cacheFunction; + _cacheEnumerator = cacheEnumerator; + } + + private KeyValuePair CachedPair(KeyValuePair next){ + return new KeyValuePair(_cacheFunction.Key(next.Key), next.Value.Value); + } + + public Bytes PeekNextKey() => _cacheFunction.Key(_cacheEnumerator.PeekNextKey()); + + public bool MoveNext() => _cacheEnumerator.MoveNext(); + + public void Reset() => _cacheEnumerator.Reset(); + + public KeyValuePair? Current + => _cacheEnumerator.Current is not null + ? CachedPair(_cacheEnumerator.Current.Value) + : null; + + object IEnumerator.Current => Current; + + public void Dispose() => _cacheEnumerator.Dispose(); + } + + private readonly IKeyValueEnumerator _cacheEnumerator; + private readonly IKeyValueEnumerator _wrappedCacheEnumerator; + private readonly Func, bool> _hasNextCondition; + + internal FilteredCacheEnumerator( + IKeyValueEnumerator cacheEnumerator, + Func,bool> hasNextCondition, + ICacheFunction segmentCacheFunction) + { + _cacheEnumerator = cacheEnumerator; + _hasNextCondition = hasNextCondition; + _wrappedCacheEnumerator = new WrappredFilteredCacheEnumerator(segmentCacheFunction, _cacheEnumerator); + } + + public Bytes PeekNextKey() => _cacheEnumerator.PeekNextKey(); + + public bool MoveNext() => _hasNextCondition(_wrappedCacheEnumerator); + + public void Reset() + => _cacheEnumerator.Reset(); + + public KeyValuePair? Current => _cacheEnumerator.Current; + + object IEnumerator.Current => Current; + + public void Dispose() + => _cacheEnumerator.Dispose(); + } +} \ No newline at end of file diff --git a/core/State/Cache/Enumerator/MergedSortedCacheWindowStoreEnumerator.cs b/core/State/Cache/Enumerator/MergedSortedCacheWindowStoreEnumerator.cs new file mode 100644 index 00000000..99236cc8 --- /dev/null +++ b/core/State/Cache/Enumerator/MergedSortedCacheWindowStoreEnumerator.cs @@ -0,0 +1,60 @@ +using System; +using System.Collections.Generic; +using Streamiz.Kafka.Net.Crosscutting; +using Streamiz.Kafka.Net.State.Enumerator; +using Streamiz.Kafka.Net.State.Helper; + +namespace Streamiz.Kafka.Net.State.Cache.Enumerator +{ + internal class MergedSortedCacheWindowStoreEnumerator : + AbstractMergedEnumerator , + IWindowStoreEnumerator + { + private readonly ICacheFunction _cacheFunction; + private readonly Func _extractStoreTimestamp; + + public MergedSortedCacheWindowStoreEnumerator( + ICacheFunction cacheFunction, + IKeyValueEnumerator cacheEnumerator, + IKeyValueEnumerator storeEnumerator, + bool forward) + : this(cacheFunction, cacheEnumerator, storeEnumerator, forward, WindowKeyHelper.ExtractStoreTimestamp) + { + + } + + private MergedSortedCacheWindowStoreEnumerator( + ICacheFunction cacheFunction, + IKeyValueEnumerator cacheEnumerator, + IKeyValueEnumerator storeEnumerator, + bool forward, + Func extractStoreTimestamp) + : base(cacheEnumerator, storeEnumerator, forward) + { + _cacheFunction = cacheFunction; + _extractStoreTimestamp = extractStoreTimestamp; + } + + protected override int Compare(Bytes cacheKey, long storeKey) + { + byte[] binaryKey = _cacheFunction.BytesFromCacheKey(cacheKey); + long cacheTimestamp = _extractStoreTimestamp(binaryKey); + return cacheTimestamp.CompareTo(storeKey); + } + + protected override long DeserializeStoreKey(long storeKey) + => storeKey; + + protected override KeyValuePair DeserializeStorePair(KeyValuePair pair) + => pair; + + protected override long DeserializeCacheKey(Bytes cacheKey) + { + byte[] binaryKey = _cacheFunction.BytesFromCacheKey(cacheKey); + return _extractStoreTimestamp(binaryKey); + } + + protected override byte[] DeserializeCacheValue(CacheEntryValue cacheEntry) + => cacheEntry.Value; + } +} \ No newline at end of file diff --git a/core/State/Cache/Enumerator/MergedSortedCacheWindowStoreKeyValueEnumerator.cs b/core/State/Cache/Enumerator/MergedSortedCacheWindowStoreKeyValueEnumerator.cs new file mode 100644 index 00000000..b3d3e8b4 --- /dev/null +++ b/core/State/Cache/Enumerator/MergedSortedCacheWindowStoreKeyValueEnumerator.cs @@ -0,0 +1,61 @@ +using System; +using System.Collections.Generic; +using System.Reflection; +using Streamiz.Kafka.Net.Crosscutting; +using Streamiz.Kafka.Net.SerDes; +using Streamiz.Kafka.Net.State.Enumerator; + +namespace Streamiz.Kafka.Net.State.Cache.Enumerator +{ + internal class MergedSortedCacheWindowStoreKeyValueEnumerator : + AbstractMergedEnumerator, Windowed, byte[], byte[]> { + + private readonly long _windowSize; + private readonly ICacheFunction _cacheFunction; + private readonly ISerDes _serdes; + private readonly string _changelogTopic; + private readonly Func, string, Windowed> _storeKeyToWindowKey; + private readonly Func _windowKeyToBytes; + + public MergedSortedCacheWindowStoreKeyValueEnumerator( + IKeyValueEnumerator cacheEnumerator, + IKeyValueEnumerator, byte[]> storeEnumerator, + long windowSize, + ICacheFunction cacheFunction, + ISerDes serdes, + string changelogTopic, + Func, string, Windowed> storeKeyToWindowKey, + Func windowKeyToBytes, + bool forward) + : base(cacheEnumerator, storeEnumerator, forward) + { + _windowSize = windowSize; + _cacheFunction = cacheFunction; + _serdes = serdes; + _changelogTopic = changelogTopic; + _storeKeyToWindowKey = storeKeyToWindowKey; + _windowKeyToBytes = windowKeyToBytes; + } + + protected override int Compare(Bytes cacheKey, Windowed storeKey) + { + var storeKeyBytes = _windowKeyToBytes(storeKey.Key, storeKey.Window.StartMs, 0); + return _cacheFunction.CompareSegmentedKeys(cacheKey, storeKeyBytes); + } + + protected override Windowed DeserializeStoreKey(Windowed storeKey) => storeKey; + + protected override KeyValuePair, byte[]> DeserializeStorePair( + KeyValuePair, byte[]> pair) + => pair; + + protected override Windowed DeserializeCacheKey(Bytes cacheKey) + { + var byteKey = _cacheFunction.Key(cacheKey).Get; + return _storeKeyToWindowKey(byteKey, _windowSize, _serdes, _changelogTopic); + } + + protected override byte[] DeserializeCacheValue(CacheEntryValue cacheEntry) + => cacheEntry.Value; + } +} \ No newline at end of file diff --git a/core/State/Cache/Enumerator/MergedStoredCacheKeyValueEnumerator.cs b/core/State/Cache/Enumerator/MergedStoredCacheKeyValueEnumerator.cs new file mode 100644 index 00000000..dfba0d07 --- /dev/null +++ b/core/State/Cache/Enumerator/MergedStoredCacheKeyValueEnumerator.cs @@ -0,0 +1,33 @@ +using System.Collections.Generic; +using Streamiz.Kafka.Net.Crosscutting; +using Streamiz.Kafka.Net.State.Enumerator; + +namespace Streamiz.Kafka.Net.State.Cache.Enumerator +{ + internal class MergedStoredCacheKeyValueEnumerator : + AbstractMergedEnumerator + { + public MergedStoredCacheKeyValueEnumerator( + IKeyValueEnumerator cacheEnumerator, + IKeyValueEnumerator storeEnumerator, + bool forward) + : base(cacheEnumerator, storeEnumerator, forward) + { + } + + protected override int Compare(Bytes cacheKey, Bytes storeKey) + => cacheKey.CompareTo(storeKey); + + protected override Bytes DeserializeStoreKey(Bytes storeKey) + => storeKey; + + protected override KeyValuePair DeserializeStorePair(KeyValuePair pair) + => pair; + + protected override Bytes DeserializeCacheKey(Bytes cacheKey) + => cacheKey; + + protected override byte[] DeserializeCacheValue(CacheEntryValue cacheEntry) + => cacheEntry.Value; + } +} \ No newline at end of file diff --git a/core/State/Cache/ICacheFunction.cs b/core/State/Cache/ICacheFunction.cs new file mode 100644 index 00000000..a0d19dc9 --- /dev/null +++ b/core/State/Cache/ICacheFunction.cs @@ -0,0 +1,16 @@ +using Streamiz.Kafka.Net.Crosscutting; + +namespace Streamiz.Kafka.Net.State.Cache +{ + internal interface ICacheFunction + { + byte[] BytesFromCacheKey(Bytes cacheKey); + Bytes Key(Bytes cacheKey); + Bytes CacheKey(Bytes cacheKey); + Bytes CacheKey(Bytes key, long segmentId); + long SegmentId(Bytes key); + long SegmentId(long timestamp); + long SegmentInterval { get; } + int CompareSegmentedKeys(Bytes cacheKey, Bytes storeKey); + } +} \ No newline at end of file diff --git a/core/State/Cache/ICachedStateStore.cs b/core/State/Cache/ICachedStateStore.cs new file mode 100644 index 00000000..2bc20e32 --- /dev/null +++ b/core/State/Cache/ICachedStateStore.cs @@ -0,0 +1,11 @@ +using System; +using System.Collections.Generic; +using Streamiz.Kafka.Net.Table.Internal; + +namespace Streamiz.Kafka.Net.State.Cache +{ + internal interface ICachedStateStore + { + bool SetFlushListener(Action>> listener, bool sendOldChanges); + } +} \ No newline at end of file diff --git a/core/State/Cache/Internal/CacheEntry.CacheEntryTokens.cs b/core/State/Cache/Internal/CacheEntry.CacheEntryTokens.cs new file mode 100644 index 00000000..048bb0ae --- /dev/null +++ b/core/State/Cache/Internal/CacheEntry.CacheEntryTokens.cs @@ -0,0 +1,60 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// This is a fork from Microsoft.Extensions.Caching.Memory.MemoryCache https://github.com/dotnet/runtime/blob/main/src/libraries/Microsoft.Extensions.Caching.Memory +// The only difference is the compaction process and eviction callback is synchronous whereas the .NET repo is asyncrhonous + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Threading; +using Microsoft.Extensions.Logging; + +namespace Streamiz.Kafka.Net.State.Cache.Internal +{ + internal sealed partial class CacheEntry + { + // this type exists just to reduce average CacheEntry size + // which typically is not using expiration tokens or callbacks + private sealed class CacheEntryTokens + { + private List>? _postEvictionCallbacks; // this is not really related to tokens, but was moved here to shrink typical CacheEntry size + + internal List> PostEvictionCallbacks => _postEvictionCallbacks ??= new List>(); + + + internal void InvokeEvictionCallbacks(CacheEntry cacheEntry) + { + if (_postEvictionCallbacks != null) + { + InvokeCallbacks(cacheEntry); + } + } + + private void InvokeCallbacks(CacheEntry entry) + { + Debug.Assert(entry._tokens != null); + List>? callbackRegistrations = Interlocked.Exchange(ref entry._tokens._postEvictionCallbacks, null); + + if (callbackRegistrations == null) + { + return; + } + + for (int i = 0; i < callbackRegistrations.Count; i++) + { + PostEvictionCallbackRegistration registration = callbackRegistrations[i]; + + try + { + registration.EvictionCallback?.Invoke(entry.Key, entry.Value, entry.EvictionReason, registration.State); + } + catch (Exception e) + { + // This will be invoked on a background thread, don't let it throw. + entry._cache.Logger.LogError(e, "EvictionCallback invoked failed"); + } + } + } + } + } +} diff --git a/core/State/Cache/Internal/CacheEntry.cs b/core/State/Cache/Internal/CacheEntry.cs new file mode 100644 index 00000000..d06ff742 --- /dev/null +++ b/core/State/Cache/Internal/CacheEntry.cs @@ -0,0 +1,115 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// This is a fork from Microsoft.Extensions.Caching.Memory.MemoryCache https://github.com/dotnet/runtime/blob/main/src/libraries/Microsoft.Extensions.Caching.Memory +// The only difference is the compaction process and eviction callback is synchronous whereas the .NET repo is asyncrhonous + +using System; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading; + +namespace Streamiz.Kafka.Net.State.Cache.Internal +{ + internal sealed partial class CacheEntry : ICacheEntry + where K : class + where V : class + { + //private static readonly Action ExpirationCallback = ExpirationTokensExpired; + private static readonly AsyncLocal?> _current = new(); + + private readonly MemoryCache _cache; + + private CacheEntryTokens? _tokens; // might be null if user is not using the tokens or callbacks + private long _size = NotSet; + private V _value; + private bool _isDisposed; + private bool _isValueSet; + private byte _evictionReason; + private bool _isExpired; + + private const int NotSet = -1; + + internal CacheEntry(K key, MemoryCache memoryCache) + { + Key = key; + _cache = memoryCache; + } + + // internal for testing + internal static CacheEntry? Current => _current.Value; + + + /// + /// Gets or sets the callbacks will be fired after the cache entry is evicted from the cache. + /// + public IList> PostEvictionCallbacks => GetOrCreateTokens().PostEvictionCallbacks; + + internal long Size => _size; + + long? ICacheEntry.Size + { + get => _size < 0 ? null : _size; + set + { + if (value < 0) + { + throw new ArgumentOutOfRangeException(nameof(value), value, $"{nameof(value)} must be non-negative."); + } + + _size = value ?? NotSet; + } + } + + public K Key { get; } + + public V Value + { + get => _value; + set + { + _value = value; + _isValueSet = true; + } + } + + internal DateTime LastAccessed { get; set; } + + internal EvictionReason EvictionReason { get => (EvictionReason)_evictionReason; private set => _evictionReason = (byte)value; } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] // added based on profiling + internal bool CheckExpired(DateTime utcNow) + => _isExpired; + + public void Dispose() + { + if (!_isDisposed) + { + _isDisposed = true; + + if (_isValueSet) + { + _cache.SetEntry(this); + } + } + } + + public void SetExpired(EvictionReason reason) + { + EvictionReason = reason; + _isExpired = true; + } + + internal void InvokeEvictionCallbacks() => _tokens?.InvokeEvictionCallbacks(this); + + private CacheEntryTokens GetOrCreateTokens() + { + if (_tokens != null) + { + return _tokens; + } + + CacheEntryTokens result = new CacheEntryTokens(); + return Interlocked.CompareExchange(ref _tokens, result, null) ?? result; + } + } +} diff --git a/core/State/Cache/Internal/CacheEntryExtensions.cs b/core/State/Cache/Internal/CacheEntryExtensions.cs new file mode 100644 index 00000000..e609039f --- /dev/null +++ b/core/State/Cache/Internal/CacheEntryExtensions.cs @@ -0,0 +1,130 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// This is a fork from Microsoft.Extensions.Caching.Memory.MemoryCache https://github.com/dotnet/runtime/blob/main/src/libraries/Microsoft.Extensions.Caching.Memory +// The only difference is the compaction process and eviction callback is synchronous whereas the .NET repo is asyncrhonous + +using System; + +namespace Streamiz.Kafka.Net.State.Cache.Internal +{ + /// + /// Provide extensions methods for operations. + /// + internal static class CacheEntryExtensions + { + /// + /// The given callback will be fired after the cache entry is evicted from the cache. + /// + /// The . + /// The callback to run after the entry is evicted. + /// The for chaining. + internal static ICacheEntry RegisterPostEvictionCallback( + this ICacheEntry entry, + PostEvictionDelegate callback) + where K : class + where V : class + { + return entry.RegisterPostEvictionCallbackNoValidation(callback, state: null); + } + + /// + /// The given callback will be fired after the cache entry is evicted from the cache. + /// + /// The . + /// The callback to run after the entry is evicted. + /// The state to pass to the post-eviction callback. + /// The for chaining. + public static ICacheEntry RegisterPostEvictionCallback( + this ICacheEntry entry, + PostEvictionDelegate callback, + MemoryCache? state) + where K : class + where V : class + { + return entry.RegisterPostEvictionCallbackNoValidation(callback, state); + } + + private static ICacheEntry RegisterPostEvictionCallbackNoValidation( + this ICacheEntry entry, + PostEvictionDelegate callback, + MemoryCache? state) + where K : class + where V : class + { + entry.PostEvictionCallbacks.Add(new PostEvictionCallbackRegistration() + { + EvictionCallback = callback, + State = state + }); + return entry; + } + + /// + /// Sets the value of the cache entry. + /// + /// The . + /// The value to set on the . + /// The for chaining. + public static ICacheEntry SetValue( + this ICacheEntry entry, + V value) + where K : class + where V : class + { + entry.Value = value; + return entry; + } + + /// + /// Sets the size of the cache entry value. + /// + /// The . + /// The size to set on the . + /// The for chaining. + public static ICacheEntry SetSize( + this ICacheEntry entry, + long size) + where K : class + where V : class + { + if (size < 0) + { + throw new ArgumentOutOfRangeException(nameof(size), size, $"{nameof(size)} must be non-negative."); + } + + entry.Size = size; + return entry; + } + + /// + /// Applies the values of an existing to the entry. + /// + /// The . + /// Set the values of these options on the . + /// The for chaining. + public static ICacheEntry SetOptions(this ICacheEntry entry, MemoryCacheEntryOptions options) + where K : class + where V : class + { + entry.Size = options.Size; + + for (int i = 0; i < options.PostEvictionCallbacks.Count; i++) + { + PostEvictionCallbackRegistration postEvictionCallback = options.PostEvictionCallbacks[i]; + if (postEvictionCallback.EvictionCallback is null) + ThrowNullCallback(i, nameof(options)); + + entry.RegisterPostEvictionCallbackNoValidation(postEvictionCallback.EvictionCallback, postEvictionCallback.State); + } + + return entry; + } + + private static void ThrowNullCallback(int index, string paramName) + { + string message = + $"MemoryCacheEntryOptions.PostEvictionCallbacks contains a PostEvictionCallbackRegistration with a null EvictionCallback at index {index}."; + throw new ArgumentException(message, paramName); + } + } +} diff --git a/core/State/Cache/Internal/EvictionReason.cs b/core/State/Cache/Internal/EvictionReason.cs new file mode 100644 index 00000000..fdf83114 --- /dev/null +++ b/core/State/Cache/Internal/EvictionReason.cs @@ -0,0 +1,38 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// This is a fork from Microsoft.Extensions.Caching.Memory.MemoryCache https://github.com/dotnet/runtime/blob/main/src/libraries/Microsoft.Extensions.Caching.Memory +// The only difference is the compaction process and eviction callback is synchronous whereas the .NET repo is asyncrhonous + +namespace Streamiz.Kafka.Net.State.Cache.Internal +{ + /// + /// Specify the reasons why an entry was evicted from the cache. + /// + internal enum EvictionReason + { + /// + /// The item was not removed from the cache or getting from the internal wrapped store + /// + None, + + /// + /// The item was adding to the cache. + /// + Setted, + + /// + /// The item was removed from the cache manually. + /// + Removed, + + /// + /// The item was removed from the cache because it was overwritten. + /// + Replaced, + + /// + /// The item was removed from the cache because it exceeded its capacity. + /// + Capacity, + } +} diff --git a/core/State/Cache/Internal/ICacheEntry.cs b/core/State/Cache/Internal/ICacheEntry.cs new file mode 100644 index 00000000..07bbc6a0 --- /dev/null +++ b/core/State/Cache/Internal/ICacheEntry.cs @@ -0,0 +1,41 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// This is a fork from Microsoft.Extensions.Caching.Memory.MemoryCache https://github.com/dotnet/runtime/blob/main/src/libraries/Microsoft.Extensions.Caching.Memory +// The only difference is the compaction process and eviction callback is synchronous whereas the .NET repo is asyncrhonous + +using System; +using System.Collections.Generic; + +namespace Streamiz.Kafka.Net.State.Cache.Internal +{ + /// + /// Represents an entry in the implementation. + /// When Disposed, is committed to the cache. + /// + internal interface ICacheEntry : IDisposable + where K : class + where V : class + { + /// + /// Gets the key of the cache entry. + /// + K Key { get; } + + /// + /// Gets or set the value of the cache entry. + /// + V Value { get; set; } + + /// + /// Gets or sets the callbacks will be fired after the cache entry is evicted from the cache. + /// + IList> PostEvictionCallbacks { get; } + + /// + /// Gets or set the size of the cache entry value. + /// + long? Size { get; set; } + + void SetExpired(EvictionReason reason); + } +} diff --git a/core/State/Cache/Internal/IClockTime.cs b/core/State/Cache/Internal/IClockTime.cs new file mode 100644 index 00000000..14b52ac1 --- /dev/null +++ b/core/State/Cache/Internal/IClockTime.cs @@ -0,0 +1,31 @@ +using System; + +namespace Streamiz.Kafka.Net.State.Cache.Internal +{ + internal interface IClockTime + { + DateTime GetCurrentTime(); + } + + internal class ClockSystemTime : IClockTime + { + public DateTime GetCurrentTime() => DateTime.Now; + } + + + internal class MockSystemTime : IClockTime + { + private DateTime initialTime; + private DateTime currentTime; + + public MockSystemTime(DateTime initialTime) + { + this.initialTime = initialTime; + currentTime = initialTime; + } + + public void AdvanceTime(TimeSpan timeSpan) => currentTime = currentTime.Add(timeSpan); + public void ReduceTime(TimeSpan timeSpan) => currentTime = currentTime.Subtract(timeSpan); + public DateTime GetCurrentTime() => currentTime; + } +} \ No newline at end of file diff --git a/core/State/Cache/Internal/IMemoryCache.cs b/core/State/Cache/Internal/IMemoryCache.cs new file mode 100644 index 00000000..2a2c172e --- /dev/null +++ b/core/State/Cache/Internal/IMemoryCache.cs @@ -0,0 +1,44 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// This is a fork from Microsoft.Extensions.Caching.Memory.MemoryCache https://github.com/dotnet/runtime/blob/main/src/libraries/Microsoft.Extensions.Caching.Memory +// The only difference is the compaction process and eviction callback is synchronous whereas the .NET repo is asyncrhonous + +using System; + +namespace Streamiz.Kafka.Net.State.Cache.Internal +{ + /// + /// Represents a local in-memory cache whose values are not serialized. + /// + internal interface IMemoryCache : IDisposable + where K : class + where V : class + { + /// + /// Gets the item associated with this key if present. + /// + /// An object identifying the requested entry. + /// The located value or null. + /// True if the key was found. + bool TryGetValue(K key, out V value); + + /// + /// Create or overwrite an entry in the cache. + /// + /// An object identifying the entry. + /// The newly created instance. + ICacheEntry CreateEntry(K key); + + /// + /// Removes the object associated with the given key. + /// + /// An object identifying the entry. + void Remove(K key); + + /// + /// Gets a snapshot of the cache statistics if available. + /// + /// An instance of containing a snapshot of the cache statistics. + MemoryCacheStatistics GetCurrentStatistics(); + } +} diff --git a/core/State/Cache/Internal/MemoryCache.cs b/core/State/Cache/Internal/MemoryCache.cs new file mode 100644 index 00000000..51f1a718 --- /dev/null +++ b/core/State/Cache/Internal/MemoryCache.cs @@ -0,0 +1,565 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// This is a fork from Microsoft.Extensions.Caching.Memory.MemoryCache https://github.com/dotnet/runtime/blob/main/src/libraries/Microsoft.Extensions.Caching.Memory +// The difference is the compaction process and eviction callback is synchronous whereas the .NET repo is asynchronous + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Streamiz.Kafka.Net.Crosscutting; + +namespace Streamiz.Kafka.Net.State.Cache.Internal +{ + /// + /// An implementation of using a dictionary to + /// store its entries. + /// + internal sealed class MemoryCache : IMemoryCache + where K : class + where V : class + { + private readonly IComparer _keyComparer; + private readonly IClockTime _clockTime; + internal readonly ILogger Logger; + + private readonly MemoryCacheOptions _options; + + private readonly List> _allStats; + private readonly Stats _accumulatedStats; + private readonly ThreadLocal _stats; + private CoherentState _coherentState; + private bool _disposed; + + /// + /// Creates a new instance. + /// + /// The options of the cache. + /// Compare the key + public MemoryCache(IOptions optionsAccessor, IComparer keyComparer) + : this(optionsAccessor, keyComparer, NullLoggerFactory.Instance, new ClockSystemTime()) + { + } + + /// + /// Creates a new instance. + /// + /// The options of the cache. + /// Compare the key + public MemoryCache(IOptions optionsAccessor, IComparer keyComparer, IClockTime clockTime) + : this(optionsAccessor, keyComparer, NullLoggerFactory.Instance, clockTime) + { + } + + /// + /// Creates a new instance. + /// + /// The options of the cache. + /// Compare the key + /// The factory used to create loggers. + /// Clock time accessor + private MemoryCache( + IOptions optionsAccessor, + IComparer keyComparer, + ILoggerFactory loggerFactory, + IClockTime clockTime) + { + Utils.CheckIfNotNull(optionsAccessor, nameof(optionsAccessor)); + Utils.CheckIfNotNull(loggerFactory, nameof(loggerFactory)); + Utils.CheckIfNotNull(keyComparer, nameof(keyComparer)); + + _keyComparer = keyComparer; + _clockTime = clockTime; + _options = optionsAccessor.Value; + Logger = loggerFactory.CreateLogger>(); + + _coherentState = new CoherentState(); + + _allStats = new List>(); + _accumulatedStats = new Stats(); + _stats = new ThreadLocal(() => new Stats(this)); + } + + private DateTime UtcNow => _clockTime.GetCurrentTime(); + + /// + /// Cleans up the background collection events. + /// + ~MemoryCache() => Dispose(false); + + /// + /// Gets the count of the current entries for diagnostic purposes. + /// + public int Count => _coherentState.Count; + + /// + /// Gets an enumerable of the all the keys in the . + /// + private IEnumerable Keys => _coherentState.Entries.Keys; + + internal IEnumerable KeySetEnumerable(bool forward) + { + return forward ? Keys : Keys.OrderByDescending(k => k, _keyComparer); + } + + // Maybe used Keys accessor .. + internal IEnumerable KeyRange(K from, K to, bool inclusive, bool forward) + { + if (from == null && to == null) + return KeySetEnumerable(forward); + + IEnumerable selectedKeys; + + if (from == null) + selectedKeys = _coherentState.Entries + .HeadMap(to, inclusive) + .Select(kv => kv.Key); + else if (to == null) + selectedKeys = _coherentState.Entries + .TailMap(from, true) + .Select(kv => kv.Key); + else if (_keyComparer.Compare(from, to) > 0) + selectedKeys = new List(); + else + selectedKeys = _coherentState.Entries + .SubMap(from, to, true, inclusive) + .Select(kv => kv.Key); + + return forward ? selectedKeys : selectedKeys.OrderByDescending(k => k, _keyComparer); + } + + /// + /// Internal accessor for Size for testing only. + /// + /// Note that this is only eventually consistent with the contents of the collection. + /// See comment on . + /// + internal long Size => _coherentState.Size; + + /// + public ICacheEntry CreateEntry(K key) + { + CheckDisposed(); + ValidateCacheKey(key); + + return new CacheEntry(key, this); + } + + internal void SetEntry(CacheEntry entry) + { + if (_disposed) + { + // No-op instead of throwing since this is called during CacheEntry.Dispose + return; + } + + DateTime utcNow = UtcNow; + + // Initialize the last access timestamp at the time the entry is added + entry.LastAccessed = utcNow; + + CoherentState coherentState = _coherentState; // Clear() can update the reference in the meantime + if (coherentState.Entries.TryGetValue(entry.Key, out CacheEntry priorEntry)) + priorEntry.SetExpired(EvictionReason.Replaced); + + if (!UpdateCacheSizeExceedsCapacity(entry, coherentState)) + { + if (priorEntry == null) + { + // Try to add the new entry if no previous entries exist. + coherentState.Entries.TryAdd(entry.Key, entry); + } + else + { + // Try to update with the new entry if a previous entries exist. + coherentState.Entries.AddOrUpdate(entry.Key, entry); + // The prior entry was removed, decrease the by the prior entry's size + Interlocked.Add(ref coherentState.CacheSize, -priorEntry.Size); + } + + priorEntry?.InvokeEvictionCallbacks(); + } + else + { + TriggerOvercapacityCompaction(); + coherentState.Entries.TryAdd(entry.Key, entry); + Interlocked.Add(ref coherentState.CacheSize, entry.Size); + } + } + + /// + public bool TryGetValue(K key, out V value) + { + if (key == null) + throw new ArgumentException(); + + CheckDisposed(); + + DateTime utcNow = UtcNow; + + CoherentState coherentState = _coherentState; // Clear() can update the reference in the meantime + if (coherentState.Entries.TryGetValue(key, out CacheEntry tmp)) + { + CacheEntry entry = tmp; + entry.LastAccessed = utcNow; + value = entry.Value; + + // Hit + lock (_allStats!) + { + if (_allStats is not null) + { + if (IntPtr.Size == 4) + Interlocked.Increment(ref GetStats().Hits); + else + GetStats().Hits++; + } + } + + return true; + } + + value = default(V); + // Miss + lock (_allStats!) + { + if (_allStats is not null) + { + if (IntPtr.Size == 4) + Interlocked.Increment(ref GetStats().Misses); + else + GetStats().Misses++; + } + } + + return false; + } + + /// + public void Remove(K key) + { + if (key == null) + throw new ArgumentException(); + + CheckDisposed(); + + CoherentState coherentState = _coherentState; // Clear() can update the reference in the meantime + if (coherentState.Entries.Remove(key, out CacheEntry entry)) + { + Interlocked.Add(ref coherentState.CacheSize, -entry.Size); + entry.SetExpired(EvictionReason.Removed); + entry.InvokeEvictionCallbacks(); + } + } + + /// + /// Removes all keys and values from the cache. + /// + public void Clear() + { + CheckDisposed(); + + CoherentState oldState = Interlocked.Exchange(ref _coherentState, new CoherentState()); + foreach (KeyValuePair> entry in oldState.Entries) + { + entry.Value.SetExpired(EvictionReason.Removed); + entry.Value.InvokeEvictionCallbacks(); + } + } + + /// + /// Gets a snapshot of the current statistics for the memory cache. + /// + /// Returns statistics tracked. + public MemoryCacheStatistics GetCurrentStatistics() + { + lock (_allStats!) + { + if (_allStats is not null) + { + (long hit, long miss) sumTotal = Sum(); + return new MemoryCacheStatistics() + { + TotalMisses = sumTotal.miss, + TotalHits = sumTotal.hit, + CurrentEntryCount = Count, + CurrentEstimatedSize = Size + }; + } + } + + return null; + } + + private (long, long) Sum() + { + lock (_allStats!) + { + long hits = _accumulatedStats!.Hits; + long misses = _accumulatedStats.Misses; + + foreach (WeakReference wr in _allStats) + { + if (wr.TryGetTarget(out Stats? stats)) + { + hits += Interlocked.Read(ref stats.Hits); + misses += Interlocked.Read(ref stats.Misses); + } + } + + return (hits, misses); + } + } + + private Stats GetStats() => _stats!.Value!; + + private sealed class Stats + { + private readonly MemoryCache _memoryCache; + public long Hits; + public long Misses; + + public Stats() { } + + public Stats(MemoryCache memoryCache) + { + _memoryCache = memoryCache; + _memoryCache.AddToStats(this); + } + + ~Stats() => _memoryCache?.RemoveFromStats(this); + } + + private void RemoveFromStats(Stats current) + { + lock (_allStats!) + { + for (int i = 0; i < _allStats.Count; i++) + { + if (!_allStats[i].TryGetTarget(out Stats? stats)) + { + _allStats.RemoveAt(i); + i--; + } + } + + _accumulatedStats!.Hits += Interlocked.Read(ref current.Hits); + _accumulatedStats.Misses += Interlocked.Read(ref current.Misses); + _allStats.TrimExcess(); + } + } + + private void AddToStats(Stats current) + { + lock (_allStats!) + { + _allStats.Add(new WeakReference(current)); + } + } + + /// + /// Returns true if increasing the cache size by the size of entry would + /// cause it to exceed any size limit on the cache, otherwise, returns false. + /// + private bool UpdateCacheSizeExceedsCapacity(CacheEntry entry, CoherentState coherentState) + { + long sizeRead = coherentState.Size; + for (int i = 0; i < 100; i++) + { + long newSize = sizeRead + entry.Size; + + if ((ulong)newSize > (ulong)_options.SizeLimit) + { + // Overflow occurred, return true without updating the cache size + return true; + } + + long original = Interlocked.CompareExchange(ref coherentState.CacheSize, newSize, sizeRead); + if (sizeRead == original) + { + return false; + } + sizeRead = original; + } + + return true; + } + + private void TriggerOvercapacityCompaction() + { + if (Logger.IsEnabled(LogLevel.Debug)) + Logger.LogDebug("Overcapacity compaction triggered"); + + OvercapacityCompaction(); + } + + private void OvercapacityCompaction() + { + CoherentState coherentState = _coherentState; // Clear() can update the reference in the meantime + long currentSize = coherentState.Size; + + if (Logger.IsEnabled(LogLevel.Debug)) + Logger.LogDebug($"Overcapacity compaction executing. Current size {currentSize}"); + + long sizeLimit = _options.SizeLimit; + if (sizeLimit >= 0) + { + long lowWatermark = sizeLimit - (long)(sizeLimit * _options.CompactionPercentage); + if (currentSize > lowWatermark) + { + Compact(currentSize - (long)lowWatermark, entry => entry.Size, coherentState); + } + } + + if (Logger.IsEnabled(LogLevel.Debug)) + Logger.LogDebug($"Overcapacity compaction executed. New size {coherentState.Size}"); + } + + /// Remove at least the given percentage (0.10 for 10%) of the total entries (or estimated memory?), according to the following policy: + /// 1. Remove all items if the percentage is more than 100% + /// 2. Least recently used objects. + public void Compact(double percentage) + { + if (percentage >= 1) // clear all the cache + Flush(); + else + { + CoherentState coherentState = _coherentState; // Clear() can update the reference in the meantime + int removalCountTarget = (int)(coherentState.Count * percentage); + Compact(removalCountTarget, _ => 1, coherentState); + } + } + + private void Flush() + { + var entriesToRemove = new List>(); + using var enumerator = _coherentState + .Entries + .OrderBy(e => e.Value.LastAccessed) + .GetEnumerator(); + + while (enumerator.MoveNext()) + entriesToRemove.Add(enumerator.Current.Value); + + foreach (CacheEntry entry in entriesToRemove) + _coherentState.RemoveEntry(entry, _options); + } + + private void Compact(long removalSizeTarget, Func, long> computeEntrySize, CoherentState coherentState) + { + var entriesToRemove = new List>(); + long removedSize = 0; + + ExpireLruBucket(ref removedSize, removalSizeTarget, computeEntrySize, entriesToRemove, coherentState.Entries); + + foreach (CacheEntry entry in entriesToRemove) + coherentState.RemoveEntry(entry, _options); + + // Expire the least recently used objects. + static void ExpireLruBucket(ref long removedSize, long removalSizeTarget, Func, long> computeEntrySize, List> entriesToRemove, SortedDictionary> priorityEntries) + { + // Do we meet our quota by just removing expired entries? + if (removalSizeTarget <= removedSize) + { + // No-op, we've met quota + return; + } + + // LRU + foreach (var keyValuePair in priorityEntries.OrderBy(e => e.Value.LastAccessed)) + { + keyValuePair.Value.SetExpired(EvictionReason.Capacity); + entriesToRemove.Add(keyValuePair.Value); + removedSize += computeEntrySize(keyValuePair.Value); + + if (removalSizeTarget <= removedSize) + { + break; + } + } + } + } + + /// + public void Dispose() + { + Dispose(true); + } + + /// + /// Dispose the cache and clear all entries. + /// + /// Dispose the object resources if true; otherwise, take no action. + private void Dispose(bool disposing) + { + if (!_disposed) + { + if (disposing) + { + _stats?.Dispose(); + GC.SuppressFinalize(this); + } + + _disposed = true; + } + } + + private void CheckDisposed() + { + if (_disposed) + { + Throw(); + } + + static void Throw() => throw new ObjectDisposedException(typeof(MemoryCache).FullName); + } + + private static void ValidateCacheKey(object key) + { + if (key == null) + throw new ArgumentException(); + } + + /// + /// Wrapper for the memory cache entries collection. + /// + /// Entries may have various sizes. If a size limit has been set, the cache keeps track of the aggregate of all the entries' sizes + /// in order to trigger compaction when the size limit is exceeded. + /// + /// For performance reasons, the size is not updated atomically with the collection, but is only made eventually consistent. + /// + /// When the memory cache is cleared, it replaces the backing collection entirely. This may occur in parallel with operations + /// like add, set, remove, and compact which may modify the collection and thus its overall size. + /// + /// To keep the overall size eventually consistent, therefore, the collection and the overall size are wrapped in this CoherentState + /// object. Individual operations take a local reference to this wrapper object while they work, and make size updates to this object. + /// Clearing the cache simply replaces the object, so that any still in progress updates do not affect the overall size value for + /// the new backing collection. + /// + private sealed class CoherentState + where K : class + where V : class + { + internal readonly SortedDictionary> Entries = new(); + internal long CacheSize; + + private ICollection>> EntriesCollection => Entries; + + internal int Count => Entries.Count; + + internal long Size => Volatile.Read(ref CacheSize); + + internal void RemoveEntry(CacheEntry entry, MemoryCacheOptions options) + { + if (EntriesCollection.Remove(new KeyValuePair>(entry.Key, entry))) + { + Interlocked.Add(ref CacheSize, -entry.Size); + entry.InvokeEvictionCallbacks(); + } + } + } + } +} diff --git a/core/State/Cache/Internal/MemoryCacheEntryExtensions.cs b/core/State/Cache/Internal/MemoryCacheEntryExtensions.cs new file mode 100644 index 00000000..d84f3d7b --- /dev/null +++ b/core/State/Cache/Internal/MemoryCacheEntryExtensions.cs @@ -0,0 +1,58 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// This is a fork from Microsoft.Extensions.Caching.Memory.MemoryCache https://github.com/dotnet/runtime/blob/main/src/libraries/Microsoft.Extensions.Caching.Memory +// The only difference is the compaction process and eviction callback is synchronous whereas the .NET repo is asyncrhonous + +using System; + +namespace Streamiz.Kafka.Net.State.Cache.Internal +{ + /// + /// Provide extensions methods for operations. + /// + internal static class MemoryCacheEntryExtensions + { + /// + /// Sets the size of the cache entry value. + /// + /// The options to set the entry size on. + /// The size to set on the . + /// The so that additional calls can be chained. + internal static MemoryCacheEntryOptions SetSize( + this MemoryCacheEntryOptions options, + long size) + where K : class + where V : class + { + if (size < 0) + { + throw new ArgumentOutOfRangeException(nameof(size), size, $"{nameof(size)} must be non-negative."); + } + + options.Size = size; + return options; + } + + /// + /// The given callback will be fired after the cache entry is evicted from the cache. + /// + /// The . + /// The callback to register for calling after an entry is evicted. + /// The state to pass to the callback. + /// The so that additional calls can be chained. + internal static MemoryCacheEntryOptions RegisterPostEvictionCallback( + this MemoryCacheEntryOptions options, + PostEvictionDelegate callback, + MemoryCache state) + where K : class + where V : class + { + options.PostEvictionCallbacks.Add(new PostEvictionCallbackRegistration() + { + EvictionCallback = callback, + State = state + }); + return options; + } + } +} diff --git a/core/State/Cache/Internal/MemoryCacheEntryOptions.cs b/core/State/Cache/Internal/MemoryCacheEntryOptions.cs new file mode 100644 index 00000000..cab859fe --- /dev/null +++ b/core/State/Cache/Internal/MemoryCacheEntryOptions.cs @@ -0,0 +1,45 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// This is a fork from Microsoft.Extensions.Caching.Memory.MemoryCache https://github.com/dotnet/runtime/blob/main/src/libraries/Microsoft.Extensions.Caching.Memory +// The only difference is the compaction process and eviction callback is synchronous whereas the .NET repo is asyncrhonous + +using System; +using System.Collections.Generic; +using Microsoft.Extensions.Primitives; + +namespace Streamiz.Kafka.Net.State.Cache.Internal +{ + /// + /// Represents the cache options applied to an entry of the instance. + /// + internal class MemoryCacheEntryOptions + where K : class + where V : class + { + private long? _size; + + + /// + /// Gets or sets the callbacks will be fired after the cache entry is evicted from the cache. + /// + public IList> PostEvictionCallbacks { get; } + = new List>(); + + /// + /// Gets or sets the size of the cache entry value. + /// + public long? Size + { + get => _size; + set + { + if (value < 0) + { + throw new ArgumentOutOfRangeException(nameof(value), value, $"{nameof(value)} must be non-negative."); + } + + _size = value; + } + } + } +} diff --git a/core/State/Cache/Internal/MemoryCacheExtensions.cs b/core/State/Cache/Internal/MemoryCacheExtensions.cs new file mode 100644 index 00000000..5b2ad3a8 --- /dev/null +++ b/core/State/Cache/Internal/MemoryCacheExtensions.cs @@ -0,0 +1,143 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// This is a fork from Microsoft.Extensions.Caching.Memory.MemoryCache https://github.com/dotnet/runtime/blob/main/src/libraries/Microsoft.Extensions.Caching.Memory +// The only difference is the compaction process and eviction callback is synchronous whereas the .NET repo is asyncrhonous + +using System; + +namespace Streamiz.Kafka.Net.State.Cache.Internal +{ + /// + /// Provide extensions methods for operations. + /// + internal static class CacheExtensions + { + /// + /// Gets the value associated with this key if present. + /// + /// The instance this method extends. + /// The key of the value to get. + /// The value associated with this key, or null if the key is not present. + internal static V Get(this IMemoryCache cache, K key) + where K : class + where V : class + { + cache.TryGetValue(key, out V value); + return value; + } + + /// + /// Try to get the value associated with the given key. + /// + /// The instance this method extends. + /// The key of the value to get. + /// The value associated with the given key. + /// true if the key was found. false otherwise. + internal static bool TryGetValue(this IMemoryCache cache, K key, out V value) + where K : class + where V : class + { + if (cache.TryGetValue(key, out V result)) + { + if (result == null) + { + value = default; + return false; // if result == null, is a delete + } + + if (result is { } item) + { + value = item; + return true; + } + } + + value = default; + return false; + } + + /// + /// Associate a value with a key in the . + /// + /// The instance this method extends. + /// The key of the entry to add. + /// The value to associate with the key. + /// The value that was set. + internal static V Set(this IMemoryCache cache, K key, V value) + where K : class + where V : class + { + using ICacheEntry entry = cache.CreateEntry(key); + entry.Value = value; + + return value; + } + + + /// + /// Sets a cache entry with the given key and value and apply the values of an existing to the created entry. + /// + /// The instance this method extends. + /// The key of the entry to add. + /// The value to associate with the key. + /// The existing instance to apply to the new entry. + /// Initial eviction reason + /// The value that was set. + internal static V Set(this IMemoryCache cache, K key, V value, MemoryCacheEntryOptions options, EvictionReason reason) + where K : class + where V : class + { + using ICacheEntry entry = cache.CreateEntry(key); + if (options != null) + { + entry.SetOptions(options); + } + + entry.Value = value; + entry.SetExpired(reason); + + return value; + } + + /// + /// Gets the value associated with this key if it exists, or generates a new entry using the provided key and a value from the given factory if the key is not found. + /// + /// The instance this method extends. + /// The key of the entry to look for or create. + /// The factory that creates the value associated with this key if the key does not exist in the cache. + /// The value associated with this key. + internal static V GetOrCreate(this IMemoryCache cache, K key, Func, V> factory) + where K : class + where V : class + { + return GetOrCreate(cache, key, factory, null); + } + + /// + /// Gets the value associated with this key if it exists, or generates a new entry using the provided key and a value from the given factory if the key is not found. + /// + /// The instance this method extends. + /// The key of the entry to look for or create. + /// The factory that creates the value associated with this key if the key does not exist in the cache. + /// The options to be applied to the if the key does not exist in the cache. + internal static V GetOrCreate(this IMemoryCache cache, K key, Func, V> factory, MemoryCacheEntryOptions? createOptions) + where K : class + where V : class + { + if (!cache.TryGetValue(key, out V result)) + { + using ICacheEntry entry = cache.CreateEntry(key); + + if (createOptions != null) + { + entry.SetOptions(createOptions); + } + + result = factory(entry); + entry.Value = result; + } + + return result; + } + } +} diff --git a/core/State/Cache/Internal/MemoryCacheOptions.cs b/core/State/Cache/Internal/MemoryCacheOptions.cs new file mode 100644 index 00000000..4c07d111 --- /dev/null +++ b/core/State/Cache/Internal/MemoryCacheOptions.cs @@ -0,0 +1,42 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// This is a fork from Microsoft.Extensions.Caching.Memory.MemoryCache https://github.com/dotnet/runtime/blob/main/src/libraries/Microsoft.Extensions.Caching.Memory +// The only difference is the compaction process and eviction callback is synchronous whereas the .NET repo is asyncrhonous + +using System; +using Microsoft.Extensions.Options; + +namespace Streamiz.Kafka.Net.State.Cache.Internal +{ + /// + /// Options class for . + /// + internal class MemoryCacheOptions : IOptions + { + private double _compactionPercentage = 0.05; + + /// + /// Gets or sets the maximum size of the cache. + /// + public long SizeLimit { get; set; } + + /// + /// Gets or sets the amount to compact the cache by when the maximum size is exceeded. + /// + public double CompactionPercentage + { + get => _compactionPercentage; + set + { + if (value is < 0 or > 1) + { + throw new ArgumentOutOfRangeException(nameof(value), value, $"{nameof(value)} must be between 0 and 1 inclusive."); + } + + _compactionPercentage = value; + } + } + + MemoryCacheOptions IOptions.Value => this; + } +} diff --git a/core/State/Cache/Internal/MemoryCacheStatistics.cs b/core/State/Cache/Internal/MemoryCacheStatistics.cs new file mode 100644 index 00000000..e28bca89 --- /dev/null +++ b/core/State/Cache/Internal/MemoryCacheStatistics.cs @@ -0,0 +1,39 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// This is a fork from Microsoft.Extensions.Caching.Memory.MemoryCache https://github.com/dotnet/runtime/blob/main/src/libraries/Microsoft.Extensions.Caching.Memory +// The only difference is the compaction process and eviction callback is synchronous whereas the .NET repo is asyncrhonous + +namespace Streamiz.Kafka.Net.State.Cache.Internal +{ + /// + /// Holds a snapshot of statistics for a memory cache. + /// + internal class MemoryCacheStatistics + { + /// + /// Initializes an instance of MemoryCacheStatistics. + /// + public MemoryCacheStatistics() { } + + /// + /// Gets the number of instances currently in the memory cache. + /// + public long CurrentEntryCount { get; set; } + + /// + /// Gets an estimated sum of all the values currently in the memory cache. + /// + /// Returns if size isn't being tracked. The common MemoryCache implementation tracks size whenever a SizeLimit is set on the cache. + public long? CurrentEstimatedSize { get; set; } + + /// + /// Gets the total number of cache misses. + /// + public long TotalMisses { get; set; } + + /// + /// Gets the total number of cache hits. + /// + public long TotalHits { get; set; } + } +} diff --git a/core/State/Cache/Internal/PostEvictionCallbackRegistration.cs b/core/State/Cache/Internal/PostEvictionCallbackRegistration.cs new file mode 100644 index 00000000..77f33a4d --- /dev/null +++ b/core/State/Cache/Internal/PostEvictionCallbackRegistration.cs @@ -0,0 +1,25 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// This is a fork from Microsoft.Extensions.Caching.Memory.MemoryCache https://github.com/dotnet/runtime/blob/main/src/libraries/Microsoft.Extensions.Caching.Memory +// The only difference is the compaction process and eviction callback is synchronous whereas the .NET repo is asyncrhonous + +namespace Streamiz.Kafka.Net.State.Cache.Internal +{ + /// + /// Represents a callback delegate that will be fired after an entry is evicted from the cache. + /// + internal class PostEvictionCallbackRegistration + where K : class + where V : class + { + /// + /// Gets or sets the callback delegate that will be fired after an entry is evicted from the cache. + /// + public PostEvictionDelegate EvictionCallback { get; set; } + + /// + /// Gets or sets the state to pass to the callback delegate. + /// + public MemoryCache? State { get; set; } + } +} diff --git a/core/State/Cache/Internal/PostEvictionDelegate.cs b/core/State/Cache/Internal/PostEvictionDelegate.cs new file mode 100644 index 00000000..86f7c8b7 --- /dev/null +++ b/core/State/Cache/Internal/PostEvictionDelegate.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// This is a fork from Microsoft.Extensions.Caching.Memory.MemoryCache https://github.com/dotnet/runtime/blob/main/src/libraries/Microsoft.Extensions.Caching.Memory +// The only difference is the compaction process and eviction callback is synchronous whereas the .NET repo is asyncrhonous + +namespace Streamiz.Kafka.Net.State.Cache.Internal +{ + /// + /// Signature of the callback which gets called when a cache entry expires. + /// + /// The key of the entry being evicted. + /// The value of the entry being evicted. + /// The . + /// The information that was passed when registering the callback. + internal delegate void PostEvictionDelegate(K key, V? value, EvictionReason reason, MemoryCache state) + where K : class + where V : class; +} diff --git a/core/State/Cache/LRUCache.cs b/core/State/Cache/LRUCache.cs deleted file mode 100644 index ae4e91b7..00000000 --- a/core/State/Cache/LRUCache.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace Streamiz.Kafka.Net.State.Cache -{ - internal class LRUCache - { - } -} diff --git a/core/State/Cache/SegmentedCacheFunction.cs b/core/State/Cache/SegmentedCacheFunction.cs new file mode 100644 index 00000000..644e703f --- /dev/null +++ b/core/State/Cache/SegmentedCacheFunction.cs @@ -0,0 +1,65 @@ +using System; +using Streamiz.Kafka.Net.Crosscutting; +using Streamiz.Kafka.Net.State.Internal; + +namespace Streamiz.Kafka.Net.State.Cache +{ + internal class SegmentedCacheFunction : ICacheFunction + { + private static int SEGMENT_ID_BYTES = 8; + + private IKeySchema keySchema; + public long SegmentInterval { get; } + + public SegmentedCacheFunction(IKeySchema keySchema, long segmentInterval) + { + this.keySchema = keySchema; + SegmentInterval = segmentInterval; + } + + public Bytes Key(Bytes cacheKey) => Bytes.Wrap(BytesFromCacheKey(cacheKey)); + + public Bytes CacheKey(Bytes cacheKey) => CacheKey(cacheKey, SegmentId(cacheKey)); + + public Bytes CacheKey(Bytes key, long segmentId) { + byte[] keyBytes = key.Get; + using var buf = ByteBuffer.Build(SEGMENT_ID_BYTES + keyBytes.Length, true); + buf.PutLong(segmentId).Put(keyBytes); + return Bytes.Wrap(buf.ToArray()); + } + + public byte[] BytesFromCacheKey( Bytes cacheKey) { + byte[] binaryKey = new byte[cacheKey.Get.Length - SEGMENT_ID_BYTES]; + Array.Copy(cacheKey.Get, SEGMENT_ID_BYTES, binaryKey, 0, binaryKey.Length); + return binaryKey; + } + + public virtual long SegmentId(Bytes key) { + return SegmentId(keySchema.SegmentTimestamp(key)); + } + + public long SegmentId( long timestamp) { + return timestamp / SegmentInterval; + } + + public int CompareSegmentedKeys( Bytes cacheKey, Bytes storeKey) { + long storeSegmentId = SegmentId(storeKey); + + using var byteBuffer = ByteBuffer.Build(cacheKey.Get, true); + long cacheSegmentId = byteBuffer.GetLong(0); + + int segmentCompare = cacheSegmentId.CompareTo(storeSegmentId); + if (segmentCompare == 0) { + byte[] cacheKeyBytes = cacheKey.Get; + byte[] storeKeyBytes = storeKey.Get; + return BytesComparer.Compare( + cacheKeyBytes + .AsSpan(SEGMENT_ID_BYTES, cacheKeyBytes.Length - SEGMENT_ID_BYTES) + .ToArray(), + storeKeyBytes); + } + + return segmentCompare; + } + } +} \ No newline at end of file diff --git a/core/State/Cache/ThreadLRUCache.cs b/core/State/Cache/ThreadLRUCache.cs deleted file mode 100644 index 918e9c5c..00000000 --- a/core/State/Cache/ThreadLRUCache.cs +++ /dev/null @@ -1,25 +0,0 @@ -using Streamiz.Kafka.Net.Crosscutting; -using System.Collections.Generic; -using Microsoft.Extensions.Logging; - -namespace Streamiz.Kafka.Net.State.Cache -{ - internal class ThreadLRUCache - { - private readonly ILogger logger; - private readonly long maxCacheSizeElements; - private readonly IDictionary caches = new Dictionary(); - - public ThreadLRUCache(string prefix, long maxCacheSizeElements) - { - this.maxCacheSizeElements = maxCacheSizeElements; - logger = Logger.GetLogger(typeof(ThreadLRUCache)); - } - - public void Flush(string @namespace) - { - } - - // public void - } -} diff --git a/core/State/Enumerator/CompositeKeyValueEnumerator.cs b/core/State/Enumerator/CompositeKeyValueEnumerator.cs index 289b852f..182c4cb5 100644 --- a/core/State/Enumerator/CompositeKeyValueEnumerator.cs +++ b/core/State/Enumerator/CompositeKeyValueEnumerator.cs @@ -110,7 +110,7 @@ public K PeekNextKey() public void Reset() { - current?.Reset(); + enumerator.ForEach(e => e.Reset()); current = null; index = 0; } diff --git a/core/State/Enumerator/EmptyEnumerator.cs b/core/State/Enumerator/EmptyEnumerator.cs index 77bc8721..fa38a64d 100644 --- a/core/State/Enumerator/EmptyEnumerator.cs +++ b/core/State/Enumerator/EmptyEnumerator.cs @@ -5,6 +5,8 @@ namespace Streamiz.Kafka.Net.State.Enumerator { internal class EmptyKeyValueEnumerator : IKeyValueEnumerator { + public static EmptyKeyValueEnumerator Empty => new(); + public KeyValuePair? Current => null; object IEnumerator.Current => null; diff --git a/core/State/Enumerator/KeyValueEnumerable.cs b/core/State/Enumerator/KeyValueEnumerable.cs new file mode 100644 index 00000000..83c9d60f --- /dev/null +++ b/core/State/Enumerator/KeyValueEnumerable.cs @@ -0,0 +1,54 @@ +using System.Collections; +using System.Collections.Generic; +using Streamiz.Kafka.Net.Crosscutting; +using Streamiz.Kafka.Net.Errors; + +namespace Streamiz.Kafka.Net.State.Enumerator +{ + internal class KeyValueEnumerable : IEnumerable> + { + #region Inner Class + private class WrappedEnumerator : IEnumerator> + { + private readonly string stateStoreName; + private readonly IKeyValueEnumerator enumerator; + + public WrappedEnumerator(string stateStoreName, IKeyValueEnumerator enumerator) + { + this.stateStoreName = stateStoreName; + this.enumerator = enumerator; + } + + public KeyValuePair Current + => enumerator.Current ?? throw new NotMoreValueException($"No more record present in your state store {stateStoreName}"); + + object IEnumerator.Current => Current; + + public void Dispose() + => enumerator.Dispose(); + + public bool MoveNext() + => enumerator.MoveNext(); + + public void Reset() + => enumerator.Reset(); + } + + #endregion + + private readonly IKeyValueEnumerator enumerator; + private readonly string stateStoreName; + + public KeyValueEnumerable(string stateStoreName, IKeyValueEnumerator enumerator) + { + this.stateStoreName = stateStoreName; + this.enumerator = enumerator; + } + + public IEnumerator> GetEnumerator() + => new WrappedEnumerator(stateStoreName, enumerator); + + IEnumerator IEnumerable.GetEnumerator() + => GetEnumerator(); + } +} \ No newline at end of file diff --git a/core/State/Enumerator/SegmentEnumerator.cs b/core/State/Enumerator/SegmentEnumerator.cs index aaba9c83..4e47d1a9 100644 --- a/core/State/Enumerator/SegmentEnumerator.cs +++ b/core/State/Enumerator/SegmentEnumerator.cs @@ -67,6 +67,9 @@ public bool MoveNext() } } + if (!hasNext && index == segmentsEnumerator.Count) // no more segment to iterate + CloseCurrentEnumerator(); + return currentEnumerator != null && hasNext; } diff --git a/core/State/Enumerator/WrapEnumerableKeyValueEnumerator.cs b/core/State/Enumerator/WrapEnumerableKeyValueEnumerator.cs index 643e156e..26bf69d6 100644 --- a/core/State/Enumerator/WrapEnumerableKeyValueEnumerator.cs +++ b/core/State/Enumerator/WrapEnumerableKeyValueEnumerator.cs @@ -7,13 +7,14 @@ namespace Streamiz.Kafka.Net.State.Enumerator internal class WrapEnumerableKeyValueEnumerator : IKeyValueEnumerator { - private readonly List> values; - private int index = 0; + private readonly IEnumerable> _enumerable; + private IEnumerator> _enumerator; private KeyValuePair? current = null; public WrapEnumerableKeyValueEnumerator(IEnumerable> enumerable) { - values = enumerable.ToList(); + _enumerable = enumerable; + _enumerator = enumerable.GetEnumerator(); } public KeyValuePair? Current => current; @@ -22,20 +23,17 @@ public WrapEnumerableKeyValueEnumerator(IEnumerable> enumerab public void Dispose() { - current = null; - index = 0; + _enumerator.Dispose(); } public bool MoveNext() { - if (values.Count > 0 && index < values.Count) - { - current = values[index]; - ++index; - return true; - } + var result = _enumerator.MoveNext(); + if (result) + current = new KeyValuePair(_enumerator.Current.Key, _enumerator.Current.Value); else - return false; + current = null; + return result; } public K PeekNextKey() @@ -43,8 +41,9 @@ public K PeekNextKey() public void Reset() { - index = 0; + _enumerator.Dispose(); current = null; + _enumerator = _enumerable.GetEnumerator(); } } } \ No newline at end of file diff --git a/core/State/InMemory/InMemoryWindowStore.cs b/core/State/InMemory/InMemoryWindowStore.cs index 13124445..997c9401 100644 --- a/core/State/InMemory/InMemoryWindowStore.cs +++ b/core/State/InMemory/InMemoryWindowStore.cs @@ -129,8 +129,6 @@ public void Close() closingCallback?.Invoke(this); disposed = true; } - else - throw new ObjectDisposedException("Enumerator was disposed"); } public void RemoveExpiredData(long time) @@ -273,8 +271,7 @@ internal class InMemoryWindowStore : IWindowStore private long observedStreamTime = -1; private int seqnum = 0; - private readonly ConcurrentDictionary> map = - new ConcurrentDictionary>(); + private readonly ConcurrentDictionary> map = new(); private readonly ISet openIterators = new HashSet(); diff --git a/core/State/InMemory/InMemoryWindowStoreSupplier.cs b/core/State/InMemory/InMemoryWindowStoreSupplier.cs index eb30ef6b..a93ca68e 100644 --- a/core/State/InMemory/InMemoryWindowStoreSupplier.cs +++ b/core/State/InMemory/InMemoryWindowStoreSupplier.cs @@ -51,6 +51,13 @@ public InMemoryWindowStoreSupplier(string storeName, TimeSpan retention, long? s /// public bool RetainDuplicates { get; set; } + /// + /// The size of the segments (in milliseconds) the store has. + /// If your store is segmented then this should be the size of segments in the underlying store. + /// It is also used to reduce the amount of data that is scanned when caching is enabled. + /// + public long SegmentInterval { get; set; } = 1; + /// /// Return a new instance. /// diff --git a/core/State/Internal/TimestampedKeyValueStoreMaterializer.cs b/core/State/Internal/TimestampedKeyValueStoreMaterializer.cs index d23b5515..1eada8f1 100644 --- a/core/State/Internal/TimestampedKeyValueStoreMaterializer.cs +++ b/core/State/Internal/TimestampedKeyValueStoreMaterializer.cs @@ -39,6 +39,10 @@ public IStoreBuilder> Materialize() { builder.WithCachingEnabled(); } + else + { + builder.WithCachingDisabled(); + } return builder; } diff --git a/core/State/RocksDb/Internal/RocksDbWindowKeySchema.cs b/core/State/Internal/WindowKeySchema.cs similarity index 94% rename from core/State/RocksDb/Internal/RocksDbWindowKeySchema.cs rename to core/State/Internal/WindowKeySchema.cs index d68d5385..d1865bb3 100644 --- a/core/State/RocksDb/Internal/RocksDbWindowKeySchema.cs +++ b/core/State/Internal/WindowKeySchema.cs @@ -6,12 +6,12 @@ using System.Collections.Generic; using System.Linq; -namespace Streamiz.Kafka.Net.State.RocksDb.Internal +namespace Streamiz.Kafka.Net.State.Internal { /// /// Like JAVA Implementation, no advantages to rewrite /// - internal class RocksDbWindowKeySchema : IKeySchema + internal class WindowKeySchema : IKeySchema { private static readonly IComparer bytesComparer = new BytesComparer(); @@ -29,7 +29,6 @@ public Func, bool> HasNextCondition(Bytes bin && time >= from && time <= to) return true; - } return false; }; diff --git a/core/State/Internal/WrappedKeyValueStore.cs b/core/State/Internal/WrappedKeyValueStore.cs index 36252e84..46c53cdc 100644 --- a/core/State/Internal/WrappedKeyValueStore.cs +++ b/core/State/Internal/WrappedKeyValueStore.cs @@ -1,11 +1,16 @@ -using Streamiz.Kafka.Net.Crosscutting; +using System; +using System.Collections.Generic; +using System.Runtime.InteropServices; +using Streamiz.Kafka.Net.Crosscutting; using Streamiz.Kafka.Net.Processors; using Streamiz.Kafka.Net.SerDes; +using Streamiz.Kafka.Net.State.Cache; +using Streamiz.Kafka.Net.Table.Internal; namespace Streamiz.Kafka.Net.State.Internal { - internal class WrappedKeyValueStore : - WrappedStateStore> + internal abstract class WrappedKeyValueStore : + WrappedStateStore>, ICachedStateStore { protected ISerDes keySerdes; protected ISerDes valueSerdes; @@ -17,7 +22,11 @@ public WrappedKeyValueStore(IKeyValueStore wrapped, ISerDes ke this.keySerdes = keySerdes; this.valueSerdes = valueSerdes; } + + public override bool IsCachedStore => wrapped is IWrappedStateStore { IsCachedStore: true }; + public abstract bool SetFlushListener(Action>> listener, bool sendOldChanges); + public virtual void InitStoreSerDes(ProcessorContext context) { if (!initStoreSerdes) diff --git a/core/State/Internal/WrappedStateStore.cs b/core/State/Internal/WrappedStateStore.cs index 3c5d3263..d262074e 100644 --- a/core/State/Internal/WrappedStateStore.cs +++ b/core/State/Internal/WrappedStateStore.cs @@ -1,15 +1,17 @@ using Confluent.Kafka; using Streamiz.Kafka.Net.Processors; using Streamiz.Kafka.Net.Processors.Internal; +using Streamiz.Kafka.Net.State.Cache; namespace Streamiz.Kafka.Net.State.Internal { internal interface IWrappedStateStore { IStateStore Wrapped { get; } + bool IsCachedStore { get; } } - internal class WrappedStore + internal static class WrappedStore { internal static bool IsTimestamped(IStateStore stateStore) { @@ -22,7 +24,9 @@ internal static bool IsTimestamped(IStateStore stateStore) } } - internal class WrappedStateStore : IStateStore, IWrappedStateStore + internal abstract class WrappedStateStore : + IStateStore, + IWrappedStateStore where S : IStateStore { protected ProcessorContext context; @@ -36,6 +40,7 @@ public WrappedStateStore(S wrapped) #region StateStore Impl + public abstract bool IsCachedStore { get; } public virtual string Name => wrapped.Name; public virtual bool Persistent => wrapped.Persistent; diff --git a/core/State/KeyValueStoreBuilder.cs b/core/State/KeyValueStoreBuilder.cs index 5991e67e..447ac3bb 100644 --- a/core/State/KeyValueStoreBuilder.cs +++ b/core/State/KeyValueStoreBuilder.cs @@ -1,6 +1,7 @@ using Streamiz.Kafka.Net.Crosscutting; using Streamiz.Kafka.Net.Errors; using Streamiz.Kafka.Net.SerDes; +using Streamiz.Kafka.Net.State.Cache; using Streamiz.Kafka.Net.State.Logging; using Streamiz.Kafka.Net.State.Metered; using Streamiz.Kafka.Net.State.Supplier; @@ -49,7 +50,7 @@ public override IKeyValueStore Build() var store = supplier.Get(); return new MeteredKeyValueStore( - WrapLogging(store), + WrapCaching(WrapLogging(store)), keySerdes, valueSerdes, supplier.MetricsScope); @@ -57,10 +58,12 @@ public override IKeyValueStore Build() private IKeyValueStore WrapLogging(IKeyValueStore inner) { - if (!LoggingEnabled) - return inner; + return !LoggingEnabled ? inner : new ChangeLoggingKeyValueBytesStore(inner); + } - return new ChangeLoggingKeyValueBytesStore(inner); + private IKeyValueStore WrapCaching(IKeyValueStore inner) + { + return !CachingEnabled ? inner : new CachingKeyValueStore(inner); } } } \ No newline at end of file diff --git a/core/State/Logging/ChangeLoggingKeyValueBytesStore.cs b/core/State/Logging/ChangeLoggingKeyValueBytesStore.cs index 9076c5c0..d306797f 100644 --- a/core/State/Logging/ChangeLoggingKeyValueBytesStore.cs +++ b/core/State/Logging/ChangeLoggingKeyValueBytesStore.cs @@ -14,6 +14,8 @@ public ChangeLoggingKeyValueBytesStore(IKeyValueStore wrapped) { } + public override bool IsCachedStore => false; + protected virtual void Publish(Bytes key, byte[] value) => context.Log(Name, key, value, context.RecordContext.Timestamp); diff --git a/core/State/Logging/ChangeLoggingWindowBytesStore.cs b/core/State/Logging/ChangeLoggingWindowBytesStore.cs index 7c9c91cf..f096cc72 100644 --- a/core/State/Logging/ChangeLoggingWindowBytesStore.cs +++ b/core/State/Logging/ChangeLoggingWindowBytesStore.cs @@ -18,6 +18,8 @@ public ChangeLoggingWindowBytesStore(IWindowStore wrapped, bool r { this.retainDuplicates = retainDuplicates; } + + public override bool IsCachedStore => false; protected virtual void Publish(Bytes key, byte[] value) => context.Log(Name, key, value, context.RecordContext.Timestamp); diff --git a/core/State/Metered/MeteredKeyValueStore.cs b/core/State/Metered/MeteredKeyValueStore.cs index 36775d33..aff60fde 100644 --- a/core/State/Metered/MeteredKeyValueStore.cs +++ b/core/State/Metered/MeteredKeyValueStore.cs @@ -8,8 +8,10 @@ using Streamiz.Kafka.Net.Metrics.Internal; using Streamiz.Kafka.Net.Processors; using Streamiz.Kafka.Net.SerDes; +using Streamiz.Kafka.Net.State.Cache; using Streamiz.Kafka.Net.State.Enumerator; using Streamiz.Kafka.Net.State.Internal; +using Streamiz.Kafka.Net.Table.Internal; using static Streamiz.Kafka.Net.Crosscutting.ActionHelper; namespace Streamiz.Kafka.Net.State.Metered @@ -57,6 +59,22 @@ protected virtual void RegisterMetrics() deleteSensor = StateStoreMetrics.DeleteSensor(context.Id, metricScope, Name, context.Metrics); } + public override bool SetFlushListener(Action>> listener, bool sendOldChanges) + { + if (wrapped is ICachedStateStore store) + { + return store.SetFlushListener( + (kv) => + { + var key = FromKey(Bytes.Wrap(kv.Key)); + var newValue = kv.Value.NewValue != null ? FromValue(kv.Value.NewValue) : default; + var oldValue = kv.Value.OldValue != null ? FromValue(kv.Value.OldValue) : default; + listener(new KeyValuePair>(key, new Change(oldValue, newValue))); + }, sendOldChanges); + } + return false; + } + // TODO : set theses methods in helper class, used in some part of the code private byte[] GetValueBytes(V value) { diff --git a/core/State/Metered/MeteredWindowStore.cs b/core/State/Metered/MeteredWindowStore.cs index 4be226f1..3fda88cb 100644 --- a/core/State/Metered/MeteredWindowStore.cs +++ b/core/State/Metered/MeteredWindowStore.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Threading; using Streamiz.Kafka.Net.Crosscutting; using Streamiz.Kafka.Net.Errors; @@ -6,15 +7,19 @@ using Streamiz.Kafka.Net.Metrics.Internal; using Streamiz.Kafka.Net.Processors; using Streamiz.Kafka.Net.SerDes; +using Streamiz.Kafka.Net.State.Cache; using Streamiz.Kafka.Net.State.Enumerator; +using Streamiz.Kafka.Net.State.Helper; using Streamiz.Kafka.Net.State.Internal; +using Streamiz.Kafka.Net.Table.Internal; using static Streamiz.Kafka.Net.Crosscutting.ActionHelper; namespace Streamiz.Kafka.Net.State.Metered { internal class MeteredWindowStore : WrappedStateStore>, - IWindowStore + IWindowStore, + ICachedStateStore, V> { private readonly long windowSizeMs; protected ISerDes keySerdes; @@ -39,6 +44,8 @@ public MeteredWindowStore( this.valueSerdes = valueSerdes; this.metricScope = metricScope; } + + public override bool IsCachedStore => ((IWrappedStateStore)wrapped).IsCachedStore; public virtual void InitStoreSerde(ProcessorContext context) { @@ -68,6 +75,21 @@ protected virtual void RegisterMetrics() flushSensor = StateStoreMetrics.FlushSensor(context.Id, metricScope, Name, context.Metrics); } + public bool SetFlushListener(Action, Change>> listener, bool sendOldChanges) + { + if (wrapped is ICachedStateStore store) + { + return store.SetFlushListener( + kv => { + var key = WindowKeyHelper.FromStoreKey(kv.Key, windowSizeMs, keySerdes, changelogTopic); + var newValue = kv.Value.NewValue != null ? FromValue(kv.Value.NewValue) : default; + var oldValue = kv.Value.OldValue != null ? FromValue(kv.Value.OldValue) : default; + listener(new KeyValuePair, Change>(key, new Change(oldValue, newValue))); + }, sendOldChanges); + } + return false; + } + private Bytes GetKeyBytes(K key) { if (keySerdes != null) diff --git a/core/State/RocksDb/IRocksDbAdapter.cs b/core/State/RocksDb/IRocksDbAdapter.cs index 0feae663..cccebe28 100644 --- a/core/State/RocksDb/IRocksDbAdapter.cs +++ b/core/State/RocksDb/IRocksDbAdapter.cs @@ -3,7 +3,7 @@ using Streamiz.Kafka.Net.State.Enumerator; using System.Collections.Generic; -namespace Streamiz.Kafka.Net.State.RocksDb +namespace Streamiz.Kafka.Net.State { internal interface IRocksDbAdapter { diff --git a/core/State/RocksDb/Internal/AbstractRocksDBSegmentedBytesStore.cs b/core/State/RocksDb/Internal/AbstractRocksDBSegmentedBytesStore.cs index 95a95b10..03a82367 100644 --- a/core/State/RocksDb/Internal/AbstractRocksDBSegmentedBytesStore.cs +++ b/core/State/RocksDb/Internal/AbstractRocksDBSegmentedBytesStore.cs @@ -8,7 +8,7 @@ using Streamiz.Kafka.Net.Metrics; using Streamiz.Kafka.Net.Metrics.Internal; -namespace Streamiz.Kafka.Net.State.RocksDb.Internal +namespace Streamiz.Kafka.Net.State.Internal { internal class AbstractRocksDBSegmentedBytesStore : ISegmentedBytesStore where S : ISegment diff --git a/core/State/RocksDb/Internal/RocksDbKeyValueSegment.cs b/core/State/RocksDb/Internal/RocksDbKeyValueSegment.cs index 7e32ae9c..2fd9a5cf 100644 --- a/core/State/RocksDb/Internal/RocksDbKeyValueSegment.cs +++ b/core/State/RocksDb/Internal/RocksDbKeyValueSegment.cs @@ -1,7 +1,7 @@ using Streamiz.Kafka.Net.State.Internal; using System; -namespace Streamiz.Kafka.Net.State.RocksDb.Internal +namespace Streamiz.Kafka.Net.State.Internal { internal class RocksDbKeyValueSegment : RocksDbKeyValueStore, IComparable, ISegment diff --git a/core/State/RocksDb/Internal/RocksDbKeyValueSegments.cs b/core/State/RocksDb/Internal/RocksDbKeyValueSegments.cs index 020bf164..5d68365f 100644 --- a/core/State/RocksDb/Internal/RocksDbKeyValueSegments.cs +++ b/core/State/RocksDb/Internal/RocksDbKeyValueSegments.cs @@ -1,6 +1,6 @@ using Streamiz.Kafka.Net.State.Internal; -namespace Streamiz.Kafka.Net.State.RocksDb.Internal +namespace Streamiz.Kafka.Net.State.Internal { internal class RocksDbKeyValueSegments : AbstractSegments { diff --git a/core/State/RocksDb/Internal/RocksDbSegmentedBytesStore.cs b/core/State/RocksDb/Internal/RocksDbSegmentedBytesStore.cs index 4f90ba96..192931fb 100644 --- a/core/State/RocksDb/Internal/RocksDbSegmentedBytesStore.cs +++ b/core/State/RocksDb/Internal/RocksDbSegmentedBytesStore.cs @@ -1,6 +1,6 @@ using Streamiz.Kafka.Net.State.Internal; -namespace Streamiz.Kafka.Net.State.RocksDb.Internal +namespace Streamiz.Kafka.Net.State.Internal { internal class RocksDbSegmentedBytesStore : AbstractRocksDBSegmentedBytesStore diff --git a/core/State/RocksDb/RocksDbEnumerator.cs b/core/State/RocksDb/RocksDbEnumerator.cs index 11474388..3eccc057 100644 --- a/core/State/RocksDb/RocksDbEnumerator.cs +++ b/core/State/RocksDb/RocksDbEnumerator.cs @@ -5,57 +5,8 @@ using System.Collections; using System.Collections.Generic; -namespace Streamiz.Kafka.Net.State.RocksDb +namespace Streamiz.Kafka.Net.State { - internal class RocksDbEnumerable : IEnumerable> - { - #region Inner Class - private class RocksDbWrappedEnumerator : IEnumerator> - { - private readonly string stateStoreName; - private readonly IKeyValueEnumerator enumerator; - - public RocksDbWrappedEnumerator(string stateStoreName, IKeyValueEnumerator enumerator) - { - this.stateStoreName = stateStoreName; - this.enumerator = enumerator; - } - - public KeyValuePair Current - => enumerator.Current.HasValue ? - enumerator.Current.Value : - throw new NotMoreValueException($"No more record present in your state store {stateStoreName}"); - - object IEnumerator.Current => Current; - - public void Dispose() - => enumerator.Dispose(); - - public bool MoveNext() - => enumerator.MoveNext(); - - public void Reset() - => enumerator.Reset(); - } - - #endregion - - private readonly IKeyValueEnumerator enumerator; - private readonly string stateStoreName; - - public RocksDbEnumerable(string stateStoreName, IKeyValueEnumerator enumerator) - { - this.stateStoreName = stateStoreName; - this.enumerator = enumerator; - } - - public IEnumerator> GetEnumerator() - => new RocksDbWrappedEnumerator(stateStoreName, enumerator); - - IEnumerator IEnumerable.GetEnumerator() - => GetEnumerator(); - } - internal class RocksDbEnumerator : IKeyValueEnumerator { protected Iterator iterator; @@ -100,7 +51,7 @@ public virtual bool MoveNext() } public Bytes PeekNextKey() - => Current.HasValue ? Current.Value.Key : null; + => Current?.Key; public void Reset() { diff --git a/core/State/RocksDb/RocksDbKeyValueBytesStoreSupplier.cs b/core/State/RocksDb/RocksDbKeyValueBytesStoreSupplier.cs index 008a43a3..610f4c09 100644 --- a/core/State/RocksDb/RocksDbKeyValueBytesStoreSupplier.cs +++ b/core/State/RocksDb/RocksDbKeyValueBytesStoreSupplier.cs @@ -1,7 +1,7 @@ using Streamiz.Kafka.Net.Crosscutting; using Streamiz.Kafka.Net.State.Supplier; -namespace Streamiz.Kafka.Net.State.RocksDb +namespace Streamiz.Kafka.Net.State { /// /// A rocksdb key/value store supplier used to create . diff --git a/core/State/RocksDb/RocksDbKeyValueStore.cs b/core/State/RocksDb/RocksDbKeyValueStore.cs index eb83ab2d..cab1a2f2 100644 --- a/core/State/RocksDb/RocksDbKeyValueStore.cs +++ b/core/State/RocksDb/RocksDbKeyValueStore.cs @@ -7,11 +7,9 @@ using System.Collections; using System.Collections.Generic; using System.IO; -using System.Linq; using Microsoft.Extensions.Logging; -using System.Collections.Concurrent; -namespace Streamiz.Kafka.Net.State.RocksDb +namespace Streamiz.Kafka.Net.State { #region RocksDb Enumerator Wrapper @@ -63,7 +61,8 @@ public class RocksDbKeyValueStore : IKeyValueStore { private static readonly ILogger log = Logger.GetLogger(typeof(RocksDbKeyValueStore)); - private readonly ConcurrentDictionary openIterators = new(); + private readonly ISet openIterators + = new HashSet(); private const Compression COMPRESSION_TYPE = Compression.No; @@ -163,8 +162,8 @@ public void Close() if (openIterators.Count != 0) { log.LogWarning("Closing {openIteratorsCount} open iterators for store {Name}", openIterators.Count, Name); - foreach (KeyValuePair entry in openIterators) - entry.Key.Dispose(); + foreach (WrappedRocksRbKeyValueEnumerator enumerator in openIterators) + enumerator.Dispose(); } IsOpen = false; @@ -439,23 +438,19 @@ private IKeyValueEnumerator Range(Bytes from, Bytes to, bool forw } CheckStateStoreOpen(); - + var rocksEnumerator = DbAdapter.Range(from, to, forward); - - Func remove = it => openIterators.TryRemove(it, out _); - var wrapped = new WrappedRocksRbKeyValueEnumerator(rocksEnumerator, remove); - openIterators.TryAdd(wrapped, true); + var wrapped = new WrappedRocksRbKeyValueEnumerator(rocksEnumerator, openIterators.Remove); + openIterators.Add(wrapped); return wrapped; } private IEnumerable> All(bool forward) { var enumerator = DbAdapter.All(forward); - - Func remove = it => openIterators.TryRemove(it, out _); - var wrapped = new WrappedRocksRbKeyValueEnumerator(enumerator, remove); - openIterators.AddOrUpdate(wrapped, true); - return new RocksDbEnumerable(Name, wrapped); + var wrapped = new WrappedRocksRbKeyValueEnumerator(enumerator, openIterators.Remove); + openIterators.Add(wrapped); + return new KeyValueEnumerable(Name, wrapped); } #endregion diff --git a/core/State/RocksDb/RocksDbOptions.cs b/core/State/RocksDb/RocksDbOptions.cs index 90877117..24f3f161 100644 --- a/core/State/RocksDb/RocksDbOptions.cs +++ b/core/State/RocksDb/RocksDbOptions.cs @@ -1,7 +1,7 @@ using RocksDbSharp; using System; -namespace Streamiz.Kafka.Net.State.RocksDb +namespace Streamiz.Kafka.Net.State { /// /// RocksDB log levels. diff --git a/core/State/RocksDb/RocksDbRangeEnumerator.cs b/core/State/RocksDb/RocksDbRangeEnumerator.cs index 0381a637..3713e0d0 100644 --- a/core/State/RocksDb/RocksDbRangeEnumerator.cs +++ b/core/State/RocksDb/RocksDbRangeEnumerator.cs @@ -4,7 +4,7 @@ using System.Collections.Generic; using System.Text; -namespace Streamiz.Kafka.Net.State.RocksDb +namespace Streamiz.Kafka.Net.State { internal class RocksDbRangeEnumerator : RocksDbEnumerator { diff --git a/core/State/RocksDb/RocksDbWindowBytesStoreSupplier.cs b/core/State/RocksDb/RocksDbWindowBytesStoreSupplier.cs index de23d561..6c6bb3de 100644 --- a/core/State/RocksDb/RocksDbWindowBytesStoreSupplier.cs +++ b/core/State/RocksDb/RocksDbWindowBytesStoreSupplier.cs @@ -1,18 +1,15 @@ using Streamiz.Kafka.Net.Crosscutting; -using Streamiz.Kafka.Net.State.RocksDb.Internal; using Streamiz.Kafka.Net.State.Supplier; using System; +using Streamiz.Kafka.Net.State.Internal; -namespace Streamiz.Kafka.Net.State.RocksDb +namespace Streamiz.Kafka.Net.State { /// /// A rocksdb key/value store supplier used to create . /// public class RocksDbWindowBytesStoreSupplier : IWindowBytesStoreSupplier { - private readonly long segmentInterval; - private readonly bool retainDuplicates; - /// /// Constructor with some arguments. /// @@ -30,9 +27,9 @@ public RocksDbWindowBytesStoreSupplier( { Name = storeName; Retention = (long)retention.TotalMilliseconds; - this.segmentInterval = segmentInterval; RetainDuplicates = retainDuplicates; WindowSize = size; + SegmentInterval = segmentInterval; } /// @@ -61,6 +58,13 @@ public RocksDbWindowBytesStoreSupplier( /// State store name /// public string Name { get; set; } + + /// + /// The size of the segments (in milliseconds) the store has. + /// If your store is segmented then this should be the size of segments in the underlying store. + /// It is also used to reduce the amount of data that is scanned when caching is enabled. + /// + public long SegmentInterval { get; set; } /// /// Build the rocksdb state store. @@ -72,8 +76,8 @@ public IWindowStore Get() new RocksDbSegmentedBytesStore( Name, Retention, - segmentInterval, - new RocksDbWindowKeySchema()), + SegmentInterval, + new WindowKeySchema()), WindowSize.HasValue ? WindowSize.Value : (long)TimeSpan.FromMinutes(1).TotalMilliseconds, RetainDuplicates); } diff --git a/core/State/RocksDb/RocksDbWindowStore.cs b/core/State/RocksDb/RocksDbWindowStore.cs index 96053d9c..0e0fdb79 100644 --- a/core/State/RocksDb/RocksDbWindowStore.cs +++ b/core/State/RocksDb/RocksDbWindowStore.cs @@ -2,10 +2,9 @@ using Streamiz.Kafka.Net.State.Enumerator; using Streamiz.Kafka.Net.State.Helper; using Streamiz.Kafka.Net.State.Internal; -using Streamiz.Kafka.Net.State.RocksDb.Internal; using System; -namespace Streamiz.Kafka.Net.State.RocksDb +namespace Streamiz.Kafka.Net.State { internal class RocksDbWindowStore : WrappedStateStore, IWindowStore @@ -24,6 +23,8 @@ public RocksDbWindowStore( this.retainDuplicates = retainDuplicates; } + public override bool IsCachedStore => false; + private void UpdateSeqNumber() { if (retainDuplicates) diff --git a/core/State/RocksDb/SingleColumnFamilyAdapter.cs b/core/State/RocksDb/SingleColumnFamilyAdapter.cs index f430c0ec..2e4d5567 100644 --- a/core/State/RocksDb/SingleColumnFamilyAdapter.cs +++ b/core/State/RocksDb/SingleColumnFamilyAdapter.cs @@ -5,7 +5,7 @@ using System; using System.Collections.Generic; -namespace Streamiz.Kafka.Net.State.RocksDb +namespace Streamiz.Kafka.Net.State { internal class SingleColumnFamilyAdapter : IRocksDbAdapter { diff --git a/core/State/RocksDb/WindowKeyBytes.cs b/core/State/RocksDb/WindowKeyBytes.cs index 592542b7..0cc65b2f 100644 --- a/core/State/RocksDb/WindowKeyBytes.cs +++ b/core/State/RocksDb/WindowKeyBytes.cs @@ -2,7 +2,7 @@ using Streamiz.Kafka.Net.State.Helper; using System.Collections.Generic; -namespace Streamiz.Kafka.Net.State.RocksDb +namespace Streamiz.Kafka.Net.State { internal class WindowKeyBytesComparer : IEqualityComparer, diff --git a/core/State/Stores.cs b/core/State/Stores.cs index e4895021..e3b7863a 100644 --- a/core/State/Stores.cs +++ b/core/State/Stores.cs @@ -1,6 +1,6 @@ using Streamiz.Kafka.Net.SerDes; using Streamiz.Kafka.Net.State.InMemory; -using Streamiz.Kafka.Net.State.RocksDb; +using Streamiz.Kafka.Net.State; using Streamiz.Kafka.Net.State.Supplier; using System; diff --git a/core/State/Supplier/IWindowBytesStoreSupplier.cs b/core/State/Supplier/IWindowBytesStoreSupplier.cs index f87291d4..db1d450c 100644 --- a/core/State/Supplier/IWindowBytesStoreSupplier.cs +++ b/core/State/Supplier/IWindowBytesStoreSupplier.cs @@ -29,5 +29,12 @@ public interface IWindowBytesStoreSupplier : IStoreSupplier public bool RetainDuplicates { get; set; } + + /// + /// The size of the segments (in milliseconds) the store has. + /// If your store is segmented then this should be the size of segments in the underlying store. + /// It is also used to reduce the amount of data that is scanned when caching is enabled. + /// + public long SegmentInterval { get; set; } } } diff --git a/core/State/TimestampedKeyValueStoreBuilder.cs b/core/State/TimestampedKeyValueStoreBuilder.cs index 12c00e01..a965d856 100644 --- a/core/State/TimestampedKeyValueStoreBuilder.cs +++ b/core/State/TimestampedKeyValueStoreBuilder.cs @@ -1,6 +1,7 @@ using Streamiz.Kafka.Net.Crosscutting; using Streamiz.Kafka.Net.Errors; using Streamiz.Kafka.Net.SerDes; +using Streamiz.Kafka.Net.State.Cache; using Streamiz.Kafka.Net.State.Logging; using Streamiz.Kafka.Net.State.Metered; using Streamiz.Kafka.Net.State.Supplier; @@ -50,7 +51,7 @@ public override ITimestampedKeyValueStore Build() var store = storeSupplier.Get(); return new MeteredTimestampedKeyValueStore( - WrapLogging(store), + WrapCaching(WrapLogging(store)), keySerdes, valueSerdes, storeSupplier.MetricsScope); @@ -58,10 +59,12 @@ public override ITimestampedKeyValueStore Build() private IKeyValueStore WrapLogging(IKeyValueStore inner) { - if (!LoggingEnabled) - return inner; + return !LoggingEnabled ? inner : new ChangeLoggingTimestampedKeyValueBytesStore(inner); + } - return new ChangeLoggingTimestampedKeyValueBytesStore(inner); + private IKeyValueStore WrapCaching(IKeyValueStore inner) + { + return !CachingEnabled ? inner : new CachingKeyValueStore(inner); } } } diff --git a/core/State/TimestampedWindowStoreBuilder.cs b/core/State/TimestampedWindowStoreBuilder.cs index 2dc9e22a..b420f181 100644 --- a/core/State/TimestampedWindowStoreBuilder.cs +++ b/core/State/TimestampedWindowStoreBuilder.cs @@ -1,5 +1,7 @@ using Streamiz.Kafka.Net.Crosscutting; using Streamiz.Kafka.Net.SerDes; +using Streamiz.Kafka.Net.State.Cache; +using Streamiz.Kafka.Net.State.Internal; using Streamiz.Kafka.Net.State.Logging; using Streamiz.Kafka.Net.State.Metered; using Streamiz.Kafka.Net.State.Supplier; @@ -46,7 +48,7 @@ public override ITimestampedWindowStore Build() { var store = supplier.Get(); return new MeteredTimestampedWindowStore( - WrapLogging(store), + WrapCaching(WrapLogging(store)), supplier.WindowSize.Value, keySerdes, valueSerdes, @@ -60,5 +62,16 @@ private IWindowStore WrapLogging(IWindowStore inne return new ChangeLoggingTimestampedWindowBytesStore(inner, supplier.RetainDuplicates); } + + private IWindowStore WrapCaching(IWindowStore inner) + { + return !CachingEnabled ? + inner : + new CachingWindowStore( + inner, + supplier.WindowSize.Value, + supplier.SegmentInterval, + new WindowKeySchema()); + } } } diff --git a/core/State/WindowStoreBuilder.cs b/core/State/WindowStoreBuilder.cs index 7c15212f..1ec759f3 100644 --- a/core/State/WindowStoreBuilder.cs +++ b/core/State/WindowStoreBuilder.cs @@ -1,5 +1,7 @@ using Streamiz.Kafka.Net.Crosscutting; using Streamiz.Kafka.Net.SerDes; +using Streamiz.Kafka.Net.State.Cache; +using Streamiz.Kafka.Net.State.Internal; using Streamiz.Kafka.Net.State.Logging; using Streamiz.Kafka.Net.State.Metered; using Streamiz.Kafka.Net.State.Supplier; @@ -47,7 +49,7 @@ public override IWindowStore Build() var store = supplier.Get(); return new MeteredWindowStore( - WrapLogging(store), + WrapCaching(WrapLogging(store)), supplier.WindowSize.Value, keySerdes, valueSerdes, @@ -61,5 +63,16 @@ private IWindowStore WrapLogging(IWindowStore inne return new ChangeLoggingWindowBytesStore(inner, supplier.RetainDuplicates); } + + private IWindowStore WrapCaching(IWindowStore inner) + { + return !CachingEnabled ? + inner : + new CachingWindowStore( + inner, + supplier.WindowSize.Value, + supplier.SegmentInterval, + new WindowKeySchema()); + } } } \ No newline at end of file diff --git a/core/State/Windowed.cs b/core/State/Windowed.cs index e6e68566..a8cfe34e 100644 --- a/core/State/Windowed.cs +++ b/core/State/Windowed.cs @@ -57,11 +57,7 @@ public override int GetHashCode() /// /// public override string ToString() - { - StringBuilder sb = new StringBuilder(); - sb.AppendLine($"Key : {Key}"); - sb.AppendLine($"Window : {Window}"); - return sb.ToString(); - } + => $"Key: {Key} | Window : {Window}"; + } } diff --git a/core/StreamBuilder.cs b/core/StreamBuilder.cs index de129ccc..84263b32 100644 --- a/core/StreamBuilder.cs +++ b/core/StreamBuilder.cs @@ -364,7 +364,7 @@ public IKTable Table(string topic, ISerDes keySerdes, ISerDes } materialized = materialized ?? Materialized>.Create(); - + var consumedInternal = new ConsumedInternal(named, keySerdes, valueSerdes, extractor); materialized.UseProvider(internalStreamBuilder, $"{topic}-").InitConsumed(consumedInternal); @@ -730,29 +730,29 @@ public IGlobalKTable GlobalTable(string topic, Materialized< public IGlobalKTable GlobalTable(string topic, Materialized> materialized, string named, ITimestampExtractor extractor) where KS : ISerDes, new() where VS : ISerDes, new() - => GlobalTable(topic, new KS(), new VS(), materialized, named, extractor); - - + => GlobalTable(topic, new KS(), new VS(), materialized, named, extractor); + + #endregion - + #endregion - + #region State Store - - /// - /// Adds a state store to the underlying . - /// It is required to connect state stores to - /// or - /// before they can be used. - /// - /// The builder used to obtain the instance. + + /// + /// Adds a state store to the underlying . + /// It is required to connect state stores to + /// or + /// before they can be used. + /// + /// The builder used to obtain the instance. public void AddStateStore(IStoreBuilder storeBuilder) - { - internalStreamBuilder.AddStateStore(storeBuilder); + { + internalStreamBuilder.AddStateStore(storeBuilder); } #endregion - + /// /// Returns the that represents the specified processing logic. /// Note that using this method means no optimizations are performed. diff --git a/core/StreamConfig.cs b/core/StreamConfig.cs index 52e6de52..3ee6378a 100644 --- a/core/StreamConfig.cs +++ b/core/StreamConfig.cs @@ -13,7 +13,6 @@ using Streamiz.Kafka.Net.Processors.Internal; using Streamiz.Kafka.Net.SerDes; using Streamiz.Kafka.Net.State; -using Streamiz.Kafka.Net.State.RocksDb; namespace Streamiz.Kafka.Net { @@ -343,6 +342,11 @@ public interface IStreamConfig : ICloneable /// TimeSpan LogProcessingSummary { get; set; } + /// + /// Maximum number of memory bytes to be used for state stores cache for a single store. (default: 5Mb) + /// + long StateStoreCacheMaxBytes { get; set; } + #endregion #region Middlewares @@ -509,6 +513,7 @@ public int GetHashCode(KeyValuePair obj) private const string deserializationExceptionHandlerCst = "deserialization.exception.handler"; private const string productionExceptionHandlerCst = "production.exception.handler"; private const string logProcessingSummaryCst = "log.processing.summary"; + private const string stateStoreCacheMaxBytesCst = "statestore.cache.max.bytes"; /// /// Default commit interval in milliseconds when exactly once is not enabled @@ -2282,6 +2287,7 @@ public StreamConfig(IDictionary properties) StartTaskDelayMs = 5000; ParallelProcessing = false; MaxDegreeOfParallelism = 8; + StateStoreCacheMaxBytes = 5 * 1024 * 1024; _consumerConfig = new ConsumerConfig(); _producerConfig = new ProducerConfig(); @@ -2749,6 +2755,17 @@ public TimeSpan LogProcessingSummary get => configProperties[logProcessingSummaryCst]; set => configProperties.AddOrUpdate(logProcessingSummaryCst, value); } + + /// + /// Maximum number of memory bytes to be used for state stores cache for a single store. (default: 5Mb) + /// + [StreamConfigProperty("" + stateStoreCacheMaxBytesCst)] + + public long StateStoreCacheMaxBytes + { + get => configProperties[stateStoreCacheMaxBytesCst]; + set => configProperties.AddOrUpdate(stateStoreCacheMaxBytesCst, value); + } /// /// Get the configs to the diff --git a/core/Streamiz.Kafka.Net.csproj b/core/Streamiz.Kafka.Net.csproj index 8a188e21..ee73700f 100644 --- a/core/Streamiz.Kafka.Net.csproj +++ b/core/Streamiz.Kafka.Net.csproj @@ -47,5 +47,4 @@ - diff --git a/core/Table/Materialized.cs b/core/Table/Materialized.cs index 91129658..59caab43 100644 --- a/core/Table/Materialized.cs +++ b/core/Table/Materialized.cs @@ -3,7 +3,6 @@ using Streamiz.Kafka.Net.SerDes; using Streamiz.Kafka.Net.State; using Streamiz.Kafka.Net.State.InMemory; -using Streamiz.Kafka.Net.State.RocksDb; using Streamiz.Kafka.Net.State.Supplier; using Streamiz.Kafka.Net.Stream.Internal; using System; @@ -214,17 +213,17 @@ public static Materialized> Create(I public IDictionary TopicConfig { get; protected set; } /// - /// Is logging enabled (default: false), Warning : will be true in next release. + /// Is logging enabled (default: true). /// public bool LoggingEnabled { get; protected set; } = true; /// - /// Is caching enabled. Not use for moment. + /// Is caching enabled (default: false) /// public bool CachingEnabled { get; protected set; } /// - /// Store suppplier use to build the state store + /// Store supplier use to build the state store /// public IStoreSupplier StoreSupplier { get; protected set; } diff --git a/docs/assets/state-store.png b/docs/assets/state-store.png new file mode 100644 index 00000000..eb8002eb Binary files /dev/null and b/docs/assets/state-store.png differ diff --git a/docs/stores.md b/docs/stores.md index 4b9da004..d3e157c8 100644 --- a/docs/stores.md +++ b/docs/stores.md @@ -6,6 +6,18 @@ - RocksDb state store is available from 1.2.0 release. - By default, a state store is tracked by a changelog topic from 1.2.0 release. (If you don't need, you have to make it explicit). +## Basics + +A stateful processor may use one or more state stores. Each task that contains a stateful processor has exclusive access to the state stores in the processor. That means a topology with two state stores and five input partitions will lead to five tasks, and each task will own two state stores resulting in 10 state stores in total for your Kafka Streams application. + +State stores in Streamiz are layered in four ways : +- The outermost layer collects metrics about the operations on the state store and serializes and deserializes the records that are written to and read from the state store. +- The next layer caches the records. If a record exists in the cache with the same key as a new record, the cache overwrites the existing record with the new record; otherwise, the cache adds a new entry for the new record. The cache is the primary serving area for lookups. If a lookup can’t find a record with a given key in the cache, it is forwarded to the next layer. If this lookup returns an entry, the entry is added to the cache. If the cache exceeds its configured size during a write, the cache evicts the records that have been least recently used and sends new and overwritten records downstream. The caching layer decreases downstream traffic because no updates are sent downstream unless the cache evicts records or is flushed. The cache’s size is configurable. If it is set to zero, the cache is disabled. (`EARLY ACCESS ONLY`) +- The changelogging layer sends each record updated in the state store to a topic in Kafka—the state’s changelog topic. The changelog topic is a compacted topic that replicates the data in the local state. Changelogging is needed for fault tolerance, as we will explain below. +- The innermost layer updates and reads the local state store. + +![state-store](./assets/state-store.png) + ## In Memory key/value store As his name, this is an inmemory key value state store which is supplied by InMemoryKeyValueBytesStoreSupplier. @@ -102,4 +114,53 @@ builder ) .ToStream() .To("output"); +``` + +## Caching + +**This feature is available in `EARLY ACCESS` only.** + +Streamiz offers robust capabilities for stream processing applications, including efficient data caching mechanisms. Caching optimizes performance by minimizing redundant computations, reducing latency, and enhancing overall throughput within stream processing pipelines. + +**Purpose of Caching** + +Caching in Streamiz serves several key purposes: +- **Reduction of Redundant Computations**: Stores intermediate results to avoid recomputing data unnecessarily. +- **Performance Optimization**: Minimizes latency by reducing the need for repeated expensive computations or data fetches. +- **Fault Tolerance**: Ensures resilience by maintaining readily accessible intermediate results for recovery during failures. + +**Types of Caching** + +Streamiz supports one primary type of caching: +1. **State Store Cache**: Enhances processing efficiency by caching state store entries, reducing the overhead of state fetches and updates. + +### Configuring Caching +To optimize caching behavior, Streamiz offers configurable parameters: +- **Cache Size**: Defines the maximum size of the cache to balance performance and memory usage. To be fair, this cache size is the maximum cache size per state store. +- **Enable caching** : By default, caching is disabled for all stateful processors in Streamiz. **This feature will be enabled by default soon.** You can enable the caching feature with `InMemory.WithCachingEnabled()`, `InMemoryWindows.WithCachingEnabled()`, `RocksDb.WithCachingEnabled()` or `RocksDbWindows.WithCachingEnabled()` + +``` csharp +// Enable caching for a rocksdb window store +RocksDbWindows.As("count-window-store") + .WithKeySerdes(new StringSerDes()) + .WithValueSerdes(new Int64SerDes()) + .WithCachingEnabled(); + +// Enable caching for an in-memory window store +InMemoryWindows.As("count-window-store") + .WithKeySerdes(new StringSerDes()) + .WithValueSerdes(new Int64SerDes()) + .WithCachingEnabled(); + +// Enable caching for an rocksdb key/value store +RocksDb.As("count-store") + .WithKeySerdes(new StringSerDes()) + .WithValueSerdes(new Int64SerDes()) + .WithCachingEnabled(); + +// Enable caching for an in-memory key/value store +InMemory.As("count-store") + .WithKeySerdes(new StringSerDes()) + .WithValueSerdes(new Int64SerDes()) + .WithCachingEnabled(); ``` \ No newline at end of file diff --git a/environment/docker-compose.yml b/environment/docker-compose.yml index 3865c52c..49fb3e24 100644 --- a/environment/docker-compose.yml +++ b/environment/docker-compose.yml @@ -2,7 +2,7 @@ version: '2' services: zookeeper: - image: confluentinc/cp-zookeeper:7.6.0.arm64 + image: confluentinc/cp-zookeeper:7.6.1 hostname: zookeeper container_name: zookeeper ports: @@ -13,7 +13,7 @@ services: KAFKA_OPTS: "-Dzookeeper.4lw.commands.whitelist=*" broker-1: - image: confluentinc/cp-server:7.6.0.arm64 + image: confluentinc/cp-server:7.6.1 hostname: broker-1 container_name: broker-1 depends_on: @@ -43,7 +43,7 @@ services: CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous' broker-2: - image: confluentinc/cp-server:7.6.0.arm64 + image: confluentinc/cp-server:7.6.1 hostname: broker-2 container_name: broker-2 depends_on: @@ -73,7 +73,7 @@ services: CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous' schema-registry: - image: confluentinc/cp-schema-registry:7.6.0.arm64 + image: confluentinc/cp-schema-registry:7.6.1 hostname: schema-registry container_name: schema-registry depends_on: diff --git a/samples/sample-stream/Program.cs b/samples/sample-stream/Program.cs index 25bffc5c..d6fcb54a 100644 --- a/samples/sample-stream/Program.cs +++ b/samples/sample-stream/Program.cs @@ -2,7 +2,10 @@ using Streamiz.Kafka.Net; using Streamiz.Kafka.Net.SerDes; using System; +using System.Collections.Generic; using System.Threading.Tasks; +using Streamiz.Kafka.Net.Metrics; +using Streamiz.Kafka.Net.Metrics.Prometheus; using Streamiz.Kafka.Net.Stream; using System.Collections.Generic; using Streamiz.Kafka.Net.Metrics.Prometheus; @@ -14,17 +17,17 @@ public static class Program { public static async Task Main(string[] args) { - var config = new StreamConfig - { + var config = new StreamConfig{ ApplicationId = $"test-app", BootstrapServers = "localhost:9092", - AutoOffsetReset = AutoOffsetReset.Earliest + AutoOffsetReset = AutoOffsetReset.Earliest, }; + config.MetricsRecording = MetricsRecordingLevel.DEBUG; config.UsePrometheusReporter(9090, true); - + var t = BuildTopology(); var stream = new KafkaStream(t, config); - + Console.CancelKeyPress += (_,_) => { stream.Dispose(); }; @@ -34,17 +37,23 @@ public static async Task Main(string[] args) private static Topology BuildTopology() { - var builder = new StreamBuilder(); + TimeSpan windowSize = TimeSpan.FromHours(1); - var globalTable = builder - .GlobalTable("Input2", + var builder = new StreamBuilder(); + builder.Stream("input") + .GroupByKey() + .WindowedBy(TumblingWindowOptions.Of(windowSize)) + .Count(RocksDbWindows.As("count-store") + .WithKeySerdes(new StringSerDes()) + .WithValueSerdes(new Int64SerDes()) + .WithCachingEnabled()) + .ToStream() + .Map((k,v) => new KeyValuePair(k.ToString(), v.ToString())) + .To("output", new StringSerDes(), - new StringSerDes(), - InMemory.As() - .WithKeySerdes() - .WithValueSerdes()); - - return builder.Build(); + new StringSerDes()); + + return builder.Build(); } } } \ No newline at end of file diff --git a/serdes/Streamiz.Kafka.Net.SerDes.CloudEvents/Streamiz.Kafka.Net.SerDes.CloudEvents.csproj b/serdes/Streamiz.Kafka.Net.SerDes.CloudEvents/Streamiz.Kafka.Net.SerDes.CloudEvents.csproj index b8c33627..b42f03c3 100644 --- a/serdes/Streamiz.Kafka.Net.SerDes.CloudEvents/Streamiz.Kafka.Net.SerDes.CloudEvents.csproj +++ b/serdes/Streamiz.Kafka.Net.SerDes.CloudEvents/Streamiz.Kafka.Net.SerDes.CloudEvents.csproj @@ -44,7 +44,7 @@ - + \ No newline at end of file diff --git a/test/Streamiz.Kafka.Net.Tests/Helpers/AssertExtensions.cs b/test/Streamiz.Kafka.Net.Tests/Helpers/AssertExtensions.cs index 17522b92..925465f8 100644 --- a/test/Streamiz.Kafka.Net.Tests/Helpers/AssertExtensions.cs +++ b/test/Streamiz.Kafka.Net.Tests/Helpers/AssertExtensions.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Threading; using Confluent.Kafka; using NUnit.Framework; @@ -23,5 +24,15 @@ public static void WaitUntil(Func condition, TimeSpan timeout, TimeSpan st Thread.Sleep((int)step.TotalMilliseconds); } } + + public static void VerifyKeyValueList(List> expected, List> actual) { + Assert.AreEqual(expected.Count, actual.Count); + for (int i = 0; i < actual.Count; i++) { + KeyValuePair expectedKv = expected[i]; + KeyValuePair actualKv = actual[i]; + Assert.AreEqual(expectedKv.Key, actualKv.Key); + Assert.AreEqual(expectedKv.Value, actualKv.Value); + } + } } } \ No newline at end of file diff --git a/test/Streamiz.Kafka.Net.Tests/Private/StoreChangelogReaderTests.cs b/test/Streamiz.Kafka.Net.Tests/Private/StoreChangelogReaderTests.cs index 67121f5e..38075372 100644 --- a/test/Streamiz.Kafka.Net.Tests/Private/StoreChangelogReaderTests.cs +++ b/test/Streamiz.Kafka.Net.Tests/Private/StoreChangelogReaderTests.cs @@ -14,7 +14,6 @@ using Streamiz.Kafka.Net.Processors.Internal; using Streamiz.Kafka.Net.SerDes; using Streamiz.Kafka.Net.State; -using Streamiz.Kafka.Net.State.RocksDb; using static Streamiz.Kafka.Net.Processors.Internal.StoreChangelogReader; namespace Streamiz.Kafka.Net.Tests.Private diff --git a/test/Streamiz.Kafka.Net.Tests/Processors/KGroupedTableCountTests.cs b/test/Streamiz.Kafka.Net.Tests/Processors/KGroupedTableCountTests.cs index 37c0b5e6..b2ce8c65 100644 --- a/test/Streamiz.Kafka.Net.Tests/Processors/KGroupedTableCountTests.cs +++ b/test/Streamiz.Kafka.Net.Tests/Processors/KGroupedTableCountTests.cs @@ -100,7 +100,6 @@ public void WithNullSerDes() IEnumerable> expected = new List> { KeyValuePair.Create("TEST", 1L), - KeyValuePair.Create("TEST", 0L), KeyValuePair.Create("TEST", 1L) }; diff --git a/test/Streamiz.Kafka.Net.Tests/Processors/KStreamGroupByTests.cs b/test/Streamiz.Kafka.Net.Tests/Processors/KStreamGroupByTests.cs index fdaaa14f..377a231b 100644 --- a/test/Streamiz.Kafka.Net.Tests/Processors/KStreamGroupByTests.cs +++ b/test/Streamiz.Kafka.Net.Tests/Processors/KStreamGroupByTests.cs @@ -4,6 +4,8 @@ using Streamiz.Kafka.Net.Stream; using System; using System.Collections.Generic; +using System.Linq; +using Streamiz.Kafka.Net.Table; namespace Streamiz.Kafka.Net.Tests.Processors { @@ -78,5 +80,52 @@ public void TestGroupByKeyOK() inputTopic.PipeInputs(data); } } + + [Test] + public void StreamGroupCachedTest() + { + var builder = new StreamBuilder(); + var data = new List>(); + data.Add(KeyValuePair.Create("key1", "1")); + data.Add(KeyValuePair.Create("key2", "2")); + data.Add(KeyValuePair.Create("key3", "3")); + data.Add(KeyValuePair.Create("key2", "4")); + data.Add(KeyValuePair.Create("key1", "5")); + data.Add(KeyValuePair.Create("key1", "6")); + data.Add(KeyValuePair.Create("key3", "7")); + + var expected = new List>(); + expected.Add(KeyValuePair.Create("key2", 2L)); + expected.Add(KeyValuePair.Create("key1", 3L)); + expected.Add(KeyValuePair.Create("key3", 2L)); + + var stream = builder.Stream("topic"); + stream + .GroupByKey() + .Count(InMemory + .As("store-count") + .WithKeySerdes(new StringSerDes()) + .WithValueSerdes(new Int64SerDes()) + .WithCachingEnabled()) + .ToStream() + .To("output",new StringSerDes(), new Int64SerDes()); + + var config = new StreamConfig(); + config.ApplicationId = "test-cached-stream-group"; + + Topology t = builder.Build(); + + using (var driver = new TopologyTestDriver(t, config)) + { + var inputTopic = driver.CreateInputTopic("topic"); + var outputTopic = driver.CreateOuputTopic("output"); + + inputTopic.PipeInputs(data); + driver.Commit(); + + var results = outputTopic.ReadKeyValueList().Select(r => KeyValuePair.Create(r.Message.Key, r.Message.Value)); + Assert.AreEqual(expected, results); + } + } } } diff --git a/test/Streamiz.Kafka.Net.Tests/Processors/KStreamKStreamJoinTests.cs b/test/Streamiz.Kafka.Net.Tests/Processors/KStreamKStreamJoinTests.cs index 76ed4fb6..4b3063dc 100644 --- a/test/Streamiz.Kafka.Net.Tests/Processors/KStreamKStreamJoinTests.cs +++ b/test/Streamiz.Kafka.Net.Tests/Processors/KStreamKStreamJoinTests.cs @@ -397,6 +397,5 @@ public void StreamStreamJoinSpecificSerdes() config.RemoveRocksDbFolderForTest(); } - } } \ No newline at end of file diff --git a/test/Streamiz.Kafka.Net.Tests/Processors/KTableGroupByTests.cs b/test/Streamiz.Kafka.Net.Tests/Processors/KTableGroupByTests.cs index 19cf2d51..a45889d4 100644 --- a/test/Streamiz.Kafka.Net.Tests/Processors/KTableGroupByTests.cs +++ b/test/Streamiz.Kafka.Net.Tests/Processors/KTableGroupByTests.cs @@ -4,6 +4,7 @@ using Streamiz.Kafka.Net.Stream; using System; using System.Collections.Generic; +using System.Linq; using Streamiz.Kafka.Net.Table; namespace Streamiz.Kafka.Net.Tests.Processors @@ -44,5 +45,57 @@ public void TestGroupOK() inputTopic.PipeInputs(data); } } + + [Test] + public void TableGroupCachedTest() + { + var builder = new StreamBuilder(); + var data = new List>(); + data.Add(KeyValuePair.Create("key1", "test123")); + data.Add(KeyValuePair.Create("key1", "test123")); + data.Add(KeyValuePair.Create("key2", "test123")); + data.Add(KeyValuePair.Create("key3", "test123")); + data.Add(KeyValuePair.Create("key1", "test123")); + data.Add(KeyValuePair.Create("key2", "test123")); + + var expected = new List>(); + expected.Add(KeyValuePair.Create("KEY3", 1L)); + expected.Add(KeyValuePair.Create("KEY1", 1L)); + expected.Add(KeyValuePair.Create("KEY2", 1L)); + + + var table = builder + .Table("topic", + InMemory + .As("table-store")); + + table + .GroupBy((k, v) => KeyValuePair.Create(k.ToUpper(), v.ToUpper())) + .Count(InMemory + .As("store-count") + .WithKeySerdes(new StringSerDes()) + .WithValueSerdes(new Int64SerDes()) + .WithCachingEnabled()) + .ToStream() + .To("output", new StringSerDes(), new Int64SerDes()); + + var config = new StreamConfig(); + config.ApplicationId = "table-test-group"; + + Topology t = builder.Build(); + + using (var driver = new TopologyTestDriver(t, config)) + { + var inputTopic = driver.CreateInputTopic("topic"); + var outputTopic = driver.CreateOuputTopic("output"); + + inputTopic.PipeInputs(data); + + driver.Commit(); + + var results = outputTopic.ReadKeyValueList().Select(r => KeyValuePair.Create(r.Message.Key, r.Message.Value)); + Assert.AreEqual(expected, results); + } + } } -} +} \ No newline at end of file diff --git a/test/Streamiz.Kafka.Net.Tests/Processors/KTableToStreamTests.cs b/test/Streamiz.Kafka.Net.Tests/Processors/KTableToStreamTests.cs index a3efcc87..dd3fb00b 100644 --- a/test/Streamiz.Kafka.Net.Tests/Processors/KTableToStreamTests.cs +++ b/test/Streamiz.Kafka.Net.Tests/Processors/KTableToStreamTests.cs @@ -157,5 +157,40 @@ public void KTableToStreamWithLastUpdate() Assert.AreEqual("c", results["key1"]); } } + + + [Test] + public void TableWithCachingTest() + { + var builder = new StreamBuilder(); + + builder.Table("table-topic", + InMemory + .As("table-topic-store") + .WithCachingEnabled()) + .ToStream((k, v) => v.ToUpper()) + .To("table-stream"); + + var config = new StreamConfig(); + config.ApplicationId = "test-map"; + + Topology t = builder.Build(); + + using var driver = new TopologyTestDriver(t, config); + var inputTopic = driver.CreateInputTopic("table-topic"); + var outputTopic = driver.CreateOuputTopic("table-stream"); + var expected = new List>(); + expected.Add(KeyValuePair.Create("D", "d")); + + inputTopic.PipeInput("key1", "a"); + inputTopic.PipeInput("key1", "b"); + inputTopic.PipeInput("key1", "c"); + inputTopic.PipeInput("key1", "d"); + + driver.Commit(); + + var results = outputTopic.ReadKeyValueList().Select(r => KeyValuePair.Create(r.Message.Key, r.Message.Value)); + Assert.AreEqual(expected, results); + } } } diff --git a/test/Streamiz.Kafka.Net.Tests/Processors/TimeWindowKStreamCountTests.cs b/test/Streamiz.Kafka.Net.Tests/Processors/TimeWindowKStreamCountTests.cs index 54a7579a..dfe8882f 100644 --- a/test/Streamiz.Kafka.Net.Tests/Processors/TimeWindowKStreamCountTests.cs +++ b/test/Streamiz.Kafka.Net.Tests/Processors/TimeWindowKStreamCountTests.cs @@ -13,349 +13,460 @@ using System; using System.Collections.Generic; using System.Linq; +using Avro.Util; using Streamiz.Kafka.Net.Metrics; using Streamiz.Kafka.Net.Tests.Helpers; -namespace Streamiz.Kafka.Net.Tests.Processors +namespace Streamiz.Kafka.Net.Tests.Processors; + +public class TimeWindowKStreamCountTests { - public class TimeWindowKStreamCountTests + internal class StringTimeWindowedSerDes : TimeWindowedSerDes { - internal class StringTimeWindowedSerDes : TimeWindowedSerDes + public StringTimeWindowedSerDes() + : base(new StringSerDes(), (long)TimeSpan.FromSeconds(10).TotalMilliseconds) { - public StringTimeWindowedSerDes() - : base(new StringSerDes(), (long)TimeSpan.FromSeconds(10).TotalMilliseconds) - { - - } } + } - [Test] - public void WithNullMaterialize() - { - // CERTIFIED THAT SAME IF Materialize is null, a state store exist for count processor with a generated namestore - var config = new StreamConfig(); - var serdes = new StringSerDes(); - - config.ApplicationId = "test-window-count"; - config.UseRandomRocksDbConfigForTest(); - - var builder = new StreamBuilder(); - Materialized> m = null; - - builder - .Stream("topic") - .GroupByKey() - .WindowedBy(TumblingWindowOptions.Of(2000)) - .Count(m); - - var topology = builder.Build(); - TaskId id = new TaskId { Id = 0, Partition = 0 }; - var processorTopology = topology.Builder.BuildTopology(id); - - var supplier = new SyncKafkaSupplier(); - var producer = supplier.GetProducer(config.ToProducerConfig()); - var consumer = supplier.GetConsumer(config.ToConsumerConfig(), null); - - var part = new TopicPartition("topic", 0); - StreamTask task = new StreamTask( - "thread-0", - id, - new List { part }, - processorTopology, - consumer, - config, - supplier, - null, - new MockChangelogRegister(), - new StreamMetricsRegistry()); - task.InitializeStateStores(); - task.InitializeTopology(); - task.RestorationIfNeeded(); - task.CompleteRestoration(); - - Assert.AreEqual(1, task.Context.States.StateStoreNames.Count()); - var nameStore = task.Context.States.StateStoreNames.ElementAt(0); - Assert.IsNotNull(nameStore); - Assert.AreNotEqual(string.Empty, nameStore); - var store = task.GetStore(nameStore); - Assert.IsInstanceOf>(store); - Assert.AreEqual(0, (store as ITimestampedWindowStore).All().ToList().Count); - config.RemoveRocksDbFolderForTest(); - } + [Test] + public void WithNullMaterialize() + { + // CERTIFIED THAT SAME IF Materialize is null, a state store exist for count processor with a generated namestore + var config = new StreamConfig(); + var serdes = new StringSerDes(); + + config.ApplicationId = "test-window-count"; + config.UseRandomRocksDbConfigForTest(); + + var builder = new StreamBuilder(); + Materialized> m = null; + + builder + .Stream("topic") + .GroupByKey() + .WindowedBy(TumblingWindowOptions.Of(2000)) + .Count(m); + + var topology = builder.Build(); + TaskId id = new TaskId { Id = 0, Partition = 0 }; + var processorTopology = topology.Builder.BuildTopology(id); + + var supplier = new SyncKafkaSupplier(); + var producer = supplier.GetProducer(config.ToProducerConfig()); + var consumer = supplier.GetConsumer(config.ToConsumerConfig(), null); + + var part = new TopicPartition("topic", 0); + StreamTask task = new StreamTask( + "thread-0", + id, + new List { part }, + processorTopology, + consumer, + config, + supplier, + null, + new MockChangelogRegister(), + new StreamMetricsRegistry()); + task.InitializeStateStores(); + task.InitializeTopology(); + task.RestorationIfNeeded(); + task.CompleteRestoration(); + + Assert.AreEqual(1, task.Context.States.StateStoreNames.Count()); + var nameStore = task.Context.States.StateStoreNames.ElementAt(0); + Assert.IsNotNull(nameStore); + Assert.AreNotEqual(string.Empty, nameStore); + var store = task.GetStore(nameStore); + Assert.IsInstanceOf>(store); + Assert.AreEqual(0, (store as ITimestampedWindowStore).All().ToList().Count); + config.RemoveRocksDbFolderForTest(); + } - [Test] - public void WithNullValueSerDes() - { - // WITH VALUE NULL SERDES, in running KeySerdes must be StringSerdes, and ValueSerdes Int64SerDes - var config = new StreamConfig(); - config.ApplicationId = "test-window-count"; - - var builder = new StreamBuilder(); - - Materialized> m = - InMemoryWindows.As("count-store"); - - builder - .Stream("topic") - .GroupByKey() - .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromSeconds(5))) - .Count(m) - .ToStream() - .To("output-topic"); - - var topology = builder.Build(); - using (var driver = new TopologyTestDriver(topology, config)) - { - var input = driver.CreateInputTopic("topic"); - var output = driver.CreateOuputTopic("output-topic", TimeSpan.FromSeconds(1), new StringTimeWindowedSerDes(), new Int64SerDes()); - input.PipeInput("test", "1"); - input.PipeInput("test-test", "30"); - var records = output.ReadKeyValueList().ToList(); - Assert.AreEqual(2, records.Count); - Assert.AreEqual("test", records[0].Message.Key.Key); - Assert.AreEqual(1, records[0].Message.Value); - Assert.AreEqual("test-test", records[1].Message.Key.Key); - Assert.AreEqual(1, records[1].Message.Value); - Assert.AreEqual(records[0].Message.Key.Window, records[1].Message.Key.Window); - } - } + [Test] + public void WithNullValueSerDes() + { + // WITH VALUE NULL SERDES, in running KeySerdes must be StringSerdes, and ValueSerdes Int64SerDes + var config = new StreamConfig(); + config.ApplicationId = "test-window-count"; + + var builder = new StreamBuilder(); - [Test] - public void TimeWindowingCount() + Materialized> m = + InMemoryWindows.As("count-store"); + + builder + .Stream("topic") + .GroupByKey() + .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromSeconds(5))) + .Count(m) + .ToStream() + .To("output-topic"); + + var topology = builder.Build(); + using (var driver = new TopologyTestDriver(topology, config)) { - var config = new StreamConfig(); - config.ApplicationId = "test-window-stream"; - - var builder = new StreamBuilder(); - builder - .Stream("topic") - .GroupByKey() - .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromSeconds(10))) - .Count(InMemoryWindows.As("count-store")) - .ToStream() - .To("output"); - - var topology = builder.Build(); - using (var driver = new TopologyTestDriver(topology, config)) - { - var input = driver.CreateInputTopic("topic"); - var output = driver.CreateOuputTopic("output", TimeSpan.FromSeconds(1), new StringTimeWindowedSerDes(), new Int64SerDes()); - input.PipeInput("test", "1"); - input.PipeInput("test", "2"); - input.PipeInput("test", "3"); - var elements = output.ReadKeyValueList().ToList(); - Assert.AreEqual(3, elements.Count); - Assert.AreEqual("test", elements[0].Message.Key.Key); - Assert.AreEqual((long)TimeSpan.FromSeconds(10).TotalMilliseconds, elements[0].Message.Key.Window.EndMs - elements[0].Message.Key.Window.StartMs); - Assert.AreEqual(1, elements[0].Message.Value); - Assert.AreEqual("test", elements[1].Message.Key.Key); - Assert.AreEqual(elements[0].Message.Key.Window, elements[1].Message.Key.Window); - Assert.AreEqual(2, elements[1].Message.Value); - Assert.AreEqual("test", elements[2].Message.Key.Key); - Assert.AreEqual(elements[0].Message.Key.Window, elements[2].Message.Key.Window); - Assert.AreEqual(3, elements[2].Message.Value); - } + var input = driver.CreateInputTopic("topic"); + var output = driver.CreateOuputTopic("output-topic", TimeSpan.FromSeconds(1), + new StringTimeWindowedSerDes(), new Int64SerDes()); + input.PipeInput("test", "1"); + input.PipeInput("test-test", "30"); + var records = output.ReadKeyValueList().ToList(); + Assert.AreEqual(2, records.Count); + Assert.AreEqual("test", records[0].Message.Key.Key); + Assert.AreEqual(1, records[0].Message.Value); + Assert.AreEqual("test-test", records[1].Message.Key.Key); + Assert.AreEqual(1, records[1].Message.Value); + Assert.AreEqual(records[0].Message.Key.Window, records[1].Message.Key.Window); } + } - [Test] - public void TimeWindowingCountWithName() + [Test] + public void TimeWindowingCount() + { + var config = new StreamConfig(); + config.ApplicationId = "test-window-stream"; + + var builder = new StreamBuilder(); + builder + .Stream("topic") + .GroupByKey() + .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromSeconds(10))) + .Count(InMemoryWindows.As("count-store")) + .ToStream() + .To("output"); + + var topology = builder.Build(); + using (var driver = new TopologyTestDriver(topology, config)) { - var config = new StreamConfig(); - config.ApplicationId = "test-window-stream"; - - var builder = new StreamBuilder(); - builder - .Stream("topic") - .GroupByKey() - .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromSeconds(10))) - .Count(InMemoryWindows.As("count-store"), "count-01") - .ToStream() - .To("output"); - - var topology = builder.Build(); - using (var driver = new TopologyTestDriver(topology, config)) - { - var input = driver.CreateInputTopic("topic"); - var output = driver.CreateOuputTopic("output", TimeSpan.FromSeconds(1), new StringTimeWindowedSerDes(), new Int64SerDes()); - input.PipeInput("test", "1"); - input.PipeInput("test", "2"); - input.PipeInput("test", "3"); - var elements = output.ReadKeyValueList().ToList(); - Assert.AreEqual(3, elements.Count); - Assert.AreEqual("test", elements[0].Message.Key.Key); - Assert.AreEqual((long)TimeSpan.FromSeconds(10).TotalMilliseconds, elements[0].Message.Key.Window.EndMs - elements[0].Message.Key.Window.StartMs); - Assert.AreEqual(1, elements[0].Message.Value); - Assert.AreEqual("test", elements[1].Message.Key.Key); - Assert.AreEqual(elements[0].Message.Key.Window, elements[1].Message.Key.Window); - Assert.AreEqual(2, elements[1].Message.Value); - Assert.AreEqual("test", elements[2].Message.Key.Key); - Assert.AreEqual(elements[0].Message.Key.Window, elements[2].Message.Key.Window); - Assert.AreEqual(3, elements[2].Message.Value); - } + var input = driver.CreateInputTopic("topic"); + var output = driver.CreateOuputTopic("output", TimeSpan.FromSeconds(1), new StringTimeWindowedSerDes(), + new Int64SerDes()); + input.PipeInput("test", "1"); + input.PipeInput("test", "2"); + input.PipeInput("test", "3"); + var elements = output.ReadKeyValueList().ToList(); + Assert.AreEqual(3, elements.Count); + Assert.AreEqual("test", elements[0].Message.Key.Key); + Assert.AreEqual((long)TimeSpan.FromSeconds(10).TotalMilliseconds, + elements[0].Message.Key.Window.EndMs - elements[0].Message.Key.Window.StartMs); + Assert.AreEqual(1, elements[0].Message.Value); + Assert.AreEqual("test", elements[1].Message.Key.Key); + Assert.AreEqual(elements[0].Message.Key.Window, elements[1].Message.Key.Window); + Assert.AreEqual(2, elements[1].Message.Value); + Assert.AreEqual("test", elements[2].Message.Key.Key); + Assert.AreEqual(elements[0].Message.Key.Window, elements[2].Message.Key.Window); + Assert.AreEqual(3, elements[2].Message.Value); } + } - [Test] - public void TimeWindowingCountWithMaterialize() + [Test] + public void TimeWindowingCountWithName() + { + var config = new StreamConfig(); + config.ApplicationId = "test-window-stream"; + + var builder = new StreamBuilder(); + builder + .Stream("topic") + .GroupByKey() + .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromSeconds(10))) + .Count(InMemoryWindows.As("count-store"), "count-01") + .ToStream() + .To("output"); + + var topology = builder.Build(); + using (var driver = new TopologyTestDriver(topology, config)) { - var config = new StreamConfig(); - config.ApplicationId = "test-window-stream"; - - var builder = new StreamBuilder(); - builder - .Stream("topic") - .GroupByKey() - .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromSeconds(10))) - .Count(InMemoryWindows.As("count-store")) - .ToStream() - .To("output"); - - var topology = builder.Build(); - using (var driver = new TopologyTestDriver(topology, config)) - { - var input = driver.CreateInputTopic("topic"); - var output = driver.CreateOuputTopic("output", TimeSpan.FromSeconds(1), new StringTimeWindowedSerDes(), new Int64SerDes()); - input.PipeInput("test", "1"); - input.PipeInput("test", "2"); - input.PipeInput("test", "3"); - var elements = output.ReadKeyValueList().ToList(); - Assert.AreEqual(3, elements.Count); - Assert.AreEqual("test", elements[0].Message.Key.Key); - Assert.AreEqual((long)TimeSpan.FromSeconds(10).TotalMilliseconds, elements[0].Message.Key.Window.EndMs - elements[0].Message.Key.Window.StartMs); - Assert.AreEqual(1, elements[0].Message.Value); - Assert.AreEqual("test", elements[1].Message.Key.Key); - Assert.AreEqual(elements[0].Message.Key.Window, elements[1].Message.Key.Window); - Assert.AreEqual(2, elements[1].Message.Value); - Assert.AreEqual("test", elements[2].Message.Key.Key); - Assert.AreEqual(elements[0].Message.Key.Window, elements[2].Message.Key.Window); - Assert.AreEqual(3, elements[2].Message.Value); - } + var input = driver.CreateInputTopic("topic"); + var output = driver.CreateOuputTopic("output", TimeSpan.FromSeconds(1), new StringTimeWindowedSerDes(), + new Int64SerDes()); + input.PipeInput("test", "1"); + input.PipeInput("test", "2"); + input.PipeInput("test", "3"); + var elements = output.ReadKeyValueList().ToList(); + Assert.AreEqual(3, elements.Count); + Assert.AreEqual("test", elements[0].Message.Key.Key); + Assert.AreEqual((long)TimeSpan.FromSeconds(10).TotalMilliseconds, + elements[0].Message.Key.Window.EndMs - elements[0].Message.Key.Window.StartMs); + Assert.AreEqual(1, elements[0].Message.Value); + Assert.AreEqual("test", elements[1].Message.Key.Key); + Assert.AreEqual(elements[0].Message.Key.Window, elements[1].Message.Key.Window); + Assert.AreEqual(2, elements[1].Message.Value); + Assert.AreEqual("test", elements[2].Message.Key.Key); + Assert.AreEqual(elements[0].Message.Key.Window, elements[2].Message.Key.Window); + Assert.AreEqual(3, elements[2].Message.Value); } + } - [Test] - public void TimeWindowingCountKeySerdesUnknownWithParallel() + [Test] + public void TimeWindowingCountWithMaterialize() + { + var config = new StreamConfig(); + config.ApplicationId = "test-window-stream"; + + var builder = new StreamBuilder(); + builder + .Stream("topic") + .GroupByKey() + .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromSeconds(10))) + .Count(InMemoryWindows.As("count-store")) + .ToStream() + .To("output"); + + var topology = builder.Build(); + using (var driver = new TopologyTestDriver(topology, config)) { - TimeWindowingCountKeySerdesUnknown(true); + var input = driver.CreateInputTopic("topic"); + var output = driver.CreateOuputTopic("output", TimeSpan.FromSeconds(1), new StringTimeWindowedSerDes(), + new Int64SerDes()); + input.PipeInput("test", "1"); + input.PipeInput("test", "2"); + input.PipeInput("test", "3"); + var elements = output.ReadKeyValueList().ToList(); + Assert.AreEqual(3, elements.Count); + Assert.AreEqual("test", elements[0].Message.Key.Key); + Assert.AreEqual((long)TimeSpan.FromSeconds(10).TotalMilliseconds, + elements[0].Message.Key.Window.EndMs - elements[0].Message.Key.Window.StartMs); + Assert.AreEqual(1, elements[0].Message.Value); + Assert.AreEqual("test", elements[1].Message.Key.Key); + Assert.AreEqual(elements[0].Message.Key.Window, elements[1].Message.Key.Window); + Assert.AreEqual(2, elements[1].Message.Value); + Assert.AreEqual("test", elements[2].Message.Key.Key); + Assert.AreEqual(elements[0].Message.Key.Window, elements[2].Message.Key.Window); + Assert.AreEqual(3, elements[2].Message.Value); } + } + + [Test] + public void TimeWindowingCountKeySerdesUnknownWithParallel() + { + TimeWindowingCountKeySerdesUnknown(true); + } - [Test] - public void TimeWindowingCountKeySerdesUnknownWithoutParallel() + [Test] + public void TimeWindowingCountKeySerdesUnknownWithoutParallel() + { + TimeWindowingCountKeySerdesUnknown(false); + } + + private void TimeWindowingCountKeySerdesUnknown(bool parallelProcessing) + { + var config = new StreamConfig { - TimeWindowingCountKeySerdesUnknown(false); - } - - private void TimeWindowingCountKeySerdesUnknown(bool parallelProcessing) + ApplicationId = "test-window-stream", + ParallelProcessing = parallelProcessing + }; + + var builder = new StreamBuilder(); + builder + .Stream("topic") + .GroupByKey() + .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromSeconds(10))) + .Count(InMemoryWindows.As("count-store")) + .ToStream() + .To("output"); + + var topology = builder.Build(); + Assert.Throws(() => { - var config = new StreamConfig - { - ApplicationId = "test-window-stream", - ParallelProcessing = parallelProcessing - }; - - var builder = new StreamBuilder(); - builder - .Stream("topic") - .GroupByKey() - .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromSeconds(10))) - .Count(InMemoryWindows.As("count-store")) - .ToStream() - .To("output"); - - var topology = builder.Build(); - Assert.Throws(() => - { - using var driver = new TopologyTestDriver(topology, config); - var input = driver.CreateInputTopic("topic"); - input.PipeInput("test", "1"); - }); - } + using var driver = new TopologyTestDriver(topology, config); + var input = driver.CreateInputTopic("topic"); + input.PipeInput("test", "1"); + }); + } - [Test] - public void TimeWindowingCountNothing() + [Test] + public void TimeWindowingCountNothing() + { + var config = new StreamConfig(); + config.ApplicationId = "test-window-stream"; + + var builder = new StreamBuilder(); + builder + .Stream("topic") + .GroupByKey() + .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromSeconds(1))) + .Count(InMemoryWindows.As("count-store")) + .ToStream() + .To("output"); + + var topology = builder.Build(); + using (var driver = new TopologyTestDriver(topology, config)) { - var config = new StreamConfig(); - config.ApplicationId = "test-window-stream"; - - var builder = new StreamBuilder(); - builder - .Stream("topic") - .GroupByKey() - .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromSeconds(1))) - .Count(InMemoryWindows.As("count-store")) - .ToStream() - .To("output"); - - var topology = builder.Build(); - using (var driver = new TopologyTestDriver(topology, config)) - { - var input = driver.CreateInputTopic("topic"); - var output = driver.CreateOuputTopic("output", TimeSpan.FromSeconds(1), new StringTimeWindowedSerDes(), new Int64SerDes()); - var elements = output.ReadKeyValueList().ToList(); - Assert.AreEqual(0, elements.Count); - } + var input = driver.CreateInputTopic("topic"); + var output = driver.CreateOuputTopic("output", TimeSpan.FromSeconds(1), new StringTimeWindowedSerDes(), + new Int64SerDes()); + var elements = output.ReadKeyValueList().ToList(); + Assert.AreEqual(0, elements.Count); } + } + + [Test] + public void TimeWindowingQueryStoreAll() + { + var config = new StreamConfig(); + config.ApplicationId = "test-window-stream"; - [Test] - public void TimeWindowingQueryStoreAll() + var builder = new StreamBuilder(); + + builder + .Stream("topic") + .GroupByKey() + .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromSeconds(10))) + .Count(InMemoryWindows.As("count-store")); + + var topology = builder.Build(); + using (var driver = new TopologyTestDriver(topology, config)) { - var config = new StreamConfig(); - config.ApplicationId = "test-window-stream"; - - var builder = new StreamBuilder(); - - builder - .Stream("topic") - .GroupByKey() - .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromSeconds(10))) - .Count(InMemoryWindows.As("count-store")); - - var topology = builder.Build(); - using (var driver = new TopologyTestDriver(topology, config)) - { - var input = driver.CreateInputTopic("topic"); - input.PipeInput("test", "1"); - input.PipeInput("test", "2"); - input.PipeInput("test", "3"); - var store = driver.GetWindowStore("count-store"); - var elements = store.All().ToList(); - Assert.AreEqual(1, elements.Count); - Assert.AreEqual("test", elements[0].Key.Key); - Assert.AreEqual((long)TimeSpan.FromSeconds(10).TotalMilliseconds, elements[0].Key.Window.EndMs - elements[0].Key.Window.StartMs); - Assert.AreEqual(3, elements[0].Value); - } + var input = driver.CreateInputTopic("topic"); + input.PipeInput("test", "1"); + input.PipeInput("test", "2"); + input.PipeInput("test", "3"); + var store = driver.GetWindowStore("count-store"); + var elements = store.All().ToList(); + Assert.AreEqual(1, elements.Count); + Assert.AreEqual("test", elements[0].Key.Key); + Assert.AreEqual((long)TimeSpan.FromSeconds(10).TotalMilliseconds, + elements[0].Key.Window.EndMs - elements[0].Key.Window.StartMs); + Assert.AreEqual(3, elements[0].Value); } + } + + [Test] + public void TimeWindowingQueryStore2Window() + { + var config = new StreamConfig(); + config.ApplicationId = "test-window-stream"; + + var builder = new StreamBuilder(); + + builder + .Stream("topic") + .GroupByKey() + .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromSeconds(5))) + .Count(InMemoryWindows.As("count-store")); - [Test] - public void TimeWindowingQueryStore2Window() + var topology = builder.Build(); + using (var driver = new TopologyTestDriver(topology, config)) { - var config = new StreamConfig(); - config.ApplicationId = "test-window-stream"; - - var builder = new StreamBuilder(); - - builder - .Stream("topic") - .GroupByKey() - .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromSeconds(5))) - .Count(InMemoryWindows.As("count-store")); - - var topology = builder.Build(); - using (var driver = new TopologyTestDriver(topology, config)) - { - DateTime dt = DateTime.Now; - var input = driver.CreateInputTopic("topic"); - input.PipeInput("test", "1", dt); - input.PipeInput("test", "2", dt); - input.PipeInput("test", "3", dt.AddMinutes(1)); - var store = driver.GetWindowStore("count-store"); - var elements = store.All().ToList(); - Assert.AreEqual(2, elements.Count); - Assert.AreEqual("test", elements[0].Key.Key); - Assert.AreEqual((long)TimeSpan.FromSeconds(5).TotalMilliseconds, elements[0].Key.Window.EndMs - elements[0].Key.Window.StartMs); - Assert.AreEqual(2, elements[0].Value); - Assert.AreEqual("test", elements[1].Key.Key); - Assert.AreEqual((long)TimeSpan.FromSeconds(5).TotalMilliseconds, elements[1].Key.Window.EndMs - elements[1].Key.Window.StartMs); - Assert.AreEqual(1, elements[1].Value); - } + DateTime dt = DateTime.Now; + var input = driver.CreateInputTopic("topic"); + input.PipeInput("test", "1", dt); + input.PipeInput("test", "2", dt); + input.PipeInput("test", "3", dt.AddMinutes(1)); + var store = driver.GetWindowStore("count-store"); + var elements = store.All().ToList(); + Assert.AreEqual(2, elements.Count); + Assert.AreEqual("test", elements[0].Key.Key); + Assert.AreEqual((long)TimeSpan.FromSeconds(5).TotalMilliseconds, + elements[0].Key.Window.EndMs - elements[0].Key.Window.StartMs); + Assert.AreEqual(2, elements[0].Value); + Assert.AreEqual("test", elements[1].Key.Key); + Assert.AreEqual((long)TimeSpan.FromSeconds(5).TotalMilliseconds, + elements[1].Key.Window.EndMs - elements[1].Key.Window.StartMs); + Assert.AreEqual(1, elements[1].Value); } } -} + + [Test] + public void WindowCachingMultipleTimesSameKey() + { + var config = new StreamConfig + { + ApplicationId = "test-window-stream" + }; + + var windowSerdes = + new TimeWindowedSerDes(new StringSerDes(), (long)TimeSpan.FromSeconds(10).TotalMilliseconds); + + var builder = new StreamBuilder(); + builder + .Stream("topic") + .GroupByKey() + .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromSeconds(10))) + .Count( + InMemoryWindows + .As("count-store") + .WithCachingEnabled()) + .ToStream() + .To("output", windowSerdes, new Int64SerDes()); + + var topology = builder.Build(); + using var driver = new TopologyTestDriver(topology, config); + var inputTopic = driver.CreateInputTopic("topic"); + var outputTopic = + driver.CreateOuputTopic("output", TimeSpan.FromSeconds(1), windowSerdes, new Int64SerDes()); + + inputTopic.PipeInput("test", "1"); + inputTopic.PipeInput("test", "2"); + inputTopic.PipeInput("test", "3"); + + driver.Commit(); + + var items = outputTopic.ReadKeyValueList().ToList(); + Assert.AreEqual(1, items.Count); + Assert.AreEqual(3, items[0].Message.Value); + Assert.AreEqual("test", items[0].Message.Key.Key); + Assert.AreEqual(TimeSpan.FromSeconds(10), items[0].Message.Key.Window.TotalTime); + } + + [Test] + public void WindowCachingMultipleKey() + { + var config = new StreamConfig + { + ApplicationId = "test-window-stream" + }; + + var windowSerdes = + new TimeWindowedSerDes(new StringSerDes(), (long)TimeSpan.FromSeconds(10).TotalMilliseconds); + + var builder = new StreamBuilder(); + builder + .Stream("topic") + .GroupByKey() + .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromSeconds(10))) + .Count( + InMemoryWindows + .As("count-store") + .WithCachingEnabled()) + .ToStream() + .To("output", windowSerdes, new Int64SerDes()); + + var topology = builder.Build(); + using var driver = new TopologyTestDriver(topology, config); + var inputTopic = driver.CreateInputTopic("topic"); + var outputTopic = + driver.CreateOuputTopic("output", TimeSpan.FromSeconds(1), windowSerdes, new Int64SerDes()); + + DateTime date = new DateTime(2024, 10, 01, 14, 00, 00, DateTimeKind.Utc); + inputTopic.PipeInput("test", "1", date); + inputTopic.PipeInput("test", "2", date.AddSeconds(1)); + inputTopic.PipeInput("test", "3", date.AddSeconds(5)); + + inputTopic.PipeInput("test2", "1", date); + inputTopic.PipeInput("test2", "2", date.AddSeconds(1)); + inputTopic.PipeInput("test2", "3", date.AddSeconds(5)); + + inputTopic.PipeInput("test", "1", date.AddSeconds(20)); + inputTopic.PipeInput("test", "2", date.AddSeconds(23)); + inputTopic.PipeInput("test", "3", date.AddSeconds(25)); + + driver.Commit(); + + var items = outputTopic.ReadKeyValueList().ToList(); + Assert.AreEqual(3, items.Count); + Assert.AreEqual(3, items[0].Message.Value); + Assert.AreEqual("test", items[0].Message.Key.Key); + Assert.AreEqual(date, items[0].Message.Key.Window.StartTime); + + Assert.AreEqual(3, items[1].Message.Value); + Assert.AreEqual("test2", items[1].Message.Key.Key); + Assert.AreEqual(date, items[1].Message.Key.Window.StartTime); + + Assert.AreEqual(3, items[2].Message.Value); + Assert.AreEqual("test", items[2].Message.Key.Key); + Assert.AreEqual(date.AddSeconds(20), items[2].Message.Key.Window.StartTime); + } +} \ No newline at end of file diff --git a/test/Streamiz.Kafka.Net.Tests/Public/RocksDbOptionsTests.cs b/test/Streamiz.Kafka.Net.Tests/Public/RocksDbOptionsTests.cs index 733fa0b5..3a3052d7 100644 --- a/test/Streamiz.Kafka.Net.Tests/Public/RocksDbOptionsTests.cs +++ b/test/Streamiz.Kafka.Net.Tests/Public/RocksDbOptionsTests.cs @@ -7,7 +7,7 @@ using Streamiz.Kafka.Net.Mock; using Streamiz.Kafka.Net.Processors; using Streamiz.Kafka.Net.Processors.Internal; -using Streamiz.Kafka.Net.State.RocksDb; +using Streamiz.Kafka.Net.State; using Streamiz.Kafka.Net.Tests.Helpers; using System; using System.Collections.Generic; diff --git a/test/Streamiz.Kafka.Net.Tests/Public/StreamJoinPropsTests.cs b/test/Streamiz.Kafka.Net.Tests/Public/StreamJoinPropsTests.cs index 56d0cfc4..784f43e7 100644 --- a/test/Streamiz.Kafka.Net.Tests/Public/StreamJoinPropsTests.cs +++ b/test/Streamiz.Kafka.Net.Tests/Public/StreamJoinPropsTests.cs @@ -18,6 +18,7 @@ public class TestSupplier : IWindowBytesStoreSupplier public string MetricsScope => "test-window"; public bool RetainDuplicates { get; set; } = false; + public long SegmentInterval { get; set; } = 1; public IWindowStore Get() => null; diff --git a/test/Streamiz.Kafka.Net.Tests/Stores/AbstractPersistentWindowStoreTests.cs b/test/Streamiz.Kafka.Net.Tests/Stores/AbstractPersistentWindowStoreTests.cs new file mode 100644 index 00000000..69a4c2aa --- /dev/null +++ b/test/Streamiz.Kafka.Net.Tests/Stores/AbstractPersistentWindowStoreTests.cs @@ -0,0 +1,288 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Confluent.Kafka; +using NUnit.Framework; +using Moq; +using Streamiz.Kafka.Net.Crosscutting; +using Streamiz.Kafka.Net.Metrics; +using Streamiz.Kafka.Net.Mock; +using Streamiz.Kafka.Net.Processors; +using Streamiz.Kafka.Net.Processors.Internal; +using Streamiz.Kafka.Net.State; +using Streamiz.Kafka.Net.State.Cache; +using Streamiz.Kafka.Net.State.Internal; +using Streamiz.Kafka.Net.Stream; +using Streamiz.Kafka.Net.Tests.Helpers; + +namespace Streamiz.Kafka.Net.Tests.Stores +{ + + public abstract class AbstractPersistentWindowStoreTests + { + protected static int MAX_CACHE_SIZE_BYTES = 150; + protected static long DEFAULT_TIMESTAMP = 10L; + protected static long WINDOW_SIZE = 10L; + protected static long SEGMENT_INTERVAL = 100L; + protected static TimeSpan RETENTION_MS = TimeSpan.FromMilliseconds(100); + + private StreamConfig config; + private IWindowStore store; + private ProcessorContext context; + private TaskId id; + private TopicPartition partition; + private ProcessorStateManager stateManager; + private Mock task; + internal WindowKeySchema keySchema; + private CachingWindowStore cachingStore; + + protected abstract IWindowStore GetBackWindowStore(); + + [SetUp] + public void Begin() + { + config = new StreamConfig(); + config.ApplicationId = "unit-test-rocksdb-w"; + config.StateStoreCacheMaxBytes = MAX_CACHE_SIZE_BYTES; + + id = new TaskId { Id = 0, Partition = 0 }; + partition = new TopicPartition("source", 0); + stateManager = new ProcessorStateManager( + id, + new List { partition }, + null, + new MockChangelogRegister(), + new MockOffsetCheckpointManager()); + + task = new Mock(); + task.Setup(k => k.Id).Returns(id); + + context = new ProcessorContext(task.Object, config, stateManager, new StreamMetricsRegistry()); + + keySchema = new WindowKeySchema(); + store = GetBackWindowStore(); + cachingStore = new CachingWindowStore(store, + WINDOW_SIZE, SEGMENT_INTERVAL, keySchema); + + if (store.Persistent) + config.UseRandomRocksDbConfigForTest(); + + cachingStore.Init(context, cachingStore); + context.SetRecordMetaData(new RecordContext(new Headers(), 0, 100, 0, "source")); + } + + [TearDown] + public void End() + { + store.Flush(); + stateManager.Close(); + if (store.Persistent) + config.RemoveRocksDbFolderForTest(); + } + + #region Private + + private static Bytes BytesKey(String key) + { + return Bytes.Wrap(Encoding.UTF8.GetBytes(key)); + } + + private String StringFrom(byte[] from) + { + return Encoding.UTF8.GetString(from); + } + + private static byte[] BytesValue(String value) + { + return Encoding.UTF8.GetBytes(value); + } + + private static void VerifyWindowedKeyValue( + KeyValuePair, byte[]>? actual, + Windowed expectedKey, + String expectedValue) + { + Assert.NotNull(actual); + Assert.AreEqual(expectedKey.Window, actual.Value.Key.Window); + Assert.AreEqual(expectedKey.Key.Get, actual.Value.Key.Key.Get); + Assert.AreEqual(expectedKey.Window, actual.Value.Key.Window); + Assert.AreEqual(expectedValue, Encoding.UTF8.GetString(actual.Value.Value)); + } + + #endregion + + #region Tests + + [Test] + public void ShouldPutFetchFromCache() + { + cachingStore.Put(BytesKey("a"), BytesValue("a"), DEFAULT_TIMESTAMP); + cachingStore.Put(BytesKey("b"), BytesValue("b"), DEFAULT_TIMESTAMP); + + Assert.AreEqual(BytesValue("a"), cachingStore.Fetch(BytesKey("a"), DEFAULT_TIMESTAMP)); + Assert.AreEqual(BytesValue("b"), cachingStore.Fetch(BytesKey("b"), DEFAULT_TIMESTAMP)); + Assert.Null(cachingStore.Fetch(BytesKey("c"), 10L)); + Assert.Null(cachingStore.Fetch(BytesKey("a"), 0L)); + + using var enumeratorA = cachingStore.Fetch(BytesKey("a"), DEFAULT_TIMESTAMP, DEFAULT_TIMESTAMP); + using var enumeratorB = cachingStore.Fetch(BytesKey("b"), DEFAULT_TIMESTAMP, DEFAULT_TIMESTAMP); + Assert.IsTrue(enumeratorA.MoveNext()); + Assert.IsTrue(enumeratorB.MoveNext()); + Assert.AreEqual(BytesValue("a"), enumeratorA.Current.Value.Value); + Assert.AreEqual(DEFAULT_TIMESTAMP, enumeratorA.Current.Value.Key); + Assert.AreEqual(BytesValue("b"), enumeratorB.Current.Value.Value); + Assert.AreEqual(DEFAULT_TIMESTAMP, enumeratorB.Current.Value.Key); + Assert.IsFalse(enumeratorA.MoveNext()); + Assert.IsFalse(enumeratorB.MoveNext()); + Assert.AreEqual(2, cachingStore.Cache.Count); + } + + [Test] + public void ShouldPutFetchWithRealTimestampFromCache() + { + DateTime now = DateTime.Now; + cachingStore.Put(BytesKey("a"), BytesValue("a"), now.GetMilliseconds()); + cachingStore.Put(BytesKey("b"), BytesValue("b"), now.GetMilliseconds() + 4); + + Assert.AreEqual(BytesValue("a"), cachingStore.Fetch(BytesKey("a"), now.GetMilliseconds())); + Assert.AreEqual(BytesValue("b"), cachingStore.Fetch(BytesKey("b"), now.GetMilliseconds() + 4)); + Assert.Null(cachingStore.Fetch(BytesKey("c"), now.GetMilliseconds() + 10)); + Assert.Null(cachingStore.Fetch(BytesKey("a"), now.GetMilliseconds() + 10)); + + using var enumeratorA = cachingStore.Fetch(BytesKey("a"), now.GetMilliseconds(), now.GetMilliseconds()); + using var enumeratorB = + cachingStore.Fetch(BytesKey("b"), now.GetMilliseconds() + 4, now.GetMilliseconds() + 4); + Assert.IsTrue(enumeratorA.MoveNext()); + Assert.IsTrue(enumeratorB.MoveNext()); + Assert.AreEqual(BytesValue("a"), enumeratorA.Current.Value.Value); + Assert.AreEqual(now.GetMilliseconds(), enumeratorA.Current.Value.Key); + Assert.AreEqual(BytesValue("b"), enumeratorB.Current.Value.Value); + Assert.AreEqual(now.GetMilliseconds() + 4, enumeratorB.Current.Value.Key); + Assert.IsFalse(enumeratorA.MoveNext()); + Assert.IsFalse(enumeratorB.MoveNext()); + Assert.AreEqual(2, cachingStore.Cache.Count); + } + + [Test] + public void ShouldFetchAllWithinTimestampRange() + { + DateTime now = DateTime.Now; + String[] array = { "a", "b", "c", "d", "e", "f", "g", "h" }; + for (int i = 0; i < array.Length; i++) + { + cachingStore.Put(BytesKey(array[i]), BytesValue(array[i]), now.AddMilliseconds(i).GetMilliseconds()); + } + + using var enumerator = cachingStore.FetchAll(now, now.AddMilliseconds(7)); + for (int i = 0; i < array.Length; i++) + { + enumerator.MoveNext(); + String str = array[i]; + VerifyWindowedKeyValue( + enumerator.Current, + new Windowed(BytesKey(str), + new TimeWindow(now.AddMilliseconds(i).GetMilliseconds(), + now.AddMilliseconds(i).GetMilliseconds() + WINDOW_SIZE)), + str); + } + + Assert.IsFalse(enumerator.MoveNext()); + + using var enumerator1 = cachingStore.FetchAll(now.AddMilliseconds(2), now.AddMilliseconds(4)); + for (int i = 2; i <= 4; i++) + { + enumerator1.MoveNext(); + String str = array[i]; + VerifyWindowedKeyValue( + enumerator1.Current, + new Windowed(BytesKey(str), + new TimeWindow(now.AddMilliseconds(i).GetMilliseconds(), + now.AddMilliseconds(i).GetMilliseconds() + WINDOW_SIZE)), + str); + } + + Assert.IsFalse(enumerator1.MoveNext()); + + using var enumerator2 = + cachingStore.FetchAll(now.AddMilliseconds(5), now.AddMilliseconds(7)); + for (int i = 5; i <= 7; i++) + { + enumerator2.MoveNext(); + String str = array[i]; + VerifyWindowedKeyValue( + enumerator2.Current, + new Windowed(BytesKey(str), + new TimeWindow(now.AddMilliseconds(i).GetMilliseconds(), + now.AddMilliseconds(i).GetMilliseconds() + WINDOW_SIZE)), + str); + } + + Assert.IsFalse(enumerator2.MoveNext()); + } + + [Test] + public void ShouldTakeValueFromCacheIfSameTimestampFlushedToRocks() + { + cachingStore.Put(BytesKey("1"), BytesValue("a"), DEFAULT_TIMESTAMP); + cachingStore.Flush(); + cachingStore.Put(BytesKey("1"), BytesValue("b"), DEFAULT_TIMESTAMP); + + using var fetch = cachingStore.Fetch(BytesKey("1"), DEFAULT_TIMESTAMP, DEFAULT_TIMESTAMP); + Assert.IsTrue(fetch.MoveNext()); + Assert.NotNull(fetch.Current); + Assert.AreEqual("b", Encoding.UTF8.GetString(fetch.Current.Value.Value)); + Assert.AreEqual(DEFAULT_TIMESTAMP, fetch.Current.Value.Key); + Assert.IsFalse(fetch.MoveNext()); + } + + [Test] + public void ShouldFetchAndIterateOverExactKeys() + { + cachingStore.Put(BytesKey("a"), BytesValue("0001"), 0); + cachingStore.Put(BytesKey("aa"), BytesValue("0002"), 0); + cachingStore.Put(BytesKey("a"), BytesValue("0003"), 1); + cachingStore.Put(BytesKey("aa"), BytesValue("0004"), 1); + cachingStore.Put(BytesKey("a"), BytesValue("0005"), SEGMENT_INTERVAL); + + List> expected = new List> + { + new(0L, BytesValue("0001")), + new(1L, BytesValue("0003")), + new(SEGMENT_INTERVAL, BytesValue("0005")) + }; + + using var enumerator = cachingStore.Fetch(BytesKey("a"), 0L, long.MaxValue); + var actual = enumerator.ToList(); + AssertExtensions.VerifyKeyValueList(expected, actual); + } + + [Test] + public void ShouldGetAllFromCache() + { + cachingStore.Put(BytesKey("a"), BytesValue("a"), DEFAULT_TIMESTAMP); + cachingStore.Put(BytesKey("b"), BytesValue("b"), DEFAULT_TIMESTAMP); + cachingStore.Put(BytesKey("c"), BytesValue("c"), DEFAULT_TIMESTAMP); + cachingStore.Put(BytesKey("d"), BytesValue("d"), DEFAULT_TIMESTAMP); + cachingStore.Put(BytesKey("e"), BytesValue("e"), DEFAULT_TIMESTAMP); + cachingStore.Put(BytesKey("f"), BytesValue("f"), DEFAULT_TIMESTAMP); + cachingStore.Put(BytesKey("g"), BytesValue("g"), DEFAULT_TIMESTAMP); + cachingStore.Put(BytesKey("h"), BytesValue("h"), DEFAULT_TIMESTAMP); + + using var enumerator = cachingStore.All(); + String[] array = { "a", "b", "c", "d", "e", "f", "g", "h" }; + foreach (var s in array) + { + enumerator.MoveNext(); + VerifyWindowedKeyValue( + enumerator.Current, + new Windowed(BytesKey(s), + new TimeWindow(DEFAULT_TIMESTAMP, DEFAULT_TIMESTAMP + WINDOW_SIZE)), + s); + } + + Assert.IsFalse(enumerator.MoveNext()); + } + + #endregion + } +} \ No newline at end of file diff --git a/test/Streamiz.Kafka.Net.Tests/Stores/CacheKeyValueStoreTests.cs b/test/Streamiz.Kafka.Net.Tests/Stores/CacheKeyValueStoreTests.cs new file mode 100644 index 00000000..b937d29d --- /dev/null +++ b/test/Streamiz.Kafka.Net.Tests/Stores/CacheKeyValueStoreTests.cs @@ -0,0 +1,350 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using Confluent.Kafka; +using Moq; +using NUnit.Framework; +using Streamiz.Kafka.Net.Crosscutting; +using Streamiz.Kafka.Net.Metrics; +using Streamiz.Kafka.Net.Metrics.Internal; +using Streamiz.Kafka.Net.Mock; +using Streamiz.Kafka.Net.Processors; +using Streamiz.Kafka.Net.Processors.Internal; +using Streamiz.Kafka.Net.SerDes; +using Streamiz.Kafka.Net.State; +using Streamiz.Kafka.Net.State.Cache; +using Streamiz.Kafka.Net.State.InMemory; + +namespace Streamiz.Kafka.Net.Tests.Stores +{ + // TODO : add test add event from internal wrapped store and flush cache store + public class CacheKeyValueStoreTests + { + private StreamConfig config; + private CachingKeyValueStore cache; + private ProcessorContext context; + private TaskId id; + private TopicPartition partition; + private ProcessorStateManager stateManager; + private Mock task; + private InMemoryKeyValueStore inMemoryKeyValue; + private StreamMetricsRegistry metricsRegistry; + private string threadId = StreamMetricsRegistry.UNKNOWN_THREAD; + + #region Tools + private Bytes ToKey(string key) + { + var serdes = new StringSerDes(); + return Bytes.Wrap(serdes.Serialize(key, SerializationContext.Empty)); + } + + private string FromKey(Bytes bytes) + { + var serdes = new StringSerDes(); + return serdes.Deserialize(bytes.Get, SerializationContext.Empty); + } + + private byte[] ToValue(string value) + { + var serdes = new StringSerDes(); + return serdes.Serialize(value, SerializationContext.Empty); + } + + private string FromValue(byte[] bytes) + { + var serdes = new StringSerDes(); + return serdes.Deserialize(bytes, SerializationContext.Empty); + } + #endregion + + [SetUp] + public void Begin() + { + config = new StreamConfig(); + config.ApplicationId = "unit-test-cachestore-kv"; + config.StateStoreCacheMaxBytes = 1000; + + threadId = Thread.CurrentThread.Name ?? StreamMetricsRegistry.UNKNOWN_THREAD; + id = new TaskId { Id = 0, Partition = 0 }; + partition = new TopicPartition("source", 0); + stateManager = new ProcessorStateManager( + id, + new List { partition }, + null, + new MockChangelogRegister(), + new MockOffsetCheckpointManager()); + + task = new Mock(); + task.Setup(k => k.Id).Returns(id); + + metricsRegistry = new StreamMetricsRegistry("test", MetricsRecordingLevel.DEBUG); + context = new ProcessorContext(task.Object, config, stateManager, metricsRegistry); + + inMemoryKeyValue = new InMemoryKeyValueStore("store"); + cache = new CachingKeyValueStore(inMemoryKeyValue); + cache.Init(context, cache); + } + + [TearDown] + public void End() + { + if (cache != null) + { + cache.Flush(); + stateManager.Close(); + } + } + + [Test] + public void ExpiryCapacityTest() + { + config.StateStoreCacheMaxBytes = 30; + cache.CreateCache(context); + bool checkListener = true; + cache.SetFlushListener(record => { + if (checkListener) + { + Assert.AreEqual(ToKey("test").Get, record.Key); + Assert.AreEqual(ToValue("value1"), record.Value.NewValue); + Assert.IsNull(record.Value.OldValue); + checkListener = false; + } + }, true); + + context.SetRecordMetaData(new RecordContext(new Headers(), 0, 100, 0, "topic")); + cache.Put(ToKey("test"), ToValue("value1")); + cache.Put(ToKey("test2"), ToValue("value2")); + Assert.AreEqual(ToValue("value1"), inMemoryKeyValue.Get(ToKey("test"))); + } + + [Test] + public void DuplicateValueSameKeyTest() + { + cache.SetFlushListener(record => { + Assert.AreEqual(ToKey("test").Get, record.Key); + Assert.AreEqual(ToValue("value2"), record.Value.NewValue); + Assert.IsNull(record.Value.OldValue); + }, true); + + context.SetRecordMetaData(new RecordContext(new Headers(), 0, 100, 0, "topic")); + cache.Put(ToKey("test"), ToValue("value1")); + cache.Put(ToKey("test"), ToValue("value2")); + cache.Flush(); + Assert.AreEqual(ToValue("value2"), inMemoryKeyValue.Get(ToKey("test"))); + } + + [Test] + public void DuplicateValueWithOldValueSameKeyTest() + { + bool checkedListener = false; + cache.SetFlushListener(record => { + if (checkedListener) + { + Assert.AreEqual(ToKey("test").Get, record.Key); + Assert.AreEqual(ToValue("value2"), record.Value.NewValue); + Assert.IsNotNull(record.Value.OldValue); + Assert.AreEqual(ToValue("value1"), record.Value.OldValue); + } + }, true); + + context.SetRecordMetaData(new RecordContext(new Headers(), 0, 100, 0, "topic")); + cache.Put(ToKey("test"), ToValue("value1")); + cache.Flush(); + checkedListener = true; + cache.Put(ToKey("test"), ToValue("value2")); + cache.Flush(); + Assert.AreEqual(ToValue("value2"), inMemoryKeyValue.Get(ToKey("test"))); + } + + [Test] + public void FlushTest() + { + cache.SetFlushListener(record => { + Assert.AreEqual(ToKey("test").Get, record.Key); + Assert.AreEqual(ToValue("value1"), record.Value.NewValue); + Assert.IsNull(record.Value.OldValue); + }, true); + + context.SetRecordMetaData(new RecordContext(new Headers(), 0, 100, 0, "topic")); + cache.Put(ToKey("test"), ToValue("value1")); + cache.Flush(); + Assert.AreEqual(ToValue("value1"), inMemoryKeyValue.Get(ToKey("test"))); + } + + [Test] + public void DeleteTest() + { + context.SetRecordMetaData(new RecordContext(new Headers(), 0, 100, 0, "topic")); + cache.Put(ToKey("test"), ToValue("value1")); + cache.Flush(); + Assert.AreEqual(ToValue("value1"), cache.Get(ToKey("test"))); + Assert.AreEqual(ToValue("value1"), inMemoryKeyValue.Get(ToKey("test"))); + cache.Delete(ToKey("test")); + Assert.IsNull(cache.Get(ToKey("test"))); + cache.Flush(); + Assert.IsNull(inMemoryKeyValue.Get(ToKey("test"))); + } + + [Test] + public void PutAllTest() + { + context.SetRecordMetaData(new RecordContext(new Headers(), 0, 100, 0, "topic")); + var input = new List> + { + new(ToKey("test1"), ToValue("value1")), + new(ToKey("test2"), ToValue("value2")), + new(ToKey("test2"), ToValue("value2bis")), + new(ToKey("test3"), ToValue("value3")), + }; + cache.PutAll(input); + Assert.AreEqual(3, cache.ApproximateNumEntries()); + cache.Flush(); + Assert.AreEqual(0, cache.ApproximateNumEntries()); + Assert.AreEqual(3, inMemoryKeyValue.ApproximateNumEntries()); + Assert.AreEqual(ToValue("value1"), inMemoryKeyValue.Get(ToKey("test1"))); + Assert.AreEqual(ToValue("value2bis"), inMemoryKeyValue.Get(ToKey("test2"))); + Assert.AreEqual(ToValue("value3"), inMemoryKeyValue.Get(ToKey("test3"))); + } + + [Test] + public void PutIfAbsentTest() + { + context.SetRecordMetaData(new RecordContext(new Headers(), 0, 100, 0, "topic")); + cache.PutIfAbsent(ToKey("test"), ToValue("value1")); + cache.PutIfAbsent(ToKey("test"), ToValue("value2")); + cache.Flush(); + Assert.AreEqual(ToValue("value1"), inMemoryKeyValue.Get(ToKey("test"))); + } + + [Test] + public void RangeTest() + { + context.SetRecordMetaData(new RecordContext(new Headers(), 0, 100, 0, "topic")); + cache.PutAll(new List> + { + new(ToKey("test1"), ToValue("value1")), + new(ToKey("test2"), ToValue("value2")), + new(ToKey("test3"), ToValue("value3")), + new(ToKey("test4"), ToValue("value4")) + }); + + Assert.AreEqual(4, cache.ApproximateNumEntries()); + + Assert.AreEqual(4, cache.All().ToList().Count); + Assert.AreEqual(4, cache.ReverseAll().ToList().Count); + Assert.AreEqual(2, cache.Range(ToKey("test"), ToKey("test2")).ToList().Count); + Assert.AreEqual(2, cache.ReverseRange(ToKey("test"), ToKey("test2")).ToList().Count); + } + + [Test] + public void DisabledCachingTest() + { + config.StateStoreCacheMaxBytes = 0; + cache.CreateCache(context); + + context.SetRecordMetaData(new RecordContext(new Headers(), 0, 100, 0, "topic")); + cache.Put(ToKey("test"), ToValue("value1")); + cache.Put(ToKey("test2"), ToValue("value2")); + + Assert.AreEqual(ToValue("value1"), cache.Get(ToKey("test"))); + Assert.AreEqual(ToValue("value2"), cache.Get(ToKey("test2"))); + Assert.AreEqual(2, cache.ApproximateNumEntries()); + + cache.Delete(ToKey("test")); + Assert.AreEqual(1, cache.ApproximateNumEntries()); + + Assert.Null(cache.Get(ToKey("test"))); + Assert.Null(cache.PutIfAbsent(ToKey("test"), ToValue("value1"))); + + cache.PutAll(new List> + { + new(ToKey("test3"), ToValue("value3")), + new(ToKey("test4"), ToValue("value4")) + }); + + Assert.AreEqual(4, cache.ApproximateNumEntries()); + + Assert.AreEqual(4, cache.All().ToList().Count); + Assert.AreEqual(4, cache.ReverseAll().ToList().Count); + Assert.AreEqual(4, cache.Range(ToKey("test"), ToKey("test4")).ToList().Count); + Assert.AreEqual(4, cache.ReverseRange(ToKey("test"), ToKey("test4")).ToList().Count); + } + + + [Test] + public void TestMetrics() + { + context.SetRecordMetaData(new RecordContext(new Headers(), 0, 100, 0, "topic")); + cache.PutAll(new List> + { + new(ToKey("test1"), ToValue("value1")), + new(ToKey("test2"), ToValue("value2")), + new(ToKey("test3"), ToValue("value3")), + new(ToKey("test4"), ToValue("value4")) + }); + + cache.Get(ToKey("test1")); + cache.Get(ToKey("test1")); + cache.Get(ToKey("test50")); // not found + cache.Get(ToKey("test100")); // not found + + var totalCacheSize = GetSensorMetric( + CachingMetrics.CACHE_SIZE_BYTES_TOTAL, + string.Empty, + StreamMetricsRegistry.STATE_STORE_LEVEL_GROUP); + + var hitRatioAvg = GetSensorMetric( + CachingMetrics.HIT_RATIO, + "-avg", + StreamMetricsRegistry.STATE_STORE_LEVEL_GROUP); + + Assert.IsTrue((double)totalCacheSize.Value > 0); + Assert.IsTrue((double)hitRatioAvg.Value > 0.5d); + } + + private StreamMetric GetSensorMetric(string sensorName, string metricSuffix, string group) + { + long now = DateTime.Now.GetMilliseconds(); + var sensor = metricsRegistry.GetSensors().FirstOrDefault(s => s.Name.Equals(GetSensorName(sensorName))); + if (sensor == null) + throw new NullReferenceException($"sensor {sensorName} not found"); + + MetricName keyMetric = MetricName.NameAndGroup( + sensorName + metricSuffix, + group); + + if (!sensor.Metrics.ContainsKey(keyMetric)) + throw new NullReferenceException($"metric {sensorName + metricSuffix}|{group} not found inside {sensorName}"); + + return sensor.Metrics[keyMetric]; + } + + [Test] + public void RehydrateCachingBasedOnWrappedStoreTest() + { + context.SetRecordMetaData(new RecordContext(new Headers(), 0, 100, 0, "topic")); + inMemoryKeyValue.Put(ToKey("test"), ToValue("value1")); + inMemoryKeyValue.Put(ToKey("test2"), ToValue("value2")); + Assert.AreEqual(0, cache.ApproximateNumEntries()); + + Assert.AreEqual(ToValue("value1"), cache.Get(ToKey("test"))); + Assert.AreEqual(ToValue("value2"), cache.Get(ToKey("test2"))); + Assert.AreEqual(2, cache.ApproximateNumEntries()); + cache.Flush(); + + cache.Delete(ToKey("test")); + Assert.AreEqual(1, cache.ApproximateNumEntries()); + Assert.AreEqual(2, inMemoryKeyValue.ApproximateNumEntries()); + + cache.Flush(); + Assert.AreEqual(1, inMemoryKeyValue.ApproximateNumEntries()); + } + + + private string GetSensorName(string sensorName) + => metricsRegistry.FullSensorName( + sensorName, + metricsRegistry.StoreSensorPrefix(threadId, id.ToString(), "store")); + } +} \ No newline at end of file diff --git a/test/Streamiz.Kafka.Net.Tests/Stores/CachingInMemoryWindowStoreTests.cs b/test/Streamiz.Kafka.Net.Tests/Stores/CachingInMemoryWindowStoreTests.cs new file mode 100644 index 00000000..76a6baaa --- /dev/null +++ b/test/Streamiz.Kafka.Net.Tests/Stores/CachingInMemoryWindowStoreTests.cs @@ -0,0 +1,19 @@ +using Streamiz.Kafka.Net.Crosscutting; +using Streamiz.Kafka.Net.State; +using Streamiz.Kafka.Net.State.InMemory; + +namespace Streamiz.Kafka.Net.Tests.Stores; + +public class CachingInMemoryWindowStoreTests + : AbstractPersistentWindowStoreTests +{ + protected override IWindowStore GetBackWindowStore() + { + return new InMemoryWindowStore( + "test-w-store", + RETENTION_MS, + WINDOW_SIZE, + false + ); + } +} \ No newline at end of file diff --git a/test/Streamiz.Kafka.Net.Tests/Stores/CachingPersistentWindowStoreTests.cs b/test/Streamiz.Kafka.Net.Tests/Stores/CachingPersistentWindowStoreTests.cs new file mode 100644 index 00000000..3e148162 --- /dev/null +++ b/test/Streamiz.Kafka.Net.Tests/Stores/CachingPersistentWindowStoreTests.cs @@ -0,0 +1,28 @@ +using Confluent.Kafka; +using Moq; +using Streamiz.Kafka.Net.Crosscutting; +using Streamiz.Kafka.Net.Processors; +using Streamiz.Kafka.Net.Processors.Internal; +using Streamiz.Kafka.Net.State; +using Streamiz.Kafka.Net.State.Cache; +using Streamiz.Kafka.Net.State.Internal; + +namespace Streamiz.Kafka.Net.Tests.Stores; + +public class CachingPersistentWindowStoreTests : + AbstractPersistentWindowStoreTests +{ + protected override IWindowStore GetBackWindowStore() + { + var bytesStore = new RocksDbSegmentedBytesStore( + "test-w-store", + (long)RETENTION_MS.TotalMilliseconds, + SEGMENT_INTERVAL, + keySchema); + + return new RocksDbWindowStore( + bytesStore, + WINDOW_SIZE + , false); + } +} \ No newline at end of file diff --git a/test/Streamiz.Kafka.Net.Tests/Stores/ChangeLoggingTimestampedKeyValueBytesStoreTests.cs b/test/Streamiz.Kafka.Net.Tests/Stores/ChangeLoggingTimestampedKeyValueBytesStoreTests.cs index 44adf928..be7e7e9b 100644 --- a/test/Streamiz.Kafka.Net.Tests/Stores/ChangeLoggingTimestampedKeyValueBytesStoreTests.cs +++ b/test/Streamiz.Kafka.Net.Tests/Stores/ChangeLoggingTimestampedKeyValueBytesStoreTests.cs @@ -1,9 +1,14 @@ -using Confluent.Kafka; +using System; +using System.Collections.Generic; +using System.Linq; +using Confluent.Kafka; using Moq; using NUnit.Framework; using Streamiz.Kafka.Net.Crosscutting; using Streamiz.Kafka.Net.Kafka; using Streamiz.Kafka.Net.Kafka.Internal; +using Streamiz.Kafka.Net.Metrics; +using Streamiz.Kafka.Net.Metrics.Internal; using Streamiz.Kafka.Net.Mock; using Streamiz.Kafka.Net.Mock.Sync; using Streamiz.Kafka.Net.Processors; @@ -12,29 +17,24 @@ using Streamiz.Kafka.Net.State; using Streamiz.Kafka.Net.State.InMemory; using Streamiz.Kafka.Net.State.Logging; -using System; -using System.Collections.Generic; -using System.Linq; -using Streamiz.Kafka.Net.Metrics; -using Streamiz.Kafka.Net.Metrics.Internal; namespace Streamiz.Kafka.Net.Tests.Stores { public class ChangeLoggingTimestampedKeyValueBytesStoreTests { - private StreamConfig config = null; - private ChangeLoggingTimestampedKeyValueBytesStore store = null; - private ProcessorContext context = null; - private TaskId id = null; - private TopicPartition partition = null; - private ProcessorStateManager stateManager = null; - private Mock task = null; + private StreamConfig config; + private ChangeLoggingTimestampedKeyValueBytesStore store; + private ProcessorContext context; + private TaskId id; + private TopicPartition partition; + private ProcessorStateManager stateManager; + private Mock task; - private SyncKafkaSupplier kafkaSupplier = null; - private IRecordCollector recordCollector = null; + private SyncKafkaSupplier kafkaSupplier; + private IRecordCollector recordCollector; - private StringSerDes stringSerDes = new StringSerDes(); + private StringSerDes stringSerDes = new(); private ValueAndTimestampSerDes valueAndTimestampSerDes; [SetUp] @@ -42,7 +42,7 @@ public void Begin() { valueAndTimestampSerDes = new ValueAndTimestampSerDes(stringSerDes); config = new StreamConfig(); - config.ApplicationId = $"unit-test-changelogging-tkv"; + config.ApplicationId = "unit-test-changelogging-tkv"; id = new TaskId { Id = 0, Partition = 0 }; partition = new TopicPartition("source", 0); diff --git a/test/Streamiz.Kafka.Net.Tests/Stores/ChangeLoggingTimestampedWindowBytesStoreTests.cs b/test/Streamiz.Kafka.Net.Tests/Stores/ChangeLoggingTimestampedWindowBytesStoreTests.cs index 5545089b..35d4da6d 100644 --- a/test/Streamiz.Kafka.Net.Tests/Stores/ChangeLoggingTimestampedWindowBytesStoreTests.cs +++ b/test/Streamiz.Kafka.Net.Tests/Stores/ChangeLoggingTimestampedWindowBytesStoreTests.cs @@ -1,9 +1,13 @@ -using Confluent.Kafka; +using System; +using System.Collections.Generic; +using Confluent.Kafka; using Moq; using NUnit.Framework; using Streamiz.Kafka.Net.Crosscutting; using Streamiz.Kafka.Net.Kafka; using Streamiz.Kafka.Net.Kafka.Internal; +using Streamiz.Kafka.Net.Metrics; +using Streamiz.Kafka.Net.Metrics.Internal; using Streamiz.Kafka.Net.Mock; using Streamiz.Kafka.Net.Mock.Sync; using Streamiz.Kafka.Net.Processors; @@ -12,27 +16,22 @@ using Streamiz.Kafka.Net.State; using Streamiz.Kafka.Net.State.InMemory; using Streamiz.Kafka.Net.State.Logging; -using System; -using System.Collections.Generic; -using Streamiz.Kafka.Net.Metrics; -using Streamiz.Kafka.Net.Metrics.Internal; - namespace Streamiz.Kafka.Net.Tests.Stores { public class ChangeLoggingTimestampedWindowBytesStoreTests { - private StreamConfig config = null; - private ChangeLoggingTimestampedWindowBytesStore store = null; - private ProcessorContext context = null; - private TaskId id = null; - private TopicPartition partition = null; - private ProcessorStateManager stateManager = null; - private Mock task = null; + private StreamConfig config; + private ChangeLoggingTimestampedWindowBytesStore store; + private ProcessorContext context; + private TaskId id; + private TopicPartition partition; + private ProcessorStateManager stateManager; + private Mock task; - private SyncKafkaSupplier kafkaSupplier = null; - private IRecordCollector recordCollector = null; + private SyncKafkaSupplier kafkaSupplier; + private IRecordCollector recordCollector; private static StringSerDes stringSerDes = new StringSerDes(); private static ValueAndTimestampSerDes valueAndTsSerDes = new ValueAndTimestampSerDes(stringSerDes); @@ -42,7 +41,7 @@ public class ChangeLoggingTimestampedWindowBytesStoreTests public void Begin() { config = new StreamConfig(); - config.ApplicationId = $"unit-test-changelogging-tw"; + config.ApplicationId = "unit-test-changelogging-tw"; id = new TaskId { Id = 0, Partition = 0 }; partition = new TopicPartition("source", 0); diff --git a/test/Streamiz.Kafka.Net.Tests/Stores/ChangeLoggingWindowBytesStoreTests.cs b/test/Streamiz.Kafka.Net.Tests/Stores/ChangeLoggingWindowBytesStoreTests.cs index 618616e3..329029f7 100644 --- a/test/Streamiz.Kafka.Net.Tests/Stores/ChangeLoggingWindowBytesStoreTests.cs +++ b/test/Streamiz.Kafka.Net.Tests/Stores/ChangeLoggingWindowBytesStoreTests.cs @@ -1,9 +1,13 @@ -using Confluent.Kafka; +using System; +using System.Collections.Generic; +using Confluent.Kafka; using Moq; using NUnit.Framework; using Streamiz.Kafka.Net.Crosscutting; using Streamiz.Kafka.Net.Kafka; using Streamiz.Kafka.Net.Kafka.Internal; +using Streamiz.Kafka.Net.Metrics; +using Streamiz.Kafka.Net.Metrics.Internal; using Streamiz.Kafka.Net.Mock; using Streamiz.Kafka.Net.Mock.Sync; using Streamiz.Kafka.Net.Processors; @@ -12,25 +16,21 @@ using Streamiz.Kafka.Net.State; using Streamiz.Kafka.Net.State.InMemory; using Streamiz.Kafka.Net.State.Logging; -using System; -using System.Collections.Generic; -using Streamiz.Kafka.Net.Metrics; -using Streamiz.Kafka.Net.Metrics.Internal; namespace Streamiz.Kafka.Net.Tests.Stores { public class ChangeLoggingWindowBytesStoreTests { - private StreamConfig config = null; - private ChangeLoggingWindowBytesStore store = null; - private ProcessorContext context = null; - private TaskId id = null; - private TopicPartition partition = null; - private ProcessorStateManager stateManager = null; - private Mock task = null; - - private SyncKafkaSupplier kafkaSupplier = null; - private IRecordCollector recordCollector = null; + private StreamConfig config; + private ChangeLoggingWindowBytesStore store; + private ProcessorContext context; + private TaskId id; + private TopicPartition partition; + private ProcessorStateManager stateManager; + private Mock task; + + private SyncKafkaSupplier kafkaSupplier; + private IRecordCollector recordCollector; private static StringSerDes stringSerDes = new StringSerDes(); private static TimeWindowedSerDes windowSerDes = new TimeWindowedSerDes(stringSerDes, TimeSpan.FromSeconds(1).Milliseconds); @@ -38,7 +38,7 @@ public class ChangeLoggingWindowBytesStoreTests public void Begin() { config = new StreamConfig(); - config.ApplicationId = $"unit-test-changelogging-w"; + config.ApplicationId = "unit-test-changelogging-w"; id = new TaskId { Id = 0, Partition = 0 }; partition = new TopicPartition("source", 0); diff --git a/test/Streamiz.Kafka.Net.Tests/Stores/InMemoryWindowStoreTests.cs b/test/Streamiz.Kafka.Net.Tests/Stores/InMemoryWindowStoreTests.cs index 7cc7c6d1..9d54315c 100644 --- a/test/Streamiz.Kafka.Net.Tests/Stores/InMemoryWindowStoreTests.cs +++ b/test/Streamiz.Kafka.Net.Tests/Stores/InMemoryWindowStoreTests.cs @@ -1,8 +1,7 @@ using System; -using System.Linq; using System.Text; using System.Threading; -using Avro.Util; +using Moq; using NUnit.Framework; using Streamiz.Kafka.Net.Crosscutting; using Streamiz.Kafka.Net.Metrics; @@ -194,7 +193,7 @@ public void FetchRangeDoesNotExist() public void TestRetention() { var metricsRegistry = new StreamMetricsRegistry(); - var mockContext = new Moq.Mock(); + var mockContext = new Mock(); mockContext.Setup(c => c.Id).Returns(new TaskId{Id = 0, Partition = 0}); mockContext.Setup(c => c.Metrics).Returns(metricsRegistry); mockContext.Setup(c => c.Timestamp).Returns(DateTime.Now.GetMilliseconds()); diff --git a/test/Streamiz.Kafka.Net.Tests/Stores/MemoryCacheTests.cs b/test/Streamiz.Kafka.Net.Tests/Stores/MemoryCacheTests.cs new file mode 100644 index 00000000..e53ac53f --- /dev/null +++ b/test/Streamiz.Kafka.Net.Tests/Stores/MemoryCacheTests.cs @@ -0,0 +1,155 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading; +using Confluent.Kafka; +using NUnit.Framework; +using Streamiz.Kafka.Net.Crosscutting; +using Streamiz.Kafka.Net.State.Cache; +using Streamiz.Kafka.Net.State.Cache.Internal; + +namespace Streamiz.Kafka.Net.Tests.Stores; + +public class MemoryCacheTests +{ + private MemoryCache memoryCache; + private const int headerSizeCacheEntry = 25; + + [SetUp] + public void Initialize() + { + var options = new MemoryCacheOptions(); + options.SizeLimit = 1000; + options.CompactionPercentage = 0.1; + + memoryCache = new MemoryCache(options, new BytesComparer()); + } + + [TearDown] + public void Dispose() + { + memoryCache?.Dispose(); + } + + private CacheEntryValue CreateValueEntry(string value) + { + return new CacheEntryValue( + Encoding.UTF8.GetBytes(value), + new Headers(), + 0, + DateTime.UtcNow.GetMilliseconds(), + 0, + "topic"); + } + + private void PutCache(Bytes key, string value, PostEvictionDelegate evictionDelegate) + { + var entry = CreateValueEntry(value); + long totalSize = key.Get.LongLength + entry.Size; + + var memoryCacheEntryOptions = new MemoryCacheEntryOptions() + .SetSize(totalSize) + .RegisterPostEvictionCallback(evictionDelegate, memoryCache); + + memoryCache.Set(key, entry, memoryCacheEntryOptions, EvictionReason.Setted); + } + + [Test] + public void KeepTrackOnSize() + { + var bytes = Bytes.Wrap(new byte[] { 1 }); + + PutCache(bytes, "coucou", (key, value, reason, state) => { }); + PutCache(bytes, "sylvain", (key, value, reason, state) => { }); + Assert.AreEqual(1 + headerSizeCacheEntry + "sylvain".Length, memoryCache.Size); + } + + [Test] + public void ShouldPutGet() { + PutCache(Bytes.Wrap(new byte[]{1}), "a", (key, value, reason, state) => { }); + PutCache(Bytes.Wrap(new byte[]{2}), "b", (key, value, reason, state) => { }); + PutCache(Bytes.Wrap(new byte[]{3}), "c", (key, value, reason, state) => { }); + + Assert.AreEqual("a", Encoding.UTF8.GetString(memoryCache.Get(Bytes.Wrap(new byte[]{1})).Value)); + Assert.AreEqual("b", Encoding.UTF8.GetString(memoryCache.Get(Bytes.Wrap(new byte[]{2})).Value)); + Assert.AreEqual("c", Encoding.UTF8.GetString(memoryCache.Get(Bytes.Wrap(new byte[]{3})).Value)); + Assert.AreEqual(3, memoryCache.GetCurrentStatistics().TotalHits); + } + + [Test] + public void ShouldDeleteAndUpdateSize() { + PutCache(Bytes.Wrap(new byte[]{1}), "a", (key, value, reason, state) => { }); + memoryCache.Remove(Bytes.Wrap(new byte[]{1})); + Assert.AreEqual(0, memoryCache.Size); + } + + [Test] + public void ShouldEvictEldestEntry() { + memoryCache?.Dispose(); + + var options = new MemoryCacheOptions(); + options.SizeLimit = 50; + options.CompactionPercentage = 0.1; + memoryCache = new MemoryCache(options, new BytesComparer()); + + PutCache(Bytes.Wrap(new byte[]{1}), "test123", (key, value, reason, state) => { + Assert.AreEqual(EvictionReason.Capacity, reason); + }); + Thread.Sleep(5); + PutCache(Bytes.Wrap(new byte[]{2}), "test456", (key, value, reason, state) => { }); + Thread.Sleep(5); + PutCache(Bytes.Wrap(new byte[]{3}), "test789", (key, value, reason, state) => { }); + + + Assert.IsNull(memoryCache.Get(Bytes.Wrap(new byte[] { 1 }))); + Assert.AreEqual(2L, memoryCache.Count); + } + + [Test] + public void ShouldEvictLRU() { + memoryCache?.Dispose(); + + var results = new List(); + var expected = new List{ + "test123", "test456", "test789" + }; + + var expectedbis = new List{ + "test456", "test789", "test123" + }; + + var clockTime = new MockSystemTime(DateTime.Now); + + var options = new MemoryCacheOptions(); + options.SizeLimit = 100000; + options.CompactionPercentage = 0.1; + memoryCache = new MemoryCache(options, new BytesComparer(), clockTime); + + PostEvictionDelegate deleg = (key, value, reason, state) => { + results.Add(Encoding.UTF8.GetString(value.Value)); + }; + + PutCache(Bytes.Wrap(new byte[]{1}), "test123", deleg); + clockTime.AdvanceTime(TimeSpan.FromMinutes(1)); + PutCache(Bytes.Wrap(new byte[]{2}), "test456", deleg); + clockTime.AdvanceTime(TimeSpan.FromMinutes(1)); + PutCache(Bytes.Wrap(new byte[]{3}), "test789", deleg); + clockTime.AdvanceTime(TimeSpan.FromMinutes(1)); + + memoryCache.Compact(1d); // total flush + + Assert.AreEqual(expected, results); + results.Clear(); + + PutCache(Bytes.Wrap(new byte[]{1}), "test123", deleg); + clockTime.AdvanceTime(TimeSpan.FromMinutes(1)); + PutCache(Bytes.Wrap(new byte[]{2}), "test456", deleg); + clockTime.AdvanceTime(TimeSpan.FromMinutes(1)); + PutCache(Bytes.Wrap(new byte[]{3}), "test789", deleg); + clockTime.AdvanceTime(TimeSpan.FromMinutes(1)); + memoryCache.Get(Bytes.Wrap(new byte[] { 1 })); + memoryCache.Compact(1d); // total + Assert.AreEqual(expectedbis, results); + } + +} \ No newline at end of file diff --git a/test/Streamiz.Kafka.Net.Tests/Stores/MergedSortedCacheWrappedWindowStoreKeyValueEnumeratorTests.cs b/test/Streamiz.Kafka.Net.Tests/Stores/MergedSortedCacheWrappedWindowStoreKeyValueEnumeratorTests.cs new file mode 100644 index 00000000..bc07c3b9 --- /dev/null +++ b/test/Streamiz.Kafka.Net.Tests/Stores/MergedSortedCacheWrappedWindowStoreKeyValueEnumeratorTests.cs @@ -0,0 +1,275 @@ +using System; +using System.Collections.Generic; +using System.Text; +using NUnit.Framework; +using Streamiz.Kafka.Net.Crosscutting; +using Streamiz.Kafka.Net.SerDes; +using Streamiz.Kafka.Net.SerDes.Internal; +using Streamiz.Kafka.Net.State; +using Streamiz.Kafka.Net.State.Cache; +using Streamiz.Kafka.Net.State.Cache.Enumerator; +using Streamiz.Kafka.Net.State.Enumerator; +using Streamiz.Kafka.Net.State.Helper; +using Streamiz.Kafka.Net.State.Internal; +using Streamiz.Kafka.Net.Stream; + +namespace Streamiz.Kafka.Net.Tests.Stores; + +public class MergedSortedCacheWrappedWindowStoreKeyValueEnumeratorTests +{ + private static IKeyValueEnumerator, byte[]> StoreKvs() + { + return new WrapEnumerableKeyValueEnumerator, byte[]>( + new List, byte[]>> + { + new(new Windowed(Bytes.Wrap(Encoding.UTF8.GetBytes(storeKey)), storeWindow), + Encoding.UTF8.GetBytes(storeKey)) + }); + } + + private static IKeyValueEnumerator CacheKvs() + { + return new WrapEnumerableKeyValueEnumerator( + new List> + { + new(segmentedCacheFunction.CacheKey(WindowKeyHelper.ToStoreKeyBinary( + new Windowed(cacheKey, cacheWindow), 0, new StringSerDes())), + new CacheEntryValue(Encoding.UTF8.GetBytes(cacheKey))) + }); + } + + private class MockSegmentedCacheFunction : SegmentedCacheFunction + { + public MockSegmentedCacheFunction(IKeySchema keySchema, long segmentInterval) + : base(keySchema, segmentInterval) + { + } + + public override long SegmentId(Bytes key) + { + return 0L; + } + } + + private static readonly int WINDOW_SIZE = 10; + private static readonly String storeKey = "a"; + private static readonly String cacheKey = "b"; + private static readonly TimeWindow storeWindow = new(0, 1); + private static readonly TimeWindow cacheWindow = new(10, 20); + + private static readonly MockSegmentedCacheFunction + segmentedCacheFunction = new MockSegmentedCacheFunction(null, -1); + + private IKeyValueEnumerator, byte[]> storeKvs; + private IKeyValueEnumerator cacheKvs; + private ISerDes serdes = new BytesSerDes(); + + private MergedSortedCacheWindowStoreKeyValueEnumerator CreateEnumerator( + IKeyValueEnumerator, byte[]> storeKvs, + IKeyValueEnumerator cacheKvs, + bool forward) + { + return new MergedSortedCacheWindowStoreKeyValueEnumerator( + cacheKvs, + storeKvs, + WINDOW_SIZE, + segmentedCacheFunction, + serdes, + "topic", + WindowKeyHelper.FromStoreKey, + WindowKeyHelper.ToStoreKeyBinary, + forward + ); + } + + private KeyValuePair, byte[]> ConvertKeyValuePair(KeyValuePair, string> pair) + { + return new KeyValuePair, byte[]>( + new Windowed(Bytes.Wrap(Encoding.UTF8.GetBytes(pair.Key.Key)), pair.Key.Window), + Encoding.UTF8.GetBytes(pair.Value)); + } + + private Windowed ConvertWindowedKey(Windowed windowed) + { + var key = Encoding.UTF8.GetString(windowed.Key.Get); + return new Windowed(key, windowed.Window); + } + + [SetUp] + public void Init() + { + storeKvs = StoreKvs(); + cacheKvs = CacheKvs(); + } + + + [Test] + public void ShouldHaveNextFromStore() { + using var enumerator = CreateEnumerator(storeKvs, EmptyKeyValueEnumerator.Empty, true); + Assert.IsTrue(enumerator.MoveNext()); + } + + [Test] + public void ShouldHaveNextFromReverseStore() { + using var enumerator = CreateEnumerator(storeKvs, EmptyKeyValueEnumerator.Empty, false); + Assert.IsTrue(enumerator.MoveNext()); + } + + [Test] + public void ShouldGetNextFromStore() { + using var enumerator = CreateEnumerator(storeKvs, EmptyKeyValueEnumerator.Empty, true); + Assert.IsTrue(enumerator.MoveNext()); + Assert.IsTrue(enumerator.Current.HasValue); + Assert.AreEqual( + ConvertKeyValuePair(new KeyValuePair, string>( + new Windowed(storeKey, storeWindow), storeKey)), + enumerator.Current.Value); + } + + [Test] + public void ShouldGetNextFromReverseStore() { + using var enumerator = CreateEnumerator(storeKvs, EmptyKeyValueEnumerator.Empty, false); + Assert.IsTrue(enumerator.MoveNext()); + Assert.IsTrue(enumerator.Current.HasValue); + Assert.AreEqual( + ConvertKeyValuePair(new KeyValuePair, string>( + new Windowed(storeKey, storeWindow), storeKey)), + enumerator.Current.Value); + } + + [Test] + public void ShouldPeekNextKeyFromStore() { + using var enumerator = CreateEnumerator(storeKvs, EmptyKeyValueEnumerator.Empty, true); + Assert.IsTrue(enumerator.MoveNext()); + Assert.AreEqual( + new Windowed(storeKey, storeWindow), + ConvertWindowedKey(enumerator.PeekNextKey())); + } + + [Test] + public void ShouldPeekNextKeyFromReverseStore() { + using var enumerator = CreateEnumerator(storeKvs, EmptyKeyValueEnumerator.Empty, false); + Assert.IsTrue(enumerator.MoveNext()); + Assert.AreEqual( + new Windowed(storeKey, storeWindow), + ConvertWindowedKey(enumerator.PeekNextKey())); + } + + [Test] + public void ShouldHaveNextFromCache() { + using var enumerator = CreateEnumerator( + EmptyKeyValueEnumerator, byte[]>.Empty, + cacheKvs, + true); + Assert.IsTrue(enumerator.MoveNext()); + } + + [Test] + public void ShouldHaveNextFromReverseCache() { + using var enumerator = CreateEnumerator( + EmptyKeyValueEnumerator, byte[]>.Empty, + cacheKvs, + false); + Assert.IsTrue(enumerator.MoveNext()); + } + + [Test] + public void ShoulGetNextFromCache() { + using var enumerator = CreateEnumerator( + EmptyKeyValueEnumerator, byte[]>.Empty, + cacheKvs, + true); + Assert.IsTrue(enumerator.MoveNext()); + Assert.IsTrue(enumerator.Current.HasValue); + Assert.AreEqual( + ConvertKeyValuePair(new KeyValuePair, string>( + new Windowed(cacheKey, cacheWindow), cacheKey)), + enumerator.Current.Value); + } + + [Test] + public void ShoulGetNextFromReverseCache() { + using var enumerator = CreateEnumerator( + EmptyKeyValueEnumerator, byte[]>.Empty, + cacheKvs, + false); + Assert.IsTrue(enumerator.MoveNext()); + Assert.IsTrue(enumerator.Current.HasValue); + Assert.AreEqual( + ConvertKeyValuePair(new KeyValuePair, string>( + new Windowed(cacheKey, cacheWindow), cacheKey)), + enumerator.Current.Value); + } + + [Test] + public void ShoulPeekNextKeyFromCache() { + using var enumerator = CreateEnumerator( + EmptyKeyValueEnumerator, byte[]>.Empty, + cacheKvs, + true); + Assert.IsTrue(enumerator.MoveNext()); + Assert.AreEqual( + new Windowed(cacheKey, cacheWindow), + ConvertWindowedKey(enumerator.PeekNextKey())); + } + + [Test] + public void ShoulPeekNextKeyFromReverseCache() { + using var enumerator = CreateEnumerator( + EmptyKeyValueEnumerator, byte[]>.Empty, + cacheKvs, + false); + Assert.IsTrue(enumerator.MoveNext()); + Assert.AreEqual( + new Windowed(cacheKey, cacheWindow), + ConvertWindowedKey(enumerator.PeekNextKey())); + } + + [Test] + public void ShouldIterateBothStoreAndCache() { + using var enumerator = CreateEnumerator( + storeKvs, + cacheKvs, + true); + + Assert.IsTrue(enumerator.MoveNext()); + Assert.IsTrue(enumerator.Current.HasValue); + Assert.AreEqual( + ConvertKeyValuePair(new KeyValuePair, string>( + new Windowed(storeKey, storeWindow), storeKey)), + enumerator.Current.Value); + + Assert.IsTrue(enumerator.MoveNext()); + Assert.IsTrue(enumerator.Current.HasValue); + Assert.AreEqual( + ConvertKeyValuePair(new KeyValuePair, string>( + new Windowed(cacheKey, cacheWindow), cacheKey)), + enumerator.Current.Value); + + Assert.IsFalse(enumerator.MoveNext()); + } + + [Test] + public void ShouldReverseIterateBothStoreAndCache() { + using var enumerator = CreateEnumerator( + storeKvs, + cacheKvs, + false); + + Assert.IsTrue(enumerator.MoveNext()); + Assert.IsTrue(enumerator.Current.HasValue); + Assert.AreEqual( + ConvertKeyValuePair(new KeyValuePair, string>( + new Windowed(cacheKey, cacheWindow), cacheKey)), + enumerator.Current.Value); + + Assert.IsTrue(enumerator.MoveNext()); + Assert.IsTrue(enumerator.Current.HasValue); + Assert.AreEqual( + ConvertKeyValuePair(new KeyValuePair, string>( + new Windowed(storeKey, storeWindow), storeKey)), + enumerator.Current.Value); + + Assert.IsFalse(enumerator.MoveNext()); + } +} \ No newline at end of file diff --git a/test/Streamiz.Kafka.Net.Tests/Stores/MergedStoredCacheKeyValueEnumeratorTest.cs b/test/Streamiz.Kafka.Net.Tests/Stores/MergedStoredCacheKeyValueEnumeratorTest.cs new file mode 100644 index 00000000..4bea6779 --- /dev/null +++ b/test/Streamiz.Kafka.Net.Tests/Stores/MergedStoredCacheKeyValueEnumeratorTest.cs @@ -0,0 +1,247 @@ +using System; +using Confluent.Kafka; +using NUnit.Framework; +using Streamiz.Kafka.Net.Crosscutting; +using Streamiz.Kafka.Net.State; +using Streamiz.Kafka.Net.State.Cache; +using Streamiz.Kafka.Net.State.Cache.Enumerator; +using Streamiz.Kafka.Net.State.Cache.Internal; +using Streamiz.Kafka.Net.State.Enumerator; +using Streamiz.Kafka.Net.State.InMemory; + +namespace Streamiz.Kafka.Net.Tests.Stores; + +public class MergedStoredCacheKeyValueEnumeratorTest +{ + private IKeyValueStore internalStore; + private MemoryCache cacheStore; + + [SetUp] + public void Begin() + { + internalStore = new InMemoryKeyValueStore("in-mem-store"); + cacheStore = new MemoryCache(new MemoryCacheOptions + { + SizeLimit = Int32.MaxValue, + CompactionPercentage = .20 + }, new BytesComparer()); + } + + [TearDown] + public void Dispose() + { + internalStore.Close(); + cacheStore.Dispose(); + } + + private MergedStoredCacheKeyValueEnumerator CreateEnumerator() + { + var cacheEnumerator = new CacheEnumerator( + cacheStore.KeySetEnumerable(true), cacheStore, () => { }); + var storeEnumerator = new WrapEnumerableKeyValueEnumerator(internalStore.All()); + return new MergedStoredCacheKeyValueEnumerator(cacheEnumerator, storeEnumerator, true); + } + + private void PutCache(Bytes key, byte[] value) + { + var cacheEntry = new CacheEntryValue( + value, + new Headers(), + 0, + 100, + 0, + "topic"); + + long totalSize = key.Get.LongLength + cacheEntry.Size; + + var memoryCacheEntryOptions = new MemoryCacheEntryOptions() + .SetSize(totalSize); + + cacheStore.Set(key, cacheEntry, memoryCacheEntryOptions, EvictionReason.Setted); + } + + [Test] + public void ShouldIterateOverRange() + { + byte[][] bytes = + { + new byte[] { 0 }, new byte[] { 1 }, new byte[] { 2 }, new byte[] { 3 }, new byte[] { 4 }, + new byte[] { 5 }, new byte[] { 6 }, new byte[] { 7 }, new byte[] { 8 }, new byte[] { 9 }, new byte[] { 10 }, + new byte[] { 11 } + }; + + for (int i = 0; i < bytes.Length; i += 2) + { + internalStore.Put(Bytes.Wrap(bytes[i]), bytes[i]); // 0 2 4 6 8 10 + PutCache(Bytes.Wrap(bytes[i + 1]), bytes[i + 1]); // 1 3 5 7 8 11 + } + + Bytes from = Bytes.Wrap(new byte[] {2}); + Bytes to = Bytes.Wrap(new byte[] {9}); + + var cacheEnumerator = new CacheEnumerator( + cacheStore.KeyRange(from, to, true, true), cacheStore, () => { }); + var storeEnumerator = internalStore.Range(from, to); + var mergedEnumerator = new MergedStoredCacheKeyValueEnumerator(cacheEnumerator, storeEnumerator, true); + + // 23456789 + + byte[][] values = new byte[8][]; + int index = 0; + int bytesIndex = 2; + while (mergedEnumerator.MoveNext()) { + Assert.NotNull(mergedEnumerator.Current); + byte[] value = mergedEnumerator.Current.Value.Value; + values[index++] = value; + Assert.AreEqual(bytes[bytesIndex++], value); + } + mergedEnumerator.Dispose(); + } + + [Test] + public void ShouldReverseIterateOverRange() { + byte[][] bytes = + { + new byte[] { 0 }, new byte[] { 1 }, new byte[] { 2 }, new byte[] { 3 }, new byte[] { 4 }, + new byte[] { 5 }, new byte[] { 6 }, new byte[] { 7 }, new byte[] { 8 }, new byte[] { 9 }, new byte[] { 10 }, + new byte[] { 11 } + }; + + for (int i = 0; i < bytes.Length; i += 2) + { + internalStore.Put(Bytes.Wrap(bytes[i]), bytes[i]); // 0 2 4 6 8 10 + PutCache(Bytes.Wrap(bytes[i + 1]), bytes[i + 1]); // 1 3 5 7 8 11 + } + + Bytes from = Bytes.Wrap(new byte[] {2}); + Bytes to = Bytes.Wrap(new byte[] {9}); + + var cacheEnumerator = new CacheEnumerator( + cacheStore.KeyRange(from, to, true, false), cacheStore, null); + var storeEnumerator = internalStore.ReverseRange(from, to); + var mergedEnumerator = new MergedStoredCacheKeyValueEnumerator(cacheEnumerator, storeEnumerator, false); + + // 98765432 + byte[][] values = new byte[8][]; + int index = 0; + int bytesIndex = 9; + while (mergedEnumerator.MoveNext()) { + Assert.NotNull(mergedEnumerator.Current); + byte[] value = mergedEnumerator.Current.Value.Value; + values[index++] = value; + Assert.AreEqual(bytes[bytesIndex--], value); + } + mergedEnumerator.Dispose(); + } + + [Test] + public void ShouldSkipLargerDeletedCacheValue() + { + byte[][] bytes = { new byte[] { 0 }, new byte[] { 1 } }; + internalStore.Put(Bytes.Wrap(bytes[0]), bytes[0]); + PutCache(Bytes.Wrap(bytes[1]), null); + using var enumerator = CreateEnumerator(); + Assert.IsTrue(enumerator.MoveNext()); + Assert.NotNull(enumerator.Current); + Assert.AreEqual(new byte[] { 0 }, enumerator.Current.Value.Key.Get); + Assert.IsFalse(enumerator.MoveNext()); + } + + [Test] + public void ShouldSkipSmallerDeletedCachedValue() + { + byte[][] bytes = { new byte[] { 0 }, new byte[] { 1 } }; + PutCache(Bytes.Wrap(bytes[0]), null); + internalStore.Put(Bytes.Wrap(bytes[1]), bytes[1]); + using var enumerator = CreateEnumerator(); + Assert.IsTrue(enumerator.MoveNext()); + Assert.NotNull(enumerator.Current); + Assert.AreEqual(new byte[] { 1 }, enumerator.Current.Value.Key.Get); + Assert.IsFalse(enumerator.MoveNext()); + } + + [Test] + public void ShouldIgnoreIfDeletedInCacheButExistsInStore() + { + byte[][] bytes = { new byte[] { 0 } }; + PutCache(Bytes.Wrap(bytes[0]), null); + internalStore.Put(Bytes.Wrap(bytes[0]), bytes[0]); + using var enumerator = CreateEnumerator(); + Assert.IsFalse(enumerator.MoveNext()); + } + + [Test] + public void ShouldNotHaveNextIfAllCachedItemsDeleted() + { + byte[][] bytes = { new byte[] { 0 }, new byte[] { 1 }, new byte[] { 2 } }; + foreach (byte[] aByte in bytes) + { + Bytes aBytes = Bytes.Wrap(aByte); + internalStore.Put(aBytes, aByte); + PutCache(aBytes, null); + } + + using var enumerator = CreateEnumerator(); + Assert.IsFalse(enumerator.MoveNext()); + } + + [Test] + public void ShouldNotHaveNextIfOnlyCacheItemsAndAllDeleted() + { + byte[][] bytes = { new byte[] { 0 }, new byte[] { 1 }, new byte[] { 2 } }; + foreach (byte[] aByte in bytes) + { + PutCache(Bytes.Wrap(aByte), null); + } + + using var enumerator = CreateEnumerator(); + Assert.IsFalse(enumerator.MoveNext()); + } + + [Test] + public void ShouldSkipAllDeletedFromCache() + { + byte[][] bytes = + { + new byte[] { 0 }, new byte[] { 1 }, new byte[] { 2 }, new byte[] { 3 }, new byte[] { 4 }, new byte[] { 5 }, + new byte[] { 6 }, new byte[] { 7 }, new byte[] { 8 }, new byte[] { 9 }, new byte[] { 10 }, new byte[] { 11 } + }; + + foreach (byte[] aByte in bytes) + { + Bytes aBytes = Bytes.Wrap(aByte); + internalStore.Put(aBytes, aByte); + PutCache(aBytes, aByte); + } + + PutCache(Bytes.Wrap(new byte[] { 1 }), null); + PutCache(Bytes.Wrap(new byte[] { 2 }), null); + PutCache(Bytes.Wrap(new byte[] { 3 }), null); + PutCache(Bytes.Wrap(new byte[] { 8 }), null); + PutCache(Bytes.Wrap(new byte[] { 11 }), null); + + using var enumerator = CreateEnumerator(); + enumerator.MoveNext(); + Assert.NotNull(enumerator.Current); + Assert.AreEqual(new byte[] { 0 }, enumerator.Current.Value.Value); + enumerator.MoveNext(); + Assert.NotNull(enumerator.Current); + Assert.AreEqual(new byte[] { 4 }, enumerator.Current.Value.Value); + enumerator.MoveNext(); + Assert.NotNull(enumerator.Current); + Assert.AreEqual(new byte[] { 5 }, enumerator.Current.Value.Value); + enumerator.MoveNext(); + Assert.NotNull(enumerator.Current); + Assert.AreEqual(new byte[] { 6 }, enumerator.Current.Value.Value); + enumerator.MoveNext(); + Assert.NotNull(enumerator.Current); + Assert.AreEqual(new byte[] { 7 }, enumerator.Current.Value.Value); + enumerator.MoveNext(); + Assert.NotNull(enumerator.Current); + Assert.AreEqual(new byte[] { 9 }, enumerator.Current.Value.Value); + enumerator.MoveNext(); + Assert.NotNull(enumerator.Current); + Assert.AreEqual(new byte[] { 10 }, enumerator.Current.Value.Value); + Assert.IsFalse(enumerator.MoveNext()); + } +} \ No newline at end of file diff --git a/test/Streamiz.Kafka.Net.Tests/Stores/RocksDbKeyValueBytesStoreSupplierTests.cs b/test/Streamiz.Kafka.Net.Tests/Stores/RocksDbKeyValueBytesStoreSupplierTests.cs index 5a9ba3c1..9586a931 100644 --- a/test/Streamiz.Kafka.Net.Tests/Stores/RocksDbKeyValueBytesStoreSupplierTests.cs +++ b/test/Streamiz.Kafka.Net.Tests/Stores/RocksDbKeyValueBytesStoreSupplierTests.cs @@ -50,7 +50,7 @@ public void StoresPersistentKeyValueStoreTest() // builder.Table("table-topic", RocksDb.As("table-topic-store")); builder.Table("table-topic", Materialized>.Create( - Streamiz.Kafka.Net.State.Stores.PersistentKeyValueStore("table-topic-store"))); + State.Stores.PersistentKeyValueStore("table-topic-store"))); var config = new StreamConfig(); config.ApplicationId = "test-map"; diff --git a/test/Streamiz.Kafka.Net.Tests/Stores/RocksDbKeyValueStoreTests.cs b/test/Streamiz.Kafka.Net.Tests/Stores/RocksDbKeyValueStoreTests.cs index 4d6bf325..e8095276 100644 --- a/test/Streamiz.Kafka.Net.Tests/Stores/RocksDbKeyValueStoreTests.cs +++ b/test/Streamiz.Kafka.Net.Tests/Stores/RocksDbKeyValueStoreTests.cs @@ -1,37 +1,37 @@ -using Confluent.Kafka; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Confluent.Kafka; using Moq; using NUnit.Framework; using Streamiz.Kafka.Net.Crosscutting; using Streamiz.Kafka.Net.Errors; +using Streamiz.Kafka.Net.Metrics; using Streamiz.Kafka.Net.Mock; using Streamiz.Kafka.Net.Processors; using Streamiz.Kafka.Net.Processors.Internal; using Streamiz.Kafka.Net.SerDes; -using Streamiz.Kafka.Net.State.RocksDb; +using Streamiz.Kafka.Net.State; using Streamiz.Kafka.Net.Tests.Helpers; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using Streamiz.Kafka.Net.Metrics; namespace Streamiz.Kafka.Net.Tests.Stores { public class RocksDbKeyValueStoreTests { - private StreamConfig config = null; - private RocksDbKeyValueStore store = null; - private ProcessorContext context = null; - private TaskId id = null; - private TopicPartition partition = null; - private ProcessorStateManager stateManager = null; - private Mock task = null; + private StreamConfig config; + private RocksDbKeyValueStore store; + private ProcessorContext context; + private TaskId id; + private TopicPartition partition; + private ProcessorStateManager stateManager; + private Mock task; [SetUp] public void Begin() { config = new StreamConfig(); - config.ApplicationId = $"unit-test-rocksdb-kv"; + config.ApplicationId = "unit-test-rocksdb-kv"; config.UseRandomRocksDbConfigForTest(); id = new TaskId { Id = 0, Partition = 0 }; diff --git a/test/Streamiz.Kafka.Net.Tests/Stores/RocksDbWindowBytesStoreSupplierTests.cs b/test/Streamiz.Kafka.Net.Tests/Stores/RocksDbWindowBytesStoreSupplierTests.cs index 548f42a8..ef88f1ed 100644 --- a/test/Streamiz.Kafka.Net.Tests/Stores/RocksDbWindowBytesStoreSupplierTests.cs +++ b/test/Streamiz.Kafka.Net.Tests/Stores/RocksDbWindowBytesStoreSupplierTests.cs @@ -1,4 +1,5 @@ -using NUnit.Framework; +using System; +using NUnit.Framework; using Streamiz.Kafka.Net.Crosscutting; using Streamiz.Kafka.Net.Mock; using Streamiz.Kafka.Net.SerDes; @@ -6,7 +7,6 @@ using Streamiz.Kafka.Net.Stream; using Streamiz.Kafka.Net.Table; using Streamiz.Kafka.Net.Tests.Helpers; -using System; namespace Streamiz.Kafka.Net.Tests.Stores { @@ -97,7 +97,7 @@ public void StoresPersistentKeyValueStoreTest() .WindowedBy(TumblingWindowOptions.Of(1000)) .Count( Materialized>.Create( - Streamiz.Kafka.Net.State.Stores.PersistentWindowStore( + State.Stores.PersistentWindowStore( "rocksdb-w-store", TimeSpan.FromDays(1), TimeSpan.FromSeconds(1)))); diff --git a/test/Streamiz.Kafka.Net.Tests/Stores/RocksDbWindowStoreDuplicateTests.cs b/test/Streamiz.Kafka.Net.Tests/Stores/RocksDbWindowStoreDuplicateTests.cs index 44895243..2b3ecfa5 100644 --- a/test/Streamiz.Kafka.Net.Tests/Stores/RocksDbWindowStoreDuplicateTests.cs +++ b/test/Streamiz.Kafka.Net.Tests/Stores/RocksDbWindowStoreDuplicateTests.cs @@ -1,22 +1,17 @@ +using System; +using System.Collections.Generic; +using System.Text; using Confluent.Kafka; using Moq; using NUnit.Framework; using Streamiz.Kafka.Net.Crosscutting; +using Streamiz.Kafka.Net.Metrics; using Streamiz.Kafka.Net.Mock; using Streamiz.Kafka.Net.Processors; using Streamiz.Kafka.Net.Processors.Internal; -using Streamiz.Kafka.Net.SerDes; using Streamiz.Kafka.Net.State; -using Streamiz.Kafka.Net.State.RocksDb; -using Streamiz.Kafka.Net.State.RocksDb.Internal; +using Streamiz.Kafka.Net.State.Internal; using Streamiz.Kafka.Net.Tests.Helpers; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading; -using Streamiz.Kafka.Net.Metrics; - namespace Streamiz.Kafka.Net.Tests.Stores { @@ -25,19 +20,19 @@ public class RocksDbWindowStoreDuplicateTests private static readonly TimeSpan defaultRetention = TimeSpan.FromMinutes(1); private static readonly TimeSpan defaultSize = TimeSpan.FromSeconds(10); - private StreamConfig config = null; - private RocksDbWindowStore store = null; - private ProcessorContext context = null; - private TaskId id = null; - private TopicPartition partition = null; - private ProcessorStateManager stateManager = null; - private Mock task = null; + private StreamConfig config; + private RocksDbWindowStore store; + private ProcessorContext context; + private TaskId id; + private TopicPartition partition; + private ProcessorStateManager stateManager; + private Mock task; [SetUp] public void Begin() { config = new StreamConfig(); - config.ApplicationId = $"unit-test-duplicate-rocksdb-window"; + config.ApplicationId = "unit-test-duplicate-rocksdb-window"; config.UseRandomRocksDbConfigForTest(); id = new TaskId { Id = 0, Partition = 0 }; @@ -55,7 +50,7 @@ public void Begin() context = new ProcessorContext(task.Object, config, stateManager, new StreamMetricsRegistry()); store = new RocksDbWindowStore( - new RocksDbSegmentedBytesStore("test-w-store", (long)defaultRetention.TotalMilliseconds, 5000, new RocksDbWindowKeySchema()), + new RocksDbSegmentedBytesStore("test-w-store", (long)defaultRetention.TotalMilliseconds, 5000, new WindowKeySchema()), (long)defaultSize.TotalMilliseconds, true); store.Init(context, store); diff --git a/test/Streamiz.Kafka.Net.Tests/Stores/RocksDbWindowStoreIssue185Tests.cs b/test/Streamiz.Kafka.Net.Tests/Stores/RocksDbWindowStoreIssue185Tests.cs index 71f6c0c8..74754af0 100644 --- a/test/Streamiz.Kafka.Net.Tests/Stores/RocksDbWindowStoreIssue185Tests.cs +++ b/test/Streamiz.Kafka.Net.Tests/Stores/RocksDbWindowStoreIssue185Tests.cs @@ -10,8 +10,7 @@ using Streamiz.Kafka.Net.Processors; using Streamiz.Kafka.Net.Processors.Internal; using Streamiz.Kafka.Net.State; -using Streamiz.Kafka.Net.State.RocksDb; -using Streamiz.Kafka.Net.State.RocksDb.Internal; +using Streamiz.Kafka.Net.State.Internal; using Streamiz.Kafka.Net.Tests.Helpers; namespace Streamiz.Kafka.Net.Tests.Stores @@ -21,19 +20,19 @@ public class RocksDbWindowStoreIssue185Tests private static readonly TimeSpan defaultRetention = TimeSpan.FromDays(1); private static readonly TimeSpan defaultSize = TimeSpan.FromHours(6); - private StreamConfig config = null; - private RocksDbWindowStore store = null; - private ProcessorContext context = null; - private TaskId id = null; - private TopicPartition partition = null; - private ProcessorStateManager stateManager = null; - private Mock task = null; + private StreamConfig config; + private RocksDbWindowStore store; + private ProcessorContext context; + private TaskId id; + private TopicPartition partition; + private ProcessorStateManager stateManager; + private Mock task; [SetUp] public void Begin() { config = new StreamConfig(); - config.ApplicationId = $"unit-test-rocksdb-w"; + config.ApplicationId = "unit-test-rocksdb-w"; config.UseRandomRocksDbConfigForTest(); id = new TaskId { Id = 0, Partition = 0 }; @@ -55,7 +54,7 @@ public void Begin() "test-w-store", (long)defaultRetention.TotalMilliseconds, 7200000, - new RocksDbWindowKeySchema()), + new WindowKeySchema()), (long)defaultSize.TotalMilliseconds, false); store.Init(context, store); diff --git a/test/Streamiz.Kafka.Net.Tests/Stores/RocksDbWindowStoreTests.cs b/test/Streamiz.Kafka.Net.Tests/Stores/RocksDbWindowStoreTests.cs index 93efc993..271d806f 100644 --- a/test/Streamiz.Kafka.Net.Tests/Stores/RocksDbWindowStoreTests.cs +++ b/test/Streamiz.Kafka.Net.Tests/Stores/RocksDbWindowStoreTests.cs @@ -1,21 +1,20 @@ -using Confluent.Kafka; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; +using Confluent.Kafka; using Moq; using NUnit.Framework; using Streamiz.Kafka.Net.Crosscutting; +using Streamiz.Kafka.Net.Metrics; using Streamiz.Kafka.Net.Mock; using Streamiz.Kafka.Net.Processors; using Streamiz.Kafka.Net.Processors.Internal; using Streamiz.Kafka.Net.SerDes; using Streamiz.Kafka.Net.State; -using Streamiz.Kafka.Net.State.RocksDb; -using Streamiz.Kafka.Net.State.RocksDb.Internal; +using Streamiz.Kafka.Net.State.Internal; using Streamiz.Kafka.Net.Tests.Helpers; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading; -using Streamiz.Kafka.Net.Metrics; namespace Streamiz.Kafka.Net.Tests.Stores { @@ -24,19 +23,19 @@ public class RocksDbWindowStoreTests private static readonly TimeSpan defaultRetention = TimeSpan.FromMinutes(1); private static readonly TimeSpan defaultSize = TimeSpan.FromSeconds(10); - private StreamConfig config = null; - private RocksDbWindowStore store = null; - private ProcessorContext context = null; - private TaskId id = null; - private TopicPartition partition = null; - private ProcessorStateManager stateManager = null; - private Mock task = null; + private StreamConfig config; + private RocksDbWindowStore store; + private ProcessorContext context; + private TaskId id; + private TopicPartition partition; + private ProcessorStateManager stateManager; + private Mock task; [SetUp] public void Begin() { config = new StreamConfig(); - config.ApplicationId = $"unit-test-rocksdb-w"; + config.ApplicationId = "unit-test-rocksdb-w"; config.UseRandomRocksDbConfigForTest(); id = new TaskId { Id = 0, Partition = 0 }; @@ -54,7 +53,7 @@ public void Begin() context = new ProcessorContext(task.Object, config, stateManager, new StreamMetricsRegistry()); store = new RocksDbWindowStore( - new RocksDbSegmentedBytesStore("test-w-store", (long)defaultRetention.TotalMilliseconds, 5000, new RocksDbWindowKeySchema()), + new RocksDbSegmentedBytesStore("test-w-store", (long)defaultRetention.TotalMilliseconds, 5000, new WindowKeySchema()), (long)defaultSize.TotalMilliseconds, false); store.Init(context, store);