diff --git a/src/Kentico.Xperience.Lucene/Enums/LuceneTaskType.cs b/src/Kentico.Xperience.Lucene/Enums/LuceneTaskType.cs index 9cac498..0091d30 100644 --- a/src/Kentico.Xperience.Lucene/Enums/LuceneTaskType.cs +++ b/src/Kentico.Xperience.Lucene/Enums/LuceneTaskType.cs @@ -25,5 +25,10 @@ public enum LuceneTaskType /// /// A task for a page which should be removed from the index. /// - DELETE + DELETE, + + /// + /// Task marks the end of indexed items, index is published after this task occurs + /// + PUBLISH_INDEX, } diff --git a/src/Kentico.Xperience.Lucene/LuceneQueueWorker.cs b/src/Kentico.Xperience.Lucene/LuceneQueueWorker.cs index a0c291f..d973d73 100644 --- a/src/Kentico.Xperience.Lucene/LuceneQueueWorker.cs +++ b/src/Kentico.Xperience.Lucene/LuceneQueueWorker.cs @@ -1,6 +1,6 @@ -using CMS.Base; +using CMS.Base; using CMS.Core; - +using CMS.DocumentEngine; using Kentico.Xperience.Lucene.Models; using Kentico.Xperience.Lucene.Services; @@ -52,6 +52,9 @@ public static void EnqueueLuceneQueueItem(LuceneQueueItem queueItem) Current.Enqueue(queueItem, false); } + public static void EnqueueIndexPublication(string indexName) + => EnqueueLuceneQueueItem(new LuceneQueueItem(null!, LuceneTaskType.PUBLISH_INDEX, indexName)); + /// protected override void Finish() => RunProcess(); diff --git a/src/Kentico.Xperience.Lucene/Models/LuceneIndex.cs b/src/Kentico.Xperience.Lucene/Models/LuceneIndex.cs index 8efa78d..975f650 100644 --- a/src/Kentico.Xperience.Lucene/Models/LuceneIndex.cs +++ b/src/Kentico.Xperience.Lucene/Models/LuceneIndex.cs @@ -1,4 +1,5 @@ using System.Reflection; +using System.Text.RegularExpressions; using Kentico.Xperience.Lucene.Attributes; using Kentico.Xperience.Lucene.Services; using Kentico.Xperience.Lucene.Services.Implementations; @@ -42,9 +43,9 @@ public string IndexName } /// - /// The filesystem path of the Lucene index. + /// Index storage context, employs picked storage strategy /// - public string IndexPath + public IndexStorageContext StorageContext { get; } @@ -78,9 +79,10 @@ internal IEnumerable IncludedPaths /// The code name of the Lucene index. /// The filesystem Lucene index. Defaults to /App_Data/LuceneSearch/[IndexName] /// Defaults to + /// Storage strategy defines how index will be stored from directory naming perspective /// /// - public LuceneIndex(Type type, Analyzer analyzer, string indexName, string? indexPath = null, ILuceneIndexingStrategy? luceneIndexingStrategy = null) + public LuceneIndex(Type type, Analyzer analyzer, string indexName, string? indexPath = null, ILuceneIndexingStrategy? luceneIndexingStrategy = null, IIndexStorageStrategy? storageStrategy = null) { if (string.IsNullOrEmpty(indexName)) { @@ -100,7 +102,9 @@ public LuceneIndex(Type type, Analyzer analyzer, string indexName, string? index Analyzer = analyzer ?? throw new ArgumentNullException(nameof(analyzer)); LuceneSearchModelType = type; IndexName = indexName; - IndexPath = indexPath ?? Path.Combine(Environment.CurrentDirectory, "App_Data", "LuceneSearch", indexName); + string indexStoragePath = indexPath ?? Path.Combine(Environment.CurrentDirectory, "App_Data", "LuceneSearch", indexName); + + StorageContext = new IndexStorageContext(storageStrategy ?? new GenerationStorageStrategy(), indexStoragePath); LuceneIndexingStrategy = luceneIndexingStrategy ?? new DefaultLuceneIndexingStrategy(); var paths = type.GetCustomAttributes(false); diff --git a/src/Kentico.Xperience.Lucene/Services/IIndexStorageStrategy.cs b/src/Kentico.Xperience.Lucene/Services/IIndexStorageStrategy.cs new file mode 100644 index 0000000..ac9a057 --- /dev/null +++ b/src/Kentico.Xperience.Lucene/Services/IIndexStorageStrategy.cs @@ -0,0 +1,163 @@ +using System.Diagnostics.CodeAnalysis; +using System.Text.RegularExpressions; + +namespace Kentico.Xperience.Lucene.Services; + +public class IndexStorageContext +{ + private readonly IIndexStorageStrategy storageStrategy; + private readonly string indexStoragePathRoot; + + public IndexStorageContext(IIndexStorageStrategy selectedStorageStrategy, string indexStoragePathRoot) + { + storageStrategy = selectedStorageStrategy; + this.indexStoragePathRoot = indexStoragePathRoot; + } + + public IndexStorageModel GetPublishedIndex() => + storageStrategy + .GetExistingIndexes(indexStoragePathRoot) + .Where(x => x.IsPublished) + .MaxBy(x => x.Generation) ?? new IndexStorageModel(storageStrategy.FormatPath(indexStoragePathRoot, 1, true), 1, true); + + /// + /// Gets next generation of index + /// + public IndexStorageModel GetNextGeneration() + { + var lastIndex = storageStrategy + .GetExistingIndexes(indexStoragePathRoot) + .MaxBy(x => x.Generation); + + var newIndex = lastIndex switch + { + var (path, generation, published) => new IndexStorageModel(path, published ? generation + 1 : generation, false), + _ => new IndexStorageModel("", 1, false) + }; + + return newIndex with { Path = storageStrategy.FormatPath(indexStoragePathRoot, newIndex.Generation, newIndex.IsPublished) }; + } + + public IndexStorageModel GetLastGeneration(bool defaultPublished) => + storageStrategy + .GetExistingIndexes(indexStoragePathRoot) + .MaxBy(x => x.Generation) + ?? new IndexStorageModel(storageStrategy.FormatPath(indexStoragePathRoot, 1, defaultPublished), 1, defaultPublished); + + /// + /// method returns last writable index storage model + /// + /// Storage model with information about writable index + /// thrown when unexpected model occurs + public IndexStorageModel GetNextOrOpenNextGeneration() + { + var lastIndex = storageStrategy + .GetExistingIndexes(indexStoragePathRoot) + .MaxBy(x => x.Generation); + + return lastIndex switch + { + { IsPublished: false } => lastIndex, + (_, var generation, true) => new IndexStorageModel(storageStrategy.FormatPath(indexStoragePathRoot, generation + 1, false), generation + 1, false), + null => + // no existing index, lets create new one + new IndexStorageModel(storageStrategy.FormatPath(indexStoragePathRoot, 1, false), 1, false), + _ => throw new ArgumentException($"Non-null last index storage with invalid settings '{lastIndex}'") + }; + } + + public void PublishIndex(IndexStorageModel storage) => storageStrategy.PublishIndex(storage); +} + +public record IndexStorageModel(string Path, int Generation, bool IsPublished); + +public interface IIndexStorageStrategy +{ + IEnumerable GetExistingIndexes(string indexStoragePath); + string FormatPath(string indexRoot, int generation, bool isPublished); + void PublishIndex(IndexStorageModel storage); +} + +public class GenerationStorageStrategy : IIndexStorageStrategy +{ + public IEnumerable GetExistingIndexes(string indexStoragePath) + { + if (Directory.Exists(indexStoragePath)) + { + foreach (string directory in Directory.GetDirectories(indexStoragePath)) + { + if (ParseIndexStorageModel(directory) is (true, var result)) + { + yield return result!; + } + } + } + } + + public string FormatPath(string indexRoot, int generation, bool isPublished) => Path.Combine(indexRoot, $"i-g{generation:0000000}-p_{isPublished}"); + + public void PublishIndex(IndexStorageModel storage) + { + string root = Path.Combine(storage.Path, ".."); + var published = storage with { IsPublished = true, Path = FormatPath(root, storage.Generation, true) }; + Directory.Move(storage.Path, published.Path); + } + + private record IndexStorageModelParsingResult( + bool Success, + [property: MemberNotNullWhen(true, "Success")] IndexStorageModel? Result + ); + + private IndexStorageModelParsingResult ParseIndexStorageModel(string directoryPath) + { + if (string.IsNullOrWhiteSpace(directoryPath)) + { + return new IndexStorageModelParsingResult(false, null); + } + + try + { + var dirInfo = new DirectoryInfo(directoryPath); + if (dirInfo.Name is { Length: > 0 } directoryName) + { + var matchResult = Regex.Match(directoryName, "i-g(?[0-9]*)-p_(?(true)|(false))", RegexOptions.IgnoreCase | RegexOptions.Singleline); + switch (matchResult) + { + case { Success: true } r + when r.Groups["generation"] is { Success: true, Value: { Length: > 0 } gen } && + r.Groups["published"] is { Success: true, Value: { Length: > 0 } pub }: + { + if (int.TryParse(gen, out int generation) && bool.TryParse(pub, out bool published)) + { + return new IndexStorageModelParsingResult(true, new IndexStorageModel(directoryPath, generation, published)); + } + + break; + } + default: + { + return new IndexStorageModelParsingResult(false, null); + } + } + } + } + catch + { + // low priority, if path cannot be parsed, it is possibly not generated index + // ignored + } + + return new IndexStorageModelParsingResult(false, null); + } +} + +public class SimpleStorageStrategy : IIndexStorageStrategy +{ + public IEnumerable GetExistingIndexes(string indexStoragePath) => new[] { new IndexStorageModel(indexStoragePath, 0, true) }; + public string FormatPath(string indexRoot, int generation, bool isPublished) => indexRoot; + + public void PublishIndex(IndexStorageModel storage) + { + // Method intentionally left empty. In this strategy, publication of index is not needed + } +} diff --git a/src/Kentico.Xperience.Lucene/Services/ILuceneIndexService.cs b/src/Kentico.Xperience.Lucene/Services/ILuceneIndexService.cs index eefd9a8..d01fc5a 100644 --- a/src/Kentico.Xperience.Lucene/Services/ILuceneIndexService.cs +++ b/src/Kentico.Xperience.Lucene/Services/ILuceneIndexService.cs @@ -6,7 +6,7 @@ namespace Kentico.Xperience.Lucene.Services; public interface ILuceneIndexService { - T UseWriter(LuceneIndex index, Func useIndexWriter, OpenMode openMode = OpenMode.CREATE_OR_APPEND); + T UseWriter(LuceneIndex index, Func useIndexWriter, IndexStorageModel storage, OpenMode openMode = OpenMode.CREATE_OR_APPEND); void ResetIndex(LuceneIndex index); diff --git a/src/Kentico.Xperience.Lucene/Services/Implementations/DefaultLuceneClient.cs b/src/Kentico.Xperience.Lucene/Services/Implementations/DefaultLuceneClient.cs index 8288cd4..2e3b75d 100644 --- a/src/Kentico.Xperience.Lucene/Services/Implementations/DefaultLuceneClient.cs +++ b/src/Kentico.Xperience.Lucene/Services/Implementations/DefaultLuceneClient.cs @@ -1,3 +1,4 @@ +using System.Collections.Concurrent; using CMS.Core; using CMS.DocumentEngine; using CMS.Helpers.Caching.Abstractions; @@ -119,8 +120,7 @@ private async Task DeleteRecordsInternal(IEnumerable objectIds, str // todo use batches writer.DeleteDocuments(booleanQuery); return "OK"; - }); - + }, index.StorageContext.GetLastGeneration(true)); } return 0; } @@ -130,6 +130,7 @@ private async Task RebuildInternal(LuceneIndex luceneIndex, CancellationToken? c { // Clear statistics cache so listing displays updated data after rebuild cacheAccessor.Remove(CACHEKEY_STATISTICS); + luceneIndexService.ResetIndex(luceneIndex); var indexedNodes = new List(); @@ -152,9 +153,9 @@ private async Task RebuildInternal(LuceneIndex luceneIndex, CancellationToken? c indexedNodes.AddRange(nodes); } - - + indexedNodes.ForEach(node => LuceneQueueWorker.EnqueueLuceneQueueItem(new LuceneQueueItem(node, LuceneTaskType.CREATE, luceneIndex.IndexName))); + LuceneQueueWorker.EnqueueIndexPublication(luceneIndex.IndexName); } private async Task UpsertRecordsInternal(IEnumerable dataObjects, string indexName) @@ -177,7 +178,7 @@ private async Task UpsertRecordsInternal(IEnumerable dat count++; } return count; - }); + }, index.StorageContext.GetLastGeneration(true)); } return 0; } diff --git a/src/Kentico.Xperience.Lucene/Services/Implementations/DefaultLuceneIndexService.cs b/src/Kentico.Xperience.Lucene/Services/Implementations/DefaultLuceneIndexService.cs index 150e5a3..7539f03 100644 --- a/src/Kentico.Xperience.Lucene/Services/Implementations/DefaultLuceneIndexService.cs +++ b/src/Kentico.Xperience.Lucene/Services/Implementations/DefaultLuceneIndexService.cs @@ -10,34 +10,37 @@ namespace Kentico.Xperience.Lucene.Services.Implementations; public class DefaultLuceneIndexService : ILuceneIndexService { private const LuceneVersion LUCENE_VERSION = LuceneVersion.LUCENE_48; - public TResult UseWriter(LuceneIndex index, Func useIndexWriter, OpenMode openMode = OpenMode.CREATE_OR_APPEND) - { - using LuceneDirectory indexDir = FSDirectory.Open(index.IndexPath); + public TResult UseWriter(LuceneIndex index, Func useIndexWriter, IndexStorageModel storage, OpenMode openMode = OpenMode.CREATE_OR_APPEND) + { + using LuceneDirectory indexDir = FSDirectory.Open(storage.Path); + //Create an index writer var indexConfig = new IndexWriterConfig(LUCENE_VERSION, index.Analyzer) { - OpenMode = openMode // create/overwrite index + OpenMode = openMode // create/overwrite index }; using var writer = new IndexWriter(indexDir, indexConfig); - + return useIndexWriter(writer); } - public void ResetIndex(LuceneIndex index) => UseWriter(index, (IndexWriter writer) => true, OpenMode.CREATE); + public void ResetIndex(LuceneIndex index) => UseWriter(index, (IndexWriter writer) => true, index.StorageContext.GetNextGeneration(), OpenMode.CREATE); public TResult UseSearcher(LuceneIndex index, Func useIndexSearcher) { - if (!System.IO.Directory.Exists(index.IndexPath)) + var storage = index.StorageContext.GetPublishedIndex(); + if (!System.IO.Directory.Exists(storage.Path)) { // ensure index UseWriter(index, (writer) => { writer.Commit(); return true; - }); + }, storage); } - using LuceneDirectory indexDir = FSDirectory.Open(index.IndexPath); + + using LuceneDirectory indexDir = FSDirectory.Open(storage.Path); using var reader = DirectoryReader.Open(indexDir); var searcher = new IndexSearcher(reader); return useIndexSearcher(searcher); diff --git a/src/Kentico.Xperience.Lucene/Services/Implementations/DefaultLuceneTaskProcessor.cs b/src/Kentico.Xperience.Lucene/Services/Implementations/DefaultLuceneTaskProcessor.cs index b11ff0e..259e8cf 100644 --- a/src/Kentico.Xperience.Lucene/Services/Implementations/DefaultLuceneTaskProcessor.cs +++ b/src/Kentico.Xperience.Lucene/Services/Implementations/DefaultLuceneTaskProcessor.cs @@ -1,4 +1,4 @@ -using CMS.Core; +using CMS.Core; using CMS.DocumentEngine; using CMS.WorkflowEngine; @@ -52,8 +52,21 @@ public async Task ProcessLuceneTasks(IEnumerable queueItem upsertData.Add(data); } - successfulOperations += await luceneClient.DeleteRecords(deleteIds, group.Key); - successfulOperations += await luceneClient.UpsertRecords(upsertData, group.Key, cancellationToken); + if (IndexStore.Instance.GetIndex(group.Key) is { } index) + { + successfulOperations += await luceneClient.DeleteRecords(deleteIds, group.Key); + successfulOperations += await luceneClient.UpsertRecords(upsertData, group.Key, cancellationToken); + + if (group.Any(t => t.TaskType == LuceneTaskType.PUBLISH_INDEX)) + { + var storage = index.StorageContext.GetNextOrOpenNextGeneration(); + index.StorageContext.PublishIndex(storage); + } + } + else + { + eventLogService.LogError(nameof(DefaultLuceneTaskProcessor), nameof(ProcessLuceneTasks), "Index instance not exists"); + } } catch (Exception ex) {