Lucene.NET storage strategy - generation

src/Kentico.Xperience.Lucene/Enums/LuceneTaskType.cs

@@ -25,5 +25,10 @@ public enum LuceneTaskType
     /// <summary>
     /// A task for a page which should be removed from the index.
     /// </summary>
-    DELETE
+    DELETE,
+
+    /// <summary>
+    /// Task marks the end of indexed items, index is published after this task occurs
+    /// </summary>
+    PUBLISH_INDEX,
 }

src/Kentico.Xperience.Lucene/LuceneQueueWorker.cs

@@ -1,6 +1,6 @@
-using CMS.Base;
+using CMS.Base;
 using CMS.Core;
-
+using CMS.DocumentEngine;
 using Kentico.Xperience.Lucene.Models;
 using Kentico.Xperience.Lucene.Services;
 
@@ -52,6 +52,9 @@ public static void EnqueueLuceneQueueItem(LuceneQueueItem queueItem)
         Current.Enqueue(queueItem, false);
     }
 
+    public static void EnqueueIndexPublication(string indexName) 
+        => EnqueueLuceneQueueItem(new LuceneQueueItem(null!, LuceneTaskType.PUBLISH_INDEX, indexName));
+
 
     /// <inheritdoc />
     protected override void Finish() => RunProcess();

src/Kentico.Xperience.Lucene/Models/LuceneIndex.cs

@@ -1,4 +1,5 @@
 using System.Reflection;
+using System.Text.RegularExpressions;
 using Kentico.Xperience.Lucene.Attributes;
 using Kentico.Xperience.Lucene.Services;
 using Kentico.Xperience.Lucene.Services.Implementations;
@@ -42,9 +43,9 @@ public string IndexName
     }
 
     /// <summary>
-    /// The filesystem path of the Lucene index.
+    /// Index storage context, employs picked storage strategy
     /// </summary>
-    public string IndexPath
+    public IndexStorageContext StorageContext
     {
         get;
     }
@@ -78,9 +79,10 @@ internal IEnumerable<IncludedPathAttribute> IncludedPaths
     /// <param name="indexName">The code name of the Lucene index.</param>
     /// <param name="indexPath">The filesystem Lucene index. Defaults to /App_Data/LuceneSearch/[IndexName]</param>
     /// <param name="luceneIndexingStrategy">Defaults to  <see cref="DefaultLuceneIndexingStrategy"/></param>
+    /// <param name="storageStrategy">Storage strategy defines how index will be stored from directory naming perspective</param>
     /// <exception cref="ArgumentNullException" />
     /// <exception cref="InvalidOperationException" />
-    public LuceneIndex(Type type, Analyzer analyzer, string indexName, string? indexPath = null, ILuceneIndexingStrategy? luceneIndexingStrategy = null)
+    public LuceneIndex(Type type, Analyzer analyzer, string indexName, string? indexPath = null, ILuceneIndexingStrategy? luceneIndexingStrategy = null, IIndexStorageStrategy? storageStrategy = null)
     {
         if (string.IsNullOrEmpty(indexName))
         {
@@ -100,7 +102,9 @@ public LuceneIndex(Type type, Analyzer analyzer, string indexName, string? index
         Analyzer = analyzer ?? throw new ArgumentNullException(nameof(analyzer));
         LuceneSearchModelType = type;
         IndexName = indexName;
-        IndexPath = indexPath ?? Path.Combine(Environment.CurrentDirectory, "App_Data", "LuceneSearch", indexName);
+        string indexStoragePath = indexPath ?? Path.Combine(Environment.CurrentDirectory, "App_Data", "LuceneSearch", indexName);
+
+        StorageContext = new IndexStorageContext(storageStrategy ?? new GenerationStorageStrategy(), indexStoragePath);
         LuceneIndexingStrategy = luceneIndexingStrategy ?? new DefaultLuceneIndexingStrategy();
 
         var paths = type.GetCustomAttributes<IncludedPathAttribute>(false);

src/Kentico.Xperience.Lucene/Services/IIndexStorageStrategy.cs

@@ -0,0 +1,163 @@
+using System.Diagnostics.CodeAnalysis;
+using System.Text.RegularExpressions;
+
+namespace Kentico.Xperience.Lucene.Services;
+
+public class IndexStorageContext
+{
+    private readonly IIndexStorageStrategy storageStrategy;
+    private readonly string indexStoragePathRoot;
+
+    public IndexStorageContext(IIndexStorageStrategy selectedStorageStrategy, string indexStoragePathRoot)
+    {
+        storageStrategy = selectedStorageStrategy;
+        this.indexStoragePathRoot = indexStoragePathRoot;
+    }
+
+    public IndexStorageModel GetPublishedIndex() =>
+        storageStrategy
+            .GetExistingIndexes(indexStoragePathRoot)
+            .Where(x => x.IsPublished)
+            .MaxBy(x => x.Generation) ?? new IndexStorageModel(storageStrategy.FormatPath(indexStoragePathRoot, 1, true), 1, true);
+
+    /// <summary>
+    /// Gets next generation of index
+    /// </summary>
+    public IndexStorageModel GetNextGeneration()
+    {
+        var lastIndex = storageStrategy
+            .GetExistingIndexes(indexStoragePathRoot)
+            .MaxBy(x => x.Generation);
+
+        var newIndex = lastIndex switch
+        {
+            var (path, generation, published) => new IndexStorageModel(path, published ? generation + 1 : generation, false),
+            _ => new IndexStorageModel("", 1, false)
+        };
+
+        return newIndex with { Path = storageStrategy.FormatPath(indexStoragePathRoot, newIndex.Generation, newIndex.IsPublished) };
+    }
+
+    public IndexStorageModel GetLastGeneration(bool defaultPublished) =>
+        storageStrategy
+            .GetExistingIndexes(indexStoragePathRoot)
+            .MaxBy(x => x.Generation)
+        ?? new IndexStorageModel(storageStrategy.FormatPath(indexStoragePathRoot, 1, defaultPublished), 1, defaultPublished);
+
+    /// <summary>
+    /// method returns last writable index storage model
+    /// </summary>
+    /// <returns>Storage model with information about writable index</returns>
+    /// <exception cref="ArgumentException">thrown when unexpected model occurs</exception>
+    public IndexStorageModel GetNextOrOpenNextGeneration()
+    {
+        var lastIndex = storageStrategy
+            .GetExistingIndexes(indexStoragePathRoot)
+            .MaxBy(x => x.Generation);
+
+        return lastIndex switch
+        {
+            { IsPublished: false } => lastIndex,
+            (_, var generation, true) => new IndexStorageModel(storageStrategy.FormatPath(indexStoragePathRoot, generation + 1, false), generation + 1, false),
+            null =>
+                // no existing index, lets create new one
+                new IndexStorageModel(storageStrategy.FormatPath(indexStoragePathRoot, 1, false), 1, false),
+            _ => throw new ArgumentException($"Non-null last index storage with invalid settings '{lastIndex}'")
+        };
+    }
+
+    public void PublishIndex(IndexStorageModel storage) => storageStrategy.PublishIndex(storage);
+}
+
+public record IndexStorageModel(string Path, int Generation, bool IsPublished);
+
+public interface IIndexStorageStrategy
+{
+    IEnumerable<IndexStorageModel> GetExistingIndexes(string indexStoragePath);
+    string FormatPath(string indexRoot, int generation, bool isPublished);
+    void PublishIndex(IndexStorageModel storage);
+}
+
+public class GenerationStorageStrategy : IIndexStorageStrategy
+{
+    public IEnumerable<IndexStorageModel> GetExistingIndexes(string indexStoragePath)
+    {
+        if (Directory.Exists(indexStoragePath))
+        {
+            foreach (string directory in Directory.GetDirectories(indexStoragePath))
+            {
+                if (ParseIndexStorageModel(directory) is (true, var result))
+                {
+                    yield return result!;
+                }
+            }
+        }
+    }
+
+    public string FormatPath(string indexRoot, int generation, bool isPublished) => Path.Combine(indexRoot, $"i-g{generation:0000000}-p_{isPublished}");
+
+    public void PublishIndex(IndexStorageModel storage)
+    {
+        string root = Path.Combine(storage.Path, "..");
+        var published = storage with { IsPublished = true, Path = FormatPath(root, storage.Generation, true) };
+        Directory.Move(storage.Path, published.Path);
+    }
+
+    private record IndexStorageModelParsingResult(
+        bool Success,
+        [property: MemberNotNullWhen(true, "Success")] IndexStorageModel? Result
+    );
+
+    private IndexStorageModelParsingResult ParseIndexStorageModel(string directoryPath)
+    {
+        if (string.IsNullOrWhiteSpace(directoryPath))
+        {
+            return new IndexStorageModelParsingResult(false, null);
+        }
+
+        try
+        {
+            var dirInfo = new DirectoryInfo(directoryPath);
+            if (dirInfo.Name is { Length: > 0 } directoryName)
+            {
+                var matchResult = Regex.Match(directoryName, "i-g(?<generation>[0-9]*)-p_(?<published>(true)|(false))", RegexOptions.IgnoreCase | RegexOptions.Singleline);
+                switch (matchResult)
+                {
+                    case { Success: true } r
+                        when r.Groups["generation"] is { Success: true, Value: { Length: > 0 } gen } &&
+                             r.Groups["published"] is { Success: true, Value: { Length: > 0 } pub }:
+                    {
+                        if (int.TryParse(gen, out int generation) && bool.TryParse(pub, out bool published))
+                        {
+                            return new IndexStorageModelParsingResult(true, new IndexStorageModel(directoryPath, generation, published));
+                        }
+
+                        break;
+                    }
+                    default:
+                    {
+                        return new IndexStorageModelParsingResult(false, null);
+                    }
+                }
+            }
+        }
+        catch
+        {
+            // low priority, if path cannot be parsed, it is possibly not generated index
+            // ignored
+        }
+
+        return new IndexStorageModelParsingResult(false, null);
+    }
+}
+
+public class SimpleStorageStrategy : IIndexStorageStrategy
+{
+    public IEnumerable<IndexStorageModel> GetExistingIndexes(string indexStoragePath) => new[] { new IndexStorageModel(indexStoragePath, 0, true) };
+    public string FormatPath(string indexRoot, int generation, bool isPublished) => indexRoot;
+
+    public void PublishIndex(IndexStorageModel storage)
+    {
+        // Method intentionally left empty. In this strategy, publication of index is not needed
+    }
+}

src/Kentico.Xperience.Lucene/Services/ILuceneIndexService.cs

@@ -6,7 +6,7 @@ namespace Kentico.Xperience.Lucene.Services;
 
 public interface ILuceneIndexService
 {
-    T UseWriter<T>(LuceneIndex index, Func<IndexWriter, T> useIndexWriter, OpenMode openMode = OpenMode.CREATE_OR_APPEND);
+    T UseWriter<T>(LuceneIndex index, Func<IndexWriter, T> useIndexWriter, IndexStorageModel storage, OpenMode openMode = OpenMode.CREATE_OR_APPEND);
 
     void ResetIndex(LuceneIndex index);
 

src/Kentico.Xperience.Lucene/Services/Implementations/DefaultLuceneClient.cs

@@ -1,3 +1,4 @@
+using System.Collections.Concurrent;
 using CMS.Core;
 using CMS.DocumentEngine;
 using CMS.Helpers.Caching.Abstractions;
@@ -20,7 +21,7 @@
     private readonly ILuceneSearchModelToDocumentMapper luceneSearchModelToDocumentMapper;
 
     private readonly ICacheAccessor cacheAccessor;
     private readonly IEventLogService eventLogService;
     private readonly IPageRetriever pageRetriever;
 
     internal const string CACHEKEY_STATISTICS = "Lucene|ListIndices";
@@ -60,7 +61,7 @@
 
 
     /// <inheritdoc/>
     public async Task<ICollection<LuceneIndexStatisticsViewModel>> GetStatistics(CancellationToken cancellationToken) =>
         IndexStore.Instance.GetAllIndexes().Select(i =>
         {
             var statistics = luceneIndexService.UseSearcher(i, s => new LuceneIndexStatisticsViewModel()
@@ -103,7 +104,7 @@
         return UpsertRecordsInternal(dataObjects, indexName);
     }
 
     private async Task<int> DeleteRecordsInternal(IEnumerable<string> objectIds, string indexName)
     {
         var index = IndexStore.Instance.GetIndex(indexName);
         if (index != null)
@@ -116,11 +117,10 @@
                     var termQuery = new TermQuery(new Term(nameof(LuceneSearchModel.ObjectID), objectId));
                     booleanQuery.Add(termQuery, Occur.SHOULD); // Match any of the object IDs
                 }
                 // todo use batches
                 writer.DeleteDocuments(booleanQuery);
                 return "OK";
-            });
-
+            }, index.StorageContext.GetLastGeneration(true));
         }
         return 0;
     }
@@ -130,6 +130,7 @@
     {
         // Clear statistics cache so listing displays updated data after rebuild
         cacheAccessor.Remove(CACHEKEY_STATISTICS);
+
         luceneIndexService.ResetIndex(luceneIndex);
 
         var indexedNodes = new List<TreeNode>();
@@ -152,12 +153,12 @@
 
             indexedNodes.AddRange(nodes);
         }
-
-
+
         indexedNodes.ForEach(node => LuceneQueueWorker.EnqueueLuceneQueueItem(new LuceneQueueItem(node, LuceneTaskType.CREATE, luceneIndex.IndexName)));
+        LuceneQueueWorker.EnqueueIndexPublication(luceneIndex.IndexName);
     }
 
     private async Task<int> UpsertRecordsInternal(IEnumerable<LuceneSearchModel> dataObjects, string indexName)
     {
         var index = IndexStore.Instance.GetIndex(indexName);
         if (index != null)
@@ -177,7 +178,7 @@
                     count++;
                 }
                 return count;
-            });
+            }, index.StorageContext.GetLastGeneration(true));
         }
         return 0;
     }

src/Kentico.Xperience.Lucene/Services/Implementations/DefaultLuceneIndexService.cs

@@ -10,34 +10,37 @@ namespace Kentico.Xperience.Lucene.Services.Implementations;
 public class DefaultLuceneIndexService : ILuceneIndexService
 {
     private const LuceneVersion LUCENE_VERSION = LuceneVersion.LUCENE_48;
-    public TResult UseWriter<TResult>(LuceneIndex index, Func<IndexWriter, TResult> useIndexWriter, OpenMode openMode = OpenMode.CREATE_OR_APPEND)
-    {
-        using LuceneDirectory indexDir = FSDirectory.Open(index.IndexPath);
 
+    public TResult UseWriter<TResult>(LuceneIndex index, Func<IndexWriter, TResult> useIndexWriter, IndexStorageModel storage, OpenMode openMode = OpenMode.CREATE_OR_APPEND)
+    {
+        using LuceneDirectory indexDir = FSDirectory.Open(storage.Path);
+
         //Create an index writer
         var indexConfig = new IndexWriterConfig(LUCENE_VERSION, index.Analyzer)
         {
-            OpenMode = openMode                             // create/overwrite index
+            OpenMode = openMode // create/overwrite index
         };
         using var writer = new IndexWriter(indexDir, indexConfig);
-
+        
         return useIndexWriter(writer);
     }
 
-    public void ResetIndex(LuceneIndex index) => UseWriter(index, (IndexWriter writer) => true, OpenMode.CREATE);
+    public void ResetIndex(LuceneIndex index) => UseWriter(index, (IndexWriter writer) => true, index.StorageContext.GetNextGeneration(), OpenMode.CREATE);
 
     public TResult UseSearcher<TResult>(LuceneIndex index, Func<IndexSearcher, TResult> useIndexSearcher)
     {
-        if (!System.IO.Directory.Exists(index.IndexPath))
+        var storage = index.StorageContext.GetPublishedIndex();
+        if (!System.IO.Directory.Exists(storage.Path))
         {
             // ensure index
             UseWriter(index, (writer) =>
             {
                 writer.Commit();
                 return true;
-            });
+            }, storage);
         }
-        using LuceneDirectory indexDir = FSDirectory.Open(index.IndexPath);
+
+        using LuceneDirectory indexDir = FSDirectory.Open(storage.Path);
         using var reader = DirectoryReader.Open(indexDir);
         var searcher = new IndexSearcher(reader);
         return useIndexSearcher(searcher);

src/Kentico.Xperience.Lucene/Services/Implementations/DefaultLuceneTaskProcessor.cs

@@ -1,4 +1,4 @@
-using CMS.Core;
+using CMS.Core;
 using CMS.DocumentEngine;
 using CMS.WorkflowEngine;
 
@@ -11,7 +11,7 @@
     private readonly ILuceneClient luceneClient;
     private readonly ILuceneModelGenerator luceneObjectGenerator;
     private readonly IEventLogService eventLogService;
     private readonly IWorkflowStepInfoProvider workflowStepInfoProvider;
     private readonly IVersionHistoryInfoProvider versionHistoryInfoProvider;
 
 
@@ -52,8 +52,21 @@
                     upsertData.Add(data);
                 }
 
-                successfulOperations += await luceneClient.DeleteRecords(deleteIds, group.Key);
-                successfulOperations += await luceneClient.UpsertRecords(upsertData, group.Key, cancellationToken);
+                if (IndexStore.Instance.GetIndex(group.Key) is { } index)
+                {
+                    successfulOperations += await luceneClient.DeleteRecords(deleteIds, group.Key);
+                    successfulOperations += await luceneClient.UpsertRecords(upsertData, group.Key, cancellationToken);
+
+                    if (group.Any(t => t.TaskType == LuceneTaskType.PUBLISH_INDEX))
+                    {
+                        var storage = index.StorageContext.GetNextOrOpenNextGeneration();
+                        index.StorageContext.PublishIndex(storage);
+                    }
+                }
+                else
+                {
+                    eventLogService.LogError(nameof(DefaultLuceneTaskProcessor), nameof(ProcessLuceneTasks), "Index instance not exists");    
+                }
             }
             catch (Exception ex)
             {