Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Optimize IndexingContentHandlers perf #13721

Closed
wants to merge 37 commits into from
Closed
Show file tree
Hide file tree
Changes from 34 commits
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
6301402
avoid retrieving content from the database multiple times
hyzx86 May 20, 2023
a31de04
Fixed multiple versions of Lucene Index storage issues
hyzx86 May 20, 2023
78032d1
Merge branch 'main' into lucenePublish
hyzx86 Jan 29, 2024
26eb712
Merge branch 'main' into pr/13721
Skrypt Mar 22, 2024
383704c
Format and typo
Skrypt Mar 22, 2024
d2fbdcb
Optimize code
Skrypt Mar 22, 2024
d3944c0
Use Context assigned contentItem
Skrypt Mar 22, 2024
88ac104
Fix when saving draft
Skrypt Mar 22, 2024
c66e3fa
Move StoreLuceneDocument
Skrypt Mar 22, 2024
1628549
Optimize them all
Skrypt Mar 23, 2024
7a5b3e2
Merge branch 'main' into lucenePublish
hyzx86 Mar 23, 2024
bf927de
Merge branch 'main' into lucenePublish
hyzx86 Mar 27, 2024
e32983f
React to Sebastien comments
Skrypt Mar 28, 2024
e46cb6c
Merge branch 'main' into lucenePublish
hyzx86 Mar 29, 2024
9708f08
add test
Mar 29, 2024
cf2a4d0
Merge branch 'lucenePublish' of https://github.com/hyzx86/OrchardCore…
Mar 29, 2024
4f4e10e
rename method
Mar 29, 2024
66c2068
update
Mar 29, 2024
0c5bfa2
update while
Mar 29, 2024
20c966e
fix unit test
Mar 29, 2024
ec3d6c7
update query template
Mar 29, 2024
dfcd821
Merge branch 'main' into lucenePublish
hyzx86 Mar 29, 2024
26acf51
Merge branch 'main' into lucenePublish
hyzx86 Apr 1, 2024
3d4d968
Merge branch 'main' into lucenePublish
hyzx86 Apr 5, 2024
5fc7c5b
Merge branch 'main' into lucenePublish
hyzx86 Apr 6, 2024
876d976
Merge branch 'main' into lucenePublish
hyzx86 Apr 23, 2024
98e694e
Merge branch 'main' into lucenePublish
hyzx86 Apr 30, 2024
5fe344e
Merge branch 'main' into lucenePublish
hyzx86 May 2, 2024
f2bfa57
FlushAsync error
hyzx86 May 2, 2024
36c7674
Merge branch 'lucenePublish' of https://github.com/hyzx86/OrchardCore…
hyzx86 May 2, 2024
eb7ab7c
fix Flush
hyzx86 May 2, 2024
759c477
restore FlushAsync method
hyzx86 May 2, 2024
11c76ad
update unit test
hyzx86 May 2, 2024
c0075be
add ContentManager Test
hyzx86 May 2, 2024
1af75f6
Merge branch 'main' into lucenePublish
hyzx86 May 2, 2024
28e9928
update RunTask
hyzx86 May 2, 2024
a894ed5
Merge branch 'lucenePublish' of https://github.com/hyzx86/OrchardCore…
hyzx86 May 2, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,8 @@ private static async Task IndexingAsync(ShellScope scope, IEnumerable<ContentCon
var services = scope.ServiceProvider;
var contentManager = services.GetRequiredService<IContentManager>();
var contentItemIndexHandlers = services.GetServices<IContentItemIndexHandler>();
var luceneIndexManager = services.GetRequiredService<LuceneIndexManager>();
var luceneIndexSettingsService = services.GetRequiredService<LuceneIndexSettingsService>();
var indexManager = services.GetRequiredService<LuceneIndexManager>();
var indexSettingsService = services.GetRequiredService<LuceneIndexSettingsService>();
var logger = services.GetRequiredService<ILogger<LuceneIndexingContentHandler>>();

// Multiple items may have been updated in the same scope, e.g through a recipe.
Expand All @@ -77,46 +77,83 @@ private static async Task IndexingAsync(ShellScope scope, IEnumerable<ContentCon
// Only process the last context.
var context = ContextsById.Last();

ContentItem published = null, latest = null;
bool publishedLoaded = false, latestLoaded = false;

foreach (var indexSettings in await luceneIndexSettingsService.GetSettingsAsync())
foreach (var indexSettings in await indexSettingsService.GetSettingsAsync())
{
var cultureAspect = await contentManager.PopulateAspectAsync<CultureAspect>(context.ContentItem);
var culture = cultureAspect.HasCulture ? cultureAspect.Culture.Name : null;
var ignoreIndexedCulture = indexSettings.Culture != "any" && culture != indexSettings.Culture;

if (indexSettings.IndexedContentTypes.Contains(context.ContentItem.ContentType) && !ignoreIndexedCulture)
{
if (!indexSettings.IndexLatest && !publishedLoaded)
{
publishedLoaded = true;
published = await contentManager.GetAsync(context.ContentItem.ContentItemId, VersionOptions.Published);
}

if (indexSettings.IndexLatest && !latestLoaded)
if (context is RemoveContentContext)
{
latestLoaded = true;
latest = await contentManager.GetAsync(context.ContentItem.ContentItemId, VersionOptions.Latest);
await indexManager.DeleteDocumentsAsync(indexSettings.IndexName, new string[] { context.ContentItem.ContentItemId });
continue;
}

var contentItem = !indexSettings.IndexLatest ? published : latest;

if (contentItem == null)
if (!indexSettings.IndexLatest)
{
await luceneIndexManager.DeleteDocumentsAsync(indexSettings.IndexName, new string[] { context.ContentItem.ContentItemId });
if (context is PublishContentContext publishContext)
{
if (publishContext.Cancel)
{
continue;
}

if (publishContext.IsPublishing)
{
await StoreDocumentAsync(publishContext.PublishingItem, contentItemIndexHandlers, indexManager, logger, indexSettings);
}
else
{
await indexManager.DeleteDocumentsAsync(indexSettings.IndexName, new string[] { context.ContentItem.ContentItemId });
}
}
else if (context is UpdateContentContext updateContext)
{
await indexManager.DeleteDocumentsAsync(indexSettings.IndexName, new string[] { context.ContentItem.ContentItemId });
}
}
else
{
var buildIndexContext = new BuildIndexContext(new DocumentIndex(contentItem.ContentItemId, contentItem.ContentItemVersionId), contentItem, new string[] { contentItem.ContentType }, new LuceneContentIndexSettings());
await contentItemIndexHandlers.InvokeAsync(x => x.BuildIndexAsync(buildIndexContext), logger);

await luceneIndexManager.DeleteDocumentsAsync(indexSettings.IndexName, new string[] { contentItem.ContentItemId });
await luceneIndexManager.StoreDocumentsAsync(indexSettings.IndexName, new DocumentIndex[] { buildIndexContext.DocumentIndex });
if (context is PublishContentContext publishContext)
{
if (publishContext.Cancel)
{
continue;
}

if (publishContext.IsPublishing)
{
await StoreDocumentAsync(publishContext.PublishingItem, contentItemIndexHandlers,indexManager, logger, indexSettings);
}
else
{
await indexManager.DeleteDocumentsAsync(indexSettings.IndexName, new string[] { context.ContentItem.ContentItemId });
}
}
else if (context is UpdateContentContext updateContext)
{
await StoreDocumentAsync(updateContext.UpdatingItem, contentItemIndexHandlers, indexManager, logger, indexSettings);
}
else if (context is CreateContentContext createContext)
{
await StoreDocumentAsync(createContext.CreatingItem, contentItemIndexHandlers, indexManager, logger, indexSettings);
}
}
Skrypt marked this conversation as resolved.
Show resolved Hide resolved
}
}
}
}

private static async Task StoreDocumentAsync(ContentItem contentItem, IEnumerable<IContentItemIndexHandler> contentItemIndexHandlers, LuceneIndexManager luceneIndexManager, ILogger logger, LuceneIndexSettings indexSettings)
{

var buildIndexContext = new BuildIndexContext(new DocumentIndex(contentItem.ContentItemId, contentItem.ContentItemVersionId), contentItem, new string[] { contentItem.ContentType }, new LuceneContentIndexSettings());
Skrypt marked this conversation as resolved.
Show resolved Hide resolved
await contentItemIndexHandlers.InvokeAsync(x => x.BuildIndexAsync(buildIndexContext), logger);

await luceneIndexManager.DeleteDocumentsAsync(indexSettings.IndexName, new string[] { contentItem.ContentItemId });
await luceneIndexManager.StoreDocumentsAsync(indexSettings.IndexName, new DocumentIndex[] { buildIndexContext.DocumentIndex });
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@ public class CreateContentContext : ContentContextBase
{
public CreateContentContext(ContentItem contentItem) : base(contentItem)
{
CreatingItem = contentItem;
}

public ContentItem CreatingItem { get; private set; }
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,18 @@ namespace OrchardCore.ContentManagement.Handlers
{
public class PublishContentContext : ContentContextBase
{
public PublishContentContext(ContentItem contentItem, ContentItem previousContentItem) : base(contentItem)
public PublishContentContext(ContentItem contentItem, ContentItem previousContentItem, bool isPublishing = true) : base(contentItem)
{
PublishingItem = contentItem;
PublishingItem = isPublishing ? contentItem : null;
PreviousItem = previousContentItem;
IsPublishing = isPublishing;
}

public ContentItem PublishingItem { get; set; }
public ContentItem PreviousItem { get; set; }
public ContentItem PublishingItem { get; private set; }
public ContentItem PreviousItem { get; private set; }

public bool Cancel { get; set; }

public bool IsPublishing { get; private set; } = true;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,6 @@ public UpdateContentContext(ContentItem contentItem) : base(contentItem)
UpdatingItem = contentItem;
}

public ContentItem UpdatingItem { get; set; }
public ContentItem UpdatingItem { get; private set; }
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -476,10 +476,7 @@ public async Task UnpublishAsync(ContentItem contentItem)
// Create a context for the item. the publishing version is null in this case
// and the previous version is the one active prior to unpublishing. handlers
// should take this null check into account
var context = new PublishContentContext(contentItem, publishedItem)
{
PublishingItem = null
};
var context = new PublishContentContext(contentItem, publishedItem, false);

await Handlers.InvokeAsync((handler, context) => handler.UnpublishingAsync(context), context, _logger);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,23 +60,19 @@ private static async Task IndexingAsync(ShellScope scope, IEnumerable<ContentCon
var services = scope.ServiceProvider;
var contentManager = services.GetRequiredService<IContentManager>();
var contentItemIndexHandlers = services.GetServices<IContentItemIndexHandler>();

var indexManager = services.GetRequiredService<AzureAIIndexDocumentManager>();
var indexSettingsService = services.GetRequiredService<AzureAISearchIndexSettingsService>();
var logger = services.GetRequiredService<ILogger<AzureAISearchIndexingContentHandler>>();
var indexDocumentManager = services.GetRequiredService<AzureAIIndexDocumentManager>();

// Multiple items may have been updated in the same scope, e.g through a recipe.
var contextsGroupById = contexts.GroupBy(c => c.ContentItem.ContentItemId);
var contextsGroupById = contexts.GroupBy(c => c.ContentItem.ContentItemId, c => c);

// Get all contexts for each content item id.
foreach (var ContextsById in contextsGroupById)
{
// Only process the last context.
var context = ContextsById.Last();

ContentItem published = null, latest = null;
bool publishedLoaded = false, latestLoaded = false;

foreach (var indexSettings in await indexSettingsService.GetSettingsAsync())
{
var cultureAspect = await contentManager.PopulateAspectAsync<CultureAspect>(context.ContentItem);
Expand All @@ -85,33 +81,74 @@ private static async Task IndexingAsync(ShellScope scope, IEnumerable<ContentCon

if (indexSettings.IndexedContentTypes.Contains(context.ContentItem.ContentType) && !ignoreIndexedCulture)
{
if (!indexSettings.IndexLatest && !publishedLoaded)
{
publishedLoaded = true;
published = await contentManager.GetAsync(context.ContentItem.ContentItemId, VersionOptions.Published);
}

if (indexSettings.IndexLatest && !latestLoaded)
if (context is RemoveContentContext)
{
latestLoaded = true;
latest = await contentManager.GetAsync(context.ContentItem.ContentItemId, VersionOptions.Latest);
await indexManager.DeleteDocumentsAsync(indexSettings.IndexName, new string[] { context.ContentItem.ContentItemId });
continue;
}

var contentItem = !indexSettings.IndexLatest ? published : latest;

if (contentItem == null)
if (!indexSettings.IndexLatest)
{
await indexDocumentManager.DeleteDocumentsAsync(indexSettings.IndexName, new string[] { context.ContentItem.ContentItemId });
if (context is PublishContentContext publishContext)
{
if (publishContext.Cancel)
{
continue;
}

if (publishContext.IsPublishing)
{
await StoreLuceneDocument(publishContext.PublishingItem, contentItemIndexHandlers, indexManager, logger, indexSettings);
}
else
{
await indexManager.DeleteDocumentsAsync(indexSettings.IndexName, new string[] { context.ContentItem.ContentItemId });
}
}
else if (context is UpdateContentContext updateContext)
{
await indexManager.DeleteDocumentsAsync(indexSettings.IndexName, new string[] { context.ContentItem.ContentItemId });
}
}
else
{
var index = new DocumentIndex(contentItem.ContentItemId, contentItem.ContentItemVersionId);
var buildIndexContext = new BuildIndexContext(index, contentItem, [contentItem.ContentType], new AzureAISearchContentIndexSettings());
await contentItemIndexHandlers.InvokeAsync(x => x.BuildIndexAsync(buildIndexContext), logger);
await indexDocumentManager.MergeOrUploadDocumentsAsync(indexSettings.IndexName, new DocumentIndex[] { buildIndexContext.DocumentIndex }, indexSettings);
if (context is PublishContentContext publishContext)
{
if (publishContext.Cancel)
{
continue;
}

if (publishContext.IsPublishing)
{
await StoreLuceneDocument(publishContext.PublishingItem, contentItemIndexHandlers, indexManager, logger, indexSettings);
}
else
{
await indexManager.DeleteDocumentsAsync(indexSettings.IndexName, new string[] { context.ContentItem.ContentItemId });
}
}
else if (context is UpdateContentContext updateContext)
{
await StoreLuceneDocument(updateContext.UpdatingItem, contentItemIndexHandlers, indexManager, logger, indexSettings);
}
else if (context is CreateContentContext createContext)
{
await StoreLuceneDocument(createContext.CreatingItem, contentItemIndexHandlers, indexManager, logger, indexSettings);
}
}
}
}
}
}

private static async Task StoreLuceneDocument(ContentItem contentItem, IEnumerable<IContentItemIndexHandler> contentItemIndexHandlers, AzureAIIndexDocumentManager indexManager, ILogger logger, AzureAISearchIndexSettings indexSettings)
{

var buildIndexContext = new BuildIndexContext(new DocumentIndex(contentItem.ContentItemId, contentItem.ContentItemVersionId), contentItem, new string[] { contentItem.ContentType }, new AzureAISearchContentIndexSettings());
await contentItemIndexHandlers.InvokeAsync(x => x.BuildIndexAsync(buildIndexContext), logger);

await indexManager.DeleteDocumentsAsync(indexSettings.IndexName, new string[] { contentItem.ContentItemId });
await indexManager.MergeOrUploadDocumentsAsync(indexSettings.IndexName, new DocumentIndex[] { buildIndexContext.DocumentIndex }, indexSettings);
}
}
Loading
Loading