yv01p/Umbraco-CMS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make the indexing batch size configurable (#20543)

Browse Source 
* Introduce configurable batch size for indexing

* Stop using Examine indexing events for reporting index rebuild operation completeness (it is volatile)
This commit is contained in:
Kenn Jacobsen
2025-10-17 15:45:03 +02:00
committed by GitHub
parent b142dcc84f
commit ae2c59b703
7 changed files with 113 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/Umbraco.Core/Configuration/Models/IndexingSettings.cs
View File

@@ -9,10 +9,16 @@ namespace Umbraco.Cms.Core.Configuration.Models;
public class IndexingSettings
{
private const bool StaticExplicitlyIndexEachNestedProperty = false;
private const int StaticBatchSize = 10000;
/// <summary>
/// Gets or sets a value for whether each nested property should have it's own indexed value. Requires a rebuild of indexes when changed.
/// </summary>
[DefaultValue(StaticExplicitlyIndexEachNestedProperty)]
public bool ExplicitlyIndexEachNestedProperty { get; set; } = StaticExplicitlyIndexEachNestedProperty;
/// <summary>
/// Gets or sets a value for how many items to index at a time.
/// </summary>
public int BatchSize { get; set; } = StaticBatchSize;
}

47
src/Umbraco.Infrastructure/Examine/ContentIndexPopulator.cs
View File

@@ -1,5 +1,9 @@
using Examine;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Umbraco.Cms.Core.Configuration.Models;
using Umbraco.Cms.Core.DependencyInjection;
using Umbraco.Cms.Core.Models;
using Umbraco.Cms.Core.Persistence.Querying;
using Umbraco.Cms.Core.Services;
@@ -20,11 +24,23 @@ public class ContentIndexPopulator : IndexPopulator<IUmbracoContentIndex>
private readonly bool _publishedValuesOnly;
private readonly IUmbracoDatabaseFactory _umbracoDatabaseFactory;
private IndexingSettings _indexingSettings;
/// <summary>
/// This is a static query, it's parameters don't change so store statically
/// </summary>
private IQuery<IContent>? _publishedQuery;
[Obsolete("Please use the non-obsolete constructor. Scheduled for removal in V19.")]
public ContentIndexPopulator(
ILogger<ContentIndexPopulator> logger,
IContentService contentService,
IUmbracoDatabaseFactory umbracoDatabaseFactory,
IContentValueSetBuilder contentValueSetBuilder)
: this(logger, false, null, contentService, umbracoDatabaseFactory, contentValueSetBuilder, StaticServiceProvider.Instance.GetRequiredService<IOptionsMonitor<IndexingSettings>>())
{
}
/// <summary>
/// Default constructor to lookup all content data
/// </summary>
@@ -32,8 +48,21 @@ public class ContentIndexPopulator : IndexPopulator<IUmbracoContentIndex>
ILogger<ContentIndexPopulator> logger,
IContentService contentService,
IUmbracoDatabaseFactory umbracoDatabaseFactory,
IContentValueSetBuilder contentValueSetBuilder)
: this(logger, false, null, contentService, umbracoDatabaseFactory, contentValueSetBuilder)
IContentValueSetBuilder contentValueSetBuilder,
IOptionsMonitor<IndexingSettings> indexingSettings)
: this(logger, false, null, contentService, umbracoDatabaseFactory, contentValueSetBuilder, indexingSettings)
{
}
[Obsolete("Please use the non-obsolete constructor. Scheduled for removal in V19.")]
public ContentIndexPopulator(
ILogger<ContentIndexPopulator> logger,
bool publishedValuesOnly,
int? parentId,
IContentService contentService,
IUmbracoDatabaseFactory umbracoDatabaseFactory,
IValueSetBuilder<IContent> contentValueSetBuilder)
: this(logger, publishedValuesOnly, parentId, contentService, umbracoDatabaseFactory, contentValueSetBuilder, StaticServiceProvider.Instance.GetRequiredService<IOptionsMonitor<IndexingSettings>>())
{
}
@@ -46,7 +75,8 @@ public class ContentIndexPopulator : IndexPopulator<IUmbracoContentIndex>
int? parentId,
IContentService contentService,
IUmbracoDatabaseFactory umbracoDatabaseFactory,
IValueSetBuilder<IContent> contentValueSetBuilder)
IValueSetBuilder<IContent> contentValueSetBuilder,
IOptionsMonitor<IndexingSettings> indexingSettings)
{
_contentService = contentService ?? throw new ArgumentNullException(nameof(contentService));
_umbracoDatabaseFactory = umbracoDatabaseFactory ?? throw new ArgumentNullException(nameof(umbracoDatabaseFactory));
@@ -54,6 +84,12 @@ public class ContentIndexPopulator : IndexPopulator<IUmbracoContentIndex>
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_publishedValuesOnly = publishedValuesOnly;
_parentId = parentId;
_indexingSettings = indexingSettings.CurrentValue;
indexingSettings.OnChange(change =>
{
_indexingSettings = change;
});
}
private IQuery<IContent> PublishedQuery => _publishedQuery ??=
@@ -75,7 +111,6 @@ public class ContentIndexPopulator : IndexPopulator<IUmbracoContentIndex>
return;
}
const int pageSize = 10000;
var pageIndex = 0;
var contentParentId = -1;
@@ -86,11 +121,11 @@ public class ContentIndexPopulator : IndexPopulator<IUmbracoContentIndex>
if (_publishedValuesOnly)
{
IndexPublishedContent(contentParentId, pageIndex, pageSize, indexes);
IndexPublishedContent(contentParentId, pageIndex, _indexingSettings.BatchSize, indexes);
}
else
{
IndexAllContent(contentParentId, pageIndex, pageSize, indexes);
IndexAllContent(contentParentId, pageIndex, _indexingSettings.BatchSize, indexes);
}
}

21
src/Umbraco.Infrastructure/Examine/DeliveryApiContentIndexHelper.cs
View File

@@ -1,5 +1,7 @@
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Options;
using Umbraco.Cms.Core.Configuration.Models;
using Umbraco.Cms.Core.DependencyInjection;
using Umbraco.Cms.Core.Models;
using Umbraco.Cms.Core.Persistence.Querying;
using Umbraco.Cms.Core.Services;
@@ -14,22 +16,33 @@ internal sealed class DeliveryApiContentIndexHelper : IDeliveryApiContentIndexHe
private readonly IUmbracoDatabaseFactory _umbracoDatabaseFactory;
private DeliveryApiSettings _deliveryApiSettings;
private IndexingSettings _indexingSettings;
[Obsolete("Please use the non-obsolete constructor. Scheduled for removal in V19.")]
public DeliveryApiContentIndexHelper(
IContentService contentService,
IUmbracoDatabaseFactory umbracoDatabaseFactory,
IOptionsMonitor<DeliveryApiSettings> deliveryApiSettings)
: this(contentService, umbracoDatabaseFactory, deliveryApiSettings, StaticServiceProvider.Instance.GetRequiredService<IOptionsMonitor<IndexingSettings>>())
{
}
public DeliveryApiContentIndexHelper(
IContentService contentService,
IUmbracoDatabaseFactory umbracoDatabaseFactory,
IOptionsMonitor<DeliveryApiSettings> deliveryApiSettings,
IOptionsMonitor<IndexingSettings> indexingSettings)
{
_contentService = contentService;
_umbracoDatabaseFactory = umbracoDatabaseFactory;
_deliveryApiSettings = deliveryApiSettings.CurrentValue;
_indexingSettings = indexingSettings.CurrentValue;
deliveryApiSettings.OnChange(settings => _deliveryApiSettings = settings);
indexingSettings.OnChange(settings => _indexingSettings = settings);
}
public void EnumerateApplicableDescendantsForContentIndex(int rootContentId, Action<IContent[]> actionToPerform)
{
const int pageSize = 10000;
EnumerateApplicableDescendantsForContentIndex(rootContentId, actionToPerform, pageSize);
}
=> EnumerateApplicableDescendantsForContentIndex(rootContentId, actionToPerform, _indexingSettings.BatchSize);
internal void EnumerateApplicableDescendantsForContentIndex(int rootContentId, Action<IContent[]> actionToPerform, int pageSize)
{

35
src/Umbraco.Infrastructure/Examine/MediaIndexPopulator.cs
View File

@@ -1,5 +1,9 @@
using Examine;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Umbraco.Cms.Core.Configuration.Models;
using Umbraco.Cms.Core.DependencyInjection;
using Umbraco.Cms.Core.Models;
using Umbraco.Cms.Core.Services;
@@ -15,23 +19,43 @@ public class MediaIndexPopulator : IndexPopulator<IUmbracoContentIndex>
private readonly IValueSetBuilder<IMedia> _mediaValueSetBuilder;
private readonly int? _parentId;
private IndexingSettings _indexingSettings;
[Obsolete("Please use the non-obsolete constructor. Scheduled for removal in V19.")]
public MediaIndexPopulator(ILogger<MediaIndexPopulator> logger, IMediaService mediaService, IValueSetBuilder<IMedia> mediaValueSetBuilder)
: this(logger, null, mediaService, mediaValueSetBuilder, StaticServiceProvider.Instance.GetRequiredService<IOptionsMonitor<IndexingSettings>>())
{
}
/// <summary>
/// Default constructor to lookup all content data
/// </summary>
public MediaIndexPopulator(ILogger<MediaIndexPopulator> logger, IMediaService mediaService, IValueSetBuilder<IMedia> mediaValueSetBuilder)
: this(logger, null, mediaService, mediaValueSetBuilder)
public MediaIndexPopulator(ILogger<MediaIndexPopulator> logger, IMediaService mediaService, IValueSetBuilder<IMedia> mediaValueSetBuilder, IOptionsMonitor<IndexingSettings> indexingSettings)
: this(logger, null, mediaService, mediaValueSetBuilder, indexingSettings)
{
}
[Obsolete("Please use the non-obsolete constructor. Scheduled for removal in V19.")]
public MediaIndexPopulator(ILogger<MediaIndexPopulator> logger, int? parentId, IMediaService mediaService, IValueSetBuilder<IMedia> mediaValueSetBuilder)
: this(logger, parentId, mediaService, mediaValueSetBuilder, StaticServiceProvider.Instance.GetRequiredService<IOptionsMonitor<IndexingSettings>>())
{
}
/// <summary>
/// Optional constructor allowing specifying custom query parameters
/// </summary>
public MediaIndexPopulator(ILogger<MediaIndexPopulator> logger, int? parentId, IMediaService mediaService, IValueSetBuilder<IMedia> mediaValueSetBuilder)
public MediaIndexPopulator(ILogger<MediaIndexPopulator> logger, int? parentId, IMediaService mediaService, IValueSetBuilder<IMedia> mediaValueSetBuilder, IOptionsMonitor<IndexingSettings> indexingSettings)
{
_logger = logger;
_parentId = parentId;
_mediaService = mediaService;
_mediaValueSetBuilder = mediaValueSetBuilder;
_indexingSettings = indexingSettings.CurrentValue;
indexingSettings.OnChange(change =>
{
_indexingSettings = change;
});
}
protected override void PopulateIndexes(IReadOnlyList<IIndex> indexes)
@@ -46,7 +70,6 @@ public class MediaIndexPopulator : IndexPopulator<IUmbracoContentIndex>
return;
}
const int pageSize = 10000;
var pageIndex = 0;
var mediaParentId = -1;
@@ -60,7 +83,7 @@ public class MediaIndexPopulator : IndexPopulator<IUmbracoContentIndex>
do
{
media = _mediaService.GetPagedDescendants(mediaParentId, pageIndex, pageSize, out _).ToArray();
media = _mediaService.GetPagedDescendants(mediaParentId, pageIndex, _indexingSettings.BatchSize, out _).ToArray();
// ReSharper disable once PossibleMultipleEnumeration
foreach (IIndex index in indexes)
@@ -70,6 +93,6 @@ public class MediaIndexPopulator : IndexPopulator<IUmbracoContentIndex>
pageIndex++;
}
while (media.Length == pageSize);
while (media.Length == _indexingSettings.BatchSize);
}
}

13
src/Umbraco.Infrastructure/Examine/PublishedContentIndexPopulator.cs
View File

@@ -1,4 +1,6 @@
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Umbraco.Cms.Core.Configuration.Models;
using Umbraco.Cms.Core.Services;
using Umbraco.Cms.Infrastructure.Persistence;
@@ -15,6 +17,7 @@ namespace Umbraco.Cms.Infrastructure.Examine;
/// </remarks>
public class PublishedContentIndexPopulator : ContentIndexPopulator
{
[Obsolete("Please use the non-obsolete constructor. Scheduled for removal in V19.")]
public PublishedContentIndexPopulator(
ILogger<PublishedContentIndexPopulator> logger,
IContentService contentService,
@@ -23,4 +26,14 @@ public class PublishedContentIndexPopulator : ContentIndexPopulator
: base(logger, true, null, contentService, umbracoDatabaseFactory, contentValueSetBuilder)
{
}
public PublishedContentIndexPopulator(
ILogger<PublishedContentIndexPopulator> logger,
IContentService contentService,
IUmbracoDatabaseFactory umbracoDatabaseFactory,
IPublishedContentValueSetBuilder contentValueSetBuilder,
IOptionsMonitor<IndexingSettings> indexingSettings)
: base(logger, true, null, contentService, umbracoDatabaseFactory, contentValueSetBuilder, indexingSettings)
{
}
}

23
src/Umbraco.Infrastructure/Services/IndexingRebuilderService.cs
View File

@@ -42,12 +42,6 @@ public class IndexingRebuilderService : IIndexingRebuilderService
/// <inheritdoc />
public async Task<bool> TryRebuildAsync(IIndex index, string indexName)
{
// Remove it in case there's a handler there already
index.IndexOperationComplete -= Indexer_IndexOperationComplete;
// Now add a single handler
index.IndexOperationComplete += Indexer_IndexOperationComplete;
try
{
Attempt<IndexRebuildResult> attempt = await _indexRebuilder.RebuildIndexAsync(indexName);
@@ -55,8 +49,6 @@ public class IndexingRebuilderService : IIndexingRebuilderService
}
catch (Exception exception)
{
// Ensure it's not listening
index.IndexOperationComplete -= Indexer_IndexOperationComplete;
_logger.LogError(exception, "An error occurred rebuilding index");
return false;
}
@@ -70,19 +62,4 @@ public class IndexingRebuilderService : IIndexingRebuilderService
/// <inheritdoc />
public Task<bool> IsRebuildingAsync(string indexName)
=> _indexRebuilder.IsRebuildingAsync(indexName);
private void Indexer_IndexOperationComplete(object? sender, EventArgs e)
{
var indexer = (IIndex?)sender;
_logger.LogDebug("Logging operation completed for index {IndexName}", indexer?.Name);
if (indexer is not null)
{
//ensure it's not listening anymore
indexer.IndexOperationComplete -= Indexer_IndexOperationComplete;
}
_logger.LogInformation("Rebuilding index '{IndexerName}' done.", indexer?.Name);
}
}

10
tests/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/IndexInitializer.cs
View File

@@ -39,6 +39,7 @@ public class IndexInitializer
private readonly IContentTypeService _contentTypeService;
private readonly IDocumentUrlService _documentUrlService;
private readonly ILanguageService _languageService;
private readonly IOptionsMonitor<IndexingSettings> _indexSettings;
public IndexInitializer(
IShortStringHelper shortStringHelper,
@@ -50,7 +51,8 @@ public class IndexInitializer
ILocalizationService localizationService,
IContentTypeService contentTypeService,
IDocumentUrlService documentUrlService,
ILanguageService languageService)
ILanguageService languageService,
IOptionsMonitor<IndexingSettings> indexSettings)
{
_shortStringHelper = shortStringHelper;
_propertyEditors = propertyEditors;
@@ -62,6 +64,7 @@ public class IndexInitializer
_contentTypeService = contentTypeService;
_documentUrlService = documentUrlService;
_languageService = languageService;
_indexSettings = indexSettings;
}
public ContentValueSetBuilder GetContentValueSetBuilder(bool publishedValuesOnly)
@@ -91,7 +94,8 @@ public class IndexInitializer
null,
contentService,
umbracoDatabaseFactory,
contentValueSetBuilder);
contentValueSetBuilder,
_indexSettings);
return contentIndexDataSource;
}
@@ -105,7 +109,7 @@ public class IndexInitializer
_shortStringHelper,
_contentSettings,
StaticServiceProvider.Instance.GetRequiredService<IContentTypeService>());
var mediaIndexDataSource = new MediaIndexPopulator(null, mediaService, mediaValueSetBuilder);
var mediaIndexDataSource = new MediaIndexPopulator(null, mediaService, mediaValueSetBuilder, _indexSettings);
return mediaIndexDataSource;
}