Use new submit and poll solution for examine index rebuild (#19707)

* Started implementing new LongRunningOperationService and adjusting tasks to use this service

This service will manage operations that require status to be synced between servers (load balanced setup).

* Missing migration to add new lock. Other simplifications.

* Add job to cleanup the LongRunningOperations entries

* Add new DatabaseCacheRebuilder.RebuildAsync method

This is both async and returns an attempt, which will fail if a rebuild operation is already running.

* Missing LongRunningOperation database table creation on clean install

* Store expire date in the long running operation. Better handling of non-background operations.

Storing an expiration date allows setting different expiration times depending on the type of operation, and whether it is running in the background or not.

* Added integration tests for LongRunningOperationRepository

* Added unit tests for LongRunningOperationService

* Add type as a parameter to more repository calls. Distinguish between expiration and deletion in `LongRunningOperationRepository.CleanOperations`.

* Fix failing unit test

* Fixed `PerformPublishBranchAsync` result not being deserialized correctly

* Remove unnecessary DatabaseCacheRebuildResult value

* Add status to `LongRunningOperationService.GetResult` attempt to inform on why a result could not be retrieved

* General improvements

* Missing rename

* Improve the handling of long running operations that are not in background and stale operations

* Fix failing unit tests

* Fixed small mismatch between interface and implementation

* Use the new submit and poll functionality for the Examine index rebuild

* Use a fire and forget task instead of the background queue

* Apply suggestions from code review

Co-authored-by: Andy Butland <abutland73@gmail.com>

* Make sure exceptions are caught when running in the background

* Alignment with other repositories (async + pagination)

* Fix build after merge

* Missing obsoletion messages

* Additional fixes

* Add Async suffix to service methods

* Missing adjustment

* Moved hardcoded settings to IOptions

* Fix issue in SQL Server where 0 is not accepted as requested number of rows

* Fix issue in SQL Server where query provided to count cannot contain orderby

* Additional SQL Server fixes

* Update method names

* Adjustments from code review

* Ignoring result of index rebuild in `IndexingNotificationHandler.Language.cs` (same behavior as before)

* Missed some obsoletion messages

---------

Co-authored-by: Andy Butland <abutland73@gmail.com>

src/Umbraco.Cms.Api.Management/Controllers/Indexer/RebuildIndexerController.cs

@@ -26,9 +26,9 @@ public class RebuildIndexerController : IndexerControllerBase
     }
 
     /// <summary>
-    ///     Rebuilds the index
+    ///     Rebuilds the index.
     /// </summary>
-    /// <param name="indexName"></param>
+    /// <param name="indexName">The name of the index to rebuild.</param>
     /// <returns></returns>
     [HttpPost("{indexName}/rebuild")]
     [MapToApiVersion("1.0")]
@@ -36,7 +36,7 @@ public class RebuildIndexerController : IndexerControllerBase
     [ProducesResponseType(typeof(ProblemDetails), StatusCodes.Status404NotFound)]
     [ProducesResponseType(typeof(ProblemDetails), StatusCodes.Status409Conflict)]
     [ProducesResponseType(StatusCodes.Status200OK)]
-    public Task<IActionResult> Rebuild(CancellationToken cancellationToken, string indexName)
+    public async Task<IActionResult> Rebuild(CancellationToken cancellationToken, string indexName)
     {
         if (!_examineManager.TryGetIndex(indexName, out IIndex? index))
         {
@@ -48,7 +48,7 @@ public class RebuildIndexerController : IndexerControllerBase
                 Type = "Error",
             };
 
-            return Task.FromResult<IActionResult>(NotFound(invalidModelProblem));
+            return NotFound(invalidModelProblem);
         }
 
         if (!_indexingRebuilderService.CanRebuild(index.Name))
@@ -57,19 +57,19 @@ public class RebuildIndexerController : IndexerControllerBase
             {
                 Title = "Could not validate the populator",
                 Detail =
-                    $"The index {index?.Name} could not be rebuilt because we could not validate its associated {typeof(IIndexPopulator)}",
+                    $"The index {index.Name} could not be rebuilt because we could not validate its associated {typeof(IIndexPopulator)}",
                 Status = StatusCodes.Status400BadRequest,
                 Type = "Error",
             };
 
-            return Task.FromResult<IActionResult>(BadRequest(invalidModelProblem));
+            return BadRequest(invalidModelProblem);
         }
 
         _logger.LogInformation("Rebuilding index '{IndexName}'", indexName);
 
-        if (_indexingRebuilderService.TryRebuild(index, indexName))
+        if (await _indexingRebuilderService.TryRebuildAsync(index, indexName))
         {
-            return Task.FromResult<IActionResult>(Ok());
+            return Ok();
         }
 
         var problemDetails = new ProblemDetails
@@ -80,6 +80,6 @@ public class RebuildIndexerController : IndexerControllerBase
             Type = "Error",
         };
 
-        return Task.FromResult<IActionResult>(Conflict(problemDetails));
+        return Conflict(problemDetails);
     }
 }

src/Umbraco.Cms.Api.Management/Factories/IIndexPresentationFactory.cs

@@ -5,5 +5,8 @@ namespace Umbraco.Cms.Api.Management.Factories;
 
 public interface IIndexPresentationFactory
 {
+    [Obsolete("Use CreateAsync() instead. Scheduled for removal in v19.")]
     IndexResponseModel Create(IIndex index);
+
+    Task<IndexResponseModel> CreateAsync(IIndex index) => Task.FromResult(Create(index));
 }

src/Umbraco.Cms.Api.Management/Factories/IndexPresentationFactory.cs

@@ -28,21 +28,17 @@ public class IndexPresentationFactory : IIndexPresentationFactory
         _logger = logger;
     }
 
-    [Obsolete("Use the non obsolete method instead. Scheduled for removal in v17")]
+    /// <inheritdoc />
-    public IndexPresentationFactory(IIndexDiagnosticsFactory indexDiagnosticsFactory, IIndexRebuilder indexRebuilder, IIndexingRebuilderService indexingRebuilderService)
+    [Obsolete("Use CreateAsync() instead. Scheduled for removal in v19.")]
-    :this(
-        indexDiagnosticsFactory,
-        indexRebuilder,
-        indexingRebuilderService,
-        StaticServiceProvider.Instance.GetRequiredService<ILogger<IndexPresentationFactory>>())
-    {
-    }
-
     public IndexResponseModel Create(IIndex index)
+        => CreateAsync(index).GetAwaiter().GetResult();
+
+    /// <inheritdoc />
+    public async Task<IndexResponseModel> CreateAsync(IIndex index)
     {
         var isCorrupt = !TryGetSearcherName(index, out var searcherName);
 
-        if (_indexingRebuilderService.IsRebuilding(index.Name))
+        if (await _indexingRebuilderService.IsRebuildingAsync(index.Name))
         {
             return new IndexResponseModel
             {
@@ -63,7 +59,7 @@ public class IndexPresentationFactory : IIndexPresentationFactory
 
         var properties = new Dictionary<string, object?>();
 
-        foreach (var property in indexDiag.Metadata)
+        foreach (KeyValuePair<string, object?> property in indexDiag.Metadata)
         {
             if (property.Value is null)
             {
@@ -71,7 +67,7 @@ public class IndexPresentationFactory : IIndexPresentationFactory
             }
             else
             {
-                var propertyType = property.Value.GetType();
+                Type propertyType = property.Value.GetType();
                 properties[property.Key] = propertyType.IsClass && !propertyType.IsArray ? property.Value?.ToString() : property.Value;
             }
         }

src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.Examine.cs

@@ -82,7 +82,7 @@ public static partial class UmbracoBuilderExtensions
         builder.AddNotificationHandler<PublicAccessCacheRefresherNotification, DeliveryApiContentIndexingNotificationHandler>();
         builder.AddNotificationHandler<MediaCacheRefresherNotification, MediaIndexingNotificationHandler>();
         builder.AddNotificationHandler<MemberCacheRefresherNotification, MemberIndexingNotificationHandler>();
-        builder.AddNotificationHandler<LanguageCacheRefresherNotification, LanguageIndexingNotificationHandler>();
+        builder.AddNotificationAsyncHandler<LanguageCacheRefresherNotification, LanguageIndexingNotificationHandler>();
 
         builder.AddNotificationHandler<UmbracoRequestBeginNotification, RebuildOnStartupHandler>();
 

src/Umbraco.Infrastructure/Examine/ExamineIndexRebuilder.cs

@@ -6,18 +6,20 @@ using Microsoft.Extensions.Logging;
 using Umbraco.Cms.Core;
 using Umbraco.Cms.Core.Runtime;
 using Umbraco.Cms.Core.Services;
-using Umbraco.Cms.Infrastructure.HostedServices;
+using Umbraco.Cms.Core.Services.OperationStatus;
+using Umbraco.Cms.Infrastructure.Models;
 
 namespace Umbraco.Cms.Infrastructure.Examine;
 
 internal class ExamineIndexRebuilder : IIndexRebuilder
 {
-    private readonly IBackgroundTaskQueue _backgroundTaskQueue;
+    private const string RebuildAllOperationTypeName = "RebuildAllExamineIndexes";
+
     private readonly IExamineManager _examineManager;
     private readonly ILogger<ExamineIndexRebuilder> _logger;
     private readonly IMainDom _mainDom;
     private readonly IEnumerable<IIndexPopulator> _populators;
-    private readonly object _rebuildLocker = new();
+    private readonly ILongRunningOperationService _longRunningOperationService;
     private readonly IRuntimeState _runtimeState;
 
     /// <summary>
@@ -29,16 +31,17 @@ internal class ExamineIndexRebuilder : IIndexRebuilder
         ILogger<ExamineIndexRebuilder> logger,
         IExamineManager examineManager,
         IEnumerable<IIndexPopulator> populators,
-        IBackgroundTaskQueue backgroundTaskQueue)
+        ILongRunningOperationService longRunningOperationService)
     {
         _mainDom = mainDom;
         _runtimeState = runtimeState;
         _logger = logger;
         _examineManager = examineManager;
         _populators = populators;
-        _backgroundTaskQueue = backgroundTaskQueue;
+        _longRunningOperationService = longRunningOperationService;
     }
 
+    /// <inheritdoc/>
     public bool CanRebuild(string indexName)
     {
         if (!_examineManager.TryGetIndex(indexName, out IIndex index))
@@ -49,180 +52,149 @@ internal class ExamineIndexRebuilder : IIndexRebuilder
         return _populators.Any(x => x.IsRegistered(index));
     }
 
+    /// <inheritdoc/>
+    [Obsolete("Use RebuildIndexAsync() instead. Scheduled for removal in v19.")]
     public virtual void RebuildIndex(string indexName, TimeSpan? delay = null, bool useBackgroundThread = true)
+        => RebuildIndexAsync(indexName, delay, useBackgroundThread).GetAwaiter().GetResult();
+
+    /// <inheritdoc/>
+    public virtual async Task<Attempt<IndexRebuildResult>> RebuildIndexAsync(string indexName, TimeSpan? delay = null, bool useBackgroundThread = true)
     {
-        if (delay == null)
+        delay ??= TimeSpan.Zero;
-        {
-            delay = TimeSpan.Zero;
-        }
 
         if (!CanRun())
         {
-            return;
+            return Attempt.Fail(IndexRebuildResult.NotAllowedToRun);
         }
 
-        if (useBackgroundThread)
+        Attempt<Guid, LongRunningOperationEnqueueStatus> attempt = await _longRunningOperationService.RunAsync(
-        {
+            GetRebuildOperationTypeName(indexName),
-            _logger.LogInformation("Starting async background thread for rebuilding index {indexName}.", indexName);
+            async ct =>
-
-            _backgroundTaskQueue.QueueBackgroundWorkItem(
-                cancellationToken =>
-                {
-                    // Do not flow AsyncLocal to the child thread
-                    using (ExecutionContext.SuppressFlow())
-                    {
-                        Task.Run(() => RebuildIndex(indexName, delay.Value, cancellationToken));
-
-                        // immediately return so the queue isn't waiting.
-                        return Task.CompletedTask;
-                    }
-                });
-        }
-        else
-        {
-            RebuildIndex(indexName, delay.Value, CancellationToken.None);
-        }
-    }
-
-    public virtual void RebuildIndexes(bool onlyEmptyIndexes, TimeSpan? delay = null, bool useBackgroundThread = true)
-    {
-        if (delay == null)
-        {
-            delay = TimeSpan.Zero;
-        }
-
-        if (!CanRun())
-        {
-            return;
-        }
-
-        if (useBackgroundThread)
-        {
-            if (_logger.IsEnabled(Microsoft.Extensions.Logging.LogLevel.Debug))
             {
-                _logger.LogDebug($"Queuing background job for {nameof(RebuildIndexes)}.");
+                await RebuildIndex(indexName, delay.Value, ct);
-            }
+                return Task.CompletedTask;
+            },
+            allowConcurrentExecution: false,
+            runInBackground: useBackgroundThread);
 
-            _backgroundTaskQueue.QueueBackgroundWorkItem(
+        if (attempt.Success)
-                cancellationToken =>
-                {
-                    // Do not flow AsyncLocal to the child thread
-                    using (ExecutionContext.SuppressFlow())
-                    {
-                        // This is a fire/forget task spawned by the background thread queue (which means we
-                        // don't need to worry about ExecutionContext flowing).
-                        Task.Run(() => RebuildIndexes(onlyEmptyIndexes, delay.Value, cancellationToken));
-
-                        // immediately return so the queue isn't waiting.
-                        return Task.CompletedTask;
-                    }
-                });
-        }
-        else
         {
-            RebuildIndexes(onlyEmptyIndexes, delay.Value, CancellationToken.None);
+            return Attempt.Succeed(IndexRebuildResult.Success);
         }
+
+        return attempt.Status switch
+        {
+            LongRunningOperationEnqueueStatus.AlreadyRunning => Attempt.Fail(IndexRebuildResult.AlreadyRebuilding),
+            _ => Attempt.Fail(IndexRebuildResult.Unknown),
+        };
     }
 
+    /// <inheritdoc/>
+    [Obsolete("Use RebuildIndexesAsync() instead. Scheduled for removal in v19.")]
+    public virtual void RebuildIndexes(bool onlyEmptyIndexes, TimeSpan? delay = null, bool useBackgroundThread = true)
+        => RebuildIndexesAsync(onlyEmptyIndexes, delay, useBackgroundThread).GetAwaiter().GetResult();
+
+    /// <inheritdoc/>
+    public virtual async Task<Attempt<IndexRebuildResult>> RebuildIndexesAsync(bool onlyEmptyIndexes, TimeSpan? delay = null, bool useBackgroundThread = true)
+    {
+        delay ??= TimeSpan.Zero;
+
+        if (!CanRun())
+        {
+            return Attempt.Fail(IndexRebuildResult.NotAllowedToRun);
+        }
+
+        Attempt<Guid, LongRunningOperationEnqueueStatus> attempt = await _longRunningOperationService.RunAsync(
+            RebuildAllOperationTypeName,
+            async ct =>
+            {
+                await RebuildIndexes(onlyEmptyIndexes, delay.Value, ct);
+                return Task.CompletedTask;
+            },
+            allowConcurrentExecution: false,
+            runInBackground: useBackgroundThread);
+
+        if (attempt.Success)
+        {
+            return Attempt.Succeed(IndexRebuildResult.Success);
+        }
+
+        return attempt.Status switch
+        {
+            LongRunningOperationEnqueueStatus.AlreadyRunning => Attempt.Fail(IndexRebuildResult.AlreadyRebuilding),
+            _ => Attempt.Fail(IndexRebuildResult.Unknown),
+        };
+    }
+
+    /// <inheritdoc/>
+    public async Task<bool> IsRebuildingAsync(string indexName)
+        => (await _longRunningOperationService.GetByTypeAsync(GetRebuildOperationTypeName(indexName), 0, 0)).Total != 0;
+
+    private static string GetRebuildOperationTypeName(string indexName) => $"RebuildExamineIndex-{indexName}";
+
     private bool CanRun() => _mainDom.IsMainDom && _runtimeState.Level == RuntimeLevel.Run;
 
-    private void RebuildIndex(string indexName, TimeSpan delay, CancellationToken cancellationToken)
+    private async Task RebuildIndex(string indexName, TimeSpan delay, CancellationToken cancellationToken)
     {
         if (delay > TimeSpan.Zero)
         {
-            Thread.Sleep(delay);
+            await Task.Delay(delay, cancellationToken);
         }
 
-        try
+        if (!_examineManager.TryGetIndex(indexName, out IIndex index))
         {
-            if (!Monitor.TryEnter(_rebuildLocker))
+            throw new InvalidOperationException($"No index found with name {indexName}");
-            {
-                _logger.LogWarning(
-                    "Call was made to RebuildIndexes but the task runner for rebuilding is already running");
-            }
-            else
-            {
-                if (!_examineManager.TryGetIndex(indexName, out IIndex index))
-                {
-                    throw new InvalidOperationException($"No index found with name {indexName}");
-                }
-
-                index.CreateIndex(); // clear the index
-                foreach (IIndexPopulator populator in _populators)
-                {
-                    if (cancellationToken.IsCancellationRequested)
-                    {
-                        return;
-                    }
-
-                    populator.Populate(index);
-                }
-            }
         }
-        finally
+
+        index.CreateIndex(); // clear the index
+        foreach (IIndexPopulator populator in _populators)
         {
-            if (Monitor.IsEntered(_rebuildLocker))
+            if (cancellationToken.IsCancellationRequested)
             {
-                Monitor.Exit(_rebuildLocker);
+                return;
             }
+
+            populator.Populate(index);
         }
     }
 
-    private void RebuildIndexes(bool onlyEmptyIndexes, TimeSpan delay, CancellationToken cancellationToken)
+    private async Task RebuildIndexes(bool onlyEmptyIndexes, TimeSpan delay, CancellationToken cancellationToken)
     {
         if (delay > TimeSpan.Zero)
         {
-            Thread.Sleep(delay);
+            await Task.Delay(delay, cancellationToken);
         }
 
-        try
+        // If an index exists but it has zero docs we'll consider it empty and rebuild
+        IIndex[] indexes = (onlyEmptyIndexes
+            ? _examineManager.Indexes.Where(ShouldRebuild)
+            : _examineManager.Indexes).ToArray();
+
+        if (indexes.Length == 0)
         {
-            if (!Monitor.TryEnter(_rebuildLocker))
+            return;
-            {
-                _logger.LogWarning(
-                    $"Call was made to {nameof(RebuildIndexes)} but the task runner for rebuilding is already running");
-            }
-            else
-            {
-                // If an index exists but it has zero docs we'll consider it empty and rebuild
-                IIndex[] indexes = (onlyEmptyIndexes
-                    ? _examineManager.Indexes.Where(ShouldRebuild)
-                    : _examineManager.Indexes).ToArray();
-
-                if (indexes.Length == 0)
-                {
-                    return;
-                }
-
-                foreach (IIndex index in indexes)
-                {
-                    index.CreateIndex(); // clear the index
-                }
-
-                // run each populator over the indexes
-                foreach (IIndexPopulator populator in _populators)
-                {
-                    if (cancellationToken.IsCancellationRequested)
-                    {
-                        return;
-                    }
-
-                    try
-                    {
-                        populator.Populate(indexes);
-                    }
-                    catch (Exception e)
-                    {
-                        _logger.LogError(e, "Index populating failed for populator {Populator}", populator.GetType());
-                    }
-                }
-            }
         }
-        finally
+
+        foreach (IIndex index in indexes)
         {
-            if (Monitor.IsEntered(_rebuildLocker))
+            index.CreateIndex(); // clear the index
+        }
+
+        // run each populator over the indexes
+        foreach (IIndexPopulator populator in _populators)
+        {
+            if (cancellationToken.IsCancellationRequested)
             {
-                Monitor.Exit(_rebuildLocker);
+                return;
+            }
+
+            try
+            {
+                populator.Populate(indexes);
+            }
+            catch (Exception e)
+            {
+                _logger.LogError(e, "Index populating failed for populator {Populator}", populator.GetType());
             }
         }
     }

src/Umbraco.Infrastructure/Examine/IIndexRebuilder.cs

@@ -1,10 +1,69 @@
+using Umbraco.Cms.Core;
+using Umbraco.Cms.Infrastructure.Models;
+
 namespace Umbraco.Cms.Infrastructure.Examine;
 
+/// <summary>
+/// Interface for rebuilding search indexes.
+/// </summary>
 public interface IIndexRebuilder
 {
+    /// <summary>
+    /// Checks if the specified index can be rebuilt.
+    /// </summary>
+    /// <param name="indexName">The name of the index to check.</param>
+    /// <returns>Whether the index can be rebuilt.</returns>
     bool CanRebuild(string indexName);
 
+    /// <summary>
+    /// Rebuilds the specified index.
+    /// </summary>
+    /// <param name="indexName">The name of the index to rebuild.</param>
+    /// <param name="delay">The delay before starting the rebuild.</param>
+    /// <param name="useBackgroundThread">Whether to use a background thread for the rebuild.</param>
+    [Obsolete("Use RebuildIndexesAsync() instead. Scheduled for removal in V19.")]
     void RebuildIndex(string indexName, TimeSpan? delay = null, bool useBackgroundThread = true);
 
+    /// <summary>
+    /// Rebuilds the specified index.
+    /// </summary>
+    /// <param name="indexName">The name of the index to rebuild.</param>
+    /// <param name="delay">The delay before starting the rebuild.</param>
+    /// <param name="useBackgroundThread">Whether to use a background thread for the rebuild.</param>
+    /// <returns>A task representing the asynchronous operation.</returns>
+    Task<Attempt<IndexRebuildResult>> RebuildIndexAsync(string indexName, TimeSpan? delay = null, bool useBackgroundThread = true)
+    {
+        RebuildIndex(indexName, delay, useBackgroundThread);
+        return Task.FromResult(Attempt.Succeed(IndexRebuildResult.Success));
+    }
+
+    /// <summary>
+    /// Rebuilds all indexes, or only those that are empty.
+    /// </summary>
+    /// <param name="onlyEmptyIndexes">Whether to only rebuild empty indexes.</param>
+    /// <param name="delay">The delay before starting the rebuild.</param>
+    /// <param name="useBackgroundThread">Whether to use a background thread for the rebuild.</param>
+    [Obsolete("Use RebuildIndexesAsync() instead. Scheduled for removal in V19.")]
     void RebuildIndexes(bool onlyEmptyIndexes, TimeSpan? delay = null, bool useBackgroundThread = true);
+
+    /// <summary>
+    /// Rebuilds all indexes, or only those that are empty.
+    /// </summary>
+    /// <param name="onlyEmptyIndexes">Whether to only rebuild empty indexes.</param>
+    /// <param name="delay">The delay before starting the rebuild.</param>
+    /// <param name="useBackgroundThread">Whether to use a background thread for the rebuild.</param>
+    /// <returns>A task representing the asynchronous operation.</returns>
+    Task<Attempt<IndexRebuildResult>> RebuildIndexesAsync(bool onlyEmptyIndexes, TimeSpan? delay = null, bool useBackgroundThread = true)
+    {
+        RebuildIndexes(onlyEmptyIndexes, delay, useBackgroundThread);
+        return Task.FromResult(Attempt.Succeed(IndexRebuildResult.Success));
+    }
+
+    /// <summary>
+    /// Checks if the specified index is currently being rebuilt.
+    /// </summary>
+    /// <param name="indexName">The name of the index to check.</param>
+    /// <returns>Whether the index is currently being rebuilt.</returns>
+    // TODO (v19): Remove the default implementation.
+    Task<bool> IsRebuildingAsync(string indexName) => throw new NotImplementedException();
 }

src/Umbraco.Infrastructure/Examine/NoopIndexRebuilder.cs

@@ -7,4 +7,6 @@ internal sealed class NoopIndexRebuilder : IIndexRebuilder
     public void RebuildIndex(string indexName, TimeSpan? delay = null, bool useBackgroundThread = true) {}
 
     public void RebuildIndexes(bool onlyEmptyIndexes, TimeSpan? delay = null, bool useBackgroundThread = true) {}
+
+    public Task<bool> IsRebuildingAsync(string indexName) => Task.FromResult(false);
 }

src/Umbraco.Infrastructure/Models/IndexRebuildResult.cs

@@ -0,0 +1,27 @@
+namespace Umbraco.Cms.Infrastructure.Models;
+
+/// <summary>
+/// Represents the status of an index rebuild trigger.
+/// </summary>
+public enum IndexRebuildResult
+{
+    /// <summary>
+    /// The rebuild was either successful or enqueued successfully.
+    /// </summary>
+    Success,
+
+    /// <summary>
+    /// The index is already being rebuilt.
+    /// </summary>
+    AlreadyRebuilding,
+
+    /// <summary>
+    /// The index rebuild was not scheduled because it's not allowed to run at this time.
+    /// </summary>
+    NotAllowedToRun,
+
+    /// <summary>
+    /// The index rebuild was not scheduled due to an unknown error.
+    /// </summary>
+    Unknown,
+}

src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.Language.cs

@@ -5,7 +5,9 @@ using Umbraco.Cms.Infrastructure.Examine;
 
 namespace Umbraco.Cms.Infrastructure.Search;
 
-public sealed class LanguageIndexingNotificationHandler : INotificationHandler<LanguageCacheRefresherNotification>
+public sealed class LanguageIndexingNotificationHandler :
+    INotificationHandler<LanguageCacheRefresherNotification>,
+    INotificationAsyncHandler<LanguageCacheRefresherNotification>
 {
     private readonly IIndexRebuilder _indexRebuilder;
     private readonly IUmbracoIndexingHandler _umbracoIndexingHandler;
@@ -19,14 +21,20 @@ public sealed class LanguageIndexingNotificationHandler : INotificationHandler<L
         _indexRebuilder = indexRebuilder ?? throw new ArgumentNullException(nameof(indexRebuilder));
     }
 
+    /// <inheritdoc />
+    [Obsolete("Use HandleAsync instead. Scheduled for removal in V19.")]
     public void Handle(LanguageCacheRefresherNotification args)
+        => HandleAsync(args, CancellationToken.None).GetAwaiter().GetResult();
+
+    /// <inheritdoc />
+    public async Task HandleAsync(LanguageCacheRefresherNotification notification, CancellationToken cancellationToken)
     {
         if (!_umbracoIndexingHandler.Enabled)
         {
             return;
         }
 
-        if (!(args.MessageObject is LanguageCacheRefresher.JsonPayload[] payloads))
+        if (notification.MessageObject is not LanguageCacheRefresher.JsonPayload[] payloads)
         {
             return;
         }
@@ -37,14 +45,14 @@ public sealed class LanguageIndexingNotificationHandler : INotificationHandler<L
         }
 
         var removedOrCultureChanged = payloads.Any(x =>
-            x.ChangeType == LanguageCacheRefresher.JsonPayload.LanguageChangeType.ChangeCulture
+            x.ChangeType is LanguageCacheRefresher.JsonPayload.LanguageChangeType.ChangeCulture
-            || x.ChangeType == LanguageCacheRefresher.JsonPayload.LanguageChangeType.Remove);
+                or LanguageCacheRefresher.JsonPayload.LanguageChangeType.Remove);
 
         if (removedOrCultureChanged)
         {
             // if a lang is removed or it's culture has changed, we need to rebuild the indexes since
             // field names and values in the index have a string culture value.
-            _indexRebuilder.RebuildIndexes(false);
+            _ = await _indexRebuilder.RebuildIndexesAsync(false);
         }
     }
 }

src/Umbraco.Infrastructure/Services/IIndexingRebuilderService.cs

@@ -2,10 +2,48 @@
 
 namespace Umbraco.Cms.Infrastructure.Services;
 
+/// <summary>
+/// Indexing rebuilder service.
+/// </summary>
 public interface IIndexingRebuilderService
 {
+    /// <summary>
+    /// Checks if the index can be rebuilt.
+    /// </summary>
+    /// <param name="indexName">The name of the index to check.</param>
+    /// <returns>Whether the index can be rebuilt.</returns>
     bool CanRebuild(string indexName);
+
+    /// <summary>
+    /// Tries to rebuild the specified index.
+    /// </summary>
+    /// <param name="index">The index to rebuild.</param>
+    /// <param name="indexName">The name of the index to rebuild.</param>
+    /// <returns>Whether the rebuild was successfully scheduled.</returns>
+    [Obsolete("Use TryRebuildAsync() instead. Scheduled for removal in V19.")]
     bool TryRebuild(IIndex index, string indexName);
 
+    /// <summary>
+    /// Tries to rebuild the specified index.
+    /// </summary>
+    /// <param name="index">The index to rebuild.</param>
+    /// <param name="indexName">The name of the index to rebuild.</param>
+    /// <returns>Whether the rebuild was successfully scheduled.</returns>
+    Task<bool> TryRebuildAsync(IIndex index, string indexName) => Task.FromResult(TryRebuild(index, indexName));
+
+    /// <summary>
+    /// Checks if the specified index is currently being rebuilt.
+    /// </summary>
+    /// <param name="indexName">The name of the index to check.</param>
+    /// <returns>Whether the index is currently being rebuilt.</returns>
+    [Obsolete("Use IsRebuildingAsync() instead. Scheduled for removal in V19.")]
     bool IsRebuilding(string indexName);
+
+    /// <summary>
+    /// Checks if the specified index is currently being rebuilt.
+    /// </summary>
+    /// <param name="indexName">The name of the index to rebuild.</param>
+    /// <returns>Whether the index is currently being rebuilt.</returns>
+    Task<bool> IsRebuildingAsync(string indexName) =>
+        Task.FromResult(IsRebuilding(indexName));
 }

src/Umbraco.Infrastructure/Services/IndexingRebuilderService.cs

@@ -1,18 +1,27 @@
 using Examine;
 using Microsoft.Extensions.Logging;
+using Umbraco.Cms.Core;
 using Umbraco.Cms.Core.Cache;
 using Umbraco.Cms.Infrastructure.Examine;
+using Umbraco.Cms.Infrastructure.Models;
 
 namespace Umbraco.Cms.Infrastructure.Services;
 
+/// <inheritdoc />
 public class IndexingRebuilderService : IIndexingRebuilderService
 {
-    private const string IsRebuildingIndexRuntimeCacheKeyPrefix = "temp_indexing_op_";
-
-    private readonly IAppPolicyCache _runtimeCache;
     private readonly IIndexRebuilder _indexRebuilder;
     private readonly ILogger<IndexingRebuilderService> _logger;
 
+    public IndexingRebuilderService(
+        IIndexRebuilder indexRebuilder,
+        ILogger<IndexingRebuilderService> logger)
+    {
+        _indexRebuilder = indexRebuilder;
+        _logger = logger;
+    }
+
+    [Obsolete("Use the non-obsolete constructor instead. Scheduled for removal in V19.")]
     public IndexingRebuilderService(
         AppCaches runtimeCache,
         IIndexRebuilder indexRebuilder,
@@ -20,12 +29,18 @@ public class IndexingRebuilderService : IIndexingRebuilderService
     {
         _indexRebuilder = indexRebuilder;
         _logger = logger;
-        _runtimeCache = runtimeCache.RuntimeCache;
     }
 
+    /// <inheritdoc />
     public bool CanRebuild(string indexName) => _indexRebuilder.CanRebuild(indexName);
 
+    /// <inheritdoc />
+    [Obsolete("Use TryRebuildAsync instead. Scheduled for removal in V19.")]
     public bool TryRebuild(IIndex index, string indexName)
+        => TryRebuildAsync(index, indexName).GetAwaiter().GetResult();
+
+    /// <inheritdoc />
+    public async Task<bool> TryRebuildAsync(IIndex index, string indexName)
     {
         // Remove it in case there's a handler there already
         index.IndexOperationComplete -= Indexer_IndexOperationComplete;
@@ -35,11 +50,10 @@ public class IndexingRebuilderService : IIndexingRebuilderService
 
         try
         {
-            Set(indexName);
+            Attempt<IndexRebuildResult> attempt = await _indexRebuilder.RebuildIndexAsync(indexName);
-            _indexRebuilder.RebuildIndex(indexName);
+            return attempt.Success;
-            return true;
         }
-        catch(Exception exception)
+        catch (Exception exception)
         {
             // Ensure it's not listening
             index.IndexOperationComplete -= Indexer_IndexOperationComplete;
@@ -48,25 +62,14 @@ public class IndexingRebuilderService : IIndexingRebuilderService
         }
     }
 
-    private void Set(string indexName)
+    /// <inheritdoc />
-    {
+    [Obsolete("Use IsRebuildingAsync() instead. Scheduled for removal in V19.")]
-        var cacheKey = IsRebuildingIndexRuntimeCacheKeyPrefix + indexName;
-
-        // put temp val in cache which is used as a rudimentary way to know when the indexing is done
-        _runtimeCache.Insert(cacheKey, () => "tempValue", TimeSpan.FromMinutes(5));
-    }
-
-    private void Clear(string? indexName)
-    {
-        var cacheKey = IsRebuildingIndexRuntimeCacheKeyPrefix + indexName;
-        _runtimeCache.Clear(cacheKey);
-    }
-
     public bool IsRebuilding(string indexName)
-    {
+        => IsRebuildingAsync(indexName).GetAwaiter().GetResult();
-        var cacheKey = IsRebuildingIndexRuntimeCacheKeyPrefix + indexName;
+
-        return _runtimeCache.Get(cacheKey) is not null;
+    /// <inheritdoc />
-    }
+    public Task<bool> IsRebuildingAsync(string indexName)
+        => _indexRebuilder.IsRebuildingAsync(indexName);
 
     private void Indexer_IndexOperationComplete(object? sender, EventArgs e)
     {
@@ -80,8 +83,6 @@ public class IndexingRebuilderService : IIndexingRebuilderService
             indexer.IndexOperationComplete -= Indexer_IndexOperationComplete;
         }
 
-        _logger.LogInformation($"Rebuilding index '{indexer?.Name}' done.");
+        _logger.LogInformation("Rebuilding index '{IndexerName}' done.", indexer?.Name);
-
-        Clear(indexer?.Name);
     }
 }

tests/Umbraco.Tests.Integration/DependencyInjection/UmbracoBuilderExtensions.cs

@@ -158,14 +158,14 @@ public static class UmbracoBuilderExtensions
             ILogger<ExamineIndexRebuilder> logger,
             IExamineManager examineManager,
             IEnumerable<IIndexPopulator> populators,
-            IBackgroundTaskQueue backgroundTaskQueue)
+            ILongRunningOperationService longRunningOperationService)
             : base(
             mainDom,
             runtimeState,
             logger,
             examineManager,
             populators,
-            backgroundTaskQueue)
+            longRunningOperationService)
         {
         }