Use new submit and poll solution for examine index rebuild (#19707)

* Started implementing new LongRunningOperationService and adjusting tasks to use this service

This service will manage operations that require status to be synced between servers (load balanced setup).

* Missing migration to add new lock. Other simplifications.

* Add job to cleanup the LongRunningOperations entries

* Add new DatabaseCacheRebuilder.RebuildAsync method

This is both async and returns an attempt, which will fail if a rebuild operation is already running.

* Missing LongRunningOperation database table creation on clean install

* Store expire date in the long running operation. Better handling of non-background operations.

Storing an expiration date allows setting different expiration times depending on the type of operation, and whether it is running in the background or not.

* Added integration tests for LongRunningOperationRepository

* Added unit tests for LongRunningOperationService

* Add type as a parameter to more repository calls. Distinguish between expiration and deletion in `LongRunningOperationRepository.CleanOperations`.

* Fix failing unit test

* Fixed `PerformPublishBranchAsync` result not being deserialized correctly

* Remove unnecessary DatabaseCacheRebuildResult value

* Add status to `LongRunningOperationService.GetResult` attempt to inform on why a result could not be retrieved

* General improvements

* Missing rename

* Improve the handling of long running operations that are not in background and stale operations

* Fix failing unit tests

* Fixed small mismatch between interface and implementation

* Use the new submit and poll functionality for the Examine index rebuild

* Use a fire and forget task instead of the background queue

* Apply suggestions from code review

Co-authored-by: Andy Butland <abutland73@gmail.com>

* Make sure exceptions are caught when running in the background

* Alignment with other repositories (async + pagination)

* Fix build after merge

* Missing obsoletion messages

* Additional fixes

* Add Async suffix to service methods

* Missing adjustment

* Moved hardcoded settings to IOptions

* Fix issue in SQL Server where 0 is not accepted as requested number of rows

* Fix issue in SQL Server where query provided to count cannot contain orderby

* Additional SQL Server fixes

* Update method names

* Adjustments from code review

* Ignoring result of index rebuild in `IndexingNotificationHandler.Language.cs` (same behavior as before)

* Missed some obsoletion messages

---------

Co-authored-by: Andy Butland <abutland73@gmail.com>

src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.Language.cs

@@ -5,7 +5,9 @@ using Umbraco.Cms.Infrastructure.Examine;
 
 namespace Umbraco.Cms.Infrastructure.Search;
 
-public sealed class LanguageIndexingNotificationHandler : INotificationHandler<LanguageCacheRefresherNotification>
+public sealed class LanguageIndexingNotificationHandler :
+    INotificationHandler<LanguageCacheRefresherNotification>,
+    INotificationAsyncHandler<LanguageCacheRefresherNotification>
 {
     private readonly IIndexRebuilder _indexRebuilder;
     private readonly IUmbracoIndexingHandler _umbracoIndexingHandler;
@@ -19,14 +21,20 @@ public sealed class LanguageIndexingNotificationHandler : INotificationHandler<L
         _indexRebuilder = indexRebuilder ?? throw new ArgumentNullException(nameof(indexRebuilder));
     }
 
+    /// <inheritdoc />
+    [Obsolete("Use HandleAsync instead. Scheduled for removal in V19.")]
     public void Handle(LanguageCacheRefresherNotification args)
+        => HandleAsync(args, CancellationToken.None).GetAwaiter().GetResult();
+
+    /// <inheritdoc />
+    public async Task HandleAsync(LanguageCacheRefresherNotification notification, CancellationToken cancellationToken)
     {
         if (!_umbracoIndexingHandler.Enabled)
         {
             return;
         }
 
-        if (!(args.MessageObject is LanguageCacheRefresher.JsonPayload[] payloads))
+        if (notification.MessageObject is not LanguageCacheRefresher.JsonPayload[] payloads)
         {
             return;
         }
@@ -37,14 +45,14 @@ public sealed class LanguageIndexingNotificationHandler : INotificationHandler<L
         }
 
         var removedOrCultureChanged = payloads.Any(x =>
-            x.ChangeType == LanguageCacheRefresher.JsonPayload.LanguageChangeType.ChangeCulture
-            || x.ChangeType == LanguageCacheRefresher.JsonPayload.LanguageChangeType.Remove);
+            x.ChangeType is LanguageCacheRefresher.JsonPayload.LanguageChangeType.ChangeCulture
+                or LanguageCacheRefresher.JsonPayload.LanguageChangeType.Remove);
 
         if (removedOrCultureChanged)
         {
             // if a lang is removed or it's culture has changed, we need to rebuild the indexes since
             // field names and values in the index have a string culture value.
-            _indexRebuilder.RebuildIndexes(false);
+            _ = await _indexRebuilder.RebuildIndexesAsync(false);
         }
     }
 }