Abstract submit and poll operations (#19688)

* Started implementing new LongRunningOperationService and adjusting tasks to use this service

This service will manage operations that require status to be synced between servers (load balanced setup).

* Missing migration to add new lock. Other simplifications.

* Add job to cleanup the LongRunningOperations entries

* Add new DatabaseCacheRebuilder.RebuildAsync method

This is both async and returns an attempt, which will fail if a rebuild operation is already running.

* Missing LongRunningOperation database table creation on clean install

* Store expire date in the long running operation. Better handling of non-background operations.

Storing an expiration date allows setting different expiration times depending on the type of operation, and whether it is running in the background or not.

* Added integration tests for LongRunningOperationRepository

* Added unit tests for LongRunningOperationService

* Add type as a parameter to more repository calls. Distinguish between expiration and deletion in `LongRunningOperationRepository.CleanOperations`.

* Fix failing unit test

* Fixed `PerformPublishBranchAsync` result not being deserialized correctly

* Remove unnecessary DatabaseCacheRebuildResult value

* Add status to `LongRunningOperationService.GetResult` attempt to inform on why a result could not be retrieved

* General improvements

* Missing rename

* Improve the handling of long running operations that are not in background and stale operations

* Fix failing unit tests

* Fixed small mismatch between interface and implementation

* Use a fire and forget task instead of the background queue

* Apply suggestions from code review

Co-authored-by: Andy Butland <abutland73@gmail.com>

* Make sure exceptions are caught when running in the background

* Alignment with other repositories (async + pagination)

* Additional fixes

* Add Async suffix to service methods

* Missing adjustment

* Moved hardcoded settings to IOptions

* Fix issue in SQL Server where 0 is not accepted as requested number of rows

* Fix issue in SQL Server where query provided to count cannot contain orderby

* Additional SQL Server fixes

---------

Co-authored-by: Andy Butland <abutland73@gmail.com>

src/Umbraco.Core/Configuration/Models/LongRunningOperationsCleanupSettings.cs

@@ -0,0 +1,27 @@
+// Copyright (c) Umbraco.
+// See LICENSE for more details.
+
+using System.ComponentModel;
+
+namespace Umbraco.Cms.Core.Configuration.Models;
+
+/// <summary>
+///     Typed configuration options for long-running operations cleanup settings.
+/// </summary>
+public class LongRunningOperationsCleanupSettings
+{
+    private const string StaticPeriod = "00:02:00";
+    private const string StaticMaxAge = "01:00:00";
+
+    /// <summary>
+    /// Gets or sets a value for the period in which long-running operations are cleaned up.
+    /// </summary>
+    [DefaultValue(StaticPeriod)]
+    public TimeSpan Period { get; set; } = TimeSpan.Parse(StaticPeriod);
+
+    /// <summary>
+    /// Gets or sets the maximum time a long-running operation entry can exist, without being updated, before it is considered for cleanup.
+    /// </summary>
+    [DefaultValue(StaticMaxAge)]
+    public TimeSpan MaxEntryAge { get; set; } = TimeSpan.Parse(StaticMaxAge);
+}

src/Umbraco.Core/Configuration/Models/LongRunningOperationsSettings.cs

@@ -0,0 +1,33 @@
+// Copyright (c) Umbraco.
+// See LICENSE for more details.
+
+using System.ComponentModel;
+
+namespace Umbraco.Cms.Core.Configuration.Models;
+
+/// <summary>
+///     Typed configuration options for long-running operations settings.
+/// </summary>
+[UmbracoOptions(Constants.Configuration.ConfigLongRunningOperations)]
+public class LongRunningOperationsSettings
+{
+    private const string StaticExpirationTime = "00:05:00";
+    private const string StaticTimeBetweenStatusChecks = "00:00:10";
+
+    /// <summary>
+    /// Gets or sets the cleanup settings for long-running operations.
+    /// </summary>
+    public LongRunningOperationsCleanupSettings Cleanup { get; set; } = new();
+
+    /// <summary>
+    /// Gets or sets the time after which a long-running operation is considered expired/stale, if not updated.
+    /// </summary>
+    [DefaultValue(StaticExpirationTime)]
+    public TimeSpan ExpirationTime { get; set; } = TimeSpan.Parse(StaticExpirationTime);
+
+    /// <summary>
+    /// Gets or sets the time between status checks for long-running operations.
+    /// </summary>
+    [DefaultValue(StaticTimeBetweenStatusChecks)]
+    public TimeSpan TimeBetweenStatusChecks { get; set; } = TimeSpan.Parse(StaticTimeBetweenStatusChecks);
+}