Load Balancing: Implement distributed background jobs (#20397)

* Start work

* Introduce dto

* Start making repository

* Add migrations

* Implement fetchable first job

* Fix up to also finish tasks

* Refactor jobs to distributed background jobs

* Filter jobs correctly on LastRun

* Hardcode delay

* Add settings to configure delay and period

* Fix formatting

* Add default data

* Add update on startup, which will update periods on startup

* Refactor service to return job directly

* Update src/Umbraco.Infrastructure/Services/Implement/DistributedJobService.cs

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update src/Umbraco.Infrastructure/BackgroundJobs/DistributedBackgroundJobHostedService.cs

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update src/Umbraco.Infrastructure/Migrations/Install/DatabaseDataCreator.cs

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update src/Umbraco.Infrastructure/Migrations/Install/DatabaseDataCreator.cs

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update src/Umbraco.Infrastructure/BackgroundJobs/DistributedBackgroundJobHostedService.cs

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Remove unused

* Move jobs and make internal

* make OpenIddictCleanupJob.cs public, as it is used elsewhere

* Minor docstring changes

* Update src/Umbraco.Core/Persistence/Constants-Locks.cs

Co-authored-by: Mole <nikolajlauridsen@protonmail.ch>

* ´Throw correct exceptions

* Update xml doc

* Remove business logic from repository

* Remove more business logic from repository into service

* Remove adding jobs from migration

* fix creation

* Rename to ExecuteAsync

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: mole <nikolajlauridsen@protonmail.ch>

src/Umbraco.Cms.Api.Common/DependencyInjection/UmbracoBuilderAuthExtensions.cs

@@ -9,8 +9,8 @@ using Umbraco.Cms.Api.Common.Security;
 using Umbraco.Cms.Core;
 using Umbraco.Cms.Core.Configuration.Models;
 using Umbraco.Cms.Core.DependencyInjection;
-using Umbraco.Cms.Infrastructure.BackgroundJobs.Jobs;
+using Umbraco.Cms.Infrastructure.BackgroundJobs;
-using Umbraco.Extensions;
+using Umbraco.Cms.Infrastructure.BackgroundJobs.Jobs.DistributedJobs;
 
 namespace Umbraco.Cms.Api.Common.DependencyInjection;
 
@@ -139,7 +139,7 @@ public static class UmbracoBuilderAuthExtensions
                 });
             });
 
-        builder.Services.AddRecurringBackgroundJob<OpenIddictCleanupJob>();
+        builder.Services.AddSingleton<IDistributedBackgroundJob, OpenIddictCleanupJob>();
         builder.Services.ConfigureOptions<ConfigureOpenIddict>();
     }
 }

src/Umbraco.Cms.Api.Management/DependencyInjection/UmbracoBuilder.BackOffice.cs

@@ -30,7 +30,7 @@ public static partial class UmbracoBuilderExtensions
         .AddMembersIdentity()
         .AddUmbracoProfiler()
         .AddMvcAndRazor(configureMvc)
-        .AddRecurringBackgroundJobs()
+        .AddBackgroundJobs()
         .AddUmbracoHybridCache()
         .AddDistributedCache()
         .AddCoreNotifications();

src/Umbraco.Core/Configuration/Models/DistributedJobSettings.cs

@@ -0,0 +1,25 @@
+using System.ComponentModel;
+
+namespace Umbraco.Cms.Core.Configuration.Models;
+
+/// <summary>
+/// Settings for distributed jobs.
+/// </summary>
+[UmbracoOptions(Constants.Configuration.ConfigDistributedJobs)]
+public class DistributedJobSettings
+{
+    internal const string StaticPeriod = "00:00:10";
+    internal const string StaticDelay = "00:01:00";
+
+    /// <summary>
+    ///     Gets or sets a value for the period of checking if there are any runnable distributed jobs.
+    /// </summary>
+    [DefaultValue(StaticPeriod)]
+    public TimeSpan Period { get; set; } = TimeSpan.Parse(StaticPeriod);
+
+    /// <summary>
+    ///     Gets or sets a value for the delay of when to start checking for distributed jobs.
+    /// </summary>
+    [DefaultValue(StaticDelay)]
+    public TimeSpan Delay { get; set; } = TimeSpan.Parse(StaticDelay);
+}

src/Umbraco.Core/Constants-Configuration.cs

@@ -65,6 +65,7 @@ public static partial class Constants
         public const string ConfigWebhook = ConfigPrefix + "Webhook";
         public const string ConfigWebhookPayloadType = ConfigWebhook + ":PayloadType";
         public const string ConfigCache = ConfigPrefix + "Cache";
+        public const string ConfigDistributedJobs = ConfigPrefix + "DistributedJobs";
 
         public static class NamedOptions
         {

src/Umbraco.Core/DependencyInjection/UmbracoBuilder.Configuration.cs

@@ -87,7 +87,8 @@ public static partial class UmbracoBuilderExtensions
             .AddUmbracoOptions<DataTypesSettings>()
             .AddUmbracoOptions<WebhookSettings>()
             .AddUmbracoOptions<CacheSettings>()
-            .AddUmbracoOptions<SystemDateMigrationSettings>();
+            .AddUmbracoOptions<SystemDateMigrationSettings>()
+            .AddUmbracoOptions<DistributedJobSettings>();
 
         // Configure connection string and ensure it's updated when the configuration changes
         builder.Services.AddSingleton<IConfigureOptions<ConnectionStrings>, ConfigureConnectionStrings>();

src/Umbraco.Core/Persistence/Constants-DatabaseSchema.cs

@@ -103,8 +103,8 @@ public static partial class Constants
             public const string Webhook2Headers = Webhook + "2Headers";
             public const string WebhookLog = Webhook + "Log";
             public const string WebhookRequest = Webhook + "Request";
-
             public const string LongRunningOperation = TableNamePrefix + "LongRunningOperation";
+            public const string DistributedJob = TableNamePrefix + "DistributedJob";
         }
     }
 }

src/Umbraco.Core/Persistence/Constants-Locks.cs

@@ -90,5 +90,10 @@ public static partial class Constants
         ///     All document URLs.
         /// </summary>
         public const int DocumentUrls = -345;
+
+        /// <summary>
+        ///     All distributed jobs.
+        /// </summary>
+        public const int DistributedJobs = -347;
     }
 }

src/Umbraco.Infrastructure/BackgroundJobs/DistributedBackgroundJobHostedService.cs

@@ -0,0 +1,103 @@
+using System.Diagnostics;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using Umbraco.Cms.Core;
+using Umbraco.Cms.Core.Configuration.Models;
+using Umbraco.Cms.Core.Services;
+using Umbraco.Cms.Infrastructure.Services;
+
+namespace Umbraco.Cms.Infrastructure.BackgroundJobs;
+
+/// <summary>
+///    A hosted service that checks for any runnable distributed background jobs on a timer.
+/// </summary>
+public class DistributedBackgroundJobHostedService : BackgroundService
+{
+    private readonly ILogger<DistributedBackgroundJobHostedService> _logger;
+    private readonly IRuntimeState _runtimeState;
+    private readonly IDistributedJobService _distributedJobService;
+    private DistributedJobSettings _distributedJobSettings;
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="DistributedBackgroundJobHostedService"/> class.
+    /// </summary>
+    /// <param name="logger"></param>
+    /// <param name="runtimeState"></param>
+    /// <param name="distributedJobService"></param>
+    /// <param name="distributedJobSettings"></param>
+    public DistributedBackgroundJobHostedService(
+        ILogger<DistributedBackgroundJobHostedService> logger,
+        IRuntimeState runtimeState,
+        IDistributedJobService distributedJobService,
+        IOptionsMonitor<DistributedJobSettings> distributedJobSettings)
+    {
+        _logger = logger;
+        _runtimeState = runtimeState;
+        _distributedJobService = distributedJobService;
+        _distributedJobSettings = distributedJobSettings.CurrentValue;
+        distributedJobSettings.OnChange(options =>
+        {
+            _distributedJobSettings = options;
+        });
+    }
+
+    /// <inheritdoc />
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        await Task.Delay(_distributedJobSettings.Delay, stoppingToken);
+
+        while (_runtimeState.Level != RuntimeLevel.Run)
+        {
+            await Task.Delay(_distributedJobSettings.Delay, stoppingToken);
+        }
+
+        // Update all jobs, periods might have changed when restarting.
+        await _distributedJobService.EnsureJobsAsync();
+
+        using PeriodicTimer timer = new(_distributedJobSettings.Period);
+
+        try
+        {
+            while (await timer.WaitForNextTickAsync(stoppingToken))
+            {
+                await RunRunnableJob();
+            }
+        }
+        catch (OperationCanceledException)
+        {
+            _logger.LogInformation("Timed Hosted Service is stopping.");
+        }
+    }
+
+    private async Task RunRunnableJob()
+    {
+        IDistributedBackgroundJob? job = await _distributedJobService.TryTakeRunnableAsync();
+
+        if (job is null)
+        {
+            // No runnable jobs for now, return
+            return;
+        }
+
+        try
+        {
+            await job.ExecuteAsync();
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "An exception occurred while running distributed background job '{JobName}'.", job.Name);
+        }
+        finally
+        {
+            try
+            {
+                await _distributedJobService.FinishAsync(job.Name);
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "An exception occurred while finishing distributed background job '{JobName}'.", job.Name);
+            }
+        }
+    }
+}

src/Umbraco.Infrastructure/BackgroundJobs/IDistributedBackgroundJob.cs

@@ -0,0 +1,23 @@
+namespace Umbraco.Cms.Infrastructure.BackgroundJobs;
+
+/// <summary>
+/// A background job that will be executed by an available server. With a single server setup this will always be the same.
+/// With a load balanced setup, the executing server might change every time this needs to be executed.
+/// </summary>
+public interface IDistributedBackgroundJob
+{
+    /// <summary>
+    /// Name of the job.
+    /// </summary>
+    string Name { get; }
+
+    /// <summary>
+    /// Timespan representing how often the task should recur.
+    /// </summary>
+    TimeSpan Period { get; }
+
+    /// <summary>
+    /// Run the job.
+    /// </summary>
+    Task ExecuteAsync();
+}

src/Umbraco.Infrastructure/BackgroundJobs/Jobs/DistributedJobs/CacheInstructionsPruningJob.cs

@@ -2,14 +2,13 @@ using Microsoft.Extensions.Options;
 using Umbraco.Cms.Core.Configuration.Models;
 using Umbraco.Cms.Core.Persistence.Repositories;
 using Umbraco.Cms.Core.Scoping;
-using Umbraco.Cms.Infrastructure.Scoping;
 
-namespace Umbraco.Cms.Infrastructure.BackgroundJobs.Jobs;
+namespace Umbraco.Cms.Infrastructure.BackgroundJobs.Jobs.DistributedJobs;
 
 /// <summary>
 /// A background job that prunes cache instructions from the database.
 /// </summary>
-public class CacheInstructionsPruningJob : IRecurringBackgroundJob
+internal class CacheInstructionsPruningJob : IDistributedBackgroundJob
 {
     private readonly IOptions<GlobalSettings> _globalSettings;
     private readonly ICacheInstructionRepository _cacheInstructionRepository;
@@ -36,18 +35,13 @@ public class CacheInstructionsPruningJob : IRecurringBackgroundJob
         Period = globalSettings.Value.DatabaseServerMessenger.TimeBetweenPruneOperations;
     }
 
-    /// <inheritdoc />
+    public string Name => "CacheInstructionsPruningJob";
-    public event EventHandler PeriodChanged
-    {
-        add { }
-        remove { }
-    }
 
     /// <inheritdoc />
     public TimeSpan Period { get; }
 
     /// <inheritdoc />
-    public Task RunJobAsync()
+    public Task ExecuteAsync()
     {
         DateTimeOffset pruneDate = _timeProvider.GetUtcNow() - _globalSettings.Value.DatabaseServerMessenger.TimeToRetainInstructions;
         using (ICoreScope scope = _scopeProvider.CreateCoreScope())

src/Umbraco.Infrastructure/BackgroundJobs/Jobs/DistributedJobs/ContentVersionCleanupJob.cs

@@ -1,24 +1,21 @@
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
-using Umbraco.Cms.Core;
 using Umbraco.Cms.Core.Configuration.Models;
-using Umbraco.Cms.Core.Runtime;
 using Umbraco.Cms.Core.Services;
-using Umbraco.Cms.Core.Sync;
-using Umbraco.Cms.Infrastructure.BackgroundJobs;
 
-namespace Umbraco.Cms.Infrastructure.BackgroundJobs.Jobs;
+namespace Umbraco.Cms.Infrastructure.BackgroundJobs.Jobs.DistributedJobs;
 
 /// <summary>
 ///     Recurring hosted service that executes the content history cleanup.
 /// </summary>
-public class ContentVersionCleanupJob : IRecurringBackgroundJob
+internal class ContentVersionCleanupJob : IDistributedBackgroundJob
 {
+    /// <inheritdoc />
+    public string Name => "ContentVersionCleanupJob";
 
+    /// <inheritdoc />
     public TimeSpan Period { get => TimeSpan.FromHours(1); }
 
-    // No-op event as the period never changes on this job
-    public event EventHandler PeriodChanged { add { } remove { } }
 
     private readonly ILogger<ContentVersionCleanupJob> _logger;
     private readonly IContentVersionService _service;
@@ -39,7 +36,7 @@ public class ContentVersionCleanupJob : IRecurringBackgroundJob
     }
 
     /// <inheritdoc />
-    public Task RunJobAsync()
+    public Task ExecuteAsync()
     {
         // Globally disabled by feature flag
         if (!_settingsMonitor.CurrentValue.ContentVersionCleanupPolicy.EnableCleanup)

src/Umbraco.Infrastructure/BackgroundJobs/Jobs/DistributedJobs/HealthCheckNotifierJob.cs

@@ -1,9 +1,7 @@
 // Copyright (c) Umbraco.
 // See LICENSE for more details.
 
-using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
-using Umbraco.Cms.Core.Configuration;
 using Umbraco.Cms.Core.Configuration.Models;
 using Umbraco.Cms.Core.Events;
 using Umbraco.Cms.Core.HealthChecks;
@@ -12,25 +10,20 @@ using Umbraco.Cms.Core.Logging;
 using Umbraco.Cms.Core.Notifications;
 using Umbraco.Cms.Core.Scoping;
 
-namespace Umbraco.Cms.Infrastructure.BackgroundJobs.Jobs;
+namespace Umbraco.Cms.Infrastructure.BackgroundJobs.Jobs.DistributedJobs;
 
 /// <summary>
 ///     Hosted service implementation for recurring health check notifications.
 /// </summary>
-public class HealthCheckNotifierJob : IRecurringBackgroundJob
+internal class HealthCheckNotifierJob : IDistributedBackgroundJob
 {
-    public TimeSpan Period { get; private set; }
+    /// <inheritdoc />
-    public TimeSpan Delay { get; private set; }
+    public string Name => "HealthCheckNotifierJob";
 
-    private event EventHandler? _periodChanged;
+    /// <inheritdoc/>
-    public event EventHandler PeriodChanged
+    public TimeSpan Period { get; private set; }
-    {
-        add { _periodChanged += value; }
-        remove { _periodChanged -= value; }
-    }
 
     private readonly HealthCheckCollection _healthChecks;
-    private readonly ILogger<HealthCheckNotifierJob> _logger;
     private readonly HealthCheckNotificationMethodCollection _notifications;
     private readonly IProfilingLogger _profilingLogger;
     private readonly IEventAggregator _eventAggregator;
@@ -53,32 +46,28 @@ public class HealthCheckNotifierJob : IRecurringBackgroundJob
         HealthCheckCollection healthChecks,
         HealthCheckNotificationMethodCollection notifications,
         ICoreScopeProvider scopeProvider,
-        ILogger<HealthCheckNotifierJob> logger,
         IProfilingLogger profilingLogger,
-        ICronTabParser cronTabParser,
         IEventAggregator eventAggregator)
     {
         _healthChecksSettings = healthChecksSettings.CurrentValue;
         _healthChecks = healthChecks;
         _notifications = notifications;
         _scopeProvider = scopeProvider;
-        _logger = logger;
         _profilingLogger = profilingLogger;
         _eventAggregator = eventAggregator;
 
         Period = healthChecksSettings.CurrentValue.Notification.Period;
-        Delay = DelayCalculator.GetDelay(healthChecksSettings.CurrentValue.Notification.FirstRunTime, cronTabParser, logger, TimeSpan.FromMinutes(3));
 
 
         healthChecksSettings.OnChange(x =>
         {
             _healthChecksSettings = x;
             Period = x.Notification.Period;
-            _periodChanged?.Invoke(this, EventArgs.Empty);
         });
     }
 
-    public async Task RunJobAsync()
+    /// <inheritdoc/>
+    public async Task ExecuteAsync()
     {
         if (_healthChecksSettings.Notification.Enabled == false)
         {

src/Umbraco.Infrastructure/BackgroundJobs/Jobs/DistributedJobs/LogScrubberJob.cs

@@ -1,17 +1,13 @@
 // Copyright (c) Umbraco.
 // See LICENSE for more details.
 
-using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
-using Umbraco.Cms.Core;
 using Umbraco.Cms.Core.Configuration.Models;
 using Umbraco.Cms.Core.Logging;
-using Umbraco.Cms.Core.Runtime;
 using Umbraco.Cms.Core.Scoping;
 using Umbraco.Cms.Core.Services;
-using Umbraco.Cms.Core.Sync;
 
-namespace Umbraco.Cms.Infrastructure.BackgroundJobs.Jobs;
+namespace Umbraco.Cms.Infrastructure.BackgroundJobs.Jobs.DistributedJobs;
 
 /// <summary>
 ///     Log scrubbing hosted service.
@@ -19,18 +15,18 @@ namespace Umbraco.Cms.Infrastructure.BackgroundJobs.Jobs;
 /// <remarks>
 ///     Will only run on non-replica servers.
 /// </remarks>
-public class LogScrubberJob : IRecurringBackgroundJob
+internal class LogScrubberJob : IDistributedBackgroundJob
 {
     private readonly IAuditService _auditService;
-    private readonly ILogger<LogScrubberJob> _logger;
     private readonly IProfilingLogger _profilingLogger;
     private readonly ICoreScopeProvider _scopeProvider;
     private LoggingSettings _settings;
 
-    public TimeSpan Period => TimeSpan.FromHours(4);
+    /// <inheritdoc />
+    public string Name => "LogScrubberJob";
 
-    // No-op event as the period never changes on this job
+    /// <inheritdoc />
-    public event EventHandler PeriodChanged { add { } remove { } }
+    public TimeSpan Period => TimeSpan.FromHours(4);
 
     /// <summary>
     ///     Initializes a new instance of the <see cref="LogScrubberJob" /> class.
@@ -44,21 +40,20 @@ public class LogScrubberJob : IRecurringBackgroundJob
         IAuditService auditService,
         IOptionsMonitor<LoggingSettings> settings,
         ICoreScopeProvider scopeProvider,
-        ILogger<LogScrubberJob> logger,
         IProfilingLogger profilingLogger)
     {
         _auditService = auditService;
         _settings = settings.CurrentValue;
         _scopeProvider = scopeProvider;
-        _logger = logger;
         _profilingLogger = profilingLogger;
         settings.OnChange(x => _settings = x);
     }
 
-    public async Task RunJobAsync()
+    /// <inheritdoc/>
+    public async Task ExecuteAsync()
     {
         // Ensure we use an explicit scope since we are running on a background thread.
-        using (ICoreScope scope = _scopeProvider.CreateCoreScope())
+        using ICoreScope scope = _scopeProvider.CreateCoreScope();
         using (_profilingLogger.DebugDuration<LogScrubberJob>("Log scrubbing executing", "Log scrubbing complete"))
         {
             await _auditService.CleanLogsAsync((int)_settings.MaxLogAge.TotalMinutes);

src/Umbraco.Infrastructure/BackgroundJobs/Jobs/DistributedJobs/LongRunningOperationsCleanupJob.cs

@@ -3,12 +3,12 @@ using Umbraco.Cms.Core.Configuration.Models;
 using Umbraco.Cms.Core.Persistence.Repositories;
 using Umbraco.Cms.Core.Scoping;
 
-namespace Umbraco.Cms.Infrastructure.BackgroundJobs.Jobs;
+namespace Umbraco.Cms.Infrastructure.BackgroundJobs.Jobs.DistributedJobs;
 
 /// <summary>
 /// Cleans up long-running operations that have exceeded a specified age.
 /// </summary>
-public class LongRunningOperationsCleanupJob : IRecurringBackgroundJob
+internal class LongRunningOperationsCleanupJob : IDistributedBackgroundJob
 {
     private readonly ICoreScopeProvider _scopeProvider;
     private readonly ILongRunningOperationRepository _longRunningOperationRepository;
@@ -36,20 +36,13 @@ public class LongRunningOperationsCleanupJob : IRecurringBackgroundJob
     }
 
     /// <inheritdoc />
-    public event EventHandler? PeriodChanged
+    public string Name => "LongRunningOperationsCleanupJob";
-    {
-        add { }
-        remove { }
-    }
 
     /// <inheritdoc />
     public TimeSpan Period { get; }
 
-    /// <inheritdoc/>
-    public TimeSpan Delay { get; } = TimeSpan.FromSeconds(10);
-
     /// <inheritdoc />
-    public async Task RunJobAsync()
+    public async Task ExecuteAsync()
     {
         using ICoreScope scope = _scopeProvider.CreateCoreScope();
         await _longRunningOperationRepository.CleanOperationsAsync(_timeProvider.GetUtcNow() - _maxEntryAge);

src/Umbraco.Infrastructure/BackgroundJobs/Jobs/DistributedJobs/OpenIddictCleanupJob.cs

@@ -1,18 +1,20 @@
-using System.Configuration;
+using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using OpenIddict.Abstractions;
 
-namespace Umbraco.Cms.Infrastructure.BackgroundJobs.Jobs;
+namespace Umbraco.Cms.Infrastructure.BackgroundJobs.Jobs.DistributedJobs;
 
-// port of the OpenIddict Quartz job for cleaning up - see https://github.com/openiddict/openiddict-core/tree/dev/src/OpenIddict.Quartz
+
-public class OpenIddictCleanupJob : IRecurringBackgroundJob
+/// <summary>
+/// Port of the OpenIddict Quartz job for cleaning up - see https://github.com/openiddict/openiddict-core/tree/dev/src/OpenIddict.Quartz
+/// </summary>
+public class OpenIddictCleanupJob : IDistributedBackgroundJob
 {
-    public TimeSpan Period { get => TimeSpan.FromHours(1); }
+    /// <inheritdoc />
-    public TimeSpan Delay { get => TimeSpan.FromMinutes(5); }
+    public string Name => "OpenIddictCleanupJob";
 
-    // No-op event as the period never changes on this job
+    /// <inheritdoc />
-    public event EventHandler PeriodChanged { add { } remove { } }
+    public TimeSpan Period => TimeSpan.FromHours(1);
 
 
     // keep tokens and authorizations in the database for 7 days
@@ -22,13 +24,19 @@ public class OpenIddictCleanupJob : IRecurringBackgroundJob
     private readonly ILogger<OpenIddictCleanupJob> _logger;
     private readonly IServiceProvider _provider;
 
+    /// <summary>
+    /// Initializes a new instance of the <see cref="OpenIddictCleanupJob"/> class.
+    /// </summary>
+    /// <param name="logger"></param>
+    /// <param name="provider"></param>
     public OpenIddictCleanupJob(ILogger<OpenIddictCleanupJob> logger, IServiceProvider provider)
     {
         _logger = logger;
         _provider = provider;
     }
 
-    public async Task RunJobAsync()
+    /// <inheritdoc />
+    public async Task ExecuteAsync()
     {
         // hosted services are registered as singletons, but this particular one consumes scoped services... so
         // we have to fetch the service dependencies manually using a new scope per invocation.

src/Umbraco.Infrastructure/BackgroundJobs/Jobs/DistributedJobs/ScheduledPublishingJob.cs

@@ -3,13 +3,12 @@
 
 using Microsoft.Extensions.Logging;
 using Umbraco.Cms.Core;
-using Umbraco.Cms.Core.Runtime;
 using Umbraco.Cms.Core.Scoping;
 using Umbraco.Cms.Core.Services;
 using Umbraco.Cms.Core.Sync;
 using Umbraco.Cms.Core.Web;
 
-namespace Umbraco.Cms.Infrastructure.BackgroundJobs.Jobs;
+namespace Umbraco.Cms.Infrastructure.BackgroundJobs.Jobs.DistributedJobs;
 
 /// <summary>
 ///     Hosted service implementation for scheduled publishing feature.
@@ -17,11 +16,13 @@ namespace Umbraco.Cms.Infrastructure.BackgroundJobs.Jobs;
 /// <remarks>
 ///     Runs only on non-replica servers.
 /// </remarks>
-public class ScheduledPublishingJob : IRecurringBackgroundJob
+internal class ScheduledPublishingJob : IDistributedBackgroundJob
 {
-    public TimeSpan Period { get => TimeSpan.FromMinutes(1); }
+    /// <inheritdoc />
-    // No-op event as the period never changes on this job
+    public string Name => "ScheduledPublishingJob";
-    public event EventHandler PeriodChanged { add { } remove { } }
+
+    /// <inheritdoc />
+    public TimeSpan Period => TimeSpan.FromMinutes(1);
 
 
     private readonly IContentService _contentService;
@@ -50,7 +51,8 @@ public class ScheduledPublishingJob : IRecurringBackgroundJob
         _timeProvider = timeProvider;
     }
 
-    public Task RunJobAsync()
+    /// <inheritdoc />
+    public Task ExecuteAsync()
     {
         if (Suspendable.ScheduledPublishing.CanRun == false)
         {

src/Umbraco.Infrastructure/BackgroundJobs/Jobs/DistributedJobs/TemporaryFileCleanupJob.cs

@@ -1,15 +1,18 @@
 using Microsoft.Extensions.Logging;
 using Umbraco.Cms.Core.Services;
 
-namespace Umbraco.Cms.Infrastructure.BackgroundJobs.Jobs;
+namespace Umbraco.Cms.Infrastructure.BackgroundJobs.Jobs.DistributedJobs;
 
-public class TemporaryFileCleanupJob : IRecurringBackgroundJob
+/// <summary>
+/// Cleans up temporary media files.
+/// </summary>
+internal class TemporaryFileCleanupJob : IDistributedBackgroundJob
 {
-    public TimeSpan Period { get => TimeSpan.FromMinutes(5); }
+    /// <inheritdoc />
-    public TimeSpan Delay { get => TimeSpan.FromMinutes(5); }
+    public string Name => "TemporaryFileCleanupJob";
 
-    // No-op event as the period never changes on this job
+    /// <inheritdoc />
-    public event EventHandler PeriodChanged { add { } remove { } }
+    public TimeSpan Period => TimeSpan.FromMinutes(5);
 
     private readonly ILogger<TemporaryFileCleanupJob> _logger;
     private readonly ITemporaryFileService _service;
@@ -26,11 +29,9 @@ public class TemporaryFileCleanupJob : IRecurringBackgroundJob
         _service = temporaryFileService;
     }
 
-    /// <summary>
+
-    /// Runs the background task to send the anonymous ID
+    /// <inheritdoc />
-    /// to telemetry service
+    public  async Task ExecuteAsync()
-    /// </summary>
-    public  async Task RunJobAsync()
     {
         var count = (await _service.CleanUpOldTempFiles()).Count();
 

src/Umbraco.Infrastructure/BackgroundJobs/Jobs/DistributedJobs/WebhookFiring.cs

@@ -8,9 +8,12 @@ using Umbraco.Cms.Core.Models;
 using Umbraco.Cms.Core.Scoping;
 using Umbraco.Cms.Core.Services;
 
-namespace Umbraco.Cms.Infrastructure.BackgroundJobs.Jobs;
+namespace Umbraco.Cms.Infrastructure.BackgroundJobs.Jobs.DistributedJobs;
 
-public class WebhookFiring : IRecurringBackgroundJob
+/// <summary>
+/// Fires pending webhooks.
+/// </summary>
+internal class WebhookFiring : IDistributedBackgroundJob
 {
     private readonly ILogger<WebhookFiring> _logger;
     private readonly IWebhookRequestService _webhookRequestService;
@@ -21,13 +24,23 @@ public class WebhookFiring : IRecurringBackgroundJob
     private readonly IHttpClientFactory _httpClientFactory;
     private WebhookSettings _webhookSettings;
 
+    /// <inheritdoc />
+    public string Name => "WebhookFiring";
+
+    /// <inheritdoc />
     public TimeSpan Period => _webhookSettings.Period;
 
-    public TimeSpan Delay { get; } = TimeSpan.FromSeconds(20);
+    /// <summary>
-
+    /// Initializes a new instance of the <see cref="WebhookFiring"/> class.
-    // No-op event as the period never changes on this job
+    /// </summary>
-    public event EventHandler PeriodChanged { add { } remove { } }
+    /// <param name="logger"></param>
-
+    /// <param name="webhookRequestService"></param>
+    /// <param name="webhookLogFactory"></param>
+    /// <param name="webhookLogService"></param>
+    /// <param name="webHookService"></param>
+    /// <param name="webhookSettings"></param>
+    /// <param name="coreScopeProvider"></param>
+    /// <param name="httpClientFactory"></param>
     public WebhookFiring(
         ILogger<WebhookFiring> logger,
         IWebhookRequestService webhookRequestService,
@@ -49,7 +62,8 @@ public class WebhookFiring : IRecurringBackgroundJob
         webhookSettings.OnChange(x => _webhookSettings = x);
     }
 
-    public async Task RunJobAsync()
+    /// <inheritdoc />
+    public async Task ExecuteAsync()
     {
         if (_webhookSettings.Enabled is false)
         {

src/Umbraco.Infrastructure/BackgroundJobs/Jobs/DistributedJobs/WebhookLoggingCleanup.cs

@@ -7,18 +7,25 @@ using Umbraco.Cms.Core.Persistence.Repositories;
 using Umbraco.Cms.Core.Scoping;
 using Umbraco.Extensions;
 
-namespace Umbraco.Cms.Infrastructure.BackgroundJobs.Jobs;
+namespace Umbraco.Cms.Infrastructure.BackgroundJobs.Jobs.DistributedJobs;
 
 /// <summary>
 /// Daily background job that removes all webhook log data older than x days as defined by <see cref="WebhookSettings.KeepLogsForDays"/>
 /// </summary>
-public class WebhookLoggingCleanup : IRecurringBackgroundJob
+internal class WebhookLoggingCleanup : IDistributedBackgroundJob
 {
     private readonly ILogger<WebhookLoggingCleanup> _logger;
     private readonly WebhookSettings _webhookSettings;
     private readonly IWebhookLogRepository _webhookLogRepository;
     private readonly ICoreScopeProvider _coreScopeProvider;
 
+    /// <summary>
+    /// Initializes a new instance of the <see cref="WebhookLoggingCleanup"/> class.
+    /// </summary>
+    /// <param name="logger"></param>
+    /// <param name="webhookSettings"></param>
+    /// <param name="webhookLogRepository"></param>
+    /// <param name="coreScopeProvider"></param>
     public WebhookLoggingCleanup(ILogger<WebhookLoggingCleanup> logger, IOptionsMonitor<WebhookSettings> webhookSettings, IWebhookLogRepository webhookLogRepository, ICoreScopeProvider coreScopeProvider)
     {
         _logger = logger;
@@ -28,20 +35,13 @@ public class WebhookLoggingCleanup : IRecurringBackgroundJob
     }
 
     /// <inheritdoc />
-    // No-op event as the period never changes on this job
+    public string Name => "WebhookLoggingCleanup";
-    public event EventHandler PeriodChanged
-    {
-        add { } remove { }
-    }
 
     /// <inheritdoc />
     public TimeSpan Period => TimeSpan.FromDays(1);
 
     /// <inheritdoc />
-    public TimeSpan Delay { get; } = TimeSpan.FromSeconds(20);
+    public async Task ExecuteAsync()
-
-    /// <inheritdoc />
-    public async Task RunJobAsync()
     {
         if (_webhookSettings.EnableLoggingCleanup is false)
         {

src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.BackgroundJobs.cs

@@ -0,0 +1,46 @@
+using Microsoft.Extensions.DependencyInjection;
+using Umbraco.Cms.Core.DependencyInjection;
+using Umbraco.Cms.Core.Notifications;
+using Umbraco.Cms.Core.Services.Navigation;
+using Umbraco.Cms.Infrastructure.BackgroundJobs;
+using Umbraco.Cms.Infrastructure.BackgroundJobs.Jobs;
+using Umbraco.Cms.Infrastructure.BackgroundJobs.Jobs.DistributedJobs;
+using Umbraco.Cms.Infrastructure.BackgroundJobs.Jobs.ServerRegistration;
+using Umbraco.Cms.Infrastructure.HostedServices;
+using Umbraco.Extensions;
+
+namespace Umbraco.Cms.Infrastructure.DependencyInjection;
+
+public static partial class UmbracoBuilderExtensions
+{
+    /// <summary>
+    /// Add Umbraco background jobs
+    /// </summary>
+    public static IUmbracoBuilder AddBackgroundJobs(this IUmbracoBuilder builder)
+    {
+        // Add background jobs
+        builder.Services.AddRecurringBackgroundJob<TempFileCleanupJob>();
+        builder.Services.AddRecurringBackgroundJob<InstructionProcessJob>();
+        builder.Services.AddRecurringBackgroundJob<TouchServerJob>();
+        builder.Services.AddRecurringBackgroundJob<ReportSiteJob>();
+
+        builder.Services.AddSingleton<IDistributedBackgroundJob, WebhookFiring>();
+        builder.Services.AddSingleton<IDistributedBackgroundJob, ContentVersionCleanupJob>();
+        builder.Services.AddSingleton<IDistributedBackgroundJob, HealthCheckNotifierJob>();
+        builder.Services.AddSingleton<IDistributedBackgroundJob, LogScrubberJob>();
+        builder.Services.AddSingleton<IDistributedBackgroundJob, ScheduledPublishingJob>();
+        builder.Services.AddSingleton<IDistributedBackgroundJob, TemporaryFileCleanupJob>();
+        builder.Services.AddSingleton<IDistributedBackgroundJob, WebhookLoggingCleanup>();
+        builder.Services.AddSingleton<IDistributedBackgroundJob, CacheInstructionsPruningJob>();
+        builder.Services.AddSingleton<IDistributedBackgroundJob, LongRunningOperationsCleanupJob>();
+        builder.Services.AddHostedService<DistributedBackgroundJobHostedService>();
+
+        builder.Services.AddSingleton(RecurringBackgroundJobHostedService.CreateHostedServiceFactory);
+        builder.Services.AddHostedService<RecurringBackgroundJobHostedServiceRunner>();
+        builder.Services.AddHostedService<QueuedHostedService>();
+        builder.AddNotificationAsyncHandler<PostRuntimePremigrationsUpgradeNotification, NavigationInitializationNotificationHandler>();
+        builder.AddNotificationAsyncHandler<PostRuntimePremigrationsUpgradeNotification, PublishStatusInitializationNotificationHandler>();
+
+        return builder;
+    }
+}

src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.Repositories.cs

@@ -83,6 +83,7 @@ public static partial class UmbracoBuilderExtensions
         builder.Services.AddUnique<INavigationRepository, ContentNavigationRepository>();
         builder.Services.AddUnique<IPublishStatusRepository, PublishStatusRepository>();
         builder.Services.AddUnique<ILongRunningOperationRepository, LongRunningOperationRepository>();
+        builder.Services.AddUnique<IDistributedJobRepository, DistributedJobRepository>();
 
         return builder;
     }

src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.Services.cs

@@ -84,6 +84,7 @@ public static partial class UmbracoBuilderExtensions
         builder.Services.TryAddTransient<IReservedFieldNamesService, ReservedFieldNamesService>();
         builder.Services.AddUnique<IContentSearchService, ContentSearchService>();
         builder.Services.AddUnique<IMediaSearchService, MediaSearchService>();
+        builder.Services.AddUnique<IDistributedJobService, DistributedJobService>();
 
         return builder;
     }

src/Umbraco.Infrastructure/Migrations/Install/DatabaseDataCreator.cs

@@ -8,6 +8,7 @@ using Umbraco.Cms.Core.Actions;
 using Umbraco.Cms.Core.Configuration;
 using Umbraco.Cms.Core.Configuration.Models;
 using Umbraco.Cms.Core.Models;
+using Umbraco.Cms.Infrastructure.BackgroundJobs;
 using Umbraco.Cms.Infrastructure.Migrations.Upgrade;
 using Umbraco.Cms.Infrastructure.Persistence;
 using Umbraco.Cms.Infrastructure.Persistence.Dtos;
@@ -1077,6 +1078,8 @@ internal sealed class DatabaseDataCreator
         _database.Insert(Constants.DatabaseSchema.Tables.Lock, "id", false, new LockDto { Id = Constants.Locks.WebhookLogs, Name = "WebhookLogs" });
         _database.Insert(Constants.DatabaseSchema.Tables.Lock, "id", false, new LockDto { Id = Constants.Locks.LongRunningOperations, Name = "LongRunningOperations" });
         _database.Insert(Constants.DatabaseSchema.Tables.Lock, "id", false, new LockDto { Id = Constants.Locks.DocumentUrls, Name = "DocumentUrls" });
+        _database.Insert(Constants.DatabaseSchema.Tables.Lock, "id", false, new LockDto { Id = Constants.Locks.DistributedJobs, Name = "DistributedJobs" });
+
     }
 
     private void CreateContentTypeData()

src/Umbraco.Infrastructure/Migrations/Install/DatabaseSchemaCreator.cs

@@ -91,6 +91,7 @@ public class DatabaseSchemaCreator
         typeof(WebhookRequestDto),
         typeof(UserDataDto),
         typeof(LongRunningOperationDto),
+        typeof(DistributedJobDto),
     };
 
     private readonly IUmbracoDatabase _database;

src/Umbraco.Infrastructure/Migrations/Upgrade/UmbracoPlan.cs

@@ -136,5 +136,6 @@ public class UmbracoPlan : MigrationPlan
         To<V_17_0_0.MigrateCheckboxListDataTypesAndPropertyData>("{EB1E50B7-CD5E-4B6B-B307-36237DD2C506}");
         To<V_17_0_0.SetDateDefaultsToUtcNow>("{1847C7FF-B021-44EB-BEB0-A77A4376A6F2}");
         To<V_17_0_0.MigrateSystemDatesToUtc>("{7208B20D-6BFC-472E-9374-85EEA817B27D}");
+        To<V_17_0_0.AddDistributedJobLock>("{263075BF-F18A-480D-92B4-4947D2EAB772}");
     }
 }

src/Umbraco.Infrastructure/Migrations/Upgrade/V_17_0_0/AddDistributedJobLock.cs

@@ -0,0 +1,51 @@
+using NPoco;
+using Umbraco.Cms.Core;
+using Umbraco.Cms.Infrastructure.BackgroundJobs;
+using Umbraco.Cms.Infrastructure.Persistence;
+using Umbraco.Cms.Infrastructure.Persistence.Dtos;
+using Umbraco.Extensions;
+
+namespace Umbraco.Cms.Infrastructure.Migrations.Upgrade.V_17_0_0;
+
+/// <summary>
+/// Adds all the distributed jobs to the database.
+/// </summary>
+public class AddDistributedJobLock : AsyncMigrationBase
+{
+    private readonly IEnumerable<IDistributedBackgroundJob> _distributedBackgroundJobs;
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="V_17_0_0.AddDistributedJobLock"/> class.
+    /// </summary>
+    /// <param name="context"></param>
+    /// <param name="distributedBackgroundJobs"></param>
+    public AddDistributedJobLock(IMigrationContext context, IEnumerable<IDistributedBackgroundJob> distributedBackgroundJobs)
+        : base(context) => _distributedBackgroundJobs = distributedBackgroundJobs;
+
+    /// <inheritdoc />
+    protected override Task MigrateAsync()
+    {
+        if (!TableExists(Constants.DatabaseSchema.Tables.DistributedJob))
+        {
+            Create.Table<DistributedJobDto>().Do();
+        }
+
+        if (!TableExists(Constants.DatabaseSchema.Tables.Lock))
+        {
+            Create.Table<LockDto>().Do();
+        }
+
+        Sql<ISqlContext> sql = Database.SqlContext.Sql()
+            .Select<LockDto>()
+            .From<LockDto>()
+            .Where<LockDto>(x => x.Id == Constants.Locks.DistributedJobs);
+
+        LockDto? existingLockDto = Database.FirstOrDefault<LockDto>(sql);
+        if (existingLockDto is null)
+        {
+            Database.Insert(Constants.DatabaseSchema.Tables.Lock, "id", false, new LockDto { Id = Constants.Locks.DistributedJobs, Name = "DistributedJobs" });
+        }
+
+        return Task.CompletedTask;
+    }
+}

src/Umbraco.Infrastructure/Models/DistributedBackgroundJobModel.cs

@@ -0,0 +1,32 @@
+namespace Umbraco.Cms.Infrastructure.Models;
+
+/// <summary>
+/// Model for distributed background jobs.
+/// </summary>
+public class DistributedBackgroundJobModel
+{
+    /// <summary>
+    /// Name of job.
+    /// </summary>
+    public required string Name { get; init; }
+
+    /// <summary>
+    /// Period of job.
+    /// </summary>
+    public TimeSpan Period { get; set; }
+
+    /// <summary>
+    /// Time of last run.
+    /// </summary>
+    public DateTime LastRun { get; set; }
+
+    /// <summary>
+    /// If the job is running.
+    /// </summary>
+    public bool IsRunning { get; set; }
+
+    /// <summary>
+    /// Time of last attempted run.
+    /// </summary>
+    public DateTime LastAttemptedRun { get; set; }
+}

src/Umbraco.Infrastructure/Persistence/Dtos/DistributedJobDto.cs

@@ -0,0 +1,36 @@
+using NPoco;
+using Umbraco.Cms.Core;
+using Umbraco.Cms.Infrastructure.Persistence.DatabaseAnnotations;
+using Umbraco.Cms.Infrastructure.Persistence.DatabaseModelDefinitions;
+
+namespace Umbraco.Cms.Infrastructure.Persistence.Dtos;
+
+[TableName(TableName)]
+[PrimaryKey("id", AutoIncrement = true)]
+[ExplicitColumns]
+internal sealed class DistributedJobDto
+{
+    public const string TableName = Constants.DatabaseSchema.Tables.DistributedJob;
+
+    [Column("id")]
+    [PrimaryKeyColumn(AutoIncrement = true)]
+    public int Id { get; set; }
+
+    [Column("Name")]
+    [NullSetting(NullSetting = NullSettings.NotNull)]
+    public required string Name { get; set; }
+
+    [Column("lastRun")]
+    [Constraint(Default = SystemMethods.CurrentUTCDateTime)]
+    public DateTime LastRun { get; set; }
+
+    [Column("period")]
+    public long Period { get; set; }
+
+    [Column("IsRunning")]
+    public bool IsRunning { get; set; }
+
+    [Column("lastAttemptedRun")]
+    [Constraint(Default = SystemMethods.CurrentUTCDateTime)]
+    public DateTime LastAttemptedRun { get; set; }
+}

src/Umbraco.Infrastructure/Persistence/Repositories/IDistributedJobRepository.cs

@@ -0,0 +1,36 @@
+using Umbraco.Cms.Infrastructure.Models;
+
+namespace Umbraco.Cms.Infrastructure.Persistence.Repositories;
+
+/// <summary>
+///   Defines a repository for managing distributed jobs.
+/// </summary>
+public interface IDistributedJobRepository
+{
+    /// <summary>
+    /// Gets a job by name.
+    /// </summary>
+    /// <returns></returns>
+    DistributedBackgroundJobModel? GetByName(string jobName);
+
+    /// <summary>
+    /// Gets all jobs.
+    /// </summary>
+    /// <returns></returns>
+    IEnumerable<DistributedBackgroundJobModel> GetAll();
+
+    /// <summary>
+    /// Updates a job.
+    /// </summary>
+    void Update(DistributedBackgroundJobModel distributedBackgroundJob);
+
+    /// <summary>
+    /// Adds a job.
+    /// </summary>
+    void Add(DistributedBackgroundJobModel distributedBackgroundJob);
+
+    /// <summary>
+    /// Deletes a job.
+    /// </summary>
+    void Delete(DistributedBackgroundJobModel distributedBackgroundJob);
+}

src/Umbraco.Infrastructure/Persistence/Repositories/Implement/DistributedJobRepository.cs

@@ -0,0 +1,109 @@
+using NPoco;
+using Umbraco.Cms.Core.Exceptions;
+using Umbraco.Cms.Infrastructure.Models;
+using Umbraco.Cms.Infrastructure.Persistence.Dtos;
+using Umbraco.Cms.Infrastructure.Scoping;
+using Umbraco.Extensions;
+
+namespace Umbraco.Cms.Infrastructure.Persistence.Repositories.Implement;
+
+/// <inheritdoc />
+internal class DistributedJobRepository(IScopeAccessor scopeAccessor) : IDistributedJobRepository
+{
+    /// <inheritdoc/>
+    public DistributedBackgroundJobModel? GetByName(string jobName)
+    {
+        if (scopeAccessor.AmbientScope is null)
+        {
+            throw new InvalidOperationException("No scope, could not get distributed jobs");
+        }
+
+        Sql<ISqlContext> sql = scopeAccessor.AmbientScope.SqlContext.Sql()
+            .Select<DistributedJobDto>()
+            .From<DistributedJobDto>()
+            .Where<DistributedJobDto>(x => x.Name == jobName);
+
+        DistributedJobDto? dto = scopeAccessor.AmbientScope.Database.FirstOrDefault<DistributedJobDto>(sql);
+        return dto is null ? null : MapFromDto(dto);
+    }
+
+    /// <inheritdoc/>
+    public IEnumerable<DistributedBackgroundJobModel> GetAll()
+    {
+        if (scopeAccessor.AmbientScope is null)
+        {
+            throw new InvalidOperationException("No scope, could not get distributed jobs");
+        }
+
+        Sql<ISqlContext> sql = scopeAccessor.AmbientScope.SqlContext.Sql()
+            .Select<DistributedJobDto>()
+            .From<DistributedJobDto>();
+
+        IUmbracoDatabase database = scopeAccessor.AmbientScope.Database;
+        List<DistributedJobDto> jobs = database.Fetch<DistributedJobDto>(sql);
+        return jobs.Select(MapFromDto);
+    }
+
+    /// <inheritdoc/>
+    public void Update(DistributedBackgroundJobModel distributedBackgroundJob)
+    {
+        if (scopeAccessor.AmbientScope is null)
+        {
+            return;
+        }
+
+        DistributedJobDto dto = MapToDto(distributedBackgroundJob);
+
+        scopeAccessor.AmbientScope.Database.Update(dto);
+    }
+
+    /// <inheritdoc/>
+    public void Add(DistributedBackgroundJobModel distributedBackgroundJob)
+    {
+        if (scopeAccessor.AmbientScope is null)
+        {
+            throw new InvalidOperationException("No scope, could not add distributed job");
+        }
+
+        DistributedJobDto dto = MapToDto(distributedBackgroundJob);
+
+        scopeAccessor.AmbientScope.Database.Insert(dto);
+    }
+
+    /// <inheritdoc/>
+    public void Delete(DistributedBackgroundJobModel distributedBackgroundJob)
+    {
+        if (scopeAccessor.AmbientScope is null)
+        {
+            throw new InvalidOperationException("No scope, could not delete distributed job");
+        }
+
+        DistributedJobDto dto = MapToDto(distributedBackgroundJob);
+
+        int rowsAffected = scopeAccessor.AmbientScope.Database.Delete(dto);
+        if (rowsAffected == 0)
+        {
+            throw new InvalidOperationException("Could not delete distributed job, it may have already been deleted");
+        }
+    }
+
+    private DistributedJobDto MapToDto(DistributedBackgroundJobModel model) =>
+        new()
+        {
+            Name = model.Name,
+            Period = model.Period.Ticks,
+            LastRun = model.LastRun,
+            IsRunning = model.IsRunning,
+            LastAttemptedRun = model.LastAttemptedRun,
+        };
+
+    private DistributedBackgroundJobModel MapFromDto(DistributedJobDto jobDto) =>
+        new()
+        {
+            Name = jobDto.Name,
+            Period = TimeSpan.FromTicks(jobDto.Period),
+            LastRun = jobDto.LastRun,
+            IsRunning = jobDto.IsRunning,
+            LastAttemptedRun = jobDto.LastAttemptedRun,
+        };
+}

src/Umbraco.Infrastructure/Services/IDistributedJobService.cs

@@ -0,0 +1,35 @@
+using Umbraco.Cms.Infrastructure.BackgroundJobs;
+
+namespace Umbraco.Cms.Infrastructure.Services;
+
+/// <summary>
+/// Service for managing distributed jobs.
+/// </summary>
+public interface IDistributedJobService
+{
+    /// <summary>
+    /// Attempts to claim a runnable distributed job for execution.
+    /// </summary>
+    /// <returns>
+    /// The claimed <see cref="IDistributedBackgroundJob"/> if available, or <see langword="null"/> if no jobs are ready to run.
+    /// </returns>
+    Task<IDistributedBackgroundJob?> TryTakeRunnableAsync();
+
+    /// <summary>
+    /// Finishes a job.
+    /// </summary>
+    Task FinishAsync(string jobName);
+
+    /// <summary>
+    /// Ensures all distributed jobs are registered in the database on startup.
+    /// </summary>
+    /// <remarks>
+    /// This method handles two scenarios:
+    /// <list type="bullet">
+    /// <item><description>Fresh install: Adds all registered jobs to the database</description></item>
+    /// <item><description>Restart: Updates existing jobs where periods have changed and adds any new jobs</description></item>
+    /// </list>
+    /// Jobs that exist in the database but are no longer registered in code will be removed.
+    /// </remarks>
+    Task EnsureJobsAsync();
+}

src/Umbraco.Infrastructure/Services/Implement/DistributedJobService.cs

@@ -0,0 +1,139 @@
+using Microsoft.Extensions.Logging;
+using Umbraco.Cms.Core;
+using Umbraco.Cms.Core.Scoping;
+using Umbraco.Cms.Infrastructure.BackgroundJobs;
+using Umbraco.Cms.Infrastructure.Models;
+using Umbraco.Cms.Infrastructure.Persistence.Repositories;
+
+namespace Umbraco.Cms.Infrastructure.Services.Implement;
+
+/// <inheritdoc />
+public class DistributedJobService : IDistributedJobService
+{
+    private readonly ICoreScopeProvider _coreScopeProvider;
+    private readonly IDistributedJobRepository _distributedJobRepository;
+    private readonly IEnumerable<IDistributedBackgroundJob> _distributedBackgroundJobs;
+    private readonly ILogger<DistributedJobService> _logger;
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="DistributedJobService"/> class.
+    /// </summary>
+    /// <param name="coreScopeProvider"></param>
+    /// <param name="distributedJobRepository"></param>
+    /// <param name="distributedBackgroundJobs"></param>
+    /// <param name="logger"></param>
+    public DistributedJobService(
+        ICoreScopeProvider coreScopeProvider,
+        IDistributedJobRepository distributedJobRepository,
+        IEnumerable<IDistributedBackgroundJob> distributedBackgroundJobs,
+        ILogger<DistributedJobService> logger)
+    {
+        _coreScopeProvider = coreScopeProvider;
+        _distributedJobRepository = distributedJobRepository;
+        _distributedBackgroundJobs = distributedBackgroundJobs;
+        _logger = logger;
+    }
+
+    /// <inheritdoc />
+    public async Task<IDistributedBackgroundJob?> TryTakeRunnableAsync()
+    {
+        using ICoreScope scope = _coreScopeProvider.CreateCoreScope();
+
+        scope.EagerWriteLock(Constants.Locks.DistributedJobs);
+
+        IEnumerable<DistributedBackgroundJobModel> jobs = _distributedJobRepository.GetAll();
+        DistributedBackgroundJobModel? job = jobs.FirstOrDefault(x => x.LastRun < DateTime.UtcNow - x.Period);
+
+        if (job is null)
+        {
+            // No runnable jobs for now.
+            return null;
+        }
+
+        job.LastAttemptedRun = DateTime.UtcNow;
+        job.IsRunning = true;
+        _distributedJobRepository.Update(job);
+
+        IDistributedBackgroundJob? distributedJob = _distributedBackgroundJobs.FirstOrDefault(x => x.Name == job.Name);
+
+        if (distributedJob is null)
+        {
+            _logger.LogWarning("Could not find a distributed job with the name '{JobName}'", job.Name);
+        }
+
+        scope.Complete();
+
+        return distributedJob;
+    }
+
+    /// <inheritdoc />
+    public async Task FinishAsync(string jobName)
+    {
+        using ICoreScope scope = _coreScopeProvider.CreateCoreScope();
+
+        scope.EagerWriteLock(Constants.Locks.DistributedJobs);
+        DistributedBackgroundJobModel? job = _distributedJobRepository.GetByName(jobName);
+
+        if (job is null)
+        {
+            _logger.LogWarning("Could not finish a distributed job with the name '{JobName}'", jobName);
+            return;
+        }
+
+        DateTime currentDateTime = DateTime.UtcNow;
+        job.LastAttemptedRun = currentDateTime;
+        job.LastRun = currentDateTime;
+        job.IsRunning = false;
+        _distributedJobRepository.Update(job);
+
+        scope.Complete();
+    }
+
+
+    /// <inheritdoc />
+    public async Task EnsureJobsAsync()
+    {
+        using ICoreScope scope = _coreScopeProvider.CreateCoreScope();
+        scope.WriteLock(Constants.Locks.DistributedJobs);
+
+        DistributedBackgroundJobModel[] existingJobs = _distributedJobRepository.GetAll().ToArray();
+        var existingJobsByName = existingJobs.ToDictionary(x => x.Name);
+
+        foreach (IDistributedBackgroundJob registeredJob in _distributedBackgroundJobs)
+        {
+            if (existingJobsByName.TryGetValue(registeredJob.Name, out DistributedBackgroundJobModel? existingJob))
+            {
+                // Update if period has changed
+                if (existingJob.Period != registeredJob.Period)
+                {
+                    existingJob.Period = registeredJob.Period;
+                    _distributedJobRepository.Update(existingJob);
+                }
+            }
+            else
+            {
+                // Add new job (fresh install or newly registered job)
+                var newJob = new DistributedBackgroundJobModel
+                {
+                    Name = registeredJob.Name,
+                    Period = registeredJob.Period,
+                    LastRun = DateTime.UtcNow,
+                    IsRunning = false,
+                    LastAttemptedRun = DateTime.UtcNow,
+                };
+                _distributedJobRepository.Add(newJob);
+            }
+        }
+
+        // Remove jobs that are no longer registered in code
+        var registeredJobNames = _distributedBackgroundJobs.Select(x => x.Name).ToHashSet();
+        IEnumerable<DistributedBackgroundJobModel> jobsToRemove = existingJobs.Where(x => registeredJobNames.Contains(x.Name) is false);
+
+        foreach (DistributedBackgroundJobModel jobToRemove in jobsToRemove)
+        {
+            _distributedJobRepository.Delete(jobToRemove);
+        }
+
+        scope.Complete();
+    }
+}

src/Umbraco.Web.Common/DependencyInjection/UmbracoBuilderExtensions.cs

@@ -1,13 +1,10 @@
 using System.Data.Common;
 using System.Net.Http.Headers;
 using System.Reflection;
-using System.Runtime.CompilerServices;
-using Microsoft.AspNetCore.Builder;
 using Microsoft.AspNetCore.DataProtection.Infrastructure;
 using Microsoft.AspNetCore.Hosting;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Mvc.ApplicationModels;
-using Microsoft.AspNetCore.Server.Kestrel.Core;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.DependencyInjection.Extensions;
@@ -36,6 +33,7 @@ using Umbraco.Cms.Core.Templates;
 using Umbraco.Cms.Core.Web;
 using Umbraco.Cms.Infrastructure.BackgroundJobs;
 using Umbraco.Cms.Infrastructure.BackgroundJobs.Jobs;
+using Umbraco.Cms.Infrastructure.BackgroundJobs.Jobs.DistributedJobs;
 using Umbraco.Cms.Infrastructure.BackgroundJobs.Jobs.ServerRegistration;
 using Umbraco.Cms.Infrastructure.DependencyInjection;
 using Umbraco.Cms.Infrastructure.HostedServices;
@@ -172,35 +170,6 @@ public static partial class UmbracoBuilderExtensions
         return builder;
     }
 
-    /// <summary>
-    ///     Add Umbraco recurring background jobs
-    /// </summary>
-    public static IUmbracoBuilder AddRecurringBackgroundJobs(this IUmbracoBuilder builder)
-    {
-        // Add background jobs
-        builder.Services.AddRecurringBackgroundJob<HealthCheckNotifierJob>();
-        builder.Services.AddRecurringBackgroundJob<LogScrubberJob>();
-        builder.Services.AddRecurringBackgroundJob<ContentVersionCleanupJob>();
-        builder.Services.AddRecurringBackgroundJob<ScheduledPublishingJob>();
-        builder.Services.AddRecurringBackgroundJob<TempFileCleanupJob>();
-        builder.Services.AddRecurringBackgroundJob<TemporaryFileCleanupJob>();
-        builder.Services.AddRecurringBackgroundJob<InstructionProcessJob>();
-        builder.Services.AddRecurringBackgroundJob<TouchServerJob>();
-        builder.Services.AddRecurringBackgroundJob<WebhookFiring>();
-        builder.Services.AddRecurringBackgroundJob<WebhookLoggingCleanup>();
-        builder.Services.AddRecurringBackgroundJob<ReportSiteJob>();
-        builder.Services.AddRecurringBackgroundJob<CacheInstructionsPruningJob>();
-        builder.Services.AddRecurringBackgroundJob<LongRunningOperationsCleanupJob>();
-
-        builder.Services.AddSingleton(RecurringBackgroundJobHostedService.CreateHostedServiceFactory);
-        builder.Services.AddHostedService<RecurringBackgroundJobHostedServiceRunner>();
-        builder.Services.AddHostedService<QueuedHostedService>();
-        builder.AddNotificationAsyncHandler<PostRuntimePremigrationsUpgradeNotification, NavigationInitializationNotificationHandler>();
-        builder.AddNotificationAsyncHandler<PostRuntimePremigrationsUpgradeNotification, PublishStatusInitializationNotificationHandler>();
-
-        return builder;
-    }
-
     /// <summary>
     ///     Adds the Umbraco request profiler
     /// </summary>

tests/Umbraco.Tests.UnitTests/Umbraco.Core/Scheduling/ContentVersionCleanupTest.cs

@@ -9,6 +9,7 @@ using Umbraco.Cms.Core.Runtime;
 using Umbraco.Cms.Core.Services;
 using Umbraco.Cms.Core.Sync;
 using Umbraco.Cms.Infrastructure.BackgroundJobs.Jobs;
+using Umbraco.Cms.Infrastructure.BackgroundJobs.Jobs.DistributedJobs;
 using Umbraco.Cms.Infrastructure.HostedServices;
 using Umbraco.Cms.Tests.UnitTests.AutoFixture;
 
@@ -35,7 +36,7 @@ internal class ContentVersionCleanupTest
         mainDom.Setup(x => x.IsMainDom).Returns(true);
         serverRoleAccessor.Setup(x => x.CurrentServerRole).Returns(ServerRole.SchedulingPublisher);
 
-        await sut.RunJobAsync();
+        await sut.ExecuteAsync();
 
         cleanupService.Verify(x => x.PerformContentVersionCleanup(It.IsAny<DateTime>()), Times.Never);
     }
@@ -59,7 +60,7 @@ internal class ContentVersionCleanupTest
         mainDom.Setup(x => x.IsMainDom).Returns(true);
         serverRoleAccessor.Setup(x => x.CurrentServerRole).Returns(ServerRole.SchedulingPublisher);
 
-        await sut.RunJobAsync();
+        await sut.ExecuteAsync();
 
         cleanupService.Verify(x => x.PerformContentVersionCleanup(It.IsAny<DateTime>()), Times.Once);
     }

tests/Umbraco.Tests.UnitTests/Umbraco.Infrastructure/BackgroundJobs/Jobs/CacheInstructionsPruningJobTests.cs

@@ -10,6 +10,7 @@ using Umbraco.Cms.Core.Events;
 using Umbraco.Cms.Core.Persistence.Repositories;
 using Umbraco.Cms.Core.Scoping;
 using Umbraco.Cms.Infrastructure.BackgroundJobs.Jobs;
+using Umbraco.Cms.Infrastructure.BackgroundJobs.Jobs.DistributedJobs;
 
 namespace Umbraco.Cms.Tests.UnitTests.Umbraco.Infrastructure.BackgroundJobs.Jobs;
 
@@ -43,7 +44,7 @@ public class CacheInstructionsPruningJobTests
 
         var job = CreateCacheInstructionsPruningJob(timeToRetainInstructions: timeToRetainInstructions);
 
-        await job.RunJobAsync();
+        await job.ExecuteAsync();
 
         _cacheInstructionRepositoryMock.Verify(repo => repo.DeleteInstructionsOlderThan(expectedPruneDate), Times.Once);
     }

tests/Umbraco.Tests.UnitTests/Umbraco.Infrastructure/BackgroundJobs/Jobs/HealthCheckNotifierJobTests.cs

@@ -1,24 +1,16 @@
 // Copyright (c) Umbraco.
 // See LICENSE for more details.
 
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
 using Moq;
 using NUnit.Framework;
-using Umbraco.Cms.Core;
 using Umbraco.Cms.Core.Configuration;
 using Umbraco.Cms.Core.Configuration.Models;
 using Umbraco.Cms.Core.Events;
 using Umbraco.Cms.Core.HealthChecks;
 using Umbraco.Cms.Core.HealthChecks.NotificationMethods;
 using Umbraco.Cms.Core.Logging;
-using Umbraco.Cms.Core.Runtime;
+using Umbraco.Cms.Infrastructure.BackgroundJobs.Jobs.DistributedJobs;
-using Umbraco.Cms.Core.Services;
-using Umbraco.Cms.Core.Sync;
-using Umbraco.Cms.Infrastructure.BackgroundJobs;
-using Umbraco.Cms.Infrastructure.BackgroundJobs.Jobs;
 using Umbraco.Cms.Infrastructure.Scoping;
 using Umbraco.Cms.Tests.Common;
 
@@ -37,7 +29,7 @@ public class HealthCheckNotifierJobTests
     public async Task Does_Not_Execute_When_Not_Enabled()
     {
         var sut = CreateHealthCheckNotifier(false);
-        await sut.RunJobAsync();
+        await sut.ExecuteAsync();
         VerifyNotificationsNotSent();
     }
 
@@ -45,7 +37,7 @@ public class HealthCheckNotifierJobTests
     public async Task Does_Not_Execute_With_No_Enabled_Notification_Methods()
     {
         var sut = CreateHealthCheckNotifier(notificationEnabled: false);
-        await sut.RunJobAsync();
+        await sut.ExecuteAsync();
         VerifyNotificationsNotSent();
     }
 
@@ -53,7 +45,7 @@ public class HealthCheckNotifierJobTests
     public async Task Executes_With_Enabled_Notification_Methods()
     {
         var sut = CreateHealthCheckNotifier();
-        await sut.RunJobAsync();
+        await sut.ExecuteAsync();
         VerifyNotificationsSent();
     }
 
@@ -61,7 +53,7 @@ public class HealthCheckNotifierJobTests
     public async Task Executes_Only_Enabled_Checks()
     {
         var sut = CreateHealthCheckNotifier();
-        await sut.RunJobAsync();
+        await sut.ExecuteAsync();
         _mockNotificationMethod.Verify(
             x => x.SendAsync(
                 It.Is<HealthCheckResults>(y =>
@@ -96,7 +88,6 @@ public class HealthCheckNotifierJobTests
 
 
         var mockScopeProvider = new Mock<IScopeProvider>();
-        var mockLogger = new Mock<ILogger<HealthCheckNotifierJob>>();
         var mockProfilingLogger = new Mock<IProfilingLogger>();
 
         return new HealthCheckNotifierJob(
@@ -104,9 +95,7 @@ public class HealthCheckNotifierJobTests
             checks,
             notifications,
             mockScopeProvider.Object,
-            mockLogger.Object,
             mockProfilingLogger.Object,
-            Mock.Of<ICronTabParser>(),
             Mock.Of<IEventAggregator>());
     }
 

tests/Umbraco.Tests.UnitTests/Umbraco.Infrastructure/BackgroundJobs/Jobs/LogScrubberJobTests.cs

@@ -2,19 +2,15 @@
 // See LICENSE for more details.
 
 using System.Data;
-using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
 using Moq;
 using NUnit.Framework;
 using Umbraco.Cms.Core.Configuration.Models;
 using Umbraco.Cms.Core.Events;
 using Umbraco.Cms.Core.Logging;
-using Umbraco.Cms.Core.Runtime;
 using Umbraco.Cms.Core.Scoping;
 using Umbraco.Cms.Core.Services;
-using Umbraco.Cms.Core.Sync;
+using Umbraco.Cms.Infrastructure.BackgroundJobs.Jobs.DistributedJobs;
-using Umbraco.Cms.Infrastructure.BackgroundJobs.Jobs;
-using Umbraco.Cms.Infrastructure.HostedServices;
 using Umbraco.Cms.Tests.Common;
 
 namespace Umbraco.Cms.Tests.UnitTests.Umbraco.Infrastructure.BackgroundJobs.Jobs;
@@ -30,7 +26,7 @@ public class LogScrubberJobTests
     public async Task Executes_And_Scrubs_Logs()
     {
         var sut = CreateLogScrubber();
-        await sut.RunJobAsync();
+        await sut.ExecuteAsync();
         VerifyLogsScrubbed();
     }
 
@@ -50,7 +46,6 @@ public class LogScrubberJobTests
                 It.IsAny<bool>(),
                 It.IsAny<bool>()))
             .Returns(mockScope.Object);
-        var mockLogger = new Mock<ILogger<LogScrubberJob>>();
         var mockProfilingLogger = new Mock<IProfilingLogger>();
 
         _mockAuditService = new Mock<IAuditService>();
@@ -59,7 +54,6 @@ public class LogScrubberJobTests
             _mockAuditService.Object,
             new TestOptionsMonitor<LoggingSettings>(settings),
             mockScopeProvider.Object,
-            mockLogger.Object,
             mockProfilingLogger.Object);
     }
 

tests/Umbraco.Tests.UnitTests/Umbraco.Infrastructure/BackgroundJobs/Jobs/ScheduledPublishingJobTests.cs

@@ -15,6 +15,7 @@ using Umbraco.Cms.Core.Sync;
 using Umbraco.Cms.Core.Web;
 using Umbraco.Cms.Infrastructure;
 using Umbraco.Cms.Infrastructure.BackgroundJobs.Jobs;
+using Umbraco.Cms.Infrastructure.BackgroundJobs.Jobs.DistributedJobs;
 using Umbraco.Cms.Infrastructure.HostedServices;
 
 namespace Umbraco.Cms.Tests.UnitTests.Umbraco.Infrastructure.BackgroundJobs.Jobs;
@@ -29,7 +30,7 @@ public class ScheduledPublishingJobTests
     public async Task Does_Not_Execute_When_Not_Enabled()
     {
         var sut = CreateScheduledPublishing(enabled: false);
-        await sut.RunJobAsync();
+        await sut.ExecuteAsync();
         VerifyScheduledPublishingNotPerformed();
     }
 
@@ -37,7 +38,7 @@ public class ScheduledPublishingJobTests
     public async Task Executes_And_Performs_Scheduled_Publishing()
     {
         var sut = CreateScheduledPublishing();
-        await sut.RunJobAsync();
+        await sut.ExecuteAsync();
         VerifyScheduledPublishingPerformed();
     }
 

tools/Umbraco.JsonSchema/UmbracoCmsSchema.cs

@@ -75,6 +75,8 @@ internal sealed class UmbracoCmsSchema
         public required WebhookSettings Webhook { get; set; }
 
         public required CacheSettings Cache { get; set; }
+
+        public required DistributedJobSettings DistributedJobSettings { get; set; }
     }
 
     public class InstallDefaultDataNamedOptions