Netcore: Health check notifier hosted service (#9295)

* Implemented health check notifier as a hosted service.
Added validation to health check settings.

* Registered health check notifier as a hosted service.
Modified health check nested settings to use concrete classes to align with other configuration models.

* Resolved issues with email sending using development server.

* PR review comments and fixed failing unit test.

* Changed period and delay millisecond and hourly values to TimeSpans.
Changed configuration of first run time for health check notifications to use H:mm format.

* Set up SecureSocketOptions as a locally defined enum.

* Tightened up time format validation to verify input is an actual time (with hours and minutes only) and not a timespan.

* Aligned naming and namespace of health check configuration related classes with other configuration classes.

* Created constants for hex colors used in formatting health check results as HTML.

* Revert "Tightened up time format validation to verify input is an actual time (with hours and minutes only) and not a timespan."

This reverts commit f9bb8a7a825bcb58146879f18b47922e09453e2d.

* Renamed method to be clear validation is of a TimeSpan and not a time.

Co-authored-by: Bjarke Berg <mail@bergmania.dk>

src/Umbraco.Infrastructure/HostedServices/HealthCheckNotifier.cs

@@ -0,0 +1,114 @@
+using System;
+using System.Linq;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using Umbraco.Core;
+using Umbraco.Core.Configuration.Models;
+using Umbraco.Core.Configuration.Models.Extensions;
+using Umbraco.Core.HealthCheck;
+using Umbraco.Core.Logging;
+using Umbraco.Core.Scoping;
+using Umbraco.Core.Sync;
+using Umbraco.Infrastructure.HealthCheck;
+using Umbraco.Web.HealthCheck;
+
+namespace Umbraco.Infrastructure.HostedServices
+{
+    /// <summary>
+    /// Hosted service implementation for recurring health check notifications.
+    /// </summary>
+    public class HealthCheckNotifier : RecurringHostedServiceBase
+    {
+        private readonly HealthChecksSettings _healthChecksSettings;
+        private readonly HealthCheckCollection _healthChecks;
+        private readonly HealthCheckNotificationMethodCollection _notifications;
+        private readonly IRuntimeState _runtimeState;
+        private readonly IServerRegistrar _serverRegistrar;
+        private readonly IMainDom _mainDom;
+        private readonly IScopeProvider _scopeProvider;
+        private readonly ILogger<HealthCheckNotifier> _logger;
+        private readonly IProfilingLogger _profilingLogger;
+
+        public HealthCheckNotifier(
+            IOptions<HealthChecksSettings> healthChecksSettings,
+            HealthCheckCollection healthChecks,
+            HealthCheckNotificationMethodCollection notifications,
+            IRuntimeState runtimeState,
+            IServerRegistrar serverRegistrar,
+            IMainDom mainDom,
+            IScopeProvider scopeProvider,
+            ILogger<HealthCheckNotifier> logger,
+            IProfilingLogger profilingLogger)
+            : base(healthChecksSettings.Value.Notification.Period,
+                   healthChecksSettings.Value.GetNotificationDelay(DateTime.Now, DefaultDelay))
+        {
+            _healthChecksSettings = healthChecksSettings.Value;
+            _healthChecks = healthChecks;
+            _notifications = notifications;
+            _runtimeState = runtimeState;
+            _serverRegistrar = serverRegistrar;
+            _mainDom = mainDom;
+            _scopeProvider = scopeProvider;
+            _logger = logger;
+            _profilingLogger = profilingLogger;
+        }
+
+       
+        public override async void ExecuteAsync(object state)
+        {
+            if (_healthChecksSettings.Notification.Enabled == false)
+            {
+                return;
+            }
+
+            if (_runtimeState.Level != RuntimeLevel.Run)
+            {
+                return;
+            }
+
+            switch (_serverRegistrar.GetCurrentServerRole())
+            {
+                case ServerRole.Replica:
+                    _logger.LogDebug("Does not run on replica servers.");
+                    return;
+                case ServerRole.Unknown:
+                    _logger.LogDebug("Does not run on servers with unknown role.");
+                    return;
+            }
+
+            // Ensure we do not run if not main domain, but do NOT lock it
+            if (_mainDom.IsMainDom == false)
+            {
+                _logger.LogDebug("Does not run if not MainDom.");
+                return;
+            }
+
+            // Ensure we use an explicit scope since we are running on a background thread and plugin health
+            // checks can be making service/database calls so we want to ensure the CallContext/Ambient scope
+            // isn't used since that can be problematic.
+            using (var scope = _scopeProvider.CreateScope())
+            using (_profilingLogger.DebugDuration<HealthCheckNotifier>("Health checks executing", "Health checks complete"))
+            {
+                // Don't notify for any checks that are disabled, nor for any disabled just for notifications.
+                var disabledCheckIds = _healthChecksSettings.Notification.DisabledChecks
+                        .Select(x => x.Id)
+                    .Union(_healthChecksSettings.DisabledChecks
+                        .Select(x => x.Id))
+                    .Distinct()
+                    .ToArray();
+
+                var checks = _healthChecks
+                    .Where(x => disabledCheckIds.Contains(x.Id) == false);
+
+                var results = new HealthCheckResults(checks);
+                results.LogResults();
+
+                // Send using registered notification methods that are enabled.
+                foreach (var notificationMethod in _notifications.Where(x => x.Enabled))
+                {
+                    await notificationMethod.SendAsync(results);
+                }
+            }
+        }
+    }
+}