diff --git a/src/Umbraco.Core/Configuration/Models/KeepAliveSettings.cs b/src/Umbraco.Core/Configuration/Models/KeepAliveSettings.cs index 01d3b36a5d..2c53407398 100644 --- a/src/Umbraco.Core/Configuration/Models/KeepAliveSettings.cs +++ b/src/Umbraco.Core/Configuration/Models/KeepAliveSettings.cs @@ -2,7 +2,7 @@ namespace Umbraco.Core.Configuration.Models { public class KeepAliveSettings { - public bool DisableKeepAliveTask => false; + public bool DisableKeepAliveTask { get; set; } = false; public string KeepAlivePingUrl => "{umbracoApplicationUrl}/api/keepalive/ping"; } diff --git a/src/Umbraco.Core/Configuration/Models/LoggingSettings.cs b/src/Umbraco.Core/Configuration/Models/LoggingSettings.cs index 414ff06b57..7d79bb83ae 100644 --- a/src/Umbraco.Core/Configuration/Models/LoggingSettings.cs +++ b/src/Umbraco.Core/Configuration/Models/LoggingSettings.cs @@ -1,7 +1,9 @@ -namespace Umbraco.Core.Configuration.Models +using System; + +namespace Umbraco.Core.Configuration.Models { public class LoggingSettings { - public int MaxLogAge { get; set; } = -1; + public TimeSpan MaxLogAge { get; set; } = TimeSpan.FromHours(24); } } diff --git a/src/Umbraco.Core/IO/CleanFolderResult.cs b/src/Umbraco.Core/IO/CleanFolderResult.cs new file mode 100644 index 0000000000..547157daff --- /dev/null +++ b/src/Umbraco.Core/IO/CleanFolderResult.cs @@ -0,0 +1,49 @@ +using System; +using System.Collections.Generic; +using System.IO; + +namespace Umbraco.Core.IO +{ + public class CleanFolderResult + { + private CleanFolderResult() + { + } + + public CleanFolderResultStatus Status { get; private set; } + + public IReadOnlyCollection Errors { get; private set; } + + public static CleanFolderResult Success() + { + return new CleanFolderResult { Status = CleanFolderResultStatus.Success }; + } + + public static CleanFolderResult FailedAsDoesNotExist() + { + return new CleanFolderResult { Status = CleanFolderResultStatus.FailedAsDoesNotExist }; + } + + public static CleanFolderResult FailedWithErrors(List errors) + { + return new CleanFolderResult + { + Status = CleanFolderResultStatus.FailedWithException, + Errors = errors.AsReadOnly(), + }; + } + + public class Error + { + public Error(Exception exception, FileInfo erroringFile) + { + Exception = exception; + ErroringFile = erroringFile; + } + + public Exception Exception { get; set; } + + public FileInfo ErroringFile { get; set; } + } + } +} diff --git a/src/Umbraco.Core/IO/CleanFolderResultStatus.cs b/src/Umbraco.Core/IO/CleanFolderResultStatus.cs new file mode 100644 index 0000000000..41bd56d53d --- /dev/null +++ b/src/Umbraco.Core/IO/CleanFolderResultStatus.cs @@ -0,0 +1,9 @@ +namespace Umbraco.Core.IO +{ + public enum CleanFolderResultStatus + { + Success, + FailedAsDoesNotExist, + FailedWithException + } +} diff --git a/src/Umbraco.Core/IO/IIOHelper.cs b/src/Umbraco.Core/IO/IIOHelper.cs index 42ca804f44..5a0f5cab45 100644 --- a/src/Umbraco.Core/IO/IIOHelper.cs +++ b/src/Umbraco.Core/IO/IIOHelper.cs @@ -1,4 +1,7 @@ +using System; using System.Collections.Generic; +using System.IO; +using Umbraco.Core.Hosting; namespace Umbraco.Core.IO { @@ -53,5 +56,20 @@ namespace Umbraco.Core.IO /// string GetRelativePath(string path); + /// + /// Retrieves array of temporary folders from the hosting environment. + /// + /// Array of instances. + DirectoryInfo[] GetTempFolders(); + + /// + /// Cleans contents of a folder by deleting all files older that the provided age. + /// If deletition of any file errors (e.g. due to a file lock) the process will continue to try to delete all that it can. + /// + /// Folder to clean. + /// Age of files within folder to delete. + /// Result of operation + CleanFolderResult CleanFolder(DirectoryInfo folder, TimeSpan age); + } } diff --git a/src/Umbraco.Core/IO/IOHelper.cs b/src/Umbraco.Core/IO/IOHelper.cs index b3ca956733..08241d28f4 100644 --- a/src/Umbraco.Core/IO/IOHelper.cs +++ b/src/Umbraco.Core/IO/IOHelper.cs @@ -188,5 +188,63 @@ namespace Umbraco.Core.IO return PathUtility.EnsurePathIsApplicationRootPrefixed(path); } + /// + /// Retrieves array of temporary folders from the hosting environment. + /// + /// Array of instances. + public DirectoryInfo[] GetTempFolders() + { + var tempFolderPaths = new[] + { + _hostingEnvironment.MapPathContentRoot(Constants.SystemDirectories.TempFileUploads) + }; + + foreach (var tempFolderPath in tempFolderPaths) + { + // Ensure it exists + Directory.CreateDirectory(tempFolderPath); + } + + return tempFolderPaths.Select(x => new DirectoryInfo(x)).ToArray(); + } + + /// + /// Cleans contents of a folder by deleting all files older that the provided age. + /// If deletition of any file errors (e.g. due to a file lock) the process will continue to try to delete all that it can. + /// + /// Folder to clean. + /// Age of files within folder to delete. + /// Result of operation. + public CleanFolderResult CleanFolder(DirectoryInfo folder, TimeSpan age) + { + folder.Refresh(); // In case it's changed during runtime. + + if (!folder.Exists) + { + return CleanFolderResult.FailedAsDoesNotExist(); + } + + var files = folder.GetFiles("*.*", SearchOption.AllDirectories); + var errors = new List(); + foreach (var file in files) + { + if (DateTime.UtcNow - file.LastWriteTimeUtc > age) + { + try + { + file.IsReadOnly = false; + file.Delete(); + } + catch (Exception ex) + { + errors.Add(new CleanFolderResult.Error(ex, file)); + } + } + } + + return errors.Any() + ? CleanFolderResult.FailedWithErrors(errors) + : CleanFolderResult.Success(); + } } } diff --git a/src/Umbraco.Core/Scheduling/TempFileCleanup.cs b/src/Umbraco.Core/Scheduling/TempFileCleanup.cs deleted file mode 100644 index 1a8ece78d1..0000000000 --- a/src/Umbraco.Core/Scheduling/TempFileCleanup.cs +++ /dev/null @@ -1,80 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using Umbraco.Core; -using Umbraco.Core.Logging; -using Microsoft.Extensions.Logging; - -namespace Umbraco.Web.Scheduling -{ - /// - /// Used to cleanup temporary file locations - /// - public class TempFileCleanup : RecurringTaskBase - { - private readonly DirectoryInfo[] _tempFolders; - private readonly TimeSpan _age; - private readonly IMainDom _mainDom; - private readonly IProfilingLogger _profilingLogger; - private readonly ILogger _logger; - - public TempFileCleanup(IBackgroundTaskRunner runner, int delayMilliseconds, int periodMilliseconds, - IEnumerable tempFolders, TimeSpan age, - IMainDom mainDom, IProfilingLogger profilingLogger, ILogger logger) - : base(runner, delayMilliseconds, periodMilliseconds) - { - //SystemDirectories.TempFileUploads - - _tempFolders = tempFolders.ToArray(); - _age = age; - _mainDom = mainDom; - _profilingLogger = profilingLogger; - _logger = logger; - } - - public override bool PerformRun() - { - // ensure we do not run if not main domain - if (_mainDom.IsMainDom == false) - { - _logger.LogDebug("Does not run if not MainDom."); - return false; // do NOT repeat, going down - } - - foreach (var dir in _tempFolders) - CleanupFolder(dir); - - return true; //repeat - } - - private void CleanupFolder(DirectoryInfo dir) - { - dir.Refresh(); //in case it's changed during runtime - if (!dir.Exists) - { - _logger.LogDebug("The cleanup folder doesn't exist {Folder}", dir.FullName); - return; - } - - var files = dir.GetFiles("*.*", SearchOption.AllDirectories); - foreach (var file in files) - { - if (DateTime.UtcNow - file.LastWriteTimeUtc > _age) - { - try - { - file.IsReadOnly = false; - file.Delete(); - } - catch (Exception ex) - { - _logger.LogError(ex, "Could not delete temp file {FileName}", file.FullName); - } - } - } - } - - public override bool IsAsync => false; - } -} diff --git a/src/Umbraco.Core/ServiceCollectionExtensions.cs b/src/Umbraco.Core/ServiceCollectionExtensions.cs index 3202cc3a38..d1c89ea17e 100644 --- a/src/Umbraco.Core/ServiceCollectionExtensions.cs +++ b/src/Umbraco.Core/ServiceCollectionExtensions.cs @@ -12,6 +12,18 @@ namespace Umbraco.Core where TImplementing : class, TService => services.Replace(ServiceDescriptor.Singleton()); + /// + /// Registers a singleton instance against multiple interfaces. + /// + public static void AddMultipleUnique(this IServiceCollection services) + where TService1 : class + where TService2 : class + where TImplementing : class, TService1, TService2 + { + services.AddUnique(); + services.AddUnique(factory => (TImplementing) factory.GetRequiredService()); + } + public static void AddUnique(this IServiceCollection services) where TImplementing : class => services.Replace(ServiceDescriptor.Singleton()); diff --git a/src/Umbraco.Infrastructure/HostedServices/HealthCheckNotifier.cs b/src/Umbraco.Infrastructure/HostedServices/HealthCheckNotifier.cs index af0a0f335c..c42892101a 100644 --- a/src/Umbraco.Infrastructure/HostedServices/HealthCheckNotifier.cs +++ b/src/Umbraco.Infrastructure/HostedServices/HealthCheckNotifier.cs @@ -1,5 +1,6 @@ using System; using System.Linq; +using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Umbraco.Core; @@ -52,9 +53,8 @@ namespace Umbraco.Infrastructure.HostedServices _logger = logger; _profilingLogger = profilingLogger; } - - public override async void ExecuteAsync(object state) + internal override async Task PerformExecuteAsync(object state) { if (_healthChecksSettings.Notification.Enabled == false) { diff --git a/src/Umbraco.Core/Scheduling/KeepAlive.cs b/src/Umbraco.Infrastructure/HostedServices/KeepAlive.cs similarity index 67% rename from src/Umbraco.Core/Scheduling/KeepAlive.cs rename to src/Umbraco.Infrastructure/HostedServices/KeepAlive.cs index 9b09a81cc3..39ac0f3d87 100644 --- a/src/Umbraco.Core/Scheduling/KeepAlive.cs +++ b/src/Umbraco.Infrastructure/HostedServices/KeepAlive.cs @@ -1,17 +1,20 @@ using System; using System.Net.Http; -using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Umbraco.Core; using Umbraco.Core.Configuration.Models; using Umbraco.Core.Logging; using Umbraco.Core.Sync; -using Microsoft.Extensions.Logging; +using Umbraco.Web; -namespace Umbraco.Web.Scheduling +namespace Umbraco.Infrastructure.HostedServices { - public class KeepAlive : RecurringTaskBase + /// + /// Hosted service implementation for keep alive feature. + /// + public class KeepAlive : RecurringHostedServiceBase { private readonly IRequestAccessor _requestAccessor; private readonly IMainDom _mainDom; @@ -19,11 +22,10 @@ namespace Umbraco.Web.Scheduling private readonly ILogger _logger; private readonly IProfilingLogger _profilingLogger; private readonly IServerRegistrar _serverRegistrar; - private static HttpClient _httpClient; + private readonly IHttpClientFactory _httpClientFactory; - public KeepAlive(IBackgroundTaskRunner runner, int delayMilliseconds, int periodMilliseconds, - IRequestAccessor requestAccessor, IMainDom mainDom, IOptions keepAliveSettings, ILogger logger, IProfilingLogger profilingLogger, IServerRegistrar serverRegistrar) - : base(runner, delayMilliseconds, periodMilliseconds) + public KeepAlive(IRequestAccessor requestAccessor, IMainDom mainDom, IOptions keepAliveSettings, ILogger logger, IProfilingLogger profilingLogger, IServerRegistrar serverRegistrar, IHttpClientFactory httpClientFactory) + : base(TimeSpan.FromMinutes(5), DefaultDelay) { _requestAccessor = requestAccessor; _mainDom = mainDom; @@ -31,30 +33,32 @@ namespace Umbraco.Web.Scheduling _logger = logger; _profilingLogger = profilingLogger; _serverRegistrar = serverRegistrar; - if (_httpClient == null) - { - _httpClient = new HttpClient(); - } + _httpClientFactory = httpClientFactory; } - public override async Task PerformRunAsync(CancellationToken token) + internal override async Task PerformExecuteAsync(object state) { - // not on replicas nor unknown role servers + if (_keepAliveSettings.DisableKeepAliveTask) + { + return; + } + + // Don't run on replicas nor unknown role servers switch (_serverRegistrar.GetCurrentServerRole()) { case ServerRole.Replica: _logger.LogDebug("Does not run on replica servers."); - return true; // role may change! + return; case ServerRole.Unknown: _logger.LogDebug("Does not run on servers with unknown role."); - return true; // role may change! + return; } - // ensure we do not run if not main domain, but do NOT lock it + // Ensure we do not run if not main domain, but do NOT lock it if (_mainDom.IsMainDom == false) { _logger.LogDebug("Does not run if not MainDom."); - return false; // do NOT repeat, going down + return; } using (_profilingLogger.DebugDuration("Keep alive executing", "Keep alive complete")) @@ -68,24 +72,21 @@ namespace Umbraco.Web.Scheduling if (umbracoAppUrl.IsNullOrWhiteSpace()) { _logger.LogWarning("No umbracoApplicationUrl for service (yet), skip."); - return true; // repeat + return; } keepAlivePingUrl = keepAlivePingUrl.Replace("{umbracoApplicationUrl}", umbracoAppUrl.TrimEnd('/')); } var request = new HttpRequestMessage(HttpMethod.Get, keepAlivePingUrl); - var result = await _httpClient.SendAsync(request, token); + var httpClient = _httpClientFactory.CreateClient(); + await httpClient.SendAsync(request); } catch (Exception ex) { _logger.LogError(ex, "Keep alive failed (at '{keepAlivePingUrl}').", keepAlivePingUrl); } } - - return true; // repeat } - - public override bool IsAsync => true; } } diff --git a/src/Umbraco.Infrastructure/HostedServices/LogScrubber.cs b/src/Umbraco.Infrastructure/HostedServices/LogScrubber.cs new file mode 100644 index 0000000000..1cf2da05f9 --- /dev/null +++ b/src/Umbraco.Infrastructure/HostedServices/LogScrubber.cs @@ -0,0 +1,70 @@ +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Umbraco.Core; +using Umbraco.Core.Configuration.Models; +using Umbraco.Core.Logging; +using Umbraco.Core.Scoping; +using Umbraco.Core.Services; +using Umbraco.Core.Sync; + +namespace Umbraco.Infrastructure.HostedServices +{ + /// + /// Log scrubbing hosted service. + /// + /// + /// Will only run on non-replica servers. + /// + public class LogScrubber : RecurringHostedServiceBase + { + private readonly IMainDom _mainDom; + private readonly IServerRegistrar _serverRegistrar; + private readonly IAuditService _auditService; + private readonly LoggingSettings _settings; + private readonly IProfilingLogger _profilingLogger; + private readonly ILogger _logger; + private readonly IScopeProvider _scopeProvider; + + public LogScrubber(IMainDom mainDom, IServerRegistrar serverRegistrar, IAuditService auditService, IOptions settings, IScopeProvider scopeProvider, ILogger logger, IProfilingLogger profilingLogger) + : base(TimeSpan.FromHours(4), DefaultDelay) + { + _mainDom = mainDom; + _serverRegistrar = serverRegistrar; + _auditService = auditService; + _settings = settings.Value; + _scopeProvider = scopeProvider; + _logger = logger; + _profilingLogger = profilingLogger; + } + + internal override async Task PerformExecuteAsync(object state) + { + switch (_serverRegistrar.GetCurrentServerRole()) + { + case ServerRole.Replica: + _logger.LogDebug("Does not run on replica servers."); + return; + case ServerRole.Unknown: + _logger.LogDebug("Does not run on servers with unknown role."); + return; + } + + // Ensure we do not run if not main domain, but do NOT lock it + if (_mainDom.IsMainDom == false) + { + _logger.LogDebug("Does not run if not MainDom."); + return; + } + + // Ensure we use an explicit scope since we are running on a background thread. + using (var scope = _scopeProvider.CreateScope()) + using (_profilingLogger.DebugDuration("Log scrubbing executing", "Log scrubbing complete")) + { + _auditService.CleanLogs((int)_settings.MaxLogAge.TotalMinutes); + scope.Complete(); + } + } + } +} diff --git a/src/Umbraco.Infrastructure/HostedServices/RecurringHostedServiceBase.cs b/src/Umbraco.Infrastructure/HostedServices/RecurringHostedServiceBase.cs index 071ada0b62..ee77326115 100644 --- a/src/Umbraco.Infrastructure/HostedServices/RecurringHostedServiceBase.cs +++ b/src/Umbraco.Infrastructure/HostedServices/RecurringHostedServiceBase.cs @@ -31,7 +31,16 @@ namespace Umbraco.Infrastructure.HostedServices return Task.CompletedTask; } - public abstract void ExecuteAsync(object state); + public async void ExecuteAsync(object state) + { + // Delegate work to method returning a task, that can be called and asserted in a unit test. + // Without this there can be behaviour where tests pass, but an error within them causes the test + // running process to crash. + // Hat-tip: https://stackoverflow.com/a/14207615/489433 + await PerformExecuteAsync(state); + } + + internal abstract Task PerformExecuteAsync(object state); public Task StopAsync(CancellationToken cancellationToken) { diff --git a/src/Umbraco.Infrastructure/Scheduling/ScheduledPublishing.cs b/src/Umbraco.Infrastructure/HostedServices/ScheduledPublishing.cs similarity index 66% rename from src/Umbraco.Infrastructure/Scheduling/ScheduledPublishing.cs rename to src/Umbraco.Infrastructure/HostedServices/ScheduledPublishing.cs index 81dd8f92af..9c32a80fe2 100644 --- a/src/Umbraco.Infrastructure/Scheduling/ScheduledPublishing.cs +++ b/src/Umbraco.Infrastructure/HostedServices/ScheduledPublishing.cs @@ -1,13 +1,20 @@ using System; using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; using Umbraco.Core; using Umbraco.Core.Services; using Umbraco.Core.Sync; -using Microsoft.Extensions.Logging; +using Umbraco.Web; -namespace Umbraco.Web.Scheduling +namespace Umbraco.Infrastructure.HostedServices { - public class ScheduledPublishing : RecurringTaskBase + /// + /// Hosted service implementation for scheduled publishing feature. + /// + /// + /// Runs only on non-replica servers. + public class ScheduledPublishing : RecurringHostedServiceBase { private readonly IContentService _contentService; private readonly ILogger _logger; @@ -18,11 +25,10 @@ namespace Umbraco.Web.Scheduling private readonly IServerRegistrar _serverRegistrar; private readonly IUmbracoContextFactory _umbracoContextFactory; - public ScheduledPublishing(IBackgroundTaskRunner runner, int delayMilliseconds, - int periodMilliseconds, + public ScheduledPublishing( IRuntimeState runtime, IMainDom mainDom, IServerRegistrar serverRegistrar, IContentService contentService, IUmbracoContextFactory umbracoContextFactory, ILogger logger, IServerMessenger serverMessenger, IBackofficeSecurityFactory backofficeSecurityFactory) - : base(runner, delayMilliseconds, periodMilliseconds) + : base(TimeSpan.FromMinutes(1), DefaultDelay) { _runtime = runtime; _mainDom = mainDom; @@ -34,35 +40,35 @@ namespace Umbraco.Web.Scheduling _backofficeSecurityFactory = backofficeSecurityFactory; } - public override bool IsAsync => false; - - public override bool PerformRun() + internal override async Task PerformExecuteAsync(object state) { if (Suspendable.ScheduledPublishing.CanRun == false) - return true; // repeat, later + { + return; + } switch (_serverRegistrar.GetCurrentServerRole()) { case ServerRole.Replica: _logger.LogDebug("Does not run on replica servers."); - return true; // DO repeat, server role can change + return; case ServerRole.Unknown: _logger.LogDebug("Does not run on servers with unknown role."); - return true; // DO repeat, server role can change + return; } - // ensure we do not run if not main domain, but do NOT lock it + // Ensure we do not run if not main domain, but do NOT lock it if (_mainDom.IsMainDom == false) { _logger.LogDebug("Does not run if not MainDom."); - return false; // do NOT repeat, going down + return; } - // do NOT run publishing if not properly running + // Do NOT run publishing if not properly running if (_runtime.Level != RuntimeLevel.Run) { _logger.LogDebug("Does not run if run level is not Run."); - return true; // repeat/wait + return; } try @@ -79,22 +85,24 @@ namespace Umbraco.Web.Scheduling // - and we should definitively *not* have to flush it here (should be auto) // _backofficeSecurityFactory.EnsureBackofficeSecurity(); - using (var contextReference = _umbracoContextFactory.EnsureUmbracoContext()) + using var contextReference = _umbracoContextFactory.EnsureUmbracoContext(); + try { - try + // Run + var result = _contentService.PerformScheduledPublish(DateTime.Now); + foreach (var grouped in result.GroupBy(x => x.Result)) { - // run - var result = _contentService.PerformScheduledPublish(DateTime.Now); - foreach (var grouped in result.GroupBy(x => x.Result)) - _logger.LogInformation( - "Scheduled publishing result: '{StatusCount}' items with status {Status}", - grouped.Count(), grouped.Key); + _logger.LogInformation( + "Scheduled publishing result: '{StatusCount}' items with status {Status}", + grouped.Count(), grouped.Key); } - finally + } + finally + { + // If running on a temp context, we have to flush the messenger + if (contextReference.IsRoot && _serverMessenger is IBatchedDatabaseServerMessenger m) { - // if running on a temp context, we have to flush the messenger - if (contextReference.IsRoot && _serverMessenger is IBatchedDatabaseServerMessenger m) - m.FlushBatch(); + m.FlushBatch(); } } } @@ -104,7 +112,7 @@ namespace Umbraco.Web.Scheduling _logger.LogError(ex, "Failed."); } - return true; // repeat + return; } } } diff --git a/src/Umbraco.Infrastructure/HostedServices/TempFileCleanup.cs b/src/Umbraco.Infrastructure/HostedServices/TempFileCleanup.cs new file mode 100644 index 0000000000..e27b83c8f6 --- /dev/null +++ b/src/Umbraco.Infrastructure/HostedServices/TempFileCleanup.cs @@ -0,0 +1,95 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Umbraco.Core; +using Umbraco.Core.IO; + +namespace Umbraco.Infrastructure.HostedServices +{ + /// + /// Used to cleanup temporary file locations. + /// + /// + /// Will run on all servers - even though file upload should only be handled on the master, this will + /// ensure that in the case it happes on replicas that they are cleaned up too. + /// + public class TempFileCleanup : RecurringHostedServiceBase + { + private readonly IIOHelper _ioHelper; + private readonly IMainDom _mainDom; + private readonly ILogger _logger; + + private readonly DirectoryInfo[] _tempFolders; + private readonly TimeSpan _age = TimeSpan.FromDays(1); + + public TempFileCleanup(IIOHelper ioHelper, IMainDom mainDom, ILogger logger) + : base(TimeSpan.FromMinutes(60), DefaultDelay) + { + _ioHelper = ioHelper; + _mainDom = mainDom; + _logger = logger; + + _tempFolders = _ioHelper.GetTempFolders(); + } + + internal override async Task PerformExecuteAsync(object state) + { + // Ensure we do not run if not main domain + if (_mainDom.IsMainDom == false) + { + _logger.LogDebug("Does not run if not MainDom."); + return; + } + + foreach (var folder in _tempFolders) + { + CleanupFolder(folder); + } + + return; + } + + private void CleanupFolder(DirectoryInfo folder) + { + var result = _ioHelper.CleanFolder(folder, _age); + switch (result.Status) + { + case CleanFolderResultStatus.FailedAsDoesNotExist: + _logger.LogDebug("The cleanup folder doesn't exist {Folder}", folder.FullName); + break; + case CleanFolderResultStatus.FailedWithException: + foreach (var error in result.Errors) + { + _logger.LogError(error.Exception, "Could not delete temp file {FileName}", error.ErroringFile.FullName); + } + + break; + } + + folder.Refresh(); // In case it's changed during runtime + if (!folder.Exists) + { + _logger.LogDebug("The cleanup folder doesn't exist {Folder}", folder.FullName); + return; + } + + var files = folder.GetFiles("*.*", SearchOption.AllDirectories); + foreach (var file in files) + { + if (DateTime.UtcNow - file.LastWriteTimeUtc > _age) + { + try + { + file.IsReadOnly = false; + file.Delete(); + } + catch (Exception ex) + { + _logger.LogError(ex, "Could not delete temp file {FileName}", file.FullName); + } + } + } + } + } +} diff --git a/src/Umbraco.Infrastructure/Scheduling/LogScrubber.cs b/src/Umbraco.Infrastructure/Scheduling/LogScrubber.cs deleted file mode 100644 index c1b0b2e6d3..0000000000 --- a/src/Umbraco.Infrastructure/Scheduling/LogScrubber.cs +++ /dev/null @@ -1,92 +0,0 @@ -using System; -using Microsoft.Extensions.Options; -using Umbraco.Core; -using Umbraco.Core.Configuration.Models; -using Umbraco.Core.Configuration.UmbracoSettings; -using Umbraco.Core.Logging; -using Umbraco.Core.Scoping; -using Umbraco.Core.Services; -using Umbraco.Core.Sync; -using Microsoft.Extensions.Logging; - -namespace Umbraco.Web.Scheduling -{ - public class LogScrubber : RecurringTaskBase - { - private readonly IMainDom _mainDom; - private readonly IServerRegistrar _serverRegistrar; - private readonly IAuditService _auditService; - private readonly LoggingSettings _settings; - private readonly IProfilingLogger _profilingLogger; - private readonly ILogger _logger; - private readonly IScopeProvider _scopeProvider; - - public LogScrubber(IBackgroundTaskRunner runner, int delayMilliseconds, int periodMilliseconds, - IMainDom mainDom, IServerRegistrar serverRegistrar, IAuditService auditService, IOptions settings, IScopeProvider scopeProvider, IProfilingLogger profilingLogger , ILogger logger) - : base(runner, delayMilliseconds, periodMilliseconds) - { - _mainDom = mainDom; - _serverRegistrar = serverRegistrar; - _auditService = auditService; - _settings = settings.Value; - _scopeProvider = scopeProvider; - _profilingLogger = profilingLogger ; - _logger = logger; - } - - // maximum age, in minutes - private int GetLogScrubbingMaximumAge(LoggingSettings settings) - { - var maximumAge = 24 * 60; // 24 hours, in minutes - try - { - if (settings.MaxLogAge > -1) - maximumAge = settings.MaxLogAge; - } - catch (Exception ex) - { - _logger.LogError(ex, "Unable to locate a log scrubbing maximum age. Defaulting to 24 hours."); - } - return maximumAge; - - } - - public static int GetLogScrubbingInterval() - { - const int interval = 4 * 60 * 60 * 1000; // 4 hours, in milliseconds - return interval; - } - - public override bool PerformRun() - { - switch (_serverRegistrar.GetCurrentServerRole()) - { - case ServerRole.Replica: - _logger.LogDebug("Does not run on replica servers."); - return true; // DO repeat, server role can change - case ServerRole.Unknown: - _logger.LogDebug("Does not run on servers with unknown role."); - return true; // DO repeat, server role can change - } - - // ensure we do not run if not main domain, but do NOT lock it - if (_mainDom.IsMainDom == false) - { - _logger.LogDebug("Does not run if not MainDom."); - return false; // do NOT repeat, going down - } - - // Ensure we use an explicit scope since we are running on a background thread. - using (var scope = _scopeProvider.CreateScope()) - using (_profilingLogger.DebugDuration("Log scrubbing executing", "Log scrubbing complete")) - { - _auditService.CleanLogs(GetLogScrubbingMaximumAge(_settings)); - scope.Complete(); - } - - return true; // repeat - } - - public override bool IsAsync => false; - } -} diff --git a/src/Umbraco.Infrastructure/Scheduling/SchedulerComponent.cs b/src/Umbraco.Infrastructure/Scheduling/SchedulerComponent.cs deleted file mode 100644 index 49999c9c56..0000000000 --- a/src/Umbraco.Infrastructure/Scheduling/SchedulerComponent.cs +++ /dev/null @@ -1,187 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using Umbraco.Core; -using Umbraco.Core.Composing; -using Umbraco.Core.Configuration.Models; -using Umbraco.Core.HealthCheck; -using Umbraco.Core.Hosting; -using Umbraco.Core.Logging; -using Umbraco.Core.Scoping; -using Umbraco.Core.Services; -using Umbraco.Core.Sync; -using Umbraco.Web.HealthCheck; -using Umbraco.Web.Routing; - -namespace Umbraco.Web.Scheduling -{ - public sealed class SchedulerComponent : IComponent - { - private const int DefaultDelayMilliseconds = 180000; // 3 mins - private const int OneMinuteMilliseconds = 60000; - private const int FiveMinuteMilliseconds = 300000; - private const int OneHourMilliseconds = 3600000; - - private readonly IRuntimeState _runtime; - private readonly IMainDom _mainDom; - private readonly IServerRegistrar _serverRegistrar; - private readonly IContentService _contentService; - private readonly IAuditService _auditService; - private readonly IProfilingLogger _profilingLogger; - private readonly ILogger _logger; - private readonly ILoggerFactory _loggerFactory; - private readonly IApplicationShutdownRegistry _applicationShutdownRegistry; - private readonly IScopeProvider _scopeProvider; - private readonly IUmbracoContextFactory _umbracoContextFactory; - private readonly IServerMessenger _serverMessenger; - private readonly IRequestAccessor _requestAccessor; - private readonly IBackofficeSecurityFactory _backofficeSecurityFactory; - private readonly LoggingSettings _loggingSettings; - private readonly KeepAliveSettings _keepAliveSettings; - private readonly IHostingEnvironment _hostingEnvironment; - - private BackgroundTaskRunner _keepAliveRunner; - private BackgroundTaskRunner _publishingRunner; - private BackgroundTaskRunner _scrubberRunner; - private BackgroundTaskRunner _fileCleanupRunner; - private BackgroundTaskRunner _healthCheckRunner; - - private bool _started; - private object _locker = new object(); - private IBackgroundTask[] _tasks; - - public SchedulerComponent(IRuntimeState runtime, IMainDom mainDom, IServerRegistrar serverRegistrar, - IContentService contentService, IAuditService auditService, - IScopeProvider scopeProvider, IUmbracoContextFactory umbracoContextFactory, IProfilingLogger profilingLogger, ILoggerFactory loggerFactory, - IApplicationShutdownRegistry applicationShutdownRegistry, - IServerMessenger serverMessenger, IRequestAccessor requestAccessor, - IOptions loggingSettings, IOptions keepAliveSettings, - IHostingEnvironment hostingEnvironment, - IBackofficeSecurityFactory backofficeSecurityFactory) - { - _runtime = runtime; - _mainDom = mainDom; - _serverRegistrar = serverRegistrar; - _contentService = contentService; - _auditService = auditService; - _scopeProvider = scopeProvider; - _profilingLogger = profilingLogger; - _loggerFactory = loggerFactory; - _logger = loggerFactory.CreateLogger(); - _applicationShutdownRegistry = applicationShutdownRegistry; - _umbracoContextFactory = umbracoContextFactory; - _serverMessenger = serverMessenger; - _requestAccessor = requestAccessor; - _backofficeSecurityFactory = backofficeSecurityFactory; - _loggingSettings = loggingSettings.Value; - _keepAliveSettings = keepAliveSettings.Value; - _hostingEnvironment = hostingEnvironment; - } - - public void Initialize() - { - var logger = _loggerFactory.CreateLogger>(); - // backgrounds runners are web aware, if the app domain dies, these tasks will wind down correctly - _keepAliveRunner = new BackgroundTaskRunner("KeepAlive", logger, _applicationShutdownRegistry); - _publishingRunner = new BackgroundTaskRunner("ScheduledPublishing", logger, _applicationShutdownRegistry); - _scrubberRunner = new BackgroundTaskRunner("LogScrubber", logger, _applicationShutdownRegistry); - _fileCleanupRunner = new BackgroundTaskRunner("TempFileCleanup", logger, _applicationShutdownRegistry); - - // we will start the whole process when a successful request is made - _requestAccessor.RouteAttempt += RegisterBackgroundTasksOnce; - } - - public void Terminate() - { - // the AppDomain / maindom / whatever takes care of stopping background task runners - } - - private void RegisterBackgroundTasksOnce(object sender, RoutableAttemptEventArgs e) - { - switch (e.Outcome) - { - case EnsureRoutableOutcome.IsRoutable: - case EnsureRoutableOutcome.NotDocumentRequest: - _requestAccessor.RouteAttempt -= RegisterBackgroundTasksOnce; - RegisterBackgroundTasks(); - break; - } - } - - private void RegisterBackgroundTasks() - { - LazyInitializer.EnsureInitialized(ref _tasks, ref _started, ref _locker, () => - { - _logger.LogDebug("Initializing the scheduler"); - - var tasks = new List(); - - if (_keepAliveSettings.DisableKeepAliveTask == false) - { - tasks.Add(RegisterKeepAlive(_keepAliveSettings)); - } - - tasks.Add(RegisterScheduledPublishing()); - tasks.Add(RegisterLogScrubber(_loggingSettings)); - tasks.Add(RegisterTempFileCleanup()); - - return tasks.ToArray(); - }); - } - - private IBackgroundTask RegisterKeepAlive(KeepAliveSettings keepAliveSettings) - { - // ping/keepalive - // on all servers - var task = new KeepAlive(_keepAliveRunner, DefaultDelayMilliseconds, FiveMinuteMilliseconds, _requestAccessor, _mainDom, Options.Create(keepAliveSettings), _loggerFactory.CreateLogger(), _profilingLogger, _serverRegistrar); - _keepAliveRunner.TryAdd(task); - return task; - } - - private IBackgroundTask RegisterScheduledPublishing() - { - // scheduled publishing/unpublishing - // install on all, will only run on non-replica servers - var task = new ScheduledPublishing(_publishingRunner, DefaultDelayMilliseconds, OneMinuteMilliseconds, _runtime, _mainDom, _serverRegistrar, _contentService, _umbracoContextFactory, _loggerFactory.CreateLogger(), _serverMessenger, _backofficeSecurityFactory); - _publishingRunner.TryAdd(task); - return task; - } - - private IBackgroundTask RegisterLogScrubber(LoggingSettings settings) - { - // log scrubbing - // install on all, will only run on non-replica servers - var task = new LogScrubber(_scrubberRunner, DefaultDelayMilliseconds, LogScrubber.GetLogScrubbingInterval(), _mainDom, _serverRegistrar, _auditService, Options.Create(settings), _scopeProvider, _profilingLogger, _loggerFactory.CreateLogger()); - _scrubberRunner.TryAdd(task); - return task; - } - - private IBackgroundTask RegisterTempFileCleanup() - { - - var tempFolderPaths = new[] - { - _hostingEnvironment.MapPathContentRoot(Constants.SystemDirectories.TempFileUploads) - }; - - foreach (var tempFolderPath in tempFolderPaths) - { - //ensure it exists - Directory.CreateDirectory(tempFolderPath); - } - - // temp file cleanup, will run on all servers - even though file upload should only be handled on the master, this will - // ensure that in the case it happes on replicas that they are cleaned up. - var task = new TempFileCleanup(_fileCleanupRunner, DefaultDelayMilliseconds, OneHourMilliseconds, - tempFolderPaths.Select(x=>new DirectoryInfo(x)), - TimeSpan.FromDays(1), //files that are over a day old - _mainDom, _profilingLogger, _loggerFactory.CreateLogger()); - _scrubberRunner.TryAdd(task); - return task; - } - } -} diff --git a/src/Umbraco.Infrastructure/Scheduling/SchedulerComposer.cs b/src/Umbraco.Infrastructure/Scheduling/SchedulerComposer.cs deleted file mode 100644 index 5c56f3d314..0000000000 --- a/src/Umbraco.Infrastructure/Scheduling/SchedulerComposer.cs +++ /dev/null @@ -1,17 +0,0 @@ -using System; -using Umbraco.Core; -using Umbraco.Core.Composing; - -namespace Umbraco.Web.Scheduling -{ - /// - /// Used to do the scheduling for tasks, publishing, etc... - /// - /// - /// All tasks are run in a background task runner which is web aware and will wind down - /// the task correctly instead of killing it completely when the app domain shuts down. - /// - [RuntimeLevel(MinLevel = RuntimeLevel.Run)] - internal sealed class SchedulerComposer : ComponentComposer, ICoreComposer - { } -} diff --git a/src/Umbraco.Infrastructure/Umbraco.Infrastructure.csproj b/src/Umbraco.Infrastructure/Umbraco.Infrastructure.csproj index 2343ea806a..54ac7817e1 100644 --- a/src/Umbraco.Infrastructure/Umbraco.Infrastructure.csproj +++ b/src/Umbraco.Infrastructure/Umbraco.Infrastructure.csproj @@ -14,9 +14,10 @@ - + + diff --git a/src/Umbraco.Tests.Integration/Testing/IntegrationTestComposer.cs b/src/Umbraco.Tests.Integration/Testing/IntegrationTestComposer.cs index 1fc086c019..f64c2c48b1 100644 --- a/src/Umbraco.Tests.Integration/Testing/IntegrationTestComposer.cs +++ b/src/Umbraco.Tests.Integration/Testing/IntegrationTestComposer.cs @@ -42,7 +42,6 @@ namespace Umbraco.Tests.Integration.Testing { base.Compose(composition); - composition.Components().Remove(); composition.Components().Remove(); composition.Services.AddUnique(); composition.Services.AddUnique(factory => Mock.Of()); diff --git a/src/Umbraco.Tests.UnitTests/Umbraco.Infrastructure/HostedServices/HealthCheckNotifierTests.cs b/src/Umbraco.Tests.UnitTests/Umbraco.Infrastructure/HostedServices/HealthCheckNotifierTests.cs index 6506c227fc..bbbb4035c5 100644 --- a/src/Umbraco.Tests.UnitTests/Umbraco.Infrastructure/HostedServices/HealthCheckNotifierTests.cs +++ b/src/Umbraco.Tests.UnitTests/Umbraco.Infrastructure/HostedServices/HealthCheckNotifierTests.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Moq; @@ -23,71 +24,71 @@ namespace Umbraco.Tests.UnitTests.Umbraco.Infrastructure.HostedServices { private Mock _mockNotificationMethod; - private const string Check1Id = "00000000-0000-0000-0000-000000000001"; - private const string Check2Id = "00000000-0000-0000-0000-000000000002"; - private const string Check3Id = "00000000-0000-0000-0000-000000000003"; + private const string _check1Id = "00000000-0000-0000-0000-000000000001"; + private const string _check2Id = "00000000-0000-0000-0000-000000000002"; + private const string _check3Id = "00000000-0000-0000-0000-000000000003"; [Test] - public void Does_Not_Execute_When_Not_Enabled() + public async Task Does_Not_Execute_When_Not_Enabled() { var sut = CreateHealthCheckNotifier(enabled: false); - sut.ExecuteAsync(null); - _mockNotificationMethod.Verify(x => x.SendAsync(It.IsAny()), Times.Never); + await sut.PerformExecuteAsync(null); + VerifyNotificationsNotSent(); } [Test] - public void Does_Not_Execute_When_Runtime_State_Is_Not_Run() + public async Task Does_Not_Execute_When_Runtime_State_Is_Not_Run() { var sut = CreateHealthCheckNotifier(runtimeLevel: RuntimeLevel.Boot); - sut.ExecuteAsync(null); - _mockNotificationMethod.Verify(x => x.SendAsync(It.IsAny()), Times.Never); + await sut.PerformExecuteAsync(null); + VerifyNotificationsNotSent(); } [Test] - public void Does_Not_Execute_When_Server_Role_Is_Replica() + public async Task Does_Not_Execute_When_Server_Role_Is_Replica() { var sut = CreateHealthCheckNotifier(serverRole: ServerRole.Replica); - sut.ExecuteAsync(null); - _mockNotificationMethod.Verify(x => x.SendAsync(It.IsAny()), Times.Never); + await sut.PerformExecuteAsync(null); + VerifyNotificationsNotSent(); } [Test] - public void Does_Not_Execute_When_Server_Role_Is_Unknown() + public async Task Does_Not_Execute_When_Server_Role_Is_Unknown() { var sut = CreateHealthCheckNotifier(serverRole: ServerRole.Unknown); - sut.ExecuteAsync(null); - _mockNotificationMethod.Verify(x => x.SendAsync(It.IsAny()), Times.Never); + await sut.PerformExecuteAsync(null); + VerifyNotificationsNotSent(); } [Test] - public void Does_Not_Execute_When_Not_Main_Dom() + public async Task Does_Not_Execute_When_Not_Main_Dom() { var sut = CreateHealthCheckNotifier(isMainDom: false); - sut.ExecuteAsync(null); - _mockNotificationMethod.Verify(x => x.SendAsync(It.IsAny()), Times.Never); + await sut.PerformExecuteAsync(null); + VerifyNotificationsNotSent(); } [Test] - public void Does_Not_Execute_With_No_Enabled_Notification_Methods() + public async Task Does_Not_Execute_With_No_Enabled_Notification_Methods() { var sut = CreateHealthCheckNotifier(notificationEnabled: false); - sut.ExecuteAsync(null); - _mockNotificationMethod.Verify(x => x.SendAsync(It.IsAny()), Times.Never); + await sut.PerformExecuteAsync(null); + VerifyNotificationsNotSent(); } [Test] - public void Executes_With_Enabled_Notification_Methods() + public async Task Executes_With_Enabled_Notification_Methods() { var sut = CreateHealthCheckNotifier(); - sut.ExecuteAsync(null); - _mockNotificationMethod.Verify(x => x.SendAsync(It.IsAny()), Times.Once); + await sut.PerformExecuteAsync(null); + VerifyNotificationsSent(); } [Test] - public void Executes_Only_Enabled_Checks() + public async Task Executes_Only_Enabled_Checks() { var sut = CreateHealthCheckNotifier(); - sut.ExecuteAsync(null); + await sut.PerformExecuteAsync(null); _mockNotificationMethod.Verify(x => x.SendAsync(It.Is( y => y.ResultsAsDictionary.Count == 1 && y.ResultsAsDictionary.ContainsKey("Check1"))), Times.Once); } @@ -106,12 +107,12 @@ namespace Umbraco.Tests.UnitTests.Umbraco.Infrastructure.HostedServices Enabled = enabled, DisabledChecks = new List { - new DisabledHealthCheckSettings { Id = Guid.Parse(Check3Id) } + new DisabledHealthCheckSettings { Id = Guid.Parse(_check3Id) } } }, DisabledChecks = new List { - new DisabledHealthCheckSettings { Id = Guid.Parse(Check2Id) } + new DisabledHealthCheckSettings { Id = Guid.Parse(_check2Id) } } }; var checks = new HealthCheckCollection(new List @@ -143,17 +144,32 @@ namespace Umbraco.Tests.UnitTests.Umbraco.Infrastructure.HostedServices mockLogger.Object, mockProfilingLogger.Object); } - [HealthCheck(Check1Id, "Check1")] + private void VerifyNotificationsNotSent() + { + VerifyNotificationsSentTimes(Times.Never()); + } + + private void VerifyNotificationsSent() + { + VerifyNotificationsSentTimes(Times.Once()); + } + + private void VerifyNotificationsSentTimes(Times times) + { + _mockNotificationMethod.Verify(x => x.SendAsync(It.IsAny()), times); + } + + [HealthCheck(_check1Id, "Check1")] private class TestHealthCheck1 : TestHealthCheck { } - [HealthCheck(Check2Id, "Check2")] + [HealthCheck(_check2Id, "Check2")] private class TestHealthCheck2 : TestHealthCheck { } - [HealthCheck(Check3Id, "Check3")] + [HealthCheck(_check3Id, "Check3")] private class TestHealthCheck3 : TestHealthCheck { } diff --git a/src/Umbraco.Tests.UnitTests/Umbraco.Infrastructure/HostedServices/KeepAliveTests.cs b/src/Umbraco.Tests.UnitTests/Umbraco.Infrastructure/HostedServices/KeepAliveTests.cs new file mode 100644 index 0000000000..9fc1454b6d --- /dev/null +++ b/src/Umbraco.Tests.UnitTests/Umbraco.Infrastructure/HostedServices/KeepAliveTests.cs @@ -0,0 +1,124 @@ +using System; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Moq; +using Moq.Protected; +using NUnit.Framework; +using Umbraco.Core; +using Umbraco.Core.Configuration.Models; +using Umbraco.Core.Logging; +using Umbraco.Core.Scoping; +using Umbraco.Core.Sync; +using Umbraco.Infrastructure.HostedServices; +using Umbraco.Web; + +namespace Umbraco.Tests.UnitTests.Umbraco.Infrastructure.HostedServices +{ + [TestFixture] + public class KeepAliveTests + { + private Mock _mockHttpMessageHandler; + + private const string _applicationUrl = "https://mysite.com"; + + [Test] + public async Task Does_Not_Execute_When_Not_Enabled() + { + var sut = CreateKeepAlive(enabled: false); + await sut.PerformExecuteAsync(null); + VerifyKeepAliveRequestNotSent(); + } + + [Test] + public async Task Does_Not_Execute_When_Server_Role_Is_Replica() + { + var sut = CreateKeepAlive(serverRole: ServerRole.Replica); + await sut.PerformExecuteAsync(null); + VerifyKeepAliveRequestNotSent(); + } + + [Test] + public async Task Does_Not_Execute_When_Server_Role_Is_Unknown() + { + var sut = CreateKeepAlive(serverRole: ServerRole.Unknown); + await sut.PerformExecuteAsync(null); + VerifyKeepAliveRequestNotSent(); + } + + [Test] + public async Task Does_Not_Execute_When_Not_Main_Dom() + { + var sut = CreateKeepAlive(isMainDom: false); + await sut.PerformExecuteAsync(null); + VerifyKeepAliveRequestNotSent(); + } + + [Test] + public async Task Executes_And_Calls_Ping_Url() + { + var sut = CreateKeepAlive(); + await sut.PerformExecuteAsync(null); + VerifyKeepAliveRequestSent(); + } + + private KeepAlive CreateKeepAlive( + bool enabled = true, + ServerRole serverRole = ServerRole.Single, + bool isMainDom = true) + { + var settings = new KeepAliveSettings + { + DisableKeepAliveTask = !enabled, + }; + + var mockRequestAccessor = new Mock(); + mockRequestAccessor.Setup(x => x.GetApplicationUrl()).Returns(new Uri(_applicationUrl)); + + var mockServerRegistrar = new Mock(); + mockServerRegistrar.Setup(x => x.GetCurrentServerRole()).Returns(serverRole); + + var mockMainDom = new Mock(); + mockMainDom.SetupGet(x => x.IsMainDom).Returns(isMainDom); + + var mockScopeProvider = new Mock(); + var mockLogger = new Mock>(); + var mockProfilingLogger = new Mock(); + + _mockHttpMessageHandler = new Mock(); + _mockHttpMessageHandler.Protected() + .Setup>("SendAsync", ItExpr.IsAny(), ItExpr.IsAny()) + .ReturnsAsync(new HttpResponseMessage(HttpStatusCode.OK)) + .Verifiable(); + _mockHttpMessageHandler.As().Setup(s => s.Dispose()); + var httpClient = new HttpClient(_mockHttpMessageHandler.Object); + + var mockHttpClientFactory = new Mock(MockBehavior.Strict); + mockHttpClientFactory.Setup(x => x.CreateClient(It.IsAny())).Returns(httpClient); + + return new KeepAlive(mockRequestAccessor.Object, mockMainDom.Object, Options.Create(settings), + mockLogger.Object, mockProfilingLogger.Object, mockServerRegistrar.Object, mockHttpClientFactory.Object); + } + + private void VerifyKeepAliveRequestNotSent() + { + VerifyKeepAliveRequestSentTimes(Times.Never()); + } + + private void VerifyKeepAliveRequestSent() + { + VerifyKeepAliveRequestSentTimes(Times.Once()); + } + + private void VerifyKeepAliveRequestSentTimes(Times times) + { + _mockHttpMessageHandler.Protected().Verify("SendAsync", + times, + ItExpr.Is(x => x.RequestUri.ToString() == $"{_applicationUrl}/api/keepalive/ping"), + ItExpr.IsAny()); + } + } +} diff --git a/src/Umbraco.Tests.UnitTests/Umbraco.Infrastructure/HostedServices/LogScrubberTests.cs b/src/Umbraco.Tests.UnitTests/Umbraco.Infrastructure/HostedServices/LogScrubberTests.cs new file mode 100644 index 0000000000..d30c83c545 --- /dev/null +++ b/src/Umbraco.Tests.UnitTests/Umbraco.Infrastructure/HostedServices/LogScrubberTests.cs @@ -0,0 +1,102 @@ +using System; +using System.Data; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Moq; +using NUnit.Framework; +using Umbraco.Core; +using Umbraco.Core.Configuration.Models; +using Umbraco.Core.Events; +using Umbraco.Core.Logging; +using Umbraco.Core.Scoping; +using Umbraco.Core.Services; +using Umbraco.Core.Sync; +using Umbraco.Infrastructure.HostedServices; + +namespace Umbraco.Tests.UnitTests.Umbraco.Infrastructure.HostedServices +{ + [TestFixture] + public class LogScrubberTests + { + private Mock _mockAuditService; + + const int _maxLogAgeInMinutes = 60; + + [Test] + public async Task Does_Not_Execute_When_Server_Role_Is_Replica() + { + var sut = CreateLogScrubber(serverRole: ServerRole.Replica); + await sut.PerformExecuteAsync(null); + VerifyLogsNotScrubbed(); + } + + [Test] + public async Task Does_Not_Execute_When_Server_Role_Is_Unknown() + { + var sut = CreateLogScrubber(serverRole: ServerRole.Unknown); + await sut.PerformExecuteAsync(null); + VerifyLogsNotScrubbed(); + } + + [Test] + public async Task Does_Not_Execute_When_Not_Main_Dom() + { + var sut = CreateLogScrubber(isMainDom: false); + await sut.PerformExecuteAsync(null); + VerifyLogsNotScrubbed(); + } + + [Test] + public async Task Executes_And_Scrubs_Logs() + { + var sut = CreateLogScrubber(); + await sut.PerformExecuteAsync(null); + VerifyLogsScrubbed(); + } + + private LogScrubber CreateLogScrubber( + ServerRole serverRole = ServerRole.Single, + bool isMainDom = true) + { + var settings = new LoggingSettings + { + MaxLogAge = TimeSpan.FromMinutes(_maxLogAgeInMinutes), + }; + + var mockServerRegistrar = new Mock(); + mockServerRegistrar.Setup(x => x.GetCurrentServerRole()).Returns(serverRole); + + var mockMainDom = new Mock(); + mockMainDom.SetupGet(x => x.IsMainDom).Returns(isMainDom); + + var mockScope = new Mock(); + var mockScopeProvider = new Mock(); + mockScopeProvider + .Setup(x => x.CreateScope(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(mockScope.Object); + var mockLogger = new Mock>(); + var mockProfilingLogger = new Mock(); + + _mockAuditService = new Mock(); + + return new LogScrubber(mockMainDom.Object, mockServerRegistrar.Object, _mockAuditService.Object, + Options.Create(settings), mockScopeProvider.Object, mockLogger.Object, mockProfilingLogger.Object); + } + + private void VerifyLogsNotScrubbed() + { + VerifyLogsScrubbed(Times.Never()); + } + + private void VerifyLogsScrubbed() + { + VerifyLogsScrubbed(Times.Once()); + } + + private void VerifyLogsScrubbed(Times times) + { + _mockAuditService.Verify(x => x.CleanLogs(It.Is(y => y == _maxLogAgeInMinutes)), times); + } + } +} diff --git a/src/Umbraco.Tests.UnitTests/Umbraco.Infrastructure/HostedServices/ScheduledPublishingTests.cs b/src/Umbraco.Tests.UnitTests/Umbraco.Infrastructure/HostedServices/ScheduledPublishingTests.cs new file mode 100644 index 0000000000..e0fd2a4acc --- /dev/null +++ b/src/Umbraco.Tests.UnitTests/Umbraco.Infrastructure/HostedServices/ScheduledPublishingTests.cs @@ -0,0 +1,124 @@ +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Moq; +using NUnit.Framework; +using Umbraco.Core; +using Umbraco.Core.Logging; +using Umbraco.Core.Services; +using Umbraco.Core.Sync; +using Umbraco.Infrastructure.HostedServices; +using Umbraco.Web; +using LogLevel = Microsoft.Extensions.Logging.LogLevel; + +namespace Umbraco.Tests.UnitTests.Umbraco.Infrastructure.HostedServices +{ + [TestFixture] + public class ScheduledPublishingTests + { + private Mock _mockContentService; + private Mock> _mockLogger; + + [Test] + public async Task Does_Not_Execute_When_Not_Enabled() + { + var sut = CreateScheduledPublishing(enabled: false); + await sut.PerformExecuteAsync(null); + VerifyScheduledPublishingNotPerformed(); + } + + [Test] + public async Task Does_Not_Execute_When_Runtime_State_Is_Not_Run() + { + var sut = CreateScheduledPublishing(runtimeLevel: RuntimeLevel.Boot); + await sut.PerformExecuteAsync(null); + VerifyScheduledPublishingNotPerformed(); + } + + [Test] + public async Task Does_Not_Execute_When_Server_Role_Is_Replica() + { + var sut = CreateScheduledPublishing(serverRole: ServerRole.Replica); + await sut.PerformExecuteAsync(null); + VerifyScheduledPublishingNotPerformed(); + } + + [Test] + public async Task Does_Not_Execute_When_Server_Role_Is_Unknown() + { + var sut = CreateScheduledPublishing(serverRole: ServerRole.Unknown); + await sut.PerformExecuteAsync(null); + VerifyScheduledPublishingNotPerformed(); + } + + [Test] + public async Task Does_Not_Execute_When_Not_Main_Dom() + { + var sut = CreateScheduledPublishing(isMainDom: false); + await sut.PerformExecuteAsync(null); + VerifyScheduledPublishingNotPerformed(); + } + + [Test] + public async Task Executes_And_Performs_Scheduled_Publishing() + { + var sut = CreateScheduledPublishing(); + await sut.PerformExecuteAsync(null); + VerifyScheduledPublishingPerformed(); + } + + private ScheduledPublishing CreateScheduledPublishing( + bool enabled = true, + RuntimeLevel runtimeLevel = RuntimeLevel.Run, + ServerRole serverRole = ServerRole.Single, + bool isMainDom = true) + { + if (enabled) + { + Suspendable.ScheduledPublishing.Resume(); + } + else + { + Suspendable.ScheduledPublishing.Suspend(); + } + + var mockRunTimeState = new Mock(); + mockRunTimeState.SetupGet(x => x.Level).Returns(runtimeLevel); + + var mockServerRegistrar = new Mock(); + mockServerRegistrar.Setup(x => x.GetCurrentServerRole()).Returns(serverRole); + + var mockMainDom = new Mock(); + mockMainDom.SetupGet(x => x.IsMainDom).Returns(isMainDom); + + _mockContentService = new Mock(); + + var mockUmbracoContextFactory = new Mock(); + mockUmbracoContextFactory.Setup(x => x.EnsureUmbracoContext()).Returns(new UmbracoContextReference(null, false, null)); + + _mockLogger = new Mock>(); + + var mockServerMessenger = new Mock(); + + var mockBackOfficeSecurityFactory = new Mock(); + + return new ScheduledPublishing(mockRunTimeState.Object, mockMainDom.Object, mockServerRegistrar.Object, _mockContentService.Object, + mockUmbracoContextFactory.Object, _mockLogger.Object, mockServerMessenger.Object, mockBackOfficeSecurityFactory.Object); + } + + private void VerifyScheduledPublishingNotPerformed() + { + VerifyScheduledPublishingPerformed(Times.Never()); + } + + private void VerifyScheduledPublishingPerformed() + { + VerifyScheduledPublishingPerformed(Times.Once()); + } + + private void VerifyScheduledPublishingPerformed(Times times) + { + _mockContentService.Verify(x => x.PerformScheduledPublish(It.IsAny()), times); + } + } +} diff --git a/src/Umbraco.Tests.UnitTests/Umbraco.Infrastructure/HostedServices/TempFileCleanupTests.cs b/src/Umbraco.Tests.UnitTests/Umbraco.Infrastructure/HostedServices/TempFileCleanupTests.cs new file mode 100644 index 0000000000..7feda1e9da --- /dev/null +++ b/src/Umbraco.Tests.UnitTests/Umbraco.Infrastructure/HostedServices/TempFileCleanupTests.cs @@ -0,0 +1,68 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Moq; +using NUnit.Framework; +using Umbraco.Core; +using Umbraco.Core.IO; +using Umbraco.Core.Logging; +using Umbraco.Infrastructure.HostedServices; + +namespace Umbraco.Tests.UnitTests.Umbraco.Infrastructure.HostedServices +{ + [TestFixture] + public class TempFileCleanupTests + { + private Mock _mockIOHelper; + private string _testPath = @"c:\test\temp\path"; + + [Test] + public async Task Does_Not_Execute_When_Not_Main_Dom() + { + var sut = CreateTempFileCleanup(isMainDom: false); + await sut.PerformExecuteAsync(null); + VerifyFilesNotCleaned(); + } + + [Test] + public async Task Executes_And_Cleans_Files() + { + var sut = CreateTempFileCleanup(); + await sut.PerformExecuteAsync(null); + VerifyFilesCleaned(); + } + + private TempFileCleanup CreateTempFileCleanup(bool isMainDom = true) + { + var mockMainDom = new Mock(); + mockMainDom.SetupGet(x => x.IsMainDom).Returns(isMainDom); + + _mockIOHelper = new Mock(); + _mockIOHelper.Setup(x => x.GetTempFolders()) + .Returns(new DirectoryInfo[] { new DirectoryInfo(_testPath) }); + _mockIOHelper.Setup(x => x.CleanFolder(It.IsAny(), It.IsAny())) + .Returns(CleanFolderResult.Success()); + + var mockLogger = new Mock>(); + var mockProfilingLogger = new Mock(); + + return new TempFileCleanup(_mockIOHelper.Object, mockMainDom.Object, mockLogger.Object); + } + + private void VerifyFilesNotCleaned() + { + VerifyFilesCleaned(Times.Never()); + } + + private void VerifyFilesCleaned() + { + VerifyFilesCleaned(Times.Once()); + } + + private void VerifyFilesCleaned(Times times) + { + _mockIOHelper.Verify(x => x.CleanFolder(It.Is(y => y.FullName == _testPath), It.IsAny()), times); + } + } +} diff --git a/src/Umbraco.Web.BackOffice/Extensions/UmbracoBuilderExtensions.cs b/src/Umbraco.Web.BackOffice/Extensions/UmbracoBuilderExtensions.cs index 649b8b76b7..c494425274 100644 --- a/src/Umbraco.Web.BackOffice/Extensions/UmbracoBuilderExtensions.cs +++ b/src/Umbraco.Web.BackOffice/Extensions/UmbracoBuilderExtensions.cs @@ -17,7 +17,8 @@ namespace Umbraco.Extensions .WithMvcAndRazor() .WithWebServer() .WithPreview() - .WithHostedServices(); + .WithHostedServices() + .WithHttpClients(); } public static IUmbracoBuilder WithBackOffice(this IUmbracoBuilder builder) diff --git a/src/Umbraco.Web.Common/Builder/UmbracoBuilderExtensions.cs b/src/Umbraco.Web.Common/Builder/UmbracoBuilderExtensions.cs index 2ac4130fe4..635594a77d 100644 --- a/src/Umbraco.Web.Common/Builder/UmbracoBuilderExtensions.cs +++ b/src/Umbraco.Web.Common/Builder/UmbracoBuilderExtensions.cs @@ -36,6 +36,9 @@ namespace Umbraco.Web.Common.Builder public static IUmbracoBuilder WithHostedServices(this IUmbracoBuilder builder) => builder.AddWith(nameof(WithHostedServices), () => builder.Services.AddUmbracoHostedServices()); + public static IUmbracoBuilder WithHttpClients(this IUmbracoBuilder builder) + => builder.AddWith(nameof(WithHttpClients), () => builder.Services.AddUmbracoHttpClients()); + public static IUmbracoBuilder WithMiniProfiler(this IUmbracoBuilder builder) => builder.AddWith(nameof(WithMiniProfiler), () => builder.Services.AddMiniProfiler(options => diff --git a/src/Umbraco.Web.Common/Extensions/UmbracoCoreServiceCollectionExtensions.cs b/src/Umbraco.Web.Common/Extensions/UmbracoCoreServiceCollectionExtensions.cs index 704a7ce066..80be943eb7 100644 --- a/src/Umbraco.Web.Common/Extensions/UmbracoCoreServiceCollectionExtensions.cs +++ b/src/Umbraco.Web.Common/Extensions/UmbracoCoreServiceCollectionExtensions.cs @@ -293,10 +293,25 @@ namespace Umbraco.Extensions public static IServiceCollection AddUmbracoHostedServices(this IServiceCollection services) { services.AddHostedService(); + services.AddHostedService(); + services.AddHostedService(); + services.AddHostedService(); + services.AddHostedService(); return services; } + /// + /// Adds HTTP clients for Umbraco. + /// + /// + /// + public static IServiceCollection AddUmbracoHttpClients(this IServiceCollection services) + { + services.AddHttpClient(); + return services; + } + private static ITypeFinder CreateTypeFinder(ILoggerFactory loggerFactory, IProfiler profiler, IWebHostEnvironment webHostEnvironment, Assembly entryAssembly, IOptionsMonitor typeFinderSettings) { var runtimeHashPaths = new RuntimeHashPaths(); diff --git a/src/Umbraco.Web.Common/Runtime/AspNetCoreComposer.cs b/src/Umbraco.Web.Common/Runtime/AspNetCoreComposer.cs index 39742dcb0d..2ba38b6e02 100644 --- a/src/Umbraco.Web.Common/Runtime/AspNetCoreComposer.cs +++ b/src/Umbraco.Web.Common/Runtime/AspNetCoreComposer.cs @@ -49,25 +49,21 @@ namespace Umbraco.Web.Common.Runtime composition.Services.AddUnique(); // Our own netcore implementations - composition.Services.AddUnique(); - composition.Services.AddUnique(); + composition.Services.AddMultipleUnique(); composition.Services.AddUnique(); // The umbraco request lifetime - composition.Services.AddUnique(); - composition.Services.AddUnique(); + composition.Services.AddMultipleUnique(); - //Password hasher + // Password hasher composition.Services.AddUnique(); - composition.Services.AddUnique(); composition.Services.AddTransient(); composition.Services.AddUnique(); - composition.Services.AddUnique(); - composition.Services.AddUnique(); + composition.Services.AddMultipleUnique(); composition.Services.AddUnique(); @@ -76,7 +72,6 @@ namespace Umbraco.Web.Common.Runtime composition.Services.AddUnique(); composition.Services.AddUnique(); - // register the umbraco context factory composition.Services.AddUnique(); composition.Services.AddUnique(); diff --git a/src/Umbraco.Web.Website/Extensions/ActionContextExtensions.cs b/src/Umbraco.Web.Website/Extensions/ActionContextExtensions.cs new file mode 100644 index 0000000000..5a79fcf188 --- /dev/null +++ b/src/Umbraco.Web.Website/Extensions/ActionContextExtensions.cs @@ -0,0 +1,29 @@ +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Mvc.Controllers; + +namespace Umbraco.Extensions +{ + public static class ActionContextExtensions + { + /// + /// Recursively gets a data token from a controller context hierarchy. + /// + /// The action context. + /// The name of the data token. + /// The data token, or null. + /// MIGRAGED TO NETCORE AS EXTENSION ON HTTPCONTEXT + internal static object GetDataTokenInViewContextHierarchy(this ActionContext actionContext, string dataTokenName) + { + + var controllerActionDescriptor = actionContext.ActionDescriptor as ControllerActionDescriptor; + while (!(controllerActionDescriptor is null)) + { + object token; + if (actionContext.RouteData.DataTokens.TryGetValue(dataTokenName, out token)) + return token; + controllerActionDescriptor = controllerActionDescriptor.ParentActionViewContext; + } + return null; + } + } +} diff --git a/src/Umbraco.Web.Website/ViewEngines/RenderViewEngine.cs b/src/Umbraco.Web.Website/ViewEngines/RenderViewEngine.cs index 2c81687091..50e70b68a7 100644 --- a/src/Umbraco.Web.Website/ViewEngines/RenderViewEngine.cs +++ b/src/Umbraco.Web.Website/ViewEngines/RenderViewEngine.cs @@ -1,11 +1,17 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Linq; +using System.Text.Encodings.Web; using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.Mvc.Razor; using Microsoft.AspNetCore.Mvc.ViewEngines; +using Microsoft.AspNetCore.Routing; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Umbraco.Core.Hosting; +using Umbraco.Extensions; using Umbraco.Web.Models; namespace Umbraco.Web.Website.ViewEngines @@ -16,13 +22,39 @@ namespace Umbraco.Web.Website.ViewEngines /// public class RenderViewEngine : RazorViewEngine { - private readonly IHostingEnvironment _hostingEnvironment; - private readonly IEnumerable _supplementedViewLocations = new[] { "/{0}.cshtml" }; - //NOTE: we will make the main view location the last to be searched since if it is the first to be searched and there is both a view and a partial - // view in both locations and the main view is rendering a partial view with the same name, we will get a stack overflow exception. - // http://issues.umbraco.org/issue/U4-1287, http://issues.umbraco.org/issue/U4-1215 - private readonly IEnumerable _supplementedPartialViewLocations = new[] { "/Partials/{0}.cshtml", "/MacroPartials/{0}.cshtml", "/{0}.cshtml" }; + public RenderViewEngine( + IRazorPageFactoryProvider pageFactory, + IRazorPageActivator pageActivator, + HtmlEncoder htmlEncoder, + ILoggerFactory loggerFactory, + DiagnosticListener diagnosticListener) + : base(pageFactory, pageActivator, htmlEncoder, OverrideViewLocations(), loggerFactory, diagnosticListener) + { + } + + private static IOptions OverrideViewLocations() + { + return Options.Create(new RazorViewEngineOptions() + { + //NOTE: we will make the main view location the last to be searched since if it is the first to be searched and there is both a view and a partial + // view in both locations and the main view is rendering a partial view with the same name, we will get a stack overflow exception. + // http://issues.umbraco.org/issue/U4-1287, http://issues.umbraco.org/issue/U4-1215 + ViewLocationFormats = + { + "/Partials/{0}.cshtml", + "/MacroPartials/{0}.cshtml", + "/{0}.cshtml" + }, + }); + } + + public new ViewEngineResult FindView(ActionContext context, string viewName, bool isMainPage) + { + return ShouldFindView(context, viewName, isMainPage) + ? base.FindView(context, viewName, isMainPage) + : ViewEngineResult.NotFound(viewName, Array.Empty()); + } // /// // /// Constructor @@ -43,44 +75,20 @@ namespace Umbraco.Web.Website.ViewEngines // EnsureFoldersAndFiles(); // } - /// - /// Ensures that the correct web.config for razor exists in the /Views folder, the partials folder exist and the ViewStartPage exists. - /// - private void EnsureFoldersAndFiles() - { - var viewFolder = _hostingEnvironment.MapPathContentRoot(Constants.ViewLocation); - // ensure the web.config file is in the ~/Views folder - Directory.CreateDirectory(viewFolder); - var webConfigPath = Path.Combine(viewFolder, "web.config"); - if (File.Exists(webConfigPath) == false) - { - using (var writer = File.CreateText(webConfigPath)) - { - writer.Write(Strings.WebConfigTemplate); - } - } - - //auto create the partials folder - var partialsFolder = Path.Combine(viewFolder, "Partials"); - Directory.CreateDirectory(partialsFolder); - - // We could create a _ViewStart page if it isn't there as well, but we may not allow editing of this page in the back office. - } - - public override ViewEngineResult FindView(ControllerContext controllerContext, string viewName, string masterName, bool useCache) - { - return ShouldFindView(controllerContext, false) - ? base.FindView(controllerContext, viewName, masterName, useCache) - : new ViewEngineResult(new string[] { }); - } - - public override ViewEngineResult FindPartialView(ControllerContext controllerContext, string partialViewName, bool useCache) - { - return ShouldFindView(controllerContext, true) - ? base.FindPartialView(controllerContext, partialViewName, useCache) - : new ViewEngineResult(new string[] { }); - } + // public override ViewEngineResult FindView(ControllerContext controllerContext, string viewName, string masterName, bool useCache) + // { + // return ShouldFindView(controllerContext, false) + // ? base.FindView(controllerContext, viewName, masterName, useCache) + // : new ViewEngineResult(new string[] { }); + // } + // + // public override ViewEngineResult FindPartialView(ControllerContext controllerContext, string partialViewName, bool useCache) + // { + // return ShouldFindView(controllerContext, true) + // ? base.FindPartialView(controllerContext, partialViewName, useCache) + // : new ViewEngineResult(new string[] { }); + // } /// /// Determines if the view should be found, this is used for view lookup performance and also to ensure @@ -90,10 +98,11 @@ namespace Umbraco.Web.Website.ViewEngines /// /// /// - private static bool ShouldFindView(ControllerContext controllerContext, bool isPartial) + private static bool ShouldFindView(ActionContext context, string viewName, bool isMainPage) { - var umbracoToken = controllerContext.GetDataTokenInViewContextHierarchy(Core.Constants.Web.UmbracoDataToken); + var umbracoToken = context.GetDataTokenInViewContextHierarchy(Core.Constants.Web.UmbracoDataToken); + context.ActionDescriptor. // first check if we're rendering a partial view for the back office, or surface controller, etc... // anything that is not ContentModel as this should only pertain to Umbraco views. if (isPartial && !(umbracoToken is ContentModel)) @@ -102,5 +111,7 @@ namespace Umbraco.Web.Website.ViewEngines // only find views if we're rendering the umbraco front end return umbracoToken is ContentModel; } + + } } diff --git a/src/Umbraco.Web/Mvc/ControllerContextExtensions.cs b/src/Umbraco.Web/Mvc/ControllerContextExtensions.cs index 4baaaac4fc..7179f54c87 100644 --- a/src/Umbraco.Web/Mvc/ControllerContextExtensions.cs +++ b/src/Umbraco.Web/Mvc/ControllerContextExtensions.cs @@ -22,6 +22,7 @@ namespace Umbraco.Web.Mvc /// The controller context. /// The name of the data token. /// The data token, or null. + /// MIGRAGED TO NETCORE AS EXTENSION ON ActionContext internal static object GetDataTokenInViewContextHierarchy(this ControllerContext controllerContext, string dataTokenName) { var context = controllerContext;