From b6695b6953177e6ec06b6e09df762914267e1d0e Mon Sep 17 00:00:00 2001 From: Gareth Evans - Cloud46 Limited Date: Thu, 20 Nov 2014 14:48:26 +0000 Subject: [PATCH 1/5] Fixed type in Step 4 install wizard - U4-3556. --- src/Umbraco.Web.UI/install/steps/defaultUser.ascx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Umbraco.Web.UI/install/steps/defaultUser.ascx b/src/Umbraco.Web.UI/install/steps/defaultUser.ascx index 4fec7e243c..0e8b18f6e8 100644 --- a/src/Umbraco.Web.UI/install/steps/defaultUser.ascx +++ b/src/Umbraco.Web.UI/install/steps/defaultUser.ascx @@ -8,7 +8,7 @@

Create User

-

You can now setup a new admin user to log into Umbraco, we recommend using a stong password for this (a password which is more than 4 characters and contains a mix of letters, numbers and symbols). +

You can now setup a new admin user to log into Umbraco, we recommend using a strong password for this (a password which is more than 4 characters and contains a mix of letters, numbers and symbols). Please make a note of the chosen password.

The password can be changed once you have completed the installation and logged into the admin interface.

From 4eb9a54fa5d4e0330b6adc178191221d41fde165 Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Sun, 7 Dec 2014 16:16:29 +0100 Subject: [PATCH 2/5] Ports v7 changes for U4-5728 - OriginalRequestUrl not initialized, KeepAlive or ScheduledPublishing run too soon #U4-5728 --- .../Sync/ServerEnvironmentHelper.cs | 36 +- .../Scheduling/BackgroundTaskRunner.cs | 363 ++++++++++++++++++ src/Umbraco.Web/Scheduling/IBackgroundTask.cs | 9 + src/Umbraco.Web/Scheduling/KeepAlive.cs | 33 +- src/Umbraco.Web/Scheduling/LogScrubber.cs | 49 +-- .../Scheduling/ScheduledPublishing.cs | 50 ++- src/Umbraco.Web/Scheduling/ScheduledTasks.cs | 64 +-- src/Umbraco.Web/Scheduling/Scheduler.cs | 108 ++++-- src/Umbraco.Web/Scheduling/TaskEventArgs.cs | 22 ++ src/Umbraco.Web/Umbraco.Web.csproj | 7 +- 10 files changed, 596 insertions(+), 145 deletions(-) create mode 100644 src/Umbraco.Web/Scheduling/BackgroundTaskRunner.cs create mode 100644 src/Umbraco.Web/Scheduling/IBackgroundTask.cs create mode 100644 src/Umbraco.Web/Scheduling/TaskEventArgs.cs diff --git a/src/Umbraco.Core/Sync/ServerEnvironmentHelper.cs b/src/Umbraco.Core/Sync/ServerEnvironmentHelper.cs index 73dca4cc98..c39b2805fc 100644 --- a/src/Umbraco.Core/Sync/ServerEnvironmentHelper.cs +++ b/src/Umbraco.Core/Sync/ServerEnvironmentHelper.cs @@ -17,15 +17,19 @@ namespace Umbraco.Core.Sync /// status. This will attempt to determine the internal umbraco base url that can be used by the current /// server to send a request to itself if it is in a load balanced environment. /// - /// The full base url including schema (i.e. http://myserver:80/umbraco ) - public static string GetCurrentServerUmbracoBaseUrl() + /// The full base url including schema (i.e. http://myserver:80/umbraco ) - or null if the url + /// cannot be determined at the moment (usually because the first request has not properly completed yet). + public static string GetCurrentServerUmbracoBaseUrl(ApplicationContext appContext) { var status = GetStatus(); if (status == CurrentServerEnvironmentStatus.Single) { - //if it's a single install, then the base url has to be the first url registered - return string.Format("http://{0}", ApplicationContext.Current.OriginalRequestUrl); + // single install, return null if no original url, else use original url as base + // use http or https as appropriate + return string.IsNullOrWhiteSpace(appContext.OriginalRequestUrl) + ? null // not initialized yet + : string.Format("http{0}://{1}", GlobalSettings.UseSSL ? "s" : "", appContext.OriginalRequestUrl); } var servers = UmbracoSettings.DistributionServers; @@ -33,8 +37,9 @@ namespace Umbraco.Core.Sync var nodes = servers.SelectNodes("./server"); if (nodes == null) { - //cannot be determined, then the base url has to be the first url registered - return string.Format("http://{0}", ApplicationContext.Current.OriginalRequestUrl); + return string.IsNullOrWhiteSpace(appContext.OriginalRequestUrl) + ? null // not initialized yet + : string.Format("http{0}://{1}", GlobalSettings.UseSSL ? "s" : "", appContext.OriginalRequestUrl); } var xmlNodes = nodes.Cast().ToList(); @@ -58,11 +63,14 @@ namespace Umbraco.Core.Sync xmlNode.InnerText, xmlNode.AttributeValue("forcePortnumber").IsNullOrWhiteSpace() ? "80" : xmlNode.AttributeValue("forcePortnumber"), IOHelper.ResolveUrl(SystemDirectories.Umbraco).TrimStart('/')); - } + } } - - //cannot be determined, then the base url has to be the first url registered - return string.Format("http://{0}", ApplicationContext.Current.OriginalRequestUrl); + + // cannot be determined, return null if no original url, else use original url as base + // use http or https as appropriate + return string.IsNullOrWhiteSpace(appContext.OriginalRequestUrl) + ? null // not initialized yet + : string.Format("http{0}://{1}", GlobalSettings.UseSSL ? "s" : "", appContext.OriginalRequestUrl); } /// @@ -85,7 +93,7 @@ namespace Umbraco.Core.Sync } var master = nodes.Cast().FirstOrDefault(); - + if (master == null) { return CurrentServerEnvironmentStatus.Unknown; @@ -104,12 +112,12 @@ namespace Umbraco.Core.Sync } if ((appId.IsNullOrWhiteSpace() == false && appId.Trim().InvariantEquals(HttpRuntime.AppDomainAppId)) - || (serverName.IsNullOrWhiteSpace() == false && serverName.Trim().InvariantEquals(NetworkHelper.MachineName))) + || (serverName.IsNullOrWhiteSpace() == false && serverName.Trim().InvariantEquals(NetworkHelper.MachineName))) { //match by appdid or server name! - return CurrentServerEnvironmentStatus.Master; + return CurrentServerEnvironmentStatus.Master; } - + return CurrentServerEnvironmentStatus.Slave; } } diff --git a/src/Umbraco.Web/Scheduling/BackgroundTaskRunner.cs b/src/Umbraco.Web/Scheduling/BackgroundTaskRunner.cs new file mode 100644 index 0000000000..d1608aee85 --- /dev/null +++ b/src/Umbraco.Web/Scheduling/BackgroundTaskRunner.cs @@ -0,0 +1,363 @@ +using System; +using System.Collections.Concurrent; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using System.Web.Hosting; +using Umbraco.Core.Logging; + +namespace Umbraco.Web.Scheduling +{ + /// + /// This is used to create a background task runner which will stay alive in the background of and complete + /// any tasks that are queued. It is web aware and will ensure that it is shutdown correctly when the app domain + /// is shutdown. + /// + /// + internal class BackgroundTaskRunner : IDisposable, IRegisteredObject + where T : IBackgroundTask + { + private readonly bool _dedicatedThread; + private readonly bool _persistentThread; + private readonly BlockingCollection _tasks = new BlockingCollection(); + private Task _consumer; + + private volatile bool _isRunning = false; + private static readonly object Locker = new object(); + private CancellationTokenSource _tokenSource; + internal event EventHandler> TaskError; + internal event EventHandler> TaskStarting; + internal event EventHandler> TaskCompleted; + internal event EventHandler> TaskCancelled; + + public BackgroundTaskRunner(bool dedicatedThread = false, bool persistentThread = false) + { + _dedicatedThread = dedicatedThread; + _persistentThread = persistentThread; + HostingEnvironment.RegisterObject(this); + } + + public int TaskCount + { + get { return _tasks.Count; } + } + + public bool IsRunning + { + get { return _isRunning; } + } + + public TaskStatus TaskStatus + { + get { return _consumer.Status; } + } + + public void Add(T task) + { + //add any tasks first + LogHelper.Debug>(" Task added {0}", () => task.GetType()); + _tasks.Add(task); + + //ensure's everything is started + StartUp(); + } + + public void StartUp() + { + if (!_isRunning) + { + lock (Locker) + { + //double check + if (!_isRunning) + { + _isRunning = true; + //Create a new token source since this is a new proces + _tokenSource = new CancellationTokenSource(); + StartConsumer(); + LogHelper.Debug>("Starting"); + } + } + } + } + + public void ShutDown() + { + lock (Locker) + { + _isRunning = false; + + try + { + if (_consumer != null) + { + //cancel all operations + _tokenSource.Cancel(); + + try + { + _consumer.Wait(); + } + catch (AggregateException e) + { + //NOTE: We are logging Debug because we are expecting these errors + + LogHelper.Debug>("AggregateException thrown with the following inner exceptions:"); + // Display information about each exception. + foreach (var v in e.InnerExceptions) + { + var exception = v as TaskCanceledException; + if (exception != null) + { + LogHelper.Debug>(" .Net TaskCanceledException: .Net Task ID {0}", () => exception.Task.Id); + } + else + { + LogHelper.Debug>(" Exception: {0}", () => v.GetType().Name); + } + } + } + } + + if (_tasks.Count > 0) + { + LogHelper.Debug>("Processing remaining tasks before shutdown: {0}", () => _tasks.Count); + + //now we need to ensure the remaining queue is processed if there's any remaining, + // this will all be processed on the current/main thread. + T remainingTask; + while (_tasks.TryTake(out remainingTask)) + { + ConsumeTaskInternal(remainingTask); + } + } + + LogHelper.Debug>("Shutdown"); + + //disposing these is really optional since they'll be disposed immediately since they are no longer running + //but we'll put this here anyways. + if (_consumer != null && (_consumer.IsCompleted || _consumer.IsCanceled)) + { + _consumer.Dispose(); + } + } + catch (Exception ex) + { + LogHelper.Error>("Error occurred shutting down task runner", ex); + } + finally + { + HostingEnvironment.UnregisterObject(this); + } + } + } + + /// + /// Starts the consumer task + /// + private void StartConsumer() + { + var token = _tokenSource.Token; + + _consumer = Task.Factory.StartNew(() => + StartThread(token), + token, + _dedicatedThread ? TaskCreationOptions.LongRunning : TaskCreationOptions.None, + TaskScheduler.Default); + + //if this is not a persistent thread, wait till it's done and shut ourselves down + // thus ending the thread or giving back to the thread pool. If another task is added + // another thread will spawn or be taken from the pool to process. + if (!_persistentThread) + { + _consumer.ContinueWith(task => ShutDown()); + } + + } + + /// + /// Invokes a new worker thread to consume tasks + /// + /// + private void StartThread(CancellationToken token) + { + // Was cancellation already requested? + if (token.IsCancellationRequested) + { + LogHelper.Info>("Thread {0} was cancelled before it got started.", () => Thread.CurrentThread.ManagedThreadId); + token.ThrowIfCancellationRequested(); + } + + TakeAndConsumeTask(token); + } + + /// + /// Trys to get a task from the queue, if there isn't one it will wait a second and try again + /// + /// + private void TakeAndConsumeTask(CancellationToken token) + { + if (token.IsCancellationRequested) + { + LogHelper.Info>("Thread {0} was cancelled.", () => Thread.CurrentThread.ManagedThreadId); + token.ThrowIfCancellationRequested(); + } + + //If this is true, the thread will stay alive and just wait until there is anything in the queue + // and process it. When there is nothing in the queue, the thread will just block until there is + // something to process. + //When this is false, the thread will process what is currently in the queue and once that is + // done, the thread will end and we will shutdown the process + + if (_persistentThread) + { + //This will iterate over the collection, if there is nothing to take + // the thread will block until there is something available. + //We need to pass our cancellation token so that the thread will + // cancel when we shutdown + foreach (var t in _tasks.GetConsumingEnumerable(token)) + { + ConsumeTaskCancellable(t, token); + } + + //recurse and keep going + TakeAndConsumeTask(token); + } + else + { + T repositoryTask; + while (_tasks.TryTake(out repositoryTask)) + { + ConsumeTaskCancellable(repositoryTask, token); + } + + //the task will end here + } + } + + internal void ConsumeTaskCancellable(T task, CancellationToken token) + { + if (token.IsCancellationRequested) + { + OnTaskCancelled(new TaskEventArgs(task)); + + //NOTE: Since the task hasn't started this is pretty pointless so leaving it out. + LogHelper.Info>("Task {0}) was cancelled.", + () => task.GetType()); + + token.ThrowIfCancellationRequested(); + } + + ConsumeTaskInternal(task); + } + + private void ConsumeTaskInternal(T task) + { + try + { + OnTaskStarting(new TaskEventArgs(task)); + + try + { + using (task) + { + task.Run(); + } + } + catch (Exception e) + { + OnTaskError(new TaskEventArgs(task, e)); + throw; + } + + OnTaskCompleted(new TaskEventArgs(task)); + } + catch (Exception ex) + { + LogHelper.Error>("An error occurred consuming task", ex); + } + } + + protected virtual void OnTaskError(TaskEventArgs e) + { + var handler = TaskError; + if (handler != null) handler(this, e); + } + + protected virtual void OnTaskStarting(TaskEventArgs e) + { + var handler = TaskStarting; + if (handler != null) handler(this, e); + } + + protected virtual void OnTaskCompleted(TaskEventArgs e) + { + var handler = TaskCompleted; + if (handler != null) handler(this, e); + } + + protected virtual void OnTaskCancelled(TaskEventArgs e) + { + var handler = TaskCancelled; + if (handler != null) handler(this, e); + } + + + #region Disposal + private readonly object _disposalLocker = new object(); + public bool IsDisposed { get; private set; } + + ~BackgroundTaskRunner() + { + this.Dispose(false); + } + + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + protected virtual void Dispose(bool disposing) + { + if (this.IsDisposed || !disposing) + return; + lock (_disposalLocker) + { + if (IsDisposed) + return; + DisposeResources(); + IsDisposed = true; + } + } + + protected virtual void DisposeResources() + { + ShutDown(); + } + #endregion + + public void Stop(bool immediate) + { + if (immediate == false) + { + LogHelper.Debug>("Application is shutting down, waiting for tasks to complete"); + Dispose(); + } + else + { + //NOTE: this will thread block the current operation if the manager + // is still shutting down because the Shutdown operation is also locked + // by this same lock instance. This would only matter if Stop is called by ASP.Net + // on two different threads though, otherwise the current thread will just block normally + // until the app is shutdown + lock (Locker) + { + LogHelper.Info>("Application is shutting down immediately"); + } + } + + } + + } +} diff --git a/src/Umbraco.Web/Scheduling/IBackgroundTask.cs b/src/Umbraco.Web/Scheduling/IBackgroundTask.cs new file mode 100644 index 0000000000..343f076b2a --- /dev/null +++ b/src/Umbraco.Web/Scheduling/IBackgroundTask.cs @@ -0,0 +1,9 @@ +using System; + +namespace Umbraco.Web.Scheduling +{ + internal interface IBackgroundTask : IDisposable + { + void Run(); + } +} \ No newline at end of file diff --git a/src/Umbraco.Web/Scheduling/KeepAlive.cs b/src/Umbraco.Web/Scheduling/KeepAlive.cs index 0ebce6878b..a6f329d13c 100644 --- a/src/Umbraco.Web/Scheduling/KeepAlive.cs +++ b/src/Umbraco.Web/Scheduling/KeepAlive.cs @@ -8,27 +8,32 @@ namespace Umbraco.Web.Scheduling { internal class KeepAlive { - public static void Start(object sender) + public static void Start(ApplicationContext appContext) { using (DisposableTimer.DebugDuration(() => "Keep alive executing", () => "Keep alive complete")) - { - var umbracoBaseUrl = ServerEnvironmentHelper.GetCurrentServerUmbracoBaseUrl(); - - var url = string.Format("{0}/ping.aspx", umbracoBaseUrl); - - try + { + var umbracoBaseUrl = ServerEnvironmentHelper.GetCurrentServerUmbracoBaseUrl(appContext); + if (string.IsNullOrWhiteSpace(umbracoBaseUrl)) { - using (var wc = new WebClient()) + LogHelper.Warn("No url for service (yet), skip."); + } + else + { + var url = string.Format("{0}/ping.aspx", umbracoBaseUrl); + + try { - wc.DownloadString(url); + using (var wc = new WebClient()) + { + wc.DownloadString(url); + } + } + catch (Exception ee) + { + LogHelper.Error("Error in ping", ee); } } - catch (Exception ee) - { - LogHelper.Error("Error in ping", ee); - } } - } } } \ No newline at end of file diff --git a/src/Umbraco.Web/Scheduling/LogScrubber.cs b/src/Umbraco.Web/Scheduling/LogScrubber.cs index cea4d1b01c..46fd36d49e 100644 --- a/src/Umbraco.Web/Scheduling/LogScrubber.cs +++ b/src/Umbraco.Web/Scheduling/LogScrubber.cs @@ -2,38 +2,20 @@ using System; using System.Web; using System.Web.Caching; using umbraco.BusinessLogic; +using Umbraco.Core; using Umbraco.Core.Logging; namespace Umbraco.Web.Scheduling { - //TODO: Refactor this to use a normal scheduling processor! - internal class LogScrubber + internal class LogScrubber : DisposableObject, IBackgroundTask { // this is a raw copy of the legacy code in all its uglyness + private readonly ApplicationContext _appContext; - CacheItemRemovedCallback _onCacheRemove; - const string LogScrubberTaskName = "ScrubLogs"; - - public void Start() + public LogScrubber(ApplicationContext appContext) { - // log scrubbing - AddTask(LogScrubberTaskName, GetLogScrubbingInterval()); - } - - private static int GetLogScrubbingInterval() - { - int interval = 24 * 60 * 60; //24 hours - try - { - if (global::umbraco.UmbracoSettings.CleaningMiliseconds > -1) - interval = global::umbraco.UmbracoSettings.CleaningMiliseconds; - } - catch (Exception e) - { - LogHelper.Error("Unable to locate a log scrubbing interval. Defaulting to 24 horus", e); - } - return interval; + _appContext = appContext; } private static int GetLogScrubbingMaximumAge() @@ -52,26 +34,19 @@ namespace Umbraco.Web.Scheduling } - private void AddTask(string name, int seconds) + /// + /// Handles the disposal of resources. Derived from abstract class which handles common required locking logic. + /// + protected override void DisposeResources() { - _onCacheRemove = new CacheItemRemovedCallback(CacheItemRemoved); - HttpRuntime.Cache.Insert(name, seconds, null, - DateTime.Now.AddSeconds(seconds), System.Web.Caching.Cache.NoSlidingExpiration, - CacheItemPriority.NotRemovable, _onCacheRemove); } - public void CacheItemRemoved(string k, object v, CacheItemRemovedReason r) + public void Run() { - if (k.Equals(LogScrubberTaskName)) + using (DisposableTimer.DebugDuration(() => "Log scrubbing executing", () => "Log scrubbing complete")) { - ScrubLogs(); + Log.CleanLogs(GetLogScrubbingMaximumAge()); } - AddTask(k, Convert.ToInt32(v)); - } - - private static void ScrubLogs() - { - Log.CleanLogs(GetLogScrubbingMaximumAge()); } } } \ No newline at end of file diff --git a/src/Umbraco.Web/Scheduling/ScheduledPublishing.cs b/src/Umbraco.Web/Scheduling/ScheduledPublishing.cs index 9d7fde0789..debb8146bc 100644 --- a/src/Umbraco.Web/Scheduling/ScheduledPublishing.cs +++ b/src/Umbraco.Web/Scheduling/ScheduledPublishing.cs @@ -10,30 +10,52 @@ using Umbraco.Web.Mvc; namespace Umbraco.Web.Scheduling { - internal class ScheduledPublishing + internal class ScheduledPublishing : DisposableObject, IBackgroundTask { + private readonly ApplicationContext _appContext; + private static bool _isPublishingRunning = false; - public void Start(ApplicationContext appContext) + public ScheduledPublishing(ApplicationContext appContext) { - if (appContext == null) return; + _appContext = appContext; + } + + /// + /// Handles the disposal of resources. Derived from abstract class which handles common required locking logic. + /// + protected override void DisposeResources() + { + } + + public void Run() + { + if (_appContext == null) return; using (DisposableTimer.DebugDuration(() => "Scheduled publishing executing", () => "Scheduled publishing complete")) - { + { if (_isPublishingRunning) return; _isPublishingRunning = true; - + try { - var umbracoBaseUrl = ServerEnvironmentHelper.GetCurrentServerUmbracoBaseUrl(); - var url = string.Format("{0}/RestServices/ScheduledPublish/Index", umbracoBaseUrl); - using (var wc = new WebClient()) - { - //pass custom the authorization header - wc.Headers.Set("Authorization", AdminTokenAuthorizeAttribute.GetAuthHeaderTokenVal(appContext)); + var umbracoBaseUrl = ServerEnvironmentHelper.GetCurrentServerUmbracoBaseUrl(_appContext); - var result = wc.UploadString(url, ""); + if (string.IsNullOrWhiteSpace(umbracoBaseUrl)) + { + LogHelper.Warn("No url for service (yet), skip."); + } + else + { + var url = string.Format("{0}/RestServices/ScheduledPublish/Index", umbracoBaseUrl); + using (var wc = new WebClient()) + { + //pass custom the authorization header + wc.Headers.Set("Authorization", AdminTokenAuthorizeAttribute.GetAuthHeaderTokenVal(_appContext)); + + var result = wc.UploadString(url, ""); + } } } catch (Exception ee) @@ -44,9 +66,7 @@ namespace Umbraco.Web.Scheduling { _isPublishingRunning = false; } - } + } } - - } } \ No newline at end of file diff --git a/src/Umbraco.Web/Scheduling/ScheduledTasks.cs b/src/Umbraco.Web/Scheduling/ScheduledTasks.cs index 6bfb6168c2..3a65e68dc1 100644 --- a/src/Umbraco.Web/Scheduling/ScheduledTasks.cs +++ b/src/Umbraco.Web/Scheduling/ScheduledTasks.cs @@ -14,39 +14,19 @@ namespace Umbraco.Web.Scheduling // would need to be a publicly available task (URL) which isn't really very good :( // We should really be using the AdminTokenAuthorizeAttribute for this stuff - internal class ScheduledTasks + internal class ScheduledTasks : DisposableObject, IBackgroundTask { + private readonly ApplicationContext _appContext; private static readonly Hashtable ScheduledTaskTimes = new Hashtable(); private static bool _isPublishingRunning = false; - - public void Start(ApplicationContext appContext) + + public ScheduledTasks(ApplicationContext appContext) { - using (DisposableTimer.DebugDuration(() => "Scheduled tasks executing", () => "Scheduled tasks complete")) - { - if (_isPublishingRunning) return; - - _isPublishingRunning = true; - - try - { - ProcessTasks(); - } - catch (Exception ee) - { - LogHelper.Error("Error executing scheduled task", ee); - } - finally - { - _isPublishingRunning = false; - } - } - + _appContext = appContext; } - private static void ProcessTasks() + private void ProcessTasks() { - - var scheduledTasks = UmbracoSettings.ScheduledTasks; if (scheduledTasks != null) { @@ -80,7 +60,7 @@ namespace Umbraco.Web.Scheduling } } - private static bool GetTaskByHttp(string url) + private bool GetTaskByHttp(string url) { var myHttpWebRequest = (HttpWebRequest)WebRequest.Create(url); @@ -98,5 +78,35 @@ namespace Umbraco.Web.Scheduling return false; } + + /// + /// Handles the disposal of resources. Derived from abstract class which handles common required locking logic. + /// + protected override void DisposeResources() + { + } + + public void Run() + { + using (DisposableTimer.DebugDuration(() => "Scheduled tasks executing", () => "Scheduled tasks complete")) + { + if (_isPublishingRunning) return; + + _isPublishingRunning = true; + + try + { + ProcessTasks(); + } + catch (Exception ee) + { + LogHelper.Error("Error executing scheduled task", ee); + } + finally + { + _isPublishingRunning = false; + } + } + } } } \ No newline at end of file diff --git a/src/Umbraco.Web/Scheduling/Scheduler.cs b/src/Umbraco.Web/Scheduling/Scheduler.cs index e18e41aeda..a36c8fde51 100644 --- a/src/Umbraco.Web/Scheduling/Scheduler.cs +++ b/src/Umbraco.Web/Scheduling/Scheduler.cs @@ -1,4 +1,6 @@ -using System.Threading; +using System; +using System.Threading; +using System.Web; using Umbraco.Core; using Umbraco.Core.Logging; using Umbraco.Core.Sync; @@ -9,60 +11,99 @@ namespace Umbraco.Web.Scheduling /// Used to do the scheduling for tasks, publishing, etc... /// /// - /// - /// TODO: Much of this code is legacy and needs to be updated, there are a few new/better ways to do scheduling - /// in a web project nowadays. - /// - /// //TODO: We need a much more robust way of handing scheduled tasks and also need to take into account app shutdowns during - /// a scheduled tasks operation - /// http://haacked.com/archive/2011/10/16/the-dangers-of-implementing-recurring-background-tasks-in-asp-net.aspx/ - /// + /// All tasks are run in a background task runner which is web aware and will wind down the task correctly instead of killing it completely when + /// the app domain shuts down. /// internal sealed class Scheduler : ApplicationEventHandler { private static Timer _pingTimer; private static Timer _schedulingTimer; - private static LogScrubber _scrubber; + private static BackgroundTaskRunner _publishingRunner; + private static BackgroundTaskRunner _tasksRunner; + private static BackgroundTaskRunner _scrubberRunner; + private static Timer _logScrubberTimer; + private static volatile bool _started = false; + private static readonly object Locker = new object(); protected override void ApplicationStarted(UmbracoApplicationBase umbracoApplication, ApplicationContext applicationContext) { if (umbracoApplication.Context == null) return; - LogHelper.Debug(() => "Initializing the scheduler"); + //subscribe to app init so we can subsribe to the application events + UmbracoApplicationBase.ApplicationInit += (sender, args) => + { + var app = (HttpApplication)sender; - // time to setup the tasks + //subscribe to the end of a successful request (a handler actually executed) + app.PostRequestHandlerExecute += (o, eventArgs) => + { + if (_started == false) + { + lock (Locker) + { + if (_started == false) + { + _started = true; + LogHelper.Debug(() => "Initializing the scheduler"); - // these are the legacy tasks - // just copied over here for backward compatibility - // of course we should have a proper scheduler, see #U4-809 + // time to setup the tasks - //NOTE: It is important to note that we need to use the ctor for a timer without the 'state' object specified, this is in order - // to ensure that the timer itself is not GC'd since internally .net will pass itself in as the state object and that will keep it alive. - // There's references to this here: http://stackoverflow.com/questions/4962172/why-does-a-system-timers-timer-survive-gc-but-not-system-threading-timer - // we also make these timers static to ensure further GC safety. + //We have 3 background runners that are web aware, if the app domain dies, these tasks will wind down correctly + _publishingRunner = new BackgroundTaskRunner(); + _tasksRunner = new BackgroundTaskRunner(); + _scrubberRunner = new BackgroundTaskRunner(); - // ping/keepalive - _pingTimer = new Timer(KeepAlive.Start); - _pingTimer.Change(60000, 300000); + //NOTE: It is important to note that we need to use the ctor for a timer without the 'state' object specified, this is in order + // to ensure that the timer itself is not GC'd since internally .net will pass itself in as the state object and that will keep it alive. + // There's references to this here: http://stackoverflow.com/questions/4962172/why-does-a-system-timers-timer-survive-gc-but-not-system-threading-timer + // we also make these timers static to ensure further GC safety. - // scheduled publishing/unpublishing - _schedulingTimer = new Timer(PerformScheduling); - _schedulingTimer.Change(30000, 60000); + // ping/keepalive - NOTE: we don't use a background runner for this because it does not need to be web aware, if the app domain dies, no problem + _pingTimer = new Timer(state => KeepAlive.Start(applicationContext)); + _pingTimer.Change(60000, 300000); - //log scrubbing - _scrubber = new LogScrubber(); - _scrubber.Start(); + // scheduled publishing/unpublishing + _schedulingTimer = new Timer(state => PerformScheduling(applicationContext)); + _schedulingTimer.Change(60000, 60000); + + //log scrubbing + _logScrubberTimer = new Timer(state => PerformLogScrub(applicationContext)); + _logScrubberTimer.Change(60000, GetLogScrubbingInterval()); + } + } + } + }; + }; } + private int GetLogScrubbingInterval() + { + int interval = 24 * 60 * 60; //24 hours + try + { + if (global::umbraco.UmbracoSettings.CleaningMiliseconds > -1) + interval = global::umbraco.UmbracoSettings.CleaningMiliseconds; + } + catch (Exception e) + { + LogHelper.Error("Unable to locate a log scrubbing interval. Defaulting to 24 horus", e); + } + return interval; + } + + private static void PerformLogScrub(ApplicationContext appContext) + { + _scrubberRunner.Add(new LogScrubber(appContext)); + } /// /// This performs all of the scheduling on the one timer /// - /// + /// /// /// No processing will be done if this server is a slave /// - private static void PerformScheduling(object sender) + private static void PerformScheduling(ApplicationContext appContext) { using (DisposableTimer.DebugDuration(() => "Scheduling interval executing", () => "Scheduling interval complete")) { @@ -78,12 +119,10 @@ namespace Umbraco.Web.Scheduling // then we will process the scheduling //do the scheduled publishing - var scheduledPublishing = new ScheduledPublishing(); - scheduledPublishing.Start(ApplicationContext.Current); + _publishingRunner.Add(new ScheduledPublishing(appContext)); //do the scheduled tasks - var scheduledTasks = new ScheduledTasks(); - scheduledTasks.Start(ApplicationContext.Current); + _tasksRunner.Add(new ScheduledTasks(appContext)); break; case CurrentServerEnvironmentStatus.Slave: @@ -96,6 +135,5 @@ namespace Umbraco.Web.Scheduling } } } - } } diff --git a/src/Umbraco.Web/Scheduling/TaskEventArgs.cs b/src/Umbraco.Web/Scheduling/TaskEventArgs.cs new file mode 100644 index 0000000000..27e5174616 --- /dev/null +++ b/src/Umbraco.Web/Scheduling/TaskEventArgs.cs @@ -0,0 +1,22 @@ +using System; + +namespace Umbraco.Web.Scheduling +{ + internal class TaskEventArgs : EventArgs + where T : IBackgroundTask + { + public T Task { get; private set; } + public Exception Exception { get; private set; } + + public TaskEventArgs(T task) + { + Task = task; + } + + public TaskEventArgs(T task, Exception exception) + { + Task = task; + Exception = exception; + } + } +} \ No newline at end of file diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index b0e93f5afa..2403084c96 100644 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -321,10 +321,13 @@ + + + @@ -750,9 +753,7 @@ - - ASPXCodeBehind - + ASPXCodeBehind From 4d8732d92526c97c0d29cbc2f0133723a4f34026 Mon Sep 17 00:00:00 2001 From: Shannon Date: Mon, 8 Dec 2014 14:59:32 +1100 Subject: [PATCH 3/5] The only way i could get this project to build and run was to change the transform to explicitly target 4.0.0.1 for mvc and webapi, otherwise nothing works and horribly cryptic build errors. Added the tests for the BackgroundTaskRunner --- .../Scheduling/BackgroundTaskRunnerTests.cs | 200 ++++++++++++++++++ src/Umbraco.Tests/Umbraco.Tests.csproj | 1 + src/Umbraco.Web.UI/Global.asax | 2 +- src/Umbraco.Web.UI/web.Template.Debug.config | 11 +- src/Umbraco.Web/Umbraco.Web.csproj | 4 +- 5 files changed, 214 insertions(+), 4 deletions(-) create mode 100644 src/Umbraco.Tests/Scheduling/BackgroundTaskRunnerTests.cs diff --git a/src/Umbraco.Tests/Scheduling/BackgroundTaskRunnerTests.cs b/src/Umbraco.Tests/Scheduling/BackgroundTaskRunnerTests.cs new file mode 100644 index 0000000000..196bce899c --- /dev/null +++ b/src/Umbraco.Tests/Scheduling/BackgroundTaskRunnerTests.cs @@ -0,0 +1,200 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; +using Umbraco.Core; +using Umbraco.Web.Scheduling; + +namespace Umbraco.Tests.Scheduling +{ + [TestFixture] + public class BackgroundTaskRunnerTests + { + + + [Test] + public void Startup_And_Shutdown() + { + BackgroundTaskRunner tManager; + using (tManager = new BackgroundTaskRunner(true, true)) + { + tManager.StartUp(); + } + + NUnit.Framework.Assert.IsFalse(tManager.IsRunning); + } + + [Test] + public void Startup_Starts_Automatically() + { + BackgroundTaskRunner tManager; + using (tManager = new BackgroundTaskRunner(true, true)) + { + tManager.Add(new MyTask()); + NUnit.Framework.Assert.IsTrue(tManager.IsRunning); + } + } + + [Test] + public void Task_Runs() + { + var myTask = new MyTask(); + var waitHandle = new ManualResetEvent(false); + BackgroundTaskRunner tManager; + using (tManager = new BackgroundTaskRunner(true, true)) + { + tManager.TaskCompleted += (sender, task) => waitHandle.Set(); + + tManager.Add(myTask); + + //wait for ITasks to complete + waitHandle.WaitOne(); + + NUnit.Framework.Assert.IsTrue(myTask.Ended != default(DateTime)); + } + } + + [Test] + public void Many_Tasks_Run() + { + var tasks = new Dictionary(); + for (var i = 0; i < 10; i++) + { + tasks.Add(new MyTask(), new ManualResetEvent(false)); + } + + BackgroundTaskRunner tManager; + using (tManager = new BackgroundTaskRunner(true, true)) + { + tManager.TaskCompleted += (sender, task) => tasks[task.Task].Set(); + + tasks.ForEach(t => tManager.Add(t.Key)); + + //wait for all ITasks to complete + WaitHandle.WaitAll(tasks.Values.Select(x => (WaitHandle)x).ToArray()); + + foreach (var task in tasks) + { + NUnit.Framework.Assert.IsTrue(task.Key.Ended != default(DateTime)); + } + } + } + + [Test] + public void Tasks_Can_Keep_Being_Added_And_Will_Execute() + { + Func> getTasks = () => + { + var newTasks = new Dictionary(); + for (var i = 0; i < 10; i++) + { + newTasks.Add(new MyTask(), new ManualResetEvent(false)); + } + return newTasks; + }; + + IDictionary tasks = getTasks(); + + BackgroundTaskRunner tManager; + using (tManager = new BackgroundTaskRunner(true, true)) + { + tManager.TaskCompleted += (sender, task) => tasks[task.Task].Set(); + + //execute first batch + tasks.ForEach(t => tManager.Add(t.Key)); + + //wait for all ITasks to complete + WaitHandle.WaitAll(tasks.Values.Select(x => (WaitHandle)x).ToArray()); + + foreach (var task in tasks) + { + NUnit.Framework.Assert.IsTrue(task.Key.Ended != default(DateTime)); + } + + //execute another batch after a bit + Thread.Sleep(2000); + + tasks = getTasks(); + tasks.ForEach(t => tManager.Add(t.Key)); + + //wait for all ITasks to complete + WaitHandle.WaitAll(tasks.Values.Select(x => (WaitHandle)x).ToArray()); + + foreach (var task in tasks) + { + NUnit.Framework.Assert.IsTrue(task.Key.Ended != default(DateTime)); + } + } + } + + [Test] + public void Task_Queue_Will_Be_Completed_Before_Shutdown() + { + var tasks = new Dictionary(); + for (var i = 0; i < 10; i++) + { + tasks.Add(new MyTask(), new ManualResetEvent(false)); + } + + BackgroundTaskRunner tManager; + using (tManager = new BackgroundTaskRunner(true, true)) + { + tManager.TaskCompleted += (sender, task) => tasks[task.Task].Set(); + + tasks.ForEach(t => tManager.Add(t.Key)); + + ////wait for all ITasks to complete + //WaitHandle.WaitAll(tasks.Values.Select(x => (WaitHandle)x).ToArray()); + + tManager.Stop(false); + //immediate stop will block until complete - but since we are running on + // a single thread this doesn't really matter as the above will just process + // until complete. + tManager.Stop(true); + + NUnit.Framework.Assert.AreEqual(0, tManager.TaskCount); + } + } + + + + + private class MyTask : BaseTask + { + public MyTask() + { + } + + public override void Run() + { + Thread.Sleep(500); + } + + public override void Cancel() + { + + } + } + + public abstract class BaseTask : IBackgroundTask + { + public Guid UniqueId { get; protected set; } + + public abstract void Run(); + public abstract void Cancel(); + + public DateTime Queued { get; set; } + public DateTime Started { get; set; } + public DateTime Ended { get; set; } + + public virtual void Dispose() + { + Ended = DateTime.Now; + } + } + + } +} diff --git a/src/Umbraco.Tests/Umbraco.Tests.csproj b/src/Umbraco.Tests/Umbraco.Tests.csproj index 6c2ee7c94d..1ce2fb7c81 100644 --- a/src/Umbraco.Tests/Umbraco.Tests.csproj +++ b/src/Umbraco.Tests/Umbraco.Tests.csproj @@ -249,6 +249,7 @@ + diff --git a/src/Umbraco.Web.UI/Global.asax b/src/Umbraco.Web.UI/Global.asax index 1627b363bc..8cb0d2d910 100644 --- a/src/Umbraco.Web.UI/Global.asax +++ b/src/Umbraco.Web.UI/Global.asax @@ -1 +1 @@ -<%@ Application Codebehind="Global.asax.cs" Inherits="Umbraco.Web.UmbracoApplication" Language="C#" %> +<%@ Application Inherits="Umbraco.Web.UmbracoApplication" Language="C#" %> diff --git a/src/Umbraco.Web.UI/web.Template.Debug.config b/src/Umbraco.Web.UI/web.Template.Debug.config index 5c5a33545d..eba925bcc6 100644 --- a/src/Umbraco.Web.UI/web.Template.Debug.config +++ b/src/Umbraco.Web.UI/web.Template.Debug.config @@ -46,7 +46,7 @@ - + @@ -64,7 +64,14 @@ xdt:Locator="Condition(_defaultNamespace:assemblyIdentity[@name='System.Web.Mvc']])"/> - + + + + + + + - + + ASPXCodeBehind + ASPXCodeBehind From 3405c5001b16551549a30aede7a908c3a0f44b68 Mon Sep 17 00:00:00 2001 From: Shannon Date: Mon, 8 Dec 2014 15:18:32 +1100 Subject: [PATCH 4/5] working on U4-5964, U4-5907, U4-5965 --- .../Configuration/UmbracoSettings.cs | 5 +++ .../Sync/ServerEnvironmentHelper.cs | 36 ++++++++++++------- src/Umbraco.Web/Scheduling/KeepAlive.cs | 10 ++++-- .../Scheduling/ScheduledPublishing.cs | 5 +-- 4 files changed, 39 insertions(+), 17 deletions(-) diff --git a/src/Umbraco.Core/Configuration/UmbracoSettings.cs b/src/Umbraco.Core/Configuration/UmbracoSettings.cs index 95e78c88a4..892f396a01 100644 --- a/src/Umbraco.Core/Configuration/UmbracoSettings.cs +++ b/src/Umbraco.Core/Configuration/UmbracoSettings.cs @@ -843,6 +843,11 @@ namespace Umbraco.Core.Configuration get { return GetKeyAsNode("/settings/scheduledTasks"); } } + internal static string ScheduledTasksBaseUrl + { + get { return GetKey("/settings/scheduledTasks/@baseUrl"); } + } + /// /// Gets a list of characters that will be replaced when generating urls /// diff --git a/src/Umbraco.Core/Sync/ServerEnvironmentHelper.cs b/src/Umbraco.Core/Sync/ServerEnvironmentHelper.cs index c39b2805fc..c0047e7b49 100644 --- a/src/Umbraco.Core/Sync/ServerEnvironmentHelper.cs +++ b/src/Umbraco.Core/Sync/ServerEnvironmentHelper.cs @@ -25,11 +25,9 @@ namespace Umbraco.Core.Sync if (status == CurrentServerEnvironmentStatus.Single) { - // single install, return null if no original url, else use original url as base + // single install, return null if no config/original url, else use config/original url as base // use http or https as appropriate - return string.IsNullOrWhiteSpace(appContext.OriginalRequestUrl) - ? null // not initialized yet - : string.Format("http{0}://{1}", GlobalSettings.UseSSL ? "s" : "", appContext.OriginalRequestUrl); + return GetBaseUrl(appContext); } var servers = UmbracoSettings.DistributionServers; @@ -37,9 +35,9 @@ namespace Umbraco.Core.Sync var nodes = servers.SelectNodes("./server"); if (nodes == null) { - return string.IsNullOrWhiteSpace(appContext.OriginalRequestUrl) - ? null // not initialized yet - : string.Format("http{0}://{1}", GlobalSettings.UseSSL ? "s" : "", appContext.OriginalRequestUrl); + // cannot be determined, return null if no config/original url, else use config/original url as base + // use http or https as appropriate + return GetBaseUrl(appContext); } var xmlNodes = nodes.Cast().ToList(); @@ -65,12 +63,10 @@ namespace Umbraco.Core.Sync IOHelper.ResolveUrl(SystemDirectories.Umbraco).TrimStart('/')); } } - - // cannot be determined, return null if no original url, else use original url as base + + // cannot be determined, return null if no config/original url, else use config/original url as base // use http or https as appropriate - return string.IsNullOrWhiteSpace(appContext.OriginalRequestUrl) - ? null // not initialized yet - : string.Format("http{0}://{1}", GlobalSettings.UseSSL ? "s" : "", appContext.OriginalRequestUrl); + return GetBaseUrl(appContext); } /// @@ -120,5 +116,21 @@ namespace Umbraco.Core.Sync return CurrentServerEnvironmentStatus.Slave; } + + private static string GetBaseUrl(ApplicationContext appContext) + { + return ( + // is config empty? + UmbracoSettings.ScheduledTasksBaseUrl.IsNullOrWhiteSpace() + // is the orig req empty? + ? appContext.OriginalRequestUrl.IsNullOrWhiteSpace() + // we've got nothing + ? null + //the orig req url is not null, use that + : string.Format("http{0}://{1}", GlobalSettings.UseSSL ? "s" : "", appContext.OriginalRequestUrl) + // the config has been specified, use that + : string.Format("http{0}://{1}", GlobalSettings.UseSSL ? "s" : "", UmbracoSettings.ScheduledTasksBaseUrl)) + .EnsureEndsWith('/'); + } } } \ No newline at end of file diff --git a/src/Umbraco.Web/Scheduling/KeepAlive.cs b/src/Umbraco.Web/Scheduling/KeepAlive.cs index a6f329d13c..d2db1ac3ff 100644 --- a/src/Umbraco.Web/Scheduling/KeepAlive.cs +++ b/src/Umbraco.Web/Scheduling/KeepAlive.cs @@ -1,6 +1,7 @@ using System; using System.Net; using Umbraco.Core; +using Umbraco.Core.Configuration; using Umbraco.Core.Logging; using Umbraco.Core.Sync; @@ -11,15 +12,17 @@ namespace Umbraco.Web.Scheduling public static void Start(ApplicationContext appContext) { using (DisposableTimer.DebugDuration(() => "Keep alive executing", () => "Keep alive complete")) - { - var umbracoBaseUrl = ServerEnvironmentHelper.GetCurrentServerUmbracoBaseUrl(appContext); + { + var umbracoBaseUrl = ServerEnvironmentHelper.GetCurrentServerUmbracoBaseUrl( + appContext); + if (string.IsNullOrWhiteSpace(umbracoBaseUrl)) { LogHelper.Warn("No url for service (yet), skip."); } else { - var url = string.Format("{0}/ping.aspx", umbracoBaseUrl); + var url = string.Format("{0}ping.aspx", umbracoBaseUrl.EnsureEndsWith('/')); try { @@ -34,6 +37,7 @@ namespace Umbraco.Web.Scheduling } } } + } } } \ No newline at end of file diff --git a/src/Umbraco.Web/Scheduling/ScheduledPublishing.cs b/src/Umbraco.Web/Scheduling/ScheduledPublishing.cs index debb8146bc..7b4969d512 100644 --- a/src/Umbraco.Web/Scheduling/ScheduledPublishing.cs +++ b/src/Umbraco.Web/Scheduling/ScheduledPublishing.cs @@ -21,6 +21,7 @@ namespace Umbraco.Web.Scheduling _appContext = appContext; } + /// /// Handles the disposal of resources. Derived from abstract class which handles common required locking logic. /// @@ -48,7 +49,7 @@ namespace Umbraco.Web.Scheduling } else { - var url = string.Format("{0}/RestServices/ScheduledPublish/Index", umbracoBaseUrl); + var url = string.Format("{0}RestServices/ScheduledPublish/Index", umbracoBaseUrl.EnsureEndsWith('/')); using (var wc = new WebClient()) { //pass custom the authorization header @@ -66,7 +67,7 @@ namespace Umbraco.Web.Scheduling { _isPublishingRunning = false; } - } + } } } } \ No newline at end of file From 864994a56e0add91fa6329765ca7bc026a560ebf Mon Sep 17 00:00:00 2001 From: Shannon Date: Mon, 8 Dec 2014 17:04:22 +1100 Subject: [PATCH 5/5] updates error msgs when sending requests to the same server pointing to docs on baseUrl --- src/Umbraco.Web/Scheduling/KeepAlive.cs | 7 ++++--- src/Umbraco.Web/Scheduling/ScheduledPublishing.cs | 8 +++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/Umbraco.Web/Scheduling/KeepAlive.cs b/src/Umbraco.Web/Scheduling/KeepAlive.cs index d2db1ac3ff..45e2558c08 100644 --- a/src/Umbraco.Web/Scheduling/KeepAlive.cs +++ b/src/Umbraco.Web/Scheduling/KeepAlive.cs @@ -13,8 +13,7 @@ namespace Umbraco.Web.Scheduling { using (DisposableTimer.DebugDuration(() => "Keep alive executing", () => "Keep alive complete")) { - var umbracoBaseUrl = ServerEnvironmentHelper.GetCurrentServerUmbracoBaseUrl( - appContext); + var umbracoBaseUrl = ServerEnvironmentHelper.GetCurrentServerUmbracoBaseUrl(appContext); if (string.IsNullOrWhiteSpace(umbracoBaseUrl)) { @@ -33,7 +32,9 @@ namespace Umbraco.Web.Scheduling } catch (Exception ee) { - LogHelper.Error("Error in ping", ee); + LogHelper.Error( + string.Format("Error in ping. The base url used in the request was: {0}, see http://our.umbraco.org/documentation/Using-Umbraco/Config-files/umbracoSettings/#ScheduledTasks documentation for details on setting a baseUrl if this is in error", umbracoBaseUrl) + , ee); } } } diff --git a/src/Umbraco.Web/Scheduling/ScheduledPublishing.cs b/src/Umbraco.Web/Scheduling/ScheduledPublishing.cs index 7b4969d512..97e26e405a 100644 --- a/src/Umbraco.Web/Scheduling/ScheduledPublishing.cs +++ b/src/Umbraco.Web/Scheduling/ScheduledPublishing.cs @@ -39,10 +39,10 @@ namespace Umbraco.Web.Scheduling _isPublishingRunning = true; + var umbracoBaseUrl = ServerEnvironmentHelper.GetCurrentServerUmbracoBaseUrl(_appContext); + try { - var umbracoBaseUrl = ServerEnvironmentHelper.GetCurrentServerUmbracoBaseUrl(_appContext); - if (string.IsNullOrWhiteSpace(umbracoBaseUrl)) { LogHelper.Warn("No url for service (yet), skip."); @@ -61,7 +61,9 @@ namespace Umbraco.Web.Scheduling } catch (Exception ee) { - LogHelper.Error("An error occurred with the scheduled publishing", ee); + LogHelper.Error( + string.Format("An error occurred with the scheduled publishing. The base url used in the request was: {0}, see http://our.umbraco.org/documentation/Using-Umbraco/Config-files/umbracoSettings/#ScheduledTasks documentation for details on setting a baseUrl if this is in error", umbracoBaseUrl) + , ee); } finally {