BackgroundTaskRunner - more fixes

src/Umbraco.Tests/Scheduling/BackgroundTaskRunnerTests.cs

@@ -21,17 +21,29 @@ namespace Umbraco.Tests.Scheduling
             TestHelper.SetupLog4NetForTests();
         }
 
+        /*
         [Test]
         public async void ShutdownWaitWhenRunning()
         {
-            using (var runner = new BackgroundTaskRunner<IBackgroundTask>(new BackgroundTaskRunnerOptions { AutoStart = true, KeepAlive = true }))
+            using (var runner = new BackgroundTaskRunner<IBackgroundTask>(new BackgroundTaskRunnerOptions
             {
-                Assert.IsTrue(runner.IsRunning);
-                Thread.Sleep(800); // for long
-                Assert.IsTrue(runner.IsRunning);
+                AutoStart = true,
+                KeepAlive = true
+            }))
+            {
+                var stopped = false;
+                runner.Stopped += (sender, args) => { stopped = true; };
+
+                Assert.IsTrue(runner.IsRunning); // because AutoStart is true
+                Thread.Sleep(500); // and because KeepAlive is true...
+                Assert.IsTrue(runner.IsRunning); // ...it keeps running
+
                 runner.Shutdown(false, true); // -force +wait
-                await runner; // wait for the entire runner operation to complete
-                Assert.IsTrue(runner.IsCompleted);
+                await runner.StoppedAwaitable; // runner stops, within test's timeout
+
+                Assert.IsTrue(runner.IsCompleted); // shutdown completes the runner
+                Assert.IsFalse(runner.IsRunning); // no more running tasks
+                Assert.IsTrue(stopped);
             }
         }
 
@@ -44,38 +56,121 @@ namespace Umbraco.Tests.Scheduling
 
                 runner.TaskStarting += (sender, args) => Console.WriteLine("starting {0}", DateTime.Now);
                 runner.TaskCompleted += (sender, args) => Console.WriteLine("completed {0}", DateTime.Now);
+                runner.Stopped += (sender, args) => Console.WriteLine("stopped {0}", DateTime.Now);
 
-                Assert.IsFalse(runner.IsRunning);
+                Assert.IsFalse(runner.IsRunning); // because AutoStart is false
 
                 Console.WriteLine("Adding task {0}", DateTime.Now);
                 runner.Add(new MyTask(5000));
+                Thread.Sleep(500);
                 Assert.IsTrue(runner.IsRunning); // is running the task
+
                 Console.WriteLine("Shutting down {0}", DateTime.Now);
                 runner.Shutdown(false, false); // -force -wait
-                Assert.IsTrue(runner.IsCompleted);
+
+                Assert.IsTrue(runner.IsCompleted); // shutdown completes the runner
                 Assert.IsTrue(runner.IsRunning); // still running that task
                 Thread.Sleep(3000);              // wait slightly less than the task takes to complete
                 Assert.IsTrue(runner.IsRunning); // still running that task
 
-                await runner; // wait for the entire runner operation to complete
-                
+                await runner.StoppedAwaitable; // runner stops, within test's timeout
+
                 Console.WriteLine("End {0}", DateTime.Now);
             }
         }
+        */
+
+        [Test]
+        public async void ShutdownWhenRunningWithWait()
+        {
+            using (var runner = new BackgroundTaskRunner<IBackgroundTask>(new BackgroundTaskRunnerOptions()))
+            {
+                var stopped = false;
+                runner.Stopped += (sender, args) => { stopped = true; };
+
+                Assert.IsFalse(runner.IsRunning); // because AutoStart is false
+                runner.Add(new MyTask(5000));
+                Assert.IsTrue(runner.IsRunning); // is running the task
+
+                runner.Shutdown(false, true); // -force +wait
+
+                // all this before we await because +wait
+                Assert.IsTrue(runner.IsCompleted); // shutdown completes the runner
+                Assert.IsFalse(runner.IsRunning); // no more running tasks
+                Assert.IsTrue(stopped);
+
+                await runner.StoppedAwaitable; // runner stops, within test's timeout
+            }
+        }
+
+        [Test]
+        public async void ShutdownWhenRunningWithoutWait()
+        {
+            using (var runner = new BackgroundTaskRunner<IBackgroundTask>(new BackgroundTaskRunnerOptions()))
+            {
+                var stopped = false;
+                runner.Stopped += (sender, args) => { stopped = true; };
+
+                Assert.IsFalse(runner.IsRunning); // because AutoStart is false
+                runner.Add(new MyTask(5000));
+                Assert.IsTrue(runner.IsRunning); // is running the task
+
+                runner.Shutdown(false, false); // -force +wait
+
+                // all this before we await because -wait
+                Assert.IsTrue(runner.IsCompleted); // shutdown completes the runner
+                Assert.IsTrue(runner.IsRunning); // still running the task
+                Assert.IsFalse(stopped);
+
+                await runner.StoppedAwaitable; // runner stops, within test's timeout
+
+                // and then...
+                Assert.IsFalse(runner.IsRunning); // no more running tasks
+                Assert.IsTrue(stopped);
+            }
+        }
+
+        [Test]
+        public async void ShutdownCompletesTheRunner()
+        {
+            using (var runner = new BackgroundTaskRunner<IBackgroundTask>(new BackgroundTaskRunnerOptions()))
+            {
+                Assert.IsFalse(runner.IsRunning); // because AutoStart is false
+
+                // shutdown -force => run all queued tasks
+                runner.Shutdown(false, false); // -force -wait
+                await runner.StoppedAwaitable; // runner stops, within test's timeout
+
+                Assert.IsFalse(runner.IsRunning); // still not running anything
+                Assert.IsTrue(runner.IsCompleted); // shutdown completes the runner
+
+                // cannot add tasks to it anymore
+                Assert.IsFalse(runner.TryAdd(new MyTask()));
+                Assert.Throws<InvalidOperationException>(() =>
+                {
+                    runner.Add(new MyTask());
+                });
+            } 
+        }
 
         [Test]
         public async void ShutdownFlushesTheQueue()
         {
             using (var runner = new BackgroundTaskRunner<IBackgroundTask>(new BackgroundTaskRunnerOptions()))
             {
-                Assert.IsFalse(runner.IsRunning);
+                MyTask t;
+
+                Assert.IsFalse(runner.IsRunning); // because AutoStart is false
                 runner.Add(new MyTask(5000));
                 runner.Add(new MyTask());
-                var t = new MyTask();
-                runner.Add(t);
-                Assert.IsTrue(runner.IsRunning); // is running the first task
+                runner.Add(t = new MyTask());
+                Assert.IsTrue(runner.IsRunning); // is running tasks
+
+                // shutdown -force => run all queued tasks
                 runner.Shutdown(false, false); // -force -wait
-                await runner; // wait for the entire runner operation to complete
+                Assert.IsTrue(runner.IsRunning); // is running tasks
+                await runner.StoppedAwaitable; // runner stops, within test's timeout
+
                 Assert.AreNotEqual(DateTime.MinValue, t.Ended); // t has run
             }
         }
@@ -85,15 +180,20 @@ namespace Umbraco.Tests.Scheduling
         {
             using (var runner = new BackgroundTaskRunner<IBackgroundTask>(new BackgroundTaskRunnerOptions()))
             {
-                Assert.IsFalse(runner.IsRunning);
+                MyTask t;
+
+                Assert.IsFalse(runner.IsRunning); // because AutoStart is false
                 runner.Add(new MyTask(5000));
                 runner.Add(new MyTask());
-                var t = new MyTask();
-                runner.Add(t);
-                Assert.IsTrue(runner.IsRunning); // is running the first task
+                runner.Add(t = new MyTask());
+                Assert.IsTrue(runner.IsRunning); // is running tasks
+
+                // shutdown +force => tries to cancel the current task, ignores queued tasks
                 runner.Shutdown(true, false); // +force -wait
-                await runner; // wait for the entire runner operation to complete
-                Assert.AreEqual(DateTime.MinValue, t.Ended); // t has not run
+                Assert.IsTrue(runner.IsRunning); // is running that long task it cannot cancel
+                await runner.StoppedAwaitable; // runner stops, within test's timeout
+
+                Assert.AreEqual(DateTime.MinValue, t.Ended); // t has *not* run
             }
         }
 
@@ -102,23 +202,106 @@ namespace Umbraco.Tests.Scheduling
         {
             using (var runner = new BackgroundTaskRunner<IBackgroundTask>(new BackgroundTaskRunnerOptions()))
             {
-                Assert.IsFalse(runner.IsRunning);
+
+                Assert.IsFalse(runner.IsRunning); // because AutoStart is false
                 runner.Add(new MyTask(5000));
                 runner.Add(new MyTask());
                 runner.Add(new MyTask());
-                Assert.IsTrue(runner.IsRunning); // is running the task
+                Assert.IsTrue(runner.IsRunning); // is running tasks
+
+                // shutdown -force => run all queued tasks
                 runner.Shutdown(false, false); // -force -wait
-                Assert.IsTrue(runner.IsCompleted);
-                Assert.IsTrue(runner.IsRunning); // still running that task
+
+                Assert.IsTrue(runner.IsCompleted); // shutdown completes the runner
+                Assert.IsTrue(runner.IsRunning); // still running a task
                 Thread.Sleep(3000);
-                Assert.IsTrue(runner.IsRunning); // still running that task
+                Assert.IsTrue(runner.IsRunning); // still running a task
+
+                // shutdown +force => tries to cancel the current task, ignores queued tasks
                 runner.Shutdown(true, false); // +force -wait
-                await runner; // wait for the entire runner operation to complete
+                await runner.StoppedAwaitable; // runner stops, within test's timeout
+            }
+        }
+
+
+        [Test]
+        public async void HostingStopNonImmediate()
+        {
+            using (var runner = new BackgroundTaskRunner<IBackgroundTask>(new BackgroundTaskRunnerOptions()))
+            {
+                MyTask t;
+
+                var stopped = false;
+                runner.Stopped += (sender, args) => { stopped = true; };
+                var terminating = false;
+                runner.Terminating += (sender, args) => { terminating = true; };
+                var terminated = false;
+                runner.Terminated += (sender, args) => { terminated = true; };
+
+                Assert.IsFalse(runner.IsRunning); // because AutoStart is false
+                runner.Add(new MyTask(5000));
+                runner.Add(new MyTask());
+                runner.Add(t = new MyTask());
+                Assert.IsTrue(runner.IsRunning); // is running the task
+
+                runner.Stop(false); // -immediate = -force, -wait
+                Assert.IsTrue(terminating); // has raised that event
+                Assert.IsFalse(terminated); // but not terminated yet
+
+                // all this before we await because -wait
+                Assert.IsTrue(runner.IsCompleted); // shutdown completes the runner
+                Assert.IsTrue(runner.IsRunning); // still running the task
+
+                await runner.StoppedAwaitable; // runner stops, within test's timeout
+                Assert.IsFalse(runner.IsRunning);
+                Assert.IsTrue(stopped);
+
+                await runner.TerminatedAwaitable; // runner terminates, within test's timeout
+                Assert.IsTrue(terminated); // has raised that event
+
+                Assert.AreNotEqual(DateTime.MinValue, t.Ended); // t has run
             }
         }
 
         [Test]
-        public void Create_IsRunning()
+        public async void HostingStopImmediate()
+        {
+            using (var runner = new BackgroundTaskRunner<IBackgroundTask>(new BackgroundTaskRunnerOptions()))
+            {
+                MyTask t;
+
+                var stopped = false;
+                runner.Stopped += (sender, args) => { stopped = true; };
+                var terminating = false;
+                runner.Terminating += (sender, args) => { terminating = true; };
+                var terminated = false;
+                runner.Terminated += (sender, args) => { terminated = true; };
+
+                Assert.IsFalse(runner.IsRunning); // because AutoStart is false
+                runner.Add(new MyTask(5000));
+                runner.Add(new MyTask());
+                runner.Add(t = new MyTask());
+                Assert.IsTrue(runner.IsRunning); // is running the task
+
+                runner.Stop(true); // +immediate = +force, +wait
+                Assert.IsTrue(terminating); // has raised that event
+                Assert.IsTrue(terminated); // and that event
+                Assert.IsTrue(stopped); // and that one
+
+                // and all this before we await because +wait
+                Assert.IsTrue(runner.IsCompleted); // shutdown completes the runner
+                Assert.IsFalse(runner.IsRunning); // done running
+
+                await runner.StoppedAwaitable; // runner stops, within test's timeout
+                await runner.TerminatedAwaitable; // runner terminates, within test's timeout
+
+                Assert.AreEqual(DateTime.MinValue, t.Ended); // t has *not* run
+            }
+        }
+
+        
+        [Test]
+        public void Create_IsNotRunning()
         {
             using (var runner = new BackgroundTaskRunner<IBackgroundTask>(new BackgroundTaskRunnerOptions()))
             {
@@ -130,10 +313,13 @@ namespace Umbraco.Tests.Scheduling
         [Test]
         public async void Create_AutoStart_IsRunning()
         {
-            using (var runner = new BackgroundTaskRunner<IBackgroundTask>(new BackgroundTaskRunnerOptions { AutoStart = true }))
+            using (var runner = new BackgroundTaskRunner<IBackgroundTask>(new BackgroundTaskRunnerOptions
             {
-                Assert.IsTrue(runner.IsRunning);
-                await runner; // wait for the entire runner operation to complete
+                AutoStart = true
+            }))
+            {
+                Assert.IsTrue(runner.IsRunning); // because AutoStart is true
+                await runner.StoppedAwaitable; // runner stops, within test's timeout
             }
         }
 
@@ -142,7 +328,7 @@ namespace Umbraco.Tests.Scheduling
         {
             using (var runner = new BackgroundTaskRunner<IBackgroundTask>(new BackgroundTaskRunnerOptions { AutoStart = true, KeepAlive = true }))
             {
-                Assert.IsTrue(runner.IsRunning);
+                Assert.IsTrue(runner.IsRunning); // because AutoStart is true
                 Thread.Sleep(800); // for long
                 Assert.IsTrue(runner.IsRunning);
                 // dispose will stop it
@@ -159,8 +345,13 @@ namespace Umbraco.Tests.Scheduling
                 // dispose will stop it
             }
 
-            await runner; // wait for the entire runner operation to complete
+            await runner.StoppedAwaitable; // runner stops, within test's timeout
+            //await runner.TerminatedAwaitable; // NO! see note below
             Assert.Throws<InvalidOperationException>(() => runner.Add(new MyTask()));
+
+            // but do NOT await on TerminatedAwaitable - disposing just shuts the runner down
+            // so that we don't have a runaway task in tests, etc - but it does NOT terminate
+            // the runner - it really is NOT a nice way to end a runner - it's there for tests
         }
       
         [Test]
@@ -188,7 +379,7 @@ namespace Umbraco.Tests.Scheduling
                 runner.Add(new MyTask());
                 Assert.IsTrue(runner.IsRunning);
                 waitHandle.WaitOne();
-                await runner; //since we are not being kept alive, it will quit
+                await runner.StoppedAwaitable; //since we are not being kept alive, it will quit
                 Assert.IsFalse(runner.IsRunning);
             }
         }
@@ -221,7 +412,7 @@ namespace Umbraco.Tests.Scheduling
                 runner.Add(task);
                 await runner.CurrentThreadingTask;   // wait for the Task operation to complete
                 Assert.IsTrue(task.Ended != default(DateTime)); // task is done
-                await runner;               // wait for the entire runner operation to complete
+                await runner.StoppedAwaitable;               // wait for the entire runner operation to complete
             }
         }
 
@@ -237,7 +428,7 @@ namespace Umbraco.Tests.Scheduling
             {
                 tasks.ForEach(runner.Add);
 
-                await runner; // wait for the entire runner operation to complete
+                await runner.StoppedAwaitable; // wait for the entire runner operation to complete
 
                 // check that tasks are done
                 Assert.IsTrue(tasks.All(x => x.Ended != default(DateTime)));
@@ -260,7 +451,7 @@ namespace Umbraco.Tests.Scheduling
                 runner.Add(task);
                 waitHandle.WaitOne(); // wait 'til task is done
                 Assert.IsTrue(task.Ended != default(DateTime)); // task is done
-                await runner; // wait for the entire runner operation to complete
+                await runner.StoppedAwaitable; // wait for the entire runner operation to complete
             }
         }
 
@@ -280,7 +471,7 @@ namespace Umbraco.Tests.Scheduling
                 WaitHandle.WaitAll(tasks.Values.Select(x => (WaitHandle)x).ToArray());
                 Assert.IsTrue(tasks.All(x => x.Key.Ended != default(DateTime)));
 
-                await runner; // wait for the entire runner operation to complete
+                await runner.StoppedAwaitable; // wait for the entire runner operation to complete
             }
         }
 
@@ -422,7 +613,7 @@ namespace Umbraco.Tests.Scheduling
         {
             using (var runner = new BackgroundTaskRunner<IBackgroundTask>(new BackgroundTaskRunnerOptions()))
             {
-                var task = new MyDelayedTask(200);
+                var task = new MyDelayedTask(200, false);
                 runner.Add(task);
                 Assert.IsTrue(runner.IsRunning);
                 Thread.Sleep(5000);
@@ -431,7 +622,7 @@ namespace Umbraco.Tests.Scheduling
                 task.Release();
                 await runner.CurrentThreadingTask; //wait for current task to complete
                 Assert.IsTrue(task.HasRun);
-                await runner; // wait for the entire runner operation to complete
+                await runner.StoppedAwaitable; // wait for the entire runner operation to complete
             }
         }
 
@@ -440,14 +631,14 @@ namespace Umbraco.Tests.Scheduling
         {
             using (var runner = new BackgroundTaskRunner<IBackgroundTask>(new BackgroundTaskRunnerOptions()))
             {
-                var task = new MyDelayedTask(200);
+                var task = new MyDelayedTask(200, true);
                 runner.Add(task);
                 Assert.IsTrue(runner.IsRunning);
                 Thread.Sleep(5000);
                 Assert.IsTrue(runner.IsRunning); // still waiting for the task to release               
                 Assert.IsFalse(task.HasRun);
                 runner.Shutdown(false, false);
-                await runner; // wait for the entire runner operation to complete
+                await runner.StoppedAwaitable; // wait for the entire runner operation to complete
                 Assert.IsTrue(task.HasRun);
             }
         }
@@ -478,6 +669,7 @@ namespace Umbraco.Tests.Scheduling
 
                 waitHandle.WaitOne();
                 Assert.GreaterOrEqual(runCount, 4);
+                Assert.IsTrue(task.HasRun);
 
                 // stops recurring
                 runner.Shutdown(false, false);
@@ -492,10 +684,27 @@ namespace Umbraco.Tests.Scheduling
                 var exceptions = new ConcurrentQueue<Exception>();
                 runner.TaskError += (sender, args) => exceptions.Enqueue(args.Exception);
 
-                var task = new MyFailingTask(false); // -async
+                var task = new MyFailingTask(false, true, false); // -async, +running, -disposing
                 runner.Add(task);
                 Assert.IsTrue(runner.IsRunning);
-                await runner; // wait for the entire runner operation to complete
+                await runner.StoppedAwaitable; // wait for the entire runner operation to complete
+
+                Assert.AreEqual(1, exceptions.Count); // traced and reported
+            }
+        }
+
+        [Test]
+        public async void FailingTaskDisposing()
+        {
+            using (var runner = new BackgroundTaskRunner<IBackgroundTask>(new BackgroundTaskRunnerOptions()))
+            {
+                var exceptions = new ConcurrentQueue<Exception>();
+                runner.TaskError += (sender, args) => exceptions.Enqueue(args.Exception);
+
+                var task = new MyFailingTask(false, false, true); // -async, -running, +disposing
+                runner.Add(task);
+                Assert.IsTrue(runner.IsRunning);
+                await runner.StoppedAwaitable; // wait for the entire runner operation to complete
 
                 Assert.AreEqual(1, exceptions.Count); // traced and reported
             }
@@ -509,10 +718,27 @@ namespace Umbraco.Tests.Scheduling
                 var exceptions = new ConcurrentQueue<Exception>();
                 runner.TaskError += (sender, args) => exceptions.Enqueue(args.Exception);
 
-                var task = new MyFailingTask(true); // +async
+                var task = new MyFailingTask(true, true, false); // +async, +running, -disposing
                 runner.Add(task);
                 Assert.IsTrue(runner.IsRunning);
-                await runner; // wait for the entire runner operation to complete
+                await runner.StoppedAwaitable; // wait for the entire runner operation to complete
+
+                Assert.AreEqual(1, exceptions.Count); // traced and reported
+            }
+        }
+
+        [Test]
+        public async void FailingTaskDisposingAsync()
+        {
+            using (var runner = new BackgroundTaskRunner<IBackgroundTask>(new BackgroundTaskRunnerOptions()))
+            {
+                var exceptions = new ConcurrentQueue<Exception>();
+                runner.TaskError += (sender, args) => exceptions.Enqueue(args.Exception);
+
+                var task = new MyFailingTask(false, false, true); // -async, -running, +disposing
+                runner.Add(task);
+                Assert.IsTrue(runner.IsRunning);
+                await runner.StoppedAwaitable; // wait for the entire runner operation to complete
 
                 Assert.AreEqual(1, exceptions.Count); // traced and reported
             }
@@ -521,22 +747,28 @@ namespace Umbraco.Tests.Scheduling
         private class MyFailingTask : IBackgroundTask
         {
             private readonly bool _isAsync;
+            private readonly bool _running;
+            private readonly bool _disposing;
 
-            public MyFailingTask(bool isAsync)
+            public MyFailingTask(bool isAsync, bool running, bool disposing)
             {
                 _isAsync = isAsync;
+                _running = running;
+                _disposing = disposing;
             }
 
             public void Run()
             {
                 Thread.Sleep(1000);
-                throw new Exception("Task has thrown.");
+                if (_running)
+                    throw new Exception("Task has thrown.");
             }
 
             public async Task RunAsync(CancellationToken token)
             {
                 await Task.Delay(1000);
-                throw new Exception("Task has thrown.");
+                if (_running)
+                    throw new Exception("Task has thrown.");
             }
 
             public bool IsAsync
@@ -544,13 +776,17 @@ namespace Umbraco.Tests.Scheduling
                 get { return _isAsync; }
             }
 
-            // fixme - must also test what happens if we throw on dispose!
             public void Dispose()
-            { }
+            {
+                if (_disposing)
+                    throw new Exception("Task has thrown.");
+            }
         }
 
         private class MyDelayedRecurringTask : DelayedRecurringTaskBase<MyDelayedRecurringTask>
         {
+            public bool HasRun { get; private set; }
+
             public MyDelayedRecurringTask(IBackgroundTaskRunner<MyDelayedRecurringTask> runner, int delayMilliseconds, int periodMilliseconds)
                 : base(runner, delayMilliseconds, periodMilliseconds)
             { }
@@ -566,7 +802,7 @@ namespace Umbraco.Tests.Scheduling
 
             public override void PerformRun()
             {
-                // nothing to do at the moment
+                HasRun = true;
             }
 
             public override Task PerformRunAsync()
@@ -583,30 +819,28 @@ namespace Umbraco.Tests.Scheduling
         private class MyDelayedTask : ILatchedBackgroundTask
         {
             private readonly int _runMilliseconds;
-            private readonly ManualResetEvent _gate;
+            private readonly ManualResetEventSlim _gate;
 
             public bool HasRun { get; private set; }
 
-            public MyDelayedTask(int runMilliseconds)
+            public MyDelayedTask(int runMilliseconds, bool runsOnShutdown)
             {
                 _runMilliseconds = runMilliseconds;
-                _gate = new ManualResetEvent(false);
+                _gate = new ManualResetEventSlim(false);
+                RunsOnShutdown = runsOnShutdown;
             }
 
             public WaitHandle Latch
             {
-                get { return _gate; }
+                get { return _gate.WaitHandle; }
             }
 
             public bool IsLatched
             {
-                get { return true; }
+                get { return _gate.IsSet == false; }
             }
 
-            public bool RunsOnShutdown
-            {
-                get { return true; }
-            }
+            public bool RunsOnShutdown { get; private set; }
 
             public void Run()
             {

src/Umbraco.Web/Scheduling/BackgroundTaskRunner.cs

@@ -25,15 +25,23 @@ namespace Umbraco.Web.Scheduling
         private readonly BackgroundTaskRunnerOptions _options;
         private readonly BlockingCollection<T> _tasks = new BlockingCollection<T>();
         private readonly object _locker = new object();
-        private readonly ManualResetEventSlim _completedEvent = new ManualResetEventSlim(false);
-        private BackgroundTaskRunnerAwaiter<T> _awaiter;
 
+        // that event is used to stop the pump when it is alive and waiting
+        // on a latched task - so it waits on the latch, the cancellation token,
+        // and the completed event
+        private readonly ManualResetEventSlim _completedEvent = new ManualResetEventSlim(false);
+
+        // fixme explain volatile here
         private volatile bool _isRunning; // is running
         private volatile bool _isCompleted; // does not accept tasks anymore, may still be running
         private Task _runningTask;
 
         private CancellationTokenSource _tokenSource;
 
+        private bool _terminating; // ensures we raise that event only once
+        private bool _terminated; // remember we've terminated
+        private TaskCompletionSource<int> _terminatedSource; // awaitable source
+
         internal event TypedEventHandler<BackgroundTaskRunner<T>, TaskEventArgs<T>> TaskError;
         internal event TypedEventHandler<BackgroundTaskRunner<T>, TaskEventArgs<T>> TaskStarting;
         internal event TypedEventHandler<BackgroundTaskRunner<T>, TaskEventArgs<T>> TaskCompleted;
@@ -42,8 +50,11 @@ namespace Umbraco.Web.Scheduling
         // triggers when the runner stops (but could start again if a task is added to it)
         internal event TypedEventHandler<BackgroundTaskRunner<T>, EventArgs> Stopped;
 
-        // triggers when the runner completes (no task can be added to it anymore)
-        internal event TypedEventHandler<BackgroundTaskRunner<T>, EventArgs> Completed;
+        // triggers when the hosting environment requests that the runner terminates
+        internal event TypedEventHandler<BackgroundTaskRunner<T>, EventArgs> Terminating;
+
+        // triggers when the runner terminates (no task can be added, no task is running)
+        internal event TypedEventHandler<BackgroundTaskRunner<T>, EventArgs> Terminated;
 
         /// <summary>
         /// Initializes a new instance of the <see cref="BackgroundTaskRunner{T}"/> class.
@@ -110,7 +121,7 @@ namespace Umbraco.Web.Scheduling
         }
 
         /// <summary>
-        /// Gets an awaiter used to await the running Threading.Task.
+        /// Gets the running task as an immutable object.
         /// </summary>
         /// <exception cref="InvalidOperationException">There is no running task.</exception>
         /// <remarks>
@@ -118,32 +129,51 @@ namespace Umbraco.Web.Scheduling
         /// a background task is added to the queue. Unless the KeepAlive option is true, there
         /// will be no running task when the queue is empty.
         /// </remarks>
-        public ThreadingTaskAwaiter CurrentThreadingTask
+        public ThreadingTaskImmutable CurrentThreadingTask
         {
             get
             {
                 if (_runningTask == null)
                     throw new InvalidOperationException("There is no current Threading.Task.");
-                return new ThreadingTaskAwaiter(_runningTask);
+                return new ThreadingTaskImmutable(_runningTask);
             }
         }
 
         /// <summary>
-        /// Gets an awaiter used to await the BackgroundTaskRunner running operation
+        /// Gets an awaitable used to await the runner running operation.
         /// </summary>
-        /// <returns>An awaiter for the BackgroundTaskRunner running operation</returns>
-        /// <remarks>
-        /// <para>This is used to wait until the background task runner is no longer running (IsRunning == false)
-        /// </para>
-        /// <para> So long as we have a method called GetAwaiter() that returns an instance of INotifyCompletion 
-        /// we can await anything. In this case we are awaiting with a custom BackgroundTaskRunnerAwaiter
-        /// which waits for the Completed event to be raised. 
-        /// ref:  http://blogs.msdn.com/b/pfxteam/archive/2011/01/13/10115642.aspx
-        /// </para>
-        /// </remarks>        
-        public BackgroundTaskRunnerAwaiter<T> GetAwaiter()
+        /// <returns>An awaitable instance.</returns>
+        /// <remarks>Used to wait until the runner is no longer running (IsRunning == false),
+        /// though the runner could be started again afterwards by adding tasks to it.</remarks>
+        public ThreadingTaskImmutable StoppedAwaitable
         {
-            return _awaiter ?? (_awaiter = new BackgroundTaskRunnerAwaiter<T>(this));
+            get
+            {
+                lock (_locker)
+                {
+                    var task = _runningTask ?? Task.FromResult(0);
+                    return new ThreadingTaskImmutable(task);
+                }
+            }
+        }
+
+        /// <summary>
+        /// Gets an awaitable used to await the runner.
+        /// </summary>
+        /// <returns>An awaitable instance.</returns>
+        /// <remarks>Used to wait until the runner is terminated.</remarks>
+        public ThreadingTaskImmutable TerminatedAwaitable
+        {
+            get
+            {
+                lock (_locker)
+                {
+                    if (_terminatedSource == null && _terminated == false)
+                        _terminatedSource = new TaskCompletionSource<int>();
+                    var task = _terminatedSource == null ? Task.FromResult(0) : _terminatedSource.Task;
+                    return new ThreadingTaskImmutable(task);
+                }
+            }
         }
 
         /// <summary>
@@ -247,7 +277,7 @@ namespace Umbraco.Web.Scheduling
             if (force)
             {
                 // we must bring everything down, now
-                Thread.Sleep(100); // give time to CompleAdding()
+                Thread.Sleep(100); // give time to CompleteAdding()
                 lock (_locker)
                 {
                     // was CompleteAdding() enough?
@@ -281,23 +311,27 @@ namespace Umbraco.Web.Scheduling
                 // because the pump does not lock, there's a race condition,
                 // the pump may stop and then we still have tasks to process,
                 // and then we must restart the pump - lock to avoid race cond
+                var onStopped = false;
                 lock (_locker)
                 {
                     if (token.IsCancellationRequested || _tasks.Count == 0)
                     {
                         LogHelper.Debug<BackgroundTaskRunner>(_logPrefix + "Stopping");
 
-                        _isRunning = false; // done
                         if (_options.PreserveRunningTask == false)
                             _runningTask = null;
 
-                        OnStopped();
-
-                        return;
+                        // stopped
+                        _isRunning = false;
+                        onStopped = true;
                     }
                 }
 
-               
+                if (onStopped)
+                {
+                    OnEvent(Stopped, "Stopped");
+                    return;
+                }
 
                 // if _runningTask is taskSource.Task then we must keep continuing it,
                 // not starting a new taskSource, else _runningTask would complete and
@@ -424,12 +458,18 @@ namespace Umbraco.Web.Scheduling
             }
             catch (Exception ex)
             {
-                LogHelper.Error<BackgroundTaskRunner>(_logPrefix + "Task has failed.", ex);
+                LogHelper.Error<BackgroundTaskRunner>(_logPrefix + "Task has failed", ex);
             }            
         }
 
         #region Events
 
+        private void OnEvent(TypedEventHandler<BackgroundTaskRunner<T>, EventArgs> handler, string name)
+        {
+            if (handler == null) return;
+            OnEvent(handler, name, EventArgs.Empty);
+        }
+
         private void OnEvent<TArgs>(TypedEventHandler<BackgroundTaskRunner<T>, TArgs> handler, string name, TArgs e)
         {
             if (handler == null) return;
@@ -467,16 +507,6 @@ namespace Umbraco.Web.Scheduling
             e.Task.Dispose();
         }
 
-        protected virtual void OnStopped()
-        {
-            OnEvent(Stopped, "Stopped", EventArgs.Empty);
-        }
-
-        protected virtual void OnCompleted()
-        {
-            OnEvent(Completed, "Completed", EventArgs.Empty);
-        }
-
         #endregion
 
         #region IDisposable
@@ -529,13 +559,30 @@ namespace Umbraco.Web.Scheduling
         /// </remarks>
         public void Stop(bool immediate)
         {
+            // the first time the hosting environment requests that the runner terminates,
+            // raise the Terminating event - that could be used to prevent any process that
+            // would expect the runner to be available from starting.
+            var onTerminating = false;
+            lock (_locker)
+            {
+                if (_terminating == false)
+                {
+                    _terminating = true;
+                    LogHelper.Info<BackgroundTaskRunner>(_logPrefix + "Terminating" + (immediate ? " (immediate)" : ""));
+                    onTerminating = true;
+                }
+            }
+
+            if (onTerminating)
+                OnEvent(Terminating, "Terminating");
+
             if (immediate == false)
             {
                 // The Stop method is first called with the immediate parameter set to false. The object can either complete
                 // processing, call the UnregisterObject method, and then return or it can return immediately and complete
                 // processing asynchronously before calling the UnregisterObject method.
 
-                LogHelper.Info<BackgroundTaskRunner>(_logPrefix + "Shutting down, waiting for tasks to complete.");
+                LogHelper.Info<BackgroundTaskRunner>(_logPrefix + "Waiting for tasks to complete");
                 Shutdown(false, false); // do not accept any more tasks, flush the queue, do not wait
 
                 // raise the completed event only after the running task has completed
@@ -544,18 +591,9 @@ namespace Umbraco.Web.Scheduling
                 lock (_locker)
                 {
                     if (_runningTask != null)
-                        _runningTask.ContinueWith(_ =>
-                        {
-                            HostingEnvironment.UnregisterObject(this);
-                            LogHelper.Info<BackgroundTaskRunner>(_logPrefix + "Down, tasks completed.");
-                            Completed.RaiseEvent(EventArgs.Empty, this);
-                        });
+                        _runningTask.ContinueWith(_ => Terminate(false));
                     else
-                    {
-                        HostingEnvironment.UnregisterObject(this);
-                        LogHelper.Info<BackgroundTaskRunner>(_logPrefix + "Down, tasks completed.");
-                        Completed.RaiseEvent(EventArgs.Empty, this);
-                    }
+                        Terminate(false);
                 }
             }
             else
@@ -565,13 +603,31 @@ namespace Umbraco.Web.Scheduling
                 // immediate parameter is true, the registered object must call the UnregisterObject method before returning;
                 // otherwise, its registration will be removed by the application manager.
 
-                LogHelper.Info<BackgroundTaskRunner>(_logPrefix + "Shutting down immediately.");
+                LogHelper.Info<BackgroundTaskRunner>(_logPrefix + "Cancelling tasks");
                 Shutdown(true, true); // cancel all tasks, wait for the current one to end
-                HostingEnvironment.UnregisterObject(this);
-                LogHelper.Info<BackgroundTaskRunner>(_logPrefix + "Down.");
-                // raise the completed event: there's no more task running
-                Completed.RaiseEvent(EventArgs.Empty, this);
+                Terminate(true);
             }
         }
+
+        private void Terminate(bool immediate)
+        {
+            // signal the environment we have terminated
+            // log
+            // raise the Terminated event
+            // complete the awaitable completion source, if any
+
+            HostingEnvironment.UnregisterObject(this);
+            LogHelper.Info<BackgroundTaskRunner>(_logPrefix + "Tasks " + (immediate ? "cancelled" : "completed") + ", terminated");
+            OnEvent(Terminated, "Terminated");
+
+            TaskCompletionSource<int> terminatedSource;
+            lock (_locker)
+            {
+                _terminated = true;
+                terminatedSource = _terminatedSource;
+            }
+            if (terminatedSource != null)
+                terminatedSource.SetResult(0);
+        }
     }
 }

src/Umbraco.Web/Scheduling/BackgroundTaskRunnerAwaiter.cs

@@ -1,79 +0,0 @@
-using System;
-using System.Runtime.CompilerServices;
-using System.Threading.Tasks;
-using Umbraco.Core.Logging;
-
-namespace Umbraco.Web.Scheduling
-{
-    /// <summary>
-    /// Custom awaiter used to await when the BackgroundTaskRunner is completed (IsRunning == false)
-    /// </summary>
-    /// <typeparam name="T"></typeparam>
-    /// <remarks>
-    /// This custom awaiter simply uses a TaskCompletionSource to set the result when the Completed event of the 
-    /// BackgroundTaskRunner executes.
-    /// A custom awaiter requires implementing INotifyCompletion as well as IsCompleted, OnCompleted and GetResult
-    /// see: http://blogs.msdn.com/b/pfxteam/archive/2011/01/13/10115642.aspx
-    /// </remarks>
-    internal class BackgroundTaskRunnerAwaiter<T> : INotifyCompletion
-        where T : class, IBackgroundTask
-    {
-        private readonly BackgroundTaskRunner<T> _runner;
-        private readonly TaskCompletionSource<int> _tcs;
-        private readonly TaskAwaiter<int> _awaiter;
-
-        public BackgroundTaskRunnerAwaiter(BackgroundTaskRunner<T> runner)
-        {            
-            if (runner == null) throw new ArgumentNullException("runner");
-            _runner = runner;
-
-            _tcs = new TaskCompletionSource<int>();
-            _awaiter = _tcs.Task.GetAwaiter();
-
-            if (_runner.IsRunning)
-            {
-                _runner.Stopped += (s, e) =>
-                {
-                    LogHelper.Debug<BackgroundTaskRunnerAwaiter<T>>("Runner has stopped.");
-                    _tcs.SetResult(0);
-                };
-            }
-            else
-            {
-                //not running, just set the result
-                LogHelper.Debug<BackgroundTaskRunnerAwaiter<T>>("Runner is stopped.");
-                _tcs.SetResult(0);
-            }
-            
-        }
-
-        public BackgroundTaskRunnerAwaiter<T> GetAwaiter()
-        {
-            return this;
-        }
-
-        /// <summary>
-        /// This is completed when the runner is finished running
-        /// </summary>
-        public bool IsCompleted
-        {
-            get
-            {
-                // FIXME I DONT UNDERSTAND
-                LogHelper.Debug<BackgroundTaskRunnerAwaiter<T>>("IsCompleted :: " + _tcs.Task.IsCompleted + ", " + (_runner.IsRunning == false));
-                //Need to check if the task is completed because it might already be done on the ctor and the runner never runs
-                return _tcs.Task.IsCompleted || _runner.IsRunning == false;
-            }
-        }
-
-        public void OnCompleted(Action continuation)
-        {
-            _awaiter.OnCompleted(continuation);
-        }
-
-        public void GetResult()
-        {
-            _awaiter.GetResult();
-        }
-    }
-}

src/Umbraco.Web/Scheduling/ILatchedBackgroundTask.cs

@@ -21,6 +21,7 @@ namespace Umbraco.Web.Scheduling
         /// <summary>
         /// Gets a value indicating whether the task is latched.
         /// </summary>
+        /// <remarks>Should return false as soon as the condition is met.</remarks>
         bool IsLatched { get; }
 
         /// <summary>

src/Umbraco.Web/Scheduling/ThreadingTaskAwaiter.cs

@@ -1,42 +0,0 @@
-using System;
-using System.Runtime.CompilerServices;
-using System.Threading.Tasks;
-
-namespace Umbraco.Web.Scheduling
-{
-    /// <summary>
-    /// This is used to return an awaitable instance from a Task without actually returning the 
-    /// underlying Task instance since it shouldn't be mutable.
-    /// </summary>
-    internal class ThreadingTaskAwaiter
-    {
-        private readonly Task _task;
-
-        public ThreadingTaskAwaiter(Task task)
-        {
-            if (task == null) throw new ArgumentNullException("task");
-            _task = task;
-        }
-
-        /// <summary>
-        /// With a GetAwaiter declared it means that this instance can be awaited on with the await keyword
-        /// </summary>
-        /// <returns></returns>
-        public TaskAwaiter GetAwaiter()
-        {
-            return _task.GetAwaiter();
-        }
-
-        /// <summary>
-        /// Gets the status of the running task.
-        /// </summary>
-        /// <exception cref="InvalidOperationException">There is no running task.</exception>
-        /// <remarks>Unless the AutoStart option is true, there will be no running task until
-        /// a background task is added to the queue. Unless the KeepAlive option is true, there
-        /// will be no running task when the queue is empty.</remarks>
-        public TaskStatus Status
-        {
-            get { return _task.Status; }
-        }
-    }
-}

src/Umbraco.Web/Scheduling/ThreadingTaskImmutable.cs

@@ -0,0 +1,44 @@
+using System;
+using System.Runtime.CompilerServices;
+using System.Threading.Tasks;
+
+namespace Umbraco.Web.Scheduling
+{
+    /// <summary>
+    /// Wraps a Task within an object that gives access to its GetAwaiter method and Status
+    /// property while ensuring that it cannot be modified in any way.
+    /// </summary>
+    internal class ThreadingTaskImmutable
+    {
+        private readonly Task _task;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="ThreadingTaskImmutable"/> class with a Task.
+        /// </summary>
+        /// <param name="task">The task.</param>
+        public ThreadingTaskImmutable(Task task)
+        {
+            if (task == null)
+                throw new ArgumentNullException("task");
+            _task = task;
+        }
+
+        /// <summary>
+        /// Gets an awaiter used to await the task.
+        /// </summary>
+        /// <returns>An awaiter instance.</returns>
+        public TaskAwaiter GetAwaiter()
+        {
+            return _task.GetAwaiter();
+        }
+
+        /// <summary>
+        /// Gets the TaskStatus of the task.
+        /// </summary>
+        /// <returns>The current TaskStatus of the task.</returns>
+        public TaskStatus Status
+        {
+            get { return _task.Status; }
+        }
+    }
+}

src/Umbraco.Web/Umbraco.Web.csproj

@@ -274,7 +274,7 @@
     <Compile Include="Mvc\DisableClientCacheAttribute.cs" />
     <Compile Include="Mvc\MvcVersionCheck.cs" />
     <Compile Include="Mvc\ReflectedFixedRazorViewEngine.cs" />
-    <Compile Include="Scheduling\ThreadingTaskAwaiter.cs" />
+    <Compile Include="Scheduling\ThreadingTaskImmutable.cs" />
     <Compile Include="Scheduling\BackgroundTaskRunner.cs" />
     <Compile Include="BatchedServerMessenger.cs" />
     <Compile Include="CacheHelperExtensions.cs" />
@@ -499,7 +499,6 @@
     <Compile Include="Mvc\UmbracoVirtualNodeRouteHandler.cs" />
     <Compile Include="Routing\CustomRouteUrlProvider.cs" />
     <Compile Include="Routing\UrlProviderExtensions.cs" />
-    <Compile Include="Scheduling\BackgroundTaskRunnerAwaiter.cs" />
     <Compile Include="Scheduling\BackgroundTaskRunnerOptions.cs" />
     <Compile Include="Scheduling\DelayedRecurringTaskBase.cs" />
     <Compile Include="Scheduling\IBackgroundTaskRunner.cs" />

src/Umbraco.Web/umbraco.presentation/content.cs

@@ -55,10 +55,19 @@ namespace umbraco
                     KeepAlive = true
                 });
 
-                // when the runner has stopped we know we will not be writing
-                // to the file anymore, so we can release the lock now - and
-                // not wait for the AppDomain unload
-                runner.Completed += (sender, args) =>
+                // when the runner is terminating we need to ensure that no modifications
+                // to content are possible anymore, as they would not be written out to
+                // the xml file - unfortunately that is not possible in 7.x because we
+                // cannot lock the content service... and so we do nothing...
+                //runner.Terminating += (sender, args) =>
+                //{
+                //};
+
+                // when the runner has terminated we know we will not be writing to the file
+                // anymore, so we can release the lock now - no need to wait for the AppDomain
+                // unload - which means any "last minute" saves will be lost - but waiting for
+                // the AppDomain to unload has issues...
+                runner.Terminated += (sender, args) =>
                 {
                     if (_fileLock == null) return; // not locking (testing?)
                     if (_fileLocked == null) return; // not locked