BackgroundTaskRunner - more fixes

src/Umbraco.Tests/Scheduling/BackgroundTaskRunnerTests.cs

@@ -21,17 +21,29 @@ namespace Umbraco.Tests.Scheduling
             TestHelper.SetupLog4NetForTests();
         }
 
+        /*
         [Test]
         public async void ShutdownWaitWhenRunning()
         {
-            using (var runner = new BackgroundTaskRunner<IBackgroundTask>(new BackgroundTaskRunnerOptions { AutoStart = true, KeepAlive = true }))
+            using (var runner = new BackgroundTaskRunner<IBackgroundTask>(new BackgroundTaskRunnerOptions
             {
-                Assert.IsTrue(runner.IsRunning);
-                Thread.Sleep(800); // for long
-                Assert.IsTrue(runner.IsRunning);
+                AutoStart = true,
+                KeepAlive = true
+            }))
+            {
+                var stopped = false;
+                runner.Stopped += (sender, args) => { stopped = true; };
+
+                Assert.IsTrue(runner.IsRunning); // because AutoStart is true
+                Thread.Sleep(500); // and because KeepAlive is true...
+                Assert.IsTrue(runner.IsRunning); // ...it keeps running
+
                 runner.Shutdown(false, true); // -force +wait
-                await runner; // wait for the entire runner operation to complete
-                Assert.IsTrue(runner.IsCompleted);
+                await runner.StoppedAwaitable; // runner stops, within test's timeout
+
+                Assert.IsTrue(runner.IsCompleted); // shutdown completes the runner
+                Assert.IsFalse(runner.IsRunning); // no more running tasks
+                Assert.IsTrue(stopped);
             }
         }
 
@@ -44,38 +56,121 @@ namespace Umbraco.Tests.Scheduling
 
                 runner.TaskStarting += (sender, args) => Console.WriteLine("starting {0}", DateTime.Now);
                 runner.TaskCompleted += (sender, args) => Console.WriteLine("completed {0}", DateTime.Now);
+                runner.Stopped += (sender, args) => Console.WriteLine("stopped {0}", DateTime.Now);
 
-                Assert.IsFalse(runner.IsRunning);
+                Assert.IsFalse(runner.IsRunning); // because AutoStart is false
 
                 Console.WriteLine("Adding task {0}", DateTime.Now);
                 runner.Add(new MyTask(5000));
+                Thread.Sleep(500);
                 Assert.IsTrue(runner.IsRunning); // is running the task
+
                 Console.WriteLine("Shutting down {0}", DateTime.Now);
                 runner.Shutdown(false, false); // -force -wait
-                Assert.IsTrue(runner.IsCompleted);
+
+                Assert.IsTrue(runner.IsCompleted); // shutdown completes the runner
                 Assert.IsTrue(runner.IsRunning); // still running that task
                 Thread.Sleep(3000);              // wait slightly less than the task takes to complete
                 Assert.IsTrue(runner.IsRunning); // still running that task
 
-                await runner; // wait for the entire runner operation to complete
-                
+                await runner.StoppedAwaitable; // runner stops, within test's timeout
+
                 Console.WriteLine("End {0}", DateTime.Now);
             }
         }
+        */
+
+        [Test]
+        public async void ShutdownWhenRunningWithWait()
+        {
+            using (var runner = new BackgroundTaskRunner<IBackgroundTask>(new BackgroundTaskRunnerOptions()))
+            {
+                var stopped = false;
+                runner.Stopped += (sender, args) => { stopped = true; };
+
+                Assert.IsFalse(runner.IsRunning); // because AutoStart is false
+                runner.Add(new MyTask(5000));
+                Assert.IsTrue(runner.IsRunning); // is running the task
+
+                runner.Shutdown(false, true); // -force +wait
+
+                // all this before we await because +wait
+                Assert.IsTrue(runner.IsCompleted); // shutdown completes the runner
+                Assert.IsFalse(runner.IsRunning); // no more running tasks
+                Assert.IsTrue(stopped);
+
+                await runner.StoppedAwaitable; // runner stops, within test's timeout
+            }
+        }
+
+        [Test]
+        public async void ShutdownWhenRunningWithoutWait()
+        {
+            using (var runner = new BackgroundTaskRunner<IBackgroundTask>(new BackgroundTaskRunnerOptions()))
+            {
+                var stopped = false;
+                runner.Stopped += (sender, args) => { stopped = true; };
+
+                Assert.IsFalse(runner.IsRunning); // because AutoStart is false
+                runner.Add(new MyTask(5000));
+                Assert.IsTrue(runner.IsRunning); // is running the task
+
+                runner.Shutdown(false, false); // -force +wait
+
+                // all this before we await because -wait
+                Assert.IsTrue(runner.IsCompleted); // shutdown completes the runner
+                Assert.IsTrue(runner.IsRunning); // still running the task
+                Assert.IsFalse(stopped);
+
+                await runner.StoppedAwaitable; // runner stops, within test's timeout
+
+                // and then...
+                Assert.IsFalse(runner.IsRunning); // no more running tasks
+                Assert.IsTrue(stopped);
+            }
+        }
+
+        [Test]
+        public async void ShutdownCompletesTheRunner()
+        {
+            using (var runner = new BackgroundTaskRunner<IBackgroundTask>(new BackgroundTaskRunnerOptions()))
+            {
+                Assert.IsFalse(runner.IsRunning); // because AutoStart is false
+
+                // shutdown -force => run all queued tasks
+                runner.Shutdown(false, false); // -force -wait
+                await runner.StoppedAwaitable; // runner stops, within test's timeout
+
+                Assert.IsFalse(runner.IsRunning); // still not running anything
+                Assert.IsTrue(runner.IsCompleted); // shutdown completes the runner
+
+                // cannot add tasks to it anymore
+                Assert.IsFalse(runner.TryAdd(new MyTask()));
+                Assert.Throws<InvalidOperationException>(() =>
+                {
+                    runner.Add(new MyTask());
+                });
+            } 
+        }
 
         [Test]
         public async void ShutdownFlushesTheQueue()
         {
             using (var runner = new BackgroundTaskRunner<IBackgroundTask>(new BackgroundTaskRunnerOptions()))
             {
-                Assert.IsFalse(runner.IsRunning);
+                MyTask t;
+
+                Assert.IsFalse(runner.IsRunning); // because AutoStart is false
                 runner.Add(new MyTask(5000));
                 runner.Add(new MyTask());
-                var t = new MyTask();
-                runner.Add(t);
-                Assert.IsTrue(runner.IsRunning); // is running the first task
+                runner.Add(t = new MyTask());
+                Assert.IsTrue(runner.IsRunning); // is running tasks
+
+                // shutdown -force => run all queued tasks
                 runner.Shutdown(false, false); // -force -wait
-                await runner; // wait for the entire runner operation to complete
+                Assert.IsTrue(runner.IsRunning); // is running tasks
+                await runner.StoppedAwaitable; // runner stops, within test's timeout
+
                 Assert.AreNotEqual(DateTime.MinValue, t.Ended); // t has run
             }
         }
@@ -85,15 +180,20 @@ namespace Umbraco.Tests.Scheduling
         {
             using (var runner = new BackgroundTaskRunner<IBackgroundTask>(new BackgroundTaskRunnerOptions()))
             {
-                Assert.IsFalse(runner.IsRunning);
+                MyTask t;
+
+                Assert.IsFalse(runner.IsRunning); // because AutoStart is false
                 runner.Add(new MyTask(5000));
                 runner.Add(new MyTask());
-                var t = new MyTask();
-                runner.Add(t);
-                Assert.IsTrue(runner.IsRunning); // is running the first task
+                runner.Add(t = new MyTask());
+                Assert.IsTrue(runner.IsRunning); // is running tasks
+
+                // shutdown +force => tries to cancel the current task, ignores queued tasks
                 runner.Shutdown(true, false); // +force -wait
-                await runner; // wait for the entire runner operation to complete
-                Assert.AreEqual(DateTime.MinValue, t.Ended); // t has not run
+                Assert.IsTrue(runner.IsRunning); // is running that long task it cannot cancel
+                await runner.StoppedAwaitable; // runner stops, within test's timeout
+
+                Assert.AreEqual(DateTime.MinValue, t.Ended); // t has *not* run
             }
         }
 
@@ -102,23 +202,106 @@ namespace Umbraco.Tests.Scheduling
         {
             using (var runner = new BackgroundTaskRunner<IBackgroundTask>(new BackgroundTaskRunnerOptions()))
             {
-                Assert.IsFalse(runner.IsRunning);
+
+                Assert.IsFalse(runner.IsRunning); // because AutoStart is false
                 runner.Add(new MyTask(5000));
                 runner.Add(new MyTask());
                 runner.Add(new MyTask());
-                Assert.IsTrue(runner.IsRunning); // is running the task
+                Assert.IsTrue(runner.IsRunning); // is running tasks
+
+                // shutdown -force => run all queued tasks
                 runner.Shutdown(false, false); // -force -wait
-                Assert.IsTrue(runner.IsCompleted);
-                Assert.IsTrue(runner.IsRunning); // still running that task
+
+                Assert.IsTrue(runner.IsCompleted); // shutdown completes the runner
+                Assert.IsTrue(runner.IsRunning); // still running a task
                 Thread.Sleep(3000);
-                Assert.IsTrue(runner.IsRunning); // still running that task
+                Assert.IsTrue(runner.IsRunning); // still running a task
+
+                // shutdown +force => tries to cancel the current task, ignores queued tasks
                 runner.Shutdown(true, false); // +force -wait
-                await runner; // wait for the entire runner operation to complete
+                await runner.StoppedAwaitable; // runner stops, within test's timeout
+            }
+        }
+
+
+        [Test]
+        public async void HostingStopNonImmediate()
+        {
+            using (var runner = new BackgroundTaskRunner<IBackgroundTask>(new BackgroundTaskRunnerOptions()))
+            {
+                MyTask t;
+
+                var stopped = false;
+                runner.Stopped += (sender, args) => { stopped = true; };
+                var terminating = false;
+                runner.Terminating += (sender, args) => { terminating = true; };
+                var terminated = false;
+                runner.Terminated += (sender, args) => { terminated = true; };
+
+                Assert.IsFalse(runner.IsRunning); // because AutoStart is false
+                runner.Add(new MyTask(5000));
+                runner.Add(new MyTask());
+                runner.Add(t = new MyTask());
+                Assert.IsTrue(runner.IsRunning); // is running the task
+
+                runner.Stop(false); // -immediate = -force, -wait
+                Assert.IsTrue(terminating); // has raised that event
+                Assert.IsFalse(terminated); // but not terminated yet
+
+                // all this before we await because -wait
+                Assert.IsTrue(runner.IsCompleted); // shutdown completes the runner
+                Assert.IsTrue(runner.IsRunning); // still running the task
+
+                await runner.StoppedAwaitable; // runner stops, within test's timeout
+                Assert.IsFalse(runner.IsRunning);
+                Assert.IsTrue(stopped);
+
+                await runner.TerminatedAwaitable; // runner terminates, within test's timeout
+                Assert.IsTrue(terminated); // has raised that event
+
+                Assert.AreNotEqual(DateTime.MinValue, t.Ended); // t has run
             }
         }
 
         [Test]
-        public void Create_IsRunning()
+        public async void HostingStopImmediate()
+        {
+            using (var runner = new BackgroundTaskRunner<IBackgroundTask>(new BackgroundTaskRunnerOptions()))
+            {
+                MyTask t;
+
+                var stopped = false;
+                runner.Stopped += (sender, args) => { stopped = true; };
+                var terminating = false;
+                runner.Terminating += (sender, args) => { terminating = true; };
+                var terminated = false;
+                runner.Terminated += (sender, args) => { terminated = true; };
+
+                Assert.IsFalse(runner.IsRunning); // because AutoStart is false
+                runner.Add(new MyTask(5000));
+                runner.Add(new MyTask());
+                runner.Add(t = new MyTask());
+                Assert.IsTrue(runner.IsRunning); // is running the task
+
+                runner.Stop(true); // +immediate = +force, +wait
+                Assert.IsTrue(terminating); // has raised that event
+                Assert.IsTrue(terminated); // and that event
+                Assert.IsTrue(stopped); // and that one
+
+                // and all this before we await because +wait
+                Assert.IsTrue(runner.IsCompleted); // shutdown completes the runner
+                Assert.IsFalse(runner.IsRunning); // done running
+
+                await runner.StoppedAwaitable; // runner stops, within test's timeout
+                await runner.TerminatedAwaitable; // runner terminates, within test's timeout
+
+                Assert.AreEqual(DateTime.MinValue, t.Ended); // t has *not* run
+            }
+        }
+
+        
+        [Test]
+        public void Create_IsNotRunning()
         {
             using (var runner = new BackgroundTaskRunner<IBackgroundTask>(new BackgroundTaskRunnerOptions()))
             {
@@ -130,10 +313,13 @@ namespace Umbraco.Tests.Scheduling
         [Test]
         public async void Create_AutoStart_IsRunning()
         {
-            using (var runner = new BackgroundTaskRunner<IBackgroundTask>(new BackgroundTaskRunnerOptions { AutoStart = true }))
+            using (var runner = new BackgroundTaskRunner<IBackgroundTask>(new BackgroundTaskRunnerOptions
             {
-                Assert.IsTrue(runner.IsRunning);
-                await runner; // wait for the entire runner operation to complete
+                AutoStart = true
+            }))
+            {
+                Assert.IsTrue(runner.IsRunning); // because AutoStart is true
+                await runner.StoppedAwaitable; // runner stops, within test's timeout
             }
         }
 
@@ -142,7 +328,7 @@ namespace Umbraco.Tests.Scheduling
         {
             using (var runner = new BackgroundTaskRunner<IBackgroundTask>(new BackgroundTaskRunnerOptions { AutoStart = true, KeepAlive = true }))
             {
-                Assert.IsTrue(runner.IsRunning);
+                Assert.IsTrue(runner.IsRunning); // because AutoStart is true
                 Thread.Sleep(800); // for long
                 Assert.IsTrue(runner.IsRunning);
                 // dispose will stop it
@@ -159,8 +345,13 @@ namespace Umbraco.Tests.Scheduling
                 // dispose will stop it
             }
 
-            await runner; // wait for the entire runner operation to complete
+            await runner.StoppedAwaitable; // runner stops, within test's timeout
+            //await runner.TerminatedAwaitable; // NO! see note below
             Assert.Throws<InvalidOperationException>(() => runner.Add(new MyTask()));
+
+            // but do NOT await on TerminatedAwaitable - disposing just shuts the runner down
+            // so that we don't have a runaway task in tests, etc - but it does NOT terminate
+            // the runner - it really is NOT a nice way to end a runner - it's there for tests
         }
       
         [Test]
@@ -188,7 +379,7 @@ namespace Umbraco.Tests.Scheduling
                 runner.Add(new MyTask());
                 Assert.IsTrue(runner.IsRunning);
                 waitHandle.WaitOne();
-                await runner; //since we are not being kept alive, it will quit
+                await runner.StoppedAwaitable; //since we are not being kept alive, it will quit
                 Assert.IsFalse(runner.IsRunning);
             }
         }
@@ -221,7 +412,7 @@ namespace Umbraco.Tests.Scheduling
                 runner.Add(task);
                 await runner.CurrentThreadingTask;   // wait for the Task operation to complete
                 Assert.IsTrue(task.Ended != default(DateTime)); // task is done
-                await runner;               // wait for the entire runner operation to complete
+                await runner.StoppedAwaitable;               // wait for the entire runner operation to complete
             }
         }
 
@@ -237,7 +428,7 @@ namespace Umbraco.Tests.Scheduling
             {
                 tasks.ForEach(runner.Add);
 
-                await runner; // wait for the entire runner operation to complete
+                await runner.StoppedAwaitable; // wait for the entire runner operation to complete
 
                 // check that tasks are done
                 Assert.IsTrue(tasks.All(x => x.Ended != default(DateTime)));
@@ -260,7 +451,7 @@ namespace Umbraco.Tests.Scheduling
                 runner.Add(task);
                 waitHandle.WaitOne(); // wait 'til task is done
                 Assert.IsTrue(task.Ended != default(DateTime)); // task is done
-                await runner; // wait for the entire runner operation to complete
+                await runner.StoppedAwaitable; // wait for the entire runner operation to complete
             }
         }
 
@@ -280,7 +471,7 @@ namespace Umbraco.Tests.Scheduling
                 WaitHandle.WaitAll(tasks.Values.Select(x => (WaitHandle)x).ToArray());
                 Assert.IsTrue(tasks.All(x => x.Key.Ended != default(DateTime)));
 
-                await runner; // wait for the entire runner operation to complete
+                await runner.StoppedAwaitable; // wait for the entire runner operation to complete
             }
         }
 
@@ -422,7 +613,7 @@ namespace Umbraco.Tests.Scheduling
         {
             using (var runner = new BackgroundTaskRunner<IBackgroundTask>(new BackgroundTaskRunnerOptions()))
             {
-                var task = new MyDelayedTask(200);
+                var task = new MyDelayedTask(200, false);
                 runner.Add(task);
                 Assert.IsTrue(runner.IsRunning);
                 Thread.Sleep(5000);
@@ -431,7 +622,7 @@ namespace Umbraco.Tests.Scheduling
                 task.Release();
                 await runner.CurrentThreadingTask; //wait for current task to complete
                 Assert.IsTrue(task.HasRun);
-                await runner; // wait for the entire runner operation to complete
+                await runner.StoppedAwaitable; // wait for the entire runner operation to complete
             }
         }
 
@@ -440,14 +631,14 @@ namespace Umbraco.Tests.Scheduling
         {
             using (var runner = new BackgroundTaskRunner<IBackgroundTask>(new BackgroundTaskRunnerOptions()))
             {
-                var task = new MyDelayedTask(200);
+                var task = new MyDelayedTask(200, true);
                 runner.Add(task);
                 Assert.IsTrue(runner.IsRunning);
                 Thread.Sleep(5000);
                 Assert.IsTrue(runner.IsRunning); // still waiting for the task to release               
                 Assert.IsFalse(task.HasRun);
                 runner.Shutdown(false, false);
-                await runner; // wait for the entire runner operation to complete
+                await runner.StoppedAwaitable; // wait for the entire runner operation to complete
                 Assert.IsTrue(task.HasRun);
             }
         }
@@ -478,6 +669,7 @@ namespace Umbraco.Tests.Scheduling
 
                 waitHandle.WaitOne();
                 Assert.GreaterOrEqual(runCount, 4);
+                Assert.IsTrue(task.HasRun);
 
                 // stops recurring
                 runner.Shutdown(false, false);
@@ -492,10 +684,27 @@ namespace Umbraco.Tests.Scheduling
                 var exceptions = new ConcurrentQueue<Exception>();
                 runner.TaskError += (sender, args) => exceptions.Enqueue(args.Exception);
 
-                var task = new MyFailingTask(false); // -async
+                var task = new MyFailingTask(false, true, false); // -async, +running, -disposing
                 runner.Add(task);
                 Assert.IsTrue(runner.IsRunning);
-                await runner; // wait for the entire runner operation to complete
+                await runner.StoppedAwaitable; // wait for the entire runner operation to complete
+
+                Assert.AreEqual(1, exceptions.Count); // traced and reported
+            }
+        }
+
+        [Test]
+        public async void FailingTaskDisposing()
+        {
+            using (var runner = new BackgroundTaskRunner<IBackgroundTask>(new BackgroundTaskRunnerOptions()))
+            {
+                var exceptions = new ConcurrentQueue<Exception>();
+                runner.TaskError += (sender, args) => exceptions.Enqueue(args.Exception);
+
+                var task = new MyFailingTask(false, false, true); // -async, -running, +disposing
+                runner.Add(task);
+                Assert.IsTrue(runner.IsRunning);
+                await runner.StoppedAwaitable; // wait for the entire runner operation to complete
 
                 Assert.AreEqual(1, exceptions.Count); // traced and reported
             }
@@ -509,10 +718,27 @@ namespace Umbraco.Tests.Scheduling
                 var exceptions = new ConcurrentQueue<Exception>();
                 runner.TaskError += (sender, args) => exceptions.Enqueue(args.Exception);
 
-                var task = new MyFailingTask(true); // +async
+                var task = new MyFailingTask(true, true, false); // +async, +running, -disposing
                 runner.Add(task);
                 Assert.IsTrue(runner.IsRunning);
-                await runner; // wait for the entire runner operation to complete
+                await runner.StoppedAwaitable; // wait for the entire runner operation to complete
+
+                Assert.AreEqual(1, exceptions.Count); // traced and reported
+            }
+        }
+
+        [Test]
+        public async void FailingTaskDisposingAsync()
+        {
+            using (var runner = new BackgroundTaskRunner<IBackgroundTask>(new BackgroundTaskRunnerOptions()))
+            {
+                var exceptions = new ConcurrentQueue<Exception>();
+                runner.TaskError += (sender, args) => exceptions.Enqueue(args.Exception);
+
+                var task = new MyFailingTask(false, false, true); // -async, -running, +disposing
+                runner.Add(task);
+                Assert.IsTrue(runner.IsRunning);
+                await runner.StoppedAwaitable; // wait for the entire runner operation to complete
 
                 Assert.AreEqual(1, exceptions.Count); // traced and reported
             }
@@ -521,22 +747,28 @@ namespace Umbraco.Tests.Scheduling
         private class MyFailingTask : IBackgroundTask
         {
             private readonly bool _isAsync;
+            private readonly bool _running;
+            private readonly bool _disposing;
 
-            public MyFailingTask(bool isAsync)
+            public MyFailingTask(bool isAsync, bool running, bool disposing)
             {
                 _isAsync = isAsync;
+                _running = running;
+                _disposing = disposing;
             }
 
             public void Run()
             {
                 Thread.Sleep(1000);
-                throw new Exception("Task has thrown.");
+                if (_running)
+                    throw new Exception("Task has thrown.");
             }
 
             public async Task RunAsync(CancellationToken token)
             {
                 await Task.Delay(1000);
-                throw new Exception("Task has thrown.");
+                if (_running)
+                    throw new Exception("Task has thrown.");
             }
 
             public bool IsAsync
@@ -544,13 +776,17 @@ namespace Umbraco.Tests.Scheduling
                 get { return _isAsync; }
             }
 
-            // fixme - must also test what happens if we throw on dispose!
             public void Dispose()
-            { }
+            {
+                if (_disposing)
+                    throw new Exception("Task has thrown.");
+            }
         }
 
         private class MyDelayedRecurringTask : DelayedRecurringTaskBase<MyDelayedRecurringTask>
         {
+            public bool HasRun { get; private set; }
+
             public MyDelayedRecurringTask(IBackgroundTaskRunner<MyDelayedRecurringTask> runner, int delayMilliseconds, int periodMilliseconds)
                 : base(runner, delayMilliseconds, periodMilliseconds)
             { }
@@ -566,7 +802,7 @@ namespace Umbraco.Tests.Scheduling
 
             public override void PerformRun()
             {
-                // nothing to do at the moment
+                HasRun = true;
             }
 
             public override Task PerformRunAsync()
@@ -583,30 +819,28 @@ namespace Umbraco.Tests.Scheduling
         private class MyDelayedTask : ILatchedBackgroundTask
         {
             private readonly int _runMilliseconds;
-            private readonly ManualResetEvent _gate;
+            private readonly ManualResetEventSlim _gate;
 
             public bool HasRun { get; private set; }
 
-            public MyDelayedTask(int runMilliseconds)
+            public MyDelayedTask(int runMilliseconds, bool runsOnShutdown)
             {
                 _runMilliseconds = runMilliseconds;
-                _gate = new ManualResetEvent(false);
+                _gate = new ManualResetEventSlim(false);
+                RunsOnShutdown = runsOnShutdown;
             }
 
             public WaitHandle Latch
             {
-                get { return _gate; }
+                get { return _gate.WaitHandle; }
             }
 
             public bool IsLatched
             {
-                get { return true; }
+                get { return _gate.IsSet == false; }
             }
 
-            public bool RunsOnShutdown
-            {
-                get { return true; }
-            }
+            public bool RunsOnShutdown { get; private set; }
 
             public void Run()
             {