Gets DB installation test working with runtime level checking

src/Umbraco.Configuration/Models/GlobalSettings.cs

@@ -12,7 +12,7 @@ namespace Umbraco.Configuration.Models
     /// </summary>
     internal class GlobalSettings : IGlobalSettings
     {
-        private const string Prefix = Constants.Configuration.ConfigPrefix + "Global:";
+        public const string Prefix = Constants.Configuration.ConfigPrefix + "Global:";
 
         internal const string
             StaticReservedPaths = "~/app_plugins/,~/install/,~/mini-profiler-resources/,"; //must end with a comma!

src/Umbraco.Configuration/Umbraco.Configuration.csproj

@@ -32,4 +32,10 @@
       <ProjectReference Include="..\Umbraco.Core\Umbraco.Core.csproj" />
     </ItemGroup>
 
+    <ItemGroup>
+        <AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo">
+            <_Parameter1>Umbraco.Tests.Integration</_Parameter1>
+        </AssemblyAttribute>
+    </ItemGroup>
+
 </Project>

src/Umbraco.Core/Configuration/UmbracoVersion.cs

@@ -13,7 +13,7 @@ namespace Umbraco.Core.Configuration
         private readonly IGlobalSettings _globalSettings;
 
         public UmbracoVersion(IGlobalSettings globalSettings)
-        :this()
+        : this()
         {
             _globalSettings = globalSettings;
         }
@@ -55,7 +55,7 @@ namespace Umbraco.Core.Configuration
         /// <para>Is the one that the CLR checks for compatibility. Therefore, it changes only on
         /// hard-breaking changes (for instance, on new major versions).</para>
         /// </remarks>
-        public Version AssemblyVersion {get; }
+        public Version AssemblyVersion { get; }
 
         /// <summary>
         /// Gets the assembly file version of the Umbraco code.
@@ -83,11 +83,13 @@ namespace Umbraco.Core.Configuration
         /// and changes during an upgrade. The executing code version changes when new code is
         /// deployed. The site/files version changes during an upgrade.</para>
         /// </remarks>
-        public SemVersion LocalVersion {
+        public SemVersion LocalVersion
+        {
             get
             {
                 var value = _globalSettings.ConfigurationStatus;
                 return value.IsNullOrWhiteSpace() ? null : SemVersion.TryParse(value, out var semver) ? semver : null;
-            } }
+            }
+        }
     }
 }

src/Umbraco.Infrastructure/Migrations/Install/DatabaseSchemaCreator.cs

@@ -130,7 +130,7 @@ namespace Umbraco.Core.Migrations.Install
 
             if (e.Cancel == false)
             {
-                var dataCreation = new DatabaseDataCreator(_database, _logger,_umbracoVersion, _globalSettings);
+                var dataCreation = new DatabaseDataCreator(_database, _logger, _umbracoVersion, _globalSettings);
                 foreach (var table in OrderedTables)
                     CreateTable(false, table, dataCreation);
             }
@@ -451,7 +451,7 @@ namespace Umbraco.Core.Migrations.Install
 
             //Execute the Create Table sql
             var created = _database.Execute(new Sql(createSql));
-                    _logger.Info<DatabaseSchemaCreator>("Create Table {TableName} ({Created}): \n {Sql}", tableName, created, createSql);
+            _logger.Info<DatabaseSchemaCreator>("Create Table {TableName} ({Created}): \n {Sql}", tableName, created, createSql);
 
             //If any statements exists for the primary key execute them here
             if (string.IsNullOrEmpty(createPrimaryKeySql) == false)
@@ -487,11 +487,11 @@ namespace Umbraco.Core.Migrations.Install
 
             if (overwrite)
             {
-                        _logger.Info<Database>("Table {TableName} was recreated", tableName);
+                _logger.Info<Database>("Table {TableName} was recreated", tableName);
             }
             else
             {
-                        _logger.Info<Database>("New table {TableName} was created", tableName);
+                _logger.Info<Database>("New table {TableName} was created", tableName);
 
             }
         }

src/Umbraco.Infrastructure/Persistence/LocalDb.cs

@@ -253,6 +253,11 @@ namespace Umbraco.Core.Persistence
                 _masterCstr = $@"Server=(localdb)\{instanceName};Integrated Security=True;";
             }
 
+            public static string GetConnectionString(string instanceName, string databaseName)
+            {
+                return $@"Server=(localdb)\{instanceName};Integrated Security=True;Database={databaseName};";
+            }
+
             /// <summary>
             /// Gets a LocalDb connection string.
             /// </summary>

src/Umbraco.Infrastructure/Persistence/UmbracoDatabase.cs

@@ -28,6 +28,7 @@ namespace Umbraco.Core.Persistence
         private readonly RetryPolicy _connectionRetryPolicy;
         private readonly RetryPolicy _commandRetryPolicy;
         private readonly Guid _instanceGuid = Guid.NewGuid();
+        private List<CommandInfo> _commands;
 
         #region Ctor
 
@@ -162,6 +163,14 @@ namespace Umbraco.Core.Persistence
         /// </summary>
         public int SqlCount { get; private set; }
 
+        internal bool LogCommands
+        {
+            get => _commands != null;
+            set => _commands = value ? new List<CommandInfo>() : null;
+        }
+
+        internal IEnumerable<CommandInfo> Commands => _commands;
+
         public int BulkInsertRecords<T>(IEnumerable<T> records)
         {
             return _bulkSqlInsertProvider.BulkInsertRecords(this, records);
@@ -267,9 +276,43 @@ namespace Umbraco.Core.Persistence
             if (_enableCount)
                 SqlCount++;
 
+            _commands?.Add(new CommandInfo(cmd));
+
             base.OnExecutedCommand(cmd);
         }
 
         #endregion
+
+        // used for tracking commands
+        public class CommandInfo
+        {
+            public CommandInfo(IDbCommand cmd)
+            {
+                Text = cmd.CommandText;
+                var parameters = new List<ParameterInfo>();
+                foreach (IDbDataParameter parameter in cmd.Parameters) parameters.Add(new ParameterInfo(parameter));
+                Parameters = parameters.ToArray();
+            }
+
+            public string Text { get; }
+            public ParameterInfo[] Parameters { get; }
+        }
+
+        // used for tracking commands
+        public class ParameterInfo
+        {
+            public ParameterInfo(IDbDataParameter parameter)
+            {
+                Name = parameter.ParameterName;
+                Value = parameter.Value;
+                DbType = parameter.DbType;
+                Size = parameter.Size;
+            }
+
+            public string Name { get; }
+            public object Value { get; }
+            public DbType DbType { get; }
+            public int Size { get; }
+        }
     }
 }

src/Umbraco.Infrastructure/Persistence/UmbracoDatabaseFactory.cs

@@ -29,7 +29,6 @@ namespace Umbraco.Core.Persistence
     {
         private readonly IDbProviderFactoryCreator _dbProviderFactoryCreator;
         private readonly IGlobalSettings _globalSettings;
-        private readonly IConnectionStrings _connectionStrings;
         private readonly Lazy<IMapperCollection> _mappers;
         private readonly ILogger _logger;
 
@@ -37,7 +36,6 @@ namespace Umbraco.Core.Persistence
 
         private DatabaseFactory _npocoDatabaseFactory;
         private IPocoDataFactory _pocoDataFactory;
-        private string _connectionString;
         private string _providerName;
         private DatabaseType _databaseType;
         private ISqlSyntaxProvider _sqlSyntax;
@@ -73,7 +71,7 @@ namespace Umbraco.Core.Persistence
         /// </summary>
         /// <remarks>Used by core runtime.</remarks>
         public UmbracoDatabaseFactory(ILogger logger, IGlobalSettings globalSettings, IConnectionStrings connectionStrings, Lazy<IMapperCollection> mappers,IDbProviderFactoryCreator dbProviderFactoryCreator)
-            : this(Constants.System.UmbracoConnectionName, globalSettings, connectionStrings, logger, mappers, dbProviderFactoryCreator)
+            : this(logger, globalSettings, connectionStrings, Constants.System.UmbracoConnectionName, mappers, dbProviderFactoryCreator)
         {
 
         }
@@ -82,13 +80,12 @@ namespace Umbraco.Core.Persistence
         /// Initializes a new instance of the <see cref="UmbracoDatabaseFactory"/>.
         /// </summary>
         /// <remarks>Used by the other ctor and in tests.</remarks>
-        public UmbracoDatabaseFactory(string connectionStringName, IGlobalSettings globalSettings, IConnectionStrings connectionStrings, ILogger logger, Lazy<IMapperCollection> mappers, IDbProviderFactoryCreator dbProviderFactoryCreator)
+        public UmbracoDatabaseFactory(ILogger logger, IGlobalSettings globalSettings, IConnectionStrings connectionStrings, string connectionStringName, Lazy<IMapperCollection> mappers, IDbProviderFactoryCreator dbProviderFactoryCreator)
         {
             if (connectionStringName == null) throw new ArgumentNullException(nameof(connectionStringName));
             if (string.IsNullOrWhiteSpace(connectionStringName)) throw new ArgumentException("Value can't be empty or consist only of white-space characters.", nameof(connectionStringName));
 
             _globalSettings = globalSettings;
-            _connectionStrings = connectionStrings;
             _mappers = mappers ?? throw new ArgumentNullException(nameof(mappers));
             _dbProviderFactoryCreator = dbProviderFactoryCreator  ?? throw new ArgumentNullException(nameof(dbProviderFactoryCreator));
             _logger = logger ?? throw new ArgumentNullException(nameof(logger));
@@ -118,7 +115,7 @@ namespace Umbraco.Core.Persistence
         /// Initializes a new instance of the <see cref="UmbracoDatabaseFactory"/>.
         /// </summary>
         /// <remarks>Used in tests.</remarks>
-        public UmbracoDatabaseFactory(string connectionString, string providerName, ILogger logger, Lazy<IMapperCollection> mappers, IDbProviderFactoryCreator dbProviderFactoryCreator)
+        public UmbracoDatabaseFactory(ILogger logger, string connectionString, string providerName, Lazy<IMapperCollection> mappers, IDbProviderFactoryCreator dbProviderFactoryCreator)
         {
             _mappers = mappers ?? throw new ArgumentNullException(nameof(mappers));
             _logger = logger ?? throw new ArgumentNullException(nameof(logger));
@@ -142,7 +139,7 @@ namespace Umbraco.Core.Persistence
             {
                 lock (_lock)
                 {
-                    return !_connectionString.IsNullOrWhiteSpace() && !_providerName.IsNullOrWhiteSpace();
+                    return !ConnectionString.IsNullOrWhiteSpace() && !_providerName.IsNullOrWhiteSpace();
                 }
             }
         }
@@ -151,13 +148,13 @@ namespace Umbraco.Core.Persistence
         public bool Initialized => Volatile.Read(ref _initialized);
 
         /// <inheritdoc />
-        public string ConnectionString => _connectionString;
+        public string ConnectionString { get; private set; }
 
         /// <inheritdoc />
         public bool CanConnect =>
             // actually tries to connect to the database (regardless of configured/initialized)
-            !_connectionString.IsNullOrWhiteSpace() && !_providerName.IsNullOrWhiteSpace() &&
-            DbConnectionExtensions.IsConnectionAvailable(_connectionString, DbProviderFactory);
+            !ConnectionString.IsNullOrWhiteSpace() && !_providerName.IsNullOrWhiteSpace() &&
+            DbConnectionExtensions.IsConnectionAvailable(ConnectionString, DbProviderFactory);
 
         private void UpdateSqlServerDatabaseType()
         {
@@ -169,7 +166,7 @@ namespace Umbraco.Core.Persistence
             if (setting.IsNullOrWhiteSpace() || !setting.StartsWith("SqlServer.")
                 || !Enum<SqlServerSyntaxProvider.VersionName>.TryParse(setting.Substring("SqlServer.".Length), out var versionName, true))
             {
-                versionName = ((SqlServerSyntaxProvider) _sqlSyntax).GetSetVersion(_connectionString, _providerName, _logger).ProductVersionName;
+                versionName = ((SqlServerSyntaxProvider) _sqlSyntax).GetSetVersion(ConnectionString, _providerName, _logger).ProductVersionName;
             }
             else
             {
@@ -233,7 +230,7 @@ namespace Umbraco.Core.Persistence
                 if (Volatile.Read(ref _initialized))
                     throw new InvalidOperationException("Already initialized.");
 
-                _connectionString = connectionString;
+                ConnectionString = connectionString;
                 _providerName = providerName;
             }
 
@@ -249,18 +246,19 @@ namespace Umbraco.Core.Persistence
         {
             _logger.Debug<UmbracoDatabaseFactory>("Initializing.");
 
-            if (_connectionString.IsNullOrWhiteSpace()) throw new InvalidOperationException("The factory has not been configured with a proper connection string.");
+            if (ConnectionString.IsNullOrWhiteSpace()) throw new InvalidOperationException("The factory has not been configured with a proper connection string.");
             if (_providerName.IsNullOrWhiteSpace()) throw new InvalidOperationException("The factory has not been configured with a proper provider name.");
 
             if (DbProviderFactory == null)
                 throw new Exception($"Can't find a provider factory for provider name \"{_providerName}\".");
 
             // cannot initialize without being able to talk to the database
-            if (!DbConnectionExtensions.IsConnectionAvailable(_connectionString, DbProviderFactory))
+            // TODO: Why not?
+            if (!DbConnectionExtensions.IsConnectionAvailable(ConnectionString, DbProviderFactory))
                 throw new Exception("Cannot connect to the database.");
 
-            _connectionRetryPolicy = RetryPolicyFactory.GetDefaultSqlConnectionRetryPolicyByConnectionString(_connectionString);
-            _commandRetryPolicy = RetryPolicyFactory.GetDefaultSqlCommandRetryPolicyByConnectionString(_connectionString);
+            _connectionRetryPolicy = RetryPolicyFactory.GetDefaultSqlConnectionRetryPolicyByConnectionString(ConnectionString);
+            _commandRetryPolicy = RetryPolicyFactory.GetDefaultSqlCommandRetryPolicyByConnectionString(ConnectionString);
 
 
             _databaseType = DatabaseType.Resolve(DbProviderFactory.GetType().Name, _providerName);
@@ -313,7 +311,7 @@ namespace Umbraco.Core.Persistence
         // method used by NPoco's UmbracoDatabaseFactory to actually create the database instance
         private UmbracoDatabase CreateDatabaseInstance()
         {
-            return new UmbracoDatabase(_connectionString, SqlContext, DbProviderFactory, _logger, _bulkSqlInsertProvider, _connectionRetryPolicy, _commandRetryPolicy);
+            return new UmbracoDatabase(ConnectionString, SqlContext, DbProviderFactory, _logger, _bulkSqlInsertProvider, _connectionRetryPolicy, _commandRetryPolicy);
         }
 
         protected override void DisposeResources()

src/Umbraco.Infrastructure/Runtime/SqlMainDomLock.cs

@@ -33,14 +33,12 @@ namespace Umbraco.Core.Runtime
             // unique id for our appdomain, this is more unique than the appdomain id which is just an INT counter to its safer
             _lockId = Guid.NewGuid().ToString();
             _logger = logger;
-            _dbFactory = new UmbracoDatabaseFactory(
-               Constants.System.UmbracoConnectionName,
+            _dbFactory = new UmbracoDatabaseFactory(_logger,
                globalSettings,
                connectionStrings,
-               _logger,
+               Constants.System.UmbracoConnectionName,
                new Lazy<IMapperCollection>(() => new MapperCollection(Enumerable.Empty<BaseMapper>())),
-               dbProviderFactoryCreator
-               );
+               dbProviderFactoryCreator);
         }
 
         public async Task<bool> AcquireLockAsync(int millisecondsTimeout)

src/Umbraco.Tests.Integration/GlobalSetupTeardown.cs

@@ -0,0 +1,29 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Text;
+using NUnit.Framework;
+using Umbraco.Tests.Integration.Testing;
+
+// this class has NO NAMESPACE
+// it applies to the whole assembly
+
+[SetUpFixture]
+// ReSharper disable once CheckNamespace
+public class TestsSetup
+{
+    private Stopwatch _stopwatch;
+
+    [OneTimeSetUp]
+    public void SetUp()
+    {
+        _stopwatch = Stopwatch.StartNew();
+    }
+
+    [OneTimeTearDown]
+    public void TearDown()
+    {
+        LocalDbTestDatabase.KillLocalDb();
+        Console.WriteLine("TOTAL TESTS DURATION: {0}", _stopwatch.Elapsed);
+    }
+}

src/Umbraco.Tests.Integration/Implementations/ApplicationBuilderExtensions.cs

@@ -0,0 +1,91 @@
+using System;
+using System.Collections.Generic;
+using System.Data.Common;
+using System.Data.SqlClient;
+using System.IO;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using NUnit.Framework;
+using Umbraco.Configuration.Models;
+using Umbraco.Core;
+using Umbraco.Core.Configuration;
+using Umbraco.Core.Logging;
+using Umbraco.Core.Persistence;
+using Umbraco.Tests.Integration.Testing;
+
+namespace Umbraco.Tests.Integration.Implementations
+{
+    public static class ApplicationBuilderExtensions
+    {
+        /// <summary>
+        /// Creates a LocalDb instance to use for the test
+        /// </summary>
+        /// <param name="app"></param>
+        /// <param name="dbFilePath"></param>
+        /// <param name="createNewDb">
+        /// Default is true - meaning a brand new database is created for this test. If this is false it will try to
+        /// re-use an existing database that was already created as part of this test fixture.
+        /// // TODO Implement the 'false' behavior
+        /// </param>
+        /// <param name="initializeSchema">
+        /// Default is true - meaning a database schema will be created for this test if it's a new database. If this is false
+        /// it will just create an empty database. 
+        /// </param>
+        /// <returns></returns>
+        public static IApplicationBuilder UseTestLocalDb(this IApplicationBuilder app,
+            string dbFilePath,
+            bool createNewDb = true,
+            bool initializeSchema = true)
+        {
+            // need to manually register this factory
+            DbProviderFactories.RegisterFactory(Constants.DbProviderNames.SqlServer, SqlClientFactory.Instance);
+
+            if (!Directory.Exists(dbFilePath))
+                Directory.CreateDirectory(dbFilePath);
+
+            var db = LocalDbTestDatabase.Get(dbFilePath,
+                app.ApplicationServices.GetRequiredService<ILogger>(),
+                app.ApplicationServices.GetRequiredService<IGlobalSettings>(),
+                app.ApplicationServices.GetRequiredService<IUmbracoDatabaseFactory>());
+
+            if (initializeSchema)
+            {
+                // New DB + Schema
+                db.AttachSchema();
+
+                // In the case that we've initialized the schema, it means that we are installed so we'll want to ensure that
+                // the runtime state is configured correctly so we'll force update the configuration flag and re-run the
+                // runtime state checker.
+                // TODO: This wouldn't be required if we don't store the Umbraco version in config
+
+                // right now we are an an 'Install' state
+                var runtimeState = (RuntimeState)app.ApplicationServices.GetRequiredService<IRuntimeState>();
+                Assert.AreEqual(RuntimeLevel.Install, runtimeState.Level);
+
+                // dynamically change the config status
+                var umbVersion = app.ApplicationServices.GetRequiredService<IUmbracoVersion>();
+                var config = app.ApplicationServices.GetRequiredService<IConfiguration>();
+                config[GlobalSettings.Prefix + "ConfigurationStatus"] = umbVersion.SemanticVersion.ToString();
+
+                // re-run the runtime level check
+                var dbFactory = app.ApplicationServices.GetRequiredService<IUmbracoDatabaseFactory>();
+                var profilingLogger = app.ApplicationServices.GetRequiredService<IProfilingLogger>();
+                runtimeState.DetermineRuntimeLevel(dbFactory, profilingLogger);
+
+                Assert.AreEqual(RuntimeLevel.Run, runtimeState.Level);
+
+            }
+            else
+            {
+                db.AttachEmpty();
+            }
+
+            return app;
+        }
+
+    }
+
+
+}

src/Umbraco.Tests.Integration/Implementations/HostBuilderExtensions.cs

@@ -1,42 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Data.Common;
-using System.Data.SqlClient;
-using System.IO;
-using Microsoft.Extensions.Configuration;
-using Microsoft.Extensions.Hosting;
-using Umbraco.Core;
-using Umbraco.Tests.Integration.Testing;
-
-namespace Umbraco.Tests.Integration.Implementations
-{
-    public static class HostBuilderExtensions
-    {
-        
-        public static IHostBuilder UseLocalDb(this IHostBuilder hostBuilder, string dbFilePath)
-        {
-            // Need to register SqlClient manually
-            // TODO: Move this to someplace central
-            DbProviderFactories.RegisterFactory(Constants.DbProviderNames.SqlServer, SqlClientFactory.Instance);
-
-            hostBuilder.ConfigureAppConfiguration(x =>
-            {
-                if (!Directory.Exists(dbFilePath))
-                    Directory.CreateDirectory(dbFilePath);
-
-                var dbName = Guid.NewGuid().ToString("N");
-                var instance = TestLocalDb.EnsureLocalDbInstanceAndDatabase(dbName, dbFilePath);
-
-                x.AddInMemoryCollection(new[]
-                {
-                    new KeyValuePair<string, string>($"ConnectionStrings:{Constants.System.UmbracoConnectionName}", instance.GetConnectionString(dbName))
-                });
-            });
-            return hostBuilder;
-        }
-
-        
-    }
-
-
-}

src/Umbraco.Tests.Integration/RuntimeTests.cs

@@ -11,9 +11,11 @@ using System.Collections.Generic;
 using System.IO;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Builder;
+using Umbraco.Configuration.Models;
 using Umbraco.Core;
 using Umbraco.Core.Composing;
 using Umbraco.Core.Composing.LightInject;
+using Umbraco.Core.Configuration;
 using Umbraco.Core.Logging;
 using Umbraco.Core.Migrations.Install;
 using Umbraco.Core.Persistence;
@@ -40,7 +42,7 @@ namespace Umbraco.Tests.Integration
         [OneTimeTearDown]
         public void FixtureTearDown()
         {
-            TestLocalDb.Cleanup();
+            
         }
 
         /// <summary>
@@ -174,8 +176,6 @@ namespace Umbraco.Tests.Integration
             var testHelper = new TestHelper();
 
             var hostBuilder = new HostBuilder()
-                //TODO: Need to have a configured umb version for the runtime state
-                .UseLocalDb(Path.Combine(testHelper.CurrentAssemblyDirectory, "LocalDb"))
                 .UseUmbraco(serviceProviderFactory)
                 .ConfigureServices((hostContext, services) =>
                 {
@@ -190,26 +190,13 @@ namespace Umbraco.Tests.Integration
             var host = await hostBuilder.StartAsync();
             var app = new ApplicationBuilder(host.Services);
 
+            // This will create a db, install the schema and ensure the app is configured to run
+            app.UseTestLocalDb(Path.Combine(testHelper.CurrentAssemblyDirectory, "LocalDb"));
+
             app.UseUmbracoCore();
 
-
-            var runtimeState = (RuntimeState)app.ApplicationServices.GetRequiredService<IRuntimeState>();
-            Assert.AreEqual(RuntimeLevel.Install, runtimeState.Level);
-
-            var dbBuilder = app.ApplicationServices.GetRequiredService<DatabaseBuilder>();
-            Assert.IsNotNull(dbBuilder);
-
-            var canConnect = dbBuilder.CanConnectToDatabase;
-            Assert.IsTrue(canConnect);
-
-            var dbResult = dbBuilder.CreateSchemaAndData();
-            Assert.IsTrue(dbResult.Success);
-
-            // TODO: Get this to work ... but to do that we need to mock or pass in a current umbraco version
-            //var dbFactory = app.ApplicationServices.GetRequiredService<IUmbracoDatabaseFactory>();
-            //var profilingLogger = app.ApplicationServices.GetRequiredService<IProfilingLogger>();
-            //runtimeState.DetermineRuntimeLevel(dbFactory, profilingLogger);
-            //Assert.AreEqual(RuntimeLevel.Run, runtimeState.Level);
+            var runtimeState = app.ApplicationServices.GetRequiredService<IRuntimeState>();
+            Assert.AreEqual(RuntimeLevel.Run, runtimeState.Level);
         }
 
         internal static LightInjectContainer GetUmbracoContainer(out UmbracoServiceProviderFactory serviceProviderFactory)

src/Umbraco.Tests.Integration/Testing/LocalDbTestDatabase.cs

@@ -0,0 +1,332 @@
+using System;
+using System.Collections.Concurrent;
+using System.Configuration;
+using System.Data;
+using System.Data.Common;
+using System.Data.SqlClient;
+using System.IO;
+using System.Linq;
+using System.Threading;
+using Umbraco.Core.Configuration;
+using Umbraco.Core.Logging;
+using Umbraco.Core.Migrations.Install;
+using Umbraco.Core.Persistence;
+
+namespace Umbraco.Tests.Integration.Testing
+{
+    /// <summary>
+    /// Manages a pool of LocalDb databases for integration testing
+    /// </summary>
+    internal class LocalDbTestDatabase 
+    {
+        public static LocalDbTestDatabase Get(string filesPath, ILogger logger, IGlobalSettings globalSettings, IUmbracoDatabaseFactory dbFactory)
+        {
+            var localDb = new LocalDb();
+            if (localDb.IsAvailable == false)
+                throw new InvalidOperationException("LocalDB is not available.");
+            return new LocalDbTestDatabase(logger, globalSettings, localDb, filesPath, dbFactory);
+        }
+
+        public const string InstanceName = "UmbracoTests";
+        public const string DatabaseName = "UmbracoTests";
+
+        private readonly ILogger _logger;
+        private readonly IGlobalSettings _globalSettings;
+        private readonly LocalDb _localDb;
+        private readonly IUmbracoVersion _umbracoVersion;
+        private static LocalDb.Instance _instance;
+        private static string _filesPath;
+        private readonly IUmbracoDatabaseFactory _dbFactory;
+        private UmbracoDatabase.CommandInfo[] _dbCommands;
+        private string _currentCstr;
+        private static DatabasePool _emptyPool;
+        private static DatabasePool _schemaPool;
+        private DatabasePool _currentPool;
+
+        public LocalDbTestDatabase(ILogger logger, IGlobalSettings globalSettings, LocalDb localDb, string filesPath, IUmbracoDatabaseFactory dbFactory)
+        {
+            _umbracoVersion = new UmbracoVersion();
+            _logger = logger;
+            _globalSettings = globalSettings;
+            _localDb = localDb;
+            _filesPath = filesPath;
+            _dbFactory = dbFactory;
+
+            _instance = _localDb.GetInstance(InstanceName);
+            if (_instance != null) return;
+
+            if (_localDb.CreateInstance(InstanceName) == false)
+                throw new Exception("Failed to create a LocalDb instance.");
+            _instance = _localDb.GetInstance(InstanceName);
+        }
+
+        public string ConnectionString => _currentCstr ?? _instance.GetAttachedConnectionString("XXXXXX", _filesPath);
+
+        private void Create()
+        {
+            var tempName = Guid.NewGuid().ToString("N");
+            _instance.CreateDatabase(tempName, _filesPath);
+            _instance.DetachDatabase(tempName);
+
+            // there's probably a sweet spot to be found for size / parallel...
+
+            var s = ConfigurationManager.AppSettings["Umbraco.Tests.LocalDbTestDatabase.EmptyPoolSize"];
+            var emptySize = s == null ? 2 : int.Parse(s);
+            s = ConfigurationManager.AppSettings["Umbraco.Tests.LocalDbTestDatabase.EmptyPoolThreadCount"];
+            var emptyParallel = s == null ? 1 : int.Parse(s);
+            s = ConfigurationManager.AppSettings["Umbraco.Tests.LocalDbTestDatabase.SchemaPoolSize"];
+            var schemaSize = s == null ? 2 : int.Parse(s);
+            s = ConfigurationManager.AppSettings["Umbraco.Tests.LocalDbTestDatabase.SchemaPoolThreadCount"];
+            var schemaParallel = s == null ? 1 : int.Parse(s);
+
+            _emptyPool = new DatabasePool(_localDb, _instance, DatabaseName + "-Empty", tempName, _filesPath, emptySize, emptyParallel);
+            _schemaPool = new DatabasePool(_localDb, _instance, DatabaseName + "-Schema", tempName, _filesPath, schemaSize, schemaParallel, delete: true, prepare: RebuildSchema);
+        }
+
+        public void AttachEmpty()
+        {
+            if (_emptyPool == null)
+                Create();
+
+            _currentCstr = _emptyPool.AttachDatabase();
+            _currentPool = _emptyPool;
+        }
+
+        public void AttachSchema()
+        {
+            if (_schemaPool == null)
+                Create();
+
+            _currentCstr = _schemaPool.AttachDatabase();
+            _currentPool = _schemaPool;
+        }
+
+        public void Detach()
+        {
+            _currentPool.DetachDatabase();
+        }
+
+        private void RebuildSchema(DbConnection conn, IDbCommand cmd)
+        {
+
+            if (_dbCommands != null)
+            {
+                foreach (var dbCommand in _dbCommands)
+                {
+
+                    if (dbCommand.Text.StartsWith("SELECT ")) continue;
+
+                    cmd.CommandText = dbCommand.Text;
+                    cmd.Parameters.Clear();
+                    foreach (var parameterInfo in dbCommand.Parameters)
+                        AddParameter(cmd, parameterInfo);
+                    cmd.ExecuteNonQuery();
+                }
+            }
+            else
+            {
+                _dbFactory.Configure(conn.ConnectionString, Umbraco.Core.Constants.DatabaseProviders.SqlServer);
+
+                using var database = (UmbracoDatabase)_dbFactory.CreateDatabase();
+                // track each db command ran as part of creating the database so we can replay these
+                database.LogCommands = true;
+
+                using var trans = database.GetTransaction();
+
+                var creator = new DatabaseSchemaCreator(database, _logger, _umbracoVersion, _globalSettings);
+                creator.InitializeDatabaseSchema();
+
+                trans.Complete(); // commit it
+
+                _dbCommands = database.Commands.ToArray();
+            }
+            
+        }
+
+        private static void AddParameter(IDbCommand cmd, UmbracoDatabase.ParameterInfo parameterInfo)
+        {
+            var p = cmd.CreateParameter();
+            p.ParameterName = parameterInfo.Name;
+            p.Value = parameterInfo.Value;
+            p.DbType = parameterInfo.DbType;
+            p.Size = parameterInfo.Size;
+            cmd.Parameters.Add(p);
+        }
+
+        public void Clear()
+        {
+            var filename = Path.Combine(_filesPath, DatabaseName).ToUpper();
+
+            foreach (var database in _instance.GetDatabases())
+            {
+                if (database.StartsWith(filename))
+                    _instance.DropDatabase(database);
+            }
+
+            foreach (var file in Directory.EnumerateFiles(_filesPath))
+            {
+                if (file.EndsWith(".mdf") == false && file.EndsWith(".ldf") == false) continue;
+                File.Delete(file);
+            }
+        }
+
+        private static void ResetLocalDb(IDbCommand cmd)
+        {
+            // https://stackoverflow.com/questions/536350
+
+            cmd.CommandType = CommandType.Text;
+            cmd.CommandText = @"
+                declare @n char(1);
+                set @n = char(10);
+                declare @stmt nvarchar(max);
+                -- check constraints
+                select @stmt = isnull( @stmt + @n, '' ) +
+                    'alter table [' + schema_name(schema_id) + '].[' + object_name( parent_object_id ) + '] drop constraint [' + name + ']'
+                from sys.check_constraints;
+                -- foreign keys
+                select @stmt = isnull( @stmt + @n, '' ) +
+                    'alter table [' + schema_name(schema_id) + '].[' + object_name( parent_object_id ) + '] drop constraint [' + name + ']'
+                from sys.foreign_keys;
+                -- tables
+                select @stmt = isnull( @stmt + @n, '' ) +
+                    'drop table [' + schema_name(schema_id) + '].[' + name + ']'
+                from sys.tables;
+                exec sp_executesql @stmt;
+            ";
+            cmd.ExecuteNonQuery();
+        }
+
+        public static void KillLocalDb()
+        {
+            _emptyPool?.Stop();
+            _schemaPool?.Stop();
+
+            if (_filesPath == null)
+                return;
+
+            var filename = Path.Combine(_filesPath, DatabaseName).ToUpper();
+
+            foreach (var database in _instance.GetDatabases())
+            {
+                if (database.StartsWith(filename))
+                    _instance.DropDatabase(database);
+            }
+
+            foreach (var file in Directory.EnumerateFiles(_filesPath))
+            {
+                if (file.EndsWith(".mdf") == false && file.EndsWith(".ldf") == false) continue;
+                try
+                {
+                    File.Delete(file);
+                }
+                catch (IOException)
+                {
+                    // ignore, must still be in use but nothing we can do
+                }
+            }
+        }
+
+        private class DatabasePool
+        {
+            private readonly LocalDb _localDb;
+            private readonly LocalDb.Instance _instance;
+            private readonly string _filesPath;
+            private readonly string _name;
+            private readonly int _size;
+            private readonly string[] _cstrs;
+            private readonly BlockingCollection<int> _prepareQueue, _readyQueue;
+            private readonly Action<DbConnection, IDbCommand> _prepare;
+            private int _current;
+
+            public DatabasePool(LocalDb localDb, LocalDb.Instance instance, string name, string tempName, string filesPath, int size, int parallel = 1, Action<DbConnection, IDbCommand> prepare = null, bool delete = false)
+            {
+                _localDb = localDb;
+                _instance = instance;
+                _filesPath = filesPath;
+                _name = name;
+                _size = size;
+                _prepare = prepare;
+                _prepareQueue = new BlockingCollection<int>();
+                _readyQueue = new BlockingCollection<int>();
+                _cstrs = new string[_size];
+
+                for (var i = 0; i < size; i++)
+                    localDb.CopyDatabaseFiles(tempName, filesPath, targetDatabaseName: name + "-" + i, overwrite: true, delete: delete && i == size - 1);
+
+                if (prepare == null)
+                {
+                    for (var i = 0; i < size; i++)
+                        _readyQueue.Add(i);
+                }
+                else
+                {
+                    for (var i = 0; i < size; i++)
+                        _prepareQueue.Add(i);
+                }
+
+                for (var i = 0; i < parallel; i++)
+                {
+                    var thread = new Thread(PrepareThread);
+                    thread.Start();
+                }
+            }
+
+            public string AttachDatabase()
+            {
+                try
+                {
+                    _current = _readyQueue.Take();
+                }
+                catch (InvalidOperationException)
+                {
+                    _current = 0;
+                    return null;
+                }
+                return ConnectionString(_current);
+            }
+
+            public void DetachDatabase()
+            {
+                _prepareQueue.Add(_current);
+            }
+
+            private string ConnectionString(int i)
+            {
+                return _cstrs[i] ?? (_cstrs[i] = _instance.GetAttachedConnectionString(_name + "-" + i, _filesPath));
+            }
+
+            private void PrepareThread()
+            {
+                while (_prepareQueue.IsCompleted == false)
+                {
+                    int i;
+                    try
+                    {
+                        i = _prepareQueue.Take();
+                    }
+                    catch (InvalidOperationException)
+                    {
+                        continue;
+                    }
+                    using (var conn = new SqlConnection(ConnectionString(i)))
+                    using (var cmd = conn.CreateCommand())
+                    {
+                        conn.Open();
+                        ResetLocalDb(cmd);
+                        _prepare?.Invoke(conn, cmd);
+                    }
+                    _readyQueue.Add(i);
+                }
+            }
+
+            public void Stop()
+            {
+                int i;
+                _prepareQueue.CompleteAdding();
+                while (_prepareQueue.TryTake(out i)) { }
+                _readyQueue.CompleteAdding();
+                while (_readyQueue.TryTake(out i)) { }
+            }
+        }
+    }
+}

src/Umbraco.Tests.Integration/Testing/TestLocalDb.cs

@@ -1,49 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
-using Umbraco.Core.Persistence;
-
-namespace Umbraco.Tests.Integration.Testing
-{
-    public static class TestLocalDb
-    {
-        private const string LocalDbInstanceName = "UmbTests";
-
-        private static LocalDb LocalDb { get; } = new LocalDb();
-
-        // TODO: We need to borrow logic from this old branch, this is the latest commit at the old branch where we had LocalDb
-        // working for tests. There's a lot of hoops to jump through to make it work 'fast'. Turns out it didn't actually run as
-        // fast as SqlCe due to the dropping/creating of DB instances since that is faster in SqlCe but this code was all heavily
-        // optimized to go as fast as possible.
-        // see https://github.com/umbraco/Umbraco-CMS/blob/3a8716ac7b1c48b51258724337086cd0712625a1/src/Umbraco.Tests/TestHelpers/LocalDbTestDatabase.cs
-        internal static LocalDb.Instance EnsureLocalDbInstanceAndDatabase(string dbName, string dbFilePath)
-        {
-            if (!LocalDb.InstanceExists(LocalDbInstanceName) && !LocalDb.CreateInstance(LocalDbInstanceName))
-            {
-                throw new InvalidOperationException(
-                    $"Failed to create LocalDb instance {LocalDbInstanceName}, assuming LocalDb is not really available.");
-            }
-
-            var instance = LocalDb.GetInstance(LocalDbInstanceName);
-
-            if (instance == null)
-            {
-                throw new InvalidOperationException(
-                    $"Failed to get LocalDb instance {LocalDbInstanceName}, assuming LocalDb is not really available.");
-            }
-
-            instance.CreateDatabase(dbName, dbFilePath);
-
-            return instance;
-        }
-
-        public static void Cleanup()
-        {
-            var instance = LocalDb.GetInstance(LocalDbInstanceName);
-            if (instance != null)
-            {
-                instance.DropDatabases();
-            }
-        }
-    }
-}

src/Umbraco.Tests/Persistence/DatabaseContextTests.cs

@@ -72,7 +72,7 @@ namespace Umbraco.Tests.Persistence
             }
 
             // re-create the database factory and database context with proper connection string
-            _databaseFactory = new UmbracoDatabaseFactory(connString, Constants.DbProviderNames.SqlCe, _logger, new Lazy<IMapperCollection>(() => Mock.Of<IMapperCollection>()), TestHelper.DbProviderFactoryCreator);
+            _databaseFactory = new UmbracoDatabaseFactory(_logger, connString, Constants.DbProviderNames.SqlCe, new Lazy<IMapperCollection>(() => Mock.Of<IMapperCollection>()), TestHelper.DbProviderFactoryCreator);
 
             // test get database type (requires an actual database)
             using (var database = _databaseFactory.CreateDatabase())

src/Umbraco.Tests/Persistence/FaultHandling/ConnectionRetryTest.cs

@@ -20,7 +20,7 @@ namespace Umbraco.Tests.Persistence.FaultHandling
         {
             const string connectionString = @"server=.\SQLEXPRESS;database=EmptyForTest;user id=x;password=umbraco";
             const string providerName = Constants.DbProviderNames.SqlServer;
-            var factory = new UmbracoDatabaseFactory(connectionString, providerName, Mock.Of<ILogger>(), new Lazy<IMapperCollection>(() => Mock.Of<IMapperCollection>()), TestHelper.DbProviderFactoryCreator);
+            var factory = new UmbracoDatabaseFactory(Mock.Of<ILogger>(), connectionString, providerName, new Lazy<IMapperCollection>(() => Mock.Of<IMapperCollection>()), TestHelper.DbProviderFactoryCreator);
 
             using (var database = factory.CreateDatabase())
             {
@@ -34,7 +34,7 @@ namespace Umbraco.Tests.Persistence.FaultHandling
         {
             const string connectionString = @"server=.\SQLEXPRESS;database=EmptyForTest;user id=umbraco;password=umbraco";
             const string providerName = Constants.DbProviderNames.SqlServer;
-            var factory = new UmbracoDatabaseFactory(connectionString, providerName, Mock.Of<ILogger>(), new Lazy<IMapperCollection>(() => Mock.Of<IMapperCollection>()), TestHelper.DbProviderFactoryCreator);
+            var factory = new UmbracoDatabaseFactory(Mock.Of<ILogger>(), connectionString, providerName, new Lazy<IMapperCollection>(() => Mock.Of<IMapperCollection>()), TestHelper.DbProviderFactoryCreator);
 
             using (var database = factory.CreateDatabase())
             {

src/Umbraco.Tests/TestHelpers/TestObjects.cs

@@ -244,11 +244,10 @@ namespace Umbraco.Tests.TestHelpers
                 // mappersBuilder.AddCore();
                 // var mappers = mappersBuilder.CreateCollection();
                 var mappers = Current.Factory.GetInstance<IMapperCollection>();
-                databaseFactory = new UmbracoDatabaseFactory(
-                    Constants.System.UmbracoConnectionName,
+                databaseFactory = new UmbracoDatabaseFactory(logger,
                     SettingsForTests.GetDefaultGlobalSettings(),
                     new ConnectionStrings(),
-                    logger,
+                    Constants.System.UmbracoConnectionName,
                     new Lazy<IMapperCollection>(() => mappers),
                     TestHelper.DbProviderFactoryCreator);
             }

src/Umbraco.Tests/TestHelpers/TestWithDatabaseBase.cs

@@ -89,7 +89,7 @@ namespace Umbraco.Tests.TestHelpers
                     return TestObjects.GetDatabaseFactoryMock();
 
                 var lazyMappers = new Lazy<IMapperCollection>(f.GetInstance<IMapperCollection>);
-                var factory = new UmbracoDatabaseFactory(GetDbConnectionString(), GetDbProviderName(), f.GetInstance<ILogger>(), lazyMappers, TestHelper.DbProviderFactoryCreator);
+                var factory = new UmbracoDatabaseFactory(f.GetInstance<ILogger>(), GetDbConnectionString(), GetDbProviderName(), lazyMappers, TestHelper.DbProviderFactoryCreator);
                 factory.ResetForTests();
                 return factory;
             });

src/Umbraco.Tests/Testing/UmbracoTestBase.cs

@@ -462,13 +462,13 @@ namespace Umbraco.Tests.Testing
             var globalSettings = TestHelper.GetConfigs().Global();
             var connectionStrings = TestHelper.GetConfigs().ConnectionStrings();
 
-            Composition.RegisterUnique<IUmbracoDatabaseFactory>(f => new UmbracoDatabaseFactory(
-                Constants.System.UmbracoConnectionName,
+            Composition.RegisterUnique<IUmbracoDatabaseFactory>(f => new UmbracoDatabaseFactory(Logger,
                 globalSettings,
                 connectionStrings,
-                Logger,
+                Constants.System.UmbracoConnectionName,
                 new Lazy<IMapperCollection>(f.GetInstance<IMapperCollection>),
                 TestHelper.DbProviderFactoryCreator));
+
             Composition.RegisterUnique(f => f.TryGetInstance<IUmbracoDatabaseFactory>().SqlContext);
 
             Composition.WithCollectionBuilder<UrlSegmentProviderCollectionBuilder>(); // empty