From 4dbe5d0c3888c664c0152eb08bbfdc78f72f2953 Mon Sep 17 00:00:00 2001 From: Paul Johnson Date: Sat, 12 Dec 2020 11:33:57 +0000 Subject: [PATCH] Consolidate LocalDbTestDatabase and SqlDeveloperTestDatabase Resolve issue with multiple empties --- .../GlobalSetupTeardown.cs | 4 +- .../Testing/BaseTestDatabase.cs | 218 +++++++++++ .../Testing/ITestDatabase.cs | 9 +- .../Testing/LocalDbTestDatabase.cs | 351 ++++-------------- .../Testing/SqlDeveloperTestDatabase.cs | 218 ++--------- .../Testing/TestDatabaseFactory.cs | 5 +- .../Testing/TestDbMeta.cs | 39 ++ .../Testing/UmbracoIntegrationTest.cs | 31 +- .../TestHelpers/TestWithDatabaseBase.cs | 3 +- 9 files changed, 367 insertions(+), 511 deletions(-) create mode 100644 src/Umbraco.Tests.Integration/Testing/BaseTestDatabase.cs create mode 100644 src/Umbraco.Tests.Integration/Testing/TestDbMeta.cs diff --git a/src/Umbraco.Tests.Integration/GlobalSetupTeardown.cs b/src/Umbraco.Tests.Integration/GlobalSetupTeardown.cs index 0fdac242d9..6e86e97770 100644 --- a/src/Umbraco.Tests.Integration/GlobalSetupTeardown.cs +++ b/src/Umbraco.Tests.Integration/GlobalSetupTeardown.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Diagnostics; using System.Text; @@ -23,7 +23,7 @@ public class TestsSetup [OneTimeTearDown] public void TearDown() { - LocalDbTestDatabase.KillLocalDb(); + LocalDbTestDatabase.Instance?.Finish(); SqlDeveloperTestDatabase.Instance?.Finish(); Console.WriteLine("TOTAL TESTS DURATION: {0}", _stopwatch.Elapsed); } diff --git a/src/Umbraco.Tests.Integration/Testing/BaseTestDatabase.cs b/src/Umbraco.Tests.Integration/Testing/BaseTestDatabase.cs new file mode 100644 index 0000000000..02a3da676a --- /dev/null +++ b/src/Umbraco.Tests.Integration/Testing/BaseTestDatabase.cs @@ -0,0 +1,218 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Data; +using System.Data.SqlClient; +using System.Diagnostics; +using System.Linq; +using System.Threading; +using Microsoft.Extensions.Logging; +using Umbraco.Core.Configuration; +using Umbraco.Core.Migrations.Install; +using Umbraco.Core.Persistence; + +namespace Umbraco.Tests.Integration.Testing +{ + public abstract class BaseTestDatabase + { + protected ILoggerFactory _loggerFactory; + protected IUmbracoDatabaseFactory _databaseFactory; + protected IEnumerable _testDatabases; + + protected UmbracoDatabase.CommandInfo[] _cachedDatabaseInitCommands; + + protected BlockingCollection _prepareQueue; + protected BlockingCollection _readySchemaQueue; + protected BlockingCollection _readyEmptyQueue; + + protected abstract void Initialize(); + + public TestDbMeta AttachEmpty() + { + if (_prepareQueue == null) + { + Initialize(); + } + + return _readyEmptyQueue.Take(); + } + + public TestDbMeta AttachSchema() + { + if (_prepareQueue == null) + { + Initialize(); + } + + return _readySchemaQueue.Take(); + } + + public void Detach(TestDbMeta meta) + { + _prepareQueue.TryAdd(meta); + } + + protected void PrepareDatabase() + { + Retry(10, () => + { + while (_prepareQueue.IsCompleted == false) + { + TestDbMeta meta; + try + { + meta = _prepareQueue.Take(); + } + catch (InvalidOperationException) + { + continue; + } + + using (var conn = new SqlConnection(meta.ConnectionString)) + using (var cmd = conn.CreateCommand()) + { + conn.Open(); + ResetTestDatabase(cmd); + + if (!meta.IsEmpty) + { + RebuildSchema(cmd, meta); + } + } + + if (!meta.IsEmpty) + { + _readySchemaQueue.TryAdd(meta); + } + else + { + _readyEmptyQueue.TryAdd(meta); + } + } + }); + } + + protected void RebuildSchema(IDbCommand command, TestDbMeta meta) + { + if (_cachedDatabaseInitCommands != null) + { + foreach (var dbCommand in _cachedDatabaseInitCommands) + { + + if (dbCommand.Text.StartsWith("SELECT ")) + { + continue; + } + + command.CommandText = dbCommand.Text; + command.Parameters.Clear(); + + foreach (var parameterInfo in dbCommand.Parameters) + { + AddParameter(command, parameterInfo); + } + + command.ExecuteNonQuery(); + } + } + else + { + _databaseFactory.Configure(meta.ConnectionString, Core.Constants.DatabaseProviders.SqlServer); + + using (var database = (UmbracoDatabase)_databaseFactory.CreateDatabase()) + { + database.LogCommands = true; + + using (var transaction = database.GetTransaction()) + { + var schemaCreator = new DatabaseSchemaCreator(database, _loggerFactory.CreateLogger(), _loggerFactory, new UmbracoVersion()); + schemaCreator.InitializeDatabaseSchema(); + + transaction.Complete(); + + _cachedDatabaseInitCommands = database.Commands.ToArray(); + } + } + } + } + + protected static void SetCommand(SqlCommand command, string sql, params object[] args) + { + command.CommandType = CommandType.Text; + command.CommandText = sql; + command.Parameters.Clear(); + + for (var i = 0; i < args.Length; i++) + { + command.Parameters.AddWithValue("@" + i, args[i]); + } + } + + protected static void AddParameter(IDbCommand cmd, UmbracoDatabase.ParameterInfo parameterInfo) + { + var p = cmd.CreateParameter(); + p.ParameterName = parameterInfo.Name; + p.Value = parameterInfo.Value; + p.DbType = parameterInfo.DbType; + p.Size = parameterInfo.Size; + cmd.Parameters.Add(p); + } + + protected static void ResetTestDatabase(IDbCommand cmd) + { + // https://stackoverflow.com/questions/536350 + + cmd.CommandType = CommandType.Text; + cmd.CommandText = @" + declare @n char(1); + set @n = char(10); + declare @stmt nvarchar(max); + -- check constraints + select @stmt = isnull( @stmt + @n, '' ) + + 'alter table [' + schema_name(schema_id) + '].[' + object_name( parent_object_id ) + '] drop constraint [' + name + ']' + from sys.check_constraints; + -- foreign keys + select @stmt = isnull( @stmt + @n, '' ) + + 'alter table [' + schema_name(schema_id) + '].[' + object_name( parent_object_id ) + '] drop constraint [' + name + ']' + from sys.foreign_keys; + -- tables + select @stmt = isnull( @stmt + @n, '' ) + + 'drop table [' + schema_name(schema_id) + '].[' + name + ']' + from sys.tables; + exec sp_executesql @stmt; + "; + + // rudimentary retry policy since a db can still be in use when we try to drop + Retry(10, () => cmd.ExecuteNonQuery()); + } + + protected static void Retry(int maxIterations, Action action) + { + for (var i = 0; i < maxIterations; i++) + { + try + { + action(); + return; + } + catch (SqlException) + { + + //Console.Error.WriteLine($"SqlException occured, but we try again {i+1}/{maxIterations}.\n{e}"); + // This can occur when there's a transaction deadlock which means (i think) that the database is still in use and hasn't been closed properly yet + // so we need to just wait a little bit + Thread.Sleep(100 * i); + if (i == maxIterations - 1) + { + Debugger.Launch(); + throw; + } + } + catch (InvalidOperationException) + { + // Ignore + } + } + } + } +} diff --git a/src/Umbraco.Tests.Integration/Testing/ITestDatabase.cs b/src/Umbraco.Tests.Integration/Testing/ITestDatabase.cs index 250e062ee4..28d7e9c8bc 100644 --- a/src/Umbraco.Tests.Integration/Testing/ITestDatabase.cs +++ b/src/Umbraco.Tests.Integration/Testing/ITestDatabase.cs @@ -1,10 +1,9 @@ -namespace Umbraco.Tests.Integration.Testing +namespace Umbraco.Tests.Integration.Testing { public interface ITestDatabase { - string ConnectionString { get; } - int AttachEmpty(); - int AttachSchema(); - void Detach(int id); + TestDbMeta AttachEmpty(); + TestDbMeta AttachSchema(); + void Detach(TestDbMeta id); } } diff --git a/src/Umbraco.Tests.Integration/Testing/LocalDbTestDatabase.cs b/src/Umbraco.Tests.Integration/Testing/LocalDbTestDatabase.cs index 0a081f48e9..a9a842cdcd 100644 --- a/src/Umbraco.Tests.Integration/Testing/LocalDbTestDatabase.cs +++ b/src/Umbraco.Tests.Integration/Testing/LocalDbTestDatabase.cs @@ -1,18 +1,8 @@ -using System; +using System; using System.Collections.Concurrent; -using System.Configuration; -using System.Data; -using System.Data.Common; -using System.Data.SqlClient; -using System.Diagnostics; using System.IO; -using System.Linq; using System.Threading; using Microsoft.Extensions.Logging; -using Umbraco.Core; -using Umbraco.Core.Configuration; -using Umbraco.Core.Configuration.Models; -using Umbraco.Core.Migrations.Install; using Umbraco.Core.Persistence; namespace Umbraco.Tests.Integration.Testing @@ -20,181 +10,104 @@ namespace Umbraco.Tests.Integration.Testing /// /// Manages a pool of LocalDb databases for integration testing /// - public class LocalDbTestDatabase : ITestDatabase + public class LocalDbTestDatabase : BaseTestDatabase, ITestDatabase { public const string InstanceName = "UmbracoTests"; public const string DatabaseName = "UmbracoTests"; - private readonly ILoggerFactory _loggerFactory; private readonly LocalDb _localDb; - private readonly IUmbracoVersion _umbracoVersion; - private static LocalDb.Instance _instance; + private static LocalDb.Instance _localDbInstance; private static string _filesPath; - private readonly IUmbracoDatabaseFactory _dbFactory; - private UmbracoDatabase.CommandInfo[] _dbCommands; - private string _currentCstr; - private static DatabasePool _emptyPool; - private static DatabasePool _schemaPool; - private DatabasePool _currentPool; + + private const int _threadCount = 2; + + public static LocalDbTestDatabase Instance { get; private set; } //It's internal because `Umbraco.Core.Persistence.LocalDb` is internal internal LocalDbTestDatabase(ILoggerFactory loggerFactory, LocalDb localDb, string filesPath, IUmbracoDatabaseFactory dbFactory) { - _umbracoVersion = new UmbracoVersion(); _loggerFactory = loggerFactory; + _databaseFactory = dbFactory; + _localDb = localDb; _filesPath = filesPath; - _dbFactory = dbFactory; - _instance = _localDb.GetInstance(InstanceName); - if (_instance != null) return; + Instance = this; // For GlobalSetupTeardown.cs + + _testDatabases = new[] + { + // With Schema + TestDbMeta.CreateWithoutConnectionString($"{DatabaseName}-1", false), + TestDbMeta.CreateWithoutConnectionString($"{DatabaseName}-2", false), + + // Empty (for migration testing etc) + TestDbMeta.CreateWithoutConnectionString($"{DatabaseName}-3", true), + TestDbMeta.CreateWithoutConnectionString($"{DatabaseName}-4", true), + }; + + _localDbInstance = _localDb.GetInstance(InstanceName); + if (_localDbInstance != null) + { + return; + } if (_localDb.CreateInstance(InstanceName) == false) + { throw new Exception("Failed to create a LocalDb instance."); - _instance = _localDb.GetInstance(InstanceName); + } + + _localDbInstance = _localDb.GetInstance(InstanceName); } - public string ConnectionString => _currentCstr ?? _instance.GetAttachedConnectionString("XXXXXX", _filesPath); - - private void Create() + protected override void Initialize() { var tempName = Guid.NewGuid().ToString("N"); - _instance.CreateDatabase(tempName, _filesPath); - _instance.DetachDatabase(tempName); + _localDbInstance.CreateDatabase(tempName, _filesPath); + _localDbInstance.DetachDatabase(tempName); + _prepareQueue = new BlockingCollection(); + _readySchemaQueue = new BlockingCollection(); + _readyEmptyQueue = new BlockingCollection(); - // there's probably a sweet spot to be found for size / parallel... - - var s = ConfigurationManager.AppSettings["Umbraco.Tests.LocalDbTestDatabase.EmptyPoolSize"]; - var emptySize = s == null ? 1 : int.Parse(s); - s = ConfigurationManager.AppSettings["Umbraco.Tests.LocalDbTestDatabase.EmptyPoolThreadCount"]; - var emptyParallel = s == null ? 1 : int.Parse(s); - s = ConfigurationManager.AppSettings["Umbraco.Tests.LocalDbTestDatabase.SchemaPoolSize"]; - var schemaSize = s == null ? 1 : int.Parse(s); - s = ConfigurationManager.AppSettings["Umbraco.Tests.LocalDbTestDatabase.SchemaPoolThreadCount"]; - var schemaParallel = s == null ? 1 : int.Parse(s); - - _emptyPool = new DatabasePool(_localDb, _instance, DatabaseName + "-Empty", tempName, _filesPath, emptySize, emptyParallel); - _schemaPool = new DatabasePool(_localDb, _instance, DatabaseName + "-Schema", tempName, _filesPath, schemaSize, schemaParallel, delete: true, prepare: RebuildSchema); - } - - public int AttachEmpty() - { - if (_emptyPool == null) - Create(); - - _currentCstr = _emptyPool.AttachDatabase(out var id); - _currentPool = _emptyPool; - return id; - } - - public int AttachSchema() - { - if (_schemaPool == null) - Create(); - - _currentCstr = _schemaPool.AttachDatabase(out var id); - _currentPool = _schemaPool; - return id; - } - - public void Detach(int id) - { - _currentPool.DetachDatabase(id); - } - - private void RebuildSchema(DbConnection conn, IDbCommand cmd) - { - - if (_dbCommands != null) + foreach (var meta in _testDatabases) { - foreach (var dbCommand in _dbCommands) - { - - if (dbCommand.Text.StartsWith("SELECT ")) continue; - - cmd.CommandText = dbCommand.Text; - cmd.Parameters.Clear(); - foreach (var parameterInfo in dbCommand.Parameters) - AddParameter(cmd, parameterInfo); - cmd.ExecuteNonQuery(); - } - } - else - { - _dbFactory.Configure(conn.ConnectionString, Constants.DatabaseProviders.SqlServer); - - using var database = (UmbracoDatabase)_dbFactory.CreateDatabase(); - // track each db command ran as part of creating the database so we can replay these - database.LogCommands = true; - - using var trans = database.GetTransaction(); - - var creator = new DatabaseSchemaCreator(database, _loggerFactory.CreateLogger(), _loggerFactory, _umbracoVersion); - creator.InitializeDatabaseSchema(); - - trans.Complete(); // commit it - - _dbCommands = database.Commands.ToArray(); + _localDb.CopyDatabaseFiles(tempName, _filesPath, targetDatabaseName: meta.Name, overwrite: true, delete: false); + meta.ConnectionString = _localDbInstance.GetAttachedConnectionString(meta.Name, _filesPath); + _prepareQueue.Add(meta); } - } - - internal static void AddParameter(IDbCommand cmd, UmbracoDatabase.ParameterInfo parameterInfo) - { - var p = cmd.CreateParameter(); - p.ParameterName = parameterInfo.Name; - p.Value = parameterInfo.Value; - p.DbType = parameterInfo.DbType; - p.Size = parameterInfo.Size; - cmd.Parameters.Add(p); - } - - - internal static void ResetLocalDb(IDbCommand cmd) - { - // https://stackoverflow.com/questions/536350 - - cmd.CommandType = CommandType.Text; - cmd.CommandText = @" - declare @n char(1); - set @n = char(10); - declare @stmt nvarchar(max); - -- check constraints - select @stmt = isnull( @stmt + @n, '' ) + - 'alter table [' + schema_name(schema_id) + '].[' + object_name( parent_object_id ) + '] drop constraint [' + name + ']' - from sys.check_constraints; - -- foreign keys - select @stmt = isnull( @stmt + @n, '' ) + - 'alter table [' + schema_name(schema_id) + '].[' + object_name( parent_object_id ) + '] drop constraint [' + name + ']' - from sys.foreign_keys; - -- tables - select @stmt = isnull( @stmt + @n, '' ) + - 'drop table [' + schema_name(schema_id) + '].[' + name + ']' - from sys.tables; - exec sp_executesql @stmt; - "; - - // rudimentary retry policy since a db can still be in use when we try to drop - Retry(10, () => + for (var i = 0; i < _threadCount; i++) { - cmd.ExecuteNonQuery(); - }); + var thread = new Thread(PrepareDatabase); + thread.Start(); + } } - public static void KillLocalDb() + public void Finish() { - _emptyPool?.Stop(); - _schemaPool?.Stop(); + if (_prepareQueue == null) + return; + + _prepareQueue.CompleteAdding(); + while (_prepareQueue.TryTake(out _)) + { } + + _readyEmptyQueue.CompleteAdding(); + while (_readyEmptyQueue.TryTake(out _)) + { } + + _readySchemaQueue.CompleteAdding(); + while (_readySchemaQueue.TryTake(out _)) + { } if (_filesPath == null) return; var filename = Path.Combine(_filesPath, DatabaseName).ToUpper(); - foreach (var database in _instance.GetDatabases()) + foreach (var database in _localDbInstance.GetDatabases()) { if (database.StartsWith(filename)) - _instance.DropDatabase(database); + _localDbInstance.DropDatabase(database); } foreach (var file in Directory.EnumerateFiles(_filesPath)) @@ -210,145 +123,5 @@ namespace Umbraco.Tests.Integration.Testing } } } - - internal static void Retry(int maxIterations, Action action) - { - for (var i = 0; i < maxIterations; i++) - { - try - { - action(); - return; - } - catch (SqlException) - { - - //Console.Error.WriteLine($"SqlException occured, but we try again {i+1}/{maxIterations}.\n{e}"); - // This can occur when there's a transaction deadlock which means (i think) that the database is still in use and hasn't been closed properly yet - // so we need to just wait a little bit - Thread.Sleep(100 * i); - if (i == maxIterations - 1) - { - Debugger.Launch(); - throw; - } - } - catch (InvalidOperationException) - { - - } - } - } - - private class DatabasePool - { - private readonly LocalDb _localDb; - private readonly LocalDb.Instance _instance; - private readonly string _filesPath; - private readonly string _name; - private readonly int _size; - private readonly string[] _cstrs; - private readonly BlockingCollection _prepareQueue, _readyQueue; - private readonly Action _prepare; - private int _current; - - public DatabasePool(LocalDb localDb, LocalDb.Instance instance, string name, string tempName, string filesPath, int size, int parallel = 1, Action prepare = null, bool delete = false) - { - _localDb = localDb; - _instance = instance; - _filesPath = filesPath; - _name = name; - _size = size; - _prepare = prepare; - _prepareQueue = new BlockingCollection(); - _readyQueue = new BlockingCollection(); - _cstrs = new string[_size]; - - for (var i = 0; i < size; i++) - localDb.CopyDatabaseFiles(tempName, filesPath, targetDatabaseName: name + "-" + i, overwrite: true, delete: delete && i == size - 1); - - if (prepare == null) - { - for (var i = 0; i < size; i++) - _readyQueue.Add(i); - } - else - { - for (var i = 0; i < size; i++) - _prepareQueue.Add(i); - } - - for (var i = 0; i < parallel; i++) - { - var thread = new Thread(PrepareThread); - thread.Start(); - } - } - - public string AttachDatabase(out int id) - { - _current = _readyQueue.Take(); - id = _current; - - return ConnectionString(_current); - } - - public void DetachDatabase(int id) - { - if (id != _current) - throw new InvalidOperationException("Cannot detatch the non-current db"); - - _prepareQueue.Add(_current); - } - - private string ConnectionString(int i) - { - return _cstrs[i] ?? (_cstrs[i] = _instance.GetAttachedConnectionString(_name + "-" + i, _filesPath)); - } - - private void PrepareThread() - { - Retry(10, () => - { - while (_prepareQueue.IsCompleted == false) - { - int i; - try - { - i = _prepareQueue.Take(); - } - catch (InvalidOperationException) - { - continue; - } - - using (var conn = new SqlConnection(ConnectionString(i))) - using (var cmd = conn.CreateCommand()) - { - conn.Open(); - ResetLocalDb(cmd); - - _prepare?.Invoke(conn, cmd); - - } - - if (!_readyQueue.IsAddingCompleted) - { - _readyQueue.Add(i); - } - } - }); - } - - public void Stop() - { - int i; - _prepareQueue.CompleteAdding(); - while (_prepareQueue.TryTake(out i)) { } - _readyQueue.CompleteAdding(); - while (_readyQueue.TryTake(out i)) { } - } - } - } } diff --git a/src/Umbraco.Tests.Integration/Testing/SqlDeveloperTestDatabase.cs b/src/Umbraco.Tests.Integration/Testing/SqlDeveloperTestDatabase.cs index 52be6d5472..4a7f602ac6 100644 --- a/src/Umbraco.Tests.Integration/Testing/SqlDeveloperTestDatabase.cs +++ b/src/Umbraco.Tests.Integration/Testing/SqlDeveloperTestDatabase.cs @@ -1,15 +1,8 @@ -using System; +using System; using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Data; using System.Data.SqlClient; -using System.Linq; -using System.Text.RegularExpressions; using System.Threading; using Microsoft.Extensions.Logging; -using Umbraco.Core; -using Umbraco.Core.Configuration; -using Umbraco.Core.Migrations.Install; using Umbraco.Core.Persistence; // ReSharper disable ConvertToUsingDeclaration @@ -19,82 +12,57 @@ namespace Umbraco.Tests.Integration.Testing /// /// It's not meant to be pretty, rushed port of LocalDb.cs + LocalDbTestDatabase.cs /// - public class SqlDeveloperTestDatabase : ITestDatabase + public class SqlDeveloperTestDatabase : BaseTestDatabase, ITestDatabase { - - // This is gross but it's how the other one works and I don't want to refactor everything. - public string ConnectionString { get; private set; } - private readonly string _masterConnectionString; - private readonly ILoggerFactory _loggerFactory; - private readonly ILogger _log; - private readonly IUmbracoDatabaseFactory _databaseFactory; - private readonly IDictionary _testDatabases; - private UmbracoDatabase.CommandInfo[] _cachedDatabaseInitCommands; + public const string DatabaseName = "UmbracoTests"; - private BlockingCollection _prepareQueue; - private BlockingCollection _readySchemaQueue; - private BlockingCollection _readyEmptyQueue; - - private const string _databasePrefix = "UmbracoTest"; private const int _threadCount = 2; - public static SqlDeveloperTestDatabase Instance; + public static SqlDeveloperTestDatabase Instance { get; private set; } public SqlDeveloperTestDatabase(ILoggerFactory loggerFactory, IUmbracoDatabaseFactory databaseFactory, string masterConnectionString) { _loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory)); _databaseFactory = databaseFactory ?? throw new ArgumentNullException(nameof(databaseFactory)); + _masterConnectionString = masterConnectionString; - _log = loggerFactory.CreateLogger(); _testDatabases = new[] { - new TestDbMeta(1, false, masterConnectionString), - new TestDbMeta(2, false, masterConnectionString), + // With Schema + TestDbMeta.CreateWithMasterConnectionString($"{DatabaseName}-1", false, masterConnectionString), + TestDbMeta.CreateWithMasterConnectionString($"{DatabaseName}-2", false, masterConnectionString), - new TestDbMeta(3, true, masterConnectionString), - }.ToDictionary(x => x.Id); + // Empty (for migration testing etc) + TestDbMeta.CreateWithMasterConnectionString($"{DatabaseName}-3", true, masterConnectionString), + TestDbMeta.CreateWithMasterConnectionString($"{DatabaseName}-4", true, masterConnectionString), + }; Instance = this; // For GlobalSetupTeardown.cs } - public int AttachEmpty() + protected override void Initialize() { - if (_prepareQueue == null) + _prepareQueue = new BlockingCollection(); + _readySchemaQueue = new BlockingCollection(); + _readyEmptyQueue = new BlockingCollection(); + + foreach (var meta in _testDatabases) { - Initialize(); + CreateDatabase(meta); + _prepareQueue.Add(meta); } - var meta = _readyEmptyQueue.Take(); - - ConnectionString = meta.ConnectionString; - - return meta.Id; - } - - public int AttachSchema() - { - if (_prepareQueue == null) + for (var i = 0; i < _threadCount; i++) { - Initialize(); + var thread = new Thread(PrepareDatabase); + thread.Start(); } - - var meta = _readySchemaQueue.Take(); - - ConnectionString = meta.ConnectionString; - - return meta.Id; - } - - public void Detach(int id) - { - _prepareQueue.TryAdd(_testDatabases[id]); } private void CreateDatabase(TestDbMeta meta) { - _log.LogInformation($"Creating database {meta.Name}"); using (var connection = new SqlConnection(_masterConnectionString)) { connection.Open(); @@ -106,91 +74,8 @@ namespace Umbraco.Tests.Integration.Testing } } - private static string ConstructConnectionString(string masterConnectionString, string databaseName) - { - var prefix = Regex.Replace(masterConnectionString, "Database=.+?;", string.Empty); - var connectionString = $"{prefix};Database={databaseName};"; - return connectionString.Replace(";;", ";"); - } - - private static void SetCommand(SqlCommand command, string sql, params object[] args) - { - command.CommandType = CommandType.Text; - command.CommandText = sql; - command.Parameters.Clear(); - - for (var i = 0; i < args.Length; i++) - { - command.Parameters.AddWithValue("@" + i, args[i]); - } - } - - private void RebuildSchema(IDbCommand command, TestDbMeta meta) - { - if (_cachedDatabaseInitCommands != null) - { - foreach (var dbCommand in _cachedDatabaseInitCommands) - { - - if (dbCommand.Text.StartsWith("SELECT ")) - { - continue; - } - - command.CommandText = dbCommand.Text; - command.Parameters.Clear(); - - foreach (var parameterInfo in dbCommand.Parameters) - { - LocalDbTestDatabase.AddParameter(command, parameterInfo); - } - - command.ExecuteNonQuery(); - } - } - else - { - _databaseFactory.Configure(meta.ConnectionString, Constants.DatabaseProviders.SqlServer); - - using (var database = (UmbracoDatabase)_databaseFactory.CreateDatabase()) - { - database.LogCommands = true; - - using (var transaction = database.GetTransaction()) - { - var schemaCreator = new DatabaseSchemaCreator(database, _loggerFactory.CreateLogger(), _loggerFactory, new UmbracoVersion()); - schemaCreator.InitializeDatabaseSchema(); - - transaction.Complete(); - - _cachedDatabaseInitCommands = database.Commands.ToArray(); - } - } - } - } - - private void Initialize() - { - _prepareQueue = new BlockingCollection(); - _readySchemaQueue = new BlockingCollection(); - _readyEmptyQueue = new BlockingCollection(); - - foreach (var meta in _testDatabases.Values) - { - CreateDatabase(meta); - _prepareQueue.Add(meta); - } - - for (var i = 0; i < _threadCount; i++) - { - var thread = new Thread(PrepareThread); - thread.Start(); - } - } - private void Drop(TestDbMeta meta) { - _log.LogInformation($"Dropping database {meta.Name}"); using (var connection = new SqlConnection(_masterConnectionString)) { connection.Open(); @@ -209,46 +94,6 @@ namespace Umbraco.Tests.Integration.Testing } } - private void PrepareThread() - { - LocalDbTestDatabase.Retry(10, () => - { - while (_prepareQueue.IsCompleted == false) - { - TestDbMeta meta; - try - { - meta = _prepareQueue.Take(); - } - catch (InvalidOperationException) - { - continue; - } - - using (var conn = new SqlConnection(meta.ConnectionString)) - using (var cmd = conn.CreateCommand()) - { - conn.Open(); - LocalDbTestDatabase.ResetLocalDb(cmd); - - if (!meta.IsEmpty) - { - RebuildSchema(cmd, meta); - } - } - - if (!meta.IsEmpty) - { - _readySchemaQueue.TryAdd(meta); - } - else - { - _readyEmptyQueue.TryAdd(meta); - } - } - }); - } - public void Finish() { if (_prepareQueue == null) @@ -263,25 +108,10 @@ namespace Umbraco.Tests.Integration.Testing _readySchemaQueue.CompleteAdding(); while (_readySchemaQueue.TryTake(out _)) { } - foreach (var testDatabase in _testDatabases.Values) + foreach (var testDatabase in _testDatabases) { Drop(testDatabase); } } - - private class TestDbMeta - { - public int Id { get; } - public string Name => $"{_databasePrefix}-{Id}"; - public bool IsEmpty { get; } - public string ConnectionString { get; } - - public TestDbMeta(int id, bool isEmpty, string masterConnectionString) - { - Id = id; - IsEmpty = isEmpty; - ConnectionString = ConstructConnectionString(masterConnectionString, Name); - } - } } } diff --git a/src/Umbraco.Tests.Integration/Testing/TestDatabaseFactory.cs b/src/Umbraco.Tests.Integration/Testing/TestDatabaseFactory.cs index 3ed2b6b49a..9bcbfa4d3a 100644 --- a/src/Umbraco.Tests.Integration/Testing/TestDatabaseFactory.cs +++ b/src/Umbraco.Tests.Integration/Testing/TestDatabaseFactory.cs @@ -1,5 +1,4 @@ -using System; -using System.Runtime.InteropServices; +using System; using Microsoft.Extensions.Logging; using Umbraco.Core.Persistence; @@ -9,7 +8,7 @@ namespace Umbraco.Tests.Integration.Testing { public static ITestDatabase Create(string filesPath, ILoggerFactory loggerFactory, TestUmbracoDatabaseFactoryProvider dbFactory) { - return RuntimeInformation.IsOSPlatform(OSPlatform.Windows) + return string.IsNullOrEmpty(Environment.GetEnvironmentVariable("UmbracoIntegrationTestConnectionString")) ? CreateLocalDb(filesPath, loggerFactory, dbFactory.Create()) : CreateSqlDeveloper(loggerFactory, dbFactory.Create()); } diff --git a/src/Umbraco.Tests.Integration/Testing/TestDbMeta.cs b/src/Umbraco.Tests.Integration/Testing/TestDbMeta.cs new file mode 100644 index 0000000000..83702db8e5 --- /dev/null +++ b/src/Umbraco.Tests.Integration/Testing/TestDbMeta.cs @@ -0,0 +1,39 @@ +using System.Text.RegularExpressions; + +namespace Umbraco.Tests.Integration.Testing +{ + public class TestDbMeta + { + + public string Name { get; } + + public bool IsEmpty { get; } + + public string ConnectionString { get; set; } + + private TestDbMeta(string name, bool isEmpty, string connectionString) + { + IsEmpty = isEmpty; + Name = name; + ConnectionString = connectionString; + } + + private static string ConstructConnectionString(string masterConnectionString, string databaseName) + { + var prefix = Regex.Replace(masterConnectionString, "Database=.+?;", string.Empty); + var connectionString = $"{prefix};Database={databaseName};"; + return connectionString.Replace(";;", ";"); + } + + public static TestDbMeta CreateWithMasterConnectionString(string name, bool isEmpty, string masterConnectionString) + { + return new TestDbMeta(name, isEmpty, ConstructConnectionString(masterConnectionString, name)); + } + + // LocalDb mdf funtimes + public static TestDbMeta CreateWithoutConnectionString(string name, bool isEmpty) + { + return new TestDbMeta(name, isEmpty, null); + } + } +} diff --git a/src/Umbraco.Tests.Integration/Testing/UmbracoIntegrationTest.cs b/src/Umbraco.Tests.Integration/Testing/UmbracoIntegrationTest.cs index 3da4ec94d6..43b2d236c7 100644 --- a/src/Umbraco.Tests.Integration/Testing/UmbracoIntegrationTest.cs +++ b/src/Umbraco.Tests.Integration/Testing/UmbracoIntegrationTest.cs @@ -244,6 +244,7 @@ namespace Umbraco.Tests.Integration.Testing private static readonly object _dbLocker = new object(); private static ITestDatabase _dbInstance; + private static TestDbMeta _fixtureDbMeta; protected void UseTestLocalDb(IServiceProvider serviceProvider) { @@ -253,8 +254,6 @@ namespace Umbraco.Tests.Integration.Testing // This will create a db, install the schema and ensure the app is configured to run InstallTestLocalDb(testDatabaseFactoryProvider, databaseFactory, serviceProvider.GetRequiredService(), state, TestHelper.WorkingDirectory); - TestDBConnectionString = databaseFactory.ConnectionString; - InMemoryConfiguration["ConnectionStrings:" + Constants.System.UmbracoConnectionName] = TestDBConnectionString; } /// @@ -312,15 +311,15 @@ namespace Umbraco.Tests.Integration.Testing case UmbracoTestOptions.Database.NewSchemaPerTest: // New DB + Schema - var newSchemaDbId = db.AttachSchema(); + var newSchemaDbMeta = db.AttachSchema(); // Add teardown callback - OnTestTearDown(() => db.Detach(newSchemaDbId)); + OnTestTearDown(() => db.Detach(newSchemaDbMeta)); // We must re-configure our current factory since attaching a new LocalDb from the pool changes connection strings if (!databaseFactory.Configured) { - databaseFactory.Configure(db.ConnectionString, Constants.DatabaseProviders.SqlServer); + databaseFactory.Configure(newSchemaDbMeta.ConnectionString, Constants.DatabaseProviders.SqlServer); } // re-run the runtime level check @@ -330,15 +329,15 @@ namespace Umbraco.Tests.Integration.Testing break; case UmbracoTestOptions.Database.NewEmptyPerTest: - var newEmptyDbId = db.AttachEmpty(); + var newEmptyDbMeta = db.AttachEmpty(); // Add teardown callback - OnTestTearDown(() => db.Detach(newEmptyDbId)); + OnTestTearDown(() => db.Detach(newEmptyDbMeta)); // We must re-configure our current factory since attaching a new LocalDb from the pool changes connection strings if (!databaseFactory.Configured) { - databaseFactory.Configure(db.ConnectionString, Constants.DatabaseProviders.SqlServer); + databaseFactory.Configure(newEmptyDbMeta.ConnectionString, Constants.DatabaseProviders.SqlServer); } // re-run the runtime level check @@ -354,16 +353,17 @@ namespace Umbraco.Tests.Integration.Testing if (FirstTestInFixture) { // New DB + Schema - var newSchemaFixtureDbId = db.AttachSchema(); + var newSchemaFixtureDbMeta = db.AttachSchema(); + _fixtureDbMeta = newSchemaFixtureDbMeta; // Add teardown callback - OnFixtureTearDown(() => db.Detach(newSchemaFixtureDbId)); + OnFixtureTearDown(() => db.Detach(newSchemaFixtureDbMeta)); } // We must re-configure our current factory since attaching a new LocalDb from the pool changes connection strings if (!databaseFactory.Configured) { - databaseFactory.Configure(db.ConnectionString, Constants.DatabaseProviders.SqlServer); + databaseFactory.Configure(_fixtureDbMeta.ConnectionString, Constants.DatabaseProviders.SqlServer); } // re-run the runtime level check @@ -377,16 +377,17 @@ namespace Umbraco.Tests.Integration.Testing if (FirstTestInFixture) { // New DB + Schema - var newEmptyFixtureDbId = db.AttachEmpty(); + var newEmptyFixtureDbMeta = db.AttachEmpty(); + _fixtureDbMeta = newEmptyFixtureDbMeta; // Add teardown callback - OnFixtureTearDown(() => db.Detach(newEmptyFixtureDbId)); + OnFixtureTearDown(() => db.Detach(newEmptyFixtureDbMeta)); } // We must re-configure our current factory since attaching a new LocalDb from the pool changes connection strings if (!databaseFactory.Configured) { - databaseFactory.Configure(db.ConnectionString, Constants.DatabaseProviders.SqlServer); + databaseFactory.Configure(_fixtureDbMeta.ConnectionString, Constants.DatabaseProviders.SqlServer); } break; @@ -407,8 +408,6 @@ namespace Umbraco.Tests.Integration.Testing public TestHelper TestHelper = new TestHelper(); - protected virtual string TestDBConnectionString { get; private set; } - protected virtual Action CustomTestSetup => services => { }; /// diff --git a/src/Umbraco.Tests/TestHelpers/TestWithDatabaseBase.cs b/src/Umbraco.Tests/TestHelpers/TestWithDatabaseBase.cs index faf387528d..8c7b9a00e2 100644 --- a/src/Umbraco.Tests/TestHelpers/TestWithDatabaseBase.cs +++ b/src/Umbraco.Tests/TestHelpers/TestWithDatabaseBase.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Configuration; using System.Data.SqlServerCe; using System.Threading; @@ -91,7 +91,6 @@ namespace Umbraco.Tests.TestHelpers var lazyMappers = new Lazy(f.GetRequiredService); var factory = new UmbracoDatabaseFactory(f.GetRequiredService>(), f.GetRequiredService(), GetDbConnectionString(), GetDbProviderName(), lazyMappers, TestHelper.DbProviderFactoryCreator); - factory.ResetForTests(); return factory; }); }