From d12602ec90a98199bf67c719c7f73f5da6e180b7 Mon Sep 17 00:00:00 2001 From: Paul Johnson Date: Fri, 1 Apr 2022 10:15:04 +0100 Subject: [PATCH] v10 - SQLite database creation enhancements (#12166) * Resolve issues creating database file for new install on app services. * Prevent accidental create of SQLite database when testing connections * Fix e2e tests against sqlite db --- build/azure-pipelines.yml | 2 + .../Services/SqliteDatabaseCreator.cs | 77 +++++++++++++++++-- .../UmbracoBuilderExtensions.cs | 27 +++++++ 3 files changed, 98 insertions(+), 8 deletions(-) diff --git a/build/azure-pipelines.yml b/build/azure-pipelines.yml index e3f4ed355b..ae92f37709 100644 --- a/build/azure-pipelines.yml +++ b/build/azure-pipelines.yml @@ -226,6 +226,8 @@ stages: value: cypress@umbraco.com - name: Umbraco__CMS__Unattended__UnattendedUserPassword value: UmbracoAcceptance123! + - name: Umbraco__CMS__Global__InstallMissingDatabase + value: true jobs: - job: Windows_Acceptance_tests variables: diff --git a/src/Umbraco.Cms.Persistence.Sqlite/Services/SqliteDatabaseCreator.cs b/src/Umbraco.Cms.Persistence.Sqlite/Services/SqliteDatabaseCreator.cs index 43980b3b77..de599529fa 100644 --- a/src/Umbraco.Cms.Persistence.Sqlite/Services/SqliteDatabaseCreator.cs +++ b/src/Umbraco.Cms.Persistence.Sqlite/Services/SqliteDatabaseCreator.cs @@ -1,5 +1,6 @@ using System.Diagnostics; using Microsoft.Data.Sqlite; +using Microsoft.Extensions.Logging; using Umbraco.Cms.Infrastructure.Persistence; namespace Umbraco.Cms.Persistence.Sqlite.Services; @@ -9,9 +10,16 @@ namespace Umbraco.Cms.Persistence.Sqlite.Services; /// public class SqliteDatabaseCreator : IDatabaseCreator { + private readonly ILogger _logger; + /// public string ProviderName => Constants.ProviderName; + public SqliteDatabaseCreator(ILogger logger) + { + _logger = logger; + } + /// /// Creates a SQLite database file. /// @@ -33,16 +41,69 @@ public class SqliteDatabaseCreator : IDatabaseCreator /// public void Create(string connectionString) { - using var connection = new SqliteConnection(connectionString); - connection.Open(); + var original = new SqliteConnectionStringBuilder(connectionString); - using SqliteCommand command = connection.CreateCommand(); - command.CommandText = "PRAGMA journal_mode = wal;"; - command.ExecuteNonQuery(); + if (original.Mode == SqliteOpenMode.Memory || original.DataSource == ":memory:") + { + // In-Memory mode - bail + return; + } - command.CommandText = "PRAGMA journal_mode"; - var mode = command.ExecuteScalar(); + if (original.DataSource.StartsWith("file:")) + { + // URI mode - bail + return; + } - Debug.Assert(mode as string == "wal", "incorrect journal_mode"); + /* Note: The following may seem a bit mad, but it has a purpose! + * + * On azure app services if we wish to ensure the database is persistent we need to write it to the persistent network share + * e.g. c:\home or /home + * + * However the network share does not play nice at all with SQLite locking for rollback mode which is the default for new databases. + * May work on windows app services with win32 vfs but not at all on linux with unix vfs. + * + * The experience is so broken in fact that we can't even create an empty sqlite database file and switch from rollback to wal. + * However once a wal database is setup it works reasonably well (perhaps a tad slower than one might like) on the persistent share. + * + * So instead of creating in the final destination, we can create in /tmp || $env:Temp, set the wal bits + * and copy the file over to its new home and finally nuke the temp file. + * + * We could try to do this only on azure e.g. detect $WEBSITE_RESOURCE_GROUP etc but there's no downside to + * always initializing in this way and it probably helps for non azure scenarios also (anytime persisting on a cifs mount for example). + */ + + var tempFile = Path.GetTempFileName(); + var tempConnectionString = new SqliteConnectionStringBuilder { DataSource = tempFile }; + + using (var connection = new SqliteConnection(tempConnectionString.ConnectionString)) + { + connection.Open(); + + using SqliteCommand command = connection.CreateCommand(); + command.CommandText = "PRAGMA journal_mode = wal;"; + command.ExecuteNonQuery(); + } + + // Copy our blank(ish) wal mode sqlite database to its final location. + try + { + File.Copy(tempFile, original.DataSource, overwrite: true); + } + catch (Exception ex) + { + _logger.LogCritical(ex, "Unable to initialize sqlite database file."); + throw; + } + + try + { + File.Delete(tempFile); + } + catch (Exception ex) + { + // We can swallow this, no worries if we can't nuke the practically empty database file. + _logger.LogWarning(ex, "Unable to cleanup temporary sqlite database file {path}", tempFile); + } } } diff --git a/src/Umbraco.Cms.Persistence.Sqlite/UmbracoBuilderExtensions.cs b/src/Umbraco.Cms.Persistence.Sqlite/UmbracoBuilderExtensions.cs index 0945b71270..ddce4788cc 100644 --- a/src/Umbraco.Cms.Persistence.Sqlite/UmbracoBuilderExtensions.cs +++ b/src/Umbraco.Cms.Persistence.Sqlite/UmbracoBuilderExtensions.cs @@ -1,12 +1,15 @@ using System.Data.Common; +using Microsoft.Data.Sqlite; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; +using Umbraco.Cms.Core.Configuration.Models; using Umbraco.Cms.Core.DependencyInjection; using Umbraco.Cms.Core.DistributedLocking; using Umbraco.Cms.Infrastructure.Persistence; using Umbraco.Cms.Infrastructure.Persistence.SqlSyntax; using Umbraco.Cms.Persistence.Sqlite.Interceptors; using Umbraco.Cms.Persistence.Sqlite.Services; +using Umbraco.Extensions; namespace Umbraco.Cms.Persistence.Sqlite; @@ -36,6 +39,30 @@ public static class UmbracoBuilderExtensions DbProviderFactories.UnregisterFactory(Constants.ProviderName); DbProviderFactories.RegisterFactory(Constants.ProviderName, Microsoft.Data.Sqlite.SqliteFactory.Instance); + + builder.Services.PostConfigure(Core.Constants.System.UmbracoConnectionName, opt => + { + if (!opt.IsConnectionStringConfigured()) + { + return; + } + + if (opt.ProviderName != Constants.ProviderName) + { + // Not us. + return; + } + + // Prevent accidental creation of database files. + var connectionStringBuilder = new SqliteConnectionStringBuilder(opt.ConnectionString); + if (connectionStringBuilder.Mode == SqliteOpenMode.ReadWriteCreate) + { + connectionStringBuilder.Mode = SqliteOpenMode.ReadWrite; + } + + opt.ConnectionString = connectionStringBuilder.ConnectionString; + }); + return builder; } }