v10 - SQLite database creation enhancements (#12166)

* Resolve issues creating database file for new install on app services.

* Prevent accidental create of SQLite database when testing connections

* Fix e2e tests against sqlite db

build/azure-pipelines.yml

@@ -226,6 +226,8 @@ stages:
             value: cypress@umbraco.com
           - name: Umbraco__CMS__Unattended__UnattendedUserPassword
             value: UmbracoAcceptance123!
+          - name: Umbraco__CMS__Global__InstallMissingDatabase
+            value: true
       jobs:
           - job: Windows_Acceptance_tests
             variables:

src/Umbraco.Cms.Persistence.Sqlite/Services/SqliteDatabaseCreator.cs

@@ -1,5 +1,6 @@
 using System.Diagnostics;
 using Microsoft.Data.Sqlite;
+using Microsoft.Extensions.Logging;
 using Umbraco.Cms.Infrastructure.Persistence;
 
 namespace Umbraco.Cms.Persistence.Sqlite.Services;
@@ -9,9 +10,16 @@ namespace Umbraco.Cms.Persistence.Sqlite.Services;
 /// </summary>
 public class SqliteDatabaseCreator : IDatabaseCreator
 {
+    private readonly ILogger<SqliteDatabaseCreator> _logger;
+
     /// <inheritdoc />
     public string ProviderName => Constants.ProviderName;
 
+    public SqliteDatabaseCreator(ILogger<SqliteDatabaseCreator> logger)
+    {
+        _logger = logger;
+    }
+
     /// <summary>
     /// Creates a SQLite database file.
     /// </summary>
@@ -33,16 +41,69 @@ public class SqliteDatabaseCreator : IDatabaseCreator
     /// </remarks>
     public void Create(string connectionString)
     {
-        using var connection = new SqliteConnection(connectionString);
-        connection.Open();
+        var original = new SqliteConnectionStringBuilder(connectionString);
 
-        using SqliteCommand command = connection.CreateCommand();
-        command.CommandText = "PRAGMA journal_mode = wal;";
-        command.ExecuteNonQuery();
+        if (original.Mode == SqliteOpenMode.Memory || original.DataSource == ":memory:")
+        {
+            // In-Memory mode - bail
+            return;
+        }
 
-        command.CommandText = "PRAGMA journal_mode";
-        var mode = command.ExecuteScalar();
+        if (original.DataSource.StartsWith("file:"))
+        {
+            // URI mode - bail
+            return;
+        }
 
-        Debug.Assert(mode as string == "wal", "incorrect journal_mode");
+        /* Note: The following may seem a bit mad, but it has a purpose!
+         *
+         * On azure app services if we wish to ensure the database is persistent we need to write it to the persistent network share
+         * e.g. c:\home or /home
+         *
+         * However the network share does not play nice at all with SQLite locking for rollback mode which is the default for new databases.
+         * May work on windows app services with win32 vfs but not at all on linux with unix vfs.
+         *
+         * The experience is so broken in fact that we can't even create an empty sqlite database file and switch from rollback to wal.
+         * However once a wal database is setup it works reasonably well (perhaps a tad slower than one might like) on the persistent share.
+         *
+         * So instead of creating in the final destination, we can create in /tmp || $env:Temp, set the wal bits
+         * and copy the file over to its new home and finally nuke the temp file.
+         *
+         * We could try to do this only on azure e.g. detect $WEBSITE_RESOURCE_GROUP etc but there's no downside to
+         * always initializing in this way and it probably helps for non azure scenarios also (anytime persisting on a cifs mount for example).
+         */
+
+        var tempFile = Path.GetTempFileName();
+        var tempConnectionString = new SqliteConnectionStringBuilder { DataSource = tempFile };
+
+        using (var connection = new SqliteConnection(tempConnectionString.ConnectionString))
+        {
+            connection.Open();
+
+            using SqliteCommand command = connection.CreateCommand();
+            command.CommandText = "PRAGMA journal_mode = wal;";
+            command.ExecuteNonQuery();
+        }
+
+        // Copy our blank(ish) wal mode sqlite database to its final location.
+        try
+        {
+            File.Copy(tempFile, original.DataSource, overwrite: true);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogCritical(ex, "Unable to initialize sqlite database file.");
+            throw;
+        }
+
+        try
+        {
+            File.Delete(tempFile);
+        }
+        catch (Exception ex)
+        {
+            // We can swallow this, no worries if we can't nuke the practically empty database file.
+            _logger.LogWarning(ex, "Unable to cleanup temporary sqlite database file {path}", tempFile);
+        }
     }
 }

src/Umbraco.Cms.Persistence.Sqlite/UmbracoBuilderExtensions.cs

@@ -1,12 +1,15 @@
 using System.Data.Common;
+using Microsoft.Data.Sqlite;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.DependencyInjection.Extensions;
+using Umbraco.Cms.Core.Configuration.Models;
 using Umbraco.Cms.Core.DependencyInjection;
 using Umbraco.Cms.Core.DistributedLocking;
 using Umbraco.Cms.Infrastructure.Persistence;
 using Umbraco.Cms.Infrastructure.Persistence.SqlSyntax;
 using Umbraco.Cms.Persistence.Sqlite.Interceptors;
 using Umbraco.Cms.Persistence.Sqlite.Services;
+using Umbraco.Extensions;
 
 namespace Umbraco.Cms.Persistence.Sqlite;
 
@@ -36,6 +39,30 @@ public static class UmbracoBuilderExtensions
         DbProviderFactories.UnregisterFactory(Constants.ProviderName);
         DbProviderFactories.RegisterFactory(Constants.ProviderName, Microsoft.Data.Sqlite.SqliteFactory.Instance);
 
+
+        builder.Services.PostConfigure<ConnectionStrings>(Core.Constants.System.UmbracoConnectionName, opt =>
+        {
+            if (!opt.IsConnectionStringConfigured())
+            {
+                return;
+            }
+
+            if (opt.ProviderName != Constants.ProviderName)
+            {
+                // Not us.
+                return;
+            }
+
+            // Prevent accidental creation of database files.
+            var connectionStringBuilder = new SqliteConnectionStringBuilder(opt.ConnectionString);
+            if (connectionStringBuilder.Mode == SqliteOpenMode.ReadWriteCreate)
+            {
+                connectionStringBuilder.Mode = SqliteOpenMode.ReadWrite;
+            }
+
+            opt.ConnectionString = connectionStringBuilder.ConnectionString;
+        });
+
         return builder;
     }
 }