diff --git a/src/Umbraco.Core/Sync/DatabaseServerMessenger.cs b/src/Umbraco.Core/Sync/DatabaseServerMessenger.cs index 49b0d23862..75ccf5e4f9 100644 --- a/src/Umbraco.Core/Sync/DatabaseServerMessenger.cs +++ b/src/Umbraco.Core/Sync/DatabaseServerMessenger.cs @@ -28,7 +28,7 @@ namespace Umbraco.Core.Sync // but only processes instructions coming from remote servers, // thus ensuring that instructions run only once // - public class DatabaseServerMessenger : ServerMessengerBase + public class DatabaseServerMessenger : ServerMessengerBase, ISyncBootStateAccessor { private readonly IRuntimeState _runtime; private readonly ManualResetEvent _syncIdle; @@ -39,9 +39,9 @@ namespace Umbraco.Core.Sync private int _lastId = -1; private DateTime _lastSync; private DateTime _lastPruned; - private bool _initialized; private bool _syncing; private bool _released; + private readonly Lazy _getSyncBootState; public DatabaseServerMessengerOptions Options { get; } @@ -59,6 +59,7 @@ namespace Umbraco.Core.Sync _lastPruned = _lastSync = DateTime.UtcNow; _syncIdle = new ManualResetEvent(true); _distCacheFilePath = new Lazy(() => GetDistCacheFilePath(globalSettings)); + _getSyncBootState = new Lazy(BootInternal); } protected ILogger Logger { get; } @@ -75,7 +76,7 @@ namespace Umbraco.Core.Sync { // we don't care if there's servers listed or not, // if distributed call is enabled we will make the call - return _initialized && DistributedEnabled; + return _getSyncBootState.IsValueCreated && DistributedEnabled; } protected override void DeliverRemote( @@ -110,14 +111,14 @@ namespace Umbraco.Core.Sync #region Sync - /// - /// Boots the messenger. - /// - /// - /// Thread safety: this is NOT thread safe. Because it is NOT meant to run multi-threaded. - /// Callers MUST ensure thread-safety. - /// + [Obsolete("This is no longer used and will be removed in future versions")] protected void Boot() + { + // if called, just forces the boot logic + _ = GetSyncBootState(); + } + + private SyncBootState BootInternal() { // weight:10, must release *before* the published snapshot service, because once released // the service will *not* be able to properly handle our notifications anymore @@ -139,7 +140,7 @@ namespace Umbraco.Core.Sync // properly releasing MainDom - a timeout here means that one refresher // is taking too much time processing, however when it's done we will // not update lastId and stop everything - var idle =_syncIdle.WaitOne(5000); + var idle = _syncIdle.WaitOne(5000); if (idle == false) { Logger.Warn("The wait lock timed out, application is shutting down. The current instruction batch will be re-processed."); @@ -147,17 +148,23 @@ namespace Umbraco.Core.Sync }, weight); + SyncBootState bootState = SyncBootState.Unknown; + if (registered == false) - return; + { + return bootState; + } ReadLastSynced(); // get _lastId using (var scope = ScopeProvider.CreateScope()) { EnsureInstructions(scope.Database); // reset _lastId if instructions are missing - Initialize(scope.Database); // boot + bootState = Initialize(scope.Database); // boot scope.Complete(); } + + return bootState; } /// @@ -167,60 +174,62 @@ namespace Umbraco.Core.Sync /// Thread safety: this is NOT thread safe. Because it is NOT meant to run multi-threaded. /// Callers MUST ensure thread-safety. /// - private void Initialize(IUmbracoDatabase database) + private SyncBootState Initialize(IUmbracoDatabase database) { - lock (_locko) - { - if (_released) return; + // could occur if shutting down immediately once starting up and before we've initialized + if (_released) return SyncBootState.Unknown; - var coldboot = false; - if (_lastId < 0) // never synced before + var coldboot = false; + if (_lastId < 0) // never synced before + { + // we haven't synced - in this case we aren't going to sync the whole thing, we will assume this is a new + // server and it will need to rebuild it's own caches, eg Lucene or the xml cache file. + Logger.Warn("No last synced Id found, this generally means this is a new server/install." + + " The server will build its caches and indexes, and then adjust its last synced Id to the latest found in" + + " the database and maintain cache updates based on that Id."); + + coldboot = true; + } + else + { + //check for how many instructions there are to process, each row contains a count of the number of instructions contained in each + //row so we will sum these numbers to get the actual count. + var count = database.ExecuteScalar("SELECT SUM(instructionCount) FROM umbracoCacheInstruction WHERE id > @lastId", new { lastId = _lastId }); + if (count > Options.MaxProcessingInstructionCount) { - // we haven't synced - in this case we aren't going to sync the whole thing, we will assume this is a new - // server and it will need to rebuild it's own caches, eg Lucene or the xml cache file. - Logger.Warn("No last synced Id found, this generally means this is a new server/install." - + " The server will build its caches and indexes, and then adjust its last synced Id to the latest found in" - + " the database and maintain cache updates based on that Id."); + //too many instructions, proceed to cold boot + Logger.Warn( + "The instruction count ({InstructionCount}) exceeds the specified MaxProcessingInstructionCount ({MaxProcessingInstructionCount})." + + " The server will skip existing instructions, rebuild its caches and indexes entirely, adjust its last synced Id" + + " to the latest found in the database and maintain cache updates based on that Id.", + count, Options.MaxProcessingInstructionCount); coldboot = true; } - else - { - //check for how many instructions there are to process, each row contains a count of the number of instructions contained in each - //row so we will sum these numbers to get the actual count. - var count = database.ExecuteScalar("SELECT SUM(instructionCount) FROM umbracoCacheInstruction WHERE id > @lastId", new {lastId = _lastId}); - if (count > Options.MaxProcessingInstructionCount) - { - //too many instructions, proceed to cold boot - Logger.Warn( - "The instruction count ({InstructionCount}) exceeds the specified MaxProcessingInstructionCount ({MaxProcessingInstructionCount})." - + " The server will skip existing instructions, rebuild its caches and indexes entirely, adjust its last synced Id" - + " to the latest found in the database and maintain cache updates based on that Id.", - count, Options.MaxProcessingInstructionCount); + } - coldboot = true; + if (coldboot) + { + // go get the last id in the db and store it + // note: do it BEFORE initializing otherwise some instructions might get lost + // when doing it before, some instructions might run twice - not an issue + var maxId = database.ExecuteScalar("SELECT MAX(id) FROM umbracoCacheInstruction"); + + //if there is a max currently, or if we've never synced + if (maxId > 0 || _lastId < 0) + SaveLastSynced(maxId); + + // execute initializing callbacks + if (Options.InitializingCallbacks != null) + { + foreach (var callback in Options.InitializingCallbacks) + { + callback(); } } - - if (coldboot) - { - // go get the last id in the db and store it - // note: do it BEFORE initializing otherwise some instructions might get lost - // when doing it before, some instructions might run twice - not an issue - var maxId = database.ExecuteScalar("SELECT MAX(id) FROM umbracoCacheInstruction"); - - //if there is a max currently, or if we've never synced - if (maxId > 0 || _lastId < 0) - SaveLastSynced(maxId); - - // execute initializing callbacks - if (Options.InitializingCallbacks != null) - foreach (var callback in Options.InitializingCallbacks) - callback(); - } - - _initialized = true; } + + return coldboot ? SyncBootState.ColdBoot : SyncBootState.WarmBoot; } /// @@ -352,7 +361,7 @@ namespace Umbraco.Core.Sync } catch (JsonException ex) { - Logger.Error(ex, "Failed to deserialize instructions ({DtoId}: '{DtoInstructions}').", + Logger.Error(ex, "Failed to deserialize instructions ({DtoId}: '{DtoInstructions}').", dto.Id, dto.Instructions); @@ -410,11 +419,11 @@ namespace Umbraco.Core.Sync //} catch (Exception ex) { - Logger.Error ( - ex, - "DISTRIBUTED CACHE IS NOT UPDATED. Failed to execute instructions ({DtoId}: '{DtoInstructions}'). Instruction is being skipped/ignored", - dto.Id, - dto.Instructions); + Logger.Error( + ex, + "DISTRIBUTED CACHE IS NOT UPDATED. Failed to execute instructions ({DtoId}: '{DtoInstructions}'). Instruction is being skipped/ignored", + dto.Id, + dto.Instructions); //we cannot throw here because this invalid instruction will just keep getting processed over and over and errors // will be thrown over and over. The only thing we can do is ignore and move on. @@ -548,6 +557,8 @@ namespace Umbraco.Core.Sync #endregion + public virtual SyncBootState GetSyncBootState() => _getSyncBootState.Value; + #region Notify refreshers private static ICacheRefresher GetRefresher(Guid id) diff --git a/src/Umbraco.Core/Sync/DatabaseServerMessengerOptions.cs b/src/Umbraco.Core/Sync/DatabaseServerMessengerOptions.cs index 6bfd6bff4a..a769709d09 100644 --- a/src/Umbraco.Core/Sync/DatabaseServerMessengerOptions.cs +++ b/src/Umbraco.Core/Sync/DatabaseServerMessengerOptions.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.ComponentModel; namespace Umbraco.Core.Sync { @@ -24,13 +25,8 @@ namespace Umbraco.Core.Sync /// public int MaxProcessingInstructionCount { get; set; } - /// - /// A list of callbacks that will be invoked if the lastsynced.txt file does not exist. - /// - /// - /// These callbacks will typically be for eg rebuilding the xml cache file, or examine indexes, based on - /// the data in the database to get this particular server node up to date. - /// + [Obsolete("This should not be used. If initialization calls need to be invoked on a cold boot, use the ISyncBootStateAccessor.Booting event.")] + [EditorBrowsable(EditorBrowsableState.Never)] public IEnumerable InitializingCallbacks { get; set; } /// diff --git a/src/Umbraco.Core/Sync/ISyncBootStateAccessor.cs b/src/Umbraco.Core/Sync/ISyncBootStateAccessor.cs new file mode 100644 index 0000000000..8d4cda093c --- /dev/null +++ b/src/Umbraco.Core/Sync/ISyncBootStateAccessor.cs @@ -0,0 +1,16 @@ +using System; + +namespace Umbraco.Core.Sync +{ + /// + /// Retrieve the state of the sync service + /// + public interface ISyncBootStateAccessor + { + /// + /// Get the boot state + /// + /// + SyncBootState GetSyncBootState(); + } +} diff --git a/src/Umbraco.Core/Sync/NonRuntimeLevelBootStateAccessor.cs b/src/Umbraco.Core/Sync/NonRuntimeLevelBootStateAccessor.cs new file mode 100644 index 0000000000..cc0b7dfaf1 --- /dev/null +++ b/src/Umbraco.Core/Sync/NonRuntimeLevelBootStateAccessor.cs @@ -0,0 +1,21 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Umbraco.Core.Sync +{ + /// + /// Boot state implementation for when umbraco is not in the run state + /// + public class NonRuntimeLevelBootStateAccessor : ISyncBootStateAccessor + { + public event EventHandler Booting; + + public SyncBootState GetSyncBootState() + { + return SyncBootState.Unknown; + } + } +} diff --git a/src/Umbraco.Core/Sync/SyncBootState.cs b/src/Umbraco.Core/Sync/SyncBootState.cs new file mode 100644 index 0000000000..45f7a87331 --- /dev/null +++ b/src/Umbraco.Core/Sync/SyncBootState.cs @@ -0,0 +1,20 @@ +namespace Umbraco.Core.Sync +{ + public enum SyncBootState + { + /// + /// Unknown state. Treat as WarmBoot + /// + Unknown = 0, + + /// + /// Cold boot. No Sync state + /// + ColdBoot = 1, + + /// + /// Warm boot. Sync state present + /// + WarmBoot = 2 + } +} diff --git a/src/Umbraco.Core/Umbraco.Core.csproj b/src/Umbraco.Core/Umbraco.Core.csproj index 0a453ad75f..ea454dbaf7 100755 --- a/src/Umbraco.Core/Umbraco.Core.csproj +++ b/src/Umbraco.Core/Umbraco.Core.csproj @@ -180,6 +180,9 @@ + + + diff --git a/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs b/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs index 834d211994..36f6b9d479 100644 --- a/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs +++ b/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs @@ -17,6 +17,7 @@ using Umbraco.Core.Scoping; using Umbraco.Core.Services; using Umbraco.Core.Services.Changes; using Umbraco.Core.Strings; +using Umbraco.Core.Sync; using Umbraco.Tests.TestHelpers; using Umbraco.Tests.Testing.Objects; using Umbraco.Tests.Testing.Objects.Accessors; @@ -155,7 +156,8 @@ namespace Umbraco.Tests.PublishedContent globalSettings, Mock.Of(), Mock.Of(), - new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() })); + new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }), + new TestSyncBootStateAccessor(SyncBootState.WarmBoot)); // invariant is the current default _variationAccesor.VariationContext = new VariationContext(); diff --git a/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs b/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs index 0e05e6baad..7cb4d97ea7 100644 --- a/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs +++ b/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs @@ -17,6 +17,7 @@ using Umbraco.Core.Scoping; using Umbraco.Core.Services; using Umbraco.Core.Services.Changes; using Umbraco.Core.Strings; +using Umbraco.Core.Sync; using Umbraco.Tests.TestHelpers; using Umbraco.Tests.Testing.Objects; using Umbraco.Tests.Testing.Objects.Accessors; @@ -201,7 +202,8 @@ namespace Umbraco.Tests.PublishedContent globalSettings, Mock.Of(), Mock.Of(), - new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() })); + new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }), + new TestSyncBootStateAccessor(SyncBootState.WarmBoot)); // invariant is the current default _variationAccesor.VariationContext = new VariationContext(); diff --git a/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs b/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs index c7c403b260..4453bfbbdf 100644 --- a/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs +++ b/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs @@ -99,7 +99,8 @@ namespace Umbraco.Tests.Scoping Factory.GetInstance(), Factory.GetInstance(), Mock.Of(), - new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() })); + new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }), + new TestSyncBootStateAccessor(SyncBootState.WarmBoot)); } protected UmbracoContext GetUmbracoContextNu(string url, int templateId = 1234, RouteData routeData = null, bool setSingleton = false, IUmbracoSettingsSection umbracoSettings = null, IEnumerable urlProviders = null) diff --git a/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs b/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs index 9391b7442f..7e3bc60411 100644 --- a/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs +++ b/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs @@ -17,6 +17,7 @@ using Umbraco.Core.PropertyEditors; using Umbraco.Core.Services; using Umbraco.Core.Strings; using Umbraco.Core.Sync; +using Umbraco.Tests.TestHelpers; using Umbraco.Tests.TestHelpers.Entities; using Umbraco.Tests.Testing; using Umbraco.Web.PublishedCache; @@ -70,7 +71,8 @@ namespace Umbraco.Tests.Services Factory.GetInstance(), Factory.GetInstance(), Mock.Of(), - new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() })); + new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }), + new TestSyncBootStateAccessor(SyncBootState.WarmBoot)); } public class LocalServerMessenger : ServerMessengerBase diff --git a/src/Umbraco.Tests/TestHelpers/TestSyncBootStateAccessor.cs b/src/Umbraco.Tests/TestHelpers/TestSyncBootStateAccessor.cs new file mode 100644 index 0000000000..3bff19ff2d --- /dev/null +++ b/src/Umbraco.Tests/TestHelpers/TestSyncBootStateAccessor.cs @@ -0,0 +1,26 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Umbraco.Core.Sync; + +namespace Umbraco.Tests.TestHelpers +{ + class TestSyncBootStateAccessor : ISyncBootStateAccessor + { + private readonly SyncBootState _syncBootState; + + public TestSyncBootStateAccessor(SyncBootState syncBootState) + { + _syncBootState = syncBootState; + } + + public event EventHandler Booting; + + public SyncBootState GetSyncBootState() + { + return _syncBootState; + } + } +} diff --git a/src/Umbraco.Tests/Umbraco.Tests.csproj b/src/Umbraco.Tests/Umbraco.Tests.csproj index 242040836b..d5b3a10f4b 100644 --- a/src/Umbraco.Tests/Umbraco.Tests.csproj +++ b/src/Umbraco.Tests/Umbraco.Tests.csproj @@ -178,6 +178,7 @@ + diff --git a/src/Umbraco.Web/BatchedDatabaseServerMessenger.cs b/src/Umbraco.Web/BatchedDatabaseServerMessenger.cs index 6596fee245..f89f62eb3d 100644 --- a/src/Umbraco.Web/BatchedDatabaseServerMessenger.cs +++ b/src/Umbraco.Web/BatchedDatabaseServerMessenger.cs @@ -26,6 +26,7 @@ namespace Umbraco.Web public class BatchedDatabaseServerMessenger : DatabaseServerMessenger { private readonly IUmbracoDatabaseFactory _databaseFactory; + private readonly Lazy _syncBootState; [Obsolete("This overload should not be used, enableDistCalls has no effect")] [EditorBrowsable(EditorBrowsableState.Never)] @@ -39,28 +40,22 @@ namespace Umbraco.Web : base(runtime, scopeProvider, sqlContext, proflog, globalSettings, true, options) { _databaseFactory = databaseFactory; - } - - // invoked by DatabaseServerRegistrarAndMessengerComponent - internal void Startup() - { - UmbracoModule.EndRequest += UmbracoModule_EndRequest; - - if (_databaseFactory.CanConnect == false) + _syncBootState = new Lazy(() => { - Logger.Warn("Cannot connect to the database, distributed calls will not be enabled for this server."); - } - else - { - Boot(); - } + if (_databaseFactory.CanConnect == false) + { + Logger.Warn("Cannot connect to the database, distributed calls will not be enabled for this server."); + return SyncBootState.Unknown; + } + else + { + return base.GetSyncBootState(); + } + }); } - private void UmbracoModule_EndRequest(object sender, UmbracoRequestEventArgs e) - { - // will clear the batch - will remain in HttpContext though - that's ok - FlushBatch(); - } + // override to deal with database connectivity + public override SyncBootState GetSyncBootState() => _syncBootState.Value; protected override void DeliverRemote(ICacheRefresher refresher, MessageType messageType, IEnumerable ids = null, string json = null) { diff --git a/src/Umbraco.Web/Compose/DatabaseServerRegistrarAndMessengerComponent.cs b/src/Umbraco.Web/Compose/DatabaseServerRegistrarAndMessengerComponent.cs index 2fa9d80779..d696ae5527 100644 --- a/src/Umbraco.Web/Compose/DatabaseServerRegistrarAndMessengerComponent.cs +++ b/src/Umbraco.Web/Compose/DatabaseServerRegistrarAndMessengerComponent.cs @@ -4,76 +4,13 @@ using Umbraco.Core; using Umbraco.Core.Composing; using Umbraco.Core.Logging; using Umbraco.Core.Services; -using Umbraco.Core.Services.Changes; using Umbraco.Core.Sync; using Umbraco.Examine; -using Umbraco.Web.Cache; using Umbraco.Web.Routing; using Umbraco.Web.Scheduling; -using Umbraco.Web.Search; -using Current = Umbraco.Web.Composing.Current; namespace Umbraco.Web.Compose { - /// - /// Ensures that servers are automatically registered in the database, when using the database server registrar. - /// - /// - /// At the moment servers are automatically registered upon first request and then on every - /// request but not more than once per (configurable) period. This really is "for information & debug" purposes so - /// we can look at the table and see what servers are registered - but the info is not used anywhere. - /// Should we actually want to use this, we would need a better and more deterministic way of figuring - /// out the "server address" ie the address to which server-to-server requests should be sent - because it - /// probably is not the "current request address" - especially in multi-domains configurations. - /// - [RuntimeLevel(MinLevel = RuntimeLevel.Run)] - - // during Initialize / Startup, we end up checking Examine, which needs to be initialized beforehand - // TODO: should not be a strong dependency on "examine" but on an "indexing component" - [ComposeAfter(typeof(ExamineComposer))] - - public sealed class DatabaseServerRegistrarAndMessengerComposer : ComponentComposer, ICoreComposer - { - public static DatabaseServerMessengerOptions GetDefaultOptions(IFactory factory) - { - var logger = factory.GetInstance(); - var indexRebuilder = factory.GetInstance(); - - return new DatabaseServerMessengerOptions - { - //These callbacks will be executed if the server has not been synced - // (i.e. it is a new server or the lastsynced.txt file has been removed) - InitializingCallbacks = new Action[] - { - //rebuild the xml cache file if the server is not synced - () => - { - // rebuild the published snapshot caches entirely, if the server is not synced - // this is equivalent to DistributedCache RefreshAll... but local only - // (we really should have a way to reuse RefreshAll... locally) - // note: refresh all content & media caches does refresh content types too - var svc = Current.PublishedSnapshotService; - svc.Notify(new[] { new DomainCacheRefresher.JsonPayload(0, DomainChangeTypes.RefreshAll) }); - svc.Notify(new[] { new ContentCacheRefresher.JsonPayload(0, null, TreeChangeTypes.RefreshAll) }, out _, out _); - svc.Notify(new[] { new MediaCacheRefresher.JsonPayload(0, null, TreeChangeTypes.RefreshAll) }, out _); - }, - - //rebuild indexes if the server is not synced - // NOTE: This will rebuild ALL indexes including the members, if developers want to target specific - // indexes then they can adjust this logic themselves. - () => { ExamineComponent.RebuildIndexes(indexRebuilder, logger, false, 5000); } - } - }; - } - - public override void Compose(Composition composition) - { - base.Compose(composition); - - composition.SetDatabaseServerMessengerOptions(GetDefaultOptions); - composition.SetServerMessenger(); - } - } public sealed class DatabaseServerRegistrarAndMessengerComponent : IComponent { @@ -87,14 +24,17 @@ namespace Umbraco.Web.Compose private readonly BackgroundTaskRunner _processTaskRunner; private bool _started; private IBackgroundTask[] _tasks; - private IndexRebuilder _indexRebuilder; - public DatabaseServerRegistrarAndMessengerComponent(IRuntimeState runtime, IServerRegistrar serverRegistrar, IServerMessenger serverMessenger, IServerRegistrationService registrationService, ILogger logger, IndexRebuilder indexRebuilder) + public DatabaseServerRegistrarAndMessengerComponent( + IRuntimeState runtime, + IServerRegistrar serverRegistrar, + IServerMessenger serverMessenger, + IServerRegistrationService registrationService, + ILogger logger) { _runtime = runtime; _logger = logger; _registrationService = registrationService; - _indexRebuilder = indexRebuilder; // create task runner for DatabaseServerRegistrar _registrar = serverRegistrar as DatabaseServerRegistrar; @@ -117,15 +57,21 @@ namespace Umbraco.Web.Compose { //We will start the whole process when a successful request is made if (_registrar != null || _messenger != null) + { UmbracoModule.RouteAttempt += RegisterBackgroundTasksOnce; - - // must come last, as it references some _variables - _messenger?.Startup(); + UmbracoModule.EndRequest += UmbracoModule_EndRequest; + } } public void Terminate() { } + private void UmbracoModule_EndRequest(object sender, UmbracoRequestEventArgs e) + { + // will clear the batch - will remain in HttpContext though - that's ok + _messenger?.FlushBatch(); + } + /// /// Handle when a request is made /// diff --git a/src/Umbraco.Web/Compose/DatabaseServerRegistrarAndMessengerComposer.cs b/src/Umbraco.Web/Compose/DatabaseServerRegistrarAndMessengerComposer.cs new file mode 100644 index 0000000000..96e538652f --- /dev/null +++ b/src/Umbraco.Web/Compose/DatabaseServerRegistrarAndMessengerComposer.cs @@ -0,0 +1,38 @@ +using Umbraco.Core; +using Umbraco.Core.Composing; +using Umbraco.Core.Sync; +using Umbraco.Web.Search; + +namespace Umbraco.Web.Compose +{ + /// + /// Ensures that servers are automatically registered in the database, when using the database server registrar. + /// + /// + /// At the moment servers are automatically registered upon first request and then on every + /// request but not more than once per (configurable) period. This really is "for information & debug" purposes so + /// we can look at the table and see what servers are registered - but the info is not used anywhere. + /// Should we actually want to use this, we would need a better and more deterministic way of figuring + /// out the "server address" ie the address to which server-to-server requests should be sent - because it + /// probably is not the "current request address" - especially in multi-domains configurations. + /// + [RuntimeLevel(MinLevel = RuntimeLevel.Run)] + // TODO: This is legacy, we no longer need to do this but we don't want to change the behavior now + [ComposeAfter(typeof(ExamineComposer))] + public sealed class DatabaseServerRegistrarAndMessengerComposer : ComponentComposer, ICoreComposer + { + public static DatabaseServerMessengerOptions GetDefaultOptions(IFactory factory) + { + return new DatabaseServerMessengerOptions(); + } + + public override void Compose(Composition composition) + { + base.Compose(composition); + + composition.SetDatabaseServerMessengerOptions(GetDefaultOptions); + composition.SetServerMessenger(); + composition.Register(factory => factory.GetInstance() as BatchedDatabaseServerMessenger, Lifetime.Singleton); + } + } +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs index f748fd555c..99f2786d49 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs @@ -1,5 +1,6 @@ using Umbraco.Core; using Umbraco.Core.Composing; +using Umbraco.Core.Sync; using Umbraco.Web.PublishedCache.NuCache.DataSource; namespace Umbraco.Web.PublishedCache.NuCache @@ -10,6 +11,9 @@ namespace Umbraco.Web.PublishedCache.NuCache { base.Compose(composition); + //Overriden on Run state in DatabaseServerRegistrarAndMessengerComposer + composition.Register(Lifetime.Singleton); + // register the NuCache database data source composition.Register(); diff --git a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs old mode 100755 new mode 100644 index 15e45cfa71..f70160cde8 --- a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs @@ -24,6 +24,7 @@ using Umbraco.Core.Services; using Umbraco.Core.Services.Changes; using Umbraco.Core.Services.Implement; using Umbraco.Core.Strings; +using Umbraco.Core.Sync; using Umbraco.Web.Cache; using Umbraco.Web.Install; using Umbraco.Web.PublishedCache.NuCache.DataSource; @@ -35,6 +36,8 @@ namespace Umbraco.Web.PublishedCache.NuCache internal class PublishedSnapshotService : PublishedSnapshotServiceBase { + private readonly PublishedSnapshotServiceOptions _options; + private readonly IMainDom _mainDom; private readonly ServiceContext _serviceContext; private readonly IPublishedContentTypeFactory _publishedContentTypeFactory; private readonly IScopeProvider _scopeProvider; @@ -49,12 +52,13 @@ namespace Umbraco.Web.PublishedCache.NuCache private readonly IDefaultCultureAccessor _defaultCultureAccessor; private readonly UrlSegmentProviderCollection _urlSegmentProviders; - // volatile because we read it with no lock - private volatile bool _isReady; + private bool _isReady; + private bool _isReadSet; + private object _isReadyLock; - private readonly ContentStore _contentStore; - private readonly ContentStore _mediaStore; - private readonly SnapDictionary _domainStore; + private ContentStore _contentStore; + private ContentStore _mediaStore; + private SnapDictionary _domainStore; private readonly object _storesLock = new object(); private readonly object _elementsLock = new object(); @@ -63,6 +67,8 @@ namespace Umbraco.Web.PublishedCache.NuCache private bool _localContentDbExists; private bool _localMediaDbExists; + private readonly ISyncBootStateAccessor _syncBootStateAccessor; + // define constant - determines whether to use cache when previewing // to store eg routes, property converted values, anything - caching // means faster execution, but uses memory - not sure if we want it @@ -81,12 +87,16 @@ namespace Umbraco.Web.PublishedCache.NuCache IDataSource dataSource, IGlobalSettings globalSettings, IEntityXmlSerializer entitySerializer, IPublishedModelFactory publishedModelFactory, - UrlSegmentProviderCollection urlSegmentProviders) + UrlSegmentProviderCollection urlSegmentProviders, + ISyncBootStateAccessor syncBootStateAccessor) : base(publishedSnapshotAccessor, variationContextAccessor) { + //if (Interlocked.Increment(ref _singletonCheck) > 1) // throw new Exception("Singleton must be instantiated only once!"); + _options = options; + _mainDom = mainDom; _serviceContext = serviceContext; _publishedContentTypeFactory = publishedContentTypeFactory; _dataSource = dataSource; @@ -99,6 +109,8 @@ namespace Umbraco.Web.PublishedCache.NuCache _globalSettings = globalSettings; _urlSegmentProviders = urlSegmentProviders; + _syncBootStateAccessor = syncBootStateAccessor; + // we need an Xml serializer here so that the member cache can support XPath, // for members this is done by navigating the serialized-to-xml member _entitySerializer = entitySerializer; @@ -117,41 +129,6 @@ namespace Umbraco.Web.PublishedCache.NuCache if (runtime.Level != RuntimeLevel.Run) return; - // lock this entire call, we only want a single thread to be accessing the stores at once and within - // the call below to mainDom.Register, a callback may occur on a threadpool thread to MainDomRelease - // at the same time as we are trying to write to the stores. MainDomRelease also locks on _storesLock so - // it will not be able to close the stores until we are done populating (if the store is empty) - lock (_storesLock) - { - if (options.IgnoreLocalDb == false) - { - var registered = mainDom.Register(MainDomRegister, MainDomRelease); - - // stores are created with a db so they can write to it, but they do not read from it, - // stores need to be populated, happens in OnResolutionFrozen which uses _localDbExists to - // figure out whether it can read the databases or it should populate them from sql - - _logger.Info("Creating the content store, localContentDbExists? {LocalContentDbExists}", _localContentDbExists); - _contentStore = new ContentStore(publishedSnapshotAccessor, variationContextAccessor, logger, _localContentDb); - _logger.Info("Creating the media store, localMediaDbExists? {LocalMediaDbExists}", _localMediaDbExists); - _mediaStore = new ContentStore(publishedSnapshotAccessor, variationContextAccessor, logger, _localMediaDb); - } - else - { - _logger.Info("Creating the content store (local db ignored)"); - _contentStore = new ContentStore(publishedSnapshotAccessor, variationContextAccessor, logger); - _logger.Info("Creating the media store (local db ignored)"); - _mediaStore = new ContentStore(publishedSnapshotAccessor, variationContextAccessor, logger); - } - - _domainStore = new SnapDictionary(); - - LoadCachesOnStartup(); - } - - Guid GetUid(ContentStore store, int id) => store.LiveSnapshot.Get(id)?.Uid ?? default; - int GetId(ContentStore store, Guid uid) => store.LiveSnapshot.Get(uid)?.Id ?? default; - if (idkMap != null) { idkMap.SetMapper(UmbracoObjectTypes.Document, id => GetUid(_contentStore, id), uid => GetId(_contentStore, uid)); @@ -159,6 +136,18 @@ namespace Umbraco.Web.PublishedCache.NuCache } } + private int GetId(ContentStore store, Guid uid) + { + EnsureCaches(); + return store.LiveSnapshot.Get(uid)?.Id ?? default; + } + + private Guid GetUid(ContentStore store, int id) + { + EnsureCaches(); + return store.LiveSnapshot.Get(id)?.Uid ?? default; + } + /// /// Install phase of /// @@ -210,47 +199,82 @@ namespace Umbraco.Web.PublishedCache.NuCache } /// - /// Populates the stores + /// Lazily populates the stores only when they are first requested /// - /// This is called inside of a lock for _storesLock - private void LoadCachesOnStartup() - { - var okContent = false; - var okMedia = false; - - try + internal void EnsureCaches() => LazyInitializer.EnsureInitialized( + ref _isReady, + ref _isReadSet, + ref _isReadyLock, + () => { - if (_localContentDbExists) + // lock this entire call, we only want a single thread to be accessing the stores at once and within + // the call below to mainDom.Register, a callback may occur on a threadpool thread to MainDomRelease + // at the same time as we are trying to write to the stores. MainDomRelease also locks on _storesLock so + // it will not be able to close the stores until we are done populating (if the store is empty) + lock (_storesLock) { - okContent = LockAndLoadContent(scope => LoadContentFromLocalDbLocked(true)); - if (!okContent) - _logger.Warn("Loading content from local db raised warnings, will reload from database."); + if (!_options.IgnoreLocalDb) + { + var registered = _mainDom.Register(MainDomRegister, MainDomRelease); + + // stores are created with a db so they can write to it, but they do not read from it, + // stores need to be populated, happens in OnResolutionFrozen which uses _localDbExists to + // figure out whether it can read the databases or it should populate them from sql + + _logger.Info("Creating the content store, localContentDbExists? {LocalContentDbExists}", _localContentDbExists); + _contentStore = new ContentStore(PublishedSnapshotAccessor, VariationContextAccessor, _logger, _localContentDb); + _logger.Info("Creating the media store, localMediaDbExists? {LocalMediaDbExists}", _localMediaDbExists); + _mediaStore = new ContentStore(PublishedSnapshotAccessor, VariationContextAccessor, _logger, _localMediaDb); + } + else + { + _logger.Info("Creating the content store (local db ignored)"); + _contentStore = new ContentStore(PublishedSnapshotAccessor, VariationContextAccessor, _logger); + _logger.Info("Creating the media store (local db ignored)"); + _mediaStore = new ContentStore(PublishedSnapshotAccessor, VariationContextAccessor, _logger); + } + + _domainStore = new SnapDictionary(); + + SyncBootState bootState = _syncBootStateAccessor.GetSyncBootState(); + + var okContent = false; + var okMedia = false; + + try + { + if (bootState != SyncBootState.ColdBoot && _localContentDbExists) + { + okContent = LockAndLoadContent(scope => LoadContentFromLocalDbLocked(true)); + if (!okContent) + _logger.Warn("Loading content from local db raised warnings, will reload from database."); + } + + if (bootState != SyncBootState.ColdBoot && _localMediaDbExists) + { + okMedia = LockAndLoadMedia(scope => LoadMediaFromLocalDbLocked(true)); + if (!okMedia) + _logger.Warn("Loading media from local db raised warnings, will reload from database."); + } + + if (!okContent) + LockAndLoadContent(scope => LoadContentFromDatabaseLocked(scope, true)); + + if (!okMedia) + LockAndLoadMedia(scope => LoadMediaFromDatabaseLocked(scope, true)); + + LockAndLoadDomains(); + } + catch (Exception ex) + { + _logger.Fatal(ex, "Panic, exception while loading cache data."); + throw; + } + + // finally, cache is ready! + return true; } - - if (_localMediaDbExists) - { - okMedia = LockAndLoadMedia(scope => LoadMediaFromLocalDbLocked(true)); - if (!okMedia) - _logger.Warn("Loading media from local db raised warnings, will reload from database."); - } - - if (!okContent) - LockAndLoadContent(scope => LoadContentFromDatabaseLocked(scope, true)); - - if (!okMedia) - LockAndLoadMedia(scope => LoadMediaFromDatabaseLocked(scope, true)); - - LockAndLoadDomains(); - } - catch (Exception ex) - { - _logger.Fatal(ex, "Panic, exception while loading cache data."); - throw; - } - - // finally, cache is ready! - _isReady = true; - } + }); private void InitializeRepositoryEvents() { @@ -1133,9 +1157,13 @@ namespace Umbraco.Web.PublishedCache.NuCache public override IPublishedSnapshot CreatePublishedSnapshot(string previewToken) { + EnsureCaches(); + // no cache, no joy - if (_isReady == false) + if (Volatile.Read(ref _isReady) == false) + { throw new InvalidOperationException("The published snapshot service has not properly initialized."); + } var preview = previewToken.IsNullOrWhiteSpace() == false; return new PublishedSnapshot(this, preview); @@ -1146,6 +1174,8 @@ namespace Umbraco.Web.PublishedCache.NuCache // even though the underlying elements may not change (store snapshots) public PublishedSnapshot.PublishedSnapshotElements GetElements(bool previewDefault) { + EnsureCaches(); + // note: using ObjectCacheAppCache for elements and snapshot caches // is not recommended because it creates an inner MemoryCache which is a heavy // thing - better use a dictionary-based cache which "just" creates a concurrent @@ -1792,6 +1822,8 @@ AND cmsContentNu.nodeId IS NULL public void Collect() { + EnsureCaches(); + var contentCollect = _contentStore.CollectAsync(); var mediaCollect = _mediaStore.CollectAsync(); System.Threading.Tasks.Task.WaitAll(contentCollect, mediaCollect); @@ -1801,8 +1833,17 @@ AND cmsContentNu.nodeId IS NULL #region Internals/Testing - internal ContentStore GetContentStore() => _contentStore; - internal ContentStore GetMediaStore() => _mediaStore; + internal ContentStore GetContentStore() + { + EnsureCaches(); + return _contentStore; + } + + internal ContentStore GetMediaStore() + { + EnsureCaches(); + return _mediaStore; + } #endregion } diff --git a/src/Umbraco.Web/Search/ExamineFinalComponent.cs b/src/Umbraco.Web/Search/ExamineFinalComponent.cs index 95000b2b46..6dd775ab64 100644 --- a/src/Umbraco.Web/Search/ExamineFinalComponent.cs +++ b/src/Umbraco.Web/Search/ExamineFinalComponent.cs @@ -3,6 +3,8 @@ using Umbraco.Core.Logging; using Umbraco.Examine; using Umbraco.Core.Composing; using Umbraco.Core; +using Umbraco.Core.Sync; +using Umbraco.Web.Routing; namespace Umbraco.Web.Search { @@ -16,23 +18,50 @@ namespace Umbraco.Web.Search private readonly IExamineManager _examineManager; BackgroundIndexRebuilder _indexRebuilder; private readonly IMainDom _mainDom; - - public ExamineFinalComponent(IProfilingLogger logger, IExamineManager examineManager, BackgroundIndexRebuilder indexRebuilder, IMainDom mainDom) + private readonly ISyncBootStateAccessor _syncBootStateAccessor; + private readonly object _locker = new object(); + private bool _initialized = false; + + public ExamineFinalComponent(IProfilingLogger logger, IExamineManager examineManager, BackgroundIndexRebuilder indexRebuilder, IMainDom mainDom, ISyncBootStateAccessor syncBootStateAccessor) { _logger = logger; _examineManager = examineManager; _indexRebuilder = indexRebuilder; _mainDom = mainDom; + _syncBootStateAccessor = syncBootStateAccessor; + } + + private void UmbracoModule_RouteAttempt(object sender, RoutableAttemptEventArgs e) + { + if (!_initialized) + { + lock (_locker) + { + // double check lock, we must only do this once + if (!_initialized) + { + _initialized = true; + + UmbracoModule.RouteAttempt -= UmbracoModule_RouteAttempt; + + if (!_mainDom.IsMainDom) return; + + var bootState = _syncBootStateAccessor.GetSyncBootState(); + + _examineManager.ConfigureIndexes(_mainDom, _logger); + + // if it's a cold boot, rebuild all indexes including non-empty ones + // delay one minute since a cold boot also triggers nucache rebuilds + _indexRebuilder.RebuildIndexes(bootState != SyncBootState.ColdBoot, 60000); + } + } + } + } public void Initialize() { - if (!_mainDom.IsMainDom) return; - - _examineManager.ConfigureIndexes(_mainDom, _logger); - - // TODO: Instead of waiting 5000 ms, we could add an event handler on to fulfilling the first request, then start? - _indexRebuilder.RebuildIndexes(true, 5000); + UmbracoModule.RouteAttempt += UmbracoModule_RouteAttempt; } public void Terminate() diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index f125ff87c4..0d7a89b97e 100644 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -132,6 +132,7 @@ +