diff --git a/src/Umbraco.Core/Sync/DatabaseServerMessenger.cs b/src/Umbraco.Core/Sync/DatabaseServerMessenger.cs index ebc77dbdca..75ccf5e4f9 100644 --- a/src/Umbraco.Core/Sync/DatabaseServerMessenger.cs +++ b/src/Umbraco.Core/Sync/DatabaseServerMessenger.cs @@ -39,9 +39,9 @@ namespace Umbraco.Core.Sync private int _lastId = -1; private DateTime _lastSync; private DateTime _lastPruned; - private bool _initialized; private bool _syncing; private bool _released; + private readonly Lazy _getSyncBootState; public DatabaseServerMessengerOptions Options { get; } @@ -59,6 +59,7 @@ namespace Umbraco.Core.Sync _lastPruned = _lastSync = DateTime.UtcNow; _syncIdle = new ManualResetEvent(true); _distCacheFilePath = new Lazy(() => GetDistCacheFilePath(globalSettings)); + _getSyncBootState = new Lazy(BootInternal); } protected ILogger Logger { get; } @@ -75,7 +76,7 @@ namespace Umbraco.Core.Sync { // we don't care if there's servers listed or not, // if distributed call is enabled we will make the call - return _initialized && DistributedEnabled; + return _getSyncBootState.IsValueCreated && DistributedEnabled; } protected override void DeliverRemote( @@ -110,14 +111,14 @@ namespace Umbraco.Core.Sync #region Sync - /// - /// Boots the messenger. - /// - /// - /// Thread safety: this is NOT thread safe. Because it is NOT meant to run multi-threaded. - /// Callers MUST ensure thread-safety. - /// + [Obsolete("This is no longer used and will be removed in future versions")] protected void Boot() + { + // if called, just forces the boot logic + _ = GetSyncBootState(); + } + + private SyncBootState BootInternal() { // weight:10, must release *before* the published snapshot service, because once released // the service will *not* be able to properly handle our notifications anymore @@ -139,7 +140,7 @@ namespace Umbraco.Core.Sync // properly releasing MainDom - a timeout here means that one refresher // is taking too much time processing, however when it's done we will // not update lastId and stop everything - var idle =_syncIdle.WaitOne(5000); + var idle = _syncIdle.WaitOne(5000); if (idle == false) { Logger.Warn("The wait lock timed out, application is shutting down. The current instruction batch will be re-processed."); @@ -147,17 +148,23 @@ namespace Umbraco.Core.Sync }, weight); + SyncBootState bootState = SyncBootState.Unknown; + if (registered == false) - return; + { + return bootState; + } ReadLastSynced(); // get _lastId using (var scope = ScopeProvider.CreateScope()) { EnsureInstructions(scope.Database); // reset _lastId if instructions are missing - Initialize(scope.Database); // boot + bootState = Initialize(scope.Database); // boot scope.Complete(); } + + return bootState; } /// @@ -167,36 +174,11 @@ namespace Umbraco.Core.Sync /// Thread safety: this is NOT thread safe. Because it is NOT meant to run multi-threaded. /// Callers MUST ensure thread-safety. /// - private void Initialize(IUmbracoDatabase database) + private SyncBootState Initialize(IUmbracoDatabase database) { - lock (_locko) - { - if (_released) return; - var coldboot = IsColdBoot(database); + // could occur if shutting down immediately once starting up and before we've initialized + if (_released) return SyncBootState.Unknown; - if (coldboot) - { - // go get the last id in the db and store it - // note: do it BEFORE initializing otherwise some instructions might get lost - // when doing it before, some instructions might run twice - not an issue - var maxId = database.ExecuteScalar("SELECT MAX(id) FROM umbracoCacheInstruction"); - - //if there is a max currently, or if we've never synced - if (maxId > 0 || _lastId < 0) - SaveLastSynced(maxId); - - // execute initializing callbacks - if (Options.InitializingCallbacks != null) - foreach (var callback in Options.InitializingCallbacks) - callback(); - } - - _initialized = true; - } - } - - private bool IsColdBoot(IUmbracoDatabase database) - { var coldboot = false; if (_lastId < 0) // never synced before { @@ -206,27 +188,48 @@ namespace Umbraco.Core.Sync + " The server will build its caches and indexes, and then adjust its last synced Id to the latest found in" + " the database and maintain cache updates based on that Id."); - coldboot = true; - } - else + coldboot = true; + } + else + { + //check for how many instructions there are to process, each row contains a count of the number of instructions contained in each + //row so we will sum these numbers to get the actual count. + var count = database.ExecuteScalar("SELECT SUM(instructionCount) FROM umbracoCacheInstruction WHERE id > @lastId", new { lastId = _lastId }); + if (count > Options.MaxProcessingInstructionCount) { - //check for how many instructions there are to process, each row contains a count of the number of instructions contained in each - //row so we will sum these numbers to get the actual count. - var count = database.ExecuteScalar("SELECT SUM(instructionCount) FROM umbracoCacheInstruction WHERE id > @lastId", new {lastId = _lastId}); - if (count > Options.MaxProcessingInstructionCount) - { - //too many instructions, proceed to cold boot - Logger.Warn( - "The instruction count ({InstructionCount}) exceeds the specified MaxProcessingInstructionCount ({MaxProcessingInstructionCount})." - + " The server will skip existing instructions, rebuild its caches and indexes entirely, adjust its last synced Id" - + " to the latest found in the database and maintain cache updates based on that Id.", - count, Options.MaxProcessingInstructionCount); + //too many instructions, proceed to cold boot + Logger.Warn( + "The instruction count ({InstructionCount}) exceeds the specified MaxProcessingInstructionCount ({MaxProcessingInstructionCount})." + + " The server will skip existing instructions, rebuild its caches and indexes entirely, adjust its last synced Id" + + " to the latest found in the database and maintain cache updates based on that Id.", + count, Options.MaxProcessingInstructionCount); coldboot = true; } } - return coldboot; + if (coldboot) + { + // go get the last id in the db and store it + // note: do it BEFORE initializing otherwise some instructions might get lost + // when doing it before, some instructions might run twice - not an issue + var maxId = database.ExecuteScalar("SELECT MAX(id) FROM umbracoCacheInstruction"); + + //if there is a max currently, or if we've never synced + if (maxId > 0 || _lastId < 0) + SaveLastSynced(maxId); + + // execute initializing callbacks + if (Options.InitializingCallbacks != null) + { + foreach (var callback in Options.InitializingCallbacks) + { + callback(); + } + } + } + + return coldboot ? SyncBootState.ColdBoot : SyncBootState.WarmBoot; } /// @@ -358,7 +361,7 @@ namespace Umbraco.Core.Sync } catch (JsonException ex) { - Logger.Error(ex, "Failed to deserialize instructions ({DtoId}: '{DtoInstructions}').", + Logger.Error(ex, "Failed to deserialize instructions ({DtoId}: '{DtoInstructions}').", dto.Id, dto.Instructions); @@ -416,11 +419,11 @@ namespace Umbraco.Core.Sync //} catch (Exception ex) { - Logger.Error ( - ex, - "DISTRIBUTED CACHE IS NOT UPDATED. Failed to execute instructions ({DtoId}: '{DtoInstructions}'). Instruction is being skipped/ignored", - dto.Id, - dto.Instructions); + Logger.Error( + ex, + "DISTRIBUTED CACHE IS NOT UPDATED. Failed to execute instructions ({DtoId}: '{DtoInstructions}'). Instruction is being skipped/ignored", + dto.Id, + dto.Instructions); //we cannot throw here because this invalid instruction will just keep getting processed over and over and errors // will be thrown over and over. The only thing we can do is ignore and move on. @@ -554,29 +557,7 @@ namespace Umbraco.Core.Sync #endregion - public SyncBootState GetSyncBootState() - { - try - { - ReadLastSynced(); // get _lastId - using (var scope = ScopeProvider.CreateScope()) - { - EnsureInstructions(scope.Database); - bool isColdBoot = IsColdBoot(scope.Database); - - if (isColdBoot) - { - return SyncBootState.ColdBoot; - } - return SyncBootState.HasSyncState; - } - } - catch(Exception ex) - { - Logger.Warn("Error determining Sync Boot State", ex); - return SyncBootState.Unknown; - } - } + public virtual SyncBootState GetSyncBootState() => _getSyncBootState.Value; #region Notify refreshers diff --git a/src/Umbraco.Core/Sync/DatabaseServerMessengerOptions.cs b/src/Umbraco.Core/Sync/DatabaseServerMessengerOptions.cs index 6bfd6bff4a..a769709d09 100644 --- a/src/Umbraco.Core/Sync/DatabaseServerMessengerOptions.cs +++ b/src/Umbraco.Core/Sync/DatabaseServerMessengerOptions.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.ComponentModel; namespace Umbraco.Core.Sync { @@ -24,13 +25,8 @@ namespace Umbraco.Core.Sync /// public int MaxProcessingInstructionCount { get; set; } - /// - /// A list of callbacks that will be invoked if the lastsynced.txt file does not exist. - /// - /// - /// These callbacks will typically be for eg rebuilding the xml cache file, or examine indexes, based on - /// the data in the database to get this particular server node up to date. - /// + [Obsolete("This should not be used. If initialization calls need to be invoked on a cold boot, use the ISyncBootStateAccessor.Booting event.")] + [EditorBrowsable(EditorBrowsableState.Never)] public IEnumerable InitializingCallbacks { get; set; } /// diff --git a/src/Umbraco.Core/Sync/ISyncBootStateAccessor.cs b/src/Umbraco.Core/Sync/ISyncBootStateAccessor.cs index 4b8500f2d9..8d4cda093c 100644 --- a/src/Umbraco.Core/Sync/ISyncBootStateAccessor.cs +++ b/src/Umbraco.Core/Sync/ISyncBootStateAccessor.cs @@ -1,8 +1,4 @@ using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace Umbraco.Core.Sync { diff --git a/src/Umbraco.Core/Sync/NonRuntimeLevelBootStateAccessor.cs b/src/Umbraco.Core/Sync/NonRuntimeLevelBootStateAccessor.cs index 70cec6cc96..cc0b7dfaf1 100644 --- a/src/Umbraco.Core/Sync/NonRuntimeLevelBootStateAccessor.cs +++ b/src/Umbraco.Core/Sync/NonRuntimeLevelBootStateAccessor.cs @@ -11,6 +11,8 @@ namespace Umbraco.Core.Sync /// public class NonRuntimeLevelBootStateAccessor : ISyncBootStateAccessor { + public event EventHandler Booting; + public SyncBootState GetSyncBootState() { return SyncBootState.Unknown; diff --git a/src/Umbraco.Core/Sync/SyncBootState.cs b/src/Umbraco.Core/Sync/SyncBootState.cs index 4abc53abba..45f7a87331 100644 --- a/src/Umbraco.Core/Sync/SyncBootState.cs +++ b/src/Umbraco.Core/Sync/SyncBootState.cs @@ -1,24 +1,20 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace Umbraco.Core.Sync +namespace Umbraco.Core.Sync { public enum SyncBootState { /// - /// Unknown state. Treat as HasSyncState + /// Unknown state. Treat as WarmBoot /// Unknown = 0, + /// /// Cold boot. No Sync state /// ColdBoot = 1, + /// /// Warm boot. Sync state present /// - HasSyncState = 2 + WarmBoot = 2 } } diff --git a/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs b/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs index 75a20ade6f..78073f7ab4 100644 --- a/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs +++ b/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs @@ -159,7 +159,7 @@ namespace Umbraco.Tests.PublishedContent Mock.Of(), Mock.Of(), new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }), - new TestSyncBootStateAccessor(SyncBootState.HasSyncState), + new TestSyncBootStateAccessor(SyncBootState.WarmBoot), _contentNestedDataSerializerFactory); // invariant is the current default diff --git a/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs b/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs index 9feb0d703b..2d7eeff098 100644 --- a/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs +++ b/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs @@ -205,7 +205,7 @@ namespace Umbraco.Tests.PublishedContent Mock.Of(), Mock.Of(), new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }), - new TestSyncBootStateAccessor(SyncBootState.HasSyncState), + new TestSyncBootStateAccessor(SyncBootState.WarmBoot), _contentNestedDataSerializerFactory); // invariant is the current default diff --git a/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs b/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs index ad372c00b9..230f490907 100644 --- a/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs +++ b/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs @@ -101,7 +101,7 @@ namespace Umbraco.Tests.Scoping Factory.GetInstance(), Mock.Of(), new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }), - new TestSyncBootStateAccessor(SyncBootState.HasSyncState), + new TestSyncBootStateAccessor(SyncBootState.WarmBoot), nestedContentDataSerializerFactory); } diff --git a/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs b/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs index b252738fee..a3735c0ac3 100644 --- a/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs +++ b/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs @@ -74,7 +74,7 @@ namespace Umbraco.Tests.Services Factory.GetInstance(), Mock.Of(), new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }), - new TestSyncBootStateAccessor(SyncBootState.HasSyncState), + new TestSyncBootStateAccessor(SyncBootState.WarmBoot), nestedContentDataSerializerFactory); } diff --git a/src/Umbraco.Tests/TestHelpers/TestSyncBootStateAccessor.cs b/src/Umbraco.Tests/TestHelpers/TestSyncBootStateAccessor.cs index e5f6989381..3bff19ff2d 100644 --- a/src/Umbraco.Tests/TestHelpers/TestSyncBootStateAccessor.cs +++ b/src/Umbraco.Tests/TestHelpers/TestSyncBootStateAccessor.cs @@ -15,6 +15,9 @@ namespace Umbraco.Tests.TestHelpers { _syncBootState = syncBootState; } + + public event EventHandler Booting; + public SyncBootState GetSyncBootState() { return _syncBootState; diff --git a/src/Umbraco.Web/BatchedDatabaseServerMessenger.cs b/src/Umbraco.Web/BatchedDatabaseServerMessenger.cs index 6596fee245..f89f62eb3d 100644 --- a/src/Umbraco.Web/BatchedDatabaseServerMessenger.cs +++ b/src/Umbraco.Web/BatchedDatabaseServerMessenger.cs @@ -26,6 +26,7 @@ namespace Umbraco.Web public class BatchedDatabaseServerMessenger : DatabaseServerMessenger { private readonly IUmbracoDatabaseFactory _databaseFactory; + private readonly Lazy _syncBootState; [Obsolete("This overload should not be used, enableDistCalls has no effect")] [EditorBrowsable(EditorBrowsableState.Never)] @@ -39,28 +40,22 @@ namespace Umbraco.Web : base(runtime, scopeProvider, sqlContext, proflog, globalSettings, true, options) { _databaseFactory = databaseFactory; - } - - // invoked by DatabaseServerRegistrarAndMessengerComponent - internal void Startup() - { - UmbracoModule.EndRequest += UmbracoModule_EndRequest; - - if (_databaseFactory.CanConnect == false) + _syncBootState = new Lazy(() => { - Logger.Warn("Cannot connect to the database, distributed calls will not be enabled for this server."); - } - else - { - Boot(); - } + if (_databaseFactory.CanConnect == false) + { + Logger.Warn("Cannot connect to the database, distributed calls will not be enabled for this server."); + return SyncBootState.Unknown; + } + else + { + return base.GetSyncBootState(); + } + }); } - private void UmbracoModule_EndRequest(object sender, UmbracoRequestEventArgs e) - { - // will clear the batch - will remain in HttpContext though - that's ok - FlushBatch(); - } + // override to deal with database connectivity + public override SyncBootState GetSyncBootState() => _syncBootState.Value; protected override void DeliverRemote(ICacheRefresher refresher, MessageType messageType, IEnumerable ids = null, string json = null) { diff --git a/src/Umbraco.Web/Compose/DatabaseServerRegistrarAndMessengerComponent.cs b/src/Umbraco.Web/Compose/DatabaseServerRegistrarAndMessengerComponent.cs index 26ba0db324..d696ae5527 100644 --- a/src/Umbraco.Web/Compose/DatabaseServerRegistrarAndMessengerComponent.cs +++ b/src/Umbraco.Web/Compose/DatabaseServerRegistrarAndMessengerComponent.cs @@ -4,77 +4,13 @@ using Umbraco.Core; using Umbraco.Core.Composing; using Umbraco.Core.Logging; using Umbraco.Core.Services; -using Umbraco.Core.Services.Changes; using Umbraco.Core.Sync; using Umbraco.Examine; -using Umbraco.Web.Cache; using Umbraco.Web.Routing; using Umbraco.Web.Scheduling; -using Umbraco.Web.Search; -using Current = Umbraco.Web.Composing.Current; namespace Umbraco.Web.Compose { - /// - /// Ensures that servers are automatically registered in the database, when using the database server registrar. - /// - /// - /// At the moment servers are automatically registered upon first request and then on every - /// request but not more than once per (configurable) period. This really is "for information & debug" purposes so - /// we can look at the table and see what servers are registered - but the info is not used anywhere. - /// Should we actually want to use this, we would need a better and more deterministic way of figuring - /// out the "server address" ie the address to which server-to-server requests should be sent - because it - /// probably is not the "current request address" - especially in multi-domains configurations. - /// - [RuntimeLevel(MinLevel = RuntimeLevel.Run)] - - // during Initialize / Startup, we end up checking Examine, which needs to be initialized beforehand - // TODO: should not be a strong dependency on "examine" but on an "indexing component" - [ComposeAfter(typeof(ExamineComposer))] - - public sealed class DatabaseServerRegistrarAndMessengerComposer : ComponentComposer, ICoreComposer - { - public static DatabaseServerMessengerOptions GetDefaultOptions(IFactory factory) - { - var logger = factory.GetInstance(); - var indexRebuilder = factory.GetInstance(); - - return new DatabaseServerMessengerOptions - { - //These callbacks will be executed if the server has not been synced - // (i.e. it is a new server or the lastsynced.txt file has been removed) - InitializingCallbacks = new Action[] - { - //rebuild the xml cache file if the server is not synced - () => - { - // rebuild the published snapshot caches entirely, if the server is not synced - // this is equivalent to DistributedCache RefreshAll... but local only - // (we really should have a way to reuse RefreshAll... locally) - // note: refresh all content & media caches does refresh content types too - var svc = Current.PublishedSnapshotService; - svc.Notify(new[] { new DomainCacheRefresher.JsonPayload(0, DomainChangeTypes.RefreshAll) }); - svc.Notify(new[] { new ContentCacheRefresher.JsonPayload(0, null, TreeChangeTypes.RefreshAll) }, out _, out _); - svc.Notify(new[] { new MediaCacheRefresher.JsonPayload(0, null, TreeChangeTypes.RefreshAll) }, out _); - }, - - //rebuild indexes if the server is not synced - // NOTE: This will rebuild ALL indexes including the members, if developers want to target specific - // indexes then they can adjust this logic themselves. - () => { ExamineComponent.RebuildIndexes(indexRebuilder, logger, false, 5000); } - } - }; - } - - public override void Compose(Composition composition) - { - base.Compose(composition); - - composition.SetDatabaseServerMessengerOptions(GetDefaultOptions); - composition.SetServerMessenger(); - composition.Register(factory=> factory.GetInstance() as BatchedDatabaseServerMessenger, Lifetime.Singleton); - } - } public sealed class DatabaseServerRegistrarAndMessengerComponent : IComponent { @@ -88,14 +24,17 @@ namespace Umbraco.Web.Compose private readonly BackgroundTaskRunner _processTaskRunner; private bool _started; private IBackgroundTask[] _tasks; - private IndexRebuilder _indexRebuilder; - public DatabaseServerRegistrarAndMessengerComponent(IRuntimeState runtime, IServerRegistrar serverRegistrar, IServerMessenger serverMessenger, IServerRegistrationService registrationService, ILogger logger, IndexRebuilder indexRebuilder) + public DatabaseServerRegistrarAndMessengerComponent( + IRuntimeState runtime, + IServerRegistrar serverRegistrar, + IServerMessenger serverMessenger, + IServerRegistrationService registrationService, + ILogger logger) { _runtime = runtime; _logger = logger; _registrationService = registrationService; - _indexRebuilder = indexRebuilder; // create task runner for DatabaseServerRegistrar _registrar = serverRegistrar as DatabaseServerRegistrar; @@ -118,15 +57,21 @@ namespace Umbraco.Web.Compose { //We will start the whole process when a successful request is made if (_registrar != null || _messenger != null) + { UmbracoModule.RouteAttempt += RegisterBackgroundTasksOnce; - - // must come last, as it references some _variables - _messenger?.Startup(); + UmbracoModule.EndRequest += UmbracoModule_EndRequest; + } } public void Terminate() { } + private void UmbracoModule_EndRequest(object sender, UmbracoRequestEventArgs e) + { + // will clear the batch - will remain in HttpContext though - that's ok + _messenger?.FlushBatch(); + } + /// /// Handle when a request is made /// diff --git a/src/Umbraco.Web/Compose/DatabaseServerRegistrarAndMessengerComposer.cs b/src/Umbraco.Web/Compose/DatabaseServerRegistrarAndMessengerComposer.cs new file mode 100644 index 0000000000..96e538652f --- /dev/null +++ b/src/Umbraco.Web/Compose/DatabaseServerRegistrarAndMessengerComposer.cs @@ -0,0 +1,38 @@ +using Umbraco.Core; +using Umbraco.Core.Composing; +using Umbraco.Core.Sync; +using Umbraco.Web.Search; + +namespace Umbraco.Web.Compose +{ + /// + /// Ensures that servers are automatically registered in the database, when using the database server registrar. + /// + /// + /// At the moment servers are automatically registered upon first request and then on every + /// request but not more than once per (configurable) period. This really is "for information & debug" purposes so + /// we can look at the table and see what servers are registered - but the info is not used anywhere. + /// Should we actually want to use this, we would need a better and more deterministic way of figuring + /// out the "server address" ie the address to which server-to-server requests should be sent - because it + /// probably is not the "current request address" - especially in multi-domains configurations. + /// + [RuntimeLevel(MinLevel = RuntimeLevel.Run)] + // TODO: This is legacy, we no longer need to do this but we don't want to change the behavior now + [ComposeAfter(typeof(ExamineComposer))] + public sealed class DatabaseServerRegistrarAndMessengerComposer : ComponentComposer, ICoreComposer + { + public static DatabaseServerMessengerOptions GetDefaultOptions(IFactory factory) + { + return new DatabaseServerMessengerOptions(); + } + + public override void Compose(Composition composition) + { + base.Compose(composition); + + composition.SetDatabaseServerMessengerOptions(GetDefaultOptions); + composition.SetServerMessenger(); + composition.Register(factory => factory.GetInstance() as BatchedDatabaseServerMessenger, Lifetime.Singleton); + } + } +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs index dd24a5db6a..c6b214102b 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs @@ -13,6 +13,9 @@ namespace Umbraco.Web.PublishedCache.NuCache { base.Compose(composition); + //Overriden on Run state in DatabaseServerRegistrarAndMessengerComposer + composition.Register(Lifetime.Singleton); + var serializer = ConfigurationManager.AppSettings[NuCacheSerializerComponent.Nucache_Serializer_Key]; if (serializer != "MsgPack") { diff --git a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs index 102501d408..f9c25b7b35 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs @@ -4,6 +4,7 @@ using System.Configuration; using System.Globalization; using System.IO; using System.Linq; +using System.Threading; using CSharpTest.Net.Collections; using Umbraco.Core; using Umbraco.Core.Cache; @@ -33,6 +34,8 @@ namespace Umbraco.Web.PublishedCache.NuCache internal class PublishedSnapshotService : PublishedSnapshotServiceBase { + private readonly PublishedSnapshotServiceOptions _options; + private readonly IMainDom _mainDom; private readonly ServiceContext _serviceContext; private readonly IPublishedContentTypeFactory _publishedContentTypeFactory; private readonly IScopeProvider _scopeProvider; @@ -49,12 +52,13 @@ namespace Umbraco.Web.PublishedCache.NuCache private readonly IContentCacheDataSerializerFactory _contentCacheDataSerializerFactory; private readonly ContentDataSerializer _contentDataSerializer; - // volatile because we read it with no lock - private volatile bool _isReady; + private bool _isReady; + private bool _isReadSet; + private object _isReadyLock; - private readonly ContentStore _contentStore; - private readonly ContentStore _mediaStore; - private readonly SnapDictionary _domainStore; + private ContentStore _contentStore; + private ContentStore _mediaStore; + private SnapDictionary _domainStore; private readonly object _storesLock = new object(); private readonly object _elementsLock = new object(); @@ -89,9 +93,12 @@ namespace Umbraco.Web.PublishedCache.NuCache ContentDataSerializer contentDataSerializer = null) : base(publishedSnapshotAccessor, variationContextAccessor) { + //if (Interlocked.Increment(ref _singletonCheck) > 1) // throw new Exception("Singleton must be instantiated only once!"); + _options = options; + _mainDom = mainDom; _serviceContext = serviceContext; _publishedContentTypeFactory = publishedContentTypeFactory; _dataSource = dataSource; @@ -108,6 +115,8 @@ namespace Umbraco.Web.PublishedCache.NuCache _syncBootStateAccessor = syncBootStateAccessor; + _syncBootStateAccessor = syncBootStateAccessor; + // we need an Xml serializer here so that the member cache can support XPath, // for members this is done by navigating the serialized-to-xml member _entitySerializer = entitySerializer; @@ -126,41 +135,6 @@ namespace Umbraco.Web.PublishedCache.NuCache if (runtime.Level != RuntimeLevel.Run) return; - // lock this entire call, we only want a single thread to be accessing the stores at once and within - // the call below to mainDom.Register, a callback may occur on a threadpool thread to MainDomRelease - // at the same time as we are trying to write to the stores. MainDomRelease also locks on _storesLock so - // it will not be able to close the stores until we are done populating (if the store is empty) - lock (_storesLock) - { - if (options.IgnoreLocalDb == false) - { - var registered = mainDom.Register(MainDomRegister, MainDomRelease); - - // stores are created with a db so they can write to it, but they do not read from it, - // stores need to be populated, happens in OnResolutionFrozen which uses _localDbExists to - // figure out whether it can read the databases or it should populate them from sql - - _logger.Info("Creating the content store, localContentDbExists? {LocalContentDbExists}", _localContentDbExists); - _contentStore = new ContentStore(publishedSnapshotAccessor, variationContextAccessor, logger, _localContentDb); - _logger.Info("Creating the media store, localMediaDbExists? {LocalMediaDbExists}", _localMediaDbExists); - _mediaStore = new ContentStore(publishedSnapshotAccessor, variationContextAccessor, logger, _localMediaDb); - } - else - { - _logger.Info("Creating the content store (local db ignored)"); - _contentStore = new ContentStore(publishedSnapshotAccessor, variationContextAccessor, logger); - _logger.Info("Creating the media store (local db ignored)"); - _mediaStore = new ContentStore(publishedSnapshotAccessor, variationContextAccessor, logger); - } - - _domainStore = new SnapDictionary(); - - LoadCachesOnStartup(); - } - - Guid GetUid(ContentStore store, int id) => store.LiveSnapshot.Get(id)?.Uid ?? default; - int GetId(ContentStore store, Guid uid) => store.LiveSnapshot.Get(uid)?.Id ?? default; - if (idkMap != null) { idkMap.SetMapper(UmbracoObjectTypes.Document, id => GetUid(_contentStore, id), uid => GetId(_contentStore, uid)); @@ -168,6 +142,18 @@ namespace Umbraco.Web.PublishedCache.NuCache } } + private int GetId(ContentStore store, Guid uid) + { + EnsureCaches(); + return store.LiveSnapshot.Get(uid)?.Id ?? default; + } + + private Guid GetUid(ContentStore store, int id) + { + EnsureCaches(); + return store.LiveSnapshot.Get(id)?.Uid ?? default; + } + /// /// Install phase of /// @@ -219,52 +205,82 @@ namespace Umbraco.Web.PublishedCache.NuCache } /// - /// Populates the stores + /// Lazily populates the stores only when they are first requested /// - /// This is called inside of a lock for _storesLock - private void LoadCachesOnStartup() - { - var okContent = false; - var okMedia = false; - if (_syncBootStateAccessor.GetSyncBootState() == SyncBootState.ColdBoot) + internal void EnsureCaches() => LazyInitializer.EnsureInitialized( + ref _isReady, + ref _isReadSet, + ref _isReadyLock, + () => { - _logger.Info("Sync Service is in a Cold Boot state. Skip LoadCachesOnStartup as the Sync Service will trigger a full reload"); - _isReady = true; - return; - } - try - { - if (_localContentDbExists) + // lock this entire call, we only want a single thread to be accessing the stores at once and within + // the call below to mainDom.Register, a callback may occur on a threadpool thread to MainDomRelease + // at the same time as we are trying to write to the stores. MainDomRelease also locks on _storesLock so + // it will not be able to close the stores until we are done populating (if the store is empty) + lock (_storesLock) { - okContent = LockAndLoadContent(scope => LoadContentFromLocalDbLocked(true)); - if (!okContent) - _logger.Warn("Loading content from local db raised warnings, will reload from database."); - } + if (!_options.IgnoreLocalDb) + { + var registered = _mainDom.Register(MainDomRegister, MainDomRelease); - if (_localMediaDbExists) - { - okMedia = LockAndLoadMedia(scope => LoadMediaFromLocalDbLocked(true)); - if (!okMedia) - _logger.Warn("Loading media from local db raised warnings, will reload from database."); - } + // stores are created with a db so they can write to it, but they do not read from it, + // stores need to be populated, happens in OnResolutionFrozen which uses _localDbExists to + // figure out whether it can read the databases or it should populate them from sql + + _logger.Info("Creating the content store, localContentDbExists? {LocalContentDbExists}", _localContentDbExists); + _contentStore = new ContentStore(PublishedSnapshotAccessor, VariationContextAccessor, _logger, _localContentDb); + _logger.Info("Creating the media store, localMediaDbExists? {LocalMediaDbExists}", _localMediaDbExists); + _mediaStore = new ContentStore(PublishedSnapshotAccessor, VariationContextAccessor, _logger, _localMediaDb); + } + else + { + _logger.Info("Creating the content store (local db ignored)"); + _contentStore = new ContentStore(PublishedSnapshotAccessor, VariationContextAccessor, _logger); + _logger.Info("Creating the media store (local db ignored)"); + _mediaStore = new ContentStore(PublishedSnapshotAccessor, VariationContextAccessor, _logger); + } + + _domainStore = new SnapDictionary(); + + SyncBootState bootState = _syncBootStateAccessor.GetSyncBootState(); + + var okContent = false; + var okMedia = false; + + try + { + if (bootState != SyncBootState.ColdBoot && _localContentDbExists) + { + okContent = LockAndLoadContent(scope => LoadContentFromLocalDbLocked(true)); + if (!okContent) + _logger.Warn("Loading content from local db raised warnings, will reload from database."); + } + + if (bootState != SyncBootState.ColdBoot && _localMediaDbExists) + { + okMedia = LockAndLoadMedia(scope => LoadMediaFromLocalDbLocked(true)); + if (!okMedia) + _logger.Warn("Loading media from local db raised warnings, will reload from database."); + } - if (!okContent) - LockAndLoadContent(scope => LoadContentFromDatabaseLocked(scope, true)); + if (!okContent) + LockAndLoadContent(scope => LoadContentFromDatabaseLocked(scope, true)); - if (!okMedia) - LockAndLoadMedia(scope => LoadMediaFromDatabaseLocked(scope, true)); + if (!okMedia) + LockAndLoadMedia(scope => LoadMediaFromDatabaseLocked(scope, true)); - LockAndLoadDomains(); - } - catch (Exception ex) - { - _logger.Fatal(ex, "Panic, exception while loading cache data."); - throw; - } + LockAndLoadDomains(); + } + catch (Exception ex) + { + _logger.Fatal(ex, "Panic, exception while loading cache data."); + throw; + } - // finally, cache is ready! - _isReady = true; - } + // finally, cache is ready! + return true; + } + }); private void InitializeRepositoryEvents() { @@ -1146,9 +1162,13 @@ namespace Umbraco.Web.PublishedCache.NuCache public override IPublishedSnapshot CreatePublishedSnapshot(string previewToken) { + EnsureCaches(); + // no cache, no joy - if (_isReady == false) + if (Volatile.Read(ref _isReady) == false) + { throw new InvalidOperationException("The published snapshot service has not properly initialized."); + } var preview = previewToken.IsNullOrWhiteSpace() == false; return new PublishedSnapshot(this, preview); @@ -1159,6 +1179,8 @@ namespace Umbraco.Web.PublishedCache.NuCache // even though the underlying elements may not change (store snapshots) public PublishedSnapshot.PublishedSnapshotElements GetElements(bool previewDefault) { + EnsureCaches(); + // note: using ObjectCacheAppCache for elements and snapshot caches // is not recommended because it creates an inner MemoryCache which is a heavy // thing - better use a dictionary-based cache which "just" creates a concurrent @@ -1826,6 +1848,8 @@ AND cmsContentNu.nodeId IS NULL public void Collect() { + EnsureCaches(); + var contentCollect = _contentStore.CollectAsync(); var mediaCollect = _mediaStore.CollectAsync(); System.Threading.Tasks.Task.WaitAll(contentCollect, mediaCollect); @@ -1835,8 +1859,17 @@ AND cmsContentNu.nodeId IS NULL #region Internals/Testing - internal ContentStore GetContentStore() => _contentStore; - internal ContentStore GetMediaStore() => _mediaStore; + internal ContentStore GetContentStore() + { + EnsureCaches(); + return _contentStore; + } + + internal ContentStore GetMediaStore() + { + EnsureCaches(); + return _mediaStore; + } #endregion } diff --git a/src/Umbraco.Web/Search/ExamineFinalComponent.cs b/src/Umbraco.Web/Search/ExamineFinalComponent.cs index 95000b2b46..6dd775ab64 100644 --- a/src/Umbraco.Web/Search/ExamineFinalComponent.cs +++ b/src/Umbraco.Web/Search/ExamineFinalComponent.cs @@ -3,6 +3,8 @@ using Umbraco.Core.Logging; using Umbraco.Examine; using Umbraco.Core.Composing; using Umbraco.Core; +using Umbraco.Core.Sync; +using Umbraco.Web.Routing; namespace Umbraco.Web.Search { @@ -16,23 +18,50 @@ namespace Umbraco.Web.Search private readonly IExamineManager _examineManager; BackgroundIndexRebuilder _indexRebuilder; private readonly IMainDom _mainDom; - - public ExamineFinalComponent(IProfilingLogger logger, IExamineManager examineManager, BackgroundIndexRebuilder indexRebuilder, IMainDom mainDom) + private readonly ISyncBootStateAccessor _syncBootStateAccessor; + private readonly object _locker = new object(); + private bool _initialized = false; + + public ExamineFinalComponent(IProfilingLogger logger, IExamineManager examineManager, BackgroundIndexRebuilder indexRebuilder, IMainDom mainDom, ISyncBootStateAccessor syncBootStateAccessor) { _logger = logger; _examineManager = examineManager; _indexRebuilder = indexRebuilder; _mainDom = mainDom; + _syncBootStateAccessor = syncBootStateAccessor; + } + + private void UmbracoModule_RouteAttempt(object sender, RoutableAttemptEventArgs e) + { + if (!_initialized) + { + lock (_locker) + { + // double check lock, we must only do this once + if (!_initialized) + { + _initialized = true; + + UmbracoModule.RouteAttempt -= UmbracoModule_RouteAttempt; + + if (!_mainDom.IsMainDom) return; + + var bootState = _syncBootStateAccessor.GetSyncBootState(); + + _examineManager.ConfigureIndexes(_mainDom, _logger); + + // if it's a cold boot, rebuild all indexes including non-empty ones + // delay one minute since a cold boot also triggers nucache rebuilds + _indexRebuilder.RebuildIndexes(bootState != SyncBootState.ColdBoot, 60000); + } + } + } + } public void Initialize() { - if (!_mainDom.IsMainDom) return; - - _examineManager.ConfigureIndexes(_mainDom, _logger); - - // TODO: Instead of waiting 5000 ms, we could add an event handler on to fulfilling the first request, then start? - _indexRebuilder.RebuildIndexes(true, 5000); + UmbracoModule.RouteAttempt += UmbracoModule_RouteAttempt; } public void Terminate() diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index a03c580c95..3028a97bbb 100644 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -138,6 +138,7 @@ + @@ -1166,9 +1167,7 @@ - - ASPXCodeBehind - +