yv01p/Umbraco-CMS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'v8/bugfix/8893-examine-startup' into v8/feature/nucache-perf

Browse Source 
# Conflicts:
#	src/Umbraco.Core/Sync/DatabaseServerMessenger.cs
#	src/Umbraco.Core/Sync/ISyncBootStateAccessor.cs
#	src/Umbraco.Core/Sync/NonRuntimeLevelBootStateAccessor.cs
#	src/Umbraco.Core/Sync/SyncBootState.cs
#	src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs
#	src/Umbraco.Tests/PublishedContent/NuCacheTests.cs
#	src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs
#	src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs
#	src/Umbraco.Tests/TestHelpers/TestSyncBootStateAccessor.cs
#	src/Umbraco.Web/Compose/DatabaseServerRegistrarAndMessengerComponent.cs
#	src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs
#	src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs
This commit is contained in:
Shannon
2021-05-24 13:55:07 -07:00
parent eb2f1e55fd eba3c82a86
commit a9508886bf
17 changed files with 305 additions and 289 deletions
Download Patch File Download Diff File Expand all files Collapse all files

149
src/Umbraco.Core/Sync/DatabaseServerMessenger.cs
View File

@@ -39,9 +39,9 @@ namespace Umbraco.Core.Sync
private int _lastId = -1;
private DateTime _lastSync;
private DateTime _lastPruned;
private bool _initialized;
private bool _syncing;
private bool _released;
private readonly Lazy<SyncBootState> _getSyncBootState;
public DatabaseServerMessengerOptions Options { get; }
@@ -59,6 +59,7 @@ namespace Umbraco.Core.Sync
_lastPruned = _lastSync = DateTime.UtcNow;
_syncIdle = new ManualResetEvent(true);
_distCacheFilePath = new Lazy<string>(() => GetDistCacheFilePath(globalSettings));
_getSyncBootState = new Lazy<SyncBootState>(BootInternal);
}
protected ILogger Logger { get; }
@@ -75,7 +76,7 @@ namespace Umbraco.Core.Sync
{
// we don't care if there's servers listed or not,
// if distributed call is enabled we will make the call
return _initialized && DistributedEnabled;
return _getSyncBootState.IsValueCreated && DistributedEnabled;
}
protected override void DeliverRemote(
@@ -110,14 +111,14 @@ namespace Umbraco.Core.Sync
#region Sync
/// <summary>
/// Boots the messenger.
/// </summary>
/// <remarks>
/// Thread safety: this is NOT thread safe. Because it is NOT meant to run multi-threaded.
/// Callers MUST ensure thread-safety.
/// </remarks>
[Obsolete("This is no longer used and will be removed in future versions")]
protected void Boot()
{
// if called, just forces the boot logic
_ = GetSyncBootState();
}
private SyncBootState BootInternal()
{
// weight:10, must release *before* the published snapshot service, because once released
// the service will *not* be able to properly handle our notifications anymore
@@ -139,7 +140,7 @@ namespace Umbraco.Core.Sync
// properly releasing MainDom - a timeout here means that one refresher
// is taking too much time processing, however when it's done we will
// not update lastId and stop everything
var idle =_syncIdle.WaitOne(5000);
var idle = _syncIdle.WaitOne(5000);
if (idle == false)
{
Logger.Warn<DatabaseServerMessenger>("The wait lock timed out, application is shutting down. The current instruction batch will be re-processed.");
@@ -147,17 +148,23 @@ namespace Umbraco.Core.Sync
},
weight);
SyncBootState bootState = SyncBootState.Unknown;
if (registered == false)
return;
{
return bootState;
}
ReadLastSynced(); // get _lastId
using (var scope = ScopeProvider.CreateScope())
{
EnsureInstructions(scope.Database); // reset _lastId if instructions are missing
Initialize(scope.Database); // boot
bootState = Initialize(scope.Database); // boot
scope.Complete();
}
return bootState;
}
/// <summary>
@@ -167,36 +174,11 @@ namespace Umbraco.Core.Sync
/// Thread safety: this is NOT thread safe. Because it is NOT meant to run multi-threaded.
/// Callers MUST ensure thread-safety.
/// </remarks>
private void Initialize(IUmbracoDatabase database)
private SyncBootState Initialize(IUmbracoDatabase database)
{
lock (_locko)
{
if (_released) return;
var coldboot = IsColdBoot(database);
// could occur if shutting down immediately once starting up and before we've initialized
if (_released) return SyncBootState.Unknown;
if (coldboot)
{
// go get the last id in the db and store it
// note: do it BEFORE initializing otherwise some instructions might get lost
// when doing it before, some instructions might run twice - not an issue
var maxId = database.ExecuteScalar<int>("SELECT MAX(id) FROM umbracoCacheInstruction");
//if there is a max currently, or if we've never synced
if (maxId > 0 || _lastId < 0)
SaveLastSynced(maxId);
// execute initializing callbacks
if (Options.InitializingCallbacks != null)
foreach (var callback in Options.InitializingCallbacks)
callback();
}
_initialized = true;
}
}
private bool IsColdBoot(IUmbracoDatabase database)
{
var coldboot = false;
if (_lastId < 0) // never synced before
{
@@ -206,27 +188,48 @@ namespace Umbraco.Core.Sync
+ " The server will build its caches and indexes, and then adjust its last synced Id to the latest found in"
+ " the database and maintain cache updates based on that Id.");
coldboot = true;
}
else
coldboot = true;
}
else
{
//check for how many instructions there are to process, each row contains a count of the number of instructions contained in each
//row so we will sum these numbers to get the actual count.
var count = database.ExecuteScalar<int>("SELECT SUM(instructionCount) FROM umbracoCacheInstruction WHERE id > @lastId", new { lastId = _lastId });
if (count > Options.MaxProcessingInstructionCount)
{
//check for how many instructions there are to process, each row contains a count of the number of instructions contained in each
//row so we will sum these numbers to get the actual count.
var count = database.ExecuteScalar<int>("SELECT SUM(instructionCount) FROM umbracoCacheInstruction WHERE id > @lastId", new {lastId = _lastId});
if (count > Options.MaxProcessingInstructionCount)
{
//too many instructions, proceed to cold boot
Logger.Warn<DatabaseServerMessenger,int,int>(
"The instruction count ({InstructionCount}) exceeds the specified MaxProcessingInstructionCount ({MaxProcessingInstructionCount})."
+ " The server will skip existing instructions, rebuild its caches and indexes entirely, adjust its last synced Id"
+ " to the latest found in the database and maintain cache updates based on that Id.",
count, Options.MaxProcessingInstructionCount);
//too many instructions, proceed to cold boot
Logger.Warn<DatabaseServerMessenger, int, int>(
"The instruction count ({InstructionCount}) exceeds the specified MaxProcessingInstructionCount ({MaxProcessingInstructionCount})."
+ " The server will skip existing instructions, rebuild its caches and indexes entirely, adjust its last synced Id"
+ " to the latest found in the database and maintain cache updates based on that Id.",
count, Options.MaxProcessingInstructionCount);
coldboot = true;
}
}
return coldboot;
if (coldboot)
{
// go get the last id in the db and store it
// note: do it BEFORE initializing otherwise some instructions might get lost
// when doing it before, some instructions might run twice - not an issue
var maxId = database.ExecuteScalar<int>("SELECT MAX(id) FROM umbracoCacheInstruction");
//if there is a max currently, or if we've never synced
if (maxId > 0 || _lastId < 0)
SaveLastSynced(maxId);
// execute initializing callbacks
if (Options.InitializingCallbacks != null)
{
foreach (var callback in Options.InitializingCallbacks)
{
callback();
}
}
}
return coldboot ? SyncBootState.ColdBoot : SyncBootState.WarmBoot;
}
/// <summary>
@@ -358,7 +361,7 @@ namespace Umbraco.Core.Sync
}
catch (JsonException ex)
{
Logger.Error<DatabaseServerMessenger,int, string>(ex, "Failed to deserialize instructions ({DtoId}: '{DtoInstructions}').",
Logger.Error<DatabaseServerMessenger, int, string>(ex, "Failed to deserialize instructions ({DtoId}: '{DtoInstructions}').",
dto.Id,
dto.Instructions);
@@ -416,11 +419,11 @@ namespace Umbraco.Core.Sync
//}
catch (Exception ex)
{
Logger.Error<DatabaseServerMessenger,int, string> (
ex,
"DISTRIBUTED CACHE IS NOT UPDATED. Failed to execute instructions ({DtoId}: '{DtoInstructions}'). Instruction is being skipped/ignored",
dto.Id,
dto.Instructions);
Logger.Error<DatabaseServerMessenger, int, string>(
ex,
"DISTRIBUTED CACHE IS NOT UPDATED. Failed to execute instructions ({DtoId}: '{DtoInstructions}'). Instruction is being skipped/ignored",
dto.Id,
dto.Instructions);
//we cannot throw here because this invalid instruction will just keep getting processed over and over and errors
// will be thrown over and over. The only thing we can do is ignore and move on.
@@ -554,29 +557,7 @@ namespace Umbraco.Core.Sync
#endregion
public SyncBootState GetSyncBootState()
{
try
{
ReadLastSynced(); // get _lastId
using (var scope = ScopeProvider.CreateScope())
{
EnsureInstructions(scope.Database);
bool isColdBoot = IsColdBoot(scope.Database);
if (isColdBoot)
{
return SyncBootState.ColdBoot;
}
return SyncBootState.HasSyncState;
}
}
catch(Exception ex)
{
Logger.Warn<DatabaseServerMessenger>("Error determining Sync Boot State", ex);
return SyncBootState.Unknown;
}
}
public virtual SyncBootState GetSyncBootState() => _getSyncBootState.Value;
#region Notify refreshers

10
src/Umbraco.Core/Sync/DatabaseServerMessengerOptions.cs
View File

@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.ComponentModel;
namespace Umbraco.Core.Sync
{
@@ -24,13 +25,8 @@ namespace Umbraco.Core.Sync
/// </summary>
public int MaxProcessingInstructionCount { get; set; }
/// <summary>
/// A list of callbacks that will be invoked if the lastsynced.txt file does not exist.
/// </summary>
/// <remarks>
/// These callbacks will typically be for eg rebuilding the xml cache file, or examine indexes, based on
/// the data in the database to get this particular server node up to date.
/// </remarks>
[Obsolete("This should not be used. If initialization calls need to be invoked on a cold boot, use the ISyncBootStateAccessor.Booting event.")]
[EditorBrowsable(EditorBrowsableState.Never)]
public IEnumerable<Action> InitializingCallbacks { get; set; }
/// <summary>

4
src/Umbraco.Core/Sync/ISyncBootStateAccessor.cs
View File

@@ -1,8 +1,4 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Umbraco.Core.Sync
{

2
src/Umbraco.Core/Sync/NonRuntimeLevelBootStateAccessor.cs
View File

@@ -11,6 +11,8 @@ namespace Umbraco.Core.Sync
/// </summary>
public class NonRuntimeLevelBootStateAccessor : ISyncBootStateAccessor
{
public event EventHandler<SyncBootState> Booting;
public SyncBootState GetSyncBootState()
{
return SyncBootState.Unknown;

14
src/Umbraco.Core/Sync/SyncBootState.cs
View File

@@ -1,24 +1,20 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Umbraco.Core.Sync
namespace Umbraco.Core.Sync
{
public enum SyncBootState
{
/// <summary>
/// Unknown state. Treat as HasSyncState
/// Unknown state. Treat as WarmBoot
/// </summary>
Unknown = 0,
/// <summary>
/// Cold boot. No Sync state
/// </summary>
ColdBoot = 1,
/// <summary>
/// Warm boot. Sync state present
/// </summary>
HasSyncState = 2
WarmBoot = 2
}
}

2
src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs
View File

@@ -159,7 +159,7 @@ namespace Umbraco.Tests.PublishedContent
Mock.Of<IEntityXmlSerializer>(),
Mock.Of<IPublishedModelFactory>(),
new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }),
new TestSyncBootStateAccessor(SyncBootState.HasSyncState),
new TestSyncBootStateAccessor(SyncBootState.WarmBoot),
_contentNestedDataSerializerFactory);
// invariant is the current default

2
src/Umbraco.Tests/PublishedContent/NuCacheTests.cs
View File

@@ -205,7 +205,7 @@ namespace Umbraco.Tests.PublishedContent
Mock.Of<IEntityXmlSerializer>(),
Mock.Of<IPublishedModelFactory>(),
new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }),
new TestSyncBootStateAccessor(SyncBootState.HasSyncState),
new TestSyncBootStateAccessor(SyncBootState.WarmBoot),
_contentNestedDataSerializerFactory);
// invariant is the current default

2
src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs
View File

@@ -101,7 +101,7 @@ namespace Umbraco.Tests.Scoping
Factory.GetInstance<IEntityXmlSerializer>(),
Mock.Of<IPublishedModelFactory>(),
new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }),
new TestSyncBootStateAccessor(SyncBootState.HasSyncState),
new TestSyncBootStateAccessor(SyncBootState.WarmBoot),
nestedContentDataSerializerFactory);
}

2
src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs
View File

@@ -74,7 +74,7 @@ namespace Umbraco.Tests.Services
Factory.GetInstance<IEntityXmlSerializer>(),
Mock.Of<IPublishedModelFactory>(),
new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }),
new TestSyncBootStateAccessor(SyncBootState.HasSyncState),
new TestSyncBootStateAccessor(SyncBootState.WarmBoot),
nestedContentDataSerializerFactory);
}

3
src/Umbraco.Tests/TestHelpers/TestSyncBootStateAccessor.cs
View File

@@ -15,6 +15,9 @@ namespace Umbraco.Tests.TestHelpers
{
_syncBootState = syncBootState;
}
public event EventHandler<SyncBootState> Booting;
public SyncBootState GetSyncBootState()
{
return _syncBootState;

33
src/Umbraco.Web/BatchedDatabaseServerMessenger.cs
View File

@@ -26,6 +26,7 @@ namespace Umbraco.Web
public class BatchedDatabaseServerMessenger : DatabaseServerMessenger
{
private readonly IUmbracoDatabaseFactory _databaseFactory;
private readonly Lazy<SyncBootState> _syncBootState;
[Obsolete("This overload should not be used, enableDistCalls has no effect")]
[EditorBrowsable(EditorBrowsableState.Never)]
@@ -39,28 +40,22 @@ namespace Umbraco.Web
: base(runtime, scopeProvider, sqlContext, proflog, globalSettings, true, options)
{
_databaseFactory = databaseFactory;
}
// invoked by DatabaseServerRegistrarAndMessengerComponent
internal void Startup()
{
UmbracoModule.EndRequest += UmbracoModule_EndRequest;
if (_databaseFactory.CanConnect == false)
_syncBootState = new Lazy<SyncBootState>(() =>
{
Logger.Warn<BatchedDatabaseServerMessenger>("Cannot connect to the database, distributed calls will not be enabled for this server.");
}
else
{
Boot();
}
if (_databaseFactory.CanConnect == false)
{
Logger.Warn<BatchedDatabaseServerMessenger>("Cannot connect to the database, distributed calls will not be enabled for this server.");
return SyncBootState.Unknown;
}
else
{
return base.GetSyncBootState();
}
});
}
private void UmbracoModule_EndRequest(object sender, UmbracoRequestEventArgs e)
{
// will clear the batch - will remain in HttpContext though - that's ok
FlushBatch();
}
// override to deal with database connectivity
public override SyncBootState GetSyncBootState() => _syncBootState.Value;
protected override void DeliverRemote(ICacheRefresher refresher, MessageType messageType, IEnumerable<object> ids = null, string json = null)
{

85
src/Umbraco.Web/Compose/DatabaseServerRegistrarAndMessengerComponent.cs
View File

@@ -4,77 +4,13 @@ using Umbraco.Core;
using Umbraco.Core.Composing;
using Umbraco.Core.Logging;
using Umbraco.Core.Services;
using Umbraco.Core.Services.Changes;
using Umbraco.Core.Sync;
using Umbraco.Examine;
using Umbraco.Web.Cache;
using Umbraco.Web.Routing;
using Umbraco.Web.Scheduling;
using Umbraco.Web.Search;
using Current = Umbraco.Web.Composing.Current;
namespace Umbraco.Web.Compose
{
/// <summary>
/// Ensures that servers are automatically registered in the database, when using the database server registrar.
/// </summary>
/// <remarks>
/// <para>At the moment servers are automatically registered upon first request and then on every
/// request but not more than once per (configurable) period. This really is "for information & debug" purposes so
/// we can look at the table and see what servers are registered - but the info is not used anywhere.</para>
/// <para>Should we actually want to use this, we would need a better and more deterministic way of figuring
/// out the "server address" ie the address to which server-to-server requests should be sent - because it
/// probably is not the "current request address" - especially in multi-domains configurations.</para>
/// </remarks>
[RuntimeLevel(MinLevel = RuntimeLevel.Run)]
// during Initialize / Startup, we end up checking Examine, which needs to be initialized beforehand
// TODO: should not be a strong dependency on "examine" but on an "indexing component"
[ComposeAfter(typeof(ExamineComposer))]
public sealed class DatabaseServerRegistrarAndMessengerComposer : ComponentComposer<DatabaseServerRegistrarAndMessengerComponent>, ICoreComposer
{
public static DatabaseServerMessengerOptions GetDefaultOptions(IFactory factory)
{
var logger = factory.GetInstance<ILogger>();
var indexRebuilder = factory.GetInstance<IndexRebuilder>();
return new DatabaseServerMessengerOptions
{
//These callbacks will be executed if the server has not been synced
// (i.e. it is a new server or the lastsynced.txt file has been removed)
InitializingCallbacks = new Action[]
{
//rebuild the xml cache file if the server is not synced
() =>
{
// rebuild the published snapshot caches entirely, if the server is not synced
// this is equivalent to DistributedCache RefreshAll... but local only
// (we really should have a way to reuse RefreshAll... locally)
// note: refresh all content & media caches does refresh content types too
var svc = Current.PublishedSnapshotService;
svc.Notify(new[] { new DomainCacheRefresher.JsonPayload(0, DomainChangeTypes.RefreshAll) });
svc.Notify(new[] { new ContentCacheRefresher.JsonPayload(0, null, TreeChangeTypes.RefreshAll) }, out _, out _);
svc.Notify(new[] { new MediaCacheRefresher.JsonPayload(0, null, TreeChangeTypes.RefreshAll) }, out _);
},
//rebuild indexes if the server is not synced
// NOTE: This will rebuild ALL indexes including the members, if developers want to target specific
// indexes then they can adjust this logic themselves.
() => { ExamineComponent.RebuildIndexes(indexRebuilder, logger, false, 5000); }
}
};
}
public override void Compose(Composition composition)
{
base.Compose(composition);
composition.SetDatabaseServerMessengerOptions(GetDefaultOptions);
composition.SetServerMessenger<BatchedDatabaseServerMessenger>();
composition.Register<ISyncBootStateAccessor>(factory=> factory.GetInstance<IServerMessenger>() as BatchedDatabaseServerMessenger, Lifetime.Singleton);
}
}
public sealed class DatabaseServerRegistrarAndMessengerComponent : IComponent
{
@@ -88,14 +24,17 @@ namespace Umbraco.Web.Compose
private readonly BackgroundTaskRunner<IBackgroundTask> _processTaskRunner;
private bool _started;
private IBackgroundTask[] _tasks;
private IndexRebuilder _indexRebuilder;
public DatabaseServerRegistrarAndMessengerComponent(IRuntimeState runtime, IServerRegistrar serverRegistrar, IServerMessenger serverMessenger, IServerRegistrationService registrationService, ILogger logger, IndexRebuilder indexRebuilder)
public DatabaseServerRegistrarAndMessengerComponent(
IRuntimeState runtime,
IServerRegistrar serverRegistrar,
IServerMessenger serverMessenger,
IServerRegistrationService registrationService,
ILogger logger)
{
_runtime = runtime;
_logger = logger;
_registrationService = registrationService;
_indexRebuilder = indexRebuilder;
// create task runner for DatabaseServerRegistrar
_registrar = serverRegistrar as DatabaseServerRegistrar;
@@ -118,15 +57,21 @@ namespace Umbraco.Web.Compose
{
//We will start the whole process when a successful request is made
if (_registrar != null || _messenger != null)
{
UmbracoModule.RouteAttempt += RegisterBackgroundTasksOnce;
// must come last, as it references some _variables
_messenger?.Startup();
UmbracoModule.EndRequest += UmbracoModule_EndRequest;
}
}
public void Terminate()
{ }
private void UmbracoModule_EndRequest(object sender, UmbracoRequestEventArgs e)
{
// will clear the batch - will remain in HttpContext though - that's ok
_messenger?.FlushBatch();
}
/// <summary>
/// Handle when a request is made
/// </summary>

38
src/Umbraco.Web/Compose/DatabaseServerRegistrarAndMessengerComposer.cs Normal file
View File

@@ -0,0 +1,38 @@
using Umbraco.Core;
using Umbraco.Core.Composing;
using Umbraco.Core.Sync;
using Umbraco.Web.Search;
namespace Umbraco.Web.Compose
{
/// <summary>
/// Ensures that servers are automatically registered in the database, when using the database server registrar.
/// </summary>
/// <remarks>
/// <para>At the moment servers are automatically registered upon first request and then on every
/// request but not more than once per (configurable) period. This really is "for information & debug" purposes so
/// we can look at the table and see what servers are registered - but the info is not used anywhere.</para>
/// <para>Should we actually want to use this, we would need a better and more deterministic way of figuring
/// out the "server address" ie the address to which server-to-server requests should be sent - because it
/// probably is not the "current request address" - especially in multi-domains configurations.</para>
/// </remarks>
[RuntimeLevel(MinLevel = RuntimeLevel.Run)]
// TODO: This is legacy, we no longer need to do this but we don't want to change the behavior now
[ComposeAfter(typeof(ExamineComposer))]
public sealed class DatabaseServerRegistrarAndMessengerComposer : ComponentComposer<DatabaseServerRegistrarAndMessengerComponent>, ICoreComposer
{
public static DatabaseServerMessengerOptions GetDefaultOptions(IFactory factory)
{
return new DatabaseServerMessengerOptions();
}
public override void Compose(Composition composition)
{
base.Compose(composition);
composition.SetDatabaseServerMessengerOptions(GetDefaultOptions);
composition.SetServerMessenger<BatchedDatabaseServerMessenger>();
composition.Register<ISyncBootStateAccessor>(factory => factory.GetInstance<IServerMessenger>() as BatchedDatabaseServerMessenger, Lifetime.Singleton);
}
}
}

3
src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs
View File

@@ -13,6 +13,9 @@ namespace Umbraco.Web.PublishedCache.NuCache
{
base.Compose(composition);
//Overriden on Run state in DatabaseServerRegistrarAndMessengerComposer
composition.Register<ISyncBootStateAccessor, NonRuntimeLevelBootStateAccessor>(Lifetime.Singleton);
var serializer = ConfigurationManager.AppSettings[NuCacheSerializerComponent.Nucache_Serializer_Key];
if (serializer != "MsgPack")
{

195
src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs
View File

@@ -4,6 +4,7 @@ using System.Configuration;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Threading;
using CSharpTest.Net.Collections;
using Umbraco.Core;
using Umbraco.Core.Cache;
@@ -33,6 +34,8 @@ namespace Umbraco.Web.PublishedCache.NuCache
internal class PublishedSnapshotService : PublishedSnapshotServiceBase
{
private readonly PublishedSnapshotServiceOptions _options;
private readonly IMainDom _mainDom;
private readonly ServiceContext _serviceContext;
private readonly IPublishedContentTypeFactory _publishedContentTypeFactory;
private readonly IScopeProvider _scopeProvider;
@@ -49,12 +52,13 @@ namespace Umbraco.Web.PublishedCache.NuCache
private readonly IContentCacheDataSerializerFactory _contentCacheDataSerializerFactory;
private readonly ContentDataSerializer _contentDataSerializer;
// volatile because we read it with no lock
private volatile bool _isReady;
private bool _isReady;
private bool _isReadSet;
private object _isReadyLock;
private readonly ContentStore _contentStore;
private readonly ContentStore _mediaStore;
private readonly SnapDictionary<int, Domain> _domainStore;
private ContentStore _contentStore;
private ContentStore _mediaStore;
private SnapDictionary<int, Domain> _domainStore;
private readonly object _storesLock = new object();
private readonly object _elementsLock = new object();
@@ -89,9 +93,12 @@ namespace Umbraco.Web.PublishedCache.NuCache
ContentDataSerializer contentDataSerializer = null)
: base(publishedSnapshotAccessor, variationContextAccessor)
{
//if (Interlocked.Increment(ref _singletonCheck) > 1)
// throw new Exception("Singleton must be instantiated only once!");
_options = options;
_mainDom = mainDom;
_serviceContext = serviceContext;
_publishedContentTypeFactory = publishedContentTypeFactory;
_dataSource = dataSource;
@@ -108,6 +115,8 @@ namespace Umbraco.Web.PublishedCache.NuCache
_syncBootStateAccessor = syncBootStateAccessor;
_syncBootStateAccessor = syncBootStateAccessor;
// we need an Xml serializer here so that the member cache can support XPath,
// for members this is done by navigating the serialized-to-xml member
_entitySerializer = entitySerializer;
@@ -126,41 +135,6 @@ namespace Umbraco.Web.PublishedCache.NuCache
if (runtime.Level != RuntimeLevel.Run)
return;
// lock this entire call, we only want a single thread to be accessing the stores at once and within
// the call below to mainDom.Register, a callback may occur on a threadpool thread to MainDomRelease
// at the same time as we are trying to write to the stores. MainDomRelease also locks on _storesLock so
// it will not be able to close the stores until we are done populating (if the store is empty)
lock (_storesLock)
{
if (options.IgnoreLocalDb == false)
{
var registered = mainDom.Register(MainDomRegister, MainDomRelease);
// stores are created with a db so they can write to it, but they do not read from it,
// stores need to be populated, happens in OnResolutionFrozen which uses _localDbExists to
// figure out whether it can read the databases or it should populate them from sql
_logger.Info<PublishedSnapshotService,bool>("Creating the content store, localContentDbExists? {LocalContentDbExists}", _localContentDbExists);
_contentStore = new ContentStore(publishedSnapshotAccessor, variationContextAccessor, logger, _localContentDb);
_logger.Info<PublishedSnapshotService,bool>("Creating the media store, localMediaDbExists? {LocalMediaDbExists}", _localMediaDbExists);
_mediaStore = new ContentStore(publishedSnapshotAccessor, variationContextAccessor, logger, _localMediaDb);
}
else
{
_logger.Info<PublishedSnapshotService>("Creating the content store (local db ignored)");
_contentStore = new ContentStore(publishedSnapshotAccessor, variationContextAccessor, logger);
_logger.Info<PublishedSnapshotService>("Creating the media store (local db ignored)");
_mediaStore = new ContentStore(publishedSnapshotAccessor, variationContextAccessor, logger);
}
_domainStore = new SnapDictionary<int, Domain>();
LoadCachesOnStartup();
}
Guid GetUid(ContentStore store, int id) => store.LiveSnapshot.Get(id)?.Uid ?? default;
int GetId(ContentStore store, Guid uid) => store.LiveSnapshot.Get(uid)?.Id ?? default;
if (idkMap != null)
{
idkMap.SetMapper(UmbracoObjectTypes.Document, id => GetUid(_contentStore, id), uid => GetId(_contentStore, uid));
@@ -168,6 +142,18 @@ namespace Umbraco.Web.PublishedCache.NuCache
}
}
private int GetId(ContentStore store, Guid uid)
{
EnsureCaches();
return store.LiveSnapshot.Get(uid)?.Id ?? default;
}
private Guid GetUid(ContentStore store, int id)
{
EnsureCaches();
return store.LiveSnapshot.Get(id)?.Uid ?? default;
}
/// <summary>
/// Install phase of <see cref="IMainDom"/>
/// </summary>
@@ -219,52 +205,82 @@ namespace Umbraco.Web.PublishedCache.NuCache
}
/// <summary>
/// Populates the stores
/// Lazily populates the stores only when they are first requested
/// </summary>
/// <remarks>This is called inside of a lock for _storesLock</remarks>
private void LoadCachesOnStartup()
{
var okContent = false;
var okMedia = false;
if (_syncBootStateAccessor.GetSyncBootState() == SyncBootState.ColdBoot)
internal void EnsureCaches() => LazyInitializer.EnsureInitialized(
ref _isReady,
ref _isReadSet,
ref _isReadyLock,
() =>
{
_logger.Info<PublishedSnapshotService>("Sync Service is in a Cold Boot state. Skip LoadCachesOnStartup as the Sync Service will trigger a full reload");
_isReady = true;
return;
}
try
{
if (_localContentDbExists)
// lock this entire call, we only want a single thread to be accessing the stores at once and within
// the call below to mainDom.Register, a callback may occur on a threadpool thread to MainDomRelease
// at the same time as we are trying to write to the stores. MainDomRelease also locks on _storesLock so
// it will not be able to close the stores until we are done populating (if the store is empty)
lock (_storesLock)
{
okContent = LockAndLoadContent(scope => LoadContentFromLocalDbLocked(true));
if (!okContent)
_logger.Warn<PublishedSnapshotService>("Loading content from local db raised warnings, will reload from database.");
}
if (!_options.IgnoreLocalDb)
{
var registered = _mainDom.Register(MainDomRegister, MainDomRelease);
if (_localMediaDbExists)
{
okMedia = LockAndLoadMedia(scope => LoadMediaFromLocalDbLocked(true));
if (!okMedia)
_logger.Warn<PublishedSnapshotService>("Loading media from local db raised warnings, will reload from database.");
}
// stores are created with a db so they can write to it, but they do not read from it,
// stores need to be populated, happens in OnResolutionFrozen which uses _localDbExists to
// figure out whether it can read the databases or it should populate them from sql
_logger.Info<PublishedSnapshotService, bool>("Creating the content store, localContentDbExists? {LocalContentDbExists}", _localContentDbExists);
_contentStore = new ContentStore(PublishedSnapshotAccessor, VariationContextAccessor, _logger, _localContentDb);
_logger.Info<PublishedSnapshotService, bool>("Creating the media store, localMediaDbExists? {LocalMediaDbExists}", _localMediaDbExists);
_mediaStore = new ContentStore(PublishedSnapshotAccessor, VariationContextAccessor, _logger, _localMediaDb);
}
else
{
_logger.Info<PublishedSnapshotService>("Creating the content store (local db ignored)");
_contentStore = new ContentStore(PublishedSnapshotAccessor, VariationContextAccessor, _logger);
_logger.Info<PublishedSnapshotService>("Creating the media store (local db ignored)");
_mediaStore = new ContentStore(PublishedSnapshotAccessor, VariationContextAccessor, _logger);
}
_domainStore = new SnapDictionary<int, Domain>();
SyncBootState bootState = _syncBootStateAccessor.GetSyncBootState();
var okContent = false;
var okMedia = false;
try
{
if (bootState != SyncBootState.ColdBoot && _localContentDbExists)
{
okContent = LockAndLoadContent(scope => LoadContentFromLocalDbLocked(true));
if (!okContent)
_logger.Warn<PublishedSnapshotService>("Loading content from local db raised warnings, will reload from database.");
}
if (bootState != SyncBootState.ColdBoot && _localMediaDbExists)
{
okMedia = LockAndLoadMedia(scope => LoadMediaFromLocalDbLocked(true));
if (!okMedia)
_logger.Warn<PublishedSnapshotService>("Loading media from local db raised warnings, will reload from database.");
}
if (!okContent)
LockAndLoadContent(scope => LoadContentFromDatabaseLocked(scope, true));
if (!okContent)
LockAndLoadContent(scope => LoadContentFromDatabaseLocked(scope, true));
if (!okMedia)
LockAndLoadMedia(scope => LoadMediaFromDatabaseLocked(scope, true));
if (!okMedia)
LockAndLoadMedia(scope => LoadMediaFromDatabaseLocked(scope, true));
LockAndLoadDomains();
}
catch (Exception ex)
{
_logger.Fatal<PublishedSnapshotService>(ex, "Panic, exception while loading cache data.");
throw;
}
LockAndLoadDomains();
}
catch (Exception ex)
{
_logger.Fatal<PublishedSnapshotService>(ex, "Panic, exception while loading cache data.");
throw;
}
// finally, cache is ready!
_isReady = true;
}
// finally, cache is ready!
return true;
}
});
private void InitializeRepositoryEvents()
{
@@ -1146,9 +1162,13 @@ namespace Umbraco.Web.PublishedCache.NuCache
public override IPublishedSnapshot CreatePublishedSnapshot(string previewToken)
{
EnsureCaches();
// no cache, no joy
if (_isReady == false)
if (Volatile.Read(ref _isReady) == false)
{
throw new InvalidOperationException("The published snapshot service has not properly initialized.");
}
var preview = previewToken.IsNullOrWhiteSpace() == false;
return new PublishedSnapshot(this, preview);
@@ -1159,6 +1179,8 @@ namespace Umbraco.Web.PublishedCache.NuCache
// even though the underlying elements may not change (store snapshots)
public PublishedSnapshot.PublishedSnapshotElements GetElements(bool previewDefault)
{
EnsureCaches();
// note: using ObjectCacheAppCache for elements and snapshot caches
// is not recommended because it creates an inner MemoryCache which is a heavy
// thing - better use a dictionary-based cache which "just" creates a concurrent
@@ -1826,6 +1848,8 @@ AND cmsContentNu.nodeId IS NULL
public void Collect()
{
EnsureCaches();
var contentCollect = _contentStore.CollectAsync();
var mediaCollect = _mediaStore.CollectAsync();
System.Threading.Tasks.Task.WaitAll(contentCollect, mediaCollect);
@@ -1835,8 +1859,17 @@ AND cmsContentNu.nodeId IS NULL
#region Internals/Testing
internal ContentStore GetContentStore() => _contentStore;
internal ContentStore GetMediaStore() => _mediaStore;
internal ContentStore GetContentStore()
{
EnsureCaches();
return _contentStore;
}
internal ContentStore GetMediaStore()
{
EnsureCaches();
return _mediaStore;
}
#endregion
}

45
src/Umbraco.Web/Search/ExamineFinalComponent.cs
View File

@@ -3,6 +3,8 @@ using Umbraco.Core.Logging;
using Umbraco.Examine;
using Umbraco.Core.Composing;
using Umbraco.Core;
using Umbraco.Core.Sync;
using Umbraco.Web.Routing;
namespace Umbraco.Web.Search
{
@@ -16,23 +18,50 @@ namespace Umbraco.Web.Search
private readonly IExamineManager _examineManager;
BackgroundIndexRebuilder _indexRebuilder;
private readonly IMainDom _mainDom;
public ExamineFinalComponent(IProfilingLogger logger, IExamineManager examineManager, BackgroundIndexRebuilder indexRebuilder, IMainDom mainDom)
private readonly ISyncBootStateAccessor _syncBootStateAccessor;
private readonly object _locker = new object();
private bool _initialized = false;
public ExamineFinalComponent(IProfilingLogger logger, IExamineManager examineManager, BackgroundIndexRebuilder indexRebuilder, IMainDom mainDom, ISyncBootStateAccessor syncBootStateAccessor)
{
_logger = logger;
_examineManager = examineManager;
_indexRebuilder = indexRebuilder;
_mainDom = mainDom;
_syncBootStateAccessor = syncBootStateAccessor;
}
private void UmbracoModule_RouteAttempt(object sender, RoutableAttemptEventArgs e)
{
if (!_initialized)
{
lock (_locker)
{
// double check lock, we must only do this once
if (!_initialized)
{
_initialized = true;
UmbracoModule.RouteAttempt -= UmbracoModule_RouteAttempt;
if (!_mainDom.IsMainDom) return;
var bootState = _syncBootStateAccessor.GetSyncBootState();
_examineManager.ConfigureIndexes(_mainDom, _logger);
// if it's a cold boot, rebuild all indexes including non-empty ones
// delay one minute since a cold boot also triggers nucache rebuilds
_indexRebuilder.RebuildIndexes(bootState != SyncBootState.ColdBoot, 60000);
}
}
}
}
public void Initialize()
{
if (!_mainDom.IsMainDom) return;
_examineManager.ConfigureIndexes(_mainDom, _logger);
// TODO: Instead of waiting 5000 ms, we could add an event handler on to fulfilling the first request, then start?
_indexRebuilder.RebuildIndexes(true, 5000);
UmbracoModule.RouteAttempt += UmbracoModule_RouteAttempt;
}
public void Terminate()

5
src/Umbraco.Web/Umbraco.Web.csproj
View File

@@ -138,6 +138,7 @@
<Compile Include="Compose\BackOfficeUserAuditEventsComposer.cs" />
<Compile Include="Compose\BlockEditorComponent.cs" />
<Compile Include="Compose\BlockEditorComposer.cs" />
<Compile Include="Compose\DatabaseServerRegistrarAndMessengerComposer.cs" />
<Compile Include="Compose\NestedContentPropertyComponent.cs" />
<Compile Include="Compose\NotificationsComposer.cs" />
<Compile Include="Compose\PublicAccessComposer.cs" />
@@ -1166,9 +1167,7 @@
<Compile Include="PublishedCache\IPublishedMediaCache.cs" />
<Compile Include="PublishedContentExtensions.cs" />
<Compile Include="ExamineExtensions.cs" />
<Compile Include="FormlessPage.cs">
<SubType>ASPXCodeBehind</SubType>
</Compile>
<Compile Include="FormlessPage.cs" />
<Compile Include="HtmlHelperRenderExtensions.cs" />
<Compile Include="Scheduling\SchedulerComponent.cs" />
<Compile Include="ModelStateExtensions.cs" />