yv01p/Umbraco-CMS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #10182 from umbraco/v8/bugfix/8893-examine-startup

Browse Source 
Fix startup issues and timing with Examine and Nucache
This commit is contained in:
Shannon Deminick
2021-06-05 02:37:33 +10:00
committed by GitHub
parent cd53d7a294 eba3c82a86
commit eb6b7cffad
19 changed files with 406 additions and 251 deletions
Download Patch File Download Diff File Expand all files Collapse all files

137
src/Umbraco.Core/Sync/DatabaseServerMessenger.cs
View File

@@ -28,7 +28,7 @@ namespace Umbraco.Core.Sync
// but only processes instructions coming from remote servers,
// thus ensuring that instructions run only once
//
public class DatabaseServerMessenger : ServerMessengerBase
public class DatabaseServerMessenger : ServerMessengerBase, ISyncBootStateAccessor
{
private readonly IRuntimeState _runtime;
private readonly ManualResetEvent _syncIdle;
@@ -39,9 +39,9 @@ namespace Umbraco.Core.Sync
private int _lastId = -1;
private DateTime _lastSync;
private DateTime _lastPruned;
private bool _initialized;
private bool _syncing;
private bool _released;
private readonly Lazy<SyncBootState> _getSyncBootState;
public DatabaseServerMessengerOptions Options { get; }
@@ -59,6 +59,7 @@ namespace Umbraco.Core.Sync
_lastPruned = _lastSync = DateTime.UtcNow;
_syncIdle = new ManualResetEvent(true);
_distCacheFilePath = new Lazy<string>(() => GetDistCacheFilePath(globalSettings));
_getSyncBootState = new Lazy<SyncBootState>(BootInternal);
}
protected ILogger Logger { get; }
@@ -75,7 +76,7 @@ namespace Umbraco.Core.Sync
{
// we don't care if there's servers listed or not,
// if distributed call is enabled we will make the call
return _initialized && DistributedEnabled;
return _getSyncBootState.IsValueCreated && DistributedEnabled;
}
protected override void DeliverRemote(
@@ -110,14 +111,14 @@ namespace Umbraco.Core.Sync
#region Sync
/// <summary>
/// Boots the messenger.
/// </summary>
/// <remarks>
/// Thread safety: this is NOT thread safe. Because it is NOT meant to run multi-threaded.
/// Callers MUST ensure thread-safety.
/// </remarks>
[Obsolete("This is no longer used and will be removed in future versions")]
protected void Boot()
{
// if called, just forces the boot logic
_ = GetSyncBootState();
}
private SyncBootState BootInternal()
{
// weight:10, must release *before* the published snapshot service, because once released
// the service will *not* be able to properly handle our notifications anymore
@@ -139,7 +140,7 @@ namespace Umbraco.Core.Sync
// properly releasing MainDom - a timeout here means that one refresher
// is taking too much time processing, however when it's done we will
// not update lastId and stop everything
var idle =_syncIdle.WaitOne(5000);
var idle = _syncIdle.WaitOne(5000);
if (idle == false)
{
Logger.Warn<DatabaseServerMessenger>("The wait lock timed out, application is shutting down. The current instruction batch will be re-processed.");
@@ -147,17 +148,23 @@ namespace Umbraco.Core.Sync
},
weight);
SyncBootState bootState = SyncBootState.Unknown;
if (registered == false)
return;
{
return bootState;
}
ReadLastSynced(); // get _lastId
using (var scope = ScopeProvider.CreateScope())
{
EnsureInstructions(scope.Database); // reset _lastId if instructions are missing
Initialize(scope.Database); // boot
bootState = Initialize(scope.Database); // boot
scope.Complete();
}
return bootState;
}
/// <summary>
@@ -167,60 +174,62 @@ namespace Umbraco.Core.Sync
/// Thread safety: this is NOT thread safe. Because it is NOT meant to run multi-threaded.
/// Callers MUST ensure thread-safety.
/// </remarks>
private void Initialize(IUmbracoDatabase database)
private SyncBootState Initialize(IUmbracoDatabase database)
{
lock (_locko)
{
if (_released) return;
// could occur if shutting down immediately once starting up and before we've initialized
if (_released) return SyncBootState.Unknown;
var coldboot = false;
if (_lastId < 0) // never synced before
var coldboot = false;
if (_lastId < 0) // never synced before
{
// we haven't synced - in this case we aren't going to sync the whole thing, we will assume this is a new
// server and it will need to rebuild it's own caches, eg Lucene or the xml cache file.
Logger.Warn<DatabaseServerMessenger>("No last synced Id found, this generally means this is a new server/install."
+ " The server will build its caches and indexes, and then adjust its last synced Id to the latest found in"
+ " the database and maintain cache updates based on that Id.");
coldboot = true;
}
else
{
//check for how many instructions there are to process, each row contains a count of the number of instructions contained in each
//row so we will sum these numbers to get the actual count.
var count = database.ExecuteScalar<int>("SELECT SUM(instructionCount) FROM umbracoCacheInstruction WHERE id > @lastId", new { lastId = _lastId });
if (count > Options.MaxProcessingInstructionCount)
{
// we haven't synced - in this case we aren't going to sync the whole thing, we will assume this is a new
// server and it will need to rebuild it's own caches, eg Lucene or the xml cache file.
Logger.Warn<DatabaseServerMessenger>("No last synced Id found, this generally means this is a new server/install."
+ " The server will build its caches and indexes, and then adjust its last synced Id to the latest found in"
+ " the database and maintain cache updates based on that Id.");
//too many instructions, proceed to cold boot
Logger.Warn<DatabaseServerMessenger, int, int>(
"The instruction count ({InstructionCount}) exceeds the specified MaxProcessingInstructionCount ({MaxProcessingInstructionCount})."
+ " The server will skip existing instructions, rebuild its caches and indexes entirely, adjust its last synced Id"
+ " to the latest found in the database and maintain cache updates based on that Id.",
count, Options.MaxProcessingInstructionCount);
coldboot = true;
}
else
{
//check for how many instructions there are to process, each row contains a count of the number of instructions contained in each
//row so we will sum these numbers to get the actual count.
var count = database.ExecuteScalar<int>("SELECT SUM(instructionCount) FROM umbracoCacheInstruction WHERE id > @lastId", new {lastId = _lastId});
if (count > Options.MaxProcessingInstructionCount)
{
//too many instructions, proceed to cold boot
Logger.Warn<DatabaseServerMessenger,int,int>(
"The instruction count ({InstructionCount}) exceeds the specified MaxProcessingInstructionCount ({MaxProcessingInstructionCount})."
+ " The server will skip existing instructions, rebuild its caches and indexes entirely, adjust its last synced Id"
+ " to the latest found in the database and maintain cache updates based on that Id.",
count, Options.MaxProcessingInstructionCount);
}
coldboot = true;
if (coldboot)
{
// go get the last id in the db and store it
// note: do it BEFORE initializing otherwise some instructions might get lost
// when doing it before, some instructions might run twice - not an issue
var maxId = database.ExecuteScalar<int>("SELECT MAX(id) FROM umbracoCacheInstruction");
//if there is a max currently, or if we've never synced
if (maxId > 0 || _lastId < 0)
SaveLastSynced(maxId);
// execute initializing callbacks
if (Options.InitializingCallbacks != null)
{
foreach (var callback in Options.InitializingCallbacks)
{
callback();
}
}
if (coldboot)
{
// go get the last id in the db and store it
// note: do it BEFORE initializing otherwise some instructions might get lost
// when doing it before, some instructions might run twice - not an issue
var maxId = database.ExecuteScalar<int>("SELECT MAX(id) FROM umbracoCacheInstruction");
//if there is a max currently, or if we've never synced
if (maxId > 0 || _lastId < 0)
SaveLastSynced(maxId);
// execute initializing callbacks
if (Options.InitializingCallbacks != null)
foreach (var callback in Options.InitializingCallbacks)
callback();
}
_initialized = true;
}
return coldboot ? SyncBootState.ColdBoot : SyncBootState.WarmBoot;
}
/// <summary>
@@ -352,7 +361,7 @@ namespace Umbraco.Core.Sync
}
catch (JsonException ex)
{
Logger.Error<DatabaseServerMessenger,int, string>(ex, "Failed to deserialize instructions ({DtoId}: '{DtoInstructions}').",
Logger.Error<DatabaseServerMessenger, int, string>(ex, "Failed to deserialize instructions ({DtoId}: '{DtoInstructions}').",
dto.Id,
dto.Instructions);
@@ -410,11 +419,11 @@ namespace Umbraco.Core.Sync
//}
catch (Exception ex)
{
Logger.Error<DatabaseServerMessenger,int, string> (
ex,
"DISTRIBUTED CACHE IS NOT UPDATED. Failed to execute instructions ({DtoId}: '{DtoInstructions}'). Instruction is being skipped/ignored",
dto.Id,
dto.Instructions);
Logger.Error<DatabaseServerMessenger, int, string>(
ex,
"DISTRIBUTED CACHE IS NOT UPDATED. Failed to execute instructions ({DtoId}: '{DtoInstructions}'). Instruction is being skipped/ignored",
dto.Id,
dto.Instructions);
//we cannot throw here because this invalid instruction will just keep getting processed over and over and errors
// will be thrown over and over. The only thing we can do is ignore and move on.
@@ -548,6 +557,8 @@ namespace Umbraco.Core.Sync
#endregion
public virtual SyncBootState GetSyncBootState() => _getSyncBootState.Value;
#region Notify refreshers
private static ICacheRefresher GetRefresher(Guid id)

10
src/Umbraco.Core/Sync/DatabaseServerMessengerOptions.cs
View File

@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.ComponentModel;
namespace Umbraco.Core.Sync
{
@@ -24,13 +25,8 @@ namespace Umbraco.Core.Sync
/// </summary>
public int MaxProcessingInstructionCount { get; set; }
/// <summary>
/// A list of callbacks that will be invoked if the lastsynced.txt file does not exist.
/// </summary>
/// <remarks>
/// These callbacks will typically be for eg rebuilding the xml cache file, or examine indexes, based on
/// the data in the database to get this particular server node up to date.
/// </remarks>
[Obsolete("This should not be used. If initialization calls need to be invoked on a cold boot, use the ISyncBootStateAccessor.Booting event.")]
[EditorBrowsable(EditorBrowsableState.Never)]
public IEnumerable<Action> InitializingCallbacks { get; set; }
/// <summary>

16
src/Umbraco.Core/Sync/ISyncBootStateAccessor.cs Normal file
View File

@@ -0,0 +1,16 @@
using System;
namespace Umbraco.Core.Sync
{
/// <summary>
/// Retrieve the state of the sync service
/// </summary>
public interface ISyncBootStateAccessor
{
/// <summary>
/// Get the boot state
/// </summary>
/// <returns></returns>
SyncBootState GetSyncBootState();
}
}

21
src/Umbraco.Core/Sync/NonRuntimeLevelBootStateAccessor.cs Normal file
View File

@@ -0,0 +1,21 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Umbraco.Core.Sync
{
/// <summary>
/// Boot state implementation for when umbraco is not in the run state
/// </summary>
public class NonRuntimeLevelBootStateAccessor : ISyncBootStateAccessor
{
public event EventHandler<SyncBootState> Booting;
public SyncBootState GetSyncBootState()
{
return SyncBootState.Unknown;
}
}
}

20
src/Umbraco.Core/Sync/SyncBootState.cs Normal file
View File

@@ -0,0 +1,20 @@
namespace Umbraco.Core.Sync
{
public enum SyncBootState
{
/// <summary>
/// Unknown state. Treat as WarmBoot
/// </summary>
Unknown = 0,
/// <summary>
/// Cold boot. No Sync state
/// </summary>
ColdBoot = 1,
/// <summary>
/// Warm boot. Sync state present
/// </summary>
WarmBoot = 2
}
}

3
src/Umbraco.Core/Umbraco.Core.csproj
View File

@@ -180,6 +180,9 @@
<Compile Include="Migrations\Upgrade\V_8_7_0\MissingDictionaryIndex.cs" />
<Compile Include="Services\IInstallationService.cs" />
<Compile Include="Services\IUpgradeService.cs" />
<Compile Include="Sync\ISyncBootStateAccessor.cs" />
<Compile Include="Sync\NonRuntimeLevelBootStateAccessor.cs" />
<Compile Include="Sync\SyncBootState.cs" />
<Compile Include="SystemLock.cs" />
<Compile Include="Attempt.cs" />
<Compile Include="AttemptOfTResult.cs" />

4
src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs
View File

@@ -17,6 +17,7 @@ using Umbraco.Core.Scoping;
using Umbraco.Core.Services;
using Umbraco.Core.Services.Changes;
using Umbraco.Core.Strings;
using Umbraco.Core.Sync;
using Umbraco.Tests.TestHelpers;
using Umbraco.Tests.Testing.Objects;
using Umbraco.Tests.Testing.Objects.Accessors;
@@ -155,7 +156,8 @@ namespace Umbraco.Tests.PublishedContent
globalSettings,
Mock.Of<IEntityXmlSerializer>(),
Mock.Of<IPublishedModelFactory>(),
new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }));
new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }),
new TestSyncBootStateAccessor(SyncBootState.WarmBoot));
// invariant is the current default
_variationAccesor.VariationContext = new VariationContext();

4
src/Umbraco.Tests/PublishedContent/NuCacheTests.cs
View File

@@ -17,6 +17,7 @@ using Umbraco.Core.Scoping;
using Umbraco.Core.Services;
using Umbraco.Core.Services.Changes;
using Umbraco.Core.Strings;
using Umbraco.Core.Sync;
using Umbraco.Tests.TestHelpers;
using Umbraco.Tests.Testing.Objects;
using Umbraco.Tests.Testing.Objects.Accessors;
@@ -201,7 +202,8 @@ namespace Umbraco.Tests.PublishedContent
globalSettings,
Mock.Of<IEntityXmlSerializer>(),
Mock.Of<IPublishedModelFactory>(),
new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }));
new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }),
new TestSyncBootStateAccessor(SyncBootState.WarmBoot));
// invariant is the current default
_variationAccesor.VariationContext = new VariationContext();

3
src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs
View File

@@ -99,7 +99,8 @@ namespace Umbraco.Tests.Scoping
Factory.GetInstance<IGlobalSettings>(),
Factory.GetInstance<IEntityXmlSerializer>(),
Mock.Of<IPublishedModelFactory>(),
new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }));
new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }),
new TestSyncBootStateAccessor(SyncBootState.WarmBoot));
}
protected UmbracoContext GetUmbracoContextNu(string url, int templateId = 1234, RouteData routeData = null, bool setSingleton = false, IUmbracoSettingsSection umbracoSettings = null, IEnumerable<IUrlProvider> urlProviders = null)

4
src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs
View File

@@ -17,6 +17,7 @@ using Umbraco.Core.PropertyEditors;
using Umbraco.Core.Services;
using Umbraco.Core.Strings;
using Umbraco.Core.Sync;
using Umbraco.Tests.TestHelpers;
using Umbraco.Tests.TestHelpers.Entities;
using Umbraco.Tests.Testing;
using Umbraco.Web.PublishedCache;
@@ -70,7 +71,8 @@ namespace Umbraco.Tests.Services
Factory.GetInstance<IGlobalSettings>(),
Factory.GetInstance<IEntityXmlSerializer>(),
Mock.Of<IPublishedModelFactory>(),
new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }));
new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }),
new TestSyncBootStateAccessor(SyncBootState.WarmBoot));
}
public class LocalServerMessenger : ServerMessengerBase

26
src/Umbraco.Tests/TestHelpers/TestSyncBootStateAccessor.cs Normal file
View File

@@ -0,0 +1,26 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Umbraco.Core.Sync;
namespace Umbraco.Tests.TestHelpers
{
class TestSyncBootStateAccessor : ISyncBootStateAccessor
{
private readonly SyncBootState _syncBootState;
public TestSyncBootStateAccessor(SyncBootState syncBootState)
{
_syncBootState = syncBootState;
}
public event EventHandler<SyncBootState> Booting;
public SyncBootState GetSyncBootState()
{
return _syncBootState;
}
}
}

1
src/Umbraco.Tests/Umbraco.Tests.csproj
View File

@@ -178,6 +178,7 @@
<Compile Include="Services\RedirectUrlServiceTests.cs" />
<Compile Include="Templates\HtmlLocalLinkParserTests.cs" />
<Compile Include="TestHelpers\RandomIdRamDirectory.cs" />
<Compile Include="TestHelpers\TestSyncBootStateAccessor.cs" />
<Compile Include="Testing\Objects\TestDataSource.cs" />
<Compile Include="Published\PublishedSnapshotTestObjects.cs" />
<Compile Include="Published\ModelTypeTests.cs" />

33
src/Umbraco.Web/BatchedDatabaseServerMessenger.cs
View File

@@ -26,6 +26,7 @@ namespace Umbraco.Web
public class BatchedDatabaseServerMessenger : DatabaseServerMessenger
{
private readonly IUmbracoDatabaseFactory _databaseFactory;
private readonly Lazy<SyncBootState> _syncBootState;
[Obsolete("This overload should not be used, enableDistCalls has no effect")]
[EditorBrowsable(EditorBrowsableState.Never)]
@@ -39,28 +40,22 @@ namespace Umbraco.Web
: base(runtime, scopeProvider, sqlContext, proflog, globalSettings, true, options)
{
_databaseFactory = databaseFactory;
}
// invoked by DatabaseServerRegistrarAndMessengerComponent
internal void Startup()
{
UmbracoModule.EndRequest += UmbracoModule_EndRequest;
if (_databaseFactory.CanConnect == false)
_syncBootState = new Lazy<SyncBootState>(() =>
{
Logger.Warn<BatchedDatabaseServerMessenger>("Cannot connect to the database, distributed calls will not be enabled for this server.");
}
else
{
Boot();
}
if (_databaseFactory.CanConnect == false)
{
Logger.Warn<BatchedDatabaseServerMessenger>("Cannot connect to the database, distributed calls will not be enabled for this server.");
return SyncBootState.Unknown;
}
else
{
return base.GetSyncBootState();
}
});
}
private void UmbracoModule_EndRequest(object sender, UmbracoRequestEventArgs e)
{
// will clear the batch - will remain in HttpContext though - that's ok
FlushBatch();
}
// override to deal with database connectivity
public override SyncBootState GetSyncBootState() => _syncBootState.Value;
protected override void DeliverRemote(ICacheRefresher refresher, MessageType messageType, IEnumerable<object> ids = null, string json = null)
{

84
src/Umbraco.Web/Compose/DatabaseServerRegistrarAndMessengerComponent.cs
View File

@@ -4,76 +4,13 @@ using Umbraco.Core;
using Umbraco.Core.Composing;
using Umbraco.Core.Logging;
using Umbraco.Core.Services;
using Umbraco.Core.Services.Changes;
using Umbraco.Core.Sync;
using Umbraco.Examine;
using Umbraco.Web.Cache;
using Umbraco.Web.Routing;
using Umbraco.Web.Scheduling;
using Umbraco.Web.Search;
using Current = Umbraco.Web.Composing.Current;
namespace Umbraco.Web.Compose
{
/// <summary>
/// Ensures that servers are automatically registered in the database, when using the database server registrar.
/// </summary>
/// <remarks>
/// <para>At the moment servers are automatically registered upon first request and then on every
/// request but not more than once per (configurable) period. This really is "for information & debug" purposes so
/// we can look at the table and see what servers are registered - but the info is not used anywhere.</para>
/// <para>Should we actually want to use this, we would need a better and more deterministic way of figuring
/// out the "server address" ie the address to which server-to-server requests should be sent - because it
/// probably is not the "current request address" - especially in multi-domains configurations.</para>
/// </remarks>
[RuntimeLevel(MinLevel = RuntimeLevel.Run)]
// during Initialize / Startup, we end up checking Examine, which needs to be initialized beforehand
// TODO: should not be a strong dependency on "examine" but on an "indexing component"
[ComposeAfter(typeof(ExamineComposer))]
public sealed class DatabaseServerRegistrarAndMessengerComposer : ComponentComposer<DatabaseServerRegistrarAndMessengerComponent>, ICoreComposer
{
public static DatabaseServerMessengerOptions GetDefaultOptions(IFactory factory)
{
var logger = factory.GetInstance<ILogger>();
var indexRebuilder = factory.GetInstance<IndexRebuilder>();
return new DatabaseServerMessengerOptions
{
//These callbacks will be executed if the server has not been synced
// (i.e. it is a new server or the lastsynced.txt file has been removed)
InitializingCallbacks = new Action[]
{
//rebuild the xml cache file if the server is not synced
() =>
{
// rebuild the published snapshot caches entirely, if the server is not synced
// this is equivalent to DistributedCache RefreshAll... but local only
// (we really should have a way to reuse RefreshAll... locally)
// note: refresh all content & media caches does refresh content types too
var svc = Current.PublishedSnapshotService;
svc.Notify(new[] { new DomainCacheRefresher.JsonPayload(0, DomainChangeTypes.RefreshAll) });
svc.Notify(new[] { new ContentCacheRefresher.JsonPayload(0, null, TreeChangeTypes.RefreshAll) }, out _, out _);
svc.Notify(new[] { new MediaCacheRefresher.JsonPayload(0, null, TreeChangeTypes.RefreshAll) }, out _);
},
//rebuild indexes if the server is not synced
// NOTE: This will rebuild ALL indexes including the members, if developers want to target specific
// indexes then they can adjust this logic themselves.
() => { ExamineComponent.RebuildIndexes(indexRebuilder, logger, false, 5000); }
}
};
}
public override void Compose(Composition composition)
{
base.Compose(composition);
composition.SetDatabaseServerMessengerOptions(GetDefaultOptions);
composition.SetServerMessenger<BatchedDatabaseServerMessenger>();
}
}
public sealed class DatabaseServerRegistrarAndMessengerComponent : IComponent
{
@@ -87,14 +24,17 @@ namespace Umbraco.Web.Compose
private readonly BackgroundTaskRunner<IBackgroundTask> _processTaskRunner;
private bool _started;
private IBackgroundTask[] _tasks;
private IndexRebuilder _indexRebuilder;
public DatabaseServerRegistrarAndMessengerComponent(IRuntimeState runtime, IServerRegistrar serverRegistrar, IServerMessenger serverMessenger, IServerRegistrationService registrationService, ILogger logger, IndexRebuilder indexRebuilder)
public DatabaseServerRegistrarAndMessengerComponent(
IRuntimeState runtime,
IServerRegistrar serverRegistrar,
IServerMessenger serverMessenger,
IServerRegistrationService registrationService,
ILogger logger)
{
_runtime = runtime;
_logger = logger;
_registrationService = registrationService;
_indexRebuilder = indexRebuilder;
// create task runner for DatabaseServerRegistrar
_registrar = serverRegistrar as DatabaseServerRegistrar;
@@ -117,15 +57,21 @@ namespace Umbraco.Web.Compose
{
//We will start the whole process when a successful request is made
if (_registrar != null || _messenger != null)
{
UmbracoModule.RouteAttempt += RegisterBackgroundTasksOnce;
// must come last, as it references some _variables
_messenger?.Startup();
UmbracoModule.EndRequest += UmbracoModule_EndRequest;
}
}
public void Terminate()
{ }
private void UmbracoModule_EndRequest(object sender, UmbracoRequestEventArgs e)
{
// will clear the batch - will remain in HttpContext though - that's ok
_messenger?.FlushBatch();
}
/// <summary>
/// Handle when a request is made
/// </summary>

38
src/Umbraco.Web/Compose/DatabaseServerRegistrarAndMessengerComposer.cs Normal file
View File

@@ -0,0 +1,38 @@
using Umbraco.Core;
using Umbraco.Core.Composing;
using Umbraco.Core.Sync;
using Umbraco.Web.Search;
namespace Umbraco.Web.Compose
{
/// <summary>
/// Ensures that servers are automatically registered in the database, when using the database server registrar.
/// </summary>
/// <remarks>
/// <para>At the moment servers are automatically registered upon first request and then on every
/// request but not more than once per (configurable) period. This really is "for information & debug" purposes so
/// we can look at the table and see what servers are registered - but the info is not used anywhere.</para>
/// <para>Should we actually want to use this, we would need a better and more deterministic way of figuring
/// out the "server address" ie the address to which server-to-server requests should be sent - because it
/// probably is not the "current request address" - especially in multi-domains configurations.</para>
/// </remarks>
[RuntimeLevel(MinLevel = RuntimeLevel.Run)]
// TODO: This is legacy, we no longer need to do this but we don't want to change the behavior now
[ComposeAfter(typeof(ExamineComposer))]
public sealed class DatabaseServerRegistrarAndMessengerComposer : ComponentComposer<DatabaseServerRegistrarAndMessengerComponent>, ICoreComposer
{
public static DatabaseServerMessengerOptions GetDefaultOptions(IFactory factory)
{
return new DatabaseServerMessengerOptions();
}
public override void Compose(Composition composition)
{
base.Compose(composition);
composition.SetDatabaseServerMessengerOptions(GetDefaultOptions);
composition.SetServerMessenger<BatchedDatabaseServerMessenger>();
composition.Register<ISyncBootStateAccessor>(factory => factory.GetInstance<IServerMessenger>() as BatchedDatabaseServerMessenger, Lifetime.Singleton);
}
}
}

4
src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs
View File

@@ -1,5 +1,6 @@
using Umbraco.Core;
using Umbraco.Core.Composing;
using Umbraco.Core.Sync;
using Umbraco.Web.PublishedCache.NuCache.DataSource;
namespace Umbraco.Web.PublishedCache.NuCache
@@ -10,6 +11,9 @@ namespace Umbraco.Web.PublishedCache.NuCache
{
base.Compose(composition);
//Overriden on Run state in DatabaseServerRegistrarAndMessengerComposer
composition.Register<ISyncBootStateAccessor, NonRuntimeLevelBootStateAccessor>(Lifetime.Singleton);
// register the NuCache database data source
composition.Register<IDataSource, DatabaseDataSource>();

203
src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs Executable file → Normal file
View File

@@ -24,6 +24,7 @@ using Umbraco.Core.Services;
using Umbraco.Core.Services.Changes;
using Umbraco.Core.Services.Implement;
using Umbraco.Core.Strings;
using Umbraco.Core.Sync;
using Umbraco.Web.Cache;
using Umbraco.Web.Install;
using Umbraco.Web.PublishedCache.NuCache.DataSource;
@@ -35,6 +36,8 @@ namespace Umbraco.Web.PublishedCache.NuCache
internal class PublishedSnapshotService : PublishedSnapshotServiceBase
{
private readonly PublishedSnapshotServiceOptions _options;
private readonly IMainDom _mainDom;
private readonly ServiceContext _serviceContext;
private readonly IPublishedContentTypeFactory _publishedContentTypeFactory;
private readonly IScopeProvider _scopeProvider;
@@ -49,12 +52,13 @@ namespace Umbraco.Web.PublishedCache.NuCache
private readonly IDefaultCultureAccessor _defaultCultureAccessor;
private readonly UrlSegmentProviderCollection _urlSegmentProviders;
// volatile because we read it with no lock
private volatile bool _isReady;
private bool _isReady;
private bool _isReadSet;
private object _isReadyLock;
private readonly ContentStore _contentStore;
private readonly ContentStore _mediaStore;
private readonly SnapDictionary<int, Domain> _domainStore;
private ContentStore _contentStore;
private ContentStore _mediaStore;
private SnapDictionary<int, Domain> _domainStore;
private readonly object _storesLock = new object();
private readonly object _elementsLock = new object();
@@ -63,6 +67,8 @@ namespace Umbraco.Web.PublishedCache.NuCache
private bool _localContentDbExists;
private bool _localMediaDbExists;
private readonly ISyncBootStateAccessor _syncBootStateAccessor;
// define constant - determines whether to use cache when previewing
// to store eg routes, property converted values, anything - caching
// means faster execution, but uses memory - not sure if we want it
@@ -81,12 +87,16 @@ namespace Umbraco.Web.PublishedCache.NuCache
IDataSource dataSource, IGlobalSettings globalSettings,
IEntityXmlSerializer entitySerializer,
IPublishedModelFactory publishedModelFactory,
UrlSegmentProviderCollection urlSegmentProviders)
UrlSegmentProviderCollection urlSegmentProviders,
ISyncBootStateAccessor syncBootStateAccessor)
: base(publishedSnapshotAccessor, variationContextAccessor)
{
//if (Interlocked.Increment(ref _singletonCheck) > 1)
// throw new Exception("Singleton must be instantiated only once!");
_options = options;
_mainDom = mainDom;
_serviceContext = serviceContext;
_publishedContentTypeFactory = publishedContentTypeFactory;
_dataSource = dataSource;
@@ -99,6 +109,8 @@ namespace Umbraco.Web.PublishedCache.NuCache
_globalSettings = globalSettings;
_urlSegmentProviders = urlSegmentProviders;
_syncBootStateAccessor = syncBootStateAccessor;
// we need an Xml serializer here so that the member cache can support XPath,
// for members this is done by navigating the serialized-to-xml member
_entitySerializer = entitySerializer;
@@ -117,41 +129,6 @@ namespace Umbraco.Web.PublishedCache.NuCache
if (runtime.Level != RuntimeLevel.Run)
return;
// lock this entire call, we only want a single thread to be accessing the stores at once and within
// the call below to mainDom.Register, a callback may occur on a threadpool thread to MainDomRelease
// at the same time as we are trying to write to the stores. MainDomRelease also locks on _storesLock so
// it will not be able to close the stores until we are done populating (if the store is empty)
lock (_storesLock)
{
if (options.IgnoreLocalDb == false)
{
var registered = mainDom.Register(MainDomRegister, MainDomRelease);
// stores are created with a db so they can write to it, but they do not read from it,
// stores need to be populated, happens in OnResolutionFrozen which uses _localDbExists to
// figure out whether it can read the databases or it should populate them from sql
_logger.Info<PublishedSnapshotService,bool>("Creating the content store, localContentDbExists? {LocalContentDbExists}", _localContentDbExists);
_contentStore = new ContentStore(publishedSnapshotAccessor, variationContextAccessor, logger, _localContentDb);
_logger.Info<PublishedSnapshotService,bool>("Creating the media store, localMediaDbExists? {LocalMediaDbExists}", _localMediaDbExists);
_mediaStore = new ContentStore(publishedSnapshotAccessor, variationContextAccessor, logger, _localMediaDb);
}
else
{
_logger.Info<PublishedSnapshotService>("Creating the content store (local db ignored)");
_contentStore = new ContentStore(publishedSnapshotAccessor, variationContextAccessor, logger);
_logger.Info<PublishedSnapshotService>("Creating the media store (local db ignored)");
_mediaStore = new ContentStore(publishedSnapshotAccessor, variationContextAccessor, logger);
}
_domainStore = new SnapDictionary<int, Domain>();
LoadCachesOnStartup();
}
Guid GetUid(ContentStore store, int id) => store.LiveSnapshot.Get(id)?.Uid ?? default;
int GetId(ContentStore store, Guid uid) => store.LiveSnapshot.Get(uid)?.Id ?? default;
if (idkMap != null)
{
idkMap.SetMapper(UmbracoObjectTypes.Document, id => GetUid(_contentStore, id), uid => GetId(_contentStore, uid));
@@ -159,6 +136,18 @@ namespace Umbraco.Web.PublishedCache.NuCache
}
}
private int GetId(ContentStore store, Guid uid)
{
EnsureCaches();
return store.LiveSnapshot.Get(uid)?.Id ?? default;
}
private Guid GetUid(ContentStore store, int id)
{
EnsureCaches();
return store.LiveSnapshot.Get(id)?.Uid ?? default;
}
/// <summary>
/// Install phase of <see cref="IMainDom"/>
/// </summary>
@@ -210,47 +199,82 @@ namespace Umbraco.Web.PublishedCache.NuCache
}
/// <summary>
/// Populates the stores
/// Lazily populates the stores only when they are first requested
/// </summary>
/// <remarks>This is called inside of a lock for _storesLock</remarks>
private void LoadCachesOnStartup()
{
var okContent = false;
var okMedia = false;
try
internal void EnsureCaches() => LazyInitializer.EnsureInitialized(
ref _isReady,
ref _isReadSet,
ref _isReadyLock,
() =>
{
if (_localContentDbExists)
// lock this entire call, we only want a single thread to be accessing the stores at once and within
// the call below to mainDom.Register, a callback may occur on a threadpool thread to MainDomRelease
// at the same time as we are trying to write to the stores. MainDomRelease also locks on _storesLock so
// it will not be able to close the stores until we are done populating (if the store is empty)
lock (_storesLock)
{
okContent = LockAndLoadContent(scope => LoadContentFromLocalDbLocked(true));
if (!okContent)
_logger.Warn<PublishedSnapshotService>("Loading content from local db raised warnings, will reload from database.");
if (!_options.IgnoreLocalDb)
{
var registered = _mainDom.Register(MainDomRegister, MainDomRelease);
// stores are created with a db so they can write to it, but they do not read from it,
// stores need to be populated, happens in OnResolutionFrozen which uses _localDbExists to
// figure out whether it can read the databases or it should populate them from sql
_logger.Info<PublishedSnapshotService, bool>("Creating the content store, localContentDbExists? {LocalContentDbExists}", _localContentDbExists);
_contentStore = new ContentStore(PublishedSnapshotAccessor, VariationContextAccessor, _logger, _localContentDb);
_logger.Info<PublishedSnapshotService, bool>("Creating the media store, localMediaDbExists? {LocalMediaDbExists}", _localMediaDbExists);
_mediaStore = new ContentStore(PublishedSnapshotAccessor, VariationContextAccessor, _logger, _localMediaDb);
}
else
{
_logger.Info<PublishedSnapshotService>("Creating the content store (local db ignored)");
_contentStore = new ContentStore(PublishedSnapshotAccessor, VariationContextAccessor, _logger);
_logger.Info<PublishedSnapshotService>("Creating the media store (local db ignored)");
_mediaStore = new ContentStore(PublishedSnapshotAccessor, VariationContextAccessor, _logger);
}
_domainStore = new SnapDictionary<int, Domain>();
SyncBootState bootState = _syncBootStateAccessor.GetSyncBootState();
var okContent = false;
var okMedia = false;
try
{
if (bootState != SyncBootState.ColdBoot && _localContentDbExists)
{
okContent = LockAndLoadContent(scope => LoadContentFromLocalDbLocked(true));
if (!okContent)
_logger.Warn<PublishedSnapshotService>("Loading content from local db raised warnings, will reload from database.");
}
if (bootState != SyncBootState.ColdBoot && _localMediaDbExists)
{
okMedia = LockAndLoadMedia(scope => LoadMediaFromLocalDbLocked(true));
if (!okMedia)
_logger.Warn<PublishedSnapshotService>("Loading media from local db raised warnings, will reload from database.");
}
if (!okContent)
LockAndLoadContent(scope => LoadContentFromDatabaseLocked(scope, true));
if (!okMedia)
LockAndLoadMedia(scope => LoadMediaFromDatabaseLocked(scope, true));
LockAndLoadDomains();
}
catch (Exception ex)
{
_logger.Fatal<PublishedSnapshotService>(ex, "Panic, exception while loading cache data.");
throw;
}
// finally, cache is ready!
return true;
}
if (_localMediaDbExists)
{
okMedia = LockAndLoadMedia(scope => LoadMediaFromLocalDbLocked(true));
if (!okMedia)
_logger.Warn<PublishedSnapshotService>("Loading media from local db raised warnings, will reload from database.");
}
if (!okContent)
LockAndLoadContent(scope => LoadContentFromDatabaseLocked(scope, true));
if (!okMedia)
LockAndLoadMedia(scope => LoadMediaFromDatabaseLocked(scope, true));
LockAndLoadDomains();
}
catch (Exception ex)
{
_logger.Fatal<PublishedSnapshotService>(ex, "Panic, exception while loading cache data.");
throw;
}
// finally, cache is ready!
_isReady = true;
}
});
private void InitializeRepositoryEvents()
{
@@ -1133,9 +1157,13 @@ namespace Umbraco.Web.PublishedCache.NuCache
public override IPublishedSnapshot CreatePublishedSnapshot(string previewToken)
{
EnsureCaches();
// no cache, no joy
if (_isReady == false)
if (Volatile.Read(ref _isReady) == false)
{
throw new InvalidOperationException("The published snapshot service has not properly initialized.");
}
var preview = previewToken.IsNullOrWhiteSpace() == false;
return new PublishedSnapshot(this, preview);
@@ -1146,6 +1174,8 @@ namespace Umbraco.Web.PublishedCache.NuCache
// even though the underlying elements may not change (store snapshots)
public PublishedSnapshot.PublishedSnapshotElements GetElements(bool previewDefault)
{
EnsureCaches();
// note: using ObjectCacheAppCache for elements and snapshot caches
// is not recommended because it creates an inner MemoryCache which is a heavy
// thing - better use a dictionary-based cache which "just" creates a concurrent
@@ -1792,6 +1822,8 @@ AND cmsContentNu.nodeId IS NULL
public void Collect()
{
EnsureCaches();
var contentCollect = _contentStore.CollectAsync();
var mediaCollect = _mediaStore.CollectAsync();
System.Threading.Tasks.Task.WaitAll(contentCollect, mediaCollect);
@@ -1801,8 +1833,17 @@ AND cmsContentNu.nodeId IS NULL
#region Internals/Testing
internal ContentStore GetContentStore() => _contentStore;
internal ContentStore GetMediaStore() => _mediaStore;
internal ContentStore GetContentStore()
{
EnsureCaches();
return _contentStore;
}
internal ContentStore GetMediaStore()
{
EnsureCaches();
return _mediaStore;
}
#endregion
}

45
src/Umbraco.Web/Search/ExamineFinalComponent.cs
View File

@@ -3,6 +3,8 @@ using Umbraco.Core.Logging;
using Umbraco.Examine;
using Umbraco.Core.Composing;
using Umbraco.Core;
using Umbraco.Core.Sync;
using Umbraco.Web.Routing;
namespace Umbraco.Web.Search
{
@@ -16,23 +18,50 @@ namespace Umbraco.Web.Search
private readonly IExamineManager _examineManager;
BackgroundIndexRebuilder _indexRebuilder;
private readonly IMainDom _mainDom;
public ExamineFinalComponent(IProfilingLogger logger, IExamineManager examineManager, BackgroundIndexRebuilder indexRebuilder, IMainDom mainDom)
private readonly ISyncBootStateAccessor _syncBootStateAccessor;
private readonly object _locker = new object();
private bool _initialized = false;
public ExamineFinalComponent(IProfilingLogger logger, IExamineManager examineManager, BackgroundIndexRebuilder indexRebuilder, IMainDom mainDom, ISyncBootStateAccessor syncBootStateAccessor)
{
_logger = logger;
_examineManager = examineManager;
_indexRebuilder = indexRebuilder;
_mainDom = mainDom;
_syncBootStateAccessor = syncBootStateAccessor;
}
private void UmbracoModule_RouteAttempt(object sender, RoutableAttemptEventArgs e)
{
if (!_initialized)
{
lock (_locker)
{
// double check lock, we must only do this once
if (!_initialized)
{
_initialized = true;
UmbracoModule.RouteAttempt -= UmbracoModule_RouteAttempt;
if (!_mainDom.IsMainDom) return;
var bootState = _syncBootStateAccessor.GetSyncBootState();
_examineManager.ConfigureIndexes(_mainDom, _logger);
// if it's a cold boot, rebuild all indexes including non-empty ones
// delay one minute since a cold boot also triggers nucache rebuilds
_indexRebuilder.RebuildIndexes(bootState != SyncBootState.ColdBoot, 60000);
}
}
}
}
public void Initialize()
{
if (!_mainDom.IsMainDom) return;
_examineManager.ConfigureIndexes(_mainDom, _logger);
// TODO: Instead of waiting 5000 ms, we could add an event handler on to fulfilling the first request, then start?
_indexRebuilder.RebuildIndexes(true, 5000);
UmbracoModule.RouteAttempt += UmbracoModule_RouteAttempt;
}
public void Terminate()

1
src/Umbraco.Web/Umbraco.Web.csproj
View File

@@ -132,6 +132,7 @@
<Compile Include="Compose\BackOfficeUserAuditEventsComposer.cs" />
<Compile Include="Compose\BlockEditorComponent.cs" />
<Compile Include="Compose\BlockEditorComposer.cs" />
<Compile Include="Compose\DatabaseServerRegistrarAndMessengerComposer.cs" />
<Compile Include="Compose\NestedContentPropertyComponent.cs" />
<Compile Include="Compose\NotificationsComposer.cs" />
<Compile Include="Compose\PublicAccessComposer.cs" />