yv01p/Umbraco-CMS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'v8/bugfix/8893-examine-startup' into v8/feature/nucache-perf

Browse Source 
# Conflicts:
#	src/Umbraco.Core/Sync/DatabaseServerMessenger.cs
#	src/Umbraco.Core/Sync/ISyncBootStateAccessor.cs
#	src/Umbraco.Core/Sync/NonRuntimeLevelBootStateAccessor.cs
#	src/Umbraco.Core/Sync/SyncBootState.cs
#	src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs
#	src/Umbraco.Tests/PublishedContent/NuCacheTests.cs
#	src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs
#	src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs
#	src/Umbraco.Tests/TestHelpers/TestSyncBootStateAccessor.cs
#	src/Umbraco.Web/Compose/DatabaseServerRegistrarAndMessengerComponent.cs
#	src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs
#	src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs
This commit is contained in:
Shannon
2021-05-24 13:55:07 -07:00
parent eb2f1e55fd eba3c82a86
commit a9508886bf
17 changed files with 305 additions and 289 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs
View File

@@ -13,6 +13,9 @@ namespace Umbraco.Web.PublishedCache.NuCache
{
base.Compose(composition);
//Overriden on Run state in DatabaseServerRegistrarAndMessengerComposer
composition.Register<ISyncBootStateAccessor, NonRuntimeLevelBootStateAccessor>(Lifetime.Singleton);
var serializer = ConfigurationManager.AppSettings[NuCacheSerializerComponent.Nucache_Serializer_Key];
if (serializer != "MsgPack")
{

195
src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs
View File

@@ -4,6 +4,7 @@ using System.Configuration;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Threading;
using CSharpTest.Net.Collections;
using Umbraco.Core;
using Umbraco.Core.Cache;
@@ -33,6 +34,8 @@ namespace Umbraco.Web.PublishedCache.NuCache
internal class PublishedSnapshotService : PublishedSnapshotServiceBase
{
private readonly PublishedSnapshotServiceOptions _options;
private readonly IMainDom _mainDom;
private readonly ServiceContext _serviceContext;
private readonly IPublishedContentTypeFactory _publishedContentTypeFactory;
private readonly IScopeProvider _scopeProvider;
@@ -49,12 +52,13 @@ namespace Umbraco.Web.PublishedCache.NuCache
private readonly IContentCacheDataSerializerFactory _contentCacheDataSerializerFactory;
private readonly ContentDataSerializer _contentDataSerializer;
// volatile because we read it with no lock
private volatile bool _isReady;
private bool _isReady;
private bool _isReadSet;
private object _isReadyLock;
private readonly ContentStore _contentStore;
private readonly ContentStore _mediaStore;
private readonly SnapDictionary<int, Domain> _domainStore;
private ContentStore _contentStore;
private ContentStore _mediaStore;
private SnapDictionary<int, Domain> _domainStore;
private readonly object _storesLock = new object();
private readonly object _elementsLock = new object();
@@ -89,9 +93,12 @@ namespace Umbraco.Web.PublishedCache.NuCache
ContentDataSerializer contentDataSerializer = null)
: base(publishedSnapshotAccessor, variationContextAccessor)
{
//if (Interlocked.Increment(ref _singletonCheck) > 1)
// throw new Exception("Singleton must be instantiated only once!");
_options = options;
_mainDom = mainDom;
_serviceContext = serviceContext;
_publishedContentTypeFactory = publishedContentTypeFactory;
_dataSource = dataSource;
@@ -108,6 +115,8 @@ namespace Umbraco.Web.PublishedCache.NuCache
_syncBootStateAccessor = syncBootStateAccessor;
_syncBootStateAccessor = syncBootStateAccessor;
// we need an Xml serializer here so that the member cache can support XPath,
// for members this is done by navigating the serialized-to-xml member
_entitySerializer = entitySerializer;
@@ -126,41 +135,6 @@ namespace Umbraco.Web.PublishedCache.NuCache
if (runtime.Level != RuntimeLevel.Run)
return;
// lock this entire call, we only want a single thread to be accessing the stores at once and within
// the call below to mainDom.Register, a callback may occur on a threadpool thread to MainDomRelease
// at the same time as we are trying to write to the stores. MainDomRelease also locks on _storesLock so
// it will not be able to close the stores until we are done populating (if the store is empty)
lock (_storesLock)
{
if (options.IgnoreLocalDb == false)
{
var registered = mainDom.Register(MainDomRegister, MainDomRelease);
// stores are created with a db so they can write to it, but they do not read from it,
// stores need to be populated, happens in OnResolutionFrozen which uses _localDbExists to
// figure out whether it can read the databases or it should populate them from sql
_logger.Info<PublishedSnapshotService,bool>("Creating the content store, localContentDbExists? {LocalContentDbExists}", _localContentDbExists);
_contentStore = new ContentStore(publishedSnapshotAccessor, variationContextAccessor, logger, _localContentDb);
_logger.Info<PublishedSnapshotService,bool>("Creating the media store, localMediaDbExists? {LocalMediaDbExists}", _localMediaDbExists);
_mediaStore = new ContentStore(publishedSnapshotAccessor, variationContextAccessor, logger, _localMediaDb);
}
else
{
_logger.Info<PublishedSnapshotService>("Creating the content store (local db ignored)");
_contentStore = new ContentStore(publishedSnapshotAccessor, variationContextAccessor, logger);
_logger.Info<PublishedSnapshotService>("Creating the media store (local db ignored)");
_mediaStore = new ContentStore(publishedSnapshotAccessor, variationContextAccessor, logger);
}
_domainStore = new SnapDictionary<int, Domain>();
LoadCachesOnStartup();
}
Guid GetUid(ContentStore store, int id) => store.LiveSnapshot.Get(id)?.Uid ?? default;
int GetId(ContentStore store, Guid uid) => store.LiveSnapshot.Get(uid)?.Id ?? default;
if (idkMap != null)
{
idkMap.SetMapper(UmbracoObjectTypes.Document, id => GetUid(_contentStore, id), uid => GetId(_contentStore, uid));
@@ -168,6 +142,18 @@ namespace Umbraco.Web.PublishedCache.NuCache
}
}
private int GetId(ContentStore store, Guid uid)
{
EnsureCaches();
return store.LiveSnapshot.Get(uid)?.Id ?? default;
}
private Guid GetUid(ContentStore store, int id)
{
EnsureCaches();
return store.LiveSnapshot.Get(id)?.Uid ?? default;
}
/// <summary>
/// Install phase of <see cref="IMainDom"/>
/// </summary>
@@ -219,52 +205,82 @@ namespace Umbraco.Web.PublishedCache.NuCache
}
/// <summary>
/// Populates the stores
/// Lazily populates the stores only when they are first requested
/// </summary>
/// <remarks>This is called inside of a lock for _storesLock</remarks>
private void LoadCachesOnStartup()
{
var okContent = false;
var okMedia = false;
if (_syncBootStateAccessor.GetSyncBootState() == SyncBootState.ColdBoot)
internal void EnsureCaches() => LazyInitializer.EnsureInitialized(
ref _isReady,
ref _isReadSet,
ref _isReadyLock,
() =>
{
_logger.Info<PublishedSnapshotService>("Sync Service is in a Cold Boot state. Skip LoadCachesOnStartup as the Sync Service will trigger a full reload");
_isReady = true;
return;
}
try
{
if (_localContentDbExists)
// lock this entire call, we only want a single thread to be accessing the stores at once and within
// the call below to mainDom.Register, a callback may occur on a threadpool thread to MainDomRelease
// at the same time as we are trying to write to the stores. MainDomRelease also locks on _storesLock so
// it will not be able to close the stores until we are done populating (if the store is empty)
lock (_storesLock)
{
okContent = LockAndLoadContent(scope => LoadContentFromLocalDbLocked(true));
if (!okContent)
_logger.Warn<PublishedSnapshotService>("Loading content from local db raised warnings, will reload from database.");
}
if (!_options.IgnoreLocalDb)
{
var registered = _mainDom.Register(MainDomRegister, MainDomRelease);
if (_localMediaDbExists)
{
okMedia = LockAndLoadMedia(scope => LoadMediaFromLocalDbLocked(true));
if (!okMedia)
_logger.Warn<PublishedSnapshotService>("Loading media from local db raised warnings, will reload from database.");
}
// stores are created with a db so they can write to it, but they do not read from it,
// stores need to be populated, happens in OnResolutionFrozen which uses _localDbExists to
// figure out whether it can read the databases or it should populate them from sql
_logger.Info<PublishedSnapshotService, bool>("Creating the content store, localContentDbExists? {LocalContentDbExists}", _localContentDbExists);
_contentStore = new ContentStore(PublishedSnapshotAccessor, VariationContextAccessor, _logger, _localContentDb);
_logger.Info<PublishedSnapshotService, bool>("Creating the media store, localMediaDbExists? {LocalMediaDbExists}", _localMediaDbExists);
_mediaStore = new ContentStore(PublishedSnapshotAccessor, VariationContextAccessor, _logger, _localMediaDb);
}
else
{
_logger.Info<PublishedSnapshotService>("Creating the content store (local db ignored)");
_contentStore = new ContentStore(PublishedSnapshotAccessor, VariationContextAccessor, _logger);
_logger.Info<PublishedSnapshotService>("Creating the media store (local db ignored)");
_mediaStore = new ContentStore(PublishedSnapshotAccessor, VariationContextAccessor, _logger);
}
_domainStore = new SnapDictionary<int, Domain>();
SyncBootState bootState = _syncBootStateAccessor.GetSyncBootState();
var okContent = false;
var okMedia = false;
try
{
if (bootState != SyncBootState.ColdBoot && _localContentDbExists)
{
okContent = LockAndLoadContent(scope => LoadContentFromLocalDbLocked(true));
if (!okContent)
_logger.Warn<PublishedSnapshotService>("Loading content from local db raised warnings, will reload from database.");
}
if (bootState != SyncBootState.ColdBoot && _localMediaDbExists)
{
okMedia = LockAndLoadMedia(scope => LoadMediaFromLocalDbLocked(true));
if (!okMedia)
_logger.Warn<PublishedSnapshotService>("Loading media from local db raised warnings, will reload from database.");
}
if (!okContent)
LockAndLoadContent(scope => LoadContentFromDatabaseLocked(scope, true));
if (!okContent)
LockAndLoadContent(scope => LoadContentFromDatabaseLocked(scope, true));
if (!okMedia)
LockAndLoadMedia(scope => LoadMediaFromDatabaseLocked(scope, true));
if (!okMedia)
LockAndLoadMedia(scope => LoadMediaFromDatabaseLocked(scope, true));
LockAndLoadDomains();
}
catch (Exception ex)
{
_logger.Fatal<PublishedSnapshotService>(ex, "Panic, exception while loading cache data.");
throw;
}
LockAndLoadDomains();
}
catch (Exception ex)
{
_logger.Fatal<PublishedSnapshotService>(ex, "Panic, exception while loading cache data.");
throw;
}
// finally, cache is ready!
_isReady = true;
}
// finally, cache is ready!
return true;
}
});
private void InitializeRepositoryEvents()
{
@@ -1146,9 +1162,13 @@ namespace Umbraco.Web.PublishedCache.NuCache
public override IPublishedSnapshot CreatePublishedSnapshot(string previewToken)
{
EnsureCaches();
// no cache, no joy
if (_isReady == false)
if (Volatile.Read(ref _isReady) == false)
{
throw new InvalidOperationException("The published snapshot service has not properly initialized.");
}
var preview = previewToken.IsNullOrWhiteSpace() == false;
return new PublishedSnapshot(this, preview);
@@ -1159,6 +1179,8 @@ namespace Umbraco.Web.PublishedCache.NuCache
// even though the underlying elements may not change (store snapshots)
public PublishedSnapshot.PublishedSnapshotElements GetElements(bool previewDefault)
{
EnsureCaches();
// note: using ObjectCacheAppCache for elements and snapshot caches
// is not recommended because it creates an inner MemoryCache which is a heavy
// thing - better use a dictionary-based cache which "just" creates a concurrent
@@ -1826,6 +1848,8 @@ AND cmsContentNu.nodeId IS NULL
public void Collect()
{
EnsureCaches();
var contentCollect = _contentStore.CollectAsync();
var mediaCollect = _mediaStore.CollectAsync();
System.Threading.Tasks.Task.WaitAll(contentCollect, mediaCollect);
@@ -1835,8 +1859,17 @@ AND cmsContentNu.nodeId IS NULL
#region Internals/Testing
internal ContentStore GetContentStore() => _contentStore;
internal ContentStore GetMediaStore() => _mediaStore;
internal ContentStore GetContentStore()
{
EnsureCaches();
return _contentStore;
}
internal ContentStore GetMediaStore()
{
EnsureCaches();
return _mediaStore;
}
#endregion
}