From e9d4b607bb5cd68b2a41adb55a2f411d431b538a Mon Sep 17 00:00:00 2001 From: Shannon Date: Thu, 15 Aug 2019 19:05:43 +1000 Subject: [PATCH] Adds POC for fast building of in-memory nucache from persisted files --- .../PublishedContent/NuCacheChildrenTests.cs | 2 +- .../PublishedContent/NuCacheTests.cs | 2 +- .../Scoping/ScopedNuCacheTests.cs | 2 +- .../ContentTypeServiceVariantsTests.cs | 2 +- .../PublishedCache/NuCache/ContentStore.cs | 146 +++++++++++++----- .../NuCache/PublishedSnapshotService.cs | 119 ++++++-------- .../PublishedCache/NuCache/Snap/LinkedNode.cs | 6 + 7 files changed, 167 insertions(+), 112 deletions(-) diff --git a/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs b/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs index eb6e7725c0..2eb3c280b2 100644 --- a/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs +++ b/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs @@ -133,7 +133,7 @@ namespace Umbraco.Tests.PublishedContent null, _snapshotAccessor, _variationAccesor, - Mock.Of(), + Mock.Of(), scopeProvider, Mock.Of(), Mock.Of(), diff --git a/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs b/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs index 399b0c1342..9f20f42993 100644 --- a/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs +++ b/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs @@ -178,7 +178,7 @@ namespace Umbraco.Tests.PublishedContent null, new TestPublishedSnapshotAccessor(), _variationAccesor, - Mock.Of(), + Mock.Of(), scopeProvider, Mock.Of(), Mock.Of(), diff --git a/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs b/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs index c974fed99e..8c14ce8219 100644 --- a/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs +++ b/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs @@ -92,7 +92,7 @@ namespace Umbraco.Tests.Scoping null, publishedSnapshotAccessor, Mock.Of(), - Logger, + ProfilingLogger, ScopeProvider, documentRepository, mediaRepository, memberRepository, DefaultCultureAccessor, diff --git a/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs b/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs index 956de186be..a9e7a86bf7 100644 --- a/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs +++ b/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs @@ -64,7 +64,7 @@ namespace Umbraco.Tests.Services null, publishedSnapshotAccessor, Mock.Of(), - Logger, + ProfilingLogger, ScopeProvider, documentRepository, mediaRepository, memberRepository, DefaultCultureAccessor, diff --git a/src/Umbraco.Web/PublishedCache/NuCache/ContentStore.cs b/src/Umbraco.Web/PublishedCache/NuCache/ContentStore.cs index 8e2cf7bc3c..ec8614f074 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/ContentStore.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/ContentStore.cs @@ -392,7 +392,7 @@ namespace Umbraco.Web.PublishedCache.NuCache var visited = new List(); foreach (var kit in kits.Where(x => refreshedIdsA.Contains(x.ContentTypeId) && - BuildKit(x))) + BuildKit(x, out _))) { // replacing the node: must preserve the parents var node = GetHead(_contentNodes, kit.Node.Id)?.Value; @@ -466,10 +466,11 @@ namespace Umbraco.Web.PublishedCache.NuCache } } - private bool BuildKit(ContentNodeKit kit) + private bool BuildKit(ContentNodeKit kit, out LinkedNode parent) { // make sure parent exists - if (!ParentExistsLocked(kit)) + parent = GetParentLink(kit.Node); + if (parent == null) { _logger.Warn($"Skip item id={kit.Node.Id}, could not find parent id={kit.Node.ParentContentId}."); return false; @@ -531,7 +532,7 @@ namespace Umbraco.Web.PublishedCache.NuCache _contentNodes.TryGetValue(kit.Node.Id, out var link); var existing = link?.Value; - if (!BuildKit(kit)) + if (!BuildKit(kit, out var parent)) return false; // moving? @@ -549,7 +550,7 @@ namespace Umbraco.Web.PublishedCache.NuCache if (existing == null) { // new, add to parent - AddNodeLocked(kit.Node); + AddNodeLocked(kit.Node, parent); } else if (moving || existing.SortOrder != kit.Node.SortOrder) { @@ -581,7 +582,74 @@ namespace Umbraco.Web.PublishedCache.NuCache _root.Value.FirstChildContentId = -1; } - // IMPORTANT kits must be sorted out by LEVEL + /// + /// Builds all kits without any sorting or generation checks + /// + /// + /// + /// + /// This requires that the collection is sorted by Level + Sort Order. + /// This should be used only on a site startup as the first generations. + /// + internal bool SetAllFastSorted(IEnumerable kits) + { + var lockInfo = new WriteLockInfo(); + var ok = true; + try + { + Lock(lockInfo); + + ClearLocked(_contentNodes); + ClearRootLocked(); + + //these are ordered by level + sort order + + // The name of the game here is to populate each kit's + // FirstChildContentId + // NextSiblingContentId + + ContentNode prev = null; + var currLevel = 0; + + foreach (var kit in kits) + { + if (currLevel != kit.Node.Level) + { + prev = null; //reset since we're on a new level + currLevel = kit.Node.Level; + } + + if (!BuildKit(kit, out var parentLink)) + { + ok = false; + continue; // skip that one + } + + _logger.Debug($"Set {kit.Node.Id} with parent {kit.Node.ParentContentId}"); + SetValueLocked(_contentNodes, kit.Node.Id, kit.Node); + + //if the parent's FirstChildContentId isn't set, then it must be the current one + if (parentLink.Value.FirstChildContentId < 0) + parentLink.Value.FirstChildContentId = kit.Node.Id; + + //if there is a previous one on the same level then set it's next sibling id to the current oen + if (prev != null) + prev.NextSiblingContentId = kit.Node.Id; + + //store the prev + prev = kit.Node; + + _xmap[kit.Node.Uid] = kit.Node.Id; + } + } + finally + { + Release(lockInfo); + } + + return ok; + } + public bool SetAll(IEnumerable kits, bool fromLocalDb = false) { var lockInfo = new WriteLockInfo(); @@ -599,7 +667,7 @@ namespace Umbraco.Web.PublishedCache.NuCache foreach (var kit in kits) { - if (!BuildKit(kit)) + if (!BuildKit(kit, out var parent)) { ok = false; continue; // skip that one @@ -609,7 +677,7 @@ namespace Umbraco.Web.PublishedCache.NuCache // don't refresh _localDb if we are reading from _localDb if (!fromLocalDb && _localDb != null) RegisterChange(kit.Node.Id, kit); - AddNodeLocked(kit.Node); + AddNodeLocked(kit.Node, parent); _xmap[kit.Node.Uid] = kit.Node.Id; } @@ -645,14 +713,14 @@ namespace Umbraco.Web.PublishedCache.NuCache // now add them all back foreach (var kit in kits) { - if (!BuildKit(kit)) + if (!BuildKit(kit, out var parent)) { ok = false; continue; // skip that one } SetValueLocked(_contentNodes, kit.Node.Id, kit.Node); if (_localDb != null) RegisterChange(kit.Node.Id, kit); - AddNodeLocked(kit.Node); + AddNodeLocked(kit.Node, parent); _xmap[kit.Node.Uid] = kit.Node.Id; } @@ -712,35 +780,49 @@ namespace Umbraco.Web.PublishedCache.NuCache var id = content.FirstChildContentId; while (id > 0) { - var link = GetLinkedNode(id, "child"); + var link = GetRequiredLinkedNode(id, "child"); ClearBranchLocked(link.Value); id = link.Value.NextSiblingContentId; } } - // gets the link node - // throws (panic) if not found, or no value - private LinkedNode GetLinkedNode(int id, string description) + /// + /// Gets the link node and if it doesn't exist throw a + /// + /// + /// + /// + private LinkedNode GetRequiredLinkedNode(int id, string description) { if (_contentNodes.TryGetValue(id, out var link) && link.Value != null) return link; - throw new Exception($"panic: failed to get {description} with id={id}"); + throw new PanicException($"panic: failed to get {description} with id={id}"); } private LinkedNode GetParentLink(ContentNode content) { - _contentNodes.TryGetValue(content.ParentContentId, out var link); // else null - //if (link == null || link.Value == null) - // throw new Exception("Panic: parent not found."); + if (content.ParentContentId < 0) return _root; + + _contentNodes.TryGetValue(content.ParentContentId, out var link); return link; } + /// + /// Gets the linked parent node and if it doesn't exist throw a + /// + /// + /// + private LinkedNode GetRequiredParentLink(ContentNode content) + { + return content.ParentContentId < 0 ? _root : GetRequiredLinkedNode(content.ParentContentId, "parent"); + } + private void RemoveNodeLocked(ContentNode content) { var parentLink = content.ParentContentId < 0 ? _root - : GetLinkedNode(content.ParentContentId, "parent"); + : GetRequiredLinkedNode(content.ParentContentId, "parent"); var parent = parentLink.Value; @@ -757,10 +839,10 @@ namespace Umbraco.Web.PublishedCache.NuCache else { // iterate children until the previous child - var link = GetLinkedNode(parent.FirstChildContentId, "first child"); + var link = GetRequiredLinkedNode(parent.FirstChildContentId, "first child"); while (link.Value.NextSiblingContentId != content.Id) - link = GetLinkedNode(link.Value.NextSiblingContentId, "next child"); + link = GetRequiredLinkedNode(link.Value.NextSiblingContentId, "next child"); // clone the previous child and replace next child var prevChild = GenCloneLocked(link); @@ -768,14 +850,6 @@ namespace Umbraco.Web.PublishedCache.NuCache } } - private bool ParentExistsLocked(ContentNodeKit kit) - { - if (kit.Node.ParentContentId < 0) - return true; - var link = GetParentLink(kit.Node); - return link?.Value != null; - } - private bool ParentPublishedLocked(ContentNodeKit kit) { if (kit.Node.ParentContentId < 0) @@ -801,11 +875,9 @@ namespace Umbraco.Web.PublishedCache.NuCache return node; } - private void AddNodeLocked(ContentNode content) + private void AddNodeLocked(ContentNode content, LinkedNode parentLink = null) { - var parentLink = content.ParentContentId < 0 - ? _root - : GetLinkedNode(content.ParentContentId, "parent"); + parentLink = parentLink ?? GetRequiredParentLink(content); var parent = parentLink.Value; @@ -818,10 +890,11 @@ namespace Umbraco.Web.PublishedCache.NuCache } // get parent's first child - var childLink = GetLinkedNode(parent.FirstChildContentId, "first child"); + var childLink = GetRequiredLinkedNode(parent.FirstChildContentId, "first child"); var child = childLink.Value; // if first, clone parent + insert as first child + // NOTE: Don't perform this check if loading from local DB since we know it's already sorted if (child.SortOrder > content.SortOrder) { content.NextSiblingContentId = parent.FirstChildContentId; @@ -834,10 +907,11 @@ namespace Umbraco.Web.PublishedCache.NuCache while (child.NextSiblingContentId > 0) { // get next child - var nextChildLink = GetLinkedNode(child.NextSiblingContentId, "next child"); + var nextChildLink = GetRequiredLinkedNode(child.NextSiblingContentId, "next child"); var nextChild = nextChildLink.Value; // if here, clone previous + append/insert + // NOTE: Don't perform this check if loading from local DB since we know it's already sorted if (nextChild.SortOrder > content.SortOrder) { content.NextSiblingContentId = nextChild.Id; @@ -946,7 +1020,7 @@ namespace Umbraco.Web.PublishedCache.NuCache while (id > 0) { - var link = GetLinkedNode(id, "sibling"); + var link = GetRequiredLinkedNode(id, "sibling"); yield return link.Value; id = link.Value.NextSiblingContentId; } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs index 6e7916c77f..068d47a004 100755 --- a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs @@ -37,7 +37,7 @@ namespace Umbraco.Web.PublishedCache.NuCache private readonly IPublishedContentTypeFactory _publishedContentTypeFactory; private readonly IScopeProvider _scopeProvider; private readonly IDataSource _dataSource; - private readonly ILogger _logger; + private readonly IProfilingLogger _logger; private readonly IDocumentRepository _documentRepository; private readonly IMediaRepository _mediaRepository; private readonly IMemberRepository _memberRepository; @@ -71,7 +71,7 @@ namespace Umbraco.Web.PublishedCache.NuCache public PublishedSnapshotService(PublishedSnapshotServiceOptions options, IMainDom mainDom, IRuntimeState runtime, ServiceContext serviceContext, IPublishedContentTypeFactory publishedContentTypeFactory, IdkMap idkMap, - IPublishedSnapshotAccessor publishedSnapshotAccessor, IVariationContextAccessor variationContextAccessor, ILogger logger, IScopeProvider scopeProvider, + IPublishedSnapshotAccessor publishedSnapshotAccessor, IVariationContextAccessor variationContextAccessor, IProfilingLogger logger, IScopeProvider scopeProvider, IDocumentRepository documentRepository, IMediaRepository mediaRepository, IMemberRepository memberRepository, IDefaultCultureAccessor defaultCultureAccessor, IDataSource dataSource, IGlobalSettings globalSettings, @@ -314,22 +314,10 @@ namespace Umbraco.Web.PublishedCache.NuCache // before I read it? NO! because the WHOLE content tree is read-locked using WithReadLocked. // don't panic. - private void LockAndLoadContent(Action action) - { - // first get a writer, then a scope - // if there already is a scope, the writer will attach to it - // otherwise, it will only exist here - cheap - using (_contentStore.GetScopedWriteLock(_scopeProvider)) - using (var scope = _scopeProvider.CreateScope()) - { - scope.ReadLock(Constants.Locks.ContentTree); - action(scope); - scope.Complete(); - } - } - private bool LockAndLoadContent(Func action) { + + // first get a writer, then a scope // if there already is a scope, the writer will attach to it // otherwise, it will only exist here - cheap @@ -343,7 +331,7 @@ namespace Umbraco.Web.PublishedCache.NuCache } } - private void LoadContentFromDatabaseLocked(IScope scope) + private bool LoadContentFromDatabaseLocked(IScope scope) { // locks: // contentStore is wlocked (1 thread) @@ -351,39 +339,38 @@ namespace Umbraco.Web.PublishedCache.NuCache var contentTypes = _serviceContext.ContentTypeService.GetAll() .Select(x => _publishedContentTypeFactory.CreateContentType(x)); + _contentStore.SetAllContentTypes(contentTypes); - // beware! at that point the cache is inconsistent, - // assuming we are going to SetAll content items! + using (_logger.TraceDuration("Loading content from database")) + { + // beware! at that point the cache is inconsistent, + // assuming we are going to SetAll content items! - _localContentDb?.Clear(); + _localContentDb?.Clear(); - _logger.Debug("Loading content from database..."); - var sw = Stopwatch.StartNew(); - // IMPORTANT GetAllContentSources sorts kits by level - var kits = _dataSource.GetAllContentSources(scope); - _contentStore.SetAll(kits); - sw.Stop(); - _logger.Debug("Loaded content from database ({Duration}ms)", sw.ElapsedMilliseconds); + // IMPORTANT GetAllContentSources sorts kits by level + var kits = _dataSource.GetAllContentSources(scope); + return _contentStore.SetAll(kits, false); + } } private bool LoadContentFromLocalDbLocked(IScope scope) { var contentTypes = _serviceContext.ContentTypeService.GetAll() - .Select(x => _publishedContentTypeFactory.CreateContentType(x)); + .Select(x => _publishedContentTypeFactory.CreateContentType(x)); _contentStore.SetAllContentTypes(contentTypes); - // beware! at that point the cache is inconsistent, - // assuming we are going to SetAll content items! + using (_logger.TraceDuration("Loading content from local cache file")) + { + // beware! at that point the cache is inconsistent, + // assuming we are going to SetAll content items! - _logger.Debug("Loading content from local db..."); - var sw = Stopwatch.StartNew(); - var kits = _localContentDb.Select(x => x.Value) - .OrderBy(x => x.Node.Level); // IMPORTANT sort by level - var ok = _contentStore.SetAll(kits, true); - sw.Stop(); - _logger.Debug("Loaded content from local db ({Duration}ms)", sw.ElapsedMilliseconds); - return ok; + var kits = _localContentDb.Select(x => x.Value) + .OrderBy(x => x.Node.Level) + .ThenBy(x => x.Node.SortOrder); // IMPORTANT sort by level + order + return _contentStore.SetAllFastSorted(kits); + } } // keep these around - might be useful @@ -408,18 +395,6 @@ namespace Umbraco.Web.PublishedCache.NuCache // _contentStore.Set(contentNode); //} - private void LockAndLoadMedia(Action action) - { - // see note in LockAndLoadContent - using (_mediaStore.GetScopedWriteLock(_scopeProvider)) - using (var scope = _scopeProvider.CreateScope()) - { - scope.ReadLock(Constants.Locks.MediaTree); - action(scope); - scope.Complete(); - } - } - private bool LockAndLoadMedia(Func action) { // see note in LockAndLoadContent @@ -433,7 +408,7 @@ namespace Umbraco.Web.PublishedCache.NuCache } } - private void LoadMediaFromDatabaseLocked(IScope scope) + private bool LoadMediaFromDatabaseLocked(IScope scope) { // locks & notes: see content @@ -441,37 +416,37 @@ namespace Umbraco.Web.PublishedCache.NuCache .Select(x => _publishedContentTypeFactory.CreateContentType(x)); _mediaStore.SetAllContentTypes(mediaTypes); - // beware! at that point the cache is inconsistent, - // assuming we are going to SetAll content items! + using (_logger.TraceDuration("Loading media from database")) + { + // beware! at that point the cache is inconsistent, + // assuming we are going to SetAll content items! - _localMediaDb?.Clear(); + _localMediaDb?.Clear(); - _logger.Debug("Loading media from database..."); - var sw = Stopwatch.StartNew(); - // IMPORTANT GetAllMediaSources sorts kits by level - var kits = _dataSource.GetAllMediaSources(scope); - _mediaStore.SetAll(kits); - sw.Stop(); - _logger.Debug("Loaded media from database ({Duration}ms)", sw.ElapsedMilliseconds); + _logger.Debug("Loading media from database..."); + // IMPORTANT GetAllMediaSources sorts kits by level + var kits = _dataSource.GetAllMediaSources(scope); + return _mediaStore.SetAll(kits); + } } private bool LoadMediaFromLocalDbLocked(IScope scope) { var mediaTypes = _serviceContext.MediaTypeService.GetAll() - .Select(x => _publishedContentTypeFactory.CreateContentType(x)); + .Select(x => _publishedContentTypeFactory.CreateContentType(x)); _mediaStore.SetAllContentTypes(mediaTypes); - // beware! at that point the cache is inconsistent, - // assuming we are going to SetAll content items! + using (_logger.TraceDuration("Loading media from local cache file")) + { + // beware! at that point the cache is inconsistent, + // assuming we are going to SetAll content items! - _logger.Debug("Loading media from local db..."); - var sw = Stopwatch.StartNew(); - var kits = _localMediaDb.Select(x => x.Value) - .OrderBy(x => x.Node.Level); // IMPORTANT sort by level - var ok = _mediaStore.SetAll(kits, true); - sw.Stop(); - _logger.Debug("Loaded media from local db ({Duration}ms)", sw.ElapsedMilliseconds); - return ok; + var kits = _localMediaDb.Select(x => x.Value) + .OrderBy(x => x.Node.Level) + .ThenBy(x => x.Node.SortOrder); // IMPORTANT sort by level + order; + return _mediaStore.SetAllFastSorted(kits); + } + } // keep these around - might be useful diff --git a/src/Umbraco.Web/PublishedCache/NuCache/Snap/LinkedNode.cs b/src/Umbraco.Web/PublishedCache/NuCache/Snap/LinkedNode.cs index 78e0ec8d33..d187996df8 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/Snap/LinkedNode.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/Snap/LinkedNode.cs @@ -1,5 +1,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.Snap { + //NOTE: This cannot be struct because it references itself + + /// + /// Used to represent an item in a linked list + /// + /// internal class LinkedNode where TValue : class {