From e9d4b607bb5cd68b2a41adb55a2f411d431b538a Mon Sep 17 00:00:00 2001 From: Shannon Date: Thu, 15 Aug 2019 19:05:43 +1000 Subject: [PATCH 01/11] Adds POC for fast building of in-memory nucache from persisted files --- .../PublishedContent/NuCacheChildrenTests.cs | 2 +- .../PublishedContent/NuCacheTests.cs | 2 +- .../Scoping/ScopedNuCacheTests.cs | 2 +- .../ContentTypeServiceVariantsTests.cs | 2 +- .../PublishedCache/NuCache/ContentStore.cs | 146 +++++++++++++----- .../NuCache/PublishedSnapshotService.cs | 119 ++++++-------- .../PublishedCache/NuCache/Snap/LinkedNode.cs | 6 + 7 files changed, 167 insertions(+), 112 deletions(-) diff --git a/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs b/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs index eb6e7725c0..2eb3c280b2 100644 --- a/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs +++ b/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs @@ -133,7 +133,7 @@ namespace Umbraco.Tests.PublishedContent null, _snapshotAccessor, _variationAccesor, - Mock.Of(), + Mock.Of(), scopeProvider, Mock.Of(), Mock.Of(), diff --git a/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs b/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs index 399b0c1342..9f20f42993 100644 --- a/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs +++ b/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs @@ -178,7 +178,7 @@ namespace Umbraco.Tests.PublishedContent null, new TestPublishedSnapshotAccessor(), _variationAccesor, - Mock.Of(), + Mock.Of(), scopeProvider, Mock.Of(), Mock.Of(), diff --git a/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs b/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs index c974fed99e..8c14ce8219 100644 --- a/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs +++ b/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs @@ -92,7 +92,7 @@ namespace Umbraco.Tests.Scoping null, publishedSnapshotAccessor, Mock.Of(), - Logger, + ProfilingLogger, ScopeProvider, documentRepository, mediaRepository, memberRepository, DefaultCultureAccessor, diff --git a/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs b/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs index 956de186be..a9e7a86bf7 100644 --- a/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs +++ b/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs @@ -64,7 +64,7 @@ namespace Umbraco.Tests.Services null, publishedSnapshotAccessor, Mock.Of(), - Logger, + ProfilingLogger, ScopeProvider, documentRepository, mediaRepository, memberRepository, DefaultCultureAccessor, diff --git a/src/Umbraco.Web/PublishedCache/NuCache/ContentStore.cs b/src/Umbraco.Web/PublishedCache/NuCache/ContentStore.cs index 8e2cf7bc3c..ec8614f074 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/ContentStore.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/ContentStore.cs @@ -392,7 +392,7 @@ namespace Umbraco.Web.PublishedCache.NuCache var visited = new List(); foreach (var kit in kits.Where(x => refreshedIdsA.Contains(x.ContentTypeId) && - BuildKit(x))) + BuildKit(x, out _))) { // replacing the node: must preserve the parents var node = GetHead(_contentNodes, kit.Node.Id)?.Value; @@ -466,10 +466,11 @@ namespace Umbraco.Web.PublishedCache.NuCache } } - private bool BuildKit(ContentNodeKit kit) + private bool BuildKit(ContentNodeKit kit, out LinkedNode parent) { // make sure parent exists - if (!ParentExistsLocked(kit)) + parent = GetParentLink(kit.Node); + if (parent == null) { _logger.Warn($"Skip item id={kit.Node.Id}, could not find parent id={kit.Node.ParentContentId}."); return false; @@ -531,7 +532,7 @@ namespace Umbraco.Web.PublishedCache.NuCache _contentNodes.TryGetValue(kit.Node.Id, out var link); var existing = link?.Value; - if (!BuildKit(kit)) + if (!BuildKit(kit, out var parent)) return false; // moving? @@ -549,7 +550,7 @@ namespace Umbraco.Web.PublishedCache.NuCache if (existing == null) { // new, add to parent - AddNodeLocked(kit.Node); + AddNodeLocked(kit.Node, parent); } else if (moving || existing.SortOrder != kit.Node.SortOrder) { @@ -581,7 +582,74 @@ namespace Umbraco.Web.PublishedCache.NuCache _root.Value.FirstChildContentId = -1; } - // IMPORTANT kits must be sorted out by LEVEL + /// + /// Builds all kits without any sorting or generation checks + /// + /// + /// + /// + /// This requires that the collection is sorted by Level + Sort Order. + /// This should be used only on a site startup as the first generations. + /// + internal bool SetAllFastSorted(IEnumerable kits) + { + var lockInfo = new WriteLockInfo(); + var ok = true; + try + { + Lock(lockInfo); + + ClearLocked(_contentNodes); + ClearRootLocked(); + + //these are ordered by level + sort order + + // The name of the game here is to populate each kit's + // FirstChildContentId + // NextSiblingContentId + + ContentNode prev = null; + var currLevel = 0; + + foreach (var kit in kits) + { + if (currLevel != kit.Node.Level) + { + prev = null; //reset since we're on a new level + currLevel = kit.Node.Level; + } + + if (!BuildKit(kit, out var parentLink)) + { + ok = false; + continue; // skip that one + } + + _logger.Debug($"Set {kit.Node.Id} with parent {kit.Node.ParentContentId}"); + SetValueLocked(_contentNodes, kit.Node.Id, kit.Node); + + //if the parent's FirstChildContentId isn't set, then it must be the current one + if (parentLink.Value.FirstChildContentId < 0) + parentLink.Value.FirstChildContentId = kit.Node.Id; + + //if there is a previous one on the same level then set it's next sibling id to the current oen + if (prev != null) + prev.NextSiblingContentId = kit.Node.Id; + + //store the prev + prev = kit.Node; + + _xmap[kit.Node.Uid] = kit.Node.Id; + } + } + finally + { + Release(lockInfo); + } + + return ok; + } + public bool SetAll(IEnumerable kits, bool fromLocalDb = false) { var lockInfo = new WriteLockInfo(); @@ -599,7 +667,7 @@ namespace Umbraco.Web.PublishedCache.NuCache foreach (var kit in kits) { - if (!BuildKit(kit)) + if (!BuildKit(kit, out var parent)) { ok = false; continue; // skip that one @@ -609,7 +677,7 @@ namespace Umbraco.Web.PublishedCache.NuCache // don't refresh _localDb if we are reading from _localDb if (!fromLocalDb && _localDb != null) RegisterChange(kit.Node.Id, kit); - AddNodeLocked(kit.Node); + AddNodeLocked(kit.Node, parent); _xmap[kit.Node.Uid] = kit.Node.Id; } @@ -645,14 +713,14 @@ namespace Umbraco.Web.PublishedCache.NuCache // now add them all back foreach (var kit in kits) { - if (!BuildKit(kit)) + if (!BuildKit(kit, out var parent)) { ok = false; continue; // skip that one } SetValueLocked(_contentNodes, kit.Node.Id, kit.Node); if (_localDb != null) RegisterChange(kit.Node.Id, kit); - AddNodeLocked(kit.Node); + AddNodeLocked(kit.Node, parent); _xmap[kit.Node.Uid] = kit.Node.Id; } @@ -712,35 +780,49 @@ namespace Umbraco.Web.PublishedCache.NuCache var id = content.FirstChildContentId; while (id > 0) { - var link = GetLinkedNode(id, "child"); + var link = GetRequiredLinkedNode(id, "child"); ClearBranchLocked(link.Value); id = link.Value.NextSiblingContentId; } } - // gets the link node - // throws (panic) if not found, or no value - private LinkedNode GetLinkedNode(int id, string description) + /// + /// Gets the link node and if it doesn't exist throw a + /// + /// + /// + /// + private LinkedNode GetRequiredLinkedNode(int id, string description) { if (_contentNodes.TryGetValue(id, out var link) && link.Value != null) return link; - throw new Exception($"panic: failed to get {description} with id={id}"); + throw new PanicException($"panic: failed to get {description} with id={id}"); } private LinkedNode GetParentLink(ContentNode content) { - _contentNodes.TryGetValue(content.ParentContentId, out var link); // else null - //if (link == null || link.Value == null) - // throw new Exception("Panic: parent not found."); + if (content.ParentContentId < 0) return _root; + + _contentNodes.TryGetValue(content.ParentContentId, out var link); return link; } + /// + /// Gets the linked parent node and if it doesn't exist throw a + /// + /// + /// + private LinkedNode GetRequiredParentLink(ContentNode content) + { + return content.ParentContentId < 0 ? _root : GetRequiredLinkedNode(content.ParentContentId, "parent"); + } + private void RemoveNodeLocked(ContentNode content) { var parentLink = content.ParentContentId < 0 ? _root - : GetLinkedNode(content.ParentContentId, "parent"); + : GetRequiredLinkedNode(content.ParentContentId, "parent"); var parent = parentLink.Value; @@ -757,10 +839,10 @@ namespace Umbraco.Web.PublishedCache.NuCache else { // iterate children until the previous child - var link = GetLinkedNode(parent.FirstChildContentId, "first child"); + var link = GetRequiredLinkedNode(parent.FirstChildContentId, "first child"); while (link.Value.NextSiblingContentId != content.Id) - link = GetLinkedNode(link.Value.NextSiblingContentId, "next child"); + link = GetRequiredLinkedNode(link.Value.NextSiblingContentId, "next child"); // clone the previous child and replace next child var prevChild = GenCloneLocked(link); @@ -768,14 +850,6 @@ namespace Umbraco.Web.PublishedCache.NuCache } } - private bool ParentExistsLocked(ContentNodeKit kit) - { - if (kit.Node.ParentContentId < 0) - return true; - var link = GetParentLink(kit.Node); - return link?.Value != null; - } - private bool ParentPublishedLocked(ContentNodeKit kit) { if (kit.Node.ParentContentId < 0) @@ -801,11 +875,9 @@ namespace Umbraco.Web.PublishedCache.NuCache return node; } - private void AddNodeLocked(ContentNode content) + private void AddNodeLocked(ContentNode content, LinkedNode parentLink = null) { - var parentLink = content.ParentContentId < 0 - ? _root - : GetLinkedNode(content.ParentContentId, "parent"); + parentLink = parentLink ?? GetRequiredParentLink(content); var parent = parentLink.Value; @@ -818,10 +890,11 @@ namespace Umbraco.Web.PublishedCache.NuCache } // get parent's first child - var childLink = GetLinkedNode(parent.FirstChildContentId, "first child"); + var childLink = GetRequiredLinkedNode(parent.FirstChildContentId, "first child"); var child = childLink.Value; // if first, clone parent + insert as first child + // NOTE: Don't perform this check if loading from local DB since we know it's already sorted if (child.SortOrder > content.SortOrder) { content.NextSiblingContentId = parent.FirstChildContentId; @@ -834,10 +907,11 @@ namespace Umbraco.Web.PublishedCache.NuCache while (child.NextSiblingContentId > 0) { // get next child - var nextChildLink = GetLinkedNode(child.NextSiblingContentId, "next child"); + var nextChildLink = GetRequiredLinkedNode(child.NextSiblingContentId, "next child"); var nextChild = nextChildLink.Value; // if here, clone previous + append/insert + // NOTE: Don't perform this check if loading from local DB since we know it's already sorted if (nextChild.SortOrder > content.SortOrder) { content.NextSiblingContentId = nextChild.Id; @@ -946,7 +1020,7 @@ namespace Umbraco.Web.PublishedCache.NuCache while (id > 0) { - var link = GetLinkedNode(id, "sibling"); + var link = GetRequiredLinkedNode(id, "sibling"); yield return link.Value; id = link.Value.NextSiblingContentId; } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs index 6e7916c77f..068d47a004 100755 --- a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs @@ -37,7 +37,7 @@ namespace Umbraco.Web.PublishedCache.NuCache private readonly IPublishedContentTypeFactory _publishedContentTypeFactory; private readonly IScopeProvider _scopeProvider; private readonly IDataSource _dataSource; - private readonly ILogger _logger; + private readonly IProfilingLogger _logger; private readonly IDocumentRepository _documentRepository; private readonly IMediaRepository _mediaRepository; private readonly IMemberRepository _memberRepository; @@ -71,7 +71,7 @@ namespace Umbraco.Web.PublishedCache.NuCache public PublishedSnapshotService(PublishedSnapshotServiceOptions options, IMainDom mainDom, IRuntimeState runtime, ServiceContext serviceContext, IPublishedContentTypeFactory publishedContentTypeFactory, IdkMap idkMap, - IPublishedSnapshotAccessor publishedSnapshotAccessor, IVariationContextAccessor variationContextAccessor, ILogger logger, IScopeProvider scopeProvider, + IPublishedSnapshotAccessor publishedSnapshotAccessor, IVariationContextAccessor variationContextAccessor, IProfilingLogger logger, IScopeProvider scopeProvider, IDocumentRepository documentRepository, IMediaRepository mediaRepository, IMemberRepository memberRepository, IDefaultCultureAccessor defaultCultureAccessor, IDataSource dataSource, IGlobalSettings globalSettings, @@ -314,22 +314,10 @@ namespace Umbraco.Web.PublishedCache.NuCache // before I read it? NO! because the WHOLE content tree is read-locked using WithReadLocked. // don't panic. - private void LockAndLoadContent(Action action) - { - // first get a writer, then a scope - // if there already is a scope, the writer will attach to it - // otherwise, it will only exist here - cheap - using (_contentStore.GetScopedWriteLock(_scopeProvider)) - using (var scope = _scopeProvider.CreateScope()) - { - scope.ReadLock(Constants.Locks.ContentTree); - action(scope); - scope.Complete(); - } - } - private bool LockAndLoadContent(Func action) { + + // first get a writer, then a scope // if there already is a scope, the writer will attach to it // otherwise, it will only exist here - cheap @@ -343,7 +331,7 @@ namespace Umbraco.Web.PublishedCache.NuCache } } - private void LoadContentFromDatabaseLocked(IScope scope) + private bool LoadContentFromDatabaseLocked(IScope scope) { // locks: // contentStore is wlocked (1 thread) @@ -351,39 +339,38 @@ namespace Umbraco.Web.PublishedCache.NuCache var contentTypes = _serviceContext.ContentTypeService.GetAll() .Select(x => _publishedContentTypeFactory.CreateContentType(x)); + _contentStore.SetAllContentTypes(contentTypes); - // beware! at that point the cache is inconsistent, - // assuming we are going to SetAll content items! + using (_logger.TraceDuration("Loading content from database")) + { + // beware! at that point the cache is inconsistent, + // assuming we are going to SetAll content items! - _localContentDb?.Clear(); + _localContentDb?.Clear(); - _logger.Debug("Loading content from database..."); - var sw = Stopwatch.StartNew(); - // IMPORTANT GetAllContentSources sorts kits by level - var kits = _dataSource.GetAllContentSources(scope); - _contentStore.SetAll(kits); - sw.Stop(); - _logger.Debug("Loaded content from database ({Duration}ms)", sw.ElapsedMilliseconds); + // IMPORTANT GetAllContentSources sorts kits by level + var kits = _dataSource.GetAllContentSources(scope); + return _contentStore.SetAll(kits, false); + } } private bool LoadContentFromLocalDbLocked(IScope scope) { var contentTypes = _serviceContext.ContentTypeService.GetAll() - .Select(x => _publishedContentTypeFactory.CreateContentType(x)); + .Select(x => _publishedContentTypeFactory.CreateContentType(x)); _contentStore.SetAllContentTypes(contentTypes); - // beware! at that point the cache is inconsistent, - // assuming we are going to SetAll content items! + using (_logger.TraceDuration("Loading content from local cache file")) + { + // beware! at that point the cache is inconsistent, + // assuming we are going to SetAll content items! - _logger.Debug("Loading content from local db..."); - var sw = Stopwatch.StartNew(); - var kits = _localContentDb.Select(x => x.Value) - .OrderBy(x => x.Node.Level); // IMPORTANT sort by level - var ok = _contentStore.SetAll(kits, true); - sw.Stop(); - _logger.Debug("Loaded content from local db ({Duration}ms)", sw.ElapsedMilliseconds); - return ok; + var kits = _localContentDb.Select(x => x.Value) + .OrderBy(x => x.Node.Level) + .ThenBy(x => x.Node.SortOrder); // IMPORTANT sort by level + order + return _contentStore.SetAllFastSorted(kits); + } } // keep these around - might be useful @@ -408,18 +395,6 @@ namespace Umbraco.Web.PublishedCache.NuCache // _contentStore.Set(contentNode); //} - private void LockAndLoadMedia(Action action) - { - // see note in LockAndLoadContent - using (_mediaStore.GetScopedWriteLock(_scopeProvider)) - using (var scope = _scopeProvider.CreateScope()) - { - scope.ReadLock(Constants.Locks.MediaTree); - action(scope); - scope.Complete(); - } - } - private bool LockAndLoadMedia(Func action) { // see note in LockAndLoadContent @@ -433,7 +408,7 @@ namespace Umbraco.Web.PublishedCache.NuCache } } - private void LoadMediaFromDatabaseLocked(IScope scope) + private bool LoadMediaFromDatabaseLocked(IScope scope) { // locks & notes: see content @@ -441,37 +416,37 @@ namespace Umbraco.Web.PublishedCache.NuCache .Select(x => _publishedContentTypeFactory.CreateContentType(x)); _mediaStore.SetAllContentTypes(mediaTypes); - // beware! at that point the cache is inconsistent, - // assuming we are going to SetAll content items! + using (_logger.TraceDuration("Loading media from database")) + { + // beware! at that point the cache is inconsistent, + // assuming we are going to SetAll content items! - _localMediaDb?.Clear(); + _localMediaDb?.Clear(); - _logger.Debug("Loading media from database..."); - var sw = Stopwatch.StartNew(); - // IMPORTANT GetAllMediaSources sorts kits by level - var kits = _dataSource.GetAllMediaSources(scope); - _mediaStore.SetAll(kits); - sw.Stop(); - _logger.Debug("Loaded media from database ({Duration}ms)", sw.ElapsedMilliseconds); + _logger.Debug("Loading media from database..."); + // IMPORTANT GetAllMediaSources sorts kits by level + var kits = _dataSource.GetAllMediaSources(scope); + return _mediaStore.SetAll(kits); + } } private bool LoadMediaFromLocalDbLocked(IScope scope) { var mediaTypes = _serviceContext.MediaTypeService.GetAll() - .Select(x => _publishedContentTypeFactory.CreateContentType(x)); + .Select(x => _publishedContentTypeFactory.CreateContentType(x)); _mediaStore.SetAllContentTypes(mediaTypes); - // beware! at that point the cache is inconsistent, - // assuming we are going to SetAll content items! + using (_logger.TraceDuration("Loading media from local cache file")) + { + // beware! at that point the cache is inconsistent, + // assuming we are going to SetAll content items! - _logger.Debug("Loading media from local db..."); - var sw = Stopwatch.StartNew(); - var kits = _localMediaDb.Select(x => x.Value) - .OrderBy(x => x.Node.Level); // IMPORTANT sort by level - var ok = _mediaStore.SetAll(kits, true); - sw.Stop(); - _logger.Debug("Loaded media from local db ({Duration}ms)", sw.ElapsedMilliseconds); - return ok; + var kits = _localMediaDb.Select(x => x.Value) + .OrderBy(x => x.Node.Level) + .ThenBy(x => x.Node.SortOrder); // IMPORTANT sort by level + order; + return _mediaStore.SetAllFastSorted(kits); + } + } // keep these around - might be useful diff --git a/src/Umbraco.Web/PublishedCache/NuCache/Snap/LinkedNode.cs b/src/Umbraco.Web/PublishedCache/NuCache/Snap/LinkedNode.cs index 78e0ec8d33..d187996df8 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/Snap/LinkedNode.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/Snap/LinkedNode.cs @@ -1,5 +1,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.Snap { + //NOTE: This cannot be struct because it references itself + + /// + /// Used to represent an item in a linked list + /// + /// internal class LinkedNode where TValue : class { From 3d56e45266c29269f805871aea1bb9ff38f89013 Mon Sep 17 00:00:00 2001 From: Shannon Date: Thu, 15 Aug 2019 19:25:28 +1000 Subject: [PATCH 02/11] Adds notes, cleans up other code --- src/Umbraco.Web/PublishedCache/NuCache/ContentStore.cs | 5 ++--- .../NuCache/DataSource/BTree.ContentNodeKitSerializer.cs | 3 ++- .../PublishedCache/NuCache/PublishedSnapshotService.cs | 4 +++- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/ContentStore.cs b/src/Umbraco.Web/PublishedCache/NuCache/ContentStore.cs index ec8614f074..dbe354d95b 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/ContentStore.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/ContentStore.cs @@ -650,7 +650,7 @@ namespace Umbraco.Web.PublishedCache.NuCache return ok; } - public bool SetAll(IEnumerable kits, bool fromLocalDb = false) + public bool SetAll(IEnumerable kits) { var lockInfo = new WriteLockInfo(); var ok = true; @@ -675,8 +675,7 @@ namespace Umbraco.Web.PublishedCache.NuCache _logger.Debug($"Set {kit.Node.Id} with parent {kit.Node.ParentContentId}"); SetValueLocked(_contentNodes, kit.Node.Id, kit.Node); - // don't refresh _localDb if we are reading from _localDb - if (!fromLocalDb && _localDb != null) RegisterChange(kit.Node.Id, kit); + if (_localDb != null) RegisterChange(kit.Node.Id, kit); AddNodeLocked(kit.Node, parent); _xmap[kit.Node.Uid] = kit.Node.Id; diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.ContentNodeKitSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.ContentNodeKitSerializer.cs index 4303f97d3c..142a2c36db 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.ContentNodeKitSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.ContentNodeKitSerializer.cs @@ -41,6 +41,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource PrimitiveSerializer.String.WriteTo(value.Node.Path, stream); PrimitiveSerializer.Int32.WriteTo(value.Node.SortOrder, stream); PrimitiveSerializer.Int32.WriteTo(value.Node.ParentContentId, stream); + //TODO: Why don't we write the FirstChildContentId + NextSiblingContentId? Then we can entirely avoid all overhead when loading on startup? PrimitiveSerializer.DateTime.WriteTo(value.Node.CreateDate, stream); PrimitiveSerializer.Int32.WriteTo(value.Node.CreatorId, stream); PrimitiveSerializer.Int32.WriteTo(value.ContentTypeId, stream); @@ -54,4 +55,4 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource DataSerializer.WriteTo(value.PublishedData, stream); } } -} \ No newline at end of file +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs index 068d47a004..5c6dc3d252 100755 --- a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs @@ -350,8 +350,9 @@ namespace Umbraco.Web.PublishedCache.NuCache _localContentDb?.Clear(); // IMPORTANT GetAllContentSources sorts kits by level + //TODO: We should sort by level + sort order and then we can use SetAllFastSorted var kits = _dataSource.GetAllContentSources(scope); - return _contentStore.SetAll(kits, false); + return _contentStore.SetAll(kits); } } @@ -425,6 +426,7 @@ namespace Umbraco.Web.PublishedCache.NuCache _logger.Debug("Loading media from database..."); // IMPORTANT GetAllMediaSources sorts kits by level + //TODO: We should sort by level + sort order and then we can use SetAllFastSorted var kits = _dataSource.GetAllMediaSources(scope); return _mediaStore.SetAll(kits); } From bbd6e1150e5b65a4c7a718917225eda437f67705 Mon Sep 17 00:00:00 2001 From: Shannon Date: Thu, 15 Aug 2019 23:59:46 +1000 Subject: [PATCH 03/11] Changes loading from DB to use SetAllFastSorted since it's already sorted by level + sort order --- .../PublishedCache/NuCache/DataSource/IDataSource.cs | 4 ++-- .../NuCache/PublishedSnapshotService.cs | 12 +++++------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IDataSource.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IDataSource.cs index d4702cf7aa..941c7d0caa 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IDataSource.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IDataSource.cs @@ -9,12 +9,12 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource internal interface IDataSource { ContentNodeKit GetContentSource(IScope scope, int id); - IEnumerable GetAllContentSources(IScope scope); // must order by level + IEnumerable GetAllContentSources(IScope scope); // must order by level, sortOrder IEnumerable GetBranchContentSources(IScope scope, int id); // must order by level, sortOrder IEnumerable GetTypeContentSources(IScope scope, IEnumerable ids); ContentNodeKit GetMediaSource(IScope scope, int id); - IEnumerable GetAllMediaSources(IScope scope); // must order by level + IEnumerable GetAllMediaSources(IScope scope); // must order by level, sortOrder IEnumerable GetBranchMediaSources(IScope scope, int id); // must order by level, sortOrder IEnumerable GetTypeMediaSources(IScope scope, IEnumerable ids); } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs index 5c6dc3d252..5a7ae9e899 100755 --- a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs @@ -349,10 +349,9 @@ namespace Umbraco.Web.PublishedCache.NuCache _localContentDb?.Clear(); - // IMPORTANT GetAllContentSources sorts kits by level - //TODO: We should sort by level + sort order and then we can use SetAllFastSorted + // IMPORTANT GetAllContentSources sorts kits by level + sortOrder var kits = _dataSource.GetAllContentSources(scope); - return _contentStore.SetAll(kits); + return _contentStore.SetAllFastSorted(kits); } } @@ -369,7 +368,7 @@ namespace Umbraco.Web.PublishedCache.NuCache var kits = _localContentDb.Select(x => x.Value) .OrderBy(x => x.Node.Level) - .ThenBy(x => x.Node.SortOrder); // IMPORTANT sort by level + order + .ThenBy(x => x.Node.SortOrder); // IMPORTANT sort by level + sortOrder return _contentStore.SetAllFastSorted(kits); } } @@ -425,10 +424,9 @@ namespace Umbraco.Web.PublishedCache.NuCache _localMediaDb?.Clear(); _logger.Debug("Loading media from database..."); - // IMPORTANT GetAllMediaSources sorts kits by level - //TODO: We should sort by level + sort order and then we can use SetAllFastSorted + // IMPORTANT GetAllMediaSources sorts kits by level + sortOrder var kits = _dataSource.GetAllMediaSources(scope); - return _mediaStore.SetAll(kits); + return _mediaStore.SetAllFastSorted(kits); } } From e9717ee86ae1c8edcb4f0a03419d22e96802d092 Mon Sep 17 00:00:00 2001 From: Shannon Date: Fri, 16 Aug 2019 16:18:58 +1000 Subject: [PATCH 04/11] changes to PanicException --- src/Umbraco.Web/PublishedCache/NuCache/ContentStore.cs | 4 ++-- .../PublishedCache/NuCache/PublishedContent.cs | 9 +++++---- src/Umbraco.Web/UmbracoInjectedModule.cs | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/ContentStore.cs b/src/Umbraco.Web/PublishedCache/NuCache/ContentStore.cs index dbe354d95b..86fc69c302 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/ContentStore.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/ContentStore.cs @@ -796,7 +796,7 @@ namespace Umbraco.Web.PublishedCache.NuCache if (_contentNodes.TryGetValue(id, out var link) && link.Value != null) return link; - throw new PanicException($"panic: failed to get {description} with id={id}"); + throw new PanicException($"failed to get {description} with id={id}"); } private LinkedNode GetParentLink(ContentNode content) @@ -827,7 +827,7 @@ namespace Umbraco.Web.PublishedCache.NuCache // must have children if (parent.FirstChildContentId < 0) - throw new Exception("panic: no children"); + throw new PanicException("no children"); // if first, clone parent + remove first child if (parent.FirstChildContentId == content.Id) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/PublishedContent.cs b/src/Umbraco.Web/PublishedCache/NuCache/PublishedContent.cs index cb7acdc164..8fc2dc1006 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/PublishedContent.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/PublishedContent.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Linq; using Umbraco.Core; using Umbraco.Core.Cache; +using Umbraco.Core.Exceptions; using Umbraco.Core.Models.PublishedContent; using Umbraco.Web.Composing; using Umbraco.Web.Models; @@ -119,7 +120,7 @@ namespace Umbraco.Web.PublishedCache.NuCache case PublishedItemType.Media: return GetMediaByIdFunc; default: - throw new Exception("panic: invalid item type"); + throw new PanicException("invalid item type"); } } @@ -194,7 +195,7 @@ namespace Umbraco.Web.PublishedCache.NuCache }; if (ContentData.CultureInfos == null) - throw new Exception("panic: _contentDate.CultureInfos is null."); + throw new PanicException("_contentDate.CultureInfos is null."); return _cultures = ContentData.CultureInfos .ToDictionary(x => x.Key, x => new PublishedCultureInfo(x.Key, x.Value.Name, x.Value.UrlSegment, x.Value.Date), StringComparer.OrdinalIgnoreCase); @@ -286,13 +287,13 @@ namespace Umbraco.Web.PublishedCache.NuCache { // but if IsPreviewing is true, we should have a child if (IsPreviewing) - throw new Exception($"panic: failed to get content with id={id}"); + throw new PanicException($"failed to get content with id={id}"); // if IsPreviewing is false, get the unpublished child nevertheless // we need it to keep enumerating children! but we don't return it content = getById(publishedSnapshot, true, id); if (content == null) - throw new Exception($"panic: failed to get content with id={id}"); + throw new PanicException($"failed to get content with id={id}"); } id = UnwrapIPublishedContent(content)._contentNode.NextSiblingContentId; diff --git a/src/Umbraco.Web/UmbracoInjectedModule.cs b/src/Umbraco.Web/UmbracoInjectedModule.cs index c973fafa81..db5372efa2 100644 --- a/src/Umbraco.Web/UmbracoInjectedModule.cs +++ b/src/Umbraco.Web/UmbracoInjectedModule.cs @@ -264,7 +264,7 @@ namespace Umbraco.Web case RuntimeLevel.Unknown: case RuntimeLevel.Boot: case RuntimeLevel.BootFailed: - throw new Exception($"panic: Unexpected runtime level: {level}."); + throw new PanicException($"Unexpected runtime level: {level}."); case RuntimeLevel.Run: // ok From 890309bc11e42e885134b3b1fde8c6bb6c68341e Mon Sep 17 00:00:00 2001 From: Shannon Date: Mon, 19 Aug 2019 17:18:45 +1000 Subject: [PATCH 05/11] Fixes SetAllFastSorted and sort orders --- .../Testing/Objects/TestDataSource.cs | 9 ++- .../PublishedCache/NuCache/ContentStore.cs | 13 ++-- .../NuCache/DataSource/DatabaseDataSource.cs | 16 ++--- .../NuCache/DataSource/IDataSource.cs | 62 ++++++++++++++++++- .../NuCache/PublishedSnapshotService.cs | 10 +-- 5 files changed, 87 insertions(+), 23 deletions(-) diff --git a/src/Umbraco.Tests/Testing/Objects/TestDataSource.cs b/src/Umbraco.Tests/Testing/Objects/TestDataSource.cs index 72fb89ab82..2d3b058421 100644 --- a/src/Umbraco.Tests/Testing/Objects/TestDataSource.cs +++ b/src/Umbraco.Tests/Testing/Objects/TestDataSource.cs @@ -29,17 +29,24 @@ namespace Umbraco.Tests.Testing.Objects public IEnumerable GetAllContentSources(IScope scope) => Kits.Values .OrderBy(x => x.Node.Level) + .ThenBy(x => x.Node.ParentContentId) + .ThenBy(x => x.Node.SortOrder) .Select(x => x.Clone()); public IEnumerable GetBranchContentSources(IScope scope, int id) => Kits.Values .Where(x => x.Node.Path.EndsWith("," + id) || x.Node.Path.Contains("," + id + ",")) - .OrderBy(x => x.Node.Level).ThenBy(x => x.Node.SortOrder) + .OrderBy(x => x.Node.Level) + .ThenBy(x => x.Node.ParentContentId) + .ThenBy(x => x.Node.SortOrder) .Select(x => x.Clone()); public IEnumerable GetTypeContentSources(IScope scope, IEnumerable ids) => Kits.Values .Where(x => ids.Contains(x.ContentTypeId)) + .OrderBy(x => x.Node.Level) + .ThenBy(x => x.Node.ParentContentId) + .ThenBy(x => x.Node.SortOrder) .Select(x => x.Clone()); public ContentNodeKit GetMediaSource(IScope scope, int id) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/ContentStore.cs b/src/Umbraco.Web/PublishedCache/NuCache/ContentStore.cs index 86fc69c302..d07602dd29 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/ContentStore.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/ContentStore.cs @@ -609,22 +609,21 @@ namespace Umbraco.Web.PublishedCache.NuCache // NextSiblingContentId ContentNode prev = null; - var currLevel = 0; + ContentNode currParent = null; foreach (var kit in kits) { - if (currLevel != kit.Node.Level) - { - prev = null; //reset since we're on a new level - currLevel = kit.Node.Level; - } - if (!BuildKit(kit, out var parentLink)) { ok = false; continue; // skip that one } + if (currParent != null && currParent.Id != parentLink.Value.Id) + prev = null; //changed parent + + currParent = parentLink.Value; + _logger.Debug($"Set {kit.Node.Id} with parent {kit.Node.ParentContentId}"); SetValueLocked(_contentNodes, kit.Node.Id, kit.Node); diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs index 3b66bd52d0..f07ea7db6b 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs @@ -67,7 +67,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource { var sql = ContentSourcesSelect(scope) .Where(x => x.NodeObjectType == Constants.ObjectTypes.Document && x.NodeId == id && !x.Trashed) - .OrderBy(x => x.Level, x => x.SortOrder); + .OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder); var dto = scope.Database.Fetch(sql).FirstOrDefault(); return dto == null ? new ContentNodeKit() : CreateContentNodeKit(dto); @@ -77,7 +77,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource { var sql = ContentSourcesSelect(scope) .Where(x => x.NodeObjectType == Constants.ObjectTypes.Document && !x.Trashed) - .OrderBy(x => x.Level, x => x.SortOrder); + .OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder); return scope.Database.Query(sql).Select(CreateContentNodeKit); } @@ -91,7 +91,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource .Where(x => x.NodeObjectType == Constants.ObjectTypes.Document && !x.Trashed) .Where(x => x.NodeId == id, "x") - .OrderBy(x => x.Level, x => x.SortOrder); + .OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder); return scope.Database.Query(sql).Select(CreateContentNodeKit); } @@ -103,7 +103,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource var sql = ContentSourcesSelect(scope) .Where(x => x.NodeObjectType == Constants.ObjectTypes.Document && !x.Trashed) .WhereIn(x => x.ContentTypeId, ids) - .OrderBy(x => x.Level, x => x.SortOrder); + .OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder); return scope.Database.Query(sql).Select(CreateContentNodeKit); } @@ -140,7 +140,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource { var sql = MediaSourcesSelect(scope) .Where(x => x.NodeObjectType == Constants.ObjectTypes.Media && x.NodeId == id && !x.Trashed) - .OrderBy(x => x.Level, x => x.SortOrder); + .OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder); var dto = scope.Database.Fetch(sql).FirstOrDefault(); return dto == null ? new ContentNodeKit() : CreateMediaNodeKit(dto); @@ -150,7 +150,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource { var sql = MediaSourcesSelect(scope) .Where(x => x.NodeObjectType == Constants.ObjectTypes.Media && !x.Trashed) - .OrderBy(x => x.Level, x => x.SortOrder); + .OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder); return scope.Database.Query(sql).Select(CreateMediaNodeKit); } @@ -164,7 +164,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource .Where(x => x.NodeObjectType == Constants.ObjectTypes.Media && !x.Trashed) .Where(x => x.NodeId == id, "x") - .OrderBy(x => x.Level, x => x.SortOrder); + .OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder); return scope.Database.Query(sql).Select(CreateMediaNodeKit); } @@ -176,7 +176,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource var sql = MediaSourcesSelect(scope) .Where(x => x.NodeObjectType == Constants.ObjectTypes.Media && !x.Trashed) .WhereIn(x => x.ContentTypeId, ids) - .OrderBy(x => x.Level, x => x.SortOrder); + .OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder); return scope.Database.Query(sql).Select(CreateMediaNodeKit); } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IDataSource.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IDataSource.cs index 941c7d0caa..ec3ab38e84 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IDataSource.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IDataSource.cs @@ -8,14 +8,70 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource /// internal interface IDataSource { + //TODO: For these required sort orders, would sorting on Path 'just work'? + ContentNodeKit GetContentSource(IScope scope, int id); - IEnumerable GetAllContentSources(IScope scope); // must order by level, sortOrder - IEnumerable GetBranchContentSources(IScope scope, int id); // must order by level, sortOrder + + /// + /// Returns all content ordered by level + sortOrder + /// + /// + /// + /// + /// MUST be ordered by level + parentId + sortOrder! + /// + IEnumerable GetAllContentSources(IScope scope); + + /// + /// Returns branch for content ordered by level + sortOrder + /// + /// + /// + /// + /// MUST be ordered by level + parentId + sortOrder! + /// + IEnumerable GetBranchContentSources(IScope scope, int id); + + /// + /// Returns content by Ids ordered by level + sortOrder + /// + /// + /// + /// + /// MUST be ordered by level + parentId + sortOrder! + /// IEnumerable GetTypeContentSources(IScope scope, IEnumerable ids); ContentNodeKit GetMediaSource(IScope scope, int id); - IEnumerable GetAllMediaSources(IScope scope); // must order by level, sortOrder + + /// + /// Returns all media ordered by level + sortOrder + /// + /// + /// + /// + /// MUST be ordered by level + parentId + sortOrder! + /// + IEnumerable GetAllMediaSources(IScope scope); + + /// + /// Returns branch for media ordered by level + sortOrder + /// + /// + /// + /// + /// MUST be ordered by level + parentId + sortOrder! + /// IEnumerable GetBranchMediaSources(IScope scope, int id); // must order by level, sortOrder + + /// + /// Returns media by Ids ordered by level + sortOrder + /// + /// + /// + /// + /// MUST be ordered by level + parentId + sortOrder! + /// IEnumerable GetTypeMediaSources(IScope scope, IEnumerable ids); } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs index 5a7ae9e899..f1141da376 100755 --- a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs @@ -349,7 +349,7 @@ namespace Umbraco.Web.PublishedCache.NuCache _localContentDb?.Clear(); - // IMPORTANT GetAllContentSources sorts kits by level + sortOrder + // IMPORTANT GetAllContentSources sorts kits by level + parentId + sortOrder var kits = _dataSource.GetAllContentSources(scope); return _contentStore.SetAllFastSorted(kits); } @@ -368,7 +368,8 @@ namespace Umbraco.Web.PublishedCache.NuCache var kits = _localContentDb.Select(x => x.Value) .OrderBy(x => x.Node.Level) - .ThenBy(x => x.Node.SortOrder); // IMPORTANT sort by level + sortOrder + .ThenBy(x => x.Node.ParentContentId) + .ThenBy(x => x.Node.SortOrder); // IMPORTANT sort by level + parentId + sortOrder return _contentStore.SetAllFastSorted(kits); } } @@ -424,7 +425,7 @@ namespace Umbraco.Web.PublishedCache.NuCache _localMediaDb?.Clear(); _logger.Debug("Loading media from database..."); - // IMPORTANT GetAllMediaSources sorts kits by level + sortOrder + // IMPORTANT GetAllMediaSources sorts kits by level + parentId + sortOrder var kits = _dataSource.GetAllMediaSources(scope); return _mediaStore.SetAllFastSorted(kits); } @@ -443,7 +444,8 @@ namespace Umbraco.Web.PublishedCache.NuCache var kits = _localMediaDb.Select(x => x.Value) .OrderBy(x => x.Node.Level) - .ThenBy(x => x.Node.SortOrder); // IMPORTANT sort by level + order; + .ThenBy(x => x.Node.ParentContentId) + .ThenBy(x => x.Node.SortOrder); // IMPORTANT sort by level + parentId + sortOrder return _mediaStore.SetAllFastSorted(kits); } From f2a22e984390f5445d13fc02981eed8c53d87ced Mon Sep 17 00:00:00 2001 From: Shannon Date: Mon, 19 Aug 2019 22:07:22 +1000 Subject: [PATCH 06/11] Ensure we throw an exception when a fatal Exception is caught instead of swallowing --- .../PublishedContent/NuCacheChildrenTests.cs | 20 ++++++++++++------- .../Testing/Objects/TestDataSource.cs | 8 ++++---- .../PublishedCache/NuCache/ContentStore.cs | 2 +- .../NuCache/PublishedSnapshotService.cs | 14 +++++++------ 4 files changed, 26 insertions(+), 18 deletions(-) diff --git a/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs b/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs index 2eb3c280b2..47572a63ee 100644 --- a/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs +++ b/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs @@ -79,12 +79,16 @@ namespace Umbraco.Tests.PublishedContent _contentTypeVariant }; - var contentTypeService = Mock.Of(); - Mock.Get(contentTypeService).Setup(x => x.GetAll()).Returns(contentTypes); - Mock.Get(contentTypeService).Setup(x => x.GetAll(It.IsAny())).Returns(contentTypes); + var contentTypeService = new Mock(); + contentTypeService.Setup(x => x.GetAll()).Returns(contentTypes); + contentTypeService.Setup(x => x.GetAll(It.IsAny())).Returns(contentTypes); - var contentTypeServiceBaseFactory = Mock.Of(); - Mock.Get(contentTypeServiceBaseFactory).Setup(x => x.For(It.IsAny())).Returns(contentTypeService); + var mediaTypeService = new Mock(); + mediaTypeService.Setup(x => x.GetAll()).Returns(Enumerable.Empty()); + mediaTypeService.Setup(x => x.GetAll(It.IsAny())).Returns(Enumerable.Empty()); + + var contentTypeServiceBaseFactory = new Mock(); + contentTypeServiceBaseFactory.Setup(x => x.For(It.IsAny())).Returns(contentTypeService.Object); var dataTypeService = Mock.Of(); Mock.Get(dataTypeService).Setup(x => x.GetAll()).Returns(dataTypes); @@ -94,8 +98,10 @@ namespace Umbraco.Tests.PublishedContent dataTypeService: dataTypeService, memberTypeService: Mock.Of(), memberService: Mock.Of(), - contentTypeService: contentTypeService, - localizationService: Mock.Of() + contentTypeService: contentTypeService.Object, + mediaTypeService: mediaTypeService.Object, + localizationService: Mock.Of(), + domainService: Mock.Of() ); // create a scope provider diff --git a/src/Umbraco.Tests/Testing/Objects/TestDataSource.cs b/src/Umbraco.Tests/Testing/Objects/TestDataSource.cs index 2d3b058421..0291715e46 100644 --- a/src/Umbraco.Tests/Testing/Objects/TestDataSource.cs +++ b/src/Umbraco.Tests/Testing/Objects/TestDataSource.cs @@ -51,22 +51,22 @@ namespace Umbraco.Tests.Testing.Objects public ContentNodeKit GetMediaSource(IScope scope, int id) { - throw new NotImplementedException(); + return default; } public IEnumerable GetAllMediaSources(IScope scope) { - throw new NotImplementedException(); + return Enumerable.Empty(); } public IEnumerable GetBranchMediaSources(IScope scope, int id) { - throw new NotImplementedException(); + return Enumerable.Empty(); } public IEnumerable GetTypeMediaSources(IScope scope, IEnumerable ids) { - throw new NotImplementedException(); + return Enumerable.Empty(); } } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/ContentStore.cs b/src/Umbraco.Web/PublishedCache/NuCache/ContentStore.cs index d07602dd29..b9d1e0edf7 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/ContentStore.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/ContentStore.cs @@ -588,7 +588,7 @@ namespace Umbraco.Web.PublishedCache.NuCache /// /// /// - /// This requires that the collection is sorted by Level + Sort Order. + /// This requires that the collection is sorted by Level + ParentId + Sort Order. /// This should be used only on a site startup as the first generations. /// internal bool SetAllFastSorted(IEnumerable kits) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs index f1141da376..6f60ed86ab 100755 --- a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs @@ -178,11 +178,12 @@ namespace Umbraco.Web.PublishedCache.NuCache { // populate the stores + + var okContent = false; + var okMedia = false; + try { - var okContent = false; - var okMedia = false; - if (_localDbExists) { okContent = LockAndLoadContent(LoadContentFromLocalDbLocked); @@ -204,6 +205,7 @@ namespace Umbraco.Web.PublishedCache.NuCache catch (Exception ex) { _logger.Fatal(ex, "Panic, exception while loading cache data."); + throw; } // finally, cache is ready! @@ -316,7 +318,7 @@ namespace Umbraco.Web.PublishedCache.NuCache private bool LockAndLoadContent(Func action) { - + // first get a writer, then a scope // if there already is a scope, the writer will attach to it @@ -343,7 +345,7 @@ namespace Umbraco.Web.PublishedCache.NuCache _contentStore.SetAllContentTypes(contentTypes); using (_logger.TraceDuration("Loading content from database")) - { + { // beware! at that point the cache is inconsistent, // assuming we are going to SetAll content items! @@ -448,7 +450,7 @@ namespace Umbraco.Web.PublishedCache.NuCache .ThenBy(x => x.Node.SortOrder); // IMPORTANT sort by level + parentId + sortOrder return _mediaStore.SetAllFastSorted(kits); } - + } // keep these around - might be useful From e18f73e74cdafdc4d4196eb2d976b344bda3eefb Mon Sep 17 00:00:00 2001 From: Shannon Date: Mon, 19 Aug 2019 22:07:46 +1000 Subject: [PATCH 07/11] removes todo --- .../NuCache/DataSource/BTree.ContentNodeKitSerializer.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.ContentNodeKitSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.ContentNodeKitSerializer.cs index 142a2c36db..f799869850 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.ContentNodeKitSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.ContentNodeKitSerializer.cs @@ -41,7 +41,6 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource PrimitiveSerializer.String.WriteTo(value.Node.Path, stream); PrimitiveSerializer.Int32.WriteTo(value.Node.SortOrder, stream); PrimitiveSerializer.Int32.WriteTo(value.Node.ParentContentId, stream); - //TODO: Why don't we write the FirstChildContentId + NextSiblingContentId? Then we can entirely avoid all overhead when loading on startup? PrimitiveSerializer.DateTime.WriteTo(value.Node.CreateDate, stream); PrimitiveSerializer.Int32.WriteTo(value.Node.CreatorId, stream); PrimitiveSerializer.Int32.WriteTo(value.ContentTypeId, stream); From fb38d9afe17a7f1b5c447f3922dbd1d9e25b57b4 Mon Sep 17 00:00:00 2001 From: Shannon Date: Mon, 19 Aug 2019 23:22:27 +1000 Subject: [PATCH 08/11] Merges - double linked list for children performance + updates --- .../PublishedCache/NuCache/ContentNode.cs | 8 ++ .../PublishedCache/NuCache/ContentStore.cs | 136 +++++++++++++----- .../NuCache/PublishedSnapshotService.cs | 32 ++--- 3 files changed, 125 insertions(+), 51 deletions(-) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/ContentNode.cs b/src/Umbraco.Web/PublishedCache/NuCache/ContentNode.cs index 1e8d5ddfc9..7379277be4 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/ContentNode.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/ContentNode.cs @@ -12,7 +12,9 @@ namespace Umbraco.Web.PublishedCache.NuCache public ContentNode() { FirstChildContentId = -1; + LastChildContentId = -1; NextSiblingContentId = -1; + PreviousSiblingContentId = -1; } // special ctor with no content data - for members @@ -58,7 +60,9 @@ namespace Umbraco.Web.PublishedCache.NuCache SortOrder = sortOrder; ParentContentId = parentContentId; FirstChildContentId = -1; + LastChildContentId = -1; NextSiblingContentId = -1; + PreviousSiblingContentId = -1; CreateDate = createDate; CreatorId = creatorId; } @@ -95,7 +99,9 @@ namespace Umbraco.Web.PublishedCache.NuCache SortOrder = origin.SortOrder; ParentContentId = origin.ParentContentId; FirstChildContentId = origin.FirstChildContentId; + LastChildContentId = origin.LastChildContentId; NextSiblingContentId = origin.NextSiblingContentId; + PreviousSiblingContentId = origin.PreviousSiblingContentId; CreateDate = origin.CreateDate; CreatorId = origin.CreatorId; @@ -119,7 +125,9 @@ namespace Umbraco.Web.PublishedCache.NuCache public readonly int SortOrder; public readonly int ParentContentId; public int FirstChildContentId; + public int LastChildContentId; public int NextSiblingContentId; + public int PreviousSiblingContentId; public readonly DateTime CreateDate; public readonly int CreatorId; diff --git a/src/Umbraco.Web/PublishedCache/NuCache/ContentStore.cs b/src/Umbraco.Web/PublishedCache/NuCache/ContentStore.cs index b9d1e0edf7..d6631b779d 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/ContentStore.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/ContentStore.cs @@ -128,7 +128,8 @@ namespace Umbraco.Web.PublishedCache.NuCache Monitor.Enter(_rlocko, ref rtaken); // see SnapDictionary - try { } finally + try { } + finally { _wlocked++; lockInfo.Count = true; @@ -277,8 +278,8 @@ namespace Umbraco.Web.PublishedCache.NuCache public void UpdateContentTypes(IEnumerable types) { - //nothing to do if this is empty, no need to lock/allocate/iterate/etc... - if (!types.Any()) return; + //nothing to do if this is empty, no need to lock/allocate/iterate/etc... + if (!types.Any()) return; var lockInfo = new WriteLockInfo(); try @@ -550,13 +551,13 @@ namespace Umbraco.Web.PublishedCache.NuCache if (existing == null) { // new, add to parent - AddNodeLocked(kit.Node, parent); + AddTreeNodeLocked(kit.Node, parent); } else if (moving || existing.SortOrder != kit.Node.SortOrder) { // moved, remove existing from its parent, add content to its parent - RemoveNodeLocked(existing); - AddNodeLocked(kit.Node); + RemoveTreeNodeLocked(existing); + AddTreeNodeLocked(kit.Node); } else { @@ -583,13 +584,16 @@ namespace Umbraco.Web.PublishedCache.NuCache } /// - /// Builds all kits without any sorting or generation checks + /// Builds all kits on startup using a fast forward only cursor /// - /// + /// + /// All kits sorted by Level + Parent Id + Sort order + /// /// /// /// This requires that the collection is sorted by Level + ParentId + Sort Order. /// This should be used only on a site startup as the first generations. + /// This CANNOT be used after startup since it bypasses all checks for Generations. /// internal bool SetAllFastSorted(IEnumerable kits) { @@ -602,11 +606,11 @@ namespace Umbraco.Web.PublishedCache.NuCache ClearLocked(_contentNodes); ClearRootLocked(); - //these are ordered by level + sort order - // The name of the game here is to populate each kit's // FirstChildContentId + // LastChildContentId // NextSiblingContentId + // PreviousSiblingContentId ContentNode prev = null; ContentNode currParent = null; @@ -620,20 +624,28 @@ namespace Umbraco.Web.PublishedCache.NuCache } if (currParent != null && currParent.Id != parentLink.Value.Id) - prev = null; //changed parent - + { + //the parent is changing so that means the prev tracked one is the last child + currParent.LastChildContentId = prev.Id; + //changed parent, reset prev + prev = null; + } + currParent = parentLink.Value; _logger.Debug($"Set {kit.Node.Id} with parent {kit.Node.ParentContentId}"); SetValueLocked(_contentNodes, kit.Node.Id, kit.Node); //if the parent's FirstChildContentId isn't set, then it must be the current one - if (parentLink.Value.FirstChildContentId < 0) - parentLink.Value.FirstChildContentId = kit.Node.Id; + if (currParent.FirstChildContentId < 0) + currParent.FirstChildContentId = kit.Node.Id; //if there is a previous one on the same level then set it's next sibling id to the current oen if (prev != null) + { prev.NextSiblingContentId = kit.Node.Id; + kit.Node.PreviousSiblingContentId = prev.Id; + } //store the prev prev = kit.Node; @@ -675,7 +687,7 @@ namespace Umbraco.Web.PublishedCache.NuCache SetValueLocked(_contentNodes, kit.Node.Id, kit.Node); if (_localDb != null) RegisterChange(kit.Node.Id, kit); - AddNodeLocked(kit.Node, parent); + AddTreeNodeLocked(kit.Node, parent); _xmap[kit.Node.Uid] = kit.Node.Id; } @@ -705,7 +717,7 @@ namespace Umbraco.Web.PublishedCache.NuCache if (existing != null) { ClearBranchLocked(existing); - RemoveNodeLocked(existing); + RemoveTreeNodeLocked(existing); } // now add them all back @@ -718,7 +730,7 @@ namespace Umbraco.Web.PublishedCache.NuCache } SetValueLocked(_contentNodes, kit.Node.Id, kit.Node); if (_localDb != null) RegisterChange(kit.Node.Id, kit); - AddNodeLocked(kit.Node, parent); + AddTreeNodeLocked(kit.Node, parent); _xmap[kit.Node.Uid] = kit.Node.Id; } @@ -750,7 +762,7 @@ namespace Umbraco.Web.PublishedCache.NuCache ClearBranchLocked(content); // manage the tree - RemoveNodeLocked(content); + RemoveTreeNodeLocked(content); return true; } @@ -798,6 +810,9 @@ namespace Umbraco.Web.PublishedCache.NuCache throw new PanicException($"failed to get {description} with id={id}"); } + /// + /// Gets the parent link node, may be null or root if ParentContentId is less than 0 + /// private LinkedNode GetParentLink(ContentNode content) { if (content.ParentContentId < 0) return _root; @@ -816,7 +831,7 @@ namespace Umbraco.Web.PublishedCache.NuCache return content.ParentContentId < 0 ? _root : GetRequiredLinkedNode(content.ParentContentId, "parent"); } - private void RemoveNodeLocked(ContentNode content) + private void RemoveTreeNodeLocked(ContentNode content) { var parentLink = content.ParentContentId < 0 ? _root @@ -828,24 +843,38 @@ namespace Umbraco.Web.PublishedCache.NuCache if (parent.FirstChildContentId < 0) throw new PanicException("no children"); - // if first, clone parent + remove first child if (parent.FirstChildContentId == content.Id) { + // if first, clone parent + remove first child parent = GenCloneLocked(parentLink); parent.FirstChildContentId = content.NextSiblingContentId; } - else + + if (parent.LastChildContentId == content.Id) { - // iterate children until the previous child - var link = GetRequiredLinkedNode(parent.FirstChildContentId, "first child"); - - while (link.Value.NextSiblingContentId != content.Id) - link = GetRequiredLinkedNode(link.Value.NextSiblingContentId, "next child"); - - // clone the previous child and replace next child - var prevChild = GenCloneLocked(link); - prevChild.NextSiblingContentId = content.NextSiblingContentId; + // if last, clone parent + remove last child + parent = GenCloneLocked(parentLink); + parent.LastChildContentId = content.PreviousSiblingContentId; } + + // maintain linked list + + if (content.NextSiblingContentId > 0) + { + var nextLink = GetRequiredLinkedNode(content.NextSiblingContentId, "next sibling"); + var next = GenCloneLocked(nextLink); + next.PreviousSiblingContentId = content.PreviousSiblingContentId; + } + + if (content.PreviousSiblingContentId > 0) + { + var prevLink = GetRequiredLinkedNode(content.PreviousSiblingContentId, "previous sibling"); + var prev = GenCloneLocked(prevLink); + prev.NextSiblingContentId = content.NextSiblingContentId; + } + + content.NextSiblingContentId = -1; + content.PreviousSiblingContentId = -1; } private bool ParentPublishedLocked(ContentNodeKit kit) @@ -873,7 +902,10 @@ namespace Umbraco.Web.PublishedCache.NuCache return node; } - private void AddNodeLocked(ContentNode content, LinkedNode parentLink = null) + /// + /// Adds a node to the tree structure. + /// + private void AddTreeNodeLocked(ContentNode content, LinkedNode parentLink = null) { parentLink = parentLink ?? GetRequiredParentLink(content); @@ -884,6 +916,7 @@ namespace Umbraco.Web.PublishedCache.NuCache { parent = GenCloneLocked(parentLink); parent.FirstChildContentId = content.Id; + parent.LastChildContentId = content.Id; return; } @@ -896,12 +929,40 @@ namespace Umbraco.Web.PublishedCache.NuCache if (child.SortOrder > content.SortOrder) { content.NextSiblingContentId = parent.FirstChildContentId; + content.PreviousSiblingContentId = -1; + parent = GenCloneLocked(parentLink); parent.FirstChildContentId = content.Id; + + child = GenCloneLocked(childLink); + child.PreviousSiblingContentId = content.Id; + return; } - // else lookup position + // get parent's last child + var lastChildLink = GetRequiredLinkedNode(parent.LastChildContentId, "last child"); + var lastChild = lastChildLink.Value; + + // if last, clone parent + append as last child + if (lastChild.SortOrder <= content.SortOrder) + { + content.PreviousSiblingContentId = parent.LastChildContentId; + content.NextSiblingContentId = -1; + + parent = GenCloneLocked(parentLink); + parent.LastChildContentId = content.Id; + + lastChild = GenCloneLocked(lastChildLink); + lastChild.NextSiblingContentId = content.Id; + + return; + } + + // else it's going somewhere in the middle, + // and this is bad, perfs-wise - we only do it when moving + // inserting in linked list is slow, optimizing would require trees + // but... that should not happen very often - and not on large amount of data while (child.NextSiblingContentId > 0) { // get next child @@ -913,8 +974,14 @@ namespace Umbraco.Web.PublishedCache.NuCache if (nextChild.SortOrder > content.SortOrder) { content.NextSiblingContentId = nextChild.Id; + content.PreviousSiblingContentId = nextChild.PreviousSiblingContentId; + child = GenCloneLocked(childLink); child.NextSiblingContentId = content.Id; + + var nnext = GenCloneLocked(nextChildLink); + nnext.PreviousSiblingContentId = content.Id; + return; } @@ -922,9 +989,8 @@ namespace Umbraco.Web.PublishedCache.NuCache child = nextChild; } - // if last, clone previous + append - child = GenCloneLocked(childLink); - child.NextSiblingContentId = content.Id; + // should never get here + throw new Exception("panic: no more children."); } // replaces the root node diff --git a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs index 6f60ed86ab..15e6574b40 100755 --- a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs @@ -160,7 +160,7 @@ namespace Umbraco.Web.PublishedCache.NuCache _domainStore = new SnapDictionary(); - publishedModelFactory.WithSafeLiveFactory(LoadCaches); + publishedModelFactory.WithSafeLiveFactory(LoadCachesOnStartup); Guid GetUid(ContentStore store, int id) => store.LiveSnapshot.Get(id)?.Uid ?? default; int GetId(ContentStore store, Guid uid) => store.LiveSnapshot.Get(uid)?.Id ?? default; @@ -172,7 +172,7 @@ namespace Umbraco.Web.PublishedCache.NuCache } } - private void LoadCaches() + private void LoadCachesOnStartup() { lock (_storesLock) { @@ -186,19 +186,19 @@ namespace Umbraco.Web.PublishedCache.NuCache { if (_localDbExists) { - okContent = LockAndLoadContent(LoadContentFromLocalDbLocked); + okContent = LockAndLoadContent(scope => LoadContentFromLocalDbLocked(true)); if (!okContent) _logger.Warn("Loading content from local db raised warnings, will reload from database."); - okMedia = LockAndLoadMedia(LoadMediaFromLocalDbLocked); + okMedia = LockAndLoadMedia(scope => LoadMediaFromLocalDbLocked(true)); if (!okMedia) _logger.Warn("Loading media from local db raised warnings, will reload from database."); } if (!okContent) - LockAndLoadContent(LoadContentFromDatabaseLocked); + LockAndLoadContent(scope => LoadContentFromDatabaseLocked(scope, true)); if (!okMedia) - LockAndLoadMedia(LoadMediaFromDatabaseLocked); + LockAndLoadMedia(scope => LoadMediaFromDatabaseLocked(scope, true)); LockAndLoadDomains(); } @@ -333,7 +333,7 @@ namespace Umbraco.Web.PublishedCache.NuCache } } - private bool LoadContentFromDatabaseLocked(IScope scope) + private bool LoadContentFromDatabaseLocked(IScope scope, bool onStartup) { // locks: // contentStore is wlocked (1 thread) @@ -353,11 +353,11 @@ namespace Umbraco.Web.PublishedCache.NuCache // IMPORTANT GetAllContentSources sorts kits by level + parentId + sortOrder var kits = _dataSource.GetAllContentSources(scope); - return _contentStore.SetAllFastSorted(kits); + return onStartup ? _contentStore.SetAllFastSorted(kits) : _contentStore.SetAll(kits); } } - private bool LoadContentFromLocalDbLocked(IScope scope) + private bool LoadContentFromLocalDbLocked(bool onStartup) { var contentTypes = _serviceContext.ContentTypeService.GetAll() .Select(x => _publishedContentTypeFactory.CreateContentType(x)); @@ -372,7 +372,7 @@ namespace Umbraco.Web.PublishedCache.NuCache .OrderBy(x => x.Node.Level) .ThenBy(x => x.Node.ParentContentId) .ThenBy(x => x.Node.SortOrder); // IMPORTANT sort by level + parentId + sortOrder - return _contentStore.SetAllFastSorted(kits); + return onStartup ? _contentStore.SetAllFastSorted(kits) : _contentStore.SetAll(kits); } } @@ -411,7 +411,7 @@ namespace Umbraco.Web.PublishedCache.NuCache } } - private bool LoadMediaFromDatabaseLocked(IScope scope) + private bool LoadMediaFromDatabaseLocked(IScope scope, bool onStartup) { // locks & notes: see content @@ -429,11 +429,11 @@ namespace Umbraco.Web.PublishedCache.NuCache _logger.Debug("Loading media from database..."); // IMPORTANT GetAllMediaSources sorts kits by level + parentId + sortOrder var kits = _dataSource.GetAllMediaSources(scope); - return _mediaStore.SetAllFastSorted(kits); + return onStartup ? _mediaStore.SetAllFastSorted(kits) : _mediaStore.SetAll(kits); } } - private bool LoadMediaFromLocalDbLocked(IScope scope) + private bool LoadMediaFromLocalDbLocked(bool onStartup) { var mediaTypes = _serviceContext.MediaTypeService.GetAll() .Select(x => _publishedContentTypeFactory.CreateContentType(x)); @@ -448,7 +448,7 @@ namespace Umbraco.Web.PublishedCache.NuCache .OrderBy(x => x.Node.Level) .ThenBy(x => x.Node.ParentContentId) .ThenBy(x => x.Node.SortOrder); // IMPORTANT sort by level + parentId + sortOrder - return _mediaStore.SetAllFastSorted(kits); + return onStartup ? _mediaStore.SetAllFastSorted(kits) : _mediaStore.SetAll(kits); } } @@ -628,7 +628,7 @@ namespace Umbraco.Web.PublishedCache.NuCache using (var scope = _scopeProvider.CreateScope()) { scope.ReadLock(Constants.Locks.ContentTree); - LoadContentFromDatabaseLocked(scope); + LoadContentFromDatabaseLocked(scope, false); scope.Complete(); } draftChanged = publishedChanged = true; @@ -721,7 +721,7 @@ namespace Umbraco.Web.PublishedCache.NuCache using (var scope = _scopeProvider.CreateScope()) { scope.ReadLock(Constants.Locks.MediaTree); - LoadMediaFromDatabaseLocked(scope); + LoadMediaFromDatabaseLocked(scope, false); scope.Complete(); } anythingChanged = true; From f4acb86ac7adec7d94efbaf13a93e88a1c5f4e12 Mon Sep 17 00:00:00 2001 From: Shannon Date: Mon, 19 Aug 2019 23:32:52 +1000 Subject: [PATCH 09/11] fixes tests --- .../PublishedContent/NuCacheTests.cs | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs b/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs index 9f20f42993..44df5c20cb 100644 --- a/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs +++ b/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Data; +using System.Linq; using Moq; using NUnit.Framework; using Umbraco.Core; @@ -128,12 +129,16 @@ namespace Umbraco.Tests.PublishedContent _contentType }; - var contentTypeService = Mock.Of(); - Mock.Get(contentTypeService).Setup(x => x.GetAll()).Returns(contentTypes); - Mock.Get(contentTypeService).Setup(x => x.GetAll(It.IsAny())).Returns(contentTypes); + var contentTypeService = new Mock(); + contentTypeService.Setup(x => x.GetAll()).Returns(contentTypes); + contentTypeService.Setup(x => x.GetAll(It.IsAny())).Returns(contentTypes); - var contentTypeServiceBaseFactory = Mock.Of(); - Mock.Get(contentTypeServiceBaseFactory).Setup(x => x.For(It.IsAny())).Returns(contentTypeService); + var mediaTypeService = new Mock(); + mediaTypeService.Setup(x => x.GetAll()).Returns(Enumerable.Empty()); + mediaTypeService.Setup(x => x.GetAll(It.IsAny())).Returns(Enumerable.Empty()); + + var contentTypeServiceBaseFactory = new Mock(); + contentTypeServiceBaseFactory.Setup(x => x.For(It.IsAny())).Returns(contentTypeService.Object); var dataTypeService = Mock.Of(); Mock.Get(dataTypeService).Setup(x => x.GetAll()).Returns(dataTypes); @@ -143,8 +148,10 @@ namespace Umbraco.Tests.PublishedContent dataTypeService: dataTypeService, memberTypeService: Mock.Of(), memberService: Mock.Of(), - contentTypeService: contentTypeService, - localizationService: Mock.Of() + contentTypeService: contentTypeService.Object, + mediaTypeService: mediaTypeService.Object, + localizationService: Mock.Of(), + domainService: Mock.Of() ); // create a scope provider From 7956ddb15a14acd7a644d6b8c32b937af8427007 Mon Sep 17 00:00:00 2001 From: Shannon Date: Tue, 20 Aug 2019 10:24:07 +1000 Subject: [PATCH 10/11] fixes tests --- src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs b/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs index a9e7a86bf7..d618bcf1ad 100644 --- a/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs +++ b/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs @@ -48,12 +48,10 @@ namespace Umbraco.Tests.Services var runtimeStateMock = new Mock(); runtimeStateMock.Setup(x => x.Level).Returns(() => RuntimeLevel.Run); - var contentTypeFactory = new PublishedContentTypeFactory(Mock.Of(), new PropertyValueConverterCollection(Array.Empty()), Mock.Of()); - //var documentRepository = Mock.Of(); + var contentTypeFactory = Factory.GetInstance(); var documentRepository = Factory.GetInstance(); var mediaRepository = Mock.Of(); var memberRepository = Mock.Of(); - var contentTypeServiceBaseFactory = Current.Services.ContentTypeBaseServices; return new PublishedSnapshotService( options, From 6f4b8dc3819a38c0bde532300f3202d69ade129c Mon Sep 17 00:00:00 2001 From: Shannon Date: Tue, 20 Aug 2019 10:26:01 +1000 Subject: [PATCH 11/11] fixes tests --- src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs b/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs index 8c14ce8219..c7c403b260 100644 --- a/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs +++ b/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs @@ -77,11 +77,10 @@ namespace Umbraco.Tests.Scoping var runtimeStateMock = new Mock(); runtimeStateMock.Setup(x => x.Level).Returns(() => RuntimeLevel.Run); - var contentTypeFactory = new PublishedContentTypeFactory(Mock.Of(), new PropertyValueConverterCollection(Array.Empty()), Mock.Of()); + var contentTypeFactory = Factory.GetInstance(); var documentRepository = Mock.Of(); var mediaRepository = Mock.Of(); var memberRepository = Mock.Of(); - var contentTypeServiceBaseFactory = Current.Services.ContentTypeBaseServices; return new PublishedSnapshotService( options,