yv01p/Umbraco-CMS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #6131 from umbraco/v8/bugfix/5925-nucache-startup

Browse Source 
Adds POC for fast building of in-memory nucache from persisted files
This commit is contained in:
Bjarke Berg
2019-09-09 15:02:49 +02:00
committed by GitHub
parent 8f32fce781 d05e79ca15
commit c121fa323d
14 changed files with 397 additions and 192 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs
View File

@@ -79,12 +79,16 @@ namespace Umbraco.Tests.PublishedContent
_contentTypeVariant
};
var contentTypeService = Mock.Of<IContentTypeService>();
Mock.Get(contentTypeService).Setup(x => x.GetAll()).Returns(contentTypes);
Mock.Get(contentTypeService).Setup(x => x.GetAll(It.IsAny<int[]>())).Returns(contentTypes);
var contentTypeService = new Mock<IContentTypeService>();
contentTypeService.Setup(x => x.GetAll()).Returns(contentTypes);
contentTypeService.Setup(x => x.GetAll(It.IsAny<int[]>())).Returns(contentTypes);
var contentTypeServiceBaseFactory = Mock.Of<IContentTypeBaseServiceProvider>();
Mock.Get(contentTypeServiceBaseFactory).Setup(x => x.For(It.IsAny<IContentBase>())).Returns(contentTypeService);
var mediaTypeService = new Mock<IMediaTypeService>();
mediaTypeService.Setup(x => x.GetAll()).Returns(Enumerable.Empty<IMediaType>());
mediaTypeService.Setup(x => x.GetAll(It.IsAny<int[]>())).Returns(Enumerable.Empty<IMediaType>());
var contentTypeServiceBaseFactory = new Mock<IContentTypeBaseServiceProvider>();
contentTypeServiceBaseFactory.Setup(x => x.For(It.IsAny<IContentBase>())).Returns(contentTypeService.Object);
var dataTypeService = Mock.Of<IDataTypeService>();
Mock.Get(dataTypeService).Setup(x => x.GetAll()).Returns(dataTypes);
@@ -94,8 +98,10 @@ namespace Umbraco.Tests.PublishedContent
dataTypeService: dataTypeService,
memberTypeService: Mock.Of<IMemberTypeService>(),
memberService: Mock.Of<IMemberService>(),
contentTypeService: contentTypeService,
localizationService: Mock.Of<ILocalizationService>()
contentTypeService: contentTypeService.Object,
mediaTypeService: mediaTypeService.Object,
localizationService: Mock.Of<ILocalizationService>(),
domainService: Mock.Of<IDomainService>()
);
// create a scope provider
@@ -133,7 +139,7 @@ namespace Umbraco.Tests.PublishedContent
null,
_snapshotAccessor,
_variationAccesor,
Mock.Of<ILogger>(),
Mock.Of<IProfilingLogger>(),
scopeProvider,
Mock.Of<IDocumentRepository>(),
Mock.Of<IMediaRepository>(),

23
src/Umbraco.Tests/PublishedContent/NuCacheTests.cs
View File

@@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Data;
using System.Linq;
using Moq;
using NUnit.Framework;
using Umbraco.Core;
@@ -128,12 +129,16 @@ namespace Umbraco.Tests.PublishedContent
_contentType
};
var contentTypeService = Mock.Of<IContentTypeService>();
Mock.Get(contentTypeService).Setup(x => x.GetAll()).Returns(contentTypes);
Mock.Get(contentTypeService).Setup(x => x.GetAll(It.IsAny<int[]>())).Returns(contentTypes);
var contentTypeService = new Mock<IContentTypeService>();
contentTypeService.Setup(x => x.GetAll()).Returns(contentTypes);
contentTypeService.Setup(x => x.GetAll(It.IsAny<int[]>())).Returns(contentTypes);
var contentTypeServiceBaseFactory = Mock.Of<IContentTypeBaseServiceProvider>();
Mock.Get(contentTypeServiceBaseFactory).Setup(x => x.For(It.IsAny<IContentBase>())).Returns(contentTypeService);
var mediaTypeService = new Mock<IMediaTypeService>();
mediaTypeService.Setup(x => x.GetAll()).Returns(Enumerable.Empty<IMediaType>());
mediaTypeService.Setup(x => x.GetAll(It.IsAny<int[]>())).Returns(Enumerable.Empty<IMediaType>());
var contentTypeServiceBaseFactory = new Mock<IContentTypeBaseServiceProvider>();
contentTypeServiceBaseFactory.Setup(x => x.For(It.IsAny<IContentBase>())).Returns(contentTypeService.Object);
var dataTypeService = Mock.Of<IDataTypeService>();
Mock.Get(dataTypeService).Setup(x => x.GetAll()).Returns(dataTypes);
@@ -143,8 +148,10 @@ namespace Umbraco.Tests.PublishedContent
dataTypeService: dataTypeService,
memberTypeService: Mock.Of<IMemberTypeService>(),
memberService: Mock.Of<IMemberService>(),
contentTypeService: contentTypeService,
localizationService: Mock.Of<ILocalizationService>()
contentTypeService: contentTypeService.Object,
mediaTypeService: mediaTypeService.Object,
localizationService: Mock.Of<ILocalizationService>(),
domainService: Mock.Of<IDomainService>()
);
// create a scope provider
@@ -178,7 +185,7 @@ namespace Umbraco.Tests.PublishedContent
null,
new TestPublishedSnapshotAccessor(),
_variationAccesor,
Mock.Of<ILogger>(),
Mock.Of<IProfilingLogger>(),
scopeProvider,
Mock.Of<IDocumentRepository>(),
Mock.Of<IMediaRepository>(),

5
src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs
View File

@@ -77,11 +77,10 @@ namespace Umbraco.Tests.Scoping
var runtimeStateMock = new Mock<IRuntimeState>();
runtimeStateMock.Setup(x => x.Level).Returns(() => RuntimeLevel.Run);
var contentTypeFactory = new PublishedContentTypeFactory(Mock.Of<IPublishedModelFactory>(), new PropertyValueConverterCollection(Array.Empty<IPropertyValueConverter>()), Mock.Of<IDataTypeService>());
var contentTypeFactory = Factory.GetInstance<IPublishedContentTypeFactory>();
var documentRepository = Mock.Of<IDocumentRepository>();
var mediaRepository = Mock.Of<IMediaRepository>();
var memberRepository = Mock.Of<IMemberRepository>();
var contentTypeServiceBaseFactory = Current.Services.ContentTypeBaseServices;
return new PublishedSnapshotService(
options,
@@ -92,7 +91,7 @@ namespace Umbraco.Tests.Scoping
null,
publishedSnapshotAccessor,
Mock.Of<IVariationContextAccessor>(),
Logger,
ProfilingLogger,
ScopeProvider,
documentRepository, mediaRepository, memberRepository,
DefaultCultureAccessor,

6
src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs
View File

@@ -48,12 +48,10 @@ namespace Umbraco.Tests.Services
var runtimeStateMock = new Mock<IRuntimeState>();
runtimeStateMock.Setup(x => x.Level).Returns(() => RuntimeLevel.Run);
var contentTypeFactory = new PublishedContentTypeFactory(Mock.Of<IPublishedModelFactory>(), new PropertyValueConverterCollection(Array.Empty<IPropertyValueConverter>()), Mock.Of<IDataTypeService>());
//var documentRepository = Mock.Of<IDocumentRepository>();
var contentTypeFactory = Factory.GetInstance<IPublishedContentTypeFactory>();
var documentRepository = Factory.GetInstance<IDocumentRepository>();
var mediaRepository = Mock.Of<IMediaRepository>();
var memberRepository = Mock.Of<IMemberRepository>();
var contentTypeServiceBaseFactory = Current.Services.ContentTypeBaseServices;
return new PublishedSnapshotService(
options,
@@ -64,7 +62,7 @@ namespace Umbraco.Tests.Services
null,
publishedSnapshotAccessor,
Mock.Of<IVariationContextAccessor>(),
Logger,
ProfilingLogger,
ScopeProvider,
documentRepository, mediaRepository, memberRepository,
DefaultCultureAccessor,

17
src/Umbraco.Tests/Testing/Objects/TestDataSource.cs
View File

@@ -29,37 +29,44 @@ namespace Umbraco.Tests.Testing.Objects
public IEnumerable<ContentNodeKit> GetAllContentSources(IScope scope)
=> Kits.Values
.OrderBy(x => x.Node.Level)
.ThenBy(x => x.Node.ParentContentId)
.ThenBy(x => x.Node.SortOrder)
.Select(x => x.Clone());
public IEnumerable<ContentNodeKit> GetBranchContentSources(IScope scope, int id)
=> Kits.Values
.Where(x => x.Node.Path.EndsWith("," + id) || x.Node.Path.Contains("," + id + ","))
.OrderBy(x => x.Node.Level).ThenBy(x => x.Node.SortOrder)
.OrderBy(x => x.Node.Level)
.ThenBy(x => x.Node.ParentContentId)
.ThenBy(x => x.Node.SortOrder)
.Select(x => x.Clone());
public IEnumerable<ContentNodeKit> GetTypeContentSources(IScope scope, IEnumerable<int> ids)
=> Kits.Values
.Where(x => ids.Contains(x.ContentTypeId))
.OrderBy(x => x.Node.Level)
.ThenBy(x => x.Node.ParentContentId)
.ThenBy(x => x.Node.SortOrder)
.Select(x => x.Clone());
public ContentNodeKit GetMediaSource(IScope scope, int id)
{
throw new NotImplementedException();
return default;
}
public IEnumerable<ContentNodeKit> GetAllMediaSources(IScope scope)
{
throw new NotImplementedException();
return Enumerable.Empty<ContentNodeKit>();
}
public IEnumerable<ContentNodeKit> GetBranchMediaSources(IScope scope, int id)
{
throw new NotImplementedException();
return Enumerable.Empty<ContentNodeKit>();
}
public IEnumerable<ContentNodeKit> GetTypeMediaSources(IScope scope, IEnumerable<int> ids)
{
throw new NotImplementedException();
return Enumerable.Empty<ContentNodeKit>();
}
}
}

8
src/Umbraco.Web/PublishedCache/NuCache/ContentNode.cs
View File

@@ -12,7 +12,9 @@ namespace Umbraco.Web.PublishedCache.NuCache
public ContentNode()
{
FirstChildContentId = -1;
LastChildContentId = -1;
NextSiblingContentId = -1;
PreviousSiblingContentId = -1;
}
// special ctor with no content data - for members
@@ -58,7 +60,9 @@ namespace Umbraco.Web.PublishedCache.NuCache
SortOrder = sortOrder;
ParentContentId = parentContentId;
FirstChildContentId = -1;
LastChildContentId = -1;
NextSiblingContentId = -1;
PreviousSiblingContentId = -1;
CreateDate = createDate;
CreatorId = creatorId;
}
@@ -95,7 +99,9 @@ namespace Umbraco.Web.PublishedCache.NuCache
SortOrder = origin.SortOrder;
ParentContentId = origin.ParentContentId;
FirstChildContentId = origin.FirstChildContentId;
LastChildContentId = origin.LastChildContentId;
NextSiblingContentId = origin.NextSiblingContentId;
PreviousSiblingContentId = origin.PreviousSiblingContentId;
CreateDate = origin.CreateDate;
CreatorId = origin.CreatorId;
@@ -119,7 +125,9 @@ namespace Umbraco.Web.PublishedCache.NuCache
public readonly int SortOrder;
public readonly int ParentContentId;
public int FirstChildContentId;
public int LastChildContentId;
public int NextSiblingContentId;
public int PreviousSiblingContentId;
public readonly DateTime CreateDate;
public readonly int CreatorId;

262
src/Umbraco.Web/PublishedCache/NuCache/ContentStore.cs
View File

@@ -128,7 +128,8 @@ namespace Umbraco.Web.PublishedCache.NuCache
Monitor.Enter(_rlocko, ref rtaken);
// see SnapDictionary
try { } finally
try { }
finally
{
_wlocked++;
lockInfo.Count = true;
@@ -277,8 +278,8 @@ namespace Umbraco.Web.PublishedCache.NuCache
public void UpdateContentTypes(IEnumerable<IPublishedContentType> types)
{
//nothing to do if this is empty, no need to lock/allocate/iterate/etc...
if (!types.Any()) return;
//nothing to do if this is empty, no need to lock/allocate/iterate/etc...
if (!types.Any()) return;
var lockInfo = new WriteLockInfo();
try
@@ -392,7 +393,7 @@ namespace Umbraco.Web.PublishedCache.NuCache
var visited = new List<int>();
foreach (var kit in kits.Where(x =>
refreshedIdsA.Contains(x.ContentTypeId) &&
BuildKit(x)))
BuildKit(x, out _)))
{
// replacing the node: must preserve the parents
var node = GetHead(_contentNodes, kit.Node.Id)?.Value;
@@ -466,10 +467,11 @@ namespace Umbraco.Web.PublishedCache.NuCache
}
}
private bool BuildKit(ContentNodeKit kit)
private bool BuildKit(ContentNodeKit kit, out LinkedNode<ContentNode> parent)
{
// make sure parent exists
if (!ParentExistsLocked(kit))
parent = GetParentLink(kit.Node);
if (parent == null)
{
_logger.Warn<ContentStore>($"Skip item id={kit.Node.Id}, could not find parent id={kit.Node.ParentContentId}.");
return false;
@@ -531,7 +533,7 @@ namespace Umbraco.Web.PublishedCache.NuCache
_contentNodes.TryGetValue(kit.Node.Id, out var link);
var existing = link?.Value;
if (!BuildKit(kit))
if (!BuildKit(kit, out var parent))
return false;
// moving?
@@ -549,13 +551,13 @@ namespace Umbraco.Web.PublishedCache.NuCache
if (existing == null)
{
// new, add to parent
AddNodeLocked(kit.Node);
AddTreeNodeLocked(kit.Node, parent);
}
else if (moving || existing.SortOrder != kit.Node.SortOrder)
{
// moved, remove existing from its parent, add content to its parent
RemoveNodeLocked(existing);
AddNodeLocked(kit.Node);
RemoveTreeNodeLocked(existing);
AddTreeNodeLocked(kit.Node);
}
else
{
@@ -581,8 +583,85 @@ namespace Umbraco.Web.PublishedCache.NuCache
_root.Value.FirstChildContentId = -1;
}
// IMPORTANT kits must be sorted out by LEVEL
public bool SetAll(IEnumerable<ContentNodeKit> kits, bool fromLocalDb = false)
/// <summary>
/// Builds all kits on startup using a fast forward only cursor
/// </summary>
/// <param name="kits">
/// All kits sorted by Level + Parent Id + Sort order
/// </param>
/// <returns></returns>
/// <remarks>
/// This requires that the collection is sorted by Level + ParentId + Sort Order.
/// This should be used only on a site startup as the first generations.
/// This CANNOT be used after startup since it bypasses all checks for Generations.
/// </remarks>
internal bool SetAllFastSorted(IEnumerable<ContentNodeKit> kits)
{
var lockInfo = new WriteLockInfo();
var ok = true;
try
{
Lock(lockInfo);
ClearLocked(_contentNodes);
ClearRootLocked();
// The name of the game here is to populate each kit's
// FirstChildContentId
// LastChildContentId
// NextSiblingContentId
// PreviousSiblingContentId
ContentNode prev = null;
ContentNode currParent = null;
foreach (var kit in kits)
{
if (!BuildKit(kit, out var parentLink))
{
ok = false;
continue; // skip that one
}
if (currParent != null && currParent.Id != parentLink.Value.Id)
{
//the parent is changing so that means the prev tracked one is the last child
currParent.LastChildContentId = prev.Id;
//changed parent, reset prev
prev = null;
}
currParent = parentLink.Value;
_logger.Debug<ContentStore>($"Set {kit.Node.Id} with parent {kit.Node.ParentContentId}");
SetValueLocked(_contentNodes, kit.Node.Id, kit.Node);
//if the parent's FirstChildContentId isn't set, then it must be the current one
if (currParent.FirstChildContentId < 0)
currParent.FirstChildContentId = kit.Node.Id;
//if there is a previous one on the same level then set it's next sibling id to the current oen
if (prev != null)
{
prev.NextSiblingContentId = kit.Node.Id;
kit.Node.PreviousSiblingContentId = prev.Id;
}
//store the prev
prev = kit.Node;
_xmap[kit.Node.Uid] = kit.Node.Id;
}
}
finally
{
Release(lockInfo);
}
return ok;
}
public bool SetAll(IEnumerable<ContentNodeKit> kits)
{
var lockInfo = new WriteLockInfo();
var ok = true;
@@ -599,7 +678,7 @@ namespace Umbraco.Web.PublishedCache.NuCache
foreach (var kit in kits)
{
if (!BuildKit(kit))
if (!BuildKit(kit, out var parent))
{
ok = false;
continue; // skip that one
@@ -607,9 +686,8 @@ namespace Umbraco.Web.PublishedCache.NuCache
_logger.Debug<ContentStore>($"Set {kit.Node.Id} with parent {kit.Node.ParentContentId}");
SetValueLocked(_contentNodes, kit.Node.Id, kit.Node);
// don't refresh _localDb if we are reading from _localDb
if (!fromLocalDb && _localDb != null) RegisterChange(kit.Node.Id, kit);
AddNodeLocked(kit.Node);
if (_localDb != null) RegisterChange(kit.Node.Id, kit);
AddTreeNodeLocked(kit.Node, parent);
_xmap[kit.Node.Uid] = kit.Node.Id;
}
@@ -639,20 +717,20 @@ namespace Umbraco.Web.PublishedCache.NuCache
if (existing != null)
{
ClearBranchLocked(existing);
RemoveNodeLocked(existing);
RemoveTreeNodeLocked(existing);
}
// now add them all back
foreach (var kit in kits)
{
if (!BuildKit(kit))
if (!BuildKit(kit, out var parent))
{
ok = false;
continue; // skip that one
}
SetValueLocked(_contentNodes, kit.Node.Id, kit.Node);
if (_localDb != null) RegisterChange(kit.Node.Id, kit);
AddNodeLocked(kit.Node);
AddTreeNodeLocked(kit.Node, parent);
_xmap[kit.Node.Uid] = kit.Node.Id;
}
@@ -684,7 +762,7 @@ namespace Umbraco.Web.PublishedCache.NuCache
ClearBranchLocked(content);
// manage the tree
RemoveNodeLocked(content);
RemoveTreeNodeLocked(content);
return true;
}
@@ -712,68 +790,91 @@ namespace Umbraco.Web.PublishedCache.NuCache
var id = content.FirstChildContentId;
while (id > 0)
{
var child = GetLinkedNode(id, "child").Value;
ClearBranchLocked(child);
id = child.NextSiblingContentId;
var link = GetRequiredLinkedNode(id, "child");
ClearBranchLocked(link.Value);
id = link.Value.NextSiblingContentId;
}
}
// gets the link node
// throws (panic) if not found, or no value
private LinkedNode<ContentNode> GetLinkedNode(int id, string description)
/// <summary>
/// Gets the link node and if it doesn't exist throw a <see cref="PanicException"/>
/// </summary>
/// <param name="id"></param>
/// <param name="description"></param>
/// <returns></returns>
private LinkedNode<ContentNode> GetRequiredLinkedNode(int id, string description)
{
if (_contentNodes.TryGetValue(id, out var link) && link.Value != null)
return link;
throw new Exception($"panic: failed to get {description} with id={id}");
throw new PanicException($"failed to get {description} with id={id}");
}
/// <summary>
/// Gets the parent link node, may be null or root if ParentContentId is less than 0
/// </summary>
private LinkedNode<ContentNode> GetParentLink(ContentNode content)
{
_contentNodes.TryGetValue(content.ParentContentId, out var link); // else null
//if (link == null || link.Value == null)
// throw new Exception("Panic: parent not found.");
if (content.ParentContentId < 0) return _root;
_contentNodes.TryGetValue(content.ParentContentId, out var link);
return link;
}
private void RemoveNodeLocked(ContentNode content)
/// <summary>
/// Gets the linked parent node and if it doesn't exist throw a <see cref="PanicException"/>
/// </summary>
/// <param name="content"></param>
/// <returns></returns>
private LinkedNode<ContentNode> GetRequiredParentLink(ContentNode content)
{
return content.ParentContentId < 0 ? _root : GetRequiredLinkedNode(content.ParentContentId, "parent");
}
private void RemoveTreeNodeLocked(ContentNode content)
{
var parentLink = content.ParentContentId < 0
? _root
: GetLinkedNode(content.ParentContentId, "parent");
: GetRequiredLinkedNode(content.ParentContentId, "parent");
var parent = parentLink.Value;
// must have children
if (parent.FirstChildContentId < 0)
throw new Exception("panic: no children");
throw new PanicException("no children");
// if first, clone parent + remove first child
if (parent.FirstChildContentId == content.Id)
{
// if first, clone parent + remove first child
parent = GenCloneLocked(parentLink);
parent.FirstChildContentId = content.NextSiblingContentId;
}
else
if (parent.LastChildContentId == content.Id)
{
// iterate children until the previous child
var link = GetLinkedNode(parent.FirstChildContentId, "first child");
while (link.Value.NextSiblingContentId != content.Id)
link = GetLinkedNode(link.Value.NextSiblingContentId, "next child");
// clone the previous child and replace next child
var prevChild = GenCloneLocked(link);
prevChild.NextSiblingContentId = content.NextSiblingContentId;
// if last, clone parent + remove last child
parent = GenCloneLocked(parentLink);
parent.LastChildContentId = content.PreviousSiblingContentId;
}
}
private bool ParentExistsLocked(ContentNodeKit kit)
{
if (kit.Node.ParentContentId < 0)
return true;
var link = GetParentLink(kit.Node);
return link?.Value != null;
// maintain linked list
if (content.NextSiblingContentId > 0)
{
var nextLink = GetRequiredLinkedNode(content.NextSiblingContentId, "next sibling");
var next = GenCloneLocked(nextLink);
next.PreviousSiblingContentId = content.PreviousSiblingContentId;
}
if (content.PreviousSiblingContentId > 0)
{
var prevLink = GetRequiredLinkedNode(content.PreviousSiblingContentId, "previous sibling");
var prev = GenCloneLocked(prevLink);
prev.NextSiblingContentId = content.NextSiblingContentId;
}
content.NextSiblingContentId = -1;
content.PreviousSiblingContentId = -1;
}
private bool ParentPublishedLocked(ContentNodeKit kit)
@@ -801,11 +902,12 @@ namespace Umbraco.Web.PublishedCache.NuCache
return node;
}
private void AddNodeLocked(ContentNode content)
/// <summary>
/// Adds a node to the tree structure.
/// </summary>
private void AddTreeNodeLocked(ContentNode content, LinkedNode<ContentNode> parentLink = null)
{
var parentLink = content.ParentContentId < 0
? _root
: GetLinkedNode(content.ParentContentId, "parent");
parentLink = parentLink ?? GetRequiredParentLink(content);
var parent = parentLink.Value;
@@ -814,35 +916,72 @@ namespace Umbraco.Web.PublishedCache.NuCache
{
parent = GenCloneLocked(parentLink);
parent.FirstChildContentId = content.Id;
parent.LastChildContentId = content.Id;
return;
}
// get parent's first child
var childLink = GetLinkedNode(parent.FirstChildContentId, "first child");
var childLink = GetRequiredLinkedNode(parent.FirstChildContentId, "first child");
var child = childLink.Value;
// if first, clone parent + insert as first child
// NOTE: Don't perform this check if loading from local DB since we know it's already sorted
if (child.SortOrder > content.SortOrder)
{
content.NextSiblingContentId = parent.FirstChildContentId;
content.PreviousSiblingContentId = -1;
parent = GenCloneLocked(parentLink);
parent.FirstChildContentId = content.Id;
child = GenCloneLocked(childLink);
child.PreviousSiblingContentId = content.Id;
return;
}
// else lookup position
// get parent's last child
var lastChildLink = GetRequiredLinkedNode(parent.LastChildContentId, "last child");
var lastChild = lastChildLink.Value;
// if last, clone parent + append as last child
if (lastChild.SortOrder <= content.SortOrder)
{
content.PreviousSiblingContentId = parent.LastChildContentId;
content.NextSiblingContentId = -1;
parent = GenCloneLocked(parentLink);
parent.LastChildContentId = content.Id;
lastChild = GenCloneLocked(lastChildLink);
lastChild.NextSiblingContentId = content.Id;
return;
}
// else it's going somewhere in the middle,
// and this is bad, perfs-wise - we only do it when moving
// inserting in linked list is slow, optimizing would require trees
// but... that should not happen very often - and not on large amount of data
while (child.NextSiblingContentId > 0)
{
// get next child
var nextChildLink = GetLinkedNode(child.NextSiblingContentId, "next child");
var nextChildLink = GetRequiredLinkedNode(child.NextSiblingContentId, "next child");
var nextChild = nextChildLink.Value;
// if here, clone previous + append/insert
// NOTE: Don't perform this check if loading from local DB since we know it's already sorted
if (nextChild.SortOrder > content.SortOrder)
{
content.NextSiblingContentId = nextChild.Id;
content.PreviousSiblingContentId = nextChild.PreviousSiblingContentId;
child = GenCloneLocked(childLink);
child.NextSiblingContentId = content.Id;
var nnext = GenCloneLocked(nextChildLink);
nnext.PreviousSiblingContentId = content.Id;
return;
}
@@ -850,9 +989,8 @@ namespace Umbraco.Web.PublishedCache.NuCache
child = nextChild;
}
// if last, clone previous + append
child = GenCloneLocked(childLink);
child.NextSiblingContentId = content.Id;
// should never get here
throw new Exception("panic: no more children.");
}
// replaces the root node
@@ -946,7 +1084,7 @@ namespace Umbraco.Web.PublishedCache.NuCache
while (id > 0)
{
var link = GetLinkedNode(id, "sibling");
var link = GetRequiredLinkedNode(id, "sibling");
yield return link.Value;
id = link.Value.NextSiblingContentId;
}

2
src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.ContentNodeKitSerializer.cs
View File

@@ -54,4 +54,4 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
DataSerializer.WriteTo(value.PublishedData, stream);
}
}
}
}

16
src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs
View File

@@ -67,7 +67,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
{
var sql = ContentSourcesSelect(scope)
.Where<NodeDto>(x => x.NodeObjectType == Constants.ObjectTypes.Document && x.NodeId == id && !x.Trashed)
.OrderBy<NodeDto>(x => x.Level, x => x.SortOrder);
.OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
var dto = scope.Database.Fetch<ContentSourceDto>(sql).FirstOrDefault();
return dto == null ? new ContentNodeKit() : CreateContentNodeKit(dto);
@@ -77,7 +77,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
{
var sql = ContentSourcesSelect(scope)
.Where<NodeDto>(x => x.NodeObjectType == Constants.ObjectTypes.Document && !x.Trashed)
.OrderBy<NodeDto>(x => x.Level, x => x.SortOrder);
.OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
return scope.Database.Query<ContentSourceDto>(sql).Select(CreateContentNodeKit);
}
@@ -91,7 +91,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
.Where<NodeDto>(x => x.NodeObjectType == Constants.ObjectTypes.Document && !x.Trashed)
.Where<NodeDto>(x => x.NodeId == id, "x")
.OrderBy<NodeDto>(x => x.Level, x => x.SortOrder);
.OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
return scope.Database.Query<ContentSourceDto>(sql).Select(CreateContentNodeKit);
}
@@ -103,7 +103,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
var sql = ContentSourcesSelect(scope)
.Where<NodeDto>(x => x.NodeObjectType == Constants.ObjectTypes.Document && !x.Trashed)
.WhereIn<ContentDto>(x => x.ContentTypeId, ids)
.OrderBy<NodeDto>(x => x.Level, x => x.SortOrder);
.OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
return scope.Database.Query<ContentSourceDto>(sql).Select(CreateContentNodeKit);
}
@@ -140,7 +140,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
{
var sql = MediaSourcesSelect(scope)
.Where<NodeDto>(x => x.NodeObjectType == Constants.ObjectTypes.Media && x.NodeId == id && !x.Trashed)
.OrderBy<NodeDto>(x => x.Level, x => x.SortOrder);
.OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
var dto = scope.Database.Fetch<ContentSourceDto>(sql).FirstOrDefault();
return dto == null ? new ContentNodeKit() : CreateMediaNodeKit(dto);
@@ -150,7 +150,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
{
var sql = MediaSourcesSelect(scope)
.Where<NodeDto>(x => x.NodeObjectType == Constants.ObjectTypes.Media && !x.Trashed)
.OrderBy<NodeDto>(x => x.Level, x => x.SortOrder);
.OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
return scope.Database.Query<ContentSourceDto>(sql).Select(CreateMediaNodeKit);
}
@@ -164,7 +164,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
.Where<NodeDto>(x => x.NodeObjectType == Constants.ObjectTypes.Media && !x.Trashed)
.Where<NodeDto>(x => x.NodeId == id, "x")
.OrderBy<NodeDto>(x => x.Level, x => x.SortOrder);
.OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
return scope.Database.Query<ContentSourceDto>(sql).Select(CreateMediaNodeKit);
}
@@ -176,7 +176,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
var sql = MediaSourcesSelect(scope)
.Where<NodeDto>(x => x.NodeObjectType == Constants.ObjectTypes.Media && !x.Trashed)
.WhereIn<ContentDto>(x => x.ContentTypeId, ids)
.OrderBy<NodeDto>(x => x.Level, x => x.SortOrder);
.OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
return scope.Database.Query<ContentSourceDto>(sql).Select(CreateMediaNodeKit);
}

62
src/Umbraco.Web/PublishedCache/NuCache/DataSource/IDataSource.cs
View File

@@ -8,14 +8,70 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
/// </summary>
internal interface IDataSource
{
//TODO: For these required sort orders, would sorting on Path 'just work'?
ContentNodeKit GetContentSource(IScope scope, int id);
IEnumerable<ContentNodeKit> GetAllContentSources(IScope scope); // must order by level
IEnumerable<ContentNodeKit> GetBranchContentSources(IScope scope, int id); // must order by level, sortOrder
/// <summary>
/// Returns all content ordered by level + sortOrder
/// </summary>
/// <param name="scope"></param>
/// <returns></returns>
/// <remarks>
/// MUST be ordered by level + parentId + sortOrder!
/// </remarks>
IEnumerable<ContentNodeKit> GetAllContentSources(IScope scope);
/// <summary>
/// Returns branch for content ordered by level + sortOrder
/// </summary>
/// <param name="scope"></param>
/// <returns></returns>
/// <remarks>
/// MUST be ordered by level + parentId + sortOrder!
/// </remarks>
IEnumerable<ContentNodeKit> GetBranchContentSources(IScope scope, int id);
/// <summary>
/// Returns content by Ids ordered by level + sortOrder
/// </summary>
/// <param name="scope"></param>
/// <returns></returns>
/// <remarks>
/// MUST be ordered by level + parentId + sortOrder!
/// </remarks>
IEnumerable<ContentNodeKit> GetTypeContentSources(IScope scope, IEnumerable<int> ids);
ContentNodeKit GetMediaSource(IScope scope, int id);
IEnumerable<ContentNodeKit> GetAllMediaSources(IScope scope); // must order by level
/// <summary>
/// Returns all media ordered by level + sortOrder
/// </summary>
/// <param name="scope"></param>
/// <returns></returns>
/// <remarks>
/// MUST be ordered by level + parentId + sortOrder!
/// </remarks>
IEnumerable<ContentNodeKit> GetAllMediaSources(IScope scope);
/// <summary>
/// Returns branch for media ordered by level + sortOrder
/// </summary>
/// <param name="scope"></param>
/// <returns></returns>
/// <remarks>
/// MUST be ordered by level + parentId + sortOrder!
/// </remarks>
IEnumerable<ContentNodeKit> GetBranchMediaSources(IScope scope, int id); // must order by level, sortOrder
/// <summary>
/// Returns media by Ids ordered by level + sortOrder
/// </summary>
/// <param name="scope"></param>
/// <returns></returns>
/// <remarks>
/// MUST be ordered by level + parentId + sortOrder!
/// </remarks>
IEnumerable<ContentNodeKit> GetTypeMediaSources(IScope scope, IEnumerable<int> ids);
}
}

9
src/Umbraco.Web/PublishedCache/NuCache/PublishedContent.cs
View File

@@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.Linq;
using Umbraco.Core;
using Umbraco.Core.Cache;
using Umbraco.Core.Exceptions;
using Umbraco.Core.Models.PublishedContent;
using Umbraco.Web.Composing;
using Umbraco.Web.Models;
@@ -119,7 +120,7 @@ namespace Umbraco.Web.PublishedCache.NuCache
case PublishedItemType.Media:
return GetMediaByIdFunc;
default:
throw new Exception("panic: invalid item type");
throw new PanicException("invalid item type");
}
}
@@ -194,7 +195,7 @@ namespace Umbraco.Web.PublishedCache.NuCache
};
if (ContentData.CultureInfos == null)
throw new Exception("panic: _contentDate.CultureInfos is null.");
throw new PanicException("_contentDate.CultureInfos is null.");
return _cultures = ContentData.CultureInfos
.ToDictionary(x => x.Key, x => new PublishedCultureInfo(x.Key, x.Value.Name, x.Value.UrlSegment, x.Value.Date), StringComparer.OrdinalIgnoreCase);
@@ -286,13 +287,13 @@ namespace Umbraco.Web.PublishedCache.NuCache
{
// but if IsPreviewing is true, we should have a child
if (IsPreviewing)
throw new Exception($"panic: failed to get content with id={id}");
throw new PanicException($"failed to get content with id={id}");
// if IsPreviewing is false, get the unpublished child nevertheless
// we need it to keep enumerating children! but we don't return it
content = getById(publishedSnapshot, true, id);
if (content == null)
throw new Exception($"panic: failed to get content with id={id}");
throw new PanicException($"failed to get content with id={id}");
}
id = UnwrapIPublishedContent(content)._contentNode.NextSiblingContentId;

149
src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs
View File

@@ -37,7 +37,7 @@ namespace Umbraco.Web.PublishedCache.NuCache
private readonly IPublishedContentTypeFactory _publishedContentTypeFactory;
private readonly IScopeProvider _scopeProvider;
private readonly IDataSource _dataSource;
private readonly ILogger _logger;
private readonly IProfilingLogger _logger;
private readonly IDocumentRepository _documentRepository;
private readonly IMediaRepository _mediaRepository;
private readonly IMemberRepository _memberRepository;
@@ -71,7 +71,7 @@ namespace Umbraco.Web.PublishedCache.NuCache
public PublishedSnapshotService(PublishedSnapshotServiceOptions options, IMainDom mainDom, IRuntimeState runtime,
ServiceContext serviceContext, IPublishedContentTypeFactory publishedContentTypeFactory, IdkMap idkMap,
IPublishedSnapshotAccessor publishedSnapshotAccessor, IVariationContextAccessor variationContextAccessor, ILogger logger, IScopeProvider scopeProvider,
IPublishedSnapshotAccessor publishedSnapshotAccessor, IVariationContextAccessor variationContextAccessor, IProfilingLogger logger, IScopeProvider scopeProvider,
IDocumentRepository documentRepository, IMediaRepository mediaRepository, IMemberRepository memberRepository,
IDefaultCultureAccessor defaultCultureAccessor,
IDataSource dataSource, IGlobalSettings globalSettings,
@@ -160,7 +160,7 @@ namespace Umbraco.Web.PublishedCache.NuCache
_domainStore = new SnapDictionary<int, Domain>();
publishedModelFactory.WithSafeLiveFactory(LoadCaches);
publishedModelFactory.WithSafeLiveFactory(LoadCachesOnStartup);
Guid GetUid(ContentStore store, int id) => store.LiveSnapshot.Get(id)?.Uid ?? default;
int GetId(ContentStore store, Guid uid) => store.LiveSnapshot.Get(uid)?.Id ?? default;
@@ -172,38 +172,40 @@ namespace Umbraco.Web.PublishedCache.NuCache
}
}
private void LoadCaches()
private void LoadCachesOnStartup()
{
lock (_storesLock)
{
// populate the stores
var okContent = false;
var okMedia = false;
try
{
var okContent = false;
var okMedia = false;
if (_localDbExists)
{
okContent = LockAndLoadContent(LoadContentFromLocalDbLocked);
okContent = LockAndLoadContent(scope => LoadContentFromLocalDbLocked(true));
if (!okContent)
_logger.Warn<PublishedSnapshotService>("Loading content from local db raised warnings, will reload from database.");
okMedia = LockAndLoadMedia(LoadMediaFromLocalDbLocked);
okMedia = LockAndLoadMedia(scope => LoadMediaFromLocalDbLocked(true));
if (!okMedia)
_logger.Warn<PublishedSnapshotService>("Loading media from local db raised warnings, will reload from database.");
}
if (!okContent)
LockAndLoadContent(LoadContentFromDatabaseLocked);
LockAndLoadContent(scope => LoadContentFromDatabaseLocked(scope, true));
if (!okMedia)
LockAndLoadMedia(LoadMediaFromDatabaseLocked);
LockAndLoadMedia(scope => LoadMediaFromDatabaseLocked(scope, true));
LockAndLoadDomains();
}
catch (Exception ex)
{
_logger.Fatal<PublishedSnapshotService>(ex, "Panic, exception while loading cache data.");
throw;
}
// finally, cache is ready!
@@ -314,22 +316,10 @@ namespace Umbraco.Web.PublishedCache.NuCache
// before I read it? NO! because the WHOLE content tree is read-locked using WithReadLocked.
// don't panic.
private void LockAndLoadContent(Action<IScope> action)
{
// first get a writer, then a scope
// if there already is a scope, the writer will attach to it
// otherwise, it will only exist here - cheap
using (_contentStore.GetScopedWriteLock(_scopeProvider))
using (var scope = _scopeProvider.CreateScope())
{
scope.ReadLock(Constants.Locks.ContentTree);
action(scope);
scope.Complete();
}
}
private bool LockAndLoadContent(Func<IScope, bool> action)
{
// first get a writer, then a scope
// if there already is a scope, the writer will attach to it
// otherwise, it will only exist here - cheap
@@ -343,7 +333,7 @@ namespace Umbraco.Web.PublishedCache.NuCache
}
}
private void LoadContentFromDatabaseLocked(IScope scope)
private bool LoadContentFromDatabaseLocked(IScope scope, bool onStartup)
{
// locks:
// contentStore is wlocked (1 thread)
@@ -351,39 +341,39 @@ namespace Umbraco.Web.PublishedCache.NuCache
var contentTypes = _serviceContext.ContentTypeService.GetAll()
.Select(x => _publishedContentTypeFactory.CreateContentType(x));
_contentStore.SetAllContentTypes(contentTypes);
// beware! at that point the cache is inconsistent,
// assuming we are going to SetAll content items!
using (_logger.TraceDuration<PublishedSnapshotService>("Loading content from database"))
{
// beware! at that point the cache is inconsistent,
// assuming we are going to SetAll content items!
_localContentDb?.Clear();
_localContentDb?.Clear();
_logger.Debug<PublishedSnapshotService>("Loading content from database...");
var sw = Stopwatch.StartNew();
// IMPORTANT GetAllContentSources sorts kits by level
var kits = _dataSource.GetAllContentSources(scope);
_contentStore.SetAll(kits);
sw.Stop();
_logger.Debug<PublishedSnapshotService>("Loaded content from database ({Duration}ms)", sw.ElapsedMilliseconds);
// IMPORTANT GetAllContentSources sorts kits by level + parentId + sortOrder
var kits = _dataSource.GetAllContentSources(scope);
return onStartup ? _contentStore.SetAllFastSorted(kits) : _contentStore.SetAll(kits);
}
}
private bool LoadContentFromLocalDbLocked(IScope scope)
private bool LoadContentFromLocalDbLocked(bool onStartup)
{
var contentTypes = _serviceContext.ContentTypeService.GetAll()
.Select(x => _publishedContentTypeFactory.CreateContentType(x));
.Select(x => _publishedContentTypeFactory.CreateContentType(x));
_contentStore.SetAllContentTypes(contentTypes);
// beware! at that point the cache is inconsistent,
// assuming we are going to SetAll content items!
using (_logger.TraceDuration<PublishedSnapshotService>("Loading content from local cache file"))
{
// beware! at that point the cache is inconsistent,
// assuming we are going to SetAll content items!
_logger.Debug<PublishedSnapshotService>("Loading content from local db...");
var sw = Stopwatch.StartNew();
var kits = _localContentDb.Select(x => x.Value)
.OrderBy(x => x.Node.Level); // IMPORTANT sort by level
var ok = _contentStore.SetAll(kits, true);
sw.Stop();
_logger.Debug<PublishedSnapshotService>("Loaded content from local db ({Duration}ms)", sw.ElapsedMilliseconds);
return ok;
var kits = _localContentDb.Select(x => x.Value)
.OrderBy(x => x.Node.Level)
.ThenBy(x => x.Node.ParentContentId)
.ThenBy(x => x.Node.SortOrder); // IMPORTANT sort by level + parentId + sortOrder
return onStartup ? _contentStore.SetAllFastSorted(kits) : _contentStore.SetAll(kits);
}
}
// keep these around - might be useful
@@ -408,18 +398,6 @@ namespace Umbraco.Web.PublishedCache.NuCache
// _contentStore.Set(contentNode);
//}
private void LockAndLoadMedia(Action<IScope> action)
{
// see note in LockAndLoadContent
using (_mediaStore.GetScopedWriteLock(_scopeProvider))
using (var scope = _scopeProvider.CreateScope())
{
scope.ReadLock(Constants.Locks.MediaTree);
action(scope);
scope.Complete();
}
}
private bool LockAndLoadMedia(Func<IScope, bool> action)
{
// see note in LockAndLoadContent
@@ -433,7 +411,7 @@ namespace Umbraco.Web.PublishedCache.NuCache
}
}
private void LoadMediaFromDatabaseLocked(IScope scope)
private bool LoadMediaFromDatabaseLocked(IScope scope, bool onStartup)
{
// locks & notes: see content
@@ -441,37 +419,38 @@ namespace Umbraco.Web.PublishedCache.NuCache
.Select(x => _publishedContentTypeFactory.CreateContentType(x));
_mediaStore.SetAllContentTypes(mediaTypes);
// beware! at that point the cache is inconsistent,
// assuming we are going to SetAll content items!
using (_logger.TraceDuration<PublishedSnapshotService>("Loading media from database"))
{
// beware! at that point the cache is inconsistent,
// assuming we are going to SetAll content items!
_localMediaDb?.Clear();
_localMediaDb?.Clear();
_logger.Debug<PublishedSnapshotService>("Loading media from database...");
var sw = Stopwatch.StartNew();
// IMPORTANT GetAllMediaSources sorts kits by level
var kits = _dataSource.GetAllMediaSources(scope);
_mediaStore.SetAll(kits);
sw.Stop();
_logger.Debug<PublishedSnapshotService>("Loaded media from database ({Duration}ms)", sw.ElapsedMilliseconds);
_logger.Debug<PublishedSnapshotService>("Loading media from database...");
// IMPORTANT GetAllMediaSources sorts kits by level + parentId + sortOrder
var kits = _dataSource.GetAllMediaSources(scope);
return onStartup ? _mediaStore.SetAllFastSorted(kits) : _mediaStore.SetAll(kits);
}
}
private bool LoadMediaFromLocalDbLocked(IScope scope)
private bool LoadMediaFromLocalDbLocked(bool onStartup)
{
var mediaTypes = _serviceContext.MediaTypeService.GetAll()
.Select(x => _publishedContentTypeFactory.CreateContentType(x));
.Select(x => _publishedContentTypeFactory.CreateContentType(x));
_mediaStore.SetAllContentTypes(mediaTypes);
// beware! at that point the cache is inconsistent,
// assuming we are going to SetAll content items!
using (_logger.TraceDuration<PublishedSnapshotService>("Loading media from local cache file"))
{
// beware! at that point the cache is inconsistent,
// assuming we are going to SetAll content items!
var kits = _localMediaDb.Select(x => x.Value)
.OrderBy(x => x.Node.Level)
.ThenBy(x => x.Node.ParentContentId)
.ThenBy(x => x.Node.SortOrder); // IMPORTANT sort by level + parentId + sortOrder
return onStartup ? _mediaStore.SetAllFastSorted(kits) : _mediaStore.SetAll(kits);
}
_logger.Debug<PublishedSnapshotService>("Loading media from local db...");
var sw = Stopwatch.StartNew();
var kits = _localMediaDb.Select(x => x.Value)
.OrderBy(x => x.Node.Level); // IMPORTANT sort by level
var ok = _mediaStore.SetAll(kits, true);
sw.Stop();
_logger.Debug<PublishedSnapshotService>("Loaded media from local db ({Duration}ms)", sw.ElapsedMilliseconds);
return ok;
}
// keep these around - might be useful
@@ -649,7 +628,7 @@ namespace Umbraco.Web.PublishedCache.NuCache
using (var scope = _scopeProvider.CreateScope())
{
scope.ReadLock(Constants.Locks.ContentTree);
LoadContentFromDatabaseLocked(scope);
LoadContentFromDatabaseLocked(scope, false);
scope.Complete();
}
draftChanged = publishedChanged = true;
@@ -742,7 +721,7 @@ namespace Umbraco.Web.PublishedCache.NuCache
using (var scope = _scopeProvider.CreateScope())
{
scope.ReadLock(Constants.Locks.MediaTree);
LoadMediaFromDatabaseLocked(scope);
LoadMediaFromDatabaseLocked(scope, false);
scope.Complete();
}
anythingChanged = true;

6
src/Umbraco.Web/PublishedCache/NuCache/Snap/LinkedNode.cs
View File

@@ -1,5 +1,11 @@
namespace Umbraco.Web.PublishedCache.NuCache.Snap
{
//NOTE: This cannot be struct because it references itself
/// <summary>
/// Used to represent an item in a linked list
/// </summary>
/// <typeparam name="TValue"></typeparam>
internal class LinkedNode<TValue>
where TValue : class
{

2
src/Umbraco.Web/UmbracoInjectedModule.cs
View File

@@ -264,7 +264,7 @@ namespace Umbraco.Web
case RuntimeLevel.Unknown:
case RuntimeLevel.Boot:
case RuntimeLevel.BootFailed:
throw new Exception($"panic: Unexpected runtime level: {level}.");
throw new PanicException($"Unexpected runtime level: {level}.");
case RuntimeLevel.Run:
// ok