yv01p/Umbraco-CMS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Adds POC for fast building of in-memory nucache from persisted files

Browse Source
This commit is contained in:
Shannon
2019-08-15 19:05:43 +10:00
parent 9d77114d50
commit e9d4b607bb
7 changed files with 167 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs
View File

@@ -133,7 +133,7 @@ namespace Umbraco.Tests.PublishedContent
null,
_snapshotAccessor,
_variationAccesor,
Mock.Of<ILogger>(),
Mock.Of<IProfilingLogger>(),
scopeProvider,
Mock.Of<IDocumentRepository>(),
Mock.Of<IMediaRepository>(),

2
src/Umbraco.Tests/PublishedContent/NuCacheTests.cs
View File

@@ -178,7 +178,7 @@ namespace Umbraco.Tests.PublishedContent
null,
new TestPublishedSnapshotAccessor(),
_variationAccesor,
Mock.Of<ILogger>(),
Mock.Of<IProfilingLogger>(),
scopeProvider,
Mock.Of<IDocumentRepository>(),
Mock.Of<IMediaRepository>(),

2
src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs
View File

@@ -92,7 +92,7 @@ namespace Umbraco.Tests.Scoping
null,
publishedSnapshotAccessor,
Mock.Of<IVariationContextAccessor>(),
Logger,
ProfilingLogger,
ScopeProvider,
documentRepository, mediaRepository, memberRepository,
DefaultCultureAccessor,

2
src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs
View File

@@ -64,7 +64,7 @@ namespace Umbraco.Tests.Services
null,
publishedSnapshotAccessor,
Mock.Of<IVariationContextAccessor>(),
Logger,
ProfilingLogger,
ScopeProvider,
documentRepository, mediaRepository, memberRepository,
DefaultCultureAccessor,

146
src/Umbraco.Web/PublishedCache/NuCache/ContentStore.cs
View File

@@ -392,7 +392,7 @@ namespace Umbraco.Web.PublishedCache.NuCache
var visited = new List<int>();
foreach (var kit in kits.Where(x =>
refreshedIdsA.Contains(x.ContentTypeId) &&
BuildKit(x)))
BuildKit(x, out _)))
{
// replacing the node: must preserve the parents
var node = GetHead(_contentNodes, kit.Node.Id)?.Value;
@@ -466,10 +466,11 @@ namespace Umbraco.Web.PublishedCache.NuCache
}
}
private bool BuildKit(ContentNodeKit kit)
private bool BuildKit(ContentNodeKit kit, out LinkedNode<ContentNode> parent)
{
// make sure parent exists
if (!ParentExistsLocked(kit))
parent = GetParentLink(kit.Node);
if (parent == null)
{
_logger.Warn<ContentStore>($"Skip item id={kit.Node.Id}, could not find parent id={kit.Node.ParentContentId}.");
return false;
@@ -531,7 +532,7 @@ namespace Umbraco.Web.PublishedCache.NuCache
_contentNodes.TryGetValue(kit.Node.Id, out var link);
var existing = link?.Value;
if (!BuildKit(kit))
if (!BuildKit(kit, out var parent))
return false;
// moving?
@@ -549,7 +550,7 @@ namespace Umbraco.Web.PublishedCache.NuCache
if (existing == null)
{
// new, add to parent
AddNodeLocked(kit.Node);
AddNodeLocked(kit.Node, parent);
}
else if (moving || existing.SortOrder != kit.Node.SortOrder)
{
@@ -581,7 +582,74 @@ namespace Umbraco.Web.PublishedCache.NuCache
_root.Value.FirstChildContentId = -1;
}
// IMPORTANT kits must be sorted out by LEVEL
/// <summary>
/// Builds all kits without any sorting or generation checks
/// </summary>
/// <param name="kits"></param>
/// <returns></returns>
/// <remarks>
/// This requires that the collection is sorted by Level + Sort Order.
/// This should be used only on a site startup as the first generations.
/// </remarks>
internal bool SetAllFastSorted(IEnumerable<ContentNodeKit> kits)
{
var lockInfo = new WriteLockInfo();
var ok = true;
try
{
Lock(lockInfo);
ClearLocked(_contentNodes);
ClearRootLocked();
//these are ordered by level + sort order
// The name of the game here is to populate each kit's
// FirstChildContentId
// NextSiblingContentId
ContentNode prev = null;
var currLevel = 0;
foreach (var kit in kits)
{
if (currLevel != kit.Node.Level)
{
prev = null; //reset since we're on a new level
currLevel = kit.Node.Level;
}
if (!BuildKit(kit, out var parentLink))
{
ok = false;
continue; // skip that one
}
_logger.Debug<ContentStore>($"Set {kit.Node.Id} with parent {kit.Node.ParentContentId}");
SetValueLocked(_contentNodes, kit.Node.Id, kit.Node);
//if the parent's FirstChildContentId isn't set, then it must be the current one
if (parentLink.Value.FirstChildContentId < 0)
parentLink.Value.FirstChildContentId = kit.Node.Id;
//if there is a previous one on the same level then set it's next sibling id to the current oen
if (prev != null)
prev.NextSiblingContentId = kit.Node.Id;
//store the prev
prev = kit.Node;
_xmap[kit.Node.Uid] = kit.Node.Id;
}
}
finally
{
Release(lockInfo);
}
return ok;
}
public bool SetAll(IEnumerable<ContentNodeKit> kits, bool fromLocalDb = false)
{
var lockInfo = new WriteLockInfo();
@@ -599,7 +667,7 @@ namespace Umbraco.Web.PublishedCache.NuCache
foreach (var kit in kits)
{
if (!BuildKit(kit))
if (!BuildKit(kit, out var parent))
{
ok = false;
continue; // skip that one
@@ -609,7 +677,7 @@ namespace Umbraco.Web.PublishedCache.NuCache
// don't refresh _localDb if we are reading from _localDb
if (!fromLocalDb && _localDb != null) RegisterChange(kit.Node.Id, kit);
AddNodeLocked(kit.Node);
AddNodeLocked(kit.Node, parent);
_xmap[kit.Node.Uid] = kit.Node.Id;
}
@@ -645,14 +713,14 @@ namespace Umbraco.Web.PublishedCache.NuCache
// now add them all back
foreach (var kit in kits)
{
if (!BuildKit(kit))
if (!BuildKit(kit, out var parent))
{
ok = false;
continue; // skip that one
}
SetValueLocked(_contentNodes, kit.Node.Id, kit.Node);
if (_localDb != null) RegisterChange(kit.Node.Id, kit);
AddNodeLocked(kit.Node);
AddNodeLocked(kit.Node, parent);
_xmap[kit.Node.Uid] = kit.Node.Id;
}
@@ -712,35 +780,49 @@ namespace Umbraco.Web.PublishedCache.NuCache
var id = content.FirstChildContentId;
while (id > 0)
{
var link = GetLinkedNode(id, "child");
var link = GetRequiredLinkedNode(id, "child");
ClearBranchLocked(link.Value);
id = link.Value.NextSiblingContentId;
}
}
// gets the link node
// throws (panic) if not found, or no value
private LinkedNode<ContentNode> GetLinkedNode(int id, string description)
/// <summary>
/// Gets the link node and if it doesn't exist throw a <see cref="PanicException"/>
/// </summary>
/// <param name="id"></param>
/// <param name="description"></param>
/// <returns></returns>
private LinkedNode<ContentNode> GetRequiredLinkedNode(int id, string description)
{
if (_contentNodes.TryGetValue(id, out var link) && link.Value != null)
return link;
throw new Exception($"panic: failed to get {description} with id={id}");
throw new PanicException($"panic: failed to get {description} with id={id}");
}
private LinkedNode<ContentNode> GetParentLink(ContentNode content)
{
_contentNodes.TryGetValue(content.ParentContentId, out var link); // else null
//if (link == null || link.Value == null)
// throw new Exception("Panic: parent not found.");
if (content.ParentContentId < 0) return _root;
_contentNodes.TryGetValue(content.ParentContentId, out var link);
return link;
}
/// <summary>
/// Gets the linked parent node and if it doesn't exist throw a <see cref="PanicException"/>
/// </summary>
/// <param name="content"></param>
/// <returns></returns>
private LinkedNode<ContentNode> GetRequiredParentLink(ContentNode content)
{
return content.ParentContentId < 0 ? _root : GetRequiredLinkedNode(content.ParentContentId, "parent");
}
private void RemoveNodeLocked(ContentNode content)
{
var parentLink = content.ParentContentId < 0
? _root
: GetLinkedNode(content.ParentContentId, "parent");
: GetRequiredLinkedNode(content.ParentContentId, "parent");
var parent = parentLink.Value;
@@ -757,10 +839,10 @@ namespace Umbraco.Web.PublishedCache.NuCache
else
{
// iterate children until the previous child
var link = GetLinkedNode(parent.FirstChildContentId, "first child");
var link = GetRequiredLinkedNode(parent.FirstChildContentId, "first child");
while (link.Value.NextSiblingContentId != content.Id)
link = GetLinkedNode(link.Value.NextSiblingContentId, "next child");
link = GetRequiredLinkedNode(link.Value.NextSiblingContentId, "next child");
// clone the previous child and replace next child
var prevChild = GenCloneLocked(link);
@@ -768,14 +850,6 @@ namespace Umbraco.Web.PublishedCache.NuCache
}
}
private bool ParentExistsLocked(ContentNodeKit kit)
{
if (kit.Node.ParentContentId < 0)
return true;
var link = GetParentLink(kit.Node);
return link?.Value != null;
}
private bool ParentPublishedLocked(ContentNodeKit kit)
{
if (kit.Node.ParentContentId < 0)
@@ -801,11 +875,9 @@ namespace Umbraco.Web.PublishedCache.NuCache
return node;
}
private void AddNodeLocked(ContentNode content)
private void AddNodeLocked(ContentNode content, LinkedNode<ContentNode> parentLink = null)
{
var parentLink = content.ParentContentId < 0
? _root
: GetLinkedNode(content.ParentContentId, "parent");
parentLink = parentLink ?? GetRequiredParentLink(content);
var parent = parentLink.Value;
@@ -818,10 +890,11 @@ namespace Umbraco.Web.PublishedCache.NuCache
}
// get parent's first child
var childLink = GetLinkedNode(parent.FirstChildContentId, "first child");
var childLink = GetRequiredLinkedNode(parent.FirstChildContentId, "first child");
var child = childLink.Value;
// if first, clone parent + insert as first child
// NOTE: Don't perform this check if loading from local DB since we know it's already sorted
if (child.SortOrder > content.SortOrder)
{
content.NextSiblingContentId = parent.FirstChildContentId;
@@ -834,10 +907,11 @@ namespace Umbraco.Web.PublishedCache.NuCache
while (child.NextSiblingContentId > 0)
{
// get next child
var nextChildLink = GetLinkedNode(child.NextSiblingContentId, "next child");
var nextChildLink = GetRequiredLinkedNode(child.NextSiblingContentId, "next child");
var nextChild = nextChildLink.Value;
// if here, clone previous + append/insert
// NOTE: Don't perform this check if loading from local DB since we know it's already sorted
if (nextChild.SortOrder > content.SortOrder)
{
content.NextSiblingContentId = nextChild.Id;
@@ -946,7 +1020,7 @@ namespace Umbraco.Web.PublishedCache.NuCache
while (id > 0)
{
var link = GetLinkedNode(id, "sibling");
var link = GetRequiredLinkedNode(id, "sibling");
yield return link.Value;
id = link.Value.NextSiblingContentId;
}

119
src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs
View File

@@ -37,7 +37,7 @@ namespace Umbraco.Web.PublishedCache.NuCache
private readonly IPublishedContentTypeFactory _publishedContentTypeFactory;
private readonly IScopeProvider _scopeProvider;
private readonly IDataSource _dataSource;
private readonly ILogger _logger;
private readonly IProfilingLogger _logger;
private readonly IDocumentRepository _documentRepository;
private readonly IMediaRepository _mediaRepository;
private readonly IMemberRepository _memberRepository;
@@ -71,7 +71,7 @@ namespace Umbraco.Web.PublishedCache.NuCache
public PublishedSnapshotService(PublishedSnapshotServiceOptions options, IMainDom mainDom, IRuntimeState runtime,
ServiceContext serviceContext, IPublishedContentTypeFactory publishedContentTypeFactory, IdkMap idkMap,
IPublishedSnapshotAccessor publishedSnapshotAccessor, IVariationContextAccessor variationContextAccessor, ILogger logger, IScopeProvider scopeProvider,
IPublishedSnapshotAccessor publishedSnapshotAccessor, IVariationContextAccessor variationContextAccessor, IProfilingLogger logger, IScopeProvider scopeProvider,
IDocumentRepository documentRepository, IMediaRepository mediaRepository, IMemberRepository memberRepository,
IDefaultCultureAccessor defaultCultureAccessor,
IDataSource dataSource, IGlobalSettings globalSettings,
@@ -314,22 +314,10 @@ namespace Umbraco.Web.PublishedCache.NuCache
// before I read it? NO! because the WHOLE content tree is read-locked using WithReadLocked.
// don't panic.
private void LockAndLoadContent(Action<IScope> action)
{
// first get a writer, then a scope
// if there already is a scope, the writer will attach to it
// otherwise, it will only exist here - cheap
using (_contentStore.GetScopedWriteLock(_scopeProvider))
using (var scope = _scopeProvider.CreateScope())
{
scope.ReadLock(Constants.Locks.ContentTree);
action(scope);
scope.Complete();
}
}
private bool LockAndLoadContent(Func<IScope, bool> action)
{
// first get a writer, then a scope
// if there already is a scope, the writer will attach to it
// otherwise, it will only exist here - cheap
@@ -343,7 +331,7 @@ namespace Umbraco.Web.PublishedCache.NuCache
}
}
private void LoadContentFromDatabaseLocked(IScope scope)
private bool LoadContentFromDatabaseLocked(IScope scope)
{
// locks:
// contentStore is wlocked (1 thread)
@@ -351,39 +339,38 @@ namespace Umbraco.Web.PublishedCache.NuCache
var contentTypes = _serviceContext.ContentTypeService.GetAll()
.Select(x => _publishedContentTypeFactory.CreateContentType(x));
_contentStore.SetAllContentTypes(contentTypes);
// beware! at that point the cache is inconsistent,
// assuming we are going to SetAll content items!
using (_logger.TraceDuration<PublishedSnapshotService>("Loading content from database"))
{
// beware! at that point the cache is inconsistent,
// assuming we are going to SetAll content items!
_localContentDb?.Clear();
_localContentDb?.Clear();
_logger.Debug<PublishedSnapshotService>("Loading content from database...");
var sw = Stopwatch.StartNew();
// IMPORTANT GetAllContentSources sorts kits by level
var kits = _dataSource.GetAllContentSources(scope);
_contentStore.SetAll(kits);
sw.Stop();
_logger.Debug<PublishedSnapshotService>("Loaded content from database ({Duration}ms)", sw.ElapsedMilliseconds);
// IMPORTANT GetAllContentSources sorts kits by level
var kits = _dataSource.GetAllContentSources(scope);
return _contentStore.SetAll(kits, false);
}
}
private bool LoadContentFromLocalDbLocked(IScope scope)
{
var contentTypes = _serviceContext.ContentTypeService.GetAll()
.Select(x => _publishedContentTypeFactory.CreateContentType(x));
.Select(x => _publishedContentTypeFactory.CreateContentType(x));
_contentStore.SetAllContentTypes(contentTypes);
// beware! at that point the cache is inconsistent,
// assuming we are going to SetAll content items!
using (_logger.TraceDuration<PublishedSnapshotService>("Loading content from local cache file"))
{
// beware! at that point the cache is inconsistent,
// assuming we are going to SetAll content items!
_logger.Debug<PublishedSnapshotService>("Loading content from local db...");
var sw = Stopwatch.StartNew();
var kits = _localContentDb.Select(x => x.Value)
.OrderBy(x => x.Node.Level); // IMPORTANT sort by level
var ok = _contentStore.SetAll(kits, true);
sw.Stop();
_logger.Debug<PublishedSnapshotService>("Loaded content from local db ({Duration}ms)", sw.ElapsedMilliseconds);
return ok;
var kits = _localContentDb.Select(x => x.Value)
.OrderBy(x => x.Node.Level)
.ThenBy(x => x.Node.SortOrder); // IMPORTANT sort by level + order
return _contentStore.SetAllFastSorted(kits);
}
}
// keep these around - might be useful
@@ -408,18 +395,6 @@ namespace Umbraco.Web.PublishedCache.NuCache
// _contentStore.Set(contentNode);
//}
private void LockAndLoadMedia(Action<IScope> action)
{
// see note in LockAndLoadContent
using (_mediaStore.GetScopedWriteLock(_scopeProvider))
using (var scope = _scopeProvider.CreateScope())
{
scope.ReadLock(Constants.Locks.MediaTree);
action(scope);
scope.Complete();
}
}
private bool LockAndLoadMedia(Func<IScope, bool> action)
{
// see note in LockAndLoadContent
@@ -433,7 +408,7 @@ namespace Umbraco.Web.PublishedCache.NuCache
}
}
private void LoadMediaFromDatabaseLocked(IScope scope)
private bool LoadMediaFromDatabaseLocked(IScope scope)
{
// locks & notes: see content
@@ -441,37 +416,37 @@ namespace Umbraco.Web.PublishedCache.NuCache
.Select(x => _publishedContentTypeFactory.CreateContentType(x));
_mediaStore.SetAllContentTypes(mediaTypes);
// beware! at that point the cache is inconsistent,
// assuming we are going to SetAll content items!
using (_logger.TraceDuration<PublishedSnapshotService>("Loading media from database"))
{
// beware! at that point the cache is inconsistent,
// assuming we are going to SetAll content items!
_localMediaDb?.Clear();
_localMediaDb?.Clear();
_logger.Debug<PublishedSnapshotService>("Loading media from database...");
var sw = Stopwatch.StartNew();
// IMPORTANT GetAllMediaSources sorts kits by level
var kits = _dataSource.GetAllMediaSources(scope);
_mediaStore.SetAll(kits);
sw.Stop();
_logger.Debug<PublishedSnapshotService>("Loaded media from database ({Duration}ms)", sw.ElapsedMilliseconds);
_logger.Debug<PublishedSnapshotService>("Loading media from database...");
// IMPORTANT GetAllMediaSources sorts kits by level
var kits = _dataSource.GetAllMediaSources(scope);
return _mediaStore.SetAll(kits);
}
}
private bool LoadMediaFromLocalDbLocked(IScope scope)
{
var mediaTypes = _serviceContext.MediaTypeService.GetAll()
.Select(x => _publishedContentTypeFactory.CreateContentType(x));
.Select(x => _publishedContentTypeFactory.CreateContentType(x));
_mediaStore.SetAllContentTypes(mediaTypes);
// beware! at that point the cache is inconsistent,
// assuming we are going to SetAll content items!
using (_logger.TraceDuration<PublishedSnapshotService>("Loading media from local cache file"))
{
// beware! at that point the cache is inconsistent,
// assuming we are going to SetAll content items!
_logger.Debug<PublishedSnapshotService>("Loading media from local db...");
var sw = Stopwatch.StartNew();
var kits = _localMediaDb.Select(x => x.Value)
.OrderBy(x => x.Node.Level); // IMPORTANT sort by level
var ok = _mediaStore.SetAll(kits, true);
sw.Stop();
_logger.Debug<PublishedSnapshotService>("Loaded media from local db ({Duration}ms)", sw.ElapsedMilliseconds);
return ok;
var kits = _localMediaDb.Select(x => x.Value)
.OrderBy(x => x.Node.Level)
.ThenBy(x => x.Node.SortOrder); // IMPORTANT sort by level + order;
return _mediaStore.SetAllFastSorted(kits);
}
}
// keep these around - might be useful

6
src/Umbraco.Web/PublishedCache/NuCache/Snap/LinkedNode.cs
View File

@@ -1,5 +1,11 @@
namespace Umbraco.Web.PublishedCache.NuCache.Snap
{
//NOTE: This cannot be struct because it references itself
/// <summary>
/// Used to represent an item in a linked list
/// </summary>
/// <typeparam name="TValue"></typeparam>
internal class LinkedNode<TValue>
where TValue : class
{