yv01p/Umbraco-CMS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

V16: Implement cache tags (#19101)

Browse Source 
* Implement tags for content cache

* Implement tags for media cache

* Refactor to only use cache and media tags

* Remove from DI

* Cleanup

* Update Nuget packages

* Apply suggestions from code review

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Change description to be more precise

* Minor code tidy: indents, static methods where possible, made tags methods a little terser.

* Fixed according to review

---------

Co-authored-by: Elitsa <elm@umbraco.dk>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Andy Butland <abutland73@gmail.com>
This commit is contained in:
Nikolaj Geisle
2025-04-23 10:08:08 +02:00
committed by GitHub
parent 4114342de2
commit d568e981ab
10 changed files with 84 additions and 176 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
Directory.Packages.props
View File

@@ -18,22 +18,22 @@
<PackageVersion Include="Microsoft.EntityFrameworkCore.Sqlite" Version="9.0.2" />
<PackageVersion Include="Microsoft.EntityFrameworkCore.SqlServer" Version="9.0.2" />
<PackageVersion Include="Microsoft.EntityFrameworkCore.Design" Version="8.0.10" />
<PackageVersion Include="Microsoft.Extensions.Caching.Abstractions" Version="9.0.2" />
<PackageVersion Include="Microsoft.Extensions.Caching.Memory" Version="9.0.2" />
<PackageVersion Include="Microsoft.Extensions.Configuration.Abstractions" Version="9.0.2" />
<PackageVersion Include="Microsoft.Extensions.Configuration.Json" Version="9.0.2" />
<PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="9.0.2" />
<PackageVersion Include="Microsoft.Extensions.FileProviders.Embedded" Version="9.0.2" />
<PackageVersion Include="Microsoft.Extensions.FileProviders.Physical" Version="9.0.2" />
<PackageVersion Include="Microsoft.Extensions.Hosting.Abstractions" Version="9.0.2" />
<PackageVersion Include="Microsoft.Extensions.Http" Version="9.0.2" />
<PackageVersion Include="Microsoft.Extensions.Identity.Core" Version="9.0.2" />
<PackageVersion Include="Microsoft.Extensions.Identity.Stores" Version="9.0.2" />
<PackageVersion Include="Microsoft.Extensions.Logging" Version="9.0.2" />
<PackageVersion Include="Microsoft.Extensions.Options" Version="9.0.2" />
<PackageVersion Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="9.0.2" />
<PackageVersion Include="Microsoft.Extensions.Options.DataAnnotations" Version="9.0.2" />
<PackageVersion Include="Microsoft.Extensions.Caching.Hybrid" Version="9.0.0-preview.9.24556.5" />
<PackageVersion Include="Microsoft.Extensions.Caching.Abstractions" Version="9.0.4" />
<PackageVersion Include="Microsoft.Extensions.Caching.Memory" Version="9.0.4" />
<PackageVersion Include="Microsoft.Extensions.Configuration.Abstractions" Version="9.0.4" />
<PackageVersion Include="Microsoft.Extensions.Configuration.Json" Version="9.0.4" />
<PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="9.0.4" />
<PackageVersion Include="Microsoft.Extensions.FileProviders.Embedded" Version="9.0.4" />
<PackageVersion Include="Microsoft.Extensions.FileProviders.Physical" Version="9.0.4" />
<PackageVersion Include="Microsoft.Extensions.Hosting.Abstractions" Version="9.0.4" />
<PackageVersion Include="Microsoft.Extensions.Http" Version="9.0.4" />
<PackageVersion Include="Microsoft.Extensions.Identity.Core" Version="9.0.4" />
<PackageVersion Include="Microsoft.Extensions.Identity.Stores" Version="9.0.4" />
<PackageVersion Include="Microsoft.Extensions.Logging" Version="9.0.4" />
<PackageVersion Include="Microsoft.Extensions.Options" Version="9.0.4" />
<PackageVersion Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="9.0.4" />
<PackageVersion Include="Microsoft.Extensions.Options.DataAnnotations" Version="9.0.4" />
<PackageVersion Include="Microsoft.Extensions.Caching.Hybrid" Version="9.4.0" />
</ItemGroup>
<!-- Umbraco packages -->
<ItemGroup>

13
src/Umbraco.Core/Constants-Cache.cs Normal file
View File

@@ -0,0 +1,13 @@
namespace Umbraco.Cms.Core;
public static partial class Constants
{
public static class Cache
{
public static class Tags
{
public const string Content = "content";
public const string Media = "media";
}
}
}

66
src/Umbraco.PublishedCache.HybridCache/Persistence/DatabaseCacheRepository.cs
View File

@@ -123,72 +123,6 @@ internal sealed class DatabaseCacheRepository : RepositoryBase, IDatabaseCacheRe
}
// assumes content tree lock
public bool VerifyContentDbCache()
{
// every document should have a corresponding row for edited properties
// and if published, may have a corresponding row for published properties
Guid contentObjectType = Constants.ObjectTypes.Document;
var count = Database.ExecuteScalar<int>(
$@"SELECT COUNT(*)
FROM umbracoNode
JOIN {Constants.DatabaseSchema.Tables.Document} ON umbracoNode.id={Constants.DatabaseSchema.Tables.Document}.nodeId
LEFT JOIN cmsContentNu nuEdited ON (umbracoNode.id=nuEdited.nodeId AND nuEdited.published=0)
LEFT JOIN cmsContentNu nuPublished ON (umbracoNode.id=nuPublished.nodeId AND nuPublished.published=1)
WHERE umbracoNode.nodeObjectType=@objType
AND nuEdited.nodeId IS NULL OR ({Constants.DatabaseSchema.Tables.Document}.published=1 AND nuPublished.nodeId IS NULL);",
new { objType = contentObjectType });
return count == 0;
}
// assumes media tree lock
public bool VerifyMediaDbCache()
{
// every media item should have a corresponding row for edited properties
Guid mediaObjectType = Constants.ObjectTypes.Media;
var count = Database.ExecuteScalar<int>(
@"SELECT COUNT(*)
FROM umbracoNode
LEFT JOIN cmsContentNu ON (umbracoNode.id=cmsContentNu.nodeId AND cmsContentNu.published=0)
WHERE umbracoNode.nodeObjectType=@objType
AND cmsContentNu.nodeId IS NULL
",
new { objType = mediaObjectType });
return count == 0;
}
public async Task<IEnumerable<Guid>> GetContentKeysAsync(Guid nodeObjectType)
{
Sql<ISqlContext> sql = Sql()
.Select<NodeDto>(x => x.UniqueId)
.From<NodeDto>()
.Where<NodeDto>(x => x.NodeObjectType == nodeObjectType);
return await Database.FetchAsync<Guid>(sql);
}
// assumes member tree lock
public bool VerifyMemberDbCache()
{
// every member item should have a corresponding row for edited properties
Guid memberObjectType = Constants.ObjectTypes.Member;
var count = Database.ExecuteScalar<int>(
@"SELECT COUNT(*)
FROM umbracoNode
LEFT JOIN cmsContentNu ON (umbracoNode.id=cmsContentNu.nodeId AND cmsContentNu.published=0)
WHERE umbracoNode.nodeObjectType=@objType
AND cmsContentNu.nodeId IS NULL
",
new { objType = memberObjectType });
return count == 0;
}
public async Task<ContentCacheNode?> GetContentSourceAsync(Guid key, bool preview = false)
{
Sql<ISqlContext>? sql = SqlContentSourcesSelect()

13
src/Umbraco.PublishedCache.HybridCache/Persistence/IDatabaseCacheRepository.cs
View File

@@ -52,17 +52,4 @@ internal interface IDatabaseCacheRepository
IReadOnlyCollection<int>? contentTypeIds = null,
IReadOnlyCollection<int>? mediaTypeIds = null,
IReadOnlyCollection<int>? memberTypeIds = null);
/// <summary>
/// Verifies the content cache by asserting that every document should have a corresponding row for edited properties and if published,
/// may have a corresponding row for published properties
/// </summary>
bool VerifyContentDbCache();
/// <summary>
/// Rebuilds the caches for content, media and/or members based on the content type ids specified
/// </summary>
bool VerifyMediaDbCache();
Task<IEnumerable<Guid>> GetContentKeysAsync(Guid nodeObjectType);
}

87
src/Umbraco.PublishedCache.HybridCache/Services/DocumentCacheService.cs
View File

@@ -124,7 +124,8 @@ internal sealed class DocumentCacheService : IDocumentCacheService
scope.Complete();
return contentCacheNode;
},
GetEntryOptions(key, preview));
GetEntryOptions(key, preview),
GenerateTags(key));
// We don't want to cache removed items, this may cause issues if the L2 serializer changes.
if (contentCacheNode is null)
@@ -158,10 +159,7 @@ internal sealed class DocumentCacheService : IDocumentCacheService
return true;
}
private bool GetPreview()
{
return _previewService.IsInPreview();
}
private bool GetPreview() => _previewService.IsInPreview();
public IEnumerable<IPublishedContent> GetByContentType(IPublishedContentType contentType)
{
@@ -176,28 +174,10 @@ internal sealed class DocumentCacheService : IDocumentCacheService
public async Task ClearMemoryCacheAsync(CancellationToken cancellationToken)
{
// TODO: This should be done with tags, however this is not implemented yet, so for now we have to naively get all content keys and clear them all.
using ICoreScope scope = _scopeProvider.CreateCoreScope();
// We have to get ALL document keys in order to be able to remove them from the cache,
IEnumerable<Guid> documentKeys = await _databaseCacheRepository.GetContentKeysAsync(Constants.ObjectTypes.Document);
foreach (Guid documentKey in documentKeys)
{
if (cancellationToken.IsCancellationRequested)
{
return;
}
// We'll remove both the draft and published cache
await _hybridCache.RemoveAsync(GetCacheKey(documentKey, false), cancellationToken);
await _hybridCache.RemoveAsync(GetCacheKey(documentKey, true), cancellationToken);
}
await _hybridCache.RemoveByTagAsync(Constants.Cache.Tags.Content, cancellationToken);
// We have to run seeding again after the cache is cleared
await SeedAsync(cancellationToken);
scope.Complete();
}
public async Task RefreshMemoryCacheAsync(Guid key)
@@ -207,13 +187,13 @@ internal sealed class DocumentCacheService : IDocumentCacheService
ContentCacheNode? draftNode = await _databaseCacheRepository.GetContentSourceAsync(key, true);
if (draftNode is not null)
{
await _hybridCache.SetAsync(GetCacheKey(draftNode.Key, true), draftNode, GetEntryOptions(draftNode.Key, true));
await _hybridCache.SetAsync(GetCacheKey(draftNode.Key, true), draftNode, GetEntryOptions(draftNode.Key, true), GenerateTags(key));
}
ContentCacheNode? publishedNode = await _databaseCacheRepository.GetContentSourceAsync(key, false);
if (publishedNode is not null && HasPublishedAncestorPath(publishedNode.Key))
{
await _hybridCache.SetAsync(GetCacheKey(publishedNode.Key, false), publishedNode, GetEntryOptions(publishedNode.Key, false));
await _hybridCache.SetAsync(GetCacheKey(publishedNode.Key, false), publishedNode, GetEntryOptions(publishedNode.Key, false), GenerateTags(key));
}
scope.Complete();
@@ -229,35 +209,37 @@ internal sealed class DocumentCacheService : IDocumentCacheService
{
foreach (Guid key in SeedKeys)
{
if(cancellationToken.IsCancellationRequested)
if (cancellationToken.IsCancellationRequested)
{
break;
}
var cacheKey = GetCacheKey(key, false);
// We'll use GetOrCreateAsync because it may be in the second level cache, in which case we don't have to re-seed.
ContentCacheNode? cachedValue = await _hybridCache.GetOrCreateAsync<ContentCacheNode?>(
cacheKey,
async cancel =>
// We'll use GetOrCreateAsync because it may be in the second level cache, in which case we don't have to re-seed.
ContentCacheNode? cachedValue = await _hybridCache.GetOrCreateAsync(
cacheKey,
async cancel =>
{
using ICoreScope scope = _scopeProvider.CreateCoreScope();
ContentCacheNode? cacheNode = await _databaseCacheRepository.GetContentSourceAsync(key);
scope.Complete();
// We don't want to seed drafts
if (cacheNode is null || cacheNode.IsDraft)
{
using ICoreScope scope = _scopeProvider.CreateCoreScope();
return null;
}
ContentCacheNode? cacheNode = await _databaseCacheRepository.GetContentSourceAsync(key, false);
return cacheNode;
},
GetSeedEntryOptions(),
GenerateTags(key),
cancellationToken: cancellationToken);
scope.Complete();
// We don't want to seed drafts
if (cacheNode is null || cacheNode.IsDraft)
{
return null;
}
return cacheNode;
},
GetSeedEntryOptions(),
cancellationToken: cancellationToken);
// If the value is null, it's likely because
// If the value is null, it's likely because
if (cachedValue is null)
{
await _hybridCache.RemoveAsync(cacheKey, cancellationToken);
@@ -290,7 +272,7 @@ internal sealed class DocumentCacheService : IDocumentCacheService
public async Task<bool> HasContentByIdAsync(int id, bool preview = false)
{
Attempt<Guid> keyAttempt = _idKeyMap.GetKeyForId(id, UmbracoObjectTypes.Document);
Attempt<Guid> keyAttempt = _idKeyMap.GetKeyForId(id, UmbracoObjectTypes.Document);
if (keyAttempt.Success is false)
{
return false;
@@ -315,7 +297,7 @@ internal sealed class DocumentCacheService : IDocumentCacheService
// Always set draft node
// We have nodes seperate in the cache, cause 99% of the time, you are only using one
// and thus we won't get too much data when retrieving from the cache.
ContentCacheNode draftCacheNode = _cacheNodeFactory.ToContentCacheNode(content, true);
var draftCacheNode = _cacheNodeFactory.ToContentCacheNode(content, true);
await _databaseCacheRepository.RefreshContentAsync(draftCacheNode, content.PublishedState);
@@ -329,13 +311,17 @@ internal sealed class DocumentCacheService : IDocumentCacheService
{
await _hybridCache.RemoveAsync(GetCacheKey(publishedCacheNode.Key, false));
}
}
scope.Complete();
}
private string GetCacheKey(Guid key, bool preview) => preview ? $"{key}+draft" : $"{key}";
private static string GetCacheKey(Guid key, bool preview) => preview ? $"{key}+draft" : $"{key}";
// Generates the cache tags for a given CacheNode
// We use the tags to be able to clear all cache entries that are related to a given content item.
// Tags for now are only content/media, but can be expanded with draft/published later.
private static HashSet<string> GenerateTags(Guid? key) => key is null ? [] : [Constants.Cache.Tags.Content];
public async Task DeleteItemAsync(IContentBase content)
{
@@ -368,6 +354,5 @@ internal sealed class DocumentCacheService : IDocumentCacheService
_hybridCache.RemoveAsync(GetCacheKey(content.Key, false)).GetAwaiter().GetResult();
}
}
}
}

41
src/Umbraco.PublishedCache.HybridCache/Services/MediaCacheService.cs
View File

@@ -103,7 +103,9 @@ internal class MediaCacheService : IMediaCacheService
ContentCacheNode? mediaCacheNode = await _databaseCacheRepository.GetMediaSourceAsync(key);
scope.Complete();
return mediaCacheNode;
}, GetEntryOptions(key));
},
GetEntryOptions(key),
GenerateTags(key));
// We don't want to cache removed items, this may cause issues if the L2 serializer changes.
if (contentCacheNode is null)
@@ -139,9 +141,6 @@ internal class MediaCacheService : IMediaCacheService
public async Task RefreshMediaAsync(IMedia media)
{
using ICoreScope scope = _scopeProvider.CreateCoreScope();
// Always set draft node
// We have nodes seperate in the cache, cause 99% of the time, you are only using one
// and thus we won't get too much data when retrieving from the cache.
var cacheNode = _cacheNodeFactory.ToContentCacheNode(media);
await _databaseCacheRepository.RefreshMediaAsync(cacheNode);
scope.Complete();
@@ -156,7 +155,6 @@ internal class MediaCacheService : IMediaCacheService
public async Task SeedAsync(CancellationToken cancellationToken)
{
foreach (Guid key in SeedKeys)
{
if (cancellationToken.IsCancellationRequested)
@@ -166,7 +164,7 @@ internal class MediaCacheService : IMediaCacheService
var cacheKey = GetCacheKey(key, false);
ContentCacheNode? cachedValue = await _hybridCache.GetOrCreateAsync<ContentCacheNode?>(
ContentCacheNode? cachedValue = await _hybridCache.GetOrCreateAsync(
cacheKey,
async cancel =>
{
@@ -175,7 +173,9 @@ internal class MediaCacheService : IMediaCacheService
scope.Complete();
return mediaCacheNode;
},
GetSeedEntryOptions());
GetSeedEntryOptions(),
GenerateTags(key),
cancellationToken: cancellationToken);
if (cachedValue is null)
{
@@ -199,27 +199,10 @@ internal class MediaCacheService : IMediaCacheService
public async Task ClearMemoryCacheAsync(CancellationToken cancellationToken)
{
// TODO: This should be done with tags, however this is not implemented yet, so for now we have to naively get all content keys and clear them all.
using ICoreScope scope = _scopeProvider.CreateCoreScope();
// We have to get ALL document keys in order to be able to remove them from the cache,
IEnumerable<Guid> documentKeys = await _databaseCacheRepository.GetContentKeysAsync(Constants.ObjectTypes.Media);
foreach (Guid documentKey in documentKeys)
{
if (cancellationToken.IsCancellationRequested)
{
return;
}
// We'll remove both the draft and published cache
await _hybridCache.RemoveAsync(GetCacheKey(documentKey, false), cancellationToken);
}
await _hybridCache.RemoveByTagAsync(Constants.Cache.Tags.Media, cancellationToken);
// We have to run seeding again after the cache is cleared
await SeedAsync(cancellationToken);
scope.Complete();
}
public async Task RemoveFromMemoryCacheAsync(Guid key)
@@ -240,6 +223,7 @@ internal class MediaCacheService : IMediaCacheService
_hybridCache.RemoveAsync(GetCacheKey(content.Key, false)).GetAwaiter().GetResult();
}
}
scope.Complete();
}
@@ -295,5 +279,10 @@ internal class MediaCacheService : IMediaCacheService
LocalCacheExpiration = _cacheSettings.Entry.Media.SeedCacheDuration,
};
private string GetCacheKey(Guid key, bool preview) => preview ? $"{key}+draft" : $"{key}";
private static string GetCacheKey(Guid key, bool preview) => preview ? $"{key}+draft" : $"{key}";
// Generates the cache tags for a given CacheNode
// We use the tags to be able to clear all cache entries that are related to a given content item.
// Tags for now are only content/media, but can be expanded with draft/published later.
private static HashSet<string> GenerateTags(Guid? key) => key is null ? [] : [Constants.Cache.Tags.Media];
}

2
src/Umbraco.Web.UI.Client/src/assets/lang/da-dk.ts
View File

@@ -2518,7 +2518,7 @@ export default {
refreshStatus: 'Refresh status',
memoryCache: 'Memory Cache',
memoryCacheDescription:
'\n This button lets you reload the in-memory cache, by entirely reloading it from the database\n cache (but it does not rebuild that database cache). This is relatively fast.\n Use it when you think that the memory cache has not been properly refreshed, after some events\n triggered&mdash;which would indicate a minor Umbraco issue.\n (note: triggers the reload on all servers in an LB environment).\n ',
'\n This button lets you reload the in-memory cache, by entirely reloading it from the database\n cache (but it does not rebuild that database cache). This is relatively fast.\n Use it when you think that the memory cache has not been properly refreshed, after some events\n triggered&mdash;which would indicate a minor Umbraco issue.\n (note: triggers the reload on all servers in an LB environment, and will clear the second level cache if you have it enabled).\n ',
reload: 'Reload',
databaseCache: 'Database Cache',
databaseCacheDescription:

2
src/Umbraco.Web.UI.Client/src/assets/lang/en-us.ts
View File

@@ -2323,7 +2323,7 @@ export default {
refreshStatus: 'Refresh status',
memoryCache: 'Memory Cache',
memoryCacheDescription:
'\n This button lets you reload the in-memory cache, by entirely reloading it from the database\n cache (but it does not rebuild that database cache). This is relatively fast.\n Use it when you think that the memory cache has not been properly refreshed, after some events\n triggered&mdash;which would indicate a minor Umbraco issue.\n (note: triggers the reload on all servers in an LB environment).\n ',
'\n This button lets you reload the in-memory cache, by entirely reloading it from the database\n cache (but it does not rebuild that database cache). This is relatively fast.\n Use it when you think that the memory cache has not been properly refreshed, after some events\n triggered&mdash;which would indicate a minor Umbraco issue.\n (note: triggers the reload on all servers in an LB environment, and will clear the second level cache if you have it enabled).\n ',
reload: 'Reload',
databaseCache: 'Database Cache',
databaseCacheDescription:

2
src/Umbraco.Web.UI.Client/src/assets/lang/en.ts
View File

@@ -2458,7 +2458,7 @@ export default {
refreshStatus: 'Refresh status',
memoryCache: 'Memory Cache',
memoryCacheDescription:
'\n This button lets you reload the in-memory cache, by entirely reloading it from the database\n cache (but it does not rebuild that database cache). This is relatively fast.\n Use it when you think that the memory cache has not been properly refreshed, after some events\n triggered&mdash;which would indicate a minor Umbraco issue.\n (note: triggers the reload on all servers in an LB environment).\n ',
'\n This button lets you reload the in-memory cache, by entirely reloading it from the database\n cache (but it does not rebuild that database cache). This is relatively fast.\n Use it when you think that the memory cache has not been properly refreshed, after some events\n triggered&mdash;which would indicate a minor Umbraco issue.\n (note: triggers the reload on all servers in an LB environment, and will clear the second level cache if you have it enabled).\n ',
reload: 'Reload',
databaseCache: 'Database Cache',
databaseCacheDescription:

2
tests/Directory.Packages.props
View File

@@ -6,7 +6,7 @@
<!-- Microsoft packages -->
<PackageVersion Include="BenchmarkDotNet" Version="0.14.0" />
<PackageVersion Include="Microsoft.AspNetCore.Mvc.Testing" Version="9.0.0" />
<PackageVersion Include="Microsoft.Extensions.Logging.Debug" Version="9.0.0" />
<PackageVersion Include="Microsoft.Extensions.Logging.Debug" Version="9.0.4" />
<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
<PackageVersion Include="System.Data.DataSetExtensions" Version="4.5.0" />
<PackageVersion Include="System.Data.Odbc" Version="9.0.0" />