yv01p/Umbraco-CMS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

V15: Refresh caches on load balanced environments (#17296)

Browse Source 
* Move DocumentCacheService

* Add clear all documentws from memory cache

* Fix RedirectTracker

* Implement refresh node/branch/all/delete

* Only update databasecache in RefreshContentAsync

* Fix tests

* Skip blueprints in cache

* Clear caches when contenttype is updated

* Clear cache on data type update

* Refresh media

* Only update memory cache from refreshers

* Fix imports

* Add named options

* Use cache entry settings in media

* Obsolete nucache settings

---------

Co-authored-by: Bjarke Berg <mail@bergmania.dk>
This commit is contained in:
Mole
2024-10-28 15:31:39 +01:00
committed by GitHub
parent 621a35f21f
commit d1799ecdd2
30 changed files with 499 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/Umbraco.PublishedCache.HybridCache/NotificationHandlers/SeedingNotificationHandler.cs
View File

@@ -3,6 +3,7 @@ using Umbraco.Cms.Core;
using Umbraco.Cms.Core.Configuration.Models;
using Umbraco.Cms.Core.Events;
using Umbraco.Cms.Core.Notifications;
using Umbraco.Cms.Core.PublishedCache;
using Umbraco.Cms.Core.Services;
using Umbraco.Cms.Infrastructure.HybridCache.Services;

30
src/Umbraco.PublishedCache.HybridCache/Persistence/DatabaseCacheRepository.cs
View File

@@ -161,6 +161,16 @@ AND cmsContentNu.nodeId IS NULL
return count == 0;
}
public async Task<IEnumerable<Guid>> GetContentKeysAsync(Guid nodeObjectType)
{
Sql<ISqlContext> sql = Sql()
.Select<NodeDto>(x => x.UniqueId)
.From<NodeDto>()
.Where<NodeDto>(x => x.NodeObjectType == nodeObjectType);
return await Database.FetchAsync<Guid>(sql);
}
// assumes member tree lock
public bool VerifyMemberDbCache()
{
@@ -235,8 +245,13 @@ AND cmsContentNu.nodeId IS NULL
return [];
}
Sql<ISqlContext>? sql = SqlContentSourcesSelect()
.InnerJoin<NodeDto>("n")
Sql<ISqlContext> sql = objectType == Constants.ObjectTypes.Document
? SqlContentSourcesSelect()
: objectType == Constants.ObjectTypes.Media
? SqlMediaSourcesSelect()
: throw new ArgumentOutOfRangeException(nameof(objectType), objectType, null);
sql.InnerJoin<NodeDto>("n")
.On<NodeDto, ContentDto>((n, c) => n.NodeId == c.ContentTypeId, "n", "umbracoContent")
.Append(SqlObjectTypeNotTrashed(SqlContext, objectType))
.WhereIn<NodeDto>(x => x.UniqueId, keys,"n")
@@ -251,7 +266,6 @@ AND cmsContentNu.nodeId IS NULL
{
ContentCacheDataSerializerEntityType.Document => Constants.ObjectTypes.Document,
ContentCacheDataSerializerEntityType.Media => Constants.ObjectTypes.Media,
ContentCacheDataSerializerEntityType.Member => Constants.ObjectTypes.Member,
_ => throw new ArgumentOutOfRangeException(nameof(entityType), entityType, null),
};
@@ -262,7 +276,15 @@ AND cmsContentNu.nodeId IS NULL
foreach (ContentSourceDto row in dtos)
{
yield return CreateContentNodeKit(row, serializer, row.Published is false);
if (entityType == ContentCacheDataSerializerEntityType.Document)
{
yield return CreateContentNodeKit(row, serializer, row.Published is false);
}
else
{
yield return CreateMediaNodeKit(row, serializer);
}
}
}

2
src/Umbraco.PublishedCache.HybridCache/Persistence/IDatabaseCacheRepository.cs
View File

@@ -67,4 +67,6 @@ internal interface IDatabaseCacheRepository
/// Rebuilds the caches for content, media and/or members based on the content type ids specified
/// </summary>
bool VerifyMediaDbCache();
Task<IEnumerable<Guid>> GetContentKeysAsync(Guid nodeObjectType);
}

133
src/Umbraco.PublishedCache.HybridCache/Services/DocumentCacheService.cs
View File

@@ -3,6 +3,7 @@ using Microsoft.Extensions.Options;
using Umbraco.Cms.Core;
using Umbraco.Cms.Core.Models;
using Umbraco.Cms.Core.Models.PublishedContent;
using Umbraco.Cms.Core.PublishedCache;
using Umbraco.Cms.Core.Scoping;
using Umbraco.Cms.Core.Services;
using Umbraco.Cms.Infrastructure.HybridCache.Factories;
@@ -23,8 +24,7 @@ internal sealed class DocumentCacheService : IDocumentCacheService
private readonly IEnumerable<IDocumentSeedKeyProvider> _seedKeyProviders;
private readonly IPublishedModelFactory _publishedModelFactory;
private readonly IPreviewService _previewService;
private readonly CacheSettings _cacheSettings;
private readonly CacheEntrySettings _cacheEntrySettings;
private HashSet<Guid>? _seedKeys;
private HashSet<Guid> SeedKeys
{
@@ -54,7 +54,7 @@ internal sealed class DocumentCacheService : IDocumentCacheService
IPublishedContentFactory publishedContentFactory,
ICacheNodeFactory cacheNodeFactory,
IEnumerable<IDocumentSeedKeyProvider> seedKeyProviders,
IOptions<CacheSettings> cacheSettings,
IOptionsMonitor<CacheEntrySettings> cacheEntrySettings,
IPublishedModelFactory publishedModelFactory,
IPreviewService previewService)
{
@@ -67,7 +67,7 @@ internal sealed class DocumentCacheService : IDocumentCacheService
_seedKeyProviders = seedKeyProviders;
_publishedModelFactory = publishedModelFactory;
_previewService = previewService;
_cacheSettings = cacheSettings.Value;
_cacheEntrySettings = cacheEntrySettings.Get(Constants.Configuration.NamedOptions.CacheEntry.Document);
}
public async Task<IPublishedContent?> GetByKeyAsync(Guid key, bool? preview = null)
@@ -82,7 +82,8 @@ internal sealed class DocumentCacheService : IDocumentCacheService
ContentCacheNode? contentCacheNode = await _databaseCacheRepository.GetContentSourceAsync(key, calculatedPreview);
scope.Complete();
return contentCacheNode;
});
},
GetEntryOptions(key));
return contentCacheNode is null ? null : _publishedContentFactory.ToIPublishedContent(contentCacheNode, calculatedPreview).CreateModel(_publishedModelFactory);
}
@@ -101,6 +102,7 @@ internal sealed class DocumentCacheService : IDocumentCacheService
}
bool calculatedPreview = preview ?? GetPreview();
Guid key = keyAttempt.Result;
ContentCacheNode? contentCacheNode = await _hybridCache.GetOrCreateAsync(
GetCacheKey(keyAttempt.Result, calculatedPreview), // Unique key to the cache entry
@@ -110,7 +112,7 @@ internal sealed class DocumentCacheService : IDocumentCacheService
ContentCacheNode? contentCacheNode = await _databaseCacheRepository.GetContentSourceAsync(id, calculatedPreview);
scope.Complete();
return contentCacheNode;
});
}, GetEntryOptions(key));
return contentCacheNode is null ? null : _publishedContentFactory.ToIPublishedContent(contentCacheNode, calculatedPreview).CreateModel(_publishedModelFactory);;
}
@@ -126,6 +128,57 @@ internal sealed class DocumentCacheService : IDocumentCacheService
.WhereNotNull();
}
public async Task ClearMemoryCacheAsync(CancellationToken cancellationToken)
{
// TODO: This should be done with tags, however this is not implemented yet, so for now we have to naively get all content keys and clear them all.
using ICoreScope scope = _scopeProvider.CreateCoreScope();
// We have to get ALL document keys in order to be able to remove them from the cache,
IEnumerable<Guid> documentKeys = await _databaseCacheRepository.GetContentKeysAsync(Constants.ObjectTypes.Document);
foreach (Guid documentKey in documentKeys)
{
if (cancellationToken.IsCancellationRequested)
{
return;
}
// We'll remove both the draft and published cache
await _hybridCache.RemoveAsync(GetCacheKey(documentKey, false), cancellationToken);
await _hybridCache.RemoveAsync(GetCacheKey(documentKey, true), cancellationToken);
}
// We have to run seeding again after the cache is cleared
await SeedAsync(cancellationToken);
scope.Complete();
}
public async Task RefreshMemoryCacheAsync(Guid key)
{
using ICoreScope scope = _scopeProvider.CreateCoreScope();
ContentCacheNode? draftNode = await _databaseCacheRepository.GetContentSourceAsync(key, true);
if (draftNode is not null)
{
await _hybridCache.SetAsync(GetCacheKey(draftNode.Key, true), draftNode, GetEntryOptions(draftNode.Key));
}
ContentCacheNode? publishedNode = await _databaseCacheRepository.GetContentSourceAsync(key, false);
if (publishedNode is not null)
{
await _hybridCache.SetAsync(GetCacheKey(publishedNode.Key, false), publishedNode, GetEntryOptions(publishedNode.Key));
}
scope.Complete();
}
public async Task RemoveFromMemoryCacheAsync(Guid key)
{
await _hybridCache.RemoveAsync(GetCacheKey(key, true));
await _hybridCache.RemoveAsync(GetCacheKey(key, false));
}
public async Task SeedAsync(CancellationToken cancellationToken)
{
foreach (Guid key in SeedKeys)
@@ -155,7 +208,8 @@ internal sealed class DocumentCacheService : IDocumentCacheService
return cacheNode;
},
GetSeedEntryOptions());
GetSeedEntryOptions(),
cancellationToken: cancellationToken);
// If the value is null, it's likely because
if (cachedValue is null)
@@ -167,10 +221,24 @@ internal sealed class DocumentCacheService : IDocumentCacheService
private HybridCacheEntryOptions GetSeedEntryOptions() => new()
{
Expiration = _cacheSettings.SeedCacheDuration,
LocalCacheExpiration = _cacheSettings.SeedCacheDuration
Expiration = _cacheEntrySettings.SeedCacheDuration,
LocalCacheExpiration = _cacheEntrySettings.SeedCacheDuration
};
private HybridCacheEntryOptions GetEntryOptions(Guid key)
{
if (SeedKeys.Contains(key))
{
return GetSeedEntryOptions();
}
return new HybridCacheEntryOptions
{
Expiration = _cacheEntrySettings.RemoteCacheDuration,
LocalCacheExpiration = _cacheEntrySettings.LocalCacheDuration,
};
}
public async Task<bool> HasContentByIdAsync(int id, bool preview = false)
{
Attempt<Guid> keyAttempt = _idKeyMap.GetKeyForId(id, UmbracoObjectTypes.Document);
@@ -195,67 +263,29 @@ internal sealed class DocumentCacheService : IDocumentCacheService
{
using ICoreScope scope = _scopeProvider.CreateCoreScope();
bool isSeeded = SeedKeys.Contains(content.Key);
// Always set draft node
// We have nodes seperate in the cache, cause 99% of the time, you are only using one
// and thus we won't get too much data when retrieving from the cache.
ContentCacheNode draftCacheNode = _cacheNodeFactory.ToContentCacheNode(content, true);
await _databaseCacheRepository.RefreshContentAsync(draftCacheNode, content.PublishedState);
_scopeProvider.Context?.Enlist($"UpdateMemoryCache_Draft_{content.Key}", completed =>
{
if(completed is false)
{
return;
}
RefreshHybridCache(draftCacheNode, GetCacheKey(content.Key, true), isSeeded).GetAwaiter().GetResult();
}, 1);
if (content.PublishedState == PublishedState.Publishing)
{
var publishedCacheNode = _cacheNodeFactory.ToContentCacheNode(content, false);
await _databaseCacheRepository.RefreshContentAsync(publishedCacheNode, content.PublishedState);
_scopeProvider.Context?.Enlist($"UpdateMemoryCache_{content.Key}", completed =>
{
if(completed is false)
{
return;
}
RefreshHybridCache(publishedCacheNode, GetCacheKey(content.Key, false), isSeeded).GetAwaiter().GetResult();
}, 1);
}
scope.Complete();
}
private async Task RefreshHybridCache(ContentCacheNode cacheNode, string cacheKey, bool isSeeded)
{
// If it's seeded we want it to stick around the cache for longer.
if (isSeeded)
{
await _hybridCache.SetAsync(
cacheKey,
cacheNode,
GetSeedEntryOptions());
}
else
{
await _hybridCache.SetAsync(cacheKey, cacheNode);
}
}
private string GetCacheKey(Guid key, bool preview) => preview ? $"{key}+draft" : $"{key}";
public async Task DeleteItemAsync(IContentBase content)
{
using ICoreScope scope = _scopeProvider.CreateCoreScope();
await _databaseCacheRepository.DeleteContentItemAsync(content.Id);
await _hybridCache.RemoveAsync(GetCacheKey(content.Key, true));
await _hybridCache.RemoveAsync(GetCacheKey(content.Key, false));
scope.Complete();
}
@@ -263,6 +293,14 @@ internal sealed class DocumentCacheService : IDocumentCacheService
{
using ICoreScope scope = _scopeProvider.CreateCoreScope();
_databaseCacheRepository.Rebuild(contentTypeIds.ToList());
RebuildMemoryCacheByContentTypeAsync(contentTypeIds).GetAwaiter().GetResult();
scope.Complete();
}
public async Task RebuildMemoryCacheByContentTypeAsync(IEnumerable<int> contentTypeIds)
{
using ICoreScope scope = _scopeProvider.CreateCoreScope();
IEnumerable<ContentCacheNode> contentByContentTypeKey = _databaseCacheRepository.GetContentByContentTypeKey(contentTypeIds.Select(x => _idKeyMap.GetKeyForId(x, UmbracoObjectTypes.DocumentType).Result), ContentCacheDataSerializerEntityType.Document);
scope.Complete();
@@ -276,6 +314,5 @@ internal sealed class DocumentCacheService : IDocumentCacheService
}
}
}
}

23
src/Umbraco.PublishedCache.HybridCache/Services/IDocumentCacheService.cs
View File

@@ -1,23 +0,0 @@
using Umbraco.Cms.Core.Models;
using Umbraco.Cms.Core.Models.PublishedContent;
namespace Umbraco.Cms.Infrastructure.HybridCache.Services;
public interface IDocumentCacheService
{
Task<IPublishedContent?> GetByKeyAsync(Guid key, bool? preview = null);
Task<IPublishedContent?> GetByIdAsync(int id, bool? preview = null);
Task SeedAsync(CancellationToken cancellationToken);
Task<bool> HasContentByIdAsync(int id, bool preview = false);
Task RefreshContentAsync(IContent content);
Task DeleteItemAsync(IContentBase content);
void Rebuild(IReadOnlyCollection<int> contentTypeIds);
internal IEnumerable<IPublishedContent> GetByContentType(IPublishedContentType contentType);
}

21
src/Umbraco.PublishedCache.HybridCache/Services/IMediaCacheService.cs
View File

@@ -1,21 +0,0 @@
using Umbraco.Cms.Core.Models;
using Umbraco.Cms.Core.Models.PublishedContent;
namespace Umbraco.Cms.Infrastructure.HybridCache.Services;
public interface IMediaCacheService
{
Task<IPublishedContent?> GetByKeyAsync(Guid key);
Task<IPublishedContent?> GetByIdAsync(int id);
Task<bool> HasContentByIdAsync(int id);
Task RefreshMediaAsync(IMedia media);
Task DeleteItemAsync(IContentBase media);
Task SeedAsync(CancellationToken cancellationToken);
void Rebuild(IReadOnlyCollection<int> contentTypeIds);
}

92
src/Umbraco.PublishedCache.HybridCache/Services/MediaCacheService.cs
View File

@@ -3,6 +3,7 @@ using Microsoft.Extensions.Options;
using Umbraco.Cms.Core;
using Umbraco.Cms.Core.Models;
using Umbraco.Cms.Core.Models.PublishedContent;
using Umbraco.Cms.Core.PublishedCache;
using Umbraco.Cms.Core.Scoping;
using Umbraco.Cms.Core.Services;
using Umbraco.Cms.Infrastructure.HybridCache.Factories;
@@ -22,7 +23,7 @@ internal class MediaCacheService : IMediaCacheService
private readonly ICacheNodeFactory _cacheNodeFactory;
private readonly IEnumerable<IMediaSeedKeyProvider> _seedKeyProviders;
private readonly IPublishedModelFactory _publishedModelFactory;
private readonly CacheSettings _cacheSettings;
private readonly CacheEntrySettings _cacheEntrySettings;
private HashSet<Guid>? _seedKeys;
private HashSet<Guid> SeedKeys
@@ -54,7 +55,7 @@ internal class MediaCacheService : IMediaCacheService
ICacheNodeFactory cacheNodeFactory,
IEnumerable<IMediaSeedKeyProvider> seedKeyProviders,
IPublishedModelFactory publishedModelFactory,
IOptions<CacheSettings> cacheSettings)
IOptionsMonitor<CacheEntrySettings> cacheEntrySettings)
{
_databaseCacheRepository = databaseCacheRepository;
_idKeyMap = idKeyMap;
@@ -64,7 +65,7 @@ internal class MediaCacheService : IMediaCacheService
_cacheNodeFactory = cacheNodeFactory;
_seedKeyProviders = seedKeyProviders;
_publishedModelFactory = publishedModelFactory;
_cacheSettings = cacheSettings.Value;
_cacheEntrySettings = cacheEntrySettings.Get(Constants.Configuration.NamedOptions.CacheEntry.Media);
}
public async Task<IPublishedContent?> GetByKeyAsync(Guid key)
@@ -83,7 +84,7 @@ internal class MediaCacheService : IMediaCacheService
ContentCacheNode? mediaCacheNode = await _databaseCacheRepository.GetMediaSourceAsync(idAttempt.Result);
scope.Complete();
return mediaCacheNode;
});
}, GetEntryOptions(key));
return contentCacheNode is null ? null : _publishedContentFactory.ToIPublishedMedia(contentCacheNode).CreateModel(_publishedModelFactory);
}
@@ -95,6 +96,7 @@ internal class MediaCacheService : IMediaCacheService
{
return null;
}
Guid key = keyAttempt.Result;
ContentCacheNode? contentCacheNode = await _hybridCache.GetOrCreateAsync(
$"{keyAttempt.Result}", // Unique key to the cache entry
@@ -104,7 +106,7 @@ internal class MediaCacheService : IMediaCacheService
ContentCacheNode? mediaCacheNode = await _databaseCacheRepository.GetMediaSourceAsync(id);
scope.Complete();
return mediaCacheNode;
});
}, GetEntryOptions(key));
return contentCacheNode is null ? null : _publishedContentFactory.ToIPublishedMedia(contentCacheNode).CreateModel(_publishedModelFactory);
}
@@ -137,7 +139,6 @@ internal class MediaCacheService : IMediaCacheService
// We have nodes seperate in the cache, cause 99% of the time, you are only using one
// and thus we won't get too much data when retrieving from the cache.
var cacheNode = _cacheNodeFactory.ToContentCacheNode(media);
await _hybridCache.SetAsync(GetCacheKey(media.Key, false), cacheNode);
await _databaseCacheRepository.RefreshMediaAsync(cacheNode);
scope.Complete();
}
@@ -146,7 +147,6 @@ internal class MediaCacheService : IMediaCacheService
{
using ICoreScope scope = _scopeProvider.CreateCoreScope();
await _databaseCacheRepository.DeleteContentItemAsync(media.Id);
await _hybridCache.RemoveAsync(media.Key.ToString());
scope.Complete();
}
@@ -180,6 +180,65 @@ internal class MediaCacheService : IMediaCacheService
}
}
public async Task RefreshMemoryCacheAsync(Guid key)
{
using ICoreScope scope = _scopeProvider.CreateCoreScope();
ContentCacheNode? publishedNode = await _databaseCacheRepository.GetMediaSourceAsync(key);
if (publishedNode is not null)
{
await _hybridCache.SetAsync(GetCacheKey(publishedNode.Key, false), publishedNode, GetEntryOptions(publishedNode.Key));
}
scope.Complete();
}
public async Task ClearMemoryCacheAsync(CancellationToken cancellationToken)
{
// TODO: This should be done with tags, however this is not implemented yet, so for now we have to naively get all content keys and clear them all.
using ICoreScope scope = _scopeProvider.CreateCoreScope();
// We have to get ALL document keys in order to be able to remove them from the cache,
IEnumerable<Guid> documentKeys = await _databaseCacheRepository.GetContentKeysAsync(Constants.ObjectTypes.Media);
foreach (Guid documentKey in documentKeys)
{
if (cancellationToken.IsCancellationRequested)
{
return;
}
// We'll remove both the draft and published cache
await _hybridCache.RemoveAsync(GetCacheKey(documentKey, false), cancellationToken);
}
// We have to run seeding again after the cache is cleared
await SeedAsync(cancellationToken);
scope.Complete();
}
public async Task RemoveFromMemoryCacheAsync(Guid key)
=> await _hybridCache.RemoveAsync(GetCacheKey(key, false));
public async Task RebuildMemoryCacheByContentTypeAsync(IEnumerable<int> mediaTypeIds)
{
using ICoreScope scope = _scopeProvider.CreateCoreScope();
IEnumerable<ContentCacheNode> contentByContentTypeKey = _databaseCacheRepository.GetContentByContentTypeKey(mediaTypeIds.Select(x => _idKeyMap.GetKeyForId(x, UmbracoObjectTypes.MediaType).Result), ContentCacheDataSerializerEntityType.Media);
foreach (ContentCacheNode content in contentByContentTypeKey)
{
_hybridCache.RemoveAsync(GetCacheKey(content.Key, true)).GetAwaiter().GetResult();
if (content.IsDraft is false)
{
_hybridCache.RemoveAsync(GetCacheKey(content.Key, false)).GetAwaiter().GetResult();
}
}
scope.Complete();
}
public void Rebuild(IReadOnlyCollection<int> contentTypeIds)
{
using ICoreScope scope = _scopeProvider.CreateCoreScope();
@@ -200,10 +259,25 @@ internal class MediaCacheService : IMediaCacheService
scope.Complete();
}
private HybridCacheEntryOptions GetEntryOptions(Guid key)
{
if (SeedKeys.Contains(key))
{
return GetSeedEntryOptions();
}
return new HybridCacheEntryOptions
{
Expiration = _cacheEntrySettings.RemoteCacheDuration,
LocalCacheExpiration = _cacheEntrySettings.LocalCacheDuration,
};
}
private HybridCacheEntryOptions GetSeedEntryOptions() => new()
{
Expiration = _cacheSettings.SeedCacheDuration,
LocalCacheExpiration = _cacheSettings.SeedCacheDuration,
Expiration = _cacheEntrySettings.SeedCacheDuration,
LocalCacheExpiration = _cacheEntrySettings.SeedCacheDuration,
};
private string GetCacheKey(Guid key, bool preview) => preview ? $"{key}+draft" : $"{key}";