V15: Refresh caches on load balanced environments (#17296)

* Move DocumentCacheService * Add clear all documentws from memory cache * Fix RedirectTracker * Implement refresh node/branch/all/delete * Only update databasecache in RefreshContentAsync * Fix tests * Skip blueprints in cache * Clear caches when contenttype is updated * Clear cache on data type update * Refresh media * Only update memory cache from refreshers * Fix imports * Add named options * Use cache entry settings in media * Obsolete nucache settings --------- Co-authored-by: Bjarke Berg <mail@bergmania.dk>
2024-10-28 15:31:39 +01:00
parent 621a35f21f
commit d1799ecdd2
30 changed files with 499 additions and 104 deletions
--- a/src/Umbraco.Core/Cache/Refreshers/Implement/ContentCacheRefresher.cs
+++ b/src/Umbraco.Core/Cache/Refreshers/Implement/ContentCacheRefresher.cs
@@ -20,6 +20,7 @@ public sealed class ContentCacheRefresher : PayloadCacheRefresherBase<ContentCac
    private readonly IDocumentNavigationQueryService _documentNavigationQueryService;
    private readonly IDocumentNavigationManagementService _documentNavigationManagementService;
    private readonly IContentService _contentService;
+    private readonly IDocumentCacheService _documentCacheService;
    private readonly IPublishStatusManagementService _publishStatusManagementService;
    private readonly IIdKeyMap _idKeyMap;

@@ -35,7 +36,8 @@ public sealed class ContentCacheRefresher : PayloadCacheRefresherBase<ContentCac
        IDocumentNavigationQueryService documentNavigationQueryService,
        IDocumentNavigationManagementService documentNavigationManagementService,
        IContentService contentService,
-        IPublishStatusManagementService publishStatusManagementService)
+        IPublishStatusManagementService publishStatusManagementService,
+        IDocumentCacheService documentCacheService)
        : base(appCaches, serializer, eventAggregator, factory)
    {
        _idKeyMap = idKeyMap;
@@ -45,6 +47,7 @@ public sealed class ContentCacheRefresher : PayloadCacheRefresherBase<ContentCac
        _documentNavigationQueryService = documentNavigationQueryService;
        _documentNavigationManagementService = documentNavigationManagementService;
        _contentService = contentService;
+        _documentCacheService = documentCacheService;
        _publishStatusManagementService = publishStatusManagementService;
    }

@@ -107,6 +110,7 @@ public sealed class ContentCacheRefresher : PayloadCacheRefresherBase<ContentCac
            }


+            HandleMemoryCache(payload);
            HandleRouting(payload);

            HandleNavigation(payload);
@@ -143,6 +147,45 @@ public sealed class ContentCacheRefresher : PayloadCacheRefresherBase<ContentCac
        base.Refresh(payloads);
    }

+    private void HandleMemoryCache(JsonPayload payload)
+    {
+        Guid key = payload.Key ?? _idKeyMap.GetKeyForId(payload.Id, UmbracoObjectTypes.Document).Result;
+
+        if (payload.Blueprint)
+        {
+            return;
+        }
+
+        if (payload.ChangeTypes.HasType(TreeChangeTypes.RefreshNode))
+        {
+            _documentCacheService.RefreshMemoryCacheAsync(key).GetAwaiter().GetResult();
+        }
+
+        if (payload.ChangeTypes.HasType(TreeChangeTypes.RefreshBranch))
+        {
+            if (_documentNavigationQueryService.TryGetDescendantsKeys(key, out IEnumerable<Guid> descendantsKeys))
+            {
+                var branchKeys = descendantsKeys.ToList();
+                branchKeys.Add(key);
+
+                foreach (Guid branchKey in branchKeys)
+                {
+                    _documentCacheService.RefreshMemoryCacheAsync(branchKey).GetAwaiter().GetResult();
+                }
+            }
+        }
+
+        if (payload.ChangeTypes.HasType(TreeChangeTypes.RefreshAll))
+        {
+            _documentCacheService.ClearMemoryCacheAsync(CancellationToken.None).GetAwaiter().GetResult();
+        }
+
+        if (payload.ChangeTypes.HasType(TreeChangeTypes.Remove))
+        {
+            _documentCacheService.RemoveFromMemoryCacheAsync(key).GetAwaiter().GetResult();
+        }
+    }
+
    private void HandleNavigation(JsonPayload payload)
    {

--- a/src/Umbraco.Core/Cache/Refreshers/Implement/ContentTypeCacheRefresher.cs
+++ b/src/Umbraco.Core/Cache/Refreshers/Implement/ContentTypeCacheRefresher.cs
@@ -3,6 +3,7 @@ using Umbraco.Cms.Core.Models;
 using Umbraco.Cms.Core.Models.PublishedContent;
 using Umbraco.Cms.Core.Notifications;
 using Umbraco.Cms.Core.Persistence.Repositories;
+using Umbraco.Cms.Core.PublishedCache;
 using Umbraco.Cms.Core.Serialization;
 using Umbraco.Cms.Core.Services;
 using Umbraco.Cms.Core.Services.Changes;
@@ -15,6 +16,9 @@ public sealed class ContentTypeCacheRefresher : PayloadCacheRefresherBase<Conten
    private readonly IContentTypeCommonRepository _contentTypeCommonRepository;
    private readonly IPublishedModelFactory _publishedModelFactory;
    private readonly IPublishedContentTypeFactory _publishedContentTypeFactory;
+    private readonly IDocumentCacheService _documentCacheService;
+    private readonly IPublishedContentTypeCache _publishedContentTypeCache;
+    private readonly IMediaCacheService _mediaCacheService;
    private readonly IIdKeyMap _idKeyMap;

    public ContentTypeCacheRefresher(
@@ -25,13 +29,19 @@ public sealed class ContentTypeCacheRefresher : PayloadCacheRefresherBase<Conten
        IEventAggregator eventAggregator,
        ICacheRefresherNotificationFactory factory,
        IPublishedModelFactory publishedModelFactory,
-        IPublishedContentTypeFactory publishedContentTypeFactory)
+        IPublishedContentTypeFactory publishedContentTypeFactory,
+        IDocumentCacheService documentCacheService,
+        IPublishedContentTypeCache publishedContentTypeCache,
+        IMediaCacheService mediaCacheService)
        : base(appCaches, serializer, eventAggregator, factory)
    {
        _idKeyMap = idKeyMap;
        _contentTypeCommonRepository = contentTypeCommonRepository;
        _publishedModelFactory = publishedModelFactory;
        _publishedContentTypeFactory = publishedContentTypeFactory;
+        _documentCacheService = documentCacheService;
+        _publishedContentTypeCache = publishedContentTypeCache;
+        _mediaCacheService = mediaCacheService;
    }

    #region Json
@@ -114,10 +124,16 @@ public sealed class ContentTypeCacheRefresher : PayloadCacheRefresherBase<Conten
            MemberCacheRefresher.RefreshMemberTypes(AppCaches);
        }

-        // TODO: We need to clear the HybridCache of any content using the ContentType, but NOT the database cache here, and this should be done within the "WithSafeLiveFactoryReset" to ensure that the factory is locked in the meantime.
-        _publishedModelFactory.WithSafeLiveFactoryReset(() => { });
-
+        _publishedContentTypeCache.ClearContentTypes(payloads.Select(x => x.Id));
        _publishedContentTypeFactory.NotifyDataTypeChanges();
+        _publishedModelFactory.WithSafeLiveFactoryReset(() =>
+        {
+            IEnumerable<int> documentTypeIds = payloads.Where(x => x.ItemType == nameof(IContentType)).Select(x => x.Id);
+            IEnumerable<int> mediaTypeIds = payloads.Where(x => x.ItemType == nameof(IMediaType)).Select(x => x.Id);
+
+            _documentCacheService.RebuildMemoryCacheByContentTypeAsync(documentTypeIds);
+            _mediaCacheService.RebuildMemoryCacheByContentTypeAsync(mediaTypeIds);
+        });

        // now we can trigger the event
        base.Refresh(payloads);
--- a/src/Umbraco.Core/Cache/Refreshers/Implement/DataTypeCacheRefresher.cs
+++ b/src/Umbraco.Core/Cache/Refreshers/Implement/DataTypeCacheRefresher.cs
@@ -3,6 +3,7 @@ using Umbraco.Cms.Core.Models;
 using Umbraco.Cms.Core.Models.PublishedContent;
 using Umbraco.Cms.Core.Notifications;
 using Umbraco.Cms.Core.Persistence.Repositories;
+using Umbraco.Cms.Core.PublishedCache;
 using Umbraco.Cms.Core.Serialization;
 using Umbraco.Cms.Core.Services;
 using Umbraco.Extensions;
@@ -14,6 +15,9 @@ public sealed class DataTypeCacheRefresher : PayloadCacheRefresherBase<DataTypeC
    private readonly IIdKeyMap _idKeyMap;
    private readonly IPublishedModelFactory _publishedModelFactory;
    private readonly IPublishedContentTypeFactory _publishedContentTypeFactory;
+    private readonly IPublishedContentTypeCache _publishedContentTypeCache;
+    private readonly IDocumentCacheService _documentCacheService;
+    private readonly IMediaCacheService _mediaCacheService;

    public DataTypeCacheRefresher(
        AppCaches appCaches,
@@ -22,12 +26,18 @@ public sealed class DataTypeCacheRefresher : PayloadCacheRefresherBase<DataTypeC
        IEventAggregator eventAggregator,
        ICacheRefresherNotificationFactory factory,
        IPublishedModelFactory publishedModelFactory,
-        IPublishedContentTypeFactory publishedContentTypeFactory)
+        IPublishedContentTypeFactory publishedContentTypeFactory,
+        IPublishedContentTypeCache publishedContentTypeCache,
+        IDocumentCacheService documentCacheService,
+        IMediaCacheService mediaCacheService)
        : base(appCaches, serializer, eventAggregator, factory)
    {
        _idKeyMap = idKeyMap;
        _publishedModelFactory = publishedModelFactory;
        _publishedContentTypeFactory = publishedContentTypeFactory;
+        _publishedContentTypeCache = publishedContentTypeCache;
+        _documentCacheService = documentCacheService;
+        _mediaCacheService = mediaCacheService;
    }

    #region Json
@@ -76,6 +86,7 @@ public sealed class DataTypeCacheRefresher : PayloadCacheRefresherBase<DataTypeC

        Attempt<IAppPolicyCache?> dataTypeCache = AppCaches.IsolatedCaches.Get<IDataType>();

+        List<IPublishedContentType> removedContentTypes = new();
        foreach (JsonPayload payload in payloads)
        {
            _idKeyMap.ClearCache(payload.Id);
@@ -84,14 +95,25 @@ public sealed class DataTypeCacheRefresher : PayloadCacheRefresherBase<DataTypeC
            {
                dataTypeCache.Result?.Clear(RepositoryCacheKeys.GetKey<IDataType, int>(payload.Id));
            }
-        }

-        // TODO: We need to clear the HybridCache of any content using the ContentType, but NOT the database cache here, and this should be done within the "WithSafeLiveFactoryReset" to ensure that the factory is locked in the meantime.
-        _publishedModelFactory.WithSafeLiveFactoryReset(() => { });
+            removedContentTypes.AddRange(_publishedContentTypeCache.ClearByDataTypeId(payload.Id));
+        }

        var changedIds = payloads.Select(x => x.Id).ToArray();
        _publishedContentTypeFactory.NotifyDataTypeChanges(changedIds);

+        _publishedModelFactory.WithSafeLiveFactoryReset(() =>
+        {
+            IEnumerable<int> documentTypeIds = removedContentTypes
+                .Where(x => x.ItemType == PublishedItemType.Content)
+                .Select(x => x.Id);
+            _documentCacheService.RebuildMemoryCacheByContentTypeAsync(documentTypeIds).GetAwaiter().GetResult();
+
+            IEnumerable<int> mediaTypeIds = removedContentTypes
+                .Where(x => x.ItemType == PublishedItemType.Media)
+                .Select(x => x.Id);
+            _mediaCacheService.RebuildMemoryCacheByContentTypeAsync(mediaTypeIds);
+        });
        base.Refresh(payloads);
    }

--- a/src/Umbraco.Core/Cache/Refreshers/Implement/MediaCacheRefresher.cs
+++ b/src/Umbraco.Core/Cache/Refreshers/Implement/MediaCacheRefresher.cs
@@ -2,6 +2,7 @@ using Umbraco.Cms.Core.Events;
 using Umbraco.Cms.Core.Models;
 using Umbraco.Cms.Core.Notifications;
 using Umbraco.Cms.Core.Persistence.Repositories;
+using Umbraco.Cms.Core.PublishedCache;
 using Umbraco.Cms.Core.Serialization;
 using Umbraco.Cms.Core.Services;
 using Umbraco.Cms.Core.Services.Changes;
@@ -16,6 +17,7 @@ public sealed class MediaCacheRefresher : PayloadCacheRefresherBase<MediaCacheRe
    private readonly IMediaNavigationQueryService _mediaNavigationQueryService;
    private readonly IMediaNavigationManagementService _mediaNavigationManagementService;
    private readonly IMediaService _mediaService;
+    private readonly IMediaCacheService _mediaCacheService;

    public MediaCacheRefresher(
        AppCaches appCaches,
@@ -25,13 +27,15 @@ public sealed class MediaCacheRefresher : PayloadCacheRefresherBase<MediaCacheRe
        ICacheRefresherNotificationFactory factory,
        IMediaNavigationQueryService mediaNavigationQueryService,
        IMediaNavigationManagementService mediaNavigationManagementService,
-        IMediaService mediaService)
+        IMediaService mediaService,
+        IMediaCacheService mediaCacheService)
        : base(appCaches, serializer, eventAggregator, factory)
    {
        _idKeyMap = idKeyMap;
        _mediaNavigationQueryService = mediaNavigationQueryService;
        _mediaNavigationManagementService = mediaNavigationManagementService;
        _mediaService = mediaService;
+        _mediaCacheService = mediaCacheService;
    }

    #region Indirect
@@ -106,6 +110,7 @@ public sealed class MediaCacheRefresher : PayloadCacheRefresherBase<MediaCacheRe
                }
            }

+            HandleMemoryCache(payload);
            HandleNavigation(payload);
        }

@@ -115,6 +120,41 @@ public sealed class MediaCacheRefresher : PayloadCacheRefresherBase<MediaCacheRe
        base.Refresh(payloads);
    }

+    private void HandleMemoryCache(JsonPayload payload)
+    {
+        Guid key = payload.Key ?? _idKeyMap.GetKeyForId(payload.Id, UmbracoObjectTypes.Document).Result;
+
+
+        if (payload.ChangeTypes.HasType(TreeChangeTypes.RefreshNode))
+        {
+            _mediaCacheService.RefreshMemoryCacheAsync(key).GetAwaiter().GetResult();
+        }
+
+        if (payload.ChangeTypes.HasType(TreeChangeTypes.RefreshBranch))
+        {
+            if (_mediaNavigationQueryService.TryGetDescendantsKeys(key, out IEnumerable<Guid> descendantsKeys))
+            {
+                var branchKeys = descendantsKeys.ToList();
+                branchKeys.Add(key);
+
+                foreach (Guid branchKey in branchKeys)
+                {
+                    _mediaCacheService.RefreshMemoryCacheAsync(branchKey).GetAwaiter().GetResult();
+                }
+            }
+        }
+
+        if (payload.ChangeTypes.HasType(TreeChangeTypes.RefreshAll))
+        {
+            _mediaCacheService.ClearMemoryCacheAsync(CancellationToken.None).GetAwaiter().GetResult();
+        }
+
+        if (payload.ChangeTypes.HasType(TreeChangeTypes.Remove))
+        {
+            _mediaCacheService.RemoveFromMemoryCacheAsync(key).GetAwaiter().GetResult();
+        }
+    }
+
    private void HandleNavigation(JsonPayload payload)
    {
        if (payload.Key is null)