V15: Only cache items if all ancestors are published (#18337)

* Introduce IsDocumentPublishedInAnyCulture

Sometimes we don't care about culture

* Check ancestor path when resolving cache items

* Fix tests

* Rebuild NavigationService

* Only set node if it has a published ancestor path

* Remove branch when unpublished

* Add tests

* Add seed test

* Consider published ancestor path when seeding documents

* Introduce MediaBreadthFirstKeyProviderTests

This is needed since the logic of document and media is no longer the same

* Remove unused services

* Move assert page to helper

* Add variant tests

* Add tests

* Filter keys in ContentTypeSeedKeyProvider

* Fix tests

* Add failing test showing refreshing issue

* Don't blow up if we can't resolve the node from navigation cache

Turns out that this can actually happen :D Should be fine to just return false

* Refactor cache refresher check

* Make NavigationQueryService service protected

* Add comment on how to refactor breadth first key provider

* Refactor if statement

src/Umbraco.PublishedCache.HybridCache/SeedKeyProviders/BreadthFirstKeyProvider.cs

@@ -4,12 +4,12 @@ namespace Umbraco.Cms.Infrastructure.HybridCache.SeedKeyProviders;
 
 public abstract class BreadthFirstKeyProvider
 {
-    private readonly INavigationQueryService _navigationQueryService;
+    protected readonly INavigationQueryService NavigationQueryService;
     private readonly int _seedCount;
 
     public BreadthFirstKeyProvider(INavigationQueryService navigationQueryService, int seedCount)
     {
-        _navigationQueryService = navigationQueryService;
+        NavigationQueryService = navigationQueryService;
         _seedCount = seedCount;
     }
 
@@ -24,7 +24,7 @@ public abstract class BreadthFirstKeyProvider
         HashSet<Guid> keys = [];
         int keyCount = 0;
 
-        if (_navigationQueryService.TryGetRootKeys(out IEnumerable<Guid> rootKeys) is false)
+        if (NavigationQueryService.TryGetRootKeys(out IEnumerable<Guid> rootKeys) is false)
         {
             return new HashSet<Guid>();
         }
@@ -44,7 +44,7 @@ public abstract class BreadthFirstKeyProvider
         {
             Guid key = keyQueue.Dequeue();
 
-            if (_navigationQueryService.TryGetChildrenKeys(key, out IEnumerable<Guid> childKeys) is false)
+            if (NavigationQueryService.TryGetChildrenKeys(key, out IEnumerable<Guid> childKeys) is false)
             {
                 continue;
             }

src/Umbraco.PublishedCache.HybridCache/SeedKeyProviders/Document/ContentTypeSeedKeyProvider.cs

@@ -1,6 +1,7 @@
 using Microsoft.Extensions.Options;
 using Umbraco.Cms.Core.Models;
 using Umbraco.Cms.Core.Scoping;
+using Umbraco.Cms.Core.Services.Navigation;
 using Umbraco.Cms.Infrastructure.HybridCache.Persistence;
 
 namespace Umbraco.Cms.Infrastructure.HybridCache.SeedKeyProviders.Document;
@@ -9,22 +10,28 @@ internal sealed class ContentTypeSeedKeyProvider : IDocumentSeedKeyProvider
 {
     private readonly ICoreScopeProvider _scopeProvider;
     private readonly IDatabaseCacheRepository _databaseCacheRepository;
+    private readonly IPublishStatusQueryService _publishStatusService;
     private readonly CacheSettings _cacheSettings;
 
     public ContentTypeSeedKeyProvider(
         ICoreScopeProvider scopeProvider,
         IDatabaseCacheRepository databaseCacheRepository,
-        IOptions<CacheSettings> cacheSettings)
+        IOptions<CacheSettings> cacheSettings,
+        IPublishStatusQueryService publishStatusService)
     {
         _scopeProvider = scopeProvider;
         _databaseCacheRepository = databaseCacheRepository;
+        _publishStatusService = publishStatusService;
         _cacheSettings = cacheSettings.Value;
     }
 
     public ISet<Guid> GetSeedKeys()
     {
         using ICoreScope scope = _scopeProvider.CreateCoreScope();
-        var documentKeys = _databaseCacheRepository.GetDocumentKeysByContentTypeKeys(_cacheSettings.ContentTypeKeys, published: true).ToHashSet();
+        var documentKeys = _databaseCacheRepository
+            .GetDocumentKeysByContentTypeKeys(_cacheSettings.ContentTypeKeys, published: true)
+            .Where(key => _publishStatusService.IsDocumentPublishedInAnyCulture(key))
+            .ToHashSet();
         scope.Complete();
 
         return documentKeys;

src/Umbraco.PublishedCache.HybridCache/SeedKeyProviders/Document/DocumentBreadthFirstKeyProvider.cs

@@ -6,10 +6,78 @@ namespace Umbraco.Cms.Infrastructure.HybridCache.SeedKeyProviders.Document;
 
 internal sealed class DocumentBreadthFirstKeyProvider : BreadthFirstKeyProvider, IDocumentSeedKeyProvider
 {
+    private readonly IPublishStatusQueryService _publishStatusService;
+    private readonly int _seedCount;
+
     public DocumentBreadthFirstKeyProvider(
         IDocumentNavigationQueryService documentNavigationQueryService,
-        IOptions<CacheSettings> cacheSettings)
+        IOptions<CacheSettings> cacheSettings,
+        IPublishStatusQueryService publishStatusService)
         : base(documentNavigationQueryService, cacheSettings.Value.DocumentBreadthFirstSeedCount)
     {
+        _publishStatusService = publishStatusService;
+        _seedCount = cacheSettings.Value.DocumentBreadthFirstSeedCount;
+    }
+
+
+    // TODO: V16 - Move this method back to the base class
+    // The main need for this is because we now need to filter the keys, based on if they have published ancestor path or not
+    // We should add `FilterKeys` virtual method on the base class that does nothing, and then override it here instead
+    // Note that it's important that we do this filtering as we're doing the search, since we want to make sure we hit the seed count
+    // For instance if you have 500 content nodes, request 100 seeded, we need to return 100 keys, even if we need to filter out 20 of them
+    public new ISet<Guid> GetSeedKeys()
+    {
+        if (_seedCount == 0)
+        {
+            return new HashSet<Guid>();
+        }
+
+        Queue<Guid> keyQueue = new();
+        HashSet<Guid> keys = [];
+        int keyCount = 0;
+
+        if (NavigationQueryService.TryGetRootKeys(out IEnumerable<Guid> rootKeys) is false)
+        {
+            return new HashSet<Guid>();
+        }
+
+        rootKeys = rootKeys.Where(x => _publishStatusService.IsDocumentPublishedInAnyCulture(x));
+
+        foreach (Guid key in rootKeys)
+        {
+            keyCount++;
+            keys.Add(key);
+            keyQueue.Enqueue(key);
+            if (keyCount == _seedCount)
+            {
+                return keys;
+            }
+        }
+
+        while (keyQueue.Count > 0 && keyCount < _seedCount)
+        {
+            Guid key = keyQueue.Dequeue();
+
+            if (NavigationQueryService.TryGetChildrenKeys(key, out IEnumerable<Guid> childKeys) is false)
+            {
+                continue;
+            }
+
+            childKeys = childKeys.Where(x => _publishStatusService.IsDocumentPublishedInAnyCulture(x));
+
+            foreach (Guid childKey in childKeys)
+            {
+                keys.Add(childKey);
+                keyCount++;
+                if (keyCount == _seedCount)
+                {
+                    return keys;
+                }
+
+                keyQueue.Enqueue(childKey);
+            }
+        }
+
+        return keys;
     }
 }