V15: Only cache items if all ancestors are published (#18337)

* Introduce IsDocumentPublishedInAnyCulture

Sometimes we don't care about culture

* Check ancestor path when resolving cache items

* Fix tests

* Rebuild NavigationService

* Only set node if it has a published ancestor path

* Remove branch when unpublished

* Add tests

* Add seed test

* Consider published ancestor path when seeding documents

* Introduce MediaBreadthFirstKeyProviderTests

This is needed since the logic of document and media is no longer the same

* Remove unused services

* Move assert page to helper

* Add variant tests

* Add tests

* Filter keys in ContentTypeSeedKeyProvider

* Fix tests

* Add failing test showing refreshing issue

* Don't blow up if we can't resolve the node from navigation cache

Turns out that this can actually happen :D Should be fine to just return false

* Refactor cache refresher check

* Make NavigationQueryService service protected

* Add comment on how to refactor breadth first key provider

* Refactor if statement

src/Umbraco.Core/Cache/Refreshers/Implement/ContentCacheRefresher.cs

@@ -172,9 +172,20 @@ public sealed class ContentCacheRefresher : PayloadCacheRefresherBase<ContentCac
                 var branchKeys = descendantsKeys.ToList();
                 branchKeys.Add(key);
 
-                foreach (Guid branchKey in branchKeys)
+                // If the branch is unpublished, we need to remove it from cache instead of refreshing it
+                if (IsBranchUnpublished(payload))
                 {
-                    _documentCacheService.RefreshMemoryCacheAsync(branchKey).GetAwaiter().GetResult();
+                    foreach (Guid branchKey in branchKeys)
+                    {
+                        _documentCacheService.RemoveFromMemoryCacheAsync(branchKey).GetAwaiter().GetResult();
+                    }
+                }
+                else
+                {
+                    foreach (Guid branchKey in branchKeys)
+                    {
+                        _documentCacheService.RefreshMemoryCacheAsync(branchKey).GetAwaiter().GetResult();
+                    }
                 }
             }
         }
@@ -190,6 +201,15 @@ public sealed class ContentCacheRefresher : PayloadCacheRefresherBase<ContentCac
         }
     }
 
+    private bool IsBranchUnpublished(JsonPayload payload)
+    {
+        // If unpublished cultures has one or more values, but published cultures does not, this means that the branch is unpublished entirely
+        // And therefore should no longer be resolve-able from the cache, so we need to remove it instead.
+        // Otherwise, some culture is still published, so it should be resolve-able from cache, and published cultures should instead be used.
+        return payload.UnpublishedCultures is not null && payload.UnpublishedCultures.Length != 0 &&
+               (payload.PublishedCultures is null || payload.PublishedCultures.Length == 0);
+    }
+
     private void HandleNavigation(JsonPayload payload)
     {
 

src/Umbraco.Core/Services/PublishStatus/IPublishStatusQueryService.cs

@@ -6,4 +6,11 @@ namespace Umbraco.Cms.Core.Services.Navigation;
 public interface IPublishStatusQueryService
 {
     bool IsDocumentPublished(Guid documentKey, string culture);
+
+    /// <summary>
+    /// Checks if a document is published in any culture.
+    /// </summary>
+    /// <param name="documentKey">Key to check for.</param>
+    /// <returns>True if document has any published culture.</returns>
+    bool IsDocumentPublishedInAnyCulture(Guid documentKey) => IsDocumentPublished(documentKey, string.Empty);
 }

src/Umbraco.Core/Services/PublishStatus/PublishStatusService.cs

@@ -76,6 +76,18 @@ public class PublishStatusService : IPublishStatusManagementService, IPublishSta
         return false;
     }
 
+    /// <inheritdoc />
+    public bool IsDocumentPublishedInAnyCulture(Guid documentKey)
+    {
+        if (_publishedCultures.TryGetValue(documentKey, out ISet<string>? publishedCultures))
+        {
+            return publishedCultures.Count > 0;
+        }
+
+        _logger.LogDebug("Document {DocumentKey} not found in the publish status cache", documentKey);
+        return false;
+    }
+
     public async Task AddOrUpdateStatusAsync(Guid documentKey, CancellationToken cancellationToken)
     {
         using ICoreScope scope = _coreScopeProvider.CreateCoreScope();

src/Umbraco.PublishedCache.HybridCache/SeedKeyProviders/BreadthFirstKeyProvider.cs

@@ -4,12 +4,12 @@ namespace Umbraco.Cms.Infrastructure.HybridCache.SeedKeyProviders;
 
 public abstract class BreadthFirstKeyProvider
 {
-    private readonly INavigationQueryService _navigationQueryService;
+    protected readonly INavigationQueryService NavigationQueryService;
     private readonly int _seedCount;
 
     public BreadthFirstKeyProvider(INavigationQueryService navigationQueryService, int seedCount)
     {
-        _navigationQueryService = navigationQueryService;
+        NavigationQueryService = navigationQueryService;
         _seedCount = seedCount;
     }
 
@@ -24,7 +24,7 @@ public abstract class BreadthFirstKeyProvider
         HashSet<Guid> keys = [];
         int keyCount = 0;
 
-        if (_navigationQueryService.TryGetRootKeys(out IEnumerable<Guid> rootKeys) is false)
+        if (NavigationQueryService.TryGetRootKeys(out IEnumerable<Guid> rootKeys) is false)
         {
             return new HashSet<Guid>();
         }
@@ -44,7 +44,7 @@ public abstract class BreadthFirstKeyProvider
         {
             Guid key = keyQueue.Dequeue();
 
-            if (_navigationQueryService.TryGetChildrenKeys(key, out IEnumerable<Guid> childKeys) is false)
+            if (NavigationQueryService.TryGetChildrenKeys(key, out IEnumerable<Guid> childKeys) is false)
             {
                 continue;
             }

src/Umbraco.PublishedCache.HybridCache/SeedKeyProviders/Document/ContentTypeSeedKeyProvider.cs

@@ -1,6 +1,7 @@
 using Microsoft.Extensions.Options;
 using Umbraco.Cms.Core.Models;
 using Umbraco.Cms.Core.Scoping;
+using Umbraco.Cms.Core.Services.Navigation;
 using Umbraco.Cms.Infrastructure.HybridCache.Persistence;
 
 namespace Umbraco.Cms.Infrastructure.HybridCache.SeedKeyProviders.Document;
@@ -9,22 +10,28 @@ internal sealed class ContentTypeSeedKeyProvider : IDocumentSeedKeyProvider
 {
     private readonly ICoreScopeProvider _scopeProvider;
     private readonly IDatabaseCacheRepository _databaseCacheRepository;
+    private readonly IPublishStatusQueryService _publishStatusService;
     private readonly CacheSettings _cacheSettings;
 
     public ContentTypeSeedKeyProvider(
         ICoreScopeProvider scopeProvider,
         IDatabaseCacheRepository databaseCacheRepository,
-        IOptions<CacheSettings> cacheSettings)
+        IOptions<CacheSettings> cacheSettings,
+        IPublishStatusQueryService publishStatusService)
     {
         _scopeProvider = scopeProvider;
         _databaseCacheRepository = databaseCacheRepository;
+        _publishStatusService = publishStatusService;
         _cacheSettings = cacheSettings.Value;
     }
 
     public ISet<Guid> GetSeedKeys()
     {
         using ICoreScope scope = _scopeProvider.CreateCoreScope();
-        var documentKeys = _databaseCacheRepository.GetDocumentKeysByContentTypeKeys(_cacheSettings.ContentTypeKeys, published: true).ToHashSet();
+        var documentKeys = _databaseCacheRepository
+            .GetDocumentKeysByContentTypeKeys(_cacheSettings.ContentTypeKeys, published: true)
+            .Where(key => _publishStatusService.IsDocumentPublishedInAnyCulture(key))
+            .ToHashSet();
         scope.Complete();
 
         return documentKeys;

src/Umbraco.PublishedCache.HybridCache/SeedKeyProviders/Document/DocumentBreadthFirstKeyProvider.cs

@@ -6,10 +6,78 @@ namespace Umbraco.Cms.Infrastructure.HybridCache.SeedKeyProviders.Document;
 
 internal sealed class DocumentBreadthFirstKeyProvider : BreadthFirstKeyProvider, IDocumentSeedKeyProvider
 {
+    private readonly IPublishStatusQueryService _publishStatusService;
+    private readonly int _seedCount;
+
     public DocumentBreadthFirstKeyProvider(
         IDocumentNavigationQueryService documentNavigationQueryService,
-        IOptions<CacheSettings> cacheSettings)
+        IOptions<CacheSettings> cacheSettings,
+        IPublishStatusQueryService publishStatusService)
         : base(documentNavigationQueryService, cacheSettings.Value.DocumentBreadthFirstSeedCount)
     {
+        _publishStatusService = publishStatusService;
+        _seedCount = cacheSettings.Value.DocumentBreadthFirstSeedCount;
+    }
+
+
+    // TODO: V16 - Move this method back to the base class
+    // The main need for this is because we now need to filter the keys, based on if they have published ancestor path or not
+    // We should add `FilterKeys` virtual method on the base class that does nothing, and then override it here instead
+    // Note that it's important that we do this filtering as we're doing the search, since we want to make sure we hit the seed count
+    // For instance if you have 500 content nodes, request 100 seeded, we need to return 100 keys, even if we need to filter out 20 of them
+    public new ISet<Guid> GetSeedKeys()
+    {
+        if (_seedCount == 0)
+        {
+            return new HashSet<Guid>();
+        }
+
+        Queue<Guid> keyQueue = new();
+        HashSet<Guid> keys = [];
+        int keyCount = 0;
+
+        if (NavigationQueryService.TryGetRootKeys(out IEnumerable<Guid> rootKeys) is false)
+        {
+            return new HashSet<Guid>();
+        }
+
+        rootKeys = rootKeys.Where(x => _publishStatusService.IsDocumentPublishedInAnyCulture(x));
+
+        foreach (Guid key in rootKeys)
+        {
+            keyCount++;
+            keys.Add(key);
+            keyQueue.Enqueue(key);
+            if (keyCount == _seedCount)
+            {
+                return keys;
+            }
+        }
+
+        while (keyQueue.Count > 0 && keyCount < _seedCount)
+        {
+            Guid key = keyQueue.Dequeue();
+
+            if (NavigationQueryService.TryGetChildrenKeys(key, out IEnumerable<Guid> childKeys) is false)
+            {
+                continue;
+            }
+
+            childKeys = childKeys.Where(x => _publishStatusService.IsDocumentPublishedInAnyCulture(x));
+
+            foreach (Guid childKey in childKeys)
+            {
+                keys.Add(childKey);
+                keyCount++;
+                if (keyCount == _seedCount)
+                {
+                    return keys;
+                }
+
+                keyQueue.Enqueue(childKey);
+            }
+        }
+
+        return keys;
     }
 }

src/Umbraco.PublishedCache.HybridCache/Services/DocumentCacheService.cs

@@ -6,6 +6,7 @@ using Umbraco.Cms.Core.Models.PublishedContent;
 using Umbraco.Cms.Core.PublishedCache;
 using Umbraco.Cms.Core.Scoping;
 using Umbraco.Cms.Core.Services;
+using Umbraco.Cms.Core.Services.Navigation;
 using Umbraco.Cms.Infrastructure.HybridCache.Factories;
 using Umbraco.Cms.Infrastructure.HybridCache.Persistence;
 using Umbraco.Cms.Infrastructure.HybridCache.Serialization;
@@ -24,8 +25,11 @@ internal sealed class DocumentCacheService : IDocumentCacheService
     private readonly IEnumerable<IDocumentSeedKeyProvider> _seedKeyProviders;
     private readonly IPublishedModelFactory _publishedModelFactory;
     private readonly IPreviewService _previewService;
+    private readonly IPublishStatusQueryService _publishStatusQueryService;
+    private readonly IDocumentNavigationQueryService _documentNavigationQueryService;
     private readonly CacheSettings _cacheSettings;
     private HashSet<Guid>? _seedKeys;
+
     private HashSet<Guid> SeedKeys
     {
         get
@@ -56,7 +60,9 @@ internal sealed class DocumentCacheService : IDocumentCacheService
         IEnumerable<IDocumentSeedKeyProvider> seedKeyProviders,
         IOptions<CacheSettings> cacheSettings,
         IPublishedModelFactory publishedModelFactory,
-        IPreviewService previewService)
+        IPreviewService previewService,
+        IPublishStatusQueryService publishStatusQueryService,
+        IDocumentNavigationQueryService documentNavigationQueryService)
     {
         _databaseCacheRepository = databaseCacheRepository;
         _idKeyMap = idKeyMap;
@@ -67,6 +73,8 @@ internal sealed class DocumentCacheService : IDocumentCacheService
         _seedKeyProviders = seedKeyProviders;
         _publishedModelFactory = publishedModelFactory;
         _previewService = previewService;
+        _publishStatusQueryService = publishStatusQueryService;
+        _documentNavigationQueryService = documentNavigationQueryService;
         _cacheSettings = cacheSettings.Value;
     }
 
@@ -101,6 +109,20 @@ internal sealed class DocumentCacheService : IDocumentCacheService
             {
                 using ICoreScope scope = _scopeProvider.CreateCoreScope();
                 ContentCacheNode? contentCacheNode = await _databaseCacheRepository.GetContentSourceAsync(key, preview);
+
+                // If we can resolve the content cache node, we still need to check if the ancestor path is published.
+                // This does cost some performance, but it's necessary to ensure that the content is actually published.
+                // When unpublishing a node, a payload with RefreshBranch is published, so we don't have to worry about this.
+                // Similarly, when a branch is published, next time the content is requested, the parent will be published,
+                // this works because we don't cache null values.
+                if (preview is false && contentCacheNode is not null)
+                {
+                    if (HasPublishedAncestorPath(contentCacheNode.Key) is false)
+                    {
+                        return null;
+                    }
+                }
+
                 scope.Complete();
                 return contentCacheNode;
             },
@@ -116,6 +138,28 @@ internal sealed class DocumentCacheService : IDocumentCacheService
         return _publishedContentFactory.ToIPublishedContent(contentCacheNode, preview).CreateModel(_publishedModelFactory);
     }
 
+    private bool HasPublishedAncestorPath(Guid contentKey)
+    {
+        var success = _documentNavigationQueryService.TryGetAncestorsKeys(contentKey, out IEnumerable<Guid> keys);
+        if (success is false)
+        {
+            // This might happen is certain cases, since 0notifications are not ordered, for instance, if you save and publish a content node in the same scope.
+            // In this case we'll try and update the node in the cache even though it hasn't been updated in the document navigation cache yet.
+            // It's okay to just return false here, since the node will be loaded later when it's actually requested.
+            return false;
+        }
+
+        foreach (Guid key in keys)
+        {
+            if (_publishStatusQueryService.IsDocumentPublishedInAnyCulture(key) is false)
+            {
+                return false;
+            }
+        }
+
+        return true;
+    }
+
     private bool GetPreview()
     {
         return _previewService.IsInPreview();
@@ -169,7 +213,7 @@ internal sealed class DocumentCacheService : IDocumentCacheService
         }
 
         ContentCacheNode? publishedNode = await _databaseCacheRepository.GetContentSourceAsync(key, false);
-        if (publishedNode is not null)
+        if (publishedNode is not null && HasPublishedAncestorPath(publishedNode.Key))
         {
             await _hybridCache.SetAsync(GetCacheKey(publishedNode.Key, false), publishedNode, GetEntryOptions(publishedNode.Key));
         }
@@ -195,7 +239,7 @@ internal sealed class DocumentCacheService : IDocumentCacheService
             var cacheKey = GetCacheKey(key, false);
 
                 // We'll use GetOrCreateAsync because it may be in the second level cache, in which case we don't have to re-seed.
-                ContentCacheNode? cachedValue = await _hybridCache.GetOrCreateAsync<ContentCacheNode?>(
+            ContentCacheNode? cachedValue = await _hybridCache.GetOrCreateAsync<ContentCacheNode?>(
                     cacheKey,
                     async cancel =>
                     {
@@ -212,17 +256,20 @@ internal sealed class DocumentCacheService : IDocumentCacheService
 
                         return cacheNode;
                     },
-                GetSeedEntryOptions(),
-                cancellationToken: cancellationToken);
+                    GetSeedEntryOptions(),
+                    cancellationToken: cancellationToken);
 
                 // If the value is null, it's likely because
-                if (cachedValue is null)
+            if (cachedValue is null)
             {
-                await _hybridCache.RemoveAsync(cacheKey);
+                await _hybridCache.RemoveAsync(cacheKey, cancellationToken);
             }
         }
     }
 
+    // Internal for test purposes.
+    internal void ResetSeedKeys() => _seedKeys = null;
+
     private HybridCacheEntryOptions GetSeedEntryOptions() => new()
     {
         Expiration = _cacheSettings.Entry.Document.SeedCacheDuration,

tests/Umbraco.Tests.Integration/Testing/UmbracoIntegrationTestWithContentEditing.cs

@@ -20,7 +20,7 @@ public abstract class UmbracoIntegrationTestWithContentEditing : UmbracoIntegrat
 
     protected ITemplateService TemplateService => GetRequiredService<ITemplateService>();
 
-    private IContentEditingService ContentEditingService => (IContentEditingService)GetRequiredService<IContentEditingService>();
+    protected IContentEditingService ContentEditingService => (IContentEditingService)GetRequiredService<IContentEditingService>();
 
     private IContentPublishingService ContentPublishingService => (IContentPublishingService)GetRequiredService<IContentPublishingService>();
 

tests/Umbraco.Tests.Integration/Umbraco.PublishedCache.HybridCache/CacheTestsHelper.cs

@@ -0,0 +1,54 @@
+using NUnit.Framework;
+using Umbraco.Cms.Core.Models;
+using Umbraco.Cms.Core.Models.PublishedContent;
+
+namespace Umbraco.Cms.Tests.Integration.Umbraco.PublishedCache.HybridCache;
+
+internal static class CacheTestsHelper
+{
+    internal static void AssertPage(IContent baseContent, IPublishedContent? comparisonContent, bool isPublished = true)
+    {
+        Assert.Multiple(() =>
+        {
+            Assert.IsNotNull(comparisonContent);
+            if (baseContent.ContentType.VariesByCulture())
+            {
+                foreach (var culture in baseContent.CultureInfos ?? Enumerable.Empty<ContentCultureInfos>())
+                {
+                    if (comparisonContent.Cultures.TryGetValue(culture.Culture, out var publishedCulture) is false)
+                    {
+                        continue;
+                    }
+
+                    Assert.That(publishedCulture.Name, Is.EqualTo(culture.Name));
+                }
+            }
+            else
+            {
+                Assert.That(comparisonContent.Name, Is.EqualTo(baseContent.Name));
+            }
+
+            Assert.That(comparisonContent.IsPublished(), Is.EqualTo(isPublished));
+        });
+
+        AssertProperties(baseContent.Properties, comparisonContent!.Properties);
+    }
+
+    internal static void AssertProperties(IPropertyCollection propertyCollection,
+        IEnumerable<IPublishedProperty> publishedProperties)
+    {
+        foreach (var prop in propertyCollection)
+        {
+            AssertProperty(prop, publishedProperties.First(x => x.Alias == prop.Alias));
+        }
+    }
+
+    internal static void AssertProperty(IProperty property, IPublishedProperty publishedProperty)
+    {
+        Assert.Multiple(() =>
+        {
+            Assert.AreEqual(property.Alias, publishedProperty.Alias);
+            Assert.AreEqual(property.PropertyType.Alias, publishedProperty.PropertyType.Alias);
+        });
+    }
+}

tests/Umbraco.Tests.Integration/Umbraco.PublishedCache.HybridCache/DocumentHybridCacheAncestryTests.cs

@@ -0,0 +1,93 @@
+using NUnit.Framework;
+using Umbraco.Cms.Core;
+using Umbraco.Cms.Core.Cache;
+using Umbraco.Cms.Core.Models;
+using Umbraco.Cms.Core.Models.PublishedContent;
+using Umbraco.Cms.Core.Notifications;
+using Umbraco.Cms.Core.PublishedCache;
+using Umbraco.Cms.Core.Services;
+using Umbraco.Cms.Core.Sync;
+using Umbraco.Cms.Infrastructure.HybridCache.Services;
+using Umbraco.Cms.Tests.Common.Builders;
+using Umbraco.Cms.Tests.Common.Testing;
+using Umbraco.Cms.Tests.Integration.Testing;
+using Umbraco.Cms.Tests.Integration.Umbraco.Infrastructure.Services;
+
+namespace Umbraco.Cms.Tests.Integration.Umbraco.PublishedCache.HybridCache;
+
+[TestFixture]
+[UmbracoTest(Database = UmbracoTestOptions.Database.NewSchemaPerTest)]
+public class DocumentHybridCacheAncestryTests : UmbracoIntegrationTestWithContent
+{
+    private IContentPublishingService ContentPublishingService => GetRequiredService<IContentPublishingService>();
+
+    private IPublishedContentCache PublishedContentCache => GetRequiredService<IPublishedContentCache>();
+
+    private IDocumentCacheService DocumentCacheService => GetRequiredService<IDocumentCacheService>();
+
+    private Content SubSubPage;
+
+    protected override void CustomTestSetup(IUmbracoBuilder builder)
+    {
+        builder.AddNotificationHandler<ContentTreeChangeNotification, ContentTreeChangeDistributedCacheNotificationHandler>();
+        builder.Services.AddUnique<IServerMessenger, ContentEventsTests.LocalServerMessenger>();
+    }
+
+    public override void Setup()
+    {
+        base.Setup();
+        // Publish documents
+        SubSubPage = ContentBuilder.CreateSimpleContent(ContentType, "SubSubPage", Subpage.Id);
+        SubSubPage.Key = Guid.Parse("E4C369B5-CCCA-4981-ADAC-389824CF6B0B");
+        ContentService.Save(SubSubPage, -1);
+    }
+
+    [Test]
+    public async Task CantGetPublishedContentIfParentIsUnpublished()
+    {
+        // Text Page
+          // Sub Page <-- Unpublished
+                // Sub Sub Page
+        await ContentPublishingService.PublishBranchAsync(Textpage.Key, Array.Empty<string>(), true, Constants.Security.SuperUserKey);
+        await ContentPublishingService.UnpublishAsync(Subpage.Key, null, Constants.Security.SuperUserKey);
+
+        var published = await PublishedContentCache.GetByIdAsync(SubSubPage.Key);
+        Assert.IsNull(published);
+    }
+
+    [Test]
+    public async Task CanGetPublishedContentIfParentIsPublished()
+    {
+        await ContentPublishingService.PublishBranchAsync(Textpage.Key, Array.Empty<string>(), true, Constants.Security.SuperUserKey);
+
+        var published = await PublishedContentCache.GetByIdAsync(SubSubPage.Key);
+        CacheTestsHelper.AssertPage(SubSubPage, published);
+    }
+
+    [Test]
+    public async Task CantGetPublishedContentAfterSeedingIfParentIsUnpublished()
+    {
+        // Text Page
+          // Sub Page <-- Unpublished
+            // Sub Sub Page
+        await ContentPublishingService.PublishBranchAsync(Textpage.Key, Array.Empty<string>(), true, Constants.Security.SuperUserKey);
+        await ContentPublishingService.UnpublishAsync(Subpage.Key, null, Constants.Security.SuperUserKey);
+
+        // Clear cache also seeds, but we have to reset the seed keys first since these are cached from test startup
+        var cacheService = DocumentCacheService as DocumentCacheService;
+        cacheService!.ResetSeedKeys();
+        await DocumentCacheService.ClearMemoryCacheAsync(CancellationToken.None);
+
+        var unpublishedSubSubPage = await PublishedContentCache.GetByIdAsync(SubSubPage.Key);
+        var unpublishedSubPage = await PublishedContentCache.GetByIdAsync(Subpage.Key);
+        Assert.IsNull(unpublishedSubSubPage);
+        Assert.IsNull(unpublishedSubPage);
+
+        // We should however be able to get the still published root Text Page
+        var publishedTextPage = await PublishedContentCache.GetByIdAsync(Textpage.Key);
+        CacheTestsHelper.AssertPage(Textpage, publishedTextPage);
+
+    }
+
+
+}

tests/Umbraco.Tests.Integration/Umbraco.PublishedCache.HybridCache/DocumentHybridCacheAncestryVariantTests.cs

@@ -0,0 +1,184 @@
+using NUnit.Framework;
+using Umbraco.Cms.Core;
+using Umbraco.Cms.Core.Cache;
+using Umbraco.Cms.Core.Models;
+using Umbraco.Cms.Core.Models.ContentPublishing;
+using Umbraco.Cms.Core.Notifications;
+using Umbraco.Cms.Core.PublishedCache;
+using Umbraco.Cms.Core.Services;
+using Umbraco.Cms.Core.Services.ContentTypeEditing;
+using Umbraco.Cms.Core.Sync;
+using Umbraco.Cms.Tests.Common.Builders;
+using Umbraco.Cms.Tests.Common.Builders.Extensions;
+using Umbraco.Cms.Tests.Common.TestHelpers;
+using Umbraco.Cms.Tests.Common.Testing;
+using Umbraco.Cms.Tests.Integration.Testing;
+using Umbraco.Cms.Tests.Integration.Umbraco.Infrastructure.Services;
+
+namespace Umbraco.Cms.Tests.Integration.Umbraco.PublishedCache.HybridCache;
+
+[TestFixture]
+[UmbracoTest(Database = UmbracoTestOptions.Database.NewSchemaPerTest)]
+public class DocumentHybridCacheAncestryVariantTests : UmbracoIntegrationTest
+{
+    private string _englishIsoCode = "en-US";
+    private string _danishIsoCode = "da-DK";
+    private string _variantTitleAlias = "variantTitle";
+    private string _variantTitleName = "Variant Title";
+    private string _invariantTitleAlias = "invariantTitle";
+    private string _invariantTitleName = "Invariant Title";
+
+    private IContent rootContent;
+    private IContent childNode;
+    private IContent grandChildNode;
+
+    private ILanguageService LanguageService => GetRequiredService<ILanguageService>();
+
+    private IContentEditingService ContentEditingService => GetRequiredService<IContentEditingService>();
+
+    private IContentTypeEditingService ContentTypeEditingService => GetRequiredService<IContentTypeEditingService>();
+
+    private IContentPublishingService ContentPublishingService => GetRequiredService<IContentPublishingService>();
+
+    private IPublishedContentCache PublishedContentCache => GetRequiredService<IPublishedContentCache>();
+
+    protected override void CustomTestSetup(IUmbracoBuilder builder)
+    {
+        builder.AddNotificationHandler<ContentTreeChangeNotification, ContentTreeChangeDistributedCacheNotificationHandler>();
+        builder.Services.AddUnique<IServerMessenger, ContentEventsTests.LocalServerMessenger>();
+    }
+
+    [SetUp]
+    public async Task Setup() => await CreateTestData();
+
+    [Test]
+    [TestCase(true)]
+    [TestCase(false)]
+    public async Task AllCulturesUnpublished(bool preview)
+    {
+        // Publish branch in all cultures
+        var publishAttempt = await ContentPublishingService.PublishBranchAsync(rootContent.Key, [_englishIsoCode, _danishIsoCode], true, Constants.Security.SuperUserKey);
+        Assert.IsTrue(publishAttempt.Success);
+        Assert.That(publishAttempt.Result.SucceededItems.Count(), Is.EqualTo(3));
+
+        // Unpublish all cultures in child
+        var unpublishAttempt = await ContentPublishingService.UnpublishAsync(childNode.Key, new HashSet<string>([_englishIsoCode, _danishIsoCode]), Constants.Security.SuperUserKey);
+        Assert.IsTrue(unpublishAttempt.Success);
+
+        var publishedGrandChild = await PublishedContentCache.GetByIdAsync(grandChildNode.Key, preview);
+
+        if (preview)
+        {
+            CacheTestsHelper.AssertPage(grandChildNode, publishedGrandChild, false);
+        }
+        else
+        {
+            Assert.IsNull(publishedGrandChild);
+        }
+    }
+
+    [Test]
+    public async Task SingleCultureUnpublished()
+    {
+        var publishAttempt = await ContentPublishingService.PublishBranchAsync(rootContent.Key, [_englishIsoCode, _danishIsoCode], true, Constants.Security.SuperUserKey);
+        Assert.IsTrue(publishAttempt.Success);
+        Assert.That(publishAttempt.Result.SucceededItems.Count(), Is.EqualTo(3));
+
+        // Unpublish only english culture
+        var unpublishAttempt = await ContentPublishingService.UnpublishAsync(childNode.Key, new HashSet<string> { _englishIsoCode }, Constants.Security.SuperUserKey);
+        Assert.IsTrue(unpublishAttempt.Success);
+
+        var publishedGrandChild = await PublishedContentCache.GetByIdAsync(grandChildNode.Key, false);
+        CacheTestsHelper.AssertPage(grandChildNode, publishedGrandChild, false);
+        Assert.IsTrue(publishedGrandChild!.IsPublished(_danishIsoCode));
+    }
+
+    [Test]
+    public async Task SingleCulturePublished()
+    {
+        var publishAttempt = await ContentPublishingService.PublishAsync(
+            rootContent.Key,
+            new List<CulturePublishScheduleModel>
+            {
+                new() { Culture = _danishIsoCode },
+                new() { Culture = _englishIsoCode },
+            },
+            Constants.Security.SuperUserKey);
+        Assert.IsTrue(publishAttempt.Success);
+
+        // Publish only single culture.
+        var publishChildAttempt = await ContentPublishingService.PublishAsync(
+            childNode.Key,
+            new List<CulturePublishScheduleModel>
+            {
+                new() { Culture = _danishIsoCode },
+            },
+            Constants.Security.SuperUserKey);
+        Assert.IsTrue(publishChildAttempt.Success);
+
+        var publishGrandChildAttempt = await ContentPublishingService.PublishAsync(
+            grandChildNode.Key,
+            new List<CulturePublishScheduleModel>
+            {
+                new() { Culture = _danishIsoCode },
+            },
+            Constants.Security.SuperUserKey);
+        Assert.IsTrue(publishGrandChildAttempt.Success);
+
+        var publishedGrandChild = await PublishedContentCache.GetByIdAsync(grandChildNode.Key, false);
+
+        CacheTestsHelper.AssertPage(grandChildNode, publishedGrandChild, false);
+        Assert.IsTrue(publishedGrandChild!.IsPublished(_danishIsoCode));
+        Assert.IsFalse(publishedGrandChild.IsPublished(_englishIsoCode));
+    }
+
+    private async Task CreateTestData()
+    {
+        var language = new LanguageBuilder()
+            .WithCultureInfo(_danishIsoCode)
+            .Build();
+        await LanguageService.CreateAsync(language, Constants.Security.SuperUserKey);
+
+        var contentTypeCreateModel = ContentTypeEditingBuilder.CreateContentTypeWithTwoPropertiesOneVariantAndOneInvariant(
+            "cultureVariationTest", "Culture Variation Test", _variantTitleAlias, _variantTitleName,
+            _invariantTitleAlias, _invariantTitleName);
+        contentTypeCreateModel.AllowedAsRoot = true;
+        var contentTypeAttempt = await ContentTypeEditingService.CreateAsync(contentTypeCreateModel, Constants.Security.SuperUserKey);
+        if (contentTypeAttempt.Success is false)
+        {
+            throw new Exception("Failed to create content type");
+        }
+
+        var contentType = contentTypeAttempt.Result!;
+        var updateModel = ContentTypeUpdateHelper.CreateContentTypeUpdateModel(contentType);
+        updateModel.AllowedContentTypes = [new ContentTypeSort { Alias = contentType.Alias, Key = contentType.Key, SortOrder = 0 }];
+        var updateAttempt = await ContentTypeEditingService.UpdateAsync(contentType, updateModel, Constants.Security.SuperUserKey);
+        if (updateAttempt.Success is false)
+        {
+            throw new Exception("Failed to update content type");
+        }
+
+        var contentCreateModel = ContentEditingBuilder.CreateContentWithTwoVariantProperties(
+            contentTypeAttempt.Result.Key,
+            _danishIsoCode,
+            _englishIsoCode,
+            _variantTitleAlias,
+            _variantTitleName);
+
+        var rootResult = await ContentEditingService.CreateAsync(contentCreateModel, Constants.Security.SuperUserKey);
+        Assert.IsTrue(rootResult.Success);
+        rootContent = rootResult.Result.Content!;
+
+        contentCreateModel.ParentKey = rootContent.Key;
+        contentCreateModel.Key = Guid.NewGuid();
+        var childResult = await ContentEditingService.CreateAsync(contentCreateModel, Constants.Security.SuperUserKey);
+        Assert.IsTrue(childResult.Success);
+        childNode = childResult.Result.Content!;
+
+        contentCreateModel.ParentKey = childNode.Key;
+        contentCreateModel.Key = Guid.NewGuid();
+        var grandChildResult = await ContentEditingService.CreateAsync(contentCreateModel, Constants.Security.SuperUserKey);
+        Assert.IsTrue(grandChildResult.Success);
+        grandChildNode = grandChildResult.Result.Content!;
+    }
+}

tests/Umbraco.Tests.Integration/Umbraco.PublishedCache.HybridCache/DocumentHybridCacheMockTests.cs

@@ -90,6 +90,9 @@ public class DocumentHybridCacheMockTests : UmbracoIntegrationTestWithContent
 
         _mockedNucacheRepository.Setup(r => r.DeleteContentItemAsync(It.IsAny<int>()));
 
+        var mockedPublishedStatusService = new Mock<IPublishStatusQueryService>();
+        mockedPublishedStatusService.Setup(x => x.IsDocumentPublishedInAnyCulture(It.IsAny<Guid>())).Returns(true);
+
         _mockDocumentCacheService = new DocumentCacheService(
             _mockedNucacheRepository.Object,
             GetRequiredService<IIdKeyMap>(),
@@ -97,10 +100,12 @@ public class DocumentHybridCacheMockTests : UmbracoIntegrationTestWithContent
             GetRequiredService<Microsoft.Extensions.Caching.Hybrid.HybridCache>(),
             GetRequiredService<IPublishedContentFactory>(),
             GetRequiredService<ICacheNodeFactory>(),
-            GetSeedProviders(),
+            GetSeedProviders(mockedPublishedStatusService.Object),
             new OptionsWrapper<CacheSettings>(new CacheSettings()),
             GetRequiredService<IPublishedModelFactory>(),
-            GetRequiredService<IPreviewService>());
+            GetRequiredService<IPreviewService>(),
+            mockedPublishedStatusService.Object,
+            GetRequiredService<IDocumentNavigationQueryService>());
 
         _mockedCache = new DocumentCache(_mockDocumentCacheService,
             GetRequiredService<IPublishedContentTypeCache>(),
@@ -111,7 +116,7 @@ public class DocumentHybridCacheMockTests : UmbracoIntegrationTestWithContent
 
     // We want to be able to alter the settings for the providers AFTER the test has started
     // So we'll manually create them with a magic options mock.
-    private IEnumerable<IDocumentSeedKeyProvider> GetSeedProviders()
+    private IEnumerable<IDocumentSeedKeyProvider> GetSeedProviders(IPublishStatusQueryService publishStatusQueryService)
     {
         _cacheSettings = new CacheSettings();
         _cacheSettings.DocumentBreadthFirstSeedCount = 0;
@@ -121,8 +126,8 @@ public class DocumentHybridCacheMockTests : UmbracoIntegrationTestWithContent
 
         return new List<IDocumentSeedKeyProvider>
         {
-            new ContentTypeSeedKeyProvider(GetRequiredService<ICoreScopeProvider>(), GetRequiredService<IDatabaseCacheRepository>(), mock.Object),
-            new DocumentBreadthFirstKeyProvider(GetRequiredService<IDocumentNavigationQueryService>(), mock.Object),
+            new ContentTypeSeedKeyProvider(GetRequiredService<ICoreScopeProvider>(), GetRequiredService<IDatabaseCacheRepository>(), mock.Object, publishStatusQueryService),
+            new DocumentBreadthFirstKeyProvider(GetRequiredService<IDocumentNavigationQueryService>(), mock.Object, publishStatusQueryService),
         };
     }
 

tests/Umbraco.Tests.Integration/Umbraco.PublishedCache.HybridCache/DocumentHybridCacheScopeTests.cs

@@ -1,10 +1,15 @@
 using NUnit.Framework;
 using Umbraco.Cms.Core;
+using Umbraco.Cms.Core.Cache;
+using Umbraco.Cms.Core.Models.ContentPublishing;
+using Umbraco.Cms.Core.Notifications;
 using Umbraco.Cms.Core.PublishedCache;
 using Umbraco.Cms.Core.Scoping;
 using Umbraco.Cms.Core.Services;
+using Umbraco.Cms.Core.Sync;
 using Umbraco.Cms.Tests.Common.Testing;
 using Umbraco.Cms.Tests.Integration.Testing;
+using Umbraco.Cms.Tests.Integration.Umbraco.Infrastructure.Services;
 
 namespace Umbraco.Cms.Tests.Integration.Umbraco.PublishedCache.HybridCache;
 
@@ -12,7 +17,11 @@ namespace Umbraco.Cms.Tests.Integration.Umbraco.PublishedCache.HybridCache;
 [UmbracoTest(Database = UmbracoTestOptions.Database.NewSchemaPerTest)]
 public class DocumentHybridCacheScopeTests : UmbracoIntegrationTestWithContentEditing
 {
-    protected override void CustomTestSetup(IUmbracoBuilder builder) => builder.AddUmbracoHybridCache();
+    protected override void CustomTestSetup(IUmbracoBuilder builder)
+    {
+        builder.AddNotificationHandler<ContentTreeChangeNotification, ContentTreeChangeDistributedCacheNotificationHandler>();
+        builder.Services.AddUnique<IServerMessenger, ContentEventsTests.LocalServerMessenger>();
+    }
 
     private IPublishedContentCache PublishedContentHybridCache => GetRequiredService<IPublishedContentCache>();
 
@@ -81,4 +90,26 @@ public class DocumentHybridCacheScopeTests : UmbracoIntegrationTestWithContentEd
         // Published page should not be in cache, as we rolled scope back.
         Assert.IsNotNull(publishedPage);
     }
+
+    [Test]
+    public async Task Can_Save_And_Publish_In_Same_Scope()
+    {
+        var key = Guid.NewGuid();
+        using (var scope = CoreScopeProvider.CreateCoreScope())
+        {
+            Textpage.Key = key;
+            var result = await ContentEditingService.CreateAsync(Textpage, Constants.Security.SuperUserKey);
+            Assert.IsTrue(result);
+            var publishResult = await ContentPublishingService.PublishAsync(Textpage.Key.Value, new List<CulturePublishScheduleModel>
+            {
+                new() { Culture = "*" },
+            }, Constants.Security.SuperUserKey);
+            Assert.IsTrue(publishResult.Success);
+
+            scope.Complete();
+        }
+
+        var published = await PublishedContentHybridCache.GetByIdAsync(key);
+        Assert.IsNotNull(published);
+    }
 }

tests/Umbraco.Tests.UnitTests/Umbraco.PublishedCache.HybridCache/DocumentBreadthFirstKeyProviderTests.cs

@@ -6,10 +6,19 @@ using Umbraco.Cms.Core.Services.Navigation;
 using Umbraco.Cms.Infrastructure.HybridCache.SeedKeyProviders.Document;
 
 namespace Umbraco.Cms.Tests.UnitTests.Umbraco.PublishedCache.HybridCache;
-
 [TestFixture]
 public class DocumentBreadthFirstKeyProviderTests
 {
+    private IPublishStatusQueryService PublishStatusQueryService
+    {
+        get
+        {
+            var mock = new Mock<IPublishStatusQueryService>();
+            mock.Setup(x => x.IsDocumentPublishedInAnyCulture(It.IsAny<Guid>())).Returns(true);
+            return mock.Object;
+        }
+    }
+
     [Test]
     public void ZeroSeedCountReturnsZeroKeys()
     {
@@ -22,7 +31,7 @@ public class DocumentBreadthFirstKeyProviderTests
         navigationQueryService.Setup(x => x.TryGetChildrenKeys(It.IsAny<Guid>(), out rootChildren)).Returns(true);
 
         var cacheSettings = new CacheSettings { DocumentBreadthFirstSeedCount = 0 };
-        var sut = new DocumentBreadthFirstKeyProvider(navigationQueryService.Object, Options.Create(cacheSettings));
+        var sut = new DocumentBreadthFirstKeyProvider(navigationQueryService.Object, Options.Create(cacheSettings), PublishStatusQueryService);
 
         var result = sut.GetSeedKeys();
 
@@ -46,7 +55,7 @@ public class DocumentBreadthFirstKeyProviderTests
 
         var expected = 3;
         var cacheSettings = new CacheSettings { DocumentBreadthFirstSeedCount = expected };
-        var sut = new DocumentBreadthFirstKeyProvider(navigationQueryService.Object, Options.Create(cacheSettings));
+        var sut = new DocumentBreadthFirstKeyProvider(navigationQueryService.Object, Options.Create(cacheSettings), PublishStatusQueryService);
 
         var result = sut.GetSeedKeys();
 
@@ -77,7 +86,7 @@ public class DocumentBreadthFirstKeyProviderTests
         // This'll get all children but no grandchildren
         var cacheSettings = new CacheSettings { DocumentBreadthFirstSeedCount = 4 };
 
-        var sut = new DocumentBreadthFirstKeyProvider(navigationQueryService.Object, Options.Create(cacheSettings));
+        var sut = new DocumentBreadthFirstKeyProvider(navigationQueryService.Object, Options.Create(cacheSettings), PublishStatusQueryService);
 
         var result = sut.GetSeedKeys();
 
@@ -105,7 +114,7 @@ public class DocumentBreadthFirstKeyProviderTests
         var settings = new CacheSettings { DocumentBreadthFirstSeedCount = int.MaxValue };
 
 
-        var sut = new DocumentBreadthFirstKeyProvider(navigationQueryService.Object, Options.Create(settings));
+        var sut = new DocumentBreadthFirstKeyProvider(navigationQueryService.Object, Options.Create(settings), PublishStatusQueryService);
 
         var result = sut.GetSeedKeys();
 

tests/Umbraco.Tests.UnitTests/Umbraco.PublishedCache.HybridCache/MediaBreadthFirstKeyProviderTests.cs

@@ -0,0 +1,115 @@
+using Microsoft.Extensions.Options;
+using Moq;
+using NUnit.Framework;
+using Umbraco.Cms.Core.Models;
+using Umbraco.Cms.Core.Services.Navigation;
+using Umbraco.Cms.Infrastructure.HybridCache.SeedKeyProviders.Media;
+
+namespace Umbraco.Cms.Tests.UnitTests.Umbraco.PublishedCache.HybridCache;
+
+[TestFixture]
+public class MediaBreadthFirstKeyProviderTests
+{
+    [Test]
+    public void ZeroSeedCountReturnsZeroKeys()
+    {
+        // The structure here doesn't matter greatly, it just matters that there is something.
+        var navigationQueryService = new Mock<IMediaNavigationQueryService>();
+        var rootKey = Guid.NewGuid();
+        IEnumerable<Guid> rootKeyList = new List<Guid> { rootKey };
+        IEnumerable<Guid> rootChildren = new List<Guid> { Guid.NewGuid(), Guid.NewGuid(), Guid.NewGuid() };
+        navigationQueryService.Setup(x => x.TryGetRootKeys(out rootKeyList)).Returns(true);
+        navigationQueryService.Setup(x => x.TryGetChildrenKeys(It.IsAny<Guid>(), out rootChildren)).Returns(true);
+
+        var cacheSettings = new CacheSettings { MediaBreadthFirstSeedCount = 0 };
+        var sut = new MediaBreadthFirstKeyProvider(navigationQueryService.Object, Options.Create(cacheSettings));
+
+        var result = sut.GetSeedKeys();
+
+        Assert.Zero(result.Count);
+    }
+
+    [Test]
+    public void OnlyReturnsKeysUpToSeedCount()
+    {
+        // Structure
+        // Root
+        // - Child1
+        // - Child2
+        // - Child3
+        var navigationQueryService = new Mock<IMediaNavigationQueryService>();
+        var rootKey = Guid.NewGuid();
+        IEnumerable<Guid> rootKeyList = new List<Guid> { rootKey };
+        IEnumerable<Guid> rootChildren = new List<Guid> { Guid.NewGuid(), Guid.NewGuid(), Guid.NewGuid() };
+        navigationQueryService.Setup(x => x.TryGetRootKeys(out rootKeyList)).Returns(true);
+        navigationQueryService.Setup(x => x.TryGetChildrenKeys(rootKey, out rootChildren)).Returns(true);
+
+        var expected = 3;
+        var cacheSettings = new CacheSettings { MediaBreadthFirstSeedCount = expected };
+        var sut = new MediaBreadthFirstKeyProvider(navigationQueryService.Object, Options.Create(cacheSettings));
+
+        var result = sut.GetSeedKeys();
+
+        Assert.That(result.Count, Is.EqualTo(expected));
+    }
+
+    [Test]
+    public void IsBreadthFirst()
+    {
+        // Structure
+        // Root
+        // - Child1
+        //    - GrandChild
+        // - Child2
+        // - Child3
+
+        var navigationQueryService = new Mock<IMediaNavigationQueryService>();
+        var rootKey = Guid.NewGuid();
+        var child1Key = Guid.NewGuid();
+        var grandChildKey = Guid.NewGuid();
+        IEnumerable<Guid> rootKeyList = new List<Guid> { rootKey };
+        IEnumerable<Guid> rootChildren = new List<Guid> { child1Key, Guid.NewGuid(), Guid.NewGuid() };
+        IEnumerable<Guid> grandChildren = new List<Guid> { grandChildKey };
+        navigationQueryService.Setup(x => x.TryGetRootKeys(out rootKeyList)).Returns(true);
+        navigationQueryService.Setup(x => x.TryGetChildrenKeys(rootKey, out rootChildren)).Returns(true);
+        navigationQueryService.Setup(x => x.TryGetChildrenKeys(child1Key, out grandChildren)).Returns(true);
+
+        // This'll get all children but no grandchildren
+        var cacheSettings = new CacheSettings { MediaBreadthFirstSeedCount = 4 };
+
+        var sut = new MediaBreadthFirstKeyProvider(navigationQueryService.Object, Options.Create(cacheSettings));
+
+        var result = sut.GetSeedKeys();
+
+        Assert.That(result.Contains(grandChildKey), Is.False);
+    }
+
+    [Test]
+    public void CanGetAll()
+    {
+        var navigationQueryService = new Mock<IMediaNavigationQueryService>();
+        var rootKey = Guid.NewGuid();
+
+
+        IEnumerable<Guid> rootKeyList = new List<Guid> { rootKey };
+        var childrenCount = 300;
+        List<Guid> rootChildren = new List<Guid>();
+        for (int i = 0; i < childrenCount; i++)
+        {
+            rootChildren.Add(Guid.NewGuid());
+        }
+
+        IEnumerable<Guid> childrenEnumerable = rootChildren;
+        navigationQueryService.Setup(x => x.TryGetRootKeys(out rootKeyList)).Returns(true);
+        navigationQueryService.Setup(x => x.TryGetChildrenKeys(rootKey, out childrenEnumerable)).Returns(true);
+        var settings = new CacheSettings { MediaBreadthFirstSeedCount = int.MaxValue };
+
+
+        var sut = new MediaBreadthFirstKeyProvider(navigationQueryService.Object, Options.Create(settings));
+
+        var result = sut.GetSeedKeys();
+
+        var expected = childrenCount + 1; // Root + children
+        Assert.That(result.Count, Is.EqualTo(expected));
+    }
+}