Handle applied public access restrictions in the Delivery API content index (#14233)

* Automatically remove content from the delivery API content index when public access restrictions are applied

* Review changes

src/Umbraco.Cms.Api.Delivery/Services/ApiContentQueryService.cs

@@ -77,7 +77,7 @@ internal sealed class ApiContentQueryService : IApiContentQueryService
 
         // Item culture must be either the requested culture or "none"
         var culture = CurrentCulture();
-        queryOperation.And().GroupedOr(new[] { "culture" }, culture.ToLowerInvariant().IfNullOrWhiteSpace(_fallbackGuidValue), "none");
+        queryOperation.And().GroupedOr(new[] { UmbracoExamineFieldNames.DeliveryApiContentIndex.Culture }, culture.ToLowerInvariant().IfNullOrWhiteSpace(_fallbackGuidValue), "none");
 
         // Handle Filtering
         var canApplyFiltering = CanHandleFiltering(filters, queryOperation);

src/Umbraco.Examine.Lucene/DeliveryApiContentIndex.cs

@@ -75,7 +75,7 @@ public class DeliveryApiContentIndex : UmbracoExamineIndex
             }
 
             // find descendants-or-self based on path and optional culture
-            var rawQuery = $"({UmbracoExamineFieldNames.IndexPathFieldName}:\\-1*,{contentId} OR {UmbracoExamineFieldNames.IndexPathFieldName}:\\-1*,{contentId},*)";
+            var rawQuery = $"({UmbracoExamineFieldNames.DeliveryApiContentIndex.Id}:{contentId} OR {UmbracoExamineFieldNames.IndexPathFieldName}:\\-1*,{contentId},*)";
             if (culture is not null)
             {
                 rawQuery = $"{rawQuery} AND culture:{culture}";

src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.Examine.cs

@@ -60,6 +60,7 @@ public static partial class UmbracoBuilderExtensions
         builder.AddNotificationHandler<ContentTypeCacheRefresherNotification, ContentTypeIndexingNotificationHandler>();
         builder.AddNotificationHandler<ContentCacheRefresherNotification, DeliveryApiContentIndexingNotificationHandler>();
         builder.AddNotificationHandler<ContentTypeCacheRefresherNotification, DeliveryApiContentIndexingNotificationHandler>();
+        builder.AddNotificationHandler<PublicAccessCacheRefresherNotification, DeliveryApiContentIndexingNotificationHandler>();
         builder.AddNotificationHandler<MediaCacheRefresherNotification, MediaIndexingNotificationHandler>();
         builder.AddNotificationHandler<MemberCacheRefresherNotification, MemberIndexingNotificationHandler>();
         builder.AddNotificationHandler<LanguageCacheRefresherNotification, LanguageIndexingNotificationHandler>();

src/Umbraco.Infrastructure/Examine/Deferred/DeliveryApiContentIndexHandleContentChanges.cs

@@ -74,10 +74,10 @@ internal sealed class DeliveryApiContentIndexHandleContentChanges : DeliveryApiC
         var existingIndexCultures = index
             .Searcher
             .CreateQuery()
-            .Field("id", content.Id)
-            .SelectField("culture")
+            .Field(UmbracoExamineFieldNames.DeliveryApiContentIndex.Id, content.Id.ToString())
+            .SelectField(UmbracoExamineFieldNames.DeliveryApiContentIndex.Culture)
             .Execute()
-            .SelectMany(f => f.GetValues("culture"))
+            .SelectMany(f => f.GetValues(UmbracoExamineFieldNames.DeliveryApiContentIndex.Culture))
             .ToArray();
 
         // index the content

src/Umbraco.Infrastructure/Examine/Deferred/DeliveryApiContentIndexHandleContentTypeChanges.cs

@@ -128,7 +128,7 @@ internal sealed class DeliveryApiContentIndexHandleContentTypeChanges : Delivery
         {
             ISearchResults? results = index.Searcher
                 .CreateQuery()
-                .Field("contentTypeId", contentTypeId)
+                .Field(UmbracoExamineFieldNames.DeliveryApiContentIndex.ContentTypeId, contentTypeId.ToString())
                 // NOTE: we need to be explicit about fetching ItemIdFieldName here, otherwise Examine will try to be
                 // clever and use the "id" field of the document (which we can't use for deletion)
                 .SelectField(UmbracoExamineFieldNames.ItemIdFieldName)

src/Umbraco.Infrastructure/Examine/Deferred/DeliveryApiContentIndexHandlePublicAccessChanges.cs

@@ -0,0 +1,90 @@
+using Examine;
+using Examine.Search;
+using Umbraco.Cms.Core.Services;
+using Umbraco.Cms.Infrastructure.HostedServices;
+using Umbraco.Extensions;
+
+namespace Umbraco.Cms.Infrastructure.Examine.Deferred;
+
+internal sealed class DeliveryApiContentIndexHandlePublicAccessChanges : DeliveryApiContentIndexDeferredBase, IDeferredAction
+{
+    private readonly IPublicAccessService _publicAccessService;
+    private readonly DeliveryApiIndexingHandler _deliveryApiIndexingHandler;
+    private readonly IBackgroundTaskQueue _backgroundTaskQueue;
+
+    public DeliveryApiContentIndexHandlePublicAccessChanges(
+        IPublicAccessService publicAccessService,
+        DeliveryApiIndexingHandler deliveryApiIndexingHandler,
+        IBackgroundTaskQueue backgroundTaskQueue)
+    {
+        _publicAccessService = publicAccessService;
+        _deliveryApiIndexingHandler = deliveryApiIndexingHandler;
+        _backgroundTaskQueue = backgroundTaskQueue;
+    }
+
+    public void Execute() => _backgroundTaskQueue.QueueBackgroundWorkItem(_ =>
+    {
+        // NOTE: at the time of implementing this, the distributed notifications for public access changes only ever
+        //       sends out "refresh all" notifications, which means we can't be clever about minimizing the work
+        //       effort to handle public access changes. instead we have to grab all protected content definitions
+        //       and handle every last one with every notification.
+
+        // NOTE: eventually the Delivery API will support protected content, but for now we need to ensure that the
+        //       index does not contain any protected content. this also means that whenever content is unprotected,
+        //       one must trigger a manual republish of said content for it to be re-added to the index. not exactly
+        //       an optimal solution, but it's the best we can do at this point, given the limitations outlined above
+        //       and without prematurely assuming the future implementation details of protected content handling.
+
+        var protectedContentIds = _publicAccessService.GetAll().Select(entry => entry.ProtectedNodeId).ToArray();
+        if (protectedContentIds.Any() is false)
+        {
+            return Task.CompletedTask;
+        }
+
+        IIndex index = _deliveryApiIndexingHandler.GetIndex() ??
+                       throw new InvalidOperationException("Could not obtain the delivery API content index");
+
+        List<string> indexIds = FindIndexIdsForContentIds(protectedContentIds, index);
+        if (indexIds.Any() is false)
+        {
+            return Task.CompletedTask;
+        }
+
+        RemoveFromIndex(indexIds, index);
+        return Task.CompletedTask;
+    });
+
+    private List<string> FindIndexIdsForContentIds(int[] contentIds, IIndex index)
+    {
+        const int pageSize = 500;
+        const int batchSize = 50;
+
+        var ids = new List<string>();
+
+        foreach (IEnumerable<int> batch in contentIds.InGroupsOf(batchSize))
+        {
+            IEnumerable<int> batchAsArray = batch as int[] ?? batch.ToArray();
+            var page = 0;
+            var total = long.MaxValue;
+
+            while (page * pageSize < total)
+            {
+                ISearchResults? results = index.Searcher
+                    .CreateQuery()
+                    .GroupedOr(new[] { UmbracoExamineFieldNames.DeliveryApiContentIndex.Id }, batchAsArray.Select(id => id.ToString()).ToArray())
+                    // NOTE: we need to be explicit about fetching ItemIdFieldName here, otherwise Examine will try to be
+                    // clever and use the "id" field of the document (which we can't use for deletion)
+                    .SelectField(UmbracoExamineFieldNames.ItemIdFieldName)
+                    .Execute(QueryOptions.SkipTake(page * pageSize, pageSize));
+                total = results.TotalItemCount;
+
+                ids.AddRange(results.Select(result => result.Id));
+
+                page++;
+            }
+        }
+
+        return ids;
+    }
+
+}

src/Umbraco.Infrastructure/Examine/DeliveryApiContentIndexFieldDefinitionBuilder.cs

@@ -32,9 +32,9 @@ internal sealed class DeliveryApiContentIndexFieldDefinitionBuilder : IDeliveryA
     // see also the field definitions in the Delivery API content index value set builder
     private void AddRequiredFieldDefinitions(ICollection<FieldDefinition> fieldDefinitions)
     {
-        fieldDefinitions.Add(new("id", FieldDefinitionTypes.Integer));
-        fieldDefinitions.Add(new("contentTypeId", FieldDefinitionTypes.Integer));
-        fieldDefinitions.Add(new("culture", FieldDefinitionTypes.Raw));
+        fieldDefinitions.Add(new(UmbracoExamineFieldNames.DeliveryApiContentIndex.Id, FieldDefinitionTypes.Raw));
+        fieldDefinitions.Add(new(UmbracoExamineFieldNames.DeliveryApiContentIndex.ContentTypeId, FieldDefinitionTypes.Raw));
+        fieldDefinitions.Add(new(UmbracoExamineFieldNames.DeliveryApiContentIndex.Culture, FieldDefinitionTypes.Raw));
         fieldDefinitions.Add(new(UmbracoExamineFieldNames.IndexPathFieldName, FieldDefinitionTypes.Raw));
         fieldDefinitions.Add(new(UmbracoExamineFieldNames.NodeNameFieldName, FieldDefinitionTypes.Raw));
     }

src/Umbraco.Infrastructure/Examine/DeliveryApiContentIndexValueSetBuilder.cs

@@ -46,9 +46,9 @@ internal sealed class DeliveryApiContentIndexValueSetBuilder : IDeliveryApiConte
                 // required index values go here
                 var indexValues = new Dictionary<string, IEnumerable<object>>(StringComparer.InvariantCultureIgnoreCase)
                 {
-                    ["id"] = new object[] { content.Id }, // required for correct publishing handling and also needed for backoffice index browsing
-                    ["contentTypeId"] = new object[] { content.ContentTypeId }, // required for correct content type change handling
-                    ["culture"] = new object[] { indexCulture }, // required for culture variant querying
+                    [UmbracoExamineFieldNames.DeliveryApiContentIndex.Id] = new object[] { content.Id.ToString() }, // required for correct publishing handling and also needed for backoffice index browsing
+                    [UmbracoExamineFieldNames.DeliveryApiContentIndex.ContentTypeId] = new object[] { content.ContentTypeId.ToString() }, // required for correct content type change handling
+                    [UmbracoExamineFieldNames.DeliveryApiContentIndex.Culture] = new object[] { indexCulture }, // required for culture variant querying
                     [UmbracoExamineFieldNames.IndexPathFieldName] = new object[] { content.Path }, // required for unpublishing/deletion handling
                     [UmbracoExamineFieldNames.NodeNameFieldName] = new object[] { content.GetPublishName(culture) ?? string.Empty }, // primarily needed for backoffice index browsing
                 };

src/Umbraco.Infrastructure/Examine/DeliveryApiIndexingHandler.cs

@@ -21,6 +21,7 @@ internal sealed class DeliveryApiIndexingHandler : IDeliveryApiIndexingHandler
 
     // these dependencies are for the deferred handling (we don't want those handlers registered in the DI)
     private readonly IContentService _contentService;
+    private readonly IPublicAccessService _publicAccessService;
     private readonly IDeliveryApiContentIndexValueSetBuilder _deliveryApiContentIndexValueSetBuilder;
     private readonly IDeliveryApiContentIndexHelper _deliveryApiContentIndexHelper;
     private readonly IBackgroundTaskQueue _backgroundTaskQueue;
@@ -31,6 +32,7 @@ internal sealed class DeliveryApiIndexingHandler : IDeliveryApiIndexingHandler
         ICoreScopeProvider scopeProvider,
         ILogger<DeliveryApiIndexingHandler> logger,
         IContentService contentService,
+        IPublicAccessService publicAccessService,
         IDeliveryApiContentIndexValueSetBuilder deliveryApiContentIndexValueSetBuilder,
         IDeliveryApiContentIndexHelper deliveryApiContentIndexHelper,
         IBackgroundTaskQueue backgroundTaskQueue)
@@ -40,6 +42,7 @@ internal sealed class DeliveryApiIndexingHandler : IDeliveryApiIndexingHandler
         _scopeProvider = scopeProvider;
         _logger = logger;
         _contentService = contentService;
+        _publicAccessService = publicAccessService;
         _deliveryApiContentIndexValueSetBuilder = deliveryApiContentIndexValueSetBuilder;
         _deliveryApiContentIndexHelper = deliveryApiContentIndexHelper;
         _backgroundTaskQueue = backgroundTaskQueue;
@@ -74,6 +77,16 @@ internal sealed class DeliveryApiIndexingHandler : IDeliveryApiIndexingHandler
         Execute(deferred);
     }
 
+    /// <inheritdoc />
+    public void HandlePublicAccessChanges()
+    {
+        var deferred = new DeliveryApiContentIndexHandlePublicAccessChanges(
+            _publicAccessService,
+            this,
+            _backgroundTaskQueue);
+        Execute(deferred);
+    }
+
     private void Execute(IDeferredAction action)
     {
         var actions = DeferredActions.Get(_scopeProvider);

src/Umbraco.Infrastructure/Examine/UmbracoExamineFieldNames.cs

@@ -25,4 +25,25 @@ public static class UmbracoExamineFieldNames
     public const string ItemIdFieldName = "__NodeId";
     public const string CategoryFieldName = "__IndexType";
     public const string ItemTypeFieldName = "__NodeTypeAlias";
+
+    /// <summary>
+    ///     Field names specifically used in the Delivery API content index
+    /// </summary>
+    public static class DeliveryApiContentIndex
+    {
+        /// <summary>
+        ///     The content ID
+        /// </summary>
+        public const string Id = "id";
+
+        /// <summary>
+        ///     The content type ID
+        /// </summary>
+        public const string ContentTypeId = "contentTypeId";
+
+        /// <summary>
+        ///     The content culture
+        /// </summary>
+        public const string Culture = "culture";
+    }
 }

src/Umbraco.Infrastructure/Search/IDeliveryApiIndexingHandler.cs

@@ -24,4 +24,13 @@ internal interface IDeliveryApiIndexingHandler
     /// </summary>
     /// <param name="changes">The list of changes by content type ID</param>
     void HandleContentTypeChanges(IList<KeyValuePair<int, ContentTypeChangeTypes>> changes);
+
+    /// <summary>
+    ///     Handles index updates for public access changes
+    /// </summary>
+    /// <remarks>
+    ///     Given the current limitations to the distributed public access notifications, this
+    ///     will remove any protected content from the index without being clever about it.
+    /// </remarks>
+    void HandlePublicAccessChanges();
 }

src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.DeliveryApi.cs

@@ -8,7 +8,9 @@ using Umbraco.Cms.Core.Sync;
 namespace Umbraco.Cms.Infrastructure.Search;
 
 internal sealed class DeliveryApiContentIndexingNotificationHandler :
-    INotificationHandler<ContentCacheRefresherNotification>, INotificationHandler<ContentTypeCacheRefresherNotification>
+    INotificationHandler<ContentCacheRefresherNotification>,
+    INotificationHandler<ContentTypeCacheRefresherNotification>,
+    INotificationHandler<PublicAccessCacheRefresherNotification>
 {
     private readonly IDeliveryApiIndexingHandler _deliveryApiIndexingHandler;
 
@@ -47,6 +49,9 @@ internal sealed class DeliveryApiContentIndexingNotificationHandler :
         _deliveryApiIndexingHandler.HandleContentTypeChanges(contentTypeChangesById);
     }
 
+    public void Handle(PublicAccessCacheRefresherNotification notification)
+        => _deliveryApiIndexingHandler.HandlePublicAccessChanges();
+
     private bool NotificationHandlingIsDisabled()
     {
         if (_deliveryApiIndexingHandler.Enabled == false)