From b7cf00ac5df1bfa1ef8f309c32264bddd2eb39a8 Mon Sep 17 00:00:00 2001 From: Kenn Jacobsen Date: Thu, 11 May 2023 11:19:34 +0200 Subject: [PATCH] Handle applied public access restrictions in the Delivery API content index (#14233) * Automatically remove content from the delivery API content index when public access restrictions are applied * Review changes --- .../Services/ApiContentQueryService.cs | 2 +- .../DeliveryApiContentIndex.cs | 2 +- .../UmbracoBuilder.Examine.cs | 1 + ...veryApiContentIndexHandleContentChanges.cs | 6 +- ...ApiContentIndexHandleContentTypeChanges.cs | 2 +- ...piContentIndexHandlePublicAccessChanges.cs | 90 +++++++++++++++++++ ...ryApiContentIndexFieldDefinitionBuilder.cs | 6 +- .../DeliveryApiContentIndexValueSetBuilder.cs | 6 +- .../Examine/DeliveryApiIndexingHandler.cs | 13 +++ .../Examine/UmbracoExamineFieldNames.cs | 21 +++++ .../Search/IDeliveryApiIndexingHandler.cs | 9 ++ ...IndexingNotificationHandler.DeliveryApi.cs | 7 +- 12 files changed, 152 insertions(+), 13 deletions(-) create mode 100644 src/Umbraco.Infrastructure/Examine/Deferred/DeliveryApiContentIndexHandlePublicAccessChanges.cs diff --git a/src/Umbraco.Cms.Api.Delivery/Services/ApiContentQueryService.cs b/src/Umbraco.Cms.Api.Delivery/Services/ApiContentQueryService.cs index 404de7a6f2..d044d774d1 100644 --- a/src/Umbraco.Cms.Api.Delivery/Services/ApiContentQueryService.cs +++ b/src/Umbraco.Cms.Api.Delivery/Services/ApiContentQueryService.cs @@ -77,7 +77,7 @@ internal sealed class ApiContentQueryService : IApiContentQueryService // Item culture must be either the requested culture or "none" var culture = CurrentCulture(); - queryOperation.And().GroupedOr(new[] { "culture" }, culture.ToLowerInvariant().IfNullOrWhiteSpace(_fallbackGuidValue), "none"); + queryOperation.And().GroupedOr(new[] { UmbracoExamineFieldNames.DeliveryApiContentIndex.Culture }, culture.ToLowerInvariant().IfNullOrWhiteSpace(_fallbackGuidValue), "none"); // Handle Filtering var canApplyFiltering = CanHandleFiltering(filters, queryOperation); diff --git a/src/Umbraco.Examine.Lucene/DeliveryApiContentIndex.cs b/src/Umbraco.Examine.Lucene/DeliveryApiContentIndex.cs index a7a7e9a7d8..0967022d77 100644 --- a/src/Umbraco.Examine.Lucene/DeliveryApiContentIndex.cs +++ b/src/Umbraco.Examine.Lucene/DeliveryApiContentIndex.cs @@ -75,7 +75,7 @@ public class DeliveryApiContentIndex : UmbracoExamineIndex } // find descendants-or-self based on path and optional culture - var rawQuery = $"({UmbracoExamineFieldNames.IndexPathFieldName}:\\-1*,{contentId} OR {UmbracoExamineFieldNames.IndexPathFieldName}:\\-1*,{contentId},*)"; + var rawQuery = $"({UmbracoExamineFieldNames.DeliveryApiContentIndex.Id}:{contentId} OR {UmbracoExamineFieldNames.IndexPathFieldName}:\\-1*,{contentId},*)"; if (culture is not null) { rawQuery = $"{rawQuery} AND culture:{culture}"; diff --git a/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.Examine.cs b/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.Examine.cs index 4106464602..0e7b0f5faa 100644 --- a/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.Examine.cs +++ b/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.Examine.cs @@ -60,6 +60,7 @@ public static partial class UmbracoBuilderExtensions builder.AddNotificationHandler(); builder.AddNotificationHandler(); builder.AddNotificationHandler(); + builder.AddNotificationHandler(); builder.AddNotificationHandler(); builder.AddNotificationHandler(); builder.AddNotificationHandler(); diff --git a/src/Umbraco.Infrastructure/Examine/Deferred/DeliveryApiContentIndexHandleContentChanges.cs b/src/Umbraco.Infrastructure/Examine/Deferred/DeliveryApiContentIndexHandleContentChanges.cs index 1ee4a0e949..c93f42b6e8 100644 --- a/src/Umbraco.Infrastructure/Examine/Deferred/DeliveryApiContentIndexHandleContentChanges.cs +++ b/src/Umbraco.Infrastructure/Examine/Deferred/DeliveryApiContentIndexHandleContentChanges.cs @@ -74,10 +74,10 @@ internal sealed class DeliveryApiContentIndexHandleContentChanges : DeliveryApiC var existingIndexCultures = index .Searcher .CreateQuery() - .Field("id", content.Id) - .SelectField("culture") + .Field(UmbracoExamineFieldNames.DeliveryApiContentIndex.Id, content.Id.ToString()) + .SelectField(UmbracoExamineFieldNames.DeliveryApiContentIndex.Culture) .Execute() - .SelectMany(f => f.GetValues("culture")) + .SelectMany(f => f.GetValues(UmbracoExamineFieldNames.DeliveryApiContentIndex.Culture)) .ToArray(); // index the content diff --git a/src/Umbraco.Infrastructure/Examine/Deferred/DeliveryApiContentIndexHandleContentTypeChanges.cs b/src/Umbraco.Infrastructure/Examine/Deferred/DeliveryApiContentIndexHandleContentTypeChanges.cs index 92cafbe670..32dc801dd3 100644 --- a/src/Umbraco.Infrastructure/Examine/Deferred/DeliveryApiContentIndexHandleContentTypeChanges.cs +++ b/src/Umbraco.Infrastructure/Examine/Deferred/DeliveryApiContentIndexHandleContentTypeChanges.cs @@ -128,7 +128,7 @@ internal sealed class DeliveryApiContentIndexHandleContentTypeChanges : Delivery { ISearchResults? results = index.Searcher .CreateQuery() - .Field("contentTypeId", contentTypeId) + .Field(UmbracoExamineFieldNames.DeliveryApiContentIndex.ContentTypeId, contentTypeId.ToString()) // NOTE: we need to be explicit about fetching ItemIdFieldName here, otherwise Examine will try to be // clever and use the "id" field of the document (which we can't use for deletion) .SelectField(UmbracoExamineFieldNames.ItemIdFieldName) diff --git a/src/Umbraco.Infrastructure/Examine/Deferred/DeliveryApiContentIndexHandlePublicAccessChanges.cs b/src/Umbraco.Infrastructure/Examine/Deferred/DeliveryApiContentIndexHandlePublicAccessChanges.cs new file mode 100644 index 0000000000..e5db4b6f1e --- /dev/null +++ b/src/Umbraco.Infrastructure/Examine/Deferred/DeliveryApiContentIndexHandlePublicAccessChanges.cs @@ -0,0 +1,90 @@ +using Examine; +using Examine.Search; +using Umbraco.Cms.Core.Services; +using Umbraco.Cms.Infrastructure.HostedServices; +using Umbraco.Extensions; + +namespace Umbraco.Cms.Infrastructure.Examine.Deferred; + +internal sealed class DeliveryApiContentIndexHandlePublicAccessChanges : DeliveryApiContentIndexDeferredBase, IDeferredAction +{ + private readonly IPublicAccessService _publicAccessService; + private readonly DeliveryApiIndexingHandler _deliveryApiIndexingHandler; + private readonly IBackgroundTaskQueue _backgroundTaskQueue; + + public DeliveryApiContentIndexHandlePublicAccessChanges( + IPublicAccessService publicAccessService, + DeliveryApiIndexingHandler deliveryApiIndexingHandler, + IBackgroundTaskQueue backgroundTaskQueue) + { + _publicAccessService = publicAccessService; + _deliveryApiIndexingHandler = deliveryApiIndexingHandler; + _backgroundTaskQueue = backgroundTaskQueue; + } + + public void Execute() => _backgroundTaskQueue.QueueBackgroundWorkItem(_ => + { + // NOTE: at the time of implementing this, the distributed notifications for public access changes only ever + // sends out "refresh all" notifications, which means we can't be clever about minimizing the work + // effort to handle public access changes. instead we have to grab all protected content definitions + // and handle every last one with every notification. + + // NOTE: eventually the Delivery API will support protected content, but for now we need to ensure that the + // index does not contain any protected content. this also means that whenever content is unprotected, + // one must trigger a manual republish of said content for it to be re-added to the index. not exactly + // an optimal solution, but it's the best we can do at this point, given the limitations outlined above + // and without prematurely assuming the future implementation details of protected content handling. + + var protectedContentIds = _publicAccessService.GetAll().Select(entry => entry.ProtectedNodeId).ToArray(); + if (protectedContentIds.Any() is false) + { + return Task.CompletedTask; + } + + IIndex index = _deliveryApiIndexingHandler.GetIndex() ?? + throw new InvalidOperationException("Could not obtain the delivery API content index"); + + List indexIds = FindIndexIdsForContentIds(protectedContentIds, index); + if (indexIds.Any() is false) + { + return Task.CompletedTask; + } + + RemoveFromIndex(indexIds, index); + return Task.CompletedTask; + }); + + private List FindIndexIdsForContentIds(int[] contentIds, IIndex index) + { + const int pageSize = 500; + const int batchSize = 50; + + var ids = new List(); + + foreach (IEnumerable batch in contentIds.InGroupsOf(batchSize)) + { + IEnumerable batchAsArray = batch as int[] ?? batch.ToArray(); + var page = 0; + var total = long.MaxValue; + + while (page * pageSize < total) + { + ISearchResults? results = index.Searcher + .CreateQuery() + .GroupedOr(new[] { UmbracoExamineFieldNames.DeliveryApiContentIndex.Id }, batchAsArray.Select(id => id.ToString()).ToArray()) + // NOTE: we need to be explicit about fetching ItemIdFieldName here, otherwise Examine will try to be + // clever and use the "id" field of the document (which we can't use for deletion) + .SelectField(UmbracoExamineFieldNames.ItemIdFieldName) + .Execute(QueryOptions.SkipTake(page * pageSize, pageSize)); + total = results.TotalItemCount; + + ids.AddRange(results.Select(result => result.Id)); + + page++; + } + } + + return ids; + } + +} diff --git a/src/Umbraco.Infrastructure/Examine/DeliveryApiContentIndexFieldDefinitionBuilder.cs b/src/Umbraco.Infrastructure/Examine/DeliveryApiContentIndexFieldDefinitionBuilder.cs index ce20716251..bfd11defde 100644 --- a/src/Umbraco.Infrastructure/Examine/DeliveryApiContentIndexFieldDefinitionBuilder.cs +++ b/src/Umbraco.Infrastructure/Examine/DeliveryApiContentIndexFieldDefinitionBuilder.cs @@ -32,9 +32,9 @@ internal sealed class DeliveryApiContentIndexFieldDefinitionBuilder : IDeliveryA // see also the field definitions in the Delivery API content index value set builder private void AddRequiredFieldDefinitions(ICollection fieldDefinitions) { - fieldDefinitions.Add(new("id", FieldDefinitionTypes.Integer)); - fieldDefinitions.Add(new("contentTypeId", FieldDefinitionTypes.Integer)); - fieldDefinitions.Add(new("culture", FieldDefinitionTypes.Raw)); + fieldDefinitions.Add(new(UmbracoExamineFieldNames.DeliveryApiContentIndex.Id, FieldDefinitionTypes.Raw)); + fieldDefinitions.Add(new(UmbracoExamineFieldNames.DeliveryApiContentIndex.ContentTypeId, FieldDefinitionTypes.Raw)); + fieldDefinitions.Add(new(UmbracoExamineFieldNames.DeliveryApiContentIndex.Culture, FieldDefinitionTypes.Raw)); fieldDefinitions.Add(new(UmbracoExamineFieldNames.IndexPathFieldName, FieldDefinitionTypes.Raw)); fieldDefinitions.Add(new(UmbracoExamineFieldNames.NodeNameFieldName, FieldDefinitionTypes.Raw)); } diff --git a/src/Umbraco.Infrastructure/Examine/DeliveryApiContentIndexValueSetBuilder.cs b/src/Umbraco.Infrastructure/Examine/DeliveryApiContentIndexValueSetBuilder.cs index 9c0f7bba6a..20942336ab 100644 --- a/src/Umbraco.Infrastructure/Examine/DeliveryApiContentIndexValueSetBuilder.cs +++ b/src/Umbraco.Infrastructure/Examine/DeliveryApiContentIndexValueSetBuilder.cs @@ -46,9 +46,9 @@ internal sealed class DeliveryApiContentIndexValueSetBuilder : IDeliveryApiConte // required index values go here var indexValues = new Dictionary>(StringComparer.InvariantCultureIgnoreCase) { - ["id"] = new object[] { content.Id }, // required for correct publishing handling and also needed for backoffice index browsing - ["contentTypeId"] = new object[] { content.ContentTypeId }, // required for correct content type change handling - ["culture"] = new object[] { indexCulture }, // required for culture variant querying + [UmbracoExamineFieldNames.DeliveryApiContentIndex.Id] = new object[] { content.Id.ToString() }, // required for correct publishing handling and also needed for backoffice index browsing + [UmbracoExamineFieldNames.DeliveryApiContentIndex.ContentTypeId] = new object[] { content.ContentTypeId.ToString() }, // required for correct content type change handling + [UmbracoExamineFieldNames.DeliveryApiContentIndex.Culture] = new object[] { indexCulture }, // required for culture variant querying [UmbracoExamineFieldNames.IndexPathFieldName] = new object[] { content.Path }, // required for unpublishing/deletion handling [UmbracoExamineFieldNames.NodeNameFieldName] = new object[] { content.GetPublishName(culture) ?? string.Empty }, // primarily needed for backoffice index browsing }; diff --git a/src/Umbraco.Infrastructure/Examine/DeliveryApiIndexingHandler.cs b/src/Umbraco.Infrastructure/Examine/DeliveryApiIndexingHandler.cs index a8654192fc..197ab58be0 100644 --- a/src/Umbraco.Infrastructure/Examine/DeliveryApiIndexingHandler.cs +++ b/src/Umbraco.Infrastructure/Examine/DeliveryApiIndexingHandler.cs @@ -21,6 +21,7 @@ internal sealed class DeliveryApiIndexingHandler : IDeliveryApiIndexingHandler // these dependencies are for the deferred handling (we don't want those handlers registered in the DI) private readonly IContentService _contentService; + private readonly IPublicAccessService _publicAccessService; private readonly IDeliveryApiContentIndexValueSetBuilder _deliveryApiContentIndexValueSetBuilder; private readonly IDeliveryApiContentIndexHelper _deliveryApiContentIndexHelper; private readonly IBackgroundTaskQueue _backgroundTaskQueue; @@ -31,6 +32,7 @@ internal sealed class DeliveryApiIndexingHandler : IDeliveryApiIndexingHandler ICoreScopeProvider scopeProvider, ILogger logger, IContentService contentService, + IPublicAccessService publicAccessService, IDeliveryApiContentIndexValueSetBuilder deliveryApiContentIndexValueSetBuilder, IDeliveryApiContentIndexHelper deliveryApiContentIndexHelper, IBackgroundTaskQueue backgroundTaskQueue) @@ -40,6 +42,7 @@ internal sealed class DeliveryApiIndexingHandler : IDeliveryApiIndexingHandler _scopeProvider = scopeProvider; _logger = logger; _contentService = contentService; + _publicAccessService = publicAccessService; _deliveryApiContentIndexValueSetBuilder = deliveryApiContentIndexValueSetBuilder; _deliveryApiContentIndexHelper = deliveryApiContentIndexHelper; _backgroundTaskQueue = backgroundTaskQueue; @@ -74,6 +77,16 @@ internal sealed class DeliveryApiIndexingHandler : IDeliveryApiIndexingHandler Execute(deferred); } + /// + public void HandlePublicAccessChanges() + { + var deferred = new DeliveryApiContentIndexHandlePublicAccessChanges( + _publicAccessService, + this, + _backgroundTaskQueue); + Execute(deferred); + } + private void Execute(IDeferredAction action) { var actions = DeferredActions.Get(_scopeProvider); diff --git a/src/Umbraco.Infrastructure/Examine/UmbracoExamineFieldNames.cs b/src/Umbraco.Infrastructure/Examine/UmbracoExamineFieldNames.cs index 5e2779e9a3..12b2eb2207 100644 --- a/src/Umbraco.Infrastructure/Examine/UmbracoExamineFieldNames.cs +++ b/src/Umbraco.Infrastructure/Examine/UmbracoExamineFieldNames.cs @@ -25,4 +25,25 @@ public static class UmbracoExamineFieldNames public const string ItemIdFieldName = "__NodeId"; public const string CategoryFieldName = "__IndexType"; public const string ItemTypeFieldName = "__NodeTypeAlias"; + + /// + /// Field names specifically used in the Delivery API content index + /// + public static class DeliveryApiContentIndex + { + /// + /// The content ID + /// + public const string Id = "id"; + + /// + /// The content type ID + /// + public const string ContentTypeId = "contentTypeId"; + + /// + /// The content culture + /// + public const string Culture = "culture"; + } } diff --git a/src/Umbraco.Infrastructure/Search/IDeliveryApiIndexingHandler.cs b/src/Umbraco.Infrastructure/Search/IDeliveryApiIndexingHandler.cs index f7081572a4..c99eda8ec9 100644 --- a/src/Umbraco.Infrastructure/Search/IDeliveryApiIndexingHandler.cs +++ b/src/Umbraco.Infrastructure/Search/IDeliveryApiIndexingHandler.cs @@ -24,4 +24,13 @@ internal interface IDeliveryApiIndexingHandler /// /// The list of changes by content type ID void HandleContentTypeChanges(IList> changes); + + /// + /// Handles index updates for public access changes + /// + /// + /// Given the current limitations to the distributed public access notifications, this + /// will remove any protected content from the index without being clever about it. + /// + void HandlePublicAccessChanges(); } diff --git a/src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.DeliveryApi.cs b/src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.DeliveryApi.cs index b1b028cde0..f4c9b22663 100644 --- a/src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.DeliveryApi.cs +++ b/src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.DeliveryApi.cs @@ -8,7 +8,9 @@ using Umbraco.Cms.Core.Sync; namespace Umbraco.Cms.Infrastructure.Search; internal sealed class DeliveryApiContentIndexingNotificationHandler : - INotificationHandler, INotificationHandler + INotificationHandler, + INotificationHandler, + INotificationHandler { private readonly IDeliveryApiIndexingHandler _deliveryApiIndexingHandler; @@ -47,6 +49,9 @@ internal sealed class DeliveryApiContentIndexingNotificationHandler : _deliveryApiIndexingHandler.HandleContentTypeChanges(contentTypeChangesById); } + public void Handle(PublicAccessCacheRefresherNotification notification) + => _deliveryApiIndexingHandler.HandlePublicAccessChanges(); + private bool NotificationHandlingIsDisabled() { if (_deliveryApiIndexingHandler.Enabled == false)