From 4cdab08910777c5c87edc3a306aac4934a80b20c Mon Sep 17 00:00:00 2001 From: Nikolaj Geisle <70372949+Zeegaan@users.noreply.github.com> Date: Tue, 9 May 2023 09:49:04 +0200 Subject: [PATCH 01/26] Fixes #14102 - NestedPropertyIndexValueFactoryBase ignores compositions (#14115) (#14219) Co-authored-by: Nuklon --- .../PropertyEditors/NestedPropertyIndexValueFactoryBase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Umbraco.Infrastructure/PropertyEditors/NestedPropertyIndexValueFactoryBase.cs b/src/Umbraco.Infrastructure/PropertyEditors/NestedPropertyIndexValueFactoryBase.cs index fa3ae836c1..4eb7051745 100644 --- a/src/Umbraco.Infrastructure/PropertyEditors/NestedPropertyIndexValueFactoryBase.cs +++ b/src/Umbraco.Infrastructure/PropertyEditors/NestedPropertyIndexValueFactoryBase.cs @@ -38,7 +38,7 @@ internal abstract class NestedPropertyIndexValueFactoryBase var propertyTypeDictionary = contentType - .PropertyGroups + .CompositionPropertyGroups .SelectMany(x => x.PropertyTypes!) .Select(propertyType => { From 9187d92a2c8c93960587217adb4181471e47391c Mon Sep 17 00:00:00 2001 From: Bjarke Berg Date: Wed, 10 May 2023 13:29:45 +0200 Subject: [PATCH 02/26] Copied back swagger for delivery api from v13 implementation (#14222) Co-authored-by: Nikolaj --- .../Attributes/MapToApiAttribute.cs | 9 ++ .../Builders/ProblemDetailsBuilder.cs | 8 - .../ConfigureApiBehaviorOptions.cs | 12 ++ .../ConfigureApiExplorerOptions.cs | 6 +- .../ConfigureApiVersioningOptions.cs | 8 +- .../ConfigureUmbracoSwaggerGenOptions.cs | 139 ++++++++++++++++++ .../Configuration/DefaultApiConfiguration.cs | 6 + .../UmbracoBuilderApiExtensions.cs | 79 ++++++++++ .../MethodInfoApiCommonExtensions.cs | 32 ++++ .../Json/NamedSystemTextJsonInputFormatter.cs | 16 +- .../NamedSystemTextJsonOutputFormatter.cs | 3 +- .../OpenApi/EnumSchemaFilter.cs | 23 +++ .../OpenApi/MimeTypeDocumentFilter.cs | 39 +++++ .../OpenApi/OperationIdRegexes.cs | 21 +++ .../OpenApi/SchemaIdGenerator.cs | 33 +++++ .../IUmbracoJsonTypeInfoResolver.cs | 8 + .../UmbracoJsonTypeInfoResolver.cs | 86 +++++++++++ .../Umbraco.Cms.Api.Common.csproj | 6 +- ...gureUmbracoDeliveryApiSwaggerGenOptions.cs | 20 +++ .../Configuration/DeliveryApiConfiguration.cs | 8 + .../Controllers/ByIdContentApiController.cs | 2 + .../ByRouteContentApiController.cs | 2 + .../Controllers/DeliveryApiControllerBase.cs | 5 +- .../Controllers/QueryContentApiController.cs | 2 + .../UmbracoBuilderExtensions.cs | 7 +- 25 files changed, 556 insertions(+), 24 deletions(-) create mode 100644 src/Umbraco.Cms.Api.Common/Attributes/MapToApiAttribute.cs create mode 100644 src/Umbraco.Cms.Api.Common/Configuration/ConfigureApiBehaviorOptions.cs create mode 100644 src/Umbraco.Cms.Api.Common/Configuration/ConfigureUmbracoSwaggerGenOptions.cs create mode 100644 src/Umbraco.Cms.Api.Common/Configuration/DefaultApiConfiguration.cs create mode 100644 src/Umbraco.Cms.Api.Common/DependencyInjection/UmbracoBuilderApiExtensions.cs create mode 100644 src/Umbraco.Cms.Api.Common/Extensions/MethodInfoApiCommonExtensions.cs create mode 100644 src/Umbraco.Cms.Api.Common/OpenApi/EnumSchemaFilter.cs create mode 100644 src/Umbraco.Cms.Api.Common/OpenApi/MimeTypeDocumentFilter.cs create mode 100644 src/Umbraco.Cms.Api.Common/OpenApi/OperationIdRegexes.cs create mode 100644 src/Umbraco.Cms.Api.Common/OpenApi/SchemaIdGenerator.cs create mode 100644 src/Umbraco.Cms.Api.Common/Serialization/IUmbracoJsonTypeInfoResolver.cs create mode 100644 src/Umbraco.Cms.Api.Common/Serialization/UmbracoJsonTypeInfoResolver.cs create mode 100644 src/Umbraco.Cms.Api.Delivery/Configuration/ConfigureUmbracoDeliveryApiSwaggerGenOptions.cs create mode 100644 src/Umbraco.Cms.Api.Delivery/Configuration/DeliveryApiConfiguration.cs diff --git a/src/Umbraco.Cms.Api.Common/Attributes/MapToApiAttribute.cs b/src/Umbraco.Cms.Api.Common/Attributes/MapToApiAttribute.cs new file mode 100644 index 0000000000..da326bfd88 --- /dev/null +++ b/src/Umbraco.Cms.Api.Common/Attributes/MapToApiAttribute.cs @@ -0,0 +1,9 @@ +namespace Umbraco.Cms.Api.Common.Attributes; + +[AttributeUsage(AttributeTargets.Class, AllowMultiple = false)] +public class MapToApiAttribute : Attribute +{ + public MapToApiAttribute(string apiName) => ApiName = apiName; + + public string ApiName { get; } +} diff --git a/src/Umbraco.Cms.Api.Common/Builders/ProblemDetailsBuilder.cs b/src/Umbraco.Cms.Api.Common/Builders/ProblemDetailsBuilder.cs index bc49851911..d3897d5377 100644 --- a/src/Umbraco.Cms.Api.Common/Builders/ProblemDetailsBuilder.cs +++ b/src/Umbraco.Cms.Api.Common/Builders/ProblemDetailsBuilder.cs @@ -7,7 +7,6 @@ public class ProblemDetailsBuilder { private string? _title; private string? _detail; - private int _status = StatusCodes.Status400BadRequest; private string? _type; public ProblemDetailsBuilder WithTitle(string title) @@ -22,12 +21,6 @@ public class ProblemDetailsBuilder return this; } - public ProblemDetailsBuilder WithStatus(int status) - { - _status = status; - return this; - } - public ProblemDetailsBuilder WithType(string type) { _type = type; @@ -39,7 +32,6 @@ public class ProblemDetailsBuilder { Title = _title, Detail = _detail, - Status = _status, Type = _type ?? "Error", }; } diff --git a/src/Umbraco.Cms.Api.Common/Configuration/ConfigureApiBehaviorOptions.cs b/src/Umbraco.Cms.Api.Common/Configuration/ConfigureApiBehaviorOptions.cs new file mode 100644 index 0000000000..ac1a98eb9a --- /dev/null +++ b/src/Umbraco.Cms.Api.Common/Configuration/ConfigureApiBehaviorOptions.cs @@ -0,0 +1,12 @@ +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Options; + +namespace Umbraco.Cms.Api.Common.Configuration; + +public class ConfigureApiBehaviorOptions : IConfigureOptions +{ + public void Configure(ApiBehaviorOptions options) => + // disable ProblemDetails as default result type for every non-success response (i.e. 404) + // - see https://learn.microsoft.com/en-us/dotnet/api/microsoft.aspnetcore.mvc.apibehavioroptions.suppressmapclienterrors + options.SuppressMapClientErrors = true; +} diff --git a/src/Umbraco.Cms.Api.Common/Configuration/ConfigureApiExplorerOptions.cs b/src/Umbraco.Cms.Api.Common/Configuration/ConfigureApiExplorerOptions.cs index 86f5bf73ae..50b1c2a93d 100644 --- a/src/Umbraco.Cms.Api.Common/Configuration/ConfigureApiExplorerOptions.cs +++ b/src/Umbraco.Cms.Api.Common/Configuration/ConfigureApiExplorerOptions.cs @@ -1,5 +1,5 @@ -using Microsoft.AspNetCore.Mvc.ApiExplorer; -using Microsoft.AspNetCore.Mvc.Versioning; +using Asp.Versioning; +using Asp.Versioning.ApiExplorer; using Microsoft.Extensions.Options; namespace Umbraco.Cms.Api.Common.Configuration; @@ -19,6 +19,6 @@ public sealed class ConfigureApiExplorerOptions : IConfigureOptions +{ + private readonly IOptions _apiVersioningOptions; + + public ConfigureUmbracoSwaggerGenOptions(IOptions apiVersioningOptions) + => _apiVersioningOptions = apiVersioningOptions; + + public void Configure(SwaggerGenOptions swaggerGenOptions) + { + swaggerGenOptions.SwaggerDoc( + DefaultApiConfiguration.ApiName, + new OpenApiInfo + { + Title = "Default API", + Version = "Latest", + Description = "All endpoints not defined under specific APIs" + }); + + swaggerGenOptions.CustomOperationIds(description => + CustomOperationId(description, _apiVersioningOptions.Value)); + + swaggerGenOptions.DocInclusionPredicate((name, api) => + { + if (string.IsNullOrWhiteSpace(api.GroupName)) + { + return false; + } + + if (api.ActionDescriptor is ControllerActionDescriptor controllerActionDescriptor) + { + return controllerActionDescriptor.MethodInfo.HasMapToApiAttribute(name); + + } + + return false; + }); + swaggerGenOptions.TagActionsBy(api => new[] { api.GroupName }); + swaggerGenOptions.OrderActionsBy(ActionOrderBy); + swaggerGenOptions.DocumentFilter(); + swaggerGenOptions.SchemaFilter(); + swaggerGenOptions.CustomSchemaIds(SchemaIdGenerator.Generate); + swaggerGenOptions.SupportNonNullableReferenceTypes(); + swaggerGenOptions.UseOneOfForPolymorphism(); + swaggerGenOptions.UseAllOfForInheritance(); + var cachedApiElementNamespace = typeof(ApiElement).Namespace ?? string.Empty; + swaggerGenOptions.SelectDiscriminatorNameUsing(type => + { + if (type.Namespace != null && type.Namespace.StartsWith(cachedApiElementNamespace)) + { + // We do not show type on delivery, as it is read only. + return null; + } + + if (type.GetInterfaces().Any()) + { + return "$type"; + } + + return null; + }); + swaggerGenOptions.SelectDiscriminatorValueUsing(x => x.Name); + } + + private static string CustomOperationId(ApiDescription api, ApiVersioningOptions apiVersioningOptions) + { + ApiVersion defaultVersion = apiVersioningOptions.DefaultApiVersion; + var httpMethod = api.HttpMethod?.ToLower().ToFirstUpper() ?? "Get"; + + // if the route info "Name" is supplied we'll use this explicitly as the operation ID + // - usage example: [HttpGet("my-api/route}", Name = "MyCustomRoute")] + if (string.IsNullOrWhiteSpace(api.ActionDescriptor.AttributeRouteInfo?.Name) == false) + { + var explicitOperationId = api.ActionDescriptor.AttributeRouteInfo!.Name; + return explicitOperationId.InvariantStartsWith(httpMethod) + ? explicitOperationId + : $"{httpMethod}{explicitOperationId}"; + } + + var relativePath = api.RelativePath; + + if (string.IsNullOrWhiteSpace(relativePath)) + { + throw new Exception( + $"There is no relative path for controller action {api.ActionDescriptor.RouteValues["controller"]}"); + } + + // Remove the prefixed base path with version, e.g. /umbraco/management/api/v1/tracked-reference/{id} => tracked-reference/{id} + var unprefixedRelativePath = OperationIdRegexes + .VersionPrefixRegex() + .Replace(relativePath, string.Empty); + + // Remove template placeholders, e.g. tracked-reference/{id} => tracked-reference/Id + var formattedOperationId = OperationIdRegexes + .TemplatePlaceholdersRegex() + .Replace(unprefixedRelativePath, m => $"By{m.Groups[1].Value.ToFirstUpper()}"); + + // Remove dashes (-) and slashes (/) and convert the following letter to uppercase with + // the word "By" in front, e.g. tracked-reference/Id => TrackedReferenceById + formattedOperationId = OperationIdRegexes + .ToCamelCaseRegex() + .Replace(formattedOperationId, m => m.Groups[1].Value.ToUpper()); + + //Get map to version attribute + string? version = null; + + if (api.ActionDescriptor is ControllerActionDescriptor controllerActionDescriptor) + { + var versionAttributeValue = controllerActionDescriptor.MethodInfo.GetMapToApiVersionAttributeValue(); + + // We only wanna add a version, if it is not the default one. + if (string.Equals(versionAttributeValue, defaultVersion.ToString()) == false) + { + version = versionAttributeValue; + } + } + + // Return the operation ID with the formatted http method verb in front, e.g. GetTrackedReferenceById + return $"{httpMethod}{formattedOperationId.ToFirstUpper()}{version}"; + } + + // see https://github.com/domaindrivendev/Swashbuckle.AspNetCore#change-operation-sort-order-eg-for-ui-sorting + private static string ActionOrderBy(ApiDescription apiDesc) + => $"{apiDesc.GroupName}_{apiDesc.ActionDescriptor.AttributeRouteInfo?.Template ?? apiDesc.ActionDescriptor.RouteValues["controller"]}_{apiDesc.ActionDescriptor.RouteValues["action"]}_{apiDesc.HttpMethod}"; + +} diff --git a/src/Umbraco.Cms.Api.Common/Configuration/DefaultApiConfiguration.cs b/src/Umbraco.Cms.Api.Common/Configuration/DefaultApiConfiguration.cs new file mode 100644 index 0000000000..64b7419d21 --- /dev/null +++ b/src/Umbraco.Cms.Api.Common/Configuration/DefaultApiConfiguration.cs @@ -0,0 +1,6 @@ +namespace Umbraco.Cms.Api.Common.Configuration; + +internal static class DefaultApiConfiguration +{ + public const string ApiName = "default"; +} diff --git a/src/Umbraco.Cms.Api.Common/DependencyInjection/UmbracoBuilderApiExtensions.cs b/src/Umbraco.Cms.Api.Common/DependencyInjection/UmbracoBuilderApiExtensions.cs new file mode 100644 index 0000000000..8b940a8b1d --- /dev/null +++ b/src/Umbraco.Cms.Api.Common/DependencyInjection/UmbracoBuilderApiExtensions.cs @@ -0,0 +1,79 @@ +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.FileProviders; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Options; +using Microsoft.OpenApi.Models; +using Swashbuckle.AspNetCore.SwaggerGen; +using Umbraco.Cms.Api.Common.Configuration; +using Umbraco.Cms.Api.Common.Serialization; +using Umbraco.Cms.Core; +using Umbraco.Cms.Core.Configuration.Models; +using Umbraco.Cms.Core.DependencyInjection; +using Umbraco.Cms.Web.Common.ApplicationBuilder; +using Umbraco.Extensions; +using IHostingEnvironment = Umbraco.Cms.Core.Hosting.IHostingEnvironment; + +namespace Umbraco.Cms.Api.Common.DependencyInjection; + +public static class UmbracoBuilderApiExtensions +{ + public static IUmbracoBuilder AddUmbracoApiOpenApiUI(this IUmbracoBuilder builder) + { + builder.Services.ConfigureOptions(); + builder.Services.ConfigureOptions(); + builder.Services.AddApiVersioning().AddApiExplorer(); + + builder.Services.AddSwaggerGen(); + builder.Services.ConfigureOptions(); + builder.Services.AddSingleton(); + + builder.Services.Configure(options => + { + options.AddFilter(new UmbracoPipelineFilter( + "UmbracoApiCommon", + applicationBuilder => + { + + }, + applicationBuilder => + { + IServiceProvider provider = applicationBuilder.ApplicationServices; + IWebHostEnvironment webHostEnvironment = provider.GetRequiredService(); + IOptions swaggerGenOptions = provider.GetRequiredService>(); + + + if (!webHostEnvironment.IsProduction()) + { + GlobalSettings? settings = provider.GetRequiredService>().Value; + IHostingEnvironment hostingEnvironment = provider.GetRequiredService(); + var umbracoPath = settings.GetBackOfficePath(hostingEnvironment); + + applicationBuilder.UseSwagger(swaggerOptions => + { + swaggerOptions.RouteTemplate = + $"{umbracoPath.TrimStart(Constants.CharArrays.ForwardSlash)}/swagger/{{documentName}}/swagger.json"; + }); + applicationBuilder.UseSwaggerUI( + swaggerUiOptions => + { + swaggerUiOptions.RoutePrefix = $"{umbracoPath.TrimStart(Constants.CharArrays.ForwardSlash)}/swagger"; + + foreach ((var name, OpenApiInfo? apiInfo) in swaggerGenOptions.Value.SwaggerGeneratorOptions.SwaggerDocs.OrderBy(x=>x.Value.Title)) + { + swaggerUiOptions.SwaggerEndpoint($"{name}/swagger.json", $"{apiInfo.Title}"); + } + }); + } + }, + applicationBuilder => + { + + })); + }); + + return builder; + } +} diff --git a/src/Umbraco.Cms.Api.Common/Extensions/MethodInfoApiCommonExtensions.cs b/src/Umbraco.Cms.Api.Common/Extensions/MethodInfoApiCommonExtensions.cs new file mode 100644 index 0000000000..9d619f5e1c --- /dev/null +++ b/src/Umbraco.Cms.Api.Common/Extensions/MethodInfoApiCommonExtensions.cs @@ -0,0 +1,32 @@ +using System.Reflection; +using Asp.Versioning; +using Umbraco.Cms.Api.Common.Attributes; +using Umbraco.Cms.Api.Common.Configuration; + +namespace Umbraco.Extensions; + +public static class MethodInfoApiCommonExtensions +{ + + public static string? GetMapToApiVersionAttributeValue(this MethodInfo methodInfo) + { + MapToApiVersionAttribute[] mapToApis = methodInfo.GetCustomAttributes(typeof(MapToApiVersionAttribute), inherit: true).Cast().ToArray(); + + return string.Join("|", mapToApis.SelectMany(x=>x.Versions)); + } + + public static string? GetMapToApiAttributeValue(this MethodInfo methodInfo) + { + MapToApiAttribute[] mapToApis = (methodInfo.DeclaringType?.GetCustomAttributes(typeof(MapToApiAttribute), inherit: true) ?? Array.Empty()).Cast().ToArray(); + + return mapToApis.SingleOrDefault()?.ApiName; + } + + public static bool HasMapToApiAttribute(this MethodInfo methodInfo, string apiName) + { + var value = methodInfo.GetMapToApiAttributeValue(); + + return value == apiName + || (value is null && apiName == DefaultApiConfiguration.ApiName); + } +} diff --git a/src/Umbraco.Cms.Api.Common/Json/NamedSystemTextJsonInputFormatter.cs b/src/Umbraco.Cms.Api.Common/Json/NamedSystemTextJsonInputFormatter.cs index 47d2e33c0c..1ae57bd8c4 100644 --- a/src/Umbraco.Cms.Api.Common/Json/NamedSystemTextJsonInputFormatter.cs +++ b/src/Umbraco.Cms.Api.Common/Json/NamedSystemTextJsonInputFormatter.cs @@ -4,7 +4,7 @@ using Microsoft.Extensions.Logging; namespace Umbraco.Cms.Api.Common.Json; -public class NamedSystemTextJsonInputFormatter : SystemTextJsonInputFormatter +internal class NamedSystemTextJsonInputFormatter : SystemTextJsonInputFormatter { private readonly string _jsonOptionsName; @@ -14,4 +14,18 @@ public class NamedSystemTextJsonInputFormatter : SystemTextJsonInputFormatter public override bool CanRead(InputFormatterContext context) => context.HttpContext.CurrentJsonOptionsName() == _jsonOptionsName && base.CanRead(context); + + public override async Task ReadAsync(InputFormatterContext context) + { + try + { + return await base.ReadAsync(context); + } + catch (NotSupportedException exception) + { + // This happens when trying to deserialize to an interface, without sending the $type as part of the request + context.ModelState.TryAddModelException(string.Empty, new InputFormatterException(exception.Message, exception)); + return await InputFormatterResult.FailureAsync(); + } + } } diff --git a/src/Umbraco.Cms.Api.Common/Json/NamedSystemTextJsonOutputFormatter.cs b/src/Umbraco.Cms.Api.Common/Json/NamedSystemTextJsonOutputFormatter.cs index 9759542828..bd1b17b81d 100644 --- a/src/Umbraco.Cms.Api.Common/Json/NamedSystemTextJsonOutputFormatter.cs +++ b/src/Umbraco.Cms.Api.Common/Json/NamedSystemTextJsonOutputFormatter.cs @@ -3,7 +3,8 @@ using Microsoft.AspNetCore.Mvc.Formatters; namespace Umbraco.Cms.Api.Common.Json; -public class NamedSystemTextJsonOutputFormatter : SystemTextJsonOutputFormatter + +internal class NamedSystemTextJsonOutputFormatter : SystemTextJsonOutputFormatter { private readonly string _jsonOptionsName; diff --git a/src/Umbraco.Cms.Api.Common/OpenApi/EnumSchemaFilter.cs b/src/Umbraco.Cms.Api.Common/OpenApi/EnumSchemaFilter.cs new file mode 100644 index 0000000000..fcd5ba8ccc --- /dev/null +++ b/src/Umbraco.Cms.Api.Common/OpenApi/EnumSchemaFilter.cs @@ -0,0 +1,23 @@ +using System.Reflection; +using System.Runtime.Serialization; +using Microsoft.OpenApi.Any; +using Microsoft.OpenApi.Models; +using Swashbuckle.AspNetCore.SwaggerGen; + +namespace Umbraco.Cms.Api.Common.OpenApi; + +public class EnumSchemaFilter : ISchemaFilter +{ + public void Apply(OpenApiSchema model, SchemaFilterContext context) + { + if (context.Type.IsEnum) + { + model.Enum.Clear(); + foreach (var name in Enum.GetNames(context.Type)) + { + var actualName = context.Type.GetField(name)?.GetCustomAttribute()?.Value ?? name; + model.Enum.Add(new OpenApiString(actualName)); + } + } + } +} diff --git a/src/Umbraco.Cms.Api.Common/OpenApi/MimeTypeDocumentFilter.cs b/src/Umbraco.Cms.Api.Common/OpenApi/MimeTypeDocumentFilter.cs new file mode 100644 index 0000000000..c1acfdadf1 --- /dev/null +++ b/src/Umbraco.Cms.Api.Common/OpenApi/MimeTypeDocumentFilter.cs @@ -0,0 +1,39 @@ +using Microsoft.OpenApi.Models; +using Swashbuckle.AspNetCore.SwaggerGen; +using Umbraco.Extensions; + +namespace Umbraco.Cms.Api.Common.OpenApi; + +/// +/// This filter explicitly removes all other mime types than application/json from the produced OpenAPI document when application/json is accepted. +/// +public class MimeTypeDocumentFilter : IDocumentFilter +{ + public void Apply(OpenApiDocument swaggerDoc, DocumentFilterContext context) + { + OpenApiOperation[] operations = swaggerDoc.Paths + .SelectMany(path => path.Value.Operations.Values) + .ToArray(); + + void RemoveUnwantedMimeTypes(IDictionary content) + { + if (content.ContainsKey("application/json")) + { + content.RemoveAll(r => r.Key != "application/json"); + } + + } + + OpenApiRequestBody[] requestBodies = operations.Select(operation => operation.RequestBody).WhereNotNull().ToArray(); + foreach (OpenApiRequestBody requestBody in requestBodies) + { + RemoveUnwantedMimeTypes(requestBody.Content); + } + + OpenApiResponse[] responses = operations.SelectMany(operation => operation.Responses.Values).WhereNotNull().ToArray(); + foreach (OpenApiResponse response in responses) + { + RemoveUnwantedMimeTypes(response.Content); + } + } +} diff --git a/src/Umbraco.Cms.Api.Common/OpenApi/OperationIdRegexes.cs b/src/Umbraco.Cms.Api.Common/OpenApi/OperationIdRegexes.cs new file mode 100644 index 0000000000..60a114c218 --- /dev/null +++ b/src/Umbraco.Cms.Api.Common/OpenApi/OperationIdRegexes.cs @@ -0,0 +1,21 @@ +using System.Text.RegularExpressions; + +namespace Umbraco.Cms.Api.Common.OpenApi; + +/// +/// This is the regexes used to generate the operation IDs, the benefit of this being partial with GeneratedRegex +/// source generators is that it will be pre-compiled at startup +/// See: https://devblogs.microsoft.com/dotnet/regular-expression-improvements-in-dotnet-7/#source-generation for more info. +/// +internal static partial class OperationIdRegexes +{ + // Your IDE may be showing errors here, this is because it's a new dotnet 7 feature (it's fixed in the EAP of Rider) + [GeneratedRegex(".*?\\/v[1-9]+/")] + public static partial Regex VersionPrefixRegex(); + + [GeneratedRegex("\\{(.*?)\\:?\\}")] + public static partial Regex TemplatePlaceholdersRegex(); + + [GeneratedRegex("[\\/\\-](\\w{1})")] + public static partial Regex ToCamelCaseRegex(); +} diff --git a/src/Umbraco.Cms.Api.Common/OpenApi/SchemaIdGenerator.cs b/src/Umbraco.Cms.Api.Common/OpenApi/SchemaIdGenerator.cs new file mode 100644 index 0000000000..b2a76cde53 --- /dev/null +++ b/src/Umbraco.Cms.Api.Common/OpenApi/SchemaIdGenerator.cs @@ -0,0 +1,33 @@ +using System.Text.RegularExpressions; +using Umbraco.Extensions; + +namespace Umbraco.Cms.Api.Common.OpenApi; + +internal static class SchemaIdGenerator +{ + public static string Generate(Type type) + { + string SanitizedTypeName(Type t) => t.Name + // first grab the "non generic" part of any generic type name (i.e. "PagedViewModel`1" becomes "PagedViewModel") + .Split('`').First() + // then remove the "ViewModel" postfix from type names + .TrimEnd("ViewModel"); + + var name = SanitizedTypeName(type); + if (type.IsGenericType) + { + // append the generic type names, ultimately turning i.e. "PagedViewModel" into "PagedRelationItem" + name = $"{name}{string.Join(string.Empty, type.GenericTypeArguments.Select(SanitizedTypeName))}"; + } + + if (name.EndsWith("Model") == false) + { + // because some models names clash with common classes in TypeScript (i.e. Document), + // we need to add a "Model" postfix to all models + name = $"{name}Model"; + } + + // make absolutely sure we don't pass any invalid named by removing all non-word chars + return Regex.Replace(name, @"[^\w]", string.Empty); + } +} diff --git a/src/Umbraco.Cms.Api.Common/Serialization/IUmbracoJsonTypeInfoResolver.cs b/src/Umbraco.Cms.Api.Common/Serialization/IUmbracoJsonTypeInfoResolver.cs new file mode 100644 index 0000000000..009f05e4e4 --- /dev/null +++ b/src/Umbraco.Cms.Api.Common/Serialization/IUmbracoJsonTypeInfoResolver.cs @@ -0,0 +1,8 @@ +using System.Text.Json.Serialization.Metadata; + +namespace Umbraco.Cms.Api.Common.Serialization; + +public interface IUmbracoJsonTypeInfoResolver : IJsonTypeInfoResolver +{ + IEnumerable FindSubTypes(Type type); +} diff --git a/src/Umbraco.Cms.Api.Common/Serialization/UmbracoJsonTypeInfoResolver.cs b/src/Umbraco.Cms.Api.Common/Serialization/UmbracoJsonTypeInfoResolver.cs new file mode 100644 index 0000000000..8ce1955a2b --- /dev/null +++ b/src/Umbraco.Cms.Api.Common/Serialization/UmbracoJsonTypeInfoResolver.cs @@ -0,0 +1,86 @@ +using System.Collections.Concurrent; +using System.Text.Json; +using System.Text.Json.Serialization.Metadata; +using Umbraco.Cms.Core.Composing; + +namespace Umbraco.Cms.Api.Common.Serialization; + +public sealed class UmbracoJsonTypeInfoResolver : DefaultJsonTypeInfoResolver, IUmbracoJsonTypeInfoResolver +{ + private readonly ITypeFinder _typeFinder; + private readonly ConcurrentDictionary> _subTypesCache = new ConcurrentDictionary>(); + + public UmbracoJsonTypeInfoResolver(ITypeFinder typeFinder) + { + _typeFinder = typeFinder; + } + + public IEnumerable FindSubTypes(Type type) + { + if (_subTypesCache.TryGetValue(type, out ISet? cachedResult)) + { + return cachedResult; + } + + var result = _typeFinder.FindClassesOfType(type).ToHashSet(); + _subTypesCache.TryAdd(type, result); + return result; + } + + public override JsonTypeInfo GetTypeInfo(Type type, JsonSerializerOptions options) + { + JsonTypeInfo result = base.GetTypeInfo(type, options); + + if (type.IsInterface) + { + return GetTypeInfoForInterface(result, type, options); + } + else + { + return GetTypeInfoForClass(result, type, options); + } + + } + + private JsonTypeInfo GetTypeInfoForClass(JsonTypeInfo result, Type type, JsonSerializerOptions options) + { + if (result.Kind != JsonTypeInfoKind.Object || !type.GetInterfaces().Any()) + { + return result; + } + + JsonPolymorphismOptions jsonPolymorphismOptions = result.PolymorphismOptions ?? new JsonPolymorphismOptions(); + + jsonPolymorphismOptions.DerivedTypes.Add(new JsonDerivedType(type, type.Name)); + + result.PolymorphismOptions = jsonPolymorphismOptions; + + return result; + } + + private JsonTypeInfo GetTypeInfoForInterface(JsonTypeInfo result, Type type, JsonSerializerOptions options) + { + IEnumerable subTypes = FindSubTypes(type); + + if (!subTypes.Any()) + { + return result; + } + + JsonPolymorphismOptions jsonPolymorphismOptions = result.PolymorphismOptions ?? new JsonPolymorphismOptions(); + + IEnumerable knownSubTypes = jsonPolymorphismOptions.DerivedTypes.Select(x => x.DerivedType); + IEnumerable subTypesToAdd = subTypes.Except(knownSubTypes); + foreach (Type subType in subTypesToAdd) + { + jsonPolymorphismOptions.DerivedTypes.Add(new JsonDerivedType( + subType, + subType.Name ?? string.Empty)); + } + + result.PolymorphismOptions = jsonPolymorphismOptions; + + + return result; + } +} diff --git a/src/Umbraco.Cms.Api.Common/Umbraco.Cms.Api.Common.csproj b/src/Umbraco.Cms.Api.Common/Umbraco.Cms.Api.Common.csproj index f979fbd49b..c6569f0262 100644 --- a/src/Umbraco.Cms.Api.Common/Umbraco.Cms.Api.Common.csproj +++ b/src/Umbraco.Cms.Api.Common/Umbraco.Cms.Api.Common.csproj @@ -9,10 +9,12 @@ - - + + + + diff --git a/src/Umbraco.Cms.Api.Delivery/Configuration/ConfigureUmbracoDeliveryApiSwaggerGenOptions.cs b/src/Umbraco.Cms.Api.Delivery/Configuration/ConfigureUmbracoDeliveryApiSwaggerGenOptions.cs new file mode 100644 index 0000000000..c61a5aad80 --- /dev/null +++ b/src/Umbraco.Cms.Api.Delivery/Configuration/ConfigureUmbracoDeliveryApiSwaggerGenOptions.cs @@ -0,0 +1,20 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using Microsoft.OpenApi.Models; +using Swashbuckle.AspNetCore.SwaggerGen; + +namespace Umbraco.Cms.Api.Delivery.Configuration; + +public class ConfigureUmbracoDeliveryApiSwaggerGenOptions: IConfigureOptions +{ + public void Configure(SwaggerGenOptions swaggerGenOptions) + { + swaggerGenOptions.SwaggerDoc( + DeliveryApiConfiguration.ApiName, + new OpenApiInfo + { + Title = DeliveryApiConfiguration.ApiTitle, + Version = "Latest", + }); + } +} diff --git a/src/Umbraco.Cms.Api.Delivery/Configuration/DeliveryApiConfiguration.cs b/src/Umbraco.Cms.Api.Delivery/Configuration/DeliveryApiConfiguration.cs new file mode 100644 index 0000000000..4f0fc17fa9 --- /dev/null +++ b/src/Umbraco.Cms.Api.Delivery/Configuration/DeliveryApiConfiguration.cs @@ -0,0 +1,8 @@ +namespace Umbraco.Cms.Api.Delivery.Configuration; + +internal static class DeliveryApiConfiguration +{ + internal const string ApiTitle = "Umbraco Delivery API"; + + internal const string ApiName = "delivery"; +} diff --git a/src/Umbraco.Cms.Api.Delivery/Controllers/ByIdContentApiController.cs b/src/Umbraco.Cms.Api.Delivery/Controllers/ByIdContentApiController.cs index 3a01f35d3a..01a5149a14 100644 --- a/src/Umbraco.Cms.Api.Delivery/Controllers/ByIdContentApiController.cs +++ b/src/Umbraco.Cms.Api.Delivery/Controllers/ByIdContentApiController.cs @@ -1,3 +1,4 @@ +using Asp.Versioning; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Umbraco.Cms.Core.DeliveryApi; @@ -7,6 +8,7 @@ using Umbraco.Cms.Core.Services; namespace Umbraco.Cms.Api.Delivery.Controllers; +[ApiVersion("1.0")] public class ByIdContentApiController : ContentApiItemControllerBase { public ByIdContentApiController( diff --git a/src/Umbraco.Cms.Api.Delivery/Controllers/ByRouteContentApiController.cs b/src/Umbraco.Cms.Api.Delivery/Controllers/ByRouteContentApiController.cs index 1f8334cefb..cc260f3930 100644 --- a/src/Umbraco.Cms.Api.Delivery/Controllers/ByRouteContentApiController.cs +++ b/src/Umbraco.Cms.Api.Delivery/Controllers/ByRouteContentApiController.cs @@ -1,3 +1,4 @@ +using Asp.Versioning; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Umbraco.Cms.Core.DeliveryApi; @@ -7,6 +8,7 @@ using Umbraco.Cms.Core.Services; namespace Umbraco.Cms.Api.Delivery.Controllers; +[ApiVersion("1.0")] public class ByRouteContentApiController : ContentApiItemControllerBase { private readonly IRequestRoutingService _requestRoutingService; diff --git a/src/Umbraco.Cms.Api.Delivery/Controllers/DeliveryApiControllerBase.cs b/src/Umbraco.Cms.Api.Delivery/Controllers/DeliveryApiControllerBase.cs index 2230e228ce..552e2e2f8b 100644 --- a/src/Umbraco.Cms.Api.Delivery/Controllers/DeliveryApiControllerBase.cs +++ b/src/Umbraco.Cms.Api.Delivery/Controllers/DeliveryApiControllerBase.cs @@ -1,14 +1,17 @@ +using Asp.Versioning; using Microsoft.AspNetCore.Mvc; +using Umbraco.Cms.Api.Common.Attributes; using Umbraco.Cms.Api.Common.Filters; +using Umbraco.Cms.Api.Delivery.Configuration; using Umbraco.Cms.Api.Delivery.Filters; using Umbraco.Cms.Core; namespace Umbraco.Cms.Api.Delivery.Controllers; [ApiController] -[ApiVersion("1.0")] [DeliveryApiAccess] [JsonOptionsName(Constants.JsonOptionsNames.DeliveryApi)] +[MapToApi(DeliveryApiConfiguration.ApiName)] public abstract class DeliveryApiControllerBase : Controller { } diff --git a/src/Umbraco.Cms.Api.Delivery/Controllers/QueryContentApiController.cs b/src/Umbraco.Cms.Api.Delivery/Controllers/QueryContentApiController.cs index 53db806c3e..8db6cdb454 100644 --- a/src/Umbraco.Cms.Api.Delivery/Controllers/QueryContentApiController.cs +++ b/src/Umbraco.Cms.Api.Delivery/Controllers/QueryContentApiController.cs @@ -1,3 +1,4 @@ +using Asp.Versioning; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Umbraco.Cms.Api.Common.ViewModels.Pagination; @@ -10,6 +11,7 @@ using Umbraco.New.Cms.Core.Models; namespace Umbraco.Cms.Api.Delivery.Controllers; +[ApiVersion("1.0")] public class QueryContentApiController : ContentApiControllerBase { private readonly IApiContentQueryService _apiContentQueryService; diff --git a/src/Umbraco.Cms.Api.Delivery/DependencyInjection/UmbracoBuilderExtensions.cs b/src/Umbraco.Cms.Api.Delivery/DependencyInjection/UmbracoBuilderExtensions.cs index 6f2ed5a2a3..b6e52ada7c 100644 --- a/src/Umbraco.Cms.Api.Delivery/DependencyInjection/UmbracoBuilderExtensions.cs +++ b/src/Umbraco.Cms.Api.Delivery/DependencyInjection/UmbracoBuilderExtensions.cs @@ -4,6 +4,7 @@ using Microsoft.Extensions.DependencyInjection; using Umbraco.Cms.Api.Common.Configuration; using Umbraco.Cms.Api.Common.DependencyInjection; using Umbraco.Cms.Api.Delivery.Accessors; +using Umbraco.Cms.Api.Delivery.Configuration; using Umbraco.Cms.Api.Delivery.Json; using Umbraco.Cms.Api.Delivery.Rendering; using Umbraco.Cms.Api.Delivery.Services; @@ -28,10 +29,8 @@ public static class UmbracoBuilderExtensions builder.Services.AddSingleton(); builder.Services.AddSingleton(); - builder.Services.ConfigureOptions(); - builder.Services.AddApiVersioning(); - builder.Services.ConfigureOptions(); - builder.Services.AddVersionedApiExplorer(); + builder.Services.ConfigureOptions(); + builder.AddUmbracoApiOpenApiUI(); builder .Services From b0f42a2c86979555309f99d6e2ec0defdb7de0d9 Mon Sep 17 00:00:00 2001 From: Kenn Jacobsen Date: Wed, 10 May 2023 14:04:42 +0200 Subject: [PATCH 03/26] Move the built-in properties back to the delivery API media model + support property expansion for other media properties (#14224) Co-authored-by: Elitsa --- .../RequestContextOutputExpansionStrategy.cs | 31 ++++- .../DeliveryApi/ApiMediaBuilder.cs | 28 ++++- .../DeliveryApi/IOutputExpansionStrategy.cs | 4 +- .../NoopOutputExpansionStrategy.cs | 9 +- .../Models/DeliveryApi/ApiMedia.cs | 14 ++- .../Models/DeliveryApi/IApiMedia.cs | 18 ++- .../Models/DeliveryApi/ApiMediaWithCrops.cs | 8 ++ .../DeliveryApi/ContentBuilderTests.cs | 1 + .../DeliveryApi/DeliveryApiTests.cs | 6 +- .../DeliveryApi/MediaBuilderTests.cs | 22 ++-- .../MediaPickerValueConverterTests.cs | 1 + ...MediaPickerWithCropsValueConverterTests.cs | 41 +++--- .../MultiNodeTreePickerValueConverterTests.cs | 2 +- .../OutputExpansionStrategyTests.cs | 117 ++++++++++++++++++ .../PropertyValueConverterTests.cs | 3 + 15 files changed, 254 insertions(+), 51 deletions(-) diff --git a/src/Umbraco.Cms.Api.Delivery/Rendering/RequestContextOutputExpansionStrategy.cs b/src/Umbraco.Cms.Api.Delivery/Rendering/RequestContextOutputExpansionStrategy.cs index b20f6ec76a..bdb451e74e 100644 --- a/src/Umbraco.Cms.Api.Delivery/Rendering/RequestContextOutputExpansionStrategy.cs +++ b/src/Umbraco.Cms.Api.Delivery/Rendering/RequestContextOutputExpansionStrategy.cs @@ -22,14 +22,33 @@ internal sealed class RequestContextOutputExpansionStrategy : IOutputExpansionSt } public IDictionary MapElementProperties(IPublishedElement element) - => MapProperties(element.Properties); - - public IDictionary MapProperties(IEnumerable properties) - => properties.ToDictionary( + => element.Properties.ToDictionary( p => p.Alias, p => p.GetDeliveryApiValue(_state == ExpansionState.Expanding)); public IDictionary MapContentProperties(IPublishedContent content) + => content.ItemType == PublishedItemType.Content + ? MapProperties(content.Properties) + : throw new ArgumentException($"Invalid item type. This method can only be used with item type {nameof(PublishedItemType.Content)}, got: {content.ItemType}"); + + public IDictionary MapMediaProperties(IPublishedContent media, bool skipUmbracoProperties = true) + { + if (media.ItemType != PublishedItemType.Media) + { + throw new ArgumentException($"Invalid item type. This method can only be used with item type {PublishedItemType.Media}, got: {media.ItemType}"); + } + + IPublishedProperty[] properties = media + .Properties + .Where(p => skipUmbracoProperties is false || p.Alias.StartsWith("umbraco") is false) + .ToArray(); + + return properties.Any() + ? MapProperties(properties) + : new Dictionary(); + } + + private IDictionary MapProperties(IEnumerable properties) { // in the initial state, content properties should always be rendered (expanded if the requests dictates it). // this corresponds to the root level of a content item, i.e. when the initial content rendering starts. @@ -37,7 +56,7 @@ internal sealed class RequestContextOutputExpansionStrategy : IOutputExpansionSt { // update state to pending so we don't end up here the next time around _state = ExpansionState.Pending; - var rendered = content.Properties.ToDictionary( + var rendered = properties.ToDictionary( property => property.Alias, property => { @@ -63,7 +82,7 @@ internal sealed class RequestContextOutputExpansionStrategy : IOutputExpansionSt if (_state == ExpansionState.Expanding) { _state = ExpansionState.Expanded; - var rendered = content.Properties.ToDictionary( + var rendered = properties.ToDictionary( property => property.Alias, property => property.GetDeliveryApiValue(false)); _state = ExpansionState.Expanding; diff --git a/src/Umbraco.Core/DeliveryApi/ApiMediaBuilder.cs b/src/Umbraco.Core/DeliveryApi/ApiMediaBuilder.cs index 81c894a635..fa74aee7b2 100644 --- a/src/Umbraco.Core/DeliveryApi/ApiMediaBuilder.cs +++ b/src/Umbraco.Core/DeliveryApi/ApiMediaBuilder.cs @@ -1,5 +1,6 @@ using Umbraco.Cms.Core.Models.DeliveryApi; using Umbraco.Cms.Core.Models.PublishedContent; +using Umbraco.Extensions; namespace Umbraco.Cms.Core.DeliveryApi; @@ -7,15 +8,18 @@ public sealed class ApiMediaBuilder : IApiMediaBuilder { private readonly IApiContentNameProvider _apiContentNameProvider; private readonly IApiMediaUrlProvider _apiMediaUrlProvider; + private readonly IPublishedValueFallback _publishedValueFallback; private readonly IOutputExpansionStrategyAccessor _outputExpansionStrategyAccessor; public ApiMediaBuilder( IApiContentNameProvider apiContentNameProvider, IApiMediaUrlProvider apiMediaUrlProvider, + IPublishedValueFallback publishedValueFallback, IOutputExpansionStrategyAccessor outputExpansionStrategyAccessor) { _apiContentNameProvider = apiContentNameProvider; _apiMediaUrlProvider = apiMediaUrlProvider; + _publishedValueFallback = publishedValueFallback; _outputExpansionStrategyAccessor = outputExpansionStrategyAccessor; } @@ -25,11 +29,27 @@ public sealed class ApiMediaBuilder : IApiMediaBuilder _apiContentNameProvider.GetName(media), media.ContentType.Alias, _apiMediaUrlProvider.GetUrl(media), + Extension(media), + Width(media), + Height(media), + Bytes(media), Properties(media)); - // map all media properties except the umbracoFile one, as we've already included the file URL etc. in the output - private IDictionary Properties(IPublishedContent media) => - _outputExpansionStrategyAccessor.TryGetValue(out IOutputExpansionStrategy? outputExpansionStrategy) - ? outputExpansionStrategy.MapProperties(media.Properties.Where(p => p.Alias != Constants.Conventions.Media.File)) + private string? Extension(IPublishedContent media) + => media.Value(_publishedValueFallback, Constants.Conventions.Media.Extension); + + private int? Width(IPublishedContent media) + => media.Value(_publishedValueFallback, Constants.Conventions.Media.Width); + + private int? Height(IPublishedContent media) + => media.Value(_publishedValueFallback, Constants.Conventions.Media.Height); + + private int? Bytes(IPublishedContent media) + => media.Value(_publishedValueFallback, Constants.Conventions.Media.Bytes); + + // map all media properties except the umbraco ones, as we've already included those in the output + private IDictionary Properties(IPublishedContent media) + => _outputExpansionStrategyAccessor.TryGetValue(out IOutputExpansionStrategy? outputExpansionStrategy) + ? outputExpansionStrategy.MapMediaProperties(media) : new Dictionary(); } diff --git a/src/Umbraco.Core/DeliveryApi/IOutputExpansionStrategy.cs b/src/Umbraco.Core/DeliveryApi/IOutputExpansionStrategy.cs index 56ed9cec73..97d0cf598b 100644 --- a/src/Umbraco.Core/DeliveryApi/IOutputExpansionStrategy.cs +++ b/src/Umbraco.Core/DeliveryApi/IOutputExpansionStrategy.cs @@ -6,7 +6,7 @@ public interface IOutputExpansionStrategy { IDictionary MapElementProperties(IPublishedElement element); - IDictionary MapProperties(IEnumerable properties); - IDictionary MapContentProperties(IPublishedContent content); + + IDictionary MapMediaProperties(IPublishedContent media, bool skipUmbracoProperties = true); } diff --git a/src/Umbraco.Core/DeliveryApi/NoopOutputExpansionStrategy.cs b/src/Umbraco.Core/DeliveryApi/NoopOutputExpansionStrategy.cs index 8ff223324a..8a2f297634 100644 --- a/src/Umbraco.Core/DeliveryApi/NoopOutputExpansionStrategy.cs +++ b/src/Umbraco.Core/DeliveryApi/NoopOutputExpansionStrategy.cs @@ -7,9 +7,12 @@ internal sealed class NoopOutputExpansionStrategy : IOutputExpansionStrategy public IDictionary MapElementProperties(IPublishedElement element) => MapProperties(element.Properties); - public IDictionary MapProperties(IEnumerable properties) - => properties.ToDictionary(p => p.Alias, p => p.GetDeliveryApiValue(true)); - public IDictionary MapContentProperties(IPublishedContent content) => MapProperties(content.Properties); + + public IDictionary MapMediaProperties(IPublishedContent media, bool skipUmbracoProperties = true) + => MapProperties(media.Properties.Where(p => skipUmbracoProperties is false || p.Alias.StartsWith("umbraco") is false)); + + private IDictionary MapProperties(IEnumerable properties) + => properties.ToDictionary(p => p.Alias, p => p.GetDeliveryApiValue(false)); } diff --git a/src/Umbraco.Core/Models/DeliveryApi/ApiMedia.cs b/src/Umbraco.Core/Models/DeliveryApi/ApiMedia.cs index df076f50a3..a161b69e2a 100644 --- a/src/Umbraco.Core/Models/DeliveryApi/ApiMedia.cs +++ b/src/Umbraco.Core/Models/DeliveryApi/ApiMedia.cs @@ -2,12 +2,16 @@ public sealed class ApiMedia : IApiMedia { - public ApiMedia(Guid id, string name, string mediaType, string url, IDictionary properties) + public ApiMedia(Guid id, string name, string mediaType, string url, string? extension, int? width, int? height, int? bytes, IDictionary properties) { Id = id; Name = name; MediaType = mediaType; Url = url; + Extension = extension; + Width = width; + Height = height; + Bytes = bytes; Properties = properties; } @@ -19,5 +23,13 @@ public sealed class ApiMedia : IApiMedia public string Url { get; } + public string? Extension { get; } + + public int? Width { get; } + + public int? Height { get; } + + public int? Bytes { get; } + public IDictionary Properties { get; } } diff --git a/src/Umbraco.Core/Models/DeliveryApi/IApiMedia.cs b/src/Umbraco.Core/Models/DeliveryApi/IApiMedia.cs index 6ae1575e61..f30b7dbc19 100644 --- a/src/Umbraco.Core/Models/DeliveryApi/IApiMedia.cs +++ b/src/Umbraco.Core/Models/DeliveryApi/IApiMedia.cs @@ -2,13 +2,21 @@ public interface IApiMedia { - public Guid Id { get; } + Guid Id { get; } - public string Name { get; } + string Name { get; } - public string MediaType { get; } + string MediaType { get; } - public string Url { get; } + string Url { get; } - public IDictionary Properties { get; } + string? Extension { get; } + + int? Width { get; } + + int? Height { get; } + + int? Bytes { get; } + + IDictionary Properties { get; } } diff --git a/src/Umbraco.Infrastructure/Models/DeliveryApi/ApiMediaWithCrops.cs b/src/Umbraco.Infrastructure/Models/DeliveryApi/ApiMediaWithCrops.cs index 6315766a1c..4aeaba3dea 100644 --- a/src/Umbraco.Infrastructure/Models/DeliveryApi/ApiMediaWithCrops.cs +++ b/src/Umbraco.Infrastructure/Models/DeliveryApi/ApiMediaWithCrops.cs @@ -24,6 +24,14 @@ internal sealed class ApiMediaWithCrops : IApiMedia public string Url => _inner.Url; + public string? Extension => _inner.Extension; + + public int? Width => _inner.Width; + + public int? Height => _inner.Height; + + public int? Bytes => _inner.Bytes; + public IDictionary Properties => _inner.Properties; public ImageCropperValue.ImageCropperFocalPoint? FocalPoint { get; } diff --git a/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/ContentBuilderTests.cs b/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/ContentBuilderTests.cs index 75f960bb0f..750ac885e0 100644 --- a/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/ContentBuilderTests.cs +++ b/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/ContentBuilderTests.cs @@ -21,6 +21,7 @@ public class ContentBuilderTests : DeliveryApiTests var contentType = new Mock(); contentType.SetupGet(c => c.Alias).Returns("thePageType"); + contentType.SetupGet(c => c.ItemType).Returns(PublishedItemType.Content); var key = Guid.NewGuid(); var urlSegment = "url-segment"; diff --git a/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/DeliveryApiTests.cs b/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/DeliveryApiTests.cs index 5398e22bc3..3bb7339bc5 100644 --- a/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/DeliveryApiTests.cs +++ b/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/DeliveryApiTests.cs @@ -56,11 +56,11 @@ public class DeliveryApiTests DefaultPropertyType = SetupPublishedPropertyType(defaultPropertyValueConverter.Object, "default", "Default.Editor"); } - protected IPublishedPropertyType SetupPublishedPropertyType(IPropertyValueConverter valueConverter, string propertyTypeAlias, string editorAlias) + protected IPublishedPropertyType SetupPublishedPropertyType(IPropertyValueConverter valueConverter, string propertyTypeAlias, string editorAlias, object? dataTypeConfiguration = null) { var mockPublishedContentTypeFactory = new Mock(); mockPublishedContentTypeFactory.Setup(x => x.GetDataType(It.IsAny())) - .Returns(new PublishedDataType(123, editorAlias, new Lazy())); + .Returns(new PublishedDataType(123, editorAlias, new Lazy(() => dataTypeConfiguration))); var publishedPropType = new PublishedPropertyType( propertyTypeAlias, @@ -100,7 +100,7 @@ public class DeliveryApiTests }); content.SetupGet(c => c.ContentType).Returns(contentType); content.SetupGet(c => c.Properties).Returns(properties); - content.SetupGet(c => c.ItemType).Returns(PublishedItemType.Content); + content.SetupGet(c => c.ItemType).Returns(contentType.ItemType); } protected string DefaultUrlSegment(string name, string? culture = null) diff --git a/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/MediaBuilderTests.cs b/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/MediaBuilderTests.cs index 80bc65d2a6..399b164783 100644 --- a/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/MediaBuilderTests.cs +++ b/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/MediaBuilderTests.cs @@ -21,19 +21,21 @@ public class MediaBuilderTests : DeliveryApiTests { { Constants.Conventions.Media.Width, 111 }, { Constants.Conventions.Media.Height, 222 }, - { Constants.Conventions.Media.Extension, ".my-ext" } + { Constants.Conventions.Media.Extension, ".my-ext" }, + { Constants.Conventions.Media.Bytes, 333 } }); - var builder = new ApiMediaBuilder(new ApiContentNameProvider(), SetupMediaUrlProvider(), CreateOutputExpansionStrategyAccessor()); + var builder = new ApiMediaBuilder(new ApiContentNameProvider(), SetupMediaUrlProvider(), Mock.Of(), CreateOutputExpansionStrategyAccessor()); var result = builder.Build(media); Assert.NotNull(result); Assert.AreEqual("The media", result.Name); Assert.AreEqual("theMediaType", result.MediaType); Assert.AreEqual("media-url:media-url-segment", result.Url); - Assert.AreEqual(3, result.Properties.Count); - Assert.AreEqual(111, result.Properties[Constants.Conventions.Media.Width]); - Assert.AreEqual(222, result.Properties[Constants.Conventions.Media.Height]); - Assert.AreEqual(".my-ext", result.Properties[Constants.Conventions.Media.Extension]); + Assert.AreEqual(key, result.Id); + Assert.AreEqual(111, result.Width); + Assert.AreEqual(222, result.Height); + Assert.AreEqual(".my-ext", result.Extension); + Assert.AreEqual(333, result.Bytes); } [Test] @@ -46,7 +48,7 @@ public class MediaBuilderTests : DeliveryApiTests new Dictionary() ); - var builder = new ApiMediaBuilder(new ApiContentNameProvider(), SetupMediaUrlProvider(), CreateOutputExpansionStrategyAccessor()); + var builder = new ApiMediaBuilder(new ApiContentNameProvider(), SetupMediaUrlProvider(), Mock.Of(), CreateOutputExpansionStrategyAccessor()); var result = builder.Build(media); Assert.NotNull(result); Assert.IsEmpty(result.Properties); @@ -61,7 +63,7 @@ public class MediaBuilderTests : DeliveryApiTests "media-url-segment", new Dictionary { { "myProperty", 123 }, { "anotherProperty", "A value goes here" } }); - var builder = new ApiMediaBuilder(new ApiContentNameProvider(), SetupMediaUrlProvider(), CreateOutputExpansionStrategyAccessor()); + var builder = new ApiMediaBuilder(new ApiContentNameProvider(), SetupMediaUrlProvider(), Mock.Of(), CreateOutputExpansionStrategyAccessor()); var result = builder.Build(media); Assert.NotNull(result); Assert.AreEqual(2, result.Properties.Count); @@ -76,7 +78,7 @@ public class MediaBuilderTests : DeliveryApiTests var mediaType = new Mock(); mediaType.SetupGet(c => c.Alias).Returns("theMediaType"); - var mediaProperties = properties.Select(kvp => SetupProperty(kvp.Key, kvp.Value, media.Object)).ToArray(); + var mediaProperties = properties.Select(kvp => SetupProperty(kvp.Key, kvp.Value)).ToArray(); media.SetupGet(c => c.Properties).Returns(mediaProperties); media.SetupGet(c => c.UrlSegment).Returns(urlSegment); @@ -89,7 +91,7 @@ public class MediaBuilderTests : DeliveryApiTests return media.Object; } - private IPublishedProperty SetupProperty(string alias, T value, IPublishedContent media) + private IPublishedProperty SetupProperty(string alias, T value) { var propertyMock = new Mock(); propertyMock.SetupGet(p => p.Alias).Returns(alias); diff --git a/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/MediaPickerValueConverterTests.cs b/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/MediaPickerValueConverterTests.cs index b629dccdb7..0bb8fa46e8 100644 --- a/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/MediaPickerValueConverterTests.cs +++ b/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/MediaPickerValueConverterTests.cs @@ -98,5 +98,6 @@ public class MediaPickerValueConverterTests : PropertyValueConverterTests new ApiMediaBuilder( new ApiContentNameProvider(), new ApiMediaUrlProvider(PublishedUrlProvider), + Mock.Of(), CreateOutputExpansionStrategyAccessor())); } diff --git a/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/MediaPickerWithCropsValueConverterTests.cs b/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/MediaPickerWithCropsValueConverterTests.cs index 957075092e..23dc5bb8b3 100644 --- a/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/MediaPickerWithCropsValueConverterTests.cs +++ b/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/MediaPickerWithCropsValueConverterTests.cs @@ -27,6 +27,7 @@ public class MediaPickerWithCropsValueConverterTests : PropertyValueConverterTes new ApiMediaBuilder( new ApiContentNameProvider(), apiUrlProvider, + Mock.Of(), CreateOutputExpansionStrategyAccessor())); } @@ -34,7 +35,7 @@ public class MediaPickerWithCropsValueConverterTests : PropertyValueConverterTes public void MediaPickerWithCropsValueConverter_InSingleMode_ConvertsValueToCollectionOfApiMedia() { var publishedPropertyType = SetupMediaPropertyType(false); - var mediaKey = SetupMedia("My media", ".jpg", 200, 400, "My alt text"); + var mediaKey = SetupMedia("My media", ".jpg", 200, 400, "My alt text", 800); var serializer = new JsonNetSerializer(); @@ -64,7 +65,15 @@ public class MediaPickerWithCropsValueConverterTests : PropertyValueConverterTes Assert.AreEqual(1, result.Count()); Assert.AreEqual("My media", result.First().Name); Assert.AreEqual("my-media", result.First().Url); + Assert.AreEqual(".jpg", result.First().Extension); + Assert.AreEqual(200, result.First().Width); + Assert.AreEqual(400, result.First().Height); + Assert.AreEqual(800, result.First().Bytes); Assert.NotNull(result.First().FocalPoint); + Assert.AreEqual(".jpg", result.First().Extension); + Assert.AreEqual(200, result.First().Width); + Assert.AreEqual(400, result.First().Height); + Assert.AreEqual(800, result.First().Bytes); Assert.AreEqual(.2m, result.First().FocalPoint.Left); Assert.AreEqual(.4m, result.First().FocalPoint.Top); Assert.NotNull(result.First().Crops); @@ -78,19 +87,16 @@ public class MediaPickerWithCropsValueConverterTests : PropertyValueConverterTes Assert.AreEqual(10m, result.First().Crops.First().Coordinates.Y1); Assert.AreEqual(20m, result.First().Crops.First().Coordinates.Y2); Assert.NotNull(result.First().Properties); - Assert.AreEqual(4, result.First().Properties.Count); + Assert.AreEqual(1, result.First().Properties.Count); Assert.AreEqual("My alt text", result.First().Properties["altText"]); - Assert.AreEqual(".jpg", result.First().Properties[Constants.Conventions.Media.Extension]); - Assert.AreEqual(200, result.First().Properties[Constants.Conventions.Media.Width]); - Assert.AreEqual(400, result.First().Properties[Constants.Conventions.Media.Height]); } [Test] public void MediaPickerWithCropsValueConverter_InMultiMode_ConvertsValueToMedias() { var publishedPropertyType = SetupMediaPropertyType(true); - var mediaKey1 = SetupMedia("My media", ".jpg", 200, 400, "My alt text"); - var mediaKey2 = SetupMedia("My other media", ".png", 800, 600, "My other alt text"); + var mediaKey1 = SetupMedia("My media", ".jpg", 200, 400, "My alt text", 800); + var mediaKey2 = SetupMedia("My other media", ".png", 800, 600, "My other alt text", 200); var serializer = new JsonNetSerializer(); @@ -135,6 +141,10 @@ public class MediaPickerWithCropsValueConverterTests : PropertyValueConverterTes Assert.AreEqual("My media", result.First().Name); Assert.AreEqual("my-media", result.First().Url); + Assert.AreEqual(".jpg", result.First().Extension); + Assert.AreEqual(200, result.First().Width); + Assert.AreEqual(400, result.First().Height); + Assert.AreEqual(800, result.First().Bytes); Assert.NotNull(result.First().FocalPoint); Assert.AreEqual(.2m, result.First().FocalPoint.Left); Assert.AreEqual(.4m, result.First().FocalPoint.Top); @@ -149,14 +159,15 @@ public class MediaPickerWithCropsValueConverterTests : PropertyValueConverterTes Assert.AreEqual(10m, result.First().Crops.First().Coordinates.Y1); Assert.AreEqual(20m, result.First().Crops.First().Coordinates.Y2); Assert.NotNull(result.First().Properties); - Assert.AreEqual(4, result.First().Properties.Count); + Assert.AreEqual(1, result.First().Properties.Count); Assert.AreEqual("My alt text", result.First().Properties["altText"]); - Assert.AreEqual(".jpg", result.First().Properties[Constants.Conventions.Media.Extension]); - Assert.AreEqual(200, result.First().Properties[Constants.Conventions.Media.Width]); - Assert.AreEqual(400, result.First().Properties[Constants.Conventions.Media.Height]); Assert.AreEqual("My other media", result.Last().Name); Assert.AreEqual("my-other-media", result.Last().Url); + Assert.AreEqual(".png", result.Last().Extension); + Assert.AreEqual(800, result.Last().Width); + Assert.AreEqual(600, result.Last().Height); + Assert.AreEqual(200, result.Last().Bytes); Assert.NotNull(result.Last().FocalPoint); Assert.AreEqual(.8m, result.Last().FocalPoint.Left); Assert.AreEqual(.6m, result.Last().FocalPoint.Top); @@ -171,11 +182,8 @@ public class MediaPickerWithCropsValueConverterTests : PropertyValueConverterTes Assert.AreEqual(2m, result.Last().Crops.Last().Coordinates.Y1); Assert.AreEqual(1m, result.Last().Crops.Last().Coordinates.Y2); Assert.NotNull(result.Last().Properties); - Assert.AreEqual(4, result.Last().Properties.Count); + Assert.AreEqual(1, result.Last().Properties.Count); Assert.AreEqual("My other alt text", result.Last().Properties["altText"]); - Assert.AreEqual(".png", result.Last().Properties[Constants.Conventions.Media.Extension]); - Assert.AreEqual(800, result.Last().Properties[Constants.Conventions.Media.Width]); - Assert.AreEqual(600, result.Last().Properties[Constants.Conventions.Media.Height]); } [TestCase("")] @@ -233,7 +241,7 @@ public class MediaPickerWithCropsValueConverterTests : PropertyValueConverterTes return publishedPropertyType.Object; } - private Guid SetupMedia(string name, string extension, int width, int height, string altText) + private Guid SetupMedia(string name, string extension, int width, int height, string altText, int bytes) { var publishedMediaType = new Mock(); publishedMediaType.SetupGet(c => c.ItemType).Returns(PublishedItemType.Media); @@ -257,6 +265,7 @@ public class MediaPickerWithCropsValueConverterTests : PropertyValueConverterTes AddProperty(Constants.Conventions.Media.Extension, extension); AddProperty(Constants.Conventions.Media.Width, width); AddProperty(Constants.Conventions.Media.Height, height); + AddProperty(Constants.Conventions.Media.Bytes, bytes); AddProperty("altText", altText); PublishedMediaCacheMock diff --git a/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/MultiNodeTreePickerValueConverterTests.cs b/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/MultiNodeTreePickerValueConverterTests.cs index ee4635f145..320f16dbf6 100644 --- a/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/MultiNodeTreePickerValueConverterTests.cs +++ b/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/MultiNodeTreePickerValueConverterTests.cs @@ -27,7 +27,7 @@ public class MultiNodeTreePickerValueConverterTests : PropertyValueConverterTest Mock.Of(), Mock.Of(), new ApiContentBuilder(contentNameProvider, routeBuilder, expansionStrategyAccessor), - new ApiMediaBuilder(contentNameProvider, apiUrProvider, expansionStrategyAccessor)); + new ApiMediaBuilder(contentNameProvider, apiUrProvider, Mock.Of(), expansionStrategyAccessor)); } private PublishedDataType MultiNodePickerPublishedDataType(bool multiSelect, string entityType) => diff --git a/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/OutputExpansionStrategyTests.cs b/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/OutputExpansionStrategyTests.cs index 86aa8215db..fe3a9bdf71 100644 --- a/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/OutputExpansionStrategyTests.cs +++ b/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/OutputExpansionStrategyTests.cs @@ -11,6 +11,7 @@ using Umbraco.Cms.Core.PropertyEditors; using Umbraco.Cms.Core.PropertyEditors.DeliveryApi; using Umbraco.Cms.Core.PropertyEditors.ValueConverters; using Umbraco.Cms.Core.PublishedCache; +using Umbraco.Cms.Infrastructure.Serialization; namespace Umbraco.Cms.Tests.UnitTests.Umbraco.Core.DeliveryApi; @@ -19,6 +20,7 @@ public class OutputExpansionStrategyTests : PropertyValueConverterTests { private IPublishedContentType _contentType; private IPublishedContentType _elementType; + private IPublishedContentType _mediaType; [SetUp] public void SetUp() @@ -31,6 +33,10 @@ public class OutputExpansionStrategyTests : PropertyValueConverterTests elementType.SetupGet(c => c.Alias).Returns("theElementType"); elementType.SetupGet(c => c.ItemType).Returns(PublishedItemType.Element); _elementType = elementType.Object; + var mediaType = new Mock(); + mediaType.SetupGet(c => c.Alias).Returns("theMediaType"); + mediaType.SetupGet(c => c.ItemType).Returns(PublishedItemType.Media); + _mediaType = mediaType.Object; } [Test] @@ -91,6 +97,47 @@ public class OutputExpansionStrategyTests : PropertyValueConverterTests Assert.AreEqual(78, contentPickerTwoOutput.Properties["numberTwo"]); } + [TestCase(false)] + [TestCase(true)] + public void OutputExpansionStrategy_CanExpandSpecificMedia(bool mediaPicker3) + { + var accessor = CreateOutputExpansionStrategyAccessor(false, new[] { "mediaPickerTwo" }); + var apiMediaBuilder = new ApiMediaBuilder( + new ApiContentNameProvider(), + new ApiMediaUrlProvider(PublishedUrlProvider), + Mock.Of(), + accessor); + + var media = new Mock(); + + var mediaPickerOneContent = CreateSimplePickedMedia(12, 34); + var mediaPickerOneProperty = mediaPicker3 + ? CreateMediaPicker3Property(media.Object, mediaPickerOneContent.Key, "mediaPickerOne", apiMediaBuilder) + : CreateMediaPickerProperty(media.Object, mediaPickerOneContent.Key, "mediaPickerOne", apiMediaBuilder); + var mediaPickerTwoContent = CreateSimplePickedMedia(56, 78); + var mediaPickerTwoProperty = mediaPicker3 + ? CreateMediaPicker3Property(media.Object, mediaPickerTwoContent.Key, "mediaPickerTwo", apiMediaBuilder) + : CreateMediaPickerProperty(media.Object, mediaPickerTwoContent.Key, "mediaPickerTwo", apiMediaBuilder); + + SetupMediaMock(media, mediaPickerOneProperty, mediaPickerTwoProperty); + + var result = apiMediaBuilder.Build(media.Object); + + Assert.AreEqual(2, result.Properties.Count); + + var mediaPickerOneOutput = (result.Properties["mediaPickerOne"] as IEnumerable)?.FirstOrDefault(); + Assert.IsNotNull(mediaPickerOneOutput); + Assert.AreEqual(mediaPickerOneContent.Key, mediaPickerOneOutput.Id); + Assert.IsEmpty(mediaPickerOneOutput.Properties); + + var mediaPickerTwoOutput = (result.Properties["mediaPickerTwo"] as IEnumerable)?.FirstOrDefault(); + Assert.IsNotNull(mediaPickerTwoOutput); + Assert.AreEqual(mediaPickerTwoContent.Key, mediaPickerTwoOutput.Id); + Assert.AreEqual(2, mediaPickerTwoOutput.Properties.Count); + Assert.AreEqual(56, mediaPickerTwoOutput.Properties["numberOne"]); + Assert.AreEqual(78, mediaPickerTwoOutput.Properties["numberTwo"]); + } + [Test] public void OutputExpansionStrategy_CanExpandAllContent() { @@ -339,6 +386,30 @@ public class OutputExpansionStrategyTests : PropertyValueConverterTests Assert.AreEqual(0, nestedContentPickerOutput.Properties.Count); } + [Test] + public void OutputExpansionStrategy_MappingContent_ThrowsOnInvalidItemType() + { + var accessor = CreateOutputExpansionStrategyAccessor(); + if (accessor.TryGetValue(out IOutputExpansionStrategy outputExpansionStrategy) is false) + { + Assert.Fail("Could not obtain the output expansion strategy"); + } + + Assert.Throws(() => outputExpansionStrategy.MapContentProperties(PublishedMedia)); + } + + [Test] + public void OutputExpansionStrategy_MappingMedia_ThrowsOnInvalidItemType() + { + var accessor = CreateOutputExpansionStrategyAccessor(); + if (accessor.TryGetValue(out IOutputExpansionStrategy outputExpansionStrategy) is false) + { + Assert.Fail("Could not obtain the output expansion strategy"); + } + + Assert.Throws(() => outputExpansionStrategy.MapMediaProperties(PublishedContent)); + } + private IOutputExpansionStrategyAccessor CreateOutputExpansionStrategyAccessor(bool expandAll = false, string[]? expandPropertyAliases = null) { var httpContextMock = new Mock(); @@ -370,6 +441,16 @@ public class OutputExpansionStrategyTests : PropertyValueConverterTests RegisterContentWithProviders(content.Object); } + private void SetupMediaMock(Mock media, params IPublishedProperty[] properties) + { + var key = Guid.NewGuid(); + var name = "The media"; + var urlSegment = "media-url-segment"; + ConfigurePublishedContentMock(media, key, name, urlSegment, _mediaType, properties); + + RegisterMediaWithProviders(media.Object); + } + private IPublishedContent CreateSimplePickedContent(int numberOneValue, int numberTwoValue) { var content = new Mock(); @@ -381,6 +462,17 @@ public class OutputExpansionStrategyTests : PropertyValueConverterTests return content.Object; } + private IPublishedContent CreateSimplePickedMedia(int numberOneValue, int numberTwoValue) + { + var media = new Mock(); + SetupMediaMock( + media, + CreateNumberProperty(media.Object, numberOneValue, "numberOne"), + CreateNumberProperty(media.Object, numberTwoValue, "numberTwo")); + + return media.Object; + } + private IPublishedContent CreateMultiLevelPickedContent(int numberValue, IPublishedContent nestedContentPickerValue, string nestedContentPickerPropertyTypeAlias, ApiContentBuilder apiContentBuilder) { var content = new Mock(); @@ -400,6 +492,31 @@ public class OutputExpansionStrategyTests : PropertyValueConverterTests return new PublishedElementPropertyBase(contentPickerPropertyType, parent, false, PropertyCacheLevel.None, new GuidUdi(Constants.UdiEntityType.Document, pickedContentKey).ToString()); } + private PublishedElementPropertyBase CreateMediaPickerProperty(IPublishedElement parent, Guid pickedMediaKey, string propertyTypeAlias, IApiMediaBuilder mediaBuilder) + { + MediaPickerValueConverter mediaPickerValueConverter = new MediaPickerValueConverter(PublishedSnapshotAccessor, Mock.Of(), mediaBuilder); + var mediaPickerPropertyType = SetupPublishedPropertyType(mediaPickerValueConverter, propertyTypeAlias, Constants.PropertyEditors.Aliases.MediaPicker, new MediaPickerConfiguration()); + + return new PublishedElementPropertyBase(mediaPickerPropertyType, parent, false, PropertyCacheLevel.None, new GuidUdi(Constants.UdiEntityType.Media, pickedMediaKey).ToString()); + } + + private PublishedElementPropertyBase CreateMediaPicker3Property(IPublishedElement parent, Guid pickedMediaKey, string propertyTypeAlias, IApiMediaBuilder mediaBuilder) + { + var serializer = new JsonNetSerializer(); + var value = serializer.Serialize(new[] + { + new MediaPicker3PropertyEditor.MediaPicker3PropertyValueEditor.MediaWithCropsDto + { + MediaKey = pickedMediaKey + } + }); + + MediaPickerWithCropsValueConverter mediaPickerValueConverter = new MediaPickerWithCropsValueConverter(PublishedSnapshotAccessor, PublishedUrlProvider, Mock.Of(), new JsonNetSerializer(), mediaBuilder); + var mediaPickerPropertyType = SetupPublishedPropertyType(mediaPickerValueConverter, propertyTypeAlias, Constants.PropertyEditors.Aliases.MediaPicker3, new MediaPicker3Configuration()); + + return new PublishedElementPropertyBase(mediaPickerPropertyType, parent, false, PropertyCacheLevel.None, value); + } + private PublishedElementPropertyBase CreateNumberProperty(IPublishedElement parent, int propertyValue, string propertyTypeAlias) { var numberPropertyType = SetupPublishedPropertyType(new IntegerValueConverter(), propertyTypeAlias, Constants.PropertyEditors.Aliases.Label); diff --git a/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/PropertyValueConverterTests.cs b/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/PropertyValueConverterTests.cs index 8ee2cac5cf..d3831ec2e4 100644 --- a/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/PropertyValueConverterTests.cs +++ b/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/PropertyValueConverterTests.cs @@ -105,5 +105,8 @@ public class PropertyValueConverterTests : DeliveryApiTests PublishedMediaCacheMock .Setup(pcc => pcc.GetById(media.Key)) .Returns(media); + PublishedMediaCacheMock + .Setup(pcc => pcc.GetById(It.IsAny(), media.Key)) + .Returns(media); } } From 2f7cb83462ad3ed5b06d52748472942a030900f7 Mon Sep 17 00:00:00 2001 From: Kenn Jacobsen Date: Wed, 10 May 2023 14:23:02 +0200 Subject: [PATCH 04/26] IAppCache implementations should not cache null values (#14218) * IAppCache implementations should not cache null values * Add comment --- src/Umbraco.Core/Cache/DictionaryAppCache.cs | 14 ++++++++++++- .../Cache/FastDictionaryAppCache.cs | 8 ++++++++ src/Umbraco.Core/Cache/IAppCache.cs | 1 + .../Umbraco.Core/Cache/AppCacheTests.cs | 19 ++++++++++++++++++ .../Cache/FastDictionaryAppCacheTests.cs | 20 +++++++++++++++++++ 5 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 tests/Umbraco.Tests.UnitTests/Umbraco.Core/Cache/FastDictionaryAppCacheTests.cs diff --git a/src/Umbraco.Core/Cache/DictionaryAppCache.cs b/src/Umbraco.Core/Cache/DictionaryAppCache.cs index 5bf5848309..fa0ec1b0e0 100644 --- a/src/Umbraco.Core/Cache/DictionaryAppCache.cs +++ b/src/Umbraco.Core/Cache/DictionaryAppCache.cs @@ -24,7 +24,19 @@ public class DictionaryAppCache : IRequestCache public virtual object? Get(string key) => _items.TryGetValue(key, out var value) ? value : null; /// - public virtual object? Get(string key, Func factory) => _items.GetOrAdd(key, _ => factory()); + public virtual object? Get(string key, Func factory) + { + var value = _items.GetOrAdd(key, _ => factory()); + + if (value is not null) + { + return value; + } + + // do not cache null values + _items.TryRemove(key, out _); + return null; + } public bool Set(string key, object? value) => _items.TryAdd(key, value); diff --git a/src/Umbraco.Core/Cache/FastDictionaryAppCache.cs b/src/Umbraco.Core/Cache/FastDictionaryAppCache.cs index 6476c76f96..e99cdad899 100644 --- a/src/Umbraco.Core/Cache/FastDictionaryAppCache.cs +++ b/src/Umbraco.Core/Cache/FastDictionaryAppCache.cs @@ -31,6 +31,14 @@ public class FastDictionaryAppCache : IAppCache Lazy? result = _items.GetOrAdd(cacheKey, k => SafeLazy.GetSafeLazy(getCacheItem)); var value = result.Value; // will not throw (safe lazy) + + if (value is null) + { + // do not cache null values + _items.TryRemove(cacheKey, out _); + return null; + } + if (!(value is SafeLazy.ExceptionHolder eh)) { return value; diff --git a/src/Umbraco.Core/Cache/IAppCache.cs b/src/Umbraco.Core/Cache/IAppCache.cs index 187ff6fc11..99207e6c10 100644 --- a/src/Umbraco.Core/Cache/IAppCache.cs +++ b/src/Umbraco.Core/Cache/IAppCache.cs @@ -18,6 +18,7 @@ public interface IAppCache /// The key of the item. /// A factory function that can create the item. /// The item. + /// Null values returned from the factory function are never cached. object? Get(string key, Func factory); /// diff --git a/tests/Umbraco.Tests.UnitTests/Umbraco.Core/Cache/AppCacheTests.cs b/tests/Umbraco.Tests.UnitTests/Umbraco.Core/Cache/AppCacheTests.cs index f818fa49e8..a207a257fe 100644 --- a/tests/Umbraco.Tests.UnitTests/Umbraco.Core/Cache/AppCacheTests.cs +++ b/tests/Umbraco.Tests.UnitTests/Umbraco.Core/Cache/AppCacheTests.cs @@ -81,6 +81,25 @@ public abstract class AppCacheTests Assert.Greater(counter, 1); } + [Test] + public void Does_Not_Cache_Null_Values() + { + var counter = 0; + + object? Factory() + { + counter++; + return counter == 3 ? "Not a null value" : null; + } + + object? Get() => AppCache.Get("Blah", Factory); + + Assert.IsNull(Get()); + Assert.IsNull(Get()); + Assert.AreEqual("Not a null value", Get()); + Assert.AreEqual(3, counter); + } + [Test] public void Ensures_Delegate_Result_Is_Cached_Once() { diff --git a/tests/Umbraco.Tests.UnitTests/Umbraco.Core/Cache/FastDictionaryAppCacheTests.cs b/tests/Umbraco.Tests.UnitTests/Umbraco.Core/Cache/FastDictionaryAppCacheTests.cs new file mode 100644 index 0000000000..efb6f0fc9e --- /dev/null +++ b/tests/Umbraco.Tests.UnitTests/Umbraco.Core/Cache/FastDictionaryAppCacheTests.cs @@ -0,0 +1,20 @@ +using NUnit.Framework; +using Umbraco.Cms.Core.Cache; + +namespace Umbraco.Cms.Tests.UnitTests.Umbraco.Core.Cache; + +[TestFixture] +public class FastDictionaryAppCacheTests : AppCacheTests +{ + public override void Setup() + { + base.Setup(); + _appCache = new FastDictionaryAppCache(); + } + + private FastDictionaryAppCache _appCache; + + internal override IAppCache AppCache => _appCache; + + protected override int GetTotalItemCount => _appCache.Count; +} From 5b73fc19eea31681905b1bd86ce72fa4e2be6adb Mon Sep 17 00:00:00 2001 From: Andreas Zerbst <73799582+andr317c@users.noreply.github.com> Date: Wed, 10 May 2023 14:31:40 +0200 Subject: [PATCH 05/26] V10: Fix NoopPublishedValuefallback varation context member not implemented (#14227) * Added missing VariationContextAccessor * Fixed identation --- .../Models/PublishedContent/NoopPublishedValueFallback.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Umbraco.Core/Models/PublishedContent/NoopPublishedValueFallback.cs b/src/Umbraco.Core/Models/PublishedContent/NoopPublishedValueFallback.cs index 1dd2fef124..1164b02753 100644 --- a/src/Umbraco.Core/Models/PublishedContent/NoopPublishedValueFallback.cs +++ b/src/Umbraco.Core/Models/PublishedContent/NoopPublishedValueFallback.cs @@ -7,7 +7,15 @@ namespace Umbraco.Cms.Core.Models.PublishedContent; /// This is for tests etc - does not implement fallback at all. /// public class NoopPublishedValueFallback : IPublishedValueFallback + { + /// + public IVariationContextAccessor VariationContextAccessor + { + get => new ThreadCultureVariationContextAccessor(); + set { } + } + /// public bool TryGetValue(IPublishedProperty property, string? culture, string? segment, Fallback fallback, object? defaultValue, out object? value) { From b0b591821b53870b0a8c28a1c44f1d5241319975 Mon Sep 17 00:00:00 2001 From: Nikolaj Geisle <70372949+Zeegaan@users.noreply.github.com> Date: Wed, 10 May 2023 14:55:06 +0200 Subject: [PATCH 06/26] V11: prevent currentuser resetting password without old password (#14189) * Implement check for current user if no old password * Add default implementation to avoid breaking change9 --- .../Controllers/CurrentUserController.cs | 2 +- .../Controllers/MemberController.cs | 2 +- .../Controllers/UsersController.cs | 2 +- .../Security/IPasswordChanger.cs | 7 +++++++ .../Security/PasswordChanger.cs | 19 ++++++++++++++++--- .../Controllers/MemberControllerUnitTests.cs | 6 ++++-- 6 files changed, 30 insertions(+), 8 deletions(-) diff --git a/src/Umbraco.Web.BackOffice/Controllers/CurrentUserController.cs b/src/Umbraco.Web.BackOffice/Controllers/CurrentUserController.cs index fb1666e522..73cd71918d 100644 --- a/src/Umbraco.Web.BackOffice/Controllers/CurrentUserController.cs +++ b/src/Umbraco.Web.BackOffice/Controllers/CurrentUserController.cs @@ -278,7 +278,7 @@ public class CurrentUserController : UmbracoAuthorizedJsonController // all current users have access to reset/manually change their password Attempt passwordChangeResult = - await _passwordChanger.ChangePasswordWithIdentityAsync(changingPasswordModel, _backOfficeUserManager); + await _passwordChanger.ChangePasswordWithIdentityAsync(changingPasswordModel, _backOfficeUserManager, currentUser); if (passwordChangeResult.Success) { diff --git a/src/Umbraco.Web.BackOffice/Controllers/MemberController.cs b/src/Umbraco.Web.BackOffice/Controllers/MemberController.cs index f06ae05f86..c190891217 100644 --- a/src/Umbraco.Web.BackOffice/Controllers/MemberController.cs +++ b/src/Umbraco.Web.BackOffice/Controllers/MemberController.cs @@ -623,7 +623,7 @@ public class MemberController : ContentControllerBase // Change and persist the password Attempt passwordChangeResult = - await _passwordChanger.ChangePasswordWithIdentityAsync(changingPasswordModel, _memberManager); + await _passwordChanger.ChangePasswordWithIdentityAsync(changingPasswordModel, _memberManager, _backOfficeSecurityAccessor.BackOfficeSecurity?.CurrentUser); if (!passwordChangeResult.Success) { diff --git a/src/Umbraco.Web.BackOffice/Controllers/UsersController.cs b/src/Umbraco.Web.BackOffice/Controllers/UsersController.cs index 02eb9cda8e..3e14154b48 100644 --- a/src/Umbraco.Web.BackOffice/Controllers/UsersController.cs +++ b/src/Umbraco.Web.BackOffice/Controllers/UsersController.cs @@ -759,7 +759,7 @@ public class UsersController : BackOfficeNotificationsController } Attempt passwordChangeResult = - await _passwordChanger.ChangePasswordWithIdentityAsync(changingPasswordModel, _userManager); + await _passwordChanger.ChangePasswordWithIdentityAsync(changingPasswordModel, _userManager, currentUser); if (passwordChangeResult.Success) { diff --git a/src/Umbraco.Web.BackOffice/Security/IPasswordChanger.cs b/src/Umbraco.Web.BackOffice/Security/IPasswordChanger.cs index 66c69d4d70..3bc5f35abf 100644 --- a/src/Umbraco.Web.BackOffice/Security/IPasswordChanger.cs +++ b/src/Umbraco.Web.BackOffice/Security/IPasswordChanger.cs @@ -1,12 +1,19 @@ using Umbraco.Cms.Core; using Umbraco.Cms.Core.Models; +using Umbraco.Cms.Core.Models.Membership; using Umbraco.Cms.Core.Security; namespace Umbraco.Cms.Web.Common.Security; public interface IPasswordChanger where TUser : UmbracoIdentityUser { + [Obsolete("Please use method that also takes a nullable IUser, scheduled for removal in v13")] public Task> ChangePasswordWithIdentityAsync( ChangingPasswordModel passwordModel, IUmbracoUserManager userMgr); + + public Task> ChangePasswordWithIdentityAsync( + ChangingPasswordModel passwordModel, + IUmbracoUserManager userMgr, + IUser? currentUser) => ChangePasswordWithIdentityAsync(passwordModel, userMgr); } diff --git a/src/Umbraco.Web.BackOffice/Security/PasswordChanger.cs b/src/Umbraco.Web.BackOffice/Security/PasswordChanger.cs index 25f0548386..8b74f6d2c3 100644 --- a/src/Umbraco.Web.BackOffice/Security/PasswordChanger.cs +++ b/src/Umbraco.Web.BackOffice/Security/PasswordChanger.cs @@ -3,6 +3,7 @@ using Microsoft.AspNetCore.Identity; using Microsoft.Extensions.Logging; using Umbraco.Cms.Core; using Umbraco.Cms.Core.Models; +using Umbraco.Cms.Core.Models.Membership; using Umbraco.Cms.Core.Security; using Umbraco.Extensions; @@ -22,16 +23,20 @@ internal class PasswordChanger : IPasswordChanger where TUser : Um /// Logger for this class public PasswordChanger(ILogger> logger) => _logger = logger; + public Task> ChangePasswordWithIdentityAsync(ChangingPasswordModel passwordModel, IUmbracoUserManager userMgr) => ChangePasswordWithIdentityAsync(passwordModel, userMgr, null); + /// /// Changes the password for a user based on the many different rules and config options /// - /// The changing password model - /// The identity manager to use to update the password + /// The changing password model. + /// The identity manager to use to update the password. + /// The user performing the operation. /// Create an adapter to pass through everything - adapting the member into a user for this functionality /// The outcome of the password changed model public async Task> ChangePasswordWithIdentityAsync( ChangingPasswordModel changingPasswordModel, - IUmbracoUserManager userMgr) + IUmbracoUserManager userMgr, + IUser? currentUser) { if (changingPasswordModel == null) { @@ -65,6 +70,14 @@ internal class PasswordChanger : IPasswordChanger where TUser : Um // Are we just changing another user/member's password? if (changingPasswordModel.OldPassword.IsNullOrWhiteSpace()) { + if (changingPasswordModel.Id == currentUser?.Id) + { + return Attempt.Fail(new PasswordChangedModel + { + ChangeError = new ValidationResult("Cannot change the password of current user without the old password", new[] { "value" }), + }); + } + // ok, we should be able to reset it var resetToken = await userMgr.GeneratePasswordResetTokenAsync(identityUser); diff --git a/tests/Umbraco.Tests.UnitTests/Umbraco.Web.BackOffice/Controllers/MemberControllerUnitTests.cs b/tests/Umbraco.Tests.UnitTests/Umbraco.Web.BackOffice/Controllers/MemberControllerUnitTests.cs index 83d39a4245..f15234b3e0 100644 --- a/tests/Umbraco.Tests.UnitTests/Umbraco.Web.BackOffice/Controllers/MemberControllerUnitTests.cs +++ b/tests/Umbraco.Tests.UnitTests/Umbraco.Web.BackOffice/Controllers/MemberControllerUnitTests.cs @@ -359,7 +359,8 @@ public class MemberControllerUnitTests Mock.Get(passwordChanger) .Setup(x => x.ChangePasswordWithIdentityAsync( It.IsAny(), - umbracoMembersUserManager)) + umbracoMembersUserManager, + null)) .ReturnsAsync(() => attempt); } else @@ -368,7 +369,8 @@ public class MemberControllerUnitTests Mock.Get(passwordChanger) .Setup(x => x.ChangePasswordWithIdentityAsync( It.IsAny(), - umbracoMembersUserManager)) + umbracoMembersUserManager, + null)) .ReturnsAsync(() => attempt); } } From eb0c7e617e16e27901f6eeb8c4a79fff33f7205d Mon Sep 17 00:00:00 2001 From: Nikolaj Geisle <70372949+Zeegaan@users.noreply.github.com> Date: Wed, 10 May 2023 15:23:00 +0200 Subject: [PATCH 07/26] Unbreak IMigrationPlanExecutor.cs (#14225) --- .../Migrations/ExecutedMigrationPlan.cs | 3 +++ .../Migrations/IMigrationPlanExecutor.cs | 12 +++++++++++- .../Migrations/MigrationPlanExecutor.cs | 2 ++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/Umbraco.Infrastructure/Migrations/ExecutedMigrationPlan.cs b/src/Umbraco.Infrastructure/Migrations/ExecutedMigrationPlan.cs index 0298939b8f..065a8e049f 100644 --- a/src/Umbraco.Infrastructure/Migrations/ExecutedMigrationPlan.cs +++ b/src/Umbraco.Infrastructure/Migrations/ExecutedMigrationPlan.cs @@ -1,3 +1,5 @@ +using System.Diagnostics.CodeAnalysis; + namespace Umbraco.Cms.Infrastructure.Migrations; public class ExecutedMigrationPlan @@ -9,6 +11,7 @@ public class ExecutedMigrationPlan FinalState = finalState ?? throw new ArgumentNullException(nameof(finalState)); } + [SetsRequiredMembers] public ExecutedMigrationPlan( MigrationPlan plan, string initialState, diff --git a/src/Umbraco.Infrastructure/Migrations/IMigrationPlanExecutor.cs b/src/Umbraco.Infrastructure/Migrations/IMigrationPlanExecutor.cs index f9f87f9e00..ee3f787c12 100644 --- a/src/Umbraco.Infrastructure/Migrations/IMigrationPlanExecutor.cs +++ b/src/Umbraco.Infrastructure/Migrations/IMigrationPlanExecutor.cs @@ -1,8 +1,18 @@ using Umbraco.Cms.Infrastructure.Migrations; +using Umbraco.Extensions; namespace Umbraco.Cms.Core.Migrations; public interface IMigrationPlanExecutor { - ExecutedMigrationPlan ExecutePlan(MigrationPlan plan, string fromState); + [Obsolete("Use ExecutePlan instead.")] + string Execute(MigrationPlan plan, string fromState); + + ExecutedMigrationPlan ExecutePlan(MigrationPlan plan, string fromState) + { + var state = Execute(plan, fromState); + + // We have no real way of knowing whether it was successfull or not here, assume true. + return new ExecutedMigrationPlan(plan, fromState, state, true, plan.Transitions.Select(x => x.Value).WhereNotNull().ToList()); + } } diff --git a/src/Umbraco.Infrastructure/Migrations/MigrationPlanExecutor.cs b/src/Umbraco.Infrastructure/Migrations/MigrationPlanExecutor.cs index 1023789012..c271088626 100644 --- a/src/Umbraco.Infrastructure/Migrations/MigrationPlanExecutor.cs +++ b/src/Umbraco.Infrastructure/Migrations/MigrationPlanExecutor.cs @@ -80,6 +80,8 @@ public class MigrationPlanExecutor : IMigrationPlanExecutor { } + public string Execute(MigrationPlan plan, string fromState) => ExecutePlan(plan, fromState).FinalState; + /// /// Executes the plan. /// From f804c8c20941da7edf364a0517417d9ff8c545a5 Mon Sep 17 00:00:00 2001 From: Kenn Jacobsen Date: Wed, 10 May 2023 15:48:00 +0200 Subject: [PATCH 08/26] Handle culture variance and add more options for indexing fields (#14185) * Handle culture variance and add more options for indexing fields * Workaround for missing DB on install * Add a document per language to the delivery API content index * Filters and Selectors must be able to match multiple values (as OR) * A few review changes * Change TODO for a note * Workaround for lazily resolved field value types in Examine * Make removal tracking more granular to cover both single and all culture deletion --------- Co-authored-by: Elitsa --- .../Filters/ContentTypeFilterIndexer.cs | 6 +- .../Selectors/AncestorsSelectorIndexer.cs | 6 +- .../Selectors/ChildrenSelectorIndexer.cs | 6 +- .../Selectors/DescendantsSelectorIndexer.cs | 12 +- .../Indexing/Sorts/CreateDateSortIndexer.cs | 6 +- .../Indexing/Sorts/LevelSortIndexer.cs | 6 +- .../Indexing/Sorts/NameSortIndexer.cs | 6 +- .../Indexing/Sorts/SortOrderSortIndexer.cs | 6 +- .../Indexing/Sorts/UpdateDateSortIndexer.cs | 13 +- .../Querying/Filters/ContentTypeFilter.cs | 21 +-- .../Querying/Filters/NameFilter.cs | 21 +-- .../Querying/Selectors/AncestorsSelector.cs | 8 +- .../Querying/Selectors/ChildrenSelector.cs | 9 +- .../Querying/Selectors/DescendantsSelector.cs | 9 +- .../Querying/Sorts/CreateDateSort.cs | 3 +- .../Querying/Sorts/LevelSort.cs | 3 +- .../Querying/Sorts/NameSort.cs | 3 +- .../Querying/Sorts/SortOrderSort.cs | 3 +- .../Querying/Sorts/UpdateDateSort.cs | 3 +- .../Services/ApiContentQueryService.cs | 84 +++++++--- src/Umbraco.Core/DeliveryApi/FieldType.cs | 3 +- src/Umbraco.Core/DeliveryApi/FilterOption.cs | 4 +- .../DeliveryApi/IContentIndexHandler.cs | 3 +- src/Umbraco.Core/DeliveryApi/IndexField.cs | 2 + .../DeliveryApi/IndexFieldValue.cs | 2 +- .../DeliveryApi/SelectorOption.cs | 2 +- src/Umbraco.Core/DeliveryApi/SortOption.cs | 4 +- .../DeliveryApiContentIndex.cs | 105 ++++++++++++- .../ConfigureIndexOptions.cs | 3 +- .../UmbracoContentIndex.cs | 46 +++++- .../UmbracoContentIndexBase.cs | 63 -------- .../UmbracoBuilder.Examine.cs | 5 + .../DeliveryApiContentIndexDeferredBase.cs | 23 +++ ...veryApiContentIndexHandleContentChanges.cs | 137 ++++++++++++++++ ...ApiContentIndexHandleContentTypeChanges.cs | 145 +++++++++++++++++ .../Examine/DeferredActions.cs | 39 +++++ ...ryApiContentIndexFieldDefinitionBuilder.cs | 91 +++++++---- .../Examine/DeliveryApiContentIndexHelper.cs | 69 ++++++++ .../DeliveryApiContentIndexPopulator.cs | 68 ++------ .../DeliveryApiContentIndexUtilites.cs | 8 + .../DeliveryApiContentIndexValueSetBuilder.cs | 104 +++++++++---- .../Examine/DeliveryApiIndexingHandler.cs | 110 +++++++++++++ .../Examine/ExamineIndexingMainDomHandler.cs | 51 ++++++ .../Examine/ExamineUmbracoIndexingHandler.cs | 147 ++++-------------- .../Examine/IDeferredAction.cs | 6 + .../Examine/IDeliveryApiContentIndexHelper.cs | 8 + .../Search/IDeliveryApiIndexingHandler.cs | 27 ++++ ...IndexingNotificationHandler.DeliveryApi.cs | 74 +++++++++ .../settings/examinemanagement.controller.js | 10 +- 49 files changed, 1179 insertions(+), 414 deletions(-) delete mode 100644 src/Umbraco.Examine.Lucene/UmbracoContentIndexBase.cs create mode 100644 src/Umbraco.Infrastructure/Examine/Deferred/DeliveryApiContentIndexDeferredBase.cs create mode 100644 src/Umbraco.Infrastructure/Examine/Deferred/DeliveryApiContentIndexHandleContentChanges.cs create mode 100644 src/Umbraco.Infrastructure/Examine/Deferred/DeliveryApiContentIndexHandleContentTypeChanges.cs create mode 100644 src/Umbraco.Infrastructure/Examine/DeferredActions.cs create mode 100644 src/Umbraco.Infrastructure/Examine/DeliveryApiContentIndexHelper.cs create mode 100644 src/Umbraco.Infrastructure/Examine/DeliveryApiContentIndexUtilites.cs create mode 100644 src/Umbraco.Infrastructure/Examine/DeliveryApiIndexingHandler.cs create mode 100644 src/Umbraco.Infrastructure/Examine/ExamineIndexingMainDomHandler.cs create mode 100644 src/Umbraco.Infrastructure/Examine/IDeferredAction.cs create mode 100644 src/Umbraco.Infrastructure/Examine/IDeliveryApiContentIndexHelper.cs create mode 100644 src/Umbraco.Infrastructure/Search/IDeliveryApiIndexingHandler.cs create mode 100644 src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.DeliveryApi.cs diff --git a/src/Umbraco.Cms.Api.Delivery/Indexing/Filters/ContentTypeFilterIndexer.cs b/src/Umbraco.Cms.Api.Delivery/Indexing/Filters/ContentTypeFilterIndexer.cs index 60f5e6a9ed..e80762403e 100644 --- a/src/Umbraco.Cms.Api.Delivery/Indexing/Filters/ContentTypeFilterIndexer.cs +++ b/src/Umbraco.Cms.Api.Delivery/Indexing/Filters/ContentTypeFilterIndexer.cs @@ -7,9 +7,9 @@ public sealed class ContentTypeFilterIndexer : IContentIndexHandler { internal const string FieldName = "contentType"; - public IEnumerable GetFieldValues(IContent content) - => new[] { new IndexFieldValue { FieldName = FieldName, Value = content.ContentType.Alias } }; + public IEnumerable GetFieldValues(IContent content, string? culture) + => new[] { new IndexFieldValue { FieldName = FieldName, Values = new object[] { content.ContentType.Alias } } }; public IEnumerable GetFields() - => new[] { new IndexField { FieldName = FieldName, FieldType = FieldType.String } }; + => new[] { new IndexField { FieldName = FieldName, FieldType = FieldType.StringRaw, VariesByCulture = false } }; } diff --git a/src/Umbraco.Cms.Api.Delivery/Indexing/Selectors/AncestorsSelectorIndexer.cs b/src/Umbraco.Cms.Api.Delivery/Indexing/Selectors/AncestorsSelectorIndexer.cs index d6b8b86c8a..e70faeab67 100644 --- a/src/Umbraco.Cms.Api.Delivery/Indexing/Selectors/AncestorsSelectorIndexer.cs +++ b/src/Umbraco.Cms.Api.Delivery/Indexing/Selectors/AncestorsSelectorIndexer.cs @@ -8,10 +8,10 @@ public sealed class AncestorsSelectorIndexer : IContentIndexHandler // NOTE: "id" is a reserved field name internal const string FieldName = "itemId"; - public IEnumerable GetFieldValues(IContent content) - => new[] { new IndexFieldValue { FieldName = FieldName, Value = content.Key } }; + public IEnumerable GetFieldValues(IContent content, string? culture) + => new[] { new IndexFieldValue { FieldName = FieldName, Values = new object[] { content.Key } } }; public IEnumerable GetFields() - => new[] { new IndexField { FieldName = FieldName, FieldType = FieldType.String } }; + => new[] { new IndexField { FieldName = FieldName, FieldType = FieldType.StringRaw, VariesByCulture = false } }; } diff --git a/src/Umbraco.Cms.Api.Delivery/Indexing/Selectors/ChildrenSelectorIndexer.cs b/src/Umbraco.Cms.Api.Delivery/Indexing/Selectors/ChildrenSelectorIndexer.cs index 9ff352a940..b1ee5b5447 100644 --- a/src/Umbraco.Cms.Api.Delivery/Indexing/Selectors/ChildrenSelectorIndexer.cs +++ b/src/Umbraco.Cms.Api.Delivery/Indexing/Selectors/ChildrenSelectorIndexer.cs @@ -14,7 +14,7 @@ public sealed class ChildrenSelectorIndexer : IContentIndexHandler internal const string FieldName = "parentId"; - public IEnumerable GetFieldValues(IContent content) + public IEnumerable GetFieldValues(IContent content, string? culture) { Guid parentKey = Guid.Empty; if (content.ParentId > 0) @@ -26,10 +26,10 @@ public sealed class ChildrenSelectorIndexer : IContentIndexHandler yield return new IndexFieldValue { FieldName = FieldName, - Value = parentKey + Values = new object[] { parentKey } }; } public IEnumerable GetFields() - => new[] { new IndexField { FieldName = FieldName, FieldType = FieldType.String } }; + => new[] { new IndexField { FieldName = FieldName, FieldType = FieldType.StringRaw, VariesByCulture = false } }; } diff --git a/src/Umbraco.Cms.Api.Delivery/Indexing/Selectors/DescendantsSelectorIndexer.cs b/src/Umbraco.Cms.Api.Delivery/Indexing/Selectors/DescendantsSelectorIndexer.cs index e61b75639d..4500ec44fb 100644 --- a/src/Umbraco.Cms.Api.Delivery/Indexing/Selectors/DescendantsSelectorIndexer.cs +++ b/src/Umbraco.Cms.Api.Delivery/Indexing/Selectors/DescendantsSelectorIndexer.cs @@ -15,22 +15,22 @@ public sealed class DescendantsSelectorIndexer : IContentIndexHandler public DescendantsSelectorIndexer(IEntityService entityService) => _entityService = entityService; - public IEnumerable GetFieldValues(IContent content) + public IEnumerable GetFieldValues(IContent content, string? culture) { - Guid[] ancestorKeys = content.GetAncestorIds()?.Select(id => + var ancestorKeys = content.GetAncestorIds()?.Select(id => { Attempt getKeyAttempt = _entityService.GetKey(id, UmbracoObjectTypes.Document); - return getKeyAttempt.Success ? getKeyAttempt.Result : Guid.Empty; - }).ToArray() ?? Array.Empty(); + return getKeyAttempt.Success ? getKeyAttempt.Result : (object)Guid.Empty; + }).ToArray() ?? Array.Empty(); yield return new IndexFieldValue { FieldName = FieldName, - Value = string.Join(" ", ancestorKeys) // TODO: investigate if search executes faster if we store this as an array + Values = ancestorKeys }; } public IEnumerable GetFields() - => new[] { new IndexField { FieldName = FieldName, FieldType = FieldType.String } }; + => new[] { new IndexField { FieldName = FieldName, FieldType = FieldType.StringRaw, VariesByCulture = false } }; } diff --git a/src/Umbraco.Cms.Api.Delivery/Indexing/Sorts/CreateDateSortIndexer.cs b/src/Umbraco.Cms.Api.Delivery/Indexing/Sorts/CreateDateSortIndexer.cs index 2d16ccb66e..11312416cd 100644 --- a/src/Umbraco.Cms.Api.Delivery/Indexing/Sorts/CreateDateSortIndexer.cs +++ b/src/Umbraco.Cms.Api.Delivery/Indexing/Sorts/CreateDateSortIndexer.cs @@ -7,9 +7,9 @@ public sealed class CreateDateSortIndexer : IContentIndexHandler { internal const string FieldName = "createDate"; - public IEnumerable GetFieldValues(IContent content) - => new[] { new IndexFieldValue { FieldName = FieldName, Value = content.CreateDate } }; + public IEnumerable GetFieldValues(IContent content, string? culture) + => new[] { new IndexFieldValue { FieldName = FieldName, Values = new object[] { content.CreateDate } } }; public IEnumerable GetFields() - => new[] { new IndexField { FieldName = FieldName, FieldType = FieldType.Date } }; + => new[] { new IndexField { FieldName = FieldName, FieldType = FieldType.Date, VariesByCulture = false } }; } diff --git a/src/Umbraco.Cms.Api.Delivery/Indexing/Sorts/LevelSortIndexer.cs b/src/Umbraco.Cms.Api.Delivery/Indexing/Sorts/LevelSortIndexer.cs index 89f3f3f0d1..03c549058e 100644 --- a/src/Umbraco.Cms.Api.Delivery/Indexing/Sorts/LevelSortIndexer.cs +++ b/src/Umbraco.Cms.Api.Delivery/Indexing/Sorts/LevelSortIndexer.cs @@ -7,9 +7,9 @@ public sealed class LevelSortIndexer : IContentIndexHandler { internal const string FieldName = "level"; - public IEnumerable GetFieldValues(IContent content) - => new[] { new IndexFieldValue { FieldName = FieldName, Value = content.Level } }; + public IEnumerable GetFieldValues(IContent content, string? culture) + => new[] { new IndexFieldValue { FieldName = FieldName, Values = new object[] { content.Level } } }; public IEnumerable GetFields() - => new[] { new IndexField { FieldName = FieldName, FieldType = FieldType.Number } }; + => new[] { new IndexField { FieldName = FieldName, FieldType = FieldType.Number, VariesByCulture = false } }; } diff --git a/src/Umbraco.Cms.Api.Delivery/Indexing/Sorts/NameSortIndexer.cs b/src/Umbraco.Cms.Api.Delivery/Indexing/Sorts/NameSortIndexer.cs index b61e01aa7e..5f5334cd05 100644 --- a/src/Umbraco.Cms.Api.Delivery/Indexing/Sorts/NameSortIndexer.cs +++ b/src/Umbraco.Cms.Api.Delivery/Indexing/Sorts/NameSortIndexer.cs @@ -7,9 +7,9 @@ public sealed class NameSortIndexer : IContentIndexHandler { internal const string FieldName = "name"; - public IEnumerable GetFieldValues(IContent content) - => new[] { new IndexFieldValue { FieldName = FieldName, Value = content.Name ?? string.Empty } }; + public IEnumerable GetFieldValues(IContent content, string? culture) + => new[] { new IndexFieldValue { FieldName = FieldName, Values = new object[] { content.GetCultureName(culture) ?? string.Empty } } }; public IEnumerable GetFields() - => new[] { new IndexField { FieldName = FieldName, FieldType = FieldType.StringSortable } }; + => new[] { new IndexField { FieldName = FieldName, FieldType = FieldType.StringSortable, VariesByCulture = true } }; } diff --git a/src/Umbraco.Cms.Api.Delivery/Indexing/Sorts/SortOrderSortIndexer.cs b/src/Umbraco.Cms.Api.Delivery/Indexing/Sorts/SortOrderSortIndexer.cs index 70d0669bda..85b928e871 100644 --- a/src/Umbraco.Cms.Api.Delivery/Indexing/Sorts/SortOrderSortIndexer.cs +++ b/src/Umbraco.Cms.Api.Delivery/Indexing/Sorts/SortOrderSortIndexer.cs @@ -7,9 +7,9 @@ public sealed class SortOrderSortIndexer : IContentIndexHandler { internal const string FieldName = "sortOrder"; - public IEnumerable GetFieldValues(IContent content) - => new[] { new IndexFieldValue { FieldName = FieldName, Value = content.SortOrder } }; + public IEnumerable GetFieldValues(IContent content, string? culture) + => new[] { new IndexFieldValue { FieldName = FieldName, Values = new object[] { content.SortOrder } } }; public IEnumerable GetFields() - => new[] { new IndexField { FieldName = FieldName, FieldType = FieldType.Number } }; + => new[] { new IndexField { FieldName = FieldName, FieldType = FieldType.Number, VariesByCulture = false } }; } diff --git a/src/Umbraco.Cms.Api.Delivery/Indexing/Sorts/UpdateDateSortIndexer.cs b/src/Umbraco.Cms.Api.Delivery/Indexing/Sorts/UpdateDateSortIndexer.cs index 073543cd5f..949a066de1 100644 --- a/src/Umbraco.Cms.Api.Delivery/Indexing/Sorts/UpdateDateSortIndexer.cs +++ b/src/Umbraco.Cms.Api.Delivery/Indexing/Sorts/UpdateDateSortIndexer.cs @@ -7,9 +7,16 @@ public sealed class UpdateDateSortIndexer : IContentIndexHandler { internal const string FieldName = "updateDate"; - public IEnumerable GetFieldValues(IContent content) - => new[] { new IndexFieldValue { FieldName = FieldName, Value = content.UpdateDate } }; + public IEnumerable GetFieldValues(IContent content, string? culture) + => new[] + { + new IndexFieldValue + { + FieldName = FieldName, + Values = new object[] { (culture is not null ? content.GetUpdateDate(culture) : null) ?? content.UpdateDate } + } + }; public IEnumerable GetFields() - => new[] { new IndexField { FieldName = FieldName, FieldType = FieldType.Date } }; + => new[] { new IndexField { FieldName = FieldName, FieldType = FieldType.Date, VariesByCulture = true } }; } diff --git a/src/Umbraco.Cms.Api.Delivery/Querying/Filters/ContentTypeFilter.cs b/src/Umbraco.Cms.Api.Delivery/Querying/Filters/ContentTypeFilter.cs index db9f4c3385..e851158b87 100644 --- a/src/Umbraco.Cms.Api.Delivery/Querying/Filters/ContentTypeFilter.cs +++ b/src/Umbraco.Cms.Api.Delivery/Querying/Filters/ContentTypeFilter.cs @@ -16,24 +16,13 @@ public sealed class ContentTypeFilter : IFilterHandler { var alias = filter.Substring(ContentTypeSpecifier.Length); - var filterOption = new FilterOption + return new FilterOption { FieldName = ContentTypeFilterIndexer.FieldName, - Value = string.Empty + Values = new[] { alias.TrimStart('!') }, + Operator = alias.StartsWith('!') + ? FilterOperation.IsNot + : FilterOperation.Is }; - - // Support negation - if (alias.StartsWith('!')) - { - filterOption.Value = alias.Substring(1); - filterOption.Operator = FilterOperation.IsNot; - } - else - { - filterOption.Value = alias; - filterOption.Operator = FilterOperation.Is; - } - - return filterOption; } } diff --git a/src/Umbraco.Cms.Api.Delivery/Querying/Filters/NameFilter.cs b/src/Umbraco.Cms.Api.Delivery/Querying/Filters/NameFilter.cs index f909a771d1..64aa5b2776 100644 --- a/src/Umbraco.Cms.Api.Delivery/Querying/Filters/NameFilter.cs +++ b/src/Umbraco.Cms.Api.Delivery/Querying/Filters/NameFilter.cs @@ -16,24 +16,13 @@ public sealed class NameFilter : IFilterHandler { var value = filter.Substring(NameSpecifier.Length); - var filterOption = new FilterOption + return new FilterOption { FieldName = NameSortIndexer.FieldName, - Value = string.Empty + Values = new[] { value.TrimStart('!') }, + Operator = value.StartsWith('!') + ? FilterOperation.IsNot + : FilterOperation.Is }; - - // Support negation - if (value.StartsWith('!')) - { - filterOption.Value = value.Substring(1); - filterOption.Operator = FilterOperation.IsNot; - } - else - { - filterOption.Value = value; - filterOption.Operator = FilterOperation.Is; - } - - return filterOption; } } diff --git a/src/Umbraco.Cms.Api.Delivery/Querying/Selectors/AncestorsSelector.cs b/src/Umbraco.Cms.Api.Delivery/Querying/Selectors/AncestorsSelector.cs index 542567bf0d..67490e38f4 100644 --- a/src/Umbraco.Cms.Api.Delivery/Querying/Selectors/AncestorsSelector.cs +++ b/src/Umbraco.Cms.Api.Delivery/Querying/Selectors/AncestorsSelector.cs @@ -22,7 +22,7 @@ public sealed class AncestorsSelector : QueryOptionBase, ISelectorHandler /// public SelectorOption BuildSelectorOption(string selector) { - var fieldValue = selector.Substring(AncestorsSpecifier.Length); + var fieldValue = selector[AncestorsSpecifier.Length..]; Guid? id = GetGuidFromQuery(fieldValue); if (id is null || @@ -35,18 +35,18 @@ public sealed class AncestorsSelector : QueryOptionBase, ISelectorHandler return new SelectorOption { FieldName = AncestorsSelectorIndexer.FieldName, - Value = string.Empty + Values = Array.Empty() }; } // With the previous check we made sure that if we reach this, we already made sure that there is a valid content item IPublishedContent contentItem = publishedSnapshot.Content.GetById((Guid)id)!; // so it can't be null - IEnumerable ancestorKeys = contentItem.Ancestors().Select(a => a.Key); + var ancestorKeys = contentItem.Ancestors().Select(a => a.Key.ToString("D")).ToArray(); return new SelectorOption { FieldName = AncestorsSelectorIndexer.FieldName, - Value = string.Join(" ", ancestorKeys) + Values = ancestorKeys }; } } diff --git a/src/Umbraco.Cms.Api.Delivery/Querying/Selectors/ChildrenSelector.cs b/src/Umbraco.Cms.Api.Delivery/Querying/Selectors/ChildrenSelector.cs index 58e965d5d2..838b5da776 100644 --- a/src/Umbraco.Cms.Api.Delivery/Querying/Selectors/ChildrenSelector.cs +++ b/src/Umbraco.Cms.Api.Delivery/Querying/Selectors/ChildrenSelector.cs @@ -1,6 +1,7 @@ using Umbraco.Cms.Api.Delivery.Indexing.Selectors; using Umbraco.Cms.Core.DeliveryApi; using Umbraco.Cms.Core.PublishedCache; +using Umbraco.Extensions; namespace Umbraco.Cms.Api.Delivery.Querying.Selectors; @@ -20,13 +21,15 @@ public sealed class ChildrenSelector : QueryOptionBase, ISelectorHandler /// public SelectorOption BuildSelectorOption(string selector) { - var fieldValue = selector.Substring(ChildrenSpecifier.Length); - Guid? id = GetGuidFromQuery(fieldValue); + var fieldValue = selector[ChildrenSpecifier.Length..]; + var id = GetGuidFromQuery(fieldValue)?.ToString("D"); return new SelectorOption { FieldName = ChildrenSelectorIndexer.FieldName, - Value = id.ToString() ?? string.Empty + Values = id.IsNullOrWhiteSpace() == false + ? new[] { id } + : Array.Empty() }; } } diff --git a/src/Umbraco.Cms.Api.Delivery/Querying/Selectors/DescendantsSelector.cs b/src/Umbraco.Cms.Api.Delivery/Querying/Selectors/DescendantsSelector.cs index c2c84dfb3e..e3c9bf33fd 100644 --- a/src/Umbraco.Cms.Api.Delivery/Querying/Selectors/DescendantsSelector.cs +++ b/src/Umbraco.Cms.Api.Delivery/Querying/Selectors/DescendantsSelector.cs @@ -1,6 +1,7 @@ using Umbraco.Cms.Api.Delivery.Indexing.Selectors; using Umbraco.Cms.Core.DeliveryApi; using Umbraco.Cms.Core.PublishedCache; +using Umbraco.Extensions; namespace Umbraco.Cms.Api.Delivery.Querying.Selectors; @@ -20,13 +21,15 @@ public sealed class DescendantsSelector : QueryOptionBase, ISelectorHandler /// public SelectorOption BuildSelectorOption(string selector) { - var fieldValue = selector.Substring(DescendantsSpecifier.Length); - Guid? id = GetGuidFromQuery(fieldValue); + var fieldValue = selector[DescendantsSpecifier.Length..]; + var id = GetGuidFromQuery(fieldValue)?.ToString("D"); return new SelectorOption { FieldName = DescendantsSelectorIndexer.FieldName, - Value = id.ToString() ?? string.Empty + Values = id.IsNullOrWhiteSpace() == false + ? new[] { id } + : Array.Empty() }; } } diff --git a/src/Umbraco.Cms.Api.Delivery/Querying/Sorts/CreateDateSort.cs b/src/Umbraco.Cms.Api.Delivery/Querying/Sorts/CreateDateSort.cs index f04b060810..1f25918aec 100644 --- a/src/Umbraco.Cms.Api.Delivery/Querying/Sorts/CreateDateSort.cs +++ b/src/Umbraco.Cms.Api.Delivery/Querying/Sorts/CreateDateSort.cs @@ -20,8 +20,7 @@ public sealed class CreateDateSort : ISortHandler return new SortOption { FieldName = CreateDateSortIndexer.FieldName, - Direction = sortDirection.StartsWith("asc") ? Direction.Ascending : Direction.Descending, - FieldType = FieldType.Date + Direction = sortDirection.StartsWith("asc") ? Direction.Ascending : Direction.Descending }; } } diff --git a/src/Umbraco.Cms.Api.Delivery/Querying/Sorts/LevelSort.cs b/src/Umbraco.Cms.Api.Delivery/Querying/Sorts/LevelSort.cs index 8b8ce4fe24..d510c7564c 100644 --- a/src/Umbraco.Cms.Api.Delivery/Querying/Sorts/LevelSort.cs +++ b/src/Umbraco.Cms.Api.Delivery/Querying/Sorts/LevelSort.cs @@ -20,8 +20,7 @@ public sealed class LevelSort : ISortHandler return new SortOption { FieldName = LevelSortIndexer.FieldName, - Direction = sortDirection.StartsWith("asc") ? Direction.Ascending : Direction.Descending, - FieldType = FieldType.Number + Direction = sortDirection.StartsWith("asc") ? Direction.Ascending : Direction.Descending }; } } diff --git a/src/Umbraco.Cms.Api.Delivery/Querying/Sorts/NameSort.cs b/src/Umbraco.Cms.Api.Delivery/Querying/Sorts/NameSort.cs index 88ba8a688a..2ec4ab3ef6 100644 --- a/src/Umbraco.Cms.Api.Delivery/Querying/Sorts/NameSort.cs +++ b/src/Umbraco.Cms.Api.Delivery/Querying/Sorts/NameSort.cs @@ -20,8 +20,7 @@ public sealed class NameSort : ISortHandler return new SortOption { FieldName = NameSortIndexer.FieldName, - Direction = sortDirection.StartsWith("asc") ? Direction.Ascending : Direction.Descending, - FieldType = FieldType.String + Direction = sortDirection.StartsWith("asc") ? Direction.Ascending : Direction.Descending }; } } diff --git a/src/Umbraco.Cms.Api.Delivery/Querying/Sorts/SortOrderSort.cs b/src/Umbraco.Cms.Api.Delivery/Querying/Sorts/SortOrderSort.cs index 3a0dd503ca..5dbf9257f2 100644 --- a/src/Umbraco.Cms.Api.Delivery/Querying/Sorts/SortOrderSort.cs +++ b/src/Umbraco.Cms.Api.Delivery/Querying/Sorts/SortOrderSort.cs @@ -20,8 +20,7 @@ public sealed class SortOrderSort : ISortHandler return new SortOption { FieldName = SortOrderSortIndexer.FieldName, - Direction = sortDirection.StartsWith("asc") ? Direction.Ascending : Direction.Descending, - FieldType = FieldType.Number + Direction = sortDirection.StartsWith("asc") ? Direction.Ascending : Direction.Descending }; } } diff --git a/src/Umbraco.Cms.Api.Delivery/Querying/Sorts/UpdateDateSort.cs b/src/Umbraco.Cms.Api.Delivery/Querying/Sorts/UpdateDateSort.cs index dd5c8262fd..06beb1e6c0 100644 --- a/src/Umbraco.Cms.Api.Delivery/Querying/Sorts/UpdateDateSort.cs +++ b/src/Umbraco.Cms.Api.Delivery/Querying/Sorts/UpdateDateSort.cs @@ -20,8 +20,7 @@ public sealed class UpdateDateSort : ISortHandler return new SortOption { FieldName = UpdateDateSortIndexer.FieldName, - Direction = sortDirection.StartsWith("asc") ? Direction.Ascending : Direction.Descending, - FieldType = FieldType.Date + Direction = sortDirection.StartsWith("asc") ? Direction.Ascending : Direction.Descending }; } } diff --git a/src/Umbraco.Cms.Api.Delivery/Services/ApiContentQueryService.cs b/src/Umbraco.Cms.Api.Delivery/Services/ApiContentQueryService.cs index efff5ef6b7..404de7a6f2 100644 --- a/src/Umbraco.Cms.Api.Delivery/Services/ApiContentQueryService.cs +++ b/src/Umbraco.Cms.Api.Delivery/Services/ApiContentQueryService.cs @@ -1,16 +1,18 @@ using Examine; using Examine.Search; +using Microsoft.Extensions.Logging; using Umbraco.Cms.Api.Delivery.Indexing.Selectors; using Umbraco.Cms.Core; using Umbraco.Cms.Core.DeliveryApi; using Umbraco.Cms.Core.Models.PublishedContent; using Umbraco.Cms.Core.Services.OperationStatus; using Umbraco.Cms.Infrastructure.Examine; +using Umbraco.Extensions; using Umbraco.New.Cms.Core.Models; namespace Umbraco.Cms.Api.Delivery.Services; -internal sealed class ApiContentQueryService : IApiContentQueryService // Examine-specific implementation - can be swapped out +internal sealed class ApiContentQueryService : IApiContentQueryService { private const string ItemIdFieldName = "itemId"; private readonly IExamineManager _examineManager; @@ -18,24 +20,38 @@ internal sealed class ApiContentQueryService : IApiContentQueryService // Examin private readonly SelectorHandlerCollection _selectorHandlers; private readonly FilterHandlerCollection _filterHandlers; private readonly SortHandlerCollection _sortHandlers; + private readonly IVariationContextAccessor _variationContextAccessor; + private readonly ILogger _logger; private readonly string _fallbackGuidValue; + private readonly Dictionary _fieldTypes; public ApiContentQueryService( IExamineManager examineManager, IRequestStartItemProviderAccessor requestStartItemProviderAccessor, SelectorHandlerCollection selectorHandlers, FilterHandlerCollection filterHandlers, - SortHandlerCollection sortHandlers) + SortHandlerCollection sortHandlers, + ContentIndexHandlerCollection indexHandlers, + ILogger logger, + IVariationContextAccessor variationContextAccessor) { _examineManager = examineManager; _requestStartItemProviderAccessor = requestStartItemProviderAccessor; _selectorHandlers = selectorHandlers; _filterHandlers = filterHandlers; _sortHandlers = sortHandlers; + _variationContextAccessor = variationContextAccessor; + _logger = logger; // A fallback value is needed for Examine queries in case we don't have a value - we can't pass null or empty string // It is set to a random guid since this would be highly unlikely to yield any results _fallbackGuidValue = Guid.NewGuid().ToString("D"); + + // build a look-up dictionary of field types by field name + _fieldTypes = indexHandlers + .SelectMany(handler => handler.GetFields()) + .DistinctBy(field => field.FieldName) + .ToDictionary(field => field.FieldName, field => field.FieldType, StringComparer.InvariantCultureIgnoreCase); } /// @@ -59,6 +75,10 @@ internal sealed class ApiContentQueryService : IApiContentQueryService // Examin return Attempt.FailWithStatus(ApiContentQueryOperationStatus.SelectorOptionNotFound, emptyResult); } + // Item culture must be either the requested culture or "none" + var culture = CurrentCulture(); + queryOperation.And().GroupedOr(new[] { "culture" }, culture.ToLowerInvariant().IfNullOrWhiteSpace(_fallbackGuidValue), "none"); + // Handle Filtering var canApplyFiltering = CanHandleFiltering(filters, queryOperation); @@ -98,7 +118,7 @@ internal sealed class ApiContentQueryService : IApiContentQueryService // Examin private IBooleanOperation? HandleSelector(string? fetch, IQuery baseQuery) { string? fieldName = null; - string? fieldValue = null; + string[] fieldValues = Array.Empty(); if (fetch is not null) { @@ -111,9 +131,9 @@ internal sealed class ApiContentQueryService : IApiContentQueryService // Examin } fieldName = selector.FieldName; - fieldValue = string.IsNullOrWhiteSpace(selector.Value) == false - ? selector.Value - : _fallbackGuidValue; + fieldValues = selector.Values.Any() + ? selector.Values + : new[] { _fallbackGuidValue }; } // Take into account the "start-item" header if present, as it defines a starting root node to query from @@ -124,19 +144,33 @@ internal sealed class ApiContentQueryService : IApiContentQueryService // Examin { // Reusing the boolean operation of the "Descendants" selector, as we want to get all the nodes from the given starting point fieldName = DescendantsSelectorIndexer.FieldName; - fieldValue = startItem.Key.ToString(); + fieldValues = new [] { startItem.Key.ToString() }; } } // If no params or no fetch value, get everything from the index - this is a way to do that with Examine fieldName ??= UmbracoExamineFieldNames.CategoryFieldName; - fieldValue ??= "content"; + fieldValues = fieldValues.Any() ? fieldValues : new [] { "content" }; - return baseQuery.Field(fieldName, fieldValue); + return fieldValues.Length == 1 + ? baseQuery.Field(fieldName, fieldValues.First()) + : baseQuery.GroupedOr(new[] { fieldName }, fieldValues); } private bool CanHandleFiltering(IEnumerable filters, IBooleanOperation queryOperation) { + void HandleExact(IQuery query, string fieldName, string[] values) + { + if (values.Length == 1) + { + query.Field(fieldName, values[0]); + } + else + { + query.GroupedOr(new[] { fieldName }, values); + } + } + foreach (var filterValue in filters) { IFilterHandler? filterHandler = _filterHandlers.FirstOrDefault(h => h.CanHandle(filterValue)); @@ -147,21 +181,19 @@ internal sealed class ApiContentQueryService : IApiContentQueryService // Examin return false; } - var value = string.IsNullOrWhiteSpace(filter.Value) == false - ? filter.Value - : _fallbackGuidValue; + var values = filter.Values.Any() + ? filter.Values + : new[] { _fallbackGuidValue }; switch (filter.Operator) { case FilterOperation.Is: - queryOperation.And().Field(filter.FieldName, - (IExamineValue)new ExamineValue(Examineness.Explicit, - value)); // TODO: doesn't work for explicit word(s) match + // TODO: test this for explicit word matching + HandleExact(queryOperation.And(), filter.FieldName, values); break; case FilterOperation.IsNot: - queryOperation.Not().Field(filter.FieldName, - (IExamineValue)new ExamineValue(Examineness.Explicit, - value)); // TODO: doesn't work for explicit word(s) match + // TODO: test this for explicit word matching + HandleExact(queryOperation.Not(), filter.FieldName, values); break; // TODO: Fix case FilterOperation.Contains: @@ -191,13 +223,20 @@ internal sealed class ApiContentQueryService : IApiContentQueryService // Examin return null; } - SortType sortType = sort.FieldType switch + if (_fieldTypes.TryGetValue(sort.FieldName, out FieldType fieldType) is false) + { + _logger.LogWarning("Sort implementation for field name {FieldName} does not match an index handler implementation, cannot resolve field type.", sort.FieldName); + continue; + } + + SortType sortType = fieldType switch { FieldType.Number => SortType.Int, FieldType.Date => SortType.Long, - FieldType.String => SortType.String, + FieldType.StringRaw => SortType.String, + FieldType.StringAnalyzed => SortType.String, FieldType.StringSortable => SortType.String, - _ => throw new ArgumentOutOfRangeException(nameof(sort.FieldType)) + _ => throw new ArgumentOutOfRangeException(nameof(fieldType)) }; orderingQuery = sort.Direction switch @@ -211,4 +250,7 @@ internal sealed class ApiContentQueryService : IApiContentQueryService // Examin // Keep the index sorting as default return orderingQuery ?? queryCriteria.OrderBy(); } + + private string CurrentCulture() + => _variationContextAccessor.VariationContext?.Culture ?? string.Empty; } diff --git a/src/Umbraco.Core/DeliveryApi/FieldType.cs b/src/Umbraco.Core/DeliveryApi/FieldType.cs index d7a77043e5..5bd7df8298 100644 --- a/src/Umbraco.Core/DeliveryApi/FieldType.cs +++ b/src/Umbraco.Core/DeliveryApi/FieldType.cs @@ -2,7 +2,8 @@ public enum FieldType { - String, + StringRaw, + StringAnalyzed, StringSortable, Number, Date diff --git a/src/Umbraco.Core/DeliveryApi/FilterOption.cs b/src/Umbraco.Core/DeliveryApi/FilterOption.cs index 5bc3ef0df4..037a3a52eb 100644 --- a/src/Umbraco.Core/DeliveryApi/FilterOption.cs +++ b/src/Umbraco.Core/DeliveryApi/FilterOption.cs @@ -4,7 +4,7 @@ public sealed class FilterOption { public required string FieldName { get; set; } - public required string Value { get; set; } + public required string[] Values { get; set; } - public FilterOperation Operator { get; set; } + public required FilterOperation Operator { get; set; } } diff --git a/src/Umbraco.Core/DeliveryApi/IContentIndexHandler.cs b/src/Umbraco.Core/DeliveryApi/IContentIndexHandler.cs index 0cccaf093e..0f894f5501 100644 --- a/src/Umbraco.Core/DeliveryApi/IContentIndexHandler.cs +++ b/src/Umbraco.Core/DeliveryApi/IContentIndexHandler.cs @@ -12,8 +12,9 @@ public interface IContentIndexHandler : IDiscoverable /// Calculates the field values for a given content item. /// /// The content item. + /// The culture to retrieve the field values for (null if the content does not vary by culture). /// The values to add to the index. - IEnumerable GetFieldValues(IContent content); + IEnumerable GetFieldValues(IContent content, string? culture); /// /// Returns the field definitions required to support the field values in the index. diff --git a/src/Umbraco.Core/DeliveryApi/IndexField.cs b/src/Umbraco.Core/DeliveryApi/IndexField.cs index 2df9005131..61092500c6 100644 --- a/src/Umbraco.Core/DeliveryApi/IndexField.cs +++ b/src/Umbraco.Core/DeliveryApi/IndexField.cs @@ -5,4 +5,6 @@ public sealed class IndexField public required string FieldName { get; set; } public required FieldType FieldType { get; set; } + + public required bool VariesByCulture { get; set; } } diff --git a/src/Umbraco.Core/DeliveryApi/IndexFieldValue.cs b/src/Umbraco.Core/DeliveryApi/IndexFieldValue.cs index 1e76eff4df..7bec3444db 100644 --- a/src/Umbraco.Core/DeliveryApi/IndexFieldValue.cs +++ b/src/Umbraco.Core/DeliveryApi/IndexFieldValue.cs @@ -4,5 +4,5 @@ public sealed class IndexFieldValue { public required string FieldName { get; set; } - public required object Value { get; set; } + public required IEnumerable Values { get; set; } } diff --git a/src/Umbraco.Core/DeliveryApi/SelectorOption.cs b/src/Umbraco.Core/DeliveryApi/SelectorOption.cs index 07620032ec..1626bc7b51 100644 --- a/src/Umbraco.Core/DeliveryApi/SelectorOption.cs +++ b/src/Umbraco.Core/DeliveryApi/SelectorOption.cs @@ -4,5 +4,5 @@ public sealed class SelectorOption { public required string FieldName { get; set; } - public required string Value { get; set; } + public required string[] Values { get; set; } } diff --git a/src/Umbraco.Core/DeliveryApi/SortOption.cs b/src/Umbraco.Core/DeliveryApi/SortOption.cs index 81670b5641..3bab63b4a1 100644 --- a/src/Umbraco.Core/DeliveryApi/SortOption.cs +++ b/src/Umbraco.Core/DeliveryApi/SortOption.cs @@ -4,7 +4,5 @@ public sealed class SortOption { public required string FieldName { get; set; } - public Direction Direction { get; set; } - - public FieldType FieldType { get; set; } + public required Direction Direction { get; set; } } diff --git a/src/Umbraco.Examine.Lucene/DeliveryApiContentIndex.cs b/src/Umbraco.Examine.Lucene/DeliveryApiContentIndex.cs index 3a7f31f32c..a7a7e9a7d8 100644 --- a/src/Umbraco.Examine.Lucene/DeliveryApiContentIndex.cs +++ b/src/Umbraco.Examine.Lucene/DeliveryApiContentIndex.cs @@ -2,13 +2,17 @@ using Examine; using Examine.Lucene; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using Umbraco.Cms.Core; using Umbraco.Cms.Core.Hosting; using Umbraco.Cms.Core.Services; +using Umbraco.Extensions; namespace Umbraco.Cms.Infrastructure.Examine; -public class DeliveryApiContentIndex : UmbracoContentIndexBase +public class DeliveryApiContentIndex : UmbracoExamineIndex { + private readonly ILogger _logger; + public DeliveryApiContentIndex( ILoggerFactory loggerFactory, string name, @@ -18,7 +22,104 @@ public class DeliveryApiContentIndex : UmbracoContentIndexBase : base(loggerFactory, name, indexOptions, hostingEnvironment, runtimeState) { PublishedValuesOnly = true; - EnableDefaultEventHandler = true; + EnableDefaultEventHandler = false; + + _logger = loggerFactory.CreateLogger(); + + // so... Examine lazily resolves the field value types, and incidentally this currently only happens at indexing time. + // however, we really must have the correct value types at boot time, so we'll forcefully resolve the value types here. + // this is, in other words, a workaround. + if (FieldValueTypeCollection.ValueTypes.Any() is false) + { + // we should never ever get here + _logger.LogError("No value types defined for the delivery API content index"); + } + } + + /// + /// + /// Deletes a node from the index. + /// + /// + /// When a content node is deleted, we also need to delete it's children from the index so we need to perform a + /// custom Lucene search to find all decendents and create Delete item queues for them too. + /// + /// ID of the node to delete + /// + protected override void PerformDeleteFromIndex(IEnumerable itemIds, Action? onComplete) + { + var removedIndexIds = new List(); + var removedContentIds = new List(); + foreach (var itemId in itemIds) + { + // if this item was already removed as a descendant of a previously removed item, skip it + if (removedIndexIds.Contains(itemId)) + { + continue; + } + + // an item ID passed to this method can be a composite of content ID and culture (like "1234|da-DK") or simply a content ID + // - when it's a composite ID, only the supplied culture of the given item should be deleted from the index + // - when it's an content ID, all cultures of the of the given item should be deleted from the index + var (contentId, culture) = ParseItemId(itemId); + if (contentId is null) + { + _logger.LogWarning("Could not parse item ID; expected integer or composite ID, got: {itemId}", itemId); + continue; + } + + // if this item was already removed as a descendant of a previously removed item (for all cultures), skip it + if (culture is null && removedContentIds.Contains(contentId)) + { + continue; + } + + // find descendants-or-self based on path and optional culture + var rawQuery = $"({UmbracoExamineFieldNames.IndexPathFieldName}:\\-1*,{contentId} OR {UmbracoExamineFieldNames.IndexPathFieldName}:\\-1*,{contentId},*)"; + if (culture is not null) + { + rawQuery = $"{rawQuery} AND culture:{culture}"; + } + + ISearchResults results = Searcher + .CreateQuery() + .NativeQuery(rawQuery) + // NOTE: we need to be explicit about fetching ItemIdFieldName here, otherwise Examine will try to be + // clever and use the "id" field of the document (which we can't use for deletion) + .SelectField(UmbracoExamineFieldNames.ItemIdFieldName) + .Execute(); + + _logger.LogDebug("DeleteFromIndex with query: {Query} (found {TotalItems} results)", rawQuery, results.TotalItemCount); + + // grab the index IDs from the index (the composite IDs) + var indexIds = results.Select(x => x.Id).ToList(); + + // remember which items we removed, so we can skip those later + removedIndexIds.AddRange(indexIds); + if (culture is null) + { + removedContentIds.AddRange(indexIds.Select(indexId => ParseItemId(indexId).ContentId).WhereNotNull()); + } + + // delete the resulting items from the index + base.PerformDeleteFromIndex(indexIds, null); + } + } + + private (string? ContentId, string? Culture) ParseItemId(string id) + { + if (int.TryParse(id, out _)) + { + return (id, null); + } + + var parts = id.Split(Constants.CharArrays.VerticalTab); + if (parts.Length == 2 && int.TryParse(parts[0], out _)) + { + return (parts[0], parts[1]); + } + + return (null, null); } protected override void OnTransformingIndexValues(IndexingItemEventArgs e) diff --git a/src/Umbraco.Examine.Lucene/DependencyInjection/ConfigureIndexOptions.cs b/src/Umbraco.Examine.Lucene/DependencyInjection/ConfigureIndexOptions.cs index 6dc6917aa6..f75d9c5889 100644 --- a/src/Umbraco.Examine.Lucene/DependencyInjection/ConfigureIndexOptions.cs +++ b/src/Umbraco.Examine.Lucene/DependencyInjection/ConfigureIndexOptions.cs @@ -59,8 +59,7 @@ public sealed class ConfigureIndexOptions : IConfigureNamedOptions /// An indexer for Umbraco content and media /// -public class UmbracoContentIndex : UmbracoContentIndexBase, IUmbracoContentIndex +public class UmbracoContentIndex : UmbracoExamineIndex, IUmbracoContentIndex { + private readonly ISet _idOnlyFieldSet = new HashSet { "id" }; + private readonly ILogger _logger; + public UmbracoContentIndex( ILoggerFactory loggerFactory, string name, @@ -25,6 +29,7 @@ public class UmbracoContentIndex : UmbracoContentIndexBase, IUmbracoContentIndex : base(loggerFactory, name, indexOptions, hostingEnvironment, runtimeState) { LanguageService = languageService; + _logger = loggerFactory.CreateLogger(); LuceneDirectoryIndexOptions namedOptions = indexOptions.Get(name); if (namedOptions == null) @@ -104,4 +109,43 @@ public class UmbracoContentIndex : UmbracoContentIndexBase, IUmbracoContentIndex onComplete(new IndexOperationEventArgs(this, 0)); } } + + /// + /// + /// Deletes a node from the index. + /// + /// + /// When a content node is deleted, we also need to delete it's children from the index so we need to perform a + /// custom Lucene search to find all decendents and create Delete item queues for them too. + /// + /// ID of the node to delete + /// + protected override void PerformDeleteFromIndex(IEnumerable itemIds, Action? onComplete) + { + var idsAsList = itemIds.ToList(); + + for (var i = 0; i < idsAsList.Count; i++) + { + var nodeId = idsAsList[i]; + + //find all descendants based on path + var descendantPath = $@"\-1\,*{nodeId}\,*"; + var rawQuery = $"{UmbracoExamineFieldNames.IndexPathFieldName}:{descendantPath}"; + IQuery? c = Searcher.CreateQuery(); + IBooleanOperation? filtered = c.NativeQuery(rawQuery); + IOrdering? selectedFields = filtered.SelectFields(_idOnlyFieldSet); + ISearchResults? results = selectedFields.Execute(); + + _logger.LogDebug("DeleteFromIndex with query: {Query} (found {TotalItems} results)", rawQuery, results.TotalItemCount); + + var toRemove = results.Select(x => x.Id).ToList(); + // delete those descendants (ensure base. is used here so we aren't calling ourselves!) + base.PerformDeleteFromIndex(toRemove, null); + + // remove any ids from our list that were part of the descendants + idsAsList.RemoveAll(x => toRemove.Contains(x)); + } + + base.PerformDeleteFromIndex(idsAsList, onComplete); + } } diff --git a/src/Umbraco.Examine.Lucene/UmbracoContentIndexBase.cs b/src/Umbraco.Examine.Lucene/UmbracoContentIndexBase.cs deleted file mode 100644 index 813d4cc8f6..0000000000 --- a/src/Umbraco.Examine.Lucene/UmbracoContentIndexBase.cs +++ /dev/null @@ -1,63 +0,0 @@ -using Examine; -using Examine.Lucene; -using Examine.Search; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using Umbraco.Cms.Core.Hosting; -using Umbraco.Cms.Core.Services; - -namespace Umbraco.Cms.Infrastructure.Examine; - -public abstract class UmbracoContentIndexBase : UmbracoExamineIndex -{ - private readonly ISet _idOnlyFieldSet = new HashSet { "id" }; - private readonly ILogger _logger; - - protected UmbracoContentIndexBase( - ILoggerFactory loggerFactory, - string name, - IOptionsMonitor indexOptions, - IHostingEnvironment hostingEnvironment, - IRuntimeState runtimeState) - : base(loggerFactory, name, indexOptions, hostingEnvironment, runtimeState) => - _logger = loggerFactory.CreateLogger(); - - /// - /// - /// Deletes a node from the index. - /// - /// - /// When a content node is deleted, we also need to delete it's children from the index so we need to perform a - /// custom Lucene search to find all decendents and create Delete item queues for them too. - /// - /// ID of the node to delete - /// - protected override void PerformDeleteFromIndex(IEnumerable itemIds, Action? onComplete) - { - var idsAsList = itemIds.ToList(); - - for (var i = 0; i < idsAsList.Count; i++) - { - var nodeId = idsAsList[i]; - - //find all descendants based on path - var descendantPath = $@"\-1\,*{nodeId}\,*"; - var rawQuery = $"{UmbracoExamineFieldNames.IndexPathFieldName}:{descendantPath}"; - IQuery? c = Searcher.CreateQuery(); - IBooleanOperation? filtered = c.NativeQuery(rawQuery); - IOrdering? selectedFields = filtered.SelectFields(_idOnlyFieldSet); - ISearchResults? results = selectedFields.Execute(); - - _logger.LogDebug("DeleteFromIndex with query: {Query} (found {TotalItems} results)", rawQuery, results.TotalItemCount); - - var toRemove = results.Select(x => x.Id).ToList(); - // delete those descendants (ensure base. is used here so we aren't calling ourselves!) - base.PerformDeleteFromIndex(toRemove, null); - - // remove any ids from our list that were part of the descendants - idsAsList.RemoveAll(x => toRemove.Contains(x)); - } - - base.PerformDeleteFromIndex(idsAsList, onComplete); - } -} diff --git a/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.Examine.cs b/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.Examine.cs index e4c35b9067..4106464602 100644 --- a/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.Examine.cs +++ b/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.Examine.cs @@ -29,6 +29,7 @@ public static partial class UmbracoBuilderExtensions builder.Services.AddSingleton(); builder.Services.AddSingleton(); + builder.Services.AddSingleton(); builder.Services.AddUnique(); builder.Services.AddUnique(); builder.Services.AddUnique(factory => @@ -51,10 +52,14 @@ public static partial class UmbracoBuilderExtensions builder.Services.AddUnique, MemberValueSetBuilder>(); builder.Services.AddUnique(); builder.Services.AddUnique(); + builder.Services.AddUnique(); + builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.AddNotificationHandler(); builder.AddNotificationHandler(); + builder.AddNotificationHandler(); + builder.AddNotificationHandler(); builder.AddNotificationHandler(); builder.AddNotificationHandler(); builder.AddNotificationHandler(); diff --git a/src/Umbraco.Infrastructure/Examine/Deferred/DeliveryApiContentIndexDeferredBase.cs b/src/Umbraco.Infrastructure/Examine/Deferred/DeliveryApiContentIndexDeferredBase.cs new file mode 100644 index 0000000000..cd9b0b85b0 --- /dev/null +++ b/src/Umbraco.Infrastructure/Examine/Deferred/DeliveryApiContentIndexDeferredBase.cs @@ -0,0 +1,23 @@ +using Examine; + +namespace Umbraco.Cms.Infrastructure.Examine.Deferred; + +internal abstract class DeliveryApiContentIndexDeferredBase +{ + protected static void RemoveFromIndex(int id, IIndex index) + => RemoveFromIndex(new[] { id }, index); + + protected static void RemoveFromIndex(IReadOnlyCollection ids, IIndex index) + => RemoveFromIndex(ids.Select(id => id.ToString()).ToArray(), index); + + protected static void RemoveFromIndex(IReadOnlyCollection ids, IIndex index) + { + if (ids.Any() is false) + { + return; + } + + // NOTE: the delivery api index implementation takes care of deleting descendants, so we don't have to do that here + index.DeleteFromIndex(ids.Select(id => id.ToString())); + } +} diff --git a/src/Umbraco.Infrastructure/Examine/Deferred/DeliveryApiContentIndexHandleContentChanges.cs b/src/Umbraco.Infrastructure/Examine/Deferred/DeliveryApiContentIndexHandleContentChanges.cs new file mode 100644 index 0000000000..1ee4a0e949 --- /dev/null +++ b/src/Umbraco.Infrastructure/Examine/Deferred/DeliveryApiContentIndexHandleContentChanges.cs @@ -0,0 +1,137 @@ +using Examine; +using Umbraco.Cms.Core.Models; +using Umbraco.Cms.Core.Services; +using Umbraco.Cms.Core.Services.Changes; +using Umbraco.Cms.Infrastructure.HostedServices; +using Umbraco.Extensions; + +namespace Umbraco.Cms.Infrastructure.Examine.Deferred; + +internal sealed class DeliveryApiContentIndexHandleContentChanges : DeliveryApiContentIndexDeferredBase, IDeferredAction +{ + private readonly IList> _changes; + private readonly IContentService _contentService; + private readonly DeliveryApiIndexingHandler _deliveryApiIndexingHandler; + private readonly IDeliveryApiContentIndexValueSetBuilder _deliveryApiContentIndexValueSetBuilder; + private readonly IDeliveryApiContentIndexHelper _deliveryApiContentIndexHelper; + private readonly IBackgroundTaskQueue _backgroundTaskQueue; + + public DeliveryApiContentIndexHandleContentChanges( + IList> changes, + DeliveryApiIndexingHandler deliveryApiIndexingHandler, + IContentService contentService, + IDeliveryApiContentIndexValueSetBuilder deliveryApiContentIndexValueSetBuilder, + IDeliveryApiContentIndexHelper deliveryApiContentIndexHelper, + IBackgroundTaskQueue backgroundTaskQueue) + { + _changes = changes; + _deliveryApiIndexingHandler = deliveryApiIndexingHandler; + _contentService = contentService; + _backgroundTaskQueue = backgroundTaskQueue; + _deliveryApiContentIndexValueSetBuilder = deliveryApiContentIndexValueSetBuilder; + _deliveryApiContentIndexHelper = deliveryApiContentIndexHelper; + } + + public void Execute() => _backgroundTaskQueue.QueueBackgroundWorkItem(_ => + { + IIndex index = _deliveryApiIndexingHandler.GetIndex() + ?? throw new InvalidOperationException("Could not obtain the delivery API content index"); + + var pendingRemovals = new List(); + foreach ((int contentId, TreeChangeTypes changeTypes) in _changes) + { + var remove = changeTypes.HasType(TreeChangeTypes.Remove); + var reindex = changeTypes.HasType(TreeChangeTypes.RefreshNode) || changeTypes.HasType(TreeChangeTypes.RefreshBranch); + + if (remove) + { + pendingRemovals.Add(contentId); + } + else if (reindex) + { + IContent? content = _contentService.GetById(contentId); + if (content == null || content.Trashed) + { + pendingRemovals.Add(contentId); + continue; + } + + RemoveFromIndex(pendingRemovals, index); + pendingRemovals.Clear(); + + Reindex(content, index); + } + } + + RemoveFromIndex(pendingRemovals, index); + + return Task.CompletedTask; + }); + + private void Reindex(IContent content, IIndex index) + { + // get the currently indexed cultures for the content + var existingIndexCultures = index + .Searcher + .CreateQuery() + .Field("id", content.Id) + .SelectField("culture") + .Execute() + .SelectMany(f => f.GetValues("culture")) + .ToArray(); + + // index the content + var indexedCultures = UpdateIndex(content, index); + if (indexedCultures.Any() is false) + { + // we likely got here because unpublishing triggered a "refresh branch" notification, now we + // need to delete every last culture of this content and all descendants + RemoveFromIndex(content.Id, index); + return; + } + + // if any of the content cultures did not exist in the index before, nor will any of its published descendants + // in those cultures be at this point, so make sure those are added as well + if (indexedCultures.Except(existingIndexCultures).Any()) + { + ReindexDescendants(content, index); + } + + // ensure that any unpublished cultures are removed from the index + var unpublishedCultures = existingIndexCultures.Except(indexedCultures).ToArray(); + if (unpublishedCultures.Any() is false) + { + return; + } + + var idsToDelete = unpublishedCultures + .Select(culture => DeliveryApiContentIndexUtilites.IndexId(content, culture)).ToArray(); + RemoveFromIndex(idsToDelete, index); + } + + private string[] UpdateIndex(IContent content, IIndex index) + { + ValueSet[] valueSets = _deliveryApiContentIndexValueSetBuilder.GetValueSets(content).ToArray(); + if (valueSets.Any() is false) + { + return Array.Empty(); + } + + index.IndexItems(valueSets); + return valueSets + .SelectMany(v => v.GetValues("culture").Select(c => c.ToString())) + .WhereNotNull() + .ToArray(); + } + + private void ReindexDescendants(IContent content, IIndex index) + => _deliveryApiContentIndexHelper.EnumerateApplicableDescendantsForContentIndex( + content.Id, + descendants => + { + foreach (IContent descendant in descendants) + { + UpdateIndex(descendant, index); + } + }); +} diff --git a/src/Umbraco.Infrastructure/Examine/Deferred/DeliveryApiContentIndexHandleContentTypeChanges.cs b/src/Umbraco.Infrastructure/Examine/Deferred/DeliveryApiContentIndexHandleContentTypeChanges.cs new file mode 100644 index 0000000000..92cafbe670 --- /dev/null +++ b/src/Umbraco.Infrastructure/Examine/Deferred/DeliveryApiContentIndexHandleContentTypeChanges.cs @@ -0,0 +1,145 @@ +using Examine; +using Examine.Search; +using Umbraco.Cms.Core; +using Umbraco.Cms.Core.Models; +using Umbraco.Cms.Core.Services; +using Umbraco.Cms.Core.Services.Changes; +using Umbraco.Cms.Infrastructure.HostedServices; +using Umbraco.Extensions; + +namespace Umbraco.Cms.Infrastructure.Examine.Deferred; + +internal sealed class DeliveryApiContentIndexHandleContentTypeChanges : DeliveryApiContentIndexDeferredBase, IDeferredAction +{ + private const int PageSize = 500; + + private readonly IList> _changes; + private readonly DeliveryApiIndexingHandler _deliveryApiIndexingHandler; + private readonly IDeliveryApiContentIndexValueSetBuilder _deliveryApiContentIndexValueSetBuilder; + private readonly IContentService _contentService; + private readonly IBackgroundTaskQueue _backgroundTaskQueue; + + public DeliveryApiContentIndexHandleContentTypeChanges( + IList> changes, + DeliveryApiIndexingHandler deliveryApiIndexingHandler, + IDeliveryApiContentIndexValueSetBuilder deliveryApiContentIndexValueSetBuilder, + IContentService contentService, + IBackgroundTaskQueue backgroundTaskQueue) + { + _changes = changes; + _deliveryApiIndexingHandler = deliveryApiIndexingHandler; + _deliveryApiContentIndexValueSetBuilder = deliveryApiContentIndexValueSetBuilder; + _contentService = contentService; + _backgroundTaskQueue = backgroundTaskQueue; + } + + public void Execute() => _backgroundTaskQueue.QueueBackgroundWorkItem(_ => + { + var updatedContentTypeIds = new List(); + + // this looks a bit cumbersome, but we must iterate the changes in turn because the order matter; i.e. if a + // content type is first changed, then deleted, we should not attempt to apply content type changes + // NOTE: clean-up after content type deletion is performed by individual content cache refresh notifications for all deleted items + foreach (KeyValuePair change in _changes) + { + if (change.Value.HasType(ContentTypeChangeTypes.Remove)) + { + updatedContentTypeIds.Remove(change.Key); + } + else if (change.Value.HasType(ContentTypeChangeTypes.RefreshMain)) + { + updatedContentTypeIds.Add(change.Key); + } + } + + if (updatedContentTypeIds.Any() is false) + { + return Task.CompletedTask; + } + + IIndex index = _deliveryApiIndexingHandler.GetIndex() ?? + throw new InvalidOperationException("Could not obtain the delivery API content index"); + + HandleUpdatedContentTypes(updatedContentTypeIds, index); + + return Task.CompletedTask; + }); + + private void HandleUpdatedContentTypes(IEnumerable updatedContentTypesIds, IIndex index) + { + foreach (var contentTypeId in updatedContentTypesIds) + { + List indexIds = FindIdsForContentType(contentTypeId, index); + + // the index can contain multiple documents per content (for culture variant content). when reindexing below, + // all documents are created "in one go", so we don't need to index the same document multiple times. + // however, we need to keep track of the mapping between content IDs and their current (composite) index + // IDs, since the index IDs can change here (if the content type culture variance is changed), and thus + // we may have to clean up the current documents after reindexing. + var indexIdsByContentIds = indexIds + .Select(id => + { + var parts = id.Split(Constants.CharArrays.VerticalTab); + return parts.Length == 2 && int.TryParse(parts[0], out var contentId) + ? (ContentId: contentId, IndexId: id) + : throw new InvalidOperationException($"Delivery API identifier should be composite of ID and culture, got: {id}"); + }) + .GroupBy(tuple => tuple.ContentId) + .ToDictionary( + group => group.Key, + group => group.Select(t => t.IndexId).ToArray()); + + // keep track of the IDs of the documents that must be removed, so we can remove them all in one go + var indexIdsToRemove = new List(); + + foreach (KeyValuePair indexIdsByContentId in indexIdsByContentIds) + { + IContent? content = _contentService.GetById(indexIdsByContentId.Key); + if (content == null) + { + // this should not happen if the rest of the indexing works as intended, but for good measure + // let's make sure we clean up all documents if the content does not exist + indexIdsToRemove.AddRange(indexIdsByContentId.Value); + continue; + } + + // reindex the documents for this content + ValueSet[] valueSets = _deliveryApiContentIndexValueSetBuilder.GetValueSets(content).ToArray(); + if (valueSets.Any()) + { + index.IndexItems(valueSets); + } + + // if any of the document IDs have changed, make sure we clean up the previous ones + indexIdsToRemove.AddRange(indexIdsByContentId.Value.Except(valueSets.Select(set => set.Id))); + } + + RemoveFromIndex(indexIdsToRemove, index); + } + } + + private List FindIdsForContentType(int contentTypeId, IIndex index) + { + var ids = new List(); + + var page = 0; + var total = long.MaxValue; + while (page * PageSize < total) + { + ISearchResults? results = index.Searcher + .CreateQuery() + .Field("contentTypeId", contentTypeId) + // NOTE: we need to be explicit about fetching ItemIdFieldName here, otherwise Examine will try to be + // clever and use the "id" field of the document (which we can't use for deletion) + .SelectField(UmbracoExamineFieldNames.ItemIdFieldName) + .Execute(QueryOptions.SkipTake(page * PageSize, PageSize)); + total = results.TotalItemCount; + + ids.AddRange(results.Select(result => result.Id)); + + page++; + } + + return ids; + } +} diff --git a/src/Umbraco.Infrastructure/Examine/DeferredActions.cs b/src/Umbraco.Infrastructure/Examine/DeferredActions.cs new file mode 100644 index 0000000000..6ef8d7d34d --- /dev/null +++ b/src/Umbraco.Infrastructure/Examine/DeferredActions.cs @@ -0,0 +1,39 @@ +using Umbraco.Cms.Core.Scoping; + +namespace Umbraco.Cms.Infrastructure.Examine; + +internal class DeferredActions +{ + // the default enlist priority is 100 + // enlist with a lower priority to ensure that anything "default" runs after us + // but greater that SafeXmlReaderWriter priority which is 60 + private const int EnlistPriority = 80; + + private readonly List _actions = new(); + + public static DeferredActions? Get(ICoreScopeProvider scopeProvider) + { + IScopeContext? scopeContext = scopeProvider.Context; + + return scopeContext?.Enlist("examineEvents", + () => new DeferredActions(), // creator + (completed, actions) => // action + { + if (completed) + { + actions?.Execute(); + } + }, + EnlistPriority); + } + + public void Add(IDeferredAction action) => _actions.Add(action); + + private void Execute() + { + foreach (IDeferredAction action in _actions) + { + action.Execute(); + } + } +} diff --git a/src/Umbraco.Infrastructure/Examine/DeliveryApiContentIndexFieldDefinitionBuilder.cs b/src/Umbraco.Infrastructure/Examine/DeliveryApiContentIndexFieldDefinitionBuilder.cs index e4bd1710f9..ce20716251 100644 --- a/src/Umbraco.Infrastructure/Examine/DeliveryApiContentIndexFieldDefinitionBuilder.cs +++ b/src/Umbraco.Infrastructure/Examine/DeliveryApiContentIndexFieldDefinitionBuilder.cs @@ -1,4 +1,5 @@ using Examine; +using Microsoft.Extensions.Logging; using Umbraco.Cms.Core.DeliveryApi; using Umbraco.Extensions; @@ -6,43 +7,71 @@ namespace Umbraco.Cms.Infrastructure.Examine; internal sealed class DeliveryApiContentIndexFieldDefinitionBuilder : IDeliveryApiContentIndexFieldDefinitionBuilder { - private readonly ContentIndexHandlerCollection _contentIndexHandlerCollection; + private readonly ContentIndexHandlerCollection _indexHandlers; + private readonly ILogger _logger; - public DeliveryApiContentIndexFieldDefinitionBuilder(ContentIndexHandlerCollection contentIndexHandlerCollection) - => _contentIndexHandlerCollection = contentIndexHandlerCollection; + public DeliveryApiContentIndexFieldDefinitionBuilder( + ContentIndexHandlerCollection indexHandlers, + ILogger logger) + { + _indexHandlers = indexHandlers; + _logger = logger; + } public FieldDefinitionCollection Build() { - // mandatory field definitions go here - // see also the field definitions in the Delivery API content index value set builder - var fieldDefinitions = new List - { - new("id", FieldDefinitionTypes.Integer), - new(UmbracoExamineFieldNames.IndexPathFieldName, FieldDefinitionTypes.Raw), - new(UmbracoExamineFieldNames.NodeNameFieldName, FieldDefinitionTypes.Raw) - }; + var fieldDefinitions = new List(); - // add custom fields from index handlers (selectors, filters, sorts) - IndexField[] fields = _contentIndexHandlerCollection - .SelectMany(handler => handler.GetFields()) - .Where(field => fieldDefinitions.Any(fieldDefinition => fieldDefinition.Name.InvariantEquals(field.FieldName)) is false) - .DistinctBy(field => field.FieldName, StringComparer.OrdinalIgnoreCase) - .ToArray(); - fieldDefinitions.AddRange( - fields.Select(field => - { - var type = field.FieldType switch - { - FieldType.Date => FieldDefinitionTypes.DateTime, - FieldType.Number => FieldDefinitionTypes.Integer, - FieldType.String => FieldDefinitionTypes.FullText, - FieldType.StringSortable => FieldDefinitionTypes.FullTextSortable, - _ => throw new ArgumentOutOfRangeException(nameof(field.FieldType)) - }; - - return new FieldDefinition(field.FieldName, type); - })); + AddRequiredFieldDefinitions(fieldDefinitions); + AddContentIndexHandlerFieldDefinitions(fieldDefinitions); return new FieldDefinitionCollection(fieldDefinitions.ToArray()); } + + // required field definitions go here + // see also the field definitions in the Delivery API content index value set builder + private void AddRequiredFieldDefinitions(ICollection fieldDefinitions) + { + fieldDefinitions.Add(new("id", FieldDefinitionTypes.Integer)); + fieldDefinitions.Add(new("contentTypeId", FieldDefinitionTypes.Integer)); + fieldDefinitions.Add(new("culture", FieldDefinitionTypes.Raw)); + fieldDefinitions.Add(new(UmbracoExamineFieldNames.IndexPathFieldName, FieldDefinitionTypes.Raw)); + fieldDefinitions.Add(new(UmbracoExamineFieldNames.NodeNameFieldName, FieldDefinitionTypes.Raw)); + } + + private void AddContentIndexHandlerFieldDefinitions(ICollection fieldDefinitions) + { + // add index fields from index handlers (selectors, filters, sorts) + foreach (IContentIndexHandler handler in _indexHandlers) + { + IndexField[] fields = handler.GetFields().ToArray(); + + foreach (IndexField field in fields) + { + if (fieldDefinitions.Any(fieldDefinition => fieldDefinition.Name.InvariantEquals(field.FieldName))) + { + _logger.LogWarning("Duplicate field definitions found for field name {FieldName} among the index handlers - first one wins.", field.FieldName); + continue; + } + + FieldDefinition fieldDefinition = CreateFieldDefinition(field); + fieldDefinitions.Add(fieldDefinition); + } + } + } + + private static FieldDefinition CreateFieldDefinition(IndexField field) + { + var indexType = field.FieldType switch + { + FieldType.Date => FieldDefinitionTypes.DateTime, + FieldType.Number => FieldDefinitionTypes.Integer, + FieldType.StringRaw => FieldDefinitionTypes.Raw, + FieldType.StringAnalyzed => FieldDefinitionTypes.FullText, + FieldType.StringSortable => FieldDefinitionTypes.FullTextSortable, + _ => throw new ArgumentOutOfRangeException(nameof(field.FieldType)) + }; + + return new FieldDefinition(field.FieldName, indexType); + } } diff --git a/src/Umbraco.Infrastructure/Examine/DeliveryApiContentIndexHelper.cs b/src/Umbraco.Infrastructure/Examine/DeliveryApiContentIndexHelper.cs new file mode 100644 index 0000000000..34897bd7af --- /dev/null +++ b/src/Umbraco.Infrastructure/Examine/DeliveryApiContentIndexHelper.cs @@ -0,0 +1,69 @@ +using Microsoft.Extensions.Options; +using Umbraco.Cms.Core.Configuration.Models; +using Umbraco.Cms.Core.Models; +using Umbraco.Cms.Core.Persistence.Querying; +using Umbraco.Cms.Core.Services; +using Umbraco.Cms.Infrastructure.Persistence; +using Umbraco.Extensions; + +namespace Umbraco.Cms.Infrastructure.Examine; + +internal sealed class DeliveryApiContentIndexHelper : IDeliveryApiContentIndexHelper +{ + private readonly IContentService _contentService; + private readonly IUmbracoDatabaseFactory _umbracoDatabaseFactory; + private DeliveryApiSettings _deliveryApiSettings; + + public DeliveryApiContentIndexHelper( + IContentService contentService, + IUmbracoDatabaseFactory umbracoDatabaseFactory, + IOptionsMonitor deliveryApiSettings) + { + _contentService = contentService; + _umbracoDatabaseFactory = umbracoDatabaseFactory; + _deliveryApiSettings = deliveryApiSettings.CurrentValue; + deliveryApiSettings.OnChange(settings => _deliveryApiSettings = settings); + } + + public void EnumerateApplicableDescendantsForContentIndex(int rootContentId, Action actionToPerform) + { + const int pageSize = 10000; + var pageIndex = 0; + var publishedContentIds = new HashSet { rootContentId }; + + IContent[] descendants; + IQuery publishedQuery = _umbracoDatabaseFactory.SqlContext.Query().Where(x => x.Published && x.Trashed == false); + do + { + descendants = _contentService.GetPagedDescendants(rootContentId, pageIndex, pageSize, out _, publishedQuery, Ordering.By("Path")).ToArray(); + + // there are a few rules we need to abide to when populating the index: + // - children of unpublished content can still be published; we need to filter them out, as they're not supposed to go into the index. + // - content of disallowed content types are not allowed in the index, but their children are + // as we're querying published content and ordering by path, we can construct a list of "allowed" published content IDs like this. + var allowedDescendants = new List(); + foreach (IContent descendant in descendants) + { + if (_deliveryApiSettings.IsDisallowedContentType(descendant.ContentType.Alias)) + { + // the content type is disallowed; make sure we consider all its children as candidates for the index anyway + publishedContentIds.Add(descendant.Id); + continue; + } + + // content at root level is by definition published, because we only fetch published content in the query above. + // content not at root level should be included only if their parents are included (unbroken chain of published content) + if (descendant.Level == 1 || publishedContentIds.Contains(descendant.ParentId)) + { + publishedContentIds.Add(descendant.Id); + allowedDescendants.Add(descendant); + } + } + + actionToPerform(allowedDescendants.ToArray()); + + pageIndex++; + } + while (descendants.Length == pageSize); + } +} diff --git a/src/Umbraco.Infrastructure/Examine/DeliveryApiContentIndexPopulator.cs b/src/Umbraco.Infrastructure/Examine/DeliveryApiContentIndexPopulator.cs index 47080971f8..12f833f3d0 100644 --- a/src/Umbraco.Infrastructure/Examine/DeliveryApiContentIndexPopulator.cs +++ b/src/Umbraco.Infrastructure/Examine/DeliveryApiContentIndexPopulator.cs @@ -1,33 +1,19 @@ using Examine; -using Microsoft.Extensions.Options; using Umbraco.Cms.Core; -using Umbraco.Cms.Core.Configuration.Models; -using Umbraco.Cms.Core.Models; -using Umbraco.Cms.Core.Persistence.Querying; -using Umbraco.Cms.Core.Services; -using Umbraco.Cms.Infrastructure.Persistence; -using Umbraco.Extensions; namespace Umbraco.Cms.Infrastructure.Examine; internal sealed class DeliveryApiContentIndexPopulator : IndexPopulator { - private readonly IContentService _contentService; - private readonly IUmbracoDatabaseFactory _umbracoDatabaseFactory; private readonly IDeliveryApiContentIndexValueSetBuilder _deliveryContentIndexValueSetBuilder; - private DeliveryApiSettings _deliveryApiSettings; + private readonly IDeliveryApiContentIndexHelper _deliveryApiContentIndexHelper; public DeliveryApiContentIndexPopulator( - IContentService contentService, IDeliveryApiContentIndexValueSetBuilder deliveryContentIndexValueSetBuilder, - IUmbracoDatabaseFactory umbracoDatabaseFactory, - IOptionsMonitor deliveryApiSettings) + IDeliveryApiContentIndexHelper deliveryApiContentIndexHelper) { - _contentService = contentService; _deliveryContentIndexValueSetBuilder = deliveryContentIndexValueSetBuilder; - _umbracoDatabaseFactory = umbracoDatabaseFactory; - _deliveryApiSettings = deliveryApiSettings.CurrentValue; - deliveryApiSettings.OnChange(settings => _deliveryApiSettings = settings); + _deliveryApiContentIndexHelper = deliveryApiContentIndexHelper; RegisterIndex(Constants.UmbracoIndexes.DeliveryApiContentIndexName); } @@ -38,49 +24,15 @@ internal sealed class DeliveryApiContentIndexPopulator : IndexPopulator return; } - const int pageSize = 10000; - var pageIndex = 0; - var publishedContentIds = new HashSet(); - - IContent[] descendants; - IQuery publishedQuery = _umbracoDatabaseFactory.SqlContext.Query().Where(x => x.Published && x.Trashed == false); - do - { - descendants = _contentService.GetPagedDescendants(Constants.System.Root, pageIndex, pageSize, out _, publishedQuery, Ordering.By("Path")).ToArray(); - - // there are a few rules we need to abide to when populating the index: - // - children of unpublished content can still be published; we need to filter them out, as they're not supposed to go into the index. - // - content of disallowed content types are not allowed in the index, but their children are - // as we're querying published content and ordering by path, we can construct a list of "allowed" published content IDs like this. - var allowedDescendants = new List(); - foreach (IContent content in descendants) + _deliveryApiContentIndexHelper.EnumerateApplicableDescendantsForContentIndex( + Constants.System.Root, + descendants => { - if (_deliveryApiSettings.IsDisallowedContentType(content.ContentType.Alias)) + ValueSet[] valueSets = _deliveryContentIndexValueSetBuilder.GetValueSets(descendants).ToArray(); + foreach (IIndex index in indexes) { - // the content type is disallowed; make sure we consider all its children as candidates for the index anyway - publishedContentIds.Add(content.Id); - continue; + index.IndexItems(valueSets); } - - // content at root level is by definition published, because we only fetch published content in the query above. - // content not at root level should be included only if their parents are included (unbroken chain of published content) - if (content.Level == 1 || publishedContentIds.Contains(content.ParentId)) - { - publishedContentIds.Add(content.Id); - allowedDescendants.Add(content); - } - } - - // now build the value sets based on the "allowed" published content only - ValueSet[] valueSets = _deliveryContentIndexValueSetBuilder.GetValueSets(allowedDescendants.ToArray()).ToArray(); - - foreach (IIndex index in indexes) - { - index.IndexItems(valueSets); - } - - pageIndex++; - } - while (descendants.Length == pageSize); + }); } } diff --git a/src/Umbraco.Infrastructure/Examine/DeliveryApiContentIndexUtilites.cs b/src/Umbraco.Infrastructure/Examine/DeliveryApiContentIndexUtilites.cs new file mode 100644 index 0000000000..2bfd3d6f80 --- /dev/null +++ b/src/Umbraco.Infrastructure/Examine/DeliveryApiContentIndexUtilites.cs @@ -0,0 +1,8 @@ +using Umbraco.Cms.Core.Models; + +namespace Umbraco.Cms.Infrastructure.Examine; + +internal static class DeliveryApiContentIndexUtilites +{ + public static string IndexId(IContent content, string culture) => $"{content.Id}|{culture}"; +} diff --git a/src/Umbraco.Infrastructure/Examine/DeliveryApiContentIndexValueSetBuilder.cs b/src/Umbraco.Infrastructure/Examine/DeliveryApiContentIndexValueSetBuilder.cs index 87d0bbf9fa..9c0f7bba6a 100644 --- a/src/Umbraco.Infrastructure/Examine/DeliveryApiContentIndexValueSetBuilder.cs +++ b/src/Umbraco.Infrastructure/Examine/DeliveryApiContentIndexValueSetBuilder.cs @@ -1,10 +1,10 @@ using Examine; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Umbraco.Cms.Core.Configuration.Models; using Umbraco.Cms.Core.DeliveryApi; using Umbraco.Cms.Core.Models; using Umbraco.Cms.Core.Services; -using Umbraco.Cms.Infrastructure.Scoping; using Umbraco.Extensions; namespace Umbraco.Cms.Infrastructure.Examine; @@ -12,19 +12,22 @@ namespace Umbraco.Cms.Infrastructure.Examine; internal sealed class DeliveryApiContentIndexValueSetBuilder : IDeliveryApiContentIndexValueSetBuilder { private readonly ContentIndexHandlerCollection _contentIndexHandlerCollection; - private readonly IScopeProvider _scopeProvider; + private readonly IContentService _contentService; private readonly IPublicAccessService _publicAccessService; + private readonly ILogger _logger; private DeliveryApiSettings _deliveryApiSettings; public DeliveryApiContentIndexValueSetBuilder( ContentIndexHandlerCollection contentIndexHandlerCollection, - IScopeProvider scopeProvider, + IContentService contentService, IPublicAccessService publicAccessService, + ILogger logger, IOptionsMonitor deliveryApiSettings) { _contentIndexHandlerCollection = contentIndexHandlerCollection; - _scopeProvider = scopeProvider; _publicAccessService = publicAccessService; + _logger = logger; + _contentService = contentService; _deliveryApiSettings = deliveryApiSettings.CurrentValue; deliveryApiSettings.OnChange(settings => _deliveryApiSettings = settings); } @@ -34,27 +37,79 @@ internal sealed class DeliveryApiContentIndexValueSetBuilder : IDeliveryApiConte { foreach (IContent content in contents.Where(CanIndex)) { - // mandatory index values go here - var indexValues = new Dictionary - { - ["id"] = content.Id, // required for unpublishing/deletion handling - [UmbracoExamineFieldNames.IndexPathFieldName] = content.Path, // required for unpublishing/deletion handling - [UmbracoExamineFieldNames.NodeNameFieldName] = content.PublishName ?? string.Empty, // primarily needed for backoffice index browsing - }; + var cultures = IndexableCultures(content); - // add custom field values from index handlers (selectors, filters, sorts) - IndexFieldValue[] fieldValues = _contentIndexHandlerCollection - .SelectMany(handler => handler.GetFieldValues(content)) - .DistinctBy(fieldValue => fieldValue.FieldName, StringComparer.OrdinalIgnoreCase) - .Where(fieldValue => indexValues.ContainsKeyIgnoreCase(fieldValue.FieldName) is false) - .ToArray(); - foreach (IndexFieldValue fieldValue in fieldValues) + foreach (var culture in cultures) { - indexValues[fieldValue.FieldName] = fieldValue.Value; + var indexCulture = culture ?? "none"; + + // required index values go here + var indexValues = new Dictionary>(StringComparer.InvariantCultureIgnoreCase) + { + ["id"] = new object[] { content.Id }, // required for correct publishing handling and also needed for backoffice index browsing + ["contentTypeId"] = new object[] { content.ContentTypeId }, // required for correct content type change handling + ["culture"] = new object[] { indexCulture }, // required for culture variant querying + [UmbracoExamineFieldNames.IndexPathFieldName] = new object[] { content.Path }, // required for unpublishing/deletion handling + [UmbracoExamineFieldNames.NodeNameFieldName] = new object[] { content.GetPublishName(culture) ?? string.Empty }, // primarily needed for backoffice index browsing + }; + + AddContentIndexHandlerFields(content, culture, indexValues); + + yield return new ValueSet(DeliveryApiContentIndexUtilites.IndexId(content, indexCulture), IndexTypes.Content, content.ContentType.Alias, indexValues); + } + } + } + + private string?[] IndexableCultures(IContent content) + { + var variesByCulture = content.ContentType.VariesByCulture(); + + // if the content varies by culture, the indexable cultures are the published + // cultures - otherwise "null" represents "no culture" + var cultures = variesByCulture + ? content.PublishedCultures.ToArray() + : new string?[] { null }; + + // now iterate all ancestors and make sure all cultures are published all the way up the tree + foreach (var ancestorId in content.GetAncestorIds() ?? Array.Empty()) + { + IContent? ancestor = _contentService.GetById(ancestorId); + if (ancestor is null || ancestor.Published is false) + { + // no published ancestor => don't index anything + cultures = Array.Empty(); + } + else if (variesByCulture && ancestor.ContentType.VariesByCulture()) + { + // both the content and the ancestor are culture variant => only index the published cultures they have in common + cultures = cultures.Intersect(ancestor.PublishedCultures).ToArray(); } - // NOTE: must use content.Id here, not content.Key - otherwise automatic clean-up i.e. on deletion or unpublishing will not work - yield return new ValueSet(content.Id.ToString(), IndexTypes.Content, content.ContentType.Alias, indexValues); + // if we've already run out of cultures to index, there is no reason to iterate the ancestors any further + if (cultures.Any() == false) + { + break; + } + } + + return cultures; + } + + private void AddContentIndexHandlerFields(IContent content, string? culture, Dictionary> indexValues) + { + foreach (IContentIndexHandler handler in _contentIndexHandlerCollection) + { + IndexFieldValue[] fieldValues = handler.GetFieldValues(content, culture).ToArray(); + foreach (IndexFieldValue fieldValue in fieldValues) + { + if (indexValues.ContainsKey(fieldValue.FieldName)) + { + _logger.LogWarning("Duplicate field value found for field name {FieldName} among the index handlers - first one wins.", fieldValue.FieldName); + continue; + } + + indexValues[fieldValue.FieldName] = fieldValue.Values.ToArray(); + } } } @@ -73,12 +128,9 @@ internal sealed class DeliveryApiContentIndexValueSetBuilder : IDeliveryApiConte } // is the content protected? - using (_scopeProvider.CreateScope(autoComplete: true)) + if (_publicAccessService.IsProtected(content.Path).Success) { - if (_publicAccessService.IsProtected(content.Path).Success) - { - return false; - } + return false; } return true; diff --git a/src/Umbraco.Infrastructure/Examine/DeliveryApiIndexingHandler.cs b/src/Umbraco.Infrastructure/Examine/DeliveryApiIndexingHandler.cs new file mode 100644 index 0000000000..a8654192fc --- /dev/null +++ b/src/Umbraco.Infrastructure/Examine/DeliveryApiIndexingHandler.cs @@ -0,0 +1,110 @@ +using Examine; +using Microsoft.Extensions.Logging; +using Umbraco.Cms.Core; +using Umbraco.Cms.Core.Scoping; +using Umbraco.Cms.Core.Services; +using Umbraco.Cms.Core.Services.Changes; +using Umbraco.Cms.Infrastructure.Examine.Deferred; +using Umbraco.Cms.Infrastructure.HostedServices; +using Umbraco.Cms.Infrastructure.Search; + +namespace Umbraco.Cms.Infrastructure.Examine; + +internal sealed class DeliveryApiIndexingHandler : IDeliveryApiIndexingHandler +{ + // these are the dependencies for this handler + private readonly ExamineIndexingMainDomHandler _mainDomHandler; + private readonly IExamineManager _examineManager; + private readonly ICoreScopeProvider _scopeProvider; + private readonly ILogger _logger; + private readonly Lazy _enabled; + + // these dependencies are for the deferred handling (we don't want those handlers registered in the DI) + private readonly IContentService _contentService; + private readonly IDeliveryApiContentIndexValueSetBuilder _deliveryApiContentIndexValueSetBuilder; + private readonly IDeliveryApiContentIndexHelper _deliveryApiContentIndexHelper; + private readonly IBackgroundTaskQueue _backgroundTaskQueue; + + public DeliveryApiIndexingHandler( + ExamineIndexingMainDomHandler mainDomHandler, + IExamineManager examineManager, + ICoreScopeProvider scopeProvider, + ILogger logger, + IContentService contentService, + IDeliveryApiContentIndexValueSetBuilder deliveryApiContentIndexValueSetBuilder, + IDeliveryApiContentIndexHelper deliveryApiContentIndexHelper, + IBackgroundTaskQueue backgroundTaskQueue) + { + _mainDomHandler = mainDomHandler; + _examineManager = examineManager; + _scopeProvider = scopeProvider; + _logger = logger; + _contentService = contentService; + _deliveryApiContentIndexValueSetBuilder = deliveryApiContentIndexValueSetBuilder; + _deliveryApiContentIndexHelper = deliveryApiContentIndexHelper; + _backgroundTaskQueue = backgroundTaskQueue; + _enabled = new Lazy(IsEnabled); + } + + /// + public bool Enabled => _enabled.Value; + + /// + public void HandleContentChanges(IList> changes) + { + var deferred = new DeliveryApiContentIndexHandleContentChanges( + changes, + this, + _contentService, + _deliveryApiContentIndexValueSetBuilder, + _deliveryApiContentIndexHelper, + _backgroundTaskQueue); + Execute(deferred); + } + + /// + public void HandleContentTypeChanges(IList> changes) + { + var deferred = new DeliveryApiContentIndexHandleContentTypeChanges( + changes, + this, + _deliveryApiContentIndexValueSetBuilder, + _contentService, + _backgroundTaskQueue); + Execute(deferred); + } + + private void Execute(IDeferredAction action) + { + var actions = DeferredActions.Get(_scopeProvider); + if (actions != null) + { + actions.Add(action); + } + else + { + action.Execute(); + } + } + + private bool IsEnabled() + { + if (_mainDomHandler.IsMainDom() == false) + { + return false; + } + + if (GetIndex() is null) + { + _logger.LogInformation("The Delivery API content index could not be found, Examine indexing is disabled."); + return false; + } + + return true; + } + + internal IIndex? GetIndex() + => _examineManager.TryGetIndex(Constants.UmbracoIndexes.DeliveryApiContentIndexName, out IIndex index) + ? index + : null; +} diff --git a/src/Umbraco.Infrastructure/Examine/ExamineIndexingMainDomHandler.cs b/src/Umbraco.Infrastructure/Examine/ExamineIndexingMainDomHandler.cs new file mode 100644 index 0000000000..fa4d184dbd --- /dev/null +++ b/src/Umbraco.Infrastructure/Examine/ExamineIndexingMainDomHandler.cs @@ -0,0 +1,51 @@ +using Examine; +using Microsoft.Extensions.Logging; +using Umbraco.Cms.Core.Logging; +using Umbraco.Cms.Core.Runtime; + +namespace Umbraco.Cms.Infrastructure.Examine; + +internal class ExamineIndexingMainDomHandler +{ + private readonly IMainDom _mainDom; + private readonly IProfilingLogger _profilingLogger; + private readonly IExamineManager _examineManager; + private readonly ILogger _logger; + private readonly Lazy _isMainDom; + + public ExamineIndexingMainDomHandler(IMainDom mainDom, IProfilingLogger profilingLogger, IExamineManager examineManager, ILogger logger) + { + _mainDom = mainDom; + _profilingLogger = profilingLogger; + _examineManager = examineManager; + _logger = logger; + _isMainDom = new Lazy(DetectMainDom); + } + + public bool IsMainDom() => _isMainDom.Value; + + private bool DetectMainDom() + { + //let's deal with shutting down Examine with MainDom + var examineShutdownRegistered = _mainDom.Register(release: () => + { + using (_profilingLogger.TraceDuration("Examine shutting down")) + { + _examineManager.Dispose(); + } + }); + + if (!examineShutdownRegistered) + { + _logger.LogInformation( + "Examine shutdown not registered, this AppDomain is not the MainDom, Examine will be disabled"); + + //if we could not register the shutdown examine ourselves, it means we are not maindom! in this case all of examine should be disabled! + Suspendable.ExamineEvents.SuspendIndexers(_logger); + return false; //exit, do not continue + } + + _logger.LogDebug("Examine shutdown registered with MainDom"); + return true; + } +} diff --git a/src/Umbraco.Infrastructure/Examine/ExamineUmbracoIndexingHandler.cs b/src/Umbraco.Infrastructure/Examine/ExamineUmbracoIndexingHandler.cs index 78fa0c7417..ea3727f31a 100644 --- a/src/Umbraco.Infrastructure/Examine/ExamineUmbracoIndexingHandler.cs +++ b/src/Umbraco.Infrastructure/Examine/ExamineUmbracoIndexingHandler.cs @@ -2,9 +2,7 @@ using System.Globalization; using Examine; using Examine.Search; using Microsoft.Extensions.Logging; -using Umbraco.Cms.Core.Logging; using Umbraco.Cms.Core.Models; -using Umbraco.Cms.Core.Runtime; using Umbraco.Cms.Core.Scoping; using Umbraco.Cms.Infrastructure.HostedServices; using Umbraco.Cms.Infrastructure.Search; @@ -17,27 +15,19 @@ namespace Umbraco.Cms.Infrastructure.Examine; /// internal class ExamineUmbracoIndexingHandler : IUmbracoIndexingHandler { - // the default enlist priority is 100 - // enlist with a lower priority to ensure that anything "default" runs after us - // but greater that SafeXmlReaderWriter priority which is 60 - private const int EnlistPriority = 80; private readonly IBackgroundTaskQueue _backgroundTaskQueue; private readonly IContentValueSetBuilder _contentValueSetBuilder; private readonly Lazy _enabled; private readonly IExamineManager _examineManager; private readonly ILogger _logger; - private readonly IMainDom _mainDom; private readonly IValueSetBuilder _mediaValueSetBuilder; private readonly IValueSetBuilder _memberValueSetBuilder; - private readonly IProfilingLogger _profilingLogger; private readonly IPublishedContentValueSetBuilder _publishedContentValueSetBuilder; - private readonly IDeliveryApiContentIndexValueSetBuilder _deliveryApiContentIndexValueSetBuilder; private readonly ICoreScopeProvider _scopeProvider; + private readonly ExamineIndexingMainDomHandler _mainDomHandler; public ExamineUmbracoIndexingHandler( - IMainDom mainDom, ILogger logger, - IProfilingLogger profilingLogger, ICoreScopeProvider scopeProvider, IExamineManager examineManager, IBackgroundTaskQueue backgroundTaskQueue, @@ -45,11 +35,9 @@ internal class ExamineUmbracoIndexingHandler : IUmbracoIndexingHandler IPublishedContentValueSetBuilder publishedContentValueSetBuilder, IValueSetBuilder mediaValueSetBuilder, IValueSetBuilder memberValueSetBuilder, - IDeliveryApiContentIndexValueSetBuilder deliveryApiContentIndexValueSetBuilder) + ExamineIndexingMainDomHandler mainDomHandler) { - _mainDom = mainDom; _logger = logger; - _profilingLogger = profilingLogger; _scopeProvider = scopeProvider; _examineManager = examineManager; _backgroundTaskQueue = backgroundTaskQueue; @@ -57,7 +45,7 @@ internal class ExamineUmbracoIndexingHandler : IUmbracoIndexingHandler _publishedContentValueSetBuilder = publishedContentValueSetBuilder; _mediaValueSetBuilder = mediaValueSetBuilder; _memberValueSetBuilder = memberValueSetBuilder; - _deliveryApiContentIndexValueSetBuilder = deliveryApiContentIndexValueSetBuilder; + _mainDomHandler = mainDomHandler; _enabled = new Lazy(IsEnabled); } @@ -67,70 +55,70 @@ internal class ExamineUmbracoIndexingHandler : IUmbracoIndexingHandler /// public void DeleteIndexForEntity(int entityId, bool keepIfUnpublished) { - var actions = DeferedActions.Get(_scopeProvider); + var actions = DeferredActions.Get(_scopeProvider); if (actions != null) { - actions.Add(new DeferedDeleteIndex(this, entityId, keepIfUnpublished)); + actions.Add(new DeferredDeleteIndex(this, entityId, keepIfUnpublished)); } else { - DeferedDeleteIndex.Execute(this, entityId, keepIfUnpublished); + DeferredDeleteIndex.Execute(this, entityId, keepIfUnpublished); } } /// public void DeleteIndexForEntities(IReadOnlyCollection entityIds, bool keepIfUnpublished) { - var actions = DeferedActions.Get(_scopeProvider); + var actions = DeferredActions.Get(_scopeProvider); if (actions != null) { - actions.Add(new DeferedDeleteIndex(this, entityIds, keepIfUnpublished)); + actions.Add(new DeferredDeleteIndex(this, entityIds, keepIfUnpublished)); } else { - DeferedDeleteIndex.Execute(this, entityIds, keepIfUnpublished); + DeferredDeleteIndex.Execute(this, entityIds, keepIfUnpublished); } } /// public void ReIndexForContent(IContent sender, bool isPublished) { - var actions = DeferedActions.Get(_scopeProvider); + var actions = DeferredActions.Get(_scopeProvider); if (actions != null) { - actions.Add(new DeferedReIndexForContent(_backgroundTaskQueue, this, sender, isPublished)); + actions.Add(new DeferredReIndexForContent(_backgroundTaskQueue, this, sender, isPublished)); } else { - DeferedReIndexForContent.Execute(_backgroundTaskQueue, this, sender, isPublished); + DeferredReIndexForContent.Execute(_backgroundTaskQueue, this, sender, isPublished); } } /// public void ReIndexForMedia(IMedia sender, bool isPublished) { - var actions = DeferedActions.Get(_scopeProvider); + var actions = DeferredActions.Get(_scopeProvider); if (actions != null) { - actions.Add(new DeferedReIndexForMedia(_backgroundTaskQueue, this, sender, isPublished)); + actions.Add(new DeferredReIndexForMedia(_backgroundTaskQueue, this, sender, isPublished)); } else { - DeferedReIndexForMedia.Execute(_backgroundTaskQueue, this, sender, isPublished); + DeferredReIndexForMedia.Execute(_backgroundTaskQueue, this, sender, isPublished); } } /// public void ReIndexForMember(IMember member) { - var actions = DeferedActions.Get(_scopeProvider); + var actions = DeferredActions.Get(_scopeProvider); if (actions != null) { - actions.Add(new DeferedReIndexForMember(_backgroundTaskQueue, this, member)); + actions.Add(new DeferredReIndexForMember(_backgroundTaskQueue, this, member)); } else { - DeferedReIndexForMember.Execute(_backgroundTaskQueue, this, member); + DeferredReIndexForMember.Execute(_backgroundTaskQueue, this, member); } } @@ -176,27 +164,11 @@ internal class ExamineUmbracoIndexingHandler : IUmbracoIndexingHandler /// private bool IsEnabled() { - //let's deal with shutting down Examine with MainDom - var examineShutdownRegistered = _mainDom.Register(release: () => + if (_mainDomHandler.IsMainDom() is false) { - using (_profilingLogger.TraceDuration("Examine shutting down")) - { - _examineManager.Dispose(); - } - }); - - if (!examineShutdownRegistered) - { - _logger.LogInformation( - "Examine shutdown not registered, this AppDomain is not the MainDom, Examine will be disabled"); - - //if we could not register the shutdown examine ourselves, it means we are not maindom! in this case all of examine should be disabled! - Suspendable.ExamineEvents.SuspendIndexers(_logger); - return false; //exit, do not continue + return false; } - _logger.LogDebug("Examine shutdown registered with MainDom"); - var registeredIndexers = _examineManager.Indexes.OfType().Count(x => x.EnableDefaultEventHandler); @@ -214,57 +186,17 @@ internal class ExamineUmbracoIndexingHandler : IUmbracoIndexingHandler #region Deferred Actions - private class DeferedActions - { - private readonly List _actions = new(); - - public static DeferedActions? Get(ICoreScopeProvider scopeProvider) - { - IScopeContext? scopeContext = scopeProvider.Context; - - return scopeContext?.Enlist("examineEvents", - () => new DeferedActions(), // creator - (completed, actions) => // action - { - if (completed) - { - actions?.Execute(); - } - }, EnlistPriority); - } - - public void Add(DeferedAction action) => _actions.Add(action); - - private void Execute() - { - foreach (DeferedAction action in _actions) - { - action.Execute(); - } - } - } - - /// - /// An action that will execute at the end of the Scope being completed - /// - private abstract class DeferedAction - { - public virtual void Execute() - { - } - } - /// /// Re-indexes an item on a background thread /// - private class DeferedReIndexForContent : DeferedAction + private class DeferredReIndexForContent : IDeferredAction { private readonly IBackgroundTaskQueue _backgroundTaskQueue; private readonly IContent _content; private readonly ExamineUmbracoIndexingHandler _examineUmbracoIndexingHandler; private readonly bool _isPublished; - public DeferedReIndexForContent(IBackgroundTaskQueue backgroundTaskQueue, + public DeferredReIndexForContent(IBackgroundTaskQueue backgroundTaskQueue, ExamineUmbracoIndexingHandler examineUmbracoIndexingHandler, IContent content, bool isPublished) { _backgroundTaskQueue = backgroundTaskQueue; @@ -273,7 +205,7 @@ internal class ExamineUmbracoIndexingHandler : IUmbracoIndexingHandler _isPublished = isPublished; } - public override void Execute() => + public void Execute() => Execute(_backgroundTaskQueue, _examineUmbracoIndexingHandler, _content, _isPublished); public static void Execute(IBackgroundTaskQueue backgroundTaskQueue, @@ -307,19 +239,6 @@ internal class ExamineUmbracoIndexingHandler : IUmbracoIndexingHandler index.IndexItems(valueSet); } - if (cancellationToken.IsCancellationRequested) - { - return Task.CompletedTask; - } - - if (isPublished && examineUmbracoIndexingHandler._examineManager.TryGetIndex( - Core.Constants.UmbracoIndexes.DeliveryApiContentIndexName, - out IIndex deliveryApiContentIndex)) - { - IEnumerable valueSets = examineUmbracoIndexingHandler._deliveryApiContentIndexValueSetBuilder.GetValueSets(content); - deliveryApiContentIndex.IndexItems(valueSets); - } - return Task.CompletedTask; }); } @@ -327,14 +246,14 @@ internal class ExamineUmbracoIndexingHandler : IUmbracoIndexingHandler /// /// Re-indexes an item on a background thread /// - private class DeferedReIndexForMedia : DeferedAction + private class DeferredReIndexForMedia : IDeferredAction { private readonly IBackgroundTaskQueue _backgroundTaskQueue; private readonly ExamineUmbracoIndexingHandler _examineUmbracoIndexingHandler; private readonly bool _isPublished; private readonly IMedia _media; - public DeferedReIndexForMedia(IBackgroundTaskQueue backgroundTaskQueue, + public DeferredReIndexForMedia(IBackgroundTaskQueue backgroundTaskQueue, ExamineUmbracoIndexingHandler examineUmbracoIndexingHandler, IMedia media, bool isPublished) { _backgroundTaskQueue = backgroundTaskQueue; @@ -343,7 +262,7 @@ internal class ExamineUmbracoIndexingHandler : IUmbracoIndexingHandler _isPublished = isPublished; } - public override void Execute() => + public void Execute() => Execute(_backgroundTaskQueue, _examineUmbracoIndexingHandler, _media, _isPublished); public static void Execute(IBackgroundTaskQueue backgroundTaskQueue, @@ -373,13 +292,13 @@ internal class ExamineUmbracoIndexingHandler : IUmbracoIndexingHandler /// /// Re-indexes an item on a background thread /// - private class DeferedReIndexForMember : DeferedAction + private class DeferredReIndexForMember : IDeferredAction { private readonly IBackgroundTaskQueue _backgroundTaskQueue; private readonly ExamineUmbracoIndexingHandler _examineUmbracoIndexingHandler; private readonly IMember _member; - public DeferedReIndexForMember(IBackgroundTaskQueue backgroundTaskQueue, + public DeferredReIndexForMember(IBackgroundTaskQueue backgroundTaskQueue, ExamineUmbracoIndexingHandler examineUmbracoIndexingHandler, IMember member) { _examineUmbracoIndexingHandler = examineUmbracoIndexingHandler; @@ -387,7 +306,7 @@ internal class ExamineUmbracoIndexingHandler : IUmbracoIndexingHandler _backgroundTaskQueue = backgroundTaskQueue; } - public override void Execute() => Execute(_backgroundTaskQueue, _examineUmbracoIndexingHandler, _member); + public void Execute() => Execute(_backgroundTaskQueue, _examineUmbracoIndexingHandler, _member); public static void Execute(IBackgroundTaskQueue backgroundTaskQueue, ExamineUmbracoIndexingHandler examineUmbracoIndexingHandler, IMember member) => @@ -412,14 +331,14 @@ internal class ExamineUmbracoIndexingHandler : IUmbracoIndexingHandler }); } - private class DeferedDeleteIndex : DeferedAction + private class DeferredDeleteIndex : IDeferredAction { private readonly ExamineUmbracoIndexingHandler _examineUmbracoIndexingHandler; private readonly int _id; private readonly IReadOnlyCollection? _ids; private readonly bool _keepIfUnpublished; - public DeferedDeleteIndex(ExamineUmbracoIndexingHandler examineUmbracoIndexingHandler, int id, + public DeferredDeleteIndex(ExamineUmbracoIndexingHandler examineUmbracoIndexingHandler, int id, bool keepIfUnpublished) { _examineUmbracoIndexingHandler = examineUmbracoIndexingHandler; @@ -427,7 +346,7 @@ internal class ExamineUmbracoIndexingHandler : IUmbracoIndexingHandler _keepIfUnpublished = keepIfUnpublished; } - public DeferedDeleteIndex(ExamineUmbracoIndexingHandler examineUmbracoIndexingHandler, + public DeferredDeleteIndex(ExamineUmbracoIndexingHandler examineUmbracoIndexingHandler, IReadOnlyCollection ids, bool keepIfUnpublished) { _examineUmbracoIndexingHandler = examineUmbracoIndexingHandler; @@ -435,7 +354,7 @@ internal class ExamineUmbracoIndexingHandler : IUmbracoIndexingHandler _keepIfUnpublished = keepIfUnpublished; } - public override void Execute() + public void Execute() { if (_ids is null) { diff --git a/src/Umbraco.Infrastructure/Examine/IDeferredAction.cs b/src/Umbraco.Infrastructure/Examine/IDeferredAction.cs new file mode 100644 index 0000000000..c31cbae86e --- /dev/null +++ b/src/Umbraco.Infrastructure/Examine/IDeferredAction.cs @@ -0,0 +1,6 @@ +namespace Umbraco.Cms.Infrastructure.Examine; + +internal interface IDeferredAction +{ + void Execute(); +} diff --git a/src/Umbraco.Infrastructure/Examine/IDeliveryApiContentIndexHelper.cs b/src/Umbraco.Infrastructure/Examine/IDeliveryApiContentIndexHelper.cs new file mode 100644 index 0000000000..1dd111ce0a --- /dev/null +++ b/src/Umbraco.Infrastructure/Examine/IDeliveryApiContentIndexHelper.cs @@ -0,0 +1,8 @@ +using Umbraco.Cms.Core.Models; + +namespace Umbraco.Cms.Infrastructure.Examine; + +internal interface IDeliveryApiContentIndexHelper +{ + void EnumerateApplicableDescendantsForContentIndex(int rootContentId, Action actionToPerform); +} diff --git a/src/Umbraco.Infrastructure/Search/IDeliveryApiIndexingHandler.cs b/src/Umbraco.Infrastructure/Search/IDeliveryApiIndexingHandler.cs new file mode 100644 index 0000000000..f7081572a4 --- /dev/null +++ b/src/Umbraco.Infrastructure/Search/IDeliveryApiIndexingHandler.cs @@ -0,0 +1,27 @@ +using Umbraco.Cms.Core.Services.Changes; + +namespace Umbraco.Cms.Infrastructure.Search; + +internal interface IDeliveryApiIndexingHandler +{ + /// + /// Returns true if the indexing handler is enabled + /// + /// + /// If this is false then there will be no data lookups executed to populate indexes + /// when service changes are made. + /// + bool Enabled { get; } + + /// + /// Handles index updates for content changes + /// + /// The list of changes by content ID + void HandleContentChanges(IList> changes); + + /// + /// Handles index updates for content type changes + /// + /// The list of changes by content type ID + void HandleContentTypeChanges(IList> changes); +} diff --git a/src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.DeliveryApi.cs b/src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.DeliveryApi.cs new file mode 100644 index 0000000000..b1b028cde0 --- /dev/null +++ b/src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.DeliveryApi.cs @@ -0,0 +1,74 @@ +using Umbraco.Cms.Core.Cache; +using Umbraco.Cms.Core.Events; +using Umbraco.Cms.Core.Models; +using Umbraco.Cms.Core.Notifications; +using Umbraco.Cms.Core.Services.Changes; +using Umbraco.Cms.Core.Sync; + +namespace Umbraco.Cms.Infrastructure.Search; + +internal sealed class DeliveryApiContentIndexingNotificationHandler : + INotificationHandler, INotificationHandler +{ + private readonly IDeliveryApiIndexingHandler _deliveryApiIndexingHandler; + + public DeliveryApiContentIndexingNotificationHandler(IDeliveryApiIndexingHandler deliveryApiIndexingHandler) + => _deliveryApiIndexingHandler = deliveryApiIndexingHandler; + + public void Handle(ContentCacheRefresherNotification notification) + { + if (NotificationHandlingIsDisabled()) + { + return; + } + + ContentCacheRefresher.JsonPayload[] payloads = GetNotificationPayloads(notification); + + var changesById = payloads + .Select(payload => new KeyValuePair(payload.Id, payload.ChangeTypes)) + .ToList(); + + _deliveryApiIndexingHandler.HandleContentChanges(changesById); + } + + public void Handle(ContentTypeCacheRefresherNotification notification) + { + if (NotificationHandlingIsDisabled()) + { + return; + } + + ContentTypeCacheRefresher.JsonPayload[] payloads = GetNotificationPayloads(notification); + + var contentTypeChangesById = payloads + .Where(payload => payload.ItemType == nameof(IContentType)) + .Select(payload => new KeyValuePair(payload.Id, payload.ChangeTypes)) + .ToList(); + _deliveryApiIndexingHandler.HandleContentTypeChanges(contentTypeChangesById); + } + + private bool NotificationHandlingIsDisabled() + { + if (_deliveryApiIndexingHandler.Enabled == false) + { + return true; + } + + if (Suspendable.ExamineEvents.CanIndex == false) + { + return true; + } + + return false; + } + + private T[] GetNotificationPayloads(CacheRefresherNotification notification) + { + if (notification.MessageType != MessageType.RefreshByPayload || notification.MessageObject is not T[] payloads) + { + throw new NotSupportedException(); + } + + return payloads; + } +} diff --git a/src/Umbraco.Web.UI.Client/src/views/dashboard/settings/examinemanagement.controller.js b/src/Umbraco.Web.UI.Client/src/views/dashboard/settings/examinemanagement.controller.js index 9588dea6eb..96caa4f8d4 100644 --- a/src/Umbraco.Web.UI.Client/src/views/dashboard/settings/examinemanagement.controller.js +++ b/src/Umbraco.Web.UI.Client/src/views/dashboard/settings/examinemanagement.controller.js @@ -85,11 +85,11 @@ function ExamineManagementController($http, $q, $timeout, umbRequestHelper, loca function nextSearchResultPage(pageNumber) { search(vm.selectedIndex ? vm.selectedIndex : vm.selectedSearcher, null, pageNumber); } - + function prevSearchResultPage(pageNumber) { search(vm.selectedIndex ? vm.selectedIndex : vm.selectedSearcher, null, pageNumber); } - + function goToPageSearchResultPage(pageNumber) { search(vm.selectedIndex ? vm.selectedIndex : vm.selectedSearcher, null, pageNumber); } @@ -131,7 +131,7 @@ function ExamineManagementController($http, $q, $timeout, umbRequestHelper, loca event.stopPropagation(); event.preventDefault(); - } + } function setViewState(state) { vm.searchResults = null; @@ -216,8 +216,8 @@ function ExamineManagementController($http, $q, $timeout, umbRequestHelper, loca switch (section) { case "content": case "media": - result.editUrl = "/" + section + "/" + section + "/edit/" + result.values["__NodeId"][0]; - result.editId = result.values["__NodeId"][0]; + result.editUrl = "/" + section + "/" + section + "/edit/" + result.id; + result.editId = result.id; result.editSection = section; break; case "member": From f4ee0d027a19f01a8e3d302f5f92c7bac00c9fdc Mon Sep 17 00:00:00 2001 From: Kenn Jacobsen Date: Thu, 11 May 2023 08:18:43 +0200 Subject: [PATCH 09/26] Move all routing tokens (incl. API versioning) to Web.Common (#14231) * Move all routing tokens (incl. API versioning) to Cms.Web.Common, so the site can start without adding the delivery API in Startup * Fixed merge * Fix backwards compat --- .../Configuration/ConfigureMvcOptions.cs | 28 ------------------- .../UmbracoBuilderApiExtensions.cs | 4 --- .../Umbraco.Cms.Api.Common.csproj | 4 +-- .../UmbracoBuilderExtensions.cs | 1 - .../Filters/ValidateStartItemAttribute.cs | 11 ++++---- .../VersionedDeliveryApiRouteAttribute.cs | 2 +- .../Umbraco.Cms.Api.Delivery.csproj | 1 + .../ConfigureApiExplorerOptions.cs | 2 +- .../ConfigureApiVersioningOptions.cs | 3 +- .../UmbracoBuilderExtensions.cs | 4 +++ .../Mvc/UmbracoMvcConfigureOptions.cs | 23 +++++++++++++++ .../Routing/BackOfficeRouteAttribute.cs | 5 ++-- .../Routing/UmbracoBackofficeToken.cs | 2 +- .../Umbraco.Web.Common.csproj | 2 ++ 14 files changed, 43 insertions(+), 49 deletions(-) delete mode 100644 src/Umbraco.Cms.Api.Common/Configuration/ConfigureMvcOptions.cs rename src/{Umbraco.Cms.Api.Common => Umbraco.Web.Common}/Configuration/ConfigureApiExplorerOptions.cs (94%) rename src/{Umbraco.Cms.Api.Common => Umbraco.Web.Common}/Configuration/ConfigureApiVersioningOptions.cs (92%) rename src/{Umbraco.Cms.Api.Common => Umbraco.Web.Common}/Routing/BackOfficeRouteAttribute.cs (72%) rename src/{Umbraco.Cms.Api.Common => Umbraco.Web.Common}/Routing/UmbracoBackofficeToken.cs (97%) diff --git a/src/Umbraco.Cms.Api.Common/Configuration/ConfigureMvcOptions.cs b/src/Umbraco.Cms.Api.Common/Configuration/ConfigureMvcOptions.cs deleted file mode 100644 index 0eefc6710c..0000000000 --- a/src/Umbraco.Cms.Api.Common/Configuration/ConfigureMvcOptions.cs +++ /dev/null @@ -1,28 +0,0 @@ -using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Options; -using Umbraco.Cms.Api.Common.Routing; -using Umbraco.Cms.Core; -using Umbraco.Cms.Core.Configuration.Models; - -namespace Umbraco.Cms.Api.Common.Configuration; - -public class ConfigureMvcOptions : IConfigureOptions -{ - private readonly IOptions _globalSettings; - - public ConfigureMvcOptions(IOptions globalSettings) => _globalSettings = globalSettings; - - public void Configure(MvcOptions options) - { - // these MVC options may be applied more than once; let's make sure we only execute once. - if (options.Conventions.Any(convention => convention is UmbracoBackofficeToken)) - { - return; - } - - // Replace the BackOfficeToken in routes. - - var backofficePath = _globalSettings.Value.UmbracoPath.TrimStart(Constants.CharArrays.TildeForwardSlash); - options.Conventions.Add(new UmbracoBackofficeToken(Constants.Web.AttributeRouting.BackOfficeToken, backofficePath)); - } -} diff --git a/src/Umbraco.Cms.Api.Common/DependencyInjection/UmbracoBuilderApiExtensions.cs b/src/Umbraco.Cms.Api.Common/DependencyInjection/UmbracoBuilderApiExtensions.cs index 8b940a8b1d..535c9b90fd 100644 --- a/src/Umbraco.Cms.Api.Common/DependencyInjection/UmbracoBuilderApiExtensions.cs +++ b/src/Umbraco.Cms.Api.Common/DependencyInjection/UmbracoBuilderApiExtensions.cs @@ -22,10 +22,6 @@ public static class UmbracoBuilderApiExtensions { public static IUmbracoBuilder AddUmbracoApiOpenApiUI(this IUmbracoBuilder builder) { - builder.Services.ConfigureOptions(); - builder.Services.ConfigureOptions(); - builder.Services.AddApiVersioning().AddApiExplorer(); - builder.Services.AddSwaggerGen(); builder.Services.ConfigureOptions(); builder.Services.AddSingleton(); diff --git a/src/Umbraco.Cms.Api.Common/Umbraco.Cms.Api.Common.csproj b/src/Umbraco.Cms.Api.Common/Umbraco.Cms.Api.Common.csproj index c6569f0262..fd5e7d4b83 100644 --- a/src/Umbraco.Cms.Api.Common/Umbraco.Cms.Api.Common.csproj +++ b/src/Umbraco.Cms.Api.Common/Umbraco.Cms.Api.Common.csproj @@ -8,9 +8,7 @@ Umbraco.Cms.Api.Common - - - + diff --git a/src/Umbraco.Cms.Api.Delivery/DependencyInjection/UmbracoBuilderExtensions.cs b/src/Umbraco.Cms.Api.Delivery/DependencyInjection/UmbracoBuilderExtensions.cs index b6e52ada7c..6370fc24f9 100644 --- a/src/Umbraco.Cms.Api.Delivery/DependencyInjection/UmbracoBuilderExtensions.cs +++ b/src/Umbraco.Cms.Api.Delivery/DependencyInjection/UmbracoBuilderExtensions.cs @@ -34,7 +34,6 @@ public static class UmbracoBuilderExtensions builder .Services - .ConfigureOptions() .AddControllers() .AddJsonOptions(Constants.JsonOptionsNames.DeliveryApi, options => { diff --git a/src/Umbraco.Cms.Api.Delivery/Filters/ValidateStartItemAttribute.cs b/src/Umbraco.Cms.Api.Delivery/Filters/ValidateStartItemAttribute.cs index 8e5d1bbb38..54f2b06b14 100644 --- a/src/Umbraco.Cms.Api.Delivery/Filters/ValidateStartItemAttribute.cs +++ b/src/Umbraco.Cms.Api.Delivery/Filters/ValidateStartItemAttribute.cs @@ -14,19 +14,20 @@ internal sealed class ValidateStartItemAttribute : TypeFilterAttribute private class ValidateStartItemFilter : IActionFilter { - private readonly IRequestStartItemProvider _requestStartItemProvider; + private readonly IRequestStartItemProviderAccessor _requestStartItemProviderAccessor; - public ValidateStartItemFilter(IRequestStartItemProvider requestStartItemProvider) - => _requestStartItemProvider = requestStartItemProvider; + public ValidateStartItemFilter(IRequestStartItemProviderAccessor requestStartItemProviderAccessor) + => _requestStartItemProviderAccessor = requestStartItemProviderAccessor; public void OnActionExecuting(ActionExecutingContext context) { - if (_requestStartItemProvider.RequestedStartItem() is null) + if (_requestStartItemProviderAccessor.TryGetValue(out IRequestStartItemProvider? requestStartItemProvider) is false + || requestStartItemProvider.RequestedStartItem() is null) { return; } - IPublishedContent? startItem = _requestStartItemProvider.GetStartItem(); + IPublishedContent? startItem = requestStartItemProvider.GetStartItem(); if (startItem is null) { diff --git a/src/Umbraco.Cms.Api.Delivery/Routing/VersionedDeliveryApiRouteAttribute.cs b/src/Umbraco.Cms.Api.Delivery/Routing/VersionedDeliveryApiRouteAttribute.cs index 468970cfe1..4853046318 100644 --- a/src/Umbraco.Cms.Api.Delivery/Routing/VersionedDeliveryApiRouteAttribute.cs +++ b/src/Umbraco.Cms.Api.Delivery/Routing/VersionedDeliveryApiRouteAttribute.cs @@ -1,4 +1,4 @@ -using Umbraco.Cms.Api.Common.Routing; +using Umbraco.Cms.Web.Common.Routing; namespace Umbraco.Cms.Api.Delivery.Routing; diff --git a/src/Umbraco.Cms.Api.Delivery/Umbraco.Cms.Api.Delivery.csproj b/src/Umbraco.Cms.Api.Delivery/Umbraco.Cms.Api.Delivery.csproj index 5efada8f00..33a3105b73 100644 --- a/src/Umbraco.Cms.Api.Delivery/Umbraco.Cms.Api.Delivery.csproj +++ b/src/Umbraco.Cms.Api.Delivery/Umbraco.Cms.Api.Delivery.csproj @@ -13,6 +13,7 @@ + diff --git a/src/Umbraco.Cms.Api.Common/Configuration/ConfigureApiExplorerOptions.cs b/src/Umbraco.Web.Common/Configuration/ConfigureApiExplorerOptions.cs similarity index 94% rename from src/Umbraco.Cms.Api.Common/Configuration/ConfigureApiExplorerOptions.cs rename to src/Umbraco.Web.Common/Configuration/ConfigureApiExplorerOptions.cs index 50b1c2a93d..ed914f8088 100644 --- a/src/Umbraco.Cms.Api.Common/Configuration/ConfigureApiExplorerOptions.cs +++ b/src/Umbraco.Web.Common/Configuration/ConfigureApiExplorerOptions.cs @@ -2,7 +2,7 @@ using Asp.Versioning; using Asp.Versioning.ApiExplorer; using Microsoft.Extensions.Options; -namespace Umbraco.Cms.Api.Common.Configuration; +namespace Umbraco.Cms.Web.Common.Configuration; public sealed class ConfigureApiExplorerOptions : IConfigureOptions { diff --git a/src/Umbraco.Cms.Api.Common/Configuration/ConfigureApiVersioningOptions.cs b/src/Umbraco.Web.Common/Configuration/ConfigureApiVersioningOptions.cs similarity index 92% rename from src/Umbraco.Cms.Api.Common/Configuration/ConfigureApiVersioningOptions.cs rename to src/Umbraco.Web.Common/Configuration/ConfigureApiVersioningOptions.cs index b00d575ab6..a15019b165 100644 --- a/src/Umbraco.Cms.Api.Common/Configuration/ConfigureApiVersioningOptions.cs +++ b/src/Umbraco.Web.Common/Configuration/ConfigureApiVersioningOptions.cs @@ -1,8 +1,7 @@ - using Asp.Versioning; using Microsoft.Extensions.Options; -namespace Umbraco.Cms.Api.Common.Configuration; +namespace Umbraco.Cms.Web.Common.Configuration; public sealed class ConfigureApiVersioningOptions : IConfigureOptions { diff --git a/src/Umbraco.Web.Common/DependencyInjection/UmbracoBuilderExtensions.cs b/src/Umbraco.Web.Common/DependencyInjection/UmbracoBuilderExtensions.cs index dc94fa3dc8..0fa1d7fc4b 100644 --- a/src/Umbraco.Web.Common/DependencyInjection/UmbracoBuilderExtensions.cs +++ b/src/Umbraco.Web.Common/DependencyInjection/UmbracoBuilderExtensions.cs @@ -46,6 +46,7 @@ using Umbraco.Cms.Infrastructure.Persistence.SqlSyntax; using Umbraco.Cms.Web.Common; using Umbraco.Cms.Web.Common.ApplicationModels; using Umbraco.Cms.Web.Common.AspNetCore; +using Umbraco.Cms.Web.Common.Configuration; using Umbraco.Cms.Web.Common.Controllers; using Umbraco.Cms.Web.Common.DependencyInjection; using Umbraco.Cms.Web.Common.FileProviders; @@ -291,6 +292,9 @@ public static partial class UmbracoBuilderExtensions options.Cookie.HttpOnly = true; }); + builder.Services.ConfigureOptions(); + builder.Services.ConfigureOptions(); + builder.Services.AddApiVersioning().AddApiExplorer(); builder.Services.ConfigureOptions(); builder.Services.ConfigureOptions(); builder.Services.TryAddEnumerable(ServiceDescriptor diff --git a/src/Umbraco.Web.Common/Mvc/UmbracoMvcConfigureOptions.cs b/src/Umbraco.Web.Common/Mvc/UmbracoMvcConfigureOptions.cs index eb4991fec3..83a2e28834 100644 --- a/src/Umbraco.Web.Common/Mvc/UmbracoMvcConfigureOptions.cs +++ b/src/Umbraco.Web.Common/Mvc/UmbracoMvcConfigureOptions.cs @@ -1,7 +1,11 @@ using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; +using Umbraco.Cms.Core.Configuration.Models; +using Umbraco.Cms.Core.DependencyInjection; using Umbraco.Cms.Web.Common.Filters; using Umbraco.Cms.Web.Common.ModelBinders; +using Umbraco.Cms.Web.Common.Routing; using Umbraco.Cms.Web.Common.Validators; namespace Umbraco.Cms.Web.Common.Mvc; @@ -15,6 +19,17 @@ namespace Umbraco.Cms.Web.Common.Mvc; /// public class UmbracoMvcConfigureOptions : IConfigureOptions { + private readonly GlobalSettings _globalSettings; + + [Obsolete("Use the constructor that accepts GlobalSettings options. Will be removed in V14.")] + public UmbracoMvcConfigureOptions() + : this(StaticServiceProvider.Instance.GetRequiredService>()) + { + } + + public UmbracoMvcConfigureOptions(IOptions globalSettings) + => _globalSettings = globalSettings.Value; + /// public void Configure(MvcOptions options) { @@ -22,5 +37,13 @@ public class UmbracoMvcConfigureOptions : IConfigureOptions options.ModelValidatorProviders.Insert(0, new BypassRenderingModelValidatorProvider()); options.ModelMetadataDetailsProviders.Add(new BypassRenderingModelValidationMetadataProvider()); options.Filters.Insert(0, new EnsurePartialViewMacroViewContextFilterAttribute()); + + // these MVC options may be applied more than once; let's make sure we only add these conventions once. + if (options.Conventions.Any(convention => convention is UmbracoBackofficeToken) is false) + { + // Replace the BackOfficeToken in routes. + var backofficePath = _globalSettings.UmbracoPath.TrimStart(Core.Constants.CharArrays.TildeForwardSlash); + options.Conventions.Add(new UmbracoBackofficeToken(Core.Constants.Web.AttributeRouting.BackOfficeToken, backofficePath)); + } } } diff --git a/src/Umbraco.Cms.Api.Common/Routing/BackOfficeRouteAttribute.cs b/src/Umbraco.Web.Common/Routing/BackOfficeRouteAttribute.cs similarity index 72% rename from src/Umbraco.Cms.Api.Common/Routing/BackOfficeRouteAttribute.cs rename to src/Umbraco.Web.Common/Routing/BackOfficeRouteAttribute.cs index 733e856aaf..f93a4f3a89 100644 --- a/src/Umbraco.Cms.Api.Common/Routing/BackOfficeRouteAttribute.cs +++ b/src/Umbraco.Web.Common/Routing/BackOfficeRouteAttribute.cs @@ -1,7 +1,6 @@ using Microsoft.AspNetCore.Mvc; -using Umbraco.Cms.Core; -namespace Umbraco.Cms.Api.Common.Routing; +namespace Umbraco.Cms.Web.Common.Routing; /// /// Routes a controller within the backoffice area, I.E /umbraco @@ -11,7 +10,7 @@ public class BackOfficeRouteAttribute : RouteAttribute // All this does is append [umbracoBackoffice]/ to the route, // this is then replaced with whatever is configures as UmbracoPath by the UmbracoBackofficeToken convention public BackOfficeRouteAttribute(string template) - : base($"[{Constants.Web.AttributeRouting.BackOfficeToken}]/" + template.TrimStart('/')) + : base($"[{Core.Constants.Web.AttributeRouting.BackOfficeToken}]/" + template.TrimStart('/')) { } } diff --git a/src/Umbraco.Cms.Api.Common/Routing/UmbracoBackofficeToken.cs b/src/Umbraco.Web.Common/Routing/UmbracoBackofficeToken.cs similarity index 97% rename from src/Umbraco.Cms.Api.Common/Routing/UmbracoBackofficeToken.cs rename to src/Umbraco.Web.Common/Routing/UmbracoBackofficeToken.cs index 1b87d5e82b..33388398bc 100644 --- a/src/Umbraco.Cms.Api.Common/Routing/UmbracoBackofficeToken.cs +++ b/src/Umbraco.Web.Common/Routing/UmbracoBackofficeToken.cs @@ -1,7 +1,7 @@ using System.Text.RegularExpressions; using Microsoft.AspNetCore.Mvc.ApplicationModels; -namespace Umbraco.Cms.Api.Common.Routing; +namespace Umbraco.Cms.Web.Common.Routing; /// /// Adds a custom template token for specifying backoffice route with attribute routing diff --git a/src/Umbraco.Web.Common/Umbraco.Web.Common.csproj b/src/Umbraco.Web.Common/Umbraco.Web.Common.csproj index 8b92241b28..739ad33aa6 100644 --- a/src/Umbraco.Web.Common/Umbraco.Web.Common.csproj +++ b/src/Umbraco.Web.Common/Umbraco.Web.Common.csproj @@ -11,6 +11,8 @@ + + From 27ae8bdba92199af04e372250807386547dd46a5 Mon Sep 17 00:00:00 2001 From: Ronald Barendse Date: Thu, 11 May 2023 11:01:03 +0200 Subject: [PATCH 10/26] v12: Add HMAC image processing protection (#14181) * Update to ImageSharp 2.1.0 and ImageSharp.Web 2.0.0-alpha.0.23 * Rename CachedNameLength to CacheHashLength and add CacheFolderDepth setting * Replace PhysicalFileSystemProvider with WebRootImageProvider * Support EXIF-orientation in image dimention extractor * Remove virtual methods on FileProviderImageProvider * Simplify FileInfoImageResolver * Update to SixLabors.ImageSharp.Web 2.0.0-alpha.0.25 and remove custom providers * Make CropWebProcessor EXIF orientation-aware * Improve width/height sanitization * Also use 'v' as cache buster value * Add WebP to supported image file types * Update to SixLabors.ImageSharp.Web 2.0.0-alpha.0.27 and fix test * Fix rounding error and add test cases * Update to newest and stable releases * Move ImageSharpImageUrlGenerator to Umbraco.Web.Common * Use IConfigureOptions to configure ImageSharp options * Implement IEquatable on ImageUrlGenerationOptions classes * Fix empty/null values in image URL generation and corresponding tests * Use IsSupportedImageFormat extension method * Remove unneeded reflection * Add HMACSecretKey setting and add token when generating image URLs * Ensure backoffice image URLs are generated by the server (and include a correct HMAC token) * Abstract HMAC generation to IImageUrlTokenGenerator * Change cache buster value to 'v' and use hexadecimal timestamp * Update comments * Fix backoffice thumbnail URL generation * Update grid media thumbnail URL generation * Remove breaking changes * Strip unknown commands from image URL token * Remove HMAC whitelisting possibility (not supported by ImageSharp) * Update to SixLabors.ImageSharp 2.1.3 * Add comment to internal constructor * Fix to support absolute image URLs * Update to SixLabors.ImageSharp.Web 2.0.3-alpha.0.3 * Remove IImageUrlTokenGenerator and use ImageSharpRequestAuthorizationUtilities * Move NuGet feed to config file * Update to ImageSharp v3 --- .../ConfigureImageSharpMiddlewareOptions.cs | 27 +- .../Media/ImageSharpImageUrlGenerator.cs | 82 +++-- .../Models/ImagingResizeSettings.cs | 6 +- .../Configuration/Models/ImagingSettings.cs | 15 +- .../Controllers/ImagesController.cs | 56 ++-- .../ImageCropperTemplateCoreExtensions.cs | 2 +- .../common/services/mediahelper.service.js | 2 +- .../blockcard/umbBlockCard.component.js | 6 +- .../fileupload/fileupload.controller.js | 12 +- .../grid/editors/media.controller.js | 52 ++-- .../imagecropper/imagecropper.controller.js | 5 +- .../Media/ImageSharpImageUrlGeneratorTests.cs | 280 ++++++++++++------ 12 files changed, 329 insertions(+), 216 deletions(-) diff --git a/src/Umbraco.Cms.Imaging.ImageSharp/ConfigureImageSharpMiddlewareOptions.cs b/src/Umbraco.Cms.Imaging.ImageSharp/ConfigureImageSharpMiddlewareOptions.cs index aaaade3544..9a1ecead89 100644 --- a/src/Umbraco.Cms.Imaging.ImageSharp/ConfigureImageSharpMiddlewareOptions.cs +++ b/src/Umbraco.Cms.Imaging.ImageSharp/ConfigureImageSharpMiddlewareOptions.cs @@ -10,7 +10,7 @@ using Umbraco.Cms.Core.Configuration.Models; namespace Umbraco.Cms.Imaging.ImageSharp; /// -/// Configures the ImageSharp middleware options. +/// Configures the ImageSharp middleware options. /// /// public sealed class ConfigureImageSharpMiddlewareOptions : IConfigureOptions @@ -19,7 +19,7 @@ public sealed class ConfigureImageSharpMiddlewareOptions : IConfigureOptions - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// /// The ImageSharp configuration. /// The Umbraco imaging settings. @@ -34,6 +34,7 @@ public sealed class ConfigureImageSharpMiddlewareOptions : IConfigureOptions { - if (context.Commands.Count == 0) + if (context.Commands.Count == 0 || _imagingSettings.HMACSecretKey.Length > 0) { + // Nothing to parse or using HMAC authentication return Task.CompletedTask; } - var width = context.Parser.ParseValue( - context.Commands.GetValueOrDefault(ResizeWebProcessor.Width), - context.Culture); + int width = context.Parser.ParseValue(context.Commands.GetValueOrDefault(ResizeWebProcessor.Width), context.Culture); if (width <= 0 || width > _imagingSettings.Resize.MaxWidth) { context.Commands.Remove(ResizeWebProcessor.Width); } - var height = context.Parser.ParseValue( - context.Commands.GetValueOrDefault(ResizeWebProcessor.Height), - context.Culture); + int height = context.Parser.ParseValue(context.Commands.GetValueOrDefault(ResizeWebProcessor.Height), context.Culture); if (height <= 0 || height > _imagingSettings.Resize.MaxHeight) { context.Commands.Remove(ResizeWebProcessor.Height); @@ -72,11 +70,16 @@ public sealed class ConfigureImageSharpMiddlewareOptions : IConfigureOptions -/// Exposes a method that generates an image URL based on the specified options that can be processed by ImageSharp. +/// Exposes a method that generates an image URL based on the specified options that can be processed by ImageSharp. /// /// public sealed class ImageSharpImageUrlGenerator : IImageUrlGenerator { - /// - public IEnumerable SupportedImageFileTypes { get; } + private readonly RequestAuthorizationUtilities? _requestAuthorizationUtilities; /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// /// The ImageSharp configuration. - public ImageSharpImageUrlGenerator(Configuration configuration) - : this(configuration.ImageFormats.SelectMany(f => f.FileExtensions).ToArray()) + /// Contains helpers that allow authorization of image requests. + public ImageSharpImageUrlGenerator(Configuration configuration, RequestAuthorizationUtilities? requestAuthorizationUtilities) + : this(configuration.ImageFormats.SelectMany(f => f.FileExtensions).ToArray(), requestAuthorizationUtilities) { } /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. + /// + /// The ImageSharp configuration. + [Obsolete("Use ctor with all params - This will be removed in Umbraco 13.")] + public ImageSharpImageUrlGenerator(Configuration configuration) + : this(configuration, StaticServiceProvider.Instance.GetService()) + { } + + /// + /// Initializes a new instance of the class. /// /// The supported image file types/extensions. + /// Contains helpers that allow authorization of image requests. /// - /// This constructor is only used for testing. + /// This constructor is only used for testing. /// - internal ImageSharpImageUrlGenerator(IEnumerable supportedImageFileTypes) => + internal ImageSharpImageUrlGenerator(IEnumerable supportedImageFileTypes, RequestAuthorizationUtilities? requestAuthorizationUtilities = null) + { SupportedImageFileTypes = supportedImageFileTypes; + _requestAuthorizationUtilities = requestAuthorizationUtilities; + } + + /// + public IEnumerable SupportedImageFileTypes { get; } /// public string? GetImageUrl(ImageUrlGenerationOptions? options) @@ -47,47 +65,44 @@ public sealed class ImageSharpImageUrlGenerator : IImageUrlGenerator var queryString = new Dictionary(); Dictionary furtherOptions = QueryHelpers.ParseQuery(options.FurtherOptions); - if (options.Crop is not null) + if (options.Crop is CropCoordinates crop) { - CropCoordinates? crop = options.Crop; - queryString.Add( - CropWebProcessor.Coordinates, - FormattableString.Invariant($"{crop.Left},{crop.Top},{crop.Right},{crop.Bottom}")); + queryString.Add(CropWebProcessor.Coordinates, FormattableString.Invariant($"{crop.Left},{crop.Top},{crop.Right},{crop.Bottom}")); } - if (options.FocalPoint is not null) + if (options.FocalPoint is FocalPointPosition focalPoint) { - queryString.Add(ResizeWebProcessor.Xy, FormattableString.Invariant($"{options.FocalPoint.Left},{options.FocalPoint.Top}")); + queryString.Add(ResizeWebProcessor.Xy, FormattableString.Invariant($"{focalPoint.Left},{focalPoint.Top}")); } - if (options.ImageCropMode is not null) + if (options.ImageCropMode is ImageCropMode imageCropMode) { - queryString.Add(ResizeWebProcessor.Mode, options.ImageCropMode.ToString()?.ToLowerInvariant()); + queryString.Add(ResizeWebProcessor.Mode, imageCropMode.ToString().ToLowerInvariant()); } - if (options.ImageCropAnchor is not null) + if (options.ImageCropAnchor is ImageCropAnchor imageCropAnchor) { - queryString.Add(ResizeWebProcessor.Anchor, options.ImageCropAnchor.ToString()?.ToLowerInvariant()); + queryString.Add(ResizeWebProcessor.Anchor, imageCropAnchor.ToString().ToLowerInvariant()); } - if (options.Width is not null) + if (options.Width is int width) { - queryString.Add(ResizeWebProcessor.Width, options.Width?.ToString(CultureInfo.InvariantCulture)); + queryString.Add(ResizeWebProcessor.Width, width.ToString(CultureInfo.InvariantCulture)); } - if (options.Height is not null) + if (options.Height is int height) { - queryString.Add(ResizeWebProcessor.Height, options.Height?.ToString(CultureInfo.InvariantCulture)); + queryString.Add(ResizeWebProcessor.Height, height.ToString(CultureInfo.InvariantCulture)); } if (furtherOptions.Remove(FormatWebProcessor.Format, out StringValues format)) { - queryString.Add(FormatWebProcessor.Format, format[0]); + queryString.Add(FormatWebProcessor.Format, format.ToString()); } - if (options.Quality is not null) + if (options.Quality is int quality) { - queryString.Add(QualityWebProcessor.Quality, options.Quality?.ToString(CultureInfo.InvariantCulture)); + queryString.Add(QualityWebProcessor.Quality, quality.ToString(CultureInfo.InvariantCulture)); } foreach (KeyValuePair kvp in furtherOptions) @@ -95,9 +110,18 @@ public sealed class ImageSharpImageUrlGenerator : IImageUrlGenerator queryString.Add(kvp.Key, kvp.Value); } - if (options.CacheBusterValue is not null && !string.IsNullOrWhiteSpace(options.CacheBusterValue)) + if (options.CacheBusterValue is string cacheBusterValue && !string.IsNullOrEmpty(cacheBusterValue)) { - queryString.Add("rnd", options.CacheBusterValue); + queryString.Add("v", cacheBusterValue); + } + + if (_requestAuthorizationUtilities is not null) + { + var uri = QueryHelpers.AddQueryString(options.ImageUrl, queryString); + if (_requestAuthorizationUtilities.ComputeHMAC(uri, CommandHandling.Sanitize) is string token && !string.IsNullOrEmpty(token)) + { + queryString.Add(RequestAuthorizationUtilities.TokenCommand, token); + } } return QueryHelpers.AddQueryString(options.ImageUrl, queryString); diff --git a/src/Umbraco.Core/Configuration/Models/ImagingResizeSettings.cs b/src/Umbraco.Core/Configuration/Models/ImagingResizeSettings.cs index dc4585bf9c..2ae7d855be 100644 --- a/src/Umbraco.Core/Configuration/Models/ImagingResizeSettings.cs +++ b/src/Umbraco.Core/Configuration/Models/ImagingResizeSettings.cs @@ -6,7 +6,7 @@ using System.ComponentModel; namespace Umbraco.Cms.Core.Configuration.Models; /// -/// Typed configuration options for image resize settings. +/// Typed configuration options for image resize settings. /// public class ImagingResizeSettings { @@ -14,13 +14,13 @@ public class ImagingResizeSettings internal const int StaticMaxHeight = 5000; /// - /// Gets or sets a value for the maximim resize width. + /// Gets or sets a value for the maximum resize width. /// [DefaultValue(StaticMaxWidth)] public int MaxWidth { get; set; } = StaticMaxWidth; /// - /// Gets or sets a value for the maximim resize height. + /// Gets or sets a value for the maximum resize height. /// [DefaultValue(StaticMaxHeight)] public int MaxHeight { get; set; } = StaticMaxHeight; diff --git a/src/Umbraco.Core/Configuration/Models/ImagingSettings.cs b/src/Umbraco.Core/Configuration/Models/ImagingSettings.cs index 8232746ead..32bfeedb51 100644 --- a/src/Umbraco.Core/Configuration/Models/ImagingSettings.cs +++ b/src/Umbraco.Core/Configuration/Models/ImagingSettings.cs @@ -4,18 +4,27 @@ namespace Umbraco.Cms.Core.Configuration.Models; /// -/// Typed configuration options for imaging settings. +/// Typed configuration options for imaging settings. /// [UmbracoOptions(Constants.Configuration.ConfigImaging)] public class ImagingSettings { /// - /// Gets or sets a value for imaging cache settings. + /// Gets or sets a value for the Hash-based Message Authentication Code (HMAC) secret key for request authentication. + /// + /// + /// Setting or updating this value will cause all existing generated URLs to become invalid and return a 400 Bad Request response code. + /// When set, the maximum resize settings are not used/validated anymore, because you can only request URLs with a valid HMAC token anyway. + /// + public byte[] HMACSecretKey { get; set; } = Array.Empty(); + + /// + /// Gets or sets a value for imaging cache settings. /// public ImagingCacheSettings Cache { get; set; } = new(); /// - /// Gets or sets a value for imaging resize settings. + /// Gets or sets a value for imaging resize settings. /// public ImagingResizeSettings Resize { get; set; } = new(); } diff --git a/src/Umbraco.Web.BackOffice/Controllers/ImagesController.cs b/src/Umbraco.Web.BackOffice/Controllers/ImagesController.cs index e718696ae3..787aa0070c 100644 --- a/src/Umbraco.Web.BackOffice/Controllers/ImagesController.cs +++ b/src/Umbraco.Web.BackOffice/Controllers/ImagesController.cs @@ -1,3 +1,4 @@ +using System.Globalization; using System.Web; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.DependencyInjection; @@ -14,24 +15,24 @@ using Umbraco.Extensions; namespace Umbraco.Cms.Web.BackOffice.Controllers; /// -/// A controller used to return images for media +/// A controller used to return images for media. /// [PluginController(Constants.Web.Mvc.BackOfficeApiArea)] public class ImagesController : UmbracoAuthorizedApiController { - private readonly IImageUrlGenerator _imageUrlGenerator; private readonly MediaFileManager _mediaFileManager; + private readonly IImageUrlGenerator _imageUrlGenerator; private ContentSettings _contentSettings; [Obsolete("Use non obsolete-constructor. Scheduled for removal in Umbraco 13.")] public ImagesController( MediaFileManager mediaFileManager, IImageUrlGenerator imageUrlGenerator) - : this(mediaFileManager, - imageUrlGenerator, - StaticServiceProvider.Instance.GetRequiredService>()) + : this( + mediaFileManager, + imageUrlGenerator, + StaticServiceProvider.Instance.GetRequiredService>()) { - } [ActivatorUtilitiesConstructor] @@ -45,30 +46,29 @@ public class ImagesController : UmbracoAuthorizedApiController _contentSettings = contentSettingsMonitor.CurrentValue; contentSettingsMonitor.OnChange(x => _contentSettings = x); - } /// - /// Gets the big thumbnail image for the original image path + /// Gets the big thumbnail image for the original image path. /// /// /// /// - /// If there is no original image is found then this will return not found. + /// If there is no original image is found then this will return not found. /// - public IActionResult GetBigThumbnail(string originalImagePath) => - string.IsNullOrWhiteSpace(originalImagePath) - ? Ok() - : GetResized(originalImagePath, 500); + public IActionResult GetBigThumbnail(string originalImagePath) + => string.IsNullOrWhiteSpace(originalImagePath) + ? Ok() + : GetResized(originalImagePath, 500); /// - /// Gets a resized image for the image at the given path + /// Gets a resized image for the image at the given path. /// /// /// /// /// - /// If there is no media, image property or image file is found then this will return not found. + /// If there is no media, image property or image file is found then this will return not found. /// public IActionResult GetResized(string imagePath, int width) { @@ -76,7 +76,6 @@ public class ImagesController : UmbracoAuthorizedApiController // We cannot use the WebUtility, as we only want to encode the path, and not the entire string var encodedImagePath = HttpUtility.UrlPathEncode(imagePath); - var ext = Path.GetExtension(encodedImagePath); // check if imagePath is local to prevent open redirect @@ -91,13 +90,13 @@ public class ImagesController : UmbracoAuthorizedApiController return NotFound(); } - // redirect to ImageProcessor thumbnail with rnd generated from last modified time of original media file + // Redirect to thumbnail with cache buster value generated from last modified time of original media file DateTimeOffset? imageLastModified = null; try { imageLastModified = _mediaFileManager.FileSystem.GetLastModified(imagePath); } - catch (Exception) + catch { // if we get an exception here it's probably because the image path being requested is an image that doesn't exist // in the local media file system. This can happen if someone is storing an absolute path to an image online, which @@ -105,12 +104,12 @@ public class ImagesController : UmbracoAuthorizedApiController // so ignore and we won't set a last modified date. } - var rnd = imageLastModified.HasValue ? $"&rnd={imageLastModified:yyyyMMddHHmmss}" : null; + var cacheBusterValue = imageLastModified.HasValue ? imageLastModified.Value.ToFileTime().ToString("x", CultureInfo.InvariantCulture) : null; var imageUrl = _imageUrlGenerator.GetImageUrl(new ImageUrlGenerationOptions(encodedImagePath) { Width = width, ImageCropMode = ImageCropMode.Max, - CacheBusterValue = rnd + CacheBusterValue = cacheBusterValue }); if (imageUrl is not null) @@ -142,7 +141,7 @@ public class ImagesController : UmbracoAuthorizedApiController } /// - /// Gets a processed image for the image at the given path + /// Gets a processed image for the image at the given path /// /// /// @@ -150,14 +149,9 @@ public class ImagesController : UmbracoAuthorizedApiController /// /// /// - /// - /// - /// - /// - /// /// /// - /// If there is no media, image property or image file is found then this will return not found. + /// If there is no media, image property or image file is found then this will return not found. /// public string? GetProcessedImageUrl( string imagePath, @@ -166,7 +160,7 @@ public class ImagesController : UmbracoAuthorizedApiController decimal? focalPointLeft = null, decimal? focalPointTop = null, ImageCropMode mode = ImageCropMode.Max, - string cacheBusterValue = "", + string? cacheBusterValue = null, decimal? cropX1 = null, decimal? cropX2 = null, decimal? cropY1 = null, @@ -182,13 +176,11 @@ public class ImagesController : UmbracoAuthorizedApiController if (focalPointLeft.HasValue && focalPointTop.HasValue) { - options.FocalPoint = - new ImageUrlGenerationOptions.FocalPointPosition(focalPointLeft.Value, focalPointTop.Value); + options.FocalPoint = new ImageUrlGenerationOptions.FocalPointPosition(focalPointLeft.Value, focalPointTop.Value); } else if (cropX1.HasValue && cropX2.HasValue && cropY1.HasValue && cropY2.HasValue) { - options.Crop = - new ImageUrlGenerationOptions.CropCoordinates(cropX1.Value, cropY1.Value, cropX2.Value, cropY2.Value); + options.Crop = new ImageUrlGenerationOptions.CropCoordinates(cropX1.Value, cropY1.Value, cropX2.Value, cropY2.Value); } return _imageUrlGenerator.GetImageUrl(options); diff --git a/src/Umbraco.Web.Common/Extensions/ImageCropperTemplateCoreExtensions.cs b/src/Umbraco.Web.Common/Extensions/ImageCropperTemplateCoreExtensions.cs index 78a01dca2d..676b05317e 100644 --- a/src/Umbraco.Web.Common/Extensions/ImageCropperTemplateCoreExtensions.cs +++ b/src/Umbraco.Web.Common/Extensions/ImageCropperTemplateCoreExtensions.cs @@ -558,7 +558,7 @@ public static class ImageCropperTemplateCoreExtensions } var cacheBusterValue = - cacheBuster ? mediaItem.UpdateDate.ToFileTimeUtc().ToString(CultureInfo.InvariantCulture) : null; + cacheBuster ? mediaItem.UpdateDate.ToFileTimeUtc().ToString("x", CultureInfo.InvariantCulture) : null; return GetCropUrl( mediaItemUrl, diff --git a/src/Umbraco.Web.UI.Client/src/common/services/mediahelper.service.js b/src/Umbraco.Web.UI.Client/src/common/services/mediahelper.service.js index e98a597e76..14de3bb1c4 100644 --- a/src/Umbraco.Web.UI.Client/src/common/services/mediahelper.service.js +++ b/src/Umbraco.Web.UI.Client/src/common/services/mediahelper.service.js @@ -313,7 +313,7 @@ function mediaHelper(umbRequestHelper, $http, $log) { var thumbnailUrl = umbRequestHelper.getApiUrl( "imagesApiBaseUrl", "GetBigThumbnail", - [{ originalImagePath: imagePath }]) + '&rnd=' + Math.random(); + [{ originalImagePath: imagePath }]); return thumbnailUrl; }, diff --git a/src/Umbraco.Web.UI.Client/src/views/components/blockcard/umbBlockCard.component.js b/src/Umbraco.Web.UI.Client/src/views/components/blockcard/umbBlockCard.component.js index 6d6872c1e7..edcec632db 100644 --- a/src/Umbraco.Web.UI.Client/src/views/components/blockcard/umbBlockCard.component.js +++ b/src/Umbraco.Web.UI.Client/src/views/components/blockcard/umbBlockCard.component.js @@ -14,7 +14,7 @@ } }); - function BlockCardController($scope, umbRequestHelper) { + function BlockCardController($scope, umbRequestHelper, mediaHelper) { const vm = this; vm.styleBackgroundImage = "none"; @@ -49,8 +49,10 @@ var path = umbRequestHelper.convertVirtualToAbsolutePath(vm.blockConfigModel.thumbnail); if (path.toLowerCase().endsWith(".svg") === false) { - path += "?width=400"; + + path = mediaHelper.getThumbnailFromPath(path); } + vm.styleBackgroundImage = `url('${path}')`; }; diff --git a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/fileupload/fileupload.controller.js b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/fileupload/fileupload.controller.js index b4d59c683c..5d4776b8f4 100644 --- a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/fileupload/fileupload.controller.js +++ b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/fileupload/fileupload.controller.js @@ -53,16 +53,8 @@ // they contain different data structures so if we need to query against it we need to be aware of this. mediaHelper.registerFileResolver("Umbraco.UploadField", function (property, entity, thumbnail) { if (thumbnail) { - if (mediaHelper.detectIfImageByExtension(property.value)) { - //get default big thumbnail from image processor - var thumbnailUrl = property.value + "?width=500&rnd=" + moment(entity.updateDate).format("YYYYMMDDHHmmss"); - return thumbnailUrl; - } - else { - return null; - } - } - else { + return mediaHelper.getThumbnailFromPath(property.value); + } else { return property.value; } }); diff --git a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/grid/editors/media.controller.js b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/grid/editors/media.controller.js index 81a548a116..71519c5245 100644 --- a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/grid/editors/media.controller.js +++ b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/grid/editors/media.controller.js @@ -1,10 +1,10 @@ angular.module("umbraco") .controller("Umbraco.PropertyEditors.Grid.MediaController", - function ($scope, userService, editorService, localizationService) { + function ($scope, userService, editorService, localizationService, mediaHelper) { $scope.control.icon = $scope.control.icon || 'icon-picture'; - $scope.thumbnailUrl = getThumbnailUrl(); + updateThumbnailUrl(); if (!$scope.model.config.startNodeId) { if ($scope.model.config.ignoreUserStartNodes === true) { @@ -61,40 +61,31 @@ angular.module("umbraco") /** * */ - function getThumbnailUrl() { - + function updateThumbnailUrl() { if ($scope.control.value && $scope.control.value.image) { - var url = $scope.control.value.image; + var options = { + width: 800 + }; - if ($scope.control.editor.config && $scope.control.editor.config.size){ - if ($scope.control.value.coordinates) { - // New way, crop by percent must come before width/height. - var coords = $scope.control.value.coordinates; - url += `?cc=${coords.x1},${coords.y1},${coords.x2},${coords.y2}`; - } else { - // Here in order not to break existing content where focalPoint were used. - if ($scope.control.value.focalPoint) { - url += `?rxy=${$scope.control.value.focalPoint.left},${$scope.control.value.focalPoint.top}`; - } else { - // Prevent black padding and no crop when focal point not set / changed from default - url += '?rxy=0.5,0.5'; - } - } - - url += '&width=' + $scope.control.editor.config.size.width; - url += '&height=' + $scope.control.editor.config.size.height; + if ($scope.control.value.coordinates) { + // Use crop + options.crop = $scope.control.value.coordinates; + } else if ($scope.control.value.focalPoint) { + // Otherwise use focal point + options.focalPoint = $scope.control.value.focalPoint; } - // set default size if no crop present (moved from the view) - if (url.includes('?') === false) - { - url += '?width=800' + if ($scope.control.editor.config && $scope.control.editor.config.size) { + options.width = $scope.control.editor.config.size.width; + options.height = $scope.control.editor.config.size.height; } - return url; + mediaHelper.getProcessedImageUrl($scope.control.value.image, options).then(imageUrl => { + $scope.thumbnailUrl = imageUrl; + }); + } else { + $scope.thumbnailUrl = null; } - - return null; } /** @@ -113,6 +104,7 @@ angular.module("umbraco") caption: selectedImage.caption, altText: selectedImage.altText }; - $scope.thumbnailUrl = getThumbnailUrl(); + + updateThumbnailUrl(); } }); diff --git a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/imagecropper/imagecropper.controller.js b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/imagecropper/imagecropper.controller.js index b5131e9938..453347bc1b 100644 --- a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/imagecropper/imagecropper.controller.js +++ b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/imagecropper/imagecropper.controller.js @@ -236,9 +236,8 @@ angular.module('umbraco') if (property.value && property.value.src) { if (thumbnail === true) { - return property.value.src + "?width=500"; - } - else { + return mediaHelper.getThumbnailFromPath(property.value.src); + } else { return property.value.src; } diff --git a/tests/Umbraco.Tests.UnitTests/Umbraco.Web.Common/Media/ImageSharpImageUrlGeneratorTests.cs b/tests/Umbraco.Tests.UnitTests/Umbraco.Web.Common/Media/ImageSharpImageUrlGeneratorTests.cs index 40f28322dc..346bbc8a32 100644 --- a/tests/Umbraco.Tests.UnitTests/Umbraco.Web.Common/Media/ImageSharpImageUrlGeneratorTests.cs +++ b/tests/Umbraco.Tests.UnitTests/Umbraco.Web.Common/Media/ImageSharpImageUrlGeneratorTests.cs @@ -1,7 +1,14 @@ // Copyright (c) Umbraco. // See LICENSE for more details. +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; using NUnit.Framework; +using SixLabors.ImageSharp.Web; +using SixLabors.ImageSharp.Web.Commands; +using SixLabors.ImageSharp.Web.Commands.Converters; +using SixLabors.ImageSharp.Web.Middleware; +using SixLabors.ImageSharp.Web.Processors; using Umbraco.Cms.Core.Models; using Umbraco.Cms.Imaging.ImageSharp.Media; @@ -14,19 +21,22 @@ namespace Umbraco.Cms.Tests.UnitTests.Umbraco.Web.Common.Media; public class ImageSharpImageUrlGeneratorTests { private const string MediaPath = "/media/1005/img_0671.jpg"; + private static readonly ImageUrlGenerationOptions.CropCoordinates _crop = new ImageUrlGenerationOptions.CropCoordinates(0.58729977382575338m, 0.055768992440203169m, 0m, 0.32457553600198386m); + private static readonly ImageUrlGenerationOptions.FocalPointPosition _focus1 = new ImageUrlGenerationOptions.FocalPointPosition(0.96m, 0.80827067669172936m); + private static readonly ImageUrlGenerationOptions.FocalPointPosition _focus2 = new ImageUrlGenerationOptions.FocalPointPosition(0.4275m, 0.41m); + private static readonly ImageSharpImageUrlGenerator _generator = new ImageSharpImageUrlGenerator(new string[0]); - private static readonly ImageUrlGenerationOptions.CropCoordinates _sCrop = new(0.58729977382575338m, 0.055768992440203169m, 0m, 0.32457553600198386m); - private static readonly ImageUrlGenerationOptions.FocalPointPosition _sFocus = new(0.96m, 0.80827067669172936m); - private static readonly ImageSharpImageUrlGenerator _sGenerator = new(Array.Empty()); - - /// - /// Tests that the media path is returned if no options are provided. - /// [Test] public void GivenMediaPath_AndNoOptions_ReturnsMediaPath() { - var actual = _sGenerator.GetImageUrl(new ImageUrlGenerationOptions(MediaPath)); - Assert.AreEqual(MediaPath, actual); + var urlString = _generator.GetImageUrl(new ImageUrlGenerationOptions(MediaPath) + { + Crop = _crop, + Width = 100, + Height = 100, + }); + + Assert.AreEqual(MediaPath + "?cc=0.58729977382575338,0.055768992440203169,0,0.32457553600198386&width=100&height=100", urlString); } /// @@ -35,8 +45,14 @@ public class ImageSharpImageUrlGeneratorTests [Test] public void GivenNullOptions_ReturnsNull() { - var actual = _sGenerator.GetImageUrl(null); - Assert.IsNull(actual); + var urlString = _generator.GetImageUrl(new ImageUrlGenerationOptions(MediaPath) + { + FocalPoint = _focus1, + Width = 200, + Height = 300, + }); + + Assert.AreEqual(MediaPath + "?rxy=0.96,0.80827067669172936&width=200&height=300", urlString); } /// @@ -45,14 +61,34 @@ public class ImageSharpImageUrlGeneratorTests [Test] public void GivenNullImageUrl_ReturnsNull() { - var actual = _sGenerator.GetImageUrl(new ImageUrlGenerationOptions(null)); - Assert.IsNull(actual); + var urlString = _generator.GetImageUrl(new ImageUrlGenerationOptions(MediaPath) + { + FocalPoint = _focus1, + Width = 100, + Height = 100, + }); + + Assert.AreEqual(MediaPath + "?rxy=0.96,0.80827067669172936&width=100&height=100", urlString); + } + + [Test] + public void GetImageUrlFurtherOptionsTest() + { + var urlString = _generator.GetImageUrl(new ImageUrlGenerationOptions(MediaPath) + { + FocalPoint = _focus1, + Width = 200, + Height = 300, + FurtherOptions = "&filter=comic&roundedcorners=radius-26|bgcolor-fff", + }); + + Assert.AreEqual(MediaPath + "?rxy=0.96,0.80827067669172936&width=200&height=300&filter=comic&roundedcorners=radius-26%7Cbgcolor-fff", urlString); } [Test] public void GetImageUrlFurtherOptionsModeAndQualityTest() { - var urlString = _sGenerator.GetImageUrl(new ImageUrlGenerationOptions(MediaPath) + var urlString = _generator.GetImageUrl(new ImageUrlGenerationOptions(MediaPath) { Quality = 10, FurtherOptions = "format=webp", @@ -66,7 +102,7 @@ public class ImageSharpImageUrlGeneratorTests [Test] public void GetImageUrlFurtherOptionsWithModeAndQualityTest() { - var urlString = _sGenerator.GetImageUrl(new ImageUrlGenerationOptions(MediaPath) + var urlString = _generator.GetImageUrl(new ImageUrlGenerationOptions(MediaPath) { FurtherOptions = "quality=10&format=webp", }); @@ -77,62 +113,101 @@ public class ImageSharpImageUrlGeneratorTests } /// - /// Test that if an empty string image url is given, null is returned. + /// Test that if options is null, the generated image URL is also null. /// [Test] public void GivenEmptyStringImageUrl_ReturnsEmptyString() { - var actual = _sGenerator.GetImageUrl(new ImageUrlGenerationOptions(string.Empty)); - Assert.AreEqual(actual, string.Empty); + var urlString = _generator.GetImageUrl(null); + Assert.AreEqual(null, urlString); } /// - /// Tests the correct query string is returned when given a crop. + /// Test that if the image URL is null, the generated image URL is also null. /// [Test] public void GivenCrop_ReturnsExpectedQueryString() { - const string expected = "?cc=0.58729977382575338,0.055768992440203169,0,0.32457553600198386"; - var actual = _sGenerator.GetImageUrl(new ImageUrlGenerationOptions(string.Empty) { Crop = _sCrop }); - Assert.AreEqual(expected, actual); + var urlString = _generator.GetImageUrl(new ImageUrlGenerationOptions(null)); + Assert.AreEqual(null, urlString); } /// - /// Tests the correct query string is returned when given a width. + /// Test that if the image URL is empty, the generated image URL is empty. /// [Test] public void GivenWidth_ReturnsExpectedQueryString() { - const string expected = "?width=200"; - var actual = _sGenerator.GetImageUrl(new ImageUrlGenerationOptions(string.Empty) { Width = 200 }); - Assert.AreEqual(expected, actual); + var urlString = _generator.GetImageUrl(new ImageUrlGenerationOptions(string.Empty)); + Assert.AreEqual(string.Empty, urlString); } /// - /// Tests the correct query string is returned when given a height. + /// Test the GetImageUrl method on the ImageCropDataSet Model /// [Test] public void GivenHeight_ReturnsExpectedQueryString() { - const string expected = "?height=200"; - var actual = _sGenerator.GetImageUrl(new ImageUrlGenerationOptions(string.Empty) { Height = 200 }); - Assert.AreEqual(expected, actual); + var urlString = _generator.GetImageUrl(new ImageUrlGenerationOptions(string.Empty) + { + Crop = _crop, + Width = 100, + Height = 100, + }); + + Assert.AreEqual("?cc=0.58729977382575338,0.055768992440203169,0,0.32457553600198386&width=100&height=100", urlString); } /// - /// Tests the correct query string is returned when provided a focal point. + /// Test that if Crop mode is specified as anything other than Crop the image doesn't use the crop /// [Test] public void GivenFocalPoint_ReturnsExpectedQueryString() { - const string expected = "?rxy=0.96,0.80827067669172936"; - var actual = _sGenerator.GetImageUrl(new ImageUrlGenerationOptions(string.Empty) { FocalPoint = _sFocus }); - Assert.AreEqual(expected, actual); + var urlStringMin = _generator.GetImageUrl(new ImageUrlGenerationOptions(MediaPath) + { + ImageCropMode = ImageCropMode.Min, + Width = 300, + Height = 150, + }); + + var urlStringBoxPad = _generator.GetImageUrl(new ImageUrlGenerationOptions(MediaPath) + { + ImageCropMode = ImageCropMode.BoxPad, + Width = 300, + Height = 150, + }); + + var urlStringPad = _generator.GetImageUrl(new ImageUrlGenerationOptions(MediaPath) + { + ImageCropMode = ImageCropMode.Pad, + Width = 300, + Height = 150, + }); + + var urlStringMax = _generator.GetImageUrl(new ImageUrlGenerationOptions(MediaPath) + { + ImageCropMode = ImageCropMode.Max, + Width = 300, + Height = 150, + }); + + var urlStringStretch = _generator.GetImageUrl(new ImageUrlGenerationOptions(MediaPath) + { + ImageCropMode = ImageCropMode.Stretch, + Width = 300, + Height = 150, + }); + + Assert.AreEqual(MediaPath + "?rmode=min&width=300&height=150", urlStringMin); + Assert.AreEqual(MediaPath + "?rmode=boxpad&width=300&height=150", urlStringBoxPad); + Assert.AreEqual(MediaPath + "?rmode=pad&width=300&height=150", urlStringPad); + Assert.AreEqual(MediaPath + "?rmode=max&width=300&height=150", urlStringMax); + Assert.AreEqual(MediaPath + "?rmode=stretch&width=300&height=150", urlStringStretch); } /// - /// Tests the correct query string is returned when given further options. - /// There are a few edge case inputs here to ensure thorough testing in future versions. + /// Test for upload property type /// [TestCase("&filter=comic&roundedcorners=radius-26%7Cbgcolor-fff", "?filter=comic&roundedcorners=radius-26%7Cbgcolor-fff")] [TestCase("testoptions", "?testoptions=")] @@ -140,100 +215,84 @@ public class ImageSharpImageUrlGeneratorTests [TestCase("should=encode&$^%()", "?should=encode&$%5E%25()=")] public void GivenFurtherOptions_ReturnsExpectedQueryString(string input, string expected) { - var actual = _sGenerator.GetImageUrl(new ImageUrlGenerationOptions(string.Empty) + var urlString = _generator.GetImageUrl(new ImageUrlGenerationOptions(MediaPath) { FurtherOptions = input, }); - Assert.AreEqual(expected, actual); + + Assert.AreEqual(MediaPath + expected, urlString); } /// - /// Test that the correct query string is returned for all image crop modes. + /// Test for preferFocalPoint when focal point is centered /// - [TestCase(ImageCropMode.Min, "?rmode=min")] - [TestCase(ImageCropMode.BoxPad, "?rmode=boxpad")] - [TestCase(ImageCropMode.Pad, "?rmode=pad")] - [TestCase(ImageCropMode.Max, "?rmode=max")] - [TestCase(ImageCropMode.Stretch, "?rmode=stretch")] - public void GivenCropMode_ReturnsExpectedQueryString(ImageCropMode cropMode, string expectedQueryString) + [Test] + public void GetImageUrl_PreferFocalPointCenter() { - var cropUrl = _sGenerator.GetImageUrl(new ImageUrlGenerationOptions(string.Empty) + var urlString = _generator.GetImageUrl(new ImageUrlGenerationOptions(MediaPath) { - ImageCropMode = cropMode, + Width = 300, + Height = 150, }); - Assert.AreEqual(expectedQueryString, cropUrl); + Assert.AreEqual(MediaPath + "?width=300&height=150", urlString); } /// - /// Test that the correct query string is returned for all image crop anchors. + /// Test to check if crop ratio is ignored if useCropDimensions is true /// - [TestCase(ImageCropAnchor.Bottom, "?ranchor=bottom")] - [TestCase(ImageCropAnchor.BottomLeft, "?ranchor=bottomleft")] - [TestCase(ImageCropAnchor.BottomRight, "?ranchor=bottomright")] - [TestCase(ImageCropAnchor.Center, "?ranchor=center")] - [TestCase(ImageCropAnchor.Left, "?ranchor=left")] - [TestCase(ImageCropAnchor.Right, "?ranchor=right")] - [TestCase(ImageCropAnchor.Top, "?ranchor=top")] - [TestCase(ImageCropAnchor.TopLeft, "?ranchor=topleft")] - [TestCase(ImageCropAnchor.TopRight, "?ranchor=topright")] - public void GivenCropAnchor_ReturnsExpectedQueryString(ImageCropAnchor imageCropAnchor, string expectedQueryString) + [Test] + public void GetImageUrl_PreDefinedCropNoCoordinatesWithWidthAndFocalPointIgnore() { - var actual = _sGenerator.GetImageUrl(new ImageUrlGenerationOptions(string.Empty) + var urlString = _generator.GetImageUrl(new ImageUrlGenerationOptions(MediaPath) { - ImageCropAnchor = imageCropAnchor, + FocalPoint = _focus2, + Width = 270, + Height = 161, }); - Assert.AreEqual(expectedQueryString, actual); + + Assert.AreEqual(MediaPath + "?rxy=0.4275,0.41&width=270&height=161", urlString); } /// - /// Tests that the quality query string always returns the input number regardless of value. + /// Test to check result when only a width parameter is passed, effectivly a resize only /// - [TestCase(int.MinValue)] - [TestCase(-50)] - [TestCase(0)] - [TestCase(50)] - [TestCase(int.MaxValue)] - public void GivenQuality_ReturnsExpectedQueryString(int quality) + [Test] + public void GetImageUrl_WidthOnlyParameter() { - var expected = "?quality=" + quality; - var actual = _sGenerator.GetImageUrl(new ImageUrlGenerationOptions(string.Empty) + var urlString = _generator.GetImageUrl(new ImageUrlGenerationOptions(MediaPath) { - Quality = quality, + Width = 200, }); - Assert.AreEqual(expected, actual); + + Assert.AreEqual(MediaPath + "?width=200", urlString); } /// - /// Tests that the correct query string is returned for cache buster. - /// There are some edge case tests here to ensure thorough testing in future versions. + /// Test to check result when only a height parameter is passed, effectivly a resize only /// - [TestCase("test-buster", "?rnd=test-buster")] - [TestCase("test-buster&&^-value", "?rnd=test-buster%26%26%5E-value")] - public void GivenCacheBusterValue_ReturnsExpectedQueryString(string input, string expected) + [Test] + public void GetImageUrl_HeightOnlyParameter() { - var actual = _sGenerator.GetImageUrl(new ImageUrlGenerationOptions(string.Empty) + var urlString = _generator.GetImageUrl(new ImageUrlGenerationOptions(MediaPath) { - CacheBusterValue = input, + Height = 200, }); - Assert.AreEqual(expected, actual); + + Assert.AreEqual(MediaPath + "?height=200", urlString); } /// - /// Tests that an expected query string is returned when all options are given. - /// This will be a good test to see if something breaks with ordering of query string parameters. + /// Test to check result when using a background color with padding /// [Test] public void GivenAllOptions_ReturnsExpectedQueryString() { - const string expected = - "/media/1005/img_0671.jpg?cc=0.58729977382575338,0.055768992440203169,0,0.32457553600198386&rxy=0.96,0.80827067669172936&rmode=stretch&ranchor=right&width=200&height=200&quality=50&more=options&rnd=buster"; - - var actual = _sGenerator.GetImageUrl(new ImageUrlGenerationOptions(MediaPath) + var urlString = _generator.GetImageUrl(new ImageUrlGenerationOptions(MediaPath) { Quality = 50, - Crop = _sCrop, - FocalPoint = _sFocus, + Crop = _crop, + FocalPoint = _focus1, CacheBusterValue = "buster", FurtherOptions = "more=options", Height = 200, @@ -242,6 +301,47 @@ public class ImageSharpImageUrlGeneratorTests ImageCropMode = ImageCropMode.Stretch, }); - Assert.AreEqual(expected, actual); + Assert.AreEqual(MediaPath + "?cc=0.58729977382575338,0.055768992440203169,0,0.32457553600198386&rxy=0.96,0.80827067669172936&rmode=stretch&ranchor=right&width=200&height=200&quality=50&more=options&v=buster", urlString); + } + + /// + /// Test to check result when using a HMAC security key. + /// + [Test] + public void GetImageUrl_HMACSecurityKey() + { + var requestAuthorizationUtilities = new RequestAuthorizationUtilities( + Options.Create(new ImageSharpMiddlewareOptions() + { + HMACSecretKey = new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 } + }), + new QueryCollectionRequestParser(), + new[] + { + new ResizeWebProcessor() + }, + new CommandParser(Enumerable.Empty()), + new ServiceCollection().BuildServiceProvider()); + + var generator = new ImageSharpImageUrlGenerator(new string[0], requestAuthorizationUtilities); + var options = new ImageUrlGenerationOptions(MediaPath) + { + Width = 400, + Height = 400, + }; + + Assert.AreEqual(MediaPath + "?width=400&height=400&hmac=6335195986da0663e23eaadfb9bb32d537375aaeec253aae66b8f4388506b4b2", generator.GetImageUrl(options)); + + // CacheBusterValue isn't included in HMAC generation + options.CacheBusterValue = "not-included-in-hmac"; + Assert.AreEqual(MediaPath + "?width=400&height=400&v=not-included-in-hmac&hmac=6335195986da0663e23eaadfb9bb32d537375aaeec253aae66b8f4388506b4b2", generator.GetImageUrl(options)); + + // Removing height should generate a different HMAC + options.Height = null; + Assert.AreEqual(MediaPath + "?width=400&v=not-included-in-hmac&hmac=5bd24a05de5ea068533579863773ddac9269482ad515575be4aace7e9e50c88c", generator.GetImageUrl(options)); + + // But adding it again using FurtherOptions should include it (and produce the same HMAC as before) + options.FurtherOptions = "height=400"; + Assert.AreEqual(MediaPath + "?width=400&height=400&v=not-included-in-hmac&hmac=6335195986da0663e23eaadfb9bb32d537375aaeec253aae66b8f4388506b4b2", generator.GetImageUrl(options)); } } From b0d19bf9d281f35194b96773ba1bb5b12adf9604 Mon Sep 17 00:00:00 2001 From: Ronald Barendse Date: Thu, 11 May 2023 11:01:59 +0200 Subject: [PATCH 11/26] v12: Rename V2 to Umbraco.Cms.Imaging.ImageSharp2 (#14223) * Rename Umbraco.Cms.Imaging.ImageSharp.V2 to Umbraco.Cms.Imaging.ImageSharp2 * Rename Umbraco.Cms.Imaging.ImageSharp to Umbraco.Cms.Imaging.ImageSharp3 * Rename Umbraco.Cms.Imaging.ImageSharp3 back to Umbraco.Cms.Imaging.ImageSharp --- .../Umbraco.Cms.Imaging.ImageSharp.csproj | 1 - .../ConfigureImageSharpMiddlewareOptions.cs | 2 +- .../ConfigurePhysicalFileSystemCacheOptions.cs | 2 +- .../ImageProcessors/CropWebProcessor.cs | 2 +- .../ImageSharpComposer.cs | 2 +- .../Media/ImageSharpDimensionExtractor.cs | 2 +- .../Media/ImageSharpImageUrlGenerator.cs | 4 ++-- .../Umbraco.Cms.Imaging.ImageSharp2.csproj} | 4 ++-- .../UmbracoBuilderExtensions.cs | 6 +++--- umbraco.sln | 8 ++++---- 10 files changed, 16 insertions(+), 17 deletions(-) rename src/{Umbraco.Cms.Imaging.ImageSharp.V2 => Umbraco.Cms.Imaging.ImageSharp2}/ConfigureImageSharpMiddlewareOptions.cs (98%) rename src/{Umbraco.Cms.Imaging.ImageSharp.V2 => Umbraco.Cms.Imaging.ImageSharp2}/ConfigurePhysicalFileSystemCacheOptions.cs (96%) rename src/{Umbraco.Cms.Imaging.ImageSharp.V2 => Umbraco.Cms.Imaging.ImageSharp2}/ImageProcessors/CropWebProcessor.cs (98%) rename src/{Umbraco.Cms.Imaging.ImageSharp.V2 => Umbraco.Cms.Imaging.ImageSharp2}/ImageSharpComposer.cs (90%) rename src/{Umbraco.Cms.Imaging.ImageSharp.V2 => Umbraco.Cms.Imaging.ImageSharp2}/Media/ImageSharpDimensionExtractor.cs (97%) rename src/{Umbraco.Cms.Imaging.ImageSharp.V2 => Umbraco.Cms.Imaging.ImageSharp2}/Media/ImageSharpImageUrlGenerator.cs (97%) rename src/{Umbraco.Cms.Imaging.ImageSharp.V2/Umbraco.Cms.Imaging.ImageSharp.V2.csproj => Umbraco.Cms.Imaging.ImageSharp2/Umbraco.Cms.Imaging.ImageSharp2.csproj} (88%) rename src/{Umbraco.Cms.Imaging.ImageSharp.V2 => Umbraco.Cms.Imaging.ImageSharp2}/UmbracoBuilderExtensions.cs (93%) diff --git a/src/Umbraco.Cms.Imaging.ImageSharp/Umbraco.Cms.Imaging.ImageSharp.csproj b/src/Umbraco.Cms.Imaging.ImageSharp/Umbraco.Cms.Imaging.ImageSharp.csproj index e4eb1cd938..c34595576f 100644 --- a/src/Umbraco.Cms.Imaging.ImageSharp/Umbraco.Cms.Imaging.ImageSharp.csproj +++ b/src/Umbraco.Cms.Imaging.ImageSharp/Umbraco.Cms.Imaging.ImageSharp.csproj @@ -2,7 +2,6 @@ Umbraco CMS - Imaging - ImageSharp Adds imaging support using ImageSharp/ImageSharp.Web to Umbraco CMS. - true diff --git a/src/Umbraco.Cms.Imaging.ImageSharp.V2/ConfigureImageSharpMiddlewareOptions.cs b/src/Umbraco.Cms.Imaging.ImageSharp2/ConfigureImageSharpMiddlewareOptions.cs similarity index 98% rename from src/Umbraco.Cms.Imaging.ImageSharp.V2/ConfigureImageSharpMiddlewareOptions.cs rename to src/Umbraco.Cms.Imaging.ImageSharp2/ConfigureImageSharpMiddlewareOptions.cs index 17b735891c..8daa1b689b 100644 --- a/src/Umbraco.Cms.Imaging.ImageSharp.V2/ConfigureImageSharpMiddlewareOptions.cs +++ b/src/Umbraco.Cms.Imaging.ImageSharp2/ConfigureImageSharpMiddlewareOptions.cs @@ -8,7 +8,7 @@ using SixLabors.ImageSharp.Web.Middleware; using SixLabors.ImageSharp.Web.Processors; using Umbraco.Cms.Core.Configuration.Models; -namespace Umbraco.Cms.Imaging.ImageSharp.V2; +namespace Umbraco.Cms.Imaging.ImageSharp; /// /// Configures the ImageSharp middleware options. diff --git a/src/Umbraco.Cms.Imaging.ImageSharp.V2/ConfigurePhysicalFileSystemCacheOptions.cs b/src/Umbraco.Cms.Imaging.ImageSharp2/ConfigurePhysicalFileSystemCacheOptions.cs similarity index 96% rename from src/Umbraco.Cms.Imaging.ImageSharp.V2/ConfigurePhysicalFileSystemCacheOptions.cs rename to src/Umbraco.Cms.Imaging.ImageSharp2/ConfigurePhysicalFileSystemCacheOptions.cs index 2c70afc3bc..3b2cd7f867 100644 --- a/src/Umbraco.Cms.Imaging.ImageSharp.V2/ConfigurePhysicalFileSystemCacheOptions.cs +++ b/src/Umbraco.Cms.Imaging.ImageSharp2/ConfigurePhysicalFileSystemCacheOptions.cs @@ -4,7 +4,7 @@ using SixLabors.ImageSharp.Web.Caching; using Umbraco.Cms.Core.Configuration.Models; using Umbraco.Cms.Core.Extensions; -namespace Umbraco.Cms.Imaging.ImageSharp.V2; +namespace Umbraco.Cms.Imaging.ImageSharp; /// /// Configures the ImageSharp physical file system cache options. diff --git a/src/Umbraco.Cms.Imaging.ImageSharp.V2/ImageProcessors/CropWebProcessor.cs b/src/Umbraco.Cms.Imaging.ImageSharp2/ImageProcessors/CropWebProcessor.cs similarity index 98% rename from src/Umbraco.Cms.Imaging.ImageSharp.V2/ImageProcessors/CropWebProcessor.cs rename to src/Umbraco.Cms.Imaging.ImageSharp2/ImageProcessors/CropWebProcessor.cs index a02f497af2..eda49fa9d0 100644 --- a/src/Umbraco.Cms.Imaging.ImageSharp.V2/ImageProcessors/CropWebProcessor.cs +++ b/src/Umbraco.Cms.Imaging.ImageSharp2/ImageProcessors/CropWebProcessor.cs @@ -8,7 +8,7 @@ using SixLabors.ImageSharp.Web; using SixLabors.ImageSharp.Web.Commands; using SixLabors.ImageSharp.Web.Processors; -namespace Umbraco.Cms.Imaging.ImageSharp.V2.ImageProcessors; +namespace Umbraco.Cms.Imaging.ImageSharp.ImageProcessors; /// /// Allows the cropping of images. diff --git a/src/Umbraco.Cms.Imaging.ImageSharp.V2/ImageSharpComposer.cs b/src/Umbraco.Cms.Imaging.ImageSharp2/ImageSharpComposer.cs similarity index 90% rename from src/Umbraco.Cms.Imaging.ImageSharp.V2/ImageSharpComposer.cs rename to src/Umbraco.Cms.Imaging.ImageSharp2/ImageSharpComposer.cs index 5ec2a1f120..9a77bc28b2 100644 --- a/src/Umbraco.Cms.Imaging.ImageSharp.V2/ImageSharpComposer.cs +++ b/src/Umbraco.Cms.Imaging.ImageSharp2/ImageSharpComposer.cs @@ -2,7 +2,7 @@ using Umbraco.Cms.Core.Composing; using Umbraco.Cms.Core.DependencyInjection; using Umbraco.Extensions; -namespace Umbraco.Cms.Imaging.ImageSharp.V2; +namespace Umbraco.Cms.Imaging.ImageSharp; /// /// Adds imaging support using ImageSharp/ImageSharp.Web. diff --git a/src/Umbraco.Cms.Imaging.ImageSharp.V2/Media/ImageSharpDimensionExtractor.cs b/src/Umbraco.Cms.Imaging.ImageSharp2/Media/ImageSharpDimensionExtractor.cs similarity index 97% rename from src/Umbraco.Cms.Imaging.ImageSharp.V2/Media/ImageSharpDimensionExtractor.cs rename to src/Umbraco.Cms.Imaging.ImageSharp2/Media/ImageSharpDimensionExtractor.cs index d96f4f7f32..409b6e2726 100644 --- a/src/Umbraco.Cms.Imaging.ImageSharp.V2/Media/ImageSharpDimensionExtractor.cs +++ b/src/Umbraco.Cms.Imaging.ImageSharp2/Media/ImageSharpDimensionExtractor.cs @@ -3,7 +3,7 @@ using SixLabors.ImageSharp.Metadata.Profiles.Exif; using Umbraco.Cms.Core.Media; using Size = System.Drawing.Size; -namespace Umbraco.Cms.Imaging.ImageSharp.V2.Media; +namespace Umbraco.Cms.Imaging.ImageSharp.Media; public sealed class ImageSharpDimensionExtractor : IImageDimensionExtractor { diff --git a/src/Umbraco.Cms.Imaging.ImageSharp.V2/Media/ImageSharpImageUrlGenerator.cs b/src/Umbraco.Cms.Imaging.ImageSharp2/Media/ImageSharpImageUrlGenerator.cs similarity index 97% rename from src/Umbraco.Cms.Imaging.ImageSharp.V2/Media/ImageSharpImageUrlGenerator.cs rename to src/Umbraco.Cms.Imaging.ImageSharp2/Media/ImageSharpImageUrlGenerator.cs index 0bcd9ad749..ad76603187 100644 --- a/src/Umbraco.Cms.Imaging.ImageSharp.V2/Media/ImageSharpImageUrlGenerator.cs +++ b/src/Umbraco.Cms.Imaging.ImageSharp2/Media/ImageSharpImageUrlGenerator.cs @@ -5,10 +5,10 @@ using SixLabors.ImageSharp; using SixLabors.ImageSharp.Web.Processors; using Umbraco.Cms.Core.Media; using Umbraco.Cms.Core.Models; -using Umbraco.Cms.Imaging.ImageSharp.V2.ImageProcessors; +using Umbraco.Cms.Imaging.ImageSharp.ImageProcessors; using static Umbraco.Cms.Core.Models.ImageUrlGenerationOptions; -namespace Umbraco.Cms.Imaging.ImageSharp.V2.Media; +namespace Umbraco.Cms.Imaging.ImageSharp.Media; /// /// Exposes a method that generates an image URL based on the specified options that can be processed by ImageSharp. diff --git a/src/Umbraco.Cms.Imaging.ImageSharp.V2/Umbraco.Cms.Imaging.ImageSharp.V2.csproj b/src/Umbraco.Cms.Imaging.ImageSharp2/Umbraco.Cms.Imaging.ImageSharp2.csproj similarity index 88% rename from src/Umbraco.Cms.Imaging.ImageSharp.V2/Umbraco.Cms.Imaging.ImageSharp.V2.csproj rename to src/Umbraco.Cms.Imaging.ImageSharp2/Umbraco.Cms.Imaging.ImageSharp2.csproj index 0eb87fbda7..dc0299defd 100644 --- a/src/Umbraco.Cms.Imaging.ImageSharp.V2/Umbraco.Cms.Imaging.ImageSharp.V2.csproj +++ b/src/Umbraco.Cms.Imaging.ImageSharp2/Umbraco.Cms.Imaging.ImageSharp2.csproj @@ -1,7 +1,7 @@ - Umbraco CMS - Imaging - ImageSharp - Adds imaging support using ImageSharp/ImageSharp.Web to Umbraco CMS. + Umbraco CMS - Imaging - ImageSharp 2 + Adds imaging support using ImageSharp/ImageSharp.Web version 2 to Umbraco CMS. false diff --git a/src/Umbraco.Cms.Imaging.ImageSharp.V2/UmbracoBuilderExtensions.cs b/src/Umbraco.Cms.Imaging.ImageSharp2/UmbracoBuilderExtensions.cs similarity index 93% rename from src/Umbraco.Cms.Imaging.ImageSharp.V2/UmbracoBuilderExtensions.cs rename to src/Umbraco.Cms.Imaging.ImageSharp2/UmbracoBuilderExtensions.cs index 078a5b61d6..4bd50034ab 100644 --- a/src/Umbraco.Cms.Imaging.ImageSharp.V2/UmbracoBuilderExtensions.cs +++ b/src/Umbraco.Cms.Imaging.ImageSharp2/UmbracoBuilderExtensions.cs @@ -7,12 +7,12 @@ using SixLabors.ImageSharp.Web.Middleware; using SixLabors.ImageSharp.Web.Providers; using Umbraco.Cms.Core.DependencyInjection; using Umbraco.Cms.Core.Media; -using Umbraco.Cms.Imaging.ImageSharp.V2.ImageProcessors; -using Umbraco.Cms.Imaging.ImageSharp.V2.Media; +using Umbraco.Cms.Imaging.ImageSharp.ImageProcessors; +using Umbraco.Cms.Imaging.ImageSharp.Media; using Umbraco.Cms.Web.Common.ApplicationBuilder; using Umbraco.Extensions; -namespace Umbraco.Cms.Imaging.ImageSharp.V2; +namespace Umbraco.Cms.Imaging.ImageSharp; public static class UmbracoBuilderExtensions { diff --git a/umbraco.sln b/umbraco.sln index b90c7d5ea2..c382c480f7 100644 --- a/umbraco.sln +++ b/umbraco.sln @@ -141,15 +141,15 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "styles", "styles", "{EA628A build\csharp-docs\umbracotemplate\styles\main.css = build\csharp-docs\umbracotemplate\styles\main.css EndProjectSection EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Umbraco.Cms.Imaging.ImageSharp.V2", "src\Umbraco.Cms.Imaging.ImageSharp.V2\Umbraco.Cms.Imaging.ImageSharp.V2.csproj", "{C280181E-597B-4AA5-82E7-D7017E928749}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Umbraco.Cms.Imaging.ImageSharp2", "src\Umbraco.Cms.Imaging.ImageSharp2\Umbraco.Cms.Imaging.ImageSharp2.csproj", "{C280181E-597B-4AA5-82E7-D7017E928749}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tools", "Tools", "{05878304-40EB-4F84-B40B-91BDB70DE094}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Umbraco.Cms.Api.Delivery", "src\Umbraco.Cms.Api.Delivery\Umbraco.Cms.Api.Delivery.csproj", "{9AA3D21F-81A9-4F27-85D1-CE850B59DC2D}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Umbraco.Cms.Api.Delivery", "src\Umbraco.Cms.Api.Delivery\Umbraco.Cms.Api.Delivery.csproj", "{9AA3D21F-81A9-4F27-85D1-CE850B59DC2D}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Umbraco.Cms.Api.Common", "src\Umbraco.Cms.Api.Common\Umbraco.Cms.Api.Common.csproj", "{D48B5D6B-82FF-4235-986C-CDE646F41DEC}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Umbraco.Cms.Api.Common", "src\Umbraco.Cms.Api.Common\Umbraco.Cms.Api.Common.csproj", "{D48B5D6B-82FF-4235-986C-CDE646F41DEC}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Umbraco.Cms.Imaging.ImageSharp", "src\Umbraco.Cms.Imaging.ImageSharp\Umbraco.Cms.Imaging.ImageSharp.csproj", "{35E3DA10-5549-41DE-B7ED-CC29355BA9FD}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Umbraco.Cms.Imaging.ImageSharp", "src\Umbraco.Cms.Imaging.ImageSharp\Umbraco.Cms.Imaging.ImageSharp.csproj", "{35E3DA10-5549-41DE-B7ED-CC29355BA9FD}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution From b7cf00ac5df1bfa1ef8f309c32264bddd2eb39a8 Mon Sep 17 00:00:00 2001 From: Kenn Jacobsen Date: Thu, 11 May 2023 11:19:34 +0200 Subject: [PATCH 12/26] Handle applied public access restrictions in the Delivery API content index (#14233) * Automatically remove content from the delivery API content index when public access restrictions are applied * Review changes --- .../Services/ApiContentQueryService.cs | 2 +- .../DeliveryApiContentIndex.cs | 2 +- .../UmbracoBuilder.Examine.cs | 1 + ...veryApiContentIndexHandleContentChanges.cs | 6 +- ...ApiContentIndexHandleContentTypeChanges.cs | 2 +- ...piContentIndexHandlePublicAccessChanges.cs | 90 +++++++++++++++++++ ...ryApiContentIndexFieldDefinitionBuilder.cs | 6 +- .../DeliveryApiContentIndexValueSetBuilder.cs | 6 +- .../Examine/DeliveryApiIndexingHandler.cs | 13 +++ .../Examine/UmbracoExamineFieldNames.cs | 21 +++++ .../Search/IDeliveryApiIndexingHandler.cs | 9 ++ ...IndexingNotificationHandler.DeliveryApi.cs | 7 +- 12 files changed, 152 insertions(+), 13 deletions(-) create mode 100644 src/Umbraco.Infrastructure/Examine/Deferred/DeliveryApiContentIndexHandlePublicAccessChanges.cs diff --git a/src/Umbraco.Cms.Api.Delivery/Services/ApiContentQueryService.cs b/src/Umbraco.Cms.Api.Delivery/Services/ApiContentQueryService.cs index 404de7a6f2..d044d774d1 100644 --- a/src/Umbraco.Cms.Api.Delivery/Services/ApiContentQueryService.cs +++ b/src/Umbraco.Cms.Api.Delivery/Services/ApiContentQueryService.cs @@ -77,7 +77,7 @@ internal sealed class ApiContentQueryService : IApiContentQueryService // Item culture must be either the requested culture or "none" var culture = CurrentCulture(); - queryOperation.And().GroupedOr(new[] { "culture" }, culture.ToLowerInvariant().IfNullOrWhiteSpace(_fallbackGuidValue), "none"); + queryOperation.And().GroupedOr(new[] { UmbracoExamineFieldNames.DeliveryApiContentIndex.Culture }, culture.ToLowerInvariant().IfNullOrWhiteSpace(_fallbackGuidValue), "none"); // Handle Filtering var canApplyFiltering = CanHandleFiltering(filters, queryOperation); diff --git a/src/Umbraco.Examine.Lucene/DeliveryApiContentIndex.cs b/src/Umbraco.Examine.Lucene/DeliveryApiContentIndex.cs index a7a7e9a7d8..0967022d77 100644 --- a/src/Umbraco.Examine.Lucene/DeliveryApiContentIndex.cs +++ b/src/Umbraco.Examine.Lucene/DeliveryApiContentIndex.cs @@ -75,7 +75,7 @@ public class DeliveryApiContentIndex : UmbracoExamineIndex } // find descendants-or-self based on path and optional culture - var rawQuery = $"({UmbracoExamineFieldNames.IndexPathFieldName}:\\-1*,{contentId} OR {UmbracoExamineFieldNames.IndexPathFieldName}:\\-1*,{contentId},*)"; + var rawQuery = $"({UmbracoExamineFieldNames.DeliveryApiContentIndex.Id}:{contentId} OR {UmbracoExamineFieldNames.IndexPathFieldName}:\\-1*,{contentId},*)"; if (culture is not null) { rawQuery = $"{rawQuery} AND culture:{culture}"; diff --git a/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.Examine.cs b/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.Examine.cs index 4106464602..0e7b0f5faa 100644 --- a/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.Examine.cs +++ b/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.Examine.cs @@ -60,6 +60,7 @@ public static partial class UmbracoBuilderExtensions builder.AddNotificationHandler(); builder.AddNotificationHandler(); builder.AddNotificationHandler(); + builder.AddNotificationHandler(); builder.AddNotificationHandler(); builder.AddNotificationHandler(); builder.AddNotificationHandler(); diff --git a/src/Umbraco.Infrastructure/Examine/Deferred/DeliveryApiContentIndexHandleContentChanges.cs b/src/Umbraco.Infrastructure/Examine/Deferred/DeliveryApiContentIndexHandleContentChanges.cs index 1ee4a0e949..c93f42b6e8 100644 --- a/src/Umbraco.Infrastructure/Examine/Deferred/DeliveryApiContentIndexHandleContentChanges.cs +++ b/src/Umbraco.Infrastructure/Examine/Deferred/DeliveryApiContentIndexHandleContentChanges.cs @@ -74,10 +74,10 @@ internal sealed class DeliveryApiContentIndexHandleContentChanges : DeliveryApiC var existingIndexCultures = index .Searcher .CreateQuery() - .Field("id", content.Id) - .SelectField("culture") + .Field(UmbracoExamineFieldNames.DeliveryApiContentIndex.Id, content.Id.ToString()) + .SelectField(UmbracoExamineFieldNames.DeliveryApiContentIndex.Culture) .Execute() - .SelectMany(f => f.GetValues("culture")) + .SelectMany(f => f.GetValues(UmbracoExamineFieldNames.DeliveryApiContentIndex.Culture)) .ToArray(); // index the content diff --git a/src/Umbraco.Infrastructure/Examine/Deferred/DeliveryApiContentIndexHandleContentTypeChanges.cs b/src/Umbraco.Infrastructure/Examine/Deferred/DeliveryApiContentIndexHandleContentTypeChanges.cs index 92cafbe670..32dc801dd3 100644 --- a/src/Umbraco.Infrastructure/Examine/Deferred/DeliveryApiContentIndexHandleContentTypeChanges.cs +++ b/src/Umbraco.Infrastructure/Examine/Deferred/DeliveryApiContentIndexHandleContentTypeChanges.cs @@ -128,7 +128,7 @@ internal sealed class DeliveryApiContentIndexHandleContentTypeChanges : Delivery { ISearchResults? results = index.Searcher .CreateQuery() - .Field("contentTypeId", contentTypeId) + .Field(UmbracoExamineFieldNames.DeliveryApiContentIndex.ContentTypeId, contentTypeId.ToString()) // NOTE: we need to be explicit about fetching ItemIdFieldName here, otherwise Examine will try to be // clever and use the "id" field of the document (which we can't use for deletion) .SelectField(UmbracoExamineFieldNames.ItemIdFieldName) diff --git a/src/Umbraco.Infrastructure/Examine/Deferred/DeliveryApiContentIndexHandlePublicAccessChanges.cs b/src/Umbraco.Infrastructure/Examine/Deferred/DeliveryApiContentIndexHandlePublicAccessChanges.cs new file mode 100644 index 0000000000..e5db4b6f1e --- /dev/null +++ b/src/Umbraco.Infrastructure/Examine/Deferred/DeliveryApiContentIndexHandlePublicAccessChanges.cs @@ -0,0 +1,90 @@ +using Examine; +using Examine.Search; +using Umbraco.Cms.Core.Services; +using Umbraco.Cms.Infrastructure.HostedServices; +using Umbraco.Extensions; + +namespace Umbraco.Cms.Infrastructure.Examine.Deferred; + +internal sealed class DeliveryApiContentIndexHandlePublicAccessChanges : DeliveryApiContentIndexDeferredBase, IDeferredAction +{ + private readonly IPublicAccessService _publicAccessService; + private readonly DeliveryApiIndexingHandler _deliveryApiIndexingHandler; + private readonly IBackgroundTaskQueue _backgroundTaskQueue; + + public DeliveryApiContentIndexHandlePublicAccessChanges( + IPublicAccessService publicAccessService, + DeliveryApiIndexingHandler deliveryApiIndexingHandler, + IBackgroundTaskQueue backgroundTaskQueue) + { + _publicAccessService = publicAccessService; + _deliveryApiIndexingHandler = deliveryApiIndexingHandler; + _backgroundTaskQueue = backgroundTaskQueue; + } + + public void Execute() => _backgroundTaskQueue.QueueBackgroundWorkItem(_ => + { + // NOTE: at the time of implementing this, the distributed notifications for public access changes only ever + // sends out "refresh all" notifications, which means we can't be clever about minimizing the work + // effort to handle public access changes. instead we have to grab all protected content definitions + // and handle every last one with every notification. + + // NOTE: eventually the Delivery API will support protected content, but for now we need to ensure that the + // index does not contain any protected content. this also means that whenever content is unprotected, + // one must trigger a manual republish of said content for it to be re-added to the index. not exactly + // an optimal solution, but it's the best we can do at this point, given the limitations outlined above + // and without prematurely assuming the future implementation details of protected content handling. + + var protectedContentIds = _publicAccessService.GetAll().Select(entry => entry.ProtectedNodeId).ToArray(); + if (protectedContentIds.Any() is false) + { + return Task.CompletedTask; + } + + IIndex index = _deliveryApiIndexingHandler.GetIndex() ?? + throw new InvalidOperationException("Could not obtain the delivery API content index"); + + List indexIds = FindIndexIdsForContentIds(protectedContentIds, index); + if (indexIds.Any() is false) + { + return Task.CompletedTask; + } + + RemoveFromIndex(indexIds, index); + return Task.CompletedTask; + }); + + private List FindIndexIdsForContentIds(int[] contentIds, IIndex index) + { + const int pageSize = 500; + const int batchSize = 50; + + var ids = new List(); + + foreach (IEnumerable batch in contentIds.InGroupsOf(batchSize)) + { + IEnumerable batchAsArray = batch as int[] ?? batch.ToArray(); + var page = 0; + var total = long.MaxValue; + + while (page * pageSize < total) + { + ISearchResults? results = index.Searcher + .CreateQuery() + .GroupedOr(new[] { UmbracoExamineFieldNames.DeliveryApiContentIndex.Id }, batchAsArray.Select(id => id.ToString()).ToArray()) + // NOTE: we need to be explicit about fetching ItemIdFieldName here, otherwise Examine will try to be + // clever and use the "id" field of the document (which we can't use for deletion) + .SelectField(UmbracoExamineFieldNames.ItemIdFieldName) + .Execute(QueryOptions.SkipTake(page * pageSize, pageSize)); + total = results.TotalItemCount; + + ids.AddRange(results.Select(result => result.Id)); + + page++; + } + } + + return ids; + } + +} diff --git a/src/Umbraco.Infrastructure/Examine/DeliveryApiContentIndexFieldDefinitionBuilder.cs b/src/Umbraco.Infrastructure/Examine/DeliveryApiContentIndexFieldDefinitionBuilder.cs index ce20716251..bfd11defde 100644 --- a/src/Umbraco.Infrastructure/Examine/DeliveryApiContentIndexFieldDefinitionBuilder.cs +++ b/src/Umbraco.Infrastructure/Examine/DeliveryApiContentIndexFieldDefinitionBuilder.cs @@ -32,9 +32,9 @@ internal sealed class DeliveryApiContentIndexFieldDefinitionBuilder : IDeliveryA // see also the field definitions in the Delivery API content index value set builder private void AddRequiredFieldDefinitions(ICollection fieldDefinitions) { - fieldDefinitions.Add(new("id", FieldDefinitionTypes.Integer)); - fieldDefinitions.Add(new("contentTypeId", FieldDefinitionTypes.Integer)); - fieldDefinitions.Add(new("culture", FieldDefinitionTypes.Raw)); + fieldDefinitions.Add(new(UmbracoExamineFieldNames.DeliveryApiContentIndex.Id, FieldDefinitionTypes.Raw)); + fieldDefinitions.Add(new(UmbracoExamineFieldNames.DeliveryApiContentIndex.ContentTypeId, FieldDefinitionTypes.Raw)); + fieldDefinitions.Add(new(UmbracoExamineFieldNames.DeliveryApiContentIndex.Culture, FieldDefinitionTypes.Raw)); fieldDefinitions.Add(new(UmbracoExamineFieldNames.IndexPathFieldName, FieldDefinitionTypes.Raw)); fieldDefinitions.Add(new(UmbracoExamineFieldNames.NodeNameFieldName, FieldDefinitionTypes.Raw)); } diff --git a/src/Umbraco.Infrastructure/Examine/DeliveryApiContentIndexValueSetBuilder.cs b/src/Umbraco.Infrastructure/Examine/DeliveryApiContentIndexValueSetBuilder.cs index 9c0f7bba6a..20942336ab 100644 --- a/src/Umbraco.Infrastructure/Examine/DeliveryApiContentIndexValueSetBuilder.cs +++ b/src/Umbraco.Infrastructure/Examine/DeliveryApiContentIndexValueSetBuilder.cs @@ -46,9 +46,9 @@ internal sealed class DeliveryApiContentIndexValueSetBuilder : IDeliveryApiConte // required index values go here var indexValues = new Dictionary>(StringComparer.InvariantCultureIgnoreCase) { - ["id"] = new object[] { content.Id }, // required for correct publishing handling and also needed for backoffice index browsing - ["contentTypeId"] = new object[] { content.ContentTypeId }, // required for correct content type change handling - ["culture"] = new object[] { indexCulture }, // required for culture variant querying + [UmbracoExamineFieldNames.DeliveryApiContentIndex.Id] = new object[] { content.Id.ToString() }, // required for correct publishing handling and also needed for backoffice index browsing + [UmbracoExamineFieldNames.DeliveryApiContentIndex.ContentTypeId] = new object[] { content.ContentTypeId.ToString() }, // required for correct content type change handling + [UmbracoExamineFieldNames.DeliveryApiContentIndex.Culture] = new object[] { indexCulture }, // required for culture variant querying [UmbracoExamineFieldNames.IndexPathFieldName] = new object[] { content.Path }, // required for unpublishing/deletion handling [UmbracoExamineFieldNames.NodeNameFieldName] = new object[] { content.GetPublishName(culture) ?? string.Empty }, // primarily needed for backoffice index browsing }; diff --git a/src/Umbraco.Infrastructure/Examine/DeliveryApiIndexingHandler.cs b/src/Umbraco.Infrastructure/Examine/DeliveryApiIndexingHandler.cs index a8654192fc..197ab58be0 100644 --- a/src/Umbraco.Infrastructure/Examine/DeliveryApiIndexingHandler.cs +++ b/src/Umbraco.Infrastructure/Examine/DeliveryApiIndexingHandler.cs @@ -21,6 +21,7 @@ internal sealed class DeliveryApiIndexingHandler : IDeliveryApiIndexingHandler // these dependencies are for the deferred handling (we don't want those handlers registered in the DI) private readonly IContentService _contentService; + private readonly IPublicAccessService _publicAccessService; private readonly IDeliveryApiContentIndexValueSetBuilder _deliveryApiContentIndexValueSetBuilder; private readonly IDeliveryApiContentIndexHelper _deliveryApiContentIndexHelper; private readonly IBackgroundTaskQueue _backgroundTaskQueue; @@ -31,6 +32,7 @@ internal sealed class DeliveryApiIndexingHandler : IDeliveryApiIndexingHandler ICoreScopeProvider scopeProvider, ILogger logger, IContentService contentService, + IPublicAccessService publicAccessService, IDeliveryApiContentIndexValueSetBuilder deliveryApiContentIndexValueSetBuilder, IDeliveryApiContentIndexHelper deliveryApiContentIndexHelper, IBackgroundTaskQueue backgroundTaskQueue) @@ -40,6 +42,7 @@ internal sealed class DeliveryApiIndexingHandler : IDeliveryApiIndexingHandler _scopeProvider = scopeProvider; _logger = logger; _contentService = contentService; + _publicAccessService = publicAccessService; _deliveryApiContentIndexValueSetBuilder = deliveryApiContentIndexValueSetBuilder; _deliveryApiContentIndexHelper = deliveryApiContentIndexHelper; _backgroundTaskQueue = backgroundTaskQueue; @@ -74,6 +77,16 @@ internal sealed class DeliveryApiIndexingHandler : IDeliveryApiIndexingHandler Execute(deferred); } + /// + public void HandlePublicAccessChanges() + { + var deferred = new DeliveryApiContentIndexHandlePublicAccessChanges( + _publicAccessService, + this, + _backgroundTaskQueue); + Execute(deferred); + } + private void Execute(IDeferredAction action) { var actions = DeferredActions.Get(_scopeProvider); diff --git a/src/Umbraco.Infrastructure/Examine/UmbracoExamineFieldNames.cs b/src/Umbraco.Infrastructure/Examine/UmbracoExamineFieldNames.cs index 5e2779e9a3..12b2eb2207 100644 --- a/src/Umbraco.Infrastructure/Examine/UmbracoExamineFieldNames.cs +++ b/src/Umbraco.Infrastructure/Examine/UmbracoExamineFieldNames.cs @@ -25,4 +25,25 @@ public static class UmbracoExamineFieldNames public const string ItemIdFieldName = "__NodeId"; public const string CategoryFieldName = "__IndexType"; public const string ItemTypeFieldName = "__NodeTypeAlias"; + + /// + /// Field names specifically used in the Delivery API content index + /// + public static class DeliveryApiContentIndex + { + /// + /// The content ID + /// + public const string Id = "id"; + + /// + /// The content type ID + /// + public const string ContentTypeId = "contentTypeId"; + + /// + /// The content culture + /// + public const string Culture = "culture"; + } } diff --git a/src/Umbraco.Infrastructure/Search/IDeliveryApiIndexingHandler.cs b/src/Umbraco.Infrastructure/Search/IDeliveryApiIndexingHandler.cs index f7081572a4..c99eda8ec9 100644 --- a/src/Umbraco.Infrastructure/Search/IDeliveryApiIndexingHandler.cs +++ b/src/Umbraco.Infrastructure/Search/IDeliveryApiIndexingHandler.cs @@ -24,4 +24,13 @@ internal interface IDeliveryApiIndexingHandler /// /// The list of changes by content type ID void HandleContentTypeChanges(IList> changes); + + /// + /// Handles index updates for public access changes + /// + /// + /// Given the current limitations to the distributed public access notifications, this + /// will remove any protected content from the index without being clever about it. + /// + void HandlePublicAccessChanges(); } diff --git a/src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.DeliveryApi.cs b/src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.DeliveryApi.cs index b1b028cde0..f4c9b22663 100644 --- a/src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.DeliveryApi.cs +++ b/src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.DeliveryApi.cs @@ -8,7 +8,9 @@ using Umbraco.Cms.Core.Sync; namespace Umbraco.Cms.Infrastructure.Search; internal sealed class DeliveryApiContentIndexingNotificationHandler : - INotificationHandler, INotificationHandler + INotificationHandler, + INotificationHandler, + INotificationHandler { private readonly IDeliveryApiIndexingHandler _deliveryApiIndexingHandler; @@ -47,6 +49,9 @@ internal sealed class DeliveryApiContentIndexingNotificationHandler : _deliveryApiIndexingHandler.HandleContentTypeChanges(contentTypeChangesById); } + public void Handle(PublicAccessCacheRefresherNotification notification) + => _deliveryApiIndexingHandler.HandlePublicAccessChanges(); + private bool NotificationHandlingIsDisabled() { if (_deliveryApiIndexingHandler.Enabled == false) From 02669e930cd2cbdbcbe50a14a65f970106d2d7e7 Mon Sep 17 00:00:00 2001 From: Nikolaj Geisle <70372949+Zeegaan@users.noreply.github.com> Date: Thu, 11 May 2023 13:32:14 +0200 Subject: [PATCH 13/26] V12: Update dependencies to latest (#14204) * Update dependencies to latest * revert npoco back to 5.5.0 * Updated Npoco and Serilog --------- Co-authored-by: Bjarke Berg --- .../Umbraco.Cms.Persistence.SqlServer.csproj | 2 +- .../Umbraco.Cms.Persistence.Sqlite.csproj | 2 +- src/Umbraco.Core/Umbraco.Core.csproj | 6 +++--- .../ModelsBuilder/RoslynCompiler.cs | 19 +++++++++++++------ .../Umbraco.Infrastructure.csproj | 16 ++++++++-------- .../Umbraco.New.Cms.Infrastructure.csproj | 2 +- .../Umbraco.PublishedCache.NuCache.csproj | 6 ++++-- .../Umbraco.Web.BackOffice.csproj | 4 ++-- .../Umbraco.Web.Common.csproj | 8 ++++---- .../Umbraco.Tests.Benchmarks.csproj | 2 +- .../Umbraco.Tests.Common.csproj | 4 ++-- .../Umbraco.Tests.Integration.csproj | 6 +++--- .../Umbraco.Tests.UnitTests.csproj | 4 ++-- .../Umbraco.JsonSchema.csproj | 2 +- 14 files changed, 46 insertions(+), 37 deletions(-) diff --git a/src/Umbraco.Cms.Persistence.SqlServer/Umbraco.Cms.Persistence.SqlServer.csproj b/src/Umbraco.Cms.Persistence.SqlServer/Umbraco.Cms.Persistence.SqlServer.csproj index 24d17d34a3..37cd0da0f7 100644 --- a/src/Umbraco.Cms.Persistence.SqlServer/Umbraco.Cms.Persistence.SqlServer.csproj +++ b/src/Umbraco.Cms.Persistence.SqlServer/Umbraco.Cms.Persistence.SqlServer.csproj @@ -5,7 +5,7 @@ - + diff --git a/src/Umbraco.Cms.Persistence.Sqlite/Umbraco.Cms.Persistence.Sqlite.csproj b/src/Umbraco.Cms.Persistence.Sqlite/Umbraco.Cms.Persistence.Sqlite.csproj index d9f9ac5123..4021d43c5a 100644 --- a/src/Umbraco.Cms.Persistence.Sqlite/Umbraco.Cms.Persistence.Sqlite.csproj +++ b/src/Umbraco.Cms.Persistence.Sqlite/Umbraco.Cms.Persistence.Sqlite.csproj @@ -5,7 +5,7 @@ - + diff --git a/src/Umbraco.Core/Umbraco.Core.csproj b/src/Umbraco.Core/Umbraco.Core.csproj index 2222b50b52..24f6c86814 100644 --- a/src/Umbraco.Core/Umbraco.Core.csproj +++ b/src/Umbraco.Core/Umbraco.Core.csproj @@ -8,12 +8,12 @@ - + - + - + diff --git a/src/Umbraco.Infrastructure/ModelsBuilder/RoslynCompiler.cs b/src/Umbraco.Infrastructure/ModelsBuilder/RoslynCompiler.cs index 4a0fcdb0e7..bbdf0cbbcc 100644 --- a/src/Umbraco.Infrastructure/ModelsBuilder/RoslynCompiler.cs +++ b/src/Umbraco.Infrastructure/ModelsBuilder/RoslynCompiler.cs @@ -35,13 +35,20 @@ public class RoslynCompiler // - not adding enough of the runtime dependencies OR // - we were explicitly adding the wrong runtime dependencies // ... at least that the gist of what I can tell. - MetadataReference[] refs = - DependencyContext.Default.CompileLibraries - .SelectMany(cl => cl.ResolveReferencePaths()) - .Select(asm => MetadataReference.CreateFromFile(asm)) - .ToArray(); + if (DependencyContext.Default != null) + { + MetadataReference[] refs = + DependencyContext.Default.CompileLibraries + .SelectMany(cl => cl.ResolveReferencePaths()) + .Select(asm => MetadataReference.CreateFromFile(asm)) + .ToArray(); - _refs = refs.ToList(); + _refs = refs.ToList(); + } + else + { + _refs = Enumerable.Empty(); + } } /// diff --git a/src/Umbraco.Infrastructure/Umbraco.Infrastructure.csproj b/src/Umbraco.Infrastructure/Umbraco.Infrastructure.csproj index d11d61c5a8..131e590991 100644 --- a/src/Umbraco.Infrastructure/Umbraco.Infrastructure.csproj +++ b/src/Umbraco.Infrastructure/Umbraco.Infrastructure.csproj @@ -13,32 +13,32 @@ - - + + - + - - + + - + - + - + diff --git a/src/Umbraco.New.Cms.Infrastructure/Umbraco.New.Cms.Infrastructure.csproj b/src/Umbraco.New.Cms.Infrastructure/Umbraco.New.Cms.Infrastructure.csproj index 82159079a4..8f527c6851 100644 --- a/src/Umbraco.New.Cms.Infrastructure/Umbraco.New.Cms.Infrastructure.csproj +++ b/src/Umbraco.New.Cms.Infrastructure/Umbraco.New.Cms.Infrastructure.csproj @@ -12,6 +12,6 @@ - + diff --git a/src/Umbraco.PublishedCache.NuCache/Umbraco.PublishedCache.NuCache.csproj b/src/Umbraco.PublishedCache.NuCache/Umbraco.PublishedCache.NuCache.csproj index 39412ef362..41c0304896 100644 --- a/src/Umbraco.PublishedCache.NuCache/Umbraco.PublishedCache.NuCache.csproj +++ b/src/Umbraco.PublishedCache.NuCache/Umbraco.PublishedCache.NuCache.csproj @@ -7,10 +7,12 @@ - + - + + + diff --git a/src/Umbraco.Web.BackOffice/Umbraco.Web.BackOffice.csproj b/src/Umbraco.Web.BackOffice/Umbraco.Web.BackOffice.csproj index 9fc5109f11..08e89c375f 100644 --- a/src/Umbraco.Web.BackOffice/Umbraco.Web.BackOffice.csproj +++ b/src/Umbraco.Web.BackOffice/Umbraco.Web.BackOffice.csproj @@ -12,8 +12,8 @@ - - + + diff --git a/src/Umbraco.Web.Common/Umbraco.Web.Common.csproj b/src/Umbraco.Web.Common/Umbraco.Web.Common.csproj index 739ad33aa6..09b4486158 100644 --- a/src/Umbraco.Web.Common/Umbraco.Web.Common.csproj +++ b/src/Umbraco.Web.Common/Umbraco.Web.Common.csproj @@ -14,11 +14,11 @@ - - + + - - + + diff --git a/tests/Umbraco.Tests.Benchmarks/Umbraco.Tests.Benchmarks.csproj b/tests/Umbraco.Tests.Benchmarks/Umbraco.Tests.Benchmarks.csproj index d8f31aa9ed..f338b443f3 100644 --- a/tests/Umbraco.Tests.Benchmarks/Umbraco.Tests.Benchmarks.csproj +++ b/tests/Umbraco.Tests.Benchmarks/Umbraco.Tests.Benchmarks.csproj @@ -7,7 +7,7 @@ - + diff --git a/tests/Umbraco.Tests.Common/Umbraco.Tests.Common.csproj b/tests/Umbraco.Tests.Common/Umbraco.Tests.Common.csproj index 214840604e..f965038fc9 100644 --- a/tests/Umbraco.Tests.Common/Umbraco.Tests.Common.csproj +++ b/tests/Umbraco.Tests.Common/Umbraco.Tests.Common.csproj @@ -8,10 +8,10 @@ - + - + diff --git a/tests/Umbraco.Tests.Integration/Umbraco.Tests.Integration.csproj b/tests/Umbraco.Tests.Integration/Umbraco.Tests.Integration.csproj index 43dd49b2a9..4a11fd91ee 100644 --- a/tests/Umbraco.Tests.Integration/Umbraco.Tests.Integration.csproj +++ b/tests/Umbraco.Tests.Integration/Umbraco.Tests.Integration.csproj @@ -11,10 +11,10 @@ - - + + - + diff --git a/tests/Umbraco.Tests.UnitTests/Umbraco.Tests.UnitTests.csproj b/tests/Umbraco.Tests.UnitTests/Umbraco.Tests.UnitTests.csproj index 10ad774c34..e8da0ec606 100644 --- a/tests/Umbraco.Tests.UnitTests/Umbraco.Tests.UnitTests.csproj +++ b/tests/Umbraco.Tests.UnitTests/Umbraco.Tests.UnitTests.csproj @@ -8,8 +8,8 @@ - - + + diff --git a/tools/Umbraco.JsonSchema/Umbraco.JsonSchema.csproj b/tools/Umbraco.JsonSchema/Umbraco.JsonSchema.csproj index 32da8f798a..c9275c6b94 100644 --- a/tools/Umbraco.JsonSchema/Umbraco.JsonSchema.csproj +++ b/tools/Umbraco.JsonSchema/Umbraco.JsonSchema.csproj @@ -7,7 +7,7 @@ - + From 36d72ac85c08fa734c1d8e26df819d82debf26a8 Mon Sep 17 00:00:00 2001 From: Nikolaj Geisle <70372949+Zeegaan@users.noreply.github.com> Date: Thu, 11 May 2023 13:50:47 +0200 Subject: [PATCH 14/26] Fallback to detailed if not telemetry level not set (#14134) Co-authored-by: Zeegaan --- .../ViewModels/Installer/InstallViewModel.cs | 2 +- src/Umbraco.Core/Services/MetricsConsentService.cs | 2 +- src/Umbraco.New.Cms.Core/Models/Installer/InstallData.cs | 2 +- .../Umbraco.Core/Telemetry/TelemetryServiceTests.cs | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Umbraco.Cms.ManagementApi/ViewModels/Installer/InstallViewModel.cs b/src/Umbraco.Cms.ManagementApi/ViewModels/Installer/InstallViewModel.cs index e6639206be..1c83375bdc 100644 --- a/src/Umbraco.Cms.ManagementApi/ViewModels/Installer/InstallViewModel.cs +++ b/src/Umbraco.Cms.ManagementApi/ViewModels/Installer/InstallViewModel.cs @@ -13,5 +13,5 @@ public class InstallViewModel public DatabaseInstallViewModel Database { get; set; } = null!; [JsonConverter(typeof(JsonStringEnumConverter))] - public TelemetryLevel TelemetryLevel { get; set; } = TelemetryLevel.Basic; + public TelemetryLevel TelemetryLevel { get; set; } = TelemetryLevel.Detailed; } diff --git a/src/Umbraco.Core/Services/MetricsConsentService.cs b/src/Umbraco.Core/Services/MetricsConsentService.cs index a3140c4f61..be30458af6 100644 --- a/src/Umbraco.Core/Services/MetricsConsentService.cs +++ b/src/Umbraco.Core/Services/MetricsConsentService.cs @@ -60,7 +60,7 @@ public class MetricsConsentService : IMetricsConsentService if (analyticsLevelString is null || Enum.TryParse(analyticsLevelString, out TelemetryLevel analyticsLevel) is false) { - return TelemetryLevel.Basic; + return TelemetryLevel.Detailed; } return analyticsLevel; diff --git a/src/Umbraco.New.Cms.Core/Models/Installer/InstallData.cs b/src/Umbraco.New.Cms.Core/Models/Installer/InstallData.cs index 2283cf2482..77d0c07477 100644 --- a/src/Umbraco.New.Cms.Core/Models/Installer/InstallData.cs +++ b/src/Umbraco.New.Cms.Core/Models/Installer/InstallData.cs @@ -8,5 +8,5 @@ public class InstallData public DatabaseInstallData Database { get; set; } = null!; - public TelemetryLevel TelemetryLevel { get; set; } = TelemetryLevel.Basic; + public TelemetryLevel TelemetryLevel { get; set; } = TelemetryLevel.Detailed; } diff --git a/tests/Umbraco.Tests.UnitTests/Umbraco.Core/Telemetry/TelemetryServiceTests.cs b/tests/Umbraco.Tests.UnitTests/Umbraco.Core/Telemetry/TelemetryServiceTests.cs index 83aa368d70..d1a41fd414 100644 --- a/tests/Umbraco.Tests.UnitTests/Umbraco.Core/Telemetry/TelemetryServiceTests.cs +++ b/tests/Umbraco.Tests.UnitTests/Umbraco.Core/Telemetry/TelemetryServiceTests.cs @@ -84,7 +84,7 @@ public class TelemetryServiceTests }; var manifestParser = CreateManifestParser(manifests); var metricsConsentService = new Mock(); - metricsConsentService.Setup(x => x.GetConsentLevel()).Returns(TelemetryLevel.Basic); + metricsConsentService.Setup(x => x.GetConsentLevel()).Returns(TelemetryLevel.Detailed); var sut = new TelemetryService( manifestParser, version, @@ -119,7 +119,7 @@ public class TelemetryServiceTests }; var manifestParser = CreateManifestParser(manifests); var metricsConsentService = new Mock(); - metricsConsentService.Setup(x => x.GetConsentLevel()).Returns(TelemetryLevel.Basic); + metricsConsentService.Setup(x => x.GetConsentLevel()).Returns(TelemetryLevel.Detailed); var sut = new TelemetryService( manifestParser, version, From e4c03e0d4e965fafac20fd9dc74b35453f0bb97c Mon Sep 17 00:00:00 2001 From: Nikolaj Geisle <70372949+Zeegaan@users.noreply.github.com> Date: Thu, 11 May 2023 13:58:57 +0200 Subject: [PATCH 15/26] V11: Obsolete action to publish (#14208) * Add obsolete message to ActionToPublish * Obsolete handler for ContentSentTOPublishNotifcation --- src/Umbraco.Core/Actions/ActionToPublish.cs | 1 + src/Umbraco.Core/Events/UserNotificationsHandler.cs | 1 + 2 files changed, 2 insertions(+) diff --git a/src/Umbraco.Core/Actions/ActionToPublish.cs b/src/Umbraco.Core/Actions/ActionToPublish.cs index e7af16bc99..25719e30fc 100644 --- a/src/Umbraco.Core/Actions/ActionToPublish.cs +++ b/src/Umbraco.Core/Actions/ActionToPublish.cs @@ -6,6 +6,7 @@ namespace Umbraco.Cms.Core.Actions; /// /// This action is invoked when children to a document is being sent to published (by an editor without publishrights). /// +[Obsolete("Scheduled for removal in v13")] public class ActionToPublish : IAction { /// diff --git a/src/Umbraco.Core/Events/UserNotificationsHandler.cs b/src/Umbraco.Core/Events/UserNotificationsHandler.cs index 042355630f..4b581788e8 100644 --- a/src/Umbraco.Core/Events/UserNotificationsHandler.cs +++ b/src/Umbraco.Core/Events/UserNotificationsHandler.cs @@ -107,6 +107,7 @@ public sealed class UserNotificationsHandler : _notifier.Notify(_actions.GetAction(), updatedEntities.ToArray()); } + [Obsolete("Scheduled for removal in v13")] public void Handle(ContentSentToPublishNotification notification) => _notifier.Notify(_actions.GetAction(), notification.Entity); From 8d4e55f8bbf5ee7662e23941f58d031a38607504 Mon Sep 17 00:00:00 2001 From: Nikolaj Geisle <70372949+Zeegaan@users.noreply.github.com> Date: Thu, 11 May 2023 14:39:21 +0200 Subject: [PATCH 16/26] Enable ForceCreateDatabase for SqlServer (#14234) --- .../Services/SqlServerDatabaseProviderMetadata.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Umbraco.Cms.Persistence.SqlServer/Services/SqlServerDatabaseProviderMetadata.cs b/src/Umbraco.Cms.Persistence.SqlServer/Services/SqlServerDatabaseProviderMetadata.cs index edb44700d8..89612b3603 100644 --- a/src/Umbraco.Cms.Persistence.SqlServer/Services/SqlServerDatabaseProviderMetadata.cs +++ b/src/Umbraco.Cms.Persistence.SqlServer/Services/SqlServerDatabaseProviderMetadata.cs @@ -50,7 +50,7 @@ public class SqlServerDatabaseProviderMetadata : IDatabaseProviderMetadata public bool RequiresConnectionTest => true; /// - public bool ForceCreateDatabase => false; + public bool ForceCreateDatabase => true; /// public bool CanRecognizeConnectionString(string? connectionString) From 06bd728c18b765d6dc509f14837af9981232a922 Mon Sep 17 00:00:00 2001 From: Kenn Jacobsen Date: Thu, 11 May 2023 14:42:57 +0200 Subject: [PATCH 17/26] Only index content for the Delivery API when it is enabled (#14235) --- .../DeliveryApiContentIndexPopulator.cs | 35 ++++++++++++++++++- ...IndexingNotificationHandler.DeliveryApi.cs | 26 ++++++++++++-- 2 files changed, 57 insertions(+), 4 deletions(-) diff --git a/src/Umbraco.Infrastructure/Examine/DeliveryApiContentIndexPopulator.cs b/src/Umbraco.Infrastructure/Examine/DeliveryApiContentIndexPopulator.cs index 12f833f3d0..7825925170 100644 --- a/src/Umbraco.Infrastructure/Examine/DeliveryApiContentIndexPopulator.cs +++ b/src/Umbraco.Infrastructure/Examine/DeliveryApiContentIndexPopulator.cs @@ -1,5 +1,8 @@ using Examine; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Umbraco.Cms.Core; +using Umbraco.Cms.Core.Configuration.Models; namespace Umbraco.Cms.Infrastructure.Examine; @@ -7,13 +10,20 @@ internal sealed class DeliveryApiContentIndexPopulator : IndexPopulator { private readonly IDeliveryApiContentIndexValueSetBuilder _deliveryContentIndexValueSetBuilder; private readonly IDeliveryApiContentIndexHelper _deliveryApiContentIndexHelper; + private readonly ILogger _logger; + private DeliveryApiSettings _deliveryApiSettings; public DeliveryApiContentIndexPopulator( IDeliveryApiContentIndexValueSetBuilder deliveryContentIndexValueSetBuilder, - IDeliveryApiContentIndexHelper deliveryApiContentIndexHelper) + IDeliveryApiContentIndexHelper deliveryApiContentIndexHelper, + ILogger logger, + IOptionsMonitor deliveryApiSettings) { _deliveryContentIndexValueSetBuilder = deliveryContentIndexValueSetBuilder; _deliveryApiContentIndexHelper = deliveryApiContentIndexHelper; + _logger = logger; + _deliveryApiSettings = deliveryApiSettings.CurrentValue; + deliveryApiSettings.OnChange(settings => _deliveryApiSettings = settings); RegisterIndex(Constants.UmbracoIndexes.DeliveryApiContentIndexName); } @@ -24,6 +34,11 @@ internal sealed class DeliveryApiContentIndexPopulator : IndexPopulator return; } + if (_deliveryApiSettings.Enabled is false) + { + return; + } + _deliveryApiContentIndexHelper.EnumerateApplicableDescendantsForContentIndex( Constants.System.Root, descendants => @@ -35,4 +50,22 @@ internal sealed class DeliveryApiContentIndexPopulator : IndexPopulator } }); } + + public override bool IsRegistered(IIndex index) + { + if (_deliveryApiSettings.Enabled) + { + return base.IsRegistered(index); + } + + // IsRegistered() is invoked for all indexes; only log a message when it's invoked for the Delivery API content index + if (index.Name is Constants.UmbracoIndexes.DeliveryApiContentIndexName) + { + // IsRegistered() is currently invoked only when Umbraco starts and when loading the Examine dashboard, + // so we won't be flooding the logs with info messages here + _logger.LogInformation("The Delivery API is not enabled, no indexing will performed for the Delivery API content index."); + } + + return false; + } } diff --git a/src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.DeliveryApi.cs b/src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.DeliveryApi.cs index f4c9b22663..1750310e71 100644 --- a/src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.DeliveryApi.cs +++ b/src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.DeliveryApi.cs @@ -1,4 +1,7 @@ -using Umbraco.Cms.Core.Cache; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Umbraco.Cms.Core.Cache; +using Umbraco.Cms.Core.Configuration.Models; using Umbraco.Cms.Core.Events; using Umbraco.Cms.Core.Models; using Umbraco.Cms.Core.Notifications; @@ -13,9 +16,19 @@ internal sealed class DeliveryApiContentIndexingNotificationHandler : INotificationHandler { private readonly IDeliveryApiIndexingHandler _deliveryApiIndexingHandler; + private readonly ILogger _logger; + private DeliveryApiSettings _deliveryApiSettings; - public DeliveryApiContentIndexingNotificationHandler(IDeliveryApiIndexingHandler deliveryApiIndexingHandler) - => _deliveryApiIndexingHandler = deliveryApiIndexingHandler; + public DeliveryApiContentIndexingNotificationHandler( + IDeliveryApiIndexingHandler deliveryApiIndexingHandler, + ILogger logger, + IOptionsMonitor deliveryApiSettings) + { + _deliveryApiIndexingHandler = deliveryApiIndexingHandler; + _logger = logger; + _deliveryApiSettings = deliveryApiSettings.CurrentValue; + deliveryApiSettings.OnChange(settings => _deliveryApiSettings = settings); + } public void Handle(ContentCacheRefresherNotification notification) { @@ -54,6 +67,13 @@ internal sealed class DeliveryApiContentIndexingNotificationHandler : private bool NotificationHandlingIsDisabled() { + if (_deliveryApiSettings.Enabled is false) + { + // using debug logging here since this happens on every content cache refresh and we don't want to flood the log + _logger.LogDebug("Delivery API index notification handling is suspended while the Delivery API is disabled."); + return true; + } + if (_deliveryApiIndexingHandler.Enabled == false) { return true; From 36fe2f7e4932bd097fd1332b9267b1c0ecae02f7 Mon Sep 17 00:00:00 2001 From: Andreas Zerbst <73799582+andr317c@users.noreply.github.com> Date: Thu, 11 May 2023 15:02:52 +0200 Subject: [PATCH 18/26] V12: Remove test.only for acceptance tests (#14232) * Remove test.only * Try quick fix for test * Fixed failing test so it now locates the correct class. Updated the locators aswell --------- Co-authored-by: Nikolaj --- .../tests/DefaultConfig/Tabs/tabs.spec.ts | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Tabs/tabs.spec.ts b/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Tabs/tabs.spec.ts index 873ae3bad6..93d8150340 100644 --- a/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Tabs/tabs.spec.ts +++ b/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Tabs/tabs.spec.ts @@ -46,12 +46,15 @@ test.describe('Tabs', () => { await openDocTypeFolder(umbracoUi, page); } - test.only('Click dashboard tabs', async ({umbracoUi, page}) => { - await umbracoUi.goToSection('content'); - await page.locator('[data-element="tab-contentRedirectManager"] > button').click(); - expect(page.locator('.redirecturlsearch')).not.toBeNull(); - await page.locator('[data-element="tab-contentIntro"] > button').click(); - await expect(page.locator('[data-element="tab-contentIntro"]')).toHaveClass('umb-tab ng-scope umb-tab--active'); + test('Click dashboard tabs', async ({umbracoUi, page}) => { + await umbracoUi.goToSection(ConstantHelper.sections.content); + await umbracoUi.clickDataElementByElementName('tab-contentRedirectManager'); + await expect(page.locator('[data-element="tab-content-contentRedirectManager"]')).toBeVisible(); + await umbracoUi.clickDataElementByElementName('tab-contentIntro'); + + // Assert + await expect(page.locator('[data-element="tab-contentIntro"]')).toHaveClass(/umb-tab--active/); + await expect(page.locator('[data-element="tab-content-contentIntro"]')).toBeVisible(); }); test('Create tab', async ({umbracoUi, umbracoApi, page}) => { From 487e85cacdaea03c85141e89d2bcf73caeab8c12 Mon Sep 17 00:00:00 2001 From: Nikolaj Geisle <70372949+Zeegaan@users.noreply.github.com> Date: Fri, 12 May 2023 09:25:19 +0200 Subject: [PATCH 19/26] Entity Framework Core Support (#14109) * Add UmbracoEFCore project * Add EFCore composer * Add Locking Mechanisms * Add scope interfaces * Add excecute scalar extension method * fix up query in locking mechanism * Add scoping * Add scoping * Add test DbContext classes * add locking test of EFCore * Creat ScopedFileSystemsTests * Add EFCoreScopeInfrastructureScopeLockTests * Add EFCoreScopeInfrastructureScopeTests * Add EFCoreScopeNotificationsTest.cs * Add EFCoreScopeTest.cs * Remake AddUmbracoEFCoreContext to use connection string * Remove unused code from extension method * Reference EFCore reference to Cms.csproj * Remove unused parameter * Dont have default implementation, breaking change instead * Add compatability suppression file * Updated EFCore packages * Use timespan for timeout * Allow overriding default EF Core actions * Option lifetime needs to be singleton * Use given timeout in database call * dont use timespan.zero, use null instead * Use variable timeout * Update test to use locking mechanism * Remove unneccesary duplicate code * Change to catch proper exception number --------- Co-authored-by: Zeegaan Co-authored-by: Bjarke Berg --- .../Extensions/DbContextExtensions.cs | 53 ++ ...mbracoEFCoreServiceCollectionExtensions.cs | 57 ++ ...ServerEFCoreDistributedLockingMechanism.cs | 185 +++++ ...SqliteEFCoreDistributedLockingMechanism.cs | 181 +++++ .../Scoping/AmbientEFCoreScopeStack.cs | 40 + .../Scoping/EFCoreDetachableScope.cs | 110 +++ .../Scoping/EFCoreScope.cs | 237 ++++++ .../Scoping/EFCoreScopeAccessor.cs | 14 + .../Scoping/EFCoreScopeProvider.cs | 207 +++++ .../Scoping/IAmbientEfCoreScopeStack.cs | 12 + .../Scoping/IEFCoreScope.cs | 30 + .../Scoping/IEFCoreScopeAccessor.cs | 10 + .../Scoping/IEFCoreScopeProvider.cs | 16 + .../Umbraco.Cms.Persistence.EFCore.csproj | 25 + src/Umbraco.Cms/Umbraco.Cms.csproj | 1 + .../CompatibilitySuppressions.xml | 9 +- src/Umbraco.Core/Scoping/CoreScope.cs | 272 +++++++ src/Umbraco.Core/Scoping/ICoreScope.cs | 2 + src/Umbraco.Core/Scoping/ILockingMechanism.cs | 58 ++ src/Umbraco.Core/Scoping/LockingMechanism.cs | 433 +++++++++++ .../Scoping/IAmbientScopeContextStack.cs | 2 +- src/Umbraco.Infrastructure/Scoping/Scope.cs | 705 +----------------- .../Scoping/ScopeContext.cs | 2 +- .../Scoping/ScopeProvider.cs | 6 +- .../UmbracoBuilderExtensions.cs | 51 +- .../DbContext/TestUmbracoDbContext.cs | 31 + .../DbContext/UmbracoLock.cs | 10 + .../Scoping/EFCoreLockTests.cs | 403 ++++++++++ ...EFCoreScopeInfrastructureScopeLockTests.cs | 139 ++++ .../EFCoreScopeInfrastructureScopeTests.cs | 208 ++++++ .../Scoping/EFCoreScopeNotificationsTest.cs | 212 ++++++ .../Scoping/EFCoreScopeTest.cs | 670 +++++++++++++++++ .../Scoping/EFCoreScopedFileSystemsTests.cs | 211 ++++++ .../Umbraco.Tests.Integration.csproj | 1 + umbraco.sln | 8 + 35 files changed, 3930 insertions(+), 681 deletions(-) create mode 100644 src/Umbraco.Cms.Persistence.EFCore/Extensions/DbContextExtensions.cs create mode 100644 src/Umbraco.Cms.Persistence.EFCore/Extensions/UmbracoEFCoreServiceCollectionExtensions.cs create mode 100644 src/Umbraco.Cms.Persistence.EFCore/Locking/SqlServerEFCoreDistributedLockingMechanism.cs create mode 100644 src/Umbraco.Cms.Persistence.EFCore/Locking/SqliteEFCoreDistributedLockingMechanism.cs create mode 100644 src/Umbraco.Cms.Persistence.EFCore/Scoping/AmbientEFCoreScopeStack.cs create mode 100644 src/Umbraco.Cms.Persistence.EFCore/Scoping/EFCoreDetachableScope.cs create mode 100644 src/Umbraco.Cms.Persistence.EFCore/Scoping/EFCoreScope.cs create mode 100644 src/Umbraco.Cms.Persistence.EFCore/Scoping/EFCoreScopeAccessor.cs create mode 100644 src/Umbraco.Cms.Persistence.EFCore/Scoping/EFCoreScopeProvider.cs create mode 100644 src/Umbraco.Cms.Persistence.EFCore/Scoping/IAmbientEfCoreScopeStack.cs create mode 100644 src/Umbraco.Cms.Persistence.EFCore/Scoping/IEFCoreScope.cs create mode 100644 src/Umbraco.Cms.Persistence.EFCore/Scoping/IEFCoreScopeAccessor.cs create mode 100644 src/Umbraco.Cms.Persistence.EFCore/Scoping/IEFCoreScopeProvider.cs create mode 100644 src/Umbraco.Cms.Persistence.EFCore/Umbraco.Cms.Persistence.EFCore.csproj create mode 100644 src/Umbraco.Core/Scoping/CoreScope.cs create mode 100644 src/Umbraco.Core/Scoping/ILockingMechanism.cs create mode 100644 src/Umbraco.Core/Scoping/LockingMechanism.cs create mode 100644 tests/Umbraco.Tests.Integration/Umbraco.Persistence.EFCore/DbContext/TestUmbracoDbContext.cs create mode 100644 tests/Umbraco.Tests.Integration/Umbraco.Persistence.EFCore/DbContext/UmbracoLock.cs create mode 100644 tests/Umbraco.Tests.Integration/Umbraco.Persistence.EFCore/Scoping/EFCoreLockTests.cs create mode 100644 tests/Umbraco.Tests.Integration/Umbraco.Persistence.EFCore/Scoping/EFCoreScopeInfrastructureScopeLockTests.cs create mode 100644 tests/Umbraco.Tests.Integration/Umbraco.Persistence.EFCore/Scoping/EFCoreScopeInfrastructureScopeTests.cs create mode 100644 tests/Umbraco.Tests.Integration/Umbraco.Persistence.EFCore/Scoping/EFCoreScopeNotificationsTest.cs create mode 100644 tests/Umbraco.Tests.Integration/Umbraco.Persistence.EFCore/Scoping/EFCoreScopeTest.cs create mode 100644 tests/Umbraco.Tests.Integration/Umbraco.Persistence.EFCore/Scoping/EFCoreScopedFileSystemsTests.cs diff --git a/src/Umbraco.Cms.Persistence.EFCore/Extensions/DbContextExtensions.cs b/src/Umbraco.Cms.Persistence.EFCore/Extensions/DbContextExtensions.cs new file mode 100644 index 0000000000..573f57e75f --- /dev/null +++ b/src/Umbraco.Cms.Persistence.EFCore/Extensions/DbContextExtensions.cs @@ -0,0 +1,53 @@ +using System.Data; +using System.Data.Common; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Storage; + +namespace Umbraco.Extensions; + +public static class DbContextExtensions +{ + /// + /// Executes a raw SQL query and returns the result. + /// + /// The database. + /// The sql query. + /// The list of db parameters. + /// The command type. + /// The amount of time to wait before the command times out. + /// the type to return. + /// Returns an object of the given type T. + public static async Task ExecuteScalarAsync(this DatabaseFacade database, string sql, List? parameters = null, CommandType commandType = CommandType.Text, TimeSpan? commandTimeOut = null) + { + ArgumentNullException.ThrowIfNull(database); + ArgumentNullException.ThrowIfNull(sql); + + await using DbCommand dbCommand = database.GetDbConnection().CreateCommand(); + + if (database.CurrentTransaction is not null) + { + dbCommand.Transaction = database.CurrentTransaction.GetDbTransaction(); + } + + if (dbCommand.Connection?.State != ConnectionState.Open) + { + await dbCommand.Connection!.OpenAsync(); + } + + dbCommand.CommandText = sql; + dbCommand.CommandType = commandType; + if (commandTimeOut is not null) + { + dbCommand.CommandTimeout = (int)commandTimeOut.Value.TotalSeconds; + } + + if (parameters != null) + { + dbCommand.Parameters.AddRange(parameters.ToArray()); + } + + var result = await dbCommand.ExecuteScalarAsync(); + return (T?)result; + } +} diff --git a/src/Umbraco.Cms.Persistence.EFCore/Extensions/UmbracoEFCoreServiceCollectionExtensions.cs b/src/Umbraco.Cms.Persistence.EFCore/Extensions/UmbracoEFCoreServiceCollectionExtensions.cs new file mode 100644 index 0000000000..857661fd83 --- /dev/null +++ b/src/Umbraco.Cms.Persistence.EFCore/Extensions/UmbracoEFCoreServiceCollectionExtensions.cs @@ -0,0 +1,57 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Umbraco.Cms.Core.DistributedLocking; +using Umbraco.Cms.Persistence.EFCore.Locking; +using Umbraco.Cms.Persistence.EFCore.Scoping; + +namespace Umbraco.Extensions; + +public static class UmbracoEFCoreServiceCollectionExtensions +{ + public delegate void DefaultEFCoreOptionsAction(DbContextOptionsBuilder options, string? providerName, string? connectionString); + + public static IServiceCollection AddUmbracoEFCoreContext(this IServiceCollection services, string connectionString, string providerName, DefaultEFCoreOptionsAction? defaultEFCoreOptionsAction = null) + where T : DbContext + { + defaultEFCoreOptionsAction ??= DefaultOptionsAction; + + services.AddDbContext( + options => + { + defaultEFCoreOptionsAction(options, providerName, connectionString); + }, + optionsLifetime: ServiceLifetime.Singleton); + + services.AddDbContextFactory(options => + { + defaultEFCoreOptionsAction(options, providerName, connectionString); + }); + + services.AddUnique, AmbientEFCoreScopeStack>(); + services.AddUnique, EFCoreScopeAccessor>(); + services.AddUnique, EFCoreScopeProvider>(); + services.AddSingleton>(); + services.AddSingleton>(); + + return services; + } + + private static void DefaultOptionsAction(DbContextOptionsBuilder options, string? providerName, string? connectionString) + { + if (connectionString.IsNullOrWhiteSpace()) + { + return; + } + + switch (providerName) + { + case "Microsoft.Data.Sqlite": + options.UseSqlite(connectionString); + break; + case "Microsoft.Data.SqlClient": + options.UseSqlServer(connectionString); + break; + } + } +} diff --git a/src/Umbraco.Cms.Persistence.EFCore/Locking/SqlServerEFCoreDistributedLockingMechanism.cs b/src/Umbraco.Cms.Persistence.EFCore/Locking/SqlServerEFCoreDistributedLockingMechanism.cs new file mode 100644 index 0000000000..d5d83f8ecf --- /dev/null +++ b/src/Umbraco.Cms.Persistence.EFCore/Locking/SqlServerEFCoreDistributedLockingMechanism.cs @@ -0,0 +1,185 @@ +using System.Data; +using Microsoft.Data.SqlClient; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Storage; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Umbraco.Cms.Core.Configuration.Models; +using Umbraco.Cms.Core.DistributedLocking; +using Umbraco.Cms.Core.DistributedLocking.Exceptions; +using Umbraco.Cms.Core.Exceptions; +using Umbraco.Cms.Persistence.EFCore.Scoping; +using Umbraco.Extensions; + +namespace Umbraco.Cms.Persistence.EFCore.Locking; + +internal class SqlServerEFCoreDistributedLockingMechanism : IDistributedLockingMechanism + where T : DbContext +{ + private readonly IOptionsMonitor _connectionStrings; + private readonly IOptionsMonitor _globalSettings; + private readonly ILogger> _logger; + private readonly Lazy> _scopeAccessor; // Hooray it's a circular dependency. + + /// + /// Initializes a new instance of the class. + /// + public SqlServerEFCoreDistributedLockingMechanism( + ILogger> logger, + Lazy> scopeAccessor, + IOptionsMonitor globalSettings, + IOptionsMonitor connectionStrings) + { + _logger = logger; + _scopeAccessor = scopeAccessor; + _globalSettings = globalSettings; + _connectionStrings = connectionStrings; + } + + public bool HasActiveRelatedScope => _scopeAccessor.Value.AmbientScope is not null; + + /// + public bool Enabled => _connectionStrings.CurrentValue.IsConnectionStringConfigured() && + string.Equals(_connectionStrings.CurrentValue.ProviderName, "Microsoft.Data.SqlClient", StringComparison.InvariantCultureIgnoreCase) && _scopeAccessor.Value.AmbientScope is not null; + + /// + public IDistributedLock ReadLock(int lockId, TimeSpan? obtainLockTimeout = null) + { + obtainLockTimeout ??= _globalSettings.CurrentValue.DistributedLockingReadLockDefaultTimeout; + return new SqlServerDistributedLock(this, lockId, DistributedLockType.ReadLock, obtainLockTimeout.Value); + } + + /// + public IDistributedLock WriteLock(int lockId, TimeSpan? obtainLockTimeout = null) + { + obtainLockTimeout ??= _globalSettings.CurrentValue.DistributedLockingWriteLockDefaultTimeout; + return new SqlServerDistributedLock(this, lockId, DistributedLockType.WriteLock, obtainLockTimeout.Value); + } + + private class SqlServerDistributedLock : IDistributedLock + { + private readonly SqlServerEFCoreDistributedLockingMechanism _parent; + private readonly TimeSpan _timeout; + + public SqlServerDistributedLock( + SqlServerEFCoreDistributedLockingMechanism parent, + int lockId, + DistributedLockType lockType, + TimeSpan timeout) + { + _parent = parent; + _timeout = timeout; + LockId = lockId; + LockType = lockType; + + _parent._logger.LogDebug("Requesting {lockType} for id {id}", LockType, LockId); + + try + { + switch (lockType) + { + case DistributedLockType.ReadLock: + ObtainReadLock(); + break; + case DistributedLockType.WriteLock: + ObtainWriteLock(); + break; + default: + throw new ArgumentOutOfRangeException(nameof(lockType), lockType, @"Unsupported lockType"); + } + } + catch (SqlException ex) when (ex.Number == 1222) + { + if (LockType == DistributedLockType.ReadLock) + { + throw new DistributedReadLockTimeoutException(LockId); + } + + throw new DistributedWriteLockTimeoutException(LockId); + } + + _parent._logger.LogDebug("Acquired {lockType} for id {id}", LockType, LockId); + } + + public int LockId { get; } + + public DistributedLockType LockType { get; } + + public void Dispose() => + // Mostly no op, cleaned up by completing transaction in scope. + _parent._logger.LogDebug("Dropped {lockType} for id {id}", LockType, LockId); + + public override string ToString() + => $"SqlServerDistributedLock({LockId}, {LockType}"; + + private void ObtainReadLock() + { + IEfCoreScope? scope = _parent._scopeAccessor.Value.AmbientScope; + + if (scope is null) + { + throw new PanicException("No ambient scope"); + } + + scope.ExecuteWithContextAsync(async dbContext => + { + if (dbContext.Database.CurrentTransaction is null) + { + throw new InvalidOperationException( + "SqlServerDistributedLockingMechanism requires a transaction to function."); + } + + if (dbContext.Database.CurrentTransaction.GetDbTransaction().IsolationLevel < + IsolationLevel.ReadCommitted) + { + throw new InvalidOperationException( + "A transaction with minimum ReadCommitted isolation level is required."); + } + + await dbContext.Database.ExecuteSqlRawAsync($"SET LOCK_TIMEOUT {(int)_timeout.TotalMilliseconds};"); + + var number = await dbContext.Database.ExecuteScalarAsync($"SELECT value FROM dbo.umbracoLock WITH (REPEATABLEREAD) WHERE id={LockId}"); + + if (number == null) + { + // ensure we are actually locking! + throw new ArgumentException(@$"LockObject with id={LockId} does not exist.", nameof(LockId)); + } + }).GetAwaiter().GetResult(); + } + + private void ObtainWriteLock() + { + IEfCoreScope? scope = _parent._scopeAccessor.Value.AmbientScope; + if (scope is null) + { + throw new PanicException("No ambient scope"); + } + + scope.ExecuteWithContextAsync(async dbContext => + { + if (dbContext.Database.CurrentTransaction is null) + { + throw new InvalidOperationException( + "SqlServerDistributedLockingMechanism requires a transaction to function."); + } + + if (dbContext.Database.CurrentTransaction.GetDbTransaction().IsolationLevel < IsolationLevel.ReadCommitted) + { + throw new InvalidOperationException( + "A transaction with minimum ReadCommitted isolation level is required."); + } + + await dbContext.Database.ExecuteSqlRawAsync($"SET LOCK_TIMEOUT {(int)_timeout.TotalMilliseconds};"); + + var rowsAffected = await dbContext.Database.ExecuteSqlAsync(@$"UPDATE umbracoLock WITH (REPEATABLEREAD) SET value = (CASE WHEN (value=1) THEN -1 ELSE 1 END) WHERE id={LockId}"); + + if (rowsAffected == 0) + { + // ensure we are actually locking! + throw new ArgumentException($"LockObject with id={LockId} does not exist."); + } + }).GetAwaiter().GetResult(); + } + } +} diff --git a/src/Umbraco.Cms.Persistence.EFCore/Locking/SqliteEFCoreDistributedLockingMechanism.cs b/src/Umbraco.Cms.Persistence.EFCore/Locking/SqliteEFCoreDistributedLockingMechanism.cs new file mode 100644 index 0000000000..8d92ec0e03 --- /dev/null +++ b/src/Umbraco.Cms.Persistence.EFCore/Locking/SqliteEFCoreDistributedLockingMechanism.cs @@ -0,0 +1,181 @@ +using System.Globalization; +using Microsoft.Data.Sqlite; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using SQLitePCL; +using Umbraco.Cms.Core.Configuration.Models; +using Umbraco.Cms.Core.DistributedLocking; +using Umbraco.Cms.Core.DistributedLocking.Exceptions; +using Umbraco.Cms.Core.Exceptions; +using Umbraco.Cms.Persistence.EFCore.Scoping; +using Umbraco.Extensions; + +namespace Umbraco.Cms.Persistence.EFCore.Locking; + +internal class SqliteEFCoreDistributedLockingMechanism : IDistributedLockingMechanism + where T : DbContext +{ + private readonly IOptionsMonitor _connectionStrings; + private readonly IOptionsMonitor _globalSettings; + private readonly ILogger> _logger; + private readonly Lazy> _efCoreScopeAccessor; + + public SqliteEFCoreDistributedLockingMechanism( + ILogger> logger, + Lazy> efCoreScopeAccessor, + IOptionsMonitor globalSettings, + IOptionsMonitor connectionStrings) + { + _logger = logger; + _efCoreScopeAccessor = efCoreScopeAccessor; + _connectionStrings = connectionStrings; + _globalSettings = globalSettings; + } + + public bool HasActiveRelatedScope => _efCoreScopeAccessor.Value.AmbientScope is not null; + + /// + public bool Enabled => _connectionStrings.CurrentValue.IsConnectionStringConfigured() && + string.Equals(_connectionStrings.CurrentValue.ProviderName, "Microsoft.Data.Sqlite", StringComparison.InvariantCultureIgnoreCase) && _efCoreScopeAccessor.Value.AmbientScope is not null; + + // With journal_mode=wal we can always read a snapshot. + public IDistributedLock ReadLock(int lockId, TimeSpan? obtainLockTimeout = null) + { + obtainLockTimeout ??= _globalSettings.CurrentValue.DistributedLockingReadLockDefaultTimeout; + return new SqliteDistributedLock(this, lockId, DistributedLockType.ReadLock, obtainLockTimeout.Value); + } + + // With journal_mode=wal only a single write transaction can exist at a time. + public IDistributedLock WriteLock(int lockId, TimeSpan? obtainLockTimeout = null) + { + obtainLockTimeout ??= _globalSettings.CurrentValue.DistributedLockingWriteLockDefaultTimeout; + return new SqliteDistributedLock(this, lockId, DistributedLockType.WriteLock, obtainLockTimeout.Value); + } + + private class SqliteDistributedLock : IDistributedLock + { + private readonly SqliteEFCoreDistributedLockingMechanism _parent; + private readonly TimeSpan _timeout; + + public SqliteDistributedLock( + SqliteEFCoreDistributedLockingMechanism parent, + int lockId, + DistributedLockType lockType, + TimeSpan timeout) + { + _parent = parent; + _timeout = timeout; + LockId = lockId; + LockType = lockType; + + _parent._logger.LogDebug("Requesting {lockType} for id {id}", LockType, LockId); + + try + { + switch (lockType) + { + case DistributedLockType.ReadLock: + ObtainReadLock(); + break; + case DistributedLockType.WriteLock: + ObtainWriteLock(); + break; + default: + throw new ArgumentOutOfRangeException(nameof(lockType), lockType, @"Unsupported lockType"); + } + } + catch (SqliteException ex) when (ex.SqliteErrorCode == SQLitePCL.raw.SQLITE_BUSY) + { + if (LockType == DistributedLockType.ReadLock) + { + throw new DistributedReadLockTimeoutException(LockId); + } + + throw new DistributedWriteLockTimeoutException(LockId); + } + + _parent._logger.LogDebug("Acquired {lockType} for id {id}", LockType, LockId); + } + + public int LockId { get; } + + public DistributedLockType LockType { get; } + + public void Dispose() => + // Mostly no op, cleaned up by completing transaction in scope. + _parent._logger.LogDebug("Dropped {lockType} for id {id}", LockType, LockId); + + public override string ToString() + => $"SqliteDistributedLock({LockId})"; + + // Can always obtain a read lock (snapshot isolation in wal mode) + // Mostly no-op just check that we didn't end up ReadUncommitted for real. + private void ObtainReadLock() + { + IEfCoreScope? efCoreScope = _parent._efCoreScopeAccessor.Value.AmbientScope; + + if (efCoreScope is null) + { + throw new PanicException("No current ambient scope"); + } + + efCoreScope.ExecuteWithContextAsync(async database => + { + if (database.Database.CurrentTransaction is null) + { + throw new InvalidOperationException( + "SqliteDistributedLockingMechanism requires a transaction to function."); + } + }); + } + + // Only one writer is possible at a time + // lock occurs for entire database as opposed to row/table. + private void ObtainWriteLock() + { + IEfCoreScope? efCoreScope = _parent._efCoreScopeAccessor.Value.AmbientScope; + + if (efCoreScope is null) + { + throw new PanicException("No ambient scope"); + } + + efCoreScope.ExecuteWithContextAsync(async database => + { + if (database.Database.CurrentTransaction is null) + { + throw new InvalidOperationException( + "SqliteDistributedLockingMechanism requires a transaction to function."); + } + + var query = @$"UPDATE umbracoLock SET value = (CASE WHEN (value=1) THEN -1 ELSE 1 END) WHERE id = {LockId.ToString(CultureInfo.InvariantCulture)}"; + + try + { + // imagine there is an existing writer, whilst elapsed time is < command timeout sqlite will busy loop + // Important to note that if this value == 0 then Command.DefaultTimeout (30s) is used. + // Math.Ceiling such that (0 < totalseconds < 1) is rounded up to 1. + database.Database.SetCommandTimeout((int)Math.Ceiling(_timeout.TotalSeconds)); + var i = await database.Database.ExecuteScalarAsync(query); + + if (i == 0) + { + // ensure we are actually locking! + throw new ArgumentException($"LockObject with id={LockId} does not exist."); + } + } + catch (SqliteException ex) when (IsBusyOrLocked(ex)) + { + throw new DistributedWriteLockTimeoutException(LockId); + } + }); + } + + private bool IsBusyOrLocked(SqliteException ex) => + ex.SqliteErrorCode + is raw.SQLITE_BUSY + or raw.SQLITE_LOCKED + or raw.SQLITE_LOCKED_SHAREDCACHE; + } +} diff --git a/src/Umbraco.Cms.Persistence.EFCore/Scoping/AmbientEFCoreScopeStack.cs b/src/Umbraco.Cms.Persistence.EFCore/Scoping/AmbientEFCoreScopeStack.cs new file mode 100644 index 0000000000..dc948f36f3 --- /dev/null +++ b/src/Umbraco.Cms.Persistence.EFCore/Scoping/AmbientEFCoreScopeStack.cs @@ -0,0 +1,40 @@ +using System.Collections.Concurrent; +using Microsoft.EntityFrameworkCore; + +namespace Umbraco.Cms.Persistence.EFCore.Scoping; + +public class AmbientEFCoreScopeStack : IAmbientEFCoreScopeStack where TDbContext : DbContext +{ + + private static AsyncLocal>> _stack = new(); + + public IEfCoreScope? AmbientScope + { + get + { + if (_stack.Value?.TryPeek(out IEfCoreScope? ambientScope) ?? false) + { + return ambientScope; + } + + return null; + } + } + + public IEfCoreScope Pop() + { + if (_stack.Value?.TryPop(out IEfCoreScope? ambientScope) ?? false) + { + return ambientScope; + } + + throw new InvalidOperationException("No AmbientScope was found."); + } + + public void Push(IEfCoreScope scope) + { + _stack.Value ??= new ConcurrentStack>(); + + _stack.Value.Push(scope); + } +} diff --git a/src/Umbraco.Cms.Persistence.EFCore/Scoping/EFCoreDetachableScope.cs b/src/Umbraco.Cms.Persistence.EFCore/Scoping/EFCoreDetachableScope.cs new file mode 100644 index 0000000000..e23a830e3f --- /dev/null +++ b/src/Umbraco.Cms.Persistence.EFCore/Scoping/EFCoreDetachableScope.cs @@ -0,0 +1,110 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using Umbraco.Cms.Core.DistributedLocking; +using Umbraco.Cms.Core.Events; +using Umbraco.Cms.Core.IO; +using Umbraco.Cms.Core.Scoping; + +namespace Umbraco.Cms.Persistence.EFCore.Scoping; + +internal class EFCoreDetachableScope : EFCoreScope where TDbContext : DbContext +{ + private readonly IEFCoreScopeAccessor _efCoreScopeAccessor; + private readonly EFCoreScopeProvider _efCoreScopeProvider; + + public EFCoreDetachableScope( + IDistributedLockingMechanismFactory distributedLockingMechanismFactory, + ILoggerFactory loggerFactory, + IEFCoreScopeAccessor efCoreScopeAccessor, + FileSystems fileSystems, + IEFCoreScopeProvider efCoreScopeProvider, + IScopeContext? scopeContext, + IEventAggregator eventAggregator, + IDbContextFactory dbContextFactory, + RepositoryCacheMode repositoryCacheMode = RepositoryCacheMode.Unspecified, + bool? scopeFileSystems = null) + : base( + distributedLockingMechanismFactory, + loggerFactory, + efCoreScopeAccessor, + fileSystems, + efCoreScopeProvider, + scopeContext, + eventAggregator, + dbContextFactory, + repositoryCacheMode, + scopeFileSystems) + { + if (scopeContext is not null) + { + throw new ArgumentException("Cannot set context on detachable scope.", nameof(scopeContext)); + } + + _efCoreScopeAccessor = efCoreScopeAccessor; + _efCoreScopeProvider = (EFCoreScopeProvider)efCoreScopeProvider; + + Detachable = true; + + ScopeContext = new ScopeContext(); + } + + public EFCoreDetachableScope( + IDistributedLockingMechanismFactory distributedLockingMechanismFactory, + ILoggerFactory loggerFactory, + IEFCoreScopeAccessor efCoreScopeAccessor, + FileSystems fileSystems, + IEFCoreScopeProvider efCoreScopeProvider, + EFCoreScope parentScope, + IScopeContext? scopeContext, + IEventAggregator eventAggregator, + IDbContextFactory dbContextFactory) + : base( + parentScope, + distributedLockingMechanismFactory, + loggerFactory, + efCoreScopeAccessor, + fileSystems, + efCoreScopeProvider, + scopeContext, + eventAggregator, + dbContextFactory) => + throw new NotImplementedException(); + + public EFCoreScope? OriginalScope { get; set; } + + public IScopeContext? OriginalContext { get; set; } + + public bool Detachable { get; } + + public bool Attached { get; set; } + + public new void Dispose() + { + HandleDetachedScopes(); + base.Dispose(); + } + + private void HandleDetachedScopes() + { + if (Detachable) + { + // get out of the way, restore original + + // TODO: Difficult to know if this is correct since this is all required + // by Deploy which I don't fully understand since there is limited tests on this in the CMS + if (OriginalScope != _efCoreScopeAccessor.AmbientScope) + { + _efCoreScopeProvider.PopAmbientScope(); + } + + if (OriginalContext != _efCoreScopeProvider.AmbientScopeContext) + { + _efCoreScopeProvider.PopAmbientScopeContext(); + } + + Attached = false; + OriginalScope = null; + OriginalContext = null; + } + } +} diff --git a/src/Umbraco.Cms.Persistence.EFCore/Scoping/EFCoreScope.cs b/src/Umbraco.Cms.Persistence.EFCore/Scoping/EFCoreScope.cs new file mode 100644 index 0000000000..461b09334c --- /dev/null +++ b/src/Umbraco.Cms.Persistence.EFCore/Scoping/EFCoreScope.cs @@ -0,0 +1,237 @@ +using System.Data.Common; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Storage; +using Microsoft.Extensions.Logging; +using Umbraco.Cms.Core.DistributedLocking; +using Umbraco.Cms.Core.Events; +using Umbraco.Cms.Core.IO; +using Umbraco.Cms.Core.Scoping; +using IScope = Umbraco.Cms.Infrastructure.Scoping.IScope; + +namespace Umbraco.Cms.Persistence.EFCore.Scoping; + +internal class EFCoreScope : CoreScope, IEfCoreScope + where TDbContext : DbContext +{ + private readonly IEFCoreScopeAccessor _efCoreScopeAccessor; + private readonly EFCoreScopeProvider _efCoreScopeProvider; + private readonly IScope? _innerScope; + private bool _disposed; + private TDbContext? _dbContext; + private IDbContextFactory _dbContextFactory; + + protected EFCoreScope( + IDistributedLockingMechanismFactory distributedLockingMechanismFactory, + ILoggerFactory loggerFactory, + IEFCoreScopeAccessor efCoreScopeAccessor, + FileSystems scopedFileSystem, + IEFCoreScopeProvider iefCoreScopeProvider, + IScopeContext? scopeContext, + IEventAggregator eventAggregator, + IDbContextFactory dbContextFactory, + RepositoryCacheMode repositoryCacheMode = RepositoryCacheMode.Unspecified, + bool? scopeFileSystems = null) + : base(distributedLockingMechanismFactory, loggerFactory, scopedFileSystem, eventAggregator, repositoryCacheMode, scopeFileSystems) + { + _efCoreScopeAccessor = efCoreScopeAccessor; + _efCoreScopeProvider = (EFCoreScopeProvider)iefCoreScopeProvider; + ScopeContext = scopeContext; + _dbContextFactory = dbContextFactory; + } + + public EFCoreScope( + IScope parentScope, + IDistributedLockingMechanismFactory distributedLockingMechanismFactory, + ILoggerFactory loggerFactory, + IEFCoreScopeAccessor efCoreScopeAccessor, + FileSystems scopedFileSystem, + IEFCoreScopeProvider iefCoreScopeProvider, + IScopeContext? scopeContext, + IEventAggregator eventAggregator, + IDbContextFactory dbContextFactory, + RepositoryCacheMode repositoryCacheMode = RepositoryCacheMode.Unspecified, + bool? scopeFileSystems = null) + : base(parentScope, distributedLockingMechanismFactory, loggerFactory, scopedFileSystem, eventAggregator, repositoryCacheMode, scopeFileSystems) + { + _efCoreScopeAccessor = efCoreScopeAccessor; + _efCoreScopeProvider = (EFCoreScopeProvider)iefCoreScopeProvider; + ScopeContext = scopeContext; + _innerScope = parentScope; + _dbContextFactory = dbContextFactory; + } + + public EFCoreScope( + EFCoreScope parentScope, + IDistributedLockingMechanismFactory distributedLockingMechanismFactory, + ILoggerFactory loggerFactory, + IEFCoreScopeAccessor efCoreScopeAccessor, + FileSystems scopedFileSystem, + IEFCoreScopeProvider iefCoreScopeProvider, + IScopeContext? scopeContext, + IEventAggregator eventAggregator, + IDbContextFactory dbContextFactory, + RepositoryCacheMode repositoryCacheMode = RepositoryCacheMode.Unspecified, + bool? scopeFileSystems = null) + : base(parentScope, distributedLockingMechanismFactory, loggerFactory, scopedFileSystem, eventAggregator, repositoryCacheMode, scopeFileSystems) + { + _efCoreScopeAccessor = efCoreScopeAccessor; + _efCoreScopeProvider = (EFCoreScopeProvider)iefCoreScopeProvider; + ScopeContext = scopeContext; + ParentScope = parentScope; + _dbContextFactory = dbContextFactory; + } + + + public EFCoreScope? ParentScope { get; } + + public IScopeContext? ScopeContext { get; set; } + + public async Task ExecuteWithContextAsync(Func> method) + { + if (_disposed) + { + throw new InvalidOperationException( + "The scope has been disposed, therefore the database is not available."); + } + + if (_dbContext is null) + { + InitializeDatabase(); + } + + return await method(_dbContext!); + } + + public async Task ExecuteWithContextAsync(Func method) => + await ExecuteWithContextAsync(async db => + { + await method(db); + return true; // Do nothing + }); + + public void Reset() => Completed = null; + + public override void Dispose() + { + if (this != _efCoreScopeAccessor.AmbientScope) + { + var failedMessage = + $"The {nameof(EFCoreScope)} {InstanceId} being disposed is not the Ambient {nameof(EFCoreScope)} {_efCoreScopeAccessor.AmbientScope?.InstanceId.ToString() ?? "NULL"}. This typically indicates that a child {nameof(EFCoreScope)} was not disposed, or flowed to a child thread that was not awaited, or concurrent threads are accessing the same {nameof(EFCoreScope)} (Ambient context) which is not supported. If using Task.Run (or similar) as a fire and forget tasks or to run threads in parallel you must suppress execution context flow with ExecutionContext.SuppressFlow() and ExecutionContext.RestoreFlow()."; + throw new InvalidOperationException(failedMessage); + } + + if (ParentScope is null) + { + DisposeEfCoreDatabase(); + } + + Locks.ClearLocks(InstanceId); + + if (ParentScope is null) + { + Locks.EnsureLocksCleared(InstanceId); + } + + _efCoreScopeProvider.PopAmbientScope(); + + HandleScopeContext(); + base.Dispose(); + + _disposed = true; + if (ParentScope is null) + { + if (Completed.HasValue && Completed.Value) + { + _innerScope?.Complete(); + } + + _innerScope?.Dispose(); + } + } + + private void InitializeDatabase() + { + if (_dbContext is null) + { + _dbContext = FindDbContext(); + } + + // Check if we are already in a transaction before starting one + if (_dbContext.Database.CurrentTransaction is null) + { + DbTransaction? transaction = _innerScope?.Database.Transaction; + _dbContext.Database.SetDbConnection(transaction?.Connection); + Locks.EnsureLocks(InstanceId); + + if (transaction is null) + { + _dbContext.Database.BeginTransaction(); + } + else + { + _dbContext.Database.UseTransaction(transaction); + } + } + } + + private TDbContext FindDbContext() + { + if (ParentScope is not null) + { + return ParentScope.FindDbContext(); + } + + return _dbContext ??= _dbContextFactory.CreateDbContext(); + } + + private void HandleScopeContext() + { + // if *we* created it, then get rid of it + if (_efCoreScopeProvider.AmbientScopeContext == ScopeContext) + { + try + { + _efCoreScopeProvider.AmbientScopeContext?.ScopeExit(Completed.HasValue && Completed.Value); + } + finally + { + // removes the ambient context (ambient scope already gone) + _efCoreScopeProvider.PopAmbientScopeContext(); + } + } + } + + private void DisposeEfCoreDatabase() + { + var completed = Completed.HasValue && Completed.Value; + { + try + { + if (_dbContext is null || _innerScope is not null) + { + return; + } + + // Transaction connection can be null here if we get chosen as the deadlock victim. + if (_dbContext.Database.CurrentTransaction?.GetDbTransaction().Connection is null) + { + return; + } + + if (completed) + { + _dbContext.Database.CommitTransaction(); + } + else + { + _dbContext.Database.RollbackTransaction(); + } + } + finally + { + _dbContext?.Dispose(); + _dbContext = null; + } + } + } +} diff --git a/src/Umbraco.Cms.Persistence.EFCore/Scoping/EFCoreScopeAccessor.cs b/src/Umbraco.Cms.Persistence.EFCore/Scoping/EFCoreScopeAccessor.cs new file mode 100644 index 0000000000..098a6957c4 --- /dev/null +++ b/src/Umbraco.Cms.Persistence.EFCore/Scoping/EFCoreScopeAccessor.cs @@ -0,0 +1,14 @@ +using Microsoft.EntityFrameworkCore; + +namespace Umbraco.Cms.Persistence.EFCore.Scoping; + +internal class EFCoreScopeAccessor : IEFCoreScopeAccessor where TDbContext : DbContext +{ + private readonly IAmbientEFCoreScopeStack _ambientEfCoreScopeStack; + + public EFCoreScopeAccessor(IAmbientEFCoreScopeStack ambientEfCoreScopeStack) => _ambientEfCoreScopeStack = ambientEfCoreScopeStack; + + public EFCoreScope? AmbientScope => (EFCoreScope?)_ambientEfCoreScopeStack.AmbientScope; + + IEfCoreScope? IEFCoreScopeAccessor.AmbientScope => _ambientEfCoreScopeStack.AmbientScope; +} diff --git a/src/Umbraco.Cms.Persistence.EFCore/Scoping/EFCoreScopeProvider.cs b/src/Umbraco.Cms.Persistence.EFCore/Scoping/EFCoreScopeProvider.cs new file mode 100644 index 0000000000..9e41eedb3c --- /dev/null +++ b/src/Umbraco.Cms.Persistence.EFCore/Scoping/EFCoreScopeProvider.cs @@ -0,0 +1,207 @@ +using System.Data; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Umbraco.Cms.Core.DependencyInjection; +using Umbraco.Cms.Core.DistributedLocking; +using Umbraco.Cms.Core.Events; +using Umbraco.Cms.Core.IO; +using Umbraco.Cms.Core.Scoping; +using Umbraco.Cms.Infrastructure.Scoping; +using IScope = Umbraco.Cms.Infrastructure.Scoping.IScope; +using IScopeProvider = Umbraco.Cms.Infrastructure.Scoping.IScopeProvider; + +namespace Umbraco.Cms.Persistence.EFCore.Scoping; + +internal class EFCoreScopeProvider : IEFCoreScopeProvider where TDbContext : DbContext +{ + private readonly IAmbientEFCoreScopeStack _ambientEfCoreScopeStack; + private readonly ILoggerFactory _loggerFactory; + private readonly IEFCoreScopeAccessor _efCoreScopeAccessor; + private readonly IAmbientScopeContextStack _ambientEfCoreScopeContextStack; + private readonly IDistributedLockingMechanismFactory _distributedLockingMechanismFactory; + private readonly IEventAggregator _eventAggregator; + private readonly FileSystems _fileSystems; + private readonly IScopeProvider _scopeProvider; + private readonly IDbContextFactory _dbContextFactory; + + // Needed for DI as IAmbientEfCoreScopeStack is internal + public EFCoreScopeProvider() + : this( + StaticServiceProvider.Instance.GetRequiredService>(), + StaticServiceProvider.Instance.GetRequiredService(), + StaticServiceProvider.Instance.GetRequiredService>(), + StaticServiceProvider.Instance.GetRequiredService(), + StaticServiceProvider.Instance.GetRequiredService(), + StaticServiceProvider.Instance.GetRequiredService(), + StaticServiceProvider.Instance.GetRequiredService(), + StaticServiceProvider.Instance.GetRequiredService(), + StaticServiceProvider.Instance.GetRequiredService>()) + { + } + + internal EFCoreScopeProvider( + IAmbientEFCoreScopeStack ambientEfCoreScopeStack, + ILoggerFactory loggerFactory, + IEFCoreScopeAccessor efCoreScopeAccessor, + IAmbientScopeContextStack ambientEfCoreScopeContextStack, + IDistributedLockingMechanismFactory distributedLockingMechanismFactory, + IEventAggregator eventAggregator, + FileSystems fileSystems, + IScopeProvider scopeProvider, + IDbContextFactory dbContextFactory) + { + _ambientEfCoreScopeStack = ambientEfCoreScopeStack; + _loggerFactory = loggerFactory; + _efCoreScopeAccessor = efCoreScopeAccessor; + _ambientEfCoreScopeContextStack = ambientEfCoreScopeContextStack; + _distributedLockingMechanismFactory = distributedLockingMechanismFactory; + _eventAggregator = eventAggregator; + _fileSystems = fileSystems; + _scopeProvider = scopeProvider; + _dbContextFactory = dbContextFactory; + _fileSystems.IsScoped = () => efCoreScopeAccessor.AmbientScope != null && ((EFCoreScope)efCoreScopeAccessor.AmbientScope).ScopedFileSystems; + } + + public IEfCoreScope CreateDetachedScope( + RepositoryCacheMode repositoryCacheMode = RepositoryCacheMode.Unspecified, + bool? scopeFileSystems = null) => + new EFCoreDetachableScope( + _distributedLockingMechanismFactory, + _loggerFactory, + _efCoreScopeAccessor, + _fileSystems, + this, + null, + _eventAggregator, + _dbContextFactory, + repositoryCacheMode, + scopeFileSystems); + + public void AttachScope(IEfCoreScope other) + { + // IScopeProvider.AttachScope works with an IEFCoreScope + // but here we can only deal with our own Scope class + if (other is not EFCoreDetachableScope otherScope) + { + throw new ArgumentException("Not a Scope instance."); + } + + if (otherScope.Detachable == false) + { + throw new ArgumentException("Not a detachable scope."); + } + + if (otherScope.Attached) + { + throw new InvalidOperationException("Already attached."); + } + + otherScope.Attached = true; + otherScope.OriginalScope = (EFCoreScope)_ambientEfCoreScopeStack.AmbientScope!; + otherScope.OriginalContext = AmbientScopeContext; + + PushAmbientScopeContext(otherScope.ScopeContext); + _ambientEfCoreScopeStack.Push(otherScope); + } + + public IEfCoreScope DetachScope() + { + if (_ambientEfCoreScopeStack.AmbientScope is not EFCoreDetachableScope ambientScope) + { + throw new InvalidOperationException("Ambient scope is not detachable"); + } + + if (ambientScope == null) + { + throw new InvalidOperationException("There is no ambient scope."); + } + + if (ambientScope.Detachable == false) + { + throw new InvalidOperationException("Ambient scope is not detachable."); + } + + PopAmbientScope(); + PopAmbientScopeContext(); + + var originalScope = (EFCoreScope)_ambientEfCoreScopeStack.AmbientScope!; + if (originalScope != ambientScope.OriginalScope) + { + throw new InvalidOperationException($"The detatched scope ({ambientScope.InstanceId}) does not match the original ({originalScope.InstanceId})"); + } + + IScopeContext? originalScopeContext = AmbientScopeContext; + if (originalScopeContext != ambientScope.OriginalContext) + { + throw new InvalidOperationException($"The detatched scope context does not match the original"); + } + + ambientScope.OriginalScope = null; + ambientScope.OriginalContext = null; + ambientScope.Attached = false; + return ambientScope; + } + + + public IScopeContext? AmbientScopeContext => _ambientEfCoreScopeContextStack.AmbientContext; + + public IEfCoreScope CreateScope( + RepositoryCacheMode repositoryCacheMode = RepositoryCacheMode.Unspecified, bool? scopeFileSystems = null) + { + if (_ambientEfCoreScopeStack.AmbientScope is null) + { + ScopeContext? newContext = _ambientEfCoreScopeContextStack.AmbientContext == null ? new ScopeContext() : null; + IScope parentScope = _scopeProvider.CreateScope(IsolationLevel.Unspecified, repositoryCacheMode, null, null, scopeFileSystems); + var ambientScope = new EFCoreScope( + parentScope, + _distributedLockingMechanismFactory, + _loggerFactory, + _efCoreScopeAccessor, + _fileSystems, + this, + newContext, + _eventAggregator, + _dbContextFactory, + repositoryCacheMode, + scopeFileSystems); + + if (newContext != null) + { + PushAmbientScopeContext(newContext); + } + + _ambientEfCoreScopeStack.Push(ambientScope); + return ambientScope; + } + + var efCoreScope = new EFCoreScope( + (EFCoreScope)_ambientEfCoreScopeStack.AmbientScope, + _distributedLockingMechanismFactory, + _loggerFactory, + _efCoreScopeAccessor, + _fileSystems, + this, + null, + _eventAggregator, + _dbContextFactory, + repositoryCacheMode, + scopeFileSystems); + + _ambientEfCoreScopeStack.Push(efCoreScope); + return efCoreScope; + } + + public void PopAmbientScope() => _ambientEfCoreScopeStack.Pop(); + + public void PushAmbientScopeContext(IScopeContext? scopeContext) + { + if (scopeContext is null) + { + throw new ArgumentNullException(nameof(scopeContext)); + } + _ambientEfCoreScopeContextStack.Push(scopeContext); + } + + public void PopAmbientScopeContext() => _ambientEfCoreScopeContextStack.Pop(); +} diff --git a/src/Umbraco.Cms.Persistence.EFCore/Scoping/IAmbientEfCoreScopeStack.cs b/src/Umbraco.Cms.Persistence.EFCore/Scoping/IAmbientEfCoreScopeStack.cs new file mode 100644 index 0000000000..01b66c7443 --- /dev/null +++ b/src/Umbraco.Cms.Persistence.EFCore/Scoping/IAmbientEfCoreScopeStack.cs @@ -0,0 +1,12 @@ +using Microsoft.EntityFrameworkCore; + +namespace Umbraco.Cms.Persistence.EFCore.Scoping; + +internal interface IAmbientEFCoreScopeStack : IEFCoreScopeAccessor where TDbContext : DbContext +{ + public IEfCoreScope? AmbientScope { get; } + + IEfCoreScope Pop(); + + void Push(IEfCoreScope scope); +} diff --git a/src/Umbraco.Cms.Persistence.EFCore/Scoping/IEFCoreScope.cs b/src/Umbraco.Cms.Persistence.EFCore/Scoping/IEFCoreScope.cs new file mode 100644 index 0000000000..5595fd5295 --- /dev/null +++ b/src/Umbraco.Cms.Persistence.EFCore/Scoping/IEFCoreScope.cs @@ -0,0 +1,30 @@ +using Umbraco.Cms.Core.Events; +using Umbraco.Cms.Core.Scoping; + +namespace Umbraco.Cms.Persistence.EFCore.Scoping; + +public interface IEfCoreScope : ICoreScope +{ + /// + /// Executes the given function on the database. + /// + /// Function to execute. + /// Type to use and return. + /// + Task ExecuteWithContextAsync(Func> method); + + public IScopeContext? ScopeContext { get; set; } + + /// + /// Executes the given function on the database. + /// + /// Function to execute. + /// Type to use and return. + /// + Task ExecuteWithContextAsync(Func method); + + /// + /// Gets the scope notification publisher + /// + IScopedNotificationPublisher Notifications { get; } +} diff --git a/src/Umbraco.Cms.Persistence.EFCore/Scoping/IEFCoreScopeAccessor.cs b/src/Umbraco.Cms.Persistence.EFCore/Scoping/IEFCoreScopeAccessor.cs new file mode 100644 index 0000000000..05db299370 --- /dev/null +++ b/src/Umbraco.Cms.Persistence.EFCore/Scoping/IEFCoreScopeAccessor.cs @@ -0,0 +1,10 @@ +namespace Umbraco.Cms.Persistence.EFCore.Scoping; + +public interface IEFCoreScopeAccessor +{ + /// + /// Gets the ambient scope. + /// + /// Returns null if there is no ambient scope. + IEfCoreScope? AmbientScope { get; } +} diff --git a/src/Umbraco.Cms.Persistence.EFCore/Scoping/IEFCoreScopeProvider.cs b/src/Umbraco.Cms.Persistence.EFCore/Scoping/IEFCoreScopeProvider.cs new file mode 100644 index 0000000000..8b872d9f14 --- /dev/null +++ b/src/Umbraco.Cms.Persistence.EFCore/Scoping/IEFCoreScopeProvider.cs @@ -0,0 +1,16 @@ +using Umbraco.Cms.Core.Scoping; + +namespace Umbraco.Cms.Persistence.EFCore.Scoping; + +public interface IEFCoreScopeProvider +{ + IEfCoreScope CreateScope(RepositoryCacheMode repositoryCacheMode = RepositoryCacheMode.Unspecified, bool? scopeFileSystems = null); + + IEfCoreScope CreateDetachedScope(RepositoryCacheMode repositoryCacheMode = RepositoryCacheMode.Unspecified, bool? scopeFileSystems = null); + + void AttachScope(IEfCoreScope other); + + IEfCoreScope DetachScope(); + + IScopeContext? AmbientScopeContext { get; } +} diff --git a/src/Umbraco.Cms.Persistence.EFCore/Umbraco.Cms.Persistence.EFCore.csproj b/src/Umbraco.Cms.Persistence.EFCore/Umbraco.Cms.Persistence.EFCore.csproj new file mode 100644 index 0000000000..af566ab67a --- /dev/null +++ b/src/Umbraco.Cms.Persistence.EFCore/Umbraco.Cms.Persistence.EFCore.csproj @@ -0,0 +1,25 @@ + + + Umbraco CMS - Persistence - EFCore + + false + + + + + + + + + + + + + + + + <_Parameter1>Umbraco.Tests.Integration + + + + diff --git a/src/Umbraco.Cms/Umbraco.Cms.csproj b/src/Umbraco.Cms/Umbraco.Cms.csproj index da6be4c30c..034c42356c 100644 --- a/src/Umbraco.Cms/Umbraco.Cms.csproj +++ b/src/Umbraco.Cms/Umbraco.Cms.csproj @@ -11,6 +11,7 @@ + diff --git a/src/Umbraco.Core/CompatibilitySuppressions.xml b/src/Umbraco.Core/CompatibilitySuppressions.xml index edd49bcf95..61c15feaf4 100644 --- a/src/Umbraco.Core/CompatibilitySuppressions.xml +++ b/src/Umbraco.Core/CompatibilitySuppressions.xml @@ -29,4 +29,11 @@ lib/net7.0/Umbraco.Core.dll true - + + CP0006 + P:Umbraco.Cms.Core.Scoping.ICoreScope.Locks + lib/net7.0/Umbraco.Core.dll + lib/net7.0/Umbraco.Core.dll + true + + \ No newline at end of file diff --git a/src/Umbraco.Core/Scoping/CoreScope.cs b/src/Umbraco.Core/Scoping/CoreScope.cs new file mode 100644 index 0000000000..a05b44f4a7 --- /dev/null +++ b/src/Umbraco.Core/Scoping/CoreScope.cs @@ -0,0 +1,272 @@ +using Microsoft.Extensions.Logging; +using Umbraco.Cms.Core.Cache; +using Umbraco.Cms.Core.DistributedLocking; +using Umbraco.Cms.Core.Events; +using Umbraco.Cms.Core.IO; + +namespace Umbraco.Cms.Core.Scoping; + +public class CoreScope : ICoreScope +{ + protected bool? Completed; + private ICompletable? _scopedFileSystem; + private IScopedNotificationPublisher? _notificationPublisher; + private IsolatedCaches? _isolatedCaches; + private ICoreScope? _parentScope; + + private readonly RepositoryCacheMode _repositoryCacheMode; + private readonly bool? _shouldScopeFileSystems; + private readonly IEventAggregator _eventAggregator; + + private bool _disposed; + + protected CoreScope( + IDistributedLockingMechanismFactory distributedLockingMechanismFactory, + ILoggerFactory loggerFactory, + FileSystems scopedFileSystem, + IEventAggregator eventAggregator, + RepositoryCacheMode repositoryCacheMode = RepositoryCacheMode.Unspecified, + bool? shouldScopeFileSystems = null, + IScopedNotificationPublisher? notificationPublisher = null) + { + _eventAggregator = eventAggregator; + InstanceId = Guid.NewGuid(); + CreatedThreadId = Environment.CurrentManagedThreadId; + Locks = ParentScope is null + ? new LockingMechanism(distributedLockingMechanismFactory, loggerFactory.CreateLogger()) + : ResolveLockingMechanism(); + _repositoryCacheMode = repositoryCacheMode; + _shouldScopeFileSystems = shouldScopeFileSystems; + _notificationPublisher = notificationPublisher; + + if (_shouldScopeFileSystems is true) + { + _scopedFileSystem = scopedFileSystem.Shadow(); + } + } + + protected CoreScope( + ICoreScope? parentScope, + IDistributedLockingMechanismFactory distributedLockingMechanismFactory, + ILoggerFactory loggerFactory, + FileSystems scopedFileSystem, + IEventAggregator eventAggregator, + RepositoryCacheMode repositoryCacheMode = RepositoryCacheMode.Unspecified, + bool? shouldScopeFileSystems = null, + IScopedNotificationPublisher? notificationPublisher = null) + { + _eventAggregator = eventAggregator; + InstanceId = Guid.NewGuid(); + CreatedThreadId = Environment.CurrentManagedThreadId; + _repositoryCacheMode = repositoryCacheMode; + _shouldScopeFileSystems = shouldScopeFileSystems; + _notificationPublisher = notificationPublisher; + + if (parentScope is null) + { + Locks = new LockingMechanism(distributedLockingMechanismFactory, loggerFactory.CreateLogger()); + if (_shouldScopeFileSystems is true) + { + _scopedFileSystem = scopedFileSystem.Shadow(); + } + + return; + } + + Locks = parentScope.Locks; + + // cannot specify a different mode! + // TODO: means that it's OK to go from L2 to None for reading purposes, but writing would be BAD! + // this is for XmlStore that wants to bypass caches when rebuilding XML (same for NuCache) + if (repositoryCacheMode != RepositoryCacheMode.Unspecified && + parentScope.RepositoryCacheMode > repositoryCacheMode) + { + throw new ArgumentException( + $"Value '{repositoryCacheMode}' cannot be lower than parent value '{parentScope.RepositoryCacheMode}'.", nameof(repositoryCacheMode)); + } + + // Only the outermost scope can specify the notification publisher + if (_notificationPublisher != null) + { + throw new ArgumentException("Value cannot be specified on nested scope.", nameof(_notificationPublisher)); + } + + _parentScope = parentScope; + + // cannot specify a different fs scope! + // can be 'true' only on outer scope (and false does not make much sense) + if (_shouldScopeFileSystems != null && ParentScope?._shouldScopeFileSystems != _shouldScopeFileSystems) + { + throw new ArgumentException( + $"Value '{_shouldScopeFileSystems.Value}' be different from parent value '{ParentScope?._shouldScopeFileSystems}'.", + nameof(_shouldScopeFileSystems)); + } + } + + private CoreScope? ParentScope => (CoreScope?)_parentScope; + + public int Depth + { + get + { + if (ParentScope == null) + { + return 0; + } + + return ParentScope.Depth + 1; + } + } + + public Guid InstanceId { get; } + + public int CreatedThreadId { get; } + + public ILockingMechanism Locks { get; } + + public IScopedNotificationPublisher Notifications + { + get + { + EnsureNotDisposed(); + if (ParentScope != null) + { + return ParentScope.Notifications; + } + + return _notificationPublisher ??= new ScopedNotificationPublisher(_eventAggregator); + } + } + + public RepositoryCacheMode RepositoryCacheMode + { + get + { + if (_repositoryCacheMode != RepositoryCacheMode.Unspecified) + { + return _repositoryCacheMode; + } + + return ParentScope?.RepositoryCacheMode ?? RepositoryCacheMode.Default; + } + } + + public IsolatedCaches IsolatedCaches + { + get + { + if (ParentScope != null) + { + return ParentScope.IsolatedCaches; + } + + return _isolatedCaches ??= new IsolatedCaches(_ => new DeepCloneAppCache(new ObjectCacheAppCache())); + } + } + + public bool ScopedFileSystems + { + get + { + if (ParentScope != null) + { + return ParentScope.ScopedFileSystems; + } + + return _scopedFileSystem != null; + } + } + + /// + /// Completes a scope + /// + /// A value indicating whether the scope is completed or not. + public bool Complete() + { + if (Completed.HasValue == false) + { + Completed = true; + } + + return Completed.Value; + } + + public void ReadLock(params int[] lockIds) => Locks.ReadLock(InstanceId, null, lockIds); + + public void WriteLock(params int[] lockIds) => Locks.WriteLock(InstanceId, null, lockIds); + + public void WriteLock(TimeSpan timeout, int lockId) => Locks.ReadLock(InstanceId, timeout, lockId); + + public void ReadLock(TimeSpan timeout, int lockId) => Locks.WriteLock(InstanceId, timeout, lockId); + + public void EagerWriteLock(params int[] lockIds) => Locks.EagerWriteLock(InstanceId, null, lockIds); + + public void EagerWriteLock(TimeSpan timeout, int lockId) => Locks.EagerWriteLock(InstanceId, timeout, lockId); + + public void EagerReadLock(TimeSpan timeout, int lockId) => Locks.EagerReadLock(InstanceId, timeout, lockId); + + public void EagerReadLock(params int[] lockIds) => Locks.EagerReadLock(InstanceId, TimeSpan.Zero, lockIds); + + public virtual void Dispose() + { + if (ParentScope is null) + { + HandleScopedFileSystems(); + HandleScopedNotifications(); + } + else + { + ParentScope.ChildCompleted(Completed); + } + + _disposed = true; + } + + protected void ChildCompleted(bool? completed) + { + // if child did not complete we cannot complete + if (completed.HasValue == false || completed.Value == false) + { + Completed = false; + } + } + + private void HandleScopedFileSystems() + { + if (_shouldScopeFileSystems == true) + { + if (Completed.HasValue && Completed.Value) + { + _scopedFileSystem?.Complete(); + } + + _scopedFileSystem?.Dispose(); + _scopedFileSystem = null; + } + } + + protected void SetParentScope(ICoreScope coreScope) + { + _parentScope = coreScope; + } + + private void HandleScopedNotifications() => _notificationPublisher?.ScopeExit(Completed.HasValue && Completed.Value); + + private void EnsureNotDisposed() + { + // We can't be disposed + if (_disposed) + { + throw new ObjectDisposedException($"The {nameof(CoreScope)} with ID ({InstanceId}) is already disposed"); + } + + // And neither can our ancestors if we're trying to be disposed since + // a child must always be disposed before it's parent. + // This is a safety check, it's actually not entirely possible that a parent can be + // disposed before the child since that will end up with a "not the Ambient" exception. + ParentScope?.EnsureNotDisposed(); + } + + private ILockingMechanism ResolveLockingMechanism() => + ParentScope is not null ? ParentScope.ResolveLockingMechanism() : Locks; +} diff --git a/src/Umbraco.Core/Scoping/ICoreScope.cs b/src/Umbraco.Core/Scoping/ICoreScope.cs index fe2a9489f3..713ecc7954 100644 --- a/src/Umbraco.Core/Scoping/ICoreScope.cs +++ b/src/Umbraco.Core/Scoping/ICoreScope.cs @@ -16,6 +16,8 @@ public interface ICoreScope : IDisposable, IInstanceIdentifiable /// public int Depth => -1; + public ILockingMechanism Locks { get; } + /// /// Gets the scope notification publisher /// diff --git a/src/Umbraco.Core/Scoping/ILockingMechanism.cs b/src/Umbraco.Core/Scoping/ILockingMechanism.cs new file mode 100644 index 0000000000..22dded1652 --- /dev/null +++ b/src/Umbraco.Core/Scoping/ILockingMechanism.cs @@ -0,0 +1,58 @@ +namespace Umbraco.Cms.Core.Scoping; + +public interface ILockingMechanism : IDisposable +{ + /// + /// Read-locks some lock objects lazily. + /// + /// Instance id of the scope who is requesting the lock + /// Array of lock object identifiers. + void ReadLock(Guid instanceId, TimeSpan? timeout = null, params int[] lockIds); + + void ReadLock(Guid instanceId, params int[] lockIds); + + /// + /// Write-locks some lock objects lazily. + /// + /// Instance id of the scope who is requesting the lock + /// Array of object identifiers. + void WriteLock(Guid instanceId, TimeSpan? timeout = null, params int[] lockIds); + + void WriteLock(Guid instanceId, params int[] lockIds); + + /// + /// Eagerly acquires a read-lock + /// + /// + /// + void EagerReadLock(Guid instanceId, TimeSpan? timeout = null, params int[] lockIds); + + void EagerReadLock(Guid instanceId, params int[] lockIds); + + /// + /// Eagerly acquires a write-lock + /// + /// + /// + void EagerWriteLock(Guid instanceId, TimeSpan? timeout = null, params int[] lockIds); + + void EagerWriteLock(Guid instanceId, params int[] lockIds); + + /// + /// Clears all the locks held + /// + /// + void ClearLocks(Guid instanceId); + + /// + /// Acquires all the non-eagerly requested locks. + /// + /// + void EnsureLocks(Guid scopeInstanceId); + + void EnsureLocksCleared(Guid instanceId); + + Dictionary>? GetReadLocks(); + + Dictionary>? GetWriteLocks(); +} diff --git a/src/Umbraco.Core/Scoping/LockingMechanism.cs b/src/Umbraco.Core/Scoping/LockingMechanism.cs new file mode 100644 index 0000000000..e41fe2d874 --- /dev/null +++ b/src/Umbraco.Core/Scoping/LockingMechanism.cs @@ -0,0 +1,433 @@ +using System.Text; +using Microsoft.Extensions.Logging; +using Umbraco.Cms.Core.Collections; +using Umbraco.Cms.Core.DistributedLocking; +using Umbraco.Extensions; + +namespace Umbraco.Cms.Core.Scoping; + +/// +/// Mechanism for handling read and write locks +/// +public class LockingMechanism : ILockingMechanism +{ + private readonly IDistributedLockingMechanismFactory _distributedLockingMechanismFactory; + private readonly ILogger _logger; + private readonly object _lockQueueLocker = new(); + private readonly object _dictionaryLocker = new(); + private StackQueue<(DistributedLockType lockType, TimeSpan timeout, Guid instanceId, int lockId)>? _queuedLocks; + private HashSet? _readLocks; + private Dictionary>? _readLocksDictionary; + private HashSet? _writeLocks; + private Dictionary>? _writeLocksDictionary; + private Queue? _acquiredLocks; + + /// + /// Constructs an instance of LockingMechanism + /// + /// + /// + public LockingMechanism(IDistributedLockingMechanismFactory distributedLockingMechanismFactory, ILogger logger) + { + _distributedLockingMechanismFactory = distributedLockingMechanismFactory; + _logger = logger; + _acquiredLocks = new Queue(); + } + + /// + public void ReadLock(Guid instanceId, TimeSpan? timeout = null, params int[] lockIds) => LazyReadLockInner(instanceId, timeout, lockIds); + + public void ReadLock(Guid instanceId, params int[] lockIds) => ReadLock(instanceId, null, lockIds); + + /// + public void WriteLock(Guid instanceId, TimeSpan? timeout = null, params int[] lockIds) => LazyWriteLockInner(instanceId, timeout, lockIds); + + public void WriteLock(Guid instanceId, params int[] lockIds) => WriteLock(instanceId, null, lockIds); + + /// + public void EagerReadLock(Guid instanceId, TimeSpan? timeout = null, params int[] lockIds) => EagerReadLockInner(instanceId, timeout, lockIds); + + public void EagerReadLock(Guid instanceId, params int[] lockIds) => + EagerReadLock(instanceId, null, lockIds); + + /// + public void EagerWriteLock(Guid instanceId, TimeSpan? timeout = null, params int[] lockIds) => EagerWriteLockInner(instanceId, timeout, lockIds); + + public void EagerWriteLock(Guid instanceId, params int[] lockIds) => + EagerWriteLock(instanceId, null, lockIds); + + /// + /// Handles acquiring a write lock with a specified timeout, will delegate it to the parent if there are any. + /// + /// Instance ID of the requesting scope. + /// Optional database timeout in milliseconds. + /// Array of lock object identifiers. + private void EagerWriteLockInner(Guid instanceId, TimeSpan? timeout, params int[] lockIds) + { + lock (_dictionaryLocker) + { + foreach (var lockId in lockIds) + { + IncrementLock(lockId, instanceId, ref _writeLocksDictionary); + + // We are the outermost scope, handle the lock request. + LockInner( + instanceId, + ref _writeLocksDictionary!, + ref _writeLocks!, + ObtainWriteLock, + timeout, + lockId); + } + } + } + + /// + /// Obtains a write lock with a custom timeout. + /// + /// Lock object identifier to lock. + /// TimeSpan specifying the timout period. + private void ObtainWriteLock(int lockId, TimeSpan? timeout) + { + if (_acquiredLocks == null) + { + throw new InvalidOperationException( + $"Cannot obtain a write lock as the {nameof(_acquiredLocks)} queue is null."); + } + + _acquiredLocks.Enqueue(_distributedLockingMechanismFactory.DistributedLockingMechanism.WriteLock(lockId, timeout)); + } + + /// + /// Handles acquiring a read lock, will delegate it to the parent if there are any. + /// + /// The id of the scope requesting the lock. + /// Optional database timeout in milliseconds. + /// Array of lock object identifiers. + private void EagerReadLockInner(Guid instanceId, TimeSpan? timeout, params int[] lockIds) + { + lock (_dictionaryLocker) + { + foreach (var lockId in lockIds) + { + IncrementLock(lockId, instanceId, ref _readLocksDictionary); + + // We are the outermost scope, handle the lock request. + LockInner( + instanceId, + ref _readLocksDictionary!, + ref _readLocks!, + ObtainReadLock, + timeout, + lockId); + } + } + } + + /// + /// Obtains a read lock with a custom timeout. + /// + /// Lock object identifier to lock. + /// TimeSpan specifying the timout period. + private void ObtainReadLock(int lockId, TimeSpan? timeout) + { + if (_acquiredLocks == null) + { + throw new InvalidOperationException( + $"Cannot obtain a read lock as the {nameof(_acquiredLocks)} queue is null."); + } + + _acquiredLocks.Enqueue( + _distributedLockingMechanismFactory.DistributedLockingMechanism.ReadLock(lockId, timeout)); + } + + /// + /// Handles acquiring a lock, this should only be called from the outermost scope. + /// + /// Instance ID of the scope requesting the lock. + /// Reference to the applicable locks dictionary (ReadLocks or WriteLocks). + /// Reference to the applicable locks hashset (_readLocks or _writeLocks). + /// Delegate used to request the lock from the locking mechanism. + /// Optional timeout parameter to specify a timeout. + /// Lock identifier. + private void LockInner( + Guid instanceId, + ref Dictionary> locks, + ref HashSet? locksSet, + Action obtainLock, + TimeSpan? timeout, + int lockId) + { + locksSet ??= new HashSet(); + + // Only acquire the lock if we haven't done so yet. + if (locksSet.Contains(lockId)) + { + return; + } + + locksSet.Add(lockId); + try + { + obtainLock(lockId, timeout); + } + catch + { + // Something went wrong and we didn't get the lock + // Since we at this point have determined that we haven't got any lock with an ID of LockID, it's safe to completely remove it instead of decrementing. + locks[instanceId].Remove(lockId); + + // It needs to be removed from the HashSet as well, because that's how we determine to acquire a lock. + locksSet.Remove(lockId); + throw; + } + } + + /// + /// Increment the counter of a locks dictionary, either ReadLocks or WriteLocks, + /// for a specific scope instance and lock identifier. Must be called within a lock. + /// + /// Lock ID to increment. + /// Instance ID of the scope requesting the lock. + /// Reference to the dictionary to increment on + private void IncrementLock(int lockId, Guid instanceId, ref Dictionary>? locks) + { + // Since we've already checked that we're the parent in the WriteLockInner method, we don't need to check again. + // If it's the very first time a lock has been requested the WriteLocks dict hasn't been instantiated yet. + locks ??= new Dictionary>(); + + // Try and get the dict associated with the scope id. + var locksDictFound = locks.TryGetValue(instanceId, out Dictionary? locksDict); + if (locksDictFound) + { + locksDict!.TryGetValue(lockId, out var value); + locksDict[lockId] = value + 1; + } + else + { + // The scope hasn't requested a lock yet, so we have to create a dict for it. + locks.Add(instanceId, new Dictionary()); + locks[instanceId][lockId] = 1; + } + } + + private void LazyWriteLockInner(Guid instanceId, TimeSpan? timeout = null, params int[] lockIds) => + LazyLockInner(DistributedLockType.WriteLock, instanceId, timeout, lockIds); + + private void LazyReadLockInner(Guid instanceId, TimeSpan? timeout = null, params int[] lockIds) => + LazyLockInner(DistributedLockType.ReadLock, instanceId, timeout, lockIds); + + private void LazyLockInner(DistributedLockType lockType, Guid instanceId, TimeSpan? timeout = null, params int[] lockIds) + { + lock (_lockQueueLocker) + { + if (_queuedLocks == null) + { + _queuedLocks = new StackQueue<(DistributedLockType, TimeSpan, Guid, int)>(); + } + + foreach (var lockId in lockIds) + { + _queuedLocks.Enqueue((lockType, timeout ?? TimeSpan.Zero, instanceId, lockId)); + } + } + } + + /// + /// Clears all lock counters for a given scope instance, signalling that the scope has been disposed. + /// + /// Instance ID of the scope to clear. + public void ClearLocks(Guid instanceId) + { + lock (_dictionaryLocker) + { + _readLocksDictionary?.Remove(instanceId); + _writeLocksDictionary?.Remove(instanceId); + + // remove any queued locks for this instance that weren't used. + while (_queuedLocks?.Count > 0) + { + // It's safe to assume that the locks on the top of the stack belong to this instance, + // since any child scopes that might have added locks to the stack must be disposed before we try and dispose this instance. + (DistributedLockType lockType, TimeSpan timeout, Guid instanceId, int lockId) top = + _queuedLocks.PeekStack(); + if (top.instanceId == instanceId) + { + _queuedLocks.Pop(); + } + else + { + break; + } + } + } + } + + public void EnsureLocksCleared(Guid instanceId) + { + while (!_acquiredLocks?.IsCollectionEmpty() ?? false) + { + _acquiredLocks?.Dequeue().Dispose(); + } + + // We're the parent scope, make sure that locks of all scopes has been cleared + // Since we're only reading we don't have to be in a lock + if (!(_readLocksDictionary?.Count > 0) && !(_writeLocksDictionary?.Count > 0)) + { + return; + } + + var exception = new InvalidOperationException( + $"All scopes has not been disposed from parent scope: {instanceId}, see log for more details."); + throw exception; + } + + /// + /// When we require a ReadLock or a WriteLock we don't immediately request these locks from the database, + /// instead we only request them when necessary (lazily). + /// To do this, we queue requests for read/write locks. + /// This is so that if there's a request for either of these + /// locks, but the service/repository returns an item from the cache, we don't end up making a DB call to make the + /// read/write lock. + /// This executes the queue of requested locks in order in an efficient way lazily whenever the database instance is + /// resolved. + /// + public void EnsureLocks(Guid scopeInstanceId) + { + lock (_lockQueueLocker) + { + if (!(_queuedLocks?.Count > 0)) + { + return; + } + + DistributedLockType currentType = DistributedLockType.ReadLock; + TimeSpan currentTimeout = TimeSpan.Zero; + Guid currentInstanceId = scopeInstanceId; + var collectedIds = new HashSet(); + + var i = 0; + while (_queuedLocks.Count > 0) + { + (DistributedLockType lockType, TimeSpan timeout, Guid instanceId, var lockId) = + _queuedLocks.Dequeue(); + + if (i == 0) + { + currentType = lockType; + currentTimeout = timeout; + currentInstanceId = instanceId; + } + else if (lockType != currentType || timeout != currentTimeout || + instanceId != currentInstanceId) + { + // the lock type, instanceId or timeout switched. + // process the lock ids collected + switch (currentType) + { + case DistributedLockType.ReadLock: + EagerReadLockInner( + currentInstanceId, + currentTimeout == TimeSpan.Zero ? null : currentTimeout, + collectedIds.ToArray()); + break; + case DistributedLockType.WriteLock: + EagerWriteLockInner( + currentInstanceId, + currentTimeout == TimeSpan.Zero ? null : currentTimeout, + collectedIds.ToArray()); + break; + } + + // clear the collected and set new type + collectedIds.Clear(); + currentType = lockType; + currentTimeout = timeout; + currentInstanceId = instanceId; + } + + collectedIds.Add(lockId); + i++; + } + + // process the remaining + switch (currentType) + { + case DistributedLockType.ReadLock: + EagerReadLockInner( + currentInstanceId, + currentTimeout == TimeSpan.Zero ? null : currentTimeout, + collectedIds.ToArray()); + break; + case DistributedLockType.WriteLock: + EagerWriteLockInner( + currentInstanceId, + currentTimeout == TimeSpan.Zero ? null : currentTimeout, + collectedIds.ToArray()); + break; + } + } + } + + + public Dictionary>? GetReadLocks() => _readLocksDictionary; + + public Dictionary>? GetWriteLocks() => _writeLocksDictionary; + + /// + public void Dispose() + { + while (!_acquiredLocks?.IsCollectionEmpty() ?? false) + { + _acquiredLocks?.Dequeue().Dispose(); + } + + // We're the parent scope, make sure that locks of all scopes has been cleared + // Since we're only reading we don't have to be in a lock + if (_readLocksDictionary?.Count > 0 || _writeLocksDictionary?.Count > 0) + { + var exception = new InvalidOperationException( + $"All locks have not been cleared, this usually means that all scopes have not been disposed from the parent scope"); + _logger.LogError(exception, GenerateUnclearedScopesLogMessage()); + throw exception; + } + } + + /// + /// Generates a log message with all scopes that hasn't cleared their locks, including how many, and what locks they + /// have requested. + /// + /// Log message. + private string GenerateUnclearedScopesLogMessage() + { + // Dump the dicts into a message for the locks. + var builder = new StringBuilder(); + builder.AppendLine( + $"Lock counters aren't empty, suggesting a scope hasn't been properly disposed"); + WriteLockDictionaryToString(_readLocksDictionary!, builder, "read locks"); + WriteLockDictionaryToString(_writeLocksDictionary!, builder, "write locks"); + return builder.ToString(); + } + + /// + /// Writes a locks dictionary to a for logging purposes. + /// + /// Lock dictionary to report on. + /// String builder to write to. + /// The name to report the dictionary as. + private void WriteLockDictionaryToString(Dictionary> dict, StringBuilder builder, string dictName) + { + if (dict?.Count > 0) + { + builder.AppendLine($"Remaining {dictName}:"); + foreach (KeyValuePair> instance in dict) + { + builder.AppendLine($"Scope {instance.Key}"); + foreach (KeyValuePair lockCounter in instance.Value) + { + builder.AppendLine($"\tLock ID: {lockCounter.Key} - times requested: {lockCounter.Value}"); + } + } + } + } +} diff --git a/src/Umbraco.Infrastructure/Scoping/IAmbientScopeContextStack.cs b/src/Umbraco.Infrastructure/Scoping/IAmbientScopeContextStack.cs index 28da9a6427..f481166d8f 100644 --- a/src/Umbraco.Infrastructure/Scoping/IAmbientScopeContextStack.cs +++ b/src/Umbraco.Infrastructure/Scoping/IAmbientScopeContextStack.cs @@ -2,7 +2,7 @@ using Umbraco.Cms.Core.Scoping; namespace Umbraco.Cms.Infrastructure.Scoping; -internal interface IAmbientScopeContextStack +public interface IAmbientScopeContextStack { IScopeContext? AmbientContext { get; } IScopeContext Pop(); diff --git a/src/Umbraco.Infrastructure/Scoping/Scope.cs b/src/Umbraco.Infrastructure/Scoping/Scope.cs index 000b6a602e..0ff1fa5d30 100644 --- a/src/Umbraco.Infrastructure/Scoping/Scope.cs +++ b/src/Umbraco.Infrastructure/Scoping/Scope.cs @@ -1,13 +1,10 @@ using System.Data; using System.Text; using Microsoft.Extensions.Logging; -using Umbraco.Cms.Core.Cache; using Umbraco.Cms.Core.Configuration.Models; using Umbraco.Cms.Core.DistributedLocking; using Umbraco.Cms.Core.Events; using Umbraco.Cms.Core.IO; -using Umbraco.Cms.Core; -using Umbraco.Cms.Core.Collections; using Umbraco.Cms.Core.Scoping; using Umbraco.Cms.Infrastructure.Persistence; using Umbraco.Extensions; @@ -18,46 +15,28 @@ namespace Umbraco.Cms.Infrastructure.Scoping /// Implements . /// /// Not thread-safe obviously. - internal class Scope : ICoreScope, IScope, Core.Scoping.IScope + internal class Scope : CoreScope, ICoreScope, IScope, Core.Scoping.IScope { private readonly bool _autoComplete; private readonly CoreDebugSettings _coreDebugSettings; - - private readonly object _dictionaryLocker; - private readonly IEventAggregator _eventAggregator; private readonly IsolationLevel _isolationLevel; - private readonly object _lockQueueLocker = new(); private readonly ILogger _logger; private readonly MediaFileManager _mediaFileManager; - private readonly RepositoryCacheMode _repositoryCacheMode; - private readonly bool? _scopeFileSystem; private readonly ScopeProvider _scopeProvider; - private bool? _completed; private IUmbracoDatabase? _database; private bool _disposed; private IEventDispatcher? _eventDispatcher; - private ICompletable? _fscope; private EventMessages? _messages; - private IsolatedCaches? _isolatedCaches; - private IScopedNotificationPublisher? _notificationPublisher; - - private StackQueue<(DistributedLockType lockType, TimeSpan timeout, Guid instanceId, int lockId)>? _queuedLocks; - - // This is all used to safely track read/write locks at given Scope levels so that - // when we dispose we can verify that everything has been cleaned up correctly. - private HashSet? _readLocks; - private Dictionary>? _readLocksDictionary; - private HashSet? _writeLocks; - private Dictionary>? _writeLocksDictionary; - private Queue? _acquiredLocks; // initializes a new scope private Scope( ScopeProvider scopeProvider, CoreDebugSettings coreDebugSettings, + IDistributedLockingMechanismFactory distributedLockingMechanismFactory, + ILoggerFactory loggerFactory, MediaFileManager mediaFileManager, IEventAggregator eventAggregator, ILogger logger, @@ -72,22 +51,26 @@ namespace Umbraco.Cms.Infrastructure.Scoping bool? scopeFileSystems = null, bool callContext = false, bool autoComplete = false) + : base( + parent, + distributedLockingMechanismFactory, + loggerFactory, + fileSystems, + eventAggregator, + repositoryCacheMode, + scopeFileSystems, + notificationPublisher) { _scopeProvider = scopeProvider; _coreDebugSettings = coreDebugSettings; _mediaFileManager = mediaFileManager; - _eventAggregator = eventAggregator; _logger = logger; Context = scopeContext; _isolationLevel = isolationLevel; - _repositoryCacheMode = repositoryCacheMode; _eventDispatcher = eventDispatcher; - _notificationPublisher = notificationPublisher; - _scopeFileSystem = scopeFileSystems; _autoComplete = autoComplete; Detachable = detachable; - _dictionaryLocker = new object(); #if DEBUG_SCOPES _scopeProvider.RegisterScope(this); @@ -114,14 +97,6 @@ namespace Umbraco.Cms.Infrastructure.Scoping // detachable creates its own scope context Context = new ScopeContext(); - // see note below - if (scopeFileSystems == true) - { - _fscope = fileSystems.Shadow(); - } - - _acquiredLocks = new Queue(); - return; } @@ -129,47 +104,11 @@ namespace Umbraco.Cms.Infrastructure.Scoping { ParentScope = parent; - // cannot specify a different mode! - // TODO: means that it's OK to go from L2 to None for reading purposes, but writing would be BAD! - // this is for XmlStore that wants to bypass caches when rebuilding XML (same for NuCache) - if (repositoryCacheMode != RepositoryCacheMode.Unspecified && - parent.RepositoryCacheMode > repositoryCacheMode) - { - throw new ArgumentException( - $"Value '{repositoryCacheMode}' cannot be lower than parent value '{parent.RepositoryCacheMode}'.", nameof(repositoryCacheMode)); - } - // cannot specify a dispatcher! if (_eventDispatcher != null) { throw new ArgumentException("Value cannot be specified on nested scope.", nameof(eventDispatcher)); } - - // Only the outermost scope can specify the notification publisher - if (_notificationPublisher != null) - { - throw new ArgumentException("Value cannot be specified on nested scope.", nameof(notificationPublisher)); - } - - // cannot specify a different fs scope! - // can be 'true' only on outer scope (and false does not make much sense) - if (scopeFileSystems != null && parent._scopeFileSystem != scopeFileSystems) - { - throw new ArgumentException( - $"Value '{scopeFileSystems.Value}' be different from parent value '{parent._scopeFileSystem}'.", nameof(scopeFileSystems)); - } - } - else - { - _acquiredLocks = new Queue(); - - // the FS scope cannot be "on demand" like the rest, because we would need to hook into - // every scoped FS to trigger the creation of shadow FS "on demand", and that would be - // pretty pointless since if scopeFileSystems is true, we *know* we want to shadow - if (scopeFileSystems == true) - { - _fscope = fileSystems.Shadow(); - } } } @@ -178,6 +117,8 @@ namespace Umbraco.Cms.Infrastructure.Scoping ScopeProvider scopeProvider, CoreDebugSettings coreDebugSettings, MediaFileManager mediaFileManager, + IDistributedLockingMechanismFactory distributedLockingMechanismFactory, + ILoggerFactory loggerFactory, IEventAggregator eventAggregator, ILogger logger, FileSystems fileSystems, @@ -193,6 +134,8 @@ namespace Umbraco.Cms.Infrastructure.Scoping : this( scopeProvider, coreDebugSettings, + distributedLockingMechanismFactory, + loggerFactory, mediaFileManager, eventAggregator, logger, @@ -215,6 +158,8 @@ namespace Umbraco.Cms.Infrastructure.Scoping ScopeProvider scopeProvider, CoreDebugSettings coreDebugSettings, MediaFileManager mediaFileManager, + IDistributedLockingMechanismFactory distributedLockingMechanismFactory, + ILoggerFactory loggerFactory, IEventAggregator eventAggregator, ILogger logger, FileSystems fileSystems, @@ -229,6 +174,8 @@ namespace Umbraco.Cms.Infrastructure.Scoping : this( scopeProvider, coreDebugSettings, + distributedLockingMechanismFactory, + loggerFactory, mediaFileManager, eventAggregator, logger, @@ -257,19 +204,6 @@ namespace Umbraco.Cms.Infrastructure.Scoping } } - public bool ScopedFileSystems - { - get - { - if (ParentScope != null) - { - return ParentScope.ScopedFileSystems; - } - - return _fscope != null; - } - } - // a value indicating whether the scope is detachable // ie whether it was created by CreateDetachedScope public bool Detachable { get; } @@ -309,10 +243,6 @@ namespace Umbraco.Cms.Infrastructure.Scoping // true if Umbraco.CoreDebugSettings.LogUncompletedScope appSetting is set to "true" private bool LogUncompletedScopes => _coreDebugSettings.LogIncompletedScopes; - public Guid InstanceId { get; } = Guid.NewGuid(); - - public int CreatedThreadId { get; } = Thread.CurrentThread.ManagedThreadId; - public ISqlContext SqlContext { get @@ -327,39 +257,6 @@ namespace Umbraco.Cms.Infrastructure.Scoping } } - /// - public RepositoryCacheMode RepositoryCacheMode - { - get - { - if (_repositoryCacheMode != RepositoryCacheMode.Unspecified) - { - return _repositoryCacheMode; - } - - if (ParentScope != null) - { - return ParentScope.RepositoryCacheMode; - } - - return RepositoryCacheMode.Default; - } - } - - /// - public IsolatedCaches IsolatedCaches - { - get - { - if (ParentScope != null) - { - return ParentScope.IsolatedCaches; - } - - return _isolatedCaches ??= new IsolatedCaches(type => new DeepCloneAppCache(new ObjectCacheAppCache())); - } - } - /// public IUmbracoDatabase Database { @@ -383,7 +280,7 @@ namespace Umbraco.Cms.Infrastructure.Scoping // UmbracoDatabase instance directly and ensure it's called when OnExecutingCommand // (so long as the executing command isn't a lock command itself!) // If we could do that, that would be the ultimate lazy executed locks. - EnsureDbLocks(); + Locks.EnsureLocks(InstanceId); return _database; } @@ -407,7 +304,7 @@ namespace Umbraco.Cms.Infrastructure.Scoping try { _database.BeginTransaction(IsolationLevel); - EnsureDbLocks(); + Locks.EnsureLocks(InstanceId); return _database; } catch @@ -442,6 +339,7 @@ namespace Umbraco.Cms.Infrastructure.Scoping } /// + [Obsolete("Will be removed in 14, please use notifications instead")] public IEventDispatcher Events { get @@ -456,45 +354,6 @@ namespace Umbraco.Cms.Infrastructure.Scoping } } - public int Depth - { - get - { - if (ParentScope == null) - { - return 0; - } - - return ParentScope.Depth + 1; - } - } - - public IScopedNotificationPublisher Notifications - { - get - { - EnsureNotDisposed(); - if (ParentScope != null) - { - return ParentScope.Notifications; - } - - return _notificationPublisher ?? - (_notificationPublisher = new ScopedNotificationPublisher(_eventAggregator)); - } - } - - /// - public bool Complete() - { - if (_completed.HasValue == false) - { - _completed = true; - } - - return _completed.Value; - } - public void Dispose() { EnsureNotDisposed(); @@ -522,24 +381,11 @@ namespace Umbraco.Cms.Infrastructure.Scoping #endif } - // Decrement the lock counters on the parent if any. - ClearLocks(InstanceId); + Locks.ClearLocks(InstanceId); + if (ParentScope is null) { - while (!_acquiredLocks?.IsCollectionEmpty() ?? false) - { - _acquiredLocks?.Dequeue().Dispose(); - } - - // We're the parent scope, make sure that locks of all scopes has been cleared - // Since we're only reading we don't have to be in a lock - if (_readLocksDictionary?.Count > 0 || _writeLocksDictionary?.Count > 0) - { - var exception = new InvalidOperationException( - $"All scopes has not been disposed from parent scope: {InstanceId}, see log for more details."); - _logger.LogError(exception, GenerateUnclearedScopesLogMessage()); - throw exception; - } + Locks.EnsureLocksCleared(InstanceId); } _scopeProvider.PopAmbientScope(); // might be null = this is how scopes are removed from context objects @@ -548,64 +394,33 @@ namespace Umbraco.Cms.Infrastructure.Scoping _scopeProvider.Disposed(this); #endif - if (_autoComplete && _completed == null) + if (_autoComplete && Completed == null) { - _completed = true; + Completed = true; } if (ParentScope != null) { - ParentScope.ChildCompleted(_completed); + ParentScope.ChildCompleted(Completed); } else { DisposeLastScope(); } - lock (_lockQueueLocker) - { - _queuedLocks?.Clear(); - } + base.Dispose(); _disposed = true; } - public void EagerReadLock(params int[] lockIds) => EagerReadLockInner(InstanceId, null, lockIds); - - /// - public void ReadLock(params int[] lockIds) => LazyReadLockInner(InstanceId, lockIds); - - public void EagerReadLock(TimeSpan timeout, int lockId) => - EagerReadLockInner(InstanceId, timeout, lockId); - - /// - public void ReadLock(TimeSpan timeout, int lockId) => LazyReadLockInner(InstanceId, timeout, lockId); - - public void EagerWriteLock(params int[] lockIds) => EagerWriteLockInner(InstanceId, null, lockIds); - - /// - public void WriteLock(params int[] lockIds) => LazyWriteLockInner(InstanceId, lockIds); - - public void EagerWriteLock(TimeSpan timeout, int lockId) => - EagerWriteLockInner(InstanceId, timeout, lockId); - - /// - public void WriteLock(TimeSpan timeout, int lockId) => LazyWriteLockInner(InstanceId, timeout, lockId); - /// /// Used for testing. Ensures and gets any queued read locks. /// /// internal Dictionary>? GetReadLocks() { - EnsureDbLocks(); - // always delegate to root/parent scope. - if (ParentScope is not null) - { - return ParentScope.GetReadLocks(); - } - - return _readLocksDictionary; + Locks.EnsureLocks(InstanceId); + return ((LockingMechanism)Locks).GetReadLocks(); } /// @@ -614,113 +429,13 @@ namespace Umbraco.Cms.Infrastructure.Scoping /// internal Dictionary>? GetWriteLocks() { - EnsureDbLocks(); - // always delegate to root/parent scope. - if (ParentScope is not null) - { - return ParentScope.GetWriteLocks(); - } - - return _writeLocksDictionary; + Locks.EnsureLocks(InstanceId); + return ((LockingMechanism)Locks).GetWriteLocks(); } - public void Reset() => _completed = null; + public void Reset() => Completed = null; - public void ChildCompleted(bool? completed) - { - // if child did not complete we cannot complete - if (completed.HasValue == false || completed.Value == false) - { - if (_coreDebugSettings.LogIncompletedScopes) - { - _logger.LogWarning("Uncompleted Child Scope at\r\n {StackTrace}", Environment.StackTrace); - } - - _completed = false; - } - } - - /// - /// When we require a ReadLock or a WriteLock we don't immediately request these locks from the database, - /// instead we only request them when necessary (lazily). - /// To do this, we queue requests for read/write locks. - /// This is so that if there's a request for either of these - /// locks, but the service/repository returns an item from the cache, we don't end up making a DB call to make the - /// read/write lock. - /// This executes the queue of requested locks in order in an efficient way lazily whenever the database instance is - /// resolved. - /// - private void EnsureDbLocks() - { - // always delegate to the root parent - if (ParentScope is not null) - { - ParentScope.EnsureDbLocks(); - } - else - { - lock (_lockQueueLocker) - { - if (_queuedLocks?.Count > 0) - { - DistributedLockType currentType = DistributedLockType.ReadLock; - TimeSpan currentTimeout = TimeSpan.Zero; - Guid currentInstanceId = InstanceId; - var collectedIds = new HashSet(); - - var i = 0; - while (_queuedLocks.Count > 0) - { - (DistributedLockType lockType, TimeSpan timeout, Guid instanceId, var lockId) = _queuedLocks.Dequeue(); - - if (i == 0) - { - currentType = lockType; - currentTimeout = timeout; - currentInstanceId = instanceId; - } - else if (lockType != currentType || timeout != currentTimeout || - instanceId != currentInstanceId) - { - // the lock type, instanceId or timeout switched. - // process the lock ids collected - switch (currentType) - { - case DistributedLockType.ReadLock: - EagerReadLockInner(currentInstanceId, currentTimeout == TimeSpan.Zero ? null : currentTimeout, collectedIds.ToArray()); - break; - case DistributedLockType.WriteLock: - EagerWriteLockInner(currentInstanceId, currentTimeout == TimeSpan.Zero ? null : currentTimeout, collectedIds.ToArray()); - break; - } - - // clear the collected and set new type - collectedIds.Clear(); - currentType = lockType; - currentTimeout = timeout; - currentInstanceId = instanceId; - } - - collectedIds.Add(lockId); - i++; - } - - // process the remaining - switch (currentType) - { - case DistributedLockType.ReadLock: - EagerReadLockInner(currentInstanceId, currentTimeout == TimeSpan.Zero ? null : currentTimeout, collectedIds.ToArray()); - break; - case DistributedLockType.WriteLock: - EagerWriteLockInner(currentInstanceId, currentTimeout == TimeSpan.Zero ? null : currentTimeout, collectedIds.ToArray()); - break; - } - } - } - } - } - - private void EnsureNotDisposed() + internal void EnsureNotDisposed() { // We can't be disposed if (_disposed) @@ -739,48 +454,10 @@ namespace Umbraco.Cms.Infrastructure.Scoping // throw new ObjectDisposedException(GetType().FullName); } - /// - /// Generates a log message with all scopes that hasn't cleared their locks, including how many, and what locks they - /// have requested. - /// - /// Log message. - private string GenerateUnclearedScopesLogMessage() - { - // Dump the dicts into a message for the locks. - var builder = new StringBuilder(); - builder.AppendLine( - $"Lock counters aren't empty, suggesting a scope hasn't been properly disposed, parent id: {InstanceId}"); - WriteLockDictionaryToString(_readLocksDictionary!, builder, "read locks"); - WriteLockDictionaryToString(_writeLocksDictionary!, builder, "write locks"); - return builder.ToString(); - } - - /// - /// Writes a locks dictionary to a for logging purposes. - /// - /// Lock dictionary to report on. - /// String builder to write to. - /// The name to report the dictionary as. - private void WriteLockDictionaryToString(Dictionary> dict, StringBuilder builder, string dictName) - { - if (dict?.Count > 0) - { - builder.AppendLine($"Remaining {dictName}:"); - foreach (KeyValuePair> instance in dict) - { - builder.AppendLine($"Scope {instance.Key}"); - foreach (KeyValuePair lockCounter in instance.Value) - { - builder.AppendLine($"\tLock ID: {lockCounter.Key} - times requested: {lockCounter.Value}"); - } - } - } - } - private void DisposeLastScope() { // figure out completed - var completed = _completed.HasValue && _completed.Value; + var completed = Completed.HasValue && Completed.Value; // deal with database var databaseException = false; @@ -837,29 +514,6 @@ namespace Umbraco.Cms.Infrastructure.Scoping completed = false; } - void HandleScopedFileSystems() - { - if (_scopeFileSystem == true) - { - if (completed) - { - _fscope?.Complete(); - } - - _fscope?.Dispose(); - _fscope = null; - } - } - - void HandleScopedNotifications() - { - if (onException == false) - { - _eventDispatcher?.ScopeExit(completed); - _notificationPublisher?.ScopeExit(completed); - } - } - void HandleScopeContext() { // if *we* created it, then get rid of it @@ -902,8 +556,6 @@ namespace Umbraco.Cms.Infrastructure.Scoping } TryFinally( - HandleScopedFileSystems, - HandleScopedNotifications, HandleScopeContext, HandleDetachedScopes); } @@ -929,288 +581,5 @@ namespace Umbraco.Cms.Infrastructure.Scoping throw new AggregateException(exceptions); } } - - /// - /// Increment the counter of a locks dictionary, either ReadLocks or WriteLocks, - /// for a specific scope instance and lock identifier. Must be called within a lock. - /// - /// Lock ID to increment. - /// Instance ID of the scope requesting the lock. - /// Reference to the dictionary to increment on - private void IncrementLock(int lockId, Guid instanceId, ref Dictionary>? locks) - { - // Since we've already checked that we're the parent in the WriteLockInner method, we don't need to check again. - // If it's the very first time a lock has been requested the WriteLocks dict hasn't been instantiated yet. - locks ??= new Dictionary>(); - - // Try and get the dict associated with the scope id. - var locksDictFound = locks.TryGetValue(instanceId, out Dictionary? locksDict); - if (locksDictFound) - { - locksDict!.TryGetValue(lockId, out var value); - locksDict[lockId] = value + 1; - } - else - { - // The scope hasn't requested a lock yet, so we have to create a dict for it. - locks.Add(instanceId, new Dictionary()); - locks[instanceId][lockId] = 1; - } - } - - /// - /// Clears all lock counters for a given scope instance, signalling that the scope has been disposed. - /// - /// Instance ID of the scope to clear. - private void ClearLocks(Guid instanceId) - { - if (ParentScope is not null) - { - ParentScope.ClearLocks(instanceId); - } - else - { - lock (_dictionaryLocker) - { - _readLocksDictionary?.Remove(instanceId); - _writeLocksDictionary?.Remove(instanceId); - - // remove any queued locks for this instance that weren't used. - while (_queuedLocks?.Count > 0) - { - // It's safe to assume that the locks on the top of the stack belong to this instance, - // since any child scopes that might have added locks to the stack must be disposed before we try and dispose this instance. - (DistributedLockType lockType, TimeSpan timeout, Guid instanceId, int lockId) top = - _queuedLocks.PeekStack(); - if (top.instanceId == instanceId) - { - _queuedLocks.Pop(); - } - else - { - break; - } - } - } - } - } - - public void LazyReadLockInner(Guid instanceId, params int[] lockIds) - { - if (ParentScope != null) - { - ParentScope.LazyReadLockInner(instanceId, lockIds); - } - else - { - LazyLockInner(DistributedLockType.ReadLock, instanceId, lockIds); - } - } - - public void LazyReadLockInner(Guid instanceId, TimeSpan timeout, int lockId) - { - if (ParentScope != null) - { - ParentScope.LazyReadLockInner(instanceId, timeout, lockId); - } - else - { - LazyLockInner(DistributedLockType.ReadLock, instanceId, timeout, lockId); - } - } - - public void LazyWriteLockInner(Guid instanceId, params int[] lockIds) - { - if (ParentScope != null) - { - ParentScope.LazyWriteLockInner(instanceId, lockIds); - } - else - { - LazyLockInner(DistributedLockType.WriteLock, instanceId, lockIds); - } - } - - public void LazyWriteLockInner(Guid instanceId, TimeSpan timeout, int lockId) - { - if (ParentScope != null) - { - ParentScope.LazyWriteLockInner(instanceId, timeout, lockId); - } - else - { - LazyLockInner(DistributedLockType.WriteLock, instanceId, timeout, lockId); - } - } - - private void LazyLockInner(DistributedLockType lockType, Guid instanceId, params int[] lockIds) - { - lock (_lockQueueLocker) - { - if (_queuedLocks == null) - { - _queuedLocks = new StackQueue<(DistributedLockType, TimeSpan, Guid, int)>(); - } - - foreach (var lockId in lockIds) - { - _queuedLocks.Enqueue((lockType, TimeSpan.Zero, instanceId, lockId)); - } - } - } - - private void LazyLockInner(DistributedLockType lockType, Guid instanceId, TimeSpan timeout, int lockId) - { - lock (_lockQueueLocker) - { - if (_queuedLocks == null) - { - _queuedLocks = new StackQueue<(DistributedLockType, TimeSpan, Guid, int)>(); - } - - - _queuedLocks.Enqueue((lockType, timeout, instanceId, lockId)); - } - } - - /// - /// Handles acquiring a read lock, will delegate it to the parent if there are any. - /// - /// Instance ID of the requesting scope. - /// Optional database timeout in milliseconds. - /// Array of lock object identifiers. - private void EagerReadLockInner(Guid instanceId, TimeSpan? timeout, params int[] lockIds) - { - if (ParentScope is not null) - { - // If we have a parent we delegate lock creation to parent. - ParentScope.EagerReadLockInner(instanceId, timeout, lockIds); - } - else - { - lock (_dictionaryLocker) - { - foreach (var lockId in lockIds) - { - IncrementLock(lockId, instanceId, ref _readLocksDictionary); - - // We are the outermost scope, handle the lock request. - LockInner( - instanceId, - ref _readLocksDictionary!, - ref _readLocks!, - ObtainReadLock, - timeout, - lockId); - } - } - } - } - - /// - /// Handles acquiring a write lock with a specified timeout, will delegate it to the parent if there are any. - /// - /// Instance ID of the requesting scope. - /// Optional database timeout in milliseconds. - /// Array of lock object identifiers. - private void EagerWriteLockInner(Guid instanceId, TimeSpan? timeout, params int[] lockIds) - { - if (ParentScope is not null) - { - // If we have a parent we delegate lock creation to parent. - ParentScope.EagerWriteLockInner(instanceId, timeout, lockIds); - } - else - { - lock (_dictionaryLocker) - { - foreach (var lockId in lockIds) - { - IncrementLock(lockId, instanceId, ref _writeLocksDictionary); - - // We are the outermost scope, handle the lock request. - LockInner( - instanceId, - ref _writeLocksDictionary!, - ref _writeLocks!, - ObtainWriteLock, - timeout, - lockId); - } - } - } - } - - /// - /// Handles acquiring a lock, this should only be called from the outermost scope. - /// - /// Instance ID of the scope requesting the lock. - /// Reference to the applicable locks dictionary (ReadLocks or WriteLocks). - /// Reference to the applicable locks hashset (_readLocks or _writeLocks). - /// Delegate used to request the lock from the locking mechanism. - /// Optional timeout parameter to specify a timeout. - /// Lock identifier. - private void LockInner( - Guid instanceId, - ref Dictionary> locks, - ref HashSet locksSet, - Action obtainLock, - TimeSpan? timeout, - int lockId) - { - locksSet ??= new HashSet(); - - // Only acquire the lock if we haven't done so yet. - if (locksSet.Contains(lockId)) - { - return; - } - - locksSet.Add(lockId); - try - { - obtainLock(lockId, timeout); - } - catch - { - // Something went wrong and we didn't get the lock - // Since we at this point have determined that we haven't got any lock with an ID of LockID, it's safe to completely remove it instead of decrementing. - locks[instanceId].Remove(lockId); - - // It needs to be removed from the HashSet as well, because that's how we determine to acquire a lock. - locksSet.Remove(lockId); - throw; - } - } - - /// - /// Obtains a read lock with a custom timeout. - /// - /// Lock object identifier to lock. - /// TimeSpan specifying the timout period. - private void ObtainReadLock(int lockId, TimeSpan? timeout) - { - if (_acquiredLocks == null) - { - throw new InvalidOperationException($"Cannot obtain a read lock as the {nameof(_acquiredLocks)} queue is null."); - } - - _acquiredLocks.Enqueue(_scopeProvider.DistributedLockingMechanismFactory.DistributedLockingMechanism.ReadLock(lockId, timeout)); - } - - /// - /// Obtains a write lock with a custom timeout. - /// - /// Lock object identifier to lock. - /// TimeSpan specifying the timout period. - private void ObtainWriteLock(int lockId, TimeSpan? timeout) - { - if (_acquiredLocks == null) - { - throw new InvalidOperationException($"Cannot obtain a write lock as the {nameof(_acquiredLocks)} queue is null."); - } - - _acquiredLocks.Enqueue(_scopeProvider.DistributedLockingMechanismFactory.DistributedLockingMechanism.WriteLock(lockId, timeout)); - } } } diff --git a/src/Umbraco.Infrastructure/Scoping/ScopeContext.cs b/src/Umbraco.Infrastructure/Scoping/ScopeContext.cs index 0008626e29..fbaad205a2 100644 --- a/src/Umbraco.Infrastructure/Scoping/ScopeContext.cs +++ b/src/Umbraco.Infrastructure/Scoping/ScopeContext.cs @@ -1,6 +1,6 @@ namespace Umbraco.Cms.Core.Scoping; -internal class ScopeContext : IScopeContext, IInstanceIdentifiable +public class ScopeContext : IScopeContext, IInstanceIdentifiable { private Dictionary? _enlisted; diff --git a/src/Umbraco.Infrastructure/Scoping/ScopeProvider.cs b/src/Umbraco.Infrastructure/Scoping/ScopeProvider.cs index 2468a7f80e..5eb367a1b0 100644 --- a/src/Umbraco.Infrastructure/Scoping/ScopeProvider.cs +++ b/src/Umbraco.Infrastructure/Scoping/ScopeProvider.cs @@ -151,7 +151,7 @@ namespace Umbraco.Cms.Infrastructure.Scoping IEventDispatcher? eventDispatcher = null, IScopedNotificationPublisher? scopedNotificationPublisher = null, bool? scopeFileSystems = null) - => new Scope(this, _coreDebugSettings, _mediaFileManager, _eventAggregator, _loggerFactory.CreateLogger(), _fileSystems, true, null, isolationLevel, repositoryCacheMode, eventDispatcher, scopedNotificationPublisher, scopeFileSystems); + => new Scope(this, _coreDebugSettings, _mediaFileManager, DistributedLockingMechanismFactory, _loggerFactory, _eventAggregator, _loggerFactory.CreateLogger(), _fileSystems, true, null, isolationLevel, repositoryCacheMode, eventDispatcher, scopedNotificationPublisher, scopeFileSystems); /// public void AttachScope(IScope other, bool callContext = false) @@ -231,7 +231,7 @@ namespace Umbraco.Cms.Infrastructure.Scoping { IScopeContext? ambientContext = AmbientContext; ScopeContext? newContext = ambientContext == null ? new ScopeContext() : null; - var scope = new Scope(this, _coreDebugSettings, _mediaFileManager, _eventAggregator, _loggerFactory.CreateLogger(), _fileSystems, false, newContext, isolationLevel, repositoryCacheMode, eventDispatcher, notificationPublisher, scopeFileSystems, callContext, autoComplete); + var scope = new Scope(this, _coreDebugSettings, _mediaFileManager, DistributedLockingMechanismFactory, _loggerFactory, _eventAggregator, _loggerFactory.CreateLogger(), _fileSystems, false, newContext, isolationLevel, repositoryCacheMode, eventDispatcher, notificationPublisher, scopeFileSystems, callContext, autoComplete); // assign only if scope creation did not throw! PushAmbientScope(scope); @@ -242,7 +242,7 @@ namespace Umbraco.Cms.Infrastructure.Scoping return scope; } - var nested = new Scope(this, _coreDebugSettings, _mediaFileManager, _eventAggregator, _loggerFactory.CreateLogger(), _fileSystems, ambientScope, isolationLevel, repositoryCacheMode, eventDispatcher, notificationPublisher, scopeFileSystems, callContext, autoComplete); + var nested = new Scope(this, _coreDebugSettings, _mediaFileManager, DistributedLockingMechanismFactory, _loggerFactory, _eventAggregator, _loggerFactory.CreateLogger(), _fileSystems, ambientScope, isolationLevel, repositoryCacheMode, eventDispatcher, notificationPublisher, scopeFileSystems, callContext, autoComplete); PushAmbientScope(nested); return nested; } diff --git a/tests/Umbraco.Tests.Integration/DependencyInjection/UmbracoBuilderExtensions.cs b/tests/Umbraco.Tests.Integration/DependencyInjection/UmbracoBuilderExtensions.cs index 1424b4bf4d..0336b05f22 100644 --- a/tests/Umbraco.Tests.Integration/DependencyInjection/UmbracoBuilderExtensions.cs +++ b/tests/Umbraco.Tests.Integration/DependencyInjection/UmbracoBuilderExtensions.cs @@ -1,12 +1,9 @@ // Copyright (c) Umbraco. // See LICENSE for more details. -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; using Examine; -using Examine.Lucene.Directories; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.FileProviders; using Microsoft.Extensions.Logging; @@ -16,7 +13,7 @@ using NUnit.Framework; using Umbraco.Cms.Core.Cache; using Umbraco.Cms.Core.Composing; using Umbraco.Cms.Core.Configuration.Models; -using Umbraco.Cms.Core.DependencyInjection; +using Umbraco.Cms.Core.DistributedLocking; using Umbraco.Cms.Core.Logging; using Umbraco.Cms.Core.Runtime; using Umbraco.Cms.Core.Services; @@ -25,9 +22,12 @@ using Umbraco.Cms.Core.WebAssets; using Umbraco.Cms.Infrastructure.Examine; using Umbraco.Cms.Infrastructure.HostedServices; using Umbraco.Cms.Infrastructure.PublishedCache; +using Umbraco.Cms.Persistence.EFCore.Locking; +using Umbraco.Cms.Persistence.EFCore.Scoping; using Umbraco.Cms.Tests.Common.TestHelpers.Stubs; using Umbraco.Cms.Tests.Integration.Implementations; -using Umbraco.Extensions; +using Umbraco.Cms.Tests.Integration.Testing; +using Umbraco.Cms.Tests.Integration.Umbraco.Persistence.EFCore.DbContext; namespace Umbraco.Cms.Tests.Integration.DependencyInjection; @@ -63,6 +63,43 @@ public static class UmbracoBuilderExtensions builder.Services.AddUnique(); builder.Services.AddUnique(); + builder.Services.AddDbContext( + (serviceProvider, options) => + { + var testDatabaseType = builder.Config.GetValue("Tests:Database:DatabaseType"); + if (testDatabaseType is TestDatabaseSettings.TestDatabaseType.Sqlite) + { + options.UseSqlite(serviceProvider.GetRequiredService>().CurrentValue.ConnectionString); + } + else + { + // If not Sqlite, assume SqlServer + options.UseSqlServer(serviceProvider.GetRequiredService>().CurrentValue.ConnectionString); + } + }, + optionsLifetime: ServiceLifetime.Singleton); + + builder.Services.AddDbContextFactory( + (serviceProvider, options) => + { + var testDatabaseType = builder.Config.GetValue("Tests:Database:DatabaseType"); + if (testDatabaseType is TestDatabaseSettings.TestDatabaseType.Sqlite) + { + options.UseSqlite(serviceProvider.GetRequiredService>().CurrentValue.ConnectionString); + } + else + { + // If not Sqlite, assume SqlServer + options.UseSqlServer(serviceProvider.GetRequiredService>().CurrentValue.ConnectionString); + } + }); + + builder.Services.AddUnique, AmbientEFCoreScopeStack>(); + builder.Services.AddUnique, EFCoreScopeAccessor>(); + builder.Services.AddUnique, EFCoreScopeProvider>(); + builder.Services.AddSingleton>(); + builder.Services.AddSingleton>(); + return builder; } diff --git a/tests/Umbraco.Tests.Integration/Umbraco.Persistence.EFCore/DbContext/TestUmbracoDbContext.cs b/tests/Umbraco.Tests.Integration/Umbraco.Persistence.EFCore/DbContext/TestUmbracoDbContext.cs new file mode 100644 index 0000000000..35759543e8 --- /dev/null +++ b/tests/Umbraco.Tests.Integration/Umbraco.Persistence.EFCore/DbContext/TestUmbracoDbContext.cs @@ -0,0 +1,31 @@ +using Microsoft.EntityFrameworkCore; + +namespace Umbraco.Cms.Tests.Integration.Umbraco.Persistence.EFCore.DbContext; + +public class TestUmbracoDbContext : Microsoft.EntityFrameworkCore.DbContext +{ + public TestUmbracoDbContext(DbContextOptions options) + : base(options) + { + } + + internal virtual DbSet UmbracoLocks { get; set; } + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity(entity => + { + entity.ToTable("umbracoLock"); + + entity.Property(e => e.Id) + .ValueGeneratedNever() + .HasColumnName("id"); + + entity.Property(e => e.Name) + .HasMaxLength(64) + .HasColumnName("name"); + + entity.Property(e => e.Value).HasColumnName("value"); + }); + } +} diff --git a/tests/Umbraco.Tests.Integration/Umbraco.Persistence.EFCore/DbContext/UmbracoLock.cs b/tests/Umbraco.Tests.Integration/Umbraco.Persistence.EFCore/DbContext/UmbracoLock.cs new file mode 100644 index 0000000000..02a3c648cb --- /dev/null +++ b/tests/Umbraco.Tests.Integration/Umbraco.Persistence.EFCore/DbContext/UmbracoLock.cs @@ -0,0 +1,10 @@ +namespace Umbraco.Cms.Tests.Integration.Umbraco.Persistence.EFCore.DbContext; + +internal class UmbracoLock +{ + public int Id { get; set; } + + public int Value { get; set; } = 1; + + public string Name { get; set; } = null!; +} diff --git a/tests/Umbraco.Tests.Integration/Umbraco.Persistence.EFCore/Scoping/EFCoreLockTests.cs b/tests/Umbraco.Tests.Integration/Umbraco.Persistence.EFCore/Scoping/EFCoreLockTests.cs new file mode 100644 index 0000000000..5103c2e2fa --- /dev/null +++ b/tests/Umbraco.Tests.Integration/Umbraco.Persistence.EFCore/Scoping/EFCoreLockTests.cs @@ -0,0 +1,403 @@ +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using Umbraco.Cms.Core; +using Umbraco.Cms.Core.DistributedLocking; +using Umbraco.Cms.Core.DistributedLocking.Exceptions; +using Umbraco.Cms.Persistence.EFCore.Locking; +using Umbraco.Cms.Persistence.EFCore.Scoping; +using Umbraco.Cms.Persistence.Sqlite.Interceptors; +using Umbraco.Cms.Tests.Common.Testing; +using Umbraco.Cms.Tests.Integration.Testing; +using Umbraco.Cms.Tests.Integration.Umbraco.Persistence.EFCore.DbContext; + +namespace Umbraco.Cms.Tests.Integration.Umbraco.Persistence.EFCore.Scoping; + +[TestFixture] +[Timeout(60000)] +[UmbracoTest(Database = UmbracoTestOptions.Database.NewSchemaPerTest, Logger = UmbracoTestOptions.Logger.Console)] +public class EFCoreLockTests : UmbracoIntegrationTest +{ + private IEFCoreScopeProvider EFScopeProvider => + GetRequiredService>(); + + protected override void ConfigureTestServices(IServiceCollection services) + { + // SQLite + retry policy makes tests fail, we retry before throwing distributed locking timeout. + services.RemoveAll(x => x.ImplementationType == typeof(SqliteAddRetryPolicyInterceptor)); + + // Remove all locking implementations to ensure we only use EFCoreDistributedLockingMechanisms + services.RemoveAll(x => x.ServiceType == typeof(IDistributedLockingMechanism)); + services.AddSingleton>(); + services.AddSingleton>(); + } + + [SetUp] + protected async Task SetUp() + { + // create a few lock objects + using var scope = EFScopeProvider.CreateScope(); + await scope.ExecuteWithContextAsync(async database => + { + database.UmbracoLocks.Add(new UmbracoLock { Id = 1, Name = "Lock.1" }); + database.UmbracoLocks.Add(new UmbracoLock { Id = 2, Name = "Lock.2" }); + database.UmbracoLocks.Add(new UmbracoLock { Id = 3, Name = "Lock.3" }); + + await database.SaveChangesAsync(); + }); + + scope.Complete(); + } + + [Test] + public void SingleEagerReadLockTest() + { + using var scope = EFScopeProvider.CreateScope(); + scope.Locks.EagerReadLock(scope.InstanceId, Constants.Locks.Servers); + scope.Complete(); + } + + [Test] + public void SingleReadLockTest() + { + using var scope = EFScopeProvider.CreateScope(); + scope.Locks.ReadLock(scope.InstanceId, Constants.Locks.Servers); + scope.Complete(); + } + + [Test] + public void SingleWriteLockTest() + { + using var scope = EFScopeProvider.CreateScope(); + scope.Locks.WriteLock(scope.InstanceId, Constants.Locks.Servers); + scope.Complete(); + } + + [Test] + public void SingleEagerWriteLockTest() + { + using var scope = EFScopeProvider.CreateScope(); + scope.Locks.EagerWriteLock(scope.InstanceId, Constants.Locks.Servers); + scope.Complete(); + } + + [Test] + public void Can_Reacquire_Read_Lock() + { + using (var scope = EFScopeProvider.CreateScope()) + { + scope.Locks.EagerReadLock(scope.InstanceId, Constants.Locks.Servers); + scope.Complete(); + } + + using (var scope = EFScopeProvider.CreateScope()) + { + scope.Locks.EagerReadLock(scope.InstanceId, Constants.Locks.Servers); + scope.Complete(); + } + } + + [Test] + public void Can_Reacquire_Write_Lock() + { + using (var scope = EFScopeProvider.CreateScope()) + { + scope.Locks.EagerWriteLock(scope.InstanceId, Constants.Locks.Servers); + scope.Complete(); + } + + using (var scope = EFScopeProvider.CreateScope()) + { + scope.Locks.EagerWriteLock(scope.InstanceId, Constants.Locks.Servers); + scope.Complete(); + } + } + + [Test] + public void ConcurrentReadersTest() + { + if (BaseTestDatabase.IsSqlite()) + { + Assert.Ignore( + "This test doesn't work with Microsoft.Data.Sqlite in EFCore as we no longer use deferred transactions"); + return; + } + + const int threadCount = 8; + var threads = new Thread[threadCount]; + var exceptions = new Exception[threadCount]; + var locker = new object(); + var acquired = 0; + var m2 = new ManualResetEventSlim(false); + var m1 = new ManualResetEventSlim(false); + + for (var i = 0; i < threadCount; i++) + { + var ic = i; // capture + threads[i] = new Thread(() => + { + using (var scope = EFScopeProvider.CreateScope()) + { + try + { + scope.Locks.EagerReadLock(scope.InstanceId, Constants.Locks.Servers); + lock (locker) + { + acquired++; + if (acquired == threadCount) + { + m2.Set(); + } + } + + m1.Wait(); + lock (locker) + { + acquired--; + } + } + catch (Exception e) + { + exceptions[ic] = e; + } + + scope.Complete(); + } + }); + } + + // ensure that current scope does not leak into starting threads + using (ExecutionContext.SuppressFlow()) + { + foreach (var thread in threads) + { + thread.Start(); + } + } + + m2.Wait(); + // all threads have locked in parallel + var maxAcquired = acquired; + m1.Set(); + + foreach (var thread in threads) + { + thread.Join(); + } + + Assert.AreEqual(threadCount, maxAcquired); + Assert.AreEqual(0, acquired); + + for (var i = 0; i < threadCount; i++) + { + Assert.IsNull(exceptions[i]); + } + } + + [Test] + public void ConcurrentWritersTest() + { + if (BaseTestDatabase.IsSqlite()) + { + Assert.Ignore( + "This test doesn't work with Microsoft.Data.Sqlite in EFCore as we no longer use deferred transactions"); + return; + } + + const int threadCount = 3; + var threads = new Thread[threadCount]; + var exceptions = new Exception[threadCount]; + var locker = new object(); + var acquired = 0; + int triedAcquiringWriteLock = 0; + var entered = 0; + var ms = new AutoResetEvent[threadCount]; + for (var i = 0; i < threadCount; i++) + { + ms[i] = new AutoResetEvent(false); + } + + var m1 = new ManualResetEventSlim(false); + var m2 = new ManualResetEventSlim(false); + + for (var i = 0; i < threadCount; i++) + { + var ic = i; // capture + threads[i] = new Thread(() => + { + using (var scope = EFScopeProvider.CreateScope()) + { + try + { + lock (locker) + { + entered++; + if (entered == threadCount) + { + m1.Set(); + } + } + + ms[ic].WaitOne(); + + lock (locker) + { + triedAcquiringWriteLock++; + if (triedAcquiringWriteLock == threadCount) + { + m2.Set(); + } + } + + scope.Locks.EagerWriteLock(scope.InstanceId, Constants.Locks.Servers); + + lock (locker) + { + acquired++; + } + + ms[ic].WaitOne(); + lock (locker) + { + acquired--; + } + } + catch (Exception e) + { + exceptions[ic] = e; + } + + scope.Complete(); + } + }); + } + + // ensure that current scope does not leak into starting threads + using (ExecutionContext.SuppressFlow()) + { + foreach (var thread in threads) + { + thread.Start(); + } + } + + m1.Wait(); + // all threads have entered + ms[0].Set(); // let 0 go + // TODO: This timing is flaky + Thread.Sleep(1000); + for (var i = 1; i < threadCount; i++) + { + ms[i].Set(); // let others go + } + + m2.Wait(); + // only 1 thread has locked + Assert.AreEqual(1, acquired); + for (var i = 0; i < threadCount; i++) + { + ms[i].Set(); // let all go + } + + foreach (var thread in threads) + { + thread.Join(); + } + + Assert.AreEqual(0, acquired); + + for (var i = 0; i < threadCount; i++) + { + Assert.IsNull(exceptions[i]); + } + } + + [Retry(10)] // TODO make this test non-flaky. + [Test] + public void DeadLockTest() + { + if (BaseTestDatabase.IsSqlite()) + { + Assert.Ignore("This test doesn't work with Microsoft.Data.Sqlite - SELECT * FROM sys.dm_tran_locks;"); + return; + } + + Exception e1 = null, e2 = null; + AutoResetEvent ev1 = new(false), ev2 = new(false); + + // testing: + // two threads will each obtain exclusive write locks over two + // identical lock objects deadlock each other + + var thread1 = new Thread(() => DeadLockTestThread(1, 2, ev1, ev2, ref e1)); + var thread2 = new Thread(() => DeadLockTestThread(2, 1, ev2, ev1, ref e2)); + + // ensure that current scope does not leak into starting threads + using (ExecutionContext.SuppressFlow()) + { + thread1.Start(); + thread2.Start(); + } + + ev2.Set(); + + thread1.Join(); + thread2.Join(); + + Assert.IsNotNull(e1); + if (e1 != null) + { + AssertIsDistributedLockingTimeoutException(e1); + } + + // the assertion below depends on timing conditions - on a fast enough environment, + // thread1 dies (deadlock) and frees thread2, which succeeds - however on a slow + // environment (CI) both threads can end up dying due to deadlock - so, cannot test + // that e2 is null - but if it's not, can test that it's a timeout + // + //Assert.IsNull(e2); + if (e2 != null) + { + AssertIsDistributedLockingTimeoutException(e2); + } + } + + private void AssertIsDistributedLockingTimeoutException(Exception e) + { + var sqlException = e as DistributedLockingTimeoutException; + Assert.IsNotNull(sqlException); + } + + private void DeadLockTestThread(int id1, int id2, EventWaitHandle myEv, WaitHandle otherEv, ref Exception exception) + { + using var scope = EFScopeProvider.CreateScope(); + try + { + otherEv.WaitOne(); + Console.WriteLine($"[{id1}] WAIT {id1}"); + scope.Locks.EagerWriteLock(scope.InstanceId, id1); + Console.WriteLine($"[{id1}] GRANT {id1}"); + myEv.Set(); + + if (id1 == 1) + { + otherEv.WaitOne(); + } + else + { + Thread.Sleep(5200); // wait for deadlock... + } + + Console.WriteLine($"[{id1}] WAIT {id2}"); + scope.Locks.EagerWriteLock(scope.InstanceId, id2); + Console.WriteLine($"[{id1}] GRANT {id2}"); + } + catch (Exception e) + { + exception = e; + } + finally + { + scope.Complete(); + } + } +} diff --git a/tests/Umbraco.Tests.Integration/Umbraco.Persistence.EFCore/Scoping/EFCoreScopeInfrastructureScopeLockTests.cs b/tests/Umbraco.Tests.Integration/Umbraco.Persistence.EFCore/Scoping/EFCoreScopeInfrastructureScopeLockTests.cs new file mode 100644 index 0000000000..262382cdda --- /dev/null +++ b/tests/Umbraco.Tests.Integration/Umbraco.Persistence.EFCore/Scoping/EFCoreScopeInfrastructureScopeLockTests.cs @@ -0,0 +1,139 @@ +using Microsoft.EntityFrameworkCore; +using NUnit.Framework; +using Umbraco.Cms.Core; +using Umbraco.Cms.Core.Events; +using Umbraco.Cms.Core.Notifications; +using Umbraco.Cms.Infrastructure.Scoping; +using Umbraco.Cms.Persistence.EFCore.Scoping; +using Umbraco.Cms.Tests.Common.Testing; +using Umbraco.Cms.Tests.Integration.Testing; +using Umbraco.Cms.Tests.Integration.Umbraco.Persistence.EFCore.DbContext; + +namespace Umbraco.Cms.Tests.Integration.Umbraco.Persistence.EFCore.Scoping; + +[TestFixture] +[UmbracoTest(Database = UmbracoTestOptions.Database.NewSchemaPerTest)] +public class EFCoreScopeInfrastructureScopeLockTests : UmbracoIntegrationTest +{ + private IEFCoreScopeProvider EfCoreScopeProvider => + GetRequiredService>(); + + private IScopeProvider InfrastructureScopeProvider => + GetRequiredService(); + + protected override void CustomTestSetup(IUmbracoBuilder builder) + { + base.CustomTestSetup(builder); + + builder.AddNotificationHandler(); + builder.AddNotificationHandler(); + } + + [Test] + public async Task ScopesCanShareNonEagerLocks() + { + using IEfCoreScope parentScope = EfCoreScopeProvider.CreateScope(); + await parentScope.ExecuteWithContextAsync(async database => + { + parentScope.Locks.WriteLock(parentScope.InstanceId, Constants.Locks.Servers); + await database.Database.ExecuteSqlAsync($"CREATE TABLE tmp3 (id INT, name NVARCHAR(64))"); + await database.Database.ExecuteSqlAsync($"INSERT INTO tmp3 (id, name) VALUES (1, 'a')"); + }); + + using (var childScope = InfrastructureScopeProvider.CreateScope()) + { + childScope.Locks.WriteLock(childScope.InstanceId, Constants.Locks.Servers); + string n = childScope.Database.ExecuteScalar("SELECT name FROM tmp3 WHERE id=1"); + Assert.AreEqual("a", n); + childScope.Complete(); + } + + parentScope.Complete(); + } + + [Test] + public async Task ScopesCanShareEagerLocks() + { + using IEfCoreScope parentScope = EfCoreScopeProvider.CreateScope(); + await parentScope.ExecuteWithContextAsync(async database => + { + parentScope.Locks.EagerWriteLock(parentScope.InstanceId, Constants.Locks.Servers); + await database.Database.ExecuteSqlAsync($"CREATE TABLE tmp3 (id INT, name NVARCHAR(64))"); + await database.Database.ExecuteSqlAsync($"INSERT INTO tmp3 (id, name) VALUES (1, 'a')"); + }); + + using (var childScope = InfrastructureScopeProvider.CreateScope()) + { + childScope.Locks.EagerWriteLock(childScope.InstanceId, Constants.Locks.Servers); + string n = childScope.Database.ExecuteScalar("SELECT name FROM tmp3 WHERE id=1"); + Assert.AreEqual("a", n); + childScope.Complete(); + } + + parentScope.Complete(); + } + + [Test] + public void EFCoreScopeAsParent_Child_Scope_Can_Send_Notification() + { + var currentAssertCount = TestContext.CurrentContext.AssertCount; + using (var scope = EfCoreScopeProvider.CreateScope()) + { + using (var childScope = InfrastructureScopeProvider.CreateScope()) + { + var savingNotification = new TestSendNotification(); + childScope.Notifications.Publish(savingNotification); + childScope.Complete(); + } + + // Assert notifications arent send on completion of scope + Assert.AreEqual(currentAssertCount, TestContext.CurrentContext.AssertCount); + + scope.Complete(); + } + + Assert.AreEqual(currentAssertCount + 2, TestContext.CurrentContext.AssertCount); + } + + [Test] + public void InfrastructureScopeAsParent_Child_Scope_Can_Send_Notification() + { + var currentAssertCount = TestContext.CurrentContext.AssertCount; + using (var scope = InfrastructureScopeProvider.CreateScope()) + { + using (var childScope = EfCoreScopeProvider.CreateScope()) + { + var savingNotification = new TestSendNotification(); + childScope.Notifications.Publish(savingNotification); + childScope.Complete(); + } + + // Assert notifications arent send on completion of scope + Assert.AreEqual(currentAssertCount, TestContext.CurrentContext.AssertCount); + + scope.Complete(); + } + + Assert.AreEqual(currentAssertCount + 2, TestContext.CurrentContext.AssertCount); + } + + private class TestSendNotification : INotification + { + } + + private class TestDoNotSendNotification : INotification + { + } + + private class TestSendNotificationHandler : INotificationHandler + { + public void Handle(TestSendNotification notification) + => Assert.IsNotNull(notification); + } + + private class TestDoNotSendNotificationHandler : INotificationHandler + { + public void Handle(TestDoNotSendNotification notification) + => Assert.Fail("Notification was sent"); + } +} diff --git a/tests/Umbraco.Tests.Integration/Umbraco.Persistence.EFCore/Scoping/EFCoreScopeInfrastructureScopeTests.cs b/tests/Umbraco.Tests.Integration/Umbraco.Persistence.EFCore/Scoping/EFCoreScopeInfrastructureScopeTests.cs new file mode 100644 index 0000000000..130b807c73 --- /dev/null +++ b/tests/Umbraco.Tests.Integration/Umbraco.Persistence.EFCore/Scoping/EFCoreScopeInfrastructureScopeTests.cs @@ -0,0 +1,208 @@ +using Microsoft.EntityFrameworkCore; +using NUnit.Framework; +using Umbraco.Cms.Infrastructure.Scoping; +using Umbraco.Cms.Persistence.EFCore.Scoping; +using Umbraco.Cms.Tests.Common.Testing; +using Umbraco.Cms.Tests.Integration.Testing; +using Umbraco.Cms.Tests.Integration.Umbraco.Persistence.EFCore.DbContext; + +namespace Umbraco.Cms.Tests.Integration.Umbraco.Persistence.EFCore.Scoping; + +[TestFixture] +[UmbracoTest(Database = UmbracoTestOptions.Database.NewEmptyPerTest)] +public class EFCoreScopeInfrastructureScopeTests : UmbracoIntegrationTest +{ + private IEFCoreScopeProvider EfCoreScopeProvider => + GetRequiredService>(); + + private IScopeProvider InfrastructureScopeProvider => + GetRequiredService(); + + private EFCoreScopeAccessor EfCoreScopeAccessor => (EFCoreScopeAccessor)GetRequiredService>(); + + private IScopeAccessor InfrastructureScopeAccessor => GetRequiredService(); + + [Test] + public void CanCreateNestedInfrastructureScope() + { + Assert.IsNull(EfCoreScopeAccessor.AmbientScope); + using (IEfCoreScope scope = EfCoreScopeProvider.CreateScope()) + { + Assert.IsInstanceOf>(scope); + Assert.IsNotNull(EfCoreScopeAccessor.AmbientScope); + Assert.IsNotNull(InfrastructureScopeAccessor.AmbientScope); + Assert.AreSame(scope, EfCoreScopeAccessor.AmbientScope); + using (var infrastructureScope = InfrastructureScopeProvider.CreateScope()) + { + Assert.AreSame(infrastructureScope, InfrastructureScopeAccessor.AmbientScope); + } + + Assert.IsNotNull(InfrastructureScopeAccessor.AmbientScope); + } + + Assert.IsNull(EfCoreScopeAccessor.AmbientScope); + Assert.IsNull(InfrastructureScopeAccessor.AmbientScope); + } + + [Test] + public async Task? TransactionWithEfCoreScopeAsParent() + { + using (IEfCoreScope parentScope = EfCoreScopeProvider.CreateScope()) + { + await parentScope.ExecuteWithContextAsync(async database => + { + await database.Database.ExecuteSqlAsync($"CREATE TABLE tmp3 (id INT, name NVARCHAR(64))"); + }); + + // This should be using same transaction, so insert data into table we're creating + using (IScope childScope = InfrastructureScopeProvider.CreateScope()) + { + childScope.Database.Execute("INSERT INTO tmp3 (id, name) VALUES (1, 'a')"); + string n = ScopeAccessor.AmbientScope.Database.ExecuteScalar( + "SELECT name FROM tmp3 WHERE id=1"); + Assert.AreEqual("a", n); + childScope.Complete(); + } + + await parentScope.ExecuteWithContextAsync(async database => + { + string? result = await database.Database.ExecuteScalarAsync("SELECT name FROM tmp3 WHERE id=1"); + Assert.AreEqual("a", result); + }); + + + parentScope.Complete(); + } + + // Check that its not rolled back + using (IEfCoreScope scope = EfCoreScopeProvider.CreateScope()) + { + await scope.ExecuteWithContextAsync(async database => + { + string? result = await database.Database.ExecuteScalarAsync("SELECT name FROM tmp3 WHERE id=1"); + Assert.IsNotNull(result); + }); + } + } + + [Test] + public async Task? TransactionWithInfrastructureScopeAsParent() + { + using (IScope parentScope = InfrastructureScopeProvider.CreateScope()) + { + parentScope.Database.Execute("CREATE TABLE tmp3 (id INT, name NVARCHAR(64))"); + + using (IEfCoreScope scope = EfCoreScopeProvider.CreateScope()) + { + await scope.ExecuteWithContextAsync(async database => + { + await database.Database.ExecuteSqlAsync($"INSERT INTO tmp3 (id, name) VALUES (1, 'a')"); + + string? result = + await database.Database.ExecuteScalarAsync("SELECT name FROM tmp3 WHERE id=1"); + Assert.AreEqual("a", result); + }); + + scope.Complete(); + } + + parentScope.Complete(); + } + + // Check that its not rolled back + using (IEfCoreScope scope = EfCoreScopeProvider.CreateScope()) + { + await scope.ExecuteWithContextAsync(async database => + { + string? result = await database.Database.ExecuteScalarAsync("SELECT name FROM tmp3 WHERE id=1"); + Assert.IsNotNull(result); + }); + } + } + + [Test] + public async Task EFCoreAsParent_DontCompleteWhenChildScopeDoesNotComplete() + { + using (IEfCoreScope scope = EfCoreScopeProvider.CreateScope()) + { + await scope.ExecuteWithContextAsync(async database => + { + await database.Database.ExecuteSqlAsync($"CREATE TABLE tmp3 (id INT, name NVARCHAR(64))"); + }); + scope.Complete(); + } + + using (IEfCoreScope parentScope = EfCoreScopeProvider.CreateScope()) + { + using (IScope scope = InfrastructureScopeProvider.CreateScope()) + { + scope.Database.Execute("INSERT INTO tmp3 (id, name) VALUES (1, 'a')"); + string n = ScopeAccessor.AmbientScope.Database.ExecuteScalar("SELECT name FROM tmp3 WHERE id=1"); + Assert.AreEqual("a", n); + } + + await parentScope.ExecuteWithContextAsync(async database => + { + // Should still be in transaction and not rolled back yet + string? result = await database.Database.ExecuteScalarAsync("SELECT name FROM tmp3 WHERE id=1"); + Assert.AreEqual("a", result); + }); + + parentScope.Complete(); + } + + // Check that its rolled back + using (IEfCoreScope scope = EfCoreScopeProvider.CreateScope()) + { + await scope.ExecuteWithContextAsync(async database => + { + // Should still be in transaction and not rolled back yet + string? result = await database.Database.ExecuteScalarAsync("SELECT name FROM tmp3 WHERE id=1"); + Assert.IsNull(result); + }); + } + } + + [Test] + public async Task InfrastructureScopeAsParent_DontCompleteWhenChildScopeDoesNotComplete() + { + using (IEfCoreScope scope = EfCoreScopeProvider.CreateScope()) + { + await scope.ExecuteWithContextAsync(async database => + { + await database.Database.ExecuteSqlAsync($"CREATE TABLE tmp3 (id INT, name NVARCHAR(64))"); + }); + + scope.Complete(); + } + + using (IScope parentScope = InfrastructureScopeProvider.CreateScope()) + { + using (IEfCoreScope scope = EfCoreScopeProvider.CreateScope()) + { + await scope.ExecuteWithContextAsync(async database => + { + await database.Database.ExecuteSqlAsync($"INSERT INTO tmp3 (id, name) VALUES (1, 'a')"); + + string? result = await database.Database.ExecuteScalarAsync("SELECT name FROM tmp3 WHERE id=1"); + Assert.AreEqual("a", result); + }); + + string n = parentScope.Database.ExecuteScalar("SELECT name FROM tmp3 WHERE id=1"); + Assert.AreEqual("a", n); + } + + parentScope.Complete(); + } + + // Check that its rolled back + using (IEfCoreScope scope = EfCoreScopeProvider.CreateScope()) + { + await scope.ExecuteWithContextAsync(async database => + { + string? result = await database.Database.ExecuteScalarAsync("SELECT name FROM tmp3 WHERE id=1"); + Assert.IsNull(result); + }); + } + } +} diff --git a/tests/Umbraco.Tests.Integration/Umbraco.Persistence.EFCore/Scoping/EFCoreScopeNotificationsTest.cs b/tests/Umbraco.Tests.Integration/Umbraco.Persistence.EFCore/Scoping/EFCoreScopeNotificationsTest.cs new file mode 100644 index 0000000000..f75d51d0e3 --- /dev/null +++ b/tests/Umbraco.Tests.Integration/Umbraco.Persistence.EFCore/Scoping/EFCoreScopeNotificationsTest.cs @@ -0,0 +1,212 @@ +using NUnit.Framework; +using Umbraco.Cms.Core.Events; +using Umbraco.Cms.Core.Notifications; +using Umbraco.Cms.Persistence.EFCore.Scoping; +using Umbraco.Cms.Tests.Common.Testing; +using Umbraco.Cms.Tests.Integration.Testing; +using Umbraco.Cms.Tests.Integration.Umbraco.Persistence.EFCore.DbContext; + +namespace Umbraco.Cms.Tests.Integration.Umbraco.Persistence.EFCore.Scoping; + +[TestFixture] +[UmbracoTest(Database = UmbracoTestOptions.Database.NewEmptyPerTest)] +public class EFCoreScopeNotificationsTest : UmbracoIntegrationTest +{ + protected override void CustomTestSetup(IUmbracoBuilder builder) + { + base.CustomTestSetup(builder); + + builder.AddNotificationHandler(); + builder.AddNotificationHandler(); + } + + private IEFCoreScopeProvider EfCoreScopeProvider => GetRequiredService>(); + + [Test] + public void Scope_Can_Send_Notification() + { + // We do asserts in the setup of Umbraco, therefore get the + // current number of asserts right how, and assert later that this + // has only gone up by 1 + var initialAssertCount = TestContext.CurrentContext.AssertCount; + + using (var scope = EfCoreScopeProvider.CreateScope()) + { + var savingNotification = new TestSendNotification(); + scope.Notifications.Publish(savingNotification); + scope.Complete(); + } + + Assert.AreEqual(initialAssertCount + 1, TestContext.CurrentContext.AssertCount); + } + + [Test] + public void Child_Scope_Can_Send_Notification() + { + var initialAssertCount = TestContext.CurrentContext.AssertCount; + using (var scope = EfCoreScopeProvider.CreateScope()) + { + using (var childScope = EfCoreScopeProvider.CreateScope()) + { + var savingNotification = new TestSendNotification(); + childScope.Notifications.Publish(savingNotification); + childScope.Complete(); + } + + scope.Complete(); + } + + Assert.AreEqual(initialAssertCount + 1, TestContext.CurrentContext.AssertCount); + } + + [Test] + public void Scope_Does_Not_Send_Notification_When_Not_Completed() + { + using var scope = EfCoreScopeProvider.CreateScope(); + + var savingNotification = new TestDoNotSendNotification(); + scope.Notifications.Publish(savingNotification); + } + + [Test] + public void Scope_Does_Not_Send_Notification_When_Suppressing() + { + using var scope = EfCoreScopeProvider.CreateScope(); + scope.Notifications.Suppress(); + var savingNotification = new TestDoNotSendNotification(); + scope.Notifications.Publish(savingNotification); + scope.Complete(); + } + + [Test] + public void Child_Scope_Cannot_Send_Suppressed_Notification() + { + using var scope = EfCoreScopeProvider.CreateScope(); + + using (var childScope = EfCoreScopeProvider.CreateScope()) + { + childScope.Notifications.Suppress(); + var savingNotification = new TestDoNotSendNotification(); + childScope.Notifications.Publish(savingNotification); + } + + scope.Complete(); + } + + [Test] + public void Parent_Scope_Can_Send_Notification_Before_Child_Suppressing() + { + var initialAssertCount = TestContext.CurrentContext.AssertCount; + + using (var scope = EfCoreScopeProvider.CreateScope()) + { + var savingParentNotification = new TestSendNotification(); + scope.Notifications.Publish(savingParentNotification); + using (var childScope = EfCoreScopeProvider.CreateScope()) + { + childScope.Notifications.Suppress(); + var savingNotification = new TestDoNotSendNotification(); + childScope.Notifications.Publish(savingNotification); + childScope.Complete(); + } + + scope.Complete(); + } + + Assert.AreEqual(initialAssertCount + 1, TestContext.CurrentContext.AssertCount); + } + + [Test] + public void Parent_Scope_Can_Send_Notification_After_Child_Suppressing() + { + var initialAssertCount = TestContext.CurrentContext.AssertCount; + + + using (var scope = EfCoreScopeProvider.CreateScope()) + { + using (var childScope = EfCoreScopeProvider.CreateScope()) + { + using (childScope.Notifications.Suppress()) + { + var savingNotification = new TestDoNotSendNotification(); + childScope.Notifications.Publish(savingNotification); + childScope.Complete(); + } + } + + var savingParentNotificationTwo = new TestSendNotification(); + scope.Notifications.Publish(savingParentNotificationTwo); + + scope.Complete(); + } + + Assert.AreEqual(initialAssertCount + 1, TestContext.CurrentContext.AssertCount); + } + + [Test] + public void Scope_Can_Send_Notification_After_Suppression_Disposed() + { + var initialAssertCount = TestContext.CurrentContext.AssertCount; + + using (var scope = EfCoreScopeProvider.CreateScope()) + { + using (scope.Notifications.Suppress()) + { + var savingNotification = new TestDoNotSendNotification(); + scope.Notifications.Publish(savingNotification); + } + + var savingParentNotificationTwo = new TestSendNotification(); + scope.Notifications.Publish(savingParentNotificationTwo); + + scope.Complete(); + } + + Assert.AreEqual(initialAssertCount + 1, TestContext.CurrentContext.AssertCount); + } + + [Test] + public void Child_Scope_Does_Not_Send_Notification_When_Parent_Suppressing() + { + using var scope = EfCoreScopeProvider.CreateScope(); + scope.Notifications.Suppress(); + + using (var childScope = EfCoreScopeProvider.CreateScope()) + { + var savingNotification = new TestDoNotSendNotification(); + childScope.Notifications.Publish(savingNotification); + childScope.Complete(); + } + + scope.Complete(); + } + + [Test] + public void Cant_Suppress_Notifactions_On_Child_When_Parent_Suppressing() + { + using var parentScope = EfCoreScopeProvider.CreateScope(); + using var parentSuppressed = parentScope.Notifications.Suppress(); + using var childScope = EfCoreScopeProvider.CreateScope(); + Assert.Throws(() => childScope.Notifications.Suppress()); + } + + private class TestSendNotification : INotification + { + } + + private class TestDoNotSendNotification : INotification + { + } + + private class TestSendNotificationHandler : INotificationHandler + { + public void Handle(TestSendNotification notification) + => Assert.IsNotNull(notification); + } + + private class TestDoNotSendNotificationHandler : INotificationHandler + { + public void Handle(TestDoNotSendNotification notification) + => Assert.Fail("Notification was sent"); + } +} diff --git a/tests/Umbraco.Tests.Integration/Umbraco.Persistence.EFCore/Scoping/EFCoreScopeTest.cs b/tests/Umbraco.Tests.Integration/Umbraco.Persistence.EFCore/Scoping/EFCoreScopeTest.cs new file mode 100644 index 0000000000..deefb0d99b --- /dev/null +++ b/tests/Umbraco.Tests.Integration/Umbraco.Persistence.EFCore/Scoping/EFCoreScopeTest.cs @@ -0,0 +1,670 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using Moq; +using NUnit.Framework; +using Umbraco.Cms.Core.Scoping; +using Umbraco.Cms.Persistence.EFCore.Scoping; +using Umbraco.Cms.Tests.Common; +using Umbraco.Cms.Tests.Common.Testing; +using Umbraco.Cms.Tests.Integration.Testing; +using Umbraco.Cms.Tests.Integration.Umbraco.Persistence.EFCore.DbContext; + +namespace Umbraco.Cms.Tests.Integration.Umbraco.Persistence.EFCore.Scoping; + +[TestFixture] +[UmbracoTest(Database = UmbracoTestOptions.Database.NewEmptyPerTest)] +public class EFCoreScopeTest : UmbracoIntegrationTest +{ + private IEFCoreScopeProvider EfCoreScopeProvider => + GetRequiredService>(); + + private EFCoreScopeAccessor EfCoreScopeAccessor => (EFCoreScopeAccessor)GetRequiredService>(); + + [Test] + public void CanCreateScope() + { + Assert.IsNull(EfCoreScopeAccessor.AmbientScope); + using (IEfCoreScope scope = EfCoreScopeProvider.CreateScope()) + { + Assert.IsInstanceOf>(scope); + Assert.IsNotNull(EfCoreScopeAccessor.AmbientScope); + Assert.AreSame(scope, EfCoreScopeAccessor.AmbientScope); + } + + Assert.IsNull(EfCoreScopeAccessor.AmbientScope); + } + + [Test] + public void CanCreateScopeTwice() => + Assert.DoesNotThrow(() => + { + using (var scope = EfCoreScopeProvider.CreateScope()) + { + scope.Complete(); + } + + using (var scopeTwo = EfCoreScopeProvider.CreateScope()) + { + scopeTwo.Complete(); + } + }); + + [Test] + public void NestedCreateScope() + { + Assert.IsNull(EfCoreScopeAccessor.AmbientScope); + using (IEfCoreScope scope = EfCoreScopeProvider.CreateScope()) + { + Assert.IsInstanceOf>(scope); + Assert.IsNotNull(EfCoreScopeAccessor.AmbientScope); + Assert.AreSame(scope, EfCoreScopeAccessor.AmbientScope); + using (IEfCoreScope nested = EfCoreScopeProvider.CreateScope()) + { + Assert.IsInstanceOf>(nested); + Assert.IsNotNull(EfCoreScopeAccessor.AmbientScope); + Assert.AreSame(nested, EfCoreScopeAccessor.AmbientScope); + Assert.AreSame(scope, ((EFCoreScope)nested).ParentScope); + } + } + + Assert.IsNull(EfCoreScopeAccessor.AmbientScope); + } + + [Test] + public async Task NestedCreateScopeInnerException() + { + bool scopeCompleted = false; + + Assert.IsNull(EfCoreScopeAccessor.AmbientScope); + try + { + using (IEfCoreScope scope = EfCoreScopeProvider.CreateScope()) + { + // scopeProvider.Context.Enlist("test", completed => scopeCompleted = completed); + await scope.ExecuteWithContextAsync(async database => + { + scope.ScopeContext!.Enlist("test", completed => scopeCompleted = completed); + Assert.IsInstanceOf>(scope); + Assert.IsNotNull(EfCoreScopeAccessor.AmbientScope); + Assert.AreSame(scope, EfCoreScopeAccessor.AmbientScope); + using (IEfCoreScope nested = EfCoreScopeProvider.CreateScope()) + { + Assert.IsInstanceOf>(nested); + Assert.IsNotNull(EfCoreScopeAccessor.AmbientScope); + Assert.AreSame(nested, EfCoreScopeAccessor.AmbientScope); + Assert.AreSame(scope, ((EFCoreScope)nested).ParentScope); + nested.Complete(); + throw new Exception("bang!"); + } + + return true; + }); + + scope.Complete(); + } + + Assert.Fail("Expected exception."); + } + catch (Exception e) + { + if (e.Message != "bang!") + { + Assert.Fail("Wrong exception."); + } + } + + Assert.IsNull(EfCoreScopeAccessor.AmbientScope); + Assert.IsFalse(scopeCompleted); + } + + [Test] + public async Task CanAccessDbContext() + { + using var scope = EfCoreScopeProvider.CreateScope(); + await scope.ExecuteWithContextAsync(async database => + { + Assert.IsTrue(await database.Database.CanConnectAsync()); + Assert.IsNotNull(database.Database.CurrentTransaction); // in a transaction + }); + scope.Complete(); + } + + [Test] + public async Task CanAccessDbContextTwice() + { + using (var scope = EfCoreScopeProvider.CreateScope()) + { + await scope.ExecuteWithContextAsync(async database => + { + Assert.IsTrue(await database.Database.CanConnectAsync()); + Assert.IsNotNull(database.Database.CurrentTransaction); // in a transaction + }); + scope.Complete(); + } + + using (var scopeTwo = EfCoreScopeProvider.CreateScope()) + { + await scopeTwo.ExecuteWithContextAsync(async database => + { + Assert.IsTrue(await database.Database.CanConnectAsync()); + Assert.IsNotNull(database.Database.CurrentTransaction); // in a transaction + }); + + scopeTwo.Complete(); + } + } + + [Test] + public async Task CanAccessNestedDbContext() + { + using (var scope = EfCoreScopeProvider.CreateScope()) + { + await scope.ExecuteWithContextAsync(async database => + { + Assert.IsTrue(await database.Database.CanConnectAsync()); + var parentTransaction = database.Database.CurrentTransaction; + + using (var nestedSCope = EfCoreScopeProvider.CreateScope()) + { + await nestedSCope.ExecuteWithContextAsync(async nestedDatabase => + { + Assert.IsTrue(await nestedDatabase.Database.CanConnectAsync()); + Assert.IsNotNull(nestedDatabase.Database.CurrentTransaction); // in a transaction + var childTransaction = nestedDatabase.Database.CurrentTransaction; + Assert.AreSame(parentTransaction, childTransaction); + }); + } + }); + scope.Complete(); + } + } + + [Test] + public void GivenUncompletedScopeOnChildThread_WhenTheParentCompletes_TheTransactionIsRolledBack() + { + Assert.IsNull(EfCoreScopeAccessor.AmbientScope); + IEfCoreScope mainScope = EfCoreScopeProvider.CreateScope(); + + var t = Task.Run(() => + { + IEfCoreScope nested = EfCoreScopeProvider.CreateScope(); + Thread.Sleep(2000); + nested.Dispose(); + }); + + Thread.Sleep(1000); // mimic some long running operation that is shorter than the other thread + mainScope.Complete(); + Assert.Throws(() => mainScope.Dispose()); + + Task.WaitAll(t); + } + + [Test] + public void GivenNonDisposedChildScope_WhenTheParentDisposes_ThenInvalidOperationExceptionThrows() + { + Assert.IsNull(EfCoreScopeAccessor.AmbientScope); + IEfCoreScope mainScope = EfCoreScopeProvider.CreateScope(); + + IEfCoreScope nested = EfCoreScopeProvider.CreateScope(); // not disposing + + InvalidOperationException ex = Assert.Throws(() => mainScope.Dispose()); + Console.WriteLine(ex); + } + + [Test] + public void GivenChildThread_WhenParentDisposedBeforeChild_ParentScopeThrows() + { + Assert.IsNull(EfCoreScopeAccessor.AmbientScope); + IEfCoreScope mainScope = EfCoreScopeProvider.CreateScope(); + + var t = Task.Run(() => + { + Console.WriteLine("Child Task start: " + EfCoreScopeAccessor.AmbientScope?.InstanceId); + + // This will push the child scope to the top of the Stack + IEfCoreScope nested = EfCoreScopeProvider.CreateScope(); + Console.WriteLine("Child Task scope created: " + EfCoreScopeAccessor.AmbientScope?.InstanceId); + Thread.Sleep(5000); // block for a bit to ensure the parent task is disposed first + Console.WriteLine("Child Task before dispose: " + EfCoreScopeAccessor.AmbientScope?.InstanceId); + nested.Dispose(); + Console.WriteLine("Child Task after dispose: " + EfCoreScopeAccessor.AmbientScope?.InstanceId); + }); + + // provide some time for the child thread to start so the ambient context is copied in AsyncLocal + Thread.Sleep(2000); + + // now dispose the main without waiting for the child thread to join + Console.WriteLine("Parent Task disposing: " + EfCoreScopeAccessor.AmbientScope?.InstanceId); + + // This will throw because at this stage a child scope has been created which means + // it is the Ambient (top) scope but here we're trying to dispose the non top scope. + Assert.Throws(() => mainScope.Dispose()); + t.Wait(); // wait for the child to dispose + mainScope.Dispose(); // now it's ok + Console.WriteLine("Parent Task disposed: " + EfCoreScopeAccessor.AmbientScope?.InstanceId); + } + + [Test] + public void GivenChildThread_WhenChildDisposedBeforeParent_OK() + { + Assert.IsNull(EfCoreScopeAccessor.AmbientScope); + IEfCoreScope mainScope = EfCoreScopeProvider.CreateScope(); + + // Task.Run will flow the execution context unless ExecutionContext.SuppressFlow() is explicitly called. + // This is what occurs in normal async behavior since it is expected to await (and join) the main thread, + // but if Task.Run is used as a fire and forget thread without being done correctly then the Scope will + // flow to that thread. + var t = Task.Run(() => + { + Console.WriteLine("Child Task start: " + EfCoreScopeAccessor.AmbientScope?.InstanceId); + IEfCoreScope nested = EfCoreScopeProvider.CreateScope(); + Console.WriteLine("Child Task before dispose: " + EfCoreScopeAccessor.AmbientScope?.InstanceId); + nested.Dispose(); + Console.WriteLine("Child Task after disposed: " + EfCoreScopeAccessor.AmbientScope?.InstanceId); + }); + + Console.WriteLine("Parent Task waiting: " + EfCoreScopeAccessor.AmbientScope?.InstanceId); + t.Wait(); + Console.WriteLine("Parent Task disposing: " + EfCoreScopeAccessor.AmbientScope?.InstanceId); + mainScope.Dispose(); + Console.WriteLine("Parent Task disposed: " + EfCoreScopeAccessor.AmbientScope?.InstanceId); + + Assert.Pass(); + } + + [Test] + public async Task Transaction() + { + using (IEfCoreScope scope = EfCoreScopeProvider.CreateScope()) + { + await scope.ExecuteWithContextAsync(async database => + { + await database.Database.ExecuteSqlAsync($"CREATE TABLE tmp3 (id INT, name NVARCHAR(64))"); + }); + scope.Complete(); + } + + using (IEfCoreScope scope = EfCoreScopeProvider.CreateScope()) + { + await scope.ExecuteWithContextAsync(async database => + { + await database.Database.ExecuteSqlAsync($"INSERT INTO tmp3 (id, name) VALUES (1, 'a')"); + + string? result = await database.Database.ExecuteScalarAsync("SELECT name FROM tmp3 WHERE id=1"); + Assert.AreEqual("a", result); + }); + } + + using (IEfCoreScope scope = EfCoreScopeProvider.CreateScope()) + { + await scope.ExecuteWithContextAsync(async database => + { + string n = await database.Database.ExecuteScalarAsync("SELECT name FROM tmp3 WHERE id=1"); + Assert.IsNull(n); + }); + } + + using (IEfCoreScope scope = EfCoreScopeProvider.CreateScope()) + { + await scope.ExecuteWithContextAsync(async database => + { + await database.Database.ExecuteSqlAsync($"INSERT INTO tmp3 (id, name) VALUES (1, 'a')"); + }); + + scope.Complete(); + } + + using (IEfCoreScope scope = EfCoreScopeProvider.CreateScope()) + { + await scope.ExecuteWithContextAsync(async database => + { + string n = await database.Database.ExecuteScalarAsync("SELECT name FROM tmp3 WHERE id=1"); + Assert.AreEqual("a", n); + }); + + scope.Complete(); + } + } + + [Test] + public async Task NestedTransactionInnerFail() + { + using (IEfCoreScope scope = EfCoreScopeProvider.CreateScope()) + { + await scope.ExecuteWithContextAsync(async database => + { + await database.Database.ExecuteSqlAsync($"CREATE TABLE tmp1 (id INT, name NVARCHAR(64))"); + }); + + scope.Complete(); + } + + using (IEfCoreScope scope = EfCoreScopeProvider.CreateScope()) + { + string n; + await scope.ExecuteWithContextAsync(async database => + { + await database.Database.ExecuteSqlAsync($"INSERT INTO tmp1 (id, name) VALUES (1, 'a')"); + n = await database.Database.ExecuteScalarAsync("SELECT name FROM tmp1 WHERE id=1"); + Assert.AreEqual("a", n); + + using (IEfCoreScope nested = EfCoreScopeProvider.CreateScope()) + { + await nested.ExecuteWithContextAsync(async nestedDatabase => + { + await nestedDatabase.Database.ExecuteSqlAsync($"INSERT INTO tmp1 (id, name) VALUES (2, 'b')"); + string nn = await nestedDatabase.Database.ExecuteScalarAsync( + "SELECT name FROM tmp1 WHERE id=2"); + Assert.AreEqual("b", nn); + }); + } + + n = await database.Database.ExecuteScalarAsync("SELECT name FROM tmp1 WHERE id=2"); + Assert.AreEqual("b", n); + }); + + scope.Complete(); + } + + using (IEfCoreScope scope = EfCoreScopeProvider.CreateScope()) + { + await scope.ExecuteWithContextAsync(async database => + { + string n = await database.Database.ExecuteScalarAsync("SELECT name FROM tmp1 WHERE id=1"); + Assert.IsNull(n); + n = await database.Database.ExecuteScalarAsync("SELECT name FROM tmp1 WHERE id=2"); + Assert.IsNull(n); + }); + } + } + + [Test] + public async Task NestedTransactionOuterFail() + { + using (IEfCoreScope scope = EfCoreScopeProvider.CreateScope()) + { + await scope.ExecuteWithContextAsync(async database => + { + await database.Database.ExecuteSqlAsync($"CREATE TABLE tmp2 (id INT, name NVARCHAR(64))"); + }); + + scope.Complete(); + } + + using (IEfCoreScope scope = EfCoreScopeProvider.CreateScope()) + { + await scope.ExecuteWithContextAsync(async database => + { + await database.Database.ExecuteSqlAsync($"INSERT INTO tmp2 (id, name) VALUES (1, 'a')"); + string n = await database.Database.ExecuteScalarAsync("SELECT name FROM tmp2 WHERE id=1"); + Assert.AreEqual("a", n); + + using (IEfCoreScope nested = EfCoreScopeProvider.CreateScope()) + { + await scope.ExecuteWithContextAsync(async nestedDatabase => + { + await nestedDatabase.Database.ExecuteSqlAsync($"INSERT INTO tmp2 (id, name) VALUES (2, 'b')"); + string nn = await nestedDatabase.Database.ExecuteScalarAsync( + "SELECT name FROM tmp2 WHERE id=2"); + Assert.AreEqual("b", nn); + }); + + nested.Complete(); + } + + n = await database.Database.ExecuteScalarAsync("SELECT name FROM tmp2 WHERE id=2"); + Assert.AreEqual("b", n); + }); + } + + using (IEfCoreScope scope = EfCoreScopeProvider.CreateScope()) + { + await scope.ExecuteWithContextAsync(async database => + { + string n = await database.Database.ExecuteScalarAsync("SELECT name FROM tmp2 WHERE id=1"); + Assert.IsNull(n); + n = await database.Database.ExecuteScalarAsync("SELECT name FROM tmp2 WHERE id=2"); + Assert.IsNull(n); + }); + } + } + + [Test] + public async Task NestedTransactionComplete() + { + using (IEfCoreScope scope = EfCoreScopeProvider.CreateScope()) + { + await scope.ExecuteWithContextAsync(async database => + { + await database.Database.ExecuteSqlAsync($"CREATE TABLE tmp (id INT, name NVARCHAR(64))"); + }); + scope.Complete(); + } + + using (IEfCoreScope scope = EfCoreScopeProvider.CreateScope()) + { + await scope.ExecuteWithContextAsync(async database => + { + await database.Database.ExecuteSqlAsync($"INSERT INTO tmp (id, name) VALUES (1, 'a')"); + string n = await database.Database.ExecuteScalarAsync("SELECT name FROM tmp WHERE id=1"); + Assert.AreEqual("a", n); + + using (IEfCoreScope nested = EfCoreScopeProvider.CreateScope()) + { + await scope.ExecuteWithContextAsync(async nestedDatabase => + { + await nestedDatabase.Database.ExecuteSqlAsync($"INSERT INTO tmp (id, name) VALUES (2, 'b')"); + string nn = + await nestedDatabase.Database.ExecuteScalarAsync("SELECT name FROM tmp WHERE id=2"); + Assert.AreEqual("b", nn); + }); + + nested.Complete(); + } + + n = await database.Database.ExecuteScalarAsync("SELECT name FROM tmp WHERE id=2"); + Assert.AreEqual("b", n); + }); + + scope.Complete(); + } + + using (IEfCoreScope scope = EfCoreScopeProvider.CreateScope()) + { + await scope.ExecuteWithContextAsync(async database => + { + string n = await database.Database.ExecuteScalarAsync("SELECT name FROM tmp WHERE id=1"); + Assert.AreEqual("a", n); + n = await database.Database.ExecuteScalarAsync("SELECT name FROM tmp WHERE id=2"); + Assert.AreEqual("b", n); + }); + } + } + + [Test] + public void CallContextScope1() + { + var taskHelper = new TaskHelper(Mock.Of>()); + using (IEfCoreScope scope = EfCoreScopeProvider.CreateScope()) + { + Assert.IsNotNull(EfCoreScopeAccessor.AmbientScope); + + // Run on another thread without a flowed context + Task t = taskHelper.ExecuteBackgroundTask(() => + { + Assert.IsNull(EfCoreScopeAccessor.AmbientScope); + + using (IEfCoreScope newScope = EfCoreScopeProvider.CreateScope()) + { + Assert.IsNotNull(EfCoreScopeAccessor.AmbientScope); + Assert.IsNull(EfCoreScopeAccessor.AmbientScope.ParentScope); + } + + Assert.IsNull(EfCoreScopeAccessor.AmbientScope); + + return Task.CompletedTask; + }); + + Task.WaitAll(t); + + Assert.IsNotNull(EfCoreScopeAccessor.AmbientScope); + Assert.AreSame(scope, EfCoreScopeAccessor.AmbientScope); + } + + Assert.IsNull(EfCoreScopeAccessor.AmbientScope); + } + + [Test] + public void CallContextScope2() + { + var taskHelper = new TaskHelper(Mock.Of>()); + Assert.IsNull(EfCoreScopeAccessor.AmbientScope); + + using (IEfCoreScope scope = EfCoreScopeProvider.CreateScope()) + { + Assert.IsNotNull(EfCoreScopeAccessor.AmbientScope); + + // Run on another thread without a flowed context + Task t = taskHelper.ExecuteBackgroundTask(() => + { + Assert.IsNull(EfCoreScopeAccessor.AmbientScope); + + using (IEfCoreScope newScope = EfCoreScopeProvider.CreateScope()) + { + Assert.IsNotNull(EfCoreScopeAccessor.AmbientScope); + Assert.IsNull(EfCoreScopeAccessor.AmbientScope.ParentScope); + } + + Assert.IsNull(EfCoreScopeAccessor.AmbientScope); + return Task.CompletedTask; + }); + + Task.WaitAll(t); + + Assert.IsNotNull(EfCoreScopeAccessor.AmbientScope); + Assert.AreSame(scope, EfCoreScopeAccessor.AmbientScope); + } + + Assert.IsNull(EfCoreScopeAccessor.AmbientScope); + } + + [TestCase(true)] + [TestCase(false)] + public void ScopeContextEnlist(bool complete) + { + bool? completed = null; + IEfCoreScope ambientScope = null; + IScopeContext ambientContext = null; + + Assert.IsNull(EfCoreScopeAccessor.AmbientScope); + using (IEfCoreScope scope = EfCoreScopeProvider.CreateScope()) + { + scope.ScopeContext.Enlist("name", c => + { + completed = c; + ambientScope = EfCoreScopeAccessor.AmbientScope; + ambientContext = EfCoreScopeProvider.AmbientScopeContext; + }); + if (complete) + { + scope.Complete(); + } + } + + Assert.IsNull(EfCoreScopeAccessor.AmbientScope); + Assert.IsNull(EfCoreScopeProvider.AmbientScopeContext); + Assert.IsNotNull(completed); + Assert.AreEqual(complete, completed.Value); + Assert.IsNull(ambientScope); // the scope is gone + Assert.IsNotNull(ambientContext); // the context is still there + } + + [TestCase(true)] + [TestCase(false)] + public void ScopeContextEnlistAgain(bool complete) + { + bool? completed = null; + bool? completed2 = null; + + Assert.IsNull(EfCoreScopeAccessor.AmbientScope); + using (IEfCoreScope scope = EfCoreScopeProvider.CreateScope()) + { + scope.ScopeContext.Enlist("name", c => + { + completed = c; + + // at that point the scope is gone, but the context is still there + IScopeContext ambientContext = EfCoreScopeProvider.AmbientScopeContext; + ambientContext.Enlist("another", c2 => completed2 = c2); + }); + if (complete) + { + scope.Complete(); + } + } + + Assert.IsNull(EfCoreScopeAccessor.AmbientScope); + Assert.IsNull(EfCoreScopeProvider.AmbientScopeContext); + Assert.IsNotNull(completed); + Assert.AreEqual(complete, completed.Value); + Assert.AreEqual(complete, completed2.Value); + } + + [Test] + public void DetachableScope() + { + Assert.IsNull(EfCoreScopeAccessor.AmbientScope); + using (IEfCoreScope scope = EfCoreScopeProvider.CreateScope()) + { + Assert.IsInstanceOf>(scope); + Assert.IsNotNull(EfCoreScopeAccessor.AmbientScope); + Assert.AreSame(scope, EfCoreScopeAccessor.AmbientScope); + + Assert.IsNotNull(EfCoreScopeProvider.AmbientScopeContext); // the ambient context + Assert.IsNotNull(scope.ScopeContext); // the ambient context too (getter only) + IScopeContext context = scope.ScopeContext; + + IEfCoreScope detached = EfCoreScopeProvider.CreateDetachedScope(); + EfCoreScopeProvider.AttachScope(detached); + + Assert.AreEqual(detached, EfCoreScopeAccessor.AmbientScope); + Assert.AreNotSame(context, EfCoreScopeProvider.AmbientScopeContext); + + // nesting under detached! + using (IEfCoreScope nested = EfCoreScopeProvider.CreateScope()) + { + Assert.Throws(() => + + // cannot detach a non-detachable scope + EfCoreScopeProvider.DetachScope()); + nested.Complete(); + } + + Assert.AreEqual(detached, EfCoreScopeAccessor.AmbientScope); + Assert.AreNotSame(context, EfCoreScopeProvider.AmbientScopeContext); + + // can detach + Assert.AreSame(detached, EfCoreScopeProvider.DetachScope()); + + Assert.AreSame(scope, EfCoreScopeAccessor.AmbientScope); + Assert.AreSame(context, EfCoreScopeProvider.AmbientScopeContext); + + Assert.Throws(() => + + // cannot disposed a non-attached scope + // in fact, only the ambient scope can be disposed + detached.Dispose()); + + EfCoreScopeProvider.AttachScope(detached); + detached.Complete(); + detached.Dispose(); + + // has self-detached, and is gone! + Assert.AreSame(scope, EfCoreScopeAccessor.AmbientScope); + Assert.AreSame(context, EfCoreScopeProvider.AmbientScopeContext); + } + + Assert.IsNull(EfCoreScopeAccessor.AmbientScope); + Assert.IsNull(EfCoreScopeProvider.AmbientScopeContext); + } +} diff --git a/tests/Umbraco.Tests.Integration/Umbraco.Persistence.EFCore/Scoping/EFCoreScopedFileSystemsTests.cs b/tests/Umbraco.Tests.Integration/Umbraco.Persistence.EFCore/Scoping/EFCoreScopedFileSystemsTests.cs new file mode 100644 index 0000000000..df91eed751 --- /dev/null +++ b/tests/Umbraco.Tests.Integration/Umbraco.Persistence.EFCore/Scoping/EFCoreScopedFileSystemsTests.cs @@ -0,0 +1,211 @@ +using Microsoft.Extensions.Logging; +using Moq; +using NUnit.Framework; +using Umbraco.Cms.Core; +using Umbraco.Cms.Core.Hosting; +using Umbraco.Cms.Core.IO; +using Umbraco.Cms.Persistence.EFCore.Scoping; +using Umbraco.Cms.Tests.Common; +using Umbraco.Cms.Tests.Common.Testing; +using Umbraco.Cms.Tests.Integration.Testing; +using Umbraco.Cms.Tests.Integration.Umbraco.Persistence.EFCore.DbContext; + +namespace Umbraco.Cms.Tests.Integration.Umbraco.Persistence.EFCore.Scoping; + +[TestFixture] +[UmbracoTest(Database = UmbracoTestOptions.Database.NewSchemaPerTest, Logger = UmbracoTestOptions.Logger.Console)] +public class EFCoreScopedFileSystemsTests : UmbracoIntegrationTest +{ + [SetUp] + public void SetUp() => ClearFiles(IOHelper); + + [TearDown] + public void Teardown() => ClearFiles(IOHelper); + + private MediaFileManager MediaFileManager => GetRequiredService(); + + private IHostingEnvironment HostingEnvironment => GetRequiredService(); + + private IEFCoreScopeProvider EfCoreScopeProvider => GetRequiredService>(); + private IEFCoreScopeAccessor EfCoreScopeAccessor => GetRequiredService>(); + + private void ClearFiles(IIOHelper ioHelper) + { + TestHelper.DeleteDirectory(ioHelper.MapPath("media")); + TestHelper.DeleteDirectory(ioHelper.MapPath("FileSysTests")); + TestHelper.DeleteDirectory(ioHelper.MapPath(Constants.SystemDirectories.TempData.EnsureEndsWith('/') + "ShadowFs")); + } + + [Test] + public void MediaFileManager_Does_Not_Write_To_Physical_File_System_When_Scoped_If_Scope_Does_Not_Complete() + { + var rootPath = HostingEnvironment.MapPathWebRoot(GlobalSettings.UmbracoMediaPhysicalRootPath); + var rootUrl = HostingEnvironment.ToAbsolute(GlobalSettings.UmbracoMediaPath); + var physMediaFileSystem = new PhysicalFileSystem(IOHelper, HostingEnvironment, GetRequiredService>(), rootPath, rootUrl); + var mediaFileManager = MediaFileManager; + + Assert.IsFalse(physMediaFileSystem.FileExists("f1.txt")); + + using (EfCoreScopeProvider.CreateScope(scopeFileSystems: true)) + { + using (var ms = new MemoryStream("foo"u8.ToArray())) + { + MediaFileManager.FileSystem.AddFile("f1.txt", ms); + } + + Assert.IsTrue(mediaFileManager.FileSystem.FileExists("f1.txt")); + Assert.IsFalse(physMediaFileSystem.FileExists("f1.txt")); + } + + // After scope is disposed ensure shadow wrapper didn't commit to physical + Assert.IsFalse(mediaFileManager.FileSystem.FileExists("f1.txt")); + Assert.IsFalse(physMediaFileSystem.FileExists("f1.txt")); + } + + [Test] + public void MediaFileManager_Writes_To_Physical_File_System_When_Scoped_And_Scope_Is_Completed() + { + var rootPath = HostingEnvironment.MapPathWebRoot(GlobalSettings.UmbracoMediaPhysicalRootPath); + var rootUrl = HostingEnvironment.ToAbsolute(GlobalSettings.UmbracoMediaPath); + var physMediaFileSystem = new PhysicalFileSystem(IOHelper, HostingEnvironment, GetRequiredService>(), rootPath, rootUrl); + var mediaFileManager = MediaFileManager; + + Assert.IsFalse(physMediaFileSystem.FileExists("f1.txt")); + + using (var scope = EfCoreScopeProvider.CreateScope(scopeFileSystems: true)) + { + using (var ms = new MemoryStream("foo"u8.ToArray())) + { + mediaFileManager.FileSystem.AddFile("f1.txt", ms); + } + + Assert.IsTrue(mediaFileManager.FileSystem.FileExists("f1.txt")); + Assert.IsFalse(physMediaFileSystem.FileExists("f1.txt")); + + scope.Complete(); + + Assert.IsTrue(mediaFileManager.FileSystem.FileExists("f1.txt")); + Assert.IsFalse(physMediaFileSystem.FileExists("f1.txt")); + } + + // After scope is disposed ensure shadow wrapper writes to physical file system + Assert.IsTrue(mediaFileManager.FileSystem.FileExists("f1.txt")); + Assert.IsTrue(physMediaFileSystem.FileExists("f1.txt")); + } + + [Test] + public void MultiThread() + { + var rootPath = HostingEnvironment.MapPathWebRoot(GlobalSettings.UmbracoMediaPhysicalRootPath); + var rootUrl = HostingEnvironment.ToAbsolute(GlobalSettings.UmbracoMediaPath); + var physMediaFileSystem = new PhysicalFileSystem(IOHelper, HostingEnvironment, GetRequiredService>(), rootPath, rootUrl); + var mediaFileManager = MediaFileManager; + var taskHelper = new TaskHelper(Mock.Of>()); + + using (EfCoreScopeProvider.CreateScope(scopeFileSystems: true)) + { + using (var ms = new MemoryStream("foo"u8.ToArray())) + { + mediaFileManager.FileSystem.AddFile("f1.txt", ms); + } + + Assert.IsTrue(mediaFileManager.FileSystem.FileExists("f1.txt")); + Assert.IsFalse(physMediaFileSystem.FileExists("f1.txt")); + + // execute on another disconnected thread (execution context will not flow) + var t = taskHelper.ExecuteBackgroundTask(() => + { + Assert.IsFalse(mediaFileManager.FileSystem.FileExists("f1.txt")); + + using (var ms = new MemoryStream("foo"u8.ToArray())) + { + mediaFileManager.FileSystem.AddFile("f2.txt", ms); + } + + Assert.IsTrue(mediaFileManager.FileSystem.FileExists("f2.txt")); + Assert.IsTrue(physMediaFileSystem.FileExists("f2.txt")); + + return Task.CompletedTask; + }); + + t.Wait(); + + Assert.IsTrue(mediaFileManager.FileSystem.FileExists("f2.txt")); + Assert.IsTrue(physMediaFileSystem.FileExists("f2.txt")); + } + } + + [Test] + public void SingleShadow() + { + var taskHelper = new TaskHelper(Mock.Of>()); + var isThrown = false; + using (EfCoreScopeProvider.CreateScope(scopeFileSystems: true)) + { + // This is testing when another thread concurrently tries to create a scoped file system + // because at the moment we don't support concurrent scoped filesystems. + var t = taskHelper.ExecuteBackgroundTask(() => + { + // ok to create a 'normal' other scope + using (var other = EfCoreScopeProvider.CreateScope()) + { + other.Complete(); + } + + // not ok to create a 'scoped filesystems' other scope + // we will get a "Already shadowing." exception. + Assert.Throws(() => + { + using var other = EfCoreScopeProvider.CreateScope(scopeFileSystems: true); + }); + + isThrown = true; + + return Task.CompletedTask; + }); + + t.Wait(); + } + + Assert.IsTrue(isThrown); + } + + [Test] + public void SingleShadowEvenDetached() + { + var taskHelper = new TaskHelper(Mock.Of>()); + using (var scope = EfCoreScopeProvider.CreateScope(scopeFileSystems: true)) + { + // This is testing when another thread concurrently tries to create a scoped file system + // because at the moment we don't support concurrent scoped filesystems. + var t = taskHelper.ExecuteBackgroundTask(() => + { + // not ok to create a 'scoped filesystems' other scope + // because at the moment we don't support concurrent scoped filesystems + // even a detached one + // we will get a "Already shadowing." exception. + Assert.Throws(() => + { + using var other = EfCoreScopeProvider.CreateDetachedScope(scopeFileSystems: true); + }); + + return Task.CompletedTask; + }); + + t.Wait(); + } + + var detached = EfCoreScopeProvider.CreateDetachedScope(scopeFileSystems: true); + + Assert.IsNull(EfCoreScopeAccessor.AmbientScope); + + Assert.Throws(() => + { + // even if there is no ambient scope, there's a single shadow + using var other = EfCoreScopeProvider.CreateScope(scopeFileSystems: true); + }); + + EfCoreScopeProvider.AttachScope(detached); + detached.Dispose(); + } +} diff --git a/tests/Umbraco.Tests.Integration/Umbraco.Tests.Integration.csproj b/tests/Umbraco.Tests.Integration/Umbraco.Tests.Integration.csproj index 4a11fd91ee..4b29b73602 100644 --- a/tests/Umbraco.Tests.Integration/Umbraco.Tests.Integration.csproj +++ b/tests/Umbraco.Tests.Integration/Umbraco.Tests.Integration.csproj @@ -21,6 +21,7 @@ + diff --git a/umbraco.sln b/umbraco.sln index c382c480f7..f6bd2d1719 100644 --- a/umbraco.sln +++ b/umbraco.sln @@ -151,6 +151,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Umbraco.Cms.Api.Common", "s EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Umbraco.Cms.Imaging.ImageSharp", "src\Umbraco.Cms.Imaging.ImageSharp\Umbraco.Cms.Imaging.ImageSharp.csproj", "{35E3DA10-5549-41DE-B7ED-CC29355BA9FD}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Umbraco.Cms.Persistence.EFCore", "src\Umbraco.Cms.Persistence.EFCore\Umbraco.Cms.Persistence.EFCore.csproj", "{9046F56E-4AC3-4603-A6A3-3ACCF632997E}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -297,6 +299,12 @@ Global {D48B5D6B-82FF-4235-986C-CDE646F41DEC}.Release|Any CPU.Build.0 = Release|Any CPU {D48B5D6B-82FF-4235-986C-CDE646F41DEC}.SkipTests|Any CPU.ActiveCfg = Debug|Any CPU {D48B5D6B-82FF-4235-986C-CDE646F41DEC}.SkipTests|Any CPU.Build.0 = Debug|Any CPU + {9046F56E-4AC3-4603-A6A3-3ACCF632997E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9046F56E-4AC3-4603-A6A3-3ACCF632997E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9046F56E-4AC3-4603-A6A3-3ACCF632997E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9046F56E-4AC3-4603-A6A3-3ACCF632997E}.Release|Any CPU.Build.0 = Release|Any CPU + {9046F56E-4AC3-4603-A6A3-3ACCF632997E}.SkipTests|Any CPU.ActiveCfg = Debug|Any CPU + {9046F56E-4AC3-4603-A6A3-3ACCF632997E}.SkipTests|Any CPU.Build.0 = Debug|Any CPU {35E3DA10-5549-41DE-B7ED-CC29355BA9FD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {35E3DA10-5549-41DE-B7ED-CC29355BA9FD}.Debug|Any CPU.Build.0 = Debug|Any CPU {35E3DA10-5549-41DE-B7ED-CC29355BA9FD}.Release|Any CPU.ActiveCfg = Release|Any CPU From 1a8dc32b02741defd3ddd06a6f5b2f2a47f1de45 Mon Sep 17 00:00:00 2001 From: Elitsa Marinovska <21998037+elit0451@users.noreply.github.com> Date: Fri, 12 May 2023 09:26:26 +0200 Subject: [PATCH 20/26] Adding Unauthorized to ProducesResponseType for single-item endpoints (#14239) --- .../Controllers/ByIdContentApiController.cs | 1 + .../Controllers/ByRouteContentApiController.cs | 1 + 2 files changed, 2 insertions(+) diff --git a/src/Umbraco.Cms.Api.Delivery/Controllers/ByIdContentApiController.cs b/src/Umbraco.Cms.Api.Delivery/Controllers/ByIdContentApiController.cs index 01a5149a14..ce870fc11b 100644 --- a/src/Umbraco.Cms.Api.Delivery/Controllers/ByIdContentApiController.cs +++ b/src/Umbraco.Cms.Api.Delivery/Controllers/ByIdContentApiController.cs @@ -27,6 +27,7 @@ public class ByIdContentApiController : ContentApiItemControllerBase [HttpGet("item/{id:guid}")] [MapToApiVersion("1.0")] [ProducesResponseType(typeof(IApiContentResponse), StatusCodes.Status200OK)] + [ProducesResponseType(StatusCodes.Status401Unauthorized)] [ProducesResponseType(StatusCodes.Status404NotFound)] public async Task ById(Guid id) { diff --git a/src/Umbraco.Cms.Api.Delivery/Controllers/ByRouteContentApiController.cs b/src/Umbraco.Cms.Api.Delivery/Controllers/ByRouteContentApiController.cs index cc260f3930..a506b0ce53 100644 --- a/src/Umbraco.Cms.Api.Delivery/Controllers/ByRouteContentApiController.cs +++ b/src/Umbraco.Cms.Api.Delivery/Controllers/ByRouteContentApiController.cs @@ -38,6 +38,7 @@ public class ByRouteContentApiController : ContentApiItemControllerBase [HttpGet("item/{*path}")] [MapToApiVersion("1.0")] [ProducesResponseType(typeof(IApiContentResponse), StatusCodes.Status200OK)] + [ProducesResponseType(StatusCodes.Status401Unauthorized)] [ProducesResponseType(StatusCodes.Status404NotFound)] public async Task ByRoute(string path = "/") { From 5df655d5990446b69656ee93283ffd6b1e735980 Mon Sep 17 00:00:00 2001 From: Bjarke Berg Date: Mon, 15 May 2023 07:58:43 +0200 Subject: [PATCH 21/26] Unbreak breaking change --- .../CompatibilitySuppressions.xml | 9 +-------- .../BlockGridPropertyValueConverter.cs | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/Umbraco.Infrastructure/CompatibilitySuppressions.xml b/src/Umbraco.Infrastructure/CompatibilitySuppressions.xml index 653a936b61..795c678729 100644 --- a/src/Umbraco.Infrastructure/CompatibilitySuppressions.xml +++ b/src/Umbraco.Infrastructure/CompatibilitySuppressions.xml @@ -29,13 +29,6 @@ lib/net7.0/Umbraco.Infrastructure.dll true - - CP0002 - M:Umbraco.Cms.Core.PropertyEditors.ValueConverters.BlockGridPropertyValueConverter.#ctor(Umbraco.Cms.Core.Logging.IProfilingLogger,Umbraco.Cms.Core.PropertyEditors.ValueConverters.BlockEditorConverter,Umbraco.Cms.Core.Serialization.IJsonSerializer) - lib/net7.0/Umbraco.Infrastructure.dll - lib/net7.0/Umbraco.Infrastructure.dll - true - CP0002 M:Umbraco.Cms.Infrastructure.Migrations.IMigrationContext.AddPostMigration``1 @@ -78,4 +71,4 @@ lib/net7.0/Umbraco.Infrastructure.dll true - \ No newline at end of file + diff --git a/src/Umbraco.Infrastructure/PropertyEditors/ValueConverters/BlockGridPropertyValueConverter.cs b/src/Umbraco.Infrastructure/PropertyEditors/ValueConverters/BlockGridPropertyValueConverter.cs index a68eebf0bc..e1c5a2052c 100644 --- a/src/Umbraco.Infrastructure/PropertyEditors/ValueConverters/BlockGridPropertyValueConverter.cs +++ b/src/Umbraco.Infrastructure/PropertyEditors/ValueConverters/BlockGridPropertyValueConverter.cs @@ -1,7 +1,9 @@ // Copyright (c) Umbraco. // See LICENSE for more details. +using Microsoft.Extensions.DependencyInjection; using Umbraco.Cms.Core.DeliveryApi; +using Umbraco.Cms.Core.DependencyInjection; using Umbraco.Cms.Core.Logging; using Umbraco.Cms.Core.Models.Blocks; using Umbraco.Cms.Core.Models.DeliveryApi; @@ -20,9 +22,20 @@ namespace Umbraco.Cms.Core.PropertyEditors.ValueConverters private readonly IJsonSerializer _jsonSerializer; private readonly IApiElementBuilder _apiElementBuilder; + [Obsolete("Please use non-obsolete cconstrutor. This will be removed in Umbraco 14.")] + public BlockGridPropertyValueConverter( + IProfilingLogger proflog, + BlockEditorConverter blockConverter, + IJsonSerializer jsonSerializer) + : this(proflog, blockConverter, jsonSerializer, StaticServiceProvider.Instance.GetRequiredService()) + { + + } + // Niels, Change: I would love if this could be general, so we don't need a specific one for each block property editor.... public BlockGridPropertyValueConverter( - IProfilingLogger proflog, BlockEditorConverter blockConverter, + IProfilingLogger proflog, + BlockEditorConverter blockConverter, IJsonSerializer jsonSerializer, IApiElementBuilder apiElementBuilder) : base(blockConverter) From 401fa7334bf3e524e80b4e4e7a2167a611d51ac7 Mon Sep 17 00:00:00 2001 From: Andreas Zerbst <73799582+andr317c@users.noreply.github.com> Date: Mon, 15 May 2023 08:04:16 +0200 Subject: [PATCH 22/26] Fixed: an block grid editor accceptance test on the pipeline for v12 (#14240) * The attribute we asserted on has been changed which failed our test. The test is updated to match with the correct url * Updated so we use encodeURI instead of replacing the values in the path --- .../Content/blockGridEditorAdvanced.spec.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/BlockGridEditor/Content/blockGridEditorAdvanced.spec.ts b/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/BlockGridEditor/Content/blockGridEditorAdvanced.spec.ts index 87b19b8776..8db631019b 100644 --- a/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/BlockGridEditor/Content/blockGridEditorAdvanced.spec.ts +++ b/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/BlockGridEditor/Content/blockGridEditorAdvanced.spec.ts @@ -44,7 +44,7 @@ test.describe('BlockGridEditorAdvancedContent', () => { await umbracoApi.media.ensureNameNotExists(customViewItemName); const imageData = await umbracoApi.media.createImageWithFile(imageName, umbracoFileValue, imageFileName, imagePath, imageMimeType); - + const customViewData = await umbracoApi.media.createFileWithFile(customViewItemName, customViewFileName, customViewPath, customViewMimeType); const customViewMediaPath = customViewData.mediaLink; @@ -101,7 +101,7 @@ test.describe('BlockGridEditorAdvancedContent', () => { await umbracoUi.navigateToContent(blockGridName); // Assert - // Checks if the block has the correct CustomView + // Checks if the block has the correct CustomView await expect(page.locator('[data-content-element-type-key="' + element['key'] + '"]').locator('[view="' + customViewMediaPath + '"]')).toBeVisible(); // Checks if the custom view updated the block by locating a name in the customView await expect(page.locator('[data-content-element-type-key="' + element['key'] + '"]').locator('[view="' + customViewMediaPath + '"]').locator('[name="BlockGridCustomView"]')).toBeVisible(); @@ -137,7 +137,7 @@ test.describe('BlockGridEditorAdvancedContent', () => { await umbracoApi.content.createDefaultContentWithABlockGridEditor(umbracoApi, element, dataType, null); await umbracoUi.navigateToContent(blockGridName); - + // Assert // Checks if the block has the correct template await expect(page.locator('umb-block-grid-entry', {hasText: elementName}).locator('umb-block-grid-block')).toHaveAttribute('stylesheet', stylesheetDataPath); @@ -345,11 +345,12 @@ test.describe('BlockGridEditorAdvancedContent', () => { // Assert // Checks if the element has the thumbnail - await expect(page.locator('umb-block-card', {hasText: elementName}).locator('.__showcase')).toHaveAttribute('style', 'background-image: url("' + imageDataPath + '?width=400");'); + const updatedImagePath = encodeURIComponent(imageDataPath); + await expect(page.locator('umb-block-card', {hasText: elementName}).locator('.__showcase')).toHaveAttribute('style', 'background-image: url("/umbraco/backoffice/umbracoapi/images/GetBigThumbnail?originalImagePath=' + updatedImagePath + '\");'); // Clean await umbracoApi.documentTypes.ensureNameNotExists(elementTwoName); await umbracoApi.media.ensureNameNotExists(imageName); }); }); -}); \ No newline at end of file +}); From 368e9f2f59bdaea5b6e6279e62568de752cb19b4 Mon Sep 17 00:00:00 2001 From: Kenn Jacobsen Date: Mon, 15 May 2023 08:49:01 +0200 Subject: [PATCH 23/26] Fix delivery API cache level for media picker property editors (#14238) --- .../ValueConverters/MediaPickerValueConverter.cs | 2 +- .../ValueConverters/MediaPickerWithCropsValueConverter.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Umbraco.Core/PropertyEditors/ValueConverters/MediaPickerValueConverter.cs b/src/Umbraco.Core/PropertyEditors/ValueConverters/MediaPickerValueConverter.cs index 1d6792f185..30ff7fb779 100644 --- a/src/Umbraco.Core/PropertyEditors/ValueConverters/MediaPickerValueConverter.cs +++ b/src/Umbraco.Core/PropertyEditors/ValueConverters/MediaPickerValueConverter.cs @@ -122,7 +122,7 @@ public class MediaPickerValueConverter : PropertyValueConverterBase, IDeliveryAp return source; } - public PropertyCacheLevel GetDeliveryApiPropertyCacheLevel(IPublishedPropertyType propertyType) => PropertyCacheLevel.Element; + public PropertyCacheLevel GetDeliveryApiPropertyCacheLevel(IPublishedPropertyType propertyType) => PropertyCacheLevel.Elements; public Type GetDeliveryApiPropertyValueType(IPublishedPropertyType propertyType) => typeof(IEnumerable); diff --git a/src/Umbraco.Infrastructure/PropertyEditors/ValueConverters/MediaPickerWithCropsValueConverter.cs b/src/Umbraco.Infrastructure/PropertyEditors/ValueConverters/MediaPickerWithCropsValueConverter.cs index ef183089e9..a33bb9870d 100644 --- a/src/Umbraco.Infrastructure/PropertyEditors/ValueConverters/MediaPickerWithCropsValueConverter.cs +++ b/src/Umbraco.Infrastructure/PropertyEditors/ValueConverters/MediaPickerWithCropsValueConverter.cs @@ -120,7 +120,7 @@ public class MediaPickerWithCropsValueConverter : PropertyValueConverterBase, ID return isMultiple ? mediaItems : mediaItems.FirstOrDefault(); } - public PropertyCacheLevel GetDeliveryApiPropertyCacheLevel(IPublishedPropertyType propertyType) => PropertyCacheLevel.Element; + public PropertyCacheLevel GetDeliveryApiPropertyCacheLevel(IPublishedPropertyType propertyType) => PropertyCacheLevel.Elements; public Type GetDeliveryApiPropertyValueType(IPublishedPropertyType propertyType) => typeof(IEnumerable); From 85d46c3e82bf99d56f0a4659b00caf8d1c5a769c Mon Sep 17 00:00:00 2001 From: Kenn Jacobsen Date: Mon, 15 May 2023 10:06:24 +0200 Subject: [PATCH 24/26] Merge local and global crops for MediaPicker3 (#14237) --- .../MediaPickerWithCropsValueConverter.cs | 11 +- ...MediaPickerWithCropsValueConverterTests.cs | 271 +++++++++++++----- 2 files changed, 205 insertions(+), 77 deletions(-) diff --git a/src/Umbraco.Infrastructure/PropertyEditors/ValueConverters/MediaPickerWithCropsValueConverter.cs b/src/Umbraco.Infrastructure/PropertyEditors/ValueConverters/MediaPickerWithCropsValueConverter.cs index a33bb9870d..19f9b5a908 100644 --- a/src/Umbraco.Infrastructure/PropertyEditors/ValueConverters/MediaPickerWithCropsValueConverter.cs +++ b/src/Umbraco.Infrastructure/PropertyEditors/ValueConverters/MediaPickerWithCropsValueConverter.cs @@ -131,7 +131,16 @@ public class MediaPickerWithCropsValueConverter : PropertyValueConverterBase, ID ApiMediaWithCrops ToApiMedia(MediaWithCrops media) { IApiMedia inner = _apiMediaBuilder.Build(media.Content); - return new ApiMediaWithCrops(inner, media.LocalCrops.FocalPoint, media.LocalCrops.Crops); + + // make sure we merge crops and focal point defined at media level with the locally defined ones (local ones take precedence in case of a conflict) + ImageCropperValue? mediaCrops = media.Content.Value(_publishedValueFallback, Constants.Conventions.Media.File); + ImageCropperValue localCrops = media.LocalCrops; + if (mediaCrops != null) + { + localCrops = localCrops.Merge(mediaCrops); + } + + return new ApiMediaWithCrops(inner, localCrops.FocalPoint, localCrops.Crops); } // NOTE: eventually we might implement this explicitly instead of piggybacking on the default object conversion. however, this only happens once per cache rebuild, diff --git a/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/MediaPickerWithCropsValueConverterTests.cs b/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/MediaPickerWithCropsValueConverterTests.cs index 23dc5bb8b3..c02526054e 100644 --- a/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/MediaPickerWithCropsValueConverterTests.cs +++ b/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/MediaPickerWithCropsValueConverterTests.cs @@ -2,7 +2,6 @@ using Moq; using NUnit.Framework; using Umbraco.Cms.Core; using Umbraco.Cms.Core.DeliveryApi; -using Umbraco.Cms.Core.DeliveryApi.Accessors; using Umbraco.Cms.Core.Models.DeliveryApi; using Umbraco.Cms.Core.Models.PublishedContent; using Umbraco.Cms.Core.PropertyEditors; @@ -63,32 +62,15 @@ public class MediaPickerWithCropsValueConverterTests : PropertyValueConverterTes var result = valueConverter.ConvertIntermediateToDeliveryApiObject(Mock.Of(), publishedPropertyType, PropertyCacheLevel.Element, inter, false) as IEnumerable; Assert.NotNull(result); Assert.AreEqual(1, result.Count()); - Assert.AreEqual("My media", result.First().Name); - Assert.AreEqual("my-media", result.First().Url); - Assert.AreEqual(".jpg", result.First().Extension); - Assert.AreEqual(200, result.First().Width); - Assert.AreEqual(400, result.First().Height); - Assert.AreEqual(800, result.First().Bytes); - Assert.NotNull(result.First().FocalPoint); - Assert.AreEqual(".jpg", result.First().Extension); - Assert.AreEqual(200, result.First().Width); - Assert.AreEqual(400, result.First().Height); - Assert.AreEqual(800, result.First().Bytes); - Assert.AreEqual(.2m, result.First().FocalPoint.Left); - Assert.AreEqual(.4m, result.First().FocalPoint.Top); - Assert.NotNull(result.First().Crops); - Assert.AreEqual(1, result.First().Crops.Count()); - Assert.AreEqual("one", result.First().Crops.First().Alias); - Assert.AreEqual(100, result.First().Crops.First().Height); - Assert.AreEqual(200, result.First().Crops.First().Width); - Assert.NotNull(result.First().Crops.First().Coordinates); - Assert.AreEqual(1m, result.First().Crops.First().Coordinates.X1); - Assert.AreEqual(2m, result.First().Crops.First().Coordinates.X2); - Assert.AreEqual(10m, result.First().Crops.First().Coordinates.Y1); - Assert.AreEqual(20m, result.First().Crops.First().Coordinates.Y2); - Assert.NotNull(result.First().Properties); - Assert.AreEqual(1, result.First().Properties.Count); - Assert.AreEqual("My alt text", result.First().Properties["altText"]); + var first = result.Single(); + ValidateMedia(first, "My media", "my-media", ".jpg", 200, 400, 800); + ValidateFocalPoint(first.FocalPoint, .2m, .4m); + Assert.NotNull(first.Crops); + Assert.AreEqual(1, first.Crops.Count()); + ValidateCrop(first.Crops.First(), "one", 200, 100, 1m, 2m, 10m, 20m); + Assert.NotNull(first.Properties); + Assert.AreEqual(1, first.Properties.Count); + Assert.AreEqual("My alt text", first.Properties["altText"]); } [Test] @@ -138,52 +120,147 @@ public class MediaPickerWithCropsValueConverterTests : PropertyValueConverterTes var result = valueConverter.ConvertIntermediateToDeliveryApiObject(Mock.Of(), publishedPropertyType, PropertyCacheLevel.Element, inter, false) as IEnumerable; Assert.NotNull(result); Assert.AreEqual(2, result.Count()); + var first = result.First(); + var last = result.Last(); - Assert.AreEqual("My media", result.First().Name); - Assert.AreEqual("my-media", result.First().Url); - Assert.AreEqual(".jpg", result.First().Extension); - Assert.AreEqual(200, result.First().Width); - Assert.AreEqual(400, result.First().Height); - Assert.AreEqual(800, result.First().Bytes); - Assert.NotNull(result.First().FocalPoint); - Assert.AreEqual(.2m, result.First().FocalPoint.Left); - Assert.AreEqual(.4m, result.First().FocalPoint.Top); - Assert.NotNull(result.First().Crops); - Assert.AreEqual(1, result.First().Crops.Count()); - Assert.AreEqual("one", result.First().Crops.First().Alias); - Assert.AreEqual(100, result.First().Crops.First().Height); - Assert.AreEqual(200, result.First().Crops.First().Width); - Assert.NotNull(result.First().Crops.First().Coordinates); - Assert.AreEqual(1m, result.First().Crops.First().Coordinates.X1); - Assert.AreEqual(2m, result.First().Crops.First().Coordinates.X2); - Assert.AreEqual(10m, result.First().Crops.First().Coordinates.Y1); - Assert.AreEqual(20m, result.First().Crops.First().Coordinates.Y2); - Assert.NotNull(result.First().Properties); - Assert.AreEqual(1, result.First().Properties.Count); - Assert.AreEqual("My alt text", result.First().Properties["altText"]); + ValidateMedia(first, "My media", "my-media", ".jpg", 200, 400, 800); + ValidateFocalPoint(first.FocalPoint, .2m, .4m); + Assert.NotNull(first.Crops); + Assert.AreEqual(1, first.Crops.Count()); + ValidateCrop(first.Crops.First(), "one", 200, 100, 1m, 2m, 10m, 20m); + Assert.NotNull(first.Properties); + Assert.AreEqual(1, first.Properties.Count); + Assert.AreEqual("My alt text", first.Properties["altText"]); - Assert.AreEqual("My other media", result.Last().Name); - Assert.AreEqual("my-other-media", result.Last().Url); - Assert.AreEqual(".png", result.Last().Extension); - Assert.AreEqual(800, result.Last().Width); - Assert.AreEqual(600, result.Last().Height); - Assert.AreEqual(200, result.Last().Bytes); - Assert.NotNull(result.Last().FocalPoint); - Assert.AreEqual(.8m, result.Last().FocalPoint.Left); - Assert.AreEqual(.6m, result.Last().FocalPoint.Top); - Assert.NotNull(result.Last().Crops); - Assert.AreEqual(1, result.Last().Crops.Count()); - Assert.AreEqual("one", result.Last().Crops.Last().Alias); - Assert.AreEqual(100, result.Last().Crops.Last().Height); - Assert.AreEqual(200, result.Last().Crops.Last().Width); - Assert.NotNull(result.Last().Crops.Last().Coordinates); - Assert.AreEqual(40m, result.Last().Crops.Last().Coordinates.X1); - Assert.AreEqual(20m, result.Last().Crops.Last().Coordinates.X2); - Assert.AreEqual(2m, result.Last().Crops.Last().Coordinates.Y1); - Assert.AreEqual(1m, result.Last().Crops.Last().Coordinates.Y2); - Assert.NotNull(result.Last().Properties); - Assert.AreEqual(1, result.Last().Properties.Count); - Assert.AreEqual("My other alt text", result.Last().Properties["altText"]); + ValidateMedia(last, "My other media", "my-other-media", ".png", 800, 600, 200); + ValidateFocalPoint(last.FocalPoint, .8m, .6m); + Assert.NotNull(last.Crops); + Assert.AreEqual(1, last.Crops.Count()); + ValidateCrop(last.Crops.First(), "one", 200, 100, 40m, 20m, 2m, 1m); + Assert.NotNull(last.Properties); + Assert.AreEqual(1, last.Properties.Count); + Assert.AreEqual("My other alt text", last.Properties["altText"]); + } + + [Test] + public void MediaPickerWithCropsValueConverter_MergesMediaCropsWithLocalCrops() + { + var publishedPropertyType = SetupMediaPropertyType(false); + var mediaCrops = new ImageCropperValue + { + Crops = new[] + { + new ImageCropperValue.ImageCropperCrop + { + Alias = "mediaOne", + Width = 111, + Height = 222, + Coordinates = new ImageCropperValue.ImageCropperCropCoordinates { X1 = 2m, X2 = 4m, Y1 = 20m, Y2 = 40m } + } + }, + FocalPoint = new ImageCropperValue.ImageCropperFocalPoint { Left = .9m, Top = .1m } + }; + var mediaKey = SetupMedia("Some media", ".123", 123, 456, "My alt text", 789, mediaCrops); + + var serializer = new JsonNetSerializer(); + + var valueConverter = MediaPickerWithCropsValueConverter(); + Assert.AreEqual(typeof(IEnumerable), valueConverter.GetDeliveryApiPropertyValueType(publishedPropertyType)); + + var inter = serializer.Serialize(new[] + { + new MediaPicker3PropertyEditor.MediaPicker3PropertyValueEditor.MediaWithCropsDto + { + Key = Guid.NewGuid(), + MediaKey = mediaKey, + Crops = new [] + { + new ImageCropperValue.ImageCropperCrop + { + Alias = "one", + Coordinates = new ImageCropperValue.ImageCropperCropCoordinates { X1 = 1m, X2 = 2m, Y1 = 10m, Y2 = 20m } + } + } + } + }); + + var result = valueConverter.ConvertIntermediateToDeliveryApiObject(Mock.Of(), publishedPropertyType, PropertyCacheLevel.Element, inter, false) as IEnumerable; + Assert.NotNull(result); + Assert.AreEqual(1, result.Count()); + var mediaWithCrops = result.Single(); + ValidateMedia(mediaWithCrops, "Some media", "some-media", ".123", 123, 456, 789); + + // no local focal point, should revert to media focal point + ValidateFocalPoint(mediaWithCrops.FocalPoint, .9m, .1m); + + // media crops should be merged with local crops + Assert.NotNull(mediaWithCrops.Crops); + Assert.AreEqual(2, mediaWithCrops.Crops.Count()); + + // local crops should be first, media crops should be last + ValidateCrop(mediaWithCrops.Crops.First(), "one", 200, 100, 1m, 2m, 10m, 20m); + ValidateCrop(mediaWithCrops.Crops.Last(), "mediaOne", 111, 222, 2m, 4m, 20m, 40m); + } + + + [Test] + public void MediaPickerWithCropsValueConverter_LocalCropsAndFocalPointTakesPrecedenceOverMediaCropsAndFocalPoint() + { + var publishedPropertyType = SetupMediaPropertyType(false); + var mediaCrops = new ImageCropperValue + { + Crops = new[] + { + new ImageCropperValue.ImageCropperCrop + { + Alias = "one", + Width = 111, + Height = 222, + Coordinates = new ImageCropperValue.ImageCropperCropCoordinates { X1 = 2m, X2 = 4m, Y1 = 20m, Y2 = 40m } + } + }, + FocalPoint = new ImageCropperValue.ImageCropperFocalPoint { Left = .9m, Top = .1m } + }; + var mediaKey = SetupMedia("Some media", ".123", 123, 456, "My alt text", 789, mediaCrops); + + var serializer = new JsonNetSerializer(); + + var valueConverter = MediaPickerWithCropsValueConverter(); + Assert.AreEqual(typeof(IEnumerable), valueConverter.GetDeliveryApiPropertyValueType(publishedPropertyType)); + + var inter = serializer.Serialize(new[] + { + new MediaPicker3PropertyEditor.MediaPicker3PropertyValueEditor.MediaWithCropsDto + { + Key = Guid.NewGuid(), + MediaKey = mediaKey, + Crops = new [] + { + new ImageCropperValue.ImageCropperCrop + { + Alias = "one", + Coordinates = new ImageCropperValue.ImageCropperCropCoordinates { X1 = 1m, X2 = 2m, Y1 = 10m, Y2 = 20m } + } + }, + FocalPoint = new ImageCropperValue.ImageCropperFocalPoint { Left = .2m, Top = .3m } + } + }); + + var result = valueConverter.ConvertIntermediateToDeliveryApiObject(Mock.Of(), publishedPropertyType, PropertyCacheLevel.Element, inter, false) as IEnumerable; + Assert.NotNull(result); + Assert.AreEqual(1, result.Count()); + var mediaWithCrops = result.Single(); + ValidateMedia(mediaWithCrops, "Some media", "some-media", ".123", 123, 456, 789); + + // local focal point should take precedence over media focal point + ValidateFocalPoint(mediaWithCrops.FocalPoint, .2m, .3m); + + // media crops should be discarded when merging with local crops (matching aliases, local ones take precedence) + Assert.NotNull(mediaWithCrops.Crops); + Assert.AreEqual(1, mediaWithCrops.Crops.Count()); + + // local crops should be first, media crops should be last + ValidateCrop(mediaWithCrops.Crops.First(), "one", 200, 100, 1m, 2m, 10m, 20m); } [TestCase("")] @@ -194,8 +271,6 @@ public class MediaPickerWithCropsValueConverterTests : PropertyValueConverterTes { var publishedPropertyType = SetupMediaPropertyType(false); - var serializer = new JsonNetSerializer(); - var valueConverter = MediaPickerWithCropsValueConverter(); var result = valueConverter.ConvertIntermediateToDeliveryApiObject(Mock.Of(), publishedPropertyType, PropertyCacheLevel.Element, inter, false) as IEnumerable; @@ -211,8 +286,6 @@ public class MediaPickerWithCropsValueConverterTests : PropertyValueConverterTes { var publishedPropertyType = SetupMediaPropertyType(true); - var serializer = new JsonNetSerializer(); - var valueConverter = MediaPickerWithCropsValueConverter(); var result = valueConverter.ConvertIntermediateToDeliveryApiObject(Mock.Of(), publishedPropertyType, PropertyCacheLevel.Element, inter, false) as IEnumerable; @@ -241,7 +314,7 @@ public class MediaPickerWithCropsValueConverterTests : PropertyValueConverterTes return publishedPropertyType.Object; } - private Guid SetupMedia(string name, string extension, int width, int height, string altText, int bytes) + private Guid SetupMedia(string name, string extension, int width, int height, string altText, int bytes, ImageCropperValue? imageCropperValue = null) { var publishedMediaType = new Mock(); publishedMediaType.SetupGet(c => c.ItemType).Returns(PublishedItemType.Media); @@ -266,6 +339,7 @@ public class MediaPickerWithCropsValueConverterTests : PropertyValueConverterTes AddProperty(Constants.Conventions.Media.Width, width); AddProperty(Constants.Conventions.Media.Height, height); AddProperty(Constants.Conventions.Media.Bytes, bytes); + AddProperty(Constants.Conventions.Media.File, imageCropperValue); AddProperty("altText", altText); PublishedMediaCacheMock @@ -281,4 +355,49 @@ public class MediaPickerWithCropsValueConverterTests : PropertyValueConverterTes return mediaKey; } + + private void ValidateMedia( + ApiMediaWithCrops actual, + string expectedName, + string expectedUrl, + string expectedExtension, + int expectedWidth, + int expectedHeight, + int expectedBytes) + { + Assert.AreEqual(expectedName, actual.Name); + Assert.AreEqual(expectedUrl, actual.Url); + Assert.AreEqual(expectedExtension, actual.Extension); + Assert.AreEqual(expectedWidth, actual.Width); + Assert.AreEqual(expectedHeight, actual.Height); + Assert.AreEqual(expectedBytes, actual.Bytes); + + } + + private void ValidateFocalPoint(ImageCropperValue.ImageCropperFocalPoint? actual, decimal expectedLeft, decimal expectedTop) + { + Assert.NotNull(actual); + Assert.AreEqual(expectedLeft, actual.Left); + Assert.AreEqual(expectedTop, actual.Top); + } + + private void ValidateCrop( + ImageCropperValue.ImageCropperCrop actual, + string expectedAlias, + int expectedWidth, + int expectedHeight, + decimal expectedX1, + decimal expectedX2, + decimal expectedY1, + decimal expectedY2) + { + Assert.AreEqual(expectedAlias, actual.Alias); + Assert.AreEqual(expectedWidth, actual.Width); + Assert.AreEqual(expectedHeight, actual.Height); + Assert.NotNull(actual.Coordinates); + Assert.AreEqual(expectedX1, actual.Coordinates.X1); + Assert.AreEqual(expectedX2, actual.Coordinates.X2); + Assert.AreEqual(expectedY1, actual.Coordinates.Y1); + Assert.AreEqual(expectedY2, actual.Coordinates.Y2); + } } From b32b8c2265185d443d155bc862f316a10d38902e Mon Sep 17 00:00:00 2001 From: Kenn Jacobsen Date: Mon, 15 May 2023 10:27:20 +0200 Subject: [PATCH 25/26] Un-routable content should never be accessible in the delivery API (#14242) --- .../Controllers/ByIdContentApiController.cs | 8 +- .../Controllers/QueryContentApiController.cs | 3 +- .../DeliveryApi/ApiContentBuilderBase.cs | 10 ++- .../DeliveryApi/ApiContentResponseBuilder.cs | 27 ++++-- .../DeliveryApi/ApiContentRouteBuilder.cs | 12 ++- .../DeliveryApi/IApiContentBuilder.cs | 2 +- .../DeliveryApi/IApiContentResponseBuilder.cs | 2 +- .../DeliveryApi/IApiContentRouteBuilder.cs | 2 +- .../DeliveryApi/ApiRichTextParser.cs | 7 +- .../MultiUrlPickerValueConverter.cs | 7 +- .../DeliveryApi/ContentBuilderTests.cs | 29 ++++++- .../DeliveryApi/ContentRouteBuilderTests.cs | 84 ++++++++++++------- .../MultiNodeTreePickerValueConverterTests.cs | 21 ++++- 13 files changed, 163 insertions(+), 51 deletions(-) diff --git a/src/Umbraco.Cms.Api.Delivery/Controllers/ByIdContentApiController.cs b/src/Umbraco.Cms.Api.Delivery/Controllers/ByIdContentApiController.cs index ce870fc11b..fc2e91e1ba 100644 --- a/src/Umbraco.Cms.Api.Delivery/Controllers/ByIdContentApiController.cs +++ b/src/Umbraco.Cms.Api.Delivery/Controllers/ByIdContentApiController.cs @@ -43,6 +43,12 @@ public class ByIdContentApiController : ContentApiItemControllerBase return Unauthorized(); } - return await Task.FromResult(Ok(ApiContentResponseBuilder.Build(contentItem))); + IApiContentResponse? apiContentResponse = ApiContentResponseBuilder.Build(contentItem); + if (apiContentResponse is null) + { + return NotFound(); + } + + return await Task.FromResult(Ok(apiContentResponse)); } } diff --git a/src/Umbraco.Cms.Api.Delivery/Controllers/QueryContentApiController.cs b/src/Umbraco.Cms.Api.Delivery/Controllers/QueryContentApiController.cs index 8db6cdb454..2b2efb8cda 100644 --- a/src/Umbraco.Cms.Api.Delivery/Controllers/QueryContentApiController.cs +++ b/src/Umbraco.Cms.Api.Delivery/Controllers/QueryContentApiController.cs @@ -7,6 +7,7 @@ using Umbraco.Cms.Core.DeliveryApi; using Umbraco.Cms.Core.Models.DeliveryApi; using Umbraco.Cms.Core.Models.PublishedContent; using Umbraco.Cms.Core.Services.OperationStatus; +using Umbraco.Extensions; using Umbraco.New.Cms.Core.Models; namespace Umbraco.Cms.Api.Delivery.Controllers; @@ -53,7 +54,7 @@ public class QueryContentApiController : ContentApiControllerBase PagedModel pagedResult = queryAttempt.Result; IEnumerable contentItems = ApiPublishedContentCache.GetByIds(pagedResult.Items); - IApiContentResponse[] apiContentItems = contentItems.Select(ApiContentResponseBuilder.Build).ToArray(); + IApiContentResponse[] apiContentItems = contentItems.Select(ApiContentResponseBuilder.Build).WhereNotNull().ToArray(); var model = new PagedViewModel { diff --git a/src/Umbraco.Core/DeliveryApi/ApiContentBuilderBase.cs b/src/Umbraco.Core/DeliveryApi/ApiContentBuilderBase.cs index 8c70c3ff5b..ae70f0fdde 100644 --- a/src/Umbraco.Core/DeliveryApi/ApiContentBuilderBase.cs +++ b/src/Umbraco.Core/DeliveryApi/ApiContentBuilderBase.cs @@ -19,8 +19,14 @@ public abstract class ApiContentBuilderBase protected abstract T Create(IPublishedContent content, Guid id, string name, string contentType, IApiContentRoute route, IDictionary properties); - public virtual T Build(IPublishedContent content) + public virtual T? Build(IPublishedContent content) { + IApiContentRoute? route = _apiContentRouteBuilder.Build(content); + if (route is null) + { + return default; + } + IDictionary properties = _outputExpansionStrategyAccessor.TryGetValue(out IOutputExpansionStrategy? outputExpansionStrategy) ? outputExpansionStrategy.MapContentProperties(content) @@ -31,7 +37,7 @@ public abstract class ApiContentBuilderBase content.Key, _apiContentNameProvider.GetName(content), content.ContentType.Alias, - _apiContentRouteBuilder.Build(content), + route, properties); } } diff --git a/src/Umbraco.Core/DeliveryApi/ApiContentResponseBuilder.cs b/src/Umbraco.Core/DeliveryApi/ApiContentResponseBuilder.cs index dfa93e55d0..eb9cea6961 100644 --- a/src/Umbraco.Core/DeliveryApi/ApiContentResponseBuilder.cs +++ b/src/Umbraco.Core/DeliveryApi/ApiContentResponseBuilder.cs @@ -1,5 +1,6 @@ using Umbraco.Cms.Core.Models.DeliveryApi; using Umbraco.Cms.Core.Models.PublishedContent; +using Umbraco.Cms.Core.Routing; using Umbraco.Extensions; namespace Umbraco.Cms.Core.DeliveryApi; @@ -14,12 +15,26 @@ public sealed class ApiContentResponseBuilder : ApiContentBuilderBase properties) { - var cultures = content.Cultures.Values - .Where(publishedCultureInfo => publishedCultureInfo.Culture.IsNullOrWhiteSpace() == false) // filter out invariant cultures - .ToDictionary( - publishedCultureInfo => publishedCultureInfo.Culture, - publishedCultureInfo => _apiContentRouteBuilder.Build(content, publishedCultureInfo.Culture)); + var routesByCulture = new Dictionary(); - return new ApiContentResponse(id, name, contentType, route, properties, cultures); + foreach (PublishedCultureInfo publishedCultureInfo in content.Cultures.Values) + { + if (publishedCultureInfo.Culture.IsNullOrWhiteSpace()) + { + // filter out invariant cultures + continue; + } + + IApiContentRoute? cultureRoute = _apiContentRouteBuilder.Build(content, publishedCultureInfo.Culture); + if (cultureRoute == null) + { + // content is un-routable in this culture + continue; + } + + routesByCulture[publishedCultureInfo.Culture] = cultureRoute; + } + + return new ApiContentResponse(id, name, contentType, route, properties, routesByCulture); } } diff --git a/src/Umbraco.Core/DeliveryApi/ApiContentRouteBuilder.cs b/src/Umbraco.Core/DeliveryApi/ApiContentRouteBuilder.cs index 34146bf5cf..80552b6488 100644 --- a/src/Umbraco.Core/DeliveryApi/ApiContentRouteBuilder.cs +++ b/src/Umbraco.Core/DeliveryApi/ApiContentRouteBuilder.cs @@ -27,7 +27,7 @@ public sealed class ApiContentRouteBuilder : IApiContentRouteBuilder _globalSettings = globalSettings.Value; } - public IApiContentRoute Build(IPublishedContent content, string? culture = null) + public IApiContentRoute? Build(IPublishedContent content, string? culture = null) { if (content.ItemType != PublishedItemType.Content) { @@ -42,11 +42,17 @@ public sealed class ApiContentRouteBuilder : IApiContentRouteBuilder // in some scenarios the published content is actually routable, but due to the built-in handling of i.e. lacking culture setup // the URL provider resolves the content URL as empty string or "#". since the Delivery API handles routing explicitly, // we can perform fallback to the content route. - if (contentPath.IsNullOrWhiteSpace() || "#".Equals(contentPath)) + if (IsInvalidContentPath(contentPath)) { contentPath = _publishedSnapshotAccessor.GetRequiredPublishedSnapshot().Content?.GetRouteById(content.Id, culture) ?? contentPath; } + // if the content path has still not been resolved as a valid path, the content is un-routable in this culture + if (IsInvalidContentPath(contentPath)) + { + return null; + } + contentPath = contentPath.EnsureStartsWith("/"); if (_globalSettings.HideTopLevelNodeFromPath == false) { @@ -55,4 +61,6 @@ public sealed class ApiContentRouteBuilder : IApiContentRouteBuilder return new ApiContentRoute(contentPath, new ApiContentStartItem(root.Key, rootPath)); } + + private static bool IsInvalidContentPath(string path) => path.IsNullOrWhiteSpace() || "#".Equals(path); } diff --git a/src/Umbraco.Core/DeliveryApi/IApiContentBuilder.cs b/src/Umbraco.Core/DeliveryApi/IApiContentBuilder.cs index 784cb29370..946a75cda7 100644 --- a/src/Umbraco.Core/DeliveryApi/IApiContentBuilder.cs +++ b/src/Umbraco.Core/DeliveryApi/IApiContentBuilder.cs @@ -5,5 +5,5 @@ namespace Umbraco.Cms.Core.DeliveryApi; public interface IApiContentBuilder { - IApiContent Build(IPublishedContent content); + IApiContent? Build(IPublishedContent content); } diff --git a/src/Umbraco.Core/DeliveryApi/IApiContentResponseBuilder.cs b/src/Umbraco.Core/DeliveryApi/IApiContentResponseBuilder.cs index 82c25f3284..1d025ecadf 100644 --- a/src/Umbraco.Core/DeliveryApi/IApiContentResponseBuilder.cs +++ b/src/Umbraco.Core/DeliveryApi/IApiContentResponseBuilder.cs @@ -5,5 +5,5 @@ namespace Umbraco.Cms.Core.DeliveryApi; public interface IApiContentResponseBuilder { - IApiContentResponse Build(IPublishedContent content); + IApiContentResponse? Build(IPublishedContent content); } diff --git a/src/Umbraco.Core/DeliveryApi/IApiContentRouteBuilder.cs b/src/Umbraco.Core/DeliveryApi/IApiContentRouteBuilder.cs index 764e7765c2..dbd1103353 100644 --- a/src/Umbraco.Core/DeliveryApi/IApiContentRouteBuilder.cs +++ b/src/Umbraco.Core/DeliveryApi/IApiContentRouteBuilder.cs @@ -5,5 +5,5 @@ namespace Umbraco.Cms.Core.DeliveryApi; public interface IApiContentRouteBuilder { - IApiContentRoute Build(IPublishedContent content, string? culture = null); + IApiContentRoute? Build(IPublishedContent content, string? culture = null); } diff --git a/src/Umbraco.Infrastructure/DeliveryApi/ApiRichTextParser.cs b/src/Umbraco.Infrastructure/DeliveryApi/ApiRichTextParser.cs index f2d2181e60..4044b37516 100644 --- a/src/Umbraco.Infrastructure/DeliveryApi/ApiRichTextParser.cs +++ b/src/Umbraco.Infrastructure/DeliveryApi/ApiRichTextParser.cs @@ -117,9 +117,12 @@ internal sealed partial class ApiRichTextParser : IApiRichTextParser { case Constants.UdiEntityType.Document: IPublishedContent? content = publishedSnapshot.Content?.GetById(udi); - if (content != null) + IApiContentRoute? route = content != null + ? _apiContentRouteBuilder.Build(content) + : null; + if (route != null) { - attributes["route"] = _apiContentRouteBuilder.Build(content); + attributes["route"] = route; } break; diff --git a/src/Umbraco.Infrastructure/PropertyEditors/ValueConverters/MultiUrlPickerValueConverter.cs b/src/Umbraco.Infrastructure/PropertyEditors/ValueConverters/MultiUrlPickerValueConverter.cs index 605c348992..821ece9b33 100644 --- a/src/Umbraco.Infrastructure/PropertyEditors/ValueConverters/MultiUrlPickerValueConverter.cs +++ b/src/Umbraco.Infrastructure/PropertyEditors/ValueConverters/MultiUrlPickerValueConverter.cs @@ -177,14 +177,17 @@ public class MultiUrlPickerValueConverter : PropertyValueConverterBase, IDeliver { case Constants.UdiEntityType.Document: IPublishedContent? content = publishedSnapshot.Content?.GetById(item.Udi.Guid); - return content == null + IApiContentRoute? route = content != null + ? _apiContentRouteBuilder.Build(content) + : null; + return content == null || route == null ? null : ApiLink.Content( item.Name.IfNullOrWhiteSpace(_apiContentNameProvider.GetName(content)), item.Target, content.Key, content.ContentType.Alias, - _apiContentRouteBuilder.Build(content)); + route); case Constants.UdiEntityType.Media: IPublishedContent? media = publishedSnapshot.Media?.GetById(item.Udi.Guid); return media == null diff --git a/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/ContentBuilderTests.cs b/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/ContentBuilderTests.cs index 750ac885e0..f214c05446 100644 --- a/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/ContentBuilderTests.cs +++ b/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/ContentBuilderTests.cs @@ -1,6 +1,7 @@ using Moq; using NUnit.Framework; using Umbraco.Cms.Core.DeliveryApi; +using Umbraco.Cms.Core.Models.DeliveryApi; using Umbraco.Cms.Core.Models.PublishedContent; using Umbraco.Cms.Core.PropertyEditors; using Umbraco.Cms.Core.PublishedCache; @@ -61,10 +62,36 @@ public class ContentBuilderTests : DeliveryApiTests var customNameProvider = new Mock(); customNameProvider.Setup(n => n.GetName(content.Object)).Returns($"Custom name for: {content.Object.Name}"); - var builder = new ApiContentBuilder(customNameProvider.Object, Mock.Of(), CreateOutputExpansionStrategyAccessor()); + var routeBuilder = new Mock(); + routeBuilder + .Setup(r => r.Build(content.Object, It.IsAny())) + .Returns(new ApiContentRoute(content.Object.UrlSegment!, new ApiContentStartItem(Guid.NewGuid(), "/"))); + + var builder = new ApiContentBuilder(customNameProvider.Object, routeBuilder.Object, CreateOutputExpansionStrategyAccessor()); var result = builder.Build(content.Object); Assert.NotNull(result); Assert.AreEqual("Custom name for: The page", result.Name); } + + [Test] + public void ContentBuilder_ReturnsNullForUnRoutableContent() + { + var content = new Mock(); + + var contentType = new Mock(); + contentType.SetupGet(c => c.Alias).Returns("thePageType"); + + ConfigurePublishedContentMock(content, Guid.NewGuid(), "The page", "the-page", contentType.Object, Array.Empty()); + + var routeBuilder = new Mock(); + routeBuilder + .Setup(r => r.Build(content.Object, It.IsAny())) + .Returns((ApiContentRoute)null); + + var builder = new ApiContentBuilder(new ApiContentNameProvider(), routeBuilder.Object, CreateOutputExpansionStrategyAccessor()); + var result = builder.Build(content.Object); + + Assert.Null(result); + } } diff --git a/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/ContentRouteBuilderTests.cs b/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/ContentRouteBuilderTests.cs index 202026ed30..90774c5e25 100644 --- a/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/ContentRouteBuilderTests.cs +++ b/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/ContentRouteBuilderTests.cs @@ -2,6 +2,7 @@ using NUnit.Framework; using Umbraco.Cms.Core.DeliveryApi; using Umbraco.Cms.Core.Models; +using Umbraco.Cms.Core.Models.DeliveryApi; using Umbraco.Cms.Core.Models.PublishedContent; using Umbraco.Cms.Core.PublishedCache; using Umbraco.Cms.Core.Routing; @@ -21,6 +22,7 @@ public class ContentRouteBuilderTests : DeliveryApiTests var builder = CreateApiContentRouteBuilder(hideTopLevelNodeFromPath); var result = builder.Build(root); + Assert.IsNotNull(result); Assert.AreEqual("/", result.Path); Assert.AreEqual(rootKey, result.StartItem.Id); Assert.AreEqual("the-root", result.StartItem.Path); @@ -38,6 +40,7 @@ public class ContentRouteBuilderTests : DeliveryApiTests var builder = CreateApiContentRouteBuilder(hideTopLevelNodeFromPath); var result = builder.Build(child); + Assert.IsNotNull(result); Assert.AreEqual("/the-child", result.Path); Assert.AreEqual(rootKey, result.StartItem.Id); Assert.AreEqual("the-root", result.StartItem.Path); @@ -58,6 +61,7 @@ public class ContentRouteBuilderTests : DeliveryApiTests var builder = CreateApiContentRouteBuilder(hideTopLevelNodeFromPath); var result = builder.Build(grandchild); + Assert.IsNotNull(result); Assert.AreEqual("/the-child/the-grandchild", result.Path); Assert.AreEqual(rootKey, result.StartItem.Id); Assert.AreEqual("the-root", result.StartItem.Path); @@ -74,11 +78,13 @@ public class ContentRouteBuilderTests : DeliveryApiTests var builder = CreateApiContentRouteBuilder(false); var result = builder.Build(child, "en-us"); + Assert.IsNotNull(result); Assert.AreEqual("/the-child-en-us", result.Path); Assert.AreEqual(rootKey, result.StartItem.Id); Assert.AreEqual("the-root-en-us", result.StartItem.Path); result = builder.Build(child, "da-dk"); + Assert.IsNotNull(result); Assert.AreEqual("/the-child-da-dk", result.Path); Assert.AreEqual(rootKey, result.StartItem.Id); Assert.AreEqual("the-root-da-dk", result.StartItem.Path); @@ -95,11 +101,13 @@ public class ContentRouteBuilderTests : DeliveryApiTests var builder = CreateApiContentRouteBuilder(false); var result = builder.Build(child, "en-us"); + Assert.IsNotNull(result); Assert.AreEqual("/the-child", result.Path); Assert.AreEqual(rootKey, result.StartItem.Id); Assert.AreEqual("the-root-en-us", result.StartItem.Path); result = builder.Build(child, "da-dk"); + Assert.IsNotNull(result); Assert.AreEqual("/the-child", result.Path); Assert.AreEqual(rootKey, result.StartItem.Id); Assert.AreEqual("the-root-da-dk", result.StartItem.Path); @@ -116,11 +124,13 @@ public class ContentRouteBuilderTests : DeliveryApiTests var builder = CreateApiContentRouteBuilder(false); var result = builder.Build(child, "en-us"); + Assert.IsNotNull(result); Assert.AreEqual("/the-child-en-us", result.Path); Assert.AreEqual(rootKey, result.StartItem.Id); Assert.AreEqual("the-root", result.StartItem.Path); result = builder.Build(child, "da-dk"); + Assert.IsNotNull(result); Assert.AreEqual("/the-child-da-dk", result.Path); Assert.AreEqual(rootKey, result.StartItem.Id); Assert.AreEqual("the-root", result.StartItem.Path); @@ -144,36 +154,18 @@ public class ContentRouteBuilderTests : DeliveryApiTests [TestCase("#")] public void FallsBackToContentPathIfUrlProviderCannotResolveUrl(string resolvedUrl) { - var publishedUrlProviderMock = new Mock(); - publishedUrlProviderMock - .Setup(p => p.GetUrl(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) - .Returns(resolvedUrl); + var result = GetUnRoutableRoute(resolvedUrl, "/the/content/route"); + Assert.IsNotNull(result); + Assert.AreEqual("/the/content/route", result.Path); + } - var publishedContentCacheMock = new Mock(); - publishedContentCacheMock - .Setup(c => c.GetRouteById(It.IsAny(), It.IsAny())) - .Returns("/the/content/route"); - - var publishedSnapshotMock = new Mock(); - publishedSnapshotMock - .SetupGet(s => s.Content) - .Returns(publishedContentCacheMock.Object); - var publishedSnapshot = publishedSnapshotMock.Object; - - var publishedSnapshotAccessorMock = new Mock(); - publishedSnapshotAccessorMock - .Setup(a => a.TryGetPublishedSnapshot(out publishedSnapshot)) - .Returns(true); - - var content = SetupVariantPublishedContent("The Content", Guid.NewGuid()); - - var builder = new ApiContentRouteBuilder( - publishedUrlProviderMock.Object, - CreateGlobalSettings(), - Mock.Of(), - publishedSnapshotAccessorMock.Object); - - Assert.AreEqual("/the/content/route", builder.Build(content).Path); + [TestCase("")] + [TestCase(" ")] + [TestCase("#")] + public void YieldsNullForUnRoutableContent(string contentPath) + { + var result = GetUnRoutableRoute(contentPath, contentPath); + Assert.IsNull(result); } [TestCase(true)] @@ -253,4 +245,38 @@ public class ContentRouteBuilderTests : DeliveryApiTests CreateGlobalSettings(hideTopLevelNodeFromPath), Mock.Of(), Mock.Of()); + + private IApiContentRoute? GetUnRoutableRoute(string publishedUrl, string routeById) + { + var publishedUrlProviderMock = new Mock(); + publishedUrlProviderMock + .Setup(p => p.GetUrl(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(publishedUrl); + + var publishedContentCacheMock = new Mock(); + publishedContentCacheMock + .Setup(c => c.GetRouteById(It.IsAny(), It.IsAny())) + .Returns(routeById); + + var publishedSnapshotMock = new Mock(); + publishedSnapshotMock + .SetupGet(s => s.Content) + .Returns(publishedContentCacheMock.Object); + var publishedSnapshot = publishedSnapshotMock.Object; + + var publishedSnapshotAccessorMock = new Mock(); + publishedSnapshotAccessorMock + .Setup(a => a.TryGetPublishedSnapshot(out publishedSnapshot)) + .Returns(true); + + var content = SetupVariantPublishedContent("The Content", Guid.NewGuid()); + + var builder = new ApiContentRouteBuilder( + publishedUrlProviderMock.Object, + CreateGlobalSettings(), + Mock.Of(), + publishedSnapshotAccessorMock.Object); + + return builder.Build(content); + } } diff --git a/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/MultiNodeTreePickerValueConverterTests.cs b/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/MultiNodeTreePickerValueConverterTests.cs index 320f16dbf6..80e175bf81 100644 --- a/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/MultiNodeTreePickerValueConverterTests.cs +++ b/tests/Umbraco.Tests.UnitTests/Umbraco.Core/DeliveryApi/MultiNodeTreePickerValueConverterTests.cs @@ -15,13 +15,13 @@ namespace Umbraco.Cms.Tests.UnitTests.Umbraco.Core.DeliveryApi; [TestFixture] public class MultiNodeTreePickerValueConverterTests : PropertyValueConverterTests { - private MultiNodeTreePickerValueConverter MultiNodeTreePickerValueConverter() + private MultiNodeTreePickerValueConverter MultiNodeTreePickerValueConverter(IApiContentRouteBuilder? routeBuilder = null) { var expansionStrategyAccessor = CreateOutputExpansionStrategyAccessor(); var contentNameProvider = new ApiContentNameProvider(); var apiUrProvider = new ApiMediaUrlProvider(PublishedUrlProvider); - var routeBuilder = new ApiContentRouteBuilder(PublishedUrlProvider, CreateGlobalSettings(), Mock.Of(), Mock.Of()); + routeBuilder = routeBuilder ?? new ApiContentRouteBuilder(PublishedUrlProvider, CreateGlobalSettings(), Mock.Of(), Mock.Of()); return new MultiNodeTreePickerValueConverter( PublishedSnapshotAccessor, Mock.Of(), @@ -274,4 +274,21 @@ public class MultiNodeTreePickerValueConverterTests : PropertyValueConverterTest Assert.NotNull(result); Assert.IsEmpty(result); } + + [Test] + public void MultiNodeTreePickerValueConverter_YieldsNothingForUnRoutableContent() + { + var publishedDataType = MultiNodePickerPublishedDataType(false, Constants.UdiEntityType.Document); + var publishedPropertyType = new Mock(); + publishedPropertyType.SetupGet(p => p.DataType).Returns(publishedDataType); + + // mocking the route builder will make it yield null values for all routes, so there is no need to setup anything on the mock + var routeBuilder = new Mock(); + var valueConverter = MultiNodeTreePickerValueConverter(routeBuilder.Object); + + var inter = new Udi[] { new GuidUdi(Constants.UdiEntityType.Document, PublishedContent.Key) }; + var result = valueConverter.ConvertIntermediateToDeliveryApiObject(Mock.Of(), publishedPropertyType.Object, PropertyCacheLevel.Element, inter, false) as IEnumerable; + Assert.NotNull(result); + Assert.IsEmpty(result); + } } From e572dcfa2da5c0921c767406dc2d2ad9456d1728 Mon Sep 17 00:00:00 2001 From: Elitsa Marinovska <21998037+elit0451@users.noreply.github.com> Date: Mon, 15 May 2023 10:46:29 +0200 Subject: [PATCH 26/26] Delivery API: Handle more unhappy paths when querying (#14245) * Handle more unhappy paths * Review comments --------- Co-authored-by: kjac --- .../Querying/Filters/ContentTypeFilter.cs | 5 ++++- .../Querying/Filters/NameFilter.cs | 5 ++++- .../Querying/QueryOptionBase.cs | 16 +++++++++------- .../Querying/Selectors/AncestorsSelector.cs | 11 ++++++----- .../Services/RequestRoutingService.cs | 5 +++++ 5 files changed, 28 insertions(+), 14 deletions(-) diff --git a/src/Umbraco.Cms.Api.Delivery/Querying/Filters/ContentTypeFilter.cs b/src/Umbraco.Cms.Api.Delivery/Querying/Filters/ContentTypeFilter.cs index e851158b87..13d9dc77ec 100644 --- a/src/Umbraco.Cms.Api.Delivery/Querying/Filters/ContentTypeFilter.cs +++ b/src/Umbraco.Cms.Api.Delivery/Querying/Filters/ContentTypeFilter.cs @@ -1,5 +1,6 @@ using Umbraco.Cms.Api.Delivery.Indexing.Filters; using Umbraco.Cms.Core.DeliveryApi; +using Umbraco.Extensions; namespace Umbraco.Cms.Api.Delivery.Querying.Filters; @@ -19,7 +20,9 @@ public sealed class ContentTypeFilter : IFilterHandler return new FilterOption { FieldName = ContentTypeFilterIndexer.FieldName, - Values = new[] { alias.TrimStart('!') }, + Values = alias.IsNullOrWhiteSpace() == false + ? new[] { alias.TrimStart('!') } + : Array.Empty(), Operator = alias.StartsWith('!') ? FilterOperation.IsNot : FilterOperation.Is diff --git a/src/Umbraco.Cms.Api.Delivery/Querying/Filters/NameFilter.cs b/src/Umbraco.Cms.Api.Delivery/Querying/Filters/NameFilter.cs index 64aa5b2776..0bf3d5e460 100644 --- a/src/Umbraco.Cms.Api.Delivery/Querying/Filters/NameFilter.cs +++ b/src/Umbraco.Cms.Api.Delivery/Querying/Filters/NameFilter.cs @@ -1,5 +1,6 @@ using Umbraco.Cms.Api.Delivery.Indexing.Sorts; using Umbraco.Cms.Core.DeliveryApi; +using Umbraco.Extensions; namespace Umbraco.Cms.Api.Delivery.Querying.Filters; @@ -19,7 +20,9 @@ public sealed class NameFilter : IFilterHandler return new FilterOption { FieldName = NameSortIndexer.FieldName, - Values = new[] { value.TrimStart('!') }, + Values = value.IsNullOrWhiteSpace() == false + ? new[] { value.TrimStart('!') } + : Array.Empty(), Operator = value.StartsWith('!') ? FilterOperation.IsNot : FilterOperation.Is diff --git a/src/Umbraco.Cms.Api.Delivery/Querying/QueryOptionBase.cs b/src/Umbraco.Cms.Api.Delivery/Querying/QueryOptionBase.cs index 8934889715..f29e0465f5 100644 --- a/src/Umbraco.Cms.Api.Delivery/Querying/QueryOptionBase.cs +++ b/src/Umbraco.Cms.Api.Delivery/Querying/QueryOptionBase.cs @@ -1,6 +1,7 @@ using Umbraco.Cms.Core.DeliveryApi; using Umbraco.Cms.Core.Models.PublishedContent; using Umbraco.Cms.Core.PublishedCache; +using Umbraco.Extensions; namespace Umbraco.Cms.Api.Delivery.Querying; @@ -16,22 +17,23 @@ public abstract class QueryOptionBase _requestRoutingService = requestRoutingService; } - public Guid? GetGuidFromQuery(string queryStringValue) + protected Guid? GetGuidFromQuery(string queryStringValue) { + if (queryStringValue.IsNullOrWhiteSpace()) + { + return null; + } + if (Guid.TryParse(queryStringValue, out Guid id)) { return id; } - if (!_publishedSnapshotAccessor.TryGetPublishedSnapshot(out IPublishedSnapshot? publishedSnapshot) || - publishedSnapshot?.Content is null) - { - return null; - } + IPublishedSnapshot publishedSnapshot = _publishedSnapshotAccessor.GetRequiredPublishedSnapshot(); // Check if the passed value is a path of a content item var contentRoute = _requestRoutingService.GetContentRoute(queryStringValue); - IPublishedContent? contentItem = publishedSnapshot.Content.GetByRoute(contentRoute); + IPublishedContent? contentItem = publishedSnapshot.Content?.GetByRoute(contentRoute); return contentItem?.Key; } diff --git a/src/Umbraco.Cms.Api.Delivery/Querying/Selectors/AncestorsSelector.cs b/src/Umbraco.Cms.Api.Delivery/Querying/Selectors/AncestorsSelector.cs index 67490e38f4..5e8e7e2019 100644 --- a/src/Umbraco.Cms.Api.Delivery/Querying/Selectors/AncestorsSelector.cs +++ b/src/Umbraco.Cms.Api.Delivery/Querying/Selectors/AncestorsSelector.cs @@ -25,9 +25,7 @@ public sealed class AncestorsSelector : QueryOptionBase, ISelectorHandler var fieldValue = selector[AncestorsSpecifier.Length..]; Guid? id = GetGuidFromQuery(fieldValue); - if (id is null || - !_publishedSnapshotAccessor.TryGetPublishedSnapshot(out IPublishedSnapshot? publishedSnapshot) || - publishedSnapshot?.Content is null) + if (id is null) { // Setting the Value to "" since that would yield no results. // It won't be appropriate to return null here since if we reached this, @@ -39,8 +37,11 @@ public sealed class AncestorsSelector : QueryOptionBase, ISelectorHandler }; } - // With the previous check we made sure that if we reach this, we already made sure that there is a valid content item - IPublishedContent contentItem = publishedSnapshot.Content.GetById((Guid)id)!; // so it can't be null + IPublishedSnapshot publishedSnapshot = _publishedSnapshotAccessor.GetRequiredPublishedSnapshot(); + + IPublishedContent contentItem = publishedSnapshot.Content?.GetById((Guid)id) + ?? throw new InvalidOperationException("Could not obtain the content cache"); + var ancestorKeys = contentItem.Ancestors().Select(a => a.Key.ToString("D")).ToArray(); return new SelectorOption diff --git a/src/Umbraco.Cms.Api.Delivery/Services/RequestRoutingService.cs b/src/Umbraco.Cms.Api.Delivery/Services/RequestRoutingService.cs index 993780680d..6bf0dbc887 100644 --- a/src/Umbraco.Cms.Api.Delivery/Services/RequestRoutingService.cs +++ b/src/Umbraco.Cms.Api.Delivery/Services/RequestRoutingService.cs @@ -22,6 +22,11 @@ internal sealed class RequestRoutingService : RoutingServiceBase, IRequestRoutin /// public string GetContentRoute(string requestedPath) { + if (requestedPath.IsNullOrWhiteSpace()) + { + return string.Empty; + } + requestedPath = requestedPath.EnsureStartsWith("/"); // do we have an explicit start item?