From bafcc2b9451c3f418cfcb960031c2a89a1a83809 Mon Sep 17 00:00:00 2001 From: Kenn Jacobsen Date: Sun, 20 Oct 2024 15:42:13 +0200 Subject: [PATCH] Block level variants - search indexing (#17239) * Support block level variance for search indexing * Rename base class --------- Co-authored-by: Elitsa --- .../DefaultPropertyIndexValueFactory.cs | 17 +- .../IPropertyIndexValueFactory.cs | 8 +- .../PropertyEditors/IndexValue.cs | 10 + .../JsonPropertyIndexValueFactoryBase.cs | 14 +- .../NoopPropertyIndexValueFactory.cs | 4 +- .../TagPropertyIndexValueFactory.cs | 32 +- .../Examine/BaseValueSetBuilder.cs | 19 +- .../BlockValuePropertyIndexValueFactory.cs | 18 +- ...BlockValuePropertyIndexValueFactoryBase.cs | 308 ++++++++++++++ .../NestedPropertyIndexValueFactoryBase.cs | 222 ---------- .../RichTextPropertyIndexValueFactory.cs | 129 ++++-- ...ListElementLevelVariationTests.Indexing.cs | 399 ++++++++++++++++++ .../PropertyIndexValueFactoryTests.cs | 36 +- .../RichTextElementLevelVariationTests.cs | 271 +++++++++++- .../Umbraco.Tests.Integration.csproj | 3 + 15 files changed, 1143 insertions(+), 347 deletions(-) create mode 100644 src/Umbraco.Core/PropertyEditors/IndexValue.cs create mode 100644 src/Umbraco.Infrastructure/PropertyEditors/BlockValuePropertyIndexValueFactoryBase.cs delete mode 100644 src/Umbraco.Infrastructure/PropertyEditors/NestedPropertyIndexValueFactoryBase.cs create mode 100644 tests/Umbraco.Tests.Integration/Umbraco.Infrastructure/PropertyEditors/BlockListElementLevelVariationTests.Indexing.cs diff --git a/src/Umbraco.Core/PropertyEditors/DefaultPropertyIndexValueFactory.cs b/src/Umbraco.Core/PropertyEditors/DefaultPropertyIndexValueFactory.cs index 6442c6686a..9d96ced00a 100644 --- a/src/Umbraco.Core/PropertyEditors/DefaultPropertyIndexValueFactory.cs +++ b/src/Umbraco.Core/PropertyEditors/DefaultPropertyIndexValueFactory.cs @@ -1,5 +1,4 @@ using Umbraco.Cms.Core.Models; -using Umbraco.Extensions; namespace Umbraco.Cms.Core.PropertyEditors; @@ -9,11 +8,15 @@ namespace Umbraco.Cms.Core.PropertyEditors; /// public class DefaultPropertyIndexValueFactory : IPropertyIndexValueFactory { - public IEnumerable>> GetIndexValues(IProperty property, string? culture, string? segment, bool published, + public IEnumerable GetIndexValues(IProperty property, string? culture, string? segment, bool published, IEnumerable availableCultures, IDictionary contentTypeDictionary) - { - yield return new KeyValuePair>( - property.Alias, - property.GetValue(culture, segment, published).Yield()); - } + => + [ + new IndexValue + { + Culture = culture, + FieldName = property.Alias, + Values = [property.GetValue(culture, segment, published)] + } + ]; } diff --git a/src/Umbraco.Core/PropertyEditors/IPropertyIndexValueFactory.cs b/src/Umbraco.Core/PropertyEditors/IPropertyIndexValueFactory.cs index ea4fbf614a..6b5942ceb7 100644 --- a/src/Umbraco.Core/PropertyEditors/IPropertyIndexValueFactory.cs +++ b/src/Umbraco.Core/PropertyEditors/IPropertyIndexValueFactory.cs @@ -12,9 +12,9 @@ public interface IPropertyIndexValueFactory /// /// /// - /// Returns key-value pairs, where keys are indexed field names. By default, that would be the property alias, - /// and there would be only one pair, but some implementations (see for instance the grid one) may return more than - /// one pair, with different indexed field names. + /// Returns index values for a given property. By default, a property uses its alias as index field name, + /// and there would be only one index value, but some implementations (see for instance the grid one) may return more than + /// one value, with different indexed field names. /// /// /// And then, values are an enumerable of objects, because each indexed field can in turn have multiple @@ -22,7 +22,7 @@ public interface IPropertyIndexValueFactory /// more than one value for a given field. /// /// - IEnumerable>> GetIndexValues( + IEnumerable GetIndexValues( IProperty property, string? culture, string? segment, diff --git a/src/Umbraco.Core/PropertyEditors/IndexValue.cs b/src/Umbraco.Core/PropertyEditors/IndexValue.cs new file mode 100644 index 0000000000..ebcc321bf9 --- /dev/null +++ b/src/Umbraco.Core/PropertyEditors/IndexValue.cs @@ -0,0 +1,10 @@ +namespace Umbraco.Cms.Core.PropertyEditors; + +public sealed class IndexValue +{ + public required string? Culture { get; set; } + + public required string FieldName { get; set; } + + public required IEnumerable Values { get; set; } +} diff --git a/src/Umbraco.Core/PropertyEditors/JsonPropertyIndexValueFactoryBase.cs b/src/Umbraco.Core/PropertyEditors/JsonPropertyIndexValueFactoryBase.cs index e4a7492c5d..bfbd38a294 100644 --- a/src/Umbraco.Core/PropertyEditors/JsonPropertyIndexValueFactoryBase.cs +++ b/src/Umbraco.Core/PropertyEditors/JsonPropertyIndexValueFactoryBase.cs @@ -27,7 +27,7 @@ public abstract class JsonPropertyIndexValueFactoryBase : IProperty indexingSettings.OnChange(newValue => _indexingSettings = newValue); } - public virtual IEnumerable>> GetIndexValues( + public virtual IEnumerable GetIndexValues( IProperty property, string? culture, string? segment, @@ -35,7 +35,7 @@ public abstract class JsonPropertyIndexValueFactoryBase : IProperty IEnumerable availableCultures, IDictionary contentTypeDictionary) { - var result = new List>>(); + var result = new List(); var propertyValue = property.GetValue(culture, segment, published); @@ -65,7 +65,7 @@ public abstract class JsonPropertyIndexValueFactoryBase : IProperty } } - IEnumerable>> summary = HandleResume(result, property, culture, segment, published); + IEnumerable summary = HandleResume(result, property, culture, segment, published); if (_indexingSettings.ExplicitlyIndexEachNestedProperty || ForceExplicitlyIndexEachNestedProperty) { result.AddRange(summary); @@ -78,17 +78,17 @@ public abstract class JsonPropertyIndexValueFactoryBase : IProperty /// /// Method to return a list of summary of the content. By default this returns an empty list /// - protected virtual IEnumerable>> HandleResume( - List>> result, + protected virtual IEnumerable HandleResume( + List result, IProperty property, string? culture, string? segment, - bool published) => Array.Empty>>(); + bool published) => Array.Empty(); /// /// Method that handle the deserialized object. /// - protected abstract IEnumerable>> Handle( + protected abstract IEnumerable Handle( TSerialized deserializedPropertyValue, IProperty property, string? culture, diff --git a/src/Umbraco.Core/PropertyEditors/NoopPropertyIndexValueFactory.cs b/src/Umbraco.Core/PropertyEditors/NoopPropertyIndexValueFactory.cs index 24711058cd..61193a5c26 100644 --- a/src/Umbraco.Core/PropertyEditors/NoopPropertyIndexValueFactory.cs +++ b/src/Umbraco.Core/PropertyEditors/NoopPropertyIndexValueFactory.cs @@ -8,7 +8,7 @@ namespace Umbraco.Cms.Core.PropertyEditors; public class NoopPropertyIndexValueFactory : IPropertyIndexValueFactory { /// - public IEnumerable>> GetIndexValues(IProperty property, string? culture, string? segment, bool published, + public IEnumerable GetIndexValues(IProperty property, string? culture, string? segment, bool published, IEnumerable availableCultures, IDictionary contentTypeDictionary) - => Array.Empty>>(); + => []; } diff --git a/src/Umbraco.Core/PropertyEditors/TagPropertyIndexValueFactory.cs b/src/Umbraco.Core/PropertyEditors/TagPropertyIndexValueFactory.cs index 1696c1550a..9f3ebb674b 100644 --- a/src/Umbraco.Core/PropertyEditors/TagPropertyIndexValueFactory.cs +++ b/src/Umbraco.Core/PropertyEditors/TagPropertyIndexValueFactory.cs @@ -19,17 +19,7 @@ public class TagPropertyIndexValueFactory : JsonPropertyIndexValueFactoryBase _indexingSettings = newValue); } - [Obsolete("Use the overload with the 'contentTypeDictionary' parameter instead, scheduled for removal in v15")] - protected IEnumerable>> Handle( - string[] deserializedPropertyValue, - IProperty property, - string? culture, - string? segment, - bool published, - IEnumerable availableCultures) - => Handle(deserializedPropertyValue, property, culture, segment, published, availableCultures, new Dictionary()); - - protected override IEnumerable>> Handle( + protected override IEnumerable Handle( string[] deserializedPropertyValue, IProperty property, string? culture, @@ -37,11 +27,17 @@ public class TagPropertyIndexValueFactory : JsonPropertyIndexValueFactoryBase availableCultures, IDictionary contentTypeDictionary) - { - yield return new KeyValuePair>(property.Alias, deserializedPropertyValue); - } + => + [ + new IndexValue + { + Culture = culture, + FieldName = property.Alias, + Values = deserializedPropertyValue + } + ]; - public override IEnumerable>> GetIndexValues( + public override IEnumerable GetIndexValues( IProperty property, string? culture, string? segment, @@ -49,13 +45,13 @@ public class TagPropertyIndexValueFactory : JsonPropertyIndexValueFactoryBase availableCultures, IDictionary contentTypeDictionary) { - IEnumerable>> jsonValues = base.GetIndexValues(property, culture, segment, published, availableCultures, contentTypeDictionary); + IEnumerable jsonValues = base.GetIndexValues(property, culture, segment, published, availableCultures, contentTypeDictionary); if (jsonValues?.Any() is true) { return jsonValues; } - var result = new List>>(); + var result = new List(); var propertyValue = property.GetValue(culture, segment, published); @@ -67,7 +63,7 @@ public class TagPropertyIndexValueFactory : JsonPropertyIndexValueFactoryBase>> summary = HandleResume(result, property, culture, segment, published); + IEnumerable summary = HandleResume(result, property, culture, segment, published); if (_indexingSettings.ExplicitlyIndexEachNestedProperty || ForceExplicitlyIndexEachNestedProperty) { result.AddRange(summary); diff --git a/src/Umbraco.Infrastructure/Examine/BaseValueSetBuilder.cs b/src/Umbraco.Infrastructure/Examine/BaseValueSetBuilder.cs index 6c3e6d189a..972c58f2f8 100644 --- a/src/Umbraco.Infrastructure/Examine/BaseValueSetBuilder.cs +++ b/src/Umbraco.Infrastructure/Examine/BaseValueSetBuilder.cs @@ -30,18 +30,19 @@ public abstract class BaseValueSetBuilder : IValueSetBuilder return; } - IEnumerable>> indexVals = + IEnumerable indexVals = editor.PropertyIndexValueFactory.GetIndexValues(property, culture, segment, PublishedValuesOnly, availableCultures, contentTypeDictionary); - foreach (KeyValuePair> keyVal in indexVals) + foreach (IndexValue indexValue in indexVals) { - if (keyVal.Key.IsNullOrWhiteSpace()) + if (indexValue.FieldName.IsNullOrWhiteSpace()) { continue; } - var cultureSuffix = culture == null ? string.Empty : "_" + culture; + var indexValueCulture = indexValue.Culture ?? culture; + var cultureSuffix = indexValueCulture == null ? string.Empty : "_" + indexValueCulture.ToLowerInvariant(); - foreach (var val in keyVal.Value) + foreach (var val in indexValue.Values) { switch (val) { @@ -55,28 +56,28 @@ public abstract class BaseValueSetBuilder : IValueSetBuilder continue; } - var key = $"{keyVal.Key}{cultureSuffix}"; + var key = $"{indexValue.FieldName}{cultureSuffix}"; if (values?.TryGetValue(key, out IEnumerable? v) ?? false) { values[key] = new List(v) { val }.ToArray(); } else { - values?.Add($"{keyVal.Key}{cultureSuffix}", val.Yield()); + values?.Add($"{indexValue.FieldName}{cultureSuffix}", val.Yield()); } } break; default: { - var key = $"{keyVal.Key}{cultureSuffix}"; + var key = $"{indexValue.FieldName}{cultureSuffix}"; if (values?.TryGetValue(key, out IEnumerable? v) ?? false) { values[key] = new List(v) { val }.ToArray(); } else { - values?.Add($"{keyVal.Key}{cultureSuffix}", val.Yield()); + values?.Add($"{indexValue.FieldName}{cultureSuffix}", val.Yield()); } } diff --git a/src/Umbraco.Infrastructure/PropertyEditors/BlockValuePropertyIndexValueFactory.cs b/src/Umbraco.Infrastructure/PropertyEditors/BlockValuePropertyIndexValueFactory.cs index b4521d3bbd..27f9d9b40a 100644 --- a/src/Umbraco.Infrastructure/PropertyEditors/BlockValuePropertyIndexValueFactory.cs +++ b/src/Umbraco.Infrastructure/PropertyEditors/BlockValuePropertyIndexValueFactory.cs @@ -3,14 +3,13 @@ using Microsoft.Extensions.Options; using Umbraco.Cms.Core.Configuration.Models; -using Umbraco.Cms.Core.Models; using Umbraco.Cms.Core.Models.Blocks; using Umbraco.Cms.Core.Serialization; namespace Umbraco.Cms.Core.PropertyEditors; -internal sealed class BlockValuePropertyIndexValueFactory : - NestedPropertyIndexValueFactoryBase, +internal class BlockValuePropertyIndexValueFactory : + BlockValuePropertyIndexValueFactoryBase, IBlockValuePropertyIndexValueFactory { public BlockValuePropertyIndexValueFactory( @@ -21,19 +20,14 @@ internal sealed class BlockValuePropertyIndexValueFactory : { } - protected override IContentType? GetContentTypeOfNestedItem(BlockItemData input, IDictionary contentTypeDictionary) - => contentTypeDictionary.TryGetValue(input.ContentTypeKey, out var result) ? result : null; - - protected override IDictionary GetRawProperty(BlockItemData blockItemData) - => blockItemData.Values - .Where(p => p.Culture is null && p.Segment is null) - .ToDictionary(p => p.Alias, p => p.Value); - - protected override IEnumerable GetDataItems(IndexValueFactoryBlockValue input) => input.ContentData; + protected override IEnumerable GetDataItems(IndexValueFactoryBlockValue input, bool published) + => GetDataItems(input.ContentData, input.Expose, published); // we only care about the content data when extracting values for indexing - not the layouts nor the settings internal class IndexValueFactoryBlockValue { public List ContentData { get; set; } = new(); + + public List Expose { get; set; } = new(); } } diff --git a/src/Umbraco.Infrastructure/PropertyEditors/BlockValuePropertyIndexValueFactoryBase.cs b/src/Umbraco.Infrastructure/PropertyEditors/BlockValuePropertyIndexValueFactoryBase.cs new file mode 100644 index 0000000000..09759246ff --- /dev/null +++ b/src/Umbraco.Infrastructure/PropertyEditors/BlockValuePropertyIndexValueFactoryBase.cs @@ -0,0 +1,308 @@ +using System.Text; +using Microsoft.Extensions.Options; +using Umbraco.Cms.Core.Configuration.Models; +using Umbraco.Cms.Core.Models; +using Umbraco.Cms.Core.Models.Blocks; +using Umbraco.Cms.Core.Serialization; +using Umbraco.Cms.Infrastructure.Examine; +using Umbraco.Extensions; + +namespace Umbraco.Cms.Core.PropertyEditors; + +internal abstract class BlockValuePropertyIndexValueFactoryBase : JsonPropertyIndexValueFactoryBase +{ + private readonly PropertyEditorCollection _propertyEditorCollection; + + protected BlockValuePropertyIndexValueFactoryBase( + PropertyEditorCollection propertyEditorCollection, + IJsonSerializer jsonSerializer, + IOptionsMonitor indexingSettings) + : base(jsonSerializer, indexingSettings) + { + _propertyEditorCollection = propertyEditorCollection; + } + + protected override IEnumerable Handle( + TSerialized deserializedPropertyValue, + IProperty property, + string? culture, + string? segment, + bool published, + IEnumerable availableCultures, + IDictionary contentTypeDictionary) + { + var result = new List(); + + var index = 0; + foreach (RawDataItem rawData in GetDataItems(deserializedPropertyValue, published)) + { + if (contentTypeDictionary.TryGetValue(rawData.ContentTypeKey, out IContentType? contentType) is false) + { + continue; + } + + var propertyTypeDictionary = + contentType + .CompositionPropertyTypes + .Select(propertyType => + { + // We want to ensure that the nested properties are set vary by culture if the parent is + // This is because it's perfectly valid to have a nested property type that's set to invariant even if the parent varies. + // For instance in a block list, the list it self can vary, but the elements can be invariant, at the same time. + if (culture is not null) + { + propertyType.Variations |= ContentVariation.Culture; + } + + if (segment is not null) + { + propertyType.Variations |= ContentVariation.Segment; + } + + return propertyType; + }) + .ToDictionary(x => x.Alias); + + result.AddRange(GetNestedResults( + $"{property.Alias}.items[{index}]", + culture, + segment, + published, + propertyTypeDictionary, + rawData, + availableCultures, + contentTypeDictionary)); + + index++; + } + + return RenameKeysToEnsureRawSegmentsIsAPrefix(result); + } + + /// + /// Rename keys that count the RAW-constant, to ensure the RAW-constant is a prefix. + /// + private IEnumerable RenameKeysToEnsureRawSegmentsIsAPrefix( + List indexContent) + { + foreach (IndexValue indexValue in indexContent) + { + // Tests if key includes the RawFieldPrefix and it is not in the start + if (indexValue.FieldName.Substring(1).Contains(UmbracoExamineFieldNames.RawFieldPrefix)) + { + indexValue.FieldName = UmbracoExamineFieldNames.RawFieldPrefix + + indexValue.FieldName.Replace(UmbracoExamineFieldNames.RawFieldPrefix, string.Empty); + } + } + + return indexContent; + } + + /// + /// Get the data items of a parent item. E.g. block list have contentData. + /// + protected abstract IEnumerable GetDataItems(TSerialized input, bool published); + + /// + /// Unwraps block item data as data items. + /// + protected IEnumerable GetDataItems(IList contentData, IList expose, bool published) + { + if (published is false) + { + return contentData.Select(ToRawData); + } + + var indexData = new List(); + foreach (BlockItemData blockItemData in contentData) + { + var exposedCultures = expose + .Where(e => e.ContentKey == blockItemData.Key) + .Select(e => e.Culture) + .ToArray(); + + if (exposedCultures.Any() is false) + { + continue; + } + + if (exposedCultures.Contains(null) + || exposedCultures.ContainsAll(blockItemData.Values.Select(v => v.Culture))) + { + indexData.Add(ToRawData(blockItemData)); + continue; + } + + indexData.Add( + ToRawData( + blockItemData.ContentTypeKey, + blockItemData.Values.Where(value => value.Culture is null || exposedCultures.Contains(value.Culture)) + ) + ); + } + + return indexData; + } + + /// + /// Index a key with the name of the property, using the relevant content of all the children. + /// + protected override IEnumerable HandleResume( + List indexedContent, + IProperty property, + string? culture, + string? segment, + bool published) + { + var indexedCultures = indexedContent + .DistinctBy(v => v.Culture) + .Select(v => v.Culture) + .WhereNotNull() + .ToArray(); + var cultures = indexedCultures.Any() + ? indexedCultures + : new string?[] { culture }; + + return cultures.Select(c => new IndexValue + { + Culture = c, FieldName = property.Alias, Values = [GetResumeFromAllContent(indexedContent, c)] + }); + } + + /// + /// Gets a resume as string of all the content in this nested type. + /// + /// All the indexed content for this property. + /// the string with all relevant content from + private static string GetResumeFromAllContent(List indexedContent, string? culture) + { + var stringBuilder = new StringBuilder(); + foreach (IndexValue indexValue in indexedContent.Where(v => v.Culture == culture || v.Culture is null)) + { + // Ignore Raw fields + if (indexValue.FieldName.Contains(UmbracoExamineFieldNames.RawFieldPrefix)) + { + continue; + } + + foreach (var value in indexValue.Values) + { + if (value is not null) + { + stringBuilder.AppendLine(value.ToString()); + } + } + } + + return stringBuilder.ToString(); + } + + /// + /// Gets the content to index for the nested type. E.g. Block list, Nested Content, etc.. + /// + private IEnumerable GetNestedResults( + string keyPrefix, + string? culture, + string? segment, + bool published, + IDictionary propertyTypeDictionary, + RawDataItem rawData, + IEnumerable availableCultures, + IDictionary contentTypeDictionary) + { + foreach (RawPropertyData rawPropertyData in rawData.Properties) + { + if (propertyTypeDictionary.TryGetValue(rawPropertyData.Alias, out IPropertyType? propertyType)) + { + IDataEditor? editor = _propertyEditorCollection[propertyType.PropertyEditorAlias]; + if (editor is null) + { + continue; + } + + IProperty subProperty = new Property(propertyType); + IEnumerable indexValues = null!; + + var propertyCulture = rawPropertyData.Culture ?? culture; + + if (propertyType.VariesByCulture() && propertyCulture is null) + { + foreach (var availableCulture in availableCultures) + { + subProperty.SetValue(rawPropertyData.Value, availableCulture, segment); + if (published) + { + subProperty.PublishValues(availableCulture, segment ?? "*"); + } + indexValues = + editor.PropertyIndexValueFactory.GetIndexValues(subProperty, availableCulture, segment, published, availableCultures, contentTypeDictionary); + } + } + else + { + subProperty.SetValue(rawPropertyData.Value, propertyCulture, segment); + if (published) + { + subProperty.PublishValues(propertyCulture ?? "*", segment ?? "*"); + } + indexValues = editor.PropertyIndexValueFactory.GetIndexValues(subProperty, propertyCulture, segment, published, availableCultures, contentTypeDictionary); + } + + var rawDataCultures = rawData.Properties.Select(property => property.Culture).Distinct().WhereNotNull().ToArray(); + foreach (IndexValue indexValue in indexValues) + { + indexValue.FieldName = $"{keyPrefix}.{indexValue.FieldName}"; + + if (indexValue.Culture is null && rawDataCultures.Any()) + { + foreach (var rawDataCulture in rawDataCultures) + { + yield return new IndexValue + { + Culture = rawDataCulture, + FieldName = indexValue.FieldName, + Values = indexValue.Values + }; + } + } + else + { + indexValue.Culture = rawDataCultures.Any() ? indexValue.Culture : null; + yield return indexValue; + } + } + } + } + } + + private RawDataItem ToRawData(BlockItemData blockItemData) + => ToRawData(blockItemData.ContentTypeKey, blockItemData.Values); + + private RawDataItem ToRawData(Guid contentTypeKey, IEnumerable values) + => new() + { + ContentTypeKey = contentTypeKey, + Properties = values.Select(value => new RawPropertyData + { + Alias = value.Alias, + Culture = value.Culture, + Value = value.Value + }) + }; + + protected class RawDataItem + { + public required Guid ContentTypeKey { get; init; } + + public required IEnumerable Properties { get; init; } + } + + protected class RawPropertyData + { + public required string Alias { get; init; } + + public required object? Value { get; init; } + + public required string? Culture { get; init; } + } +} diff --git a/src/Umbraco.Infrastructure/PropertyEditors/NestedPropertyIndexValueFactoryBase.cs b/src/Umbraco.Infrastructure/PropertyEditors/NestedPropertyIndexValueFactoryBase.cs deleted file mode 100644 index 40e691b8d7..0000000000 --- a/src/Umbraco.Infrastructure/PropertyEditors/NestedPropertyIndexValueFactoryBase.cs +++ /dev/null @@ -1,222 +0,0 @@ -using System.Text; -using Microsoft.Extensions.Options; -using Umbraco.Cms.Core.Configuration.Models; -using Umbraco.Cms.Core.DependencyInjection; -using Umbraco.Cms.Core.Models; -using Umbraco.Cms.Core.Serialization; -using Umbraco.Cms.Infrastructure.Examine; -using Umbraco.Extensions; - -namespace Umbraco.Cms.Core.PropertyEditors; - -internal abstract class NestedPropertyIndexValueFactoryBase : JsonPropertyIndexValueFactoryBase -{ - private readonly PropertyEditorCollection _propertyEditorCollection; - - protected NestedPropertyIndexValueFactoryBase( - PropertyEditorCollection propertyEditorCollection, - IJsonSerializer jsonSerializer, - IOptionsMonitor indexingSettings) - : base(jsonSerializer, indexingSettings) - { - _propertyEditorCollection = propertyEditorCollection; - } - - protected override IEnumerable>> Handle( - TSerialized deserializedPropertyValue, - IProperty property, - string? culture, - string? segment, - bool published, - IEnumerable availableCultures, - IDictionary contentTypeDictionary) - { - var result = new List>>(); - - var index = 0; - foreach (TItem nestedContentRowValue in GetDataItems(deserializedPropertyValue)) - { - IContentType? contentType = GetContentTypeOfNestedItem(nestedContentRowValue, contentTypeDictionary); - - if (contentType is null) - { - continue; - } - - var propertyTypeDictionary = - contentType - .CompositionPropertyGroups - .SelectMany(x => x.PropertyTypes!) - .Select(propertyType => - { - // We want to ensure that the nested properties are set vary by culture if the parent is - // This is because it's perfectly valid to have a nested property type that's set to invariant even if the parent varies. - // For instance in a block list, the list it self can vary, but the elements can be invariant, at the same time. - if (culture is not null) - { - propertyType.Variations |= ContentVariation.Culture; - } - - if (segment is not null) - { - propertyType.Variations |= ContentVariation.Segment; - } - - return propertyType; - }) - .ToDictionary(x => x.Alias); - - result.AddRange(GetNestedResults( - $"{property.Alias}.items[{index}]", - culture, - segment, - published, - propertyTypeDictionary, - nestedContentRowValue, - availableCultures, - contentTypeDictionary)); - - index++; - } - - return RenameKeysToEnsureRawSegmentsIsAPrefix(result); - } - - /// - /// Rename keys that count the RAW-constant, to ensure the RAW-constant is a prefix. - /// - private IEnumerable>> RenameKeysToEnsureRawSegmentsIsAPrefix( - List>> indexContent) - { - foreach (KeyValuePair> indexedKeyValuePair in indexContent) - { - // Tests if key includes the RawFieldPrefix and it is not in the start - if (indexedKeyValuePair.Key.Substring(1).Contains(UmbracoExamineFieldNames.RawFieldPrefix)) - { - var newKey = UmbracoExamineFieldNames.RawFieldPrefix + - indexedKeyValuePair.Key.Replace(UmbracoExamineFieldNames.RawFieldPrefix, string.Empty); - yield return new KeyValuePair>(newKey, indexedKeyValuePair.Value); - } - else - { - yield return indexedKeyValuePair; - } - } - } - - /// - /// Gets the content type using the nested item. - /// - protected abstract IContentType? GetContentTypeOfNestedItem(TItem nestedItem, IDictionary contentTypeDictionary); - - /// - /// Gets the raw data from a nested item. - /// - protected abstract IDictionary GetRawProperty(TItem nestedItem); - - /// - /// Get the data times of a parent item. E.g. block list have contentData. - /// - protected abstract IEnumerable GetDataItems(TSerialized input); - - /// - /// Index a key with the name of the property, using the relevant content of all the children. - /// - protected override IEnumerable>> HandleResume( - List>> indexedContent, - IProperty property, - string? culture, - string? segment, - bool published) - { - yield return new KeyValuePair>( - property.Alias, - GetResumeFromAllContent(indexedContent).Yield()); - } - - /// - /// Gets a resume as string of all the content in this nested type. - /// - /// All the indexed content for this property. - /// the string with all relevant content from - private static string GetResumeFromAllContent(List>> indexedContent) - { - var stringBuilder = new StringBuilder(); - foreach ((var indexKey, IEnumerable? indexedValue) in indexedContent) - { - // Ignore Raw fields - if (indexKey.Contains(UmbracoExamineFieldNames.RawFieldPrefix)) - { - continue; - } - - foreach (var value in indexedValue) - { - if (value is not null) - { - stringBuilder.AppendLine(value.ToString()); - } - } - } - - return stringBuilder.ToString(); - } - - /// - /// Gets the content to index for the nested type. E.g. Block list, Nested Content, etc.. - /// - private IEnumerable>> GetNestedResults( - string keyPrefix, - string? culture, - string? segment, - bool published, - IDictionary propertyTypeDictionary, - TItem nestedContentRowValue, - IEnumerable availableCultures, - IDictionary contentTypeDictionary) - { - foreach ((var propertyAlias, var propertyValue) in GetRawProperty(nestedContentRowValue)) - { - if (propertyTypeDictionary.TryGetValue(propertyAlias, out IPropertyType? propertyType)) - { - IDataEditor? editor = _propertyEditorCollection[propertyType.PropertyEditorAlias]; - if (editor is null) - { - continue; - } - - IProperty subProperty = new Property(propertyType); - IEnumerable>> indexValues = null!; - - if (propertyType.VariesByCulture() && culture is null) - { - foreach (var availableCulture in availableCultures) - { - subProperty.SetValue(propertyValue, availableCulture, segment); - if (published) - { - subProperty.PublishValues(availableCulture, segment ?? "*"); - } - indexValues = - editor.PropertyIndexValueFactory.GetIndexValues(subProperty, availableCulture, segment, published, availableCultures, contentTypeDictionary); - } - } - else - { - subProperty.SetValue(propertyValue, culture, segment); - if (published) - { - subProperty.PublishValues(culture ?? "*", segment ?? "*"); - } - indexValues = editor.PropertyIndexValueFactory.GetIndexValues(subProperty, culture, segment, published, availableCultures, contentTypeDictionary); - } - - foreach ((var nestedAlias, IEnumerable nestedValue) in indexValues) - { - yield return new KeyValuePair>( - $"{keyPrefix}.{nestedAlias}", nestedValue!); - } - } - } - } -} diff --git a/src/Umbraco.Infrastructure/PropertyEditors/RichTextPropertyIndexValueFactory.cs b/src/Umbraco.Infrastructure/PropertyEditors/RichTextPropertyIndexValueFactory.cs index e2236e2976..0eb1ee257a 100644 --- a/src/Umbraco.Infrastructure/PropertyEditors/RichTextPropertyIndexValueFactory.cs +++ b/src/Umbraco.Infrastructure/PropertyEditors/RichTextPropertyIndexValueFactory.cs @@ -2,15 +2,13 @@ using Microsoft.Extensions.Options; using Umbraco.Cms.Core.Configuration.Models; using Umbraco.Cms.Core.Models; -using Umbraco.Cms.Core.Models.Blocks; using Umbraco.Cms.Core.Serialization; -using Umbraco.Cms.Core.Services; using Umbraco.Cms.Infrastructure.Examine; using Umbraco.Extensions; namespace Umbraco.Cms.Core.PropertyEditors; -internal class RichTextPropertyIndexValueFactory : NestedPropertyIndexValueFactoryBase, IRichTextPropertyIndexValueFactory +internal class RichTextPropertyIndexValueFactory : BlockValuePropertyIndexValueFactoryBase, IRichTextPropertyIndexValueFactory { private readonly IJsonSerializer _jsonSerializer; private readonly ILogger _logger; @@ -26,18 +24,7 @@ internal class RichTextPropertyIndexValueFactory : NestedPropertyIndexValueFacto _logger = logger; } - [Obsolete("Use constructor that doesn't take IContentTypeService, scheduled for removal in V15")] - public RichTextPropertyIndexValueFactory( - PropertyEditorCollection propertyEditorCollection, - IJsonSerializer jsonSerializer, - IOptionsMonitor indexingSettings, - IContentTypeService contentTypeService, - ILogger logger) - : this(propertyEditorCollection, jsonSerializer, indexingSettings, logger) - { - } - - public new IEnumerable>> GetIndexValues( + public override IEnumerable GetIndexValues( IProperty property, string? culture, string? segment, @@ -48,33 +35,101 @@ internal class RichTextPropertyIndexValueFactory : NestedPropertyIndexValueFacto var val = property.GetValue(culture, segment, published); if (RichTextPropertyEditorHelper.TryParseRichTextEditorValue(val, _jsonSerializer, _logger, out RichTextEditorValue? richTextEditorValue) is false) { - yield break; + return []; + } + + // always index the "raw" value + var indexValues = new List + { + new IndexValue + { + Culture = culture, + FieldName = $"{UmbracoExamineFieldNames.RawFieldPrefix}{property.Alias}", + Values = [richTextEditorValue.Markup] + } + }; + + // the actual content (RTE content without markup, i.e. the actual words) must be indexed under the property alias + var richTextWithoutMarkup = richTextEditorValue.Markup.StripHtml(); + if (richTextEditorValue.Blocks?.ContentData.Any() is not true) + { + // no blocks; index the content for the culture and be done with it + indexValues.Add(new IndexValue + { + Culture = culture, + FieldName = property.Alias, + Values = [richTextWithoutMarkup] + }); + return indexValues; } // the "blocks values resume" (the combined searchable text values from all blocks) is stored as a string value under the property alias by the base implementation - var blocksIndexValues = base.GetIndexValues(property, culture, segment, published, availableCultures, contentTypeDictionary).ToDictionary(pair => pair.Key, pair => pair.Value); - var blocksIndexValuesResume = blocksIndexValues.TryGetValue(property.Alias, out IEnumerable? blocksIndexValuesResumeValue) - ? blocksIndexValuesResumeValue.FirstOrDefault() as string - : null; + var blocksIndexValuesResumes = base + .GetIndexValues(property, culture, segment, published, availableCultures, contentTypeDictionary) + .Where(value => value.FieldName == property.Alias) + .GroupBy(value => value.Culture?.ToLowerInvariant()) + .Select(group => new + { + Culture = group.Key, + Resume = string.Join(Environment.NewLine, group.Select(v => v.Values.FirstOrDefault() as string)) + }) + .ToArray(); - // index the stripped HTML values combined with "blocks values resume" value - yield return new KeyValuePair>( - property.Alias, - new object[] { $"{richTextEditorValue.Markup.StripHtml()} {blocksIndexValuesResume}" }); + // is this RTE sat on culture variant content? + if (culture is not null) + { + // yes, append the "block values resume" for the specific culture only (if present) + var blocksResume = blocksIndexValuesResumes + .FirstOrDefault(r => r.Culture.InvariantEquals(culture))? + .Resume; - // store the raw value - yield return new KeyValuePair>( - $"{UmbracoExamineFieldNames.RawFieldPrefix}{property.Alias}", new object[] { richTextEditorValue.Markup }); + indexValues.Add(new IndexValue + { + Culture = culture, + FieldName = property.Alias, + Values = [$"{richTextWithoutMarkup}{Environment.NewLine}{blocksResume}"] + }); + return indexValues; + } + + // is there an invariant "block values resume"? this might happen for purely invariant blocks or in a culture invariant context + var invariantResume = blocksIndexValuesResumes + .FirstOrDefault(r => r.Culture is null) + ?.Resume; + if (invariantResume != null) + { + // yes, append the invariant "block values resume" + indexValues.Add(new IndexValue + { + Culture = culture, + FieldName = property.Alias, + Values = [$"{richTextWithoutMarkup}{Environment.NewLine}{invariantResume}"] + }); + return indexValues; + } + + // at this point we have encountered block level variance - add explicit index values for all "block values resume" cultures found + indexValues.AddRange(blocksIndexValuesResumes.Select(resume => + new IndexValue + { + Culture = resume.Culture, + FieldName = property.Alias, + Values = [$"{richTextWithoutMarkup}{Environment.NewLine}{resume.Resume}"] + })); + + // if one or more cultures did not have any (exposed) blocks, ensure that the RTE content is still indexed for those cultures + IEnumerable missingBlocksResumeCultures = availableCultures.Except(blocksIndexValuesResumes.Select(r => r.Culture), StringComparer.CurrentCultureIgnoreCase); + indexValues.AddRange(missingBlocksResumeCultures.Select(missingResumeCulture => + new IndexValue + { + Culture = missingResumeCulture, + FieldName = property.Alias, + Values = [richTextWithoutMarkup] + })); + + return indexValues; } - protected override IContentType? GetContentTypeOfNestedItem(BlockItemData nestedItem, IDictionary contentTypeDictionary) - => contentTypeDictionary.TryGetValue(nestedItem.ContentTypeKey, out var result) ? result : null; - - protected override IDictionary GetRawProperty(BlockItemData blockItemData) - => blockItemData.Values - .Where(p => p.Culture is null && p.Segment is null) - .ToDictionary(p => p.Alias, p => p.Value); - - protected override IEnumerable GetDataItems(RichTextEditorValue input) - => input.Blocks?.ContentData ?? new List(); + protected override IEnumerable GetDataItems(RichTextEditorValue input, bool published) + => GetDataItems(input.Blocks?.ContentData ?? [], input.Blocks?.Expose ?? [], published); } diff --git a/tests/Umbraco.Tests.Integration/Umbraco.Infrastructure/PropertyEditors/BlockListElementLevelVariationTests.Indexing.cs b/tests/Umbraco.Tests.Integration/Umbraco.Infrastructure/PropertyEditors/BlockListElementLevelVariationTests.Indexing.cs new file mode 100644 index 0000000000..6058676f69 --- /dev/null +++ b/tests/Umbraco.Tests.Integration/Umbraco.Infrastructure/PropertyEditors/BlockListElementLevelVariationTests.Indexing.cs @@ -0,0 +1,399 @@ +using NUnit.Framework; +using Umbraco.Cms.Core; +using Umbraco.Cms.Core.Models; +using Umbraco.Cms.Core.Models.Blocks; +using Umbraco.Cms.Tests.Common.Builders; +using Umbraco.Cms.Tests.Common.Builders.Extensions; + +namespace Umbraco.Cms.Tests.Integration.Umbraco.Infrastructure.PropertyEditors; + +public partial class BlockListElementLevelVariationTests +{ + [Test] + public async Task Can_Index_Cultures_Independently_Invariant_Blocks() + { + var elementType = CreateElementType(ContentVariation.Culture); + var blockListDataType = await CreateBlockListDataType(elementType); + var contentType = CreateContentType(ContentVariation.Culture, blockListDataType); + + var content = CreateContent( + contentType, + elementType, + new List + { + new() { Alias = "invariantText", Value = "The invariant content value" }, + new() { Alias = "variantText", Value = "The content value (en-US)", Culture = "en-US" }, + new() { Alias = "variantText", Value = "The content value (da-DK)", Culture = "da-DK" }, + }, + new List + { + new() { Alias = "invariantText", Value = "The invariant settings value" }, + new() { Alias = "variantText", Value = "The settings value (en-US)", Culture = "en-US" }, + new() { Alias = "variantText", Value = "The settings value (da-DK)", Culture = "da-DK" }, + }, + true); + + var editor = blockListDataType.Editor!; + var indexValues = editor.PropertyIndexValueFactory.GetIndexValues( + content.Properties["blocks"]!, + culture: null, + segment: null, + published: true, + availableCultures: ["en-US", "da-DK"], + contentTypeDictionary: new Dictionary + { + { elementType.Key, elementType }, { contentType.Key, contentType } + }); + + Assert.AreEqual(2, indexValues.Count()); + + AssertIndexValues("en-US"); + AssertIndexValues("da-DK"); + + void AssertIndexValues(string culture) + { + var indexValue = indexValues.FirstOrDefault(v => v.Culture == culture); + Assert.IsNotNull(indexValue); + Assert.AreEqual(1, indexValue.Values.Count()); + var indexedValue = indexValue.Values.First() as string; + Assert.IsNotNull(indexedValue); + var values = indexedValue.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries); + Assert.AreEqual(2, values.Length); + Assert.Contains($"The content value ({culture})", values); + Assert.Contains("The invariant content value", values); + } + } + + [Test] + public async Task Can_Index_Cultures_Independently_Variant_Blocks() + { + var elementType = CreateElementType(ContentVariation.Culture); + var blockListDataType = await CreateBlockListDataType(elementType); + var contentType = CreateContentType(ContentVariation.Culture, blockListDataType, ContentVariation.Culture); + + var content = CreateContent( + contentType, + elementType, + new [] + { + new BlockProperty( + new List + { + new() { Alias = "invariantText", Value = "en-US invariantText content value" }, + new() { Alias = "variantText", Value = "en-US variantText content value" } + }, + new List + { + new() { Alias = "invariantText", Value = "en-US invariantText settings value" }, + new() { Alias = "variantText", Value = "en-US variantText settings value" } + }, + "en-US", + null), + new BlockProperty( + new List + { + new() { Alias = "invariantText", Value = "da-DK invariantText content value" }, + new() { Alias = "variantText", Value = "da-DK variantText content value" } + }, + new List + { + new() { Alias = "invariantText", Value = "da-DK invariantText settings value" }, + new() { Alias = "variantText", Value = "da-DK variantText settings value" } + }, + "da-DK", + null) + }, + true); + + AssertIndexValues("en-US"); + AssertIndexValues("da-DK"); + + void AssertIndexValues(string culture) + { + var editor = blockListDataType.Editor!; + var indexValues = editor.PropertyIndexValueFactory.GetIndexValues( + content.Properties["blocks"]!, + culture: culture, + segment: null, + published: true, + availableCultures: ["en-US", "da-DK"], + contentTypeDictionary: new Dictionary + { + { elementType.Key, elementType }, { contentType.Key, contentType } + }); + + Assert.AreEqual(1, indexValues.Count()); + + var indexValue = indexValues.FirstOrDefault(v => v.Culture == culture); + Assert.IsNotNull(indexValue); + Assert.AreEqual(1, indexValue.Values.Count()); + var indexedValue = indexValue.Values.First() as string; + Assert.IsNotNull(indexedValue); + Assert.AreEqual($"{culture} invariantText content value {culture} variantText content value", TrimAndStripNewlines(indexedValue)); + } + } + + [TestCase(true)] + [TestCase(false)] + public async Task Can_Handle_Unexposed_Blocks(bool published) + { + var elementType = CreateElementType(ContentVariation.Culture); + var blockListDataType = await CreateBlockListDataType(elementType); + var contentType = CreateContentType(ContentVariation.Culture, blockListDataType); + + var content = CreateContent(contentType, elementType, [], false); + var blockListValue = BlockListPropertyValue( + elementType, + [ + ( + Guid.NewGuid(), + Guid.NewGuid(), + new BlockProperty( + new List { + new() { Alias = "invariantText", Value = "#1: The invariant content value" }, + new() { Alias = "variantText", Value = "#1: The content value in English", Culture = "en-US" }, + new() { Alias = "variantText", Value = "#1: The content value in Danish", Culture = "da-DK" } + }, + [], + null, + null + ) + ), + ( + Guid.NewGuid(), + Guid.NewGuid(), + new BlockProperty( + new List { + new() { Alias = "invariantText", Value = "#2: The invariant content value" }, + new() { Alias = "variantText", Value = "#2: The content value in English", Culture = "en-US" }, + new() { Alias = "variantText", Value = "#2: The content value in Danish", Culture = "da-DK" } + }, + [], + null, + null + ) + ) + ] + ); + + // only expose the first block in English and the second block in Danish (to make a difference between published and unpublished index values) + blockListValue.Expose = + [ + new() { ContentKey = blockListValue.ContentData[0].Key, Culture = "en-US" }, + new() { ContentKey = blockListValue.ContentData[1].Key, Culture = "da-DK" }, + ]; + + content.Properties["blocks"]!.SetValue(JsonSerializer.Serialize(blockListValue)); + ContentService.Save(content); + + PublishContent(content, contentType, ["en-US", "da-DK"]); + + var editor = blockListDataType.Editor!; + var indexValues = editor.PropertyIndexValueFactory.GetIndexValues( + content.Properties["blocks"]!, + culture: null, + segment: null, + published: published, + availableCultures: ["en-US", "da-DK"], + contentTypeDictionary: new Dictionary + { + { elementType.Key, elementType }, { contentType.Key, contentType } + }); + + Assert.AreEqual(2, indexValues.Count()); + var indexValue = indexValues.FirstOrDefault(v => v.Culture == "da-DK"); + Assert.IsNotNull(indexValue); + Assert.AreEqual(1, indexValue.Values.Count()); + var indexedValue = indexValue.Values.First() as string; + Assert.IsNotNull(indexedValue); + if (published) + { + Assert.AreEqual("#2: The invariant content value #2: The content value in Danish", TrimAndStripNewlines(indexedValue)); + } + else + { + Assert.AreEqual("#1: The invariant content value #1: The content value in Danish #2: The invariant content value #2: The content value in Danish", TrimAndStripNewlines(indexedValue)); + } + + indexValue = indexValues.FirstOrDefault(v => v.Culture == "en-US"); + Assert.IsNotNull(indexValue); + Assert.AreEqual(1, indexValue.Values.Count()); + indexedValue = indexValue.Values.First() as string; + Assert.IsNotNull(indexedValue); + if (published) + { + Assert.AreEqual("#1: The content value in English #1: The invariant content value", TrimAndStripNewlines(indexedValue)); + } + else + { + Assert.AreEqual("#1: The invariant content value #1: The content value in English #2: The invariant content value #2: The content value in English", TrimAndStripNewlines(indexedValue)); + } + } + + [TestCase(ContentVariation.Nothing)] + [TestCase(ContentVariation.Culture)] + public async Task Can_Index_Invariant(ContentVariation elementTypeVariation) + { + var elementType = CreateElementType(elementTypeVariation); + var blockListDataType = await CreateBlockListDataType(elementType); + var contentType = CreateContentType(ContentVariation.Nothing, blockListDataType); + + var content = CreateContent( + contentType, + elementType, + new List + { + new() { Alias = "invariantText", Value = "The invariant content value" }, + new() { Alias = "variantText", Value = "Another invariant content value" } + }, + new List + { + new() { Alias = "invariantText", Value = "The invariant settings value" }, + new() { Alias = "variantText", Value = "Another invariant settings value" } + }, + true); + + var editor = blockListDataType.Editor!; + var indexValues = editor.PropertyIndexValueFactory.GetIndexValues( + content.Properties["blocks"]!, + culture: null, + segment: null, + published: true, + availableCultures: ["en-US"], + contentTypeDictionary: new Dictionary + { + { elementType.Key, elementType }, { contentType.Key, contentType } + }); + + Assert.AreEqual(1, indexValues.Count()); + var indexValue = indexValues.FirstOrDefault(v => v.Culture is null); + Assert.IsNotNull(indexValue); + Assert.AreEqual(1, indexValue.Values.Count()); + var indexedValue = indexValue.Values.First() as string; + Assert.IsNotNull(indexedValue); + var values = indexedValue.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries); + Assert.AreEqual(2, values.Length); + Assert.Contains("The invariant content value", values); + Assert.Contains("Another invariant content value", values); + } + + [Test] + public async Task Can_Index_Cultures_Independently_Nested_Invariant_Blocks() + { + var nestedElementType = CreateElementType(ContentVariation.Culture); + var nestedBlockListDataType = await CreateBlockListDataType(nestedElementType); + + var rootElementType = new ContentTypeBuilder() + .WithAlias("myRootElementType") + .WithName("My Root Element Type") + .WithIsElement(true) + .WithContentVariation(ContentVariation.Culture) + .AddPropertyType() + .WithAlias("invariantText") + .WithName("Invariant text") + .WithDataTypeId(Constants.DataTypes.Textbox) + .WithPropertyEditorAlias(Constants.PropertyEditors.Aliases.TextBox) + .WithValueStorageType(ValueStorageType.Nvarchar) + .WithVariations(ContentVariation.Nothing) + .Done() + .AddPropertyType() + .WithAlias("variantText") + .WithName("Variant text") + .WithDataTypeId(Constants.DataTypes.Textbox) + .WithPropertyEditorAlias(Constants.PropertyEditors.Aliases.TextBox) + .WithValueStorageType(ValueStorageType.Nvarchar) + .WithVariations(ContentVariation.Culture) + .Done() + .AddPropertyType() + .WithAlias("nestedBlocks") + .WithName("Nested blocks") + .WithDataTypeId(nestedBlockListDataType.Id) + .WithPropertyEditorAlias(Constants.PropertyEditors.Aliases.BlockList) + .WithValueStorageType(ValueStorageType.Ntext) + .WithVariations(ContentVariation.Nothing) + .Done() + .Build(); + ContentTypeService.Save(rootElementType); + var rootBlockListDataType = await CreateBlockListDataType(rootElementType); + var contentType = CreateContentType(ContentVariation.Culture, rootBlockListDataType); + + var nestedElementContentKey = Guid.NewGuid(); + var nestedElementSettingsKey = Guid.NewGuid(); + var content = CreateContent( + contentType, + rootElementType, + new List + { + new() + { + Alias = "nestedBlocks", + Value = BlockListPropertyValue( + nestedElementType, + nestedElementContentKey, + nestedElementSettingsKey, + new BlockProperty( + new List + { + new() { Alias = "invariantText", Value = "The first nested invariant content value" }, + new() { Alias = "variantText", Value = "The first nested content value in English", Culture = "en-US" }, + new() { Alias = "variantText", Value = "The first nested content value in Danish", Culture = "da-DK" }, + }, + new List + { + new() { Alias = "invariantText", Value = "The first nested invariant settings value" }, + new() { Alias = "variantText", Value = "The first nested settings value in English", Culture = "en-US" }, + new() { Alias = "variantText", Value = "The first nested settings value in Danish", Culture = "da-DK" }, + }, + null, + null)) + }, + new() { Alias = "invariantText", Value = "The first root invariant content value" }, + new() { Alias = "variantText", Value = "The first root content value in English", Culture = "en-US" }, + new() { Alias = "variantText", Value = "The first root content value in Danish", Culture = "da-DK" }, + }, + [], + true); + + var editor = rootBlockListDataType.Editor!; + var indexValues = editor.PropertyIndexValueFactory.GetIndexValues( + content.Properties["blocks"]!, + culture: null, + segment: null, + published: true, + availableCultures: ["en-US"], + contentTypeDictionary: new Dictionary + { + { nestedElementType.Key, nestedElementType }, { rootElementType.Key, rootElementType }, { contentType.Key, contentType } + }); + Assert.AreEqual(2, indexValues.Count()); + + AssertIndexedValues( + "en-US", + "The first root invariant content value", + "The first root content value in English", + "The first nested invariant content value", + "The first nested content value in English"); + + AssertIndexedValues( + "da-DK", + "The first root invariant content value", + "The first root content value in Danish", + "The first nested invariant content value", + "The first nested content value in Danish"); + + void AssertIndexedValues(string culture, params string[] expectedIndexedValues) + { + var indexValue = indexValues.FirstOrDefault(v => v.Culture == culture); + Assert.IsNotNull(indexValue); + Assert.AreEqual(1, indexValue.Values.Count()); + var indexedValue = indexValue.Values.First() as string; + Assert.IsNotNull(indexedValue); + var values = indexedValue.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries); + Assert.AreEqual(expectedIndexedValues.Length, values.Length); + Assert.IsTrue(values.ContainsAll(expectedIndexedValues)); + } + } + + private string TrimAndStripNewlines(string value) + => value.Replace(Environment.NewLine, " ").Trim(); +} diff --git a/tests/Umbraco.Tests.Integration/Umbraco.Infrastructure/PropertyEditors/PropertyIndexValueFactoryTests.cs b/tests/Umbraco.Tests.Integration/Umbraco.Infrastructure/PropertyEditors/PropertyIndexValueFactoryTests.cs index ac386f10d2..b97af90ee1 100644 --- a/tests/Umbraco.Tests.Integration/Umbraco.Infrastructure/PropertyEditors/PropertyIndexValueFactoryTests.cs +++ b/tests/Umbraco.Tests.Integration/Umbraco.Infrastructure/PropertyEditors/PropertyIndexValueFactoryTests.cs @@ -83,12 +83,13 @@ public class PropertyIndexValueFactoryTests : UmbracoIntegrationTest contentTypeDictionary: new Dictionary { { elementType.Key, elementType }, { contentType.Key, contentType } - }).ToDictionary(); + }); - Assert.IsTrue(indexValues.TryGetValue("bodyText", out var bodyTextIndexValues)); + var indexValue = indexValues.FirstOrDefault(v => v.FieldName == "bodyText"); + Assert.IsNotNull(indexValue); - Assert.AreEqual(1, bodyTextIndexValues.Count()); - var bodyTextIndexValue = bodyTextIndexValues.First() as string; + Assert.AreEqual(1, indexValue.Values.Count()); + var bodyTextIndexValue = indexValue.Values.First() as string; Assert.IsNotNull(bodyTextIndexValue); Assert.Multiple(() => @@ -122,12 +123,13 @@ public class PropertyIndexValueFactoryTests : UmbracoIntegrationTest contentTypeDictionary: new Dictionary { { contentType.Key, contentType } - }).ToDictionary(); + }); - Assert.IsTrue(indexValues.TryGetValue("bodyText", out var bodyTextIndexValues)); + var indexValue = indexValues.FirstOrDefault(v => v.FieldName == "bodyText"); + Assert.IsNotNull(indexValue); - Assert.AreEqual(1, bodyTextIndexValues.Count()); - var bodyTextIndexValue = bodyTextIndexValues.First() as string; + Assert.AreEqual(1, indexValue.Values.Count()); + var bodyTextIndexValue = indexValue.Values.First() as string; Assert.IsNotNull(bodyTextIndexValue); Assert.IsTrue(bodyTextIndexValue.Contains("This is some markup")); } @@ -205,12 +207,13 @@ public class PropertyIndexValueFactoryTests : UmbracoIntegrationTest contentTypeDictionary: new Dictionary { { elementType.Key, elementType }, { contentType.Key, contentType } - }).ToDictionary(); + }); - Assert.IsTrue(indexValues.TryGetValue("blocks", out var blocksIndexValues)); + var indexValue = indexValues.FirstOrDefault(v => v.FieldName == "blocks"); + Assert.IsNotNull(indexValue); - Assert.AreEqual(1, blocksIndexValues.Count()); - var blockIndexValue = blocksIndexValues.First() as string; + Assert.AreEqual(1, indexValue.Values.Count()); + var blockIndexValue = indexValue.Values.First() as string; Assert.IsNotNull(blockIndexValue); Assert.Multiple(() => @@ -333,12 +336,13 @@ public class PropertyIndexValueFactoryTests : UmbracoIntegrationTest contentTypeDictionary: new Dictionary { { elementType.Key, elementType }, { contentType.Key, contentType } - }).ToDictionary(); + }); - Assert.IsTrue(indexValues.TryGetValue("blocks", out var blocksIndexValues)); + var indexValue = indexValues.FirstOrDefault(v => v.FieldName == "blocks"); + Assert.IsNotNull(indexValue); - Assert.AreEqual(1, blocksIndexValues.Count()); - var blockIndexValue = blocksIndexValues.First() as string; + Assert.AreEqual(1, indexValue.Values.Count()); + var blockIndexValue = indexValue.Values.First() as string; Assert.IsNotNull(blockIndexValue); Assert.Multiple(() => diff --git a/tests/Umbraco.Tests.Integration/Umbraco.Infrastructure/PropertyEditors/RichTextElementLevelVariationTests.cs b/tests/Umbraco.Tests.Integration/Umbraco.Infrastructure/PropertyEditors/RichTextElementLevelVariationTests.cs index ad40548aac..80d02caa15 100644 --- a/tests/Umbraco.Tests.Integration/Umbraco.Infrastructure/PropertyEditors/RichTextElementLevelVariationTests.cs +++ b/tests/Umbraco.Tests.Integration/Umbraco.Infrastructure/PropertyEditors/RichTextElementLevelVariationTests.cs @@ -5,6 +5,7 @@ using Umbraco.Cms.Core.Models.Blocks; using Umbraco.Cms.Core.Models.DeliveryApi; using Umbraco.Cms.Core.PropertyEditors; using Umbraco.Cms.Core.Serialization; +using Umbraco.Cms.Infrastructure.Examine; using Umbraco.Cms.Tests.Common.Builders; using Umbraco.Cms.Tests.Common.Builders.Extensions; @@ -22,8 +23,8 @@ public class RichTextElementLevelVariationTests : BlockEditorElementVariationTes { var elementType = CreateElementType(ContentVariation.Culture); - var blockGridDataType = await CreateRichTextDataType(elementType); - var contentType = CreateContentType(blockGridDataType); + var rteDataType = await CreateRichTextDataType(elementType); + var contentType = CreateContentType(rteDataType); var richTextValue = CreateRichTextValue(elementType); var content = CreateContent(contentType, richTextValue); @@ -180,8 +181,8 @@ public class RichTextElementLevelVariationTests : BlockEditorElementVariationTes { var elementType = CreateElementType(ContentVariation.Culture); - var blockGridDataType = await CreateRichTextDataType(elementType); - var contentType = CreateContentType(blockGridDataType); + var rteDataType = await CreateRichTextDataType(elementType); + var contentType = CreateContentType(rteDataType); var richTextValue = CreateRichTextValue(elementType); var content = CreateContent(contentType, richTextValue); @@ -285,8 +286,8 @@ public class RichTextElementLevelVariationTests : BlockEditorElementVariationTes { var elementType = CreateElementType(ContentVariation.Culture); - var blockGridDataType = await CreateRichTextDataType(elementType); - var contentType = CreateContentType(blockGridDataType); + var rteDataType = await CreateRichTextDataType(elementType); + var contentType = CreateContentType(rteDataType); var richTextValue = CreateRichTextValue(elementType); var content = CreateContent(contentType, richTextValue); @@ -369,8 +370,8 @@ public class RichTextElementLevelVariationTests : BlockEditorElementVariationTes { var elementType = CreateElementType(ContentVariation.Culture); - var blockGridDataType = await CreateRichTextDataType(elementType); - var contentType = CreateContentType(blockGridDataType); + var rteDataType = await CreateRichTextDataType(elementType); + var contentType = CreateContentType(rteDataType); var richTextValue = new RichTextEditorValue { Markup = "

Markup here

", Blocks = null }; var content = CreateContent(contentType, richTextValue); @@ -398,8 +399,8 @@ public class RichTextElementLevelVariationTests : BlockEditorElementVariationTes { var elementType = CreateElementType(ContentVariation.Culture); - var blockGridDataType = await CreateRichTextDataType(elementType); - var contentType = CreateContentType(ContentVariation.Nothing, blockGridDataType); + var rteDataType = await CreateRichTextDataType(elementType); + var contentType = CreateContentType(ContentVariation.Nothing, rteDataType); var richTextValue = new RichTextEditorValue { Markup = "

Markup here

", Blocks = null }; var content = CreateContent(contentType, richTextValue); @@ -422,6 +423,247 @@ public class RichTextElementLevelVariationTests : BlockEditorElementVariationTes } } + [Test] + public async Task Can_Index_Cultures_Independently_Invariant_Blocks() + { + var elementType = CreateElementType(ContentVariation.Culture); + + var rteDataType = await CreateRichTextDataType(elementType); + var contentType = CreateContentType(rteDataType); + var richTextValue = CreateRichTextValue(elementType); + var content = CreateContent(contentType, richTextValue); + PublishContent(content, ["en-US", "da-DK"]); + + var editor = rteDataType.Editor!; + var indexValues = editor.PropertyIndexValueFactory.GetIndexValues( + content.Properties["blocks"]!, + culture: null, + segment: null, + published: true, + availableCultures: ["en-US", "da-DK"], + contentTypeDictionary: new Dictionary + { + { elementType.Key, elementType }, { contentType.Key, contentType } + }); + + Assert.AreEqual(3, indexValues.Count()); + Assert.NotNull(indexValues.FirstOrDefault(value => value.FieldName.StartsWith(UmbracoExamineFieldNames.RawFieldPrefix))); + + AssertIndexedValues( + "en-US", + "Some text.", + "More text.", + "Even more text.", + "The end.", + "#1: The first invariant content value", + "#1: The first content value in English", + "#2: The first invariant content value", + "#2: The first content value in English", + "#3: The first invariant content value", + "#3: The first content value in English"); + + AssertIndexedValues( + "da-DK", + "Some text.", + "More text.", + "Even more text.", + "The end.", + "#1: The first invariant content value", + "#1: The first content value in Danish", + "#2: The first invariant content value", + "#2: The first content value in Danish", + "#3: The first invariant content value", + "#3: The first content value in Danish"); + + void AssertIndexedValues(string culture, params string[] expectedIndexedValues) + { + var indexValue = indexValues.FirstOrDefault(v => v.Culture.InvariantEquals(culture)); + Assert.IsNotNull(indexValue); + Assert.AreEqual(1, indexValue.Values.Count()); + var indexedValue = indexValue.Values.First() as string; + Assert.IsNotNull(indexedValue); + var values = indexedValue.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries).Select(s => s.Trim()).ToArray(); + Assert.AreEqual(expectedIndexedValues.Length, values.Length); + Assert.IsTrue(values.ContainsAll(expectedIndexedValues)); + } + } + + [TestCase(true)] + [TestCase(false)] + public async Task Can_Index_With_Unexposed_Blocks(bool published) + { + var elementType = CreateElementType(ContentVariation.Culture); + + var rteDataType = await CreateRichTextDataType(elementType); + var contentType = CreateContentType(rteDataType); + var richTextValue = CreateRichTextValue(elementType); + richTextValue.Blocks!.Expose.RemoveAll(e => e.Culture == "da-DK"); + + var content = CreateContent(contentType, richTextValue); + PublishContent(content, ["en-US", "da-DK"]); + + var editor = rteDataType.Editor!; + var indexValues = editor.PropertyIndexValueFactory.GetIndexValues( + content.Properties["blocks"]!, + culture: null, + segment: null, + published: published, + availableCultures: ["en-US", "da-DK"], + contentTypeDictionary: new Dictionary + { + { elementType.Key, elementType }, { contentType.Key, contentType } + }); + + Assert.AreEqual(3, indexValues.Count()); + Assert.NotNull(indexValues.FirstOrDefault(value => value.FieldName.StartsWith(UmbracoExamineFieldNames.RawFieldPrefix))); + + if (published) + { + AssertIndexedValues( + "da-DK", + "Some text.", + "More text.", + "Even more text.", + "The end."); + } + else + { + AssertIndexedValues( + "da-DK", + "Some text.", + "More text.", + "Even more text.", + "The end.", + "#1: The first invariant content value", + "#1: The first content value in Danish", + "#2: The first invariant content value", + "#2: The first content value in Danish", + "#3: The first invariant content value", + "#3: The first content value in Danish"); + } + + AssertIndexedValues( + "en-US", + "Some text.", + "More text.", + "Even more text.", + "The end.", + "#1: The first invariant content value", + "#1: The first content value in English", + "#2: The first invariant content value", + "#2: The first content value in English", + "#3: The first invariant content value", + "#3: The first content value in English"); + + void AssertIndexedValues(string culture, params string[] expectedIndexedValues) + { + var indexValue = indexValues.FirstOrDefault(v => v.Culture.InvariantEquals(culture)); + Assert.IsNotNull(indexValue); + Assert.AreEqual(1, indexValue.Values.Count()); + var indexedValue = indexValue.Values.First() as string; + Assert.IsNotNull(indexedValue); + var values = indexedValue.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries).Select(s => s.Trim()).ToArray(); + Assert.AreEqual(expectedIndexedValues.Length, values.Length); + Assert.IsTrue(values.ContainsAll(expectedIndexedValues)); + } + } + + [TestCase(ContentVariation.Culture)] + [TestCase(ContentVariation.Nothing)] + public async Task Can_Index_Cultures_Independently_Variant_Blocks(ContentVariation elementTypeVariation) + { + var elementType = CreateElementType(elementTypeVariation); + + var rteDataType = await CreateRichTextDataType(elementType); + var contentType = CreateContentType(ContentVariation.Culture, rteDataType, ContentVariation.Culture); + + var englishRichTextValue = CreateInvariantRichTextValue("en-US"); + var danishRichTextValue = CreateInvariantRichTextValue("da-DK"); + + var content = CreateContent(contentType); + content.Properties["blocks"]!.SetValue(JsonSerializer.Serialize(englishRichTextValue), "en-US"); + content.Properties["blocks"]!.SetValue(JsonSerializer.Serialize(danishRichTextValue), "da-DK"); + ContentService.Save(content); + + PublishContent(content, ["en-US", "da-DK"]); + + var editor = rteDataType.Editor!; + + AssertIndexedValues( + "en-US", + "Some text for en-US.", + "More text for en-US.", + "invariantText value for en-US", + "variantText value for en-US"); + + AssertIndexedValues( + "da-DK", + "Some text for da-DK.", + "More text for da-DK.", + "invariantText value for da-DK", + "variantText value for da-DK"); + + void AssertIndexedValues(string culture, params string[] expectedIndexedValues) + { + var indexValues = editor.PropertyIndexValueFactory.GetIndexValues( + content.Properties["blocks"]!, + culture: culture, + segment: null, + published: true, + availableCultures: ["en-US", "da-DK"], + contentTypeDictionary: new Dictionary + { + { elementType.Key, elementType }, { contentType.Key, contentType } + }); + + Assert.AreEqual(2, indexValues.Count()); + Assert.NotNull(indexValues.FirstOrDefault(value => value.FieldName.StartsWith(UmbracoExamineFieldNames.RawFieldPrefix))); + + var indexValue = indexValues.FirstOrDefault(v => v.Culture.InvariantEquals(culture) && v.FieldName == "blocks"); + Assert.IsNotNull(indexValue); + Assert.AreEqual(1, indexValue.Values.Count()); + var indexedValue = indexValue.Values.First() as string; + Assert.IsNotNull(indexedValue); + var values = indexedValue.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries).Select(s => s.Trim()).ToArray(); + Assert.AreEqual(expectedIndexedValues.Length, values.Length); + Assert.IsTrue(values.ContainsAll(expectedIndexedValues)); + } + + RichTextEditorValue CreateInvariantRichTextValue(string culture) + { + var contentElementKey = Guid.NewGuid(); + return new RichTextEditorValue + { + Markup = $""" +

Some text for {culture}.

+ +

More text for {culture}.

+ """, + Blocks = new RichTextBlockValue([ + new RichTextBlockLayoutItem(contentElementKey) + ]) + { + ContentData = + [ + new(contentElementKey, elementType.Key, elementType.Alias) + { + Values = + [ + new() { Alias = "invariantText", Value = $"invariantText value for {culture}" }, + new() { Alias = "variantText", Value = $"variantText value for {culture}" } + ] + } + ], + SettingsData = [], + Expose = + [ + new(contentElementKey, culture, null), + ] + } + }; + } + } + private async Task CreateRichTextDataType(IContentType elementType) => await CreateBlockEditorDataType( Constants.PropertyEditors.Aliases.RichText, @@ -536,7 +778,7 @@ public class RichTextElementLevelVariationTests : BlockEditorElementVariationTes }; } - private IContent CreateContent(IContentType contentType, RichTextEditorValue richTextValue) + private IContent CreateContent(IContentType contentType, RichTextEditorValue? richTextValue = null) { var contentBuilder = new ContentBuilder() .WithContentType(contentType); @@ -554,8 +796,11 @@ public class RichTextElementLevelVariationTests : BlockEditorElementVariationTes var content = contentBuilder.Build(); - var propertyValue = JsonSerializer.Serialize(richTextValue); - content.Properties["blocks"]!.SetValue(propertyValue); + if (richTextValue is not null) + { + var propertyValue = JsonSerializer.Serialize(richTextValue); + content.Properties["blocks"]!.SetValue(propertyValue); + } ContentService.Save(content); return content; diff --git a/tests/Umbraco.Tests.Integration/Umbraco.Tests.Integration.csproj b/tests/Umbraco.Tests.Integration/Umbraco.Tests.Integration.csproj index ab9d579c9d..705eb29f87 100644 --- a/tests/Umbraco.Tests.Integration/Umbraco.Tests.Integration.csproj +++ b/tests/Umbraco.Tests.Integration/Umbraco.Tests.Integration.csproj @@ -158,6 +158,9 @@ MediaTypeEditingServiceTests.cs + + BlockListElementLevelVariationTests.cs + BlockListElementLevelVariationTests.cs