diff --git a/src/Umbraco.Core/PropertyEditors/DataEditor.cs b/src/Umbraco.Core/PropertyEditors/DataEditor.cs index 2d0b34a849..f3a9c7f4c6 100644 --- a/src/Umbraco.Core/PropertyEditors/DataEditor.cs +++ b/src/Umbraco.Core/PropertyEditors/DataEditor.cs @@ -153,6 +153,11 @@ namespace Umbraco.Core.PropertyEditors set => _defaultConfiguration = value; } + /// + /// Returns the value indexer for this editor + /// + public virtual IValueIndexer ValueIndexer => new DefaultValueIndexer(); + /// /// Creates a value editor instance. /// diff --git a/src/Umbraco.Core/PropertyEditors/DefaultValueIndexer.cs b/src/Umbraco.Core/PropertyEditors/DefaultValueIndexer.cs new file mode 100644 index 0000000000..d46f15771c --- /dev/null +++ b/src/Umbraco.Core/PropertyEditors/DefaultValueIndexer.cs @@ -0,0 +1,16 @@ +using System.Collections.Generic; +using Umbraco.Core.Models; + +namespace Umbraco.Core.PropertyEditors +{ + /// + /// Returns a single field to index containing the property value + /// + public class DefaultValueIndexer : IValueIndexer + { + public IEnumerable> GetIndexValues(Property property, string culture) + { + yield return new KeyValuePair(property.Alias, new[] { property.GetValue(culture) }); + } + } +} diff --git a/src/Umbraco.Core/PropertyEditors/IDataEditor.cs b/src/Umbraco.Core/PropertyEditors/IDataEditor.cs index 8137101826..f967f6f269 100644 --- a/src/Umbraco.Core/PropertyEditors/IDataEditor.cs +++ b/src/Umbraco.Core/PropertyEditors/IDataEditor.cs @@ -3,6 +3,7 @@ using Umbraco.Core.Composing; namespace Umbraco.Core.PropertyEditors { + /// /// Represents a data editor. /// @@ -65,5 +66,7 @@ namespace Umbraco.Core.PropertyEditors /// Is expected to throw if the editor does not support being configured, e.g. for most parameter editors. /// IConfigurationEditor GetConfigurationEditor(); + + IValueIndexer ValueIndexer { get; } } -} \ No newline at end of file +} diff --git a/src/Umbraco.Core/PropertyEditors/IValueIndexer.cs b/src/Umbraco.Core/PropertyEditors/IValueIndexer.cs new file mode 100644 index 0000000000..fdf3a9234f --- /dev/null +++ b/src/Umbraco.Core/PropertyEditors/IValueIndexer.cs @@ -0,0 +1,14 @@ +using System.Collections.Generic; +using Umbraco.Core.Models; + +namespace Umbraco.Core.PropertyEditors +{ + /// + /// Returns indexable data for the property + /// + public interface IValueIndexer + { + //fixme: What about segments and whether we want the published value? + IEnumerable> GetIndexValues(Property property, string culture); + } +} diff --git a/src/Umbraco.Core/Umbraco.Core.csproj b/src/Umbraco.Core/Umbraco.Core.csproj index 23b90aaf3c..c54f14423b 100755 --- a/src/Umbraco.Core/Umbraco.Core.csproj +++ b/src/Umbraco.Core/Umbraco.Core.csproj @@ -439,6 +439,7 @@ + @@ -447,6 +448,7 @@ + diff --git a/src/Umbraco.Examine/Umbraco.Examine.csproj b/src/Umbraco.Examine/Umbraco.Examine.csproj index 8065cc799c..50561f7d8f 100644 --- a/src/Umbraco.Examine/Umbraco.Examine.csproj +++ b/src/Umbraco.Examine/Umbraco.Examine.csproj @@ -74,6 +74,7 @@ Properties\SolutionInfo.cs + diff --git a/src/Umbraco.Examine/UmbracoContentIndexer.cs b/src/Umbraco.Examine/UmbracoContentIndexer.cs index 541ecd1b2b..ae0c93a6ae 100644 --- a/src/Umbraco.Examine/UmbracoContentIndexer.cs +++ b/src/Umbraco.Examine/UmbracoContentIndexer.cs @@ -22,6 +22,7 @@ using Umbraco.Examine.Config; using IContentService = Umbraco.Core.Services.IContentService; using IMediaService = Umbraco.Core.Services.IMediaService; using Examine.LuceneEngine; +using Umbraco.Core.PropertyEditors; namespace Umbraco.Examine { @@ -30,12 +31,11 @@ namespace Umbraco.Examine /// public class UmbracoContentIndexer : UmbracoExamineIndexer { + protected UmbracoValueSetBuilder ValueSetBuilder { get; } protected IContentService ContentService { get; } protected IMediaService MediaService { get; } - protected IUserService UserService { get; } protected ILocalizationService LanguageService { get; } - private readonly IEnumerable _urlSegmentProviders; private int? _parentId; #region Constructors @@ -48,10 +48,9 @@ namespace Umbraco.Examine { ContentService = Current.Services.ContentService; MediaService = Current.Services.MediaService; - UserService = Current.Services.UserService; LanguageService = Current.Services.LocalizationService; - _urlSegmentProviders = Current.UrlSegmentProviders; + ValueSetBuilder = new UmbracoValueSetBuilder(Current.PropertyEditors, Current.UrlSegmentProviders, Current.Services.UserService); InitializeQueries(Current.SqlContext); } @@ -66,9 +65,7 @@ namespace Umbraco.Examine /// /// /// - /// /// - /// /// /// /// @@ -78,12 +75,11 @@ namespace Umbraco.Examine Directory luceneDirectory, Analyzer defaultAnalyzer, ProfilingLogger profilingLogger, + UmbracoValueSetBuilder valueSetBuilder, IContentService contentService, IMediaService mediaService, - IUserService userService, ILocalizationService languageService, ISqlContext sqlContext, - IEnumerable urlSegmentProviders, IValueSetValidator validator, UmbracoContentIndexerOptions options, IReadOnlyDictionary> indexValueTypes = null) @@ -95,12 +91,10 @@ namespace Umbraco.Examine SupportProtectedContent = options.SupportProtectedContent; SupportUnpublishedContent = options.SupportUnpublishedContent; ParentId = options.ParentId; - + ValueSetBuilder = valueSetBuilder ?? throw new ArgumentNullException(nameof(valueSetBuilder)); ContentService = contentService ?? throw new ArgumentNullException(nameof(contentService)); MediaService = mediaService ?? throw new ArgumentNullException(nameof(mediaService)); - UserService = userService ?? throw new ArgumentNullException(nameof(userService)); LanguageService = languageService ?? throw new ArgumentNullException(nameof(languageService)); - _urlSegmentProviders = urlSegmentProviders ?? throw new ArgumentNullException(nameof(urlSegmentProviders)); InitializeQueries(sqlContext); } @@ -292,7 +286,7 @@ namespace Umbraco.Examine content = descendants.ToArray(); } - IndexItems(GetValueSets(_urlSegmentProviders, UserService, content)); + IndexItems(ValueSetBuilder.GetValueSets(content)); pageIndex++; } while (content.Length == pageSize); @@ -327,7 +321,7 @@ namespace Umbraco.Examine media = descendants.ToArray(); } - IndexItems(GetValueSets(_urlSegmentProviders, UserService, media)); + IndexItems(ValueSetBuilder.GetValueSets(media)); pageIndex++; } while (media.Length == pageSize); @@ -336,146 +330,7 @@ namespace Umbraco.Examine } } - /// - /// Creates a collection of for a collection - /// - /// - /// - /// - /// Yield returns - public static IEnumerable GetValueSets(IEnumerable urlSegmentProviders, IUserService userService, params IContent[] content) - { - //TODO: There is a lot of boxing going on here and ultimately all values will be boxed by Lucene anyways - // but I wonder if there's a way to reduce the boxing that we have to do or if it will matter in the end since - // Lucene will do it no matter what? One idea was to create a `FieldValue` struct which would contain `object`, `object[]`, `ValueType` and `ValueType[]` - // references and then each array is an array of `FieldValue[]` and values are assigned accordingly. Not sure if it will make a difference or not. - - foreach (var c in content) - { - var isVariant = c.ContentType.VariesByCulture(); - - var urlValue = c.GetUrlSegment(urlSegmentProviders); //Always add invariant urlName - var values = new Dictionary - { - {"icon", new [] {c.ContentType.Icon}}, - {PublishedFieldName, new object[] {c.Published ? 1 : 0}}, //Always add invariant published value - {"id", new object[] {c.Id}}, - {"key", new object[] {c.Key}}, - {"parentID", new object[] {c.Level > 1 ? c.ParentId : -1}}, - {"level", new object[] {c.Level}}, - {"creatorID", new object[] {c.CreatorId}}, - {"sortOrder", new object[] {c.SortOrder}}, - {"createDate", new object[] {c.CreateDate}}, //Always add invariant createDate - {"updateDate", new object[] {c.UpdateDate}}, //Always add invariant updateDate - {"nodeName", new object[] {c.Name}}, //Always add invariant nodeName - {"urlName", new object[] {urlValue}}, //Always add invariant urlName - {"path", new object[] {c.Path}}, - {"nodeType", new object[] {c.ContentType.Id}}, - {"creatorName", new object[] {c.GetCreatorProfile(userService)?.Name ?? "??"}}, - {"writerName", new object[] {c.GetWriterProfile(userService)?.Name ?? "??"}}, - {"writerID", new object[] {c.WriterId}}, - {"template", new object[] {c.Template?.Id ?? 0}}, - {$"{SpecialFieldPrefix}VariesByCulture", new object[] {0}}, - }; - - if (isVariant) - { - values[$"{SpecialFieldPrefix}VariesByCulture"] = new object[] { 1 }; - - foreach(var culture in c.AvailableCultures) - { - var variantUrl = c.GetUrlSegment(urlSegmentProviders, culture); - var lowerCulture = culture.ToLowerInvariant(); - values[$"urlName_{lowerCulture}"] = new object[] { variantUrl }; - values[$"nodeName_{lowerCulture}"] = new object[] { c.GetCultureName(culture) }; - values[$"{PublishedFieldName}_{lowerCulture}"] = new object[] { c.IsCulturePublished(culture) ? 1 : 0 }; - values[$"updateDate_{lowerCulture}"] = new object[] { c.GetUpdateDate(culture) }; - } - } - - foreach (var property in c.Properties) - { - if (!property.PropertyType.VariesByCulture()) - { - AddPropertyValue(null, c, property, values); - } - else - { - foreach (var culture in c.AvailableCultures) - AddPropertyValue(culture.ToLowerInvariant(), c, property, values); - } - } - - var vs = new ValueSet(c.Id.ToInvariantString(), IndexTypes.Content, c.ContentType.Alias, values); - - yield return vs; - } - } - - private static void AddPropertyValue(string culture, IContent c, Property property, IDictionary values) - { - var val = property.GetValue(culture); - var cultureSuffix = culture == null ? string.Empty : "_" + culture; - switch (val) - { - //only add the value if its not null or empty (we'll check for string explicitly here too) - case null: - return; - case string strVal: - if (strVal.IsNullOrWhiteSpace()) return; - values.Add($"{property.Alias}{cultureSuffix}", new[] { val }); - break; - default: - values.Add($"{property.Alias}{cultureSuffix}", new[] { val }); - break; - } - } - - public static IEnumerable GetValueSets(IEnumerable urlSegmentProviders, IUserService userService, params IMedia[] media) - { - foreach (var m in media) - { - var urlValue = m.GetUrlSegment(urlSegmentProviders); - var values = new Dictionary - { - {"icon", new object[] {m.ContentType.Icon}}, - {"id", new object[] {m.Id}}, - {"key", new object[] {m.Key}}, - {"parentID", new object[] {m.Level > 1 ? m.ParentId : -1}}, - {"level", new object[] {m.Level}}, - {"creatorID", new object[] {m.CreatorId}}, - {"sortOrder", new object[] {m.SortOrder}}, - {"createDate", new object[] {m.CreateDate}}, - {"updateDate", new object[] {m.UpdateDate}}, - {"nodeName", new object[] {m.Name}}, - {"urlName", new object[] {urlValue}}, - {"path", new object[] {m.Path}}, - {"nodeType", new object[] {m.ContentType.Id}}, - {"creatorName", new object[] {m.GetCreatorProfile(userService).Name}} - }; - - foreach (var property in m.Properties) - { - //only add the value if its not null or empty (we'll check for string explicitly here too) - var val = property.GetValue(); - switch (val) - { - case null: - continue; - case string strVal when strVal.IsNullOrWhiteSpace() == false: - values.Add(property.Alias, new[] { val }); - break; - default: - values.Add(property.Alias, new[] { val }); - break; - } - } - - var vs = new ValueSet(m.Id.ToInvariantString(), IndexTypes.Media, m.ContentType.Alias, values); - - yield return vs; - } - } + #endregion diff --git a/src/Umbraco.Examine/UmbracoContentValueSetValidator.cs b/src/Umbraco.Examine/UmbracoContentValueSetValidator.cs index fb5c26d3c1..1881511ee3 100644 --- a/src/Umbraco.Examine/UmbracoContentValueSetValidator.cs +++ b/src/Umbraco.Examine/UmbracoContentValueSetValidator.cs @@ -7,6 +7,7 @@ using Umbraco.Core.Services; namespace Umbraco.Examine { + /// /// Used to validate a ValueSet for content - based on permissions, parent id, etc.... /// diff --git a/src/Umbraco.Examine/UmbracoExamineIndexer.cs b/src/Umbraco.Examine/UmbracoExamineIndexer.cs index 1fb3b0c3a3..3ede45c60a 100644 --- a/src/Umbraco.Examine/UmbracoExamineIndexer.cs +++ b/src/Umbraco.Examine/UmbracoExamineIndexer.cs @@ -419,30 +419,6 @@ namespace Umbraco.Examine e.IndexItem.ValueSet.Set(IndexPathFieldName, path); } - //strip html of all users fields if we detect it has HTML in it. - //if that is the case, we'll create a duplicate 'raw' copy of it so that we can return - //the value of the field 'as-is'. - foreach (var value in e.IndexItem.ValueSet.Values.ToList()) //ToList here to make a diff collection else we'll get collection modified errors - { - if (value.Value == null) continue; - - if (value.Value.Count > 0) - { - if (value.Value.First() is string str) - { - if (XmlHelper.CouldItBeXml(str)) - { - //First save the raw value to a raw field, we will change the policy of this field by detecting the prefix later - e.IndexItem.ValueSet.Values[string.Concat(RawFieldPrefix, value.Key)] = new List { str }; - - //now replace the original value with the stripped html - //TODO: This should be done with an analzer?! - e.IndexItem.ValueSet.Values[value.Key] = new List { str.StripHtml() }; - } - } - } - } - //icon if (e.IndexItem.ValueSet.Values.TryGetValue("icon", out var icon) && e.IndexItem.ValueSet.Values.ContainsKey(IconFieldName) == false) { diff --git a/src/Umbraco.Examine/UmbracoMemberIndexer.cs b/src/Umbraco.Examine/UmbracoMemberIndexer.cs index 62d9a7a1d0..f78348a255 100644 --- a/src/Umbraco.Examine/UmbracoMemberIndexer.cs +++ b/src/Umbraco.Examine/UmbracoMemberIndexer.cs @@ -23,6 +23,7 @@ namespace Umbraco.Examine /// public class UmbracoMemberIndexer : UmbracoExamineIndexer { + private readonly UmbracoValueSetBuilder _valueSetBuilder; private readonly IMemberService _memberService; /// @@ -32,6 +33,7 @@ namespace Umbraco.Examine public UmbracoMemberIndexer() { _memberService = Current.Services.MemberService; + _valueSetBuilder = new UmbracoValueSetBuilder(Current.PropertyEditors, null, null); } /// @@ -50,10 +52,12 @@ namespace Umbraco.Examine Directory luceneDirectory, Analyzer analyzer, ProfilingLogger profilingLogger, + UmbracoValueSetBuilder valueSetBuilder, IMemberService memberService, IValueSetValidator validator = null) : base(name, fieldDefinitions, luceneDirectory, analyzer, profilingLogger, validator) { + _valueSetBuilder = valueSetBuilder ?? throw new ArgumentNullException(nameof(valueSetBuilder)); _memberService = memberService ?? throw new ArgumentNullException(nameof(memberService)); } @@ -99,7 +103,7 @@ namespace Umbraco.Examine { members = _memberService.GetAll(pageIndex, pageSize, out _, "LoginName", Direction.Ascending, true, null, nodeType).ToArray(); - IndexItems(GetValueSets(members)); + IndexItems(_valueSetBuilder.GetValueSets(members)); pageIndex++; } while (members.Length == pageSize); @@ -112,58 +116,14 @@ namespace Umbraco.Examine { members = _memberService.GetAll(pageIndex, pageSize, out _).ToArray(); - IndexItems(GetValueSets(members)); + IndexItems(_valueSetBuilder.GetValueSets(members)); pageIndex++; } while (members.Length == pageSize); } } - public static IEnumerable GetValueSets(params IMember[] members) - { - foreach (var m in members) - { - var values = new Dictionary - { - {"icon", new object[] {m.ContentType.Icon}}, - {"id", new object[] {m.Id}}, - {"key", new object[] {m.Key}}, - {"parentID", new object[] {m.Level > 1 ? m.ParentId : -1}}, - {"level", new object[] {m.Level}}, - {"creatorID", new object[] {m.CreatorId}}, - {"sortOrder", new object[] {m.SortOrder}}, - {"createDate", new object[] {m.CreateDate}}, - {"updateDate", new object[] {m.UpdateDate}}, - {"nodeName", new object[] {m.Name}}, - {"path", new object[] {m.Path}}, - {"nodeType", new object[] {m.ContentType.Id}}, - {"loginName", new object[] {m.Username}}, - {"email", new object[] {m.Email}}, - }; - - foreach (var property in m.Properties) - { - //only add the value if its not null or empty (we'll check for string explicitly here too) - var val = property.GetValue(); - switch (val) - { - case null: - continue; - case string strVal: - if (strVal.IsNullOrWhiteSpace()) continue; - values.Add(property.Alias, new[] { val }); - break; - default: - values.Add(property.Alias, new[] { val }); - break; - } - } - - var vs = new ValueSet(m.Id.ToInvariantString(), IndexTypes.Content, m.ContentType.Alias, values); - - yield return vs; - } - } + /// /// Ensure some custom values are added to the index diff --git a/src/Umbraco.Examine/UmbracoValueSetBuilder.cs b/src/Umbraco.Examine/UmbracoValueSetBuilder.cs new file mode 100644 index 0000000000..7bee1ecf3d --- /dev/null +++ b/src/Umbraco.Examine/UmbracoValueSetBuilder.cs @@ -0,0 +1,202 @@ +using Examine; +using System.Collections.Generic; +using Umbraco.Core; +using Umbraco.Core.Models; +using Umbraco.Core.PropertyEditors; +using Umbraco.Core.Services; +using Umbraco.Core.Strings; + +namespace Umbraco.Examine +{ + public class UmbracoValueSetBuilder + { + private readonly PropertyEditorCollection _propertyEditors; + private readonly IEnumerable _urlSegmentProviders; + private readonly IUserService _userService; + + public UmbracoValueSetBuilder(PropertyEditorCollection propertyEditors, + IEnumerable urlSegmentProviders, + IUserService userService) + { + _propertyEditors = propertyEditors; + _urlSegmentProviders = urlSegmentProviders; + _userService = userService; + } + + /// + /// Creates a collection of for a collection + /// + /// + /// + /// + /// Yield returns + public IEnumerable GetValueSets(params IContent[] content) + { + //TODO: There is a lot of boxing going on here and ultimately all values will be boxed by Lucene anyways + // but I wonder if there's a way to reduce the boxing that we have to do or if it will matter in the end since + // Lucene will do it no matter what? One idea was to create a `FieldValue` struct which would contain `object`, `object[]`, `ValueType` and `ValueType[]` + // references and then each array is an array of `FieldValue[]` and values are assigned accordingly. Not sure if it will make a difference or not. + + foreach (var c in content) + { + var isVariant = c.ContentType.VariesByCulture(); + + var urlValue = c.GetUrlSegment(_urlSegmentProviders); //Always add invariant urlName + var values = new Dictionary + { + {"icon", new [] {c.ContentType.Icon}}, + {UmbracoExamineIndexer.PublishedFieldName, new object[] {c.Published ? 1 : 0}}, //Always add invariant published value + {"id", new object[] {c.Id}}, + {"key", new object[] {c.Key}}, + {"parentID", new object[] {c.Level > 1 ? c.ParentId : -1}}, + {"level", new object[] {c.Level}}, + {"creatorID", new object[] {c.CreatorId}}, + {"sortOrder", new object[] {c.SortOrder}}, + {"createDate", new object[] {c.CreateDate}}, //Always add invariant createDate + {"updateDate", new object[] {c.UpdateDate}}, //Always add invariant updateDate + {"nodeName", new object[] {c.Name}}, //Always add invariant nodeName + {"urlName", new object[] {urlValue}}, //Always add invariant urlName + {"path", new object[] {c.Path}}, + {"nodeType", new object[] {c.ContentType.Id}}, + {"creatorName", new object[] {c.GetCreatorProfile(_userService)?.Name ?? "??"}}, + {"writerName", new object[] {c.GetWriterProfile(_userService)?.Name ?? "??"}}, + {"writerID", new object[] {c.WriterId}}, + {"template", new object[] {c.Template?.Id ?? 0}}, + {$"{UmbracoExamineIndexer.SpecialFieldPrefix}VariesByCulture", new object[] {0}}, + }; + + if (isVariant) + { + values[$"{UmbracoExamineIndexer.SpecialFieldPrefix}VariesByCulture"] = new object[] { 1 }; + + foreach (var culture in c.AvailableCultures) + { + var variantUrl = c.GetUrlSegment(_urlSegmentProviders, culture); + var lowerCulture = culture.ToLowerInvariant(); + values[$"urlName_{lowerCulture}"] = new object[] { variantUrl }; + values[$"nodeName_{lowerCulture}"] = new object[] { c.GetCultureName(culture) }; + values[$"{UmbracoExamineIndexer.PublishedFieldName}_{lowerCulture}"] = new object[] { c.IsCulturePublished(culture) ? 1 : 0 }; + values[$"updateDate_{lowerCulture}"] = new object[] { c.GetUpdateDate(culture) }; + } + } + + foreach (var property in c.Properties) + { + if (!property.PropertyType.VariesByCulture()) + { + AddPropertyValue(null, property, values); + } + else + { + foreach (var culture in c.AvailableCultures) + AddPropertyValue(culture.ToLowerInvariant(), property, values); + } + } + + var vs = new ValueSet(c.Id.ToInvariantString(), IndexTypes.Content, c.ContentType.Alias, values); + + yield return vs; + } + } + + public IEnumerable GetValueSets(params IMedia[] media) + { + foreach (var m in media) + { + var urlValue = m.GetUrlSegment(_urlSegmentProviders); + var values = new Dictionary + { + {"icon", new object[] {m.ContentType.Icon}}, + {"id", new object[] {m.Id}}, + {"key", new object[] {m.Key}}, + {"parentID", new object[] {m.Level > 1 ? m.ParentId : -1}}, + {"level", new object[] {m.Level}}, + {"creatorID", new object[] {m.CreatorId}}, + {"sortOrder", new object[] {m.SortOrder}}, + {"createDate", new object[] {m.CreateDate}}, + {"updateDate", new object[] {m.UpdateDate}}, + {"nodeName", new object[] {m.Name}}, + {"urlName", new object[] {urlValue}}, + {"path", new object[] {m.Path}}, + {"nodeType", new object[] {m.ContentType.Id}}, + {"creatorName", new object[] {m.GetCreatorProfile(_userService).Name}} + }; + + foreach (var property in m.Properties) + { + AddPropertyValue(null, property, values); + } + + var vs = new ValueSet(m.Id.ToInvariantString(), IndexTypes.Media, m.ContentType.Alias, values); + + yield return vs; + } + } + + public IEnumerable GetValueSets(params IMember[] members) + { + foreach (var m in members) + { + var values = new Dictionary + { + {"icon", new object[] {m.ContentType.Icon}}, + {"id", new object[] {m.Id}}, + {"key", new object[] {m.Key}}, + {"parentID", new object[] {m.Level > 1 ? m.ParentId : -1}}, + {"level", new object[] {m.Level}}, + {"creatorID", new object[] {m.CreatorId}}, + {"sortOrder", new object[] {m.SortOrder}}, + {"createDate", new object[] {m.CreateDate}}, + {"updateDate", new object[] {m.UpdateDate}}, + {"nodeName", new object[] {m.Name}}, + {"path", new object[] {m.Path}}, + {"nodeType", new object[] {m.ContentType.Id}}, + {"loginName", new object[] {m.Username}}, + {"email", new object[] {m.Email}}, + }; + + foreach (var property in m.Properties) + { + AddPropertyValue(null, property, values); + } + + var vs = new ValueSet(m.Id.ToInvariantString(), IndexTypes.Content, m.ContentType.Alias, values); + + yield return vs; + } + } + + private void AddPropertyValue(string culture, Property property, IDictionary values) + { + var editor = _propertyEditors[property.Alias]; + if (editor == null) return; + + var indexVals = editor.ValueIndexer.GetIndexValues(property, culture); + foreach(var keyVal in indexVals) + { + if (keyVal.Key.IsNullOrWhiteSpace()) continue; + + var cultureSuffix = culture == null ? string.Empty : "_" + culture; + + foreach(var val in keyVal.Value) + { + switch (val) + { + //only add the value if its not null or empty (we'll check for string explicitly here too) + case null: + continue; + case string strVal: + if (strVal.IsNullOrWhiteSpace()) return; + values.Add($"{keyVal.Key}{cultureSuffix}", new[] { val }); + break; + default: + values.Add($"{keyVal.Key}{cultureSuffix}", new[] { val }); + break; + } + } + } + + + } + } +} diff --git a/src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs b/src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs index 0f69563dd6..2e24b1ee41 100644 --- a/src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs +++ b/src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs @@ -183,12 +183,12 @@ namespace Umbraco.Tests.UmbracoExamine luceneDir, analyzer, profilingLogger, + //fixme: need a property editor collection here + new UmbracoValueSetBuilder(null, new[] { new DefaultUrlSegmentProvider() }, userService), contentService, mediaService, - userService, languageService, sqlContext, - new[] {new DefaultUrlSegmentProvider()}, new UmbracoContentValueSetValidator(options, Mock.Of()), options); diff --git a/src/Umbraco.Web/PropertyEditors/GridPropertyEditor.cs b/src/Umbraco.Web/PropertyEditors/GridPropertyEditor.cs index 9b5443c649..1002261ca4 100644 --- a/src/Umbraco.Web/PropertyEditors/GridPropertyEditor.cs +++ b/src/Umbraco.Web/PropertyEditors/GridPropertyEditor.cs @@ -1,15 +1,9 @@ -using System; -using System.Linq; -using System.Text; +using System.Linq; using Umbraco.Core.Logging; using Examine; using Lucene.Net.Documents; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; using Umbraco.Core; using Umbraco.Core.PropertyEditors; -using Umbraco.Core.Xml; -using Umbraco.Examine; namespace Umbraco.Web.PropertyEditors { @@ -25,85 +19,7 @@ namespace Umbraco.Web.PropertyEditors : base(logger) { } - //TODO: Change this to use a native way of indexing data: https://github.com/umbraco/Umbraco-CMS/issues/3531 - internal void DocumentWriting(object sender, Examine.LuceneEngine.DocumentWritingEventArgs e) - { - foreach (var value in e.ValueSet.Values) - { - //if there is a value, it's a string and it's detected as json - if (value.Value.Count > 0 && value.Value[0] != null && (value.Value[0] is string firstVal) && firstVal.DetectIsJson()) - { - try - { - //TODO: We should deserialize this to Umbraco.Core.Models.GridValue instead of doing the below - var json = JsonConvert.DeserializeObject(firstVal); - - //check if this is formatted for grid json - if (json.HasValues && json.TryGetValue("name", out _) && json.TryGetValue("sections", out _)) - { - //get all values and put them into a single field (using JsonPath) - var sb = new StringBuilder(); - foreach (var row in json.SelectTokens("$.sections[*].rows[*]")) - { - var rowName = row["name"].Value(); - var areaVals = row.SelectTokens("$.areas[*].controls[*].value"); - - foreach (var areaVal in areaVals) - { - //TODO: If it's not a string, then it's a json formatted value - - // we cannot really index this in a smart way since it could be 'anything' - if (areaVal.Type == JTokenType.String) - { - var str = areaVal.Value(); - str = XmlHelper.CouldItBeXml(str) ? str.StripHtml() : str; - sb.Append(str); - sb.Append(" "); - - //add the row name as an individual field - e.Document.Add( - new Field( - $"{value.Key}.{rowName}", str, Field.Store.YES, Field.Index.ANALYZED)); - } - - } - } - - if (sb.Length > 0) - { - //First save the raw value to a raw field - e.Document.Add( - new Field( - $"{UmbracoExamineIndexer.RawFieldPrefix}{value.Key}", - firstVal, Field.Store.YES, Field.Index.NO, Field.TermVector.NO)); - - //now replace the original value with the combined/cleaned value - e.Document.RemoveField(value.Key); - e.Document.Add( - new Field( - value.Key, - sb.ToString(), Field.Store.YES, Field.Index.ANALYZED)); - } - } - } - catch (InvalidCastException) - { - //swallow...on purpose, there's a chance that this isn't the json format we are looking for - // and we don't want that to affect the website. - } - catch (JsonException) - { - //swallow...on purpose, there's a chance that this isn't json and we don't want that to affect - // the website. - } - catch (ArgumentException) - { - //swallow on purpose to prevent this error: - // Can not add Newtonsoft.Json.Linq.JValue to Newtonsoft.Json.Linq.JObject. - } - } - - } - } + public override IValueIndexer ValueIndexer => new GridValueIndexer(); /// /// Overridden to ensure that the value is validated diff --git a/src/Umbraco.Web/PropertyEditors/GridValueIndexer.cs b/src/Umbraco.Web/PropertyEditors/GridValueIndexer.cs new file mode 100644 index 0000000000..5160b09743 --- /dev/null +++ b/src/Umbraco.Web/PropertyEditors/GridValueIndexer.cs @@ -0,0 +1,91 @@ +using System; +using System.Text; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using Umbraco.Core; +using Umbraco.Core.PropertyEditors; +using Umbraco.Core.Xml; +using Umbraco.Examine; + +namespace Umbraco.Web.PropertyEditors +{ + using System.Collections.Generic; + using Umbraco.Core.Models; + + /// + /// Parses the grid value into indexable values + /// + public class GridValueIndexer : IValueIndexer + { + public IEnumerable> GetIndexValues(Property property, string culture) + { + var result = new Dictionary(); + + var val = property.GetValue(culture); + + //if there is a value, it's a string and it's detected as json + if (val is string rawVal && rawVal.DetectIsJson()) + { + try + { + //TODO: We should deserialize this to Umbraco.Core.Models.GridValue instead of doing the below + var json = JsonConvert.DeserializeObject(rawVal); + + //check if this is formatted for grid json + if (json.HasValues && json.TryGetValue("name", out _) && json.TryGetValue("sections", out _)) + { + //get all values and put them into a single field (using JsonPath) + var sb = new StringBuilder(); + foreach (var row in json.SelectTokens("$.sections[*].rows[*]")) + { + var rowName = row["name"].Value(); + var areaVals = row.SelectTokens("$.areas[*].controls[*].value"); + + foreach (var areaVal in areaVals) + { + //TODO: If it's not a string, then it's a json formatted value - + // we cannot really index this in a smart way since it could be 'anything' + if (areaVal.Type == JTokenType.String) + { + var str = areaVal.Value(); + str = XmlHelper.CouldItBeXml(str) ? str.StripHtml() : str; + sb.Append(str); + sb.Append(" "); + + //add the row name as an individual field + result.Add($"{property.Alias}.{rowName}", new[] { str }); + } + } + } + + if (sb.Length > 0) + { + //First save the raw value to a raw field + result.Add($"{UmbracoExamineIndexer.RawFieldPrefix}{property.Alias}", new[] { rawVal }); + + //index the property with the combined/cleaned value + result.Add(property.Alias, new[] { sb.ToString() }); + } + } + } + catch (InvalidCastException) + { + //swallow...on purpose, there's a chance that this isn't the json format we are looking for + // and we don't want that to affect the website. + } + catch (JsonException) + { + //swallow...on purpose, there's a chance that this isn't json and we don't want that to affect + // the website. + } + catch (ArgumentException) + { + //swallow on purpose to prevent this error: + // Can not add Newtonsoft.Json.Linq.JValue to Newtonsoft.Json.Linq.JObject. + } + } + + return result; + } + } +} diff --git a/src/Umbraco.Web/PropertyEditors/NestedContentPropertyEditor.cs b/src/Umbraco.Web/PropertyEditors/NestedContentPropertyEditor.cs index f7d886f637..c4978e62bc 100644 --- a/src/Umbraco.Web/PropertyEditors/NestedContentPropertyEditor.cs +++ b/src/Umbraco.Web/PropertyEditors/NestedContentPropertyEditor.cs @@ -42,6 +42,8 @@ namespace Umbraco.Web.PropertyEditors : Current.Services.ContentTypeService.Get(contentTypeAlias); } + //fixme: Need to add a custom IValueIndexer for this editor + #region Pre Value Editor protected override IConfigurationEditor CreateConfigurationEditor() => new NestedContentConfigurationEditor(); diff --git a/src/Umbraco.Web/Search/ExamineComponent.cs b/src/Umbraco.Web/Search/ExamineComponent.cs index b355f6bd91..a4f67d40af 100644 --- a/src/Umbraco.Web/Search/ExamineComponent.cs +++ b/src/Umbraco.Web/Search/ExamineComponent.cs @@ -38,6 +38,7 @@ namespace Umbraco.Web.Search public sealed class ExamineComponent : UmbracoComponentBase, IUmbracoCoreComponent { private IExamineManager _examineManager; + private UmbracoValueSetBuilder _valueSetBuilder; private static bool _disableExamineIndexing = false; private static volatile bool _isConfigured = false; private static readonly object IsConfiguredLocker = new object(); @@ -59,12 +60,13 @@ namespace Umbraco.Web.Search composition.Container.RegisterSingleton(); } - internal void Initialize(IRuntimeState runtime, MainDom mainDom, PropertyEditorCollection propertyEditors, IExamineManager examineManager, ProfilingLogger profilingLogger, IScopeProvider scopeProvider, IUmbracoIndexesBuilder indexBuilder, ServiceContext services, IEnumerable urlSegmentProviders) + internal void Initialize(IRuntimeState runtime, MainDom mainDom, PropertyEditorCollection propertyEditors, IExamineManager examineManager, ProfilingLogger profilingLogger, IScopeProvider scopeProvider, IUmbracoIndexesBuilder indexBuilder, ServiceContext services, IEnumerable urlSegmentProviders, UmbracoValueSetBuilder valueSetBuilder) { _services = services; _scopeProvider = scopeProvider; _examineManager = examineManager; _urlSegmentProviders = urlSegmentProviders; + _valueSetBuilder = valueSetBuilder; //We want to manage Examine's appdomain shutdown sequence ourselves so first we'll disable Examine's default behavior //and then we'll use MainDom to control Examine's shutdown @@ -114,8 +116,6 @@ namespace Umbraco.Web.Search if (registeredIndexers == 0) return; - BindGridToExamine(profilingLogger.Logger, examineManager, propertyEditors); - // bind to distributed cache events - this ensures that this logic occurs on ALL servers // that are taking part in a load balanced environment. ContentCacheRefresher.CacheUpdated += ContentCacheRefresherUpdated; @@ -197,26 +197,7 @@ namespace Umbraco.Web.Search } } } - - //TODO: Change this to use a native way of indexing data: https://github.com/umbraco/Umbraco-CMS/issues/3531 - private static void BindGridToExamine(ILogger logger, IExamineManager examineManager, IEnumerable propertyEditors) - { - //bind the grid property editors - this is a hack until http://issues.umbraco.org/issue/U4-8437 - try - { - var grid = propertyEditors.OfType().FirstOrDefault(); - if (grid != null) - { - foreach (var i in examineManager.IndexProviders.Values.OfType()) - i.DocumentWriting += grid.DocumentWriting; - } - } - catch (Exception ex) - { - logger.Error(ex, "Failed to bind grid property editor."); - } - } - + #region Cache refresher updated event handlers private void MemberCacheRefresherUpdated(MemberCacheRefresher sender, CacheRefresherEventArgs args) { @@ -698,7 +679,7 @@ namespace Umbraco.Web.Search public static void Execute(ExamineComponent examineComponent, IContent content, bool? supportUnpublished) { - var valueSet = UmbracoContentIndexer.GetValueSets(examineComponent._urlSegmentProviders, examineComponent._services.UserService, content); + var valueSet = examineComponent._valueSetBuilder.GetValueSets(content); examineComponent._examineManager.IndexItems( valueSet.ToArray(), @@ -729,7 +710,7 @@ namespace Umbraco.Web.Search public static void Execute(ExamineComponent examineComponent, IMedia media, bool isPublished) { - var valueSet = UmbracoContentIndexer.GetValueSets(examineComponent._urlSegmentProviders, examineComponent._services.UserService, media); + var valueSet = examineComponent._valueSetBuilder.GetValueSets(media); examineComponent._examineManager.IndexItems( valueSet.ToArray(), @@ -759,7 +740,7 @@ namespace Umbraco.Web.Search public static void Execute(ExamineComponent examineComponent, IMember member) { - var valueSet = UmbracoMemberIndexer.GetValueSets(member); + var valueSet = examineComponent._valueSetBuilder.GetValueSets(member); examineComponent._examineManager.IndexItems( valueSet.ToArray(), diff --git a/src/Umbraco.Web/Search/UmbracoIndexesBuilder.cs b/src/Umbraco.Web/Search/UmbracoIndexesBuilder.cs index 19dbc034a1..c3e940857b 100644 --- a/src/Umbraco.Web/Search/UmbracoIndexesBuilder.cs +++ b/src/Umbraco.Web/Search/UmbracoIndexesBuilder.cs @@ -25,35 +25,32 @@ namespace Umbraco.Web.Search //TODO: we should inject the different IValueSetValidator so devs can just register them instead of overriding this class? public UmbracoIndexesBuilder(ProfilingLogger profilingLogger, + UmbracoValueSetBuilder valueSetBuilder, IContentService contentService, IMediaService mediaService, - IUserService userService, ILocalizationService languageService, IPublicAccessService publicAccessService, IMemberService memberService, - ISqlContext sqlContext, - IEnumerable urlSegmentProviders) + ISqlContext sqlContext) { ProfilingLogger = profilingLogger ?? throw new System.ArgumentNullException(nameof(profilingLogger)); + ValueSetBuilder = valueSetBuilder ?? throw new System.ArgumentNullException(nameof(valueSetBuilder)); ContentService = contentService ?? throw new System.ArgumentNullException(nameof(contentService)); MediaService = mediaService ?? throw new System.ArgumentNullException(nameof(mediaService)); - UserService = userService ?? throw new System.ArgumentNullException(nameof(userService)); LanguageService = languageService ?? throw new System.ArgumentNullException(nameof(languageService)); PublicAccessService = publicAccessService ?? throw new System.ArgumentNullException(nameof(publicAccessService)); MemberService = memberService ?? throw new System.ArgumentNullException(nameof(memberService)); SqlContext = sqlContext ?? throw new System.ArgumentNullException(nameof(sqlContext)); - UrlSegmentProviders = urlSegmentProviders ?? throw new System.ArgumentNullException(nameof(urlSegmentProviders)); } protected ProfilingLogger ProfilingLogger { get; } + protected UmbracoValueSetBuilder ValueSetBuilder { get; } protected IContentService ContentService { get; } protected IMediaService MediaService { get; } - protected IUserService UserService { get; } protected ILocalizationService LanguageService { get; } protected IPublicAccessService PublicAccessService { get; } protected IMemberService MemberService { get; } protected ISqlContext SqlContext { get; } - protected IEnumerable UrlSegmentProviders { get; } public const string InternalIndexPath = "Internal"; public const string ExternalIndexPath = "External"; @@ -87,7 +84,8 @@ namespace Umbraco.Web.Search UmbracoExamineIndexer.UmbracoIndexFieldDefinitions, GetFileSystemLuceneDirectory(name), analyzer, - ProfilingLogger, ContentService, MediaService, UserService, LanguageService, SqlContext, UrlSegmentProviders, + ProfilingLogger, ValueSetBuilder, + ContentService, MediaService, LanguageService, SqlContext, GetContentValueSetValidator(options), options); return index; @@ -102,7 +100,7 @@ namespace Umbraco.Web.Search UmbracoExamineIndexer.UmbracoIndexFieldDefinitions, GetFileSystemLuceneDirectory(MembersIndexPath), new CultureInvariantWhitespaceAnalyzer(), - ProfilingLogger, MemberService, + ProfilingLogger, ValueSetBuilder, MemberService, GetMemberValueSetValidator()); return index; } diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index 2f8613df73..2bc6969969 100755 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -159,6 +159,7 @@ +