yv01p/Umbraco-CMS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

New IValueIndexer and implements it for grid, new UmbracoValueSetBuilder to create the value sets, no more event handling for grid index data

Browse Source
This commit is contained in:
Shannon
2018-11-26 13:33:15 +11:00
parent 4f7de35e04
commit 9a9fcab0e9
18 changed files with 372 additions and 348 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/Umbraco.Core/PropertyEditors/DataEditor.cs
View File

@@ -153,6 +153,11 @@ namespace Umbraco.Core.PropertyEditors
set => _defaultConfiguration = value;
}
/// <summary>
/// Returns the value indexer for this editor
/// </summary>
public virtual IValueIndexer ValueIndexer => new DefaultValueIndexer();
/// <summary>
/// Creates a value editor instance.
/// </summary>

16
src/Umbraco.Core/PropertyEditors/DefaultValueIndexer.cs Normal file
View File

@@ -0,0 +1,16 @@
using System.Collections.Generic;
using Umbraco.Core.Models;
namespace Umbraco.Core.PropertyEditors
{
/// <summary>
/// Returns a single field to index containing the property value
/// </summary>
public class DefaultValueIndexer : IValueIndexer
{
public IEnumerable<KeyValuePair<string, object[]>> GetIndexValues(Property property, string culture)
{
yield return new KeyValuePair<string, object[]>(property.Alias, new[] { property.GetValue(culture) });
}
}
}

5
src/Umbraco.Core/PropertyEditors/IDataEditor.cs
View File

@@ -3,6 +3,7 @@ using Umbraco.Core.Composing;
namespace Umbraco.Core.PropertyEditors
{
/// <summary>
/// Represents a data editor.
/// </summary>
@@ -65,5 +66,7 @@ namespace Umbraco.Core.PropertyEditors
/// <para>Is expected to throw if the editor does not support being configured, e.g. for most parameter editors.</para>
/// </remarks>
IConfigurationEditor GetConfigurationEditor();
IValueIndexer ValueIndexer { get; }
}
}
}

14
src/Umbraco.Core/PropertyEditors/IValueIndexer.cs Normal file
View File

@@ -0,0 +1,14 @@
using System.Collections.Generic;
using Umbraco.Core.Models;
namespace Umbraco.Core.PropertyEditors
{
/// <summary>
/// Returns indexable data for the property
/// </summary>
public interface IValueIndexer
{
//fixme: What about segments and whether we want the published value?
IEnumerable<KeyValuePair<string, object[]>> GetIndexValues(Property property, string culture);
}
}

2
src/Umbraco.Core/Umbraco.Core.csproj
View File

@@ -439,6 +439,7 @@
<Compile Include="PropertyEditors\ConfigurationEditorOfTConfiguration.cs" />
<Compile Include="PropertyEditors\DataEditorAttribute.cs" />
<Compile Include="PropertyEditors\ConfigurationEditor.cs" />
<Compile Include="PropertyEditors\DefaultValueIndexer.cs" />
<Compile Include="PropertyEditors\DropDownFlexibleConfiguration.cs" />
<Compile Include="PropertyEditors\EditorType.cs" />
<Compile Include="PropertyEditors\IConfigurationEditor.cs" />
@@ -447,6 +448,7 @@
<Compile Include="PropertyEditors\ImageCropperConfiguration.cs" />
<Compile Include="PropertyEditors\IManifestValueValidator.cs" />
<Compile Include="PropertyEditors\IValueFormatValidator.cs" />
<Compile Include="PropertyEditors\IValueIndexer.cs" />
<Compile Include="PropertyEditors\IValueRequiredValidator.cs" />
<Compile Include="PropertyEditors\LabelConfiguration.cs" />
<Compile Include="PropertyEditors\LabelConfigurationEditor.cs" />

1
src/Umbraco.Examine/Umbraco.Examine.csproj
View File

@@ -74,6 +74,7 @@
<Compile Include="..\SolutionInfo.cs">
<Link>Properties\SolutionInfo.cs</Link>
</Compile>
<Compile Include="UmbracoValueSetBuilder.cs" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Umbraco.Core\Umbraco.Core.csproj">

161
src/Umbraco.Examine/UmbracoContentIndexer.cs
View File

@@ -22,6 +22,7 @@ using Umbraco.Examine.Config;
using IContentService = Umbraco.Core.Services.IContentService;
using IMediaService = Umbraco.Core.Services.IMediaService;
using Examine.LuceneEngine;
using Umbraco.Core.PropertyEditors;
namespace Umbraco.Examine
{
@@ -30,12 +31,11 @@ namespace Umbraco.Examine
/// </summary>
public class UmbracoContentIndexer : UmbracoExamineIndexer
{
protected UmbracoValueSetBuilder ValueSetBuilder { get; }
protected IContentService ContentService { get; }
protected IMediaService MediaService { get; }
protected IUserService UserService { get; }
protected ILocalizationService LanguageService { get; }
private readonly IEnumerable<IUrlSegmentProvider> _urlSegmentProviders;
private int? _parentId;
#region Constructors
@@ -48,10 +48,9 @@ namespace Umbraco.Examine
{
ContentService = Current.Services.ContentService;
MediaService = Current.Services.MediaService;
UserService = Current.Services.UserService;
LanguageService = Current.Services.LocalizationService;
_urlSegmentProviders = Current.UrlSegmentProviders;
ValueSetBuilder = new UmbracoValueSetBuilder(Current.PropertyEditors, Current.UrlSegmentProviders, Current.Services.UserService);
InitializeQueries(Current.SqlContext);
}
@@ -66,9 +65,7 @@ namespace Umbraco.Examine
/// <param name="profilingLogger"></param>
/// <param name="contentService"></param>
/// <param name="mediaService"></param>
/// <param name="userService"></param>
/// <param name="sqlContext"></param>
/// <param name="urlSegmentProviders"></param>
/// <param name="validator"></param>
/// <param name="options"></param>
/// <param name="indexValueTypes"></param>
@@ -78,12 +75,11 @@ namespace Umbraco.Examine
Directory luceneDirectory,
Analyzer defaultAnalyzer,
ProfilingLogger profilingLogger,
UmbracoValueSetBuilder valueSetBuilder,
IContentService contentService,
IMediaService mediaService,
IUserService userService,
ILocalizationService languageService,
ISqlContext sqlContext,
IEnumerable<IUrlSegmentProvider> urlSegmentProviders,
IValueSetValidator validator,
UmbracoContentIndexerOptions options,
IReadOnlyDictionary<string, Func<string, IIndexValueType>> indexValueTypes = null)
@@ -95,12 +91,10 @@ namespace Umbraco.Examine
SupportProtectedContent = options.SupportProtectedContent;
SupportUnpublishedContent = options.SupportUnpublishedContent;
ParentId = options.ParentId;
ValueSetBuilder = valueSetBuilder ?? throw new ArgumentNullException(nameof(valueSetBuilder));
ContentService = contentService ?? throw new ArgumentNullException(nameof(contentService));
MediaService = mediaService ?? throw new ArgumentNullException(nameof(mediaService));
UserService = userService ?? throw new ArgumentNullException(nameof(userService));
LanguageService = languageService ?? throw new ArgumentNullException(nameof(languageService));
_urlSegmentProviders = urlSegmentProviders ?? throw new ArgumentNullException(nameof(urlSegmentProviders));
InitializeQueries(sqlContext);
}
@@ -292,7 +286,7 @@ namespace Umbraco.Examine
content = descendants.ToArray();
}
IndexItems(GetValueSets(_urlSegmentProviders, UserService, content));
IndexItems(ValueSetBuilder.GetValueSets(content));
pageIndex++;
} while (content.Length == pageSize);
@@ -327,7 +321,7 @@ namespace Umbraco.Examine
media = descendants.ToArray();
}
IndexItems(GetValueSets(_urlSegmentProviders, UserService, media));
IndexItems(ValueSetBuilder.GetValueSets(media));
pageIndex++;
} while (media.Length == pageSize);
@@ -336,146 +330,7 @@ namespace Umbraco.Examine
}
}
/// <summary>
/// Creates a collection of <see cref="ValueSet"/> for a <see cref="IContent"/> collection
/// </summary>
/// <param name="urlSegmentProviders"></param>
/// <param name="userService"></param>
/// <param name="content"></param>
/// <returns>Yield returns <see cref="ValueSet"/></returns>
public static IEnumerable<ValueSet> GetValueSets(IEnumerable<IUrlSegmentProvider> urlSegmentProviders, IUserService userService, params IContent[] content)
{
//TODO: There is a lot of boxing going on here and ultimately all values will be boxed by Lucene anyways
// but I wonder if there's a way to reduce the boxing that we have to do or if it will matter in the end since
// Lucene will do it no matter what? One idea was to create a `FieldValue` struct which would contain `object`, `object[]`, `ValueType` and `ValueType[]`
// references and then each array is an array of `FieldValue[]` and values are assigned accordingly. Not sure if it will make a difference or not.
foreach (var c in content)
{
var isVariant = c.ContentType.VariesByCulture();
var urlValue = c.GetUrlSegment(urlSegmentProviders); //Always add invariant urlName
var values = new Dictionary<string, object[]>
{
{"icon", new [] {c.ContentType.Icon}},
{PublishedFieldName, new object[] {c.Published ? 1 : 0}}, //Always add invariant published value
{"id", new object[] {c.Id}},
{"key", new object[] {c.Key}},
{"parentID", new object[] {c.Level > 1 ? c.ParentId : -1}},
{"level", new object[] {c.Level}},
{"creatorID", new object[] {c.CreatorId}},
{"sortOrder", new object[] {c.SortOrder}},
{"createDate", new object[] {c.CreateDate}}, //Always add invariant createDate
{"updateDate", new object[] {c.UpdateDate}}, //Always add invariant updateDate
{"nodeName", new object[] {c.Name}}, //Always add invariant nodeName
{"urlName", new object[] {urlValue}}, //Always add invariant urlName
{"path", new object[] {c.Path}},
{"nodeType", new object[] {c.ContentType.Id}},
{"creatorName", new object[] {c.GetCreatorProfile(userService)?.Name ?? "??"}},
{"writerName", new object[] {c.GetWriterProfile(userService)?.Name ?? "??"}},
{"writerID", new object[] {c.WriterId}},
{"template", new object[] {c.Template?.Id ?? 0}},
{$"{SpecialFieldPrefix}VariesByCulture", new object[] {0}},
};
if (isVariant)
{
values[$"{SpecialFieldPrefix}VariesByCulture"] = new object[] { 1 };
foreach(var culture in c.AvailableCultures)
{
var variantUrl = c.GetUrlSegment(urlSegmentProviders, culture);
var lowerCulture = culture.ToLowerInvariant();
values[$"urlName_{lowerCulture}"] = new object[] { variantUrl };
values[$"nodeName_{lowerCulture}"] = new object[] { c.GetCultureName(culture) };
values[$"{PublishedFieldName}_{lowerCulture}"] = new object[] { c.IsCulturePublished(culture) ? 1 : 0 };
values[$"updateDate_{lowerCulture}"] = new object[] { c.GetUpdateDate(culture) };
}
}
foreach (var property in c.Properties)
{
if (!property.PropertyType.VariesByCulture())
{
AddPropertyValue(null, c, property, values);
}
else
{
foreach (var culture in c.AvailableCultures)
AddPropertyValue(culture.ToLowerInvariant(), c, property, values);
}
}
var vs = new ValueSet(c.Id.ToInvariantString(), IndexTypes.Content, c.ContentType.Alias, values);
yield return vs;
}
}
private static void AddPropertyValue(string culture, IContent c, Property property, IDictionary<string, object[]> values)
{
var val = property.GetValue(culture);
var cultureSuffix = culture == null ? string.Empty : "_" + culture;
switch (val)
{
//only add the value if its not null or empty (we'll check for string explicitly here too)
case null:
return;
case string strVal:
if (strVal.IsNullOrWhiteSpace()) return;
values.Add($"{property.Alias}{cultureSuffix}", new[] { val });
break;
default:
values.Add($"{property.Alias}{cultureSuffix}", new[] { val });
break;
}
}
public static IEnumerable<ValueSet> GetValueSets(IEnumerable<IUrlSegmentProvider> urlSegmentProviders, IUserService userService, params IMedia[] media)
{
foreach (var m in media)
{
var urlValue = m.GetUrlSegment(urlSegmentProviders);
var values = new Dictionary<string, object[]>
{
{"icon", new object[] {m.ContentType.Icon}},
{"id", new object[] {m.Id}},
{"key", new object[] {m.Key}},
{"parentID", new object[] {m.Level > 1 ? m.ParentId : -1}},
{"level", new object[] {m.Level}},
{"creatorID", new object[] {m.CreatorId}},
{"sortOrder", new object[] {m.SortOrder}},
{"createDate", new object[] {m.CreateDate}},
{"updateDate", new object[] {m.UpdateDate}},
{"nodeName", new object[] {m.Name}},
{"urlName", new object[] {urlValue}},
{"path", new object[] {m.Path}},
{"nodeType", new object[] {m.ContentType.Id}},
{"creatorName", new object[] {m.GetCreatorProfile(userService).Name}}
};
foreach (var property in m.Properties)
{
//only add the value if its not null or empty (we'll check for string explicitly here too)
var val = property.GetValue();
switch (val)
{
case null:
continue;
case string strVal when strVal.IsNullOrWhiteSpace() == false:
values.Add(property.Alias, new[] { val });
break;
default:
values.Add(property.Alias, new[] { val });
break;
}
}
var vs = new ValueSet(m.Id.ToInvariantString(), IndexTypes.Media, m.ContentType.Alias, values);
yield return vs;
}
}
#endregion

1
src/Umbraco.Examine/UmbracoContentValueSetValidator.cs
View File

@@ -7,6 +7,7 @@ using Umbraco.Core.Services;
namespace Umbraco.Examine
{
/// <summary>
/// Used to validate a ValueSet for content - based on permissions, parent id, etc....
/// </summary>

24
src/Umbraco.Examine/UmbracoExamineIndexer.cs
View File

@@ -419,30 +419,6 @@ namespace Umbraco.Examine
e.IndexItem.ValueSet.Set(IndexPathFieldName, path);
}
//strip html of all users fields if we detect it has HTML in it.
//if that is the case, we'll create a duplicate 'raw' copy of it so that we can return
//the value of the field 'as-is'.
foreach (var value in e.IndexItem.ValueSet.Values.ToList()) //ToList here to make a diff collection else we'll get collection modified errors
{
if (value.Value == null) continue;
if (value.Value.Count > 0)
{
if (value.Value.First() is string str)
{
if (XmlHelper.CouldItBeXml(str))
{
//First save the raw value to a raw field, we will change the policy of this field by detecting the prefix later
e.IndexItem.ValueSet.Values[string.Concat(RawFieldPrefix, value.Key)] = new List<object> { str };
//now replace the original value with the stripped html
//TODO: This should be done with an analzer?!
e.IndexItem.ValueSet.Values[value.Key] = new List<object> { str.StripHtml() };
}
}
}
}
//icon
if (e.IndexItem.ValueSet.Values.TryGetValue("icon", out var icon) && e.IndexItem.ValueSet.Values.ContainsKey(IconFieldName) == false)
{

54
src/Umbraco.Examine/UmbracoMemberIndexer.cs
View File

@@ -23,6 +23,7 @@ namespace Umbraco.Examine
/// </summary>
public class UmbracoMemberIndexer : UmbracoExamineIndexer
{
private readonly UmbracoValueSetBuilder _valueSetBuilder;
private readonly IMemberService _memberService;
/// <summary>
@@ -32,6 +33,7 @@ namespace Umbraco.Examine
public UmbracoMemberIndexer()
{
_memberService = Current.Services.MemberService;
_valueSetBuilder = new UmbracoValueSetBuilder(Current.PropertyEditors, null, null);
}
/// <summary>
@@ -50,10 +52,12 @@ namespace Umbraco.Examine
Directory luceneDirectory,
Analyzer analyzer,
ProfilingLogger profilingLogger,
UmbracoValueSetBuilder valueSetBuilder,
IMemberService memberService,
IValueSetValidator validator = null) :
base(name, fieldDefinitions, luceneDirectory, analyzer, profilingLogger, validator)
{
_valueSetBuilder = valueSetBuilder ?? throw new ArgumentNullException(nameof(valueSetBuilder));
_memberService = memberService ?? throw new ArgumentNullException(nameof(memberService));
}
@@ -99,7 +103,7 @@ namespace Umbraco.Examine
{
members = _memberService.GetAll(pageIndex, pageSize, out _, "LoginName", Direction.Ascending, true, null, nodeType).ToArray();
IndexItems(GetValueSets(members));
IndexItems(_valueSetBuilder.GetValueSets(members));
pageIndex++;
} while (members.Length == pageSize);
@@ -112,58 +116,14 @@ namespace Umbraco.Examine
{
members = _memberService.GetAll(pageIndex, pageSize, out _).ToArray();
IndexItems(GetValueSets(members));
IndexItems(_valueSetBuilder.GetValueSets(members));
pageIndex++;
} while (members.Length == pageSize);
}
}
public static IEnumerable<ValueSet> GetValueSets(params IMember[] members)
{
foreach (var m in members)
{
var values = new Dictionary<string, object[]>
{
{"icon", new object[] {m.ContentType.Icon}},
{"id", new object[] {m.Id}},
{"key", new object[] {m.Key}},
{"parentID", new object[] {m.Level > 1 ? m.ParentId : -1}},
{"level", new object[] {m.Level}},
{"creatorID", new object[] {m.CreatorId}},
{"sortOrder", new object[] {m.SortOrder}},
{"createDate", new object[] {m.CreateDate}},
{"updateDate", new object[] {m.UpdateDate}},
{"nodeName", new object[] {m.Name}},
{"path", new object[] {m.Path}},
{"nodeType", new object[] {m.ContentType.Id}},
{"loginName", new object[] {m.Username}},
{"email", new object[] {m.Email}},
};
foreach (var property in m.Properties)
{
//only add the value if its not null or empty (we'll check for string explicitly here too)
var val = property.GetValue();
switch (val)
{
case null:
continue;
case string strVal:
if (strVal.IsNullOrWhiteSpace()) continue;
values.Add(property.Alias, new[] { val });
break;
default:
values.Add(property.Alias, new[] { val });
break;
}
}
var vs = new ValueSet(m.Id.ToInvariantString(), IndexTypes.Content, m.ContentType.Alias, values);
yield return vs;
}
}
/// <summary>
/// Ensure some custom values are added to the index

202
src/Umbraco.Examine/UmbracoValueSetBuilder.cs Normal file
View File

@@ -0,0 +1,202 @@
using Examine;
using System.Collections.Generic;
using Umbraco.Core;
using Umbraco.Core.Models;
using Umbraco.Core.PropertyEditors;
using Umbraco.Core.Services;
using Umbraco.Core.Strings;
namespace Umbraco.Examine
{
public class UmbracoValueSetBuilder
{
private readonly PropertyEditorCollection _propertyEditors;
private readonly IEnumerable<IUrlSegmentProvider> _urlSegmentProviders;
private readonly IUserService _userService;
public UmbracoValueSetBuilder(PropertyEditorCollection propertyEditors,
IEnumerable<IUrlSegmentProvider> urlSegmentProviders,
IUserService userService)
{
_propertyEditors = propertyEditors;
_urlSegmentProviders = urlSegmentProviders;
_userService = userService;
}
/// <summary>
/// Creates a collection of <see cref="ValueSet"/> for a <see cref="IContent"/> collection
/// </summary>
/// <param name="urlSegmentProviders"></param>
/// <param name="userService"></param>
/// <param name="content"></param>
/// <returns>Yield returns <see cref="ValueSet"/></returns>
public IEnumerable<ValueSet> GetValueSets(params IContent[] content)
{
//TODO: There is a lot of boxing going on here and ultimately all values will be boxed by Lucene anyways
// but I wonder if there's a way to reduce the boxing that we have to do or if it will matter in the end since
// Lucene will do it no matter what? One idea was to create a `FieldValue` struct which would contain `object`, `object[]`, `ValueType` and `ValueType[]`
// references and then each array is an array of `FieldValue[]` and values are assigned accordingly. Not sure if it will make a difference or not.
foreach (var c in content)
{
var isVariant = c.ContentType.VariesByCulture();
var urlValue = c.GetUrlSegment(_urlSegmentProviders); //Always add invariant urlName
var values = new Dictionary<string, object[]>
{
{"icon", new [] {c.ContentType.Icon}},
{UmbracoExamineIndexer.PublishedFieldName, new object[] {c.Published ? 1 : 0}}, //Always add invariant published value
{"id", new object[] {c.Id}},
{"key", new object[] {c.Key}},
{"parentID", new object[] {c.Level > 1 ? c.ParentId : -1}},
{"level", new object[] {c.Level}},
{"creatorID", new object[] {c.CreatorId}},
{"sortOrder", new object[] {c.SortOrder}},
{"createDate", new object[] {c.CreateDate}}, //Always add invariant createDate
{"updateDate", new object[] {c.UpdateDate}}, //Always add invariant updateDate
{"nodeName", new object[] {c.Name}}, //Always add invariant nodeName
{"urlName", new object[] {urlValue}}, //Always add invariant urlName
{"path", new object[] {c.Path}},
{"nodeType", new object[] {c.ContentType.Id}},
{"creatorName", new object[] {c.GetCreatorProfile(_userService)?.Name ?? "??"}},
{"writerName", new object[] {c.GetWriterProfile(_userService)?.Name ?? "??"}},
{"writerID", new object[] {c.WriterId}},
{"template", new object[] {c.Template?.Id ?? 0}},
{$"{UmbracoExamineIndexer.SpecialFieldPrefix}VariesByCulture", new object[] {0}},
};
if (isVariant)
{
values[$"{UmbracoExamineIndexer.SpecialFieldPrefix}VariesByCulture"] = new object[] { 1 };
foreach (var culture in c.AvailableCultures)
{
var variantUrl = c.GetUrlSegment(_urlSegmentProviders, culture);
var lowerCulture = culture.ToLowerInvariant();
values[$"urlName_{lowerCulture}"] = new object[] { variantUrl };
values[$"nodeName_{lowerCulture}"] = new object[] { c.GetCultureName(culture) };
values[$"{UmbracoExamineIndexer.PublishedFieldName}_{lowerCulture}"] = new object[] { c.IsCulturePublished(culture) ? 1 : 0 };
values[$"updateDate_{lowerCulture}"] = new object[] { c.GetUpdateDate(culture) };
}
}
foreach (var property in c.Properties)
{
if (!property.PropertyType.VariesByCulture())
{
AddPropertyValue(null, property, values);
}
else
{
foreach (var culture in c.AvailableCultures)
AddPropertyValue(culture.ToLowerInvariant(), property, values);
}
}
var vs = new ValueSet(c.Id.ToInvariantString(), IndexTypes.Content, c.ContentType.Alias, values);
yield return vs;
}
}
public IEnumerable<ValueSet> GetValueSets(params IMedia[] media)
{
foreach (var m in media)
{
var urlValue = m.GetUrlSegment(_urlSegmentProviders);
var values = new Dictionary<string, object[]>
{
{"icon", new object[] {m.ContentType.Icon}},
{"id", new object[] {m.Id}},
{"key", new object[] {m.Key}},
{"parentID", new object[] {m.Level > 1 ? m.ParentId : -1}},
{"level", new object[] {m.Level}},
{"creatorID", new object[] {m.CreatorId}},
{"sortOrder", new object[] {m.SortOrder}},
{"createDate", new object[] {m.CreateDate}},
{"updateDate", new object[] {m.UpdateDate}},
{"nodeName", new object[] {m.Name}},
{"urlName", new object[] {urlValue}},
{"path", new object[] {m.Path}},
{"nodeType", new object[] {m.ContentType.Id}},
{"creatorName", new object[] {m.GetCreatorProfile(_userService).Name}}
};
foreach (var property in m.Properties)
{
AddPropertyValue(null, property, values);
}
var vs = new ValueSet(m.Id.ToInvariantString(), IndexTypes.Media, m.ContentType.Alias, values);
yield return vs;
}
}
public IEnumerable<ValueSet> GetValueSets(params IMember[] members)
{
foreach (var m in members)
{
var values = new Dictionary<string, object[]>
{
{"icon", new object[] {m.ContentType.Icon}},
{"id", new object[] {m.Id}},
{"key", new object[] {m.Key}},
{"parentID", new object[] {m.Level > 1 ? m.ParentId : -1}},
{"level", new object[] {m.Level}},
{"creatorID", new object[] {m.CreatorId}},
{"sortOrder", new object[] {m.SortOrder}},
{"createDate", new object[] {m.CreateDate}},
{"updateDate", new object[] {m.UpdateDate}},
{"nodeName", new object[] {m.Name}},
{"path", new object[] {m.Path}},
{"nodeType", new object[] {m.ContentType.Id}},
{"loginName", new object[] {m.Username}},
{"email", new object[] {m.Email}},
};
foreach (var property in m.Properties)
{
AddPropertyValue(null, property, values);
}
var vs = new ValueSet(m.Id.ToInvariantString(), IndexTypes.Content, m.ContentType.Alias, values);
yield return vs;
}
}
private void AddPropertyValue(string culture, Property property, IDictionary<string, object[]> values)
{
var editor = _propertyEditors[property.Alias];
if (editor == null) return;
var indexVals = editor.ValueIndexer.GetIndexValues(property, culture);
foreach(var keyVal in indexVals)
{
if (keyVal.Key.IsNullOrWhiteSpace()) continue;
var cultureSuffix = culture == null ? string.Empty : "_" + culture;
foreach(var val in keyVal.Value)
{
switch (val)
{
//only add the value if its not null or empty (we'll check for string explicitly here too)
case null:
continue;
case string strVal:
if (strVal.IsNullOrWhiteSpace()) return;
values.Add($"{keyVal.Key}{cultureSuffix}", new[] { val });
break;
default:
values.Add($"{keyVal.Key}{cultureSuffix}", new[] { val });
break;
}
}
}
}
}
}

4
src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs
View File

@@ -183,12 +183,12 @@ namespace Umbraco.Tests.UmbracoExamine
luceneDir,
analyzer,
profilingLogger,
//fixme: need a property editor collection here
new UmbracoValueSetBuilder(null, new[] { new DefaultUrlSegmentProvider() }, userService),
contentService,
mediaService,
userService,
languageService,
sqlContext,
new[] {new DefaultUrlSegmentProvider()},
new UmbracoContentValueSetValidator(options, Mock.Of<IPublicAccessService>()),
options);

88
src/Umbraco.Web/PropertyEditors/GridPropertyEditor.cs
View File

@@ -1,15 +1,9 @@
using System;
using System.Linq;
using System.Text;
using System.Linq;
using Umbraco.Core.Logging;
using Examine;
using Lucene.Net.Documents;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Umbraco.Core;
using Umbraco.Core.PropertyEditors;
using Umbraco.Core.Xml;
using Umbraco.Examine;
namespace Umbraco.Web.PropertyEditors
{
@@ -25,85 +19,7 @@ namespace Umbraco.Web.PropertyEditors
: base(logger)
{ }
//TODO: Change this to use a native way of indexing data: https://github.com/umbraco/Umbraco-CMS/issues/3531
internal void DocumentWriting(object sender, Examine.LuceneEngine.DocumentWritingEventArgs e)
{
foreach (var value in e.ValueSet.Values)
{
//if there is a value, it's a string and it's detected as json
if (value.Value.Count > 0 && value.Value[0] != null && (value.Value[0] is string firstVal) && firstVal.DetectIsJson())
{
try
{
//TODO: We should deserialize this to Umbraco.Core.Models.GridValue instead of doing the below
var json = JsonConvert.DeserializeObject<JObject>(firstVal);
//check if this is formatted for grid json
if (json.HasValues && json.TryGetValue("name", out _) && json.TryGetValue("sections", out _))
{
//get all values and put them into a single field (using JsonPath)
var sb = new StringBuilder();
foreach (var row in json.SelectTokens("$.sections[*].rows[*]"))
{
var rowName = row["name"].Value<string>();
var areaVals = row.SelectTokens("$.areas[*].controls[*].value");
foreach (var areaVal in areaVals)
{
//TODO: If it's not a string, then it's a json formatted value -
// we cannot really index this in a smart way since it could be 'anything'
if (areaVal.Type == JTokenType.String)
{
var str = areaVal.Value<string>();
str = XmlHelper.CouldItBeXml(str) ? str.StripHtml() : str;
sb.Append(str);
sb.Append(" ");
//add the row name as an individual field
e.Document.Add(
new Field(
$"{value.Key}.{rowName}", str, Field.Store.YES, Field.Index.ANALYZED));
}
}
}
if (sb.Length > 0)
{
//First save the raw value to a raw field
e.Document.Add(
new Field(
$"{UmbracoExamineIndexer.RawFieldPrefix}{value.Key}",
firstVal, Field.Store.YES, Field.Index.NO, Field.TermVector.NO));
//now replace the original value with the combined/cleaned value
e.Document.RemoveField(value.Key);
e.Document.Add(
new Field(
value.Key,
sb.ToString(), Field.Store.YES, Field.Index.ANALYZED));
}
}
}
catch (InvalidCastException)
{
//swallow...on purpose, there's a chance that this isn't the json format we are looking for
// and we don't want that to affect the website.
}
catch (JsonException)
{
//swallow...on purpose, there's a chance that this isn't json and we don't want that to affect
// the website.
}
catch (ArgumentException)
{
//swallow on purpose to prevent this error:
// Can not add Newtonsoft.Json.Linq.JValue to Newtonsoft.Json.Linq.JObject.
}
}
}
}
public override IValueIndexer ValueIndexer => new GridValueIndexer();
/// <summary>
/// Overridden to ensure that the value is validated

91
src/Umbraco.Web/PropertyEditors/GridValueIndexer.cs Normal file
View File

@@ -0,0 +1,91 @@
using System;
using System.Text;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Umbraco.Core;
using Umbraco.Core.PropertyEditors;
using Umbraco.Core.Xml;
using Umbraco.Examine;
namespace Umbraco.Web.PropertyEditors
{
using System.Collections.Generic;
using Umbraco.Core.Models;
/// <summary>
/// Parses the grid value into indexable values
/// </summary>
public class GridValueIndexer : IValueIndexer
{
public IEnumerable<KeyValuePair<string, object[]>> GetIndexValues(Property property, string culture)
{
var result = new Dictionary<string, object[]>();
var val = property.GetValue(culture);
//if there is a value, it's a string and it's detected as json
if (val is string rawVal && rawVal.DetectIsJson())
{
try
{
//TODO: We should deserialize this to Umbraco.Core.Models.GridValue instead of doing the below
var json = JsonConvert.DeserializeObject<JObject>(rawVal);
//check if this is formatted for grid json
if (json.HasValues && json.TryGetValue("name", out _) && json.TryGetValue("sections", out _))
{
//get all values and put them into a single field (using JsonPath)
var sb = new StringBuilder();
foreach (var row in json.SelectTokens("$.sections[*].rows[*]"))
{
var rowName = row["name"].Value<string>();
var areaVals = row.SelectTokens("$.areas[*].controls[*].value");
foreach (var areaVal in areaVals)
{
//TODO: If it's not a string, then it's a json formatted value -
// we cannot really index this in a smart way since it could be 'anything'
if (areaVal.Type == JTokenType.String)
{
var str = areaVal.Value<string>();
str = XmlHelper.CouldItBeXml(str) ? str.StripHtml() : str;
sb.Append(str);
sb.Append(" ");
//add the row name as an individual field
result.Add($"{property.Alias}.{rowName}", new[] { str });
}
}
}
if (sb.Length > 0)
{
//First save the raw value to a raw field
result.Add($"{UmbracoExamineIndexer.RawFieldPrefix}{property.Alias}", new[] { rawVal });
//index the property with the combined/cleaned value
result.Add(property.Alias, new[] { sb.ToString() });
}
}
}
catch (InvalidCastException)
{
//swallow...on purpose, there's a chance that this isn't the json format we are looking for
// and we don't want that to affect the website.
}
catch (JsonException)
{
//swallow...on purpose, there's a chance that this isn't json and we don't want that to affect
// the website.
}
catch (ArgumentException)
{
//swallow on purpose to prevent this error:
// Can not add Newtonsoft.Json.Linq.JValue to Newtonsoft.Json.Linq.JObject.
}
}
return result;
}
}
}

2
src/Umbraco.Web/PropertyEditors/NestedContentPropertyEditor.cs
View File

@@ -42,6 +42,8 @@ namespace Umbraco.Web.PropertyEditors
: Current.Services.ContentTypeService.Get(contentTypeAlias);
}
//fixme: Need to add a custom IValueIndexer for this editor
#region Pre Value Editor
protected override IConfigurationEditor CreateConfigurationEditor() => new NestedContentConfigurationEditor();

33
src/Umbraco.Web/Search/ExamineComponent.cs
View File

@@ -38,6 +38,7 @@ namespace Umbraco.Web.Search
public sealed class ExamineComponent : UmbracoComponentBase, IUmbracoCoreComponent
{
private IExamineManager _examineManager;
private UmbracoValueSetBuilder _valueSetBuilder;
private static bool _disableExamineIndexing = false;
private static volatile bool _isConfigured = false;
private static readonly object IsConfiguredLocker = new object();
@@ -59,12 +60,13 @@ namespace Umbraco.Web.Search
composition.Container.RegisterSingleton<IUmbracoIndexesBuilder, UmbracoIndexesBuilder>();
}
internal void Initialize(IRuntimeState runtime, MainDom mainDom, PropertyEditorCollection propertyEditors, IExamineManager examineManager, ProfilingLogger profilingLogger, IScopeProvider scopeProvider, IUmbracoIndexesBuilder indexBuilder, ServiceContext services, IEnumerable<IUrlSegmentProvider> urlSegmentProviders)
internal void Initialize(IRuntimeState runtime, MainDom mainDom, PropertyEditorCollection propertyEditors, IExamineManager examineManager, ProfilingLogger profilingLogger, IScopeProvider scopeProvider, IUmbracoIndexesBuilder indexBuilder, ServiceContext services, IEnumerable<IUrlSegmentProvider> urlSegmentProviders, UmbracoValueSetBuilder valueSetBuilder)
{
_services = services;
_scopeProvider = scopeProvider;
_examineManager = examineManager;
_urlSegmentProviders = urlSegmentProviders;
_valueSetBuilder = valueSetBuilder;
//We want to manage Examine's appdomain shutdown sequence ourselves so first we'll disable Examine's default behavior
//and then we'll use MainDom to control Examine's shutdown
@@ -114,8 +116,6 @@ namespace Umbraco.Web.Search
if (registeredIndexers == 0)
return;
BindGridToExamine(profilingLogger.Logger, examineManager, propertyEditors);
// bind to distributed cache events - this ensures that this logic occurs on ALL servers
// that are taking part in a load balanced environment.
ContentCacheRefresher.CacheUpdated += ContentCacheRefresherUpdated;
@@ -197,26 +197,7 @@ namespace Umbraco.Web.Search
}
}
}
//TODO: Change this to use a native way of indexing data: https://github.com/umbraco/Umbraco-CMS/issues/3531
private static void BindGridToExamine(ILogger logger, IExamineManager examineManager, IEnumerable propertyEditors)
{
//bind the grid property editors - this is a hack until http://issues.umbraco.org/issue/U4-8437
try
{
var grid = propertyEditors.OfType<GridPropertyEditor>().FirstOrDefault();
if (grid != null)
{
foreach (var i in examineManager.IndexProviders.Values.OfType<UmbracoExamineIndexer>())
i.DocumentWriting += grid.DocumentWriting;
}
}
catch (Exception ex)
{
logger.Error<ExamineComponent>(ex, "Failed to bind grid property editor.");
}
}
#region Cache refresher updated event handlers
private void MemberCacheRefresherUpdated(MemberCacheRefresher sender, CacheRefresherEventArgs args)
{
@@ -698,7 +679,7 @@ namespace Umbraco.Web.Search
public static void Execute(ExamineComponent examineComponent, IContent content, bool? supportUnpublished)
{
var valueSet = UmbracoContentIndexer.GetValueSets(examineComponent._urlSegmentProviders, examineComponent._services.UserService, content);
var valueSet = examineComponent._valueSetBuilder.GetValueSets(content);
examineComponent._examineManager.IndexItems(
valueSet.ToArray(),
@@ -729,7 +710,7 @@ namespace Umbraco.Web.Search
public static void Execute(ExamineComponent examineComponent, IMedia media, bool isPublished)
{
var valueSet = UmbracoContentIndexer.GetValueSets(examineComponent._urlSegmentProviders, examineComponent._services.UserService, media);
var valueSet = examineComponent._valueSetBuilder.GetValueSets(media);
examineComponent._examineManager.IndexItems(
valueSet.ToArray(),
@@ -759,7 +740,7 @@ namespace Umbraco.Web.Search
public static void Execute(ExamineComponent examineComponent, IMember member)
{
var valueSet = UmbracoMemberIndexer.GetValueSets(member);
var valueSet = examineComponent._valueSetBuilder.GetValueSets(member);
examineComponent._examineManager.IndexItems(
valueSet.ToArray(),

16
src/Umbraco.Web/Search/UmbracoIndexesBuilder.cs
View File

@@ -25,35 +25,32 @@ namespace Umbraco.Web.Search
//TODO: we should inject the different IValueSetValidator so devs can just register them instead of overriding this class?
public UmbracoIndexesBuilder(ProfilingLogger profilingLogger,
UmbracoValueSetBuilder valueSetBuilder,
IContentService contentService,
IMediaService mediaService,
IUserService userService,
ILocalizationService languageService,
IPublicAccessService publicAccessService,
IMemberService memberService,
ISqlContext sqlContext,
IEnumerable<IUrlSegmentProvider> urlSegmentProviders)
ISqlContext sqlContext)
{
ProfilingLogger = profilingLogger ?? throw new System.ArgumentNullException(nameof(profilingLogger));
ValueSetBuilder = valueSetBuilder ?? throw new System.ArgumentNullException(nameof(valueSetBuilder));
ContentService = contentService ?? throw new System.ArgumentNullException(nameof(contentService));
MediaService = mediaService ?? throw new System.ArgumentNullException(nameof(mediaService));
UserService = userService ?? throw new System.ArgumentNullException(nameof(userService));
LanguageService = languageService ?? throw new System.ArgumentNullException(nameof(languageService));
PublicAccessService = publicAccessService ?? throw new System.ArgumentNullException(nameof(publicAccessService));
MemberService = memberService ?? throw new System.ArgumentNullException(nameof(memberService));
SqlContext = sqlContext ?? throw new System.ArgumentNullException(nameof(sqlContext));
UrlSegmentProviders = urlSegmentProviders ?? throw new System.ArgumentNullException(nameof(urlSegmentProviders));
}
protected ProfilingLogger ProfilingLogger { get; }
protected UmbracoValueSetBuilder ValueSetBuilder { get; }
protected IContentService ContentService { get; }
protected IMediaService MediaService { get; }
protected IUserService UserService { get; }
protected ILocalizationService LanguageService { get; }
protected IPublicAccessService PublicAccessService { get; }
protected IMemberService MemberService { get; }
protected ISqlContext SqlContext { get; }
protected IEnumerable<IUrlSegmentProvider> UrlSegmentProviders { get; }
public const string InternalIndexPath = "Internal";
public const string ExternalIndexPath = "External";
@@ -87,7 +84,8 @@ namespace Umbraco.Web.Search
UmbracoExamineIndexer.UmbracoIndexFieldDefinitions,
GetFileSystemLuceneDirectory(name),
analyzer,
ProfilingLogger, ContentService, MediaService, UserService, LanguageService, SqlContext, UrlSegmentProviders,
ProfilingLogger, ValueSetBuilder,
ContentService, MediaService, LanguageService, SqlContext,
GetContentValueSetValidator(options),
options);
return index;
@@ -102,7 +100,7 @@ namespace Umbraco.Web.Search
UmbracoExamineIndexer.UmbracoIndexFieldDefinitions,
GetFileSystemLuceneDirectory(MembersIndexPath),
new CultureInvariantWhitespaceAnalyzer(),
ProfilingLogger, MemberService,
ProfilingLogger, ValueSetBuilder, MemberService,
GetMemberValueSetValidator());
return index;
}

1
src/Umbraco.Web/Umbraco.Web.csproj
View File

@@ -159,6 +159,7 @@
<Compile Include="ContentApps\ContentInfoContentAppDefinition.cs" />
<Compile Include="ContentApps\ListViewContentAppDefinition.cs" />
<Compile Include="Models\Mapping\ScheduledPublishDateResolver.cs" />
<Compile Include="PropertyEditors\GridValueIndexer.cs" />
<Compile Include="Search\IUmbracoIndexesBuilder.cs" />
<Compile Include="Search\UmbracoIndexesBuilder.cs" />
<Compile Include="Security\ActiveDirectoryBackOfficeUserPasswordChecker.cs" />