yv01p/Umbraco-CMS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Gets search tools working for explicitly defined searchers, have tested with a multi index searcher, refactors SupportsUnpublished vs SupportSoftDelete vs OnlyPublishedContent booleans and made some sense out of the reindexing items in ExamineComponent which didn't really work, cleans up IPublishedContentQuery implementation

Browse Source
This commit is contained in:
Shannon
2018-12-05 16:53:25 +11:00
parent 1717f94fcf
commit b48a7fa4a9
36 changed files with 517 additions and 568 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/Umbraco.Core/Constants-Indexes.cs
View File

@@ -8,9 +8,9 @@ namespace Umbraco.Core
public static class UmbracoIndexes
{
public const string InternalIndexName = InternalIndexPath + "Indexer";
public const string ExternalIndexName = ExternalIndexPath + "Indexer";
public const string MembersIndexName = MembersIndexPath + "Indexer";
public const string InternalIndexName = InternalIndexPath + "Index";
public const string ExternalIndexName = ExternalIndexPath + "Index";
public const string MembersIndexName = MembersIndexPath + "Index";
public const string InternalIndexPath = "Internal";
public const string ExternalIndexPath = "External";

2
src/Umbraco.Core/Models/PagedResult.cs
View File

@@ -19,7 +19,7 @@ namespace Umbraco.Core.Models
if (pageSize > 0)
{
TotalPages = (long)Math.Ceiling(totalItems / (Decimal)pageSize);
TotalPages = (long)Math.Ceiling(totalItems / (decimal)pageSize);
}
else
{

6
src/Umbraco.Core/PropertyEditors/DefaultPropertyIndexValues.cs
View File

@@ -8,9 +8,11 @@ namespace Umbraco.Core.PropertyEditors
/// </summary>
public class DefaultPropertyIndexValues : IPropertyIndexValues
{
public IEnumerable<KeyValuePair<string, object[]>> GetIndexValues(Property property, string culture, string segment)
public IEnumerable<KeyValuePair<string, IEnumerable<object>>> GetIndexValues(Property property, string culture, string segment, bool published)
{
yield return new KeyValuePair<string, object[]>(property.Alias, new[] { property.GetValue(culture, segment) });
yield return new KeyValuePair<string, IEnumerable<object>>(
property.Alias,
property.GetValue(culture, segment, published).Yield());
}
}
}

2
src/Umbraco.Core/PropertyEditors/IPropertyIndexValues.cs
View File

@@ -8,6 +8,6 @@ namespace Umbraco.Core.PropertyEditors
/// </summary>
public interface IPropertyIndexValues
{
IEnumerable<KeyValuePair<string, object[]>> GetIndexValues(Property property, string culture, string segment);
IEnumerable<KeyValuePair<string, IEnumerable<object>>> GetIndexValues(Property property, string culture, string segment, bool published);
}
}

6
src/Umbraco.Examine/BaseValueSetBuilder.cs
View File

@@ -12,10 +12,12 @@ namespace Umbraco.Examine
public abstract class BaseValueSetBuilder<TContent> : IValueSetBuilder<TContent>
where TContent : IContentBase
{
protected bool PublishedValuesOnly { get; }
private readonly PropertyEditorCollection _propertyEditors;
protected BaseValueSetBuilder(PropertyEditorCollection propertyEditors)
protected BaseValueSetBuilder(PropertyEditorCollection propertyEditors, bool publishedValuesOnly)
{
PublishedValuesOnly = publishedValuesOnly;
_propertyEditors = propertyEditors ?? throw new System.ArgumentNullException(nameof(propertyEditors));
}
@@ -27,7 +29,7 @@ namespace Umbraco.Examine
var editor = _propertyEditors[property.PropertyType.PropertyEditorAlias];
if (editor == null) return;
var indexVals = editor.PropertyIndexValues.GetIndexValues(property, culture, segment);
var indexVals = editor.PropertyIndexValues.GetIndexValues(property, culture, segment, PublishedValuesOnly);
foreach (var keyVal in indexVals)
{
if (keyVal.Key.IsNullOrWhiteSpace()) continue;

42
src/Umbraco.Examine/Config/IndexSet.cs
View File

@@ -11,48 +11,6 @@ namespace Umbraco.Examine.Config
[ConfigurationProperty("SetName", IsRequired = true, IsKey = true)]
public string SetName => (string)this["SetName"];
private string _indexPath = "";
/// <summary>
/// The folder path of where the lucene index is stored
/// </summary>
/// <value>The index path.</value>
/// <remarks>
/// This can be set at runtime but will not be persisted to the configuration file
/// </remarks>
[ConfigurationProperty("IndexPath", IsRequired = true, IsKey = false)]
public string IndexPath
{
get
{
if (string.IsNullOrEmpty(_indexPath))
_indexPath = (string)this["IndexPath"];
return _indexPath;
}
set => _indexPath = value;
}
/// <summary>
/// Returns the DirectoryInfo object for the index path.
/// </summary>
/// <value>The index directory.</value>
public DirectoryInfo IndexDirectory
{
get
{
//TODO: Get this out of the index set. We need to use the Indexer's DataService to lookup the folder so it can be unit tested. Probably need DataServices on the searcher then too
//we need to de-couple the context
if (HttpContext.Current != null)
return new DirectoryInfo(HttpContext.Current.Server.MapPath(this.IndexPath));
else if (HostingEnvironment.ApplicationID != null)
return new DirectoryInfo(HostingEnvironment.MapPath(this.IndexPath));
else
return new DirectoryInfo(this.IndexPath);
}
}
/// <summary>
/// When this property is set, the indexing will only index documents that are descendants of this node.
/// </summary>

14
src/Umbraco.Examine/ContentIndexPopulator.cs
View File

@@ -25,7 +25,7 @@ namespace Umbraco.Examine
/// </summary>
private static IQuery<IContent> _publishedQuery;
private readonly bool _supportUnpublishedContent;
private readonly bool _publishedValuesOnly;
private readonly int? _parentId;
/// <summary>
@@ -34,27 +34,27 @@ namespace Umbraco.Examine
/// <param name="contentService"></param>
/// <param name="sqlContext"></param>
/// <param name="contentValueSetBuilder"></param>
public ContentIndexPopulator(IContentService contentService, ISqlContext sqlContext, IValueSetBuilder<IContent> contentValueSetBuilder)
: this(true, null, contentService, sqlContext, contentValueSetBuilder)
public ContentIndexPopulator(IContentService contentService, ISqlContext sqlContext, IContentValueSetBuilder contentValueSetBuilder)
: this(false, null, contentService, sqlContext, contentValueSetBuilder)
{
}
/// <summary>
/// Optional constructor allowing specifying custom query parameters
/// </summary>
/// <param name="supportUnpublishedContent"></param>
/// <param name="publishedValuesOnly"></param>
/// <param name="parentId"></param>
/// <param name="contentService"></param>
/// <param name="sqlContext"></param>
/// <param name="contentValueSetBuilder"></param>
public ContentIndexPopulator(bool supportUnpublishedContent, int? parentId, IContentService contentService, ISqlContext sqlContext, IValueSetBuilder<IContent> contentValueSetBuilder)
public ContentIndexPopulator(bool publishedValuesOnly, int? parentId, IContentService contentService, ISqlContext sqlContext, IValueSetBuilder<IContent> contentValueSetBuilder)
{
if (sqlContext == null) throw new ArgumentNullException(nameof(sqlContext));
_contentService = contentService ?? throw new ArgumentNullException(nameof(contentService));
_contentValueSetBuilder = contentValueSetBuilder ?? throw new ArgumentNullException(nameof(contentValueSetBuilder));
if (_publishedQuery != null)
_publishedQuery = sqlContext.Query<IContent>().Where(x => x.Published);
_supportUnpublishedContent = supportUnpublishedContent;
_publishedValuesOnly = publishedValuesOnly;
_parentId = parentId;
RegisterIndex(Constants.UmbracoIndexes.InternalIndexName);
@@ -75,7 +75,7 @@ namespace Umbraco.Examine
do
{
if (_supportUnpublishedContent)
if (!_publishedValuesOnly)
{
content = _contentService.GetPagedDescendants(contentParentId, pageIndex, pageSize, out _).ToArray();
}

26
src/Umbraco.Examine/ContentValueSetBuilder.cs
View File

@@ -8,15 +8,19 @@ using Umbraco.Core.Strings;
namespace Umbraco.Examine
{
public class ContentValueSetBuilder : BaseValueSetBuilder<IContent>
/// <summary>
/// Builds <see cref="ValueSet"/>s for <see cref="IContent"/> items
/// </summary>
public class ContentValueSetBuilder : BaseValueSetBuilder<IContent>, IContentValueSetBuilder, IPublishedContentValueSetBuilder
{
private readonly IEnumerable<IUrlSegmentProvider> _urlSegmentProviders;
private readonly IUserService _userService;
public ContentValueSetBuilder(PropertyEditorCollection propertyEditors,
IEnumerable<IUrlSegmentProvider> urlSegmentProviders,
IUserService userService)
: base(propertyEditors)
IUserService userService,
bool publishedValuesOnly)
: base(propertyEditors, publishedValuesOnly)
{
_urlSegmentProviders = urlSegmentProviders;
_userService = userService;
@@ -47,8 +51,10 @@ namespace Umbraco.Examine
{"sortOrder", new object[] {c.SortOrder}},
{"createDate", new object[] {c.CreateDate}}, //Always add invariant createDate
{"updateDate", new object[] {c.UpdateDate}}, //Always add invariant updateDate
{"nodeName", c.Name.Yield()}, //Always add invariant nodeName
{"urlName", urlValue.Yield()}, //Always add invariant urlName
{"nodeName", PublishedValuesOnly //Always add invariant nodeName
? c.PublishName.Yield()
: c.Name.Yield()},
{"urlName", urlValue.Yield()}, //Always add invariant urlName
{"path", c.Path.Yield()},
{"nodeType", new object[] {c.ContentType.Id}},
{"creatorName", (c.GetCreatorProfile(_userService)?.Name ?? "??").Yield() },
@@ -67,9 +73,13 @@ namespace Umbraco.Examine
var variantUrl = c.GetUrlSegment(_urlSegmentProviders, culture);
var lowerCulture = culture.ToLowerInvariant();
values[$"urlName_{lowerCulture}"] = variantUrl.Yield();
values[$"nodeName_{lowerCulture}"] = c.GetCultureName(culture).Yield();
values[$"{UmbracoExamineIndexer.PublishedFieldName}_{lowerCulture}"] = new object[] { c.IsCulturePublished(culture) ? 1 : 0 };
values[$"updateDate_{lowerCulture}"] = new object[] { c.GetUpdateDate(culture) };
values[$"nodeName_{lowerCulture}"] = PublishedValuesOnly
? c.GetPublishName(culture).Yield()
: c.GetCultureName(culture).Yield();
values[$"{UmbracoExamineIndexer.PublishedFieldName}_{lowerCulture}"] = (c.IsCulturePublished(culture) ? 1 : 0).Yield<object>();
values[$"updateDate_{lowerCulture}"] = PublishedValuesOnly
? c.GetPublishDate(culture).Yield<object>()
: c.GetUpdateDate(culture).Yield<object>();
}
}

14
src/Umbraco.Examine/ContentValueSetValidator.cs
View File

@@ -20,7 +20,7 @@ namespace Umbraco.Examine
private static readonly IEnumerable<string> ValidCategories = new[] {IndexTypes.Content, IndexTypes.Media};
protected override IEnumerable<string> ValidIndexCategories => ValidCategories;
public bool SupportUnpublishedContent { get; }
public bool PublishedValuesOnly { get; }
public bool SupportProtectedContent { get; }
public int? ParentId { get; }
@@ -43,7 +43,7 @@ namespace Umbraco.Examine
var recycleBinId = category == IndexTypes.Content ? Constants.System.RecycleBinContent : Constants.System.RecycleBinMedia;
//check for recycle bin
if (!SupportUnpublishedContent)
if (!PublishedValuesOnly)
{
if (path.Contains(string.Concat(",", recycleBinId, ",")))
return false;
@@ -64,18 +64,18 @@ namespace Umbraco.Examine
return true;
}
public ContentValueSetValidator(bool supportUnpublishedContent, int? parentId = null,
public ContentValueSetValidator(bool publishedValuesOnly, int? parentId = null,
IEnumerable<string> includeItemTypes = null, IEnumerable<string> excludeItemTypes = null)
: this(supportUnpublishedContent, true, null, parentId, includeItemTypes, excludeItemTypes)
: this(publishedValuesOnly, true, null, parentId, includeItemTypes, excludeItemTypes)
{
}
public ContentValueSetValidator(bool supportUnpublishedContent, bool supportProtectedContent,
public ContentValueSetValidator(bool publishedValuesOnly, bool supportProtectedContent,
IPublicAccessService publicAccessService, int? parentId = null,
IEnumerable<string> includeItemTypes = null, IEnumerable<string> excludeItemTypes = null)
: base(includeItemTypes, excludeItemTypes, null, null)
{
SupportUnpublishedContent = supportUnpublishedContent;
PublishedValuesOnly = publishedValuesOnly;
SupportProtectedContent = supportProtectedContent;
ParentId = parentId;
_publicAccessService = publicAccessService;
@@ -90,7 +90,7 @@ namespace Umbraco.Examine
var isFiltered = baseValidate == ValueSetValidationResult.Filtered;
//check for published content
if (valueSet.Category == IndexTypes.Content && !SupportUnpublishedContent)
if (valueSet.Category == IndexTypes.Content && !PublishedValuesOnly)
{
if (!valueSet.Values.TryGetValue(UmbracoExamineIndexer.PublishedFieldName, out var published))
return ValueSetValidationResult.Failed;

13
src/Umbraco.Examine/IContentValueSetBuilder.cs Normal file
View File

@@ -0,0 +1,13 @@
using Examine;
using Umbraco.Core.Models;
namespace Umbraco.Examine
{
/// <inheritdoc />
/// <summary>
/// Marker interface for a <see cref="T:Examine.ValueSet" /> builder for supporting unpublished content
/// </summary>
public interface IContentValueSetBuilder : IValueSetBuilder<IContent>
{
}
}

15
src/Umbraco.Examine/IContentValueSetValidator.cs
View File

@@ -7,8 +7,21 @@ namespace Umbraco.Examine
/// </summary>
public interface IContentValueSetValidator : IValueSetValidator
{
bool SupportUnpublishedContent { get; }
/// <summary>
/// When set to true the index will only retain published values
/// </summary>
/// <remarks>
/// Any non-published values will not be put or kept in the index:
/// * Deleted, Trashed, non-published Content items
/// * non-published Variants
/// </remarks>
bool PublishedValuesOnly { get; }
/// <summary>
/// If true, protected content will be indexed otherwise it will not be put or kept in the index
/// </summary>
bool SupportProtectedContent { get; }
int? ParentId { get; }
bool ValidatePath(string path, string category);

12
src/Umbraco.Examine/IPublishedContentValueSetBuilder.cs Normal file
View File

@@ -0,0 +1,12 @@
using Examine;
using Umbraco.Core.Models;
namespace Umbraco.Examine
{
/// <summary>
/// Marker interface for a <see cref="ValueSet"/> builder for only published content
/// </summary>
public interface IPublishedContentValueSetBuilder : IValueSetBuilder<IContent>
{
}
}

9
src/Umbraco.Examine/IUmbracoIndexer.cs
View File

@@ -13,8 +13,13 @@ namespace Umbraco.Examine
bool EnableDefaultEventHandler { get; }
/// <summary>
/// When set to true data will not be deleted from the index if the data is being soft deleted (unpublished or trashed)
/// When set to true the index will only retain published values
/// </summary>
bool SupportSoftDelete { get; }
/// <remarks>
/// Any non-published values will not be put or kept in the index:
/// * Deleted, Trashed, non-published Content items
/// * non-published Variants
/// </remarks>
bool PublishedValuesOnly { get; }
}
}

2
src/Umbraco.Examine/MediaValueSetBuilder.cs
View File

@@ -16,7 +16,7 @@ namespace Umbraco.Examine
public MediaValueSetBuilder(PropertyEditorCollection propertyEditors,
IEnumerable<IUrlSegmentProvider> urlSegmentProviders,
IUserService userService)
: base(propertyEditors)
: base(propertyEditors, false)
{
_urlSegmentProviders = urlSegmentProviders;
_userService = userService;

2
src/Umbraco.Examine/MemberValueSetBuilder.cs
View File

@@ -10,7 +10,7 @@ namespace Umbraco.Examine
public class MemberValueSetBuilder : BaseValueSetBuilder<IMember>
{
public MemberValueSetBuilder(PropertyEditorCollection propertyEditors)
: base(propertyEditors)
: base(propertyEditors, false)
{
}

4
src/Umbraco.Examine/PublishedContentIndexPopulator.cs
View File

@@ -14,8 +14,8 @@ namespace Umbraco.Examine
/// </remarks>
public class PublishedContentIndexPopulator : ContentIndexPopulator
{
public PublishedContentIndexPopulator(IContentService contentService, ISqlContext sqlContext, IValueSetBuilder<IContent> contentValueSetBuilder) :
base(false, null, contentService, sqlContext, contentValueSetBuilder)
public PublishedContentIndexPopulator(IContentService contentService, ISqlContext sqlContext, IPublishedContentValueSetBuilder contentValueSetBuilder) :
base(true, null, contentService, sqlContext, contentValueSetBuilder)
{
}
}

4
src/Umbraco.Examine/Umbraco.Examine.csproj
View File

@@ -48,7 +48,7 @@
</ItemGroup>
<ItemGroup>
<!-- note: NuGet deals with transitive references now -->
<PackageReference Include="Examine" Version="1.0.0-beta043" />
<PackageReference Include="Examine" Version="1.0.0-beta045" />
<PackageReference Include="Newtonsoft.Json" Version="11.0.2" />
<PackageReference Include="NPoco" Version="3.9.4" />
</ItemGroup>
@@ -64,11 +64,13 @@
<Compile Include="ContentIndexPopulator.cs" />
<Compile Include="ContentValueSetBuilder.cs" />
<Compile Include="ExamineExtensions.cs" />
<Compile Include="IContentValueSetBuilder.cs" />
<Compile Include="IContentValueSetValidator.cs" />
<Compile Include="IIndexDiagnostics.cs" />
<Compile Include="IIndexPopulator.cs" />
<Compile Include="IndexPopulator.cs" />
<Compile Include="IndexRebuilder.cs" />
<Compile Include="IPublishedContentValueSetBuilder.cs" />
<Compile Include="IUmbracoIndexer.cs" />
<Compile Include="IValueSetBuilder.cs" />
<Compile Include="MediaIndexPopulator.cs" />

6
src/Umbraco.Examine/UmbracoContentIndexer.cs
View File

@@ -64,8 +64,8 @@ namespace Umbraco.Examine
if (validator == null) throw new ArgumentNullException(nameof(validator));
LanguageService = languageService ?? throw new ArgumentNullException(nameof(languageService));
if (validator is ContentValueSetValidator contentValueSetValidator)
SupportSoftDelete = contentValueSetValidator.SupportUnpublishedContent;
if (validator is IContentValueSetValidator contentValueSetValidator)
PublishedValuesOnly = contentValueSetValidator.PublishedValuesOnly;
}
#endregion
@@ -124,7 +124,7 @@ namespace Umbraco.Examine
parentId,
ConfigIndexCriteria.IncludeItemTypes, ConfigIndexCriteria.ExcludeItemTypes);
SupportSoftDelete = supportUnpublished;
PublishedValuesOnly = supportUnpublished;
}
#endregion

15
src/Umbraco.Examine/UmbracoExamineExtensions.cs
View File

@@ -76,17 +76,6 @@ namespace Umbraco.Examine
return fieldQuery;
}
/// <summary>
/// Used to replace any available tokens in the index path before the lucene directory is assigned to the path
/// </summary>
/// <param name="indexSet"></param>
internal static void ReplaceTokensInIndexPath(this IndexSet indexSet)
{
if (indexSet == null) return;
indexSet.IndexPath = indexSet.IndexPath
.Replace("{machinename}", NetworkHelper.FileSafeMachineName)
.Replace("{appdomainappid}", (HttpRuntime.AppDomainAppId ?? string.Empty).ReplaceNonAlphanumericChars(""))
.EnsureEndsWith('/');
}
}
}
}

4
src/Umbraco.Examine/UmbracoExamineIndexDiagnostics.cs
View File

@@ -70,7 +70,7 @@ namespace Umbraco.Examine
_index.LuceneIndexFolder == null
? string.Empty
: _index.LuceneIndexFolder.ToString().ToLowerInvariant().TrimStart(IOHelper.MapPath(SystemDirectories.Root).ToLowerInvariant()).Replace("\\", "/").EnsureStartsWith('/'),
[nameof(UmbracoExamineIndexer.SupportSoftDelete)] = _index.SupportSoftDelete,
[nameof(UmbracoExamineIndexer.PublishedValuesOnly)] = _index.PublishedValuesOnly,
//There's too much info here
//[nameof(UmbracoExamineIndexer.FieldDefinitionCollection)] = _index.FieldDefinitionCollection,
};
@@ -85,7 +85,7 @@ namespace Umbraco.Examine
if (_index.ValueSetValidator is ContentValueSetValidator cvsv)
{
d[nameof(ContentValueSetValidator.SupportUnpublishedContent)] = cvsv.SupportUnpublishedContent;
d[nameof(ContentValueSetValidator.PublishedValuesOnly)] = cvsv.PublishedValuesOnly;
d[nameof(ContentValueSetValidator.SupportProtectedContent)] = cvsv.SupportProtectedContent;
d[nameof(ContentValueSetValidator.ParentId)] = cvsv.ParentId;
}

55
src/Umbraco.Examine/UmbracoExamineIndexer.cs
View File

@@ -1,7 +1,6 @@
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.IO;
using System.Linq;
using Examine.LuceneEngine.Providers;
using Lucene.Net.Analysis;
@@ -14,7 +13,6 @@ using Examine.LuceneEngine.Indexing;
using Lucene.Net.Store;
using Umbraco.Core.Composing;
using Umbraco.Core.Logging;
using Umbraco.Core.Xml;
using Umbraco.Examine.Config;
using Directory = Lucene.Net.Store.Directory;
@@ -147,7 +145,7 @@ namespace Umbraco.Examine
/// </summary>
public bool EnableDefaultEventHandler { get; set; } = true;
public bool SupportSoftDelete { get; protected set; } = false;
public bool PublishedValuesOnly { get; protected set; } = false;
protected ConfigIndexCriteria ConfigIndexCriteria { get; private set; }
@@ -177,38 +175,35 @@ namespace Umbraco.Examine
}
//Need to check if the index set or IndexerData is specified...
if (config["indexSet"] == null && (FieldDefinitionCollection.Count == 0))
if (config["indexSet"] == null && FieldDefinitionCollection.Count == 0)
{
//if we don't have either, then we'll try to set the index set by naming conventions
var found = false;
if (name.EndsWith("Indexer"))
var possibleSuffixes = new[] {"Index", "Indexer"};
foreach (var suffix in possibleSuffixes)
{
var setNameByConvension = name.Remove(name.LastIndexOf("Indexer")) + "IndexSet";
if (!name.EndsWith(suffix)) continue;
var setNameByConvension = name.Remove(name.LastIndexOf(suffix, StringComparison.Ordinal)) + "IndexSet";
//check if we can assign the index set by naming convention
var set = IndexSets.Instance.Sets.Cast<IndexSet>().SingleOrDefault(x => x.SetName == setNameByConvension);
if (set != null)
if (set == null) continue;
//we've found an index set by naming conventions :)
IndexSetName = set.SetName;
var indexSet = IndexSets.Instance.Sets[IndexSetName];
//get the index criteria and ensure folder
ConfigIndexCriteria = CreateFieldDefinitionsFromConfig(indexSet);
foreach (var fieldDefinition in ConfigIndexCriteria.StandardFields.Union(ConfigIndexCriteria.UserFields))
{
//we've found an index set by naming conventions :)
IndexSetName = set.SetName;
var indexSet = IndexSets.Instance.Sets[IndexSetName];
//if tokens are declared in the path, then use them (i.e. {machinename} )
indexSet.ReplaceTokensInIndexPath();
//get the index criteria and ensure folder
ConfigIndexCriteria = CreateFieldDefinitionsFromConfig(indexSet);
foreach (var fieldDefinition in ConfigIndexCriteria.StandardFields.Union(ConfigIndexCriteria.UserFields))
{
FieldDefinitionCollection.TryAdd(fieldDefinition.Name, fieldDefinition);
}
//now set the index folder
LuceneIndexFolder = new DirectoryInfo(Path.Combine(IndexSets.Instance.Sets[IndexSetName].IndexDirectory.FullName, "Index"));
found = true;
FieldDefinitionCollection.TryAdd(fieldDefinition.Name, fieldDefinition);
}
found = true;
break;
}
if (!found)
@@ -229,24 +224,18 @@ namespace Umbraco.Examine
var indexSet = IndexSets.Instance.Sets[IndexSetName];
//if tokens are declared in the path, then use them (i.e. {machinename} )
indexSet.ReplaceTokensInIndexPath();
//get the index criteria and ensure folder
ConfigIndexCriteria = CreateFieldDefinitionsFromConfig(indexSet);
foreach (var fieldDefinition in ConfigIndexCriteria.StandardFields.Union(ConfigIndexCriteria.UserFields))
{
FieldDefinitionCollection.TryAdd(fieldDefinition.Name, fieldDefinition);
}
//now set the index folder
LuceneIndexFolder = new DirectoryInfo(Path.Combine(IndexSets.Instance.Sets[IndexSetName].IndexDirectory.FullName, "Index"));
}
}
base.Initialize(name, config);
}
#endregion
/// <summary>

189
src/Umbraco.Examine/UmbracoExamineSearcher.cs
View File

@@ -1,145 +1,52 @@
using System;
using System.ComponentModel;
using System.IO;
using System.Linq;
using Umbraco.Core;
using Examine.LuceneEngine.Providers;
using Lucene.Net.Analysis;
using Lucene.Net.Index;
using Umbraco.Core.Composing;
using Umbraco.Examine.Config;
using Directory = Lucene.Net.Store.Directory;
using Examine.LuceneEngine;
//using System;
//using System.ComponentModel;
//using System.IO;
//using System.Linq;
//using Umbraco.Core;
//using Examine.LuceneEngine.Providers;
//using Lucene.Net.Analysis;
//using Lucene.Net.Index;
//using Umbraco.Core.Composing;
//using Umbraco.Examine.Config;
//using Directory = Lucene.Net.Store.Directory;
//using Examine.LuceneEngine;
namespace Umbraco.Examine
{
/// <summary>
/// An Examine searcher which uses Lucene.Net as the
/// </summary>
public class UmbracoExamineSearcher : LuceneSearcher
{
private readonly bool _configBased = false;
//namespace Umbraco.Examine
//{
// /// <summary>
// /// An Examine searcher which uses Lucene.Net as the
// /// </summary>
// public class UmbracoExamineSearcher : LuceneSearcher
// {
// /// <summary>
// /// Constructor to allow for creating an indexer at runtime
// /// </summary>
// /// <param name="name"></param>
// /// <param name="luceneDirectory"></param>
// /// <param name="analyzer"></param>
// public UmbracoExamineSearcher(string name, Directory luceneDirectory, Analyzer analyzer, FieldValueTypeCollection fieldValueTypeCollection)
// : base(name, luceneDirectory, analyzer, fieldValueTypeCollection)
// {
// }
/// <summary>
/// Default constructor for config based construction
/// </summary>
[EditorBrowsable(EditorBrowsableState.Never)]
public UmbracoExamineSearcher()
{
_configBased = true;
}
// /// <inheritdoc />
// public UmbracoExamineSearcher(string name, IndexWriter writer, Analyzer analyzer, FieldValueTypeCollection fieldValueTypeCollection)
// : base(name, writer, analyzer, fieldValueTypeCollection)
// {
// }
/// <summary>
/// Constructor to allow for creating an indexer at runtime
/// </summary>
/// <param name="name"></param>
/// <param name="luceneDirectory"></param>
/// <param name="analyzer"></param>
public UmbracoExamineSearcher(string name, Directory luceneDirectory, Analyzer analyzer, FieldValueTypeCollection fieldValueTypeCollection)
: base(name, luceneDirectory, analyzer, fieldValueTypeCollection)
{
_configBased = false;
}
// /// <summary>
// /// Returns a list of fields to search on, this will also exclude the IndexPathFieldName and node type alias
// /// </summary>
// /// <returns></returns>
// public override string[] GetAllIndexedFields()
// {
// var fields = base.GetAllIndexedFields();
// return fields
// .Where(x => x != UmbracoExamineIndexer.IndexPathFieldName)
// .Where(x => x != LuceneIndex.ItemTypeFieldName)
// .ToArray();
// }
/// <inheritdoc />
public UmbracoExamineSearcher(string name, IndexWriter writer, Analyzer analyzer, FieldValueTypeCollection fieldValueTypeCollection)
: base(name, writer, analyzer, fieldValueTypeCollection)
{
_configBased = false;
}
/// <summary>
/// Name of the Lucene.NET index set
/// </summary>
public string IndexSetName { get; private set; }
/// <summary>
/// Method used for initializing based on a configuration based searcher
/// </summary>
/// <param name="name"></param>
/// <param name="config"></param>
public override void Initialize(string name, System.Collections.Specialized.NameValueCollection config)
{
if (string.IsNullOrWhiteSpace(name)) throw new ArgumentException("Value cannot be null or whitespace.", nameof(name));
//We need to check if we actually can initialize, if not then don't continue
if (CanInitialize() == false)
{
return;
}
//need to check if the index set is specified, if it's not, we'll see if we can find one by convension
//if the folder is not null and the index set is null, we'll assume that this has been created at runtime.
//NOTE: Don't proceed if the _luceneDirectory is set since we already know where to look.
var luceneDirectory = GetLuceneDirectory();
if (config["indexSet"] == null && (LuceneIndexFolder == null && luceneDirectory == null))
{
//if we don't have either, then we'll try to set the index set by naming convensions
var found = false;
if (name.EndsWith("Searcher"))
{
var setNameByConvension = name.Remove(name.LastIndexOf("Searcher")) + "IndexSet";
//check if we can assign the index set by naming convension
var set = IndexSets.Instance.Sets.Cast<IndexSet>().SingleOrDefault(x => x.SetName == setNameByConvension);
if (set != null)
{
set.ReplaceTokensInIndexPath();
//we've found an index set by naming convensions :)
IndexSetName = set.SetName;
found = true;
}
}
if (!found)
throw new ArgumentNullException("indexSet on LuceneExamineIndexer provider has not been set in configuration");
//get the folder to index
LuceneIndexFolder = new DirectoryInfo(Path.Combine(IndexSets.Instance.Sets[IndexSetName].IndexDirectory.FullName, "Index"));
}
else if (config["indexSet"] != null && luceneDirectory == null)
{
if (IndexSets.Instance.Sets[config["indexSet"]] == null)
throw new ArgumentException("The indexSet specified for the LuceneExamineIndexer provider does not exist");
IndexSetName = config["indexSet"];
var indexSet = IndexSets.Instance.Sets[IndexSetName];
indexSet.ReplaceTokensInIndexPath();
//get the folder to index
LuceneIndexFolder = new DirectoryInfo(Path.Combine(indexSet.IndexDirectory.FullName, "Index"));
}
base.Initialize(name, config);
}
/// <summary>
/// Returns true if the Umbraco application is in a state that we can initialize the examine indexes
/// </summary>
protected bool CanInitialize()
{
// only affects indexers that are config file based, if an index was created via code then
// this has no effect, it is assumed the index would not be created if it could not be initialized
return _configBased == false || Current.RuntimeState.Level == RuntimeLevel.Run;
}
/// <summary>
/// Returns a list of fields to search on, this will also exclude the IndexPathFieldName and node type alias
/// </summary>
/// <returns></returns>
public override string[] GetAllIndexedFields()
{
var fields = base.GetAllIndexedFields();
return fields
.Where(x => x != UmbracoExamineIndexer.IndexPathFieldName)
.Where(x => x != LuceneIndex.ItemTypeFieldName)
.ToArray();
}
}
}
// }
//}

2
src/Umbraco.Tests/Umbraco.Tests.csproj
View File

@@ -77,7 +77,7 @@
<ItemGroup>
<PackageReference Include="AutoMapper" Version="7.0.1" />
<PackageReference Include="Castle.Core" Version="4.2.1" />
<PackageReference Include="Examine" Version="1.0.0-beta043" />
<PackageReference Include="Examine" Version="1.0.0-beta045" />
<PackageReference Include="HtmlAgilityPack">
<Version>1.8.9</Version>
</PackageReference>

4
src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs
View File

@@ -31,7 +31,7 @@ namespace Umbraco.Tests.UmbracoExamine
{
public static ContentValueSetBuilder GetContentValueSetBuilder(PropertyEditorCollection propertyEditors)
{
var contentValueSetBuilder = new ContentValueSetBuilder(propertyEditors, new[] { new DefaultUrlSegmentProvider() }, IndexInitializer.GetMockUserService());
var contentValueSetBuilder = new ContentValueSetBuilder(propertyEditors, new[] { new DefaultUrlSegmentProvider() }, GetMockUserService(), true);
return contentValueSetBuilder;
}
@@ -44,7 +44,7 @@ namespace Umbraco.Tests.UmbracoExamine
public static MediaIndexPopulator GetMediaIndexRebuilder(PropertyEditorCollection propertyEditors, IMediaService mediaService)
{
var mediaValueSetBuilder = new MediaValueSetBuilder(propertyEditors, new[] { new DefaultUrlSegmentProvider() }, IndexInitializer.GetMockUserService());
var mediaValueSetBuilder = new MediaValueSetBuilder(propertyEditors, new[] { new DefaultUrlSegmentProvider() }, GetMockUserService());
var mediaIndexDataSource = new MediaIndexPopulator(null, mediaService, mediaValueSetBuilder);
return mediaIndexDataSource;
}

3
src/Umbraco.Web.UI.Client/src/views/dashboard/settings/examinemanagement.controller.js
View File

@@ -176,9 +176,6 @@ function ExamineManagementController($scope, umbRequestHelper, $http, $q, $timeo
'Failed to retrieve searcher details')
.then(data => {
vm.searcherDetails = data;
for (var s in vm.searcherDetails) {
vm.searcherDetails[s].searchType = "text";
}
})
])
.then(() => { vm.loading = false });

165
src/Umbraco.Web.UI.Client/src/views/dashboard/settings/examinemanagement.html
View File

@@ -86,8 +86,107 @@
</div>
<div ng-if="vm.viewState === 'searcher-details'">
<!-- TODO: Add searcher properties -->
<!-- TODO: Add search tools -->
<umb-editor-sub-header>
<umb-editor-sub-header-content-left>
<a class="umb-healthcheck-back-link" href="" ng-click="vm.setViewState('list');">&larr; Back to overview</a>
</umb-editor-sub-header-content-left>
</umb-editor-sub-header>
<div class="umb-healthcheck-group__details">
<div class="umb-healthcheck-group__details-group">
<div class="umb-healthcheck-group__details-group-title">
<div class="umb-healthcheck-group__details-group-name">{{ vm.selectedSearcher.name }}</div>
</div>
<div class="umb-healthcheck-group__details-checks">
<!-- Search Tool -->
<div class="umb-healthcheck-group__details-check">
<div class="umb-healthcheck-group__details-check-title">
<div class="umb-healthcheck-group__details-check-name">Search</div>
<div class="umb-healthcheck-group__details-check-description">Search the index and view the results</div>
</div>
<div class="umb-healthcheck-group__details-status">
<div class="umb-healthcheck-group__details-status-content">
<div class="umb-healthcheck-group__details-status-actions">
<div class="umb-healthcheck-group__details-status-action">
<ng-form name="searchTools">
<div class="row form-search">
<div>
<input type="text" class="search-query"
ng-model="vm.searchText" no-dirty-check
ng-keypress="vm.search(vm.selectedSearcher, $event)" />
<umb-button disabled="vm.selectedSearcher.isProcessing"
type="button"
button-style="success"
action="vm.search(vm.selectedSearcher)"
label="Search">
</umb-button>
</div>
</div>
<div ng-if="!vm.selectedSearcher.isProcessing && vm.searchResults">
<br />
<table class="table table-bordered table-condensed">
<thead>
<tr>
<th class="score">Score</th>
<th class="id">Id</th>
<th>Name</th>
</tr>
</thead>
<tbody>
<tr ng-repeat="result in vm.searchResults.results track by $index">
<td>{{result.score}}</td>
<td>{{result.id}}</td>
<td>
<span>{{result.values['nodeName']}}</span>&nbsp;
<a class="color-green" href="" ng-click="vm.showSearchResultDialog(result.values)">
<em>({{result.fieldCount}} fields)</em>
</a>
</td>
</tr>
</tbody>
</table>
<div class="flex justify-center">
<umb-pagination page-number="vm.searchResults.pageNumber"
total-pages="vm.searchResults.totalPages"
on-next="vm.nextSearchResultPage"
on-prev="vm.prevSearchResultPage"
on-go-to-page="vm.goToPageSearchResultPage">
</umb-pagination>
</div>
</div>
</ng-form>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
<div ng-if="vm.viewState === 'index-details'">
@@ -140,33 +239,6 @@
</div>
<!-- Index Stats -->
<div class="umb-healthcheck-group__details-check">
<div class="umb-healthcheck-group__details-check-title">
<div class="umb-healthcheck-group__details-check-name">Index info</div>
<div class="umb-healthcheck-group__details-check-description">Lists the properties of the index</div>
</div>
<div class="umb-healthcheck-group__details-status">
<div class="umb-healthcheck-group__details-status-content">
<table class="table table-bordered table-condensed">
<caption>&nbsp;</caption>
<tr ng-repeat="(key, val) in vm.selectedIndex.providerProperties track by $index">
<th>{{key}}</th>
<td>{{val}}</td>
</tr>
</table>
</div>
</div>
</div>
<!-- Search Tool -->
<div class="umb-healthcheck-group__details-check">
@@ -200,7 +272,7 @@
</div>
</div>
<div ng-hide="vm.selectedIndex.isProcessing || !vm.searchResults">
<div ng-if="!vm.selectedIndex.isProcessing && vm.searchResults">
<br />
<table class="table table-bordered table-condensed">
@@ -212,7 +284,7 @@
</tr>
</thead>
<tbody>
<tr ng-repeat="result in vm.searchResults.results track by result.id">
<tr ng-repeat="result in vm.searchResults.results track by $index">
<td>{{result.score}}</td>
<td>{{result.id}}</td>
<td>
@@ -246,6 +318,35 @@
</div>
<!-- Index Stats -->
<div class="umb-healthcheck-group__details-check">
<div class="umb-healthcheck-group__details-check-title">
<div class="umb-healthcheck-group__details-check-name">Index info</div>
<div class="umb-healthcheck-group__details-check-description">Lists the properties of the index</div>
</div>
<div class="umb-healthcheck-group__details-status">
<div class="umb-healthcheck-group__details-status-content">
<table class="table table-bordered table-condensed">
<caption>&nbsp;</caption>
<tr ng-repeat="(key, val) in vm.selectedIndex.providerProperties track by $index">
<th>{{key}}</th>
<td>{{val}}</td>
</tr>
</table>
</div>
</div>
</div>
<!-- Rebuild -->
<div class="umb-healthcheck-group__details-check">
<div class="umb-healthcheck-group__details-check-title">
@@ -292,8 +393,6 @@
</div>
</div>
</div>

2
src/Umbraco.Web.UI/Umbraco.Web.UI.csproj
View File

@@ -88,7 +88,7 @@
<PackageReference Include="CSharpTest.Net.Collections" Version="14.906.1403.1082" />
<PackageReference Include="ClientDependency" Version="1.9.7" />
<PackageReference Include="ClientDependency-Mvc5" Version="1.8.0.0" />
<PackageReference Include="Examine" Version="1.0.0-beta043" />
<PackageReference Include="Examine" Version="1.0.0-beta045" />
<PackageReference Include="ImageProcessor.Web" Version="4.9.3.25" />
<PackageReference Include="ImageProcessor.Web.Config" Version="2.4.1.19" />
<PackageReference Include="Microsoft.AspNet.Identity.Owin" Version="2.2.2" />

6
src/Umbraco.Web.UI/config/ExamineIndex.config
View File

@@ -4,7 +4,11 @@ Umbraco examine is an extensible indexer and search engine.
This configuration file can be extended to create your own index sets.
Index/Search providers can be defined in the UmbracoSettings.config
More information and documentation can be found on GitHub: https://github.com/Shazwazza/Examine/
More information and documentation can be found on Our:
https://our.umbraco.com/Documentation/Reference/Searching/Examine/
https://our.umbraco.com/Documentation/Reference/Config/ExamineIndex/
https://our.umbraco.com/Documentation/Reference/Config/ExamineSettings/
-->
<ExamineLuceneIndexSets>
</ExamineLuceneIndexSets>

8
src/Umbraco.Web.UI/config/ExamineSettings.config
View File

@@ -4,7 +4,10 @@ Umbraco examine is an extensible indexer and search engine.
This configuration file can be extended to add your own search/index providers.
Index sets can be defined in the ExamineIndex.config if you're using the standard provider model.
More information and documentation can be found on GitHub: https://github.com/Shazwazza/Examine/
More information and documentation can be found on Our:
https://our.umbraco.com/Documentation/Reference/Searching/Examine/
https://our.umbraco.com/Documentation/Reference/Config/ExamineIndex/
https://our.umbraco.com/Documentation/Reference/Config/ExamineSettings/
-->
<Examine>
<ExamineIndexProviders>
@@ -14,7 +17,8 @@ More information and documentation can be found on GitHub: https://github.com/Sh
<ExamineSearchProviders>
<providers>
<add name="MultiIndexSearcher" type="Examine.LuceneEngine.Providers.MultiIndexSearcher, Examine" indexes="InternalIndex,ExternalIndex" />
</providers>
</ExamineSearchProviders>
</Examine>
</Examine>

3
src/Umbraco.Web/Editors/ExamineManagementController.cs
View File

@@ -86,7 +86,8 @@ namespace Umbraco.Web.Editors
{
Id = x.Id,
Score = x.Score,
Values = x.Values
//order the values by key
Values = new Dictionary<string, string>(x.Values.OrderBy(y => y.Key).ToDictionary(y => y.Key, y => y.Value))
})
};
}

12
src/Umbraco.Web/PropertyEditors/GridPropertyIndexValues.cs
View File

@@ -18,11 +18,11 @@ namespace Umbraco.Web.PropertyEditors
/// </summary>
public class GridPropertyIndexValues : IPropertyIndexValues
{
public IEnumerable<KeyValuePair<string, object[]>> GetIndexValues(Property property, string culture, string segment)
public IEnumerable<KeyValuePair<string, IEnumerable<object>>> GetIndexValues(Property property, string culture, string segment, bool published)
{
var result = new List<KeyValuePair<string, object[]>>();
var result = new List<KeyValuePair<string, IEnumerable<object>>>();
var val = property.GetValue(culture, segment);
var val = property.GetValue(culture, segment, published);
//if there is a value, it's a string and it's detected as json
if (val is string rawVal && rawVal.DetectIsJson())
@@ -51,7 +51,7 @@ namespace Umbraco.Web.PropertyEditors
sb.Append(" ");
//add the row name as an individual field
result.Add(new KeyValuePair<string, object[]>($"{property.Alias}.{rowName}", new[] { str }));
result.Add(new KeyValuePair<string, IEnumerable<object>>($"{property.Alias}.{rowName}", new[] { str }));
}
}
}
@@ -59,10 +59,10 @@ namespace Umbraco.Web.PropertyEditors
if (sb.Length > 0)
{
//First save the raw value to a raw field
result.Add(new KeyValuePair<string, object[]>($"{UmbracoExamineIndexer.RawFieldPrefix}{property.Alias}", new[] { rawVal }));
result.Add(new KeyValuePair<string, IEnumerable<object>>($"{UmbracoExamineIndexer.RawFieldPrefix}{property.Alias}", new[] { rawVal }));
//index the property with the combined/cleaned value
result.Add(new KeyValuePair<string, object[]>(property.Alias, new[] { sb.ToString() }));
result.Add(new KeyValuePair<string, IEnumerable<object>>(property.Alias, new[] { sb.ToString() }));
}
}
catch (InvalidCastException)

8
src/Umbraco.Web/PropertyEditors/RichTextPropertyEditor.cs
View File

@@ -95,16 +95,16 @@ namespace Umbraco.Web.PropertyEditors
internal class RichTextPropertyIndexValues : IPropertyIndexValues
{
public IEnumerable<KeyValuePair<string, object[]>> GetIndexValues(Property property, string culture, string segment)
public IEnumerable<KeyValuePair<string, IEnumerable<object>>> GetIndexValues(Property property, string culture, string segment, bool published)
{
var val = property.GetValue(culture, segment);
var val = property.GetValue(culture, segment, published);
if (!(val is string strVal)) yield break;
//index the stripped html values
yield return new KeyValuePair<string, object[]>(property.Alias, new object[] { strVal.StripHtml() });
yield return new KeyValuePair<string, IEnumerable<object>>(property.Alias, new object[] { strVal.StripHtml() });
//store the raw value
yield return new KeyValuePair<string, object[]>($"{UmbracoExamineIndexer.RawFieldPrefix}{property.Alias}", new object[] { strVal });
yield return new KeyValuePair<string, IEnumerable<object>>($"{UmbracoExamineIndexer.RawFieldPrefix}{property.Alias}", new object[] { strVal });
}
}
}

35
src/Umbraco.Web/PublishedContentQuery.cs
View File

@@ -242,15 +242,12 @@ namespace Umbraco.Web
var searcher = index.GetSearcher();
if (skip == 0 && take == 0)
{
var results = searcher.Search(term, useWildCards);
totalRecords = results.TotalItemCount;
return results.ToPublishedSearchResults(_contentCache);
}
var results = skip == 0 && take == 0
? searcher.Search(term, true)
: searcher.Search(term, true, maxResults: skip + take);
var criteria = SearchAllFields(term, useWildCards, searcher, index);
return Search(skip, take, out totalRecords, criteria, searcher);
totalRecords = results.TotalItemCount;
return results.ToPublishedSearchResults(_contentCache);
}
/// <inheritdoc />
@@ -280,28 +277,6 @@ namespace Umbraco.Web
return results.ToPublishedSearchResults(_contentCache);
}
/// <summary>
/// Creates an ISearchCriteria for searching all fields in a <see cref="BaseLuceneSearcher"/>.
/// </summary>
private ISearchCriteria SearchAllFields(string searchText, bool useWildcards, ISearcher searcher, IIndex indexer)
{
var sc = searcher.CreateCriteria();
//if we're dealing with a lucene searcher, we can get all of it's indexed fields,
//else we can get the defined fields for the index.
var searchFields = (searcher is BaseLuceneSearcher luceneSearcher)
? luceneSearcher.GetAllIndexedFields()
: indexer.FieldDefinitionCollection.Keys;
//this is what Examine does internally to create ISearchCriteria for searching all fields
var strArray = searchText.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
sc = useWildcards == false
? sc.GroupedOr(searchFields, strArray).Compile()
: sc.GroupedOr(searchFields, strArray.Select(x => (IExamineValue)new ExamineValue(Examineness.ComplexWildcard, x.MultipleCharacterWildcard().Value)).ToArray()).Compile();
return sc;
}
#endregion
}

312
src/Umbraco.Web/Search/ExamineComponent.cs
View File

@@ -22,8 +22,10 @@ using Umbraco.Examine;
using Umbraco.Core.Persistence.DatabaseModelDefinitions;
using Umbraco.Web.Scheduling;
using System.Threading.Tasks;
using Examine.LuceneEngine.Directories;
using LightInject;
using Umbraco.Core.Composing;
using Umbraco.Core.Strings;
namespace Umbraco.Web.Search
{
@@ -34,7 +36,8 @@ namespace Umbraco.Web.Search
public sealed class ExamineComponent : UmbracoComponentBase, IUmbracoCoreComponent
{
private IExamineManager _examineManager;
private IValueSetBuilder<IContent> _contentValueSetBuilder;
private IContentValueSetBuilder _contentValueSetBuilder;
private IPublishedContentValueSetBuilder _publishedContentValueSetBuilder;
private IValueSetBuilder<IMedia> _mediaValueSetBuilder;
private IValueSetBuilder<IMember> _memberValueSetBuilder;
private static bool _disableExamineIndexing = false;
@@ -71,7 +74,18 @@ namespace Umbraco.Web.Search
composition.Container.RegisterSingleton<IndexRebuilder>();
composition.Container.RegisterSingleton<IUmbracoIndexesCreator, UmbracoIndexesCreator>();
composition.Container.RegisterSingleton<IValueSetBuilder<IContent>, ContentValueSetBuilder>();
composition.Container.Register<IPublishedContentValueSetBuilder, PerContainerLifetime>(factory =>
new ContentValueSetBuilder(
factory.GetInstance<PropertyEditorCollection>(),
factory.GetInstance<IEnumerable<IUrlSegmentProvider>>(),
factory.GetInstance<IUserService>(),
false));
composition.Container.Register<IContentValueSetBuilder, PerContainerLifetime>(factory =>
new ContentValueSetBuilder(
factory.GetInstance<PropertyEditorCollection>(),
factory.GetInstance<IEnumerable<IUrlSegmentProvider>>(),
factory.GetInstance<IUserService>(),
true));
composition.Container.RegisterSingleton<IValueSetBuilder<IMedia>, MediaValueSetBuilder>();
composition.Container.RegisterSingleton<IValueSetBuilder<IMember>, MemberValueSetBuilder>();
}
@@ -80,7 +94,8 @@ namespace Umbraco.Web.Search
IExamineManager examineManager, ProfilingLogger profilingLogger,
IScopeProvider scopeProvider, IUmbracoIndexesCreator indexCreator,
IndexRebuilder indexRebuilder, ServiceContext services,
IValueSetBuilder<IContent> contentValueSetBuilder,
IContentValueSetBuilder contentValueSetBuilder,
IPublishedContentValueSetBuilder publishedContentValueSetBuilder,
IValueSetBuilder<IMedia> mediaValueSetBuilder,
IValueSetBuilder<IMember> memberValueSetBuilder)
{
@@ -88,6 +103,7 @@ namespace Umbraco.Web.Search
_scopeProvider = scopeProvider;
_examineManager = examineManager;
_contentValueSetBuilder = contentValueSetBuilder;
_publishedContentValueSetBuilder = publishedContentValueSetBuilder;
_mediaValueSetBuilder = mediaValueSetBuilder;
_memberValueSetBuilder = memberValueSetBuilder;
@@ -98,7 +114,7 @@ namespace Umbraco.Web.Search
//we want to tell examine to use a different fs lock instead of the default NativeFSFileLock which could cause problems if the appdomain
//terminates and in some rare cases would only allow unlocking of the file if IIS is forcefully terminated. Instead we'll rely on the simplefslock
//which simply checks the existence of the lock file
DirectoryTracker.DefaultLockFactory = d =>
DirectoryFactory.DefaultLockFactory = d =>
{
var simpleFsLockFactory = new NoPrefixSimpleFsLockFactory(d);
return simpleFsLockFactory;
@@ -221,8 +237,106 @@ namespace Umbraco.Web.Search
}
}
}
#region Cache refresher updated event handlers
/// <summary>
/// Updates indexes based on content changes
/// </summary>
/// <param name="sender"></param>
/// <param name="args"></param>
private void ContentCacheRefresherUpdated(ContentCacheRefresher sender, CacheRefresherEventArgs args)
{
if (Suspendable.ExamineEvents.CanIndex == false)
return;
if (args.MessageType != MessageType.RefreshByPayload)
throw new NotSupportedException();
var contentService = _services.ContentService;
foreach (var payload in (ContentCacheRefresher.JsonPayload[])args.MessageObject)
{
if (payload.ChangeTypes.HasType(TreeChangeTypes.Remove))
{
// delete content entirely (with descendants)
// false: remove entirely from all indexes
DeleteIndexForEntity(payload.Id, false);
}
else if (payload.ChangeTypes.HasType(TreeChangeTypes.RefreshAll))
{
// ExamineEvents does not support RefreshAll
// just ignore that payload
// so what?!
//fixme: Rebuild the index at this point?
}
else // RefreshNode or RefreshBranch (maybe trashed)
{
// don't try to be too clever - refresh entirely
// there has to be race conds in there ;-(
var content = contentService.GetById(payload.Id);
if (content == null)
{
// gone fishing, remove entirely from all indexes (with descendants)
DeleteIndexForEntity(payload.Id, false);
continue;
}
IContent published = null;
if (content.Published && contentService.IsPathPublished(content))
published = content;
if (published == null)
DeleteIndexForEntity(payload.Id, true);
// just that content
ReIndexForContent(content, published != null);
// branch
if (payload.ChangeTypes.HasType(TreeChangeTypes.RefreshBranch))
{
var masked = published == null ? null : new List<int>();
const int pageSize = 500;
var page = 0;
var total = long.MaxValue;
while (page * pageSize < total)
{
var descendants = contentService.GetPagedDescendants(content.Id, page++, pageSize, out total,
//order by shallowest to deepest, this allows us to check it's published state without checking every item
ordering: Ordering.By("Path", Direction.Ascending));
foreach (var descendant in descendants)
{
published = null;
if (masked != null) // else everything is masked
{
if (masked.Contains(descendant.ParentId) || !descendant.Published)
masked.Add(descendant.Id);
else
published = descendant;
}
ReIndexForContent(descendant, published != null);
}
}
}
}
// NOTE
//
// DeleteIndexForEntity is handled by UmbracoContentIndexer.DeleteFromIndex() which takes
// care of also deleting the descendants
//
// ReIndexForContent is NOT taking care of descendants so we have to reload everything
// again in order to process the branch - we COULD improve that by just reloading the
// XML from database instead of reloading content & re-serializing!
//
// BUT ... pretty sure it is! see test "Index_Delete_Index_Item_Ensure_Heirarchy_Removed"
}
}
private void MemberCacheRefresherUpdated(MemberCacheRefresher sender, CacheRefresherEventArgs args)
{
if (Suspendable.ExamineEvents.CanIndex == false)
@@ -292,15 +406,18 @@ namespace Umbraco.Web.Search
else // RefreshNode or RefreshBranch (maybe trashed)
{
var media = mediaService.GetById(payload.Id);
if (media == null || media.Trashed)
if (media == null)
{
// gone fishing, remove entirely
DeleteIndexForEntity(payload.Id, false);
continue;
}
if (media.Trashed)
DeleteIndexForEntity(payload.Id, true);
// just that media
ReIndexForMedia(media, media.Trashed == false);
ReIndexForMedia(media, !media.Trashed);
// branch
if (payload.ChangeTypes.HasType(TreeChangeTypes.RefreshBranch))
@@ -313,7 +430,7 @@ namespace Umbraco.Web.Search
var descendants = mediaService.GetPagedDescendants(media.Id, page++, pageSize, out total);
foreach (var descendant in descendants)
{
ReIndexForMedia(descendant, descendant.Trashed == false);
ReIndexForMedia(descendant, !descendant.Trashed);
}
}
}
@@ -461,155 +578,33 @@ namespace Umbraco.Web.Search
foreach (var c in contentToRefresh)
{
IContent published = null;
var isPublished = false;
if (c.Published)
{
if (publishChecked.TryGetValue(c.ParentId, out var isPublished))
{
//if the parent's published path has already been verified then this is published
if (isPublished)
published = c;
}
else
if (!publishChecked.TryGetValue(c.ParentId, out isPublished))
{
//nothing by parent id, so query the service and cache the result for the next child to check against
isPublished = _services.ContentService.IsPathPublished(c);
publishChecked[c.Id] = isPublished;
if (isPublished)
published = c;
}
}
ReIndexForContent(c, published);
ReIndexForContent(c, isPublished);
}
}
}
/// <summary>
/// Updates indexes based on content changes
/// </summary>
/// <param name="sender"></param>
/// <param name="args"></param>
private void ContentCacheRefresherUpdated(ContentCacheRefresher sender, CacheRefresherEventArgs args)
{
if (Suspendable.ExamineEvents.CanIndex == false)
return;
if (args.MessageType != MessageType.RefreshByPayload)
throw new NotSupportedException();
var contentService = _services.ContentService;
foreach (var payload in (ContentCacheRefresher.JsonPayload[])args.MessageObject)
{
if (payload.ChangeTypes.HasType(TreeChangeTypes.Remove))
{
// delete content entirely (with descendants)
// false: remove entirely from all indexes
DeleteIndexForEntity(payload.Id, false);
}
else if (payload.ChangeTypes.HasType(TreeChangeTypes.RefreshAll))
{
// ExamineEvents does not support RefreshAll
// just ignore that payload
// so what?!
//fixme: Rebuild the index at this point?
}
else // RefreshNode or RefreshBranch (maybe trashed)
{
// don't try to be too clever - refresh entirely
// there has to be race conds in there ;-(
var content = contentService.GetById(payload.Id);
if (content == null || content.Trashed)
{
// gone fishing, remove entirely from all indexes (with descendants)
DeleteIndexForEntity(payload.Id, false);
continue;
}
IContent published = null;
if (content.Published && contentService.IsPathPublished(content))
published = content;
// just that content
ReIndexForContent(content, published);
// branch
if (payload.ChangeTypes.HasType(TreeChangeTypes.RefreshBranch))
{
var masked = published == null ? null : new List<int>();
const int pageSize = 500;
var page = 0;
var total = long.MaxValue;
while (page * pageSize < total)
{
var descendants = contentService.GetPagedDescendants(content.Id, page++, pageSize, out total,
//order by shallowest to deepest, this allows us to check it's published state without checking every item
ordering: Ordering.By("Path", Direction.Ascending));
foreach (var descendant in descendants)
{
published = null;
if (masked != null) // else everything is masked
{
if (masked.Contains(descendant.ParentId) || !descendant.Published)
masked.Add(descendant.Id);
else
published = descendant;
}
ReIndexForContent(descendant, published);
}
}
}
}
// NOTE
//
// DeleteIndexForEntity is handled by UmbracoContentIndexer.DeleteFromIndex() which takes
// care of also deleting the descendants
//
// ReIndexForContent is NOT taking care of descendants so we have to reload everything
// again in order to process the branch - we COULD improve that by just reloading the
// XML from database instead of reloading content & re-serializing!
//
// BUT ... pretty sure it is! see test "Index_Delete_Index_Item_Ensure_Heirarchy_Removed"
}
}
#endregion
#region ReIndex/Delete for entity
private void ReIndexForContent(IContent content, IContent published)
{
if (published != null && content.VersionId == published.VersionId)
{
ReIndexForContent(content); // same = both
}
else
{
if (published == null)
{
// remove 'published' - keep 'draft'
DeleteIndexForEntity(content.Id, true);
}
else
{
// index 'published' - don't overwrite 'draft'
ReIndexForContent(published, false);
}
ReIndexForContent(content, true); // index 'draft'
}
}
private void ReIndexForContent(IContent sender, bool? supportUnpublished = null)
private void ReIndexForContent(IContent sender, bool isPublished)
{
var actions = DeferedActions.Get(_scopeProvider);
if (actions != null)
actions.Add(new DeferedReIndexForContent(this, sender, supportUnpublished));
actions.Add(new DeferedReIndexForContent(this, sender, isPublished));
else
DeferedReIndexForContent.Execute(this, sender, supportUnpublished);
DeferedReIndexForContent.Execute(this, sender, isPublished);
}
private void ReIndexForMember(IMember member)
@@ -621,17 +616,17 @@ namespace Umbraco.Web.Search
DeferedReIndexForMember.Execute(this, member);
}
private void ReIndexForMedia(IMedia sender, bool isMediaPublished)
private void ReIndexForMedia(IMedia sender, bool isPublished)
{
var actions = DeferedActions.Get(_scopeProvider);
if (actions != null)
actions.Add(new DeferedReIndexForMedia(this, sender, isMediaPublished));
actions.Add(new DeferedReIndexForMedia(this, sender, isPublished));
else
DeferedReIndexForMedia.Execute(this, sender, isMediaPublished);
DeferedReIndexForMedia.Execute(this, sender, isPublished);
}
/// <summary>
/// Remove items from any index that doesn't support unpublished content
/// Remove items from an index
/// </summary>
/// <param name="entityId"></param>
/// <param name="keepIfUnpublished">
@@ -687,30 +682,33 @@ namespace Umbraco.Web.Search
{
private readonly ExamineComponent _examineComponent;
private readonly IContent _content;
private readonly bool? _supportUnpublished;
private readonly bool _isPublished;
public DeferedReIndexForContent(ExamineComponent examineComponent, IContent content, bool? supportUnpublished)
public DeferedReIndexForContent(ExamineComponent examineComponent, IContent content, bool isPublished)
{
_examineComponent = examineComponent;
_content = content;
_supportUnpublished = supportUnpublished;
_isPublished = isPublished;
}
public override void Execute()
{
Execute(_examineComponent, _content, _supportUnpublished);
Execute(_examineComponent, _content, _isPublished);
}
public static void Execute(ExamineComponent examineComponent, IContent content, bool? supportUnpublished)
public static void Execute(ExamineComponent examineComponent, IContent content, bool isPublished)
{
var valueSet = examineComponent._contentValueSetBuilder.GetValueSets(content).ToList();
foreach (var index in examineComponent._examineManager.Indexes.OfType<IUmbracoIndexer>()
// only for the specified indexers
.Where(x => supportUnpublished.HasValue == false || supportUnpublished.Value == x.SupportSoftDelete)
//filter the indexers
.Where(x => isPublished || !x.PublishedValuesOnly)
.Where(x => x.EnableDefaultEventHandler))
{
index.IndexItems(valueSet);
//for content we have a different builder for published vs unpublished
var builder = index.PublishedValuesOnly
? examineComponent._publishedContentValueSetBuilder
: (IValueSetBuilder<IContent>)examineComponent._contentValueSetBuilder;
index.IndexItems(builder.GetValueSets(content));
}
}
}
@@ -738,9 +736,8 @@ namespace Umbraco.Web.Search
var valueSet = examineComponent._mediaValueSetBuilder.GetValueSets(media).ToList();
foreach (var index in examineComponent._examineManager.Indexes.OfType<IUmbracoIndexer>()
// index this item for all indexers if the media is not trashed, otherwise if the item is trashed
// then only index this for indexers supporting unpublished media
.Where(x => isPublished || (x.SupportSoftDelete))
//filter the indexers
.Where(x => isPublished || !x.PublishedValuesOnly)
.Where(x => x.EnableDefaultEventHandler))
{
index.IndexItems(valueSet);
@@ -768,7 +765,7 @@ namespace Umbraco.Web.Search
{
var valueSet = examineComponent._memberValueSetBuilder.GetValueSets(member).ToList();
foreach (var index in examineComponent._examineManager.Indexes.OfType<IUmbracoIndexer>()
//ensure that only the providers are flagged to listen execute
//filter the indexers
.Where(x => x.EnableDefaultEventHandler))
{
index.IndexItems(valueSet);
@@ -798,9 +795,8 @@ namespace Umbraco.Web.Search
{
var strId = id.ToString(CultureInfo.InvariantCulture);
foreach (var index in examineComponent._examineManager.Indexes.OfType<IUmbracoIndexer>()
// if keepIfUnpublished == true then only delete this item from indexes not supporting unpublished content,
// otherwise if keepIfUnpublished == false then remove from all indexes
.Where(x => keepIfUnpublished == false || x.SupportSoftDelete == false)
.Where(x => (keepIfUnpublished && !x.PublishedValuesOnly) || !keepIfUnpublished)
.Where(x => x.EnableDefaultEventHandler))
{
index.DeleteFromIndex(strId);

2
src/Umbraco.Web/Umbraco.Web.csproj
View File

@@ -62,7 +62,7 @@
<PackageReference Include="AutoMapper" Version="7.0.1" />
<PackageReference Include="ClientDependency" Version="1.9.7" />
<PackageReference Include="CSharpTest.Net.Collections" Version="14.906.1403.1082" />
<PackageReference Include="Examine" Version="1.0.0-beta043" />
<PackageReference Include="Examine" Version="1.0.0-beta045" />
<PackageReference Include="HtmlAgilityPack" Version="1.8.9" />
<PackageReference Include="ImageProcessor">
<Version>2.6.2.25</Version>

75
src/Umbraco.Web/UmbracoHelper.cs
View File

@@ -30,13 +30,11 @@ namespace Umbraco.Web
private readonly IPublishedContent _currentPage;
private readonly IPublishedContentQuery _iQuery;
private readonly ServiceContext _services;
private readonly CacheHelper _appCache;
private IUmbracoComponentRenderer _componentRenderer;
private PublishedContentQuery _query;
private MembershipHelper _membershipHelper;
private ITagQuery _tag;
private IDataTypeService _dataTypeService;
private ICultureDictionary _cultureDictionary;
#region Constructors
@@ -48,34 +46,21 @@ namespace Umbraco.Web
internal UmbracoHelper(UmbracoContext umbracoContext, IPublishedContent content,
IPublishedContentQuery query,
ITagQuery tagQuery,
IDataTypeService dataTypeService,
ICultureDictionary cultureDictionary,
IUmbracoComponentRenderer componentRenderer,
MembershipHelper membershipHelper,
ServiceContext services,
CacheHelper appCache)
ServiceContext services)
{
if (umbracoContext == null) throw new ArgumentNullException(nameof(umbracoContext));
if (content == null) throw new ArgumentNullException(nameof(content));
if (query == null) throw new ArgumentNullException(nameof(query));
if (tagQuery == null) throw new ArgumentNullException(nameof(tagQuery));
if (dataTypeService == null) throw new ArgumentNullException(nameof(dataTypeService));
if (cultureDictionary == null) throw new ArgumentNullException(nameof(cultureDictionary));
if (componentRenderer == null) throw new ArgumentNullException(nameof(componentRenderer));
if (membershipHelper == null) throw new ArgumentNullException(nameof(membershipHelper));
if (services == null) throw new ArgumentNullException(nameof(services));
if (appCache == null) throw new ArgumentNullException(nameof(appCache));
_umbracoContext = umbracoContext;
_umbracoContext = umbracoContext ?? throw new ArgumentNullException(nameof(umbracoContext));
_tag = new TagQuery(tagQuery);
_dataTypeService = dataTypeService;
_cultureDictionary = cultureDictionary;
_componentRenderer = componentRenderer;
_membershipHelper = membershipHelper;
_currentPage = content;
_iQuery = query;
_services = services;
_appCache = appCache;
_cultureDictionary = cultureDictionary ?? throw new ArgumentNullException(nameof(cultureDictionary));
_componentRenderer = componentRenderer ?? throw new ArgumentNullException(nameof(componentRenderer));
_membershipHelper = membershipHelper ?? throw new ArgumentNullException(nameof(membershipHelper));
_currentPage = content ?? throw new ArgumentNullException(nameof(content));
_iQuery = query ?? throw new ArgumentNullException(nameof(query));
_services = services ?? throw new ArgumentNullException(nameof(services));
}
/// <summary>
@@ -93,13 +78,11 @@ namespace Umbraco.Web
/// <param name="umbracoContext">An Umbraco context.</param>
/// <param name="content">A content item.</param>
/// <param name="services">A services context.</param>
/// <param name="appCache">An application cache helper.</param>
/// <remarks>Sets the current page to the supplied content item.</remarks>
public UmbracoHelper(UmbracoContext umbracoContext, ServiceContext services, CacheHelper appCache, IPublishedContent content)
: this(umbracoContext, services, appCache)
public UmbracoHelper(UmbracoContext umbracoContext, ServiceContext services, IPublishedContent content)
: this(umbracoContext, services)
{
if (content == null) throw new ArgumentNullException(nameof(content));
_currentPage = content;
_currentPage = content ?? throw new ArgumentNullException(nameof(content));
}
/// <summary>
@@ -107,19 +90,13 @@ namespace Umbraco.Web
/// </summary>
/// <param name="umbracoContext">An Umbraco context.</param>
/// <param name="services">A services context.</param>
/// <param name="appCache">An application cache helper.</param>
/// <remarks>Sets the current page to the context's published content request's content item.</remarks>
public UmbracoHelper(UmbracoContext umbracoContext, ServiceContext services, CacheHelper appCache)
public UmbracoHelper(UmbracoContext umbracoContext, ServiceContext services)
{
if (umbracoContext == null) throw new ArgumentNullException(nameof(umbracoContext));
if (services == null) throw new ArgumentNullException(nameof(services));
if (appCache == null) throw new ArgumentNullException(nameof(appCache));
_umbracoContext = umbracoContext;
_services = services ?? throw new ArgumentNullException(nameof(services));
_umbracoContext = umbracoContext ?? throw new ArgumentNullException(nameof(umbracoContext));
if (_umbracoContext.IsFrontEndUmbracoRequest)
_currentPage = _umbracoContext.PublishedRequest.PublishedContent;
_services = services;
_appCache = appCache;
}
#endregion
@@ -133,7 +110,7 @@ namespace Umbraco.Web
/// <summary>
/// Gets the query context.
/// </summary>
public PublishedContentQuery ContentQuery => _query ??
public IPublishedContentQuery ContentQuery => _query ??
(_query = _iQuery != null
? new PublishedContentQuery(_iQuery)
: new PublishedContentQuery(UmbracoContext.ContentCache, UmbracoContext.MediaCache));
@@ -162,12 +139,6 @@ namespace Umbraco.Web
/// </summary>
public UrlProvider UrlProvider => UmbracoContext.UrlProvider;
/// <summary>
/// Gets the datatype service.
/// </summary>
private IDataTypeService DataTypeService => _dataTypeService
?? (_dataTypeService = _services.DataTypeService);
/// <summary>
/// Gets the component renderer.
/// </summary>
@@ -819,11 +790,11 @@ namespace Umbraco.Web
/// </summary>
/// <param name="term"></param>
/// <param name="useWildCards"></param>
/// <param name="searchProvider"></param>
/// <param name="indexName"></param>
/// <returns></returns>
public IEnumerable<PublishedSearchResult> Search(string term, bool useWildCards = true, string searchProvider = null)
public IEnumerable<PublishedSearchResult> Search(string term, bool useWildCards = true, string indexName = null)
{
return ContentQuery.Search(term, useWildCards, searchProvider);
return ContentQuery.Search(term, useWildCards, indexName);
}
/// <summary>
@@ -834,11 +805,11 @@ namespace Umbraco.Web
/// <param name="totalRecords"></param>
/// <param name="term"></param>
/// <param name="useWildCards"></param>
/// <param name="searchProvider"></param>
/// <param name="indexName"></param>
/// <returns></returns>
public IEnumerable<PublishedSearchResult> TypedSearch(int skip, int take, out long totalRecords, string term, bool useWildCards = true, string searchProvider = null)
public IEnumerable<PublishedSearchResult> Search(int skip, int take, out long totalRecords, string term, bool useWildCards = true, string indexName = null)
{
return ContentQuery.Search(skip, take, out totalRecords, term, useWildCards, searchProvider);
return ContentQuery.Search(skip, take, out totalRecords, term, useWildCards, indexName);
}
/// <summary>
@@ -850,7 +821,7 @@ namespace Umbraco.Web
/// <param name="criteria"></param>
/// <param name="searchProvider"></param>
/// <returns></returns>
public IEnumerable<PublishedSearchResult> TypedSearch(int skip, int take, out long totalRecords, Examine.SearchCriteria.ISearchCriteria criteria, Examine.Providers.BaseSearchProvider searchProvider = null)
public IEnumerable<PublishedSearchResult> Search(int skip, int take, out long totalRecords, Examine.SearchCriteria.ISearchCriteria criteria, Examine.Providers.BaseSearchProvider searchProvider = null)
{
return ContentQuery.Search(skip, take, out totalRecords, criteria, searchProvider);
}