diff --git a/src/Umbraco.Core/Constants-Indexes.cs b/src/Umbraco.Core/Constants-Indexes.cs index d9d86884c4..ecf143e83f 100644 --- a/src/Umbraco.Core/Constants-Indexes.cs +++ b/src/Umbraco.Core/Constants-Indexes.cs @@ -8,9 +8,9 @@ namespace Umbraco.Core public static class UmbracoIndexes { - public const string InternalIndexName = InternalIndexPath + "Indexer"; - public const string ExternalIndexName = ExternalIndexPath + "Indexer"; - public const string MembersIndexName = MembersIndexPath + "Indexer"; + public const string InternalIndexName = InternalIndexPath + "Index"; + public const string ExternalIndexName = ExternalIndexPath + "Index"; + public const string MembersIndexName = MembersIndexPath + "Index"; public const string InternalIndexPath = "Internal"; public const string ExternalIndexPath = "External"; diff --git a/src/Umbraco.Core/Models/PagedResult.cs b/src/Umbraco.Core/Models/PagedResult.cs index 653712d9f8..ef4d4efdfd 100644 --- a/src/Umbraco.Core/Models/PagedResult.cs +++ b/src/Umbraco.Core/Models/PagedResult.cs @@ -19,7 +19,7 @@ namespace Umbraco.Core.Models if (pageSize > 0) { - TotalPages = (long)Math.Ceiling(totalItems / (Decimal)pageSize); + TotalPages = (long)Math.Ceiling(totalItems / (decimal)pageSize); } else { diff --git a/src/Umbraco.Core/PropertyEditors/DefaultPropertyIndexValues.cs b/src/Umbraco.Core/PropertyEditors/DefaultPropertyIndexValues.cs index bf39adf93b..c6caab86d8 100644 --- a/src/Umbraco.Core/PropertyEditors/DefaultPropertyIndexValues.cs +++ b/src/Umbraco.Core/PropertyEditors/DefaultPropertyIndexValues.cs @@ -8,9 +8,11 @@ namespace Umbraco.Core.PropertyEditors /// public class DefaultPropertyIndexValues : IPropertyIndexValues { - public IEnumerable> GetIndexValues(Property property, string culture, string segment) + public IEnumerable>> GetIndexValues(Property property, string culture, string segment, bool published) { - yield return new KeyValuePair(property.Alias, new[] { property.GetValue(culture, segment) }); + yield return new KeyValuePair>( + property.Alias, + property.GetValue(culture, segment, published).Yield()); } } } diff --git a/src/Umbraco.Core/PropertyEditors/IPropertyIndexValues.cs b/src/Umbraco.Core/PropertyEditors/IPropertyIndexValues.cs index 5ebd0a29b7..15580ccb80 100644 --- a/src/Umbraco.Core/PropertyEditors/IPropertyIndexValues.cs +++ b/src/Umbraco.Core/PropertyEditors/IPropertyIndexValues.cs @@ -8,6 +8,6 @@ namespace Umbraco.Core.PropertyEditors /// public interface IPropertyIndexValues { - IEnumerable> GetIndexValues(Property property, string culture, string segment); + IEnumerable>> GetIndexValues(Property property, string culture, string segment, bool published); } } diff --git a/src/Umbraco.Examine/BaseValueSetBuilder.cs b/src/Umbraco.Examine/BaseValueSetBuilder.cs index 6191297aff..f0261f7cd9 100644 --- a/src/Umbraco.Examine/BaseValueSetBuilder.cs +++ b/src/Umbraco.Examine/BaseValueSetBuilder.cs @@ -12,10 +12,12 @@ namespace Umbraco.Examine public abstract class BaseValueSetBuilder : IValueSetBuilder where TContent : IContentBase { + protected bool PublishedValuesOnly { get; } private readonly PropertyEditorCollection _propertyEditors; - protected BaseValueSetBuilder(PropertyEditorCollection propertyEditors) + protected BaseValueSetBuilder(PropertyEditorCollection propertyEditors, bool publishedValuesOnly) { + PublishedValuesOnly = publishedValuesOnly; _propertyEditors = propertyEditors ?? throw new System.ArgumentNullException(nameof(propertyEditors)); } @@ -27,7 +29,7 @@ namespace Umbraco.Examine var editor = _propertyEditors[property.PropertyType.PropertyEditorAlias]; if (editor == null) return; - var indexVals = editor.PropertyIndexValues.GetIndexValues(property, culture, segment); + var indexVals = editor.PropertyIndexValues.GetIndexValues(property, culture, segment, PublishedValuesOnly); foreach (var keyVal in indexVals) { if (keyVal.Key.IsNullOrWhiteSpace()) continue; diff --git a/src/Umbraco.Examine/Config/IndexSet.cs b/src/Umbraco.Examine/Config/IndexSet.cs index 33b67412f2..65b1cd5aec 100644 --- a/src/Umbraco.Examine/Config/IndexSet.cs +++ b/src/Umbraco.Examine/Config/IndexSet.cs @@ -11,48 +11,6 @@ namespace Umbraco.Examine.Config [ConfigurationProperty("SetName", IsRequired = true, IsKey = true)] public string SetName => (string)this["SetName"]; - private string _indexPath = ""; - - /// - /// The folder path of where the lucene index is stored - /// - /// The index path. - /// - /// This can be set at runtime but will not be persisted to the configuration file - /// - [ConfigurationProperty("IndexPath", IsRequired = true, IsKey = false)] - public string IndexPath - { - get - { - if (string.IsNullOrEmpty(_indexPath)) - _indexPath = (string)this["IndexPath"]; - - return _indexPath; - } - set => _indexPath = value; - } - - /// - /// Returns the DirectoryInfo object for the index path. - /// - /// The index directory. - public DirectoryInfo IndexDirectory - { - get - { - //TODO: Get this out of the index set. We need to use the Indexer's DataService to lookup the folder so it can be unit tested. Probably need DataServices on the searcher then too - - //we need to de-couple the context - if (HttpContext.Current != null) - return new DirectoryInfo(HttpContext.Current.Server.MapPath(this.IndexPath)); - else if (HostingEnvironment.ApplicationID != null) - return new DirectoryInfo(HostingEnvironment.MapPath(this.IndexPath)); - else - return new DirectoryInfo(this.IndexPath); - } - } - /// /// When this property is set, the indexing will only index documents that are descendants of this node. /// diff --git a/src/Umbraco.Examine/ContentIndexPopulator.cs b/src/Umbraco.Examine/ContentIndexPopulator.cs index 8127a8c3d3..d1255b4d46 100644 --- a/src/Umbraco.Examine/ContentIndexPopulator.cs +++ b/src/Umbraco.Examine/ContentIndexPopulator.cs @@ -25,7 +25,7 @@ namespace Umbraco.Examine /// private static IQuery _publishedQuery; - private readonly bool _supportUnpublishedContent; + private readonly bool _publishedValuesOnly; private readonly int? _parentId; /// @@ -34,27 +34,27 @@ namespace Umbraco.Examine /// /// /// - public ContentIndexPopulator(IContentService contentService, ISqlContext sqlContext, IValueSetBuilder contentValueSetBuilder) - : this(true, null, contentService, sqlContext, contentValueSetBuilder) + public ContentIndexPopulator(IContentService contentService, ISqlContext sqlContext, IContentValueSetBuilder contentValueSetBuilder) + : this(false, null, contentService, sqlContext, contentValueSetBuilder) { } /// /// Optional constructor allowing specifying custom query parameters /// - /// + /// /// /// /// /// - public ContentIndexPopulator(bool supportUnpublishedContent, int? parentId, IContentService contentService, ISqlContext sqlContext, IValueSetBuilder contentValueSetBuilder) + public ContentIndexPopulator(bool publishedValuesOnly, int? parentId, IContentService contentService, ISqlContext sqlContext, IValueSetBuilder contentValueSetBuilder) { if (sqlContext == null) throw new ArgumentNullException(nameof(sqlContext)); _contentService = contentService ?? throw new ArgumentNullException(nameof(contentService)); _contentValueSetBuilder = contentValueSetBuilder ?? throw new ArgumentNullException(nameof(contentValueSetBuilder)); if (_publishedQuery != null) _publishedQuery = sqlContext.Query().Where(x => x.Published); - _supportUnpublishedContent = supportUnpublishedContent; + _publishedValuesOnly = publishedValuesOnly; _parentId = parentId; RegisterIndex(Constants.UmbracoIndexes.InternalIndexName); @@ -75,7 +75,7 @@ namespace Umbraco.Examine do { - if (_supportUnpublishedContent) + if (!_publishedValuesOnly) { content = _contentService.GetPagedDescendants(contentParentId, pageIndex, pageSize, out _).ToArray(); } diff --git a/src/Umbraco.Examine/ContentValueSetBuilder.cs b/src/Umbraco.Examine/ContentValueSetBuilder.cs index 056e137c07..39ffab98d9 100644 --- a/src/Umbraco.Examine/ContentValueSetBuilder.cs +++ b/src/Umbraco.Examine/ContentValueSetBuilder.cs @@ -8,15 +8,19 @@ using Umbraco.Core.Strings; namespace Umbraco.Examine { - public class ContentValueSetBuilder : BaseValueSetBuilder + /// + /// Builds s for items + /// + public class ContentValueSetBuilder : BaseValueSetBuilder, IContentValueSetBuilder, IPublishedContentValueSetBuilder { private readonly IEnumerable _urlSegmentProviders; private readonly IUserService _userService; public ContentValueSetBuilder(PropertyEditorCollection propertyEditors, IEnumerable urlSegmentProviders, - IUserService userService) - : base(propertyEditors) + IUserService userService, + bool publishedValuesOnly) + : base(propertyEditors, publishedValuesOnly) { _urlSegmentProviders = urlSegmentProviders; _userService = userService; @@ -47,8 +51,10 @@ namespace Umbraco.Examine {"sortOrder", new object[] {c.SortOrder}}, {"createDate", new object[] {c.CreateDate}}, //Always add invariant createDate {"updateDate", new object[] {c.UpdateDate}}, //Always add invariant updateDate - {"nodeName", c.Name.Yield()}, //Always add invariant nodeName - {"urlName", urlValue.Yield()}, //Always add invariant urlName + {"nodeName", PublishedValuesOnly //Always add invariant nodeName + ? c.PublishName.Yield() + : c.Name.Yield()}, + {"urlName", urlValue.Yield()}, //Always add invariant urlName {"path", c.Path.Yield()}, {"nodeType", new object[] {c.ContentType.Id}}, {"creatorName", (c.GetCreatorProfile(_userService)?.Name ?? "??").Yield() }, @@ -67,9 +73,13 @@ namespace Umbraco.Examine var variantUrl = c.GetUrlSegment(_urlSegmentProviders, culture); var lowerCulture = culture.ToLowerInvariant(); values[$"urlName_{lowerCulture}"] = variantUrl.Yield(); - values[$"nodeName_{lowerCulture}"] = c.GetCultureName(culture).Yield(); - values[$"{UmbracoExamineIndexer.PublishedFieldName}_{lowerCulture}"] = new object[] { c.IsCulturePublished(culture) ? 1 : 0 }; - values[$"updateDate_{lowerCulture}"] = new object[] { c.GetUpdateDate(culture) }; + values[$"nodeName_{lowerCulture}"] = PublishedValuesOnly + ? c.GetPublishName(culture).Yield() + : c.GetCultureName(culture).Yield(); + values[$"{UmbracoExamineIndexer.PublishedFieldName}_{lowerCulture}"] = (c.IsCulturePublished(culture) ? 1 : 0).Yield(); + values[$"updateDate_{lowerCulture}"] = PublishedValuesOnly + ? c.GetPublishDate(culture).Yield() + : c.GetUpdateDate(culture).Yield(); } } diff --git a/src/Umbraco.Examine/ContentValueSetValidator.cs b/src/Umbraco.Examine/ContentValueSetValidator.cs index 46e4f48c9a..d671d101d6 100644 --- a/src/Umbraco.Examine/ContentValueSetValidator.cs +++ b/src/Umbraco.Examine/ContentValueSetValidator.cs @@ -20,7 +20,7 @@ namespace Umbraco.Examine private static readonly IEnumerable ValidCategories = new[] {IndexTypes.Content, IndexTypes.Media}; protected override IEnumerable ValidIndexCategories => ValidCategories; - public bool SupportUnpublishedContent { get; } + public bool PublishedValuesOnly { get; } public bool SupportProtectedContent { get; } public int? ParentId { get; } @@ -43,7 +43,7 @@ namespace Umbraco.Examine var recycleBinId = category == IndexTypes.Content ? Constants.System.RecycleBinContent : Constants.System.RecycleBinMedia; //check for recycle bin - if (!SupportUnpublishedContent) + if (!PublishedValuesOnly) { if (path.Contains(string.Concat(",", recycleBinId, ","))) return false; @@ -64,18 +64,18 @@ namespace Umbraco.Examine return true; } - public ContentValueSetValidator(bool supportUnpublishedContent, int? parentId = null, + public ContentValueSetValidator(bool publishedValuesOnly, int? parentId = null, IEnumerable includeItemTypes = null, IEnumerable excludeItemTypes = null) - : this(supportUnpublishedContent, true, null, parentId, includeItemTypes, excludeItemTypes) + : this(publishedValuesOnly, true, null, parentId, includeItemTypes, excludeItemTypes) { } - public ContentValueSetValidator(bool supportUnpublishedContent, bool supportProtectedContent, + public ContentValueSetValidator(bool publishedValuesOnly, bool supportProtectedContent, IPublicAccessService publicAccessService, int? parentId = null, IEnumerable includeItemTypes = null, IEnumerable excludeItemTypes = null) : base(includeItemTypes, excludeItemTypes, null, null) { - SupportUnpublishedContent = supportUnpublishedContent; + PublishedValuesOnly = publishedValuesOnly; SupportProtectedContent = supportProtectedContent; ParentId = parentId; _publicAccessService = publicAccessService; @@ -90,7 +90,7 @@ namespace Umbraco.Examine var isFiltered = baseValidate == ValueSetValidationResult.Filtered; //check for published content - if (valueSet.Category == IndexTypes.Content && !SupportUnpublishedContent) + if (valueSet.Category == IndexTypes.Content && !PublishedValuesOnly) { if (!valueSet.Values.TryGetValue(UmbracoExamineIndexer.PublishedFieldName, out var published)) return ValueSetValidationResult.Failed; diff --git a/src/Umbraco.Examine/IContentValueSetBuilder.cs b/src/Umbraco.Examine/IContentValueSetBuilder.cs new file mode 100644 index 0000000000..fed706d592 --- /dev/null +++ b/src/Umbraco.Examine/IContentValueSetBuilder.cs @@ -0,0 +1,13 @@ +using Examine; +using Umbraco.Core.Models; + +namespace Umbraco.Examine +{ + /// + /// + /// Marker interface for a builder for supporting unpublished content + /// + public interface IContentValueSetBuilder : IValueSetBuilder + { + } +} diff --git a/src/Umbraco.Examine/IContentValueSetValidator.cs b/src/Umbraco.Examine/IContentValueSetValidator.cs index a7164773bb..fa85a0d32b 100644 --- a/src/Umbraco.Examine/IContentValueSetValidator.cs +++ b/src/Umbraco.Examine/IContentValueSetValidator.cs @@ -7,8 +7,21 @@ namespace Umbraco.Examine /// public interface IContentValueSetValidator : IValueSetValidator { - bool SupportUnpublishedContent { get; } + /// + /// When set to true the index will only retain published values + /// + /// + /// Any non-published values will not be put or kept in the index: + /// * Deleted, Trashed, non-published Content items + /// * non-published Variants + /// + bool PublishedValuesOnly { get; } + + /// + /// If true, protected content will be indexed otherwise it will not be put or kept in the index + /// bool SupportProtectedContent { get; } + int? ParentId { get; } bool ValidatePath(string path, string category); diff --git a/src/Umbraco.Examine/IPublishedContentValueSetBuilder.cs b/src/Umbraco.Examine/IPublishedContentValueSetBuilder.cs new file mode 100644 index 0000000000..c337a7a1e6 --- /dev/null +++ b/src/Umbraco.Examine/IPublishedContentValueSetBuilder.cs @@ -0,0 +1,12 @@ +using Examine; +using Umbraco.Core.Models; + +namespace Umbraco.Examine +{ + /// + /// Marker interface for a builder for only published content + /// + public interface IPublishedContentValueSetBuilder : IValueSetBuilder + { + } +} \ No newline at end of file diff --git a/src/Umbraco.Examine/IUmbracoIndexer.cs b/src/Umbraco.Examine/IUmbracoIndexer.cs index c3b4a5d629..00f772b1e2 100644 --- a/src/Umbraco.Examine/IUmbracoIndexer.cs +++ b/src/Umbraco.Examine/IUmbracoIndexer.cs @@ -13,8 +13,13 @@ namespace Umbraco.Examine bool EnableDefaultEventHandler { get; } /// - /// When set to true data will not be deleted from the index if the data is being soft deleted (unpublished or trashed) + /// When set to true the index will only retain published values /// - bool SupportSoftDelete { get; } + /// + /// Any non-published values will not be put or kept in the index: + /// * Deleted, Trashed, non-published Content items + /// * non-published Variants + /// + bool PublishedValuesOnly { get; } } } diff --git a/src/Umbraco.Examine/MediaValueSetBuilder.cs b/src/Umbraco.Examine/MediaValueSetBuilder.cs index af41f574fb..8df51570c1 100644 --- a/src/Umbraco.Examine/MediaValueSetBuilder.cs +++ b/src/Umbraco.Examine/MediaValueSetBuilder.cs @@ -16,7 +16,7 @@ namespace Umbraco.Examine public MediaValueSetBuilder(PropertyEditorCollection propertyEditors, IEnumerable urlSegmentProviders, IUserService userService) - : base(propertyEditors) + : base(propertyEditors, false) { _urlSegmentProviders = urlSegmentProviders; _userService = userService; diff --git a/src/Umbraco.Examine/MemberValueSetBuilder.cs b/src/Umbraco.Examine/MemberValueSetBuilder.cs index 85a8fa13c5..9864aba18d 100644 --- a/src/Umbraco.Examine/MemberValueSetBuilder.cs +++ b/src/Umbraco.Examine/MemberValueSetBuilder.cs @@ -10,7 +10,7 @@ namespace Umbraco.Examine public class MemberValueSetBuilder : BaseValueSetBuilder { public MemberValueSetBuilder(PropertyEditorCollection propertyEditors) - : base(propertyEditors) + : base(propertyEditors, false) { } diff --git a/src/Umbraco.Examine/PublishedContentIndexPopulator.cs b/src/Umbraco.Examine/PublishedContentIndexPopulator.cs index a29031ca3c..143e2db630 100644 --- a/src/Umbraco.Examine/PublishedContentIndexPopulator.cs +++ b/src/Umbraco.Examine/PublishedContentIndexPopulator.cs @@ -14,8 +14,8 @@ namespace Umbraco.Examine /// public class PublishedContentIndexPopulator : ContentIndexPopulator { - public PublishedContentIndexPopulator(IContentService contentService, ISqlContext sqlContext, IValueSetBuilder contentValueSetBuilder) : - base(false, null, contentService, sqlContext, contentValueSetBuilder) + public PublishedContentIndexPopulator(IContentService contentService, ISqlContext sqlContext, IPublishedContentValueSetBuilder contentValueSetBuilder) : + base(true, null, contentService, sqlContext, contentValueSetBuilder) { } } diff --git a/src/Umbraco.Examine/Umbraco.Examine.csproj b/src/Umbraco.Examine/Umbraco.Examine.csproj index 5c18da00ef..0b0e02df49 100644 --- a/src/Umbraco.Examine/Umbraco.Examine.csproj +++ b/src/Umbraco.Examine/Umbraco.Examine.csproj @@ -48,7 +48,7 @@ - + @@ -64,11 +64,13 @@ + + diff --git a/src/Umbraco.Examine/UmbracoContentIndexer.cs b/src/Umbraco.Examine/UmbracoContentIndexer.cs index feea0a4efd..e39a9d5990 100644 --- a/src/Umbraco.Examine/UmbracoContentIndexer.cs +++ b/src/Umbraco.Examine/UmbracoContentIndexer.cs @@ -64,8 +64,8 @@ namespace Umbraco.Examine if (validator == null) throw new ArgumentNullException(nameof(validator)); LanguageService = languageService ?? throw new ArgumentNullException(nameof(languageService)); - if (validator is ContentValueSetValidator contentValueSetValidator) - SupportSoftDelete = contentValueSetValidator.SupportUnpublishedContent; + if (validator is IContentValueSetValidator contentValueSetValidator) + PublishedValuesOnly = contentValueSetValidator.PublishedValuesOnly; } #endregion @@ -124,7 +124,7 @@ namespace Umbraco.Examine parentId, ConfigIndexCriteria.IncludeItemTypes, ConfigIndexCriteria.ExcludeItemTypes); - SupportSoftDelete = supportUnpublished; + PublishedValuesOnly = supportUnpublished; } #endregion diff --git a/src/Umbraco.Examine/UmbracoExamineExtensions.cs b/src/Umbraco.Examine/UmbracoExamineExtensions.cs index ab8819b276..8be5a6c1e3 100644 --- a/src/Umbraco.Examine/UmbracoExamineExtensions.cs +++ b/src/Umbraco.Examine/UmbracoExamineExtensions.cs @@ -76,17 +76,6 @@ namespace Umbraco.Examine return fieldQuery; } - /// - /// Used to replace any available tokens in the index path before the lucene directory is assigned to the path - /// - /// - internal static void ReplaceTokensInIndexPath(this IndexSet indexSet) - { - if (indexSet == null) return; - indexSet.IndexPath = indexSet.IndexPath - .Replace("{machinename}", NetworkHelper.FileSafeMachineName) - .Replace("{appdomainappid}", (HttpRuntime.AppDomainAppId ?? string.Empty).ReplaceNonAlphanumericChars("")) - .EnsureEndsWith('/'); - } + } -} \ No newline at end of file +} diff --git a/src/Umbraco.Examine/UmbracoExamineIndexDiagnostics.cs b/src/Umbraco.Examine/UmbracoExamineIndexDiagnostics.cs index 7e8bd13323..0812d93931 100644 --- a/src/Umbraco.Examine/UmbracoExamineIndexDiagnostics.cs +++ b/src/Umbraco.Examine/UmbracoExamineIndexDiagnostics.cs @@ -70,7 +70,7 @@ namespace Umbraco.Examine _index.LuceneIndexFolder == null ? string.Empty : _index.LuceneIndexFolder.ToString().ToLowerInvariant().TrimStart(IOHelper.MapPath(SystemDirectories.Root).ToLowerInvariant()).Replace("\\", "/").EnsureStartsWith('/'), - [nameof(UmbracoExamineIndexer.SupportSoftDelete)] = _index.SupportSoftDelete, + [nameof(UmbracoExamineIndexer.PublishedValuesOnly)] = _index.PublishedValuesOnly, //There's too much info here //[nameof(UmbracoExamineIndexer.FieldDefinitionCollection)] = _index.FieldDefinitionCollection, }; @@ -85,7 +85,7 @@ namespace Umbraco.Examine if (_index.ValueSetValidator is ContentValueSetValidator cvsv) { - d[nameof(ContentValueSetValidator.SupportUnpublishedContent)] = cvsv.SupportUnpublishedContent; + d[nameof(ContentValueSetValidator.PublishedValuesOnly)] = cvsv.PublishedValuesOnly; d[nameof(ContentValueSetValidator.SupportProtectedContent)] = cvsv.SupportProtectedContent; d[nameof(ContentValueSetValidator.ParentId)] = cvsv.ParentId; } diff --git a/src/Umbraco.Examine/UmbracoExamineIndexer.cs b/src/Umbraco.Examine/UmbracoExamineIndexer.cs index 05958e185d..84c8a7d8c5 100644 --- a/src/Umbraco.Examine/UmbracoExamineIndexer.cs +++ b/src/Umbraco.Examine/UmbracoExamineIndexer.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.ComponentModel; -using System.IO; using System.Linq; using Examine.LuceneEngine.Providers; using Lucene.Net.Analysis; @@ -14,7 +13,6 @@ using Examine.LuceneEngine.Indexing; using Lucene.Net.Store; using Umbraco.Core.Composing; using Umbraco.Core.Logging; -using Umbraco.Core.Xml; using Umbraco.Examine.Config; using Directory = Lucene.Net.Store.Directory; @@ -147,7 +145,7 @@ namespace Umbraco.Examine /// public bool EnableDefaultEventHandler { get; set; } = true; - public bool SupportSoftDelete { get; protected set; } = false; + public bool PublishedValuesOnly { get; protected set; } = false; protected ConfigIndexCriteria ConfigIndexCriteria { get; private set; } @@ -177,38 +175,35 @@ namespace Umbraco.Examine } //Need to check if the index set or IndexerData is specified... - if (config["indexSet"] == null && (FieldDefinitionCollection.Count == 0)) + if (config["indexSet"] == null && FieldDefinitionCollection.Count == 0) { //if we don't have either, then we'll try to set the index set by naming conventions var found = false; - if (name.EndsWith("Indexer")) + + var possibleSuffixes = new[] {"Index", "Indexer"}; + foreach (var suffix in possibleSuffixes) { - var setNameByConvension = name.Remove(name.LastIndexOf("Indexer")) + "IndexSet"; + if (!name.EndsWith(suffix)) continue; + + var setNameByConvension = name.Remove(name.LastIndexOf(suffix, StringComparison.Ordinal)) + "IndexSet"; //check if we can assign the index set by naming convention var set = IndexSets.Instance.Sets.Cast().SingleOrDefault(x => x.SetName == setNameByConvension); - if (set != null) + if (set == null) continue; + + //we've found an index set by naming conventions :) + IndexSetName = set.SetName; + + var indexSet = IndexSets.Instance.Sets[IndexSetName]; + + //get the index criteria and ensure folder + ConfigIndexCriteria = CreateFieldDefinitionsFromConfig(indexSet); + foreach (var fieldDefinition in ConfigIndexCriteria.StandardFields.Union(ConfigIndexCriteria.UserFields)) { - //we've found an index set by naming conventions :) - IndexSetName = set.SetName; - - var indexSet = IndexSets.Instance.Sets[IndexSetName]; - - //if tokens are declared in the path, then use them (i.e. {machinename} ) - indexSet.ReplaceTokensInIndexPath(); - - //get the index criteria and ensure folder - ConfigIndexCriteria = CreateFieldDefinitionsFromConfig(indexSet); - foreach (var fieldDefinition in ConfigIndexCriteria.StandardFields.Union(ConfigIndexCriteria.UserFields)) - { - FieldDefinitionCollection.TryAdd(fieldDefinition.Name, fieldDefinition); - } - - //now set the index folder - LuceneIndexFolder = new DirectoryInfo(Path.Combine(IndexSets.Instance.Sets[IndexSetName].IndexDirectory.FullName, "Index")); - - found = true; + FieldDefinitionCollection.TryAdd(fieldDefinition.Name, fieldDefinition); } + found = true; + break; } if (!found) @@ -229,24 +224,18 @@ namespace Umbraco.Examine var indexSet = IndexSets.Instance.Sets[IndexSetName]; - //if tokens are declared in the path, then use them (i.e. {machinename} ) - indexSet.ReplaceTokensInIndexPath(); - //get the index criteria and ensure folder ConfigIndexCriteria = CreateFieldDefinitionsFromConfig(indexSet); foreach (var fieldDefinition in ConfigIndexCriteria.StandardFields.Union(ConfigIndexCriteria.UserFields)) { FieldDefinitionCollection.TryAdd(fieldDefinition.Name, fieldDefinition); } - - //now set the index folder - LuceneIndexFolder = new DirectoryInfo(Path.Combine(IndexSets.Instance.Sets[IndexSetName].IndexDirectory.FullName, "Index")); } } base.Initialize(name, config); } - + #endregion /// diff --git a/src/Umbraco.Examine/UmbracoExamineSearcher.cs b/src/Umbraco.Examine/UmbracoExamineSearcher.cs index 2474bc6429..50c6c21e37 100644 --- a/src/Umbraco.Examine/UmbracoExamineSearcher.cs +++ b/src/Umbraco.Examine/UmbracoExamineSearcher.cs @@ -1,145 +1,52 @@ -using System; -using System.ComponentModel; -using System.IO; -using System.Linq; -using Umbraco.Core; -using Examine.LuceneEngine.Providers; -using Lucene.Net.Analysis; -using Lucene.Net.Index; -using Umbraco.Core.Composing; -using Umbraco.Examine.Config; -using Directory = Lucene.Net.Store.Directory; -using Examine.LuceneEngine; +//using System; +//using System.ComponentModel; +//using System.IO; +//using System.Linq; +//using Umbraco.Core; +//using Examine.LuceneEngine.Providers; +//using Lucene.Net.Analysis; +//using Lucene.Net.Index; +//using Umbraco.Core.Composing; +//using Umbraco.Examine.Config; +//using Directory = Lucene.Net.Store.Directory; +//using Examine.LuceneEngine; -namespace Umbraco.Examine -{ - /// - /// An Examine searcher which uses Lucene.Net as the - /// - public class UmbracoExamineSearcher : LuceneSearcher - { - private readonly bool _configBased = false; +//namespace Umbraco.Examine +//{ +// /// +// /// An Examine searcher which uses Lucene.Net as the +// /// +// public class UmbracoExamineSearcher : LuceneSearcher +// { +// /// +// /// Constructor to allow for creating an indexer at runtime +// /// +// /// +// /// +// /// +// public UmbracoExamineSearcher(string name, Directory luceneDirectory, Analyzer analyzer, FieldValueTypeCollection fieldValueTypeCollection) +// : base(name, luceneDirectory, analyzer, fieldValueTypeCollection) +// { +// } - /// - /// Default constructor for config based construction - /// - [EditorBrowsable(EditorBrowsableState.Never)] - public UmbracoExamineSearcher() - { - _configBased = true; - } +// /// +// public UmbracoExamineSearcher(string name, IndexWriter writer, Analyzer analyzer, FieldValueTypeCollection fieldValueTypeCollection) +// : base(name, writer, analyzer, fieldValueTypeCollection) +// { +// } - /// - /// Constructor to allow for creating an indexer at runtime - /// - /// - /// - /// - public UmbracoExamineSearcher(string name, Directory luceneDirectory, Analyzer analyzer, FieldValueTypeCollection fieldValueTypeCollection) - : base(name, luceneDirectory, analyzer, fieldValueTypeCollection) - { - _configBased = false; - } +// /// +// /// Returns a list of fields to search on, this will also exclude the IndexPathFieldName and node type alias +// /// +// /// +// public override string[] GetAllIndexedFields() +// { +// var fields = base.GetAllIndexedFields(); +// return fields +// .Where(x => x != UmbracoExamineIndexer.IndexPathFieldName) +// .Where(x => x != LuceneIndex.ItemTypeFieldName) +// .ToArray(); +// } - /// - public UmbracoExamineSearcher(string name, IndexWriter writer, Analyzer analyzer, FieldValueTypeCollection fieldValueTypeCollection) - : base(name, writer, analyzer, fieldValueTypeCollection) - { - _configBased = false; - } - - /// - /// Name of the Lucene.NET index set - /// - public string IndexSetName { get; private set; } - - /// - /// Method used for initializing based on a configuration based searcher - /// - /// - /// - public override void Initialize(string name, System.Collections.Specialized.NameValueCollection config) - { - if (string.IsNullOrWhiteSpace(name)) throw new ArgumentException("Value cannot be null or whitespace.", nameof(name)); - - //We need to check if we actually can initialize, if not then don't continue - if (CanInitialize() == false) - { - return; - } - - //need to check if the index set is specified, if it's not, we'll see if we can find one by convension - //if the folder is not null and the index set is null, we'll assume that this has been created at runtime. - //NOTE: Don't proceed if the _luceneDirectory is set since we already know where to look. - var luceneDirectory = GetLuceneDirectory(); - if (config["indexSet"] == null && (LuceneIndexFolder == null && luceneDirectory == null)) - { - //if we don't have either, then we'll try to set the index set by naming convensions - var found = false; - if (name.EndsWith("Searcher")) - { - var setNameByConvension = name.Remove(name.LastIndexOf("Searcher")) + "IndexSet"; - //check if we can assign the index set by naming convension - var set = IndexSets.Instance.Sets.Cast().SingleOrDefault(x => x.SetName == setNameByConvension); - - if (set != null) - { - set.ReplaceTokensInIndexPath(); - - //we've found an index set by naming convensions :) - IndexSetName = set.SetName; - found = true; - } - } - - if (!found) - throw new ArgumentNullException("indexSet on LuceneExamineIndexer provider has not been set in configuration"); - - //get the folder to index - LuceneIndexFolder = new DirectoryInfo(Path.Combine(IndexSets.Instance.Sets[IndexSetName].IndexDirectory.FullName, "Index")); - } - else if (config["indexSet"] != null && luceneDirectory == null) - { - if (IndexSets.Instance.Sets[config["indexSet"]] == null) - throw new ArgumentException("The indexSet specified for the LuceneExamineIndexer provider does not exist"); - - IndexSetName = config["indexSet"]; - - var indexSet = IndexSets.Instance.Sets[IndexSetName]; - - indexSet.ReplaceTokensInIndexPath(); - - //get the folder to index - LuceneIndexFolder = new DirectoryInfo(Path.Combine(indexSet.IndexDirectory.FullName, "Index")); - } - - base.Initialize(name, config); - - } - - /// - /// Returns true if the Umbraco application is in a state that we can initialize the examine indexes - /// - - protected bool CanInitialize() - { - // only affects indexers that are config file based, if an index was created via code then - // this has no effect, it is assumed the index would not be created if it could not be initialized - return _configBased == false || Current.RuntimeState.Level == RuntimeLevel.Run; - } - - /// - /// Returns a list of fields to search on, this will also exclude the IndexPathFieldName and node type alias - /// - /// - public override string[] GetAllIndexedFields() - { - var fields = base.GetAllIndexedFields(); - return fields - .Where(x => x != UmbracoExamineIndexer.IndexPathFieldName) - .Where(x => x != LuceneIndex.ItemTypeFieldName) - .ToArray(); - } - - } -} +// } +//} diff --git a/src/Umbraco.Tests/Umbraco.Tests.csproj b/src/Umbraco.Tests/Umbraco.Tests.csproj index 3fd6393a3c..3f9ab2ddbd 100644 --- a/src/Umbraco.Tests/Umbraco.Tests.csproj +++ b/src/Umbraco.Tests/Umbraco.Tests.csproj @@ -77,7 +77,7 @@ - + 1.8.9 diff --git a/src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs b/src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs index b478b8f8fc..8b54619c1e 100644 --- a/src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs +++ b/src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs @@ -31,7 +31,7 @@ namespace Umbraco.Tests.UmbracoExamine { public static ContentValueSetBuilder GetContentValueSetBuilder(PropertyEditorCollection propertyEditors) { - var contentValueSetBuilder = new ContentValueSetBuilder(propertyEditors, new[] { new DefaultUrlSegmentProvider() }, IndexInitializer.GetMockUserService()); + var contentValueSetBuilder = new ContentValueSetBuilder(propertyEditors, new[] { new DefaultUrlSegmentProvider() }, GetMockUserService(), true); return contentValueSetBuilder; } @@ -44,7 +44,7 @@ namespace Umbraco.Tests.UmbracoExamine public static MediaIndexPopulator GetMediaIndexRebuilder(PropertyEditorCollection propertyEditors, IMediaService mediaService) { - var mediaValueSetBuilder = new MediaValueSetBuilder(propertyEditors, new[] { new DefaultUrlSegmentProvider() }, IndexInitializer.GetMockUserService()); + var mediaValueSetBuilder = new MediaValueSetBuilder(propertyEditors, new[] { new DefaultUrlSegmentProvider() }, GetMockUserService()); var mediaIndexDataSource = new MediaIndexPopulator(null, mediaService, mediaValueSetBuilder); return mediaIndexDataSource; } diff --git a/src/Umbraco.Web.UI.Client/src/views/dashboard/settings/examinemanagement.controller.js b/src/Umbraco.Web.UI.Client/src/views/dashboard/settings/examinemanagement.controller.js index 4b54663a4d..a5e4125742 100644 --- a/src/Umbraco.Web.UI.Client/src/views/dashboard/settings/examinemanagement.controller.js +++ b/src/Umbraco.Web.UI.Client/src/views/dashboard/settings/examinemanagement.controller.js @@ -176,9 +176,6 @@ function ExamineManagementController($scope, umbRequestHelper, $http, $q, $timeo 'Failed to retrieve searcher details') .then(data => { vm.searcherDetails = data; - for (var s in vm.searcherDetails) { - vm.searcherDetails[s].searchType = "text"; - } }) ]) .then(() => { vm.loading = false }); diff --git a/src/Umbraco.Web.UI.Client/src/views/dashboard/settings/examinemanagement.html b/src/Umbraco.Web.UI.Client/src/views/dashboard/settings/examinemanagement.html index b685396ffd..5f226771e9 100644 --- a/src/Umbraco.Web.UI.Client/src/views/dashboard/settings/examinemanagement.html +++ b/src/Umbraco.Web.UI.Client/src/views/dashboard/settings/examinemanagement.html @@ -86,8 +86,107 @@
- - + + + + ← Back to overview + + + +
+ +
+ +
+
{{ vm.selectedSearcher.name }}
+
+ +
+ + + +
+ +
+
Search
+
Search the index and view the results
+
+ +
+ +
+ +
+
+ + +
+
+ + + + + +
+
+ +
+
+ + + + + + + + + + + + + + + + +
ScoreIdName
{{result.score}}{{result.id}} + {{result.values['nodeName']}}  + + ({{result.fieldCount}} fields) + +
+ +
+ + +
+ +
+ + +
+
+ +
+ +
+ +
+ +
+ +
+ +
@@ -140,33 +239,6 @@
- - -
- -
-
Index info
-
Lists the properties of the index
-
- -
- -
- - - - - - - -
 
{{key}}{{val}}
- -
- -
- -
-
@@ -200,7 +272,7 @@
-
+

@@ -212,7 +284,7 @@ - +
{{result.score}} {{result.id}} @@ -246,6 +318,35 @@ + + +
+ +
+
Index info
+
Lists the properties of the index
+
+ +
+ +
+ + + + + + + +
 
{{key}}{{val}}
+ +
+ +
+ +
+ + +
@@ -292,8 +393,6 @@
- -
diff --git a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj index 13db7ce7cb..a16e9544bb 100644 --- a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj +++ b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj @@ -88,7 +88,7 @@ - + diff --git a/src/Umbraco.Web.UI/config/ExamineIndex.config b/src/Umbraco.Web.UI/config/ExamineIndex.config index ec0d18aa2d..833a65c8d1 100644 --- a/src/Umbraco.Web.UI/config/ExamineIndex.config +++ b/src/Umbraco.Web.UI/config/ExamineIndex.config @@ -4,7 +4,11 @@ Umbraco examine is an extensible indexer and search engine. This configuration file can be extended to create your own index sets. Index/Search providers can be defined in the UmbracoSettings.config -More information and documentation can be found on GitHub: https://github.com/Shazwazza/Examine/ +More information and documentation can be found on Our: +https://our.umbraco.com/Documentation/Reference/Searching/Examine/ +https://our.umbraco.com/Documentation/Reference/Config/ExamineIndex/ +https://our.umbraco.com/Documentation/Reference/Config/ExamineSettings/ + --> diff --git a/src/Umbraco.Web.UI/config/ExamineSettings.config b/src/Umbraco.Web.UI/config/ExamineSettings.config index 265e5ff788..09a8f90207 100644 --- a/src/Umbraco.Web.UI/config/ExamineSettings.config +++ b/src/Umbraco.Web.UI/config/ExamineSettings.config @@ -4,7 +4,10 @@ Umbraco examine is an extensible indexer and search engine. This configuration file can be extended to add your own search/index providers. Index sets can be defined in the ExamineIndex.config if you're using the standard provider model. -More information and documentation can be found on GitHub: https://github.com/Shazwazza/Examine/ +More information and documentation can be found on Our: +https://our.umbraco.com/Documentation/Reference/Searching/Examine/ +https://our.umbraco.com/Documentation/Reference/Config/ExamineIndex/ +https://our.umbraco.com/Documentation/Reference/Config/ExamineSettings/ --> @@ -14,7 +17,8 @@ More information and documentation can be found on GitHub: https://github.com/Sh + - + diff --git a/src/Umbraco.Web/Editors/ExamineManagementController.cs b/src/Umbraco.Web/Editors/ExamineManagementController.cs index f624f2cee3..c72b4439f0 100644 --- a/src/Umbraco.Web/Editors/ExamineManagementController.cs +++ b/src/Umbraco.Web/Editors/ExamineManagementController.cs @@ -86,7 +86,8 @@ namespace Umbraco.Web.Editors { Id = x.Id, Score = x.Score, - Values = x.Values + //order the values by key + Values = new Dictionary(x.Values.OrderBy(y => y.Key).ToDictionary(y => y.Key, y => y.Value)) }) }; } diff --git a/src/Umbraco.Web/PropertyEditors/GridPropertyIndexValues.cs b/src/Umbraco.Web/PropertyEditors/GridPropertyIndexValues.cs index 4c03fe2ccd..04876ec118 100644 --- a/src/Umbraco.Web/PropertyEditors/GridPropertyIndexValues.cs +++ b/src/Umbraco.Web/PropertyEditors/GridPropertyIndexValues.cs @@ -18,11 +18,11 @@ namespace Umbraco.Web.PropertyEditors /// public class GridPropertyIndexValues : IPropertyIndexValues { - public IEnumerable> GetIndexValues(Property property, string culture, string segment) + public IEnumerable>> GetIndexValues(Property property, string culture, string segment, bool published) { - var result = new List>(); + var result = new List>>(); - var val = property.GetValue(culture, segment); + var val = property.GetValue(culture, segment, published); //if there is a value, it's a string and it's detected as json if (val is string rawVal && rawVal.DetectIsJson()) @@ -51,7 +51,7 @@ namespace Umbraco.Web.PropertyEditors sb.Append(" "); //add the row name as an individual field - result.Add(new KeyValuePair($"{property.Alias}.{rowName}", new[] { str })); + result.Add(new KeyValuePair>($"{property.Alias}.{rowName}", new[] { str })); } } } @@ -59,10 +59,10 @@ namespace Umbraco.Web.PropertyEditors if (sb.Length > 0) { //First save the raw value to a raw field - result.Add(new KeyValuePair($"{UmbracoExamineIndexer.RawFieldPrefix}{property.Alias}", new[] { rawVal })); + result.Add(new KeyValuePair>($"{UmbracoExamineIndexer.RawFieldPrefix}{property.Alias}", new[] { rawVal })); //index the property with the combined/cleaned value - result.Add(new KeyValuePair(property.Alias, new[] { sb.ToString() })); + result.Add(new KeyValuePair>(property.Alias, new[] { sb.ToString() })); } } catch (InvalidCastException) diff --git a/src/Umbraco.Web/PropertyEditors/RichTextPropertyEditor.cs b/src/Umbraco.Web/PropertyEditors/RichTextPropertyEditor.cs index fa490433ba..6dd63e6123 100644 --- a/src/Umbraco.Web/PropertyEditors/RichTextPropertyEditor.cs +++ b/src/Umbraco.Web/PropertyEditors/RichTextPropertyEditor.cs @@ -95,16 +95,16 @@ namespace Umbraco.Web.PropertyEditors internal class RichTextPropertyIndexValues : IPropertyIndexValues { - public IEnumerable> GetIndexValues(Property property, string culture, string segment) + public IEnumerable>> GetIndexValues(Property property, string culture, string segment, bool published) { - var val = property.GetValue(culture, segment); + var val = property.GetValue(culture, segment, published); if (!(val is string strVal)) yield break; //index the stripped html values - yield return new KeyValuePair(property.Alias, new object[] { strVal.StripHtml() }); + yield return new KeyValuePair>(property.Alias, new object[] { strVal.StripHtml() }); //store the raw value - yield return new KeyValuePair($"{UmbracoExamineIndexer.RawFieldPrefix}{property.Alias}", new object[] { strVal }); + yield return new KeyValuePair>($"{UmbracoExamineIndexer.RawFieldPrefix}{property.Alias}", new object[] { strVal }); } } } diff --git a/src/Umbraco.Web/PublishedContentQuery.cs b/src/Umbraco.Web/PublishedContentQuery.cs index 1883010582..c2fb84a3da 100644 --- a/src/Umbraco.Web/PublishedContentQuery.cs +++ b/src/Umbraco.Web/PublishedContentQuery.cs @@ -242,15 +242,12 @@ namespace Umbraco.Web var searcher = index.GetSearcher(); - if (skip == 0 && take == 0) - { - var results = searcher.Search(term, useWildCards); - totalRecords = results.TotalItemCount; - return results.ToPublishedSearchResults(_contentCache); - } + var results = skip == 0 && take == 0 + ? searcher.Search(term, true) + : searcher.Search(term, true, maxResults: skip + take); - var criteria = SearchAllFields(term, useWildCards, searcher, index); - return Search(skip, take, out totalRecords, criteria, searcher); + totalRecords = results.TotalItemCount; + return results.ToPublishedSearchResults(_contentCache); } /// @@ -280,28 +277,6 @@ namespace Umbraco.Web return results.ToPublishedSearchResults(_contentCache); } - /// - /// Creates an ISearchCriteria for searching all fields in a . - /// - private ISearchCriteria SearchAllFields(string searchText, bool useWildcards, ISearcher searcher, IIndex indexer) - { - var sc = searcher.CreateCriteria(); - - //if we're dealing with a lucene searcher, we can get all of it's indexed fields, - //else we can get the defined fields for the index. - var searchFields = (searcher is BaseLuceneSearcher luceneSearcher) - ? luceneSearcher.GetAllIndexedFields() - : indexer.FieldDefinitionCollection.Keys; - - //this is what Examine does internally to create ISearchCriteria for searching all fields - var strArray = searchText.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries); - - sc = useWildcards == false - ? sc.GroupedOr(searchFields, strArray).Compile() - : sc.GroupedOr(searchFields, strArray.Select(x => (IExamineValue)new ExamineValue(Examineness.ComplexWildcard, x.MultipleCharacterWildcard().Value)).ToArray()).Compile(); - return sc; - } - #endregion } diff --git a/src/Umbraco.Web/Search/ExamineComponent.cs b/src/Umbraco.Web/Search/ExamineComponent.cs index fa22da5ae1..1a842525a4 100644 --- a/src/Umbraco.Web/Search/ExamineComponent.cs +++ b/src/Umbraco.Web/Search/ExamineComponent.cs @@ -22,8 +22,10 @@ using Umbraco.Examine; using Umbraco.Core.Persistence.DatabaseModelDefinitions; using Umbraco.Web.Scheduling; using System.Threading.Tasks; +using Examine.LuceneEngine.Directories; using LightInject; using Umbraco.Core.Composing; +using Umbraco.Core.Strings; namespace Umbraco.Web.Search { @@ -34,7 +36,8 @@ namespace Umbraco.Web.Search public sealed class ExamineComponent : UmbracoComponentBase, IUmbracoCoreComponent { private IExamineManager _examineManager; - private IValueSetBuilder _contentValueSetBuilder; + private IContentValueSetBuilder _contentValueSetBuilder; + private IPublishedContentValueSetBuilder _publishedContentValueSetBuilder; private IValueSetBuilder _mediaValueSetBuilder; private IValueSetBuilder _memberValueSetBuilder; private static bool _disableExamineIndexing = false; @@ -71,7 +74,18 @@ namespace Umbraco.Web.Search composition.Container.RegisterSingleton(); composition.Container.RegisterSingleton(); - composition.Container.RegisterSingleton, ContentValueSetBuilder>(); + composition.Container.Register(factory => + new ContentValueSetBuilder( + factory.GetInstance(), + factory.GetInstance>(), + factory.GetInstance(), + false)); + composition.Container.Register(factory => + new ContentValueSetBuilder( + factory.GetInstance(), + factory.GetInstance>(), + factory.GetInstance(), + true)); composition.Container.RegisterSingleton, MediaValueSetBuilder>(); composition.Container.RegisterSingleton, MemberValueSetBuilder>(); } @@ -80,7 +94,8 @@ namespace Umbraco.Web.Search IExamineManager examineManager, ProfilingLogger profilingLogger, IScopeProvider scopeProvider, IUmbracoIndexesCreator indexCreator, IndexRebuilder indexRebuilder, ServiceContext services, - IValueSetBuilder contentValueSetBuilder, + IContentValueSetBuilder contentValueSetBuilder, + IPublishedContentValueSetBuilder publishedContentValueSetBuilder, IValueSetBuilder mediaValueSetBuilder, IValueSetBuilder memberValueSetBuilder) { @@ -88,6 +103,7 @@ namespace Umbraco.Web.Search _scopeProvider = scopeProvider; _examineManager = examineManager; _contentValueSetBuilder = contentValueSetBuilder; + _publishedContentValueSetBuilder = publishedContentValueSetBuilder; _mediaValueSetBuilder = mediaValueSetBuilder; _memberValueSetBuilder = memberValueSetBuilder; @@ -98,7 +114,7 @@ namespace Umbraco.Web.Search //we want to tell examine to use a different fs lock instead of the default NativeFSFileLock which could cause problems if the appdomain //terminates and in some rare cases would only allow unlocking of the file if IIS is forcefully terminated. Instead we'll rely on the simplefslock //which simply checks the existence of the lock file - DirectoryTracker.DefaultLockFactory = d => + DirectoryFactory.DefaultLockFactory = d => { var simpleFsLockFactory = new NoPrefixSimpleFsLockFactory(d); return simpleFsLockFactory; @@ -221,8 +237,106 @@ namespace Umbraco.Web.Search } } } - + #region Cache refresher updated event handlers + + /// + /// Updates indexes based on content changes + /// + /// + /// + private void ContentCacheRefresherUpdated(ContentCacheRefresher sender, CacheRefresherEventArgs args) + { + if (Suspendable.ExamineEvents.CanIndex == false) + return; + + if (args.MessageType != MessageType.RefreshByPayload) + throw new NotSupportedException(); + + var contentService = _services.ContentService; + + foreach (var payload in (ContentCacheRefresher.JsonPayload[])args.MessageObject) + { + if (payload.ChangeTypes.HasType(TreeChangeTypes.Remove)) + { + // delete content entirely (with descendants) + // false: remove entirely from all indexes + DeleteIndexForEntity(payload.Id, false); + } + else if (payload.ChangeTypes.HasType(TreeChangeTypes.RefreshAll)) + { + // ExamineEvents does not support RefreshAll + // just ignore that payload + // so what?! + + //fixme: Rebuild the index at this point? + } + else // RefreshNode or RefreshBranch (maybe trashed) + { + // don't try to be too clever - refresh entirely + // there has to be race conds in there ;-( + + var content = contentService.GetById(payload.Id); + if (content == null) + { + // gone fishing, remove entirely from all indexes (with descendants) + DeleteIndexForEntity(payload.Id, false); + continue; + } + + IContent published = null; + if (content.Published && contentService.IsPathPublished(content)) + published = content; + + if (published == null) + DeleteIndexForEntity(payload.Id, true); + + // just that content + ReIndexForContent(content, published != null); + + // branch + if (payload.ChangeTypes.HasType(TreeChangeTypes.RefreshBranch)) + { + var masked = published == null ? null : new List(); + const int pageSize = 500; + var page = 0; + var total = long.MaxValue; + while (page * pageSize < total) + { + var descendants = contentService.GetPagedDescendants(content.Id, page++, pageSize, out total, + //order by shallowest to deepest, this allows us to check it's published state without checking every item + ordering: Ordering.By("Path", Direction.Ascending)); + + foreach (var descendant in descendants) + { + published = null; + if (masked != null) // else everything is masked + { + if (masked.Contains(descendant.ParentId) || !descendant.Published) + masked.Add(descendant.Id); + else + published = descendant; + } + + ReIndexForContent(descendant, published != null); + } + } + } + } + + // NOTE + // + // DeleteIndexForEntity is handled by UmbracoContentIndexer.DeleteFromIndex() which takes + // care of also deleting the descendants + // + // ReIndexForContent is NOT taking care of descendants so we have to reload everything + // again in order to process the branch - we COULD improve that by just reloading the + // XML from database instead of reloading content & re-serializing! + // + // BUT ... pretty sure it is! see test "Index_Delete_Index_Item_Ensure_Heirarchy_Removed" + } + } + private void MemberCacheRefresherUpdated(MemberCacheRefresher sender, CacheRefresherEventArgs args) { if (Suspendable.ExamineEvents.CanIndex == false) @@ -292,15 +406,18 @@ namespace Umbraco.Web.Search else // RefreshNode or RefreshBranch (maybe trashed) { var media = mediaService.GetById(payload.Id); - if (media == null || media.Trashed) + if (media == null) { // gone fishing, remove entirely DeleteIndexForEntity(payload.Id, false); continue; } + if (media.Trashed) + DeleteIndexForEntity(payload.Id, true); + // just that media - ReIndexForMedia(media, media.Trashed == false); + ReIndexForMedia(media, !media.Trashed); // branch if (payload.ChangeTypes.HasType(TreeChangeTypes.RefreshBranch)) @@ -313,7 +430,7 @@ namespace Umbraco.Web.Search var descendants = mediaService.GetPagedDescendants(media.Id, page++, pageSize, out total); foreach (var descendant in descendants) { - ReIndexForMedia(descendant, descendant.Trashed == false); + ReIndexForMedia(descendant, !descendant.Trashed); } } } @@ -461,155 +578,33 @@ namespace Umbraco.Web.Search foreach (var c in contentToRefresh) { - IContent published = null; + var isPublished = false; if (c.Published) { - if (publishChecked.TryGetValue(c.ParentId, out var isPublished)) - { - //if the parent's published path has already been verified then this is published - if (isPublished) - published = c; - } - else + if (!publishChecked.TryGetValue(c.ParentId, out isPublished)) { //nothing by parent id, so query the service and cache the result for the next child to check against isPublished = _services.ContentService.IsPathPublished(c); publishChecked[c.Id] = isPublished; - if (isPublished) - published = c; } } - ReIndexForContent(c, published); + ReIndexForContent(c, isPublished); } } } - /// - /// Updates indexes based on content changes - /// - /// - /// - private void ContentCacheRefresherUpdated(ContentCacheRefresher sender, CacheRefresherEventArgs args) - { - if (Suspendable.ExamineEvents.CanIndex == false) - return; - - if (args.MessageType != MessageType.RefreshByPayload) - throw new NotSupportedException(); - - var contentService = _services.ContentService; - - foreach (var payload in (ContentCacheRefresher.JsonPayload[])args.MessageObject) - { - if (payload.ChangeTypes.HasType(TreeChangeTypes.Remove)) - { - // delete content entirely (with descendants) - // false: remove entirely from all indexes - DeleteIndexForEntity(payload.Id, false); - } - else if (payload.ChangeTypes.HasType(TreeChangeTypes.RefreshAll)) - { - // ExamineEvents does not support RefreshAll - // just ignore that payload - // so what?! - - //fixme: Rebuild the index at this point? - } - else // RefreshNode or RefreshBranch (maybe trashed) - { - // don't try to be too clever - refresh entirely - // there has to be race conds in there ;-( - - var content = contentService.GetById(payload.Id); - if (content == null || content.Trashed) - { - // gone fishing, remove entirely from all indexes (with descendants) - DeleteIndexForEntity(payload.Id, false); - continue; - } - - IContent published = null; - if (content.Published && contentService.IsPathPublished(content)) - published = content; - - // just that content - ReIndexForContent(content, published); - - // branch - if (payload.ChangeTypes.HasType(TreeChangeTypes.RefreshBranch)) - { - var masked = published == null ? null : new List(); - const int pageSize = 500; - var page = 0; - var total = long.MaxValue; - while (page * pageSize < total) - { - var descendants = contentService.GetPagedDescendants(content.Id, page++, pageSize, out total, - //order by shallowest to deepest, this allows us to check it's published state without checking every item - ordering: Ordering.By("Path", Direction.Ascending)); - - foreach (var descendant in descendants) - { - published = null; - if (masked != null) // else everything is masked - { - if (masked.Contains(descendant.ParentId) || !descendant.Published) - masked.Add(descendant.Id); - else - published = descendant; - } - - ReIndexForContent(descendant, published); - } - } - } - } - - // NOTE - // - // DeleteIndexForEntity is handled by UmbracoContentIndexer.DeleteFromIndex() which takes - // care of also deleting the descendants - // - // ReIndexForContent is NOT taking care of descendants so we have to reload everything - // again in order to process the branch - we COULD improve that by just reloading the - // XML from database instead of reloading content & re-serializing! - // - // BUT ... pretty sure it is! see test "Index_Delete_Index_Item_Ensure_Heirarchy_Removed" - } - } + #endregion #region ReIndex/Delete for entity - private void ReIndexForContent(IContent content, IContent published) - { - if (published != null && content.VersionId == published.VersionId) - { - ReIndexForContent(content); // same = both - } - else - { - if (published == null) - { - // remove 'published' - keep 'draft' - DeleteIndexForEntity(content.Id, true); - } - else - { - // index 'published' - don't overwrite 'draft' - ReIndexForContent(published, false); - } - ReIndexForContent(content, true); // index 'draft' - } - } - - private void ReIndexForContent(IContent sender, bool? supportUnpublished = null) + private void ReIndexForContent(IContent sender, bool isPublished) { var actions = DeferedActions.Get(_scopeProvider); if (actions != null) - actions.Add(new DeferedReIndexForContent(this, sender, supportUnpublished)); + actions.Add(new DeferedReIndexForContent(this, sender, isPublished)); else - DeferedReIndexForContent.Execute(this, sender, supportUnpublished); + DeferedReIndexForContent.Execute(this, sender, isPublished); } private void ReIndexForMember(IMember member) @@ -621,17 +616,17 @@ namespace Umbraco.Web.Search DeferedReIndexForMember.Execute(this, member); } - private void ReIndexForMedia(IMedia sender, bool isMediaPublished) + private void ReIndexForMedia(IMedia sender, bool isPublished) { var actions = DeferedActions.Get(_scopeProvider); if (actions != null) - actions.Add(new DeferedReIndexForMedia(this, sender, isMediaPublished)); + actions.Add(new DeferedReIndexForMedia(this, sender, isPublished)); else - DeferedReIndexForMedia.Execute(this, sender, isMediaPublished); + DeferedReIndexForMedia.Execute(this, sender, isPublished); } /// - /// Remove items from any index that doesn't support unpublished content + /// Remove items from an index /// /// /// @@ -687,30 +682,33 @@ namespace Umbraco.Web.Search { private readonly ExamineComponent _examineComponent; private readonly IContent _content; - private readonly bool? _supportUnpublished; + private readonly bool _isPublished; - public DeferedReIndexForContent(ExamineComponent examineComponent, IContent content, bool? supportUnpublished) + public DeferedReIndexForContent(ExamineComponent examineComponent, IContent content, bool isPublished) { _examineComponent = examineComponent; _content = content; - _supportUnpublished = supportUnpublished; + _isPublished = isPublished; } public override void Execute() { - Execute(_examineComponent, _content, _supportUnpublished); + Execute(_examineComponent, _content, _isPublished); } - public static void Execute(ExamineComponent examineComponent, IContent content, bool? supportUnpublished) + public static void Execute(ExamineComponent examineComponent, IContent content, bool isPublished) { - var valueSet = examineComponent._contentValueSetBuilder.GetValueSets(content).ToList(); - foreach (var index in examineComponent._examineManager.Indexes.OfType() - // only for the specified indexers - .Where(x => supportUnpublished.HasValue == false || supportUnpublished.Value == x.SupportSoftDelete) + //filter the indexers + .Where(x => isPublished || !x.PublishedValuesOnly) .Where(x => x.EnableDefaultEventHandler)) { - index.IndexItems(valueSet); + //for content we have a different builder for published vs unpublished + var builder = index.PublishedValuesOnly + ? examineComponent._publishedContentValueSetBuilder + : (IValueSetBuilder)examineComponent._contentValueSetBuilder; + + index.IndexItems(builder.GetValueSets(content)); } } } @@ -738,9 +736,8 @@ namespace Umbraco.Web.Search var valueSet = examineComponent._mediaValueSetBuilder.GetValueSets(media).ToList(); foreach (var index in examineComponent._examineManager.Indexes.OfType() - // index this item for all indexers if the media is not trashed, otherwise if the item is trashed - // then only index this for indexers supporting unpublished media - .Where(x => isPublished || (x.SupportSoftDelete)) + //filter the indexers + .Where(x => isPublished || !x.PublishedValuesOnly) .Where(x => x.EnableDefaultEventHandler)) { index.IndexItems(valueSet); @@ -768,7 +765,7 @@ namespace Umbraco.Web.Search { var valueSet = examineComponent._memberValueSetBuilder.GetValueSets(member).ToList(); foreach (var index in examineComponent._examineManager.Indexes.OfType() - //ensure that only the providers are flagged to listen execute + //filter the indexers .Where(x => x.EnableDefaultEventHandler)) { index.IndexItems(valueSet); @@ -798,9 +795,8 @@ namespace Umbraco.Web.Search { var strId = id.ToString(CultureInfo.InvariantCulture); foreach (var index in examineComponent._examineManager.Indexes.OfType() - // if keepIfUnpublished == true then only delete this item from indexes not supporting unpublished content, - // otherwise if keepIfUnpublished == false then remove from all indexes - .Where(x => keepIfUnpublished == false || x.SupportSoftDelete == false) + + .Where(x => (keepIfUnpublished && !x.PublishedValuesOnly) || !keepIfUnpublished) .Where(x => x.EnableDefaultEventHandler)) { index.DeleteFromIndex(strId); diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index 1ebb54248d..63908a23f9 100755 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -62,7 +62,7 @@ - + 2.6.2.25 diff --git a/src/Umbraco.Web/UmbracoHelper.cs b/src/Umbraco.Web/UmbracoHelper.cs index 357c8a30a6..fbb739b5c2 100644 --- a/src/Umbraco.Web/UmbracoHelper.cs +++ b/src/Umbraco.Web/UmbracoHelper.cs @@ -30,13 +30,11 @@ namespace Umbraco.Web private readonly IPublishedContent _currentPage; private readonly IPublishedContentQuery _iQuery; private readonly ServiceContext _services; - private readonly CacheHelper _appCache; - + private IUmbracoComponentRenderer _componentRenderer; private PublishedContentQuery _query; private MembershipHelper _membershipHelper; private ITagQuery _tag; - private IDataTypeService _dataTypeService; private ICultureDictionary _cultureDictionary; #region Constructors @@ -48,34 +46,21 @@ namespace Umbraco.Web internal UmbracoHelper(UmbracoContext umbracoContext, IPublishedContent content, IPublishedContentQuery query, ITagQuery tagQuery, - IDataTypeService dataTypeService, ICultureDictionary cultureDictionary, IUmbracoComponentRenderer componentRenderer, MembershipHelper membershipHelper, - ServiceContext services, - CacheHelper appCache) + ServiceContext services) { - if (umbracoContext == null) throw new ArgumentNullException(nameof(umbracoContext)); - if (content == null) throw new ArgumentNullException(nameof(content)); - if (query == null) throw new ArgumentNullException(nameof(query)); if (tagQuery == null) throw new ArgumentNullException(nameof(tagQuery)); - if (dataTypeService == null) throw new ArgumentNullException(nameof(dataTypeService)); - if (cultureDictionary == null) throw new ArgumentNullException(nameof(cultureDictionary)); - if (componentRenderer == null) throw new ArgumentNullException(nameof(componentRenderer)); - if (membershipHelper == null) throw new ArgumentNullException(nameof(membershipHelper)); - if (services == null) throw new ArgumentNullException(nameof(services)); - if (appCache == null) throw new ArgumentNullException(nameof(appCache)); - _umbracoContext = umbracoContext; + _umbracoContext = umbracoContext ?? throw new ArgumentNullException(nameof(umbracoContext)); _tag = new TagQuery(tagQuery); - _dataTypeService = dataTypeService; - _cultureDictionary = cultureDictionary; - _componentRenderer = componentRenderer; - _membershipHelper = membershipHelper; - _currentPage = content; - _iQuery = query; - _services = services; - _appCache = appCache; + _cultureDictionary = cultureDictionary ?? throw new ArgumentNullException(nameof(cultureDictionary)); + _componentRenderer = componentRenderer ?? throw new ArgumentNullException(nameof(componentRenderer)); + _membershipHelper = membershipHelper ?? throw new ArgumentNullException(nameof(membershipHelper)); + _currentPage = content ?? throw new ArgumentNullException(nameof(content)); + _iQuery = query ?? throw new ArgumentNullException(nameof(query)); + _services = services ?? throw new ArgumentNullException(nameof(services)); } /// @@ -93,13 +78,11 @@ namespace Umbraco.Web /// An Umbraco context. /// A content item. /// A services context. - /// An application cache helper. /// Sets the current page to the supplied content item. - public UmbracoHelper(UmbracoContext umbracoContext, ServiceContext services, CacheHelper appCache, IPublishedContent content) - : this(umbracoContext, services, appCache) + public UmbracoHelper(UmbracoContext umbracoContext, ServiceContext services, IPublishedContent content) + : this(umbracoContext, services) { - if (content == null) throw new ArgumentNullException(nameof(content)); - _currentPage = content; + _currentPage = content ?? throw new ArgumentNullException(nameof(content)); } /// @@ -107,19 +90,13 @@ namespace Umbraco.Web /// /// An Umbraco context. /// A services context. - /// An application cache helper. /// Sets the current page to the context's published content request's content item. - public UmbracoHelper(UmbracoContext umbracoContext, ServiceContext services, CacheHelper appCache) + public UmbracoHelper(UmbracoContext umbracoContext, ServiceContext services) { - if (umbracoContext == null) throw new ArgumentNullException(nameof(umbracoContext)); - if (services == null) throw new ArgumentNullException(nameof(services)); - if (appCache == null) throw new ArgumentNullException(nameof(appCache)); - - _umbracoContext = umbracoContext; + _services = services ?? throw new ArgumentNullException(nameof(services)); + _umbracoContext = umbracoContext ?? throw new ArgumentNullException(nameof(umbracoContext)); if (_umbracoContext.IsFrontEndUmbracoRequest) _currentPage = _umbracoContext.PublishedRequest.PublishedContent; - _services = services; - _appCache = appCache; } #endregion @@ -133,7 +110,7 @@ namespace Umbraco.Web /// /// Gets the query context. /// - public PublishedContentQuery ContentQuery => _query ?? + public IPublishedContentQuery ContentQuery => _query ?? (_query = _iQuery != null ? new PublishedContentQuery(_iQuery) : new PublishedContentQuery(UmbracoContext.ContentCache, UmbracoContext.MediaCache)); @@ -162,12 +139,6 @@ namespace Umbraco.Web /// public UrlProvider UrlProvider => UmbracoContext.UrlProvider; - /// - /// Gets the datatype service. - /// - private IDataTypeService DataTypeService => _dataTypeService - ?? (_dataTypeService = _services.DataTypeService); - /// /// Gets the component renderer. /// @@ -819,11 +790,11 @@ namespace Umbraco.Web /// /// /// - /// + /// /// - public IEnumerable Search(string term, bool useWildCards = true, string searchProvider = null) + public IEnumerable Search(string term, bool useWildCards = true, string indexName = null) { - return ContentQuery.Search(term, useWildCards, searchProvider); + return ContentQuery.Search(term, useWildCards, indexName); } /// @@ -834,11 +805,11 @@ namespace Umbraco.Web /// /// /// - /// + /// /// - public IEnumerable TypedSearch(int skip, int take, out long totalRecords, string term, bool useWildCards = true, string searchProvider = null) + public IEnumerable Search(int skip, int take, out long totalRecords, string term, bool useWildCards = true, string indexName = null) { - return ContentQuery.Search(skip, take, out totalRecords, term, useWildCards, searchProvider); + return ContentQuery.Search(skip, take, out totalRecords, term, useWildCards, indexName); } /// @@ -850,7 +821,7 @@ namespace Umbraco.Web /// /// /// - public IEnumerable TypedSearch(int skip, int take, out long totalRecords, Examine.SearchCriteria.ISearchCriteria criteria, Examine.Providers.BaseSearchProvider searchProvider = null) + public IEnumerable Search(int skip, int take, out long totalRecords, Examine.SearchCriteria.ISearchCriteria criteria, Examine.Providers.BaseSearchProvider searchProvider = null) { return ContentQuery.Search(skip, take, out totalRecords, criteria, searchProvider); }