From 6ab5d6ec9fdac787f1792ad6d13ffb5df2351c12 Mon Sep 17 00:00:00 2001 From: Shannon Date: Wed, 28 Nov 2018 18:39:03 +1100 Subject: [PATCH] More cleanup, removes options since it was irrelevant --- .../ContentValueSetValidator.cs | 50 +++++----- src/Umbraco.Examine/IUmbracoIndexer.cs | 7 +- .../MemberValueSetValidator.cs | 32 +++++++ src/Umbraco.Examine/Umbraco.Examine.csproj | 3 +- src/Umbraco.Examine/UmbracoContentIndexer.cs | 62 ++++--------- .../UmbracoContentIndexerOptions.cs | 52 ----------- src/Umbraco.Examine/UmbracoExamineIndexer.cs | 26 +----- src/Umbraco.Examine/UmbracoMemberIndexer.cs | 8 +- src/Umbraco.Examine/ValueSetValidator.cs | 91 +++++++++++++++++++ .../PublishedContent/PublishedMediaTests.cs | 28 +++--- .../UmbracoExamine/EventsTest.cs | 4 +- .../UmbracoExamine/IndexInitializer.cs | 9 +- src/Umbraco.Tests/UmbracoExamine/IndexTest.cs | 14 +-- .../UmbracoContentValueSetValidatorTests.cs | 48 +++------- src/Umbraco.Web/Search/ExamineComponent.cs | 10 +- .../Search/UmbracoIndexesBuilder.cs | 56 ++++++------ 16 files changed, 259 insertions(+), 241 deletions(-) create mode 100644 src/Umbraco.Examine/MemberValueSetValidator.cs delete mode 100644 src/Umbraco.Examine/UmbracoContentIndexerOptions.cs create mode 100644 src/Umbraco.Examine/ValueSetValidator.cs diff --git a/src/Umbraco.Examine/ContentValueSetValidator.cs b/src/Umbraco.Examine/ContentValueSetValidator.cs index 4fbb039bae..9355d6d695 100644 --- a/src/Umbraco.Examine/ContentValueSetValidator.cs +++ b/src/Umbraco.Examine/ContentValueSetValidator.cs @@ -9,31 +9,45 @@ using Umbraco.Core.Services; namespace Umbraco.Examine { - /// /// Used to validate a ValueSet for content/media - based on permissions, parent id, etc.... /// - public class ContentValueSetValidator : IValueSetValidator + public class ContentValueSetValidator : ValueSetValidator { - private readonly UmbracoContentIndexerOptions _options; private readonly IPublicAccessService _publicAccessService; private const string PathKey = "path"; - private static readonly IEnumerable ValidIndexTypes = new[] {IndexTypes.Content, IndexTypes.Media}; + private static readonly IEnumerable ValidCategories = new[] {IndexTypes.Content, IndexTypes.Media}; + protected override IEnumerable ValidIndexCategories => ValidCategories; - public ContentValueSetValidator(UmbracoContentIndexerOptions options, IPublicAccessService publicAccessService) + public bool SupportUnpublishedContent { get; } + public bool SupportProtectedContent { get; } + public int? ParentId { get; } + + public ContentValueSetValidator(bool supportUnpublishedContent, int? parentId = null, + IEnumerable includeItemTypes = null, IEnumerable excludeItemTypes = null) + : this(supportUnpublishedContent, true, null, parentId, includeItemTypes, excludeItemTypes) { - _options = options; + } + + public ContentValueSetValidator(bool supportUnpublishedContent, bool supportProtectedContent, + IPublicAccessService publicAccessService, int? parentId = null, + IEnumerable includeItemTypes = null, IEnumerable excludeItemTypes = null) + : base(includeItemTypes, excludeItemTypes, null, null) + { + SupportUnpublishedContent = supportUnpublishedContent; + SupportProtectedContent = supportProtectedContent; + ParentId = parentId; _publicAccessService = publicAccessService; } - public bool Validate(ValueSet valueSet) + public override bool Validate(ValueSet valueSet) { - if (!ValidIndexTypes.Contains(valueSet.Category)) + if (!ValidCategories.Contains(valueSet.Category)) return false; //check for published content - if (valueSet.Category == IndexTypes.Content && !_options.SupportUnpublishedContent) + if (valueSet.Category == IndexTypes.Content && !SupportUnpublishedContent) { if (!valueSet.Values.TryGetValue(UmbracoExamineIndexer.PublishedFieldName, out var published)) return false; @@ -70,36 +84,28 @@ namespace Umbraco.Examine // return nothing if we're not supporting protected content and it is protected, and we're not supporting unpublished content if (valueSet.Category == IndexTypes.Content - && !_options.SupportProtectedContent + && !SupportProtectedContent && _publicAccessService.IsProtected(path)) { return false; } //check if this document is a descendent of the parent - if (_options.ParentId.HasValue && _options.ParentId.Value > 0) + if (ParentId.HasValue && ParentId.Value > 0) { - if (!path.Contains(string.Concat(",", _options.ParentId.Value, ","))) + if (!path.Contains(string.Concat(",", ParentId.Value, ","))) return false; } //check for recycle bin - if (!_options.SupportUnpublishedContent) + if (!SupportUnpublishedContent) { var recycleBinId = valueSet.Category == IndexTypes.Content ? Constants.System.RecycleBinContent : Constants.System.RecycleBinMedia; if (path.Contains(string.Concat(",", recycleBinId, ","))) return false; } - //check if this document is of a correct type of node type alias - if (_options.IncludeContentTypes != null && !_options.IncludeContentTypes.Contains(valueSet.ItemType)) - return false; - - //if this node type is part of our exclusion list - if (_options.ExcludeContentTypes != null && _options.ExcludeContentTypes.Contains(valueSet.ItemType)) - return false; - - return true; + return base.Validate(valueSet); } } } diff --git a/src/Umbraco.Examine/IUmbracoIndexer.cs b/src/Umbraco.Examine/IUmbracoIndexer.cs index 936c78c71d..1785cd5a92 100644 --- a/src/Umbraco.Examine/IUmbracoIndexer.cs +++ b/src/Umbraco.Examine/IUmbracoIndexer.cs @@ -14,11 +14,8 @@ namespace Umbraco.Examine bool EnableDefaultEventHandler { get; } /// - /// When set to true data will not be deleted from the index if the data is being unpublished (not deleted) + /// When set to true data will not be deleted from the index if the data is being soft deleted (unpublished or trashed) /// - /// - /// Generally used only for published content - /// - bool SupportUnpublishedContent { get; } + bool SupportSoftDelete { get; } } } diff --git a/src/Umbraco.Examine/MemberValueSetValidator.cs b/src/Umbraco.Examine/MemberValueSetValidator.cs new file mode 100644 index 0000000000..b8e96dc68d --- /dev/null +++ b/src/Umbraco.Examine/MemberValueSetValidator.cs @@ -0,0 +1,32 @@ +using System.Collections.Generic; +using System.Linq; +using Examine; + +namespace Umbraco.Examine +{ + public class MemberValueSetValidator : ValueSetValidator + { + public MemberValueSetValidator() : base(null, null, DefaultMemberIndexFields, null) + { + } + + public MemberValueSetValidator(IEnumerable includeItemTypes, IEnumerable excludeItemTypes) + : base(includeItemTypes, excludeItemTypes, DefaultMemberIndexFields, null) + { + } + + public MemberValueSetValidator(IEnumerable includeItemTypes, IEnumerable excludeItemTypes, IEnumerable includeFields, IEnumerable excludeFields) + : base(includeItemTypes, excludeItemTypes, includeFields, excludeFields) + { + } + + /// + /// By default these are the member fields we index + /// + public static readonly string[] DefaultMemberIndexFields = { "id", "nodeName", "updateDate", "writerName", "loginName", "email", "nodeTypeAlias" }; + + private static readonly IEnumerable ValidCategories = new[] { IndexTypes.Member }; + protected override IEnumerable ValidIndexCategories => ValidCategories; + + } +} diff --git a/src/Umbraco.Examine/Umbraco.Examine.csproj b/src/Umbraco.Examine/Umbraco.Examine.csproj index e36f911f47..5c206250b1 100644 --- a/src/Umbraco.Examine/Umbraco.Examine.csproj +++ b/src/Umbraco.Examine/Umbraco.Examine.csproj @@ -74,13 +74,13 @@ + - @@ -88,6 +88,7 @@ Properties\SolutionInfo.cs + diff --git a/src/Umbraco.Examine/UmbracoContentIndexer.cs b/src/Umbraco.Examine/UmbracoContentIndexer.cs index 8e8208479c..ddf5f23b4f 100644 --- a/src/Umbraco.Examine/UmbracoContentIndexer.cs +++ b/src/Umbraco.Examine/UmbracoContentIndexer.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Collections.Specialized; using System.ComponentModel; +using System.Linq; using Examine; using Umbraco.Core; using Umbraco.Core.Services; @@ -22,11 +23,9 @@ namespace Umbraco.Examine /// public class UmbracoContentIndexer : UmbracoExamineIndexer, IUmbracoContentIndexer, IUmbracoMediaIndexer { - public const string VariesByCultureFieldName = UmbracoExamineIndexer.SpecialFieldPrefix + "VariesByCulture"; + public const string VariesByCultureFieldName = SpecialFieldPrefix + "VariesByCulture"; protected ILocalizationService LanguageService { get; } - private int? _parentId; - #region Constructors /// @@ -36,6 +35,8 @@ namespace Umbraco.Examine public UmbracoContentIndexer() { LanguageService = Current.Services.LocalizationService; + + //note: The validator for this config based indexer is set in the Initialize method } /// @@ -46,8 +47,8 @@ namespace Umbraco.Examine /// /// /// + /// /// - /// /// public UmbracoContentIndexer( string name, @@ -57,19 +58,15 @@ namespace Umbraco.Examine ProfilingLogger profilingLogger, ILocalizationService languageService, IValueSetValidator validator, - UmbracoContentIndexerOptions options, IReadOnlyDictionary> indexValueTypes = null) : base(name, fieldDefinitions, luceneDirectory, defaultAnalyzer, profilingLogger, validator, indexValueTypes) { if (validator == null) throw new ArgumentNullException(nameof(validator)); - if (options == null) throw new ArgumentNullException(nameof(options)); - - SupportProtectedContent = options.SupportProtectedContent; - SupportUnpublishedContent = options.SupportUnpublishedContent; - ParentId = options.ParentId; LanguageService = languageService ?? throw new ArgumentNullException(nameof(languageService)); - } + if (validator is ContentValueSetValidator contentValueSetValidator) + SupportSoftDelete = contentValueSetValidator.SupportUnpublishedContent; + } #endregion @@ -97,19 +94,19 @@ namespace Umbraco.Examine { base.Initialize(name, config); + var supportUnpublished = false; + var supportProtected = false; + //check if there's a flag specifying to support unpublished content, //if not, set to false; - if (config["supportUnpublished"] != null && bool.TryParse(config["supportUnpublished"], out var supportUnpublished)) - SupportUnpublishedContent = supportUnpublished; - else - SupportUnpublishedContent = false; + if (config["supportUnpublished"] != null) + bool.TryParse(config["supportUnpublished"], out supportUnpublished); //check if there's a flag specifying to support protected content, //if not, set to false; - if (config["supportProtected"] != null && bool.TryParse(config["supportProtected"], out var supportProtected)) - SupportProtectedContent = supportProtected; - else - SupportProtectedContent = false; + if (config["supportProtected"] != null) + bool.TryParse(config["supportProtected"], out supportProtected); + //now we need to build up the indexer options so we can create our validator int? parentId = null; @@ -120,32 +117,13 @@ namespace Umbraco.Examine } ValueSetValidator = new ContentValueSetValidator( - new UmbracoContentIndexerOptions( - SupportUnpublishedContent, SupportProtectedContent, parentId, - ConfigIndexCriteria.IncludeItemTypes, ConfigIndexCriteria.ExcludeItemTypes), + supportUnpublished, supportProtected, //Using a singleton here, we can't inject this when using config based providers and we don't use this //anywhere else in this class - Current.Services.PublicAccessService); + Current.Services.PublicAccessService, + parentId, ConfigIndexCriteria.IncludeItemTypes, ConfigIndexCriteria.ExcludeItemTypes); - } - - #endregion - - #region Properties - - /// - /// By default this is false, if set to true then the indexer will include indexing content that is flagged as publicly protected. - /// This property is ignored if SupportUnpublishedContent is set to true. - /// - public bool SupportProtectedContent { get; protected set; } - - /// - /// If set this will filter the content items allowed to be indexed - /// - public int? ParentId - { - get => _parentId ?? ConfigIndexCriteria?.ParentNodeId; - protected set => _parentId = value; + SupportSoftDelete = supportUnpublished; } #endregion diff --git a/src/Umbraco.Examine/UmbracoContentIndexerOptions.cs b/src/Umbraco.Examine/UmbracoContentIndexerOptions.cs deleted file mode 100644 index 55fd35e012..0000000000 --- a/src/Umbraco.Examine/UmbracoContentIndexerOptions.cs +++ /dev/null @@ -1,52 +0,0 @@ -using System; -using System.Collections.Generic; - -namespace Umbraco.Examine -{ - - /// - /// Options used to configure the umbraco content indexer - /// - public class UmbracoContentIndexerOptions - { - public bool SupportUnpublishedContent { get; private set; } - public bool SupportProtectedContent { get; private set; } - public int? ParentId { get; private set; } - - /// - /// Optional inclusion list of content types to index - /// - /// - /// All other types will be ignored if they do not match this list - /// - public IEnumerable IncludeContentTypes { get; private set; } - - /// - /// Optional exclusion list of content types to ignore - /// - /// - /// Any content type alias matched in this will not be included in the index - /// - public IEnumerable ExcludeContentTypes { get; private set; } - - /// - /// Creates a new - /// - /// If the index supports unpublished content - /// If the index supports protected content - /// Optional value indicating to only index content below this ID - /// Optional content type alias inclusion list - /// Optional content type alias exclusion list - public UmbracoContentIndexerOptions(bool supportUnpublishedContent, bool supportProtectedContent, - int? parentId = null, IEnumerable includeContentTypes = null, IEnumerable excludeContentTypes = null) - { - SupportUnpublishedContent = supportUnpublishedContent; - SupportProtectedContent = supportProtectedContent; - ParentId = parentId; - IncludeContentTypes = includeContentTypes; - ExcludeContentTypes = excludeContentTypes; - } - - - } -} diff --git a/src/Umbraco.Examine/UmbracoExamineIndexer.cs b/src/Umbraco.Examine/UmbracoExamineIndexer.cs index 19929d15fe..b62f527450 100644 --- a/src/Umbraco.Examine/UmbracoExamineIndexer.cs +++ b/src/Umbraco.Examine/UmbracoExamineIndexer.cs @@ -54,23 +54,6 @@ namespace Umbraco.Examine { ProfilingLogger = Current.ProfilingLogger; _configBased = true; - - //This is using the config so we'll validate based on that - ValueSetValidator = new ValueSetValidatorDelegate(set => - { - - //check if this document is of a correct type of node type alias - if (ConfigIndexCriteria.IncludeItemTypes.Any()) - if (!ConfigIndexCriteria.IncludeItemTypes.Contains(set.ItemType)) - return false; - - //if this node type is part of our exclusion list, do not validate - if (ConfigIndexCriteria.ExcludeItemTypes.Any()) - if (ConfigIndexCriteria.ExcludeItemTypes.Contains(set.ItemType)) - return false; - - return true; - }); } /// @@ -135,7 +118,6 @@ namespace Umbraco.Examine /// /// Overridden to ensure that the umbraco system field definitions are in place /// - /// /// /// protected override FieldValueTypeCollection CreateFieldValueTypes(IReadOnlyDictionary> indexValueTypesFactory = null) @@ -158,13 +140,7 @@ namespace Umbraco.Examine /// public bool EnableDefaultEventHandler { get; set; } = true; - /// - /// When set to true data will not be deleted from the index if the data is being unpublished (not deleted) - /// - /// - /// Generally used only for published content - /// - public bool SupportUnpublishedContent { get; protected set; } = false; + public bool SupportSoftDelete { get; protected set; } = false; protected ConfigIndexCriteria ConfigIndexCriteria { get; private set; } diff --git a/src/Umbraco.Examine/UmbracoMemberIndexer.cs b/src/Umbraco.Examine/UmbracoMemberIndexer.cs index 4c59b4d4ca..2b0d3a0c66 100644 --- a/src/Umbraco.Examine/UmbracoMemberIndexer.cs +++ b/src/Umbraco.Examine/UmbracoMemberIndexer.cs @@ -5,6 +5,7 @@ using Umbraco.Core.Models; using Umbraco.Core.Persistence.DatabaseModelDefinitions; using Umbraco.Core.Services; using System.Collections.Generic; +using System.Collections.Specialized; using System.ComponentModel; using Examine; using Examine.LuceneEngine; @@ -51,11 +52,16 @@ namespace Umbraco.Examine { } + public override void Initialize(string name, NameValueCollection config) + { + base.Initialize(name, config); + + ValueSetValidator = new MemberValueSetValidator(ConfigIndexCriteria.IncludeItemTypes, ConfigIndexCriteria.ExcludeItemTypes); + } /// /// Overridden to ensure that the umbraco system field definitions are in place /// - /// /// /// protected override FieldValueTypeCollection CreateFieldValueTypes(IReadOnlyDictionary> indexValueTypesFactory = null) diff --git a/src/Umbraco.Examine/ValueSetValidator.cs b/src/Umbraco.Examine/ValueSetValidator.cs new file mode 100644 index 0000000000..1bf7ec2d7a --- /dev/null +++ b/src/Umbraco.Examine/ValueSetValidator.cs @@ -0,0 +1,91 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Examine; +using Examine.LuceneEngine.Providers; +using Umbraco.Core; + +namespace Umbraco.Examine +{ + /// + /// Performing basic validation of a value set + /// + public abstract class ValueSetValidator : IValueSetValidator + { + protected ValueSetValidator( + IEnumerable includeItemTypes, + IEnumerable excludeItemTypes, + IEnumerable includeFields, + IEnumerable excludeFields) + { + IncludeItemTypes = includeItemTypes; + ExcludeItemTypes = excludeItemTypes; + IncludeFields = includeFields; + ExcludeFields = excludeFields; + } + + protected abstract IEnumerable ValidIndexCategories { get; } + + /// + /// Optional inclusion list of content types to index + /// + /// + /// All other types will be ignored if they do not match this list + /// + public IEnumerable IncludeItemTypes { get; } + + /// + /// Optional exclusion list of content types to ignore + /// + /// + /// Any content type alias matched in this will not be included in the index + /// + public IEnumerable ExcludeItemTypes { get; } + + /// + /// Optional inclusion list of index fields to index + /// + /// + /// If specified, all other fields in a will be filtered + /// + public IEnumerable IncludeFields { get; } + + /// + /// Optional exclusion list of index fields + /// + /// + /// If specified, all fields matching these field names will be filtered from the + /// + public IEnumerable ExcludeFields { get; } + + public virtual bool Validate(ValueSet valueSet) + { + if (!ValidIndexCategories.InvariantContains(valueSet.Category)) + return false; + + //check if this document is of a correct type of node type alias + if (IncludeItemTypes != null && !IncludeItemTypes.InvariantContains(valueSet.ItemType)) + return false; + + //if this node type is part of our exclusion list + if (ExcludeItemTypes != null && ExcludeItemTypes.InvariantContains(valueSet.ItemType)) + return false; + + //filter based on the fields provided (if any) + if (IncludeFields != null || ExcludeFields != null) + { + foreach (var key in valueSet.Values.Keys.ToList()) + { + if (IncludeFields != null && !IncludeFields.InvariantContains(key)) + valueSet.Values.Remove(key); //remove any value with a key that doesn't match the inclusion list + + if (ExcludeFields != null && ExcludeFields.InvariantContains(key)) + valueSet.Values.Remove(key); //remove any value with a key that matches the exclusion list + } + } + + + return true; + } + } +} diff --git a/src/Umbraco.Tests/PublishedContent/PublishedMediaTests.cs b/src/Umbraco.Tests/PublishedContent/PublishedMediaTests.cs index 4b49fad3e4..784f534af4 100644 --- a/src/Umbraco.Tests/PublishedContent/PublishedMediaTests.cs +++ b/src/Umbraco.Tests/PublishedContent/PublishedMediaTests.cs @@ -116,8 +116,8 @@ namespace Umbraco.Tests.PublishedContent var rebuilder = IndexInitializer.GetMediaIndexRebuilder(Container.GetInstance(), IndexInitializer.GetMockMediaService()); using (var luceneDir = new RandomIdRamDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, - options: new UmbracoContentIndexerOptions(true, false, null))) + using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, + validator: new ContentValueSetValidator(true))) { rebuilder.Populate(indexer); @@ -143,9 +143,9 @@ namespace Umbraco.Tests.PublishedContent var rebuilder = IndexInitializer.GetMediaIndexRebuilder(Container.GetInstance(), IndexInitializer.GetMockMediaService()); using (var luceneDir = new RandomIdRamDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, + using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, //include unpublished content since this uses the 'internal' indexer, it's up to the media cache to filter - options: new UmbracoContentIndexerOptions(true, false, null))) + validator: new ContentValueSetValidator(true))) using (indexer.ProcessNonAsync()) { rebuilder.Populate(indexer); @@ -190,8 +190,8 @@ namespace Umbraco.Tests.PublishedContent var rebuilder = IndexInitializer.GetMediaIndexRebuilder(Container.GetInstance(), IndexInitializer.GetMockMediaService()); using (var luceneDir = new RandomIdRamDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, - options: new UmbracoContentIndexerOptions(true, false, null))) + using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, + validator: new ContentValueSetValidator(true))) using (indexer.ProcessNonAsync()) { rebuilder.Populate(indexer); @@ -217,8 +217,8 @@ namespace Umbraco.Tests.PublishedContent var rebuilder = IndexInitializer.GetMediaIndexRebuilder(Container.GetInstance(), IndexInitializer.GetMockMediaService()); using (var luceneDir = new RandomIdRamDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, - options: new UmbracoContentIndexerOptions(true, false, null))) + using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, + validator: new ContentValueSetValidator(true))) using (indexer.ProcessNonAsync()) { rebuilder.Populate(indexer); @@ -244,8 +244,8 @@ namespace Umbraco.Tests.PublishedContent var rebuilder = IndexInitializer.GetMediaIndexRebuilder(Container.GetInstance(), IndexInitializer.GetMockMediaService()); using (var luceneDir = new RandomIdRamDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, - options: new UmbracoContentIndexerOptions(true, false, null))) + using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, + validator: new ContentValueSetValidator(true))) using (indexer.ProcessNonAsync()) { rebuilder.Populate(indexer); @@ -272,8 +272,8 @@ namespace Umbraco.Tests.PublishedContent using (var luceneDir = new RandomIdRamDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, - options: new UmbracoContentIndexerOptions(true, false, null))) + using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, + validator: new ContentValueSetValidator(true))) using (indexer.ProcessNonAsync()) { rebuilder.Populate(indexer); @@ -296,8 +296,8 @@ namespace Umbraco.Tests.PublishedContent var rebuilder = IndexInitializer.GetMediaIndexRebuilder(Container.GetInstance(), IndexInitializer.GetMockMediaService()); using (var luceneDir = new RandomIdRamDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, - options: new UmbracoContentIndexerOptions(true, false, null))) + using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, + validator: new ContentValueSetValidator(true))) using (indexer.ProcessNonAsync()) { rebuilder.Populate(indexer); diff --git a/src/Umbraco.Tests/UmbracoExamine/EventsTest.cs b/src/Umbraco.Tests/UmbracoExamine/EventsTest.cs index 7f613753a5..47fa32cbaa 100644 --- a/src/Umbraco.Tests/UmbracoExamine/EventsTest.cs +++ b/src/Umbraco.Tests/UmbracoExamine/EventsTest.cs @@ -19,9 +19,9 @@ namespace Umbraco.Tests.UmbracoExamine public void Events_Ignoring_Node() { using (var luceneDir = new RandomIdRamDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, + using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, //make parent id 999 so all are ignored - options: new UmbracoContentIndexerOptions(false, false, 999))) + validator: new ContentValueSetValidator(false, 999))) using (indexer.ProcessNonAsync()) { var searcher = indexer.GetSearcher(); diff --git a/src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs b/src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs index f66e55cbbd..e5d7598080 100644 --- a/src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs +++ b/src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs @@ -151,7 +151,7 @@ namespace Umbraco.Tests.UmbracoExamine Directory luceneDir, Analyzer analyzer = null, ILocalizationService languageService = null, - UmbracoContentIndexerOptions options = null) + IValueSetValidator validator = null) { if (languageService == null) languageService = GetMockLocalizationService(); @@ -159,8 +159,8 @@ namespace Umbraco.Tests.UmbracoExamine if (analyzer == null) analyzer = new StandardAnalyzer(Version.LUCENE_30); - if (options == null) - options = new UmbracoContentIndexerOptions(false, false, null); + if (validator == null) + validator = new ContentValueSetValidator(false); var i = new UmbracoContentIndexer( "testIndexer", @@ -169,8 +169,7 @@ namespace Umbraco.Tests.UmbracoExamine analyzer, profilingLogger, languageService, - new ContentValueSetValidator(options, Mock.Of()), - options); + validator); i.IndexingError += IndexingError; diff --git a/src/Umbraco.Tests/UmbracoExamine/IndexTest.cs b/src/Umbraco.Tests/UmbracoExamine/IndexTest.cs index 6ef3cc376f..2132c9b19e 100644 --- a/src/Umbraco.Tests/UmbracoExamine/IndexTest.cs +++ b/src/Umbraco.Tests/UmbracoExamine/IndexTest.cs @@ -33,8 +33,8 @@ namespace Umbraco.Tests.UmbracoExamine var contentValueSetBuilder = IndexInitializer.GetContentValueSetBuilder(Container.GetInstance()); using (var luceneDir = new RandomIdRamDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, - options: new UmbracoContentIndexerOptions(true, false, null))) + using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, + validator: new ContentValueSetValidator(true))) using (indexer.ProcessNonAsync()) { indexer.CreateIndex(); @@ -126,8 +126,8 @@ namespace Umbraco.Tests.UmbracoExamine var mediaRebuilder = IndexInitializer.GetMediaIndexRebuilder(Container.GetInstance(), IndexInitializer.GetMockMediaService()); using (var luceneDir = new RandomIdRamDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, - options: new UmbracoContentIndexerOptions(true, false, null))) + using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, + validator: new ContentValueSetValidator(true))) using (indexer.ProcessNonAsync()) { contentRebuilder.RegisterIndex(indexer.Name); @@ -191,7 +191,7 @@ namespace Umbraco.Tests.UmbracoExamine using (var luceneDir = new RandomIdRamDirectory()) using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, //make parent id 1116 - options: new UmbracoContentIndexerOptions(false, false, 1116))) + validator: new ContentValueSetValidator(false, 1116))) using (indexer.ProcessNonAsync()) { var searcher = indexer.GetSearcher(); @@ -233,7 +233,7 @@ namespace Umbraco.Tests.UmbracoExamine using (var luceneDir = new RandomIdRamDirectory()) using (var indexer1 = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, //make parent id 2222 - options: new UmbracoContentIndexerOptions(false, false, 2222))) + validator: new ContentValueSetValidator(false, 2222))) using (indexer1.ProcessNonAsync()) { var searcher = indexer1.GetSearcher(); @@ -283,7 +283,7 @@ namespace Umbraco.Tests.UmbracoExamine var rebuilder = IndexInitializer.GetContentIndexRebuilder(Container.GetInstance(), IndexInitializer.GetMockContentService(), ScopeProvider.SqlContext); using (var luceneDir = new RandomIdRamDirectory()) using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, - options: new UmbracoContentIndexerOptions(true, false, null))) + validator: new ContentValueSetValidator(true))) using (indexer.ProcessNonAsync()) { rebuilder.RegisterIndex(indexer.Name); diff --git a/src/Umbraco.Tests/UmbracoExamine/UmbracoContentValueSetValidatorTests.cs b/src/Umbraco.Tests/UmbracoExamine/UmbracoContentValueSetValidatorTests.cs index 63ca0141f2..daac5f5778 100644 --- a/src/Umbraco.Tests/UmbracoExamine/UmbracoContentValueSetValidatorTests.cs +++ b/src/Umbraco.Tests/UmbracoExamine/UmbracoContentValueSetValidatorTests.cs @@ -17,9 +17,7 @@ namespace Umbraco.Tests.UmbracoExamine [Test] public void Invalid_Category() { - var validator = new ContentValueSetValidator( - new UmbracoContentIndexerOptions(true, true, null), - Mock.Of()); + var validator = new ContentValueSetValidator(true, true, Mock.Of()); var result = validator.Validate(new ValueSet("555", IndexTypes.Content, new { hello = "world", path = "-1,555" })); Assert.IsTrue(result); @@ -35,9 +33,7 @@ namespace Umbraco.Tests.UmbracoExamine [Test] public void Must_Have_Path() { - var validator = new ContentValueSetValidator( - new UmbracoContentIndexerOptions(true, true, null), - Mock.Of()); + var validator = new ContentValueSetValidator(true, true, Mock.Of()); var result = validator.Validate(new ValueSet("555", IndexTypes.Content, new { hello = "world" })); Assert.IsFalse(result); @@ -49,9 +45,7 @@ namespace Umbraco.Tests.UmbracoExamine [Test] public void Parent_Id() { - var validator = new ContentValueSetValidator( - new UmbracoContentIndexerOptions(true, true, 555), - Mock.Of()); + var validator = new ContentValueSetValidator(true, true, Mock.Of(), 555); var result = validator.Validate(new ValueSet("555", IndexTypes.Content, new { hello = "world", path = "-1,555" })); Assert.IsFalse(result); @@ -69,9 +63,8 @@ namespace Umbraco.Tests.UmbracoExamine [Test] public void Inclusion_List() { - var validator = new ContentValueSetValidator( - new UmbracoContentIndexerOptions(true, true, includeContentTypes: new List { "include-content" }), - Mock.Of()); + var validator = new ContentValueSetValidator(true, true, Mock.Of(), + includeItemTypes: new List { "include-content" }); var result = validator.Validate(new ValueSet("555", IndexTypes.Content, "test-content", new { hello = "world", path = "-1,555" })); Assert.IsFalse(result); @@ -86,9 +79,8 @@ namespace Umbraco.Tests.UmbracoExamine [Test] public void Exclusion_List() { - var validator = new ContentValueSetValidator( - new UmbracoContentIndexerOptions(true, true, excludeContentTypes: new List { "exclude-content" }), - Mock.Of()); + var validator = new ContentValueSetValidator(true, true, Mock.Of(), + excludeItemTypes: new List { "exclude-content" }); var result = validator.Validate(new ValueSet("555", IndexTypes.Content, "test-content", new { hello = "world", path = "-1,555" })); Assert.IsTrue(result); @@ -103,12 +95,10 @@ namespace Umbraco.Tests.UmbracoExamine [Test] public void Inclusion_Exclusion_List() { - var validator = new ContentValueSetValidator( - new UmbracoContentIndexerOptions(true, true, - includeContentTypes: new List { "include-content", "exclude-content" }, - excludeContentTypes: new List { "exclude-content" }), - Mock.Of()); - + var validator = new ContentValueSetValidator(true, true, Mock.Of(), + includeItemTypes: new List { "include-content", "exclude-content" }, + excludeItemTypes: new List { "exclude-content" }); + var result = validator.Validate(new ValueSet("555", IndexTypes.Content, "test-content", new { hello = "world", path = "-1,555" })); Assert.IsFalse(result); @@ -125,9 +115,7 @@ namespace Umbraco.Tests.UmbracoExamine [Test] public void Recycle_Bin() { - var validator = new ContentValueSetValidator( - new UmbracoContentIndexerOptions(false, true, null), - Mock.Of()); + var validator = new ContentValueSetValidator(false, true, Mock.Of()); var result = validator.Validate(new ValueSet("555", IndexTypes.Content, new { hello = "world", path = "-1,-20,555" })); Assert.IsFalse(result); @@ -151,9 +139,7 @@ namespace Umbraco.Tests.UmbracoExamine [Test] public void Published_Only() { - var validator = new ContentValueSetValidator( - new UmbracoContentIndexerOptions(false, true, null), - Mock.Of()); + var validator = new ContentValueSetValidator(false, true, Mock.Of()); var result = validator.Validate(new ValueSet("555", IndexTypes.Content, new { hello = "world", path = "-1,555" })); Assert.IsFalse(result); @@ -180,9 +166,7 @@ namespace Umbraco.Tests.UmbracoExamine [Test] public void Published_Only_With_Variants() { - var validator = new ContentValueSetValidator( - new UmbracoContentIndexerOptions(false, true, null), - Mock.Of()); + var validator = new ContentValueSetValidator(false, true, Mock.Of()); var result = validator.Validate(new ValueSet("555", IndexTypes.Content, new Dictionary @@ -240,9 +224,7 @@ namespace Umbraco.Tests.UmbracoExamine .Returns(Attempt.Succeed(new PublicAccessEntry(Guid.NewGuid(), 555, 444, 333, Enumerable.Empty()))); publicAccessService.Setup(x => x.IsProtected("-1,777")) .Returns(Attempt.Fail()); - var validator = new ContentValueSetValidator( - new UmbracoContentIndexerOptions(true, false, null), - publicAccessService.Object); + var validator = new ContentValueSetValidator(true, false, publicAccessService.Object); var result = validator.Validate(new ValueSet("555", IndexTypes.Content, new { hello = "world", path = "-1,555" })); Assert.IsFalse(result); diff --git a/src/Umbraco.Web/Search/ExamineComponent.cs b/src/Umbraco.Web/Search/ExamineComponent.cs index 752440a4dc..8d1bf2898a 100644 --- a/src/Umbraco.Web/Search/ExamineComponent.cs +++ b/src/Umbraco.Web/Search/ExamineComponent.cs @@ -695,9 +695,9 @@ namespace Umbraco.Web.Search { var valueSet = examineComponent._contentValueSetBuilder.GetValueSets(content).ToList(); - foreach (var index in examineComponent._examineManager.IndexProviders.Values.OfType() + foreach (var index in examineComponent._examineManager.IndexProviders.Values.OfType() // only for the specified indexers - .Where(x => supportUnpublished.HasValue == false || supportUnpublished.Value == x.SupportUnpublishedContent) + .Where(x => supportUnpublished.HasValue == false || supportUnpublished.Value == x.SupportSoftDelete) .Where(x => x.EnableDefaultEventHandler)) { index.IndexItems(valueSet); @@ -727,10 +727,10 @@ namespace Umbraco.Web.Search { var valueSet = examineComponent._mediaValueSetBuilder.GetValueSets(media).ToList(); - foreach (var index in examineComponent._examineManager.IndexProviders.Values.OfType() + foreach (var index in examineComponent._examineManager.IndexProviders.Values.OfType() // index this item for all indexers if the media is not trashed, otherwise if the item is trashed // then only index this for indexers supporting unpublished media - .Where(x => isPublished || (x.SupportUnpublishedContent)) + .Where(x => isPublished || (x.SupportSoftDelete)) .Where(x => x.EnableDefaultEventHandler)) { index.IndexItems(valueSet); @@ -790,7 +790,7 @@ namespace Umbraco.Web.Search foreach (var index in examineComponent._examineManager.IndexProviders.Values.OfType() // if keepIfUnpublished == true then only delete this item from indexes not supporting unpublished content, // otherwise if keepIfUnpublished == false then remove from all indexes - .Where(x => keepIfUnpublished == false || x.SupportUnpublishedContent == false) + .Where(x => keepIfUnpublished == false || x.SupportSoftDelete == false) .Where(x => x.EnableDefaultEventHandler)) { index.DeleteFromIndex(strId); diff --git a/src/Umbraco.Web/Search/UmbracoIndexesBuilder.cs b/src/Umbraco.Web/Search/UmbracoIndexesBuilder.cs index bdfb259c10..fbe4d618e6 100644 --- a/src/Umbraco.Web/Search/UmbracoIndexesBuilder.cs +++ b/src/Umbraco.Web/Search/UmbracoIndexesBuilder.cs @@ -41,11 +41,6 @@ namespace Umbraco.Web.Search protected IPublicAccessService PublicAccessService { get; } protected IMemberService MemberService { get; } - /// - /// By default these are the member fields we index - /// - public static readonly string[] DefaultMemberIndexFields = new[] { "id", "nodeName", "updateDate", "writerName", "loginName", "email", "nodeTypeAlias" }; - /// /// Creates the Umbraco indexes /// @@ -54,24 +49,37 @@ namespace Umbraco.Web.Search { return new [] { - CreateContentIndex(Constants.UmbracoIndexes.InternalIndexName, Constants.UmbracoIndexes.InternalIndexPath, new UmbracoContentIndexerOptions(true, true, null), new CultureInvariantWhitespaceAnalyzer()), - CreateContentIndex(Constants.UmbracoIndexes.ExternalIndexName, Constants.UmbracoIndexes.ExternalIndexPath, new UmbracoContentIndexerOptions(false, false, null), new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30)), + CreateInternalIndex(), + CreateExternalIndex(), CreateMemberIndex() }; } - private IIndexer CreateContentIndex(string name, string path, UmbracoContentIndexerOptions options, Analyzer analyzer) + private IIndexer CreateInternalIndex() { var index = new UmbracoContentIndexer( - name, + Constants.UmbracoIndexes.InternalIndexName, //fixme - how to deal with languages like in UmbracoContentIndexer.CreateFieldValueTypes UmbracoExamineIndexer.UmbracoIndexFieldDefinitions, - GetFileSystemLuceneDirectory(path), - analyzer, + GetFileSystemLuceneDirectory(Constants.UmbracoIndexes.InternalIndexPath), + new CultureInvariantWhitespaceAnalyzer(), ProfilingLogger, LanguageService, - GetContentValueSetValidator(options), - options); + GetContentValueSetValidator()); + return index; + } + + private IIndexer CreateExternalIndex() + { + var index = new UmbracoContentIndexer( + Constants.UmbracoIndexes.ExternalIndexName, + //fixme - how to deal with languages like in UmbracoContentIndexer.CreateFieldValueTypes + UmbracoExamineIndexer.UmbracoIndexFieldDefinitions, + GetFileSystemLuceneDirectory(Constants.UmbracoIndexes.ExternalIndexPath), + new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30), + ProfilingLogger, + LanguageService, + GetPublishedContentValueSetValidator()); return index; } @@ -102,9 +110,14 @@ namespace Umbraco.Web.Search return luceneDir; } - public virtual IValueSetValidator GetContentValueSetValidator(UmbracoContentIndexerOptions options) + public virtual IValueSetValidator GetContentValueSetValidator() { - return new ContentValueSetValidator(options, PublicAccessService); + return new ContentValueSetValidator(true, true, PublicAccessService); + } + + public virtual IValueSetValidator GetPublishedContentValueSetValidator() + { + return new ContentValueSetValidator(false, false, PublicAccessService); } /// @@ -113,18 +126,7 @@ namespace Umbraco.Web.Search /// public virtual IValueSetValidator GetMemberValueSetValidator() { - //This validator is used purely to filter the value set - return new ValueSetValidatorDelegate(valueSet => - { - - foreach(var key in valueSet.Values.Keys.ToList()) - { - if (!DefaultMemberIndexFields.InvariantContains(key)) - valueSet.Values.Remove(key); //remove any value with a key that doesn't match our list - } - - return true; - }); + return new MemberValueSetValidator(); } }