From 3fd0aef18ed13e2d107436c1451dfef9714947d1 Mon Sep 17 00:00:00 2001 From: Shannon Date: Wed, 28 Nov 2018 01:23:02 +1100 Subject: [PATCH] decouples data lookup from indexers and rebuilding logic --- .../Config/IndexFieldCollection.cs | 47 ++++ .../Config/IndexSetCollection.cs | 44 --- src/Umbraco.Examine/ContentIndexPopulator.cs | 97 +++++++ src/Umbraco.Examine/IIndexPopulator.cs | 16 ++ src/Umbraco.Examine/IUmbracoContentIndexer.cs | 12 + src/Umbraco.Examine/IUmbracoIndexer.cs | 1 + src/Umbraco.Examine/IUmbracoMediaIndexer.cs | 12 + src/Umbraco.Examine/IndexRebuilder.cs | 48 ++++ src/Umbraco.Examine/MediaIndexPopulator.cs | 68 +++++ src/Umbraco.Examine/MemberIndexPopulator.cs | 42 +++ .../PublishedContentIndexPopulator.cs | 22 ++ src/Umbraco.Examine/Umbraco.Examine.csproj | 11 +- src/Umbraco.Examine/UmbracoContentIndexer.cs | 215 +-------------- src/Umbraco.Examine/UmbracoExamineIndexer.cs | 63 ----- src/Umbraco.Examine/UmbracoMemberIndexer.cs | 59 ---- .../PublishedContent/PublishedMediaTests.cs | 49 ++-- .../TestHelpers/Stubs/TestExamineManager.cs | 42 +-- src/Umbraco.Tests/Umbraco.Tests.csproj | 2 +- .../UmbracoExamine/EventsTest.cs | 2 +- .../UmbracoExamine/IndexInitializer.cs | 258 ++++++++---------- src/Umbraco.Tests/UmbracoExamine/IndexTest.cs | 50 ++-- .../UmbracoExamine/SearchTests.cs | 13 +- src/Umbraco.Web.UI/Umbraco.Web.UI.csproj | 2 +- ...aseServerRegistrarAndMessengerComponent.cs | 9 +- .../Editors/ExamineManagementController.cs | 4 +- src/Umbraco.Web/Search/ExamineComponent.cs | 117 ++++---- .../Search/IUmbracoIndexesBuilder.cs | 2 +- .../Search/UmbracoIndexesBuilder.cs | 46 ++-- src/Umbraco.Web/Suspendable.cs | 12 +- src/Umbraco.Web/Umbraco.Web.csproj | 2 +- 30 files changed, 670 insertions(+), 697 deletions(-) create mode 100644 src/Umbraco.Examine/Config/IndexFieldCollection.cs create mode 100644 src/Umbraco.Examine/ContentIndexPopulator.cs create mode 100644 src/Umbraco.Examine/IIndexPopulator.cs create mode 100644 src/Umbraco.Examine/IUmbracoContentIndexer.cs create mode 100644 src/Umbraco.Examine/IUmbracoMediaIndexer.cs create mode 100644 src/Umbraco.Examine/IndexRebuilder.cs create mode 100644 src/Umbraco.Examine/MediaIndexPopulator.cs create mode 100644 src/Umbraco.Examine/MemberIndexPopulator.cs create mode 100644 src/Umbraco.Examine/PublishedContentIndexPopulator.cs diff --git a/src/Umbraco.Examine/Config/IndexFieldCollection.cs b/src/Umbraco.Examine/Config/IndexFieldCollection.cs new file mode 100644 index 0000000000..063c157dbe --- /dev/null +++ b/src/Umbraco.Examine/Config/IndexFieldCollection.cs @@ -0,0 +1,47 @@ +using System.Configuration; + +namespace Umbraco.Examine.Config +{ + public sealed class IndexFieldCollection : ConfigurationElementCollection + { + #region Overridden methods to define collection + protected override ConfigurationElement CreateNewElement() + { + return new ConfigIndexField(); + } + protected override object GetElementKey(ConfigurationElement element) + { + ConfigIndexField field = (ConfigIndexField)element; + return field.Name; + } + + public override bool IsReadOnly() + { + return false; + } + #endregion + + /// + /// Adds an index field to the collection + /// + /// + public void Add(ConfigIndexField field) + { + BaseAdd(field, true); + } + + /// + /// Default property for accessing an IndexField definition + /// + /// Field Name + /// + public new ConfigIndexField this[string name] + { + get + { + return (ConfigIndexField)this.BaseGet(name); + } + } + + } +} \ No newline at end of file diff --git a/src/Umbraco.Examine/Config/IndexSetCollection.cs b/src/Umbraco.Examine/Config/IndexSetCollection.cs index 5ba2382563..bfce8e4cce 100644 --- a/src/Umbraco.Examine/Config/IndexSetCollection.cs +++ b/src/Umbraco.Examine/Config/IndexSetCollection.cs @@ -3,50 +3,6 @@ namespace Umbraco.Examine.Config { - public sealed class IndexFieldCollection : ConfigurationElementCollection - { - #region Overridden methods to define collection - protected override ConfigurationElement CreateNewElement() - { - return new ConfigIndexField(); - } - protected override object GetElementKey(ConfigurationElement element) - { - ConfigIndexField field = (ConfigIndexField)element; - return field.Name; - } - - public override bool IsReadOnly() - { - return false; - } - #endregion - - /// - /// Adds an index field to the collection - /// - /// - public void Add(ConfigIndexField field) - { - BaseAdd(field, true); - } - - /// - /// Default property for accessing an IndexField definition - /// - /// Field Name - /// - public new ConfigIndexField this[string name] - { - get - { - return (ConfigIndexField)this.BaseGet(name); - } - } - - } - - public sealed class IndexSetCollection : ConfigurationElementCollection { #region Overridden methods to define collection diff --git a/src/Umbraco.Examine/ContentIndexPopulator.cs b/src/Umbraco.Examine/ContentIndexPopulator.cs new file mode 100644 index 0000000000..211e2d51de --- /dev/null +++ b/src/Umbraco.Examine/ContentIndexPopulator.cs @@ -0,0 +1,97 @@ +using System; +using System.Linq; +using Examine; +using Umbraco.Core.Models; +using Umbraco.Core.Services; +using Umbraco.Core.Persistence; +using Umbraco.Core.Persistence.DatabaseModelDefinitions; +using Umbraco.Core.Persistence.Querying; + +namespace Umbraco.Examine +{ + + /// + /// Performs the data lookups required to rebuild a content index + /// + public class ContentIndexPopulator : IIndexPopulator + { + private readonly IContentService _contentService; + private readonly IValueSetBuilder _contentValueSetBuilder; + + /// + /// This is a static query, it's parameters don't change so store statically + /// + private static IQuery _publishedQuery; + + private readonly bool _supportUnpublishedContent; + private readonly int? _parentId; + + /// + /// Default constructor to lookup all content data + /// + /// + /// + /// + public ContentIndexPopulator(IContentService contentService, ISqlContext sqlContext, IValueSetBuilder contentValueSetBuilder) + : this(true, null, contentService, sqlContext, contentValueSetBuilder) + { + } + + /// + /// Optional constructor allowing specifying custom query parameters + /// + /// + /// + /// + /// + /// + public ContentIndexPopulator(bool supportUnpublishedContent, int? parentId, IContentService contentService, ISqlContext sqlContext, IValueSetBuilder contentValueSetBuilder) + { + if (sqlContext == null) throw new ArgumentNullException(nameof(sqlContext)); + _contentService = contentService ?? throw new ArgumentNullException(nameof(contentService)); + _contentValueSetBuilder = contentValueSetBuilder ?? throw new ArgumentNullException(nameof(contentValueSetBuilder)); + if (_publishedQuery != null) + _publishedQuery = sqlContext.Query().Where(x => x.Published); + _supportUnpublishedContent = supportUnpublishedContent; + _parentId = parentId; + } + + public void Populate(params IIndexer[] indexes) + { + const int pageSize = 10000; + var pageIndex = 0; + + var contentParentId = -1; + if (_parentId.HasValue && _parentId.Value > 0) + { + contentParentId = _parentId.Value; + } + IContent[] content; + + do + { + long total; + + if (_supportUnpublishedContent) + { + content = _contentService.GetPagedDescendants(contentParentId, pageIndex, pageSize, out total).ToArray(); + } + else + { + //add the published filter + //note: We will filter for published variants in the validator + content = _contentService.GetPagedDescendants(contentParentId, pageIndex, pageSize, out total, + _publishedQuery, Ordering.By("Path", Direction.Ascending)).ToArray(); + } + + if (content.Length > 0) + { + foreach (var index in indexes) + index.IndexItems(_contentValueSetBuilder.GetValueSets(content)); + } + + pageIndex++; + } while (content.Length == pageSize); + } + } +} diff --git a/src/Umbraco.Examine/IIndexPopulator.cs b/src/Umbraco.Examine/IIndexPopulator.cs new file mode 100644 index 0000000000..fd71b3bb53 --- /dev/null +++ b/src/Umbraco.Examine/IIndexPopulator.cs @@ -0,0 +1,16 @@ +using Examine; + +namespace Umbraco.Examine +{ + /// + /// Populates indexes with data + /// + public interface IIndexPopulator + { + /// + /// Populates indexes with data + /// + /// + void Populate(params IIndexer[] indexes); + } +} diff --git a/src/Umbraco.Examine/IUmbracoContentIndexer.cs b/src/Umbraco.Examine/IUmbracoContentIndexer.cs new file mode 100644 index 0000000000..a7cc9f9c2f --- /dev/null +++ b/src/Umbraco.Examine/IUmbracoContentIndexer.cs @@ -0,0 +1,12 @@ +using Examine; + +namespace Umbraco.Examine +{ + //TODO: Rethink this, need a better way of rebuilding + /// + /// A Marker interface for defining an Umbraco content indexer + /// + public interface IUmbracoContentIndexer : IIndexer + { + } +} diff --git a/src/Umbraco.Examine/IUmbracoIndexer.cs b/src/Umbraco.Examine/IUmbracoIndexer.cs index c926bfb2fe..936c78c71d 100644 --- a/src/Umbraco.Examine/IUmbracoIndexer.cs +++ b/src/Umbraco.Examine/IUmbracoIndexer.cs @@ -2,6 +2,7 @@ namespace Umbraco.Examine { + /// /// A Marker interface for defining an Umbraco indexer /// diff --git a/src/Umbraco.Examine/IUmbracoMediaIndexer.cs b/src/Umbraco.Examine/IUmbracoMediaIndexer.cs new file mode 100644 index 0000000000..05ab46bec3 --- /dev/null +++ b/src/Umbraco.Examine/IUmbracoMediaIndexer.cs @@ -0,0 +1,12 @@ +using Examine; + +namespace Umbraco.Examine +{ + //TODO: Rethink this, need a better way of rebuilding + /// + /// A Marker interface for defining an Umbraco media indexer + /// + public interface IUmbracoMediaIndexer : IIndexer + { + } +} diff --git a/src/Umbraco.Examine/IndexRebuilder.cs b/src/Umbraco.Examine/IndexRebuilder.cs new file mode 100644 index 0000000000..5488fa2583 --- /dev/null +++ b/src/Umbraco.Examine/IndexRebuilder.cs @@ -0,0 +1,48 @@ +using System.Linq; +using Examine; + +namespace Umbraco.Examine +{ + /// + /// Utility to rebuild all indexes ensuring minimal data queries + /// + public class IndexRebuilder + { + public IExamineManager ExamineManager { get; } + private readonly ContentIndexPopulator _contentIndexPopulator; + private readonly MediaIndexPopulator _mediaIndexPopulator; + + public IndexRebuilder(IExamineManager examineManager, ContentIndexPopulator contentIndexPopulator, MediaIndexPopulator mediaIndexPopulator) + { + ExamineManager = examineManager; + _contentIndexPopulator = contentIndexPopulator; + _mediaIndexPopulator = mediaIndexPopulator; + } + + public void RebuildIndexes(bool onlyEmptyIndexes) + { + var indexes = (onlyEmptyIndexes + ? ExamineManager.IndexProviders.Values.Where(x => x.IndexExists()) + : ExamineManager.IndexProviders.Values).ToList(); + + var contentIndexes = indexes.Where(x => x is IUmbracoContentIndexer).ToArray(); + var mediaIndexes = indexes.Where(x => x is IUmbracoContentIndexer).ToArray(); + var nonUmbracoIndexes = indexes.Except(contentIndexes).Except(mediaIndexes).ToArray(); + + foreach(var index in indexes) + index.CreateIndex(); // clear the index + + //reindex all content/media indexes with the same data source/lookup + _contentIndexPopulator.Populate(contentIndexes); + _mediaIndexPopulator.Populate(mediaIndexes); + + //then do the rest + foreach (var index in nonUmbracoIndexes) + { + index.CreateIndex(); + //TODO: How to rebuild? + } + + } + } +} diff --git a/src/Umbraco.Examine/MediaIndexPopulator.cs b/src/Umbraco.Examine/MediaIndexPopulator.cs new file mode 100644 index 0000000000..0dfd9b1bc5 --- /dev/null +++ b/src/Umbraco.Examine/MediaIndexPopulator.cs @@ -0,0 +1,68 @@ +using System.Linq; +using Examine; +using Umbraco.Core.Models; +using Umbraco.Core.Services; + +namespace Umbraco.Examine +{ + /// + /// Performs the data lookups required to rebuild a media index + /// + public class MediaIndexPopulator : IIndexPopulator + { + private readonly int? _parentId; + private readonly IMediaService _mediaService; + private readonly IValueSetBuilder _mediaValueSetBuilder; + + /// + /// Default constructor to lookup all content data + /// + /// + /// + public MediaIndexPopulator(IMediaService mediaService, IValueSetBuilder mediaValueSetBuilder) + : this(null, mediaService, mediaValueSetBuilder) + { + } + + /// + /// Optional constructor allowing specifying custom query parameters + /// + /// + /// + /// + public MediaIndexPopulator(int? parentId, IMediaService mediaService, IValueSetBuilder mediaValueSetBuilder) + { + _parentId = parentId; + _mediaService = mediaService; + _mediaValueSetBuilder = mediaValueSetBuilder; + } + + public void Populate(params IIndexer[] indexes) + { + const int pageSize = 10000; + var pageIndex = 0; + + var mediaParentId = -1; + + if (_parentId.HasValue && _parentId.Value > 0) + { + mediaParentId = _parentId.Value; + } + + IMedia[] media; + + do + { + media = _mediaService.GetPagedDescendants(mediaParentId, pageIndex, pageSize, out var total).ToArray(); + + if (media.Length > 0) + { + foreach (var index in indexes) + index.IndexItems(_mediaValueSetBuilder.GetValueSets(media)); + } + + pageIndex++; + } while (media.Length == pageSize); + } + } +} diff --git a/src/Umbraco.Examine/MemberIndexPopulator.cs b/src/Umbraco.Examine/MemberIndexPopulator.cs new file mode 100644 index 0000000000..cbe763e2c1 --- /dev/null +++ b/src/Umbraco.Examine/MemberIndexPopulator.cs @@ -0,0 +1,42 @@ +using System.Linq; +using Examine; +using Umbraco.Core.Models; +using Umbraco.Core.Services; + +namespace Umbraco.Examine +{ + public class MemberIndexPopulator: IIndexPopulator + { + private readonly IMemberService _memberService; + private readonly IValueSetBuilder _valueSetBuilder; + + public MemberIndexPopulator(IMemberService memberService, IValueSetBuilder valueSetBuilder) + { + _memberService = memberService; + _valueSetBuilder = valueSetBuilder; + } + public void Populate(params IIndexer[] indexes) + { + const int pageSize = 1000; + var pageIndex = 0; + + IMember[] members; + + //TODO: Add validators for member indexers for ConfigIndexCriteria.IncludeItemTypes + + //no node types specified, do all members + do + { + members = _memberService.GetAll(pageIndex, pageSize, out _).ToArray(); + + if (members.Length > 0) + { + foreach (var index in indexes) + index.IndexItems(_valueSetBuilder.GetValueSets(members)); + } + + pageIndex++; + } while (members.Length == pageSize); + } + } +} diff --git a/src/Umbraco.Examine/PublishedContentIndexPopulator.cs b/src/Umbraco.Examine/PublishedContentIndexPopulator.cs new file mode 100644 index 0000000000..a29031ca3c --- /dev/null +++ b/src/Umbraco.Examine/PublishedContentIndexPopulator.cs @@ -0,0 +1,22 @@ +using Umbraco.Core.Models; +using Umbraco.Core.Services; +using Umbraco.Core.Persistence; + +namespace Umbraco.Examine +{ + /// + /// Performs the data lookups required to rebuild a content index containing only published content + /// + /// + /// The published (external) index will still rebuild just fine using the default which is what + /// is used when rebuilding all indexes, but this will be used when the single index is rebuilt and will go a little bit faster + /// since the data query is more specific. + /// + public class PublishedContentIndexPopulator : ContentIndexPopulator + { + public PublishedContentIndexPopulator(IContentService contentService, ISqlContext sqlContext, IValueSetBuilder contentValueSetBuilder) : + base(false, null, contentService, sqlContext, contentValueSetBuilder) + { + } + } +} diff --git a/src/Umbraco.Examine/Umbraco.Examine.csproj b/src/Umbraco.Examine/Umbraco.Examine.csproj index 83a9ef0d7d..f5c85fada3 100644 --- a/src/Umbraco.Examine/Umbraco.Examine.csproj +++ b/src/Umbraco.Examine/Umbraco.Examine.csproj @@ -48,7 +48,7 @@ - + @@ -56,15 +56,24 @@ + + + + + + + + + diff --git a/src/Umbraco.Examine/UmbracoContentIndexer.cs b/src/Umbraco.Examine/UmbracoContentIndexer.cs index 984f70edcd..0433a75d41 100644 --- a/src/Umbraco.Examine/UmbracoContentIndexer.cs +++ b/src/Umbraco.Examine/UmbracoContentIndexer.cs @@ -2,10 +2,8 @@ using System.Collections.Generic; using System.Collections.Specialized; using System.ComponentModel; -using System.Linq; using Examine; using Umbraco.Core; -using Umbraco.Core.Models; using Umbraco.Core.Services; using Umbraco.Core.Strings; using Examine.LuceneEngine.Indexing; @@ -14,29 +12,17 @@ using Lucene.Net.Analysis; using Lucene.Net.Store; using Umbraco.Core.Composing; using Umbraco.Core.Logging; -using Umbraco.Core.Persistence; -using Umbraco.Core.Persistence.DatabaseModelDefinitions; -using Umbraco.Core.Persistence.Querying; -using Umbraco.Core.Scoping; using Umbraco.Examine.Config; -using IContentService = Umbraco.Core.Services.IContentService; -using IMediaService = Umbraco.Core.Services.IMediaService; using Examine.LuceneEngine; -using Umbraco.Core.PropertyEditors; namespace Umbraco.Examine { /// /// An indexer for Umbraco content and media /// - public class UmbracoContentIndexer : UmbracoExamineIndexer + public class UmbracoContentIndexer : UmbracoExamineIndexer, IUmbracoContentIndexer, IUmbracoMediaIndexer { public const string VariesByCultureFieldName = UmbracoExamineIndexer.SpecialFieldPrefix + "VariesByCulture"; - - public IValueSetBuilder MediaValueSetBuilder { get; } - public IValueSetBuilder ContentValueSetBuilder { get; } - protected IContentService ContentService { get; } - protected IMediaService MediaService { get; } protected ILocalizationService LanguageService { get; } private int? _parentId; @@ -49,14 +35,7 @@ namespace Umbraco.Examine [EditorBrowsable(EditorBrowsableState.Never)] public UmbracoContentIndexer() { - ContentService = Current.Services.ContentService; - MediaService = Current.Services.MediaService; LanguageService = Current.Services.LocalizationService; - - ContentValueSetBuilder = new ContentValueSetBuilder(Current.PropertyEditors, Current.UrlSegmentProviders, Current.Services.UserService); - MediaValueSetBuilder = new MediaValueSetBuilder(Current.PropertyEditors, Current.UrlSegmentProviders, Current.Services.UserService); - - InitializeQueries(Current.SqlContext); } /// @@ -67,9 +46,6 @@ namespace Umbraco.Examine /// /// /// - /// - /// - /// /// /// /// @@ -79,12 +55,7 @@ namespace Umbraco.Examine Directory luceneDirectory, Analyzer defaultAnalyzer, ProfilingLogger profilingLogger, - IValueSetBuilder contentValueSetBuilder, - IValueSetBuilder mediaValueSetBuilder, - IContentService contentService, - IMediaService mediaService, ILocalizationService languageService, - ISqlContext sqlContext, IValueSetValidator validator, UmbracoContentIndexerOptions options, IReadOnlyDictionary> indexValueTypes = null) @@ -96,21 +67,9 @@ namespace Umbraco.Examine SupportProtectedContent = options.SupportProtectedContent; SupportUnpublishedContent = options.SupportUnpublishedContent; ParentId = options.ParentId; - ContentValueSetBuilder = contentValueSetBuilder ?? throw new ArgumentNullException(nameof(contentValueSetBuilder)); - MediaValueSetBuilder = mediaValueSetBuilder ?? throw new ArgumentNullException(nameof(mediaValueSetBuilder)); - ContentService = contentService ?? throw new ArgumentNullException(nameof(contentService)); - MediaService = mediaService ?? throw new ArgumentNullException(nameof(mediaService)); LanguageService = languageService ?? throw new ArgumentNullException(nameof(languageService)); - - InitializeQueries(sqlContext); } - private void InitializeQueries(ISqlContext sqlContext) - { - if (sqlContext == null) throw new ArgumentNullException(nameof(sqlContext)); - if (_publishedQuery == null) - _publishedQuery = sqlContext.Query().Where(x => x.Published); - } #endregion @@ -133,9 +92,11 @@ namespace Umbraco.Examine /// /// An attempt is made to call on a provider after the provider has already been initialized. /// - + [EditorBrowsable(EditorBrowsableState.Never)] public override void Initialize(string name, NameValueCollection config) { + base.Initialize(name, config); + //check if there's a flag specifying to support unpublished content, //if not, set to false; if (config["supportUnpublished"] != null && bool.TryParse(config["supportUnpublished"], out var supportUnpublished)) @@ -143,7 +104,6 @@ namespace Umbraco.Examine else SupportUnpublishedContent = false; - //check if there's a flag specifying to support protected content, //if not, set to false; if (config["supportProtected"] != null && bool.TryParse(config["supportProtected"], out var supportProtected)) @@ -151,8 +111,6 @@ namespace Umbraco.Examine else SupportProtectedContent = false; - base.Initialize(name, config); - //now we need to build up the indexer options so we can create our validator int? parentId = null; if (IndexSetName.IsNullOrWhiteSpace() == false) @@ -160,11 +118,15 @@ namespace Umbraco.Examine var indexSet = IndexSets.Instance.Sets[IndexSetName]; parentId = indexSet.IndexParentId; } + ValueSetValidator = new UmbracoContentValueSetValidator( - new UmbracoContentIndexerOptions(SupportUnpublishedContent, SupportProtectedContent, parentId), + new UmbracoContentIndexerOptions( + SupportUnpublishedContent, SupportProtectedContent, parentId, + ConfigIndexCriteria.IncludeItemTypes, ConfigIndexCriteria.ExcludeItemTypes), //Using a singleton here, we can't inject this when using config based providers and we don't use this //anywhere else in this class Current.Services.PublicAccessService); + } #endregion @@ -186,8 +148,6 @@ namespace Umbraco.Examine protected set => _parentId = value; } - protected override IEnumerable SupportedTypes => new[] {IndexTypes.Content, IndexTypes.Media}; - #endregion #region Public methods @@ -245,164 +205,7 @@ namespace Umbraco.Examine return base.CreateFieldValueTypes(indexValueTypesFactory); } - /// - /// This is a static query, it's parameters don't change so store statically - /// - private static IQuery _publishedQuery; - - protected override void PerformIndexAll(string type) - { - const int pageSize = 10000; - var pageIndex = 0; - - switch (type) - { - case IndexTypes.Content: - var contentParentId = -1; - if (ParentId.HasValue && ParentId.Value > 0) - { - contentParentId = ParentId.Value; - } - IContent[] content; - - do - { - long total; - - IEnumerable descendants; - if (SupportUnpublishedContent) - { - descendants = ContentService.GetPagedDescendants(contentParentId, pageIndex, pageSize, out total); - } - else - { - //add the published filter - descendants = ContentService.GetPagedDescendants(contentParentId, pageIndex, pageSize, out total, - _publishedQuery, Ordering.By("Path", Direction.Ascending)); - } - - //if specific types are declared we need to post filter them - //TODO: Update the service layer to join the cmsContentType table so we can query by content type too - if (ConfigIndexCriteria != null && ConfigIndexCriteria.IncludeItemTypes.Any()) - { - content = descendants.Where(x => ConfigIndexCriteria.IncludeItemTypes.Contains(x.ContentType.Alias)).ToArray(); - } - else - { - content = descendants.ToArray(); - } - - IndexItems(ContentValueSetBuilder.GetValueSets(content)); - - pageIndex++; - } while (content.Length == pageSize); - - break; - case IndexTypes.Media: - var mediaParentId = -1; - - if (ParentId.HasValue && ParentId.Value > 0) - { - mediaParentId = ParentId.Value; - } - - IMedia[] media; - - do - { - var descendants = MediaService.GetPagedDescendants(mediaParentId, pageIndex, pageSize, out _); - - //if specific types are declared we need to post filter them - //TODO: Update the service layer to join the cmsContentType table so we can query by content type too - if (ConfigIndexCriteria != null && ConfigIndexCriteria.IncludeItemTypes.Any()) - { - media = descendants.Where(x => ConfigIndexCriteria.IncludeItemTypes.Contains(x.ContentType.Alias)).ToArray(); - } - else - { - media = descendants.ToArray(); - } - - IndexItems(MediaValueSetBuilder.GetValueSets(media)); - - pageIndex++; - } while (media.Length == pageSize); - - break; - } - } - - //TODO: We want to make a public method that iterates a data set, potentially with callbacks so that we can iterate - // a single data set once but populate multiple indexes with it. This is for Startup performance when no indexes exist. - // This could be used for any indexer of type UmbracoContentIndexer - BUT that means we need to make another interface - // for content indexers since UmbracoContentIndexer is strongly tied to lucene, so maybe we have a more generic interface - // or add to the current IUmbracoIndexer interface - #endregion } - - public class ContentIndexDataSource - { - public ContentIndexDataSource(bool supportUnpublishedContent, int? parentId, - IContentService contentService, ISqlContext sqlContext, - IValueSetBuilder contentValueSetBuilder) - { - SupportUnpublishedContent = supportUnpublishedContent; - ParentId = parentId; - ContentService = contentService; - _contentValueSetBuilder = contentValueSetBuilder; - if (sqlContext == null) throw new ArgumentNullException(nameof(sqlContext)); - _publishedQuery = sqlContext.Query().Where(x => x.Published); - - } - - public bool SupportUnpublishedContent { get; } - public int? ParentId { get; } - public IContentService ContentService { get; } - - /// - /// This is a static query, it's parameters don't change so store statically - /// - private static IQuery _publishedQuery; - private readonly IValueSetBuilder _contentValueSetBuilder; - - public void Index(params IIndexer[] indexes) - { - const int pageSize = 10000; - var pageIndex = 0; - - var contentParentId = -1; - if (ParentId.HasValue && ParentId.Value > 0) - { - contentParentId = ParentId.Value; - } - IContent[] content; - - do - { - long total; - - IEnumerable descendants; - if (SupportUnpublishedContent) - { - descendants = ContentService.GetPagedDescendants(contentParentId, pageIndex, pageSize, out total); - } - else - { - //add the published filter - //note: We will filter for published variants in the validator - descendants = ContentService.GetPagedDescendants(contentParentId, pageIndex, pageSize, out total, - _publishedQuery, Ordering.By("Path", Direction.Ascending)); - } - - content = descendants.ToArray(); - - foreach(var index in indexes) - index.IndexItems(_contentValueSetBuilder.GetValueSets(content)); - - pageIndex++; - } while (content.Length == pageSize); - } - } } diff --git a/src/Umbraco.Examine/UmbracoExamineIndexer.cs b/src/Umbraco.Examine/UmbracoExamineIndexer.cs index 3ede45c60a..19929d15fe 100644 --- a/src/Umbraco.Examine/UmbracoExamineIndexer.cs +++ b/src/Umbraco.Examine/UmbracoExamineIndexer.cs @@ -166,11 +166,6 @@ namespace Umbraco.Examine /// public bool SupportUnpublishedContent { get; protected set; } = false; - /// - /// the supported indexable types - /// - protected abstract IEnumerable SupportedTypes { get; } - protected ConfigIndexCriteria ConfigIndexCriteria { get; private set; } /// @@ -271,53 +266,6 @@ namespace Umbraco.Examine #endregion - - /// - /// override to check if we can actually initialize. - /// - /// - /// This check is required since the base examine lib will try to rebuild on startup - /// - public override void RebuildIndex() - { - if (CanInitialize()) - { - ProfilingLogger.Logger.Debug(GetType(), "Rebuilding index"); - using (new SafeCallContext()) - { - base.RebuildIndex(); - } - } - } - - /// - /// override to check if we can actually initialize. - /// - /// - /// This check is required since the base examine lib will try to rebuild on startup - /// - public override void IndexAll(string type) - { - if (CanInitialize()) - { - using (new SafeCallContext()) - { - base.IndexAll(type); - } - } - } - - public override void IndexItems(IEnumerable nodes) - { - if (CanInitialize()) - { - using (new SafeCallContext()) - { - base.IndexItems(nodes); - } - } - } - /// /// override to check if we can actually initialize. /// @@ -346,17 +294,6 @@ namespace Umbraco.Examine return _configBased == false || Current.RuntimeState.Level == RuntimeLevel.Run; } - /// - /// Reindexes all supported types - /// - protected override void PerformIndexRebuild() - { - foreach (var t in SupportedTypes) - { - IndexAll(t); - } - } - /// /// overridden for logging /// diff --git a/src/Umbraco.Examine/UmbracoMemberIndexer.cs b/src/Umbraco.Examine/UmbracoMemberIndexer.cs index 4fa924f995..4c59b4d4ca 100644 --- a/src/Umbraco.Examine/UmbracoMemberIndexer.cs +++ b/src/Umbraco.Examine/UmbracoMemberIndexer.cs @@ -23,17 +23,12 @@ namespace Umbraco.Examine /// public class UmbracoMemberIndexer : UmbracoExamineIndexer { - private readonly IValueSetBuilder _valueSetBuilder; - private readonly IMemberService _memberService; - /// /// Constructor for config/provider based indexes /// [EditorBrowsable(EditorBrowsableState.Never)] public UmbracoMemberIndexer() { - _memberService = Current.Services.MemberService; - _valueSetBuilder = new MemberValueSetBuilder(Current.PropertyEditors); } /// @@ -44,7 +39,6 @@ namespace Umbraco.Examine /// /// /// - /// /// public UmbracoMemberIndexer( string name, @@ -52,13 +46,9 @@ namespace Umbraco.Examine Directory luceneDirectory, Analyzer analyzer, ProfilingLogger profilingLogger, - IValueSetBuilder valueSetBuilder, - IMemberService memberService, IValueSetValidator validator = null) : base(name, fieldDefinitions, luceneDirectory, analyzer, profilingLogger, validator) { - _valueSetBuilder = valueSetBuilder ?? throw new ArgumentNullException(nameof(valueSetBuilder)); - _memberService = memberService ?? throw new ArgumentNullException(nameof(memberService)); } @@ -76,55 +66,6 @@ namespace Umbraco.Examine return base.CreateFieldValueTypes(indexValueTypesFactory); } - /// - protected override IEnumerable SupportedTypes => new[] {IndexTypes.Member}; - - /// - /// Reindex all members - /// - /// - protected override void PerformIndexAll(string type) - { - //This only supports members - if (SupportedTypes.Contains(type) == false) - return; - - const int pageSize = 1000; - var pageIndex = 0; - - IMember[] members; - - if (ConfigIndexCriteria != null && ConfigIndexCriteria.IncludeItemTypes.Any()) - { - //if there are specific node types then just index those - foreach (var nodeType in ConfigIndexCriteria.IncludeItemTypes) - { - do - { - members = _memberService.GetAll(pageIndex, pageSize, out _, "LoginName", Direction.Ascending, true, null, nodeType).ToArray(); - - IndexItems(_valueSetBuilder.GetValueSets(members)); - - pageIndex++; - } while (members.Length == pageSize); - } - } - else - { - //no node types specified, do all members - do - { - members = _memberService.GetAll(pageIndex, pageSize, out _).ToArray(); - - IndexItems(_valueSetBuilder.GetValueSets(members)); - - pageIndex++; - } while (members.Length == pageSize); - } - } - - - /// /// Ensure some custom values are added to the index /// diff --git a/src/Umbraco.Tests/PublishedContent/PublishedMediaTests.cs b/src/Umbraco.Tests/PublishedContent/PublishedMediaTests.cs index 2a3b60fcdf..4b49fad3e4 100644 --- a/src/Umbraco.Tests/PublishedContent/PublishedMediaTests.cs +++ b/src/Umbraco.Tests/PublishedContent/PublishedMediaTests.cs @@ -113,12 +113,13 @@ namespace Umbraco.Tests.PublishedContent [Test] public void Ensure_Children_Sorted_With_Examine() { + var rebuilder = IndexInitializer.GetMediaIndexRebuilder(Container.GetInstance(), IndexInitializer.GetMockMediaService()); + using (var luceneDir = new RandomIdRamDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, Container.GetInstance(), + using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, options: new UmbracoContentIndexerOptions(true, false, null))) { - - indexer.RebuildIndex(); + rebuilder.Populate(indexer); var searcher = indexer.GetSearcher(); var ctx = GetUmbracoContext("/test"); @@ -139,14 +140,15 @@ namespace Umbraco.Tests.PublishedContent [Test] public void Do_Not_Find_In_Recycle_Bin() { + var rebuilder = IndexInitializer.GetMediaIndexRebuilder(Container.GetInstance(), IndexInitializer.GetMockMediaService()); + using (var luceneDir = new RandomIdRamDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, Container.GetInstance(), + using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, //include unpublished content since this uses the 'internal' indexer, it's up to the media cache to filter options: new UmbracoContentIndexerOptions(true, false, null))) using (indexer.ProcessNonAsync()) { - indexer.RebuildIndex(); - + rebuilder.Populate(indexer); var searcher = indexer.GetSearcher(); var ctx = GetUmbracoContext("/test"); @@ -185,13 +187,14 @@ namespace Umbraco.Tests.PublishedContent [Test] public void Children_With_Examine() { + var rebuilder = IndexInitializer.GetMediaIndexRebuilder(Container.GetInstance(), IndexInitializer.GetMockMediaService()); + using (var luceneDir = new RandomIdRamDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, Container.GetInstance(), + using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, options: new UmbracoContentIndexerOptions(true, false, null))) using (indexer.ProcessNonAsync()) { - indexer.RebuildIndex(); - + rebuilder.Populate(indexer); var searcher = indexer.GetSearcher(); var ctx = GetUmbracoContext("/test"); @@ -211,13 +214,14 @@ namespace Umbraco.Tests.PublishedContent [Test] public void Descendants_With_Examine() { + var rebuilder = IndexInitializer.GetMediaIndexRebuilder(Container.GetInstance(), IndexInitializer.GetMockMediaService()); + using (var luceneDir = new RandomIdRamDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, Container.GetInstance(), + using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, options: new UmbracoContentIndexerOptions(true, false, null))) using (indexer.ProcessNonAsync()) { - indexer.RebuildIndex(); - + rebuilder.Populate(indexer); var searcher = indexer.GetSearcher(); var ctx = GetUmbracoContext("/test"); @@ -237,13 +241,14 @@ namespace Umbraco.Tests.PublishedContent [Test] public void DescendantsOrSelf_With_Examine() { + var rebuilder = IndexInitializer.GetMediaIndexRebuilder(Container.GetInstance(), IndexInitializer.GetMockMediaService()); + using (var luceneDir = new RandomIdRamDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, Container.GetInstance(), + using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, options: new UmbracoContentIndexerOptions(true, false, null))) using (indexer.ProcessNonAsync()) { - indexer.RebuildIndex(); - + rebuilder.Populate(indexer); var searcher = indexer.GetSearcher(); var ctx = GetUmbracoContext("/test"); @@ -263,13 +268,15 @@ namespace Umbraco.Tests.PublishedContent [Test] public void Ancestors_With_Examine() { + var rebuilder = IndexInitializer.GetMediaIndexRebuilder(Container.GetInstance(), IndexInitializer.GetMockMediaService()); + + using (var luceneDir = new RandomIdRamDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, Container.GetInstance(), + using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, options: new UmbracoContentIndexerOptions(true, false, null))) using (indexer.ProcessNonAsync()) { - indexer.RebuildIndex(); - + rebuilder.Populate(indexer); var ctx = GetUmbracoContext("/test"); var searcher = indexer.GetSearcher(); @@ -286,12 +293,14 @@ namespace Umbraco.Tests.PublishedContent [Test] public void AncestorsOrSelf_With_Examine() { + var rebuilder = IndexInitializer.GetMediaIndexRebuilder(Container.GetInstance(), IndexInitializer.GetMockMediaService()); + using (var luceneDir = new RandomIdRamDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, Container.GetInstance(), + using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, options: new UmbracoContentIndexerOptions(true, false, null))) using (indexer.ProcessNonAsync()) { - indexer.RebuildIndex(); + rebuilder.Populate(indexer); var ctx = GetUmbracoContext("/test"); diff --git a/src/Umbraco.Tests/TestHelpers/Stubs/TestExamineManager.cs b/src/Umbraco.Tests/TestHelpers/Stubs/TestExamineManager.cs index 14c6a08c8b..f0abe053ac 100644 --- a/src/Umbraco.Tests/TestHelpers/Stubs/TestExamineManager.cs +++ b/src/Umbraco.Tests/TestHelpers/Stubs/TestExamineManager.cs @@ -9,26 +9,16 @@ namespace Umbraco.Tests.TestHelpers.Stubs private readonly ConcurrentDictionary _indexers = new ConcurrentDictionary(); private readonly ConcurrentDictionary _searchers = new ConcurrentDictionary(); - public void AddIndexer(string name, IIndexer indexer) + public void AddIndexer(IIndexer indexer) { - _indexers.TryAdd(name, indexer); + _indexers.TryAdd(indexer.Name, indexer); } - public void AddSearcher(string name, ISearcher searcher) + public void AddSearcher(ISearcher searcher) { - _searchers.TryAdd(name, searcher); + _searchers.TryAdd(searcher.Name, searcher); } - - public void DeleteFromIndexes(string nodeId) - { - //noop - } - - public void DeleteFromIndexes(string nodeId, IEnumerable providers) - { - //noop - } - + public void Dispose() { //noop @@ -39,31 +29,11 @@ namespace Umbraco.Tests.TestHelpers.Stubs return _indexers.TryGetValue(indexerName, out var indexer) ? indexer : null; } - public ISearcher GetRegisteredSearcher(string searcherName) + public ISearcher GetSearcher(string searcherName) { return _searchers.TryGetValue(searcherName, out var indexer) ? indexer : null; } - public void IndexAll(string indexCategory) - { - //noop - } - - public void IndexItems(ValueSet[] nodes) - { - //noop - } - - public void IndexItems(ValueSet[] nodes, IEnumerable providers) - { - //noop - } - - public void RebuildIndexes() - { - //noop - } - public IReadOnlyDictionary IndexProviders => _indexers; } } diff --git a/src/Umbraco.Tests/Umbraco.Tests.csproj b/src/Umbraco.Tests/Umbraco.Tests.csproj index 10ba02b314..5fa0cf3469 100644 --- a/src/Umbraco.Tests/Umbraco.Tests.csproj +++ b/src/Umbraco.Tests/Umbraco.Tests.csproj @@ -77,7 +77,7 @@ - + 1.8.9 diff --git a/src/Umbraco.Tests/UmbracoExamine/EventsTest.cs b/src/Umbraco.Tests/UmbracoExamine/EventsTest.cs index 217ce307f0..7f613753a5 100644 --- a/src/Umbraco.Tests/UmbracoExamine/EventsTest.cs +++ b/src/Umbraco.Tests/UmbracoExamine/EventsTest.cs @@ -19,7 +19,7 @@ namespace Umbraco.Tests.UmbracoExamine public void Events_Ignoring_Node() { using (var luceneDir = new RandomIdRamDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, Container.GetInstance(), + using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, //make parent id 999 so all are ignored options: new UmbracoContentIndexerOptions(false, false, 999))) using (indexer.ProcessNonAsync()) diff --git a/src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs b/src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs index 6357447d54..c8a5cbceb4 100644 --- a/src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs +++ b/src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs @@ -29,168 +29,146 @@ namespace Umbraco.Tests.UmbracoExamine /// internal static class IndexInitializer { + public static ContentValueSetBuilder GetContentValueSetBuilder(PropertyEditorCollection propertyEditors) + { + var contentValueSetBuilder = new ContentValueSetBuilder(propertyEditors, new[] { new DefaultUrlSegmentProvider() }, IndexInitializer.GetMockUserService()); + return contentValueSetBuilder; + } + + public static ContentIndexPopulator GetContentIndexRebuilder(PropertyEditorCollection propertyEditors, IContentService contentService, ISqlContext sqlContext) + { + var contentValueSetBuilder = GetContentValueSetBuilder(propertyEditors); + var contentIndexDataSource = new ContentIndexPopulator(true, null, contentService, sqlContext, contentValueSetBuilder); + return contentIndexDataSource; + } + + public static MediaIndexPopulator GetMediaIndexRebuilder(PropertyEditorCollection propertyEditors, IMediaService mediaService) + { + var mediaValueSetBuilder = new MediaValueSetBuilder(propertyEditors, new[] { new DefaultUrlSegmentProvider() }, IndexInitializer.GetMockUserService()); + var mediaIndexDataSource = new MediaIndexPopulator(null, mediaService, mediaValueSetBuilder); + return mediaIndexDataSource; + } + + public static IContentService GetMockContentService() + { + long longTotalRecs; + var demoData = new ExamineDemoDataContentService(); + + var allRecs = demoData.GetLatestContentByXPath("//*[@isDoc]") + .Root + .Elements() + .Select(x => Mock.Of( + m => + m.Id == (int)x.Attribute("id") && + m.ParentId == (int)x.Attribute("parentID") && + m.Level == (int)x.Attribute("level") && + m.CreatorId == 0 && + m.SortOrder == (int)x.Attribute("sortOrder") && + m.CreateDate == (DateTime)x.Attribute("createDate") && + m.UpdateDate == (DateTime)x.Attribute("updateDate") && + m.Name == (string)x.Attribute("nodeName") && + m.GetCultureName(It.IsAny()) == (string)x.Attribute("nodeName") && + m.Path == (string)x.Attribute("path") && + m.Properties == new PropertyCollection() && + m.ContentType == Mock.Of(mt => + mt.Icon == "test" && + mt.Alias == x.Name.LocalName && + mt.Id == (int)x.Attribute("nodeType")))) + .ToArray(); + + + return Mock.Of( + x => x.GetPagedDescendants( + It.IsAny(), It.IsAny(), It.IsAny(), out longTotalRecs, It.IsAny>(), It.IsAny()) + == allRecs); + } + + public static IUserService GetMockUserService() + { + return Mock.Of(x => x.GetProfileById(It.IsAny()) == Mock.Of(p => p.Id == 0 && p.Name == "admin")); + } + + public static IMediaService GetMockMediaService() + { + long totalRecs; + + var demoData = new ExamineDemoDataMediaService(); + + var allRecs = demoData.GetLatestMediaByXpath("//node") + .Root + .Elements() + .Select(x => Mock.Of( + m => + m.Id == (int)x.Attribute("id") && + m.ParentId == (int)x.Attribute("parentID") && + m.Level == (int)x.Attribute("level") && + m.CreatorId == 0 && + m.SortOrder == (int)x.Attribute("sortOrder") && + m.CreateDate == (DateTime)x.Attribute("createDate") && + m.UpdateDate == (DateTime)x.Attribute("updateDate") && + m.Name == (string)x.Attribute("nodeName") && + m.GetCultureName(It.IsAny()) == (string)x.Attribute("nodeName") && + m.Path == (string)x.Attribute("path") && + m.Properties == new PropertyCollection() && + m.ContentType == Mock.Of(mt => + mt.Alias == (string)x.Attribute("nodeTypeAlias") && + mt.Id == (int)x.Attribute("nodeType")))) + .ToArray(); + + // MOCK! + var mediaServiceMock = new Mock(); + + mediaServiceMock + .Setup(x => x.GetPagedDescendants( + It.IsAny(), It.IsAny(), It.IsAny(), out totalRecs, It.IsAny>(), It.IsAny()) + ).Returns(() => allRecs); + + //mediaServiceMock.Setup(service => service.GetPagedXmlEntries(It.IsAny(), It.IsAny(), It.IsAny(), out longTotalRecs)) + // .Returns(() => allRecs.Select(x => x.ToXml())); + + return mediaServiceMock.Object; + } + + public static ILocalizationService GetMockLocalizationService() + { + return Mock.Of(x => x.GetAllLanguages() == Array.Empty()); + } + + public static IMediaTypeService GetMockMediaTypeService() + { + var mediaTypeServiceMock = new Mock(); + mediaTypeServiceMock.Setup(x => x.GetAll()) + .Returns(new List + { + new MediaType(-1) {Alias = "Folder", Name = "Folder", Id = 1031, Icon = "icon-folder"}, + new MediaType(-1) {Alias = "Image", Name = "Image", Id = 1032, Icon = "icon-picture"} + }); + return mediaTypeServiceMock.Object; + } + public static UmbracoContentIndexer GetUmbracoIndexer( ProfilingLogger profilingLogger, Directory luceneDir, - ISqlContext sqlContext, - PropertyEditorCollection propertyEditors, Analyzer analyzer = null, - IContentService contentService = null, - IMediaService mediaService = null, - IMemberService memberService = null, - IUserService userService = null, ILocalizationService languageService = null, - IContentTypeService contentTypeService = null, - IMediaTypeService mediaTypeService = null, UmbracoContentIndexerOptions options = null) { if (languageService == null) - { - languageService = Mock.Of( - x => x.GetAllLanguages() == Array.Empty()); - } - - if (contentService == null) - { - long longTotalRecs; - var demoData = new ExamineDemoDataContentService(); - - var allRecs = demoData.GetLatestContentByXPath("//*[@isDoc]") - .Root - .Elements() - .Select(x => Mock.Of( - m => - m.Id == (int)x.Attribute("id") && - m.ParentId == (int)x.Attribute("parentID") && - m.Level == (int)x.Attribute("level") && - m.CreatorId == 0 && - m.SortOrder == (int)x.Attribute("sortOrder") && - m.CreateDate == (DateTime)x.Attribute("createDate") && - m.UpdateDate == (DateTime)x.Attribute("updateDate") && - m.Name == (string)x.Attribute("nodeName") && - m.GetCultureName(It.IsAny()) == (string)x.Attribute("nodeName") && - m.Path == (string)x.Attribute("path") && - m.Properties == new PropertyCollection() && - m.ContentType == Mock.Of(mt => - mt.Icon == "test" && - mt.Alias == x.Name.LocalName && - mt.Id == (int)x.Attribute("nodeType")))) - .ToArray(); - - - contentService = Mock.Of( - x => x.GetPagedDescendants( - It.IsAny(), It.IsAny(), It.IsAny(), out longTotalRecs, It.IsAny>(), It.IsAny()) - == - allRecs); - } - if (userService == null) - { - userService = Mock.Of(x => x.GetProfileById(It.IsAny()) == Mock.Of(p => p.Id == 0 && p.Name == "admin")); - } - - if (mediaService == null) - { - long totalRecs; - - var demoData = new ExamineDemoDataMediaService(); - - var allRecs = demoData.GetLatestMediaByXpath("//node") - .Root - .Elements() - .Select(x => Mock.Of( - m => - m.Id == (int) x.Attribute("id") && - m.ParentId == (int) x.Attribute("parentID") && - m.Level == (int) x.Attribute("level") && - m.CreatorId == 0 && - m.SortOrder == (int) x.Attribute("sortOrder") && - m.CreateDate == (DateTime) x.Attribute("createDate") && - m.UpdateDate == (DateTime) x.Attribute("updateDate") && - m.Name == (string) x.Attribute("nodeName") && - m.GetCultureName(It.IsAny()) == (string)x.Attribute("nodeName") && - m.Path == (string) x.Attribute("path") && - m.Properties == new PropertyCollection() && - m.ContentType == Mock.Of(mt => - mt.Alias == (string) x.Attribute("nodeTypeAlias") && - mt.Id == (int) x.Attribute("nodeType")))) - .ToArray(); - - // MOCK! - var mediaServiceMock = new Mock(); - - mediaServiceMock - .Setup(x => x.GetPagedDescendants( - It.IsAny(), It.IsAny(), It.IsAny(), out totalRecs, It.IsAny>(), It.IsAny()) - ).Returns(() => allRecs); - - //mediaServiceMock.Setup(service => service.GetPagedXmlEntries(It.IsAny(), It.IsAny(), It.IsAny(), out longTotalRecs)) - // .Returns(() => allRecs.Select(x => x.ToXml())); - - mediaService = mediaServiceMock.Object; - - } + languageService = GetMockLocalizationService(); if (analyzer == null) - { analyzer = new StandardAnalyzer(Version.LUCENE_30); - } - - //var indexSet = new IndexSet(); - // var indexCriteria = indexSet.ToIndexCriteria(dataService, UmbracoContentIndexer.IndexFieldPolicies); - - //var i = new UmbracoContentIndexer(indexCriteria, - // luceneDir, //custom lucene directory - // dataService, - // contentService, - // mediaService, - // dataTypeService, - // userService, - // new[] { new DefaultUrlSegmentProvider() }, - // analyzer, - // false); - - //i.IndexSecondsInterval = 1; if (options == null) - { options = new UmbracoContentIndexerOptions(false, false, null); - } - - if (mediaTypeService == null) - { - var mediaTypeServiceMock = new Mock(); - mediaTypeServiceMock.Setup(x => x.GetAll()) - .Returns(new List - { - new MediaType(-1) {Alias = "Folder", Name = "Folder", Id = 1031, Icon = "icon-folder"}, - new MediaType(-1) {Alias = "Image", Name = "Image", Id = 1032, Icon = "icon-picture"} - }); - mediaTypeService = mediaTypeServiceMock.Object; - } - - // fixme oops?! - //var query = new Mock>(); - //query - // .Setup(x => x.GetWhereClauses()) - // .Returns(new List> { new Tuple($"{Constants.DatabaseSchema.Tables.Document}.published", new object[] { 1 }) }); - //scopeProvider - // .Setup(x => x.Query()) - // .Returns(query.Object); - var i = new UmbracoContentIndexer( "testIndexer", UmbracoExamineIndexer.UmbracoIndexFieldDefinitions, luceneDir, analyzer, profilingLogger, - new ContentValueSetBuilder(propertyEditors, new[] { new DefaultUrlSegmentProvider() }, userService), - new MediaValueSetBuilder(propertyEditors, new[] { new DefaultUrlSegmentProvider() }, userService), - contentService, - mediaService, languageService, - sqlContext, new UmbracoContentValueSetValidator(options, Mock.Of()), options); diff --git a/src/Umbraco.Tests/UmbracoExamine/IndexTest.cs b/src/Umbraco.Tests/UmbracoExamine/IndexTest.cs index e454fe4d39..5ad542bf19 100644 --- a/src/Umbraco.Tests/UmbracoExamine/IndexTest.cs +++ b/src/Umbraco.Tests/UmbracoExamine/IndexTest.cs @@ -30,13 +30,14 @@ namespace Umbraco.Tests.UmbracoExamine [Test] public void Index_Property_Data_With_Value_Indexer() { + var contentValueSetBuilder = IndexInitializer.GetContentValueSetBuilder(Container.GetInstance()); + using (var luceneDir = new RandomIdRamDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer( - ProfilingLogger, luceneDir, ScopeProvider.SqlContext, Container.GetInstance(), + using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, options: new UmbracoContentIndexerOptions(true, false, null))) using (indexer.ProcessNonAsync()) { - indexer.EnsureIndex(true); + indexer.CreateIndex(); var contentType = MockedContentTypes.CreateBasicContentType(); contentType.AddPropertyType(new PropertyType("test", ValueStorageType.Ntext) @@ -99,8 +100,7 @@ namespace Umbraco.Tests.UmbracoExamine var json = JsonConvert.SerializeObject(gridVal); content.Properties["grid"].SetValue(json); - - var valueSet = indexer.ContentValueSetBuilder.GetValueSets(content); + var valueSet = contentValueSetBuilder.GetValueSets(content); indexer.IndexItems(valueSet); var searcher = indexer.GetSearcher(); @@ -122,15 +122,19 @@ namespace Umbraco.Tests.UmbracoExamine [Test] public void Rebuild_Index() { + var contentRebuilder = IndexInitializer.GetContentIndexRebuilder(Container.GetInstance(), IndexInitializer.GetMockContentService(), ScopeProvider.SqlContext); + var mediaRebuilder = IndexInitializer.GetMediaIndexRebuilder(Container.GetInstance(), IndexInitializer.GetMockMediaService()); + using (var luceneDir = new RandomIdRamDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, Container.GetInstance(), + using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, options: new UmbracoContentIndexerOptions(true, false, null))) using (indexer.ProcessNonAsync()) { var searcher = indexer.GetSearcher(); //create the whole thing - indexer.RebuildIndex(); + contentRebuilder.Populate(indexer); + mediaRebuilder.Populate(indexer); var result = searcher.Search(searcher.CreateCriteria().All().Compile()); @@ -145,13 +149,17 @@ namespace Umbraco.Tests.UmbracoExamine [Test] public void Index_Protected_Content_Not_Indexed() { + var rebuilder = IndexInitializer.GetContentIndexRebuilder(Container.GetInstance(), IndexInitializer.GetMockContentService(), ScopeProvider.SqlContext); + + using (var luceneDir = new RandomIdRamDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, Container.GetInstance())) + using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir)) using (indexer.ProcessNonAsync()) using (var searcher = ((LuceneSearcher)indexer.GetSearcher()).GetLuceneSearcher()) { //create the whole thing - indexer.RebuildIndex(); + rebuilder.Populate(indexer); + var protectedQuery = new BooleanQuery(); protectedQuery.Add( @@ -176,8 +184,9 @@ namespace Umbraco.Tests.UmbracoExamine [Test] public void Index_Move_Media_From_Non_Indexable_To_Indexable_ParentID() { + using (var luceneDir = new RandomIdRamDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, Container.GetInstance(), + using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, //make parent id 1116 options: new UmbracoContentIndexerOptions(false, false, 1116))) using (indexer.ProcessNonAsync()) @@ -219,7 +228,7 @@ namespace Umbraco.Tests.UmbracoExamine public void Index_Move_Media_To_Non_Indexable_ParentID() { using (var luceneDir = new RandomIdRamDirectory()) - using (var indexer1 = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, Container.GetInstance(), + using (var indexer1 = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, //make parent id 2222 options: new UmbracoContentIndexerOptions(false, false, 2222))) using (indexer1.ProcessNonAsync()) @@ -268,16 +277,17 @@ namespace Umbraco.Tests.UmbracoExamine [Test] public void Index_Reindex_Content() { + var rebuilder = IndexInitializer.GetContentIndexRebuilder(Container.GetInstance(), IndexInitializer.GetMockContentService(), ScopeProvider.SqlContext); + using (var luceneDir = new RandomIdRamDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, Container.GetInstance(), + using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, options: new UmbracoContentIndexerOptions(true, false, null))) using (indexer.ProcessNonAsync()) { var searcher = indexer.GetSearcher(); //create the whole thing - indexer.RebuildIndex(); - + rebuilder.Populate(indexer); var result = searcher.Search(searcher.CreateCriteria().Field(LuceneIndexer.CategoryFieldName, IndexTypes.Content).Compile()); Assert.AreEqual(21, result.TotalItemCount); @@ -294,9 +304,7 @@ namespace Umbraco.Tests.UmbracoExamine Assert.AreEqual(0, result.TotalItemCount); //call our indexing methods - indexer.IndexAll(IndexTypes.Content); - - + rebuilder.Populate(indexer); result = searcher.Search(searcher.CreateCriteria().Field(LuceneIndexer.CategoryFieldName, IndexTypes.Content).Compile()); Assert.AreEqual(21, result.TotalItemCount); @@ -309,15 +317,17 @@ namespace Umbraco.Tests.UmbracoExamine [Test] public void Index_Delete_Index_Item_Ensure_Heirarchy_Removed() { + + var rebuilder = IndexInitializer.GetContentIndexRebuilder(Container.GetInstance(), IndexInitializer.GetMockContentService(), ScopeProvider.SqlContext); + using (var luceneDir = new RandomIdRamDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, Container.GetInstance())) + using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir)) using (indexer.ProcessNonAsync()) { var searcher = indexer.GetSearcher(); //create the whole thing - indexer.RebuildIndex(); - + rebuilder.Populate(indexer); //now delete a node that has children diff --git a/src/Umbraco.Tests/UmbracoExamine/SearchTests.cs b/src/Umbraco.Tests/UmbracoExamine/SearchTests.cs index 58df17c052..fc95592d3d 100644 --- a/src/Umbraco.Tests/UmbracoExamine/SearchTests.cs +++ b/src/Umbraco.Tests/UmbracoExamine/SearchTests.cs @@ -7,11 +7,13 @@ using NUnit.Framework; using Examine.LuceneEngine.SearchCriteria; using Moq; using Umbraco.Core.Models; +using Umbraco.Core.Persistence; using Umbraco.Core.Persistence.Querying; using Umbraco.Core.Services; using Umbraco.Examine; using Umbraco.Tests.Testing; using Umbraco.Core.PropertyEditors; +using Umbraco.Core.Strings; namespace Umbraco.Tests.UmbracoExamine { @@ -53,15 +55,16 @@ namespace Umbraco.Tests.UmbracoExamine == allRecs); + var propertyEditors = Container.GetInstance(); + var rebuilder = IndexInitializer.GetContentIndexRebuilder(propertyEditors, contentService, ScopeProvider.SqlContext); + using (var luceneDir = new RandomIdRamDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, - Container.GetInstance(), - contentService: contentService)) + using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir)) using (indexer.ProcessNonAsync()) { - indexer.RebuildIndex(); + indexer.CreateIndex(); + rebuilder.Populate(indexer); - var searcher = indexer.GetSearcher(); var numberSortedCriteria = searcher.CreateCriteria() diff --git a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj index 832760bde2..f33df18a54 100644 --- a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj +++ b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj @@ -88,7 +88,7 @@ - + diff --git a/src/Umbraco.Web/Components/DatabaseServerRegistrarAndMessengerComponent.cs b/src/Umbraco.Web/Components/DatabaseServerRegistrarAndMessengerComponent.cs index 9a2ca3513c..6d35580afa 100644 --- a/src/Umbraco.Web/Components/DatabaseServerRegistrarAndMessengerComponent.cs +++ b/src/Umbraco.Web/Components/DatabaseServerRegistrarAndMessengerComponent.cs @@ -11,6 +11,7 @@ using Umbraco.Core.Scoping; using Umbraco.Core.Services; using Umbraco.Core.Services.Changes; using Umbraco.Core.Sync; +using Umbraco.Examine; using Umbraco.Web.Cache; using Umbraco.Web.Composing; using Umbraco.Web.Routing; @@ -48,7 +49,7 @@ namespace Umbraco.Web.Components private BackgroundTaskRunner _processTaskRunner; private bool _started; private IBackgroundTask[] _tasks; - private IExamineManager _examineManager; + private IndexRebuilder _indexRebuilder; public override void Compose(Composition composition) { @@ -87,13 +88,13 @@ namespace Umbraco.Web.Components //rebuild indexes if the server is not synced // NOTE: This will rebuild ALL indexes including the members, if developers want to target specific // indexes then they can adjust this logic themselves. - () => ExamineComponent.RebuildIndexes(_examineManager, _logger, false, 5000) + () => ExamineComponent.RebuildIndexes(_indexRebuilder, _logger, false, 5000) } }); }); } - public void Initialize(IRuntimeState runtime, IServerRegistrar serverRegistrar, IServerMessenger serverMessenger, IServerRegistrationService registrationService, ILogger logger, IExamineManager examineManager) + public void Initialize(IRuntimeState runtime, IServerRegistrar serverRegistrar, IServerMessenger serverMessenger, IServerRegistrationService registrationService, ILogger logger, IndexRebuilder indexRebuilder) { _registrar = serverRegistrar as DatabaseServerRegistrar; if (_registrar == null) throw new Exception("panic: registar."); @@ -106,7 +107,7 @@ namespace Umbraco.Web.Components _runtime = runtime; _logger = logger; _registrationService = registrationService; - _examineManager = examineManager; + _indexRebuilder = indexRebuilder; _touchTaskRunner = new BackgroundTaskRunner("ServerRegistration", new BackgroundTaskRunnerOptions { AutoStart = true }, logger); diff --git a/src/Umbraco.Web/Editors/ExamineManagementController.cs b/src/Umbraco.Web/Editors/ExamineManagementController.cs index 829c4ccf74..2c29f63613 100644 --- a/src/Umbraco.Web/Editors/ExamineManagementController.cs +++ b/src/Umbraco.Web/Editors/ExamineManagementController.cs @@ -165,7 +165,9 @@ namespace Umbraco.Web.Editors try { - indexer.RebuildIndex(); + //TODO: Rebuilding isn't build directly into an index, we need a new IRebuildIndex or similar interface that can be registered + throw new NotImplementedException("Implement rebuilding!"); + //indexer.RebuildIndex(); } catch (Exception ex) { diff --git a/src/Umbraco.Web/Search/ExamineComponent.cs b/src/Umbraco.Web/Search/ExamineComponent.cs index a8bc1649e5..64a69a43fc 100644 --- a/src/Umbraco.Web/Search/ExamineComponent.cs +++ b/src/Umbraco.Web/Search/ExamineComponent.cs @@ -42,6 +42,8 @@ namespace Umbraco.Web.Search private IValueSetBuilder _contentValueSetBuilder; private IValueSetBuilder _mediaValueSetBuilder; private IValueSetBuilder _memberValueSetBuilder; + private ContentIndexPopulator _contentIndexPopulator; + private MediaIndexPopulator _mediaIndexPopulator; private static bool _disableExamineIndexing = false; private static volatile bool _isConfigured = false; private static readonly object IsConfiguredLocker = new object(); @@ -49,7 +51,6 @@ namespace Umbraco.Web.Search private ServiceContext _services; private static BackgroundTaskRunner _rebuildOnStartupRunner; private static readonly object _rebuildLocker = new object(); - private IEnumerable _urlSegmentProviders; // the default enlist priority is 100 // enlist with a lower priority to ensure that anything "default" runs after us @@ -60,8 +61,11 @@ namespace Umbraco.Web.Search { base.Compose(composition); + composition.Container.RegisterSingleton(); + composition.Container.RegisterSingleton(); + composition.Container.RegisterSingleton(); + composition.Container.RegisterSingleton(); composition.Container.RegisterSingleton(); - composition.Container.RegisterSingleton(); composition.Container.RegisterSingleton, ContentValueSetBuilder>(); composition.Container.RegisterSingleton, MediaValueSetBuilder>(); composition.Container.RegisterSingleton, MemberValueSetBuilder>(); @@ -69,17 +73,19 @@ namespace Umbraco.Web.Search internal void Initialize(IRuntimeState runtime, MainDom mainDom, PropertyEditorCollection propertyEditors, IExamineManager examineManager, ProfilingLogger profilingLogger, - IScopeProvider scopeProvider, IUmbracoIndexesBuilder indexBuilder, ServiceContext services, - IEnumerable urlSegmentProviders, + IScopeProvider scopeProvider, IUmbracoIndexesBuilder indexBuilder, + IndexRebuilder indexRebuilder, ServiceContext services, IValueSetBuilder contentValueSetBuilder, IValueSetBuilder mediaValueSetBuilder, - IValueSetBuilder memberValueSetBuilder) + IValueSetBuilder memberValueSetBuilder, + ContentIndexPopulator contentIndexPopulator, PublishedContentIndexPopulator publishedContentIndexPopulator, + MediaIndexPopulator mediaIndexPopulator) { _services = services; _scopeProvider = scopeProvider; _examineManager = examineManager; - _urlSegmentProviders = urlSegmentProviders; - + _contentIndexPopulator = contentIndexPopulator; + _mediaIndexPopulator = mediaIndexPopulator; _contentValueSetBuilder = contentValueSetBuilder; _mediaValueSetBuilder = mediaValueSetBuilder; _memberValueSetBuilder = memberValueSetBuilder; @@ -111,7 +117,7 @@ namespace Umbraco.Web.Search profilingLogger.Logger.Debug("Examine shutdown not registered, this appdomain is not the MainDom, Examine will be disabled"); //if we could not register the shutdown examine ourselves, it means we are not maindom! in this case all of examine should be disabled! - Suspendable.ExamineEvents.SuspendIndexers(); + Suspendable.ExamineEvents.SuspendIndexers(profilingLogger.Logger); _disableExamineIndexing = true; return; //exit, do not continue } @@ -119,7 +125,7 @@ namespace Umbraco.Web.Search //create the indexes and register them with the manager foreach(var index in indexBuilder.Create()) { - _examineManager.AddIndexer(index.Key, index.Value); + _examineManager.AddIndexer(index); } profilingLogger.Logger.Debug("Examine shutdown registered with MainDom"); @@ -141,7 +147,7 @@ namespace Umbraco.Web.Search EnsureUnlocked(profilingLogger.Logger, examineManager); - RebuildIndexes(examineManager, profilingLogger.Logger, true, 5000); + RebuildIndexes(indexRebuilder, profilingLogger.Logger, true, 5000); } @@ -149,7 +155,7 @@ namespace Umbraco.Web.Search /// Called to rebuild empty indexes on startup /// /// - public static void RebuildIndexes(IExamineManager examineManager, ILogger logger, bool onlyEmptyIndexes, int waitMilliseconds = 0) + public static void RebuildIndexes(IndexRebuilder indexRebuilder, ILogger logger, bool onlyEmptyIndexes, int waitMilliseconds = 0) { //TODO: need a way to disable rebuilding on startup @@ -163,7 +169,7 @@ namespace Umbraco.Web.Search logger.Info("Starting initialize async background thread."); //do the rebuild on a managed background thread - var task = new RebuildOnStartupTask(examineManager, logger, onlyEmptyIndexes, waitMilliseconds); + var task = new RebuildOnStartupTask(indexRebuilder, logger, onlyEmptyIndexes, waitMilliseconds); _rebuildOnStartupRunner = new BackgroundTaskRunner( "RebuildIndexesOnStartup", @@ -695,14 +701,15 @@ namespace Umbraco.Web.Search public static void Execute(ExamineComponent examineComponent, IContent content, bool? supportUnpublished) { - var valueSet = examineComponent._contentValueSetBuilder.GetValueSets(content); + var valueSet = examineComponent._contentValueSetBuilder.GetValueSets(content).ToList(); - examineComponent._examineManager.IndexItems( - valueSet.ToArray(), - examineComponent._examineManager.IndexProviders.Values.OfType() - // only for the specified indexers - .Where(x => supportUnpublished.HasValue == false || supportUnpublished.Value == x.SupportUnpublishedContent) - .Where(x => x.EnableDefaultEventHandler)); + foreach (var index in examineComponent._examineManager.IndexProviders.Values.OfType() + // only for the specified indexers + .Where(x => supportUnpublished.HasValue == false || supportUnpublished.Value == x.SupportUnpublishedContent) + .Where(x => x.EnableDefaultEventHandler)) + { + index.IndexItems(valueSet); + } } } @@ -726,15 +733,16 @@ namespace Umbraco.Web.Search public static void Execute(ExamineComponent examineComponent, IMedia media, bool isPublished) { - var valueSet = examineComponent._mediaValueSetBuilder.GetValueSets(media); + var valueSet = examineComponent._mediaValueSetBuilder.GetValueSets(media).ToList(); - examineComponent._examineManager.IndexItems( - valueSet.ToArray(), - examineComponent._examineManager.IndexProviders.Values.OfType() - // index this item for all indexers if the media is not trashed, otherwise if the item is trashed - // then only index this for indexers supporting unpublished media - .Where(x => isPublished || (x.SupportUnpublishedContent)) - .Where(x => x.EnableDefaultEventHandler)); + foreach (var index in examineComponent._examineManager.IndexProviders.Values.OfType() + // index this item for all indexers if the media is not trashed, otherwise if the item is trashed + // then only index this for indexers supporting unpublished media + .Where(x => isPublished || (x.SupportUnpublishedContent)) + .Where(x => x.EnableDefaultEventHandler)) + { + index.IndexItems(valueSet); + } } } @@ -756,13 +764,13 @@ namespace Umbraco.Web.Search public static void Execute(ExamineComponent examineComponent, IMember member) { - var valueSet = examineComponent._memberValueSetBuilder.GetValueSets(member); - - examineComponent._examineManager.IndexItems( - valueSet.ToArray(), - examineComponent._examineManager.IndexProviders.Values.OfType() - //ensure that only the providers are flagged to listen execute - .Where(x => x.EnableDefaultEventHandler)); + var valueSet = examineComponent._memberValueSetBuilder.GetValueSets(member).ToList(); + foreach (var index in examineComponent._examineManager.IndexProviders.Values.OfType() + //ensure that only the providers are flagged to listen execute + .Where(x => x.EnableDefaultEventHandler)) + { + index.IndexItems(valueSet); + } } } @@ -786,13 +794,15 @@ namespace Umbraco.Web.Search public static void Execute(ExamineComponent examineComponent, int id, bool keepIfUnpublished) { - examineComponent._examineManager.DeleteFromIndexes( - id.ToString(CultureInfo.InvariantCulture), - examineComponent._examineManager.IndexProviders.Values.OfType() - // if keepIfUnpublished == true then only delete this item from indexes not supporting unpublished content, - // otherwise if keepIfUnpublished == false then remove from all indexes - .Where(x => keepIfUnpublished == false || x.SupportUnpublishedContent == false) - .Where(x => x.EnableDefaultEventHandler)); + var strId = id.ToString(CultureInfo.InvariantCulture); + foreach (var index in examineComponent._examineManager.IndexProviders.Values.OfType() + // if keepIfUnpublished == true then only delete this item from indexes not supporting unpublished content, + // otherwise if keepIfUnpublished == false then remove from all indexes + .Where(x => keepIfUnpublished == false || x.SupportUnpublishedContent == false) + .Where(x => x.EnableDefaultEventHandler)) + { + index.DeleteFromIndex(strId); + } } } #endregion @@ -802,15 +812,15 @@ namespace Umbraco.Web.Search /// private class RebuildOnStartupTask : IBackgroundTask { - private readonly IExamineManager _examineManager; + private readonly IndexRebuilder _indexRebuilder; private readonly ILogger _logger; private readonly bool _onlyEmptyIndexes; private readonly int _waitMilliseconds; - public RebuildOnStartupTask(IExamineManager examineManager, ILogger logger, bool onlyEmptyIndexes, int waitMilliseconds = 0) + public RebuildOnStartupTask(IndexRebuilder indexRebuilder, ILogger logger, bool onlyEmptyIndexes, int waitMilliseconds = 0) { - _examineManager = examineManager; - _logger = logger; + _indexRebuilder = indexRebuilder ?? throw new ArgumentNullException(nameof(indexRebuilder)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _onlyEmptyIndexes = onlyEmptyIndexes; _waitMilliseconds = waitMilliseconds; } @@ -843,7 +853,6 @@ namespace Umbraco.Web.Search /// /// Used to rebuild indexes on startup or cold boot /// - /// private void RebuildIndexes() { //do not attempt to do this if this has been disabled since we are not the main dom. @@ -853,20 +862,8 @@ namespace Umbraco.Web.Search if (_waitMilliseconds > 0) Thread.Sleep(_waitMilliseconds); - EnsureUnlocked(_logger, _examineManager); - - if (_onlyEmptyIndexes) - { - foreach (var indexer in _examineManager.IndexProviders.Values.Where(x => x.IsIndexNew())) - { - indexer.RebuildIndex(); - } - } - else - { - //do all of them - _examineManager.RebuildIndexes(); - } + EnsureUnlocked(_logger, _indexRebuilder.ExamineManager); + _indexRebuilder.RebuildIndexes(_onlyEmptyIndexes); } } } diff --git a/src/Umbraco.Web/Search/IUmbracoIndexesBuilder.cs b/src/Umbraco.Web/Search/IUmbracoIndexesBuilder.cs index 012f119238..8193117d69 100644 --- a/src/Umbraco.Web/Search/IUmbracoIndexesBuilder.cs +++ b/src/Umbraco.Web/Search/IUmbracoIndexesBuilder.cs @@ -5,6 +5,6 @@ namespace Umbraco.Web.Search { internal interface IUmbracoIndexesBuilder { - IReadOnlyDictionary Create(); + IEnumerable Create(); } } diff --git a/src/Umbraco.Web/Search/UmbracoIndexesBuilder.cs b/src/Umbraco.Web/Search/UmbracoIndexesBuilder.cs index 25ba64828f..f8a6a16ec5 100644 --- a/src/Umbraco.Web/Search/UmbracoIndexesBuilder.cs +++ b/src/Umbraco.Web/Search/UmbracoIndexesBuilder.cs @@ -26,39 +26,33 @@ namespace Umbraco.Web.Search //TODO: we should inject the different IValueSetValidator so devs can just register them instead of overriding this class? public UmbracoIndexesBuilder(ProfilingLogger profilingLogger, - IValueSetBuilder contentValueSetBuilder, - IValueSetBuilder mediaValueSetBuilder, IValueSetBuilder memberValueSetBuilder, - IContentService contentService, - IMediaService mediaService, + ContentIndexPopulator contentIndexPopulator, + PublishedContentIndexPopulator publishedContentIndexPopulator, + MediaIndexPopulator mediaIndexPopulator, ILocalizationService languageService, IPublicAccessService publicAccessService, - IMemberService memberService, - ISqlContext sqlContext) + IMemberService memberService) { ProfilingLogger = profilingLogger ?? throw new System.ArgumentNullException(nameof(profilingLogger)); - ContentValueSetBuilder = contentValueSetBuilder ?? throw new System.ArgumentNullException(nameof(contentValueSetBuilder)); - MediaValueSetBuilder = mediaValueSetBuilder ?? throw new System.ArgumentNullException(nameof(mediaValueSetBuilder)); MemberValueSetBuilder = memberValueSetBuilder ?? throw new System.ArgumentNullException(nameof(memberValueSetBuilder)); - ContentService = contentService ?? throw new System.ArgumentNullException(nameof(contentService)); - MediaService = mediaService ?? throw new System.ArgumentNullException(nameof(mediaService)); + ContentIndexPopulator = contentIndexPopulator; + PublishedContentIndexPopulator = publishedContentIndexPopulator; + MediaIndexPopulator = mediaIndexPopulator; LanguageService = languageService ?? throw new System.ArgumentNullException(nameof(languageService)); PublicAccessService = publicAccessService ?? throw new System.ArgumentNullException(nameof(publicAccessService)); MemberService = memberService ?? throw new System.ArgumentNullException(nameof(memberService)); - SqlContext = sqlContext ?? throw new System.ArgumentNullException(nameof(sqlContext)); } protected ProfilingLogger ProfilingLogger { get; } - protected IValueSetBuilder ContentValueSetBuilder { get; } - protected IValueSetBuilder MediaValueSetBuilder { get; } protected IValueSetBuilder MemberValueSetBuilder { get; } - protected IContentService ContentService { get; } - protected IMediaService MediaService { get; } + protected ContentIndexPopulator ContentIndexPopulator { get; } + protected PublishedContentIndexPopulator PublishedContentIndexPopulator { get; } + protected MediaIndexPopulator MediaIndexPopulator { get; } protected ILocalizationService LanguageService { get; } protected IPublicAccessService PublicAccessService { get; } protected IMemberService MemberService { get; } - protected ISqlContext SqlContext { get; } - + public const string InternalIndexPath = "Internal"; public const string ExternalIndexPath = "External"; public const string MembersIndexPath = "Members"; @@ -72,27 +66,26 @@ namespace Umbraco.Web.Search /// Creates the Umbraco indexes /// /// - public IReadOnlyDictionary Create() + public IEnumerable Create() { - return new Dictionary + return new [] { - [InternalIndexPath] = CreateContentIndex(InternalIndexPath, new UmbracoContentIndexerOptions(true, true, null), new CultureInvariantWhitespaceAnalyzer()), - [ExternalIndexPath] = CreateContentIndex(ExternalIndexPath, new UmbracoContentIndexerOptions(false, false, null), new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30)), - [MembersIndexPath] = CreateMemberIndex() + CreateContentIndex(InternalIndexPath, new UmbracoContentIndexerOptions(true, true, null), new CultureInvariantWhitespaceAnalyzer()), + CreateContentIndex(ExternalIndexPath, new UmbracoContentIndexerOptions(false, false, null), new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30)), + CreateMemberIndex() }; } private IIndexer CreateContentIndex(string name, UmbracoContentIndexerOptions options, Analyzer analyzer) { - var index = new UmbracoContentIndexer( $"{name}Indexer", //fixme - how to deal with languages like in UmbracoContentIndexer.CreateFieldValueTypes UmbracoExamineIndexer.UmbracoIndexFieldDefinitions, GetFileSystemLuceneDirectory(name), analyzer, - ProfilingLogger, ContentValueSetBuilder, MediaValueSetBuilder, - ContentService, MediaService, LanguageService, SqlContext, + ProfilingLogger, + LanguageService, GetContentValueSetValidator(options), options); return index; @@ -100,14 +93,13 @@ namespace Umbraco.Web.Search private IIndexer CreateMemberIndex() { - var appData = Path.Combine(IOHelper.MapPath(SystemDirectories.Data), "TEMP", "ExamineIndexes", MembersIndexPath); var index = new UmbracoMemberIndexer( $"{MembersIndexPath}Indexer", //fixme - how to deal with languages like in UmbracoContentIndexer.CreateFieldValueTypes UmbracoExamineIndexer.UmbracoIndexFieldDefinitions, GetFileSystemLuceneDirectory(MembersIndexPath), new CultureInvariantWhitespaceAnalyzer(), - ProfilingLogger, MemberValueSetBuilder, MemberService, + ProfilingLogger, GetMemberValueSetValidator()); return index; } diff --git a/src/Umbraco.Web/Suspendable.cs b/src/Umbraco.Web/Suspendable.cs index 9bbbef9742..3317afe7a4 100644 --- a/src/Umbraco.Web/Suspendable.cs +++ b/src/Umbraco.Web/Suspendable.cs @@ -1,6 +1,7 @@ using System; using Examine; using Examine.Providers; +using Umbraco.Core.Logging; using Umbraco.Core.Composing; using Umbraco.Examine; using Umbraco.Web.Cache; @@ -63,24 +64,23 @@ namespace Umbraco.Web } } - public static void SuspendIndexers() + public static void SuspendIndexers(ILogger logger) { - Current.ProfilingLogger.Logger.Info(typeof (ExamineEvents), "Suspend indexers."); + logger.Info(typeof (ExamineEvents), "Suspend indexers."); _suspended = true; } - public static void ResumeIndexers() + public static void ResumeIndexers(IndexRebuilder indexRebuilder, ILogger logger) { _suspended = false; - Current.ProfilingLogger.Logger.Info(typeof (ExamineEvents), "Resume indexers (rebuild:{Tried}).", _tried); + logger.Info(typeof (ExamineEvents), "Resume indexers (rebuild:{Tried}).", _tried); if (_tried == false) return; _tried = false; //TODO: when resuming do we always want a full rebuild of all indexes? - // fixme - can we inject these somehow? - ExamineComponent.RebuildIndexes(ExamineManager.Instance, Current.Logger, false); + ExamineComponent.RebuildIndexes(indexRebuilder, logger, false); } } diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index 0e18f07a2b..a626cd8ebd 100755 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -62,7 +62,7 @@ - + 2.6.2.25