From 5010cbbf3079a828c058ee780a11311327210be9 Mon Sep 17 00:00:00 2001 From: Shannon Date: Thu, 29 Mar 2018 15:39:48 +1100 Subject: [PATCH] Gets the examine stuff re-merged and updated after the great merge --- ...aseServerRegistrarAndMessengerComponent.cs | 4 +- src/Umbraco.Web/IPublishedContentQuery.cs | 4 +- .../XmlPublishedCache/PublishedMediaCache.cs | 23 +- src/Umbraco.Web/PublishedContentQuery.cs | 82 +++--- src/Umbraco.Web/Search/ExamineComponent.cs | 238 +++++++++++++++--- 5 files changed, 241 insertions(+), 110 deletions(-) diff --git a/src/Umbraco.Web/Components/DatabaseServerRegistrarAndMessengerComponent.cs b/src/Umbraco.Web/Components/DatabaseServerRegistrarAndMessengerComponent.cs index 9a2f7a4ade..59fa8e899a 100644 --- a/src/Umbraco.Web/Components/DatabaseServerRegistrarAndMessengerComponent.cs +++ b/src/Umbraco.Web/Components/DatabaseServerRegistrarAndMessengerComponent.cs @@ -93,20 +93,20 @@ namespace Umbraco.Web.Components }); } - public void Initialize(IRuntimeState runtime, IServerRegistrar serverRegistrar, IServerRegistrationService registrationService, IUmbracoDatabaseFactory databaseFactory, ILogger logger, IExamineIndexCollectionAccessor indexCollection) + public void Initialize(IRuntimeState runtime, IServerRegistrar serverRegistrar, IServerMessenger serverMessenger, IServerRegistrationService registrationService, ILogger logger, IExamineIndexCollectionAccessor indexes) { if (UmbracoConfig.For.UmbracoSettings().DistributedCall.Enabled) return; _registrar = serverRegistrar as DatabaseServerRegistrar; if (_registrar == null) throw new Exception("panic: registar."); - _indexCollection = indexCollection; _messenger = serverMessenger as BatchedDatabaseServerMessenger; if (_messenger == null) throw new Exception("panic: messenger"); _runtime = runtime; _logger = logger; _registrationService = registrationService; + _indexCollection = indexes; _touchTaskRunner = new BackgroundTaskRunner("ServerRegistration", new BackgroundTaskRunnerOptions { AutoStart = true }, logger); diff --git a/src/Umbraco.Web/IPublishedContentQuery.cs b/src/Umbraco.Web/IPublishedContentQuery.cs index 68b0961425..055dcbe181 100644 --- a/src/Umbraco.Web/IPublishedContentQuery.cs +++ b/src/Umbraco.Web/IPublishedContentQuery.cs @@ -39,7 +39,7 @@ namespace Umbraco.Web /// /// Searches content. /// - IEnumerable Search(int skip, int take, out int totalRecords, string term, bool useWildCards = true, string searchProvider = null); + IEnumerable Search(int skip, int take, out int totalRecords, string term, bool useWildCards = true, string indexName = null); /// /// Searches content. @@ -49,6 +49,6 @@ namespace Umbraco.Web /// /// Searches content. /// - IEnumerable Search(int skip, int take, out int totalrecords, Examine.SearchCriteria.ISearchCriteria criteria, Examine.Providers.BaseSearchProvider searchProvider = null); + IEnumerable Search(int skip, int take, out int totalRecords, Examine.SearchCriteria.ISearchCriteria criteria, Examine.ISearcher searchProvider = null); } } diff --git a/src/Umbraco.Web/PublishedCache/XmlPublishedCache/PublishedMediaCache.cs b/src/Umbraco.Web/PublishedCache/XmlPublishedCache/PublishedMediaCache.cs index 51d0aa12d5..bfe8851800 100644 --- a/src/Umbraco.Web/PublishedCache/XmlPublishedCache/PublishedMediaCache.cs +++ b/src/Umbraco.Web/PublishedCache/XmlPublishedCache/PublishedMediaCache.cs @@ -51,10 +51,8 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache public PublishedMediaCache(XmlStore xmlStore, IMediaService mediaService, IUserService userService, ICacheProvider cacheProvider, PublishedContentTypeCache contentTypeCache) : base(false) { - if (mediaService == null) throw new ArgumentNullException(nameof(mediaService)); - if (userService == null) throw new ArgumentNullException(nameof(userService)); - _mediaService = mediaService; - _userService = userService; + _mediaService = mediaService ?? throw new ArgumentNullException(nameof(mediaService)); + _userService = userService ?? throw new ArgumentNullException(nameof(userService)); _cacheProvider = cacheProvider; _xmlStore = xmlStore; @@ -73,15 +71,10 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache internal PublishedMediaCache(IMediaService mediaService, IUserService userService, ISearcher searchProvider, BaseIndexProvider indexProvider, ICacheProvider cacheProvider, PublishedContentTypeCache contentTypeCache) : base(false) { - if (mediaService == null) throw new ArgumentNullException(nameof(mediaService)); - if (userService == null) throw new ArgumentNullException(nameof(userService)); - if (searchProvider == null) throw new ArgumentNullException(nameof(searchProvider)); - if (indexProvider == null) throw new ArgumentNullException(nameof(indexProvider)); - - _mediaService = mediaService; - _userService = userService; - _searchProvider = searchProvider; - _indexProvider = indexProvider; + _mediaService = mediaService ?? throw new ArgumentNullException(nameof(mediaService)); + _userService = userService ?? throw new ArgumentNullException(nameof(userService)); + _searchProvider = searchProvider ?? throw new ArgumentNullException(nameof(searchProvider)); + _indexProvider = indexProvider ?? throw new ArgumentNullException(nameof(indexProvider)); _cacheProvider = cacheProvider; _contentTypeCache = contentTypeCache; } @@ -117,8 +110,8 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache // first check in Examine for the cache values // +(+parentID:-1) +__IndexType:media - var criteria = searchProvider.CreateSearchCriteria("media"); - var filter = criteria.ParentId(-1).Not().Field(UmbracoContentIndexer.IndexPathFieldName, "-1,-21,".MultipleCharacterWildcard()); + var criteria = searchProvider.CreateCriteria("media"); + var filter = criteria.ParentId(-1).Not().Field(UmbracoExamineIndexer.IndexPathFieldName, "-1,-21,".MultipleCharacterWildcard()); var result = searchProvider.Search(filter.Compile()); if (result != null) diff --git a/src/Umbraco.Web/PublishedContentQuery.cs b/src/Umbraco.Web/PublishedContentQuery.cs index e7e6f766f5..f6007e5d2c 100644 --- a/src/Umbraco.Web/PublishedContentQuery.cs +++ b/src/Umbraco.Web/PublishedContentQuery.cs @@ -221,18 +221,24 @@ namespace Umbraco.Web /// public IEnumerable Search(string term, bool useWildCards = true, string indexName = null) { - return Search(0, 0, out _, term, useWildCards, searchProvider); + return Search(0, 0, out _, term, useWildCards, indexName); } /// - public IEnumerable Search(int skip, int take, out int totalRecords, string term, bool useWildCards = true, string searchProvider = null) - { - if (_query != null) return _query.Search(skip, take, out totalRecords, term, useWildCards, searchProvider); + public IEnumerable Search(int skip, int take, out int totalRecords, string term, bool useWildCards = true, string indexName = null) + { + //TODO: Can we inject/Ioc into this class for ExamineManager or the IExamineIndexAccessor? + + if (_query != null) return _query.Search(skip, take, out totalRecords, term, useWildCards, indexName); + + var indexer = string.IsNullOrEmpty(indexName) + ? Examine.ExamineManager.Instance.IndexProviders[Constants.Examine.ExternalIndexer] + : Examine.ExamineManager.Instance.IndexProviders[indexName]; + + if (indexer == null) throw new InvalidOperationException("No index found by name " + indexName); + + var searcher = indexer.GetSearcher(); - var searcher = string.IsNullOrWhiteSpace(searchProvider) - ? Examine.ExamineManager.Instance.DefaultSearchProvider - : Examine.ExamineManager.Instance.SearchProviderCollection[searchProvider]; - if (skip == 0 && take == 0) { var results = searcher.Search(term, useWildCards); @@ -240,30 +246,22 @@ namespace Umbraco.Web return results.ToPublishedSearchResults(_contentCache); } - if (!(searcher is BaseLuceneSearcher luceneSearcher)) - { - var results = searcher.Search(term, useWildCards); - totalRecords = results.TotalItemCount; - // Examine skip, Linq take - return results.Skip(skip).ToPublishedSearchResults(_contentCache).Take(take); - } - - var criteria = SearchAllFields(term, useWildCards, luceneSearcher); + var criteria = SearchAllFields(term, useWildCards, searcher, indexer); return Search(skip, take, out totalRecords, criteria, searcher); } /// - public IEnumerable Search(Examine.SearchCriteria.ISearchCriteria criteria, Examine.Providers.BaseSearchProvider searchProvider = null) + public IEnumerable Search(ISearchCriteria criteria, Examine.ISearcher searchProvider = null) { return Search(0, 0, out _, criteria, searchProvider); } /// - public IEnumerable Search(int skip, int take, out int totalRecords, Examine.SearchCriteria.ISearchCriteria criteria, Examine.Providers.BaseSearchProvider searchProvider = null) + public IEnumerable Search(int skip, int take, out int totalRecords, ISearchCriteria criteria, Examine.ISearcher searchProvider = null) { if (_query != null) return _query.Search(skip, take, out totalRecords, criteria, searchProvider); - var searcher = searchProvider ?? Examine.ExamineManager.Instance.DefaultSearchProvider; + var searcher = searchProvider ?? Examine.ExamineManager.Instance.GetIndexSearcher(Constants.Examine.ExternalIndexer); var results = skip == 0 && take == 0 ? searcher.Search(criteria) @@ -276,45 +274,25 @@ namespace Umbraco.Web /// /// Creates an ISearchCriteria for searching all fields in a . /// - /// - /// This is here because some of this stuff is internal in Examine. - /// - private ISearchCriteria SearchAllFields(string searchText, bool useWildcards, BaseLuceneSearcher searcher) + private ISearchCriteria SearchAllFields(string searchText, bool useWildcards, Examine.ISearcher searcher, Examine.IIndexer indexer) { - var sc = searcher.CreateSearchCriteria(); - - if (_examineGetSearchFields == null) - { - //get the GetSearchFields method from BaseLuceneSearcher - _examineGetSearchFields = typeof(BaseLuceneSearcher).GetMethod("GetSearchFields", BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Static); - } - - //get the results of searcher.BaseLuceneSearcher() using ugly reflection since it's not public - var searchFields = (IEnumerable) _examineGetSearchFields.Invoke(searcher, null); + var sc = searcher.CreateCriteria(); + + //if we're dealing with a lucene searcher, we can get all of it's indexed fields, + //else we can get the defined fields for the index. + var searchFields = (searcher is BaseLuceneSearcher luceneSearcher) + ? luceneSearcher.GetAllIndexedFields() + : indexer.FieldDefinitionCollection.Keys; //this is what Examine does internally to create ISearchCriteria for searching all fields - var strArray = searchText.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries); + var strArray = searchText.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries); + sc = useWildcards == false ? sc.GroupedOr(searchFields, strArray).Compile() - : sc.GroupedOr(searchFields, strArray.Select(x => new CustomExamineValue(Examineness.ComplexWildcard, x.MultipleCharacterWildcard().Value)).ToArray()).Compile(); + : sc.GroupedOr(searchFields, strArray.Select(x => (IExamineValue)new ExamineValue(Examineness.ComplexWildcard, x.MultipleCharacterWildcard().Value)).ToArray()).Compile(); return sc; - } + } - private static MethodInfo _examineGetSearchFields; - - //support class since Examine doesn't expose it's own ExamineValue class publicly - private class CustomExamineValue : IExamineValue - { - public CustomExamineValue(Examineness vagueness, string value) - { - this.Examineness = vagueness; - this.Value = value; - this.Level = 1f; - } - public Examineness Examineness { get; private set; } - public string Value { get; private set; } - public float Level { get; private set; } - } #endregion } diff --git a/src/Umbraco.Web/Search/ExamineComponent.cs b/src/Umbraco.Web/Search/ExamineComponent.cs index c1f1bd0346..497df0b691 100644 --- a/src/Umbraco.Web/Search/ExamineComponent.cs +++ b/src/Umbraco.Web/Search/ExamineComponent.cs @@ -16,8 +16,11 @@ using Umbraco.Core.Components; using Umbraco.Core.Logging; using Umbraco.Core.Models; using Umbraco.Core.PropertyEditors; +using Umbraco.Core.Scoping; +using Umbraco.Core.Services; using Umbraco.Core.Services.Changes; using Umbraco.Core.Services.Implement; +using Umbraco.Core.Strings; using Umbraco.Core.Sync; using Umbraco.Web.Cache; using Umbraco.Web.Composing; @@ -35,11 +38,22 @@ namespace Umbraco.Web.Search //fixme - we are injecting this which is nice, but we still use ExamineManager everywhere, we could instead interface IExamineManager? private IExamineIndexCollectionAccessor _indexCollection; private static bool _disableExamineIndexing = false; - private static volatile bool __isConfigured = false; + private static volatile bool _isConfigured = false; private static readonly object IsConfiguredLocker = new object(); + private IScopeProvider _scopeProvider; + private UrlSegmentProviderCollection _urlSegmentProviders; + private ServiceContext _services; - public void Initialize(IRuntimeState runtime, PropertyEditorCollection propertyEditors, IExamineIndexCollectionAccessor indexCollection, ProfilingLogger profilingLogger) + // the default enlist priority is 100 + // enlist with a lower priority to ensure that anything "default" runs after us + // but greater that SafeXmlReaderWriter priority which is 60 + private const int EnlistPriority = 80; + + public void Initialize(IRuntimeState runtime, PropertyEditorCollection propertyEditors, IExamineIndexCollectionAccessor indexCollection, ProfilingLogger profilingLogger, IScopeProvider scopeProvider, UrlSegmentProviderCollection urlSegmentProviderCollection, ServiceContext services) { + _services = services; + _urlSegmentProviders = urlSegmentProviderCollection; + _scopeProvider = scopeProvider; _indexCollection = indexCollection; //fixme we cannot inject MainDom since it's internal, so thsi is the only way we can get it, alternatively we can add the container to the container and resolve @@ -174,14 +188,14 @@ namespace Umbraco.Web.Search private static void EnsureUnlocked(ILogger logger, IExamineIndexCollectionAccessor indexCollection) { if (_disableExamineIndexing) return; - if (__isConfigured) return; + if (_isConfigured) return; lock (IsConfiguredLocker) { //double chekc - if (__isConfigured) return; + if (_isConfigured) return; - __isConfigured = true; + _isConfigured = true; foreach (var luceneIndexer in indexCollection.Indexes.Values.OfType()) { @@ -228,7 +242,7 @@ namespace Umbraco.Web.Search switch (args.MessageType) { case MessageType.RefreshById: - var c1 = Current.Services.MemberService.GetById((int)args.MessageObject); + var c1 = _services.MemberService.GetById((int)args.MessageObject); if (c1 != null) { ReIndexForMember(c1); @@ -271,7 +285,7 @@ namespace Umbraco.Web.Search if (args.MessageType != MessageType.RefreshByPayload) throw new NotSupportedException(); - var mediaService = Current.Services.MediaService; + var mediaService = _services.MediaService; foreach (var payload in (MediaCacheRefresher.JsonPayload[]) args.MessageObject) { @@ -320,7 +334,7 @@ namespace Umbraco.Web.Search if (args.MessageType != MessageType.RefreshByPayload) throw new NotSupportedException(); - var contentService = Current.Services.ContentService; + var contentService = _services.ContentService; foreach (var payload in (ContentCacheRefresher.JsonPayload[]) args.MessageObject) { @@ -385,6 +399,8 @@ namespace Umbraco.Web.Search // ReIndexForContent is NOT taking care of descendants so we have to reload everything // again in order to process the branch - we COULD improve that by just reloading the // XML from database instead of reloading content & re-serializing! + // + // BUT ... pretty sure it is! see test "Index_Delete_Index_Item_Ensure_Heirarchy_Removed" } } @@ -412,38 +428,29 @@ namespace Umbraco.Web.Search private void ReIndexForContent(IContent sender, bool? supportUnpublished = null) { - var valueSet = UmbracoContentIndexer.GetValueSets(Current.UrlSegmentProviders, Current.Services.UserService, sender); - - ExamineManager.Instance.IndexItems( - valueSet.ToArray(), - _indexCollection.Indexes.Values.OfType() - // only for the specified indexers - .Where(x => supportUnpublished.HasValue == false || supportUnpublished.Value == x.SupportUnpublishedContent) - .Where(x => x.EnableDefaultEventHandler)); + var actions = DeferedActions.Get(_scopeProvider); + if (actions != null) + actions.Add(new DeferedReIndexForContent(this, sender, supportUnpublished)); + else + DeferedReIndexForContent.Execute(this, sender, supportUnpublished); } private void ReIndexForMember(IMember member) { - var valueSet = UmbracoMemberIndexer.GetValueSets(member); - - ExamineManager.Instance.IndexItems( - valueSet.ToArray(), - _indexCollection.Indexes.Values.OfType() - //ensure that only the providers are flagged to listen execute - .Where(x => x.EnableDefaultEventHandler)); + var actions = DeferedActions.Get(_scopeProvider); + if (actions != null) + actions.Add(new DeferedReIndexForMember(this, member)); + else + DeferedReIndexForMember.Execute(this, member); } private void ReIndexForMedia(IMedia sender, bool isMediaPublished) { - var valueSet = UmbracoContentIndexer.GetValueSets(Current.UrlSegmentProviders, Current.Services.UserService, sender); - - ExamineManager.Instance.IndexItems( - valueSet.ToArray(), - _indexCollection.Indexes.Values.OfType() - // index this item for all indexers if the media is not trashed, otherwise if the item is trashed - // then only index this for indexers supporting unpublished media - .Where(x => isMediaPublished || (x.SupportUnpublishedContent)) - .Where(x => x.EnableDefaultEventHandler)); + var actions = DeferedActions.Get(_scopeProvider); + if (actions != null) + actions.Add(new DeferedReIndexForMedia(this, sender, isMediaPublished)); + else + DeferedReIndexForMedia.Execute(this, sender, isMediaPublished); } /// @@ -456,13 +463,166 @@ namespace Umbraco.Web.Search /// private void DeleteIndexForEntity(int entityId, bool keepIfUnpublished) { - ExamineManager.Instance.DeleteFromIndexes( - entityId.ToString(CultureInfo.InvariantCulture), - _indexCollection.Indexes.Values.OfType() - // if keepIfUnpublished == true then only delete this item from indexes not supporting unpublished content, - // otherwise if keepIfUnpublished == false then remove from all indexes - .Where(x => keepIfUnpublished == false || (x is UmbracoContentIndexer && ((UmbracoContentIndexer)x).SupportUnpublishedContent == false)) - .Where(x => x.EnableDefaultEventHandler)); + var actions = DeferedActions.Get(_scopeProvider); + if (actions != null) + actions.Add(new DeferedDeleteIndex(this, entityId, keepIfUnpublished)); + else + DeferedDeleteIndex.Execute(this, entityId, keepIfUnpublished); + } + + private class DeferedActions + { + private readonly List _actions = new List(); + + public static DeferedActions Get(IScopeProvider scopeProvider) + { + var scopeContext = scopeProvider.Context; + + return scopeContext?.Enlist("examineEvents", + () => new DeferedActions(), // creator + (completed, actions) => // action + { + if (completed) actions.Execute(); + }, EnlistPriority); + } + + public void Add(DeferedAction action) + { + _actions.Add(action); + } + + private void Execute() + { + foreach (var action in _actions) + action.Execute(); + } + } + + private abstract class DeferedAction + { + public virtual void Execute() + { } + } + + private class DeferedReIndexForContent : DeferedAction + { + private readonly ExamineComponent _examineComponent; + private readonly IContent _content; + private readonly bool? _supportUnpublished; + + public DeferedReIndexForContent(ExamineComponent examineComponent, IContent content, bool? supportUnpublished) + { + _examineComponent = examineComponent; + _content = content; + _supportUnpublished = supportUnpublished; + } + + public override void Execute() + { + Execute(_examineComponent, _content, _supportUnpublished); + } + + public static void Execute(ExamineComponent examineComponent, IContent content, bool? supportUnpublished) + { + var valueSet = UmbracoContentIndexer.GetValueSets(examineComponent._urlSegmentProviders, examineComponent._services.UserService, content); + + ExamineManager.Instance.IndexItems( + valueSet.ToArray(), + examineComponent._indexCollection.Indexes.Values.OfType() + // only for the specified indexers + .Where(x => supportUnpublished.HasValue == false || supportUnpublished.Value == x.SupportUnpublishedContent) + .Where(x => x.EnableDefaultEventHandler)); + } + } + + private class DeferedReIndexForMedia : DeferedAction + { + private readonly ExamineComponent _examineComponent; + private readonly IMedia _media; + private readonly bool _isPublished; + + public DeferedReIndexForMedia(ExamineComponent examineComponent, IMedia media, bool isPublished) + { + _examineComponent = examineComponent; + _media = media; + _isPublished = isPublished; + } + + public override void Execute() + { + Execute(_examineComponent, _media, _isPublished); + } + + public static void Execute(ExamineComponent examineComponent, IMedia media, bool isPublished) + { + var valueSet = UmbracoContentIndexer.GetValueSets(examineComponent._urlSegmentProviders, examineComponent._services.UserService, media); + + ExamineManager.Instance.IndexItems( + valueSet.ToArray(), + examineComponent._indexCollection.Indexes.Values.OfType() + // index this item for all indexers if the media is not trashed, otherwise if the item is trashed + // then only index this for indexers supporting unpublished media + .Where(x => isPublished || (x.SupportUnpublishedContent)) + .Where(x => x.EnableDefaultEventHandler)); + } + } + + private class DeferedReIndexForMember : DeferedAction + { + private readonly ExamineComponent _examineComponent; + private readonly IMember _member; + + public DeferedReIndexForMember(ExamineComponent examineComponent, IMember member) + { + _examineComponent = examineComponent; + _member = member; + } + + public override void Execute() + { + Execute(_examineComponent, _member); + } + + public static void Execute(ExamineComponent examineComponent, IMember member) + { + var valueSet = UmbracoMemberIndexer.GetValueSets(member); + + ExamineManager.Instance.IndexItems( + valueSet.ToArray(), + examineComponent._indexCollection.Indexes.Values.OfType() + //ensure that only the providers are flagged to listen execute + .Where(x => x.EnableDefaultEventHandler)); + } + } + + private class DeferedDeleteIndex : DeferedAction + { + private readonly ExamineComponent _examineComponent; + private readonly int _id; + private readonly bool _keepIfUnpublished; + + public DeferedDeleteIndex(ExamineComponent examineComponent, int id, bool keepIfUnpublished) + { + _examineComponent = examineComponent; + _id = id; + _keepIfUnpublished = keepIfUnpublished; + } + + public override void Execute() + { + Execute(_examineComponent, _id, _keepIfUnpublished); + } + + public static void Execute(ExamineComponent examineComponent, int id, bool keepIfUnpublished) + { + ExamineManager.Instance.DeleteFromIndexes( + id.ToString(CultureInfo.InvariantCulture), + examineComponent._indexCollection.Indexes.Values.OfType() + // if keepIfUnpublished == true then only delete this item from indexes not supporting unpublished content, + // otherwise if keepIfUnpublished == false then remove from all indexes + .Where(x => keepIfUnpublished == false || (x is UmbracoContentIndexer && ((UmbracoContentIndexer)x).SupportUnpublishedContent == false)) + .Where(x => x.EnableDefaultEventHandler)); + } } }