From fce5b29b1c20a8a53f360e96d4710bdad7d083e6 Mon Sep 17 00:00:00 2001 From: Shannon Date: Mon, 3 Dec 2018 23:15:18 +1100 Subject: [PATCH] index rebuilding updates --- src/Umbraco.Examine/IIndexPopulator.cs | 2 + src/Umbraco.Examine/IndexRebuilder.cs | 33 +++++++++--- src/Umbraco.Examine/UmbracoContentIndexer.cs | 52 +++++++++++-------- .../Editors/ExamineManagementController.cs | 20 +++---- 4 files changed, 70 insertions(+), 37 deletions(-) diff --git a/src/Umbraco.Examine/IIndexPopulator.cs b/src/Umbraco.Examine/IIndexPopulator.cs index 9e73de7260..aeb3514578 100644 --- a/src/Umbraco.Examine/IIndexPopulator.cs +++ b/src/Umbraco.Examine/IIndexPopulator.cs @@ -3,6 +3,8 @@ using Examine; namespace Umbraco.Examine { + + public interface IIndexPopulator { bool IsRegistered(string indexName); diff --git a/src/Umbraco.Examine/IndexRebuilder.cs b/src/Umbraco.Examine/IndexRebuilder.cs index 525b57cc7d..39b29c5a2b 100644 --- a/src/Umbraco.Examine/IndexRebuilder.cs +++ b/src/Umbraco.Examine/IndexRebuilder.cs @@ -1,5 +1,7 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Linq; +using System.Threading.Tasks; using Examine; namespace Umbraco.Examine @@ -18,19 +20,36 @@ namespace Umbraco.Examine ExamineManager = examineManager; } + public bool CanRebuild(string indexName) + { + return _populators.Any(x => x.IsRegistered(indexName)); + } + + public void RebuildIndex(string indexName) + { + if (!ExamineManager.TryGetIndex(indexName, out var index)) + throw new InvalidOperationException($"No index found with name {indexName}"); + index.CreateIndex(); // clear the index + foreach (var populator in _populators) + { + populator.Populate(index); + } + } + public void RebuildIndexes(bool onlyEmptyIndexes) { var indexes = (onlyEmptyIndexes ? ExamineManager.Indexes.Where(x => !x.IndexExists()) : ExamineManager.Indexes).ToArray(); - foreach(var index in indexes) - index.CreateIndex(); // clear the index - - foreach (var populator in _populators) + foreach (var index in indexes) { - populator.Populate(indexes); - } + index.CreateIndex(); // clear the index + } + + //run the populators in parallel against all indexes + Parallel.ForEach(_populators, populator => populator.Populate(indexes)); } + } } diff --git a/src/Umbraco.Examine/UmbracoContentIndexer.cs b/src/Umbraco.Examine/UmbracoContentIndexer.cs index f37429a5c7..feea0a4efd 100644 --- a/src/Umbraco.Examine/UmbracoContentIndexer.cs +++ b/src/Umbraco.Examine/UmbracoContentIndexer.cs @@ -136,33 +136,43 @@ namespace Umbraco.Examine /// protected override void PerformIndexItems(IEnumerable values, Action onComplete) { - var valid = true; - - // ReSharper disable once PossibleMultipleEnumeration - foreach (var v in values) + //We don't want to re-enumerate this list, but we need to split it into 2x enumerables: invalid and valid items. + // The Invalid items will be deleted, these are items that have invalid paths (i.e. moved to the recycle bin, etc...) + // Then we'll index the Value group all together. + // We return 0 or 1 here so we can order the results and do the invalid first and then the valid. + var invalidOrValid = values.GroupBy(v => { - if (v.Values.TryGetValue("path", out var paths) && paths.Count > 0 && paths[0] != null) - { - //we know this is an IContentValueSetValidator - var validator = (IContentValueSetValidator) ValueSetValidator; - var path = paths[0].ToString(); - - if (!validator.ValidatePath(path, v.Category) + if (!v.Values.TryGetValue("path", out var paths) || paths.Count <= 0 || paths[0] == null) + return 0; + + //we know this is an IContentValueSetValidator + var validator = (IContentValueSetValidator)ValueSetValidator; + var path = paths[0].ToString(); + + return (!validator.ValidatePath(path, v.Category) || !validator.ValidateRecycleBin(path, v.Category) || !validator.ValidateProtectedContent(path, v.Category)) + ? 0 + : 1; + }); + + foreach (var group in invalidOrValid.OrderBy(x => x.Key)) + { + if (group.Key == 0) + { + //these are the invalid items so we'll delete them + //since the path is not valid we need to delete this item in case it exists in the index already and has now + //been moved to an invalid parent. + foreach (var i in group) { - //since the path is not valid we need to delete this item in case it exists in the index already and has now - //been moved to an invalid parent. - PerformDeleteFromIndex(v.Id, x => { /*noop*/ }); - valid = false; + PerformDeleteFromIndex(i.Id, x => { /*noop*/ }); } } - } - - if (valid) - { - // ReSharper disable once PossibleMultipleEnumeration - base.PerformIndexItems(values, onComplete); + else + { + //these are the valid ones, so just index them all at once + base.PerformIndexItems(group, onComplete); + } } } diff --git a/src/Umbraco.Web/Editors/ExamineManagementController.cs b/src/Umbraco.Web/Editors/ExamineManagementController.cs index c166162930..ab60f64e49 100644 --- a/src/Umbraco.Web/Editors/ExamineManagementController.cs +++ b/src/Umbraco.Web/Editors/ExamineManagementController.cs @@ -25,16 +25,17 @@ namespace Umbraco.Web.Editors private readonly IExamineManager _examineManager; private readonly ILogger _logger; private readonly IRuntimeCacheProvider _runtimeCacheProvider; - private readonly IEnumerable _populators; + private readonly IndexRebuilder _indexRebuilder; + public ExamineManagementController(IExamineManager examineManager, ILogger logger, IRuntimeCacheProvider runtimeCacheProvider, - IEnumerable populators) + IndexRebuilder indexRebuilder) { _examineManager = examineManager; _logger = logger; _runtimeCacheProvider = runtimeCacheProvider; - _populators = populators; + _indexRebuilder = indexRebuilder; } /// @@ -151,9 +152,11 @@ namespace Umbraco.Web.Editors //clear and replace index.CreateIndex(); - //populate it - foreach (var populator in _populators.Where(x => x.IsRegistered(indexName))) - populator.Populate(index); + _indexRebuilder.RebuildIndex(indexName); + + ////populate it + //foreach (var populator in _populators.Where(x => x.IsRegistered(indexName))) + // populator.Populate(index); return Request.CreateResponse(HttpStatusCode.OK); } @@ -195,7 +198,7 @@ namespace Umbraco.Web.Editors Name = indexName, HealthStatus = isHealth.Success ? (isHealth.Result ?? "Healthy") : (isHealth.Result ?? "Unhealthy"), ProviderProperties = properties, - CanRebuild = _populators.Any(x => x.IsRegistered(indexName)) + CanRebuild = _indexRebuilder.CanRebuild(indexName) }; @@ -239,7 +242,7 @@ namespace Umbraco.Web.Editors private HttpResponseMessage ValidatePopulator(string indexName) { - if (_populators.Any(x => x.IsRegistered(indexName))) + if (_indexRebuilder.CanRebuild(indexName)) return Request.CreateResponse(HttpStatusCode.OK); var response = Request.CreateResponse(HttpStatusCode.BadRequest); @@ -264,7 +267,6 @@ namespace Umbraco.Web.Editors return response; } - //static listener so it's not GC'd private void Indexer_IndexOperationComplete(object sender, EventArgs e) { var indexer = (LuceneIndex) sender;