From 52aa02668f1a4f7fb5aafdb91bd7e155e5408dcc Mon Sep 17 00:00:00 2001 From: Shannon Date: Wed, 28 Mar 2018 16:12:43 +1100 Subject: [PATCH] Gets examine working with proper events, tests are working, etc... --- src/Umbraco.Examine/Umbraco.Examine.csproj | 2 +- src/Umbraco.Examine/UmbracoContentIndexer.cs | 36 ++- src/Umbraco.Examine/UmbracoExamineIndexer.cs | 15 +- .../UmbracoExamineMultiIndexSearcher.cs | 112 +++++---- src/Umbraco.Examine/UmbracoExamineSearcher.cs | 31 ++- src/Umbraco.Examine/UmbracoMemberIndexer.cs | 21 +- src/Umbraco.Tests/Umbraco.Tests.csproj | 2 +- .../UmbracoExamine/IndexInitializer.cs | 5 +- src/Umbraco.Tests/packages.config | 2 +- src/Umbraco.Web.UI/Umbraco.Web.UI.csproj | 2 +- .../config/ExamineSettings.Release.config | 8 +- .../config/ExamineSettings.config | 9 +- src/Umbraco.Web.UI/packages.config | 2 +- src/Umbraco.Web/Search/ExamineComponent.cs | 230 +++++++++++++----- ...aseServerRegistrarAndMessengerComponent.cs | 23 +- src/Umbraco.Web/Suspendable.cs | 9 +- src/Umbraco.Web/Umbraco.Web.csproj | 2 +- .../ExamineManagementApiController.cs | 7 +- 18 files changed, 335 insertions(+), 183 deletions(-) diff --git a/src/Umbraco.Examine/Umbraco.Examine.csproj b/src/Umbraco.Examine/Umbraco.Examine.csproj index 2622bf9c3e..813b3709ee 100644 --- a/src/Umbraco.Examine/Umbraco.Examine.csproj +++ b/src/Umbraco.Examine/Umbraco.Examine.csproj @@ -47,7 +47,7 @@ - 1.0.0-beta012 + 1.0.0-beta017 diff --git a/src/Umbraco.Examine/UmbracoContentIndexer.cs b/src/Umbraco.Examine/UmbracoContentIndexer.cs index 918f33d79c..2f177e2466 100644 --- a/src/Umbraco.Examine/UmbracoContentIndexer.cs +++ b/src/Umbraco.Examine/UmbracoContentIndexer.cs @@ -57,6 +57,7 @@ namespace Umbraco.Examine /// /// Create an index at runtime /// + /// /// /// /// @@ -70,6 +71,7 @@ namespace Umbraco.Examine /// /// public UmbracoContentIndexer( + string name, IEnumerable fieldDefinitions, Directory luceneDirectory, Analyzer defaultAnalyzer, @@ -82,7 +84,7 @@ namespace Umbraco.Examine IValueSetValidator validator, UmbracoContentIndexerOptions options, IReadOnlyDictionary> indexValueTypes = null) - : base(fieldDefinitions, luceneDirectory, defaultAnalyzer, profilingLogger, validator, indexValueTypes) + : base(name, fieldDefinitions, luceneDirectory, defaultAnalyzer, profilingLogger, validator, indexValueTypes) { if (validator == null) throw new ArgumentNullException(nameof(validator)); if (options == null) throw new ArgumentNullException(nameof(options)); @@ -342,9 +344,21 @@ namespace Umbraco.Examine {"template", new object[] {c.Template?.Id ?? 0}} }; - foreach (var property in c.Properties.Where(p => p?.GetValue() != null && p.GetValue().ToString().IsNullOrWhiteSpace() == false)) + foreach (var property in c.Properties) { - values.Add(property.Alias, new[] {property.GetValue() }); + //only add the value if its not null or empty (we'll check for string explicitly here too) + var val = property.GetValue(); + switch (val) + { + case null: + continue; + case string strVal when strVal.IsNullOrWhiteSpace() == false: + values.Add(property.Alias, new[] { val }); + break; + default: + values.Add(property.Alias, new[] { val }); + break; + } } var vs = new ValueSet(c.Id.ToInvariantString(), IndexTypes.Content, c.ContentType.Alias, values); @@ -376,9 +390,21 @@ namespace Umbraco.Examine {"creatorName", new object[] {m.GetCreatorProfile(userService).Name}} }; - foreach (var property in m.Properties.Where(p => p?.GetValue() != null && p.GetValue().ToString().IsNullOrWhiteSpace() == false)) + foreach (var property in m.Properties) { - values.Add(property.Alias, new[] { property.GetValue() }); + //only add the value if its not null or empty (we'll check for string explicitly here too) + var val = property.GetValue(); + switch (val) + { + case null: + continue; + case string strVal when strVal.IsNullOrWhiteSpace() == false: + values.Add(property.Alias, new[] { val }); + break; + default: + values.Add(property.Alias, new[] { val }); + break; + } } var vs = new ValueSet(m.Id.ToInvariantString(), IndexTypes.Media, m.ContentType.Alias, values); diff --git a/src/Umbraco.Examine/UmbracoExamineIndexer.cs b/src/Umbraco.Examine/UmbracoExamineIndexer.cs index 3717b4cde2..bec6415a2e 100644 --- a/src/Umbraco.Examine/UmbracoExamineIndexer.cs +++ b/src/Umbraco.Examine/UmbracoExamineIndexer.cs @@ -72,13 +72,14 @@ namespace Umbraco.Examine } protected UmbracoExamineIndexer( + string name, IEnumerable fieldDefinitions, Directory luceneDirectory, Analyzer defaultAnalyzer, ProfilingLogger profilingLogger, IValueSetValidator validator = null, IReadOnlyDictionary> indexValueTypes = null) - : base(fieldDefinitions, luceneDirectory, defaultAnalyzer, validator, indexValueTypes) + : base(name, fieldDefinitions, luceneDirectory, defaultAnalyzer, validator, indexValueTypes) { ProfilingLogger = profilingLogger ?? throw new ArgumentNullException(nameof(profilingLogger)); } @@ -131,10 +132,10 @@ namespace Umbraco.Examine return base.CreateFieldValueTypes(x, indexValueTypesFactory); } - //TODO: Remove this? - [Obsolete("This should not be used, it is used by the configuration based indexes but instead to disable Examine event handlers use the ExamineEvents class instead.")] - [EditorBrowsable(EditorBrowsableState.Never)] - public bool EnableDefaultEventHandler { get; protected set; } + /// + /// When set to true Umbraco will keep the index in sync with Umbraco data automatically + /// + public bool EnableDefaultEventHandler { get; set; } = true; /// /// the supported indexable types @@ -163,9 +164,7 @@ namespace Umbraco.Examine { ProfilingLogger.Logger.Debug(GetType(), "{0} indexer initializing", () => name); - EnableDefaultEventHandler = true; //set to true by default - bool enabled; - if (bool.TryParse(config["enableDefaultEventHandler"], out enabled)) + if (config["enableDefaultEventHandler"] != null && bool.TryParse(config["enableDefaultEventHandler"], out var enabled)) { EnableDefaultEventHandler = enabled; } diff --git a/src/Umbraco.Examine/UmbracoExamineMultiIndexSearcher.cs b/src/Umbraco.Examine/UmbracoExamineMultiIndexSearcher.cs index a5e63a1acd..e8d2f90cd3 100644 --- a/src/Umbraco.Examine/UmbracoExamineMultiIndexSearcher.cs +++ b/src/Umbraco.Examine/UmbracoExamineMultiIndexSearcher.cs @@ -1,61 +1,65 @@ -using System; -using System.Collections.Generic; -using System.Collections.Specialized; -using System.IO; -using System.Linq; -using Examine.LuceneEngine.Providers; -using Lucene.Net.Analysis; -using Umbraco.Examine.Config; -using Directory = Lucene.Net.Store.Directory; +//using System; +//using System.Collections.Generic; +//using System.Collections.Specialized; +//using System.IO; +//using System.Linq; +//using Examine.LuceneEngine.Providers; +//using Lucene.Net.Analysis; +//using Umbraco.Examine.Config; +//using Directory = Lucene.Net.Store.Directory; -namespace Umbraco.Examine -{ - public class UmbracoExamineMultiIndexSearcher : MultiIndexSearcher - { - public UmbracoExamineMultiIndexSearcher() - { - } +//namespace Umbraco.Examine +//{ +// public class UmbracoExamineMultiIndexSearcher : MultiIndexSearcher +// { +// public UmbracoExamineMultiIndexSearcher() +// { +// } - public UmbracoExamineMultiIndexSearcher(IEnumerable indexPath, Analyzer analyzer) : base(indexPath, analyzer) - { - } +// public UmbracoExamineMultiIndexSearcher(string name, IEnumerable indexPath, Analyzer analyzer) +// : base(name, indexPath, analyzer) +// { +// } - public UmbracoExamineMultiIndexSearcher(IEnumerable luceneDirs, Analyzer analyzer) : base(luceneDirs, analyzer) - { - } +// public UmbracoExamineMultiIndexSearcher(string name, IEnumerable luceneDirs, Analyzer analyzer) +// : base(name, luceneDirs, analyzer) +// { +// } - public override void Initialize(string name, NameValueCollection config) - { - base.Initialize(name, config); +// public override void Initialize(string name, NameValueCollection config) +// { +// base.Initialize(name, config); - //need to check if the index set is specified, if it's not, we'll see if we can find one by convension - //if the folder is not null and the index set is null, we'll assume that this has been created at runtime. - if (config["indexSets"] == null) - { - throw new ArgumentNullException("indexSets on MultiIndexSearcher provider has not been set in configuration"); - } +// //need to check if the index set is specified, if it's not, we'll see if we can find one by convension +// //if the folder is not null and the index set is null, we'll assume that this has been created at runtime. +// if (config["indexSets"] == null) +// { +// throw new ArgumentNullException("indexSets on MultiIndexSearcher provider has not been set in configuration"); +// } - var toSearch = new List(); - var sets = IndexSets.Instance.Sets.Cast(); - foreach (var i in config["indexSets"].Split(',')) - { - var s = sets.SingleOrDefault(x => x.SetName == i); - if (s == null) - { - throw new ArgumentException("The index set " + i + " does not exist"); - } - toSearch.Add(s); - } +// var toSearch = new List(); +// var sets = IndexSets.Instance.Sets.Cast(); +// foreach (var i in config["indexSets"].Split(',')) +// { +// var s = sets.SingleOrDefault(x => x.SetName == i); +// if (s == null) +// { +// throw new ArgumentException("The index set " + i + " does not exist"); +// } +// toSearch.Add(s); +// } - //create the searchers - var analyzer = DefaultLuceneAnalyzer; - var searchers = new List(); - //DO NOT PUT THIS INTO LINQ BECAUSE THE SECURITY ACCCESS SHIT WONT WORK - foreach (var s in toSearch) - { - searchers.Add(new LuceneSearcher(s.IndexDirectory, analyzer)); - } - Searchers = searchers; - } - } -} \ No newline at end of file +// //create the searchers +// var analyzer = LuceneAnalyzer; +// var searchers = new List(); + +// var n = 0; +// foreach (var s in toSearch) +// { +// searchers.Add(new LuceneSearcher(Name + "_" + n, s.IndexDirectory, analyzer)); +// n++; +// } +// Searchers = searchers; +// } +// } +//} \ No newline at end of file diff --git a/src/Umbraco.Examine/UmbracoExamineSearcher.cs b/src/Umbraco.Examine/UmbracoExamineSearcher.cs index f6c2bca963..7a06e47925 100644 --- a/src/Umbraco.Examine/UmbracoExamineSearcher.cs +++ b/src/Umbraco.Examine/UmbracoExamineSearcher.cs @@ -5,6 +5,7 @@ using System.Linq; using Umbraco.Core; using Examine.LuceneEngine.Providers; using Lucene.Net.Analysis; +using Lucene.Net.Index; using Umbraco.Core.Composing; using Umbraco.Examine.Config; using Directory = Lucene.Net.Store.Directory; @@ -17,8 +18,6 @@ namespace Umbraco.Examine /// public class UmbracoExamineSearcher : LuceneSearcher { - - private string _name; private readonly bool _configBased = false; /// @@ -26,7 +25,6 @@ namespace Umbraco.Examine /// [EditorBrowsable(EditorBrowsableState.Never)] public UmbracoExamineSearcher() - : base() { _configBased = true; } @@ -34,10 +32,11 @@ namespace Umbraco.Examine /// /// Constructor to allow for creating an indexer at runtime /// + /// /// /// - public UmbracoExamineSearcher(DirectoryInfo indexPath, Analyzer analyzer) - : base(indexPath, analyzer) + public UmbracoExamineSearcher(string name, DirectoryInfo indexPath, Analyzer analyzer) + : base(name, indexPath, analyzer) { _configBased = false; } @@ -45,21 +44,20 @@ namespace Umbraco.Examine /// /// Constructor to allow for creating an indexer at runtime /// + /// /// /// - public UmbracoExamineSearcher(Directory luceneDirectory, Analyzer analyzer) - : base(luceneDirectory, analyzer) + public UmbracoExamineSearcher(string name, Directory luceneDirectory, Analyzer analyzer) + : base(name, luceneDirectory, analyzer) { _configBased = false; } - //TODO: What about the NRT ctor? - - /// - /// we override name because we need to manually set it if !CanInitialize() - /// since we cannot call base.Initialize in that case. - /// - public override string Name => _name; + /// + public UmbracoExamineSearcher(string name, IndexWriter writer, Analyzer analyzer) : base(name, writer, analyzer) + { + _configBased = false; + } /// /// Name of the Lucene.NET index set @@ -73,9 +71,8 @@ namespace Umbraco.Examine /// public override void Initialize(string name, System.Collections.Specialized.NameValueCollection config) { - //ensure name is set - _name = name ?? throw new ArgumentNullException(nameof(name)); - + if (string.IsNullOrWhiteSpace(name)) throw new ArgumentException("Value cannot be null or whitespace.", nameof(name)); + //We need to check if we actually can initialize, if not then don't continue if (CanInitialize() == false) { diff --git a/src/Umbraco.Examine/UmbracoMemberIndexer.cs b/src/Umbraco.Examine/UmbracoMemberIndexer.cs index 3b47be2d9d..b7cbcc19bc 100644 --- a/src/Umbraco.Examine/UmbracoMemberIndexer.cs +++ b/src/Umbraco.Examine/UmbracoMemberIndexer.cs @@ -28,7 +28,6 @@ namespace Umbraco.Examine /// /// Constructor for config/provider based indexes /// - /// [EditorBrowsable(EditorBrowsableState.Never)] public UmbracoMemberIndexer() { @@ -38,6 +37,7 @@ namespace Umbraco.Examine /// /// Constructor to allow for creating an indexer at runtime /// + /// /// /// /// @@ -45,13 +45,14 @@ namespace Umbraco.Examine /// /// public UmbracoMemberIndexer( + string name, IEnumerable fieldDefinitions, Directory luceneDirectory, Analyzer analyzer, ProfilingLogger profilingLogger, IValueSetValidator validator, IMemberService memberService) : - base(fieldDefinitions, luceneDirectory, analyzer, profilingLogger, validator) + base(name, fieldDefinitions, luceneDirectory, analyzer, profilingLogger, validator) { _memberService = memberService ?? throw new ArgumentNullException(nameof(memberService)); } @@ -140,9 +141,21 @@ namespace Umbraco.Examine {"email", new object[] {m.Email}}, }; - foreach (var property in m.Properties.Where(p => p?.GetValue() != null && p.GetValue().ToString().IsNullOrWhiteSpace() == false)) + foreach (var property in m.Properties) { - values.Add(property.Alias, new[] { property.GetValue() }); + //only add the value if its not null or empty (we'll check for string explicitly here too) + var val = property.GetValue(); + switch (val) + { + case null: + continue; + case string strVal when strVal.IsNullOrWhiteSpace() == false: + values.Add(property.Alias, new[] { val }); + break; + default: + values.Add(property.Alias, new[] { val }); + break; + } } var vs = new ValueSet(m.Id.ToInvariantString(), IndexTypes.Content, m.ContentType.Alias, values); diff --git a/src/Umbraco.Tests/Umbraco.Tests.csproj b/src/Umbraco.Tests/Umbraco.Tests.csproj index ff47667daa..5c64d9cdb7 100644 --- a/src/Umbraco.Tests/Umbraco.Tests.csproj +++ b/src/Umbraco.Tests/Umbraco.Tests.csproj @@ -60,7 +60,7 @@ ..\packages\Castle.Core.4.1.1\lib\net45\Castle.Core.dll - ..\packages\Examine.1.0.0-beta012\lib\net45\Examine.dll + ..\packages\Examine.1.0.0-beta017\lib\net45\Examine.dll ..\packages\SharpZipLib.0.86.0\lib\20\ICSharpCode.SharpZipLib.dll diff --git a/src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs b/src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs index 61f3758656..30bcc3766d 100644 --- a/src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs +++ b/src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs @@ -178,6 +178,7 @@ namespace Umbraco.Tests.UmbracoExamine // .Returns(query.Object); var i = new UmbracoContentIndexer( + "testIndexer", Enumerable.Empty(), luceneDir, analyzer, @@ -197,12 +198,12 @@ namespace Umbraco.Tests.UmbracoExamine public static LuceneSearcher GetLuceneSearcher(Directory luceneDir) { - return new LuceneSearcher(luceneDir, new StandardAnalyzer(Version.LUCENE_29)); + return new LuceneSearcher("testSearcher", luceneDir, new StandardAnalyzer(Version.LUCENE_29)); } public static MultiIndexSearcher GetMultiSearcher(Directory pdfDir, Directory simpleDir, Directory conventionDir, Directory cwsDir) { - var i = new MultiIndexSearcher(new[] { pdfDir, simpleDir, conventionDir, cwsDir }, new StandardAnalyzer(Version.LUCENE_29)); + var i = new MultiIndexSearcher("testSearcher", new[] { pdfDir, simpleDir, conventionDir, cwsDir }, new StandardAnalyzer(Version.LUCENE_29)); return i; } diff --git a/src/Umbraco.Tests/packages.config b/src/Umbraco.Tests/packages.config index 23e2104bbf..42248360fb 100644 --- a/src/Umbraco.Tests/packages.config +++ b/src/Umbraco.Tests/packages.config @@ -2,7 +2,7 @@ - + diff --git a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj index cef0b4666c..7da7d6303d 100644 --- a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj +++ b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj @@ -55,7 +55,7 @@ ..\packages\AutoMapper.6.1.1\lib\net45\AutoMapper.dll - ..\packages\Examine.1.0.0-beta012\lib\net45\Examine.dll + ..\packages\Examine.1.0.0-beta017\lib\net45\Examine.dll ..\packages\ImageProcessor.Web.4.8.4\lib\net45\ImageProcessor.Web.dll diff --git a/src/Umbraco.Web.UI/config/ExamineSettings.Release.config b/src/Umbraco.Web.UI/config/ExamineSettings.Release.config index 49287b21c3..ac0f43ce1a 100644 --- a/src/Umbraco.Web.UI/config/ExamineSettings.Release.config +++ b/src/Umbraco.Web.UI/config/ExamineSettings.Release.config @@ -12,12 +12,12 @@ More information and documentation can be found on GitHub: https://github.com/Sh + analyzer="Examine.LuceneEngine.CultureInvariantWhitespaceAnalyzer, Examine"/> + analyzer="Examine.LuceneEngine.CultureInvariantStandardAnalyzer, Examine"/> @@ -25,7 +25,7 @@ More information and documentation can be found on GitHub: https://github.com/Sh - + @@ -33,7 +33,7 @@ More information and documentation can be found on GitHub: https://github.com/Sh + analyzer="Examine.LuceneEngine.CultureInvariantStandardAnalyzer, Examine" /> diff --git a/src/Umbraco.Web.UI/config/ExamineSettings.config b/src/Umbraco.Web.UI/config/ExamineSettings.config index 40586ee854..68208427cf 100644 --- a/src/Umbraco.Web.UI/config/ExamineSettings.config +++ b/src/Umbraco.Web.UI/config/ExamineSettings.config @@ -12,12 +12,12 @@ More information and documentation can be found on GitHub: https://github.com/Sh + analyzer="Examine.LuceneEngine.CultureInvariantWhitespaceAnalyzer, Examine"/> + analyzer="Examine.LuceneEngine.CultureInvariantStandardAnalyzer, Examine"/> @@ -25,7 +25,7 @@ More information and documentation can be found on GitHub: https://github.com/Sh - + @@ -33,8 +33,7 @@ More information and documentation can be found on GitHub: https://github.com/Sh + analyzer="Examine.LuceneEngine.CultureInvariantStandardAnalyzer, Examine" /> diff --git a/src/Umbraco.Web.UI/packages.config b/src/Umbraco.Web.UI/packages.config index 61a17ac0e5..140ae9803d 100644 --- a/src/Umbraco.Web.UI/packages.config +++ b/src/Umbraco.Web.UI/packages.config @@ -4,7 +4,7 @@ - + diff --git a/src/Umbraco.Web/Search/ExamineComponent.cs b/src/Umbraco.Web/Search/ExamineComponent.cs index 69358e59eb..c1f1bd0346 100644 --- a/src/Umbraco.Web/Search/ExamineComponent.cs +++ b/src/Umbraco.Web/Search/ExamineComponent.cs @@ -1,4 +1,5 @@ using System; +using System.Collections; using System.Collections.Generic; using System.Globalization; using System.Linq; @@ -6,7 +7,9 @@ using System.Threading; using System.Xml.Linq; using Examine; using Examine.LuceneEngine; +using Examine.LuceneEngine.Providers; using Lucene.Net.Documents; +using Lucene.Net.Index; using Umbraco.Core; using Umbraco.Core.Cache; using Umbraco.Core.Components; @@ -29,51 +32,68 @@ namespace Umbraco.Web.Search [RuntimeLevel(MinLevel = RuntimeLevel.Run)] public sealed class ExamineComponent : UmbracoComponentBase, IUmbracoCoreComponent { - public void Initialize(IRuntimeState runtime, PropertyEditorCollection propertyEditors, IExamineIndexCollectionAccessor indexCollection, ILogger logger) + //fixme - we are injecting this which is nice, but we still use ExamineManager everywhere, we could instead interface IExamineManager? + private IExamineIndexCollectionAccessor _indexCollection; + private static bool _disableExamineIndexing = false; + private static volatile bool __isConfigured = false; + private static readonly object IsConfiguredLocker = new object(); + + public void Initialize(IRuntimeState runtime, PropertyEditorCollection propertyEditors, IExamineIndexCollectionAccessor indexCollection, ProfilingLogger profilingLogger) { - logger.Info("Starting initialize async background thread."); + _indexCollection = indexCollection; - // make it async in order not to slow down the boot - // fixme - should be a proper background task else we cannot stop it! - var bg = new Thread(() => + //fixme we cannot inject MainDom since it's internal, so thsi is the only way we can get it, alternatively we can add the container to the container and resolve + //directly from the container but that's not nice either + if (!(runtime is RuntimeState coreRuntime)) + throw new NotSupportedException($"Unsupported IRuntimeState implementation {runtime.GetType().FullName}, expecting {typeof(RuntimeState).FullName}."); + + //We want to manage Examine's appdomain shutdown sequence ourselves so first we'll disable Examine's default behavior + //and then we'll use MainDom to control Examine's shutdown + ExamineManager.DisableDefaultHostingEnvironmentRegistration(); + + //we want to tell examine to use a different fs lock instead of the default NativeFSFileLock which could cause problems if the appdomain + //terminates and in some rare cases would only allow unlocking of the file if IIS is forcefully terminated. Instead we'll rely on the simplefslock + //which simply checks the existence of the lock file + DirectoryTracker.DefaultLockFactory = d => { - try + var simpleFsLockFactory = new NoPrefixSimpleFsLockFactory(d); + return simpleFsLockFactory; + }; + + //let's deal with shutting down Examine with MainDom + var examineShutdownRegistered = coreRuntime.MainDom.Register(() => + { + using (profilingLogger.TraceDuration("Examine shutting down")) { - // from WebRuntimeComponent - // rebuilds any empty indexes - RebuildIndexes(true); - } - catch (Exception e) - { - logger.Error("Failed to rebuild empty indexes.", e); - } - - try - { - // from PropertyEditorsComponent - var grid = propertyEditors.OfType().FirstOrDefault(); - if (grid != null) BindGridToExamine(grid, indexCollection); - } - catch (Exception e) - { - logger.Error("Failed to bind grid property editor.", e); + //Due to the way Examine's own IRegisteredObject works, we'll first run it with immediate=false and then true so that + //it's correct subroutines are executed (otherwise we'd have to run this logic manually ourselves) + ExamineManager.Instance.Stop(false); + ExamineManager.Instance.Stop(true); } }); - bg.Start(); - // the rest is the original Examine event handler + if (!examineShutdownRegistered) + { + profilingLogger.Logger.Debug("Examine shutdown not registered, this appdomain is not the MainDom, Examine will be disabled"); - logger.Info("Initialize and bind to business logic events."); - - var registeredProviders = ExamineManager.Instance.IndexProviders - .OfType().Count(x => x.EnableDefaultEventHandler); + //if we could not register the shutdown examine ourselves, it means we are not maindom! in this case all of examine should be disabled! + Suspendable.ExamineEvents.SuspendIndexers(); + _disableExamineIndexing = true; + return; //exit, do not continue + } - logger.Info($"Adding examine event handlers for {registeredProviders} index providers."); + profilingLogger.Logger.Debug("Examine shutdown registered with MainDom"); + + var registeredIndexers = indexCollection.Indexes.Values.OfType().Count(x => x.EnableDefaultEventHandler); + + profilingLogger.Logger.Info($"Adding examine event handlers for {registeredIndexers} index providers."); // don't bind event handlers if we're not suppose to listen - if (registeredProviders == 0) + if (registeredIndexers == 0) return; + BindGridToExamine(profilingLogger.Logger, indexCollection, propertyEditors); + // bind to distributed cache events - this ensures that this logic occurs on ALL servers // that are taking part in a load balanced environment. ContentCacheRefresher.CacheUpdated += ContentCacheRefresherUpdated; @@ -83,26 +103,124 @@ namespace Umbraco.Web.Search // fixme - content type? // events handling removed in ef013f9d3b945d0a48a306ff1afbd49c10c3fff8 // because, could not make sense of it? + + EnsureUnlocked(profilingLogger.Logger, indexCollection); + + RebuildIndexesOnStartup(profilingLogger.Logger); } - private static void RebuildIndexes(bool onlyEmptyIndexes) + /// + /// Called to rebuild empty indexes on startup + /// + /// + private void RebuildIndexesOnStartup(ILogger logger) { - var indexers = ExamineManager.Instance.IndexProviders; + //TODO: need a way to disable rebuilding on startup + + logger.Info("Starting initialize async background thread."); + + // make it async in order not to slow down the boot + // fixme - should be a proper background task else we cannot stop it! + var bg = new Thread(() => + { + try + { + // rebuilds any empty indexes + RebuildIndexes(true, _indexCollection, logger); + } + catch (Exception e) + { + logger.Error("Failed to rebuild empty indexes.", e); + } + }); + bg.Start(); + } + + /// + /// Used to rebuild indexes on startup or cold boot + /// + /// + /// + /// + internal static void RebuildIndexes(bool onlyEmptyIndexes, IExamineIndexCollectionAccessor indexCollection, ILogger logger) + { + //do not attempt to do this if this has been disabled since we are not the main dom. + //this can be called during a cold boot + if (_disableExamineIndexing) return; + + EnsureUnlocked(logger, indexCollection); + if (onlyEmptyIndexes) - indexers = indexers.Where(x => x.Value.IsIndexNew()).ToDictionary(x => x.Key, x => x.Value); - foreach (var indexer in indexers) - indexer.Value.RebuildIndex(); + { + foreach (var indexer in indexCollection.Indexes.Values.Where(x => x.IsIndexNew())) + { + indexer.RebuildIndex(); + } + } + else + { + //do all of them + ExamineManager.Instance.RebuildIndexes(); + } } - private static void BindGridToExamine(GridPropertyEditor grid, IExamineIndexCollectionAccessor indexCollection) + /// + /// Must be called to each index is unlocked before any indexing occurs + /// + /// + /// Indexing rebuilding can occur on a normal boot if the indexes are empty or on a cold boot by the database server messenger. Before + /// either of these happens, we need to configure the indexes. + /// + private static void EnsureUnlocked(ILogger logger, IExamineIndexCollectionAccessor indexCollection) { - var indexes = indexCollection.Indexes; - if (indexes == null) return; - foreach (var i in indexes.Values.OfType()) - i.DocumentWriting += grid.DocumentWriting; + if (_disableExamineIndexing) return; + if (__isConfigured) return; + + lock (IsConfiguredLocker) + { + //double chekc + if (__isConfigured) return; + + __isConfigured = true; + + foreach (var luceneIndexer in indexCollection.Indexes.Values.OfType()) + { + //We now need to disable waiting for indexing for Examine so that the appdomain is shutdown immediately and doesn't wait for pending + //indexing operations. We used to wait for indexing operations to complete but this can cause more problems than that is worth because + //that could end up halting shutdown for a very long time causing overlapping appdomains and many other problems. + luceneIndexer.WaitForIndexQueueOnShutdown = false; + + //we should check if the index is locked ... it shouldn't be! We are using simple fs lock now and we are also ensuring that + //the indexes are not operational unless MainDom is true + var dir = luceneIndexer.GetLuceneDirectory(); + if (IndexWriter.IsLocked(dir)) + { + logger.Info("Forcing index " + luceneIndexer.Name + " to be unlocked since it was left in a locked state"); + IndexWriter.Unlock(dir); + } + } + } } - static void MemberCacheRefresherUpdated(MemberCacheRefresher sender, CacheRefresherEventArgs args) + private static void BindGridToExamine(ILogger logger, IExamineIndexCollectionAccessor indexCollection, IEnumerable propertyEditors) + { + //bind the grid property editors - this is a hack until http://issues.umbraco.org/issue/U4-8437 + try + { + var grid = propertyEditors.OfType().FirstOrDefault(); + if (grid != null) + { + foreach (var i in indexCollection.Indexes.Values.OfType()) + i.DocumentWriting += grid.DocumentWriting; + } + } + catch (Exception e) + { + logger.Error("Failed to bind grid property editor.", e); + } + } + + private void MemberCacheRefresherUpdated(MemberCacheRefresher sender, CacheRefresherEventArgs args) { if (Suspendable.ExamineEvents.CanIndex == false) return; @@ -145,7 +263,7 @@ namespace Umbraco.Web.Search } } - static void MediaCacheRefresherUpdated(MediaCacheRefresher sender, CacheRefresherEventArgs args) + private void MediaCacheRefresherUpdated(MediaCacheRefresher sender, CacheRefresherEventArgs args) { if (Suspendable.ExamineEvents.CanIndex == false) return; @@ -194,7 +312,7 @@ namespace Umbraco.Web.Search } } - static void ContentCacheRefresherUpdated(ContentCacheRefresher sender, CacheRefresherEventArgs args) + private void ContentCacheRefresherUpdated(ContentCacheRefresher sender, CacheRefresherEventArgs args) { if (Suspendable.ExamineEvents.CanIndex == false) return; @@ -270,7 +388,7 @@ namespace Umbraco.Web.Search } } - private static void ReIndexForContent(IContent content, IContent published) + private void ReIndexForContent(IContent content, IContent published) { if (published != null && content.VersionId == published.VersionId) { @@ -292,36 +410,36 @@ namespace Umbraco.Web.Search } } - private static void ReIndexForContent(IContent sender, bool? supportUnpublished = null) + private void ReIndexForContent(IContent sender, bool? supportUnpublished = null) { var valueSet = UmbracoContentIndexer.GetValueSets(Current.UrlSegmentProviders, Current.Services.UserService, sender); ExamineManager.Instance.IndexItems( - valueSet.ToArray(), - ExamineManager.Instance.IndexProviders.OfType() + valueSet.ToArray(), + _indexCollection.Indexes.Values.OfType() // only for the specified indexers .Where(x => supportUnpublished.HasValue == false || supportUnpublished.Value == x.SupportUnpublishedContent) .Where(x => x.EnableDefaultEventHandler)); } - private static void ReIndexForMember(IMember member) + private void ReIndexForMember(IMember member) { var valueSet = UmbracoMemberIndexer.GetValueSets(member); ExamineManager.Instance.IndexItems( valueSet.ToArray(), - ExamineManager.Instance.IndexProviders.OfType() + _indexCollection.Indexes.Values.OfType() //ensure that only the providers are flagged to listen execute .Where(x => x.EnableDefaultEventHandler)); } - private static void ReIndexForMedia(IMedia sender, bool isMediaPublished) + private void ReIndexForMedia(IMedia sender, bool isMediaPublished) { var valueSet = UmbracoContentIndexer.GetValueSets(Current.UrlSegmentProviders, Current.Services.UserService, sender); ExamineManager.Instance.IndexItems( valueSet.ToArray(), - ExamineManager.Instance.IndexProviders.OfType() + _indexCollection.Indexes.Values.OfType() // index this item for all indexers if the media is not trashed, otherwise if the item is trashed // then only index this for indexers supporting unpublished media .Where(x => isMediaPublished || (x.SupportUnpublishedContent)) @@ -336,11 +454,11 @@ namespace Umbraco.Web.Search /// If true, indicates that we will only delete this item from indexes that don't support unpublished content. /// If false it will delete this from all indexes regardless. /// - private static void DeleteIndexForEntity(int entityId, bool keepIfUnpublished) + private void DeleteIndexForEntity(int entityId, bool keepIfUnpublished) { - ExamineManager.Instance.DeleteFromIndex( + ExamineManager.Instance.DeleteFromIndexes( entityId.ToString(CultureInfo.InvariantCulture), - ExamineManager.Instance.IndexProviders.OfType() + _indexCollection.Indexes.Values.OfType() // if keepIfUnpublished == true then only delete this item from indexes not supporting unpublished content, // otherwise if keepIfUnpublished == false then remove from all indexes .Where(x => keepIfUnpublished == false || (x is UmbracoContentIndexer && ((UmbracoContentIndexer)x).SupportUnpublishedContent == false)) diff --git a/src/Umbraco.Web/Strategies/DatabaseServerRegistrarAndMessengerComponent.cs b/src/Umbraco.Web/Strategies/DatabaseServerRegistrarAndMessengerComponent.cs index 75ad252640..e8f2bc95d8 100644 --- a/src/Umbraco.Web/Strategies/DatabaseServerRegistrarAndMessengerComponent.cs +++ b/src/Umbraco.Web/Strategies/DatabaseServerRegistrarAndMessengerComponent.cs @@ -18,7 +18,9 @@ using Umbraco.Web.Routing; using Umbraco.Web.Scheduling; using LightInject; using Umbraco.Core.Exceptions; +using Umbraco.Examine; using Umbraco.Web.Composing; +using Umbraco.Web.Search; namespace Umbraco.Web.Strategies { @@ -45,6 +47,7 @@ namespace Umbraco.Web.Strategies private bool _started; private TouchServerTask _task; private IUmbracoDatabaseFactory _databaseFactory; + private IExamineIndexCollectionAccessor _indexCollection; public override void Compose(Composition composition) { @@ -77,38 +80,28 @@ namespace Umbraco.Web.Strategies // (we really should have a way to reuse RefreshAll... locally) // note: refresh all content & media caches does refresh content types too var svc = Current.PublishedSnapshotService; - bool ignored1, ignored2; svc.Notify(new[] { new DomainCacheRefresher.JsonPayload(0, DomainChangeTypes.RefreshAll) }); - svc.Notify(new[] { new ContentCacheRefresher.JsonPayload(0, TreeChangeTypes.RefreshAll) }, out ignored1, out ignored2); - svc.Notify(new[] { new MediaCacheRefresher.JsonPayload(0, TreeChangeTypes.RefreshAll) }, out ignored1); + svc.Notify(new[] { new ContentCacheRefresher.JsonPayload(0, TreeChangeTypes.RefreshAll) }, out _, out _); + svc.Notify(new[] { new MediaCacheRefresher.JsonPayload(0, TreeChangeTypes.RefreshAll) }, out _); }, //rebuild indexes if the server is not synced // NOTE: This will rebuild ALL indexes including the members, if developers want to target specific // indexes then they can adjust this logic themselves. - () => RebuildIndexes(false) + () => ExamineComponent.RebuildIndexes(false, _indexCollection, _logger) } }); }); } - // fixme - this should move to something else, we should not depend on Examine here! - private static void RebuildIndexes(bool onlyEmptyIndexes) - { - var indexers = (IEnumerable>) ExamineManager.Instance.IndexProviders; - if (onlyEmptyIndexes) - indexers = indexers.Where(x => x.Value.IsIndexNew()); - foreach (var indexer in indexers) - indexer.Value.RebuildIndex(); - } - - public void Initialize(IRuntimeState runtime, IServerRegistrar serverRegistrar, IServerRegistrationService registrationService, IUmbracoDatabaseFactory databaseFactory, ILogger logger) + public void Initialize(IRuntimeState runtime, IServerRegistrar serverRegistrar, IServerRegistrationService registrationService, IUmbracoDatabaseFactory databaseFactory, ILogger logger, IExamineIndexCollectionAccessor indexCollection) { if (UmbracoConfig.For.UmbracoSettings().DistributedCall.Enabled) return; _registrar = serverRegistrar as DatabaseServerRegistrar; if (_registrar == null) throw new Exception("panic: registar."); + _indexCollection = indexCollection; _runtime = runtime; _databaseFactory = databaseFactory; _logger = logger; diff --git a/src/Umbraco.Web/Suspendable.cs b/src/Umbraco.Web/Suspendable.cs index 877c9797e5..c59e090664 100644 --- a/src/Umbraco.Web/Suspendable.cs +++ b/src/Umbraco.Web/Suspendable.cs @@ -2,7 +2,9 @@ using Examine; using Examine.Providers; using Umbraco.Core.Composing; +using Umbraco.Examine; using Umbraco.Web.Cache; +using Umbraco.Web.Search; namespace Umbraco.Web { @@ -77,10 +79,9 @@ namespace Umbraco.Web _tried = false; // fixme - could we fork this on a background thread? - foreach (BaseIndexProvider indexer in ExamineManager.Instance.IndexProviderCollection) - { - indexer.RebuildIndex(); - } + //TODO: when resuming do we always want a full rebuild of all indexes? + //We are calling into the ExamineComponent since this rebuilds only when MainDom is active + ExamineComponent.RebuildIndexes(false, new ExamineIndexCollectionAccessor(), Current.Logger); } } diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index 7844b8b7fb..220aaf709c 100644 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -66,7 +66,7 @@ - + diff --git a/src/Umbraco.Web/WebServices/ExamineManagementApiController.cs b/src/Umbraco.Web/WebServices/ExamineManagementApiController.cs index 58d854524d..8c7c97c478 100644 --- a/src/Umbraco.Web/WebServices/ExamineManagementApiController.cs +++ b/src/Umbraco.Web/WebServices/ExamineManagementApiController.cs @@ -23,9 +23,10 @@ namespace Umbraco.Web.WebServices [ValidateAngularAntiForgeryToken] public class ExamineManagementApiController : UmbracoAuthorizedApiController { - public ExamineManagementApiController(ExamineManager examineManager, ILogger logger, IRuntimeCacheProvider runtimeCacheProvider) + public ExamineManagementApiController(ILogger logger, IRuntimeCacheProvider runtimeCacheProvider) { - _examineManager = examineManager; + //fixme can we inject this? we'll need an IExamineManager + _examineManager = ExamineManager.Instance; _logger = logger; _runtimeCacheProvider = runtimeCacheProvider; } @@ -114,7 +115,7 @@ namespace Umbraco.Web.WebServices .OrderBy(x => x.Name); foreach (var p in props) { - indexerModel.ProviderProperties.Add(p.Name, p.GetValue(searcher, null).ToString()); + indexerModel.ProviderProperties.Add(p.Name, p.GetValue(searcher, null)?.ToString()); } return indexerModel; }).OrderBy(x =>