diff --git a/NuGet.Config b/NuGet.Config index 92eaf83792..f8dc68d26a 100644 --- a/NuGet.Config +++ b/NuGet.Config @@ -9,7 +9,6 @@ - diff --git a/src/Umbraco.Core/Composing/TypeLoader.cs b/src/Umbraco.Core/Composing/TypeLoader.cs index c53e6ee4d2..d248be19b7 100644 --- a/src/Umbraco.Core/Composing/TypeLoader.cs +++ b/src/Umbraco.Core/Composing/TypeLoader.cs @@ -316,7 +316,7 @@ namespace Umbraco.Cms.Core.Composing /// private string GetFileBasePath() { - var fileBasePath = Path.Combine(_localTempPath.FullName, "TypesCache", "umbraco-types." + NetworkHelper.FileSafeMachineName); + var fileBasePath = Path.Combine(_localTempPath.FullName, "TypesCache", "umbraco-types." + EnvironmentHelper.FileSafeMachineName); // ensure that the folder exists var directory = Path.GetDirectoryName(fileBasePath); diff --git a/src/Umbraco.Core/Constants-Indexes.cs b/src/Umbraco.Core/Constants-Indexes.cs index 8384faa08d..fcf2e7ed14 100644 --- a/src/Umbraco.Core/Constants-Indexes.cs +++ b/src/Umbraco.Core/Constants-Indexes.cs @@ -1,16 +1,12 @@ -namespace Umbraco.Cms.Core +namespace Umbraco.Cms.Core { public static partial class Constants { public static class UmbracoIndexes { - public const string InternalIndexName = InternalIndexPath + "Index"; - public const string ExternalIndexName = ExternalIndexPath + "Index"; - public const string MembersIndexName = MembersIndexPath + "Index"; - - public const string InternalIndexPath = "Internal"; - public const string ExternalIndexPath = "External"; - public const string MembersIndexPath = "Members"; + public const string InternalIndexName = "InternalIndex"; + public const string ExternalIndexName = "ExternalIndex"; + public const string MembersIndexName = "MembersIndex"; } } } diff --git a/src/Umbraco.Core/DependencyInjection/UmbracoBuilder.cs b/src/Umbraco.Core/DependencyInjection/UmbracoBuilder.cs index d0fe41dfc3..316dba75c1 100644 --- a/src/Umbraco.Core/DependencyInjection/UmbracoBuilder.cs +++ b/src/Umbraco.Core/DependencyInjection/UmbracoBuilder.cs @@ -225,6 +225,8 @@ namespace Umbraco.Cms.Core.DependencyInjection Services .AddNotificationHandler() .AddNotificationHandler(); + + Services.AddSingleton(); } } } diff --git a/src/Umbraco.Core/EnvironmentHelper.cs b/src/Umbraco.Core/EnvironmentHelper.cs new file mode 100644 index 0000000000..097ffc9629 --- /dev/null +++ b/src/Umbraco.Core/EnvironmentHelper.cs @@ -0,0 +1,17 @@ +using System; +using Umbraco.Extensions; + +namespace Umbraco.Cms.Core +{ + /// + /// Currently just used to get the machine name for use with file names + /// + internal class EnvironmentHelper + { + /// + /// Returns the machine name that is safe to use in file paths. + /// + public static string FileSafeMachineName => Environment.MachineName.ReplaceNonAlphanumericChars('-'); + + } +} diff --git a/src/Umbraco.Core/Extensions/StringExtensions.cs b/src/Umbraco.Core/Extensions/StringExtensions.cs index 3b42426cd6..e815f219ca 100644 --- a/src/Umbraco.Core/Extensions/StringExtensions.cs +++ b/src/Umbraco.Core/Extensions/StringExtensions.cs @@ -616,12 +616,7 @@ namespace Umbraco.Extensions /// /// Refers to itself /// The hashed string - public static string GenerateHash(this string str) - { - return CryptoConfig.AllowOnlyFipsAlgorithms - ? str.ToSHA1() - : str.ToMd5(); - } + public static string GenerateHash(this string str) => str.ToSHA1(); /// /// Generate a hash of a string based on the specified hash algorithm. @@ -632,30 +627,14 @@ namespace Umbraco.Extensions /// The hashed string. /// public static string GenerateHash(this string str) - where T : HashAlgorithm - { - return str.GenerateHash(typeof(T).FullName); - } - - /// - /// Converts the string to MD5 - /// - /// Refers to itself - /// The MD5 hashed string - public static string ToMd5(this string stringToConvert) - { - return stringToConvert.GenerateHash("MD5"); - } + where T : HashAlgorithm => str.GenerateHash(typeof(T).FullName); /// /// Converts the string to SHA1 /// /// refers to itself /// The SHA1 hashed string - public static string ToSHA1(this string stringToConvert) - { - return stringToConvert.GenerateHash("SHA1"); - } + public static string ToSHA1(this string stringToConvert) => stringToConvert.GenerateHash("SHA1"); /// Generate a hash of a string based on the hashType passed in /// diff --git a/src/Umbraco.Core/Hosting/IHostingEnvironment.cs b/src/Umbraco.Core/Hosting/IHostingEnvironment.cs index 311d7559d0..dedf809230 100644 --- a/src/Umbraco.Core/Hosting/IHostingEnvironment.cs +++ b/src/Umbraco.Core/Hosting/IHostingEnvironment.cs @@ -6,6 +6,21 @@ namespace Umbraco.Cms.Core.Hosting { string SiteName { get; } + /// + /// The unique application ID for this Umbraco website. + /// + /// + /// + /// The returned value will be the same consistent value for an Umbraco website on a specific server and will the same + /// between restarts of that Umbraco website/application on that specific server. + /// + /// + /// The value of this does not necesarily distinguish between unique workers/servers for this Umbraco application. + /// Usage of this must take into account that the same may be returned for the same + /// Umbraco website hosted on different servers. Similarly the usage of this must take into account that a different + /// may be returned for the same Umbraco website hosted on different servers. + /// + /// string ApplicationId { get; } /// diff --git a/src/Umbraco.Core/NetworkHelper.cs b/src/Umbraco.Core/NetworkHelper.cs deleted file mode 100644 index 8e1bfaea92..0000000000 --- a/src/Umbraco.Core/NetworkHelper.cs +++ /dev/null @@ -1,48 +0,0 @@ -using System; -using Umbraco.Extensions; - -namespace Umbraco.Cms.Core -{ - /// - /// Currently just used to get the machine name in med trust and to format a machine name for use with file names - /// - public class NetworkHelper - { - /// - /// Returns the machine name that is safe to use in file paths. - /// - public static string FileSafeMachineName - { - get { return MachineName.ReplaceNonAlphanumericChars('-'); } - } - - /// - /// Returns the current machine name - /// - /// - /// Tries to resolve the machine name, if it cannot it uses the config section. - /// - public static string MachineName - { - get - { - try - { - return Environment.MachineName; - } - catch - { - try - { - return System.Net.Dns.GetHostName(); - } - catch - { - //if we get here it means we cannot access the machine name - throw new ApplicationException("Cannot resolve the current machine name either by Environment.MachineName or by Dns.GetHostname()"); - } - } - } - } - } -} diff --git a/src/Umbraco.Core/Notifications/UmbracoApplicationStartingNotification.cs b/src/Umbraco.Core/Notifications/UmbracoApplicationStartingNotification.cs index 9d324a4ca5..4cbf0a55c6 100644 --- a/src/Umbraco.Core/Notifications/UmbracoApplicationStartingNotification.cs +++ b/src/Umbraco.Core/Notifications/UmbracoApplicationStartingNotification.cs @@ -3,7 +3,10 @@ namespace Umbraco.Cms.Core.Notifications { - + /// + /// Notification that occurs at the very end of the Umbraco boot + /// process and after all initialize. + /// public class UmbracoApplicationStartingNotification : INotification { /// diff --git a/src/Umbraco.Core/Notifications/UmbracoRequestEndNotification.cs b/src/Umbraco.Core/Notifications/UmbracoRequestEndNotification.cs index 84aad8a3b3..27fb6ff09d 100644 --- a/src/Umbraco.Core/Notifications/UmbracoRequestEndNotification.cs +++ b/src/Umbraco.Core/Notifications/UmbracoRequestEndNotification.cs @@ -1,4 +1,4 @@ -// Copyright (c) Umbraco. +// Copyright (c) Umbraco. // See LICENSE for more details. using Umbraco.Cms.Core.Web; diff --git a/src/Umbraco.Core/Runtime/MainDom.cs b/src/Umbraco.Core/Runtime/MainDom.cs index adab97ed6d..ec4e56df1b 100644 --- a/src/Umbraco.Core/Runtime/MainDom.cs +++ b/src/Umbraco.Core/Runtime/MainDom.cs @@ -31,7 +31,7 @@ namespace Umbraco.Cms.Core.Runtime private bool _isInitialized; // indicates whether... - private bool _isMainDom; // we are the main domain + private bool? _isMainDom; // we are the main domain private volatile bool _signaled; // we have been signaled // actions to run before releasing the main domain @@ -64,7 +64,7 @@ namespace Umbraco.Cms.Core.Runtime { hostingEnvironment.RegisterObject(this); return Acquire(); - }); + }).Value; } /// @@ -85,7 +85,11 @@ namespace Umbraco.Cms.Core.Runtime return false; } - if (_isMainDom == false) + if (_isMainDom.HasValue == false) + { + throw new InvalidOperationException("Register called when MainDom has not been acquired"); + } + else if (_isMainDom == false) { _logger.LogWarning("Register called when MainDom has not been acquired"); return false; @@ -215,7 +219,17 @@ namespace Umbraco.Cms.Core.Runtime /// /// Acquire must be called first else this will always return false /// - public bool IsMainDom => _isMainDom; + public bool IsMainDom + { + get + { + if (!_isMainDom.HasValue) + { + throw new InvalidOperationException("MainDom has not been acquired yet"); + } + return _isMainDom.Value; + } + } // IRegisteredObject void IRegisteredObject.Stop(bool immediate) diff --git a/src/Umbraco.Core/Services/CacheInstructionServiceProcessInstructionsResult.cs b/src/Umbraco.Core/Services/CacheInstructionServiceProcessInstructionsResult.cs deleted file mode 100644 index 84116584a2..0000000000 --- a/src/Umbraco.Core/Services/CacheInstructionServiceProcessInstructionsResult.cs +++ /dev/null @@ -1,26 +0,0 @@ -using System; - -namespace Umbraco.Cms.Core.Services -{ - /// - /// Defines a result object for the operation. - /// - public class CacheInstructionServiceProcessInstructionsResult - { - private CacheInstructionServiceProcessInstructionsResult() - { - } - - public int NumberOfInstructionsProcessed { get; private set; } - - public int LastId { get; private set; } - - public bool InstructionsWerePruned { get; private set; } - - public static CacheInstructionServiceProcessInstructionsResult AsCompleted(int numberOfInstructionsProcessed, int lastId) => - new CacheInstructionServiceProcessInstructionsResult { NumberOfInstructionsProcessed = numberOfInstructionsProcessed, LastId = lastId }; - - public static CacheInstructionServiceProcessInstructionsResult AsCompletedAndPruned(int numberOfInstructionsProcessed, int lastId) => - new CacheInstructionServiceProcessInstructionsResult { NumberOfInstructionsProcessed = numberOfInstructionsProcessed, LastId = lastId, InstructionsWerePruned = true }; - }; -} diff --git a/src/Umbraco.Core/Services/ICacheInstructionService.cs b/src/Umbraco.Core/Services/ICacheInstructionService.cs index faf05f2237..c884b8bed8 100644 --- a/src/Umbraco.Core/Services/ICacheInstructionService.cs +++ b/src/Umbraco.Core/Services/ICacheInstructionService.cs @@ -1,5 +1,7 @@ using System; using System.Collections.Generic; +using System.Threading; +using Umbraco.Cms.Core.Cache; using Umbraco.Cms.Core.Sync; namespace Umbraco.Cms.Core.Services @@ -40,6 +42,12 @@ namespace Umbraco.Cms.Core.Services /// Local identity of the executing AppDomain. /// Date of last prune operation. /// Id of the latest processed instruction - CacheInstructionServiceProcessInstructionsResult ProcessInstructions(bool released, string localIdentity, DateTime lastPruned, int lastId); + ProcessInstructionsResult ProcessInstructions( + CacheRefresherCollection cacheRefreshers, + ServerRole serverRole, + CancellationToken cancellationToken, + string localIdentity, + DateTime lastPruned, + int lastId); } } diff --git a/src/Umbraco.Core/Services/ProcessInstructionsResult.cs b/src/Umbraco.Core/Services/ProcessInstructionsResult.cs new file mode 100644 index 0000000000..9a368dab7e --- /dev/null +++ b/src/Umbraco.Core/Services/ProcessInstructionsResult.cs @@ -0,0 +1,26 @@ +using System; + +namespace Umbraco.Cms.Core.Services +{ + /// + /// Defines a result object for the operation. + /// + public class ProcessInstructionsResult + { + private ProcessInstructionsResult() + { + } + + public int NumberOfInstructionsProcessed { get; private set; } + + public int LastId { get; private set; } + + public bool InstructionsWerePruned { get; private set; } + + public static ProcessInstructionsResult AsCompleted(int numberOfInstructionsProcessed, int lastId) => + new ProcessInstructionsResult { NumberOfInstructionsProcessed = numberOfInstructionsProcessed, LastId = lastId }; + + public static ProcessInstructionsResult AsCompletedAndPruned(int numberOfInstructionsProcessed, int lastId) => + new ProcessInstructionsResult { NumberOfInstructionsProcessed = numberOfInstructionsProcessed, LastId = lastId, InstructionsWerePruned = true }; + }; +} diff --git a/src/Umbraco.Core/Sync/DatabaseServerMessengerCallbacks.cs b/src/Umbraco.Core/Sync/DatabaseServerMessengerCallbacks.cs deleted file mode 100644 index 2107bbde20..0000000000 --- a/src/Umbraco.Core/Sync/DatabaseServerMessengerCallbacks.cs +++ /dev/null @@ -1,20 +0,0 @@ -using System; -using System.Collections.Generic; - -namespace Umbraco.Cms.Core.Sync -{ - /// - /// Holds a list of callbacks associated with implementations of . - /// - public class DatabaseServerMessengerCallbacks - { - /// - /// A list of callbacks that will be invoked if the lastsynced.txt file does not exist. - /// - /// - /// These callbacks will typically be for e.g. rebuilding the xml cache file, or examine indexes, based on - /// the data in the database to get this particular server node up to date. - /// - public IEnumerable InitializingCallbacks { get; set; } - } -} diff --git a/src/Umbraco.Core/Sync/IServerAddress.cs b/src/Umbraco.Core/Sync/IServerAddress.cs index c9333f33b8..a177454886 100644 --- a/src/Umbraco.Core/Sync/IServerAddress.cs +++ b/src/Umbraco.Core/Sync/IServerAddress.cs @@ -1,4 +1,4 @@ -namespace Umbraco.Cms.Core.Sync +namespace Umbraco.Cms.Core.Sync { /// /// Provides the address of a server. diff --git a/src/Umbraco.Core/Sync/ISyncBootStateAccessor.cs b/src/Umbraco.Core/Sync/ISyncBootStateAccessor.cs new file mode 100644 index 0000000000..4ced4acf83 --- /dev/null +++ b/src/Umbraco.Core/Sync/ISyncBootStateAccessor.cs @@ -0,0 +1,14 @@ +namespace Umbraco.Cms.Core.Sync +{ + /// + /// Retrieve the for the application during startup + /// + public interface ISyncBootStateAccessor + { + /// + /// Get the + /// + /// + SyncBootState GetSyncBootState(); + } +} diff --git a/src/Umbraco.Core/Sync/NonRuntimeLevelBootStateAccessor.cs b/src/Umbraco.Core/Sync/NonRuntimeLevelBootStateAccessor.cs new file mode 100644 index 0000000000..0dcfa471db --- /dev/null +++ b/src/Umbraco.Core/Sync/NonRuntimeLevelBootStateAccessor.cs @@ -0,0 +1,10 @@ +namespace Umbraco.Cms.Core.Sync +{ + /// + /// Boot state implementation for when umbraco is not in the run state + /// + public sealed class NonRuntimeLevelBootStateAccessor : ISyncBootStateAccessor + { + public SyncBootState GetSyncBootState() => SyncBootState.Unknown; + } +} diff --git a/src/Umbraco.Core/Sync/SyncBootState.cs b/src/Umbraco.Core/Sync/SyncBootState.cs new file mode 100644 index 0000000000..e07898486f --- /dev/null +++ b/src/Umbraco.Core/Sync/SyncBootState.cs @@ -0,0 +1,20 @@ +namespace Umbraco.Cms.Core.Sync +{ + public enum SyncBootState + { + /// + /// Unknown state. Treat as WarmBoot + /// + Unknown = 0, + + /// + /// Cold boot. No Sync state + /// + ColdBoot = 1, + + /// + /// Warm boot. Sync state present + /// + WarmBoot = 2 + } +} diff --git a/src/Umbraco.Examine.Lucene/BackOfficeExamineSearcher.cs b/src/Umbraco.Examine.Lucene/BackOfficeExamineSearcher.cs index d05bb8f07f..0a99c4d6ef 100644 --- a/src/Umbraco.Examine.Lucene/BackOfficeExamineSearcher.cs +++ b/src/Umbraco.Examine.Lucene/BackOfficeExamineSearcher.cs @@ -1,4 +1,4 @@ -// Copyright (c) Umbraco. +// Copyright (c) Umbraco. // See LICENSE for more details. using System; @@ -7,6 +7,8 @@ using System.Linq; using System.Text; using System.Text.RegularExpressions; using Examine; +using Examine.Search; +using Lucene.Net.QueryParsers.Classic; using Umbraco.Cms.Core; using Umbraco.Cms.Core.Cache; using Umbraco.Cms.Core.Mapping; @@ -103,17 +105,16 @@ namespace Umbraco.Cms.Infrastructure.Examine if (!_examineManager.TryGetIndex(indexName, out var index)) throw new InvalidOperationException("No index found by name " + indexName); - var internalSearcher = index.GetSearcher(); - if (!BuildQuery(sb, query, searchFrom, fields, type)) { totalFound = 0; return Enumerable.Empty(); } - var result = internalSearcher.CreateQuery().NativeQuery(sb.ToString()) - //only return the number of items specified to read up to the amount of records to fill from 0 -> the number of items on the page requested - .Execute(Convert.ToInt32(pageSize * (pageIndex + 1))); + var result = index.Searcher + .CreateQuery() + .NativeQuery(sb.ToString()) + .Execute(QueryOptions.SkipTake(Convert.ToInt32(pageSize * pageIndex), pageSize)); totalFound = result.TotalItemCount; @@ -143,7 +144,7 @@ namespace Umbraco.Cms.Infrastructure.Examine //strip quotes, escape string, the replace again query = query.Trim(Constants.CharArrays.DoubleQuoteSingleQuote); - query = Lucene.Net.QueryParsers.QueryParser.Escape(query); + query = QueryParser.Escape(query); //nothing to search if (searchFrom.IsNullOrWhiteSpace() && query.IsNullOrWhiteSpace()) @@ -186,7 +187,7 @@ namespace Umbraco.Cms.Infrastructure.Examine //update the query with the query term if (trimmed.IsNullOrWhiteSpace() == false) { - query = Lucene.Net.QueryParsers.QueryParser.Escape(query); + query = QueryParser.Escape(query); var querywords = query.Split(Constants.CharArrays.Space, StringSplitOptions.RemoveEmptyEntries); @@ -355,6 +356,8 @@ namespace Umbraco.Cms.Infrastructure.Examine sb.Append("\\,*"); } + // TODO: When/Where is this used? + /// /// Returns a collection of entities for media based on search results /// @@ -389,6 +392,8 @@ namespace Umbraco.Cms.Infrastructure.Examine } } + // TODO: When/Where is this used? + /// /// Returns a collection of entities for media based on search results /// @@ -397,6 +402,8 @@ namespace Umbraco.Cms.Infrastructure.Examine private IEnumerable MediaFromSearchResults(IEnumerable results) => _umbracoMapper.Map>(results); + // TODO: When/Where is this used? + /// /// Returns a collection of entities for content based on search results /// diff --git a/src/Umbraco.Examine.Lucene/ConfigurationEnabledDirectoryFactory.cs b/src/Umbraco.Examine.Lucene/ConfigurationEnabledDirectoryFactory.cs new file mode 100644 index 0000000000..e6e2ff9a82 --- /dev/null +++ b/src/Umbraco.Examine.Lucene/ConfigurationEnabledDirectoryFactory.cs @@ -0,0 +1,83 @@ +// Copyright (c) Umbraco. +// See LICENSE for more details. + +using System; +using System.IO; +using Examine; +using Examine.Lucene.Directories; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using Umbraco.Cms.Core.Composing; +using Umbraco.Cms.Core.Configuration.Models; +using Umbraco.Cms.Core.Hosting; +using Umbraco.Extensions; +using Constants = Umbraco.Cms.Core.Constants; + +namespace Umbraco.Cms.Infrastructure.Examine +{ + public class ConfigurationEnabledDirectoryFactory : IDirectoryFactory + { + private readonly IServiceProvider _services; + private readonly ITypeFinder _typeFinder; + private readonly IHostingEnvironment _hostingEnvironment; + private readonly ILockFactory _lockFactory; + private readonly IApplicationRoot _applicationRoot; + private readonly IndexCreatorSettings _settings; + + public ConfigurationEnabledDirectoryFactory( + IServiceProvider services, + ITypeFinder typeFinder, + IHostingEnvironment hostingEnvironment, + ILockFactory lockFactory, + IOptions settings, + IApplicationRoot applicationRoot) + { + _services = services; + _typeFinder = typeFinder; + _hostingEnvironment = hostingEnvironment; + _lockFactory = lockFactory; + _applicationRoot = applicationRoot; + _settings = settings.Value; + } + + public Lucene.Net.Store.Directory CreateDirectory(string indexName) => CreateFileSystemLuceneDirectory(indexName); + + /// + /// Creates a file system based Lucene with the correct locking guidelines for Umbraco + /// + /// + /// The folder name to store the index (single word, not a fully qualified folder) (i.e. Internal) + /// + /// + public virtual Lucene.Net.Store.Directory CreateFileSystemLuceneDirectory(string indexName) + { + var dirInfo = _applicationRoot.ApplicationRoot; + + if (!dirInfo.Exists) + { + Directory.CreateDirectory(dirInfo.FullName); + } + + //check if there's a configured directory factory, if so create it and use that to create the lucene dir + var configuredDirectoryFactory = _settings.LuceneDirectoryFactory; + + if (!configuredDirectoryFactory.IsNullOrWhiteSpace()) + { + //this should be a fully qualified type + Type factoryType = _typeFinder.GetTypeByName(configuredDirectoryFactory); + if (factoryType == null) + { + throw new InvalidOperationException("No directory type found for value: " + configuredDirectoryFactory); + } + + var directoryFactory = (IDirectoryFactory)ActivatorUtilities.CreateInstance(_services, factoryType); + + return directoryFactory.CreateDirectory(indexName); + } + + var fileSystemDirectoryFactory = new FileSystemDirectoryFactory(dirInfo, _lockFactory); + return fileSystemDirectoryFactory.CreateDirectory(indexName); + + } + } +} diff --git a/src/Umbraco.Examine.Lucene/DependencyInjection/ConfigureIndexOptions.cs b/src/Umbraco.Examine.Lucene/DependencyInjection/ConfigureIndexOptions.cs new file mode 100644 index 0000000000..677167f0ff --- /dev/null +++ b/src/Umbraco.Examine.Lucene/DependencyInjection/ConfigureIndexOptions.cs @@ -0,0 +1,46 @@ +using System; +using Examine; +using Examine.Lucene; +using Examine.Lucene.Analyzers; +using Lucene.Net.Analysis.Standard; +using Microsoft.Extensions.Options; +using Umbraco.Cms.Core; + +namespace Umbraco.Cms.Infrastructure.Examine.DependencyInjection +{ + /// + /// Configures the index options to construct the Examine indexes + /// + public sealed class ConfigureIndexOptions : IConfigureNamedOptions + { + private readonly IUmbracoIndexConfig _umbracoIndexConfig; + + public ConfigureIndexOptions(IUmbracoIndexConfig umbracoIndexConfig) + => _umbracoIndexConfig = umbracoIndexConfig; + + public void Configure(string name, LuceneDirectoryIndexOptions options) + { + switch (name) + { + case Constants.UmbracoIndexes.InternalIndexName: + options.Analyzer = new CultureInvariantWhitespaceAnalyzer(); + options.Validator = _umbracoIndexConfig.GetContentValueSetValidator(); + options.FieldDefinitions = new UmbracoFieldDefinitionCollection(); + break; + case Constants.UmbracoIndexes.ExternalIndexName: + options.Analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); + options.Validator = _umbracoIndexConfig.GetPublishedContentValueSetValidator(); + options.FieldDefinitions = new UmbracoFieldDefinitionCollection(); + break; + case Constants.UmbracoIndexes.MembersIndexName: + options.Analyzer = new CultureInvariantWhitespaceAnalyzer(); + options.Validator = _umbracoIndexConfig.GetMemberValueSetValidator(); + options.FieldDefinitions = new UmbracoFieldDefinitionCollection(); + break; + } + } + + public void Configure(LuceneDirectoryIndexOptions options) + => throw new NotImplementedException("This is never called and is just part of the interface"); + } +} diff --git a/src/Umbraco.Examine.Lucene/DependencyInjection/UmbracoBuilderExtensions.cs b/src/Umbraco.Examine.Lucene/DependencyInjection/UmbracoBuilderExtensions.cs new file mode 100644 index 0000000000..8eafde1a38 --- /dev/null +++ b/src/Umbraco.Examine.Lucene/DependencyInjection/UmbracoBuilderExtensions.cs @@ -0,0 +1,40 @@ +using Examine; +using Examine.Lucene.Directories; +using Microsoft.Extensions.DependencyInjection; +using Umbraco.Cms.Core; +using Umbraco.Cms.Core.DependencyInjection; +using Umbraco.Cms.Infrastructure.DependencyInjection; + +namespace Umbraco.Cms.Infrastructure.Examine.DependencyInjection +{ + public static class UmbracoBuilderExtensions + { + /// + /// Adds the Examine indexes for Umbraco + /// + /// + /// + public static IUmbracoBuilder AddExamineIndexes(this IUmbracoBuilder umbracoBuilder) + { + IServiceCollection services = umbracoBuilder.Services; + + services.AddSingleton(); + services.AddSingleton(); + + services.AddExamine(); + + // Create the indexes + services + .AddExamineLuceneIndex(Constants.UmbracoIndexes.InternalIndexName) + .AddExamineLuceneIndex(Constants.UmbracoIndexes.ExternalIndexName) + .AddExamineLuceneIndex(Constants.UmbracoIndexes.MembersIndexName) + .ConfigureOptions(); + + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + + return umbracoBuilder; + } + } +} diff --git a/src/Umbraco.Examine.Lucene/ExamineLuceneComponent.cs b/src/Umbraco.Examine.Lucene/ExamineLuceneComponent.cs deleted file mode 100644 index fe1826c989..0000000000 --- a/src/Umbraco.Examine.Lucene/ExamineLuceneComponent.cs +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright (c) Umbraco. -// See LICENSE for more details. - -using Examine; -using Examine.LuceneEngine.Directories; -using Microsoft.Extensions.Logging; -using Umbraco.Cms.Core.Composing; -using Umbraco.Cms.Core.Runtime; -using Umbraco.Extensions; - -namespace Umbraco.Cms.Infrastructure.Examine -{ - public sealed class ExamineLuceneComponent : IComponent - { - private readonly IndexRebuilder _indexRebuilder; - private readonly IExamineManager _examineManager; - private readonly IMainDom _mainDom; - private readonly ILoggerFactory _loggerFactory; - - public ExamineLuceneComponent(IndexRebuilder indexRebuilder, IExamineManager examineManager, IMainDom mainDom, ILoggerFactory loggerFactory) - { - _indexRebuilder = indexRebuilder; - _examineManager = examineManager; - _mainDom = mainDom; - _loggerFactory = loggerFactory; - } - - public void Initialize() - { - //we want to tell examine to use a different fs lock instead of the default NativeFSFileLock which could cause problems if the AppDomain - //terminates and in some rare cases would only allow unlocking of the file if IIS is forcefully terminated. Instead we'll rely on the simplefslock - //which simply checks the existence of the lock file - DirectoryFactory.DefaultLockFactory = d => - { - var simpleFsLockFactory = new NoPrefixSimpleFsLockFactory(d); - return simpleFsLockFactory; - }; - - _indexRebuilder.RebuildingIndexes += IndexRebuilder_RebuildingIndexes; - } - - /// - /// Handles event to ensure that all lucene based indexes are properly configured before rebuilding - /// - /// - /// - private void IndexRebuilder_RebuildingIndexes(object sender, IndexRebuildingEventArgs e) => _examineManager.ConfigureIndexes(_mainDom, _loggerFactory.CreateLogger()); - - public void Terminate() - { - } - } -} diff --git a/src/Umbraco.Examine.Lucene/ExamineLuceneComposer.cs b/src/Umbraco.Examine.Lucene/ExamineLuceneComposer.cs deleted file mode 100644 index 327ac4b4ba..0000000000 --- a/src/Umbraco.Examine.Lucene/ExamineLuceneComposer.cs +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) Umbraco. -// See LICENSE for more details. - -using System.Runtime.InteropServices; -using Umbraco.Cms.Core.Composing; -using Umbraco.Cms.Core.DependencyInjection; -using Umbraco.Extensions; - -namespace Umbraco.Cms.Infrastructure.Examine -{ - // We want to run after core composers since we are replacing some items - [ComposeAfter(typeof(ICoreComposer))] - public sealed class ExamineLuceneComposer : ComponentComposer - { - public override void Compose(IUmbracoBuilder builder) - { - var isWindows = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); - if(!isWindows) return; - - - base.Compose(builder); - - builder.Services.AddUnique(); - builder.Services.AddUnique(); - builder.Services.AddUnique(); - builder.Services.AddUnique(); - } - } -} diff --git a/src/Umbraco.Examine.Lucene/ExamineLuceneFinalComponent.cs b/src/Umbraco.Examine.Lucene/ExamineLuceneFinalComponent.cs deleted file mode 100644 index b95165b121..0000000000 --- a/src/Umbraco.Examine.Lucene/ExamineLuceneFinalComponent.cs +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright (c) Umbraco. -// See LICENSE for more details. - -using Examine; -using Microsoft.Extensions.Logging; -using Umbraco.Cms.Core.Composing; -using Umbraco.Cms.Core.Runtime; -using Umbraco.Extensions; - -namespace Umbraco.Cms.Infrastructure.Examine -{ - public class ExamineLuceneFinalComponent : IComponent - { - private readonly ILoggerFactory _loggerFactory; - private readonly IExamineManager _examineManager; - private readonly IMainDom _mainDom; - - public ExamineLuceneFinalComponent(ILoggerFactory loggerFactory, IExamineManager examineManager, IMainDom mainDom) - { - _loggerFactory = loggerFactory; - _examineManager = examineManager; - _mainDom = mainDom; - } - - public void Initialize() - { - if (!_mainDom.IsMainDom) return; - - // Ensures all lucene based indexes are unlocked and ready to go - _examineManager.ConfigureIndexes(_mainDom, _loggerFactory.CreateLogger()); - } - - public void Terminate() - { - } - } -} diff --git a/src/Umbraco.Examine.Lucene/ExamineLuceneFinalComposer.cs b/src/Umbraco.Examine.Lucene/ExamineLuceneFinalComposer.cs deleted file mode 100644 index 518ffc2db8..0000000000 --- a/src/Umbraco.Examine.Lucene/ExamineLuceneFinalComposer.cs +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright (c) Umbraco. -// See LICENSE for more details. - -using Umbraco.Cms.Core.Composing; - -namespace Umbraco.Cms.Infrastructure.Examine -{ - // examine's Lucene final composer composes after all user composers - // and *also* after ICoreComposer (in case IUserComposer is disabled) - [ComposeAfter(typeof(IUserComposer))] - [ComposeAfter(typeof(ICoreComposer))] - public class ExamineLuceneFinalComposer : ComponentComposer - { } -} diff --git a/src/Umbraco.Examine.Lucene/Extensions/ExamineExtensions.cs b/src/Umbraco.Examine.Lucene/Extensions/ExamineExtensions.cs index 9307c4cbf4..1a8bb36baa 100644 --- a/src/Umbraco.Examine.Lucene/Extensions/ExamineExtensions.cs +++ b/src/Umbraco.Examine.Lucene/Extensions/ExamineExtensions.cs @@ -1,19 +1,17 @@ -// Copyright (c) Umbraco. +// Copyright (c) Umbraco. // See LICENSE for more details. using System; using System.Linq; using System.Threading; using Examine; -using Examine.LuceneEngine.Providers; -using Lucene.Net.Analysis; +using Examine.Lucene.Providers; +using Lucene.Net.Analysis.Core; using Lucene.Net.Index; -using Lucene.Net.QueryParsers; -using Lucene.Net.Search; +using Lucene.Net.QueryParsers.Classic; using Microsoft.Extensions.Logging; using Umbraco.Cms.Core.Runtime; using Umbraco.Cms.Infrastructure.Examine; -using Version = Lucene.Net.Util.Version; namespace Umbraco.Extensions { @@ -22,40 +20,19 @@ namespace Umbraco.Extensions /// public static class ExamineExtensions { - private static bool _isConfigured = false; - private static object _configuredInit = null; - private static object _isConfiguredLocker = new object(); - - /// - /// Called on startup to configure each index. - /// - /// - /// Configures and unlocks all Lucene based indexes registered with the . - /// - internal static void ConfigureIndexes(this IExamineManager examineManager, IMainDom mainDom, ILogger logger) - { - LazyInitializer.EnsureInitialized( - ref _configuredInit, - ref _isConfigured, - ref _isConfiguredLocker, - () => - { - examineManager.ConfigureLuceneIndexes(logger, !mainDom.IsMainDom); - return null; - }); - } - internal static bool TryParseLuceneQuery(string query) { // TODO: I'd assume there would be a more strict way to parse the query but not that i can find yet, for now we'll // also do this rudimentary check if (!query.Contains(":")) + { return false; + } try { //This will pass with a plain old string without any fields, need to figure out a way to have it properly parse - var parsed = new QueryParser(Version.LUCENE_30, UmbracoExamineFieldNames.NodeNameFieldName, new KeywordAnalyzer()).Parse(query); + var parsed = new QueryParser(LuceneInfo.CurrentVersion, UmbracoExamineFieldNames.NodeNameFieldName, new KeywordAnalyzer()).Parse(query); return true; } catch (ParseException) @@ -68,34 +45,6 @@ namespace Umbraco.Extensions } } - /// - /// Forcibly unlocks all lucene based indexes - /// - /// - /// This is not thread safe, use with care - /// - private static void ConfigureLuceneIndexes(this IExamineManager examineManager, ILogger logger, bool disableExamineIndexing) - { - foreach (var luceneIndexer in examineManager.Indexes.OfType()) - { - //We now need to disable waiting for indexing for Examine so that the appdomain is shutdown immediately and doesn't wait for pending - //indexing operations. We used to wait for indexing operations to complete but this can cause more problems than that is worth because - //that could end up halting shutdown for a very long time causing overlapping appdomains and many other problems. - luceneIndexer.WaitForIndexQueueOnShutdown = false; - - if (disableExamineIndexing) continue; //exit if not enabled, we don't need to unlock them if we're not maindom - - //we should check if the index is locked ... it shouldn't be! We are using simple fs lock now and we are also ensuring that - //the indexes are not operational unless MainDom is true - var dir = luceneIndexer.GetLuceneDirectory(); - if (IndexWriter.IsLocked(dir)) - { - logger.LogDebug("Forcing index {IndexerName} to be unlocked since it was left in a locked state", luceneIndexer.Name); - IndexWriter.Unlock(dir); - } - } - } - /// /// Checks if the index can be read/opened /// @@ -106,7 +55,7 @@ namespace Umbraco.Extensions { try { - using (indexer.GetIndexWriter().GetReader()) + using (indexer.IndexWriter.IndexWriter.GetReader(false)) { ex = null; return true; @@ -119,38 +68,5 @@ namespace Umbraco.Extensions } } - /// - /// Return the number of indexed documents in Lucene - /// - /// - /// - public static int GetIndexDocumentCount(this LuceneIndex indexer) - { - if (!((indexer.GetSearcher() as LuceneSearcher)?.GetLuceneSearcher() is IndexSearcher searcher)) - return 0; - - using (searcher) - using (var reader = searcher.IndexReader) - { - return reader.NumDocs(); - } - } - - /// - /// Return the total number of fields in the index - /// - /// - /// - public static int GetIndexFieldCount(this LuceneIndex indexer) - { - if (!((indexer.GetSearcher() as LuceneSearcher)?.GetLuceneSearcher() is IndexSearcher searcher)) - return 0; - - using (searcher) - using (var reader = searcher.IndexReader) - { - return reader.GetFieldNames(IndexReader.FieldOption.ALL).Count; - } - } } } diff --git a/src/Umbraco.Examine.Lucene/ILuceneDirectoryFactory.cs b/src/Umbraco.Examine.Lucene/ILuceneDirectoryFactory.cs deleted file mode 100644 index 70f3825667..0000000000 --- a/src/Umbraco.Examine.Lucene/ILuceneDirectoryFactory.cs +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright (c) Umbraco. -// See LICENSE for more details. - -namespace Umbraco.Cms.Infrastructure.Examine -{ - public interface ILuceneDirectoryFactory - { - Lucene.Net.Store.Directory CreateDirectory(string indexName); - } -} diff --git a/src/Umbraco.Examine.Lucene/LuceneFileSystemDirectoryFactory.cs b/src/Umbraco.Examine.Lucene/LuceneFileSystemDirectoryFactory.cs deleted file mode 100644 index 9e09c7e96e..0000000000 --- a/src/Umbraco.Examine.Lucene/LuceneFileSystemDirectoryFactory.cs +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright (c) Umbraco. -// See LICENSE for more details. - -using System; -using System.IO; -using Examine.LuceneEngine.Directories; -using Lucene.Net.Store; -using Microsoft.Extensions.Options; -using Umbraco.Cms.Core.Composing; -using Umbraco.Cms.Core.Configuration.Models; -using Umbraco.Cms.Core.Hosting; -using Umbraco.Extensions; -using Constants = Umbraco.Cms.Core.Constants; - -namespace Umbraco.Cms.Infrastructure.Examine -{ - public class LuceneFileSystemDirectoryFactory : ILuceneDirectoryFactory - { - private readonly ITypeFinder _typeFinder; - private readonly IHostingEnvironment _hostingEnvironment; - private readonly IndexCreatorSettings _settings; - - public LuceneFileSystemDirectoryFactory(ITypeFinder typeFinder, IHostingEnvironment hostingEnvironment, IOptions settings) - { - _typeFinder = typeFinder; - _hostingEnvironment = hostingEnvironment; - _settings = settings.Value; - } - - public Lucene.Net.Store.Directory CreateDirectory(string indexName) => CreateFileSystemLuceneDirectory(indexName); - - /// - /// Creates a file system based Lucene with the correct locking guidelines for Umbraco - /// - /// - /// The folder name to store the index (single word, not a fully qualified folder) (i.e. Internal) - /// - /// - public virtual Lucene.Net.Store.Directory CreateFileSystemLuceneDirectory(string folderName) - { - - var dirInfo = new DirectoryInfo(Path.Combine(_hostingEnvironment.MapPathContentRoot(Constants.SystemDirectories.TempData), "ExamineIndexes", folderName)); - if (!dirInfo.Exists) - System.IO.Directory.CreateDirectory(dirInfo.FullName); - - //check if there's a configured directory factory, if so create it and use that to create the lucene dir - var configuredDirectoryFactory = _settings.LuceneDirectoryFactory; - - if (!configuredDirectoryFactory.IsNullOrWhiteSpace()) - { - //this should be a fully qualified type - var factoryType = _typeFinder.GetTypeByName(configuredDirectoryFactory); - if (factoryType == null) throw new NullReferenceException("No directory type found for value: " + configuredDirectoryFactory); - var directoryFactory = (IDirectoryFactory)Activator.CreateInstance(factoryType); - return directoryFactory.CreateDirectory(dirInfo); - } - - //no dir factory, just create a normal fs directory - - var luceneDir = new SimpleFSDirectory(dirInfo); - - //we want to tell examine to use a different fs lock instead of the default NativeFSFileLock which could cause problems if the appdomain - //terminates and in some rare cases would only allow unlocking of the file if IIS is forcefully terminated. Instead we'll rely on the simplefslock - //which simply checks the existence of the lock file - // The full syntax of this is: new NoPrefixSimpleFsLockFactory(dirInfo) - // however, we are setting the DefaultLockFactory in startup so we'll use that instead since it can be managed globally. - luceneDir.SetLockFactory(DirectoryFactory.DefaultLockFactory(dirInfo)); - return luceneDir; - - - } - } -} diff --git a/src/Umbraco.Examine.Lucene/LuceneIndexCreator.cs b/src/Umbraco.Examine.Lucene/LuceneIndexCreator.cs deleted file mode 100644 index dc2acfa66d..0000000000 --- a/src/Umbraco.Examine.Lucene/LuceneIndexCreator.cs +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) Umbraco. -// See LICENSE for more details. - -using System.Collections.Generic; -using Examine; -using Microsoft.Extensions.Options; -using Umbraco.Cms.Core.Composing; -using Umbraco.Cms.Core.Configuration.Models; -using Umbraco.Cms.Core.Hosting; - -namespace Umbraco.Cms.Infrastructure.Examine -{ - /// - /// - /// Abstract class for creating Lucene based Indexes - /// - public abstract class LuceneIndexCreator : IIndexCreator - { - private readonly ITypeFinder _typeFinder; - private readonly IHostingEnvironment _hostingEnvironment; - private readonly IndexCreatorSettings _settings; - - protected LuceneIndexCreator(ITypeFinder typeFinder, IHostingEnvironment hostingEnvironment, IOptions settings) - { - _typeFinder = typeFinder; - _hostingEnvironment = hostingEnvironment; - _settings = settings.Value; - } - - public abstract IEnumerable Create(); - } -} diff --git a/src/Umbraco.Examine.Lucene/LuceneIndexDiagnostics.cs b/src/Umbraco.Examine.Lucene/LuceneIndexDiagnostics.cs index 1cba0767eb..6ad23b5992 100644 --- a/src/Umbraco.Examine.Lucene/LuceneIndexDiagnostics.cs +++ b/src/Umbraco.Examine.Lucene/LuceneIndexDiagnostics.cs @@ -1,8 +1,10 @@ -// Copyright (c) Umbraco. +// Copyright (c) Umbraco. // See LICENSE for more details. +using System; using System.Collections.Generic; -using Examine.LuceneEngine.Providers; +using System.Threading.Tasks; +using Examine.Lucene.Providers; using Lucene.Net.Store; using Microsoft.Extensions.Logging; using Umbraco.Cms.Core; @@ -25,37 +27,7 @@ namespace Umbraco.Cms.Infrastructure.Examine public LuceneIndex Index { get; } public ILogger Logger { get; } - public int DocumentCount - { - get - { - try - { - return Index.GetIndexDocumentCount(); - } - catch (AlreadyClosedException) - { - Logger.LogWarning("Cannot get GetIndexDocumentCount, the writer is already closed"); - return 0; - } - } - } - - public int FieldCount - { - get - { - try - { - return Index.GetIndexFieldCount(); - } - catch (AlreadyClosedException) - { - Logger.LogWarning("Cannot get GetIndexFieldCount, the writer is already closed"); - return 0; - } - } - } + public Attempt IsHealthy() { @@ -63,6 +35,10 @@ namespace Umbraco.Cms.Infrastructure.Examine return isHealthy ? Attempt.Succeed() : Attempt.Fail(indexError.Message); } + public long GetDocumentCount() => Index.GetDocumentCount(); + + public IEnumerable GetFieldNames() => Index.GetFieldNames(); + public virtual IReadOnlyDictionary Metadata { get diff --git a/src/Umbraco.Examine.Lucene/LuceneIndexDiagnosticsFactory.cs b/src/Umbraco.Examine.Lucene/LuceneIndexDiagnosticsFactory.cs index 322da710dc..bdfc299121 100644 --- a/src/Umbraco.Examine.Lucene/LuceneIndexDiagnosticsFactory.cs +++ b/src/Umbraco.Examine.Lucene/LuceneIndexDiagnosticsFactory.cs @@ -1,8 +1,8 @@ -// Copyright (c) Umbraco. +// Copyright (c) Umbraco. // See LICENSE for more details. using Examine; -using Examine.LuceneEngine.Providers; +using Examine.Lucene.Providers; using Microsoft.Extensions.Logging; using Umbraco.Cms.Core.Hosting; @@ -28,9 +28,13 @@ namespace Umbraco.Cms.Infrastructure.Examine if (!(index is IIndexDiagnostics indexDiag)) { if (index is LuceneIndex luceneIndex) + { indexDiag = new LuceneIndexDiagnostics(luceneIndex, _loggerFactory.CreateLogger(), _hostingEnvironment); + } else + { indexDiag = base.Create(index); + } } return indexDiag; } diff --git a/src/Umbraco.Examine.Lucene/LuceneRAMDirectoryFactory.cs b/src/Umbraco.Examine.Lucene/LuceneRAMDirectoryFactory.cs index 1f9802f072..2b25350f09 100644 --- a/src/Umbraco.Examine.Lucene/LuceneRAMDirectoryFactory.cs +++ b/src/Umbraco.Examine.Lucene/LuceneRAMDirectoryFactory.cs @@ -1,27 +1,26 @@ -// Copyright (c) Umbraco. +// Copyright (c) Umbraco. // See LICENSE for more details. using System; +using System.IO; +using Examine.Lucene.Directories; using Lucene.Net.Store; +using Directory = Lucene.Net.Store.Directory; namespace Umbraco.Cms.Infrastructure.Examine { - public class LuceneRAMDirectoryFactory : ILuceneDirectoryFactory + public class LuceneRAMDirectoryFactory : IDirectoryFactory { public LuceneRAMDirectoryFactory() { - } - public Lucene.Net.Store.Directory CreateDirectory(string indexName) => new RandomIdRAMDirectory(); + public Directory CreateDirectory(string indexName) => new RandomIdRAMDirectory(); private class RandomIdRAMDirectory : RAMDirectory { private readonly string _lockId = Guid.NewGuid().ToString(); - public override string GetLockId() - { - return _lockId; - } + public override string GetLockID() => _lockId; } } } diff --git a/src/Umbraco.Examine.Lucene/NoPrefixSimpleFsLockFactory.cs b/src/Umbraco.Examine.Lucene/NoPrefixSimpleFsLockFactory.cs index 38d704e681..ed6f47c882 100644 --- a/src/Umbraco.Examine.Lucene/NoPrefixSimpleFsLockFactory.cs +++ b/src/Umbraco.Examine.Lucene/NoPrefixSimpleFsLockFactory.cs @@ -6,6 +6,7 @@ using Lucene.Net.Store; namespace Umbraco.Cms.Infrastructure.Examine { + /// /// A custom that ensures a prefixless lock prefix /// diff --git a/src/Umbraco.Examine.Lucene/Umbraco.Examine.Lucene.csproj b/src/Umbraco.Examine.Lucene/Umbraco.Examine.Lucene.csproj index ef67c424d8..2417178d69 100644 --- a/src/Umbraco.Examine.Lucene/Umbraco.Examine.Lucene.csproj +++ b/src/Umbraco.Examine.Lucene/Umbraco.Examine.Lucene.csproj @@ -1,56 +1,42 @@ - - - net472 - Umbraco.Cms.Infrastructure.Examine - Umbraco CMS - Umbraco.Examine.Lucene - - - false - - Umbraco.Cms.Examine.Lucene - - - - true - bin\Release\Umbraco.Examine.Lucene.xml - - - - - - - - - - - - - - - - - - - - - - - 1.0.0 - runtime; build; native; contentfiles; analyzers; buildtransitive - all - - - - - 3.5.4 - runtime; build; native; contentfiles; analyzers - all - - - all - - - - + + netstandard2.0 + Umbraco.Cms.Infrastructure.Examine + Umbraco CMS + Umbraco.Examine.Lucene + + Umbraco.Cms.Examine.Lucene + + + true + bin\Release\Umbraco.Examine.Lucene.xml + + + + + + + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + 3.5.4 + runtime; build; native; contentfiles; analyzers + all + + + all + + + \ No newline at end of file diff --git a/src/Umbraco.Examine.Lucene/UmbracoApplicationRoot.cs b/src/Umbraco.Examine.Lucene/UmbracoApplicationRoot.cs new file mode 100644 index 0000000000..e99f986176 --- /dev/null +++ b/src/Umbraco.Examine.Lucene/UmbracoApplicationRoot.cs @@ -0,0 +1,23 @@ +using System.IO; +using Examine; +using Umbraco.Cms.Core.Hosting; + +namespace Umbraco.Cms.Infrastructure.Examine +{ + /// + /// Sets the Examine to be ExamineIndexes sub directory of the Umbraco TEMP folder + /// + public class UmbracoApplicationRoot : IApplicationRoot + { + private readonly IHostingEnvironment _hostingEnvironment; + + public UmbracoApplicationRoot(IHostingEnvironment hostingEnvironment) + => _hostingEnvironment = hostingEnvironment; + + public DirectoryInfo ApplicationRoot + => new DirectoryInfo( + Path.Combine( + _hostingEnvironment.MapPathContentRoot(Core.Constants.SystemDirectories.TempData), + "ExamineIndexes")); + } +} diff --git a/src/Umbraco.Examine.Lucene/UmbracoContentIndex.cs b/src/Umbraco.Examine.Lucene/UmbracoContentIndex.cs index 18b9945a6e..b3852254af 100644 --- a/src/Umbraco.Examine.Lucene/UmbracoContentIndex.cs +++ b/src/Umbraco.Examine.Lucene/UmbracoContentIndex.cs @@ -5,12 +5,10 @@ using System; using System.Collections.Generic; using System.Linq; using Examine; -using Examine.LuceneEngine; -using Lucene.Net.Analysis; -using Lucene.Net.Store; +using Examine.Lucene; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Umbraco.Cms.Core.Hosting; -using Umbraco.Cms.Core.Logging; using Umbraco.Cms.Core.Services; namespace Umbraco.Cms.Infrastructure.Examine @@ -21,49 +19,38 @@ namespace Umbraco.Cms.Infrastructure.Examine public class UmbracoContentIndex : UmbracoExamineIndex, IUmbracoContentIndex, IDisposable { private readonly ILogger _logger; - protected ILocalizationService LanguageService { get; } - #region Constructors - - /// - /// Create an index at runtime - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// public UmbracoContentIndex( - string name, - Directory luceneDirectory, - FieldDefinitionCollection fieldDefinitions, - Analyzer defaultAnalyzer, - IProfilingLogger profilingLogger, - ILogger logger, ILoggerFactory loggerFactory, + string name, + IOptionsSnapshot indexOptions, IHostingEnvironment hostingEnvironment, IRuntimeState runtimeState, - ILocalizationService languageService, - IContentValueSetValidator validator, - IReadOnlyDictionary indexValueTypes = null) - : base(name, luceneDirectory, fieldDefinitions, defaultAnalyzer, profilingLogger, logger, loggerFactory ,hostingEnvironment, runtimeState, validator, indexValueTypes) + ILocalizationService languageService = null) + : base(loggerFactory, name, indexOptions, hostingEnvironment, runtimeState) { - if (validator == null) throw new ArgumentNullException(nameof(validator)); - _logger = logger; - LanguageService = languageService ?? throw new ArgumentNullException(nameof(languageService)); + LanguageService = languageService; + _logger = loggerFactory.CreateLogger(); - if (validator is IContentValueSetValidator contentValueSetValidator) + LuceneDirectoryIndexOptions namedOptions = indexOptions.Get(name); + if (namedOptions == null) + { + throw new InvalidOperationException($"No named {typeof(LuceneDirectoryIndexOptions)} options with name {name}"); + } + + if (namedOptions.Validator is IContentValueSetValidator contentValueSetValidator) + { PublishedValuesOnly = contentValueSetValidator.PublishedValuesOnly; + } } - #endregion + protected ILocalizationService LanguageService { get; } + + /// + /// Explicitly override because we need to do validation differently than the underlying logic + /// + /// + void IIndex.IndexItems(IEnumerable values) => PerformIndexItems(values, OnIndexOperationComplete); /// /// Special check for invalid paths @@ -75,45 +62,48 @@ namespace Umbraco.Cms.Infrastructure.Examine // We don't want to re-enumerate this list, but we need to split it into 2x enumerables: invalid and valid items. // The Invalid items will be deleted, these are items that have invalid paths (i.e. moved to the recycle bin, etc...) // Then we'll index the Value group all together. - // We return 0 or 1 here so we can order the results and do the invalid first and then the valid. var invalidOrValid = values.GroupBy(v => { - if (!v.Values.TryGetValue("path", out var paths) || paths.Count <= 0 || paths[0] == null) - return 0; + if (!v.Values.TryGetValue("path", out List paths) || paths.Count <= 0 || paths[0] == null) + { + return ValueSetValidationResult.Failed; + } - //we know this is an IContentValueSetValidator - var validator = (IContentValueSetValidator)ValueSetValidator; - var path = paths[0].ToString(); + ValueSetValidationResult validationResult = ValueSetValidator.Validate(v); - return (!validator.ValidatePath(path, v.Category) - || !validator.ValidateRecycleBin(path, v.Category) - || !validator.ValidateProtectedContent(path, v.Category)) - ? 0 - : 1; + return validationResult; }).ToList(); var hasDeletes = false; var hasUpdates = false; - foreach (var group in invalidOrValid.OrderBy(x => x.Key)) - { - if (group.Key == 0) - { - hasDeletes = true; - //these are the invalid items so we'll delete them - //since the path is not valid we need to delete this item in case it exists in the index already and has now - //been moved to an invalid parent. - base.PerformDeleteFromIndex(group.Select(x => x.Id), args => { /*noop*/ }); - } - else + // ordering by descending so that Filtered/Failed processes first + foreach (IGrouping group in invalidOrValid.OrderByDescending(x => x.Key)) + { + switch (group.Key) { - hasUpdates = true; - //these are the valid ones, so just index them all at once - base.PerformIndexItems(group.ToList(), onComplete); + case ValueSetValidationResult.Valid: + hasUpdates = true; + + //these are the valid ones, so just index them all at once + base.PerformIndexItems(group.ToList(), onComplete); + break; + case ValueSetValidationResult.Failed: + // don't index anything that is invalid + break; + case ValueSetValidationResult.Filtered: + hasDeletes = true; + + // these are the invalid/filtered items so we'll delete them + // since the path is not valid we need to delete this item in + // case it exists in the index already and has now + // been moved to an invalid parent. + base.PerformDeleteFromIndex(group.Select(x => x.Id), null); + break; } } - if (hasDeletes && !hasUpdates || !hasDeletes && !hasUpdates) + if ((hasDeletes && !hasUpdates) || (!hasDeletes && !hasUpdates)) { //we need to manually call the completed method onComplete(new IndexOperationEventArgs(this, 0)); @@ -133,21 +123,27 @@ namespace Umbraco.Cms.Infrastructure.Examine protected override void PerformDeleteFromIndex(IEnumerable itemIds, Action onComplete) { var idsAsList = itemIds.ToList(); - foreach (var nodeId in idsAsList) + + for (int i = 0; i < idsAsList.Count; i++) { + string nodeId = idsAsList[i]; + //find all descendants based on path var descendantPath = $@"\-1\,*{nodeId}\,*"; var rawQuery = $"{UmbracoExamineFieldNames.IndexPathFieldName}:{descendantPath}"; - var searcher = GetSearcher(); - var c = searcher.CreateQuery(); + var c = Searcher.CreateQuery(); var filtered = c.NativeQuery(rawQuery); var results = filtered.Execute(); _logger. LogDebug("DeleteFromIndex with query: {Query} (found {TotalItems} results)", rawQuery, results.TotalItemCount); - //need to queue a delete item for each one found - QueueIndexOperation(results.Select(r => new IndexOperation(new ValueSet(r.Id), IndexOperationType.Delete))); + var toRemove = results.Select(x => x.Id).ToList(); + // delete those descendants (ensure base. is used here so we aren't calling ourselves!) + base.PerformDeleteFromIndex(toRemove, null); + + // remove any ids from our list that were part of the descendants + idsAsList.RemoveAll(x => toRemove.Contains(x)); } base.PerformDeleteFromIndex(idsAsList, onComplete); diff --git a/src/Umbraco.Examine.Lucene/UmbracoExamineIndex.cs b/src/Umbraco.Examine.Lucene/UmbracoExamineIndex.cs index 851dfbd152..5ebcb4877a 100644 --- a/src/Umbraco.Examine.Lucene/UmbracoExamineIndex.cs +++ b/src/Umbraco.Examine.Lucene/UmbracoExamineIndex.cs @@ -4,20 +4,17 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Threading; using Examine; -using Examine.LuceneEngine; -using Examine.LuceneEngine.Providers; -using Lucene.Net.Analysis; +using Examine.Lucene; +using Examine.Lucene.Providers; using Lucene.Net.Documents; using Lucene.Net.Index; using Lucene.Net.Store; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Umbraco.Cms.Core; using Umbraco.Cms.Core.Hosting; -using Umbraco.Cms.Core.Logging; using Umbraco.Cms.Core.Services; -using Directory = Lucene.Net.Store.Directory; namespace Umbraco.Cms.Infrastructure.Examine { @@ -27,61 +24,24 @@ namespace Umbraco.Cms.Infrastructure.Examine /// public abstract class UmbracoExamineIndex : LuceneIndex, IUmbracoIndex, IIndexDiagnostics { - private readonly ILogger _logger; - private readonly ILoggerFactory _loggerFactory; - + private readonly UmbracoExamineIndexDiagnostics _diagnostics; private readonly IRuntimeState _runtimeState; + private bool _hasLoggedInitLog = false; + private readonly ILogger _logger; - // note - // wrapping all operations that end up calling base.SafelyProcessQueueItems in a safe call - // context because they will fork a thread/task/whatever which should *not* capture our - // call context (and the database it can contain)! - // TODO: FIX Examine to not flow the ExecutionContext so callers don't need to worry about this! - - /// - /// Create a new - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// protected UmbracoExamineIndex( - string name, - Directory luceneDirectory, - FieldDefinitionCollection fieldDefinitions, - Analyzer defaultAnalyzer, - IProfilingLogger profilingLogger, - ILogger logger, ILoggerFactory loggerFactory, + string name, + IOptionsSnapshot indexOptions, IHostingEnvironment hostingEnvironment, - IRuntimeState runtimeState, - IValueSetValidator validator = null, - IReadOnlyDictionary indexValueTypes = null) - : base(name, luceneDirectory, fieldDefinitions, defaultAnalyzer, validator, indexValueTypes) + IRuntimeState runtimeState) + : base(loggerFactory, name, indexOptions) { - _logger = logger; - _loggerFactory = loggerFactory; _runtimeState = runtimeState; - ProfilingLogger = profilingLogger ?? throw new ArgumentNullException(nameof(profilingLogger)); - - //try to set the value of `LuceneIndexFolder` for diagnostic reasons - if (luceneDirectory is FSDirectory fsDir) - LuceneIndexFolder = fsDir.Directory; - - _diagnostics = new UmbracoExamineIndexDiagnostics(this, _loggerFactory.CreateLogger(), hostingEnvironment); + _diagnostics = new UmbracoExamineIndexDiagnostics(this, loggerFactory.CreateLogger(), hostingEnvironment); + _logger = loggerFactory.CreateLogger(); } - private readonly bool _configBased = false; - - protected IProfilingLogger ProfilingLogger { get; } - /// /// When set to true Umbraco will keep the index in sync with Umbraco data automatically /// @@ -89,14 +49,6 @@ namespace Umbraco.Cms.Infrastructure.Examine public bool PublishedValuesOnly { get; protected set; } = false; - /// - public IEnumerable GetFields() - { - //we know this is a LuceneSearcher - var searcher = (LuceneSearcher)GetSearcher(); - return searcher.GetAllIndexedFields(); - } - /// /// override to check if we can actually initialize. /// @@ -107,13 +59,7 @@ namespace Umbraco.Cms.Infrastructure.Examine { if (CanInitialize()) { - // Use ExecutionContext.SuppressFlow to prevent the current Execution Context (AsyncLocal) flow to child - // tasks executed in the base class so we don't leak Scopes. - // TODO: See notes at the top of this class - using (ExecutionContext.SuppressFlow()) - { - base.PerformDeleteFromIndex(itemIds, onComplete); - } + base.PerformDeleteFromIndex(itemIds, onComplete); } } @@ -121,13 +67,7 @@ namespace Umbraco.Cms.Infrastructure.Examine { if (CanInitialize()) { - // Use ExecutionContext.SuppressFlow to prevent the current Execution Context (AsyncLocal) flow to child - // tasks executed in the base class so we don't leak Scopes. - // TODO: See notes at the top of this class - using (ExecutionContext.SuppressFlow()) - { - base.PerformIndexItems(values, onComplete); - } + base.PerformIndexItems(values, onComplete); } } @@ -137,19 +77,15 @@ namespace Umbraco.Cms.Infrastructure.Examine /// protected bool CanInitialize() { - // only affects indexers that are config file based, if an index was created via code then - // this has no effect, it is assumed the index would not be created if it could not be initialized - return _configBased == false || _runtimeState.Level == RuntimeLevel.Run; - } + var canInit = _runtimeState.Level == RuntimeLevel.Run; - /// - /// overridden for logging - /// - /// - protected override void OnIndexingError(IndexingErrorEventArgs ex) - { - _logger.LogError(ex.InnerException, ex.Message); - base.OnIndexingError(ex); + if (!canInit && !_hasLoggedInitLog) + { + _hasLoggedInitLog = true; + _logger.LogWarning("Runtime state is not " + RuntimeLevel.Run + ", no indexing will occur"); + } + + return canInit; } /// @@ -167,31 +103,16 @@ namespace Umbraco.Cms.Infrastructure.Examine //remove the original value so we can store it the correct way d.RemoveField(f.Key); - d.Add(new Field( + d.Add(new StringField( f.Key, f.Value[0].ToString(), - Field.Store.YES, - Field.Index.NO, //don't index this field, we never want to search by it - Field.TermVector.NO)); + Field.Store.YES)); } } base.OnDocumentWriting(docArgs); } - /// - /// Overridden for logging. - /// - protected override void AddDocument(Document doc, ValueSet valueSet, IndexWriter writer) - { - _logger.LogDebug("Write lucene doc id:{DocumentId}, category:{DocumentCategory}, type:{DocumentItemType}", - valueSet.Id, - valueSet.Category, - valueSet.ItemType); - - base.AddDocument(doc, valueSet, writer); - } - protected override void OnTransformingIndexValues(IndexingItemEventArgs e) { base.OnTransformingIndexValues(e); @@ -210,15 +131,7 @@ namespace Umbraco.Cms.Infrastructure.Examine } } - #region IIndexDiagnostics - - private readonly UmbracoExamineIndexDiagnostics _diagnostics; - - public int DocumentCount => _diagnostics.DocumentCount; - public int FieldCount => _diagnostics.FieldCount; public Attempt IsHealthy() => _diagnostics.IsHealthy(); public virtual IReadOnlyDictionary Metadata => _diagnostics.Metadata; - - #endregion } } diff --git a/src/Umbraco.Examine.Lucene/UmbracoIndexesCreator.cs b/src/Umbraco.Examine.Lucene/UmbracoIndexesCreator.cs deleted file mode 100644 index aa7b30677f..0000000000 --- a/src/Umbraco.Examine.Lucene/UmbracoIndexesCreator.cs +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright (c) Umbraco. -// See LICENSE for more details. - -using System.Collections.Generic; -using Examine; -using Examine.LuceneEngine; -using Lucene.Net.Analysis.Standard; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using Umbraco.Cms.Core.Composing; -using Umbraco.Cms.Core.Configuration.Models; -using Umbraco.Cms.Core.Hosting; -using Umbraco.Cms.Core.Logging; -using Umbraco.Cms.Core.Services; -using Constants = Umbraco.Cms.Core.Constants; - -namespace Umbraco.Cms.Infrastructure.Examine -{ - /// - /// Creates the indexes used by Umbraco - /// - public class UmbracoIndexesCreator : LuceneIndexCreator, IUmbracoIndexesCreator - { - // TODO: we should inject the different IValueSetValidator so devs can just register them instead of overriding this class? - - public UmbracoIndexesCreator( - ITypeFinder typeFinder, - IProfilingLogger profilingLogger, - ILoggerFactory loggerFactory, - ILocalizationService languageService, - IPublicAccessService publicAccessService, - IMemberService memberService, - IUmbracoIndexConfig umbracoIndexConfig, - IHostingEnvironment hostingEnvironment, - IRuntimeState runtimeState, - IOptions settings, - ILuceneDirectoryFactory directoryFactory) : base(typeFinder, hostingEnvironment, settings) - { - ProfilingLogger = profilingLogger ?? throw new System.ArgumentNullException(nameof(profilingLogger)); - LoggerFactory = loggerFactory; - LanguageService = languageService ?? throw new System.ArgumentNullException(nameof(languageService)); - PublicAccessService = publicAccessService ?? throw new System.ArgumentNullException(nameof(publicAccessService)); - MemberService = memberService ?? throw new System.ArgumentNullException(nameof(memberService)); - UmbracoIndexConfig = umbracoIndexConfig; - HostingEnvironment = hostingEnvironment ?? throw new System.ArgumentNullException(nameof(hostingEnvironment)); - RuntimeState = runtimeState ?? throw new System.ArgumentNullException(nameof(runtimeState)); - DirectoryFactory = directoryFactory; - } - - protected IProfilingLogger ProfilingLogger { get; } - protected ILoggerFactory LoggerFactory { get; } - protected IHostingEnvironment HostingEnvironment { get; } - protected IRuntimeState RuntimeState { get; } - protected ILuceneDirectoryFactory DirectoryFactory { get; } - protected ILocalizationService LanguageService { get; } - protected IPublicAccessService PublicAccessService { get; } - protected IMemberService MemberService { get; } - protected IUmbracoIndexConfig UmbracoIndexConfig { get; } - - /// - /// Creates the Umbraco indexes - /// - /// - public override IEnumerable Create() - { - return new[] - { - CreateInternalIndex(), - CreateExternalIndex(), - CreateMemberIndex() - }; - } - - private IIndex CreateInternalIndex() - { - var index = new UmbracoContentIndex( - Constants.UmbracoIndexes.InternalIndexName, - DirectoryFactory.CreateDirectory(Constants.UmbracoIndexes.InternalIndexPath), - new UmbracoFieldDefinitionCollection(), - new CultureInvariantWhitespaceAnalyzer(), - ProfilingLogger, - LoggerFactory.CreateLogger(), - LoggerFactory, - HostingEnvironment, - RuntimeState, - LanguageService, - UmbracoIndexConfig.GetContentValueSetValidator() - ); - return index; - } - - private IIndex CreateExternalIndex() - { - var index = new UmbracoContentIndex( - Constants.UmbracoIndexes.ExternalIndexName, - DirectoryFactory.CreateDirectory(Constants.UmbracoIndexes.ExternalIndexPath), - new UmbracoFieldDefinitionCollection(), - new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30), - ProfilingLogger, - LoggerFactory.CreateLogger(), - LoggerFactory, - HostingEnvironment, - RuntimeState, - LanguageService, - UmbracoIndexConfig.GetPublishedContentValueSetValidator()); - return index; - } - - private IIndex CreateMemberIndex() - { - var index = new UmbracoMemberIndex( - Constants.UmbracoIndexes.MembersIndexName, - new UmbracoFieldDefinitionCollection(), - DirectoryFactory.CreateDirectory(Constants.UmbracoIndexes.MembersIndexPath), - new CultureInvariantWhitespaceAnalyzer(), - ProfilingLogger, - LoggerFactory.CreateLogger(), - LoggerFactory, - HostingEnvironment, - RuntimeState, - UmbracoIndexConfig.GetMemberValueSetValidator() - ); - return index; - } - } -} diff --git a/src/Umbraco.Examine.Lucene/UmbracoLockFactory.cs b/src/Umbraco.Examine.Lucene/UmbracoLockFactory.cs new file mode 100644 index 0000000000..89f61c1e53 --- /dev/null +++ b/src/Umbraco.Examine.Lucene/UmbracoLockFactory.cs @@ -0,0 +1,15 @@ +// Copyright (c) Umbraco. +// See LICENSE for more details. + +using System.IO; +using Examine.Lucene.Directories; +using Lucene.Net.Store; + +namespace Umbraco.Cms.Infrastructure.Examine +{ + public class UmbracoLockFactory : ILockFactory + { + public LockFactory GetLockFactory(DirectoryInfo directory) + => new NoPrefixSimpleFsLockFactory(directory); + } +} diff --git a/src/Umbraco.Examine.Lucene/UmbracoMemberIndex.cs b/src/Umbraco.Examine.Lucene/UmbracoMemberIndex.cs index 3889209fdb..0792dd8a6f 100644 --- a/src/Umbraco.Examine.Lucene/UmbracoMemberIndex.cs +++ b/src/Umbraco.Examine.Lucene/UmbracoMemberIndex.cs @@ -1,13 +1,11 @@ -// Copyright (c) Umbraco. +// Copyright (c) Umbraco. // See LICENSE for more details. -using Examine; -using Lucene.Net.Analysis; +using Examine.Lucene; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Umbraco.Cms.Core.Hosting; -using Umbraco.Cms.Core.Logging; using Umbraco.Cms.Core.Services; -using Directory = Lucene.Net.Store.Directory; namespace Umbraco.Cms.Infrastructure.Examine { @@ -16,31 +14,14 @@ namespace Umbraco.Cms.Infrastructure.Examine /// public class UmbracoMemberIndex : UmbracoExamineIndex, IUmbracoMemberIndex { - /// - /// Constructor to allow for creating an indexer at runtime - /// - /// - /// - /// - /// - /// - /// - /// - /// public UmbracoMemberIndex( - string name, - FieldDefinitionCollection fieldDefinitions, - Directory luceneDirectory, - Analyzer analyzer, - IProfilingLogger profilingLogger, - ILogger logger, ILoggerFactory loggerFactory, + string name, + IOptionsSnapshot indexOptions, IHostingEnvironment hostingEnvironment, - IRuntimeState runtimeState, - IValueSetValidator validator = null) : - base(name, luceneDirectory, fieldDefinitions, analyzer, profilingLogger, logger, loggerFactory, hostingEnvironment, runtimeState, validator) + IRuntimeState runtimeState) + : base(loggerFactory, name, indexOptions, hostingEnvironment, runtimeState) { } - } } diff --git a/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.CoreServices.cs b/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.CoreServices.cs index 6eb08bd4d5..14457e9687 100644 --- a/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.CoreServices.cs +++ b/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.CoreServices.cs @@ -154,8 +154,6 @@ namespace Umbraco.Cms.Infrastructure.DependencyInjection builder.Services.AddUnique(); - // Register noop versions for examine to be overridden by examine - builder.Services.AddUnique(); builder.Services.AddUnique(); builder.Services.AddUnique(); @@ -170,7 +168,6 @@ namespace Umbraco.Cms.Infrastructure.DependencyInjection // Services required to run background jobs (with out the handler) builder.Services.AddUnique(); - builder.Services.AddUnique(); return builder; } diff --git a/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.DistributedCache.cs b/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.DistributedCache.cs index 0757f2c725..05dba2cc0f 100644 --- a/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.DistributedCache.cs +++ b/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.DistributedCache.cs @@ -27,7 +27,8 @@ namespace Umbraco.Cms.Infrastructure.DependencyInjection /// public static IUmbracoBuilder AddDistributedCache(this IUmbracoBuilder builder) { - builder.SetDatabaseServerMessengerCallbacks(GetCallbacks); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); builder.SetServerMessenger(); builder.AddNotificationHandler(); builder.AddNotificationHandler(); @@ -59,24 +60,6 @@ namespace Umbraco.Cms.Infrastructure.DependencyInjection public static void SetServerRegistrar(this IUmbracoBuilder builder, IServerRoleAccessor registrar) => builder.Services.AddUnique(registrar); - /// - /// Sets the database server messenger options. - /// - /// The builder. - /// A function creating the options. - /// Use DatabaseServerRegistrarAndMessengerComposer.GetDefaultOptions to get the options that Umbraco would use by default. - public static void SetDatabaseServerMessengerCallbacks(this IUmbracoBuilder builder, Func factory) - => builder.Services.AddUnique(factory); - - /// - /// Sets the database server messenger options. - /// - /// The builder. - /// Options. - /// Use DatabaseServerRegistrarAndMessengerComposer.GetDefaultOptions to get the options that Umbraco would use by default. - public static void SetDatabaseServerMessengerOptions(this IUmbracoBuilder builder, DatabaseServerMessengerCallbacks options) - => builder.Services.AddUnique(options); - /// /// Sets the server messenger. /// @@ -101,36 +84,5 @@ namespace Umbraco.Cms.Infrastructure.DependencyInjection /// A server messenger. public static void SetServerMessenger(this IUmbracoBuilder builder, IServerMessenger registrar) => builder.Services.AddUnique(registrar); - - private static DatabaseServerMessengerCallbacks GetCallbacks(IServiceProvider factory) => new DatabaseServerMessengerCallbacks - { - // These callbacks will be executed if the server has not been synced - // (i.e. it is a new server or the lastsynced.txt file has been removed) - InitializingCallbacks = new Action[] - { - // rebuild the xml cache file if the server is not synced - () => - { - IPublishedSnapshotService publishedSnapshotService = factory.GetRequiredService(); - - // rebuild the published snapshot caches entirely, if the server is not synced - // this is equivalent to DistributedCache RefreshAll... but local only - // (we really should have a way to reuse RefreshAll... locally) - // note: refresh all content & media caches does refresh content types too - publishedSnapshotService.Notify(new[] { new DomainCacheRefresher.JsonPayload(0, DomainChangeTypes.RefreshAll) }); - publishedSnapshotService.Notify(new[] { new ContentCacheRefresher.JsonPayload(0, null, TreeChangeTypes.RefreshAll) }, out _, out _); - publishedSnapshotService.Notify(new[] { new MediaCacheRefresher.JsonPayload(0, null, TreeChangeTypes.RefreshAll) }, out _); - }, - - // rebuild indexes if the server is not synced - // NOTE: This will rebuild ALL indexes including the members, if developers want to target specific - // indexes then they can adjust this logic themselves. - () => - { - var indexRebuilder = factory.GetRequiredService(); - indexRebuilder.RebuildIndexes(false, TimeSpan.FromSeconds(5)); - } - } - }; } } diff --git a/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.Examine.cs b/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.Examine.cs index 103d7a198d..d061a4372c 100644 --- a/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.Examine.cs +++ b/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.Examine.cs @@ -28,7 +28,8 @@ namespace Umbraco.Cms.Infrastructure.DependencyInjection builder.Services.AddSingleton(); builder.Services.AddSingleton(); - builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); builder.Services.AddUnique(); builder.Services.AddUnique(); builder.Services.AddUnique(factory => @@ -49,14 +50,15 @@ namespace Umbraco.Cms.Infrastructure.DependencyInjection false)); builder.Services.AddUnique, MediaValueSetBuilder>(); builder.Services.AddUnique, MemberValueSetBuilder>(); - builder.Services.AddUnique(); + builder.Services.AddUnique(); - builder.AddNotificationHandler(); - builder.AddNotificationHandler(); - builder.AddNotificationHandler(); - builder.AddNotificationHandler(); - builder.AddNotificationHandler(); - builder.AddNotificationHandler(); + builder.AddNotificationHandler(); + builder.AddNotificationHandler(); + builder.AddNotificationHandler(); + builder.AddNotificationHandler(); + builder.AddNotificationHandler(); + + builder.AddNotificationHandler(); return builder; } diff --git a/src/Umbraco.Infrastructure/Examine/ContentIndexPopulator.cs b/src/Umbraco.Infrastructure/Examine/ContentIndexPopulator.cs index d9fd10f1d7..bd205e2009 100644 --- a/src/Umbraco.Infrastructure/Examine/ContentIndexPopulator.cs +++ b/src/Umbraco.Infrastructure/Examine/ContentIndexPopulator.cs @@ -1,7 +1,8 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using Examine; +using Microsoft.Extensions.Logging; using Umbraco.Cms.Core; using Umbraco.Cms.Core.Models; using Umbraco.Cms.Core.Persistence.Querying; @@ -27,6 +28,7 @@ namespace Umbraco.Cms.Infrastructure.Examine private readonly bool _publishedValuesOnly; private readonly int? _parentId; + private readonly ILogger _logger; /// /// Default constructor to lookup all content data @@ -34,20 +36,30 @@ namespace Umbraco.Cms.Infrastructure.Examine /// /// /// - public ContentIndexPopulator(IContentService contentService, IUmbracoDatabaseFactory umbracoDatabaseFactory, IContentValueSetBuilder contentValueSetBuilder) - : this(false, null, contentService, umbracoDatabaseFactory, contentValueSetBuilder) + public ContentIndexPopulator( + ILogger logger, + IContentService contentService, + IUmbracoDatabaseFactory umbracoDatabaseFactory, + IContentValueSetBuilder contentValueSetBuilder) + : this(logger, false, null, contentService, umbracoDatabaseFactory, contentValueSetBuilder) { } /// /// Optional constructor allowing specifying custom query parameters /// - public ContentIndexPopulator(bool publishedValuesOnly, int? parentId, IContentService contentService, IUmbracoDatabaseFactory umbracoDatabaseFactory, IValueSetBuilder contentValueSetBuilder) + public ContentIndexPopulator( + ILogger logger, + bool publishedValuesOnly, + int? parentId, + IContentService contentService, + IUmbracoDatabaseFactory umbracoDatabaseFactory, + IValueSetBuilder contentValueSetBuilder) { - if (umbracoDatabaseFactory == null) throw new ArgumentNullException(nameof(umbracoDatabaseFactory)); _contentService = contentService ?? throw new ArgumentNullException(nameof(contentService)); - _umbracoDatabaseFactory = umbracoDatabaseFactory; + _umbracoDatabaseFactory = umbracoDatabaseFactory ?? throw new ArgumentNullException(nameof(umbracoDatabaseFactory)); _contentValueSetBuilder = contentValueSetBuilder ?? throw new ArgumentNullException(nameof(contentValueSetBuilder)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _publishedValuesOnly = publishedValuesOnly; _parentId = parentId; } @@ -60,7 +72,11 @@ namespace Umbraco.Cms.Infrastructure.Examine protected override void PopulateIndexes(IReadOnlyList indexes) { - if (indexes.Count == 0) return; + if (indexes.Count == 0) + { + _logger.LogDebug($"{nameof(PopulateIndexes)} called with no indexes to populate. Typically means no index is registered with this populator."); + return; + } const int pageSize = 10000; var pageIndex = 0; @@ -144,9 +160,10 @@ namespace Umbraco.Cms.Infrastructure.Examine var valueSets = _contentValueSetBuilder.GetValueSets(indexableContent.ToArray()).ToList(); - // ReSharper disable once PossibleMultipleEnumeration - foreach (var index in indexes) + foreach (IIndex index in indexes) + { index.IndexItems(valueSets); + } } pageIndex++; diff --git a/src/Umbraco.Infrastructure/Examine/ContentValueSetValidator.cs b/src/Umbraco.Infrastructure/Examine/ContentValueSetValidator.cs index 463e8dee26..39d260a24d 100644 --- a/src/Umbraco.Infrastructure/Examine/ContentValueSetValidator.cs +++ b/src/Umbraco.Infrastructure/Examine/ContentValueSetValidator.cs @@ -1,4 +1,4 @@ -using System.Collections.Generic; +using System.Collections.Generic; using System.Linq; using Examine; using Umbraco.Cms.Core.Scoping; @@ -106,10 +106,14 @@ namespace Umbraco.Cms.Infrastructure.Examine if (valueSet.Category == IndexTypes.Content && PublishedValuesOnly) { if (!valueSet.Values.TryGetValue(UmbracoExamineFieldNames.PublishedFieldName, out var published)) + { return ValueSetValidationResult.Failed; + } if (!published[0].Equals("y")) + { return ValueSetValidationResult.Failed; + } //deal with variants, if there are unpublished variants than we need to remove them from the value set if (valueSet.Values.TryGetValue(UmbracoExamineFieldNames.VariesByCultureFieldName, out var variesByCulture) diff --git a/src/Umbraco.Infrastructure/Search/ExamineIndexModel.cs b/src/Umbraco.Infrastructure/Examine/ExamineIndexModel.cs similarity index 88% rename from src/Umbraco.Infrastructure/Search/ExamineIndexModel.cs rename to src/Umbraco.Infrastructure/Examine/ExamineIndexModel.cs index d14cef8ccf..ff9f499217 100644 --- a/src/Umbraco.Infrastructure/Search/ExamineIndexModel.cs +++ b/src/Umbraco.Infrastructure/Examine/ExamineIndexModel.cs @@ -1,7 +1,7 @@ -using System.Collections.Generic; +using System.Collections.Generic; using System.Runtime.Serialization; -namespace Umbraco.Cms.Infrastructure.Search +namespace Umbraco.Cms.Infrastructure.Examine { [DataContract(Name = "indexer", Namespace = "")] public class ExamineIndexModel diff --git a/src/Umbraco.Infrastructure/Examine/ExamineIndexRebuilder.cs b/src/Umbraco.Infrastructure/Examine/ExamineIndexRebuilder.cs new file mode 100644 index 0000000000..d7719cfd40 --- /dev/null +++ b/src/Umbraco.Infrastructure/Examine/ExamineIndexRebuilder.cs @@ -0,0 +1,207 @@ +// Copyright (c) Umbraco. +// See LICENSE for more details. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Examine; +using Microsoft.Extensions.Logging; +using Umbraco.Cms.Core; +using Umbraco.Cms.Core.Runtime; +using Umbraco.Cms.Core.Services; +using Umbraco.Cms.Infrastructure.HostedServices; + +namespace Umbraco.Cms.Infrastructure.Examine +{ + public class ExamineIndexRebuilder : IIndexRebuilder + { + private readonly IBackgroundTaskQueue _backgroundTaskQueue; + private readonly IMainDom _mainDom; + private readonly IRuntimeState _runtimeState; + private readonly ILogger _logger; + private readonly IExamineManager _examineManager; + private readonly IEnumerable _populators; + private readonly object _rebuildLocker = new(); + + /// + /// Initializes a new instance of the class. + /// + public ExamineIndexRebuilder( + IMainDom mainDom, + IRuntimeState runtimeState, + ILogger logger, + IExamineManager examineManager, + IEnumerable populators, + IBackgroundTaskQueue backgroundTaskQueue) + { + _mainDom = mainDom; + _runtimeState = runtimeState; + _logger = logger; + _examineManager = examineManager; + _populators = populators; + _backgroundTaskQueue = backgroundTaskQueue; + } + + public bool CanRebuild(string indexName) + { + if (!_examineManager.TryGetIndex(indexName, out IIndex index)) + { + throw new InvalidOperationException("No index found by name " + indexName); + } + + return _populators.Any(x => x.IsRegistered(index)); + } + + public virtual void RebuildIndex(string indexName, TimeSpan? delay = null, bool useBackgroundThread = true) + { + if (delay == null) + { + delay = TimeSpan.Zero; + } + + if (!CanRun()) + { + return; + } + + if (useBackgroundThread) + { + _logger.LogInformation($"Starting async background thread for rebuilding index {indexName}."); + + _backgroundTaskQueue.QueueBackgroundWorkItem( + cancellationToken => Task.Run(() => RebuildIndex(indexName, delay.Value, cancellationToken))); + } + else + { + RebuildIndex(indexName, delay.Value, CancellationToken.None); + } + } + + public virtual void RebuildIndexes(bool onlyEmptyIndexes, TimeSpan? delay = null, bool useBackgroundThread = true) + { + if (delay == null) + { + delay = TimeSpan.Zero; + } + + if (!CanRun()) + { + return; + } + + if (useBackgroundThread) + { + _logger.LogInformation($"Starting async background thread for {nameof(RebuildIndexes)}."); + + _backgroundTaskQueue.QueueBackgroundWorkItem( + cancellationToken => Task.Run(() => RebuildIndexes(onlyEmptyIndexes, delay.Value, cancellationToken))); + } + else + { + RebuildIndexes(onlyEmptyIndexes, delay.Value, CancellationToken.None); + } + } + + private bool CanRun() => _mainDom.IsMainDom && _runtimeState.Level >= RuntimeLevel.Run; + + private void RebuildIndex(string indexName, TimeSpan delay, CancellationToken cancellationToken) + { + if (delay > TimeSpan.Zero) + { + Thread.Sleep(delay); + } + + try + { + if (!Monitor.TryEnter(_rebuildLocker)) + { + _logger.LogWarning("Call was made to RebuildIndexes but the task runner for rebuilding is already running"); + } + else + { + if (!_examineManager.TryGetIndex(indexName, out IIndex index)) + { + throw new InvalidOperationException($"No index found with name {indexName}"); + } + + index.CreateIndex(); // clear the index + foreach (IIndexPopulator populator in _populators) + { + if (cancellationToken.IsCancellationRequested) + { + return; + } + + populator.Populate(index); + } + } + } + finally + { + if (Monitor.IsEntered(_rebuildLocker)) + { + Monitor.Exit(_rebuildLocker); + } + } + } + + private void RebuildIndexes(bool onlyEmptyIndexes, TimeSpan delay, CancellationToken cancellationToken) + { + if (delay > TimeSpan.Zero) + { + Thread.Sleep(delay); + } + + try + { + if (!Monitor.TryEnter(_rebuildLocker)) + { + _logger.LogWarning($"Call was made to {nameof(RebuildIndexes)} but the task runner for rebuilding is already running"); + } + else + { + IIndex[] indexes = (onlyEmptyIndexes + ? _examineManager.Indexes.Where(x => !x.IndexExists()) + : _examineManager.Indexes).ToArray(); + + if (indexes.Length == 0) + { + return; + } + + foreach (IIndex index in indexes) + { + index.CreateIndex(); // clear the index + } + + // run each populator over the indexes + foreach (IIndexPopulator populator in _populators) + { + if (cancellationToken.IsCancellationRequested) + { + return; + } + + try + { + populator.Populate(indexes); + } + catch (Exception e) + { + _logger.LogError(e, "Index populating failed for populator {Populator}", populator.GetType()); + } + } + } + } + finally + { + if (Monitor.IsEntered(_rebuildLocker)) + { + Monitor.Exit(_rebuildLocker); + } + } + } + } +} diff --git a/src/Umbraco.Infrastructure/Search/ExamineSearcherModel.cs b/src/Umbraco.Infrastructure/Examine/ExamineSearcherModel.cs similarity index 74% rename from src/Umbraco.Infrastructure/Search/ExamineSearcherModel.cs rename to src/Umbraco.Infrastructure/Examine/ExamineSearcherModel.cs index 8e6ea30c0c..c4b602e430 100644 --- a/src/Umbraco.Infrastructure/Search/ExamineSearcherModel.cs +++ b/src/Umbraco.Infrastructure/Examine/ExamineSearcherModel.cs @@ -1,6 +1,6 @@ -using System.Runtime.Serialization; +using System.Runtime.Serialization; -namespace Umbraco.Cms.Infrastructure.Search +namespace Umbraco.Cms.Infrastructure.Examine { [DataContract(Name = "searcher", Namespace = "")] public class ExamineSearcherModel diff --git a/src/Umbraco.Infrastructure/Examine/ExamineUmbracoIndexingHandler.cs b/src/Umbraco.Infrastructure/Examine/ExamineUmbracoIndexingHandler.cs new file mode 100644 index 0000000000..6c6d209e5a --- /dev/null +++ b/src/Umbraco.Infrastructure/Examine/ExamineUmbracoIndexingHandler.cs @@ -0,0 +1,394 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Threading.Tasks; +using Examine; +using Examine.Search; +using Microsoft.Extensions.Logging; +using Umbraco.Cms.Core.Logging; +using Umbraco.Cms.Core.Models; +using Umbraco.Cms.Core.Runtime; +using Umbraco.Cms.Core.Scoping; +using Umbraco.Cms.Infrastructure.HostedServices; +using Umbraco.Cms.Infrastructure.Search; +using Umbraco.Extensions; + +namespace Umbraco.Cms.Infrastructure.Examine +{ + /// + /// Indexing handler for Examine indexes + /// + internal class ExamineUmbracoIndexingHandler : IUmbracoIndexingHandler + { + // the default enlist priority is 100 + // enlist with a lower priority to ensure that anything "default" runs after us + // but greater that SafeXmlReaderWriter priority which is 60 + private const int EnlistPriority = 80; + private readonly IMainDom _mainDom; + private readonly ILogger _logger; + private readonly IProfilingLogger _profilingLogger; + private readonly IScopeProvider _scopeProvider; + private readonly IExamineManager _examineManager; + private readonly IBackgroundTaskQueue _backgroundTaskQueue; + private readonly IContentValueSetBuilder _contentValueSetBuilder; + private readonly IPublishedContentValueSetBuilder _publishedContentValueSetBuilder; + private readonly IValueSetBuilder _mediaValueSetBuilder; + private readonly IValueSetBuilder _memberValueSetBuilder; + private readonly Lazy _enabled; + + public ExamineUmbracoIndexingHandler( + IMainDom mainDom, + ILogger logger, + IProfilingLogger profilingLogger, + IScopeProvider scopeProvider, + IExamineManager examineManager, + IBackgroundTaskQueue backgroundTaskQueue, + IContentValueSetBuilder contentValueSetBuilder, + IPublishedContentValueSetBuilder publishedContentValueSetBuilder, + IValueSetBuilder mediaValueSetBuilder, + IValueSetBuilder memberValueSetBuilder) + { + _mainDom = mainDom; + _logger = logger; + _profilingLogger = profilingLogger; + _scopeProvider = scopeProvider; + _examineManager = examineManager; + _backgroundTaskQueue = backgroundTaskQueue; + _contentValueSetBuilder = contentValueSetBuilder; + _publishedContentValueSetBuilder = publishedContentValueSetBuilder; + _mediaValueSetBuilder = mediaValueSetBuilder; + _memberValueSetBuilder = memberValueSetBuilder; + _enabled = new Lazy(IsEnabled); + } + + /// + /// Used to lazily check if Examine Index handling is enabled + /// + /// + private bool IsEnabled() + { + //let's deal with shutting down Examine with MainDom + var examineShutdownRegistered = _mainDom.Register(release: () => + { + using (_profilingLogger.TraceDuration("Examine shutting down")) + { + _examineManager.Dispose(); + } + }); + + if (!examineShutdownRegistered) + { + _logger.LogInformation("Examine shutdown not registered, this AppDomain is not the MainDom, Examine will be disabled"); + + //if we could not register the shutdown examine ourselves, it means we are not maindom! in this case all of examine should be disabled! + Suspendable.ExamineEvents.SuspendIndexers(_logger); + return false; //exit, do not continue + } + + _logger.LogDebug("Examine shutdown registered with MainDom"); + + var registeredIndexers = _examineManager.Indexes.OfType().Count(x => x.EnableDefaultEventHandler); + + _logger.LogInformation("Adding examine event handlers for {RegisteredIndexers} index providers.", registeredIndexers); + + // don't bind event handlers if we're not suppose to listen + if (registeredIndexers == 0) + { + return false; + } + + return true; + } + + /// + public bool Enabled => _enabled.Value; + + /// + public void DeleteIndexForEntity(int entityId, bool keepIfUnpublished) + { + var actions = DeferedActions.Get(_scopeProvider); + if (actions != null) + { + actions.Add(new DeferedDeleteIndex(this, entityId, keepIfUnpublished)); + } + else + { + DeferedDeleteIndex.Execute(this, entityId, keepIfUnpublished); + } + } + + /// + public void ReIndexForContent(IContent sender, bool isPublished) + { + var actions = DeferedActions.Get(_scopeProvider); + if (actions != null) + { + actions.Add(new DeferedReIndexForContent(_backgroundTaskQueue, this, sender, isPublished)); + } + else + { + DeferedReIndexForContent.Execute(_backgroundTaskQueue, this, sender, isPublished); + } + } + + /// + public void ReIndexForMedia(IMedia sender, bool isPublished) + { + var actions = DeferedActions.Get(_scopeProvider); + if (actions != null) + { + actions.Add(new DeferedReIndexForMedia(_backgroundTaskQueue, this, sender, isPublished)); + } + else + { + DeferedReIndexForMedia.Execute(_backgroundTaskQueue, this, sender, isPublished); + } + } + + /// + public void ReIndexForMember(IMember member) + { + var actions = DeferedActions.Get(_scopeProvider); + if (actions != null) + { + actions.Add(new DeferedReIndexForMember(_backgroundTaskQueue, this, member)); + } + else + { + DeferedReIndexForMember.Execute(_backgroundTaskQueue, this, member); + } + } + + /// + public void DeleteDocumentsForContentTypes(IReadOnlyCollection removedContentTypes) + { + const int pageSize = 500; + + //Delete all content of this content/media/member type that is in any content indexer by looking up matched examine docs + foreach (var id in removedContentTypes) + { + foreach (var index in _examineManager.Indexes.OfType()) + { + var page = 0; + var total = long.MaxValue; + while (page * pageSize < total) + { + //paging with examine, see https://shazwazza.com/post/paging-with-examine/ + var results = index.Searcher + .CreateQuery() + .Field("nodeType", id.ToInvariantString()) + .Execute(QueryOptions.SkipTake(page * pageSize, pageSize)); + total = results.TotalItemCount; + var paged = results.Skip(page * pageSize); + + foreach (ISearchResult item in paged) + { + if (int.TryParse(item.Id, out int contentId)) + { + DeleteIndexForEntity(contentId, false); + } + } + + page++; + } + } + } + } + + #region Deferred Actions + private class DeferedActions + { + private readonly List _actions = new List(); + + public static DeferedActions Get(IScopeProvider scopeProvider) + { + IScopeContext scopeContext = scopeProvider.Context; + + return scopeContext?.Enlist("examineEvents", + () => new DeferedActions(), // creator + (completed, actions) => // action + { + if (completed) + { + actions.Execute(); + } + }, EnlistPriority); + } + + public void Add(DeferedAction action) => _actions.Add(action); + + private void Execute() + { + foreach (DeferedAction action in _actions) + { + action.Execute(); + } + } + } + + /// + /// An action that will execute at the end of the Scope being completed + /// + private abstract class DeferedAction + { + public virtual void Execute() + { } + } + + /// + /// Re-indexes an item on a background thread + /// + private class DeferedReIndexForContent : DeferedAction + { + private readonly IBackgroundTaskQueue _backgroundTaskQueue; + private readonly ExamineUmbracoIndexingHandler _ExamineUmbracoIndexingHandler; + private readonly IContent _content; + private readonly bool _isPublished; + + public DeferedReIndexForContent(IBackgroundTaskQueue backgroundTaskQueue, ExamineUmbracoIndexingHandler ExamineUmbracoIndexingHandler, IContent content, bool isPublished) + { + _backgroundTaskQueue = backgroundTaskQueue; + _ExamineUmbracoIndexingHandler = ExamineUmbracoIndexingHandler; + _content = content; + _isPublished = isPublished; + } + + public override void Execute() => Execute(_backgroundTaskQueue, _ExamineUmbracoIndexingHandler, _content, _isPublished); + + public static void Execute(IBackgroundTaskQueue backgroundTaskQueue, ExamineUmbracoIndexingHandler ExamineUmbracoIndexingHandler, IContent content, bool isPublished) + => backgroundTaskQueue.QueueBackgroundWorkItem(cancellationToken => + { + using IScope scope = ExamineUmbracoIndexingHandler._scopeProvider.CreateScope(autoComplete: true); + + // for content we have a different builder for published vs unpublished + // we don't want to build more value sets than is needed so we'll lazily build 2 one for published one for non-published + var builders = new Dictionary>> + { + [true] = new Lazy>(() => ExamineUmbracoIndexingHandler._publishedContentValueSetBuilder.GetValueSets(content).ToList()), + [false] = new Lazy>(() => ExamineUmbracoIndexingHandler._contentValueSetBuilder.GetValueSets(content).ToList()) + }; + + foreach (IUmbracoIndex index in ExamineUmbracoIndexingHandler._examineManager.Indexes.OfType() + //filter the indexers + .Where(x => isPublished || !x.PublishedValuesOnly) + .Where(x => x.EnableDefaultEventHandler)) + { + if (cancellationToken.IsCancellationRequested) + { + return Task.CompletedTask; + } + + List valueSet = builders[index.PublishedValuesOnly].Value; + index.IndexItems(valueSet); + } + + return Task.CompletedTask; + }); + } + + /// + /// Re-indexes an item on a background thread + /// + private class DeferedReIndexForMedia : DeferedAction + { + private readonly IBackgroundTaskQueue _backgroundTaskQueue; + private readonly ExamineUmbracoIndexingHandler _ExamineUmbracoIndexingHandler; + private readonly IMedia _media; + private readonly bool _isPublished; + + public DeferedReIndexForMedia(IBackgroundTaskQueue backgroundTaskQueue, ExamineUmbracoIndexingHandler ExamineUmbracoIndexingHandler, IMedia media, bool isPublished) + { + _backgroundTaskQueue = backgroundTaskQueue; + _ExamineUmbracoIndexingHandler = ExamineUmbracoIndexingHandler; + _media = media; + _isPublished = isPublished; + } + + public override void Execute() => Execute(_backgroundTaskQueue, _ExamineUmbracoIndexingHandler, _media, _isPublished); + + public static void Execute(IBackgroundTaskQueue backgroundTaskQueue, ExamineUmbracoIndexingHandler ExamineUmbracoIndexingHandler, IMedia media, bool isPublished) => + // perform the ValueSet lookup on a background thread + backgroundTaskQueue.QueueBackgroundWorkItem(cancellationToken => + { + using IScope scope = ExamineUmbracoIndexingHandler._scopeProvider.CreateScope(autoComplete: true); + + var valueSet = ExamineUmbracoIndexingHandler._mediaValueSetBuilder.GetValueSets(media).ToList(); + + foreach (IUmbracoIndex index in ExamineUmbracoIndexingHandler._examineManager.Indexes.OfType() + //filter the indexers + .Where(x => isPublished || !x.PublishedValuesOnly) + .Where(x => x.EnableDefaultEventHandler)) + { + index.IndexItems(valueSet); + } + + return Task.CompletedTask; + }); + } + + /// + /// Re-indexes an item on a background thread + /// + private class DeferedReIndexForMember : DeferedAction + { + private readonly ExamineUmbracoIndexingHandler _ExamineUmbracoIndexingHandler; + private readonly IMember _member; + private readonly IBackgroundTaskQueue _backgroundTaskQueue; + + public DeferedReIndexForMember(IBackgroundTaskQueue backgroundTaskQueue, ExamineUmbracoIndexingHandler ExamineUmbracoIndexingHandler, IMember member) + { + _ExamineUmbracoIndexingHandler = ExamineUmbracoIndexingHandler; + _member = member; + _backgroundTaskQueue = backgroundTaskQueue; + } + + public override void Execute() => Execute(_backgroundTaskQueue, _ExamineUmbracoIndexingHandler, _member); + + public static void Execute(IBackgroundTaskQueue backgroundTaskQueue, ExamineUmbracoIndexingHandler ExamineUmbracoIndexingHandler, IMember member) => + // perform the ValueSet lookup on a background thread + backgroundTaskQueue.QueueBackgroundWorkItem(cancellationToken => + { + using IScope scope = ExamineUmbracoIndexingHandler._scopeProvider.CreateScope(autoComplete: true); + + var valueSet = ExamineUmbracoIndexingHandler._memberValueSetBuilder.GetValueSets(member).ToList(); + foreach (IUmbracoIndex index in ExamineUmbracoIndexingHandler._examineManager.Indexes.OfType() + //filter the indexers + .Where(x => x.EnableDefaultEventHandler)) + { + index.IndexItems(valueSet); + } + + return Task.CompletedTask; + }); + } + + private class DeferedDeleteIndex : DeferedAction + { + private readonly ExamineUmbracoIndexingHandler _ExamineUmbracoIndexingHandler; + private readonly int _id; + private readonly bool _keepIfUnpublished; + + public DeferedDeleteIndex(ExamineUmbracoIndexingHandler ExamineUmbracoIndexingHandler, int id, bool keepIfUnpublished) + { + _ExamineUmbracoIndexingHandler = ExamineUmbracoIndexingHandler; + _id = id; + _keepIfUnpublished = keepIfUnpublished; + } + + public override void Execute() => Execute(_ExamineUmbracoIndexingHandler, _id, _keepIfUnpublished); + + public static void Execute(ExamineUmbracoIndexingHandler ExamineUmbracoIndexingHandler, int id, bool keepIfUnpublished) + { + var strId = id.ToString(CultureInfo.InvariantCulture); + foreach (var index in ExamineUmbracoIndexingHandler._examineManager.Indexes.OfType() + .Where(x => x.PublishedValuesOnly || !keepIfUnpublished) + .Where(x => x.EnableDefaultEventHandler)) + { + index.DeleteFromIndex(strId); + } + } + } + #endregion + } +} diff --git a/src/Umbraco.Infrastructure/Examine/GenericIndexDiagnostics.cs b/src/Umbraco.Infrastructure/Examine/GenericIndexDiagnostics.cs index 8c05926483..2ff01c51dc 100644 --- a/src/Umbraco.Infrastructure/Examine/GenericIndexDiagnostics.cs +++ b/src/Umbraco.Infrastructure/Examine/GenericIndexDiagnostics.cs @@ -1,6 +1,7 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; +using System.Threading.Tasks; using Examine; using Umbraco.Cms.Core; using Umbraco.Cms.Core.Composing; @@ -15,12 +16,9 @@ namespace Umbraco.Cms.Infrastructure.Examine public class GenericIndexDiagnostics : IIndexDiagnostics { private readonly IIndex _index; - private static readonly string[] IgnoreProperties = { "Description" }; + private static readonly string[] s_ignoreProperties = { "Description" }; - public GenericIndexDiagnostics(IIndex index) - { - _index = index; - } + public GenericIndexDiagnostics(IIndex index) => _index = index; public int DocumentCount => -1; //unknown @@ -33,8 +31,7 @@ namespace Umbraco.Cms.Infrastructure.Examine try { - var searcher = _index.GetSearcher(); - var result = searcher.Search("test"); + var result = _index.Searcher.Search("test"); return Attempt.Succeed(); //if we can search we'll assume it's healthy } catch (Exception e) @@ -43,6 +40,10 @@ namespace Umbraco.Cms.Infrastructure.Examine } } + public long GetDocumentCount() => -1L; + + public IEnumerable GetFieldNames() => Enumerable.Empty(); + public IReadOnlyDictionary Metadata { get @@ -50,7 +51,7 @@ namespace Umbraco.Cms.Infrastructure.Examine var result = new Dictionary(); var props = TypeHelper.CachedDiscoverableProperties(_index.GetType(), mustWrite: false) - .Where(x => IgnoreProperties.InvariantContains(x.Name) == false) + .Where(x => s_ignoreProperties.InvariantContains(x.Name) == false) .OrderBy(x => x.Name); foreach (var p in props) diff --git a/src/Umbraco.Infrastructure/Examine/IIndexCreator.cs b/src/Umbraco.Infrastructure/Examine/IIndexCreator.cs deleted file mode 100644 index aadaa00f46..0000000000 --- a/src/Umbraco.Infrastructure/Examine/IIndexCreator.cs +++ /dev/null @@ -1,13 +0,0 @@ -using System.Collections.Generic; -using Examine; - -namespace Umbraco.Cms.Infrastructure.Examine -{ - /// - /// Creates 's - /// - public interface IIndexCreator - { - IEnumerable Create(); - } -} diff --git a/src/Umbraco.Infrastructure/Examine/IIndexDiagnostics.cs b/src/Umbraco.Infrastructure/Examine/IIndexDiagnostics.cs index a4e1c0ca4f..716b7731eb 100644 --- a/src/Umbraco.Infrastructure/Examine/IIndexDiagnostics.cs +++ b/src/Umbraco.Infrastructure/Examine/IIndexDiagnostics.cs @@ -1,4 +1,6 @@ -using System.Collections.Generic; +using System.Collections.Generic; +using System.Threading.Tasks; +using Examine; using Umbraco.Cms.Core; namespace Umbraco.Cms.Infrastructure.Examine @@ -7,18 +9,8 @@ namespace Umbraco.Cms.Infrastructure.Examine /// /// Exposes diagnostic information about an index /// - public interface IIndexDiagnostics + public interface IIndexDiagnostics : IIndexStats { - /// - /// The number of documents in the index - /// - int DocumentCount { get; } - - /// - /// The number of fields in the index - /// - int FieldCount { get; } - /// /// If the index can be open/read /// diff --git a/src/Umbraco.Infrastructure/Examine/IIndexRebuilder.cs b/src/Umbraco.Infrastructure/Examine/IIndexRebuilder.cs new file mode 100644 index 0000000000..127a20d685 --- /dev/null +++ b/src/Umbraco.Infrastructure/Examine/IIndexRebuilder.cs @@ -0,0 +1,14 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Examine; + +namespace Umbraco.Cms.Infrastructure.Examine +{ + public interface IIndexRebuilder + { + bool CanRebuild(string indexName); + void RebuildIndex(string indexName, TimeSpan? delay = null, bool useBackgroundThread = true); + void RebuildIndexes(bool onlyEmptyIndexes, TimeSpan? delay = null, bool useBackgroundThread = true); + } +} diff --git a/src/Umbraco.Infrastructure/Examine/IUmbracoIndex.cs b/src/Umbraco.Infrastructure/Examine/IUmbracoIndex.cs index 8dfdf6d812..f2221e5c91 100644 --- a/src/Umbraco.Infrastructure/Examine/IUmbracoIndex.cs +++ b/src/Umbraco.Infrastructure/Examine/IUmbracoIndex.cs @@ -1,4 +1,4 @@ -using System.Collections.Generic; +using System.Collections.Generic; using Examine; namespace Umbraco.Cms.Infrastructure.Examine @@ -6,7 +6,7 @@ namespace Umbraco.Cms.Infrastructure.Examine /// /// A Marker interface for defining an Umbraco indexer /// - public interface IUmbracoIndex : IIndex + public interface IUmbracoIndex : IIndex, IIndexStats { /// /// When set to true Umbraco will keep the index in sync with Umbraco data automatically @@ -22,11 +22,5 @@ namespace Umbraco.Cms.Infrastructure.Examine /// * non-published Variants /// bool PublishedValuesOnly { get; } - - /// - /// Returns a list of all indexed fields - /// - /// - IEnumerable GetFields(); } } diff --git a/src/Umbraco.Infrastructure/Examine/IUmbracoIndexesCreator.cs b/src/Umbraco.Infrastructure/Examine/IUmbracoIndexesCreator.cs deleted file mode 100644 index df61901dba..0000000000 --- a/src/Umbraco.Infrastructure/Examine/IUmbracoIndexesCreator.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace Umbraco.Cms.Infrastructure.Examine -{ - /// - /// - /// Used to create the Umbraco indexes - /// - public interface IUmbracoIndexesCreator : IIndexCreator - { - } -} diff --git a/src/Umbraco.Infrastructure/Examine/IndexDiagnosticsFactory.cs b/src/Umbraco.Infrastructure/Examine/IndexDiagnosticsFactory.cs index ca2e732071..a60a373e65 100644 --- a/src/Umbraco.Infrastructure/Examine/IndexDiagnosticsFactory.cs +++ b/src/Umbraco.Infrastructure/Examine/IndexDiagnosticsFactory.cs @@ -1,4 +1,4 @@ -using Examine; +using Examine; namespace Umbraco.Cms.Infrastructure.Examine { @@ -9,8 +9,11 @@ namespace Umbraco.Cms.Infrastructure.Examine { public virtual IIndexDiagnostics Create(IIndex index) { - if (!(index is IIndexDiagnostics indexDiag)) + if (index is not IIndexDiagnostics indexDiag) + { indexDiag = new GenericIndexDiagnostics(index); + } + return indexDiag; } } diff --git a/src/Umbraco.Infrastructure/Examine/IndexPopulator.cs b/src/Umbraco.Infrastructure/Examine/IndexPopulator.cs index 2feac0710a..d32470d875 100644 --- a/src/Umbraco.Infrastructure/Examine/IndexPopulator.cs +++ b/src/Umbraco.Infrastructure/Examine/IndexPopulator.cs @@ -1,4 +1,4 @@ -using System.Collections.Generic; +using System.Collections.Generic; using System.Linq; using Examine; using Umbraco.Cms.Core.Collections; diff --git a/src/Umbraco.Infrastructure/Examine/IndexRebuilder.cs b/src/Umbraco.Infrastructure/Examine/IndexRebuilder.cs deleted file mode 100644 index 9e4fe6fed0..0000000000 --- a/src/Umbraco.Infrastructure/Examine/IndexRebuilder.cs +++ /dev/null @@ -1,81 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using Examine; -using Microsoft.Extensions.Logging; -using Umbraco.Cms.Core.Logging; - -namespace Umbraco.Cms.Infrastructure.Examine -{ - - /// - /// Utility to rebuild all indexes ensuring minimal data queries - /// - public class IndexRebuilder - { - private readonly IProfilingLogger _profilingLogger; - private readonly ILogger _logger; - private readonly IEnumerable _populators; - public IExamineManager ExamineManager { get; } - - public IndexRebuilder(IProfilingLogger profilingLogger , ILogger logger, IExamineManager examineManager, IEnumerable populators) - { - _profilingLogger = profilingLogger ; - _populators = populators; - _logger = logger; - ExamineManager = examineManager; - } - - public bool CanRebuild(IIndex index) - { - return _populators.Any(x => x.IsRegistered(index)); - } - - public void RebuildIndex(string indexName) - { - if (!ExamineManager.TryGetIndex(indexName, out var index)) - throw new InvalidOperationException($"No index found with name {indexName}"); - index.CreateIndex(); // clear the index - foreach (var populator in _populators) - { - populator.Populate(index); - } - } - - public void RebuildIndexes(bool onlyEmptyIndexes) - { - var indexes = (onlyEmptyIndexes - ? ExamineManager.Indexes.Where(x => !x.IndexExists()) - : ExamineManager.Indexes).ToArray(); - - if (indexes.Length == 0) return; - - OnRebuildingIndexes(new IndexRebuildingEventArgs(indexes)); - - foreach (var index in indexes) - { - index.CreateIndex(); // clear the index - } - - // run each populator over the indexes - foreach(var populator in _populators) - { - try - { - populator.Populate(indexes); - } - catch (Exception e) - { - _logger.LogError(e, "Index populating failed for populator {Populator}", populator.GetType()); - } - } - } - - /// - /// Event raised when indexes are being rebuilt - /// - public event EventHandler RebuildingIndexes; - - private void OnRebuildingIndexes(IndexRebuildingEventArgs args) => RebuildingIndexes?.Invoke(this, args); - } -} diff --git a/src/Umbraco.Infrastructure/Examine/IndexRebuildingEventArgs.cs b/src/Umbraco.Infrastructure/Examine/IndexRebuildingEventArgs.cs deleted file mode 100644 index fbe3dbcbe3..0000000000 --- a/src/Umbraco.Infrastructure/Examine/IndexRebuildingEventArgs.cs +++ /dev/null @@ -1,19 +0,0 @@ -using System; -using System.Collections.Generic; -using Examine; - -namespace Umbraco.Cms.Infrastructure.Examine -{ - public class IndexRebuildingEventArgs : EventArgs - { - public IndexRebuildingEventArgs(IEnumerable indexes) - { - Indexes = indexes; - } - - /// - /// The indexes being rebuilt - /// - public IEnumerable Indexes { get; } - } -} diff --git a/src/Umbraco.Infrastructure/Examine/MediaIndexPopulator.cs b/src/Umbraco.Infrastructure/Examine/MediaIndexPopulator.cs index 429285fa85..9f6e33f8dd 100644 --- a/src/Umbraco.Infrastructure/Examine/MediaIndexPopulator.cs +++ b/src/Umbraco.Infrastructure/Examine/MediaIndexPopulator.cs @@ -1,6 +1,7 @@ -using System.Collections.Generic; +using System.Collections.Generic; using System.Linq; using Examine; +using Microsoft.Extensions.Logging; using Umbraco.Cms.Core.Models; using Umbraco.Cms.Core.Services; @@ -11,6 +12,7 @@ namespace Umbraco.Cms.Infrastructure.Examine /// public class MediaIndexPopulator : IndexPopulator { + private readonly ILogger _logger; private readonly int? _parentId; private readonly IMediaService _mediaService; private readonly IValueSetBuilder _mediaValueSetBuilder; @@ -20,8 +22,8 @@ namespace Umbraco.Cms.Infrastructure.Examine /// /// /// - public MediaIndexPopulator(IMediaService mediaService, IValueSetBuilder mediaValueSetBuilder) - : this(null, mediaService, mediaValueSetBuilder) + public MediaIndexPopulator(ILogger logger, IMediaService mediaService, IValueSetBuilder mediaValueSetBuilder) + : this(logger, null, mediaService, mediaValueSetBuilder) { } @@ -31,8 +33,9 @@ namespace Umbraco.Cms.Infrastructure.Examine /// /// /// - public MediaIndexPopulator(int? parentId, IMediaService mediaService, IValueSetBuilder mediaValueSetBuilder) + public MediaIndexPopulator(ILogger logger, int? parentId, IMediaService mediaService, IValueSetBuilder mediaValueSetBuilder) { + _logger = logger; _parentId = parentId; _mediaService = mediaService; _mediaValueSetBuilder = mediaValueSetBuilder; @@ -40,7 +43,11 @@ namespace Umbraco.Cms.Infrastructure.Examine protected override void PopulateIndexes(IReadOnlyList indexes) { - if (indexes.Count == 0) return; + if (indexes.Count == 0) + { + _logger.LogDebug($"{nameof(PopulateIndexes)} called with no indexes to populate. Typically means no index is registered with this populator."); + return; + } const int pageSize = 10000; var pageIndex = 0; diff --git a/src/Umbraco.Infrastructure/Examine/NoopUmbracoIndexesCreator.cs b/src/Umbraco.Infrastructure/Examine/NoopUmbracoIndexesCreator.cs deleted file mode 100644 index e84fb96a74..0000000000 --- a/src/Umbraco.Infrastructure/Examine/NoopUmbracoIndexesCreator.cs +++ /dev/null @@ -1,14 +0,0 @@ -using System.Collections.Generic; -using System.Linq; -using Examine; - -namespace Umbraco.Cms.Infrastructure.Examine -{ - public class NoopUmbracoIndexesCreator : IUmbracoIndexesCreator - { - public IEnumerable Create() - { - return Enumerable.Empty(); - } - } -} diff --git a/src/Umbraco.Infrastructure/Examine/PublishedContentIndexPopulator.cs b/src/Umbraco.Infrastructure/Examine/PublishedContentIndexPopulator.cs index 4b55337670..f9ccaffdbc 100644 --- a/src/Umbraco.Infrastructure/Examine/PublishedContentIndexPopulator.cs +++ b/src/Umbraco.Infrastructure/Examine/PublishedContentIndexPopulator.cs @@ -1,4 +1,5 @@ -using Umbraco.Cms.Core.Services; +using Microsoft.Extensions.Logging; +using Umbraco.Cms.Core.Services; using Umbraco.Cms.Infrastructure.Persistence; namespace Umbraco.Cms.Infrastructure.Examine @@ -13,8 +14,8 @@ namespace Umbraco.Cms.Infrastructure.Examine /// public class PublishedContentIndexPopulator : ContentIndexPopulator { - public PublishedContentIndexPopulator(IContentService contentService, IUmbracoDatabaseFactory umbracoDatabaseFactory, IPublishedContentValueSetBuilder contentValueSetBuilder) : - base(true, null, contentService, umbracoDatabaseFactory, contentValueSetBuilder) + public PublishedContentIndexPopulator(ILogger logger, IContentService contentService, IUmbracoDatabaseFactory umbracoDatabaseFactory, IPublishedContentValueSetBuilder contentValueSetBuilder) : + base(logger, true, null, contentService, umbracoDatabaseFactory, contentValueSetBuilder) { } } diff --git a/src/Umbraco.Infrastructure/Examine/RebuildOnStartupHandler.cs b/src/Umbraco.Infrastructure/Examine/RebuildOnStartupHandler.cs new file mode 100644 index 0000000000..60f7478c3f --- /dev/null +++ b/src/Umbraco.Infrastructure/Examine/RebuildOnStartupHandler.cs @@ -0,0 +1,60 @@ +using System; +using System.Threading; +using Umbraco.Cms.Core.Events; +using Umbraco.Cms.Core.Notifications; +using Umbraco.Cms.Core.Sync; + +namespace Umbraco.Cms.Infrastructure.Examine +{ + /// + /// Handles how the indexes are rebuilt on startup + /// + /// + /// On the first HTTP request this will rebuild the Examine indexes if they are empty. + /// If it is a cold boot, they are all rebuilt. + /// + public sealed class RebuildOnStartupHandler : INotificationHandler + { + private readonly ISyncBootStateAccessor _syncBootStateAccessor; + private readonly ExamineIndexRebuilder _backgroundIndexRebuilder; + + // These must be static because notification handlers are transient. + // this does unfortunatley mean that one RebuildOnStartupHandler instance + // will be created for each front-end request even though we only use the first one. + // TODO: Is there a better way to acheive this without allocating? We cannot remove + // a handler from the notification system. It's not a huge deal but would be better + // with less objects. + private static bool _isReady; + private static bool _isReadSet; + private static object _isReadyLock; + + public RebuildOnStartupHandler( + ISyncBootStateAccessor syncBootStateAccessor, + ExamineIndexRebuilder backgroundIndexRebuilder) + { + _syncBootStateAccessor = syncBootStateAccessor; + _backgroundIndexRebuilder = backgroundIndexRebuilder; + } + + /// + /// On first http request schedule an index rebuild for any empty indexes (or all if it's a cold boot) + /// + /// + public void Handle(UmbracoRequestBeginNotification notification) + => LazyInitializer.EnsureInitialized( + ref _isReady, + ref _isReadSet, + ref _isReadyLock, + () => + { + SyncBootState bootState = _syncBootStateAccessor.GetSyncBootState(); + + _backgroundIndexRebuilder.RebuildIndexes( + // if it's not a cold boot, only rebuild empty ones + bootState != SyncBootState.ColdBoot, + TimeSpan.FromMinutes(1)); + + return true; + }); + } +} diff --git a/src/Umbraco.Infrastructure/Examine/UmbracoExamineExtensions.cs b/src/Umbraco.Infrastructure/Examine/UmbracoExamineExtensions.cs index 90f012f08a..0d341d1d9b 100644 --- a/src/Umbraco.Infrastructure/Examine/UmbracoExamineExtensions.cs +++ b/src/Umbraco.Infrastructure/Examine/UmbracoExamineExtensions.cs @@ -1,5 +1,6 @@ -using System.Collections.Generic; +using System.Collections.Generic; using System.Text.RegularExpressions; +using System.Threading.Tasks; using Examine; using Examine.Search; using Umbraco.Cms.Infrastructure.Examine; @@ -16,8 +17,6 @@ namespace Umbraco.Extensions /// internal static readonly Regex CultureIsoCodeFieldNameMatchExpression = new Regex("^([_\\w]+)_([a-z]{2}-[a-z0-9]{2,4})$", RegexOptions.Compiled); - - //TODO: We need a public method here to just match a field name against CultureIsoCodeFieldNameMatchExpression /// @@ -28,14 +27,19 @@ namespace Umbraco.Extensions /// public static IEnumerable GetCultureFields(this IUmbracoIndex index, string culture) { - var allFields = index.GetFields(); - // ReSharper disable once LoopCanBeConvertedToQuery + IEnumerable allFields = index.GetFieldNames(); + + var results = new List(); foreach (var field in allFields) { var match = CultureIsoCodeFieldNameMatchExpression.Match(field); if (match.Success && match.Groups.Count == 3 && culture.InvariantEquals(match.Groups[2].Value)) - yield return field; + { + results.Add(field); + } } + + return results; } /// @@ -46,8 +50,8 @@ namespace Umbraco.Extensions /// public static IEnumerable GetCultureAndInvariantFields(this IUmbracoIndex index, string culture) { - var allFields = index.GetFields(); - // ReSharper disable once LoopCanBeConvertedToQuery + IEnumerable allFields = index.GetFieldNames(); + foreach (var field in allFields) { var match = CultureIsoCodeFieldNameMatchExpression.Match(field); @@ -59,7 +63,6 @@ namespace Umbraco.Extensions { yield return field; //matches no culture field (invariant) } - } } diff --git a/src/Umbraco.Infrastructure/HostedServices/QueuedHostedService.cs b/src/Umbraco.Infrastructure/HostedServices/QueuedHostedService.cs index db933fec31..c15a37855e 100644 --- a/src/Umbraco.Infrastructure/HostedServices/QueuedHostedService.cs +++ b/src/Umbraco.Infrastructure/HostedServices/QueuedHostedService.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Hosting; @@ -35,8 +35,7 @@ namespace Umbraco.Cms.Infrastructure.HostedServices { while (!stoppingToken.IsCancellationRequested) { - var workItem = - await TaskQueue.DequeueAsync(stoppingToken); + Func workItem = await TaskQueue.DequeueAsync(stoppingToken); try { diff --git a/src/Umbraco.Infrastructure/HostedServices/RecurringHostedServiceBase.cs b/src/Umbraco.Infrastructure/HostedServices/RecurringHostedServiceBase.cs index 1ec13f334e..131b81322a 100644 --- a/src/Umbraco.Infrastructure/HostedServices/RecurringHostedServiceBase.cs +++ b/src/Umbraco.Infrastructure/HostedServices/RecurringHostedServiceBase.cs @@ -47,12 +47,25 @@ namespace Umbraco.Cms.Infrastructure.HostedServices /// Executes the task. /// /// The task state. - public async void ExecuteAsync(object state) => - // Delegate work to method returning a task, that can be called and asserted in a unit test. - // Without this there can be behaviour where tests pass, but an error within them causes the test - // running process to crash. - // Hat-tip: https://stackoverflow.com/a/14207615/489433 - await PerformExecuteAsync(state); + public async void ExecuteAsync(object state) + { + try + { + // First, stop the timer, we do not want tasks to execute in parallel + _timer?.Change(Timeout.Infinite, 0); + + // Delegate work to method returning a task, that can be called and asserted in a unit test. + // Without this there can be behaviour where tests pass, but an error within them causes the test + // running process to crash. + // Hat-tip: https://stackoverflow.com/a/14207615/489433 + await PerformExecuteAsync(state); + } + finally + { + // Resume now that the task is complete + _timer?.Change((int)_delay.TotalMilliseconds, (int)_period.TotalMilliseconds); + } + } public abstract Task PerformExecuteAsync(object state); diff --git a/src/Umbraco.Infrastructure/PublishedContentQuery.cs b/src/Umbraco.Infrastructure/PublishedContentQuery.cs index 1d13748aeb..47b98d8dc0 100644 --- a/src/Umbraco.Infrastructure/PublishedContentQuery.cs +++ b/src/Umbraco.Infrastructure/PublishedContentQuery.cs @@ -1,7 +1,8 @@ -using System; +using System; using System.Collections; using System.Collections.Generic; using System.Linq; +using System.Threading.Tasks; using System.Xml.XPath; using Examine; using Examine.Search; @@ -260,7 +261,7 @@ namespace Umbraco.Cms.Infrastructure $"No index found by name {indexName} or is not of type {typeof(IUmbracoIndex)}"); } - var query = umbIndex.GetSearcher().CreateQuery(IndexTypes.Content); + var query = umbIndex.Searcher.CreateQuery(IndexTypes.Content); IQueryExecutor queryExecutor; if (culture == "*") @@ -286,7 +287,7 @@ namespace Umbraco.Cms.Infrastructure var results = skip == 0 && take == 0 ? queryExecutor.Execute() - : queryExecutor.Execute(skip + take); + : queryExecutor.Execute(QueryOptions.SkipTake(skip, take)); totalRecords = results.TotalItemCount; @@ -316,7 +317,7 @@ namespace Umbraco.Cms.Infrastructure var results = skip == 0 && take == 0 ? query.Execute() - : query.Execute(skip + take); + : query.Execute(QueryOptions.SkipTake(skip, take)); totalRecords = results.TotalItemCount; diff --git a/src/Umbraco.Infrastructure/Runtime/CoreRuntime.cs b/src/Umbraco.Infrastructure/Runtime/CoreRuntime.cs index e004313ac3..e2b20ced8f 100644 --- a/src/Umbraco.Infrastructure/Runtime/CoreRuntime.cs +++ b/src/Umbraco.Infrastructure/Runtime/CoreRuntime.cs @@ -119,10 +119,10 @@ namespace Umbraco.Cms.Infrastructure.Runtime } - await _eventAggregator.PublishAsync(new UmbracoApplicationStartingNotification(State.Level), cancellationToken); - // create & initialize the components _components.Initialize(); + + await _eventAggregator.PublishAsync(new UmbracoApplicationStartingNotification(State.Level), cancellationToken); } private void DoUnattendedUpgrade() diff --git a/src/Umbraco.Infrastructure/Runtime/SqlMainDomLock.cs b/src/Umbraco.Infrastructure/Runtime/SqlMainDomLock.cs index c2c3262047..1c02898334 100644 --- a/src/Umbraco.Infrastructure/Runtime/SqlMainDomLock.cs +++ b/src/Umbraco.Infrastructure/Runtime/SqlMainDomLock.cs @@ -56,7 +56,7 @@ namespace Umbraco.Cms.Infrastructure.Runtime dbProviderFactoryCreator, databaseSchemaCreatorFactory); - MainDomKey = MainDomKeyPrefix + "-" + (NetworkHelper.MachineName + MainDom.GetMainDomId(_hostingEnvironment)).GenerateHash(); + MainDomKey = MainDomKeyPrefix + "-" + (Environment.MachineName + MainDom.GetMainDomId(_hostingEnvironment)).GenerateHash(); } public async Task AcquireLockAsync(int millisecondsTimeout) diff --git a/src/Umbraco.Infrastructure/Search/BackgroundIndexRebuilder.cs b/src/Umbraco.Infrastructure/Search/BackgroundIndexRebuilder.cs deleted file mode 100644 index 2fbceb2f9a..0000000000 --- a/src/Umbraco.Infrastructure/Search/BackgroundIndexRebuilder.cs +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright (c) Umbraco. -// See LICENSE for more details. - -using System; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Umbraco.Cms.Core.Runtime; -using Umbraco.Cms.Infrastructure.Examine; -using Umbraco.Cms.Infrastructure.HostedServices; - -namespace Umbraco.Cms.Infrastructure.Search -{ - /// - /// Utility to rebuild all indexes on a background thread - /// - public class BackgroundIndexRebuilder - { - private readonly IndexRebuilder _indexRebuilder; - private readonly IBackgroundTaskQueue _backgroundTaskQueue; - - private readonly IMainDom _mainDom; - private readonly ILogger _logger; - - private volatile bool _isRunning = false; - private static readonly object s_rebuildLocker = new object(); - - /// - /// Initializes a new instance of the class. - /// - public BackgroundIndexRebuilder( - IMainDom mainDom, - ILogger logger, - IndexRebuilder indexRebuilder, - IBackgroundTaskQueue backgroundTaskQueue) - { - _mainDom = mainDom; - _logger = logger; - _indexRebuilder = indexRebuilder; - _backgroundTaskQueue = backgroundTaskQueue; - } - - - /// - /// Called to rebuild empty indexes on startup - /// - public virtual void RebuildIndexes(bool onlyEmptyIndexes, TimeSpan? delay = null) - { - - lock (s_rebuildLocker) - { - if (_isRunning) - { - _logger.LogWarning("Call was made to RebuildIndexes but the task runner for rebuilding is already running"); - return; - } - - _logger.LogInformation("Starting initialize async background thread."); - - _backgroundTaskQueue.QueueBackgroundWorkItem(cancellationToken => RebuildIndexes(onlyEmptyIndexes, delay ?? TimeSpan.Zero, cancellationToken)); - - } - } - - private Task RebuildIndexes(bool onlyEmptyIndexes, TimeSpan delay, CancellationToken cancellationToken) - { - if (!_mainDom.IsMainDom) - { - return Task.CompletedTask; - } - - if (delay > TimeSpan.Zero) - { - Thread.Sleep(delay); - } - - _isRunning = true; - _indexRebuilder.RebuildIndexes(onlyEmptyIndexes); - _isRunning = false; - return Task.CompletedTask; - } - } -} diff --git a/src/Umbraco.Infrastructure/Search/ExamineNotificationHandler.cs b/src/Umbraco.Infrastructure/Search/ExamineNotificationHandler.cs deleted file mode 100644 index d0541cfd97..0000000000 --- a/src/Umbraco.Infrastructure/Search/ExamineNotificationHandler.cs +++ /dev/null @@ -1,838 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Globalization; -using System.Linq; -using System.Threading.Tasks; -using Examine; -using Microsoft.Extensions.Logging; -using Umbraco.Cms.Core; -using Umbraco.Cms.Core.Cache; -using Umbraco.Cms.Core.Events; -using Umbraco.Cms.Core.Logging; -using Umbraco.Cms.Core.Models; -using Umbraco.Cms.Core.Notifications; -using Umbraco.Cms.Core.Runtime; -using Umbraco.Cms.Core.Scoping; -using Umbraco.Cms.Core.Services; -using Umbraco.Cms.Core.Services.Changes; -using Umbraco.Cms.Core.Sync; -using Umbraco.Cms.Infrastructure.Examine; -using Umbraco.Extensions; - -namespace Umbraco.Cms.Infrastructure.Search -{ - public sealed class ExamineNotificationHandler : - INotificationHandler, - INotificationHandler, - INotificationHandler, - INotificationHandler, - INotificationHandler, - INotificationHandler - { - private readonly IExamineManager _examineManager; - private readonly IContentValueSetBuilder _contentValueSetBuilder; - private readonly IPublishedContentValueSetBuilder _publishedContentValueSetBuilder; - private readonly IValueSetBuilder _mediaValueSetBuilder; - private readonly IValueSetBuilder _memberValueSetBuilder; - private readonly BackgroundIndexRebuilder _backgroundIndexRebuilder; - private readonly TaskHelper _taskHelper; - private readonly IRuntimeState _runtimeState; - private readonly IScopeProvider _scopeProvider; - private readonly ServiceContext _services; - private readonly IMainDom _mainDom; - private readonly IProfilingLogger _profilingLogger; - private readonly ILogger _logger; - private readonly IUmbracoIndexesCreator _indexCreator; - private static bool s_deactivate_handlers; - - // the default enlist priority is 100 - // enlist with a lower priority to ensure that anything "default" runs after us - // but greater that SafeXmlReaderWriter priority which is 60 - private const int EnlistPriority = 80; - - public ExamineNotificationHandler(IMainDom mainDom, - IExamineManager examineManager, - IProfilingLogger profilingLogger, - ILogger logger, - IScopeProvider scopeProvider, - IUmbracoIndexesCreator indexCreator, - ServiceContext services, - IContentValueSetBuilder contentValueSetBuilder, - IPublishedContentValueSetBuilder publishedContentValueSetBuilder, - IValueSetBuilder mediaValueSetBuilder, - IValueSetBuilder memberValueSetBuilder, - BackgroundIndexRebuilder backgroundIndexRebuilder, - TaskHelper taskHelper, - IRuntimeState runtimeState) - { - _services = services; - _scopeProvider = scopeProvider; - _examineManager = examineManager; - _contentValueSetBuilder = contentValueSetBuilder; - _publishedContentValueSetBuilder = publishedContentValueSetBuilder; - _mediaValueSetBuilder = mediaValueSetBuilder; - _memberValueSetBuilder = memberValueSetBuilder; - _backgroundIndexRebuilder = backgroundIndexRebuilder; - _taskHelper = taskHelper; - _runtimeState = runtimeState; - _mainDom = mainDom; - _profilingLogger = profilingLogger; - _logger = logger; - _indexCreator = indexCreator; - } - public void Handle(UmbracoApplicationStartingNotification notification) - { - //let's deal with shutting down Examine with MainDom - var examineShutdownRegistered = _mainDom.Register(release: () => - { - using (_profilingLogger.TraceDuration("Examine shutting down")) - { - _examineManager.Dispose(); - } - }); - - if (!examineShutdownRegistered) - { - _logger.LogInformation("Examine shutdown not registered, this AppDomain is not the MainDom, Examine will be disabled"); - - //if we could not register the shutdown examine ourselves, it means we are not maindom! in this case all of examine should be disabled! - Suspendable.ExamineEvents.SuspendIndexers(_logger); - return; //exit, do not continue - } - - //create the indexes and register them with the manager - foreach (IIndex index in _indexCreator.Create()) - { - _examineManager.AddIndex(index); - } - - _logger.LogDebug("Examine shutdown registered with MainDom"); - - var registeredIndexers = _examineManager.Indexes.OfType().Count(x => x.EnableDefaultEventHandler); - - _logger.LogInformation("Adding examine event handlers for {RegisteredIndexers} index providers.", registeredIndexers); - - // don't bind event handlers if we're not suppose to listen - if (registeredIndexers == 0) - { - s_deactivate_handlers = true; - } - - if (_mainDom.IsMainDom && _runtimeState.Level >= RuntimeLevel.Run) - { - _backgroundIndexRebuilder.RebuildIndexes(true); - } - } - - - #region Cache refresher updated event handlers - - /// - /// Updates indexes based on content changes - /// - /// - /// - public void Handle(ContentCacheRefresherNotification args) - { - if (s_deactivate_handlers) - { - return; - } - if (Suspendable.ExamineEvents.CanIndex == false) - { - return; - } - - if (args.MessageType != MessageType.RefreshByPayload) - { - throw new NotSupportedException(); - } - - var contentService = _services.ContentService; - - foreach (var payload in (ContentCacheRefresher.JsonPayload[])args.MessageObject) - { - if (payload.ChangeTypes.HasType(TreeChangeTypes.Remove)) - { - // delete content entirely (with descendants) - // false: remove entirely from all indexes - DeleteIndexForEntity(payload.Id, false); - } - else if (payload.ChangeTypes.HasType(TreeChangeTypes.RefreshAll)) - { - // ExamineEvents does not support RefreshAll - // just ignore that payload - // so what?! - - // TODO: Rebuild the index at this point? - } - else // RefreshNode or RefreshBranch (maybe trashed) - { - // don't try to be too clever - refresh entirely - // there has to be race conditions in there ;-( - - var content = contentService.GetById(payload.Id); - if (content == null) - { - // gone fishing, remove entirely from all indexes (with descendants) - DeleteIndexForEntity(payload.Id, false); - continue; - } - - IContent published = null; - if (content.Published && contentService.IsPathPublished(content)) - { - published = content; - } - - if (published == null) - { - DeleteIndexForEntity(payload.Id, true); - } - - // just that content - ReIndexForContent(content, published != null); - - // branch - if (payload.ChangeTypes.HasType(TreeChangeTypes.RefreshBranch)) - { - var masked = published == null ? null : new List(); - const int pageSize = 500; - var page = 0; - var total = long.MaxValue; - while (page * pageSize < total) - { - var descendants = contentService.GetPagedDescendants(content.Id, page++, pageSize, out total, - //order by shallowest to deepest, this allows us to check it's published state without checking every item - ordering: Ordering.By("Path", Direction.Ascending)); - - foreach (var descendant in descendants) - { - published = null; - if (masked != null) // else everything is masked - { - if (masked.Contains(descendant.ParentId) || !descendant.Published) - { - masked.Add(descendant.Id); - } - else - { - published = descendant; - } - } - - ReIndexForContent(descendant, published != null); - } - } - } - } - - // NOTE - // - // DeleteIndexForEntity is handled by UmbracoContentIndexer.DeleteFromIndex() which takes - // care of also deleting the descendants - // - // ReIndexForContent is NOT taking care of descendants so we have to reload everything - // again in order to process the branch - we COULD improve that by just reloading the - // XML from database instead of reloading content & re-serializing! - // - // BUT ... pretty sure it is! see test "Index_Delete_Index_Item_Ensure_Heirarchy_Removed" - } - } - - public void Handle(MemberCacheRefresherNotification args) - { - if (s_deactivate_handlers) - { - return; - } - - if (Suspendable.ExamineEvents.CanIndex == false) - { - return; - } - - switch (args.MessageType) - { - case MessageType.RefreshById: - var c1 = _services.MemberService.GetById((int)args.MessageObject); - if (c1 != null) - { - ReIndexForMember(c1); - } - break; - case MessageType.RemoveById: - - // This is triggered when the item is permanently deleted - - DeleteIndexForEntity((int)args.MessageObject, false); - break; - case MessageType.RefreshByInstance: - if (args.MessageObject is IMember c3) - { - ReIndexForMember(c3); - } - break; - case MessageType.RemoveByInstance: - - // This is triggered when the item is permanently deleted - - if (args.MessageObject is IMember c4) - { - DeleteIndexForEntity(c4.Id, false); - } - break; - case MessageType.RefreshByPayload: - var payload = (MemberCacheRefresher.JsonPayload[])args.MessageObject; - foreach (var p in payload) - { - if (p.Removed) - { - DeleteIndexForEntity(p.Id, false); - } - else - { - var m = _services.MemberService.GetById(p.Id); - if (m != null) - { - ReIndexForMember(m); - } - } - } - break; - case MessageType.RefreshAll: - case MessageType.RefreshByJson: - default: - //We don't support these, these message types will not fire for unpublished content - break; - } - } - - public void Handle(MediaCacheRefresherNotification args) - { - if (s_deactivate_handlers) - { - return; - } - - if (Suspendable.ExamineEvents.CanIndex == false) - { - return; - } - - if (args.MessageType != MessageType.RefreshByPayload) - { - throw new NotSupportedException(); - } - - var mediaService = _services.MediaService; - - foreach (var payload in (MediaCacheRefresher.JsonPayload[])args.MessageObject) - { - if (payload.ChangeTypes.HasType(TreeChangeTypes.Remove)) - { - // remove from *all* indexes - DeleteIndexForEntity(payload.Id, false); - } - else if (payload.ChangeTypes.HasType(TreeChangeTypes.RefreshAll)) - { - // ExamineEvents does not support RefreshAll - // just ignore that payload - // so what?! - } - else // RefreshNode or RefreshBranch (maybe trashed) - { - var media = mediaService.GetById(payload.Id); - if (media == null) - { - // gone fishing, remove entirely - DeleteIndexForEntity(payload.Id, false); - continue; - } - - if (media.Trashed) - { - DeleteIndexForEntity(payload.Id, true); - } - - // just that media - ReIndexForMedia(media, !media.Trashed); - - // branch - if (payload.ChangeTypes.HasType(TreeChangeTypes.RefreshBranch)) - { - const int pageSize = 500; - var page = 0; - var total = long.MaxValue; - while (page * pageSize < total) - { - var descendants = mediaService.GetPagedDescendants(media.Id, page++, pageSize, out total); - foreach (var descendant in descendants) - { - ReIndexForMedia(descendant, !descendant.Trashed); - } - } - } - } - } - } - - public void Handle(LanguageCacheRefresherNotification args) - { - if (s_deactivate_handlers) - { - return; - } - - if (!(args.MessageObject is LanguageCacheRefresher.JsonPayload[] payloads)) - { - return; - } - - if (payloads.Length == 0) - { - return; - } - - var removedOrCultureChanged = payloads.Any(x => - x.ChangeType == LanguageCacheRefresher.JsonPayload.LanguageChangeType.ChangeCulture - || x.ChangeType == LanguageCacheRefresher.JsonPayload.LanguageChangeType.Remove); - - if (removedOrCultureChanged) - { - //if a lang is removed or it's culture has changed, we need to rebuild the indexes since - //field names and values in the index have a string culture value. - _backgroundIndexRebuilder.RebuildIndexes(false); - } - } - - /// - /// Updates indexes based on content type changes - /// - /// - /// - public void Handle(ContentTypeCacheRefresherNotification args) - { - if (s_deactivate_handlers) - { - return; - } - - if (Suspendable.ExamineEvents.CanIndex == false) - { - return; - } - - if (args.MessageType != MessageType.RefreshByPayload) - { - throw new NotSupportedException(); - } - - var changedIds = new Dictionary removedIds, List refreshedIds, List otherIds)>(); - - foreach (var payload in (ContentTypeCacheRefresher.JsonPayload[])args.MessageObject) - { - if (!changedIds.TryGetValue(payload.ItemType, out var idLists)) - { - idLists = (removedIds: new List(), refreshedIds: new List(), otherIds: new List()); - changedIds.Add(payload.ItemType, idLists); - } - - if (payload.ChangeTypes.HasType(ContentTypeChangeTypes.Remove)) - { - idLists.removedIds.Add(payload.Id); - } - else if (payload.ChangeTypes.HasType(ContentTypeChangeTypes.RefreshMain)) - { - idLists.refreshedIds.Add(payload.Id); - } - else if (payload.ChangeTypes.HasType(ContentTypeChangeTypes.RefreshOther)) - { - idLists.otherIds.Add(payload.Id); - } - } - - const int pageSize = 500; - - foreach (var ci in changedIds) - { - if (ci.Value.refreshedIds.Count > 0 || ci.Value.otherIds.Count > 0) - { - switch (ci.Key) - { - case var itemType when itemType == typeof(IContentType).Name: - RefreshContentOfContentTypes(ci.Value.refreshedIds.Concat(ci.Value.otherIds).Distinct().ToArray()); - break; - case var itemType when itemType == typeof(IMediaType).Name: - RefreshMediaOfMediaTypes(ci.Value.refreshedIds.Concat(ci.Value.otherIds).Distinct().ToArray()); - break; - case var itemType when itemType == typeof(IMemberType).Name: - RefreshMemberOfMemberTypes(ci.Value.refreshedIds.Concat(ci.Value.otherIds).Distinct().ToArray()); - break; - } - } - - //Delete all content of this content/media/member type that is in any content indexer by looking up matched examine docs - foreach (var id in ci.Value.removedIds) - { - foreach (var index in _examineManager.Indexes.OfType()) - { - var searcher = index.GetSearcher(); - - var page = 0; - var total = long.MaxValue; - while (page * pageSize < total) - { - //paging with examine, see https://shazwazza.com/post/paging-with-examine/ - var results = searcher.CreateQuery().Field("nodeType", id.ToInvariantString()).Execute(maxResults: pageSize * (page + 1)); - total = results.TotalItemCount; - var paged = results.Skip(page * pageSize); - - foreach (ISearchResult item in paged) - { - if (int.TryParse(item.Id, out int contentId)) - { - DeleteIndexForEntity(contentId, false); - } - } - - page++; - } - } - } - } - } - - private void RefreshMemberOfMemberTypes(int[] memberTypeIds) - { - const int pageSize = 500; - - IEnumerable memberTypes = _services.MemberTypeService.GetAll(memberTypeIds); - foreach (IMemberType memberType in memberTypes) - { - var page = 0; - var total = long.MaxValue; - while (page * pageSize < total) - { - IEnumerable memberToRefresh = _services.MemberService.GetAll( - page++, pageSize, out total, "LoginName", Direction.Ascending, - memberType.Alias); - - foreach (IMember c in memberToRefresh) - { - ReIndexForMember(c); - } - } - } - } - - private void RefreshMediaOfMediaTypes(int[] mediaTypeIds) - { - const int pageSize = 500; - var page = 0; - var total = long.MaxValue; - while (page * pageSize < total) - { - IEnumerable mediaToRefresh = _services.MediaService.GetPagedOfTypes( - //Re-index all content of these types - mediaTypeIds, - page++, pageSize, out total, null, - Ordering.By("Path", Direction.Ascending)); - - foreach (IMedia c in mediaToRefresh) - { - ReIndexForMedia(c, c.Trashed == false); - } - } - } - - private void RefreshContentOfContentTypes(int[] contentTypeIds) - { - const int pageSize = 500; - var page = 0; - var total = long.MaxValue; - while (page * pageSize < total) - { - IEnumerable contentToRefresh = _services.ContentService.GetPagedOfTypes( - //Re-index all content of these types - contentTypeIds, - page++, pageSize, out total, null, - //order by shallowest to deepest, this allows us to check it's published state without checking every item - Ordering.By("Path", Direction.Ascending)); - - //track which Ids have their paths are published - var publishChecked = new Dictionary(); - - foreach (IContent c in contentToRefresh) - { - var isPublished = false; - if (c.Published) - { - if (!publishChecked.TryGetValue(c.ParentId, out isPublished)) - { - //nothing by parent id, so query the service and cache the result for the next child to check against - isPublished = _services.ContentService.IsPathPublished(c); - publishChecked[c.Id] = isPublished; - } - } - - ReIndexForContent(c, isPublished); - } - } - } - - #endregion - - #region ReIndex/Delete for entity - private void ReIndexForContent(IContent sender, bool isPublished) - { - var actions = DeferedActions.Get(_scopeProvider); - if (actions != null) - { - actions.Add(new DeferedReIndexForContent(_taskHelper, this, sender, isPublished)); - } - else - { - DeferedReIndexForContent.Execute(_taskHelper, this, sender, isPublished); - } - } - - private void ReIndexForMember(IMember member) - { - var actions = DeferedActions.Get(_scopeProvider); - if (actions != null) - { - actions.Add(new DeferedReIndexForMember(_taskHelper, this, member)); - } - else - { - DeferedReIndexForMember.Execute(_taskHelper, this, member); - } - } - - private void ReIndexForMedia(IMedia sender, bool isPublished) - { - var actions = DeferedActions.Get(_scopeProvider); - if (actions != null) - { - actions.Add(new DeferedReIndexForMedia(_taskHelper, this, sender, isPublished)); - } - else - { - DeferedReIndexForMedia.Execute(_taskHelper, this, sender, isPublished); - } - } - - /// - /// Remove items from an index - /// - /// - /// - /// If true, indicates that we will only delete this item from indexes that don't support unpublished content. - /// If false it will delete this from all indexes regardless. - /// - private void DeleteIndexForEntity(int entityId, bool keepIfUnpublished) - { - var actions = DeferedActions.Get(_scopeProvider); - if (actions != null) - { - actions.Add(new DeferedDeleteIndex(this, entityId, keepIfUnpublished)); - } - else - { - DeferedDeleteIndex.Execute(this, entityId, keepIfUnpublished); - } - } - #endregion - - #region Deferred Actions - private class DeferedActions - { - private readonly List _actions = new List(); - - public static DeferedActions Get(IScopeProvider scopeProvider) - { - IScopeContext scopeContext = scopeProvider.Context; - - return scopeContext?.Enlist("examineEvents", - () => new DeferedActions(), // creator - (completed, actions) => // action - { - if (completed) - { - actions.Execute(); - } - }, EnlistPriority); - } - - public void Add(DeferedAction action) => _actions.Add(action); - - private void Execute() - { - foreach (DeferedAction action in _actions) - { - action.Execute(); - } - } - } - - /// - /// An action that will execute at the end of the Scope being completed - /// - private abstract class DeferedAction - { - public virtual void Execute() - { } - } - - /// - /// Re-indexes an item on a background thread - /// - private class DeferedReIndexForContent : DeferedAction - { - private readonly TaskHelper _taskHelper; - private readonly ExamineNotificationHandler _ExamineNotificationHandler; - private readonly IContent _content; - private readonly bool _isPublished; - - public DeferedReIndexForContent(TaskHelper taskHelper, ExamineNotificationHandler ExamineNotificationHandler, IContent content, bool isPublished) - { - _taskHelper = taskHelper; - _ExamineNotificationHandler = ExamineNotificationHandler; - _content = content; - _isPublished = isPublished; - } - - public override void Execute() => Execute(_taskHelper, _ExamineNotificationHandler, _content, _isPublished); - - public static void Execute(TaskHelper taskHelper, ExamineNotificationHandler ExamineNotificationHandler, IContent content, bool isPublished) - => taskHelper.RunBackgroundTask(() => - { - using IScope scope = ExamineNotificationHandler._scopeProvider.CreateScope(autoComplete: true); - - // for content we have a different builder for published vs unpublished - // we don't want to build more value sets than is needed so we'll lazily build 2 one for published one for non-published - var builders = new Dictionary>> - { - [true] = new Lazy>(() => ExamineNotificationHandler._publishedContentValueSetBuilder.GetValueSets(content).ToList()), - [false] = new Lazy>(() => ExamineNotificationHandler._contentValueSetBuilder.GetValueSets(content).ToList()) - }; - - foreach (IUmbracoIndex index in ExamineNotificationHandler._examineManager.Indexes.OfType() - //filter the indexers - .Where(x => isPublished || !x.PublishedValuesOnly) - .Where(x => x.EnableDefaultEventHandler)) - { - List valueSet = builders[index.PublishedValuesOnly].Value; - index.IndexItems(valueSet); - } - - return Task.CompletedTask; - }); - } - - /// - /// Re-indexes an item on a background thread - /// - private class DeferedReIndexForMedia : DeferedAction - { - private readonly TaskHelper _taskHelper; - private readonly ExamineNotificationHandler _ExamineNotificationHandler; - private readonly IMedia _media; - private readonly bool _isPublished; - - public DeferedReIndexForMedia(TaskHelper taskHelper, ExamineNotificationHandler ExamineNotificationHandler, IMedia media, bool isPublished) - { - _taskHelper = taskHelper; - _ExamineNotificationHandler = ExamineNotificationHandler; - _media = media; - _isPublished = isPublished; - } - - public override void Execute() => Execute(_taskHelper, _ExamineNotificationHandler, _media, _isPublished); - - public static void Execute(TaskHelper taskHelper, ExamineNotificationHandler ExamineNotificationHandler, IMedia media, bool isPublished) => - // perform the ValueSet lookup on a background thread - taskHelper.RunBackgroundTask(() => - { - using IScope scope = ExamineNotificationHandler._scopeProvider.CreateScope(autoComplete: true); - - var valueSet = ExamineNotificationHandler._mediaValueSetBuilder.GetValueSets(media).ToList(); - - foreach (IUmbracoIndex index in ExamineNotificationHandler._examineManager.Indexes.OfType() - //filter the indexers - .Where(x => isPublished || !x.PublishedValuesOnly) - .Where(x => x.EnableDefaultEventHandler)) - { - index.IndexItems(valueSet); - } - - return Task.CompletedTask; - }); - } - - /// - /// Re-indexes an item on a background thread - /// - private class DeferedReIndexForMember : DeferedAction - { - private readonly ExamineNotificationHandler _ExamineNotificationHandler; - private readonly IMember _member; - private readonly TaskHelper _taskHelper; - - public DeferedReIndexForMember(TaskHelper taskHelper, ExamineNotificationHandler ExamineNotificationHandler, IMember member) - { - _ExamineNotificationHandler = ExamineNotificationHandler; - _member = member; - _taskHelper = taskHelper; - } - - public override void Execute() => Execute(_taskHelper, _ExamineNotificationHandler, _member); - - public static void Execute(TaskHelper taskHelper, ExamineNotificationHandler ExamineNotificationHandler, IMember member) => - // perform the ValueSet lookup on a background thread - taskHelper.RunBackgroundTask(() => - { - using IScope scope = ExamineNotificationHandler._scopeProvider.CreateScope(autoComplete: true); - - var valueSet = ExamineNotificationHandler._memberValueSetBuilder.GetValueSets(member).ToList(); - foreach (IUmbracoIndex index in ExamineNotificationHandler._examineManager.Indexes.OfType() - //filter the indexers - .Where(x => x.EnableDefaultEventHandler)) - { - index.IndexItems(valueSet); - } - - return Task.CompletedTask; - }); - } - - private class DeferedDeleteIndex : DeferedAction - { - private readonly ExamineNotificationHandler _ExamineNotificationHandler; - private readonly int _id; - private readonly bool _keepIfUnpublished; - - public DeferedDeleteIndex(ExamineNotificationHandler ExamineNotificationHandler, int id, bool keepIfUnpublished) - { - _ExamineNotificationHandler = ExamineNotificationHandler; - _id = id; - _keepIfUnpublished = keepIfUnpublished; - } - - public override void Execute() => Execute(_ExamineNotificationHandler, _id, _keepIfUnpublished); - - public static void Execute(ExamineNotificationHandler ExamineNotificationHandler, int id, bool keepIfUnpublished) - { - var strId = id.ToString(CultureInfo.InvariantCulture); - foreach (var index in ExamineNotificationHandler._examineManager.Indexes.OfType() - .Where(x => x.PublishedValuesOnly || !keepIfUnpublished) - .Where(x => x.EnableDefaultEventHandler)) - { - index.DeleteFromIndex(strId); - } - } - } - #endregion - } -} diff --git a/src/Umbraco.Infrastructure/Search/ExamineUserComponent.cs b/src/Umbraco.Infrastructure/Search/ExamineUserComponent.cs deleted file mode 100644 index 6c39da44c7..0000000000 --- a/src/Umbraco.Infrastructure/Search/ExamineUserComponent.cs +++ /dev/null @@ -1,37 +0,0 @@ -using Umbraco.Cms.Core.Composing; -using Umbraco.Cms.Core.Runtime; - -namespace Umbraco.Cms.Infrastructure.Search -{ - /// - /// An abstract class for custom index authors to inherit from - /// - public abstract class ExamineUserComponent : IComponent - { - private readonly IMainDom _mainDom; - - public ExamineUserComponent(IMainDom mainDom) - { - _mainDom = mainDom; - } - - /// - /// Initialize the component, eagerly exits if ExamineComponent.ExamineEnabled == false - /// - public void Initialize() - { - if (!_mainDom.IsMainDom) return; - - InitializeComponent(); - } - - /// - /// Abstract method which executes to initialize this component if ExamineComponent.ExamineEnabled == true - /// - protected abstract void InitializeComponent(); - - public virtual void Terminate() - { - } - } -} diff --git a/src/Umbraco.Infrastructure/Search/IUmbracoIndexingHandler.cs b/src/Umbraco.Infrastructure/Search/IUmbracoIndexingHandler.cs new file mode 100644 index 0000000000..24c82c055d --- /dev/null +++ b/src/Umbraco.Infrastructure/Search/IUmbracoIndexingHandler.cs @@ -0,0 +1,37 @@ +using System.Collections.Generic; +using Umbraco.Cms.Core.Models; + +namespace Umbraco.Cms.Infrastructure.Search +{ + public interface IUmbracoIndexingHandler + { + /// + /// Returns true if the indexing handler is enabled + /// + /// + /// If this is false then there will be no data lookups executed to populate indexes + /// when service changes are made. + /// + bool Enabled { get; } + + void ReIndexForContent(IContent sender, bool isPublished); + void ReIndexForMember(IMember member); + void ReIndexForMedia(IMedia sender, bool isPublished); + + /// + /// Deletes all documents for the content type Ids + /// + /// + void DeleteDocumentsForContentTypes(IReadOnlyCollection removedContentTypes); + + /// + /// Remove items from an index + /// + /// + /// + /// If true, indicates that we will only delete this item from indexes that don't support unpublished content. + /// If false it will delete this from all indexes regardless. + /// + void DeleteIndexForEntity(int entityId, bool keepIfUnpublished); + } +} diff --git a/src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.Content.cs b/src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.Content.cs new file mode 100644 index 0000000000..ebebdb7f34 --- /dev/null +++ b/src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.Content.cs @@ -0,0 +1,137 @@ +using System; +using System.Collections.Generic; +using Umbraco.Cms.Core; +using Umbraco.Cms.Core.Cache; +using Umbraco.Cms.Core.Events; +using Umbraco.Cms.Core.Models; +using Umbraco.Cms.Core.Notifications; +using Umbraco.Cms.Core.Services; +using Umbraco.Cms.Core.Services.Changes; +using Umbraco.Cms.Core.Sync; +using Umbraco.Extensions; + +namespace Umbraco.Cms.Infrastructure.Search +{ + public sealed class ContentIndexingNotificationHandler : INotificationHandler + { + private readonly IUmbracoIndexingHandler _umbracoIndexingHandler; + private readonly IContentService _contentService; + + public ContentIndexingNotificationHandler(IUmbracoIndexingHandler umbracoIndexingHandler, IContentService contentService) + { + _umbracoIndexingHandler = umbracoIndexingHandler ?? throw new ArgumentNullException(nameof(umbracoIndexingHandler)); + _contentService = contentService ?? throw new ArgumentNullException(nameof(contentService)); + } + + /// + /// Updates indexes based on content changes + /// + /// + /// + public void Handle(ContentCacheRefresherNotification args) + { + if (!_umbracoIndexingHandler.Enabled) + { + return; + } + if (Suspendable.ExamineEvents.CanIndex == false) + { + return; + } + + if (args.MessageType != MessageType.RefreshByPayload) + { + throw new NotSupportedException(); + } + + foreach (var payload in (ContentCacheRefresher.JsonPayload[])args.MessageObject) + { + if (payload.ChangeTypes.HasType(TreeChangeTypes.Remove)) + { + // delete content entirely (with descendants) + // false: remove entirely from all indexes + _umbracoIndexingHandler.DeleteIndexForEntity(payload.Id, false); + } + else if (payload.ChangeTypes.HasType(TreeChangeTypes.RefreshAll)) + { + // ExamineEvents does not support RefreshAll + // just ignore that payload + // so what?! + + // TODO: Rebuild the index at this point? + } + else // RefreshNode or RefreshBranch (maybe trashed) + { + // don't try to be too clever - refresh entirely + // there has to be race conditions in there ;-( + + var content = _contentService.GetById(payload.Id); + if (content == null) + { + // gone fishing, remove entirely from all indexes (with descendants) + _umbracoIndexingHandler.DeleteIndexForEntity(payload.Id, false); + continue; + } + + IContent published = null; + if (content.Published && _contentService.IsPathPublished(content)) + { + published = content; + } + + if (published == null) + { + _umbracoIndexingHandler.DeleteIndexForEntity(payload.Id, true); + } + + // just that content + _umbracoIndexingHandler.ReIndexForContent(content, published != null); + + // branch + if (payload.ChangeTypes.HasType(TreeChangeTypes.RefreshBranch)) + { + var masked = published == null ? null : new List(); + const int pageSize = 500; + var page = 0; + var total = long.MaxValue; + while (page * pageSize < total) + { + var descendants = _contentService.GetPagedDescendants(content.Id, page++, pageSize, out total, + //order by shallowest to deepest, this allows us to check it's published state without checking every item + ordering: Ordering.By("Path", Direction.Ascending)); + + foreach (var descendant in descendants) + { + published = null; + if (masked != null) // else everything is masked + { + if (masked.Contains(descendant.ParentId) || !descendant.Published) + { + masked.Add(descendant.Id); + } + else + { + published = descendant; + } + } + + _umbracoIndexingHandler.ReIndexForContent(descendant, published != null); + } + } + } + } + + // NOTE + // + // DeleteIndexForEntity is handled by UmbracoContentIndexer.DeleteFromIndex() which takes + // care of also deleting the descendants + // + // ReIndexForContent is NOT taking care of descendants so we have to reload everything + // again in order to process the branch - we COULD improve that by just reloading the + // XML from database instead of reloading content & re-serializing! + // + // BUT ... pretty sure it is! see test "Index_Delete_Index_Item_Ensure_Heirarchy_Removed" + } + } + } +} diff --git a/src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.ContentType.cs b/src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.ContentType.cs new file mode 100644 index 0000000000..9bdc9fa3c4 --- /dev/null +++ b/src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.ContentType.cs @@ -0,0 +1,180 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Umbraco.Cms.Core; +using Umbraco.Cms.Core.Cache; +using Umbraco.Cms.Core.Events; +using Umbraco.Cms.Core.Models; +using Umbraco.Cms.Core.Notifications; +using Umbraco.Cms.Core.Services; +using Umbraco.Cms.Core.Services.Changes; +using Umbraco.Cms.Core.Sync; +using Umbraco.Extensions; + +namespace Umbraco.Cms.Infrastructure.Search +{ + public sealed class ContentTypeIndexingNotificationHandler : INotificationHandler + { + private readonly IUmbracoIndexingHandler _umbracoIndexingHandler; + private readonly IContentService _contentService; + private readonly IMemberService _memberService; + private readonly IMediaService _mediaService; + private readonly IMemberTypeService _memberTypeService; + + public ContentTypeIndexingNotificationHandler(IUmbracoIndexingHandler umbracoIndexingHandler, IContentService contentService, IMemberService memberService, IMediaService mediaService, IMemberTypeService memberTypeService) + { + _umbracoIndexingHandler = umbracoIndexingHandler ?? throw new ArgumentNullException(nameof(umbracoIndexingHandler)); + _contentService = contentService ?? throw new ArgumentNullException(nameof(contentService)); + _memberService = memberService ?? throw new ArgumentNullException(nameof(memberService)); + _mediaService = mediaService ?? throw new ArgumentNullException(nameof(mediaService)); + _memberTypeService = memberTypeService ?? throw new ArgumentNullException(nameof(memberTypeService)); + } + + /// + /// Updates indexes based on content type changes + /// + /// + /// + public void Handle(ContentTypeCacheRefresherNotification args) + { + if (!_umbracoIndexingHandler.Enabled) + { + return; + } + + if (Suspendable.ExamineEvents.CanIndex == false) + { + return; + } + + if (args.MessageType != MessageType.RefreshByPayload) + { + throw new NotSupportedException(); + } + + var changedIds = new Dictionary removedIds, List refreshedIds, List otherIds)>(); + + foreach (var payload in (ContentTypeCacheRefresher.JsonPayload[])args.MessageObject) + { + if (!changedIds.TryGetValue(payload.ItemType, out var idLists)) + { + idLists = (removedIds: new List(), refreshedIds: new List(), otherIds: new List()); + changedIds.Add(payload.ItemType, idLists); + } + + if (payload.ChangeTypes.HasType(ContentTypeChangeTypes.Remove)) + { + idLists.removedIds.Add(payload.Id); + } + else if (payload.ChangeTypes.HasType(ContentTypeChangeTypes.RefreshMain)) + { + idLists.refreshedIds.Add(payload.Id); + } + else if (payload.ChangeTypes.HasType(ContentTypeChangeTypes.RefreshOther)) + { + idLists.otherIds.Add(payload.Id); + } + } + + foreach (var ci in changedIds) + { + if (ci.Value.refreshedIds.Count > 0 || ci.Value.otherIds.Count > 0) + { + switch (ci.Key) + { + case var itemType when itemType == typeof(IContentType).Name: + RefreshContentOfContentTypes(ci.Value.refreshedIds.Concat(ci.Value.otherIds).Distinct().ToArray()); + break; + case var itemType when itemType == typeof(IMediaType).Name: + RefreshMediaOfMediaTypes(ci.Value.refreshedIds.Concat(ci.Value.otherIds).Distinct().ToArray()); + break; + case var itemType when itemType == typeof(IMemberType).Name: + RefreshMemberOfMemberTypes(ci.Value.refreshedIds.Concat(ci.Value.otherIds).Distinct().ToArray()); + break; + } + } + + //Delete all content of this content/media/member type that is in any content indexer by looking up matched examine docs + _umbracoIndexingHandler.DeleteDocumentsForContentTypes(ci.Value.removedIds); + } + } + + private void RefreshMemberOfMemberTypes(int[] memberTypeIds) + { + const int pageSize = 500; + + IEnumerable memberTypes = _memberTypeService.GetAll(memberTypeIds); + foreach (IMemberType memberType in memberTypes) + { + var page = 0; + var total = long.MaxValue; + while (page * pageSize < total) + { + IEnumerable memberToRefresh = _memberService.GetAll( + page++, pageSize, out total, "LoginName", Direction.Ascending, + memberType.Alias); + + foreach (IMember c in memberToRefresh) + { + _umbracoIndexingHandler.ReIndexForMember(c); + } + } + } + } + + private void RefreshMediaOfMediaTypes(int[] mediaTypeIds) + { + const int pageSize = 500; + var page = 0; + var total = long.MaxValue; + while (page * pageSize < total) + { + IEnumerable mediaToRefresh = _mediaService.GetPagedOfTypes( + //Re-index all content of these types + mediaTypeIds, + page++, pageSize, out total, null, + Ordering.By("Path", Direction.Ascending)); + + foreach (IMedia c in mediaToRefresh) + { + _umbracoIndexingHandler.ReIndexForMedia(c, c.Trashed == false); + } + } + } + + private void RefreshContentOfContentTypes(int[] contentTypeIds) + { + const int pageSize = 500; + var page = 0; + var total = long.MaxValue; + while (page * pageSize < total) + { + IEnumerable contentToRefresh = _contentService.GetPagedOfTypes( + //Re-index all content of these types + contentTypeIds, + page++, pageSize, out total, null, + //order by shallowest to deepest, this allows us to check it's published state without checking every item + Ordering.By("Path", Direction.Ascending)); + + //track which Ids have their paths are published + var publishChecked = new Dictionary(); + + foreach (IContent c in contentToRefresh) + { + var isPublished = false; + if (c.Published) + { + if (!publishChecked.TryGetValue(c.ParentId, out isPublished)) + { + //nothing by parent id, so query the service and cache the result for the next child to check against + isPublished = _contentService.IsPathPublished(c); + publishChecked[c.Id] = isPublished; + } + } + + _umbracoIndexingHandler.ReIndexForContent(c, isPublished); + } + } + } + } +} diff --git a/src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.Language.cs b/src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.Language.cs new file mode 100644 index 0000000000..2f7d5f66ca --- /dev/null +++ b/src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.Language.cs @@ -0,0 +1,50 @@ +using System; +using System.Linq; +using Umbraco.Cms.Core.Cache; +using Umbraco.Cms.Core.Events; +using Umbraco.Cms.Core.Notifications; +using Umbraco.Cms.Infrastructure.Examine; + +namespace Umbraco.Cms.Infrastructure.Search +{ + public sealed class LanguageIndexingNotificationHandler : INotificationHandler + { + private readonly IUmbracoIndexingHandler _umbracoIndexingHandler; + private readonly IIndexRebuilder _indexRebuilder; + + public LanguageIndexingNotificationHandler(IUmbracoIndexingHandler umbracoIndexingHandler, IIndexRebuilder indexRebuilder) + { + _umbracoIndexingHandler = umbracoIndexingHandler ?? throw new ArgumentNullException(nameof(umbracoIndexingHandler)); + _indexRebuilder = indexRebuilder ?? throw new ArgumentNullException(nameof(indexRebuilder)); + } + + public void Handle(LanguageCacheRefresherNotification args) + { + if (!_umbracoIndexingHandler.Enabled) + { + return; + } + + if (!(args.MessageObject is LanguageCacheRefresher.JsonPayload[] payloads)) + { + return; + } + + if (payloads.Length == 0) + { + return; + } + + var removedOrCultureChanged = payloads.Any(x => + x.ChangeType == LanguageCacheRefresher.JsonPayload.LanguageChangeType.ChangeCulture + || x.ChangeType == LanguageCacheRefresher.JsonPayload.LanguageChangeType.Remove); + + if (removedOrCultureChanged) + { + //if a lang is removed or it's culture has changed, we need to rebuild the indexes since + //field names and values in the index have a string culture value. + _indexRebuilder.RebuildIndexes(false); + } + } + } +} diff --git a/src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.Media.cs b/src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.Media.cs new file mode 100644 index 0000000000..8b37d047de --- /dev/null +++ b/src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.Media.cs @@ -0,0 +1,90 @@ +using System; +using Umbraco.Cms.Core.Cache; +using Umbraco.Cms.Core.Events; +using Umbraco.Cms.Core.Notifications; +using Umbraco.Cms.Core.Services; +using Umbraco.Cms.Core.Services.Changes; +using Umbraco.Cms.Core.Sync; +using Umbraco.Extensions; + +namespace Umbraco.Cms.Infrastructure.Search +{ + public sealed class MediaIndexingNotificationHandler : INotificationHandler + { + private readonly IUmbracoIndexingHandler _umbracoIndexingHandler; + private readonly IMediaService _mediaService; + + public MediaIndexingNotificationHandler(IUmbracoIndexingHandler umbracoIndexingHandler, IMediaService mediaService) + { + _umbracoIndexingHandler = umbracoIndexingHandler ?? throw new ArgumentNullException(nameof(umbracoIndexingHandler)); + _mediaService = mediaService ?? throw new ArgumentNullException(nameof(mediaService)); + } + + public void Handle(MediaCacheRefresherNotification args) + { + if (!_umbracoIndexingHandler.Enabled) + { + return; + } + + if (Suspendable.ExamineEvents.CanIndex == false) + { + return; + } + + if (args.MessageType != MessageType.RefreshByPayload) + { + throw new NotSupportedException(); + } + + foreach (var payload in (MediaCacheRefresher.JsonPayload[])args.MessageObject) + { + if (payload.ChangeTypes.HasType(TreeChangeTypes.Remove)) + { + // remove from *all* indexes + _umbracoIndexingHandler.DeleteIndexForEntity(payload.Id, false); + } + else if (payload.ChangeTypes.HasType(TreeChangeTypes.RefreshAll)) + { + // ExamineEvents does not support RefreshAll + // just ignore that payload + // so what?! + } + else // RefreshNode or RefreshBranch (maybe trashed) + { + var media = _mediaService.GetById(payload.Id); + if (media == null) + { + // gone fishing, remove entirely + _umbracoIndexingHandler.DeleteIndexForEntity(payload.Id, false); + continue; + } + + if (media.Trashed) + { + _umbracoIndexingHandler.DeleteIndexForEntity(payload.Id, true); + } + + // just that media + _umbracoIndexingHandler.ReIndexForMedia(media, !media.Trashed); + + // branch + if (payload.ChangeTypes.HasType(TreeChangeTypes.RefreshBranch)) + { + const int pageSize = 500; + var page = 0; + var total = long.MaxValue; + while (page * pageSize < total) + { + var descendants = _mediaService.GetPagedDescendants(media.Id, page++, pageSize, out total); + foreach (var descendant in descendants) + { + _umbracoIndexingHandler.ReIndexForMedia(descendant, !descendant.Trashed); + } + } + } + } + } + } + } +} diff --git a/src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.Member.cs b/src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.Member.cs new file mode 100644 index 0000000000..389b839c67 --- /dev/null +++ b/src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.Member.cs @@ -0,0 +1,90 @@ +using System; +using Umbraco.Cms.Core.Cache; +using Umbraco.Cms.Core.Events; +using Umbraco.Cms.Core.Models; +using Umbraco.Cms.Core.Notifications; +using Umbraco.Cms.Core.Services; +using Umbraco.Cms.Core.Sync; + +namespace Umbraco.Cms.Infrastructure.Search +{ + public sealed class MemberIndexingNotificationHandler : INotificationHandler + { + private readonly IUmbracoIndexingHandler _umbracoIndexingHandler; + private readonly IMemberService _memberService; + + public MemberIndexingNotificationHandler(IUmbracoIndexingHandler umbracoIndexingHandler, IMemberService memberService) + { + _umbracoIndexingHandler = umbracoIndexingHandler ?? throw new ArgumentNullException(nameof(umbracoIndexingHandler)); + _memberService = memberService ?? throw new ArgumentNullException(nameof(memberService)); + } + + public void Handle(MemberCacheRefresherNotification args) + { + if (!_umbracoIndexingHandler.Enabled) + { + return; + } + + if (Suspendable.ExamineEvents.CanIndex == false) + { + return; + } + + switch (args.MessageType) + { + case MessageType.RefreshById: + var c1 = _memberService.GetById((int)args.MessageObject); + if (c1 != null) + { + _umbracoIndexingHandler.ReIndexForMember(c1); + } + break; + case MessageType.RemoveById: + + // This is triggered when the item is permanently deleted + + _umbracoIndexingHandler.DeleteIndexForEntity((int)args.MessageObject, false); + break; + case MessageType.RefreshByInstance: + if (args.MessageObject is IMember c3) + { + _umbracoIndexingHandler.ReIndexForMember(c3); + } + break; + case MessageType.RemoveByInstance: + + // This is triggered when the item is permanently deleted + + if (args.MessageObject is IMember c4) + { + _umbracoIndexingHandler.DeleteIndexForEntity(c4.Id, false); + } + break; + case MessageType.RefreshByPayload: + var payload = (MemberCacheRefresher.JsonPayload[])args.MessageObject; + foreach (var p in payload) + { + if (p.Removed) + { + _umbracoIndexingHandler.DeleteIndexForEntity(p.Id, false); + } + else + { + var m = _memberService.GetById(p.Id); + if (m != null) + { + _umbracoIndexingHandler.ReIndexForMember(m); + } + } + } + break; + case MessageType.RefreshAll: + case MessageType.RefreshByJson: + default: + //We don't support these, these message types will not fire for unpublished content + break; + } + } + } +} diff --git a/src/Umbraco.Infrastructure/Services/Implement/CacheInstructionService.cs b/src/Umbraco.Infrastructure/Services/Implement/CacheInstructionService.cs index 0a6e945a23..e0c0f56244 100644 --- a/src/Umbraco.Infrastructure/Services/Implement/CacheInstructionService.cs +++ b/src/Umbraco.Infrastructure/Services/Implement/CacheInstructionService.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Threading; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Newtonsoft.Json; @@ -22,8 +23,6 @@ namespace Umbraco.Cms.Core.Services.Implement /// public class CacheInstructionService : RepositoryService, ICacheInstructionService { - private readonly IServerRoleAccessor _serverRoleAccessor; - private readonly CacheRefresherCollection _cacheRefreshers; private readonly ICacheInstructionRepository _cacheInstructionRepository; private readonly IProfilingLogger _profilingLogger; private readonly ILogger _logger; @@ -36,16 +35,12 @@ namespace Umbraco.Cms.Core.Services.Implement IScopeProvider provider, ILoggerFactory loggerFactory, IEventMessagesFactory eventMessagesFactory, - IServerRoleAccessor serverRoleAccessor, - CacheRefresherCollection cacheRefreshers, ICacheInstructionRepository cacheInstructionRepository, IProfilingLogger profilingLogger, ILogger logger, IOptions globalSettings) : base(provider, loggerFactory, eventMessagesFactory) { - _serverRoleAccessor = serverRoleAccessor; - _cacheRefreshers = cacheRefreshers; _cacheInstructionRepository = cacheInstructionRepository; _profilingLogger = profilingLogger; _logger = logger; @@ -57,7 +52,7 @@ namespace Umbraco.Cms.Core.Services.Implement { using (IScope scope = ScopeProvider.CreateScope(autoComplete: true)) { - if (lastId == 0) + if (lastId <= 0) { var count = _cacheInstructionRepository.CountAll(); @@ -79,7 +74,6 @@ namespace Umbraco.Cms.Core.Services.Implement return false; } } - /// public bool IsInstructionCountOverLimit(int lastId, int limit, out int count) { @@ -133,22 +127,28 @@ namespace Umbraco.Cms.Core.Services.Implement new CacheInstruction(0, DateTime.UtcNow, JsonConvert.SerializeObject(instructions, Formatting.None), localIdentity, instructions.Sum(x => x.JsonIdCount)); /// - public CacheInstructionServiceProcessInstructionsResult ProcessInstructions(bool released, string localIdentity, DateTime lastPruned, int lastId) + public ProcessInstructionsResult ProcessInstructions( + CacheRefresherCollection cacheRefreshers, + ServerRole serverRole, + CancellationToken cancellationToken, + string localIdentity, + DateTime lastPruned, + int lastId) { using (_profilingLogger.DebugDuration("Syncing from database...")) using (IScope scope = ScopeProvider.CreateScope()) { - var numberOfInstructionsProcessed = ProcessDatabaseInstructions(released, localIdentity, ref lastId); + var numberOfInstructionsProcessed = ProcessDatabaseInstructions(cacheRefreshers, cancellationToken, localIdentity, ref lastId); // Check for pruning throttling. - if (released || (DateTime.UtcNow - lastPruned) <= _globalSettings.DatabaseServerMessenger.TimeBetweenPruneOperations) + if (cancellationToken.IsCancellationRequested || (DateTime.UtcNow - lastPruned) <= _globalSettings.DatabaseServerMessenger.TimeBetweenPruneOperations) { scope.Complete(); - return CacheInstructionServiceProcessInstructionsResult.AsCompleted(numberOfInstructionsProcessed, lastId); + return ProcessInstructionsResult.AsCompleted(numberOfInstructionsProcessed, lastId); } var instructionsWerePruned = false; - switch (_serverRoleAccessor.CurrentServerRole) + switch (serverRole) { case ServerRole.Single: case ServerRole.Master: @@ -160,8 +160,8 @@ namespace Umbraco.Cms.Core.Services.Implement scope.Complete(); return instructionsWerePruned - ? CacheInstructionServiceProcessInstructionsResult.AsCompletedAndPruned(numberOfInstructionsProcessed, lastId) - : CacheInstructionServiceProcessInstructionsResult.AsCompleted(numberOfInstructionsProcessed, lastId); + ? ProcessInstructionsResult.AsCompletedAndPruned(numberOfInstructionsProcessed, lastId) + : ProcessInstructionsResult.AsCompleted(numberOfInstructionsProcessed, lastId); } } @@ -172,7 +172,7 @@ namespace Umbraco.Cms.Core.Services.Implement /// Thread safety: this is NOT thread safe. Because it is NOT meant to run multi-threaded. /// /// Number of instructions processed. - private int ProcessDatabaseInstructions(bool released, string localIdentity, ref int lastId) + private int ProcessDatabaseInstructions(CacheRefresherCollection cacheRefreshers, CancellationToken cancellationToken, string localIdentity, ref int lastId) { // NOTE: // We 'could' recurse to ensure that no remaining instructions are pending in the table before proceeding but I don't think that @@ -205,7 +205,7 @@ namespace Umbraco.Cms.Core.Services.Implement { // If this flag gets set it means we're shutting down! In this case, we need to exit asap and cannot // continue processing anything otherwise we'll hold up the app domain shutdown. - if (released) + if (cancellationToken.IsCancellationRequested) { break; } @@ -227,7 +227,7 @@ namespace Umbraco.Cms.Core.Services.Implement List instructionBatch = GetAllInstructions(jsonInstructions); // Process as per-normal. - var success = ProcessDatabaseInstructions(instructionBatch, instruction, processed, released, ref lastId); + var success = ProcessDatabaseInstructions(cacheRefreshers, instructionBatch, instruction, processed, cancellationToken, ref lastId); // If they couldn't be all processed (i.e. we're shutting down) then exit. if (success == false) @@ -295,12 +295,18 @@ namespace Umbraco.Cms.Core.Services.Implement /// /// Returns true if all instructions in the batch were processed, otherwise false if they could not be due to the app being shut down /// - private bool ProcessDatabaseInstructions(IReadOnlyCollection instructionBatch, CacheInstruction instruction, HashSet processed, bool released, ref int lastId) + private bool ProcessDatabaseInstructions( + CacheRefresherCollection cacheRefreshers, + IReadOnlyCollection instructionBatch, + CacheInstruction instruction, + HashSet processed, + CancellationToken cancellationToken, + ref int lastId) { // Execute remote instructions & update lastId. try { - var result = NotifyRefreshers(instructionBatch, processed, released); + var result = NotifyRefreshers(cacheRefreshers, instructionBatch, processed, cancellationToken); if (result) { // If all instructions were processed, set the last id. @@ -330,12 +336,16 @@ namespace Umbraco.Cms.Core.Services.Implement /// /// Returns true if all instructions were processed, otherwise false if the processing was interrupted (i.e. by app shutdown). /// - private bool NotifyRefreshers(IEnumerable instructions, HashSet processed, bool released) + private bool NotifyRefreshers( + CacheRefresherCollection cacheRefreshers, + IEnumerable instructions, + HashSet processed, + CancellationToken cancellationToken) { foreach (RefreshInstruction instruction in instructions) { // Check if the app is shutting down, we need to exit if this happens. - if (released) + if (cancellationToken.IsCancellationRequested) { return false; } @@ -349,22 +359,22 @@ namespace Umbraco.Cms.Core.Services.Implement switch (instruction.RefreshType) { case RefreshMethodType.RefreshAll: - RefreshAll(instruction.RefresherId); + RefreshAll(cacheRefreshers, instruction.RefresherId); break; case RefreshMethodType.RefreshByGuid: - RefreshByGuid(instruction.RefresherId, instruction.GuidId); + RefreshByGuid(cacheRefreshers, instruction.RefresherId, instruction.GuidId); break; case RefreshMethodType.RefreshById: - RefreshById(instruction.RefresherId, instruction.IntId); + RefreshById(cacheRefreshers, instruction.RefresherId, instruction.IntId); break; case RefreshMethodType.RefreshByIds: - RefreshByIds(instruction.RefresherId, instruction.JsonIds); + RefreshByIds(cacheRefreshers, instruction.RefresherId, instruction.JsonIds); break; case RefreshMethodType.RefreshByJson: - RefreshByJson(instruction.RefresherId, instruction.JsonPayload); + RefreshByJson(cacheRefreshers, instruction.RefresherId, instruction.JsonPayload); break; case RefreshMethodType.RemoveById: - RemoveById(instruction.RefresherId, instruction.IntId); + RemoveById(cacheRefreshers, instruction.RefresherId, instruction.IntId); break; } @@ -374,48 +384,48 @@ namespace Umbraco.Cms.Core.Services.Implement return true; } - private void RefreshAll(Guid uniqueIdentifier) + private void RefreshAll(CacheRefresherCollection cacheRefreshers, Guid uniqueIdentifier) { - ICacheRefresher refresher = GetRefresher(uniqueIdentifier); + ICacheRefresher refresher = GetRefresher(cacheRefreshers, uniqueIdentifier); refresher.RefreshAll(); } - private void RefreshByGuid(Guid uniqueIdentifier, Guid id) + private void RefreshByGuid(CacheRefresherCollection cacheRefreshers, Guid uniqueIdentifier, Guid id) { - ICacheRefresher refresher = GetRefresher(uniqueIdentifier); + ICacheRefresher refresher = GetRefresher(cacheRefreshers, uniqueIdentifier); refresher.Refresh(id); } - private void RefreshById(Guid uniqueIdentifier, int id) + private void RefreshById(CacheRefresherCollection cacheRefreshers, Guid uniqueIdentifier, int id) { - ICacheRefresher refresher = GetRefresher(uniqueIdentifier); + ICacheRefresher refresher = GetRefresher(cacheRefreshers, uniqueIdentifier); refresher.Refresh(id); } - private void RefreshByIds(Guid uniqueIdentifier, string jsonIds) + private void RefreshByIds(CacheRefresherCollection cacheRefreshers, Guid uniqueIdentifier, string jsonIds) { - ICacheRefresher refresher = GetRefresher(uniqueIdentifier); + ICacheRefresher refresher = GetRefresher(cacheRefreshers, uniqueIdentifier); foreach (var id in JsonConvert.DeserializeObject(jsonIds)) { refresher.Refresh(id); } } - private void RefreshByJson(Guid uniqueIdentifier, string jsonPayload) + private void RefreshByJson(CacheRefresherCollection cacheRefreshers, Guid uniqueIdentifier, string jsonPayload) { - IJsonCacheRefresher refresher = GetJsonRefresher(uniqueIdentifier); + IJsonCacheRefresher refresher = GetJsonRefresher(cacheRefreshers, uniqueIdentifier); refresher.Refresh(jsonPayload); } - private void RemoveById(Guid uniqueIdentifier, int id) + private void RemoveById(CacheRefresherCollection cacheRefreshers, Guid uniqueIdentifier, int id) { - ICacheRefresher refresher = GetRefresher(uniqueIdentifier); + ICacheRefresher refresher = GetRefresher(cacheRefreshers, uniqueIdentifier); refresher.Remove(id); } - private ICacheRefresher GetRefresher(Guid id) + private ICacheRefresher GetRefresher(CacheRefresherCollection cacheRefreshers, Guid id) { - ICacheRefresher refresher = _cacheRefreshers[id]; + ICacheRefresher refresher = cacheRefreshers[id]; if (refresher == null) { throw new InvalidOperationException("Cache refresher with ID \"" + id + "\" does not exist."); @@ -424,7 +434,7 @@ namespace Umbraco.Cms.Core.Services.Implement return refresher; } - private IJsonCacheRefresher GetJsonRefresher(Guid id) => GetJsonRefresher(GetRefresher(id)); + private IJsonCacheRefresher GetJsonRefresher(CacheRefresherCollection cacheRefreshers, Guid id) => GetJsonRefresher(GetRefresher(cacheRefreshers, id)); private static IJsonCacheRefresher GetJsonRefresher(ICacheRefresher refresher) { diff --git a/src/Umbraco.Infrastructure/Services/Implement/ServerRegistrationService.cs b/src/Umbraco.Infrastructure/Services/Implement/ServerRegistrationService.cs index 9c03f9aabc..248de428a8 100644 --- a/src/Umbraco.Infrastructure/Services/Implement/ServerRegistrationService.cs +++ b/src/Umbraco.Infrastructure/Services/Implement/ServerRegistrationService.cs @@ -154,7 +154,7 @@ namespace Umbraco.Cms.Core.Services.Implement /// /// Gets the local server identity. /// - private string GetCurrentServerIdentity() => NetworkHelper.MachineName // eg DOMAIN\SERVER + private string GetCurrentServerIdentity() => Environment.MachineName // eg DOMAIN\SERVER + "/" + _hostingEnvironment.ApplicationId; // eg /LM/S3SVC/11/ROOT; } } diff --git a/src/Umbraco.Infrastructure/Suspendable.cs b/src/Umbraco.Infrastructure/Suspendable.cs index e96baa44e4..022a641094 100644 --- a/src/Umbraco.Infrastructure/Suspendable.cs +++ b/src/Umbraco.Infrastructure/Suspendable.cs @@ -2,7 +2,6 @@ using Microsoft.Extensions.Logging; using Umbraco.Cms.Core; using Umbraco.Cms.Core.Cache; using Umbraco.Cms.Infrastructure.Examine; -using Umbraco.Cms.Infrastructure.Search; namespace Umbraco.Cms.Infrastructure { @@ -81,7 +80,7 @@ namespace Umbraco.Cms.Infrastructure s_suspended = true; } - public static void ResumeIndexers(IndexRebuilder indexRebuilder, ILogger logger, BackgroundIndexRebuilder backgroundIndexRebuilder) + public static void ResumeIndexers(ExamineIndexRebuilder backgroundIndexRebuilder) { s_suspended = false; diff --git a/src/Umbraco.Infrastructure/Sync/BatchedDatabaseServerMessenger.cs b/src/Umbraco.Infrastructure/Sync/BatchedDatabaseServerMessenger.cs index 940ebfe0cd..9bae34cf3e 100644 --- a/src/Umbraco.Infrastructure/Sync/BatchedDatabaseServerMessenger.cs +++ b/src/Umbraco.Infrastructure/Sync/BatchedDatabaseServerMessenger.cs @@ -27,15 +27,18 @@ namespace Umbraco.Cms.Infrastructure.Sync /// public BatchedDatabaseServerMessenger( IMainDom mainDom, + CacheRefresherCollection cacheRefreshers, + IServerRoleAccessor serverRoleAccessor, ILogger logger, - DatabaseServerMessengerCallbacks callbacks, + ISyncBootStateAccessor syncBootStateAccessor, IHostingEnvironment hostingEnvironment, ICacheInstructionService cacheInstructionService, IJsonSerializer jsonSerializer, IRequestCache requestCache, IRequestAccessor requestAccessor, + LastSyncedFileManager lastSyncedFileManager, IOptions globalSettings) - : base(mainDom, logger, true, callbacks, hostingEnvironment, cacheInstructionService, jsonSerializer, globalSettings) + : base(mainDom, cacheRefreshers, serverRoleAccessor, logger, true, syncBootStateAccessor, hostingEnvironment, cacheInstructionService, jsonSerializer, lastSyncedFileManager, globalSettings) { _requestCache = requestCache; _requestAccessor = requestAccessor; diff --git a/src/Umbraco.Infrastructure/Sync/DatabaseServerMessenger.cs b/src/Umbraco.Infrastructure/Sync/DatabaseServerMessenger.cs index 0b2076a3a7..ee8793f5c9 100644 --- a/src/Umbraco.Infrastructure/Sync/DatabaseServerMessenger.cs +++ b/src/Umbraco.Infrastructure/Sync/DatabaseServerMessenger.cs @@ -1,8 +1,6 @@ using System; using System.Collections.Generic; using System.Diagnostics; -using System.Globalization; -using System.IO; using System.Linq; using System.Threading; using Microsoft.Extensions.Logging; @@ -15,7 +13,6 @@ using Umbraco.Cms.Core.Runtime; using Umbraco.Cms.Core.Serialization; using Umbraco.Cms.Core.Services; using Umbraco.Cms.Core.Sync; -using Umbraco.Extensions; namespace Umbraco.Cms.Infrastructure.Sync { @@ -31,57 +28,61 @@ namespace Umbraco.Cms.Infrastructure.Sync */ private readonly IMainDom _mainDom; + private readonly CacheRefresherCollection _cacheRefreshers; + private readonly IServerRoleAccessor _serverRoleAccessor; + private readonly ISyncBootStateAccessor _syncBootStateAccessor; private readonly ManualResetEvent _syncIdle; private readonly object _locko = new object(); private readonly IHostingEnvironment _hostingEnvironment; - - private readonly Lazy _distCacheFilePath; - private int _lastId = -1; + private readonly LastSyncedFileManager _lastSyncedFileManager; private DateTime _lastSync; private DateTime _lastPruned; - private readonly Lazy _initialized; + private readonly Lazy _initialized; private bool _syncing; - private bool _released; + private readonly CancellationTokenSource _cancellationTokenSource = new CancellationTokenSource(); + private readonly CancellationToken _cancellationToken; /// /// Initializes a new instance of the class. /// protected DatabaseServerMessenger( IMainDom mainDom, + CacheRefresherCollection cacheRefreshers, + IServerRoleAccessor serverRoleAccessor, ILogger logger, bool distributedEnabled, - DatabaseServerMessengerCallbacks callbacks, + ISyncBootStateAccessor syncBootStateAccessor, IHostingEnvironment hostingEnvironment, ICacheInstructionService cacheInstructionService, IJsonSerializer jsonSerializer, + LastSyncedFileManager lastSyncedFileManager, IOptions globalSettings) : base(distributedEnabled) { + _cancellationToken = _cancellationTokenSource.Token; _mainDom = mainDom; + _cacheRefreshers = cacheRefreshers; + _serverRoleAccessor = serverRoleAccessor; _hostingEnvironment = hostingEnvironment; Logger = logger; - Callbacks = callbacks ?? throw new ArgumentNullException(nameof(callbacks)); + _syncBootStateAccessor = syncBootStateAccessor; CacheInstructionService = cacheInstructionService; JsonSerializer = jsonSerializer; + _lastSyncedFileManager = lastSyncedFileManager; GlobalSettings = globalSettings.Value; _lastPruned = _lastSync = DateTime.UtcNow; _syncIdle = new ManualResetEvent(true); - _distCacheFilePath = new Lazy(() => GetDistCacheFilePath(hostingEnvironment)); // See notes on _localIdentity - LocalIdentity = NetworkHelper.MachineName // eg DOMAIN\SERVER + LocalIdentity = Environment.MachineName // eg DOMAIN\SERVER + "/" + hostingEnvironment.ApplicationId // eg /LM/S3SVC/11/ROOT + " [P" + Process.GetCurrentProcess().Id // eg 1234 + "/D" + AppDomain.CurrentDomain.Id // eg 22 + "] " + Guid.NewGuid().ToString("N").ToUpper(); // make it truly unique - _initialized = new Lazy(EnsureInitialized); + _initialized = new Lazy(InitializeWithMainDom); } - private string DistCacheFilePath => _distCacheFilePath.Value; - - public DatabaseServerMessengerCallbacks Callbacks { get; } - public GlobalSettings GlobalSettings { get; } protected ILogger Logger { get; } @@ -102,12 +103,17 @@ namespace Umbraco.Cms.Infrastructure.Sync /// protected string LocalIdentity { get; } + /// + /// Returns true if initialization was successfull (i.e. Is MainDom) + /// + protected bool EnsureInitialized() => _initialized.Value.HasValue; + #region Messenger // we don't care if there are servers listed or not, // if distributed call is enabled we will make the call protected override bool RequiresDistributed(ICacheRefresher refresher, MessageType dispatchType) - => _initialized.Value && DistributedEnabled; + => EnsureInitialized() && DistributedEnabled; protected override void DeliverRemote( ICacheRefresher refresher, @@ -134,7 +140,7 @@ namespace Umbraco.Cms.Infrastructure.Sync /// /// Boots the messenger. /// - private bool EnsureInitialized() + private SyncBootState? InitializeWithMainDom() { // weight:10, must release *before* the published snapshot service, because once released // the service will *not* be able to properly handle our notifications anymore. @@ -145,15 +151,15 @@ namespace Umbraco.Cms.Infrastructure.Sync { lock (_locko) { - _released = true; // no more syncs + _cancellationTokenSource.Cancel(); // no more syncs } - // Wait a max of 5 seconds and then return, so that we don't block + // Wait a max of 3 seconds and then return, so that we don't block // the entire MainDom callbacks chain and prevent the AppDomain from // properly releasing MainDom - a timeout here means that one refresher // is taking too much time processing, however when it's done we will // not update lastId and stop everything. - var idle = _syncIdle.WaitOne(5000); + var idle = _syncIdle.WaitOne(3000); if (idle == false) { Logger.LogWarning("The wait lock timed out, application is shutting down. The current instruction batch will be re-processed."); @@ -163,17 +169,11 @@ namespace Umbraco.Cms.Infrastructure.Sync if (registered == false) { - return false; + // return null if we cannot initialize + return null; } - ReadLastSynced(); // get _lastId - - if (CacheInstructionService.IsColdBootRequired(_lastId)) - { - _lastId = -1; // reset _lastId if instructions are missing - } - - return Initialize(); // boot + return InitializeColdBootState(); } // @@ -183,70 +183,32 @@ namespace Umbraco.Cms.Infrastructure.Sync /// Thread safety: this is NOT thread safe. Because it is NOT meant to run multi-threaded. /// Callers MUST ensure thread-safety. /// - private bool Initialize() + private SyncBootState InitializeColdBootState() { lock (_locko) { - if (_released) + if (_cancellationToken.IsCancellationRequested) { - return false; + return SyncBootState.Unknown; } - var coldboot = false; + SyncBootState syncState = _syncBootStateAccessor.GetSyncBootState(); - // Never synced before. - if (_lastId < 0) - { - // We haven't synced - in this case we aren't going to sync the whole thing, we will assume this is a new - // server and it will need to rebuild it's own caches, e.g. Lucene or the XML cache file. - Logger.LogWarning("No last synced Id found, this generally means this is a new server/install." - + " The server will build its caches and indexes, and then adjust its last synced Id to the latest found in" - + " the database and maintain cache updates based on that Id."); - - coldboot = true; - } - else - { - // Check for how many instructions there are to process, each row contains a count of the number of instructions contained in each - // row so we will sum these numbers to get the actual count. - var limit = GlobalSettings.DatabaseServerMessenger.MaxProcessingInstructionCount; - if (CacheInstructionService.IsInstructionCountOverLimit(_lastId, limit, out int count)) - { - // Too many instructions, proceed to cold boot. - Logger.LogWarning( - "The instruction count ({InstructionCount}) exceeds the specified MaxProcessingInstructionCount ({MaxProcessingInstructionCount})." - + " The server will skip existing instructions, rebuild its caches and indexes entirely, adjust its last synced Id" - + " to the latest found in the database and maintain cache updates based on that Id.", - count, limit); - - coldboot = true; - } - } - - if (coldboot) + if (syncState == SyncBootState.ColdBoot) { // Get the last id in the db and store it. // Note: Do it BEFORE initializing otherwise some instructions might get lost // when doing it before. Some instructions might run twice but this is not an issue. var maxId = CacheInstructionService.GetMaxInstructionId(); - // If there is a max currently, or if we've never synced. - if (maxId > 0 || _lastId < 0) + // if there is a max currently, or if we've never synced + if (maxId > 0 || _lastSyncedFileManager.LastSyncedId < 0) { - SaveLastSynced(maxId); - } - - // Execute initializing callbacks. - if (Callbacks.InitializingCallbacks != null) - { - foreach (Action callback in Callbacks.InitializingCallbacks) - { - callback(); - } + _lastSyncedFileManager.SaveLastSyncedId(maxId); } } - return true; + return syncState; } } @@ -255,7 +217,7 @@ namespace Umbraco.Cms.Infrastructure.Sync /// public override void Sync() { - if (!_initialized.Value) + if (!EnsureInitialized()) { return; } @@ -268,7 +230,7 @@ namespace Umbraco.Cms.Infrastructure.Sync } // Don't continue if we are released - if (_released) + if (_cancellationToken.IsCancellationRequested) { return; } @@ -286,7 +248,14 @@ namespace Umbraco.Cms.Infrastructure.Sync try { - CacheInstructionServiceProcessInstructionsResult result = CacheInstructionService.ProcessInstructions(_released, LocalIdentity, _lastPruned, _lastId); + ProcessInstructionsResult result = CacheInstructionService.ProcessInstructions( + _cacheRefreshers, + _serverRoleAccessor.CurrentServerRole, + _cancellationToken, + LocalIdentity, + _lastPruned, + _lastSyncedFileManager.LastSyncedId); + if (result.InstructionsWerePruned) { _lastPruned = _lastSync; @@ -294,7 +263,7 @@ namespace Umbraco.Cms.Infrastructure.Sync if (result.LastId > 0) { - SaveLastSynced(result.LastId); + _lastSyncedFileManager.SaveLastSyncedId(result.LastId); } } finally @@ -309,60 +278,6 @@ namespace Umbraco.Cms.Infrastructure.Sync } } - /// - /// Reads the last-synced id from file into memory. - /// - /// - /// Thread safety: this is NOT thread safe. Because it is NOT meant to run multi-threaded. - /// - private void ReadLastSynced() - { - if (File.Exists(DistCacheFilePath) == false) - { - return; - } - - var content = File.ReadAllText(DistCacheFilePath); - if (int.TryParse(content, out var last)) - { - _lastId = last; - } - } - - /// - /// Updates the in-memory last-synced id and persists it to file. - /// - /// The id. - /// - /// Thread safety: this is NOT thread safe. Because it is NOT meant to run multi-threaded. - /// - private void SaveLastSynced(int id) - { - File.WriteAllText(DistCacheFilePath, id.ToString(CultureInfo.InvariantCulture)); - _lastId = id; - } - - private string GetDistCacheFilePath(IHostingEnvironment hostingEnvironment) - { - var fileName = _hostingEnvironment.ApplicationId.ReplaceNonAlphanumericChars(string.Empty) + "-lastsynced.txt"; - - var distCacheFilePath = Path.Combine(hostingEnvironment.LocalTempPath, "DistCache", fileName); - - //ensure the folder exists - var folder = Path.GetDirectoryName(distCacheFilePath); - if (folder == null) - { - throw new InvalidOperationException("The folder could not be determined for the file " + distCacheFilePath); - } - - if (Directory.Exists(folder) == false) - { - Directory.CreateDirectory(folder); - } - - return distCacheFilePath; - } - #endregion } } diff --git a/src/Umbraco.Infrastructure/Sync/LastSyncedFileManager.cs b/src/Umbraco.Infrastructure/Sync/LastSyncedFileManager.cs new file mode 100644 index 0000000000..3b3351fd93 --- /dev/null +++ b/src/Umbraco.Infrastructure/Sync/LastSyncedFileManager.cs @@ -0,0 +1,89 @@ +using System; +using System.Globalization; +using System.IO; +using System.Threading; +using Umbraco.Cms.Core.Hosting; +using Umbraco.Extensions; + +namespace Umbraco.Cms.Infrastructure.Sync +{ + public sealed class LastSyncedFileManager + { + private string _distCacheFile; + private bool _lastIdReady; + private object _lastIdLock; + private int _lastId; + private readonly IHostingEnvironment _hostingEnvironment; + + public LastSyncedFileManager(IHostingEnvironment hostingEnvironment) + => _hostingEnvironment = hostingEnvironment; + + /// + /// Persists the last-synced id to file. + /// + /// The id. + public void SaveLastSyncedId(int id) + { + lock (_lastIdLock) + { + if (!_lastIdReady) + { + throw new InvalidOperationException("Cannot save the last synced id before it is read"); + } + + File.WriteAllText(DistCacheFilePath, id.ToString(CultureInfo.InvariantCulture)); + _lastId = id; + } + } + + /// + /// Returns the last-synced id. + /// + public int LastSyncedId => LazyInitializer.EnsureInitialized( + ref _lastId, + ref _lastIdReady, + ref _lastIdLock, + () => + { + // On first load, read from file, else it will return the in-memory _lastId value + + var distCacheFilePath = DistCacheFilePath; + + if (File.Exists(distCacheFilePath)) + { + var content = File.ReadAllText(distCacheFilePath); + if (int.TryParse(content, out var last)) + { + return last; + } + } + + return -1; + }); + + /// + /// Gets the dist cache file path (once). + /// + /// + public string DistCacheFilePath => LazyInitializer.EnsureInitialized(ref _distCacheFile, () => + { + var fileName = (Environment.MachineName + _hostingEnvironment.ApplicationId).GenerateHash() + "-lastsynced.txt"; + + var distCacheFilePath = Path.Combine(_hostingEnvironment.LocalTempPath, "DistCache", fileName); + + //ensure the folder exists + var folder = Path.GetDirectoryName(distCacheFilePath); + if (folder == null) + { + throw new InvalidOperationException("The folder could not be determined for the file " + distCacheFilePath); + } + + if (Directory.Exists(folder) == false) + { + Directory.CreateDirectory(folder); + } + + return distCacheFilePath; + }); + } +} diff --git a/src/Umbraco.Infrastructure/Sync/SyncBootStateAccessor.cs b/src/Umbraco.Infrastructure/Sync/SyncBootStateAccessor.cs new file mode 100644 index 0000000000..9a77c57965 --- /dev/null +++ b/src/Umbraco.Infrastructure/Sync/SyncBootStateAccessor.cs @@ -0,0 +1,84 @@ +using System.Threading; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Umbraco.Cms.Core.Configuration.Models; +using Umbraco.Cms.Core.Services; +using Umbraco.Cms.Core.Sync; + +namespace Umbraco.Cms.Infrastructure.Sync +{ + public class SyncBootStateAccessor : ISyncBootStateAccessor + { + private readonly ILogger _logger; + private readonly LastSyncedFileManager _lastSyncedFileManager; + private readonly GlobalSettings _globalSettings; + private readonly ICacheInstructionService _cacheInstructionService; + + private SyncBootState _syncBootState; + private bool _syncBootStateReady; + private object _syncBootStateLock; + + public SyncBootStateAccessor( + ILogger logger, + LastSyncedFileManager lastSyncedFileManager, + IOptions globalSettings, + ICacheInstructionService cacheInstructionService) + { + _logger = logger; + _lastSyncedFileManager = lastSyncedFileManager; + _globalSettings = globalSettings.Value; + _cacheInstructionService = cacheInstructionService; + } + + public SyncBootState GetSyncBootState() + => LazyInitializer.EnsureInitialized( + ref _syncBootState, + ref _syncBootStateReady, + ref _syncBootStateLock, + () => InitializeColdBootState(_lastSyncedFileManager.LastSyncedId)); + + private SyncBootState InitializeColdBootState(int lastId) + { + var coldboot = false; + + // Never synced before. + if (lastId < 0) + { + // We haven't synced - in this case we aren't going to sync the whole thing, we will assume this is a new + // server and it will need to rebuild it's own caches, e.g. Lucene or the XML cache file. + _logger.LogWarning("No last synced Id found, this generally means this is a new server/install. " + + "A cold boot will be triggered."); + + coldboot = true; + } + else + { + if (_cacheInstructionService.IsColdBootRequired(lastId)) + { + _logger.LogWarning("Last synced Id found {LastSyncedId} but was not found in the database. This generally means this server/install " + + " has been idle for too long and the instructions in the database have been pruned. A cold boot will be triggered.", lastId); + + coldboot = true; + } + else + { + // Check for how many instructions there are to process, each row contains a count of the number of instructions contained in each + // row so we will sum these numbers to get the actual count. + var limit = _globalSettings.DatabaseServerMessenger.MaxProcessingInstructionCount; + if (_cacheInstructionService.IsInstructionCountOverLimit(lastId, limit, out int count)) + { + // Too many instructions, proceed to cold boot. + _logger.LogWarning( + "The instruction count ({InstructionCount}) exceeds the specified MaxProcessingInstructionCount ({MaxProcessingInstructionCount}). " + + "A cold boot will be triggered.", + count, limit); + + coldboot = true; + } + } + } + + return coldboot ? SyncBootState.ColdBoot : SyncBootState.WarmBoot; + } + } +} diff --git a/src/Umbraco.Infrastructure/Umbraco.Infrastructure.csproj b/src/Umbraco.Infrastructure/Umbraco.Infrastructure.csproj index 9dbacfa267..54970e58a9 100644 --- a/src/Umbraco.Infrastructure/Umbraco.Infrastructure.csproj +++ b/src/Umbraco.Infrastructure/Umbraco.Infrastructure.csproj @@ -49,7 +49,7 @@ - + all diff --git a/src/Umbraco.PublishedCache.NuCache/DependencyInjection/UmbracoBuilderExtensions.cs b/src/Umbraco.PublishedCache.NuCache/DependencyInjection/UmbracoBuilderExtensions.cs index 82e62b2328..113a2245d8 100644 --- a/src/Umbraco.PublishedCache.NuCache/DependencyInjection/UmbracoBuilderExtensions.cs +++ b/src/Umbraco.PublishedCache.NuCache/DependencyInjection/UmbracoBuilderExtensions.cs @@ -29,9 +29,6 @@ namespace Umbraco.Extensions // must register default options, required in the service ctor builder.Services.TryAddTransient(factory => new PublishedSnapshotServiceOptions()); builder.SetPublishedSnapshotService(); - - // Add as itself - builder.Services.TryAddSingleton(); builder.Services.TryAddSingleton(); // replace this service since we want to improve the content/media diff --git a/src/Umbraco.PublishedCache.NuCache/PublishedSnapshot.cs b/src/Umbraco.PublishedCache.NuCache/PublishedSnapshot.cs index ead279a199..fc4c64d552 100644 --- a/src/Umbraco.PublishedCache.NuCache/PublishedSnapshot.cs +++ b/src/Umbraco.PublishedCache.NuCache/PublishedSnapshot.cs @@ -14,9 +14,9 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache #region Constructors - public PublishedSnapshot(PublishedSnapshotService service, bool defaultPreview) + public PublishedSnapshot(IPublishedSnapshotService service, bool defaultPreview) { - _service = service; + _service = service as PublishedSnapshotService; _defaultPreview = defaultPreview; } @@ -38,7 +38,17 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache } } - private PublishedSnapshotElements Elements => _elements ?? (_elements = _service.GetElements(_defaultPreview)); + private PublishedSnapshotElements Elements + { + get + { + if (_service == null) + { + throw new InvalidOperationException($"The {typeof(PublishedSnapshot)} cannot be used when the {typeof(IPublishedSnapshotService)} is not the default type {typeof(PublishedSnapshotService)}"); + } + return _elements ??= _service.GetElements(_defaultPreview); + } + } public void Resync() { diff --git a/src/Umbraco.PublishedCache.NuCache/PublishedSnapshotService.cs b/src/Umbraco.PublishedCache.NuCache/PublishedSnapshotService.cs index 916fb2da5e..9c08a2fc5a 100644 --- a/src/Umbraco.PublishedCache.NuCache/PublishedSnapshotService.cs +++ b/src/Umbraco.PublishedCache.NuCache/PublishedSnapshotService.cs @@ -19,6 +19,7 @@ using Umbraco.Cms.Core.Runtime; using Umbraco.Cms.Core.Scoping; using Umbraco.Cms.Core.Services; using Umbraco.Cms.Core.Services.Changes; +using Umbraco.Cms.Core.Sync; using Umbraco.Cms.Infrastructure.PublishedCache.DataSource; using Umbraco.Cms.Infrastructure.PublishedCache.Persistence; using Umbraco.Extensions; @@ -29,6 +30,9 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache { internal class PublishedSnapshotService : IPublishedSnapshotService { + private readonly PublishedSnapshotServiceOptions _options; + private readonly ISyncBootStateAccessor _syncBootStateAccessor; + private readonly IMainDom _mainDom; private readonly ServiceContext _serviceContext; private readonly IPublishedContentTypeFactory _publishedContentTypeFactory; private readonly IPublishedSnapshotAccessor _publishedSnapshotAccessor; @@ -39,7 +43,6 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache private readonly ILogger _logger; private readonly ILoggerFactory _loggerFactory; private readonly GlobalSettings _globalSettings; - private readonly IEntityXmlSerializer _entitySerializer; private readonly IPublishedModelFactory _publishedModelFactory; private readonly IDefaultCultureAccessor _defaultCultureAccessor; private readonly IHostingEnvironment _hostingEnvironment; @@ -49,9 +52,9 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache private bool _isReadSet; private object _isReadyLock; - private readonly ContentStore _contentStore; - private readonly ContentStore _mediaStore; - private readonly SnapDictionary _domainStore; + private ContentStore _contentStore; + private ContentStore _mediaStore; + private SnapDictionary _domainStore; private readonly object _storesLock = new object(); private readonly object _elementsLock = new object(); @@ -73,6 +76,7 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache public PublishedSnapshotService( PublishedSnapshotServiceOptions options, + ISyncBootStateAccessor syncBootStateAccessor, IMainDom mainDom, ServiceContext serviceContext, IPublishedContentTypeFactory publishedContentTypeFactory, @@ -84,11 +88,13 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache INuCacheContentService publishedContentService, IDefaultCultureAccessor defaultCultureAccessor, IOptions globalSettings, - IEntityXmlSerializer entitySerializer, IPublishedModelFactory publishedModelFactory, IHostingEnvironment hostingEnvironment, IOptions config) { + _options = options; + _syncBootStateAccessor = syncBootStateAccessor; + _mainDom = mainDom; _serviceContext = serviceContext; _publishedContentTypeFactory = publishedContentTypeFactory; _publishedSnapshotAccessor = publishedSnapshotAccessor; @@ -102,41 +108,7 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache _globalSettings = globalSettings.Value; _hostingEnvironment = hostingEnvironment; _config = config.Value; - - // we need an Xml serializer here so that the member cache can support XPath, - // for members this is done by navigating the serialized-to-xml member - _entitySerializer = entitySerializer; _publishedModelFactory = publishedModelFactory; - - // lock this entire call, we only want a single thread to be accessing the stores at once and within - // the call below to mainDom.Register, a callback may occur on a threadpool thread to MainDomRelease - // at the same time as we are trying to write to the stores. MainDomRelease also locks on _storesLock so - // it will not be able to close the stores until we are done populating (if the store is empty) - lock (_storesLock) - { - if (!options.IgnoreLocalDb) - { - mainDom.Register(MainDomRegister, MainDomRelease); - - // stores are created with a db so they can write to it, but they do not read from it, - // stores need to be populated, happens in OnResolutionFrozen which uses _localDbExists to - // figure out whether it can read the databases or it should populate them from sql - - _logger.LogInformation("Creating the content store, localContentDbExists? {LocalContentDbExists}", _localContentDbExists); - _contentStore = new ContentStore(_publishedSnapshotAccessor, _variationContextAccessor, _loggerFactory.CreateLogger("ContentStore"), _loggerFactory, _publishedModelFactory, _localContentDb); - _logger.LogInformation("Creating the media store, localMediaDbExists? {LocalMediaDbExists}", _localMediaDbExists); - _mediaStore = new ContentStore(_publishedSnapshotAccessor, _variationContextAccessor, _loggerFactory.CreateLogger("ContentStore"), _loggerFactory, _publishedModelFactory, _localMediaDb); - } - else - { - _logger.LogInformation("Creating the content store (local db ignored)"); - _contentStore = new ContentStore(_publishedSnapshotAccessor, _variationContextAccessor, _loggerFactory.CreateLogger("ContentStore"), _loggerFactory, _publishedModelFactory); - _logger.LogInformation("Creating the media store (local db ignored)"); - _mediaStore = new ContentStore(_publishedSnapshotAccessor, _variationContextAccessor, _loggerFactory.CreateLogger("ContentStore"), _loggerFactory, _publishedModelFactory); - } - - _domainStore = new SnapDictionary(); - } } protected PublishedSnapshot CurrentPublishedSnapshot => (PublishedSnapshot)_publishedSnapshotAccessor.PublishedSnapshot; @@ -144,13 +116,29 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache // NOTE: These aren't used within this object but are made available internally to improve the IdKey lookup performance // when nucache is enabled. // TODO: Does this need to be here? - internal int GetDocumentId(Guid udi) => GetId(_contentStore, udi); + internal int GetDocumentId(Guid udi) + { + EnsureCaches(); + return GetId(_contentStore, udi); + } - internal int GetMediaId(Guid udi) => GetId(_mediaStore, udi); + internal int GetMediaId(Guid udi) + { + EnsureCaches(); + return GetId(_mediaStore, udi); + } - internal Guid GetDocumentUid(int id) => GetUid(_contentStore, id); + internal Guid GetDocumentUid(int id) + { + EnsureCaches(); + return GetUid(_contentStore, id); + } - internal Guid GetMediaUid(int id) => GetUid(_mediaStore, id); + internal Guid GetMediaUid(int id) + { + EnsureCaches(); + return GetUid(_mediaStore, id); + } private int GetId(ContentStore store, Guid uid) => store.LiveSnapshot.Get(uid)?.Id ?? 0; @@ -249,7 +237,7 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache } /// - /// Populates the stores + /// Lazily populates the stores only when they are first requested /// internal void EnsureCaches() => LazyInitializer.EnsureInitialized( ref _isReady, @@ -257,15 +245,43 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache ref _isReadyLock, () => { - // even though we are ready locked here we want to ensure that the stores lock is also locked + // lock this entire call, we only want a single thread to be accessing the stores at once and within + // the call below to mainDom.Register, a callback may occur on a threadpool thread to MainDomRelease + // at the same time as we are trying to write to the stores. MainDomRelease also locks on _storesLock so + // it will not be able to close the stores until we are done populating (if the store is empty) lock (_storesLock) { + if (!_options.IgnoreLocalDb) + { + _mainDom.Register(MainDomRegister, MainDomRelease); + + // stores are created with a db so they can write to it, but they do not read from it, + // stores need to be populated, happens in OnResolutionFrozen which uses _localDbExists to + // figure out whether it can read the databases or it should populate them from sql + + _logger.LogInformation("Creating the content store, localContentDbExists? {LocalContentDbExists}", _localContentDbExists); + _contentStore = new ContentStore(_publishedSnapshotAccessor, _variationContextAccessor, _loggerFactory.CreateLogger("ContentStore"), _loggerFactory, _publishedModelFactory, _localContentDb); + _logger.LogInformation("Creating the media store, localMediaDbExists? {LocalMediaDbExists}", _localMediaDbExists); + _mediaStore = new ContentStore(_publishedSnapshotAccessor, _variationContextAccessor, _loggerFactory.CreateLogger("ContentStore"), _loggerFactory, _publishedModelFactory, _localMediaDb); + } + else + { + _logger.LogInformation("Creating the content store (local db ignored)"); + _contentStore = new ContentStore(_publishedSnapshotAccessor, _variationContextAccessor, _loggerFactory.CreateLogger("ContentStore"), _loggerFactory, _publishedModelFactory); + _logger.LogInformation("Creating the media store (local db ignored)"); + _mediaStore = new ContentStore(_publishedSnapshotAccessor, _variationContextAccessor, _loggerFactory.CreateLogger("ContentStore"), _loggerFactory, _publishedModelFactory); + } + + _domainStore = new SnapDictionary(); + var okContent = false; var okMedia = false; + SyncBootState bootState = _syncBootStateAccessor.GetSyncBootState(); + try { - if (_localContentDbExists) + if (bootState != SyncBootState.ColdBoot && _localContentDbExists) { okContent = LockAndLoadContent(() => LoadContentFromLocalDbLocked(true)); if (!okContent) @@ -274,7 +290,7 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache } } - if (_localMediaDbExists) + if (bootState != SyncBootState.ColdBoot && _localMediaDbExists) { okMedia = LockAndLoadMedia(() => LoadMediaFromLocalDbLocked(true)); if (!okMedia) diff --git a/src/Umbraco.PublishedCache.NuCache/PublishedSnapshotStatus.cs b/src/Umbraco.PublishedCache.NuCache/PublishedSnapshotStatus.cs index df4f803006..6a75e3e021 100644 --- a/src/Umbraco.PublishedCache.NuCache/PublishedSnapshotStatus.cs +++ b/src/Umbraco.PublishedCache.NuCache/PublishedSnapshotStatus.cs @@ -11,9 +11,9 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache private readonly PublishedSnapshotService _service; private readonly INuCacheContentService _publishedContentService; - public PublishedSnapshotStatus(PublishedSnapshotService service, INuCacheContentService publishedContentService) + public PublishedSnapshotStatus(IPublishedSnapshotService service, INuCacheContentService publishedContentService) { - _service = service; + _service = service as PublishedSnapshotService; _publishedContentService = publishedContentService; } @@ -23,6 +23,12 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache /// public string GetStatus() { + if (_service == null) + { + return $"The current {typeof(IPublishedSnapshotService)} is not the default type. A status cannot be determined."; + } + + // TODO: This should be private _service.EnsureCaches(); var dbCacheIsOk = _publishedContentService.VerifyContentDbCache() diff --git a/src/Umbraco.Core/TaskHelper.cs b/src/Umbraco.Tests.Common/TaskHelper.cs similarity index 95% rename from src/Umbraco.Core/TaskHelper.cs rename to src/Umbraco.Tests.Common/TaskHelper.cs index ba9f865eba..8b22f7b47d 100644 --- a/src/Umbraco.Core/TaskHelper.cs +++ b/src/Umbraco.Tests.Common/TaskHelper.cs @@ -7,7 +7,7 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -namespace Umbraco.Cms.Core +namespace Umbraco.Cms.Tests.Common { /// /// Helper class to not repeat common patterns with Task. @@ -24,7 +24,7 @@ namespace Umbraco.Cms.Core public void RunBackgroundTask(Func fn) => ExecuteBackgroundTask(fn); // for tests, returning the Task as a public API indicates it can be awaited that is not what we want to do - internal Task ExecuteBackgroundTask(Func fn) + public Task ExecuteBackgroundTask(Func fn) { // it is also possible to use UnsafeQueueUserWorkItem which does not flow the execution context, // however that seems more difficult to use for async operations. @@ -45,7 +45,7 @@ namespace Umbraco.Cms.Core public void RunLongRunningBackgroundTask(Func fn) => ExecuteLongRunningBackgroundTask(fn); // for tests, returning the Task as a public API indicates it can be awaited that is not what we want to do - internal Task ExecuteLongRunningBackgroundTask(Func fn) + public Task ExecuteLongRunningBackgroundTask(Func fn) { // it is also possible to use UnsafeQueueUserWorkItem which does not flow the execution context, // however that seems more difficult to use for async operations. diff --git a/src/Umbraco.Tests.Integration/Implementations/TestHostingEnvironment.cs b/src/Umbraco.Tests.Common/Testing/TestHostingEnvironment.cs similarity index 62% rename from src/Umbraco.Tests.Integration/Implementations/TestHostingEnvironment.cs rename to src/Umbraco.Tests.Common/Testing/TestHostingEnvironment.cs index 8980a91cff..e34161a3c2 100644 --- a/src/Umbraco.Tests.Integration/Implementations/TestHostingEnvironment.cs +++ b/src/Umbraco.Tests.Common/Testing/TestHostingEnvironment.cs @@ -7,15 +7,22 @@ using Umbraco.Cms.Core.Configuration.Models; using Umbraco.Cms.Web.Common.AspNetCore; using IHostingEnvironment = Umbraco.Cms.Core.Hosting.IHostingEnvironment; -namespace Umbraco.Cms.Tests.Integration.Implementations +namespace Umbraco.Cms.Tests.Common.Testing { - public class TestHostingEnvironment : AspNetCoreHostingEnvironment, Cms.Core.Hosting.IHostingEnvironment + public class TestHostingEnvironment : AspNetCoreHostingEnvironment, IHostingEnvironment { - public TestHostingEnvironment(IOptionsMonitor hostingSettings,IOptionsMonitor webRoutingSettings, IWebHostEnvironment webHostEnvironment) - : base(hostingSettings,webRoutingSettings, webHostEnvironment) + public TestHostingEnvironment( + IOptionsMonitor hostingSettings, + IOptionsMonitor webRoutingSettings, + IWebHostEnvironment webHostEnvironment) + : base(null, hostingSettings, webRoutingSettings, webHostEnvironment) { } + // override + string IHostingEnvironment.ApplicationId { get; } = "TestApplication"; + + /// /// Gets a value indicating whether we are hosted. /// diff --git a/src/Umbraco.Tests.Common/Umbraco.Tests.Common.csproj b/src/Umbraco.Tests.Common/Umbraco.Tests.Common.csproj index 373c319218..58527c4cfa 100644 --- a/src/Umbraco.Tests.Common/Umbraco.Tests.Common.csproj +++ b/src/Umbraco.Tests.Common/Umbraco.Tests.Common.csproj @@ -1,7 +1,7 @@ - + - netstandard2.0 + net5.0 Umbraco.Cms.Tests.Common Umbraco.Cms.Tests Umbraco CMS Test Tools @@ -24,5 +24,6 @@ + diff --git a/src/Umbraco.Tests.Integration/ComponentRuntimeTests.cs b/src/Umbraco.Tests.Integration/ComponentRuntimeTests.cs index ddac52872f..baa194b17e 100644 --- a/src/Umbraco.Tests.Integration/ComponentRuntimeTests.cs +++ b/src/Umbraco.Tests.Integration/ComponentRuntimeTests.cs @@ -10,6 +10,7 @@ using Umbraco.Cms.Core.Runtime; using Umbraco.Cms.Core.Services; using Umbraco.Cms.Tests.Common.Testing; using Umbraco.Cms.Tests.Integration.Testing; +using Umbraco.Extensions; namespace Umbraco.Cms.Tests.Integration { @@ -18,7 +19,11 @@ namespace Umbraco.Cms.Tests.Integration public class ComponentRuntimeTests : UmbracoIntegrationTest { // ensure composers are added - protected override void CustomTestSetup(IUmbracoBuilder builder) => builder.AddComposers(); + protected override void CustomTestSetup(IUmbracoBuilder builder) + { + builder.AddNuCache(); + builder.AddComposers(); + } /// /// This will boot up umbraco with components enabled to show they initialize and shutdown diff --git a/src/Umbraco.Tests.Integration/DependencyInjection/UmbracoBuilderExtensions.cs b/src/Umbraco.Tests.Integration/DependencyInjection/UmbracoBuilderExtensions.cs index 5cc94ce4c9..bb0da4d08a 100644 --- a/src/Umbraco.Tests.Integration/DependencyInjection/UmbracoBuilderExtensions.cs +++ b/src/Umbraco.Tests.Integration/DependencyInjection/UmbracoBuilderExtensions.cs @@ -2,8 +2,11 @@ // See LICENSE for more details. using System; +using System.Collections.Generic; using System.IO; using System.Linq; +using Examine; +using Examine.Lucene.Directories; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -21,7 +24,6 @@ using Umbraco.Cms.Core.WebAssets; using Umbraco.Cms.Infrastructure.Examine; using Umbraco.Cms.Infrastructure.HostedServices; using Umbraco.Cms.Infrastructure.PublishedCache; -using Umbraco.Cms.Infrastructure.Search; using Umbraco.Cms.Tests.Common.TestHelpers.Stubs; using Umbraco.Cms.Tests.Integration.Implementations; using Umbraco.Extensions; @@ -43,7 +45,7 @@ namespace Umbraco.Cms.Tests.Integration.DependencyInjection builder.Services.AddUnique(Mock.Of()); builder.Services.AddUnique(testHelper.MainDom); - builder.Services.AddUnique(); + builder.Services.AddUnique(); builder.Services.AddUnique(factory => Mock.Of()); // we don't want persisted nucache files in tests @@ -51,7 +53,7 @@ namespace Umbraco.Cms.Tests.Integration.DependencyInjection #if IS_WINDOWS // ensure all lucene indexes are using RAM directory (no file system) - builder.Services.AddUnique(); + builder.Services.AddUnique(); #endif // replace this service so that it can lookup the correct file locations @@ -97,18 +99,18 @@ namespace Umbraco.Cms.Tests.Integration.DependencyInjection } // replace the default so there is no background index rebuilder - private class TestBackgroundIndexRebuilder : BackgroundIndexRebuilder + private class TestBackgroundIndexRebuilder : ExamineIndexRebuilder { - public TestBackgroundIndexRebuilder( - IMainDom mainDom, - ILogger logger, - IndexRebuilder indexRebuilder, - IBackgroundTaskQueue backgroundTaskQueue) - : base(mainDom, logger, indexRebuilder, backgroundTaskQueue) + public TestBackgroundIndexRebuilder(IMainDom mainDom, IRuntimeState runtimeState, ILogger logger, IExamineManager examineManager, IEnumerable populators, IBackgroundTaskQueue backgroundTaskQueue) : base(mainDom, runtimeState, logger, examineManager, populators, backgroundTaskQueue) { } - public override void RebuildIndexes(bool onlyEmptyIndexes, TimeSpan? delay = null) + public override void RebuildIndex(string indexName, TimeSpan? delay = null, bool useBackgroundThread = true) + { + // noop + } + + public override void RebuildIndexes(bool onlyEmptyIndexes, TimeSpan? delay = null, bool useBackgroundThread = true) { // noop } diff --git a/src/Umbraco.Tests.Integration/Implementations/TestHelper.cs b/src/Umbraco.Tests.Integration/Implementations/TestHelper.cs index 8e897011d2..90c5e0eb02 100644 --- a/src/Umbraco.Tests.Integration/Implementations/TestHelper.cs +++ b/src/Umbraco.Tests.Integration/Implementations/TestHelper.cs @@ -12,8 +12,10 @@ using System.Reflection; using System.Threading; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.FileProviders; using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Hosting.Internal; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Moq; @@ -31,6 +33,7 @@ using Umbraco.Cms.Core.PropertyEditors; using Umbraco.Cms.Core.Runtime; using Umbraco.Cms.Infrastructure.Persistence; using Umbraco.Cms.Tests.Common; +using Umbraco.Cms.Tests.Common.Testing; using Umbraco.Cms.Web.Common.AspNetCore; using Umbraco.Extensions; using File = System.IO.File; diff --git a/src/Umbraco.Tests.Integration/TestServerTest/UmbracoTestServerTestBase.cs b/src/Umbraco.Tests.Integration/TestServerTest/UmbracoTestServerTestBase.cs index 635a17a2b1..b91a034420 100644 --- a/src/Umbraco.Tests.Integration/TestServerTest/UmbracoTestServerTestBase.cs +++ b/src/Umbraco.Tests.Integration/TestServerTest/UmbracoTestServerTestBase.cs @@ -150,6 +150,7 @@ namespace Umbraco.Cms.Tests.Integration.TestServerTest .AddConfiguration() .AddUmbracoCore() .AddWebComponents() + .AddNuCache() .AddRuntimeMinifier() .AddBackOfficeCore() .AddBackOfficeAuthentication() diff --git a/src/Umbraco.Tests.Integration/Testing/IntegrationTestComponent.cs b/src/Umbraco.Tests.Integration/Testing/IntegrationTestComponent.cs index 277510fc9e..c0b490e0e5 100644 --- a/src/Umbraco.Tests.Integration/Testing/IntegrationTestComponent.cs +++ b/src/Umbraco.Tests.Integration/Testing/IntegrationTestComponent.cs @@ -2,7 +2,7 @@ // See LICENSE for more details. using Examine; -using Examine.LuceneEngine.Providers; +using Examine.Lucene.Providers; using Umbraco.Cms.Core.Composing; namespace Umbraco.Cms.Tests.Integration.Testing @@ -31,7 +31,7 @@ namespace Umbraco.Cms.Tests.Integration.Testing { if (index is LuceneIndex luceneIndex) { - luceneIndex.ProcessNonAsync(); + luceneIndex.WithThreadingMode(IndexThreadingMode.Synchronous); } } } diff --git a/src/Umbraco.Tests.Integration/Testing/UmbracoIntegrationTest.cs b/src/Umbraco.Tests.Integration/Testing/UmbracoIntegrationTest.cs index dbf047cf48..f0eac637fd 100644 --- a/src/Umbraco.Tests.Integration/Testing/UmbracoIntegrationTest.cs +++ b/src/Umbraco.Tests.Integration/Testing/UmbracoIntegrationTest.cs @@ -138,12 +138,13 @@ namespace Umbraco.Cms.Tests.Integration.Testing Log.Logger = new LoggerConfiguration() .WriteTo.File(path, rollingInterval: RollingInterval.Day) + .MinimumLevel.Debug() .CreateLogger(); builder.AddSerilog(Log.Logger); }); case UmbracoTestOptions.Logger.Console: - return Microsoft.Extensions.Logging.LoggerFactory.Create(builder => builder.AddConsole()); + return Microsoft.Extensions.Logging.LoggerFactory.Create(builder => builder.AddConsole().SetMinimumLevel(LogLevel.Debug)); } } catch diff --git a/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/ExamineBaseTest.cs b/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/ExamineBaseTest.cs new file mode 100644 index 0000000000..8840988ac6 --- /dev/null +++ b/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/ExamineBaseTest.cs @@ -0,0 +1,135 @@ +using System; +using System.Data; +using Examine.Lucene.Providers; +using Examine.Search; +using Microsoft.Extensions.DependencyInjection; +using Moq; +using NPoco; +using NUnit.Framework; +using Umbraco.Cms.Core; +using Umbraco.Cms.Core.Events; +using Umbraco.Cms.Core.Hosting; +using Umbraco.Cms.Core.Models; +using Umbraco.Cms.Core.Persistence.Querying; +using Umbraco.Cms.Core.Scoping; +using Umbraco.Cms.Core.Services; +using Umbraco.Cms.Infrastructure.Examine; +using Umbraco.Cms.Infrastructure.Persistence; +using Umbraco.Cms.Infrastructure.Persistence.SqlSyntax; +using Umbraco.Cms.Tests.Integration.Testing; + +namespace Umbraco.Cms.Tests.Integration.Umbraco.Examine.Lucene.UmbracoExamine +{ + [TestFixture] + public abstract class ExamineBaseTest : UmbracoIntegrationTest + { + protected IndexInitializer IndexInitializer => Services.GetRequiredService(); + + protected IHostingEnvironment HostingEnvironment => Services.GetRequiredService(); + + protected IRuntimeState RunningRuntimeState { get; } = Mock.Of(x => x.Level == RuntimeLevel.Run); + + public override void ConfigureServices(IServiceCollection services) + { + base.ConfigureServices(services); + services.AddSingleton(); + } + + /// + /// Used to create and manage a testable index + /// + /// + /// + /// + /// + /// + /// + protected IDisposable GetSynchronousContentIndex( + bool publishedValuesOnly, + out UmbracoContentIndex index, + out ContentIndexPopulator contentRebuilder, + out ContentValueSetBuilder contentValueSetBuilder, + int? parentId = null, + IContentService contentService = null) + { + contentValueSetBuilder = IndexInitializer.GetContentValueSetBuilder(publishedValuesOnly); + + ISqlContext sqlContext = Mock.Of(x => x.Query() == Mock.Of>()); + IUmbracoDatabaseFactory dbFactory = Mock.Of(x => x.SqlContext == sqlContext); + + if (contentService == null) + { + contentService = IndexInitializer.GetMockContentService(); + } + + contentRebuilder = IndexInitializer.GetContentIndexRebuilder(contentService, publishedValuesOnly, dbFactory); + + var luceneDir = new RandomIdRAMDirectory(); + + ContentValueSetValidator validator; + + // if only published values then we'll change the validator for tests to + // ensure we don't support protected nodes and that we + // mock the public access service for the special protected node. + if (publishedValuesOnly) + { + var publicAccessServiceMock = new Mock(); + publicAccessServiceMock.Setup(x => x.IsProtected(It.IsAny())) + .Returns((string path) => + { + if (path.EndsWith("," + ExamineDemoDataContentService.ProtectedNode)) + { + return Attempt.Succeed(); + } + return Attempt.Fail(); + }); + + var scopeProviderMock = new Mock(); + scopeProviderMock.Setup(x => x.CreateScope( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(Mock.Of); + + validator = new ContentValueSetValidator( + publishedValuesOnly, + false, + publicAccessServiceMock.Object, + scopeProviderMock.Object, + parentId); + } + else + { + validator = new ContentValueSetValidator(publishedValuesOnly, parentId); + } + + index = IndexInitializer.GetUmbracoIndexer( + HostingEnvironment, + RunningRuntimeState, + luceneDir, + validator: validator); + + IDisposable syncMode = index.WithThreadingMode(IndexThreadingMode.Synchronous); + + return new DisposableWrapper(syncMode, index, luceneDir); + } + + private class DisposableWrapper : IDisposable + { + private readonly IDisposable[] _disposables; + + public DisposableWrapper(params IDisposable[] disposables) => _disposables = disposables; + + public void Dispose() + { + foreach (IDisposable d in _disposables) + { + d.Dispose(); + } + } + } + } +} diff --git a/src/Umbraco.Tests/UmbracoExamine/ExamineDemoDataContentService.cs b/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/ExamineDemoDataContentService.cs similarity index 94% rename from src/Umbraco.Tests/UmbracoExamine/ExamineDemoDataContentService.cs rename to src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/ExamineDemoDataContentService.cs index ca11680f68..8323acf9bf 100644 --- a/src/Umbraco.Tests/UmbracoExamine/ExamineDemoDataContentService.cs +++ b/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/ExamineDemoDataContentService.cs @@ -1,7 +1,7 @@ -using System.Xml.Linq; +using System.Xml.Linq; using System.Xml.XPath; -namespace Umbraco.Tests.UmbracoExamine +namespace Umbraco.Cms.Tests.Integration.Umbraco.Examine.Lucene.UmbracoExamine { // TODO: This is ultra hack and still left over from legacy but still works for testing atm public class ExamineDemoDataContentService diff --git a/src/Umbraco.Tests/UmbracoExamine/ExamineDemoDataMediaService.cs b/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/ExamineDemoDataMediaService.cs similarity index 88% rename from src/Umbraco.Tests/UmbracoExamine/ExamineDemoDataMediaService.cs rename to src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/ExamineDemoDataMediaService.cs index 035a31b240..a7172248db 100644 --- a/src/Umbraco.Tests/UmbracoExamine/ExamineDemoDataMediaService.cs +++ b/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/ExamineDemoDataMediaService.cs @@ -1,11 +1,11 @@ -using System; +using System; using System.IO; using System.Linq; using System.Text; using System.Xml.Linq; using System.Xml.XPath; -namespace Umbraco.Tests.UmbracoExamine +namespace Umbraco.Cms.Tests.Integration.Umbraco.Examine.Lucene.UmbracoExamine { // TODO: This is ultra hack and still left over from legacy but still works for testing atm internal class ExamineDemoDataMediaService diff --git a/src/Umbraco.Tests/UmbracoExamine/ExamineExtensions.cs b/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/ExamineExtensions.cs similarity index 98% rename from src/Umbraco.Tests/UmbracoExamine/ExamineExtensions.cs rename to src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/ExamineExtensions.cs index 9cca58719e..ee8aea385f 100644 --- a/src/Umbraco.Tests/UmbracoExamine/ExamineExtensions.cs +++ b/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/ExamineExtensions.cs @@ -1,11 +1,11 @@ -using Examine; +using Examine; using System; using System.Collections.Generic; using System.ComponentModel; using System.Linq; using System.Xml.Linq; -namespace Umbraco.Tests.UmbracoExamine +namespace Umbraco.Cms.Tests.Integration.Umbraco.Examine.Lucene.UmbracoExamine { /// /// LEGACY!! Static methods to help query umbraco xml diff --git a/src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs b/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/IndexInitializer.cs similarity index 53% rename from src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs rename to src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/IndexInitializer.cs index 6dfc0a39ce..b7aa9fafe1 100644 --- a/src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs +++ b/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/IndexInitializer.cs @@ -1,11 +1,14 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using Examine; +using Examine.Lucene; +using Examine.Lucene.Directories; using Lucene.Net.Analysis; using Lucene.Net.Analysis.Standard; using Lucene.Net.Store; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Moq; using Umbraco.Cms.Core.Hosting; using Umbraco.Cms.Core.Logging; @@ -14,48 +17,74 @@ using Umbraco.Cms.Core.Models.Membership; using Umbraco.Cms.Core.Persistence.Querying; using Umbraco.Cms.Core.PropertyEditors; using Umbraco.Cms.Core.Scoping; +using Umbraco.Cms.Core.Serialization; using Umbraco.Cms.Core.Services; using Umbraco.Cms.Core.Strings; using Umbraco.Cms.Infrastructure.Examine; using Umbraco.Cms.Infrastructure.Persistence; -using Umbraco.Tests.TestHelpers; using IContentService = Umbraco.Cms.Core.Services.IContentService; using IMediaService = Umbraco.Cms.Core.Services.IMediaService; -using Version = Lucene.Net.Util.Version; -namespace Umbraco.Tests.UmbracoExamine +namespace Umbraco.Cms.Tests.Integration.Umbraco.Examine.Lucene.UmbracoExamine { /// /// Used internally by test classes to initialize a new index from the template /// - internal static class IndexInitializer + public class IndexInitializer { - public static ContentValueSetBuilder GetContentValueSetBuilder(PropertyEditorCollection propertyEditors, IScopeProvider scopeProvider, bool publishedValuesOnly) + private readonly IShortStringHelper _shortStringHelper; + private readonly IJsonSerializer _jsonSerializer; + private readonly PropertyEditorCollection _propertyEditors; + private readonly IScopeProvider _scopeProvider; + private readonly ILoggerFactory _loggerFactory; + + public IndexInitializer( + IShortStringHelper shortStringHelper, + IJsonSerializer jsonSerializer, + PropertyEditorCollection propertyEditors, + IScopeProvider scopeProvider, + ILoggerFactory loggerFactory) + { + _shortStringHelper = shortStringHelper; + _jsonSerializer = jsonSerializer; + _propertyEditors = propertyEditors; + _scopeProvider = scopeProvider; + _loggerFactory = loggerFactory; + } + + public ContentValueSetBuilder GetContentValueSetBuilder(bool publishedValuesOnly) { var contentValueSetBuilder = new ContentValueSetBuilder( - propertyEditors, - new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider(TestHelper.ShortStringHelper) }), + _propertyEditors, + new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider(_shortStringHelper) }), GetMockUserService(), - TestHelper.ShortStringHelper, - scopeProvider, + _shortStringHelper, + _scopeProvider, publishedValuesOnly); return contentValueSetBuilder; } - public static ContentIndexPopulator GetContentIndexRebuilder(PropertyEditorCollection propertyEditors, IContentService contentService, IScopeProvider scopeProvider, IUmbracoDatabaseFactory umbracoDatabaseFactory, bool publishedValuesOnly) + public ContentIndexPopulator GetContentIndexRebuilder(IContentService contentService, bool publishedValuesOnly, IUmbracoDatabaseFactory umbracoDatabaseFactory) { - var contentValueSetBuilder = GetContentValueSetBuilder(propertyEditors, scopeProvider, publishedValuesOnly); - var contentIndexDataSource = new ContentIndexPopulator(publishedValuesOnly, null, contentService, umbracoDatabaseFactory, contentValueSetBuilder); + var contentValueSetBuilder = GetContentValueSetBuilder(publishedValuesOnly); + var contentIndexDataSource = new ContentIndexPopulator( + _loggerFactory.CreateLogger(), + publishedValuesOnly, + null, + contentService, + umbracoDatabaseFactory, + contentValueSetBuilder); return contentIndexDataSource; } - public static MediaIndexPopulator GetMediaIndexRebuilder(PropertyEditorCollection propertyEditors, IMediaService mediaService) + public MediaIndexPopulator GetMediaIndexRebuilder(IMediaService mediaService) { - var mediaValueSetBuilder = new MediaValueSetBuilder(propertyEditors, new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider(TestHelper.ShortStringHelper) }), GetMockUserService(), Mock.Of>(), TestHelper.ShortStringHelper, TestHelper.JsonSerializer); + var mediaValueSetBuilder = new MediaValueSetBuilder(_propertyEditors, new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider(_shortStringHelper) }), GetMockUserService(), Mock.Of>(), _shortStringHelper, _jsonSerializer); var mediaIndexDataSource = new MediaIndexPopulator(null, mediaService, mediaValueSetBuilder); return mediaIndexDataSource; } + public static IContentService GetMockContentService() { long longTotalRecs; @@ -64,23 +93,25 @@ namespace Umbraco.Tests.UmbracoExamine var allRecs = demoData.GetLatestContentByXPath("//*[@isDoc]") .Root .Elements() - .Select(x => Mock.Of( + .Select((xmlElement, index) => Mock.Of( m => - m.Id == (int)x.Attribute("id") && - m.ParentId == (int)x.Attribute("parentID") && - m.Level == (int)x.Attribute("level") && + m.Id == (int)xmlElement.Attribute("id") && + // have every second one published and include the special one + m.Published == ((ExamineDemoDataContentService.ProtectedNode == (int)xmlElement.Attribute("id")) || (index % 2 == 0 ? true : false)) && + m.ParentId == (int)xmlElement.Attribute("parentID") && + m.Level == (int)xmlElement.Attribute("level") && m.CreatorId == 0 && - m.SortOrder == (int)x.Attribute("sortOrder") && - m.CreateDate == (DateTime)x.Attribute("createDate") && - m.UpdateDate == (DateTime)x.Attribute("updateDate") && - m.Name == (string)x.Attribute(UmbracoExamineFieldNames.NodeNameFieldName) && - m.GetCultureName(It.IsAny()) == (string)x.Attribute(UmbracoExamineFieldNames.NodeNameFieldName) && - m.Path == (string)x.Attribute("path") && + m.SortOrder == (int)xmlElement.Attribute("sortOrder") && + m.CreateDate == (DateTime)xmlElement.Attribute("createDate") && + m.UpdateDate == (DateTime)xmlElement.Attribute("updateDate") && + m.Name == (string)xmlElement.Attribute(UmbracoExamineFieldNames.NodeNameFieldName) && + m.GetCultureName(It.IsAny()) == (string)xmlElement.Attribute(UmbracoExamineFieldNames.NodeNameFieldName) && + m.Path == (string)xmlElement.Attribute("path") && m.Properties == new PropertyCollection() && m.ContentType == Mock.Of(mt => mt.Icon == "test" && - mt.Alias == x.Name.LocalName && - mt.Id == (int)x.Attribute("nodeType")))) + mt.Alias == xmlElement.Name.LocalName && + mt.Id == (int)xmlElement.Attribute("nodeType")))) .ToArray(); @@ -90,10 +121,7 @@ namespace Umbraco.Tests.UmbracoExamine == allRecs); } - public static IUserService GetMockUserService() - { - return Mock.Of(x => x.GetProfileById(It.IsAny()) == Mock.Of(p => p.Id == 0 && p.Name == "admin")); - } + public IUserService GetMockUserService() => Mock.Of(x => x.GetProfileById(It.IsAny()) == Mock.Of(p => p.Id == 0 && p.Name == "admin")); public static IMediaService GetMockMediaService() { @@ -136,30 +164,23 @@ namespace Umbraco.Tests.UmbracoExamine return mediaServiceMock.Object; } - public static ILocalizationService GetMockLocalizationService() - { - return Mock.Of(x => x.GetAllLanguages() == Array.Empty()); - } + public ILocalizationService GetMockLocalizationService() => Mock.Of(x => x.GetAllLanguages() == Array.Empty()); - public static IMediaTypeService GetMockMediaTypeService() + public static IMediaTypeService GetMockMediaTypeService(IShortStringHelper shortStringHelper) { var mediaTypeServiceMock = new Mock(); mediaTypeServiceMock.Setup(x => x.GetAll()) .Returns(new List { - new MediaType(TestHelper.ShortStringHelper, -1) {Alias = "Folder", Name = "Folder", Id = 1031, Icon = "icon-folder"}, - new MediaType(TestHelper.ShortStringHelper, -1) {Alias = "Image", Name = "Image", Id = 1032, Icon = "icon-picture"} + new MediaType(shortStringHelper, -1) {Alias = "Folder", Name = "Folder", Id = 1031, Icon = "icon-folder"}, + new MediaType(shortStringHelper, -1) {Alias = "Image", Name = "Image", Id = 1032, Icon = "icon-picture"} }); return mediaTypeServiceMock.Object; } - public static IProfilingLogger GetMockProfilingLogger() - { - return new ProfilingLogger(Mock.Of>(), Mock.Of()); - } + public IProfilingLogger GetMockProfilingLogger() => new ProfilingLogger(Mock.Of>(), Mock.Of()); - public static UmbracoContentIndex GetUmbracoIndexer( - IProfilingLogger profilingLogger, + public UmbracoContentIndex GetUmbracoIndexer( IHostingEnvironment hostingEnvironment, IRuntimeState runtimeState, Directory luceneDir, @@ -171,40 +192,50 @@ namespace Umbraco.Tests.UmbracoExamine languageService = GetMockLocalizationService(); if (analyzer == null) - analyzer = new StandardAnalyzer(Version.LUCENE_30); + analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); if (validator == null) validator = new ContentValueSetValidator(true); - var i = new UmbracoContentIndex( + var options = GetOptions( "testIndexer", - luceneDir, - new UmbracoFieldDefinitionCollection(), - analyzer, - profilingLogger, - Mock.Of>(), - Mock.Of(), + new LuceneDirectoryIndexOptions + { + Analyzer = analyzer, + Validator = validator, + DirectoryFactory = new GenericDirectoryFactory(s => luceneDir), + FieldDefinitions = new UmbracoFieldDefinitionCollection() + }); + + var i = new UmbracoContentIndex( + _loggerFactory, + "testIndexer", + options, hostingEnvironment, runtimeState, - languageService, - validator); + languageService); i.IndexingError += IndexingError; + i.IndexOperationComplete += I_IndexOperationComplete; return i; } + private void I_IndexOperationComplete(object sender, IndexOperationEventArgs e) + { + + } + //public static MultiIndexSearcher GetMultiSearcher(Directory pdfDir, Directory simpleDir, Directory conventionDir, Directory cwsDir) //{ // var i = new MultiIndexSearcher("testSearcher", new[] { pdfDir, simpleDir, conventionDir, cwsDir }, new StandardAnalyzer(Version.LUCENE_29)); // return i; //} + public static IOptionsSnapshot GetOptions(string indexName, LuceneDirectoryIndexOptions options) + => Mock.Of>(x => x.Get(indexName) == options); - internal static void IndexingError(object sender, IndexingErrorEventArgs e) - { - throw new ApplicationException(e.Message, e.InnerException); - } + internal void IndexingError(object sender, IndexingErrorEventArgs e) => throw new ApplicationException(e.Message, e.Exception); } diff --git a/src/Umbraco.Tests/UmbracoExamine/IndexTest.cs b/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/IndexTest.cs similarity index 54% rename from src/Umbraco.Tests/UmbracoExamine/IndexTest.cs rename to src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/IndexTest.cs index 3daf185cd4..f6362a8156 100644 --- a/src/Umbraco.Tests/UmbracoExamine/IndexTest.cs +++ b/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/IndexTest.cs @@ -1,52 +1,67 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; -using System.Threading; using Examine; -using Examine.LuceneEngine.Providers; -using Lucene.Net.Index; -using Lucene.Net.Search; -using Microsoft.Extensions.DependencyInjection; using Newtonsoft.Json; using NUnit.Framework; using Umbraco.Cms.Core.Models; -using Umbraco.Cms.Core.PropertyEditors; using Umbraco.Cms.Infrastructure.Examine; +using Umbraco.Cms.Tests.Common.Builders; using Umbraco.Cms.Tests.Common.Testing; using Umbraco.Extensions; -using Umbraco.Tests.TestHelpers; -using Umbraco.Tests.TestHelpers.Entities; -namespace Umbraco.Tests.UmbracoExamine +namespace Umbraco.Cms.Tests.Integration.Umbraco.Examine.Lucene.UmbracoExamine { + /// /// Tests the standard indexing capabilities /// [TestFixture] - [Apartment(ApartmentState.STA)] - [UmbracoTest(Database = UmbracoTestOptions.Database.NewSchemaPerTest)] + [UmbracoTest(Database = UmbracoTestOptions.Database.None)] public class IndexTest : ExamineBaseTest { [Test] - public void Index_Property_Data_With_Value_Indexer() + public void GivenValidationParentNode_WhenContentIndexedUnderDifferentParent_DocumentIsNotIndexed() { - var contentValueSetBuilder = IndexInitializer.GetContentValueSetBuilder(Factory.GetRequiredService(), ScopeProvider, false); - - using (var luceneDir = new RandomIdRamDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, HostingEnvironment, RuntimeState, luceneDir, - validator: new ContentValueSetValidator(false))) - using (indexer.ProcessNonAsync()) + using (GetSynchronousContentIndex(false, out UmbracoContentIndex index, out _, out _, 999)) { - indexer.CreateIndex(); + var searcher = index.Searcher; - var contentType = MockedContentTypes.CreateBasicContentType(); + var contentService = new ExamineDemoDataContentService(); + //get a node from the data repo + var node = contentService.GetPublishedContentByXPath("//*[string-length(@id)>0 and number(@id)>0]") + .Root + .Elements() + .First(); + + ValueSet valueSet = node.ConvertToValueSet(IndexTypes.Content); + + // Ignored since the path isn't under 999 + index.IndexItems(new[] { valueSet }); + Assert.AreEqual(0, searcher.CreateQuery().Id(valueSet.Id).Execute().TotalItemCount); + + // Change so that it's under 999 and verify + valueSet.Values["path"] = new List { "-1,999," + valueSet.Id }; + index.IndexItems(new[] { valueSet }); + Assert.AreEqual(1, searcher.CreateQuery().Id(valueSet.Id).Execute().TotalItemCount); + } + } + + [Test] + public void GivenIndexingDocument_WhenGridPropertyData_ThenDataIndexedInSegregatedFields() + { + using (GetSynchronousContentIndex(false, out UmbracoContentIndex index, out _, out ContentValueSetBuilder contentValueSetBuilder, null)) + { + index.CreateIndex(); + + ContentType contentType = ContentTypeBuilder.CreateBasicContentType(); contentType.AddPropertyType(new PropertyType(TestHelper.ShortStringHelper, "test", ValueStorageType.Ntext) { Alias = "grid", Name = "Grid", PropertyEditorAlias = Cms.Core.Constants.PropertyEditors.Aliases.Grid }); - var content = MockedContent.CreateBasicContent(contentType); + Content content = ContentBuilder.CreateBasicContent(contentType); content.Id = 555; content.Path = "-1,555"; var gridVal = new GridValue @@ -100,15 +115,13 @@ namespace Umbraco.Tests.UmbracoExamine var json = JsonConvert.SerializeObject(gridVal); content.Properties["grid"].SetValue(json); - var valueSet = contentValueSetBuilder.GetValueSets(content); - indexer.IndexItems(valueSet); + IEnumerable valueSet = contentValueSetBuilder.GetValueSets(content); + index.IndexItems(valueSet); - var searcher = indexer.GetSearcher(); - - var results = searcher.CreateQuery().Id(555).Execute(); + ISearchResults results = index.Searcher.CreateQuery().Id(555).Execute(); Assert.AreEqual(1, results.TotalItemCount); - var result = results.First(); + ISearchResult result = results.First(); Assert.IsTrue(result.Values.ContainsKey("grid.row1")); Assert.AreEqual("value1", result.AllValues["grid.row1"][0]); Assert.AreEqual("value2", result.AllValues["grid.row1"][1]); @@ -120,95 +133,64 @@ namespace Umbraco.Tests.UmbracoExamine } [Test] - public void Rebuild_Index() - { - var contentRebuilder = IndexInitializer.GetContentIndexRebuilder(Factory.GetRequiredService(), IndexInitializer.GetMockContentService(), ScopeProvider, UmbracoDatabaseFactory,false); - var mediaRebuilder = IndexInitializer.GetMediaIndexRebuilder(Factory.GetRequiredService(), IndexInitializer.GetMockMediaService()); + public void GivenEmptyIndex_WhenUsingWithContentAndMediaPopulators_ThenIndexPopulated() + { + var mediaRebuilder = IndexInitializer.GetMediaIndexRebuilder(IndexInitializer.GetMockMediaService()); - using (var luceneDir = new RandomIdRamDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, HostingEnvironment, RuntimeState, luceneDir, - validator: new ContentValueSetValidator(false))) - using (indexer.ProcessNonAsync()) + using (GetSynchronousContentIndex(false, out UmbracoContentIndex index, out ContentIndexPopulator contentRebuilder, out _, null)) { - - var searcher = indexer.GetSearcher(); - //create the whole thing - contentRebuilder.Populate(indexer); - mediaRebuilder.Populate(indexer); + contentRebuilder.Populate(index); + mediaRebuilder.Populate(index); - var result = searcher.CreateQuery().All().Execute(); + var result = index.Searcher.CreateQuery().All().Execute(); Assert.AreEqual(29, result.TotalItemCount); } } - ///// /// /// Check that the node signalled as protected in the content service is not present in the index. /// [Test] - public void Index_Protected_Content_Not_Indexed() + public void GivenPublishedContentIndex_WhenProtectedContentIndexed_ThenItIsIgnored() { - var rebuilder = IndexInitializer.GetContentIndexRebuilder(Factory.GetRequiredService(), IndexInitializer.GetMockContentService(), ScopeProvider, UmbracoDatabaseFactory,false); - - - using (var luceneDir = new RandomIdRamDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, HostingEnvironment, RuntimeState, luceneDir)) - using (indexer.ProcessNonAsync()) - using (var searcher = ((LuceneSearcher)indexer.GetSearcher()).GetLuceneSearcher()) + using (GetSynchronousContentIndex(true, out UmbracoContentIndex index, out ContentIndexPopulator contentRebuilder, out _, null)) { //create the whole thing - rebuilder.Populate(indexer); + contentRebuilder.Populate(index); + Assert.Greater( + index.Searcher.CreateQuery().All().Execute().TotalItemCount, + 0); - var protectedQuery = new BooleanQuery(); - protectedQuery.Add( - new BooleanClause( - new TermQuery(new Term(ExamineFieldNames.CategoryFieldName, IndexTypes.Content)), - Occur.MUST)); - - protectedQuery.Add( - new BooleanClause( - new TermQuery(new Term(ExamineFieldNames.ItemIdFieldName, ExamineDemoDataContentService.ProtectedNode.ToString())), - Occur.MUST)); - - var collector = TopScoreDocCollector.Create(100, true); - - searcher.Search(protectedQuery, collector); - - Assert.AreEqual(0, collector.TotalHits, "Protected node should not be indexed"); + Assert.AreEqual( + 0, + index.Searcher.CreateQuery().Id(ExamineDemoDataContentService.ProtectedNode.ToString()).Execute().TotalItemCount); } - } [Test] - public void Index_Move_Media_From_Non_Indexable_To_Indexable_ParentID() + public void GivenMediaUnderNonIndexableParent_WhenMediaMovedUnderIndexableParent_ThenItIsIncludedInTheIndex() { // create a validator with // publishedValuesOnly false - // parentId 1116 (only content under that parent will be indexed) - var validator = new ContentValueSetValidator(false, 1116); - - using (var luceneDir = new RandomIdRamDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, HostingEnvironment, RuntimeState, luceneDir, validator: validator)) - using (indexer.ProcessNonAsync()) + // parentId 1116 (only content under that parent will be indexed) + using (GetSynchronousContentIndex(false, out UmbracoContentIndex index, out ContentIndexPopulator contentRebuilder, out _, 1116)) { - var searcher = indexer.GetSearcher(); - //get a node from the data repo (this one exists underneath 2222) var node = _mediaService.GetLatestMediaByXpath("//*[string-length(@id)>0 and number(@id)>0]") .Root.Elements() - .First(x => (int) x.Attribute("id") == 2112); + .First(x => (int)x.Attribute("id") == 2112); var currPath = (string)node.Attribute("path"); //should be : -1,1111,2222,2112 Assert.AreEqual("-1,1111,2222,2112", currPath); //ensure it's indexed - indexer.IndexItem(node.ConvertToValueSet(IndexTypes.Media)); + index.IndexItem(node.ConvertToValueSet(IndexTypes.Media)); //it will not exist because it exists under 2222 - var results = searcher.CreateQuery().Id(2112).Execute(); + var results = index.Searcher.CreateQuery().Id(2112).Execute(); Assert.AreEqual(0, results.Count()); //now mimic moving 2112 to 1116 @@ -217,38 +199,34 @@ namespace Umbraco.Tests.UmbracoExamine node.SetAttributeValue("parentID", "1116"); //now reindex the node, this should first delete it and then WILL add it because of the parent id constraint - indexer.IndexItems(new[] { node.ConvertToValueSet(IndexTypes.Media) }); + index.IndexItems(new[] { node.ConvertToValueSet(IndexTypes.Media) }); //now ensure it exists - results = searcher.CreateQuery().Id(2112).Execute(); + results = index.Searcher.CreateQuery().Id(2112).Execute(); Assert.AreEqual(1, results.Count()); } } [Test] - public void Index_Move_Media_To_Non_Indexable_ParentID() + public void GivenMediaUnderIndexableParent_WhenMediaMovedUnderNonIndexableParent_ThenItIsRemovedFromTheIndex() { // create a validator with // publishedValuesOnly false // parentId 2222 (only content under that parent will be indexed) - var validator = new ContentValueSetValidator(false, 2222); - - using (var luceneDir = new RandomIdRamDirectory()) - using (var indexer1 = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, HostingEnvironment, RuntimeState, luceneDir, validator: validator)) - using (indexer1.ProcessNonAsync()) + using (GetSynchronousContentIndex(false, out UmbracoContentIndex index, out ContentIndexPopulator contentRebuilder, out _, 2222)) { - var searcher = indexer1.GetSearcher(); + var searcher = index.Searcher; //get a node from the data repo (this one exists underneath 2222) var node = _mediaService.GetLatestMediaByXpath("//*[string-length(@id)>0 and number(@id)>0]") .Root.Elements() - .First(x => (int) x.Attribute("id") == 2112); + .First(x => (int)x.Attribute("id") == 2112); var currPath = (string)node.Attribute("path"); //should be : -1,1111,2222,2112 Assert.AreEqual("-1,1111,2222,2112", currPath); //ensure it's indexed - indexer1.IndexItem(node.ConvertToValueSet(IndexTypes.Media)); + index.IndexItem(node.ConvertToValueSet(IndexTypes.Media)); //it will exist because it exists under 2222 var results = searcher.CreateQuery().Id(2112).Execute(); @@ -259,7 +237,7 @@ namespace Umbraco.Tests.UmbracoExamine node.SetAttributeValue("parentID", "1116"); //now reindex the node, this should first delete it and then NOT add it because of the parent id constraint - indexer1.IndexItems(new[] { node.ConvertToValueSet(IndexTypes.Media) }); + index.IndexItems(new[] { node.ConvertToValueSet(IndexTypes.Media) }); //now ensure it's deleted results = searcher.CreateQuery().Id(2112).Execute(); @@ -273,38 +251,34 @@ namespace Umbraco.Tests.UmbracoExamine /// We then call the Examine method to re-index Content and do some comparisons to ensure that it worked correctly. /// [Test] - public void Index_Reindex_Content() + public void GivenEmptyIndex_WhenIndexedWithContentPopulator_ThenTheIndexIsPopulated() { - var rebuilder = IndexInitializer.GetContentIndexRebuilder(Factory.GetRequiredService(), IndexInitializer.GetMockContentService(), ScopeProvider, UmbracoDatabaseFactory,false); - using (var luceneDir = new RandomIdRamDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, HostingEnvironment, RuntimeState, luceneDir, - validator: new ContentValueSetValidator(false))) - using (indexer.ProcessNonAsync()) + using (GetSynchronousContentIndex(false, out UmbracoContentIndex index, out ContentIndexPopulator contentRebuilder, out _, null)) { - - var searcher = indexer.GetSearcher(); - //create the whole thing - rebuilder.Populate(indexer); + contentRebuilder.Populate(index); - var result = searcher.CreateQuery().Field(ExamineFieldNames.CategoryFieldName, IndexTypes.Content).Execute(); + var result = index.Searcher + .CreateQuery() + .Field(ExamineFieldNames.CategoryFieldName, IndexTypes.Content) + .Execute(); Assert.AreEqual(21, result.TotalItemCount); //delete all content - foreach (var r in result) - { - indexer.DeleteFromIndex(r.Id); - } - + index.DeleteFromIndex(result.Select(x => x.Id)); //ensure it's all gone - result = searcher.CreateQuery().Field(ExamineFieldNames.CategoryFieldName, IndexTypes.Content).Execute(); + result = index.Searcher.CreateQuery().Field(ExamineFieldNames.CategoryFieldName, IndexTypes.Content).Execute(); Assert.AreEqual(0, result.TotalItemCount); //call our indexing methods - rebuilder.Populate(indexer); + contentRebuilder.Populate(index); + + result = index.Searcher + .CreateQuery() + .Field(ExamineFieldNames.CategoryFieldName, IndexTypes.Content) + .Execute(); - result = searcher.CreateQuery().Field(ExamineFieldNames.CategoryFieldName, IndexTypes.Content).Execute(); Assert.AreEqual(21, result.TotalItemCount); } } @@ -313,26 +287,24 @@ namespace Umbraco.Tests.UmbracoExamine /// This will delete an item from the index and ensure that all children of the node are deleted too! /// [Test] - public void Index_Delete_Index_Item_Ensure_Heirarchy_Removed() + public void GivenPopulatedIndex_WhenDocumentDeleted_ThenItsHierarchyIsAlsoDeleted() { - - var rebuilder = IndexInitializer.GetContentIndexRebuilder(Factory.GetRequiredService(), IndexInitializer.GetMockContentService(), ScopeProvider, UmbracoDatabaseFactory,false); - - using (var luceneDir = new RandomIdRamDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, HostingEnvironment, RuntimeState, luceneDir)) - using (indexer.ProcessNonAsync()) + using (GetSynchronousContentIndex(false, out UmbracoContentIndex index, out ContentIndexPopulator contentRebuilder, out _, null)) { - var searcher = indexer.GetSearcher(); + var searcher = index.Searcher; //create the whole thing - rebuilder.Populate(indexer); + contentRebuilder.Populate(index); + + var results = searcher.CreateQuery().Id(1141).Execute(); + Assert.AreEqual(1, results.Count()); //now delete a node that has children - indexer.DeleteFromIndex(1140.ToString()); + index.DeleteFromIndex(1140.ToString()); //this node had children: 1141 & 1142, let's ensure they are also removed - var results = searcher.CreateQuery().Id(1141).Execute(); + results = searcher.CreateQuery().Id(1141).Execute(); Assert.AreEqual(0, results.Count()); results = searcher.CreateQuery().Id(1142).Execute(); diff --git a/src/Umbraco.Tests/Web/PublishedContentQueryTests.cs b/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/PublishedContentQueryTests.cs similarity index 62% rename from src/Umbraco.Tests/Web/PublishedContentQueryTests.cs rename to src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/PublishedContentQueryTests.cs index cc34cd4aba..f2269916a4 100644 --- a/src/Umbraco.Tests/Web/PublishedContentQueryTests.cs +++ b/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/PublishedContentQueryTests.cs @@ -1,33 +1,43 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using Examine; -using Examine.LuceneEngine.Providers; +using Examine.Lucene; +using Examine.Lucene.Directories; +using Examine.Lucene.Providers; using Lucene.Net.Store; +using Microsoft.Extensions.Logging; using Moq; using NUnit.Framework; using Umbraco.Cms.Core.Models.PublishedContent; using Umbraco.Cms.Core.PublishedCache; using Umbraco.Cms.Infrastructure; using Umbraco.Cms.Infrastructure.Examine; -using Umbraco.Tests.TestHelpers; -using Umbraco.Web; +using Umbraco.Cms.Tests.Common.Testing; -namespace Umbraco.Tests.Web +namespace Umbraco.Cms.Tests.Integration.Umbraco.Examine.Lucene.UmbracoExamine { [TestFixture] - public class PublishedContentQueryTests + [UmbracoTest(Database = UmbracoTestOptions.Database.None)] + public class PublishedContentQueryTests : ExamineBaseTest { private class TestIndex : LuceneIndex, IUmbracoIndex { private readonly string[] _fieldNames; - public TestIndex(string name, Directory luceneDirectory, string[] fieldNames) - : base(name, luceneDirectory, null, null, null, null) + public TestIndex(ILoggerFactory loggerFactory, string name, Directory luceneDirectory, string[] fieldNames) + : base( + loggerFactory, + name, + IndexInitializer.GetOptions(name, new LuceneDirectoryIndexOptions + { + DirectoryFactory = new GenericDirectoryFactory(s => luceneDirectory) + })) { _fieldNames = fieldNames; } + public bool EnableDefaultEventHandler => throw new NotImplementedException(); public bool PublishedValuesOnly => throw new NotImplementedException(); public IEnumerable GetFields() => _fieldNames; @@ -35,29 +45,29 @@ namespace Umbraco.Tests.Web private TestIndex CreateTestIndex(Directory luceneDirectory, string[] fieldNames) { - var indexer = new TestIndex("TestIndex", luceneDirectory, fieldNames); + var index = new TestIndex(LoggerFactory, "TestIndex", luceneDirectory, fieldNames); - using (indexer.ProcessNonAsync()) + using (index.WithThreadingMode(IndexThreadingMode.Synchronous)) { //populate with some test data - indexer.IndexItem(new ValueSet("1", "content", new Dictionary + index.IndexItem(new ValueSet("1", "content", new Dictionary { [fieldNames[0]] = "Hello world, there are products here", [UmbracoExamineFieldNames.VariesByCultureFieldName] = "n" })); - indexer.IndexItem(new ValueSet("2", "content", new Dictionary + index.IndexItem(new ValueSet("2", "content", new Dictionary { [fieldNames[1]] = "Hello world, there are products here", [UmbracoExamineFieldNames.VariesByCultureFieldName] = "y" })); - indexer.IndexItem(new ValueSet("3", "content", new Dictionary + index.IndexItem(new ValueSet("3", "content", new Dictionary { [fieldNames[2]] = "Hello world, there are products here", [UmbracoExamineFieldNames.VariesByCultureFieldName] = "y" })); } - return indexer; + return index; } private PublishedContentQuery CreatePublishedContentQuery(IIndex indexer) @@ -75,10 +85,10 @@ namespace Umbraco.Tests.Web return new PublishedContentQuery(snapshot, variationContextAccessor, examineManager.Object); } - [TestCase("fr-fr", ExpectedResult = "1, 3", TestName = "Search Culture: fr-fr. Must return both fr-fr and invariant results")] - [TestCase("en-us", ExpectedResult = "1, 2", TestName = "Search Culture: en-us. Must return both en-us and invariant results")] - [TestCase("*", ExpectedResult = "1, 2, 3", TestName = "Search Culture: *. Must return all cultures and all invariant results")] - [TestCase(null, ExpectedResult = "1", TestName = "Search Culture: null. Must return only invariant results")] + [TestCase("fr-fr", ExpectedResult = "1, 3", Description = "Search Culture: fr-fr. Must return both fr-fr and invariant results")] + [TestCase("en-us", ExpectedResult = "1, 2", Description = "Search Culture: en-us. Must return both en-us and invariant results")] + [TestCase("*", ExpectedResult = "1, 2, 3", Description = "Search Culture: *. Must return all cultures and all invariant results")] + [TestCase(null, ExpectedResult = "1", Description = "Search Culture: null. Must return only invariant results")] public string Search(string culture) { using (var luceneDir = new RandomIdRAMDirectory()) diff --git a/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/RandomIdRAMDirectory.cs b/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/RandomIdRAMDirectory.cs new file mode 100644 index 0000000000..3d8fc1f192 --- /dev/null +++ b/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/RandomIdRAMDirectory.cs @@ -0,0 +1,11 @@ +using System; +using Lucene.Net.Store; + +namespace Umbraco.Cms.Tests.Integration.Umbraco.Examine.Lucene.UmbracoExamine +{ + public class RandomIdRAMDirectory : RAMDirectory + { + private readonly string _lockId = Guid.NewGuid().ToString(); + public override string GetLockID() => _lockId; + } +} diff --git a/src/Umbraco.Tests/UmbracoExamine/SearchTests.cs b/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/SearchTests.cs similarity index 69% rename from src/Umbraco.Tests/UmbracoExamine/SearchTests.cs rename to src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/SearchTests.cs index c4698fcdf2..2aefc593db 100644 --- a/src/Umbraco.Tests/UmbracoExamine/SearchTests.cs +++ b/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/SearchTests.cs @@ -1,23 +1,22 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using Examine; +using Examine.Lucene.Providers; using Examine.Search; -using Microsoft.Extensions.DependencyInjection; using Moq; using NUnit.Framework; using Umbraco.Cms.Core.Models; using Umbraco.Cms.Core.Persistence.Querying; -using Umbraco.Cms.Core.PropertyEditors; using Umbraco.Cms.Core.Services; using Umbraco.Cms.Infrastructure.Examine; using Umbraco.Cms.Tests.Common.Testing; using Umbraco.Extensions; -namespace Umbraco.Tests.UmbracoExamine +namespace Umbraco.Cms.Tests.Integration.Umbraco.Examine.Lucene.UmbracoExamine { [TestFixture] - [UmbracoTest(Database = UmbracoTestOptions.Database.NewSchemaPerTest)] + [UmbracoTest(Database = UmbracoTestOptions.Database.NewSchemaPerTest, Logger = UmbracoTestOptions.Logger.Console)] public class SearchTests : ExamineBaseTest { @@ -54,17 +53,14 @@ namespace Umbraco.Tests.UmbracoExamine == allRecs); - var propertyEditors = Factory.GetRequiredService(); - var rebuilder = IndexInitializer.GetContentIndexRebuilder(propertyEditors, contentService, ScopeProvider, UmbracoDatabaseFactory,true); - - using (var luceneDir = new RandomIdRamDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, HostingEnvironment, RuntimeState, luceneDir)) - using (indexer.ProcessNonAsync()) + using (GetSynchronousContentIndex(false, out UmbracoContentIndex index, out ContentIndexPopulator contentRebuilder, out _, null, contentService)) { - indexer.CreateIndex(); - rebuilder.Populate(indexer); + index.CreateIndex(); + contentRebuilder.Populate(index); - var searcher = indexer.GetSearcher(); + var searcher = index.Searcher; + + Assert.Greater(searcher.CreateQuery().All().Execute().TotalItemCount, 0); var numberSortedCriteria = searcher.CreateQuery() .ParentId(1148) @@ -99,23 +95,5 @@ namespace Umbraco.Tests.UmbracoExamine return true; } - //[Test] - //public void Test_Index_Type_With_German_Analyzer() - //{ - // using (var luceneDir = new RandomIdRamDirectory()) - // { - // var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir, - // new GermanAnalyzer()); - // indexer.RebuildIndex(); - // var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir); - // } - //} - - //private readonly TestContentService _contentService = new TestContentService(); - //private readonly TestMediaService _mediaService = new TestMediaService(); - //private static UmbracoExamineSearcher _searcher; - //private static UmbracoContentIndexer _indexer; - //private Lucene.Net.Store.Directory _luceneDir; - } } diff --git a/src/Umbraco.Tests/UmbracoExamine/TestFiles.Designer.cs b/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/TestFiles.Designer.cs similarity index 93% rename from src/Umbraco.Tests/UmbracoExamine/TestFiles.Designer.cs rename to src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/TestFiles.Designer.cs index b60dc487de..166d329208 100644 --- a/src/Umbraco.Tests/UmbracoExamine/TestFiles.Designer.cs +++ b/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/TestFiles.Designer.cs @@ -8,7 +8,7 @@ // //------------------------------------------------------------------------------ -namespace Umbraco.Tests.UmbracoExamine { +namespace Umbraco.Cms.Tests.Integration.Umbraco.Examine.Lucene.UmbracoExamine { using System; @@ -19,7 +19,7 @@ namespace Umbraco.Tests.UmbracoExamine { // class via a tool like ResGen or Visual Studio. // To add or remove a member, edit your .ResX file then rerun ResGen // with the /str option, or rebuild your VS project. - [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "15.0.0.0")] + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "16.0.0.0")] [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] internal class TestFiles { @@ -39,7 +39,7 @@ namespace Umbraco.Tests.UmbracoExamine { internal static global::System.Resources.ResourceManager ResourceManager { get { if (object.ReferenceEquals(resourceMan, null)) { - global::System.Resources.ResourceManager temp = new global::System.Resources.ResourceManager("Umbraco.Tests.UmbracoExamine.TestFiles", typeof(TestFiles).Assembly); + global::System.Resources.ResourceManager temp = new global::System.Resources.ResourceManager("Umbraco.Cms.Tests.Integration.Umbraco.Examine.Lucene.UmbracoExamine.TestFiles", typeof(TestFiles).Assembly); resourceMan = temp; } return resourceMan; diff --git a/src/Umbraco.Tests/UmbracoExamine/TestFiles.resx b/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/TestFiles.resx similarity index 96% rename from src/Umbraco.Tests/UmbracoExamine/TestFiles.resx rename to src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/TestFiles.resx index e23540252a..b5ed853136 100644 --- a/src/Umbraco.Tests/UmbracoExamine/TestFiles.resx +++ b/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/TestFiles.resx @@ -1,4 +1,4 @@ - +