From eba6373a1239d89770fa02287665b7ebfa3bd5db Mon Sep 17 00:00:00 2001 From: Shannon Deminick Date: Tue, 18 May 2021 18:31:38 +1000 Subject: [PATCH] Examine 2.0 integration (#10241) * Init commit for examine 2.0 work, most old umb examine tests working, probably a lot that doesn't * Gets Umbraco Examine tests passing and makes some sense out of them, fixes some underlying issues. * Large refactor, remove TaskHelper, rename Notifications to be consistent, Gets all examine/lucene indexes building and startup ordered in the correct way, removes old files, creates new IUmbracoIndexingHandler for abstracting out all index operations for umbraco data, abstracts out IIndexRebuilder, Fixes Stack overflow with LiveModelsProvider and loading assemblies, ports some changes from v8 for startup handling with cold boots, refactors out LastSyncedFileManager * fix up issues with rebuilding and management dashboard. * removes old files, removes NetworkHelper, fixes LastSyncedFileManager implementation to ensure the machine name is used, fix up logging with cold boot state. * Makes MainDom safer to use and makes PublishedSnapshotService lazily register with MainDom * lazily acquire application id (fix unit tests) * Fixes resource casing and missing test file * Ensures caches when requiring internal services for PublishedSnapshotService, UseNuCache is a separate call, shouldn't be buried in AddWebComponents, was also causing issues in integration tests since nucache was being used for the Id2Key service. * For UmbracoTestServerTestBase enable nucache services * Fixing tests * Fix another test * Fixes tests, use TestHostingEnvironment, make Tests.Common use net5, remove old Lucene.Net.Contrib ref. * Fixes up some review notes * Fixes issue with doubly registering PublishedSnapshotService meanig there could be 2x instances of it * Checks for parseexception when executing the query * Use application root instead of duplicating functionality. * Added Examine project to netcore only solution file * Fixed casing issue with LazyLoad, that is not lowercase. * uses cancellationToken instead of bool flag, fixes always reading lastId from the LastSyncedFileManager, fixes RecurringHostedServiceBase so that there isn't an overlapping thread for the same task type * Fix tests * remove legacy test project from solution file * Fix test Co-authored-by: Bjarke Berg --- NuGet.Config | 1 - src/Umbraco.Core/Composing/TypeLoader.cs | 2 +- src/Umbraco.Core/Constants-Indexes.cs | 12 +- .../DependencyInjection/UmbracoBuilder.cs | 2 + src/Umbraco.Core/EnvironmentHelper.cs | 17 + .../Extensions/StringExtensions.cs | 27 +- .../Hosting/IHostingEnvironment.cs | 15 + src/Umbraco.Core/NetworkHelper.cs | 48 - .../UmbracoApplicationStartingNotification.cs | 5 +- .../UmbracoRequestEndNotification.cs | 2 +- src/Umbraco.Core/Runtime/MainDom.cs | 22 +- ...ructionServiceProcessInstructionsResult.cs | 26 - .../Services/ICacheInstructionService.cs | 10 +- .../Services/ProcessInstructionsResult.cs | 26 + .../Sync/DatabaseServerMessengerCallbacks.cs | 20 - src/Umbraco.Core/Sync/IServerAddress.cs | 2 +- .../Sync/ISyncBootStateAccessor.cs | 14 + .../Sync/NonRuntimeLevelBootStateAccessor.cs | 10 + src/Umbraco.Core/Sync/SyncBootState.cs | 20 + .../BackOfficeExamineSearcher.cs | 23 +- .../ConfigurationEnabledDirectoryFactory.cs | 83 ++ .../ConfigureIndexOptions.cs | 46 + .../UmbracoBuilderExtensions.cs | 40 + .../ExamineLuceneComponent.cs | 53 -- .../ExamineLuceneComposer.cs | 29 - .../ExamineLuceneFinalComponent.cs | 37 - .../ExamineLuceneFinalComposer.cs | 14 - .../Extensions/ExamineExtensions.cs | 100 +-- .../ILuceneDirectoryFactory.cs | 10 - .../LuceneFileSystemDirectoryFactory.cs | 73 -- .../LuceneIndexCreator.cs | 32 - .../LuceneIndexDiagnostics.cs | 42 +- .../LuceneIndexDiagnosticsFactory.cs | 8 +- .../LuceneRAMDirectoryFactory.cs | 15 +- .../NoPrefixSimpleFsLockFactory.cs | 1 + .../Umbraco.Examine.Lucene.csproj | 96 +- .../UmbracoApplicationRoot.cs | 23 + .../UmbracoContentIndex.cs | 132 ++- .../UmbracoExamineIndex.cs | 135 +-- .../UmbracoIndexesCreator.cs | 126 --- .../UmbracoLockFactory.cs | 15 + .../UmbracoMemberIndex.cs | 33 +- .../UmbracoBuilder.CoreServices.cs | 3 - .../UmbracoBuilder.DistributedCache.cs | 52 +- .../UmbracoBuilder.Examine.cs | 18 +- .../Examine/ContentIndexPopulator.cs | 35 +- .../Examine/ContentValueSetValidator.cs | 6 +- .../{Search => Examine}/ExamineIndexModel.cs | 4 +- .../Examine/ExamineIndexRebuilder.cs | 207 +++++ .../ExamineSearcherModel.cs | 4 +- .../Examine/ExamineUmbracoIndexingHandler.cs | 394 ++++++++ .../Examine/GenericIndexDiagnostics.cs | 19 +- .../Examine/IIndexCreator.cs | 13 - .../Examine/IIndexDiagnostics.cs | 16 +- .../Examine/IIndexRebuilder.cs | 14 + .../Examine/IUmbracoIndex.cs | 10 +- .../Examine/IUmbracoIndexesCreator.cs | 10 - .../Examine/IndexDiagnosticsFactory.cs | 7 +- .../Examine/IndexPopulator.cs | 2 +- .../Examine/IndexRebuilder.cs | 81 -- .../Examine/IndexRebuildingEventArgs.cs | 19 - .../Examine/MediaIndexPopulator.cs | 17 +- .../Examine/NoopUmbracoIndexesCreator.cs | 14 - .../Examine/PublishedContentIndexPopulator.cs | 7 +- .../Examine/RebuildOnStartupHandler.cs | 60 ++ .../Examine/UmbracoExamineExtensions.cs | 21 +- .../HostedServices/QueuedHostedService.cs | 5 +- .../RecurringHostedServiceBase.cs | 25 +- .../PublishedContentQuery.cs | 9 +- .../Runtime/CoreRuntime.cs | 4 +- .../Runtime/SqlMainDomLock.cs | 2 +- .../Search/BackgroundIndexRebuilder.cs | 83 -- .../Search/ExamineNotificationHandler.cs | 838 ------------------ .../Search/ExamineUserComponent.cs | 37 - .../Search/IUmbracoIndexingHandler.cs | 37 + .../IndexingNotificationHandler.Content.cs | 137 +++ ...IndexingNotificationHandler.ContentType.cs | 180 ++++ .../IndexingNotificationHandler.Language.cs | 50 ++ .../IndexingNotificationHandler.Media.cs | 90 ++ .../IndexingNotificationHandler.Member.cs | 90 ++ .../Implement/CacheInstructionService.cs | 96 +- .../Implement/ServerRegistrationService.cs | 2 +- src/Umbraco.Infrastructure/Suspendable.cs | 3 +- .../Sync/BatchedDatabaseServerMessenger.cs | 7 +- .../Sync/DatabaseServerMessenger.cs | 187 ++-- .../Sync/LastSyncedFileManager.cs | 89 ++ .../Sync/SyncBootStateAccessor.cs | 84 ++ .../Umbraco.Infrastructure.csproj | 2 +- .../UmbracoBuilderExtensions.cs | 3 - .../PublishedSnapshot.cs | 16 +- .../PublishedSnapshotService.cs | 110 ++- .../PublishedSnapshotStatus.cs | 10 +- .../TaskHelper.cs | 6 +- .../Testing}/TestHostingEnvironment.cs | 15 +- .../Umbraco.Tests.Common.csproj | 5 +- .../ComponentRuntimeTests.cs | 7 +- .../UmbracoBuilderExtensions.cs | 24 +- .../Implementations/TestHelper.cs | 3 + .../UmbracoTestServerTestBase.cs | 1 + .../Testing/IntegrationTestComponent.cs | 4 +- .../Testing/UmbracoIntegrationTest.cs | 3 +- .../UmbracoExamine/ExamineBaseTest.cs | 135 +++ .../ExamineDemoDataContentService.cs | 4 +- .../ExamineDemoDataMediaService.cs | 4 +- .../UmbracoExamine/ExamineExtensions.cs | 4 +- .../UmbracoExamine/IndexInitializer.cs | 147 +-- .../UmbracoExamine/IndexTest.cs | 218 ++--- .../PublishedContentQueryTests.cs | 46 +- .../UmbracoExamine/RandomIdRAMDirectory.cs | 11 + .../UmbracoExamine/SearchTests.cs | 42 +- .../UmbracoExamine/TestFiles.Designer.cs | 6 +- .../UmbracoExamine/TestFiles.resx | 10 +- .../UmbracoExamine/TestFiles/media.xml | 0 .../TestFiles/umbraco-sort.config | 0 .../UmbracoExamine/TestFiles/umbraco.config | 0 .../Scoping/ScopeFileSystemsTests.cs | 2 +- .../Scoping/ScopeTests.cs | 2 +- .../Scoping/ScopedRepositoryTests.cs | 7 +- .../Services/CacheInstructionServiceTests.cs | 35 +- .../Services/ContentEventsTests.cs | 1 + .../ContentTypeServiceVariantsTests.cs | 1 + .../Services/TrackRelationsTests.cs | 8 + .../Umbraco.Tests.Integration.csproj | 21 +- .../TestHelpers/TestHelper.cs | 4 +- .../Models/GlobalSettingsTests.cs | 7 +- .../Extensions/UriExtensionsTests.cs | 23 - .../Routing/UmbracoRequestPathsTests.cs | 12 +- .../Umbraco.Core/TaskHelperTests.cs | 2 +- .../Umbraco.Tests.UnitTests.csproj | 1 - .../PublishedMediaCache.cs | 6 +- .../PublishedContent/PublishedMediaTests.cs | 1 - .../TestHelpers/RandomIdRamDirectory.cs | 22 - src/Umbraco.Tests/Umbraco.Tests.csproj | 28 - .../UmbracoExamine/EventsTest.cs | 45 - .../UmbracoExamine/ExamineBaseTest.cs | 38 - .../UmbracoExamine/RandomIdRamDirectory.cs | 17 - .../ExamineManagementController.cs | 68 +- .../UmbracoBuilderExtensions.cs | 5 +- .../AspNetCoreHostingEnvironment.cs | 31 +- .../UmbracoBuilderExtensions.cs | 16 +- .../Extensions/PublishedContentExtensions.cs | 8 +- .../ModelsBuilder/PureLiveModelFactory.cs | 4 +- .../Umbraco.Web.Common.csproj | 1 + .../UmbracoBackOffice/AuthorizeUpgrade.cshtml | 2 +- src/umbraco-netcore-only.sln | 7 + src/umbraco.sln | 7 - 146 files changed, 2899 insertions(+), 2904 deletions(-) create mode 100644 src/Umbraco.Core/EnvironmentHelper.cs delete mode 100644 src/Umbraco.Core/NetworkHelper.cs delete mode 100644 src/Umbraco.Core/Services/CacheInstructionServiceProcessInstructionsResult.cs create mode 100644 src/Umbraco.Core/Services/ProcessInstructionsResult.cs delete mode 100644 src/Umbraco.Core/Sync/DatabaseServerMessengerCallbacks.cs create mode 100644 src/Umbraco.Core/Sync/ISyncBootStateAccessor.cs create mode 100644 src/Umbraco.Core/Sync/NonRuntimeLevelBootStateAccessor.cs create mode 100644 src/Umbraco.Core/Sync/SyncBootState.cs create mode 100644 src/Umbraco.Examine.Lucene/ConfigurationEnabledDirectoryFactory.cs create mode 100644 src/Umbraco.Examine.Lucene/DependencyInjection/ConfigureIndexOptions.cs create mode 100644 src/Umbraco.Examine.Lucene/DependencyInjection/UmbracoBuilderExtensions.cs delete mode 100644 src/Umbraco.Examine.Lucene/ExamineLuceneComponent.cs delete mode 100644 src/Umbraco.Examine.Lucene/ExamineLuceneComposer.cs delete mode 100644 src/Umbraco.Examine.Lucene/ExamineLuceneFinalComponent.cs delete mode 100644 src/Umbraco.Examine.Lucene/ExamineLuceneFinalComposer.cs delete mode 100644 src/Umbraco.Examine.Lucene/ILuceneDirectoryFactory.cs delete mode 100644 src/Umbraco.Examine.Lucene/LuceneFileSystemDirectoryFactory.cs delete mode 100644 src/Umbraco.Examine.Lucene/LuceneIndexCreator.cs create mode 100644 src/Umbraco.Examine.Lucene/UmbracoApplicationRoot.cs delete mode 100644 src/Umbraco.Examine.Lucene/UmbracoIndexesCreator.cs create mode 100644 src/Umbraco.Examine.Lucene/UmbracoLockFactory.cs rename src/Umbraco.Infrastructure/{Search => Examine}/ExamineIndexModel.cs (88%) create mode 100644 src/Umbraco.Infrastructure/Examine/ExamineIndexRebuilder.cs rename src/Umbraco.Infrastructure/{Search => Examine}/ExamineSearcherModel.cs (74%) create mode 100644 src/Umbraco.Infrastructure/Examine/ExamineUmbracoIndexingHandler.cs delete mode 100644 src/Umbraco.Infrastructure/Examine/IIndexCreator.cs create mode 100644 src/Umbraco.Infrastructure/Examine/IIndexRebuilder.cs delete mode 100644 src/Umbraco.Infrastructure/Examine/IUmbracoIndexesCreator.cs delete mode 100644 src/Umbraco.Infrastructure/Examine/IndexRebuilder.cs delete mode 100644 src/Umbraco.Infrastructure/Examine/IndexRebuildingEventArgs.cs delete mode 100644 src/Umbraco.Infrastructure/Examine/NoopUmbracoIndexesCreator.cs create mode 100644 src/Umbraco.Infrastructure/Examine/RebuildOnStartupHandler.cs delete mode 100644 src/Umbraco.Infrastructure/Search/BackgroundIndexRebuilder.cs delete mode 100644 src/Umbraco.Infrastructure/Search/ExamineNotificationHandler.cs delete mode 100644 src/Umbraco.Infrastructure/Search/ExamineUserComponent.cs create mode 100644 src/Umbraco.Infrastructure/Search/IUmbracoIndexingHandler.cs create mode 100644 src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.Content.cs create mode 100644 src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.ContentType.cs create mode 100644 src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.Language.cs create mode 100644 src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.Media.cs create mode 100644 src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.Member.cs create mode 100644 src/Umbraco.Infrastructure/Sync/LastSyncedFileManager.cs create mode 100644 src/Umbraco.Infrastructure/Sync/SyncBootStateAccessor.cs rename src/{Umbraco.Core => Umbraco.Tests.Common}/TaskHelper.cs (95%) rename src/{Umbraco.Tests.Integration/Implementations => Umbraco.Tests.Common/Testing}/TestHostingEnvironment.cs (62%) create mode 100644 src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/ExamineBaseTest.cs rename src/{Umbraco.Tests => Umbraco.Tests.Integration/Umbraco.Examine.Lucene}/UmbracoExamine/ExamineDemoDataContentService.cs (94%) rename src/{Umbraco.Tests => Umbraco.Tests.Integration/Umbraco.Examine.Lucene}/UmbracoExamine/ExamineDemoDataMediaService.cs (88%) rename src/{Umbraco.Tests => Umbraco.Tests.Integration/Umbraco.Examine.Lucene}/UmbracoExamine/ExamineExtensions.cs (98%) rename src/{Umbraco.Tests => Umbraco.Tests.Integration/Umbraco.Examine.Lucene}/UmbracoExamine/IndexInitializer.cs (53%) rename src/{Umbraco.Tests => Umbraco.Tests.Integration/Umbraco.Examine.Lucene}/UmbracoExamine/IndexTest.cs (54%) rename src/{Umbraco.Tests/Web => Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine}/PublishedContentQueryTests.cs (62%) create mode 100644 src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/RandomIdRAMDirectory.cs rename src/{Umbraco.Tests => Umbraco.Tests.Integration/Umbraco.Examine.Lucene}/UmbracoExamine/SearchTests.cs (69%) rename src/{Umbraco.Tests => Umbraco.Tests.Integration/Umbraco.Examine.Lucene}/UmbracoExamine/TestFiles.Designer.cs (93%) rename src/{Umbraco.Tests => Umbraco.Tests.Integration/Umbraco.Examine.Lucene}/UmbracoExamine/TestFiles.resx (96%) rename src/{Umbraco.Tests => Umbraco.Tests.Integration/Umbraco.Examine.Lucene}/UmbracoExamine/TestFiles/media.xml (100%) rename src/{Umbraco.Tests => Umbraco.Tests.Integration/Umbraco.Examine.Lucene}/UmbracoExamine/TestFiles/umbraco-sort.config (100%) rename src/{Umbraco.Tests => Umbraco.Tests.Integration/Umbraco.Examine.Lucene}/UmbracoExamine/TestFiles/umbraco.config (100%) delete mode 100644 src/Umbraco.Tests/TestHelpers/RandomIdRamDirectory.cs delete mode 100644 src/Umbraco.Tests/UmbracoExamine/EventsTest.cs delete mode 100644 src/Umbraco.Tests/UmbracoExamine/ExamineBaseTest.cs delete mode 100644 src/Umbraco.Tests/UmbracoExamine/RandomIdRamDirectory.cs diff --git a/NuGet.Config b/NuGet.Config index 92eaf83792..f8dc68d26a 100644 --- a/NuGet.Config +++ b/NuGet.Config @@ -9,7 +9,6 @@ - diff --git a/src/Umbraco.Core/Composing/TypeLoader.cs b/src/Umbraco.Core/Composing/TypeLoader.cs index c53e6ee4d2..d248be19b7 100644 --- a/src/Umbraco.Core/Composing/TypeLoader.cs +++ b/src/Umbraco.Core/Composing/TypeLoader.cs @@ -316,7 +316,7 @@ namespace Umbraco.Cms.Core.Composing /// private string GetFileBasePath() { - var fileBasePath = Path.Combine(_localTempPath.FullName, "TypesCache", "umbraco-types." + NetworkHelper.FileSafeMachineName); + var fileBasePath = Path.Combine(_localTempPath.FullName, "TypesCache", "umbraco-types." + EnvironmentHelper.FileSafeMachineName); // ensure that the folder exists var directory = Path.GetDirectoryName(fileBasePath); diff --git a/src/Umbraco.Core/Constants-Indexes.cs b/src/Umbraco.Core/Constants-Indexes.cs index 8384faa08d..fcf2e7ed14 100644 --- a/src/Umbraco.Core/Constants-Indexes.cs +++ b/src/Umbraco.Core/Constants-Indexes.cs @@ -1,16 +1,12 @@ -namespace Umbraco.Cms.Core +namespace Umbraco.Cms.Core { public static partial class Constants { public static class UmbracoIndexes { - public const string InternalIndexName = InternalIndexPath + "Index"; - public const string ExternalIndexName = ExternalIndexPath + "Index"; - public const string MembersIndexName = MembersIndexPath + "Index"; - - public const string InternalIndexPath = "Internal"; - public const string ExternalIndexPath = "External"; - public const string MembersIndexPath = "Members"; + public const string InternalIndexName = "InternalIndex"; + public const string ExternalIndexName = "ExternalIndex"; + public const string MembersIndexName = "MembersIndex"; } } } diff --git a/src/Umbraco.Core/DependencyInjection/UmbracoBuilder.cs b/src/Umbraco.Core/DependencyInjection/UmbracoBuilder.cs index d0fe41dfc3..316dba75c1 100644 --- a/src/Umbraco.Core/DependencyInjection/UmbracoBuilder.cs +++ b/src/Umbraco.Core/DependencyInjection/UmbracoBuilder.cs @@ -225,6 +225,8 @@ namespace Umbraco.Cms.Core.DependencyInjection Services .AddNotificationHandler() .AddNotificationHandler(); + + Services.AddSingleton(); } } } diff --git a/src/Umbraco.Core/EnvironmentHelper.cs b/src/Umbraco.Core/EnvironmentHelper.cs new file mode 100644 index 0000000000..097ffc9629 --- /dev/null +++ b/src/Umbraco.Core/EnvironmentHelper.cs @@ -0,0 +1,17 @@ +using System; +using Umbraco.Extensions; + +namespace Umbraco.Cms.Core +{ + /// + /// Currently just used to get the machine name for use with file names + /// + internal class EnvironmentHelper + { + /// + /// Returns the machine name that is safe to use in file paths. + /// + public static string FileSafeMachineName => Environment.MachineName.ReplaceNonAlphanumericChars('-'); + + } +} diff --git a/src/Umbraco.Core/Extensions/StringExtensions.cs b/src/Umbraco.Core/Extensions/StringExtensions.cs index 3b42426cd6..e815f219ca 100644 --- a/src/Umbraco.Core/Extensions/StringExtensions.cs +++ b/src/Umbraco.Core/Extensions/StringExtensions.cs @@ -616,12 +616,7 @@ namespace Umbraco.Extensions /// /// Refers to itself /// The hashed string - public static string GenerateHash(this string str) - { - return CryptoConfig.AllowOnlyFipsAlgorithms - ? str.ToSHA1() - : str.ToMd5(); - } + public static string GenerateHash(this string str) => str.ToSHA1(); /// /// Generate a hash of a string based on the specified hash algorithm. @@ -632,30 +627,14 @@ namespace Umbraco.Extensions /// The hashed string. /// public static string GenerateHash(this string str) - where T : HashAlgorithm - { - return str.GenerateHash(typeof(T).FullName); - } - - /// - /// Converts the string to MD5 - /// - /// Refers to itself - /// The MD5 hashed string - public static string ToMd5(this string stringToConvert) - { - return stringToConvert.GenerateHash("MD5"); - } + where T : HashAlgorithm => str.GenerateHash(typeof(T).FullName); /// /// Converts the string to SHA1 /// /// refers to itself /// The SHA1 hashed string - public static string ToSHA1(this string stringToConvert) - { - return stringToConvert.GenerateHash("SHA1"); - } + public static string ToSHA1(this string stringToConvert) => stringToConvert.GenerateHash("SHA1"); /// Generate a hash of a string based on the hashType passed in /// diff --git a/src/Umbraco.Core/Hosting/IHostingEnvironment.cs b/src/Umbraco.Core/Hosting/IHostingEnvironment.cs index 311d7559d0..dedf809230 100644 --- a/src/Umbraco.Core/Hosting/IHostingEnvironment.cs +++ b/src/Umbraco.Core/Hosting/IHostingEnvironment.cs @@ -6,6 +6,21 @@ namespace Umbraco.Cms.Core.Hosting { string SiteName { get; } + /// + /// The unique application ID for this Umbraco website. + /// + /// + /// + /// The returned value will be the same consistent value for an Umbraco website on a specific server and will the same + /// between restarts of that Umbraco website/application on that specific server. + /// + /// + /// The value of this does not necesarily distinguish between unique workers/servers for this Umbraco application. + /// Usage of this must take into account that the same may be returned for the same + /// Umbraco website hosted on different servers. Similarly the usage of this must take into account that a different + /// may be returned for the same Umbraco website hosted on different servers. + /// + /// string ApplicationId { get; } /// diff --git a/src/Umbraco.Core/NetworkHelper.cs b/src/Umbraco.Core/NetworkHelper.cs deleted file mode 100644 index 8e1bfaea92..0000000000 --- a/src/Umbraco.Core/NetworkHelper.cs +++ /dev/null @@ -1,48 +0,0 @@ -using System; -using Umbraco.Extensions; - -namespace Umbraco.Cms.Core -{ - /// - /// Currently just used to get the machine name in med trust and to format a machine name for use with file names - /// - public class NetworkHelper - { - /// - /// Returns the machine name that is safe to use in file paths. - /// - public static string FileSafeMachineName - { - get { return MachineName.ReplaceNonAlphanumericChars('-'); } - } - - /// - /// Returns the current machine name - /// - /// - /// Tries to resolve the machine name, if it cannot it uses the config section. - /// - public static string MachineName - { - get - { - try - { - return Environment.MachineName; - } - catch - { - try - { - return System.Net.Dns.GetHostName(); - } - catch - { - //if we get here it means we cannot access the machine name - throw new ApplicationException("Cannot resolve the current machine name either by Environment.MachineName or by Dns.GetHostname()"); - } - } - } - } - } -} diff --git a/src/Umbraco.Core/Notifications/UmbracoApplicationStartingNotification.cs b/src/Umbraco.Core/Notifications/UmbracoApplicationStartingNotification.cs index 9d324a4ca5..4cbf0a55c6 100644 --- a/src/Umbraco.Core/Notifications/UmbracoApplicationStartingNotification.cs +++ b/src/Umbraco.Core/Notifications/UmbracoApplicationStartingNotification.cs @@ -3,7 +3,10 @@ namespace Umbraco.Cms.Core.Notifications { - + /// + /// Notification that occurs at the very end of the Umbraco boot + /// process and after all initialize. + /// public class UmbracoApplicationStartingNotification : INotification { /// diff --git a/src/Umbraco.Core/Notifications/UmbracoRequestEndNotification.cs b/src/Umbraco.Core/Notifications/UmbracoRequestEndNotification.cs index 84aad8a3b3..27fb6ff09d 100644 --- a/src/Umbraco.Core/Notifications/UmbracoRequestEndNotification.cs +++ b/src/Umbraco.Core/Notifications/UmbracoRequestEndNotification.cs @@ -1,4 +1,4 @@ -// Copyright (c) Umbraco. +// Copyright (c) Umbraco. // See LICENSE for more details. using Umbraco.Cms.Core.Web; diff --git a/src/Umbraco.Core/Runtime/MainDom.cs b/src/Umbraco.Core/Runtime/MainDom.cs index adab97ed6d..ec4e56df1b 100644 --- a/src/Umbraco.Core/Runtime/MainDom.cs +++ b/src/Umbraco.Core/Runtime/MainDom.cs @@ -31,7 +31,7 @@ namespace Umbraco.Cms.Core.Runtime private bool _isInitialized; // indicates whether... - private bool _isMainDom; // we are the main domain + private bool? _isMainDom; // we are the main domain private volatile bool _signaled; // we have been signaled // actions to run before releasing the main domain @@ -64,7 +64,7 @@ namespace Umbraco.Cms.Core.Runtime { hostingEnvironment.RegisterObject(this); return Acquire(); - }); + }).Value; } /// @@ -85,7 +85,11 @@ namespace Umbraco.Cms.Core.Runtime return false; } - if (_isMainDom == false) + if (_isMainDom.HasValue == false) + { + throw new InvalidOperationException("Register called when MainDom has not been acquired"); + } + else if (_isMainDom == false) { _logger.LogWarning("Register called when MainDom has not been acquired"); return false; @@ -215,7 +219,17 @@ namespace Umbraco.Cms.Core.Runtime /// /// Acquire must be called first else this will always return false /// - public bool IsMainDom => _isMainDom; + public bool IsMainDom + { + get + { + if (!_isMainDom.HasValue) + { + throw new InvalidOperationException("MainDom has not been acquired yet"); + } + return _isMainDom.Value; + } + } // IRegisteredObject void IRegisteredObject.Stop(bool immediate) diff --git a/src/Umbraco.Core/Services/CacheInstructionServiceProcessInstructionsResult.cs b/src/Umbraco.Core/Services/CacheInstructionServiceProcessInstructionsResult.cs deleted file mode 100644 index 84116584a2..0000000000 --- a/src/Umbraco.Core/Services/CacheInstructionServiceProcessInstructionsResult.cs +++ /dev/null @@ -1,26 +0,0 @@ -using System; - -namespace Umbraco.Cms.Core.Services -{ - /// - /// Defines a result object for the operation. - /// - public class CacheInstructionServiceProcessInstructionsResult - { - private CacheInstructionServiceProcessInstructionsResult() - { - } - - public int NumberOfInstructionsProcessed { get; private set; } - - public int LastId { get; private set; } - - public bool InstructionsWerePruned { get; private set; } - - public static CacheInstructionServiceProcessInstructionsResult AsCompleted(int numberOfInstructionsProcessed, int lastId) => - new CacheInstructionServiceProcessInstructionsResult { NumberOfInstructionsProcessed = numberOfInstructionsProcessed, LastId = lastId }; - - public static CacheInstructionServiceProcessInstructionsResult AsCompletedAndPruned(int numberOfInstructionsProcessed, int lastId) => - new CacheInstructionServiceProcessInstructionsResult { NumberOfInstructionsProcessed = numberOfInstructionsProcessed, LastId = lastId, InstructionsWerePruned = true }; - }; -} diff --git a/src/Umbraco.Core/Services/ICacheInstructionService.cs b/src/Umbraco.Core/Services/ICacheInstructionService.cs index faf05f2237..c884b8bed8 100644 --- a/src/Umbraco.Core/Services/ICacheInstructionService.cs +++ b/src/Umbraco.Core/Services/ICacheInstructionService.cs @@ -1,5 +1,7 @@ using System; using System.Collections.Generic; +using System.Threading; +using Umbraco.Cms.Core.Cache; using Umbraco.Cms.Core.Sync; namespace Umbraco.Cms.Core.Services @@ -40,6 +42,12 @@ namespace Umbraco.Cms.Core.Services /// Local identity of the executing AppDomain. /// Date of last prune operation. /// Id of the latest processed instruction - CacheInstructionServiceProcessInstructionsResult ProcessInstructions(bool released, string localIdentity, DateTime lastPruned, int lastId); + ProcessInstructionsResult ProcessInstructions( + CacheRefresherCollection cacheRefreshers, + ServerRole serverRole, + CancellationToken cancellationToken, + string localIdentity, + DateTime lastPruned, + int lastId); } } diff --git a/src/Umbraco.Core/Services/ProcessInstructionsResult.cs b/src/Umbraco.Core/Services/ProcessInstructionsResult.cs new file mode 100644 index 0000000000..9a368dab7e --- /dev/null +++ b/src/Umbraco.Core/Services/ProcessInstructionsResult.cs @@ -0,0 +1,26 @@ +using System; + +namespace Umbraco.Cms.Core.Services +{ + /// + /// Defines a result object for the operation. + /// + public class ProcessInstructionsResult + { + private ProcessInstructionsResult() + { + } + + public int NumberOfInstructionsProcessed { get; private set; } + + public int LastId { get; private set; } + + public bool InstructionsWerePruned { get; private set; } + + public static ProcessInstructionsResult AsCompleted(int numberOfInstructionsProcessed, int lastId) => + new ProcessInstructionsResult { NumberOfInstructionsProcessed = numberOfInstructionsProcessed, LastId = lastId }; + + public static ProcessInstructionsResult AsCompletedAndPruned(int numberOfInstructionsProcessed, int lastId) => + new ProcessInstructionsResult { NumberOfInstructionsProcessed = numberOfInstructionsProcessed, LastId = lastId, InstructionsWerePruned = true }; + }; +} diff --git a/src/Umbraco.Core/Sync/DatabaseServerMessengerCallbacks.cs b/src/Umbraco.Core/Sync/DatabaseServerMessengerCallbacks.cs deleted file mode 100644 index 2107bbde20..0000000000 --- a/src/Umbraco.Core/Sync/DatabaseServerMessengerCallbacks.cs +++ /dev/null @@ -1,20 +0,0 @@ -using System; -using System.Collections.Generic; - -namespace Umbraco.Cms.Core.Sync -{ - /// - /// Holds a list of callbacks associated with implementations of . - /// - public class DatabaseServerMessengerCallbacks - { - /// - /// A list of callbacks that will be invoked if the lastsynced.txt file does not exist. - /// - /// - /// These callbacks will typically be for e.g. rebuilding the xml cache file, or examine indexes, based on - /// the data in the database to get this particular server node up to date. - /// - public IEnumerable InitializingCallbacks { get; set; } - } -} diff --git a/src/Umbraco.Core/Sync/IServerAddress.cs b/src/Umbraco.Core/Sync/IServerAddress.cs index c9333f33b8..a177454886 100644 --- a/src/Umbraco.Core/Sync/IServerAddress.cs +++ b/src/Umbraco.Core/Sync/IServerAddress.cs @@ -1,4 +1,4 @@ -namespace Umbraco.Cms.Core.Sync +namespace Umbraco.Cms.Core.Sync { /// /// Provides the address of a server. diff --git a/src/Umbraco.Core/Sync/ISyncBootStateAccessor.cs b/src/Umbraco.Core/Sync/ISyncBootStateAccessor.cs new file mode 100644 index 0000000000..4ced4acf83 --- /dev/null +++ b/src/Umbraco.Core/Sync/ISyncBootStateAccessor.cs @@ -0,0 +1,14 @@ +namespace Umbraco.Cms.Core.Sync +{ + /// + /// Retrieve the for the application during startup + /// + public interface ISyncBootStateAccessor + { + /// + /// Get the + /// + /// + SyncBootState GetSyncBootState(); + } +} diff --git a/src/Umbraco.Core/Sync/NonRuntimeLevelBootStateAccessor.cs b/src/Umbraco.Core/Sync/NonRuntimeLevelBootStateAccessor.cs new file mode 100644 index 0000000000..0dcfa471db --- /dev/null +++ b/src/Umbraco.Core/Sync/NonRuntimeLevelBootStateAccessor.cs @@ -0,0 +1,10 @@ +namespace Umbraco.Cms.Core.Sync +{ + /// + /// Boot state implementation for when umbraco is not in the run state + /// + public sealed class NonRuntimeLevelBootStateAccessor : ISyncBootStateAccessor + { + public SyncBootState GetSyncBootState() => SyncBootState.Unknown; + } +} diff --git a/src/Umbraco.Core/Sync/SyncBootState.cs b/src/Umbraco.Core/Sync/SyncBootState.cs new file mode 100644 index 0000000000..e07898486f --- /dev/null +++ b/src/Umbraco.Core/Sync/SyncBootState.cs @@ -0,0 +1,20 @@ +namespace Umbraco.Cms.Core.Sync +{ + public enum SyncBootState + { + /// + /// Unknown state. Treat as WarmBoot + /// + Unknown = 0, + + /// + /// Cold boot. No Sync state + /// + ColdBoot = 1, + + /// + /// Warm boot. Sync state present + /// + WarmBoot = 2 + } +} diff --git a/src/Umbraco.Examine.Lucene/BackOfficeExamineSearcher.cs b/src/Umbraco.Examine.Lucene/BackOfficeExamineSearcher.cs index d05bb8f07f..0a99c4d6ef 100644 --- a/src/Umbraco.Examine.Lucene/BackOfficeExamineSearcher.cs +++ b/src/Umbraco.Examine.Lucene/BackOfficeExamineSearcher.cs @@ -1,4 +1,4 @@ -// Copyright (c) Umbraco. +// Copyright (c) Umbraco. // See LICENSE for more details. using System; @@ -7,6 +7,8 @@ using System.Linq; using System.Text; using System.Text.RegularExpressions; using Examine; +using Examine.Search; +using Lucene.Net.QueryParsers.Classic; using Umbraco.Cms.Core; using Umbraco.Cms.Core.Cache; using Umbraco.Cms.Core.Mapping; @@ -103,17 +105,16 @@ namespace Umbraco.Cms.Infrastructure.Examine if (!_examineManager.TryGetIndex(indexName, out var index)) throw new InvalidOperationException("No index found by name " + indexName); - var internalSearcher = index.GetSearcher(); - if (!BuildQuery(sb, query, searchFrom, fields, type)) { totalFound = 0; return Enumerable.Empty(); } - var result = internalSearcher.CreateQuery().NativeQuery(sb.ToString()) - //only return the number of items specified to read up to the amount of records to fill from 0 -> the number of items on the page requested - .Execute(Convert.ToInt32(pageSize * (pageIndex + 1))); + var result = index.Searcher + .CreateQuery() + .NativeQuery(sb.ToString()) + .Execute(QueryOptions.SkipTake(Convert.ToInt32(pageSize * pageIndex), pageSize)); totalFound = result.TotalItemCount; @@ -143,7 +144,7 @@ namespace Umbraco.Cms.Infrastructure.Examine //strip quotes, escape string, the replace again query = query.Trim(Constants.CharArrays.DoubleQuoteSingleQuote); - query = Lucene.Net.QueryParsers.QueryParser.Escape(query); + query = QueryParser.Escape(query); //nothing to search if (searchFrom.IsNullOrWhiteSpace() && query.IsNullOrWhiteSpace()) @@ -186,7 +187,7 @@ namespace Umbraco.Cms.Infrastructure.Examine //update the query with the query term if (trimmed.IsNullOrWhiteSpace() == false) { - query = Lucene.Net.QueryParsers.QueryParser.Escape(query); + query = QueryParser.Escape(query); var querywords = query.Split(Constants.CharArrays.Space, StringSplitOptions.RemoveEmptyEntries); @@ -355,6 +356,8 @@ namespace Umbraco.Cms.Infrastructure.Examine sb.Append("\\,*"); } + // TODO: When/Where is this used? + /// /// Returns a collection of entities for media based on search results /// @@ -389,6 +392,8 @@ namespace Umbraco.Cms.Infrastructure.Examine } } + // TODO: When/Where is this used? + /// /// Returns a collection of entities for media based on search results /// @@ -397,6 +402,8 @@ namespace Umbraco.Cms.Infrastructure.Examine private IEnumerable MediaFromSearchResults(IEnumerable results) => _umbracoMapper.Map>(results); + // TODO: When/Where is this used? + /// /// Returns a collection of entities for content based on search results /// diff --git a/src/Umbraco.Examine.Lucene/ConfigurationEnabledDirectoryFactory.cs b/src/Umbraco.Examine.Lucene/ConfigurationEnabledDirectoryFactory.cs new file mode 100644 index 0000000000..e6e2ff9a82 --- /dev/null +++ b/src/Umbraco.Examine.Lucene/ConfigurationEnabledDirectoryFactory.cs @@ -0,0 +1,83 @@ +// Copyright (c) Umbraco. +// See LICENSE for more details. + +using System; +using System.IO; +using Examine; +using Examine.Lucene.Directories; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using Umbraco.Cms.Core.Composing; +using Umbraco.Cms.Core.Configuration.Models; +using Umbraco.Cms.Core.Hosting; +using Umbraco.Extensions; +using Constants = Umbraco.Cms.Core.Constants; + +namespace Umbraco.Cms.Infrastructure.Examine +{ + public class ConfigurationEnabledDirectoryFactory : IDirectoryFactory + { + private readonly IServiceProvider _services; + private readonly ITypeFinder _typeFinder; + private readonly IHostingEnvironment _hostingEnvironment; + private readonly ILockFactory _lockFactory; + private readonly IApplicationRoot _applicationRoot; + private readonly IndexCreatorSettings _settings; + + public ConfigurationEnabledDirectoryFactory( + IServiceProvider services, + ITypeFinder typeFinder, + IHostingEnvironment hostingEnvironment, + ILockFactory lockFactory, + IOptions settings, + IApplicationRoot applicationRoot) + { + _services = services; + _typeFinder = typeFinder; + _hostingEnvironment = hostingEnvironment; + _lockFactory = lockFactory; + _applicationRoot = applicationRoot; + _settings = settings.Value; + } + + public Lucene.Net.Store.Directory CreateDirectory(string indexName) => CreateFileSystemLuceneDirectory(indexName); + + /// + /// Creates a file system based Lucene with the correct locking guidelines for Umbraco + /// + /// + /// The folder name to store the index (single word, not a fully qualified folder) (i.e. Internal) + /// + /// + public virtual Lucene.Net.Store.Directory CreateFileSystemLuceneDirectory(string indexName) + { + var dirInfo = _applicationRoot.ApplicationRoot; + + if (!dirInfo.Exists) + { + Directory.CreateDirectory(dirInfo.FullName); + } + + //check if there's a configured directory factory, if so create it and use that to create the lucene dir + var configuredDirectoryFactory = _settings.LuceneDirectoryFactory; + + if (!configuredDirectoryFactory.IsNullOrWhiteSpace()) + { + //this should be a fully qualified type + Type factoryType = _typeFinder.GetTypeByName(configuredDirectoryFactory); + if (factoryType == null) + { + throw new InvalidOperationException("No directory type found for value: " + configuredDirectoryFactory); + } + + var directoryFactory = (IDirectoryFactory)ActivatorUtilities.CreateInstance(_services, factoryType); + + return directoryFactory.CreateDirectory(indexName); + } + + var fileSystemDirectoryFactory = new FileSystemDirectoryFactory(dirInfo, _lockFactory); + return fileSystemDirectoryFactory.CreateDirectory(indexName); + + } + } +} diff --git a/src/Umbraco.Examine.Lucene/DependencyInjection/ConfigureIndexOptions.cs b/src/Umbraco.Examine.Lucene/DependencyInjection/ConfigureIndexOptions.cs new file mode 100644 index 0000000000..677167f0ff --- /dev/null +++ b/src/Umbraco.Examine.Lucene/DependencyInjection/ConfigureIndexOptions.cs @@ -0,0 +1,46 @@ +using System; +using Examine; +using Examine.Lucene; +using Examine.Lucene.Analyzers; +using Lucene.Net.Analysis.Standard; +using Microsoft.Extensions.Options; +using Umbraco.Cms.Core; + +namespace Umbraco.Cms.Infrastructure.Examine.DependencyInjection +{ + /// + /// Configures the index options to construct the Examine indexes + /// + public sealed class ConfigureIndexOptions : IConfigureNamedOptions + { + private readonly IUmbracoIndexConfig _umbracoIndexConfig; + + public ConfigureIndexOptions(IUmbracoIndexConfig umbracoIndexConfig) + => _umbracoIndexConfig = umbracoIndexConfig; + + public void Configure(string name, LuceneDirectoryIndexOptions options) + { + switch (name) + { + case Constants.UmbracoIndexes.InternalIndexName: + options.Analyzer = new CultureInvariantWhitespaceAnalyzer(); + options.Validator = _umbracoIndexConfig.GetContentValueSetValidator(); + options.FieldDefinitions = new UmbracoFieldDefinitionCollection(); + break; + case Constants.UmbracoIndexes.ExternalIndexName: + options.Analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); + options.Validator = _umbracoIndexConfig.GetPublishedContentValueSetValidator(); + options.FieldDefinitions = new UmbracoFieldDefinitionCollection(); + break; + case Constants.UmbracoIndexes.MembersIndexName: + options.Analyzer = new CultureInvariantWhitespaceAnalyzer(); + options.Validator = _umbracoIndexConfig.GetMemberValueSetValidator(); + options.FieldDefinitions = new UmbracoFieldDefinitionCollection(); + break; + } + } + + public void Configure(LuceneDirectoryIndexOptions options) + => throw new NotImplementedException("This is never called and is just part of the interface"); + } +} diff --git a/src/Umbraco.Examine.Lucene/DependencyInjection/UmbracoBuilderExtensions.cs b/src/Umbraco.Examine.Lucene/DependencyInjection/UmbracoBuilderExtensions.cs new file mode 100644 index 0000000000..8eafde1a38 --- /dev/null +++ b/src/Umbraco.Examine.Lucene/DependencyInjection/UmbracoBuilderExtensions.cs @@ -0,0 +1,40 @@ +using Examine; +using Examine.Lucene.Directories; +using Microsoft.Extensions.DependencyInjection; +using Umbraco.Cms.Core; +using Umbraco.Cms.Core.DependencyInjection; +using Umbraco.Cms.Infrastructure.DependencyInjection; + +namespace Umbraco.Cms.Infrastructure.Examine.DependencyInjection +{ + public static class UmbracoBuilderExtensions + { + /// + /// Adds the Examine indexes for Umbraco + /// + /// + /// + public static IUmbracoBuilder AddExamineIndexes(this IUmbracoBuilder umbracoBuilder) + { + IServiceCollection services = umbracoBuilder.Services; + + services.AddSingleton(); + services.AddSingleton(); + + services.AddExamine(); + + // Create the indexes + services + .AddExamineLuceneIndex(Constants.UmbracoIndexes.InternalIndexName) + .AddExamineLuceneIndex(Constants.UmbracoIndexes.ExternalIndexName) + .AddExamineLuceneIndex(Constants.UmbracoIndexes.MembersIndexName) + .ConfigureOptions(); + + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + + return umbracoBuilder; + } + } +} diff --git a/src/Umbraco.Examine.Lucene/ExamineLuceneComponent.cs b/src/Umbraco.Examine.Lucene/ExamineLuceneComponent.cs deleted file mode 100644 index fe1826c989..0000000000 --- a/src/Umbraco.Examine.Lucene/ExamineLuceneComponent.cs +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright (c) Umbraco. -// See LICENSE for more details. - -using Examine; -using Examine.LuceneEngine.Directories; -using Microsoft.Extensions.Logging; -using Umbraco.Cms.Core.Composing; -using Umbraco.Cms.Core.Runtime; -using Umbraco.Extensions; - -namespace Umbraco.Cms.Infrastructure.Examine -{ - public sealed class ExamineLuceneComponent : IComponent - { - private readonly IndexRebuilder _indexRebuilder; - private readonly IExamineManager _examineManager; - private readonly IMainDom _mainDom; - private readonly ILoggerFactory _loggerFactory; - - public ExamineLuceneComponent(IndexRebuilder indexRebuilder, IExamineManager examineManager, IMainDom mainDom, ILoggerFactory loggerFactory) - { - _indexRebuilder = indexRebuilder; - _examineManager = examineManager; - _mainDom = mainDom; - _loggerFactory = loggerFactory; - } - - public void Initialize() - { - //we want to tell examine to use a different fs lock instead of the default NativeFSFileLock which could cause problems if the AppDomain - //terminates and in some rare cases would only allow unlocking of the file if IIS is forcefully terminated. Instead we'll rely on the simplefslock - //which simply checks the existence of the lock file - DirectoryFactory.DefaultLockFactory = d => - { - var simpleFsLockFactory = new NoPrefixSimpleFsLockFactory(d); - return simpleFsLockFactory; - }; - - _indexRebuilder.RebuildingIndexes += IndexRebuilder_RebuildingIndexes; - } - - /// - /// Handles event to ensure that all lucene based indexes are properly configured before rebuilding - /// - /// - /// - private void IndexRebuilder_RebuildingIndexes(object sender, IndexRebuildingEventArgs e) => _examineManager.ConfigureIndexes(_mainDom, _loggerFactory.CreateLogger()); - - public void Terminate() - { - } - } -} diff --git a/src/Umbraco.Examine.Lucene/ExamineLuceneComposer.cs b/src/Umbraco.Examine.Lucene/ExamineLuceneComposer.cs deleted file mode 100644 index 327ac4b4ba..0000000000 --- a/src/Umbraco.Examine.Lucene/ExamineLuceneComposer.cs +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) Umbraco. -// See LICENSE for more details. - -using System.Runtime.InteropServices; -using Umbraco.Cms.Core.Composing; -using Umbraco.Cms.Core.DependencyInjection; -using Umbraco.Extensions; - -namespace Umbraco.Cms.Infrastructure.Examine -{ - // We want to run after core composers since we are replacing some items - [ComposeAfter(typeof(ICoreComposer))] - public sealed class ExamineLuceneComposer : ComponentComposer - { - public override void Compose(IUmbracoBuilder builder) - { - var isWindows = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); - if(!isWindows) return; - - - base.Compose(builder); - - builder.Services.AddUnique(); - builder.Services.AddUnique(); - builder.Services.AddUnique(); - builder.Services.AddUnique(); - } - } -} diff --git a/src/Umbraco.Examine.Lucene/ExamineLuceneFinalComponent.cs b/src/Umbraco.Examine.Lucene/ExamineLuceneFinalComponent.cs deleted file mode 100644 index b95165b121..0000000000 --- a/src/Umbraco.Examine.Lucene/ExamineLuceneFinalComponent.cs +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright (c) Umbraco. -// See LICENSE for more details. - -using Examine; -using Microsoft.Extensions.Logging; -using Umbraco.Cms.Core.Composing; -using Umbraco.Cms.Core.Runtime; -using Umbraco.Extensions; - -namespace Umbraco.Cms.Infrastructure.Examine -{ - public class ExamineLuceneFinalComponent : IComponent - { - private readonly ILoggerFactory _loggerFactory; - private readonly IExamineManager _examineManager; - private readonly IMainDom _mainDom; - - public ExamineLuceneFinalComponent(ILoggerFactory loggerFactory, IExamineManager examineManager, IMainDom mainDom) - { - _loggerFactory = loggerFactory; - _examineManager = examineManager; - _mainDom = mainDom; - } - - public void Initialize() - { - if (!_mainDom.IsMainDom) return; - - // Ensures all lucene based indexes are unlocked and ready to go - _examineManager.ConfigureIndexes(_mainDom, _loggerFactory.CreateLogger()); - } - - public void Terminate() - { - } - } -} diff --git a/src/Umbraco.Examine.Lucene/ExamineLuceneFinalComposer.cs b/src/Umbraco.Examine.Lucene/ExamineLuceneFinalComposer.cs deleted file mode 100644 index 518ffc2db8..0000000000 --- a/src/Umbraco.Examine.Lucene/ExamineLuceneFinalComposer.cs +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright (c) Umbraco. -// See LICENSE for more details. - -using Umbraco.Cms.Core.Composing; - -namespace Umbraco.Cms.Infrastructure.Examine -{ - // examine's Lucene final composer composes after all user composers - // and *also* after ICoreComposer (in case IUserComposer is disabled) - [ComposeAfter(typeof(IUserComposer))] - [ComposeAfter(typeof(ICoreComposer))] - public class ExamineLuceneFinalComposer : ComponentComposer - { } -} diff --git a/src/Umbraco.Examine.Lucene/Extensions/ExamineExtensions.cs b/src/Umbraco.Examine.Lucene/Extensions/ExamineExtensions.cs index 9307c4cbf4..1a8bb36baa 100644 --- a/src/Umbraco.Examine.Lucene/Extensions/ExamineExtensions.cs +++ b/src/Umbraco.Examine.Lucene/Extensions/ExamineExtensions.cs @@ -1,19 +1,17 @@ -// Copyright (c) Umbraco. +// Copyright (c) Umbraco. // See LICENSE for more details. using System; using System.Linq; using System.Threading; using Examine; -using Examine.LuceneEngine.Providers; -using Lucene.Net.Analysis; +using Examine.Lucene.Providers; +using Lucene.Net.Analysis.Core; using Lucene.Net.Index; -using Lucene.Net.QueryParsers; -using Lucene.Net.Search; +using Lucene.Net.QueryParsers.Classic; using Microsoft.Extensions.Logging; using Umbraco.Cms.Core.Runtime; using Umbraco.Cms.Infrastructure.Examine; -using Version = Lucene.Net.Util.Version; namespace Umbraco.Extensions { @@ -22,40 +20,19 @@ namespace Umbraco.Extensions /// public static class ExamineExtensions { - private static bool _isConfigured = false; - private static object _configuredInit = null; - private static object _isConfiguredLocker = new object(); - - /// - /// Called on startup to configure each index. - /// - /// - /// Configures and unlocks all Lucene based indexes registered with the . - /// - internal static void ConfigureIndexes(this IExamineManager examineManager, IMainDom mainDom, ILogger logger) - { - LazyInitializer.EnsureInitialized( - ref _configuredInit, - ref _isConfigured, - ref _isConfiguredLocker, - () => - { - examineManager.ConfigureLuceneIndexes(logger, !mainDom.IsMainDom); - return null; - }); - } - internal static bool TryParseLuceneQuery(string query) { // TODO: I'd assume there would be a more strict way to parse the query but not that i can find yet, for now we'll // also do this rudimentary check if (!query.Contains(":")) + { return false; + } try { //This will pass with a plain old string without any fields, need to figure out a way to have it properly parse - var parsed = new QueryParser(Version.LUCENE_30, UmbracoExamineFieldNames.NodeNameFieldName, new KeywordAnalyzer()).Parse(query); + var parsed = new QueryParser(LuceneInfo.CurrentVersion, UmbracoExamineFieldNames.NodeNameFieldName, new KeywordAnalyzer()).Parse(query); return true; } catch (ParseException) @@ -68,34 +45,6 @@ namespace Umbraco.Extensions } } - /// - /// Forcibly unlocks all lucene based indexes - /// - /// - /// This is not thread safe, use with care - /// - private static void ConfigureLuceneIndexes(this IExamineManager examineManager, ILogger logger, bool disableExamineIndexing) - { - foreach (var luceneIndexer in examineManager.Indexes.OfType()) - { - //We now need to disable waiting for indexing for Examine so that the appdomain is shutdown immediately and doesn't wait for pending - //indexing operations. We used to wait for indexing operations to complete but this can cause more problems than that is worth because - //that could end up halting shutdown for a very long time causing overlapping appdomains and many other problems. - luceneIndexer.WaitForIndexQueueOnShutdown = false; - - if (disableExamineIndexing) continue; //exit if not enabled, we don't need to unlock them if we're not maindom - - //we should check if the index is locked ... it shouldn't be! We are using simple fs lock now and we are also ensuring that - //the indexes are not operational unless MainDom is true - var dir = luceneIndexer.GetLuceneDirectory(); - if (IndexWriter.IsLocked(dir)) - { - logger.LogDebug("Forcing index {IndexerName} to be unlocked since it was left in a locked state", luceneIndexer.Name); - IndexWriter.Unlock(dir); - } - } - } - /// /// Checks if the index can be read/opened /// @@ -106,7 +55,7 @@ namespace Umbraco.Extensions { try { - using (indexer.GetIndexWriter().GetReader()) + using (indexer.IndexWriter.IndexWriter.GetReader(false)) { ex = null; return true; @@ -119,38 +68,5 @@ namespace Umbraco.Extensions } } - /// - /// Return the number of indexed documents in Lucene - /// - /// - /// - public static int GetIndexDocumentCount(this LuceneIndex indexer) - { - if (!((indexer.GetSearcher() as LuceneSearcher)?.GetLuceneSearcher() is IndexSearcher searcher)) - return 0; - - using (searcher) - using (var reader = searcher.IndexReader) - { - return reader.NumDocs(); - } - } - - /// - /// Return the total number of fields in the index - /// - /// - /// - public static int GetIndexFieldCount(this LuceneIndex indexer) - { - if (!((indexer.GetSearcher() as LuceneSearcher)?.GetLuceneSearcher() is IndexSearcher searcher)) - return 0; - - using (searcher) - using (var reader = searcher.IndexReader) - { - return reader.GetFieldNames(IndexReader.FieldOption.ALL).Count; - } - } } } diff --git a/src/Umbraco.Examine.Lucene/ILuceneDirectoryFactory.cs b/src/Umbraco.Examine.Lucene/ILuceneDirectoryFactory.cs deleted file mode 100644 index 70f3825667..0000000000 --- a/src/Umbraco.Examine.Lucene/ILuceneDirectoryFactory.cs +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright (c) Umbraco. -// See LICENSE for more details. - -namespace Umbraco.Cms.Infrastructure.Examine -{ - public interface ILuceneDirectoryFactory - { - Lucene.Net.Store.Directory CreateDirectory(string indexName); - } -} diff --git a/src/Umbraco.Examine.Lucene/LuceneFileSystemDirectoryFactory.cs b/src/Umbraco.Examine.Lucene/LuceneFileSystemDirectoryFactory.cs deleted file mode 100644 index 9e09c7e96e..0000000000 --- a/src/Umbraco.Examine.Lucene/LuceneFileSystemDirectoryFactory.cs +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright (c) Umbraco. -// See LICENSE for more details. - -using System; -using System.IO; -using Examine.LuceneEngine.Directories; -using Lucene.Net.Store; -using Microsoft.Extensions.Options; -using Umbraco.Cms.Core.Composing; -using Umbraco.Cms.Core.Configuration.Models; -using Umbraco.Cms.Core.Hosting; -using Umbraco.Extensions; -using Constants = Umbraco.Cms.Core.Constants; - -namespace Umbraco.Cms.Infrastructure.Examine -{ - public class LuceneFileSystemDirectoryFactory : ILuceneDirectoryFactory - { - private readonly ITypeFinder _typeFinder; - private readonly IHostingEnvironment _hostingEnvironment; - private readonly IndexCreatorSettings _settings; - - public LuceneFileSystemDirectoryFactory(ITypeFinder typeFinder, IHostingEnvironment hostingEnvironment, IOptions settings) - { - _typeFinder = typeFinder; - _hostingEnvironment = hostingEnvironment; - _settings = settings.Value; - } - - public Lucene.Net.Store.Directory CreateDirectory(string indexName) => CreateFileSystemLuceneDirectory(indexName); - - /// - /// Creates a file system based Lucene with the correct locking guidelines for Umbraco - /// - /// - /// The folder name to store the index (single word, not a fully qualified folder) (i.e. Internal) - /// - /// - public virtual Lucene.Net.Store.Directory CreateFileSystemLuceneDirectory(string folderName) - { - - var dirInfo = new DirectoryInfo(Path.Combine(_hostingEnvironment.MapPathContentRoot(Constants.SystemDirectories.TempData), "ExamineIndexes", folderName)); - if (!dirInfo.Exists) - System.IO.Directory.CreateDirectory(dirInfo.FullName); - - //check if there's a configured directory factory, if so create it and use that to create the lucene dir - var configuredDirectoryFactory = _settings.LuceneDirectoryFactory; - - if (!configuredDirectoryFactory.IsNullOrWhiteSpace()) - { - //this should be a fully qualified type - var factoryType = _typeFinder.GetTypeByName(configuredDirectoryFactory); - if (factoryType == null) throw new NullReferenceException("No directory type found for value: " + configuredDirectoryFactory); - var directoryFactory = (IDirectoryFactory)Activator.CreateInstance(factoryType); - return directoryFactory.CreateDirectory(dirInfo); - } - - //no dir factory, just create a normal fs directory - - var luceneDir = new SimpleFSDirectory(dirInfo); - - //we want to tell examine to use a different fs lock instead of the default NativeFSFileLock which could cause problems if the appdomain - //terminates and in some rare cases would only allow unlocking of the file if IIS is forcefully terminated. Instead we'll rely on the simplefslock - //which simply checks the existence of the lock file - // The full syntax of this is: new NoPrefixSimpleFsLockFactory(dirInfo) - // however, we are setting the DefaultLockFactory in startup so we'll use that instead since it can be managed globally. - luceneDir.SetLockFactory(DirectoryFactory.DefaultLockFactory(dirInfo)); - return luceneDir; - - - } - } -} diff --git a/src/Umbraco.Examine.Lucene/LuceneIndexCreator.cs b/src/Umbraco.Examine.Lucene/LuceneIndexCreator.cs deleted file mode 100644 index dc2acfa66d..0000000000 --- a/src/Umbraco.Examine.Lucene/LuceneIndexCreator.cs +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) Umbraco. -// See LICENSE for more details. - -using System.Collections.Generic; -using Examine; -using Microsoft.Extensions.Options; -using Umbraco.Cms.Core.Composing; -using Umbraco.Cms.Core.Configuration.Models; -using Umbraco.Cms.Core.Hosting; - -namespace Umbraco.Cms.Infrastructure.Examine -{ - /// - /// - /// Abstract class for creating Lucene based Indexes - /// - public abstract class LuceneIndexCreator : IIndexCreator - { - private readonly ITypeFinder _typeFinder; - private readonly IHostingEnvironment _hostingEnvironment; - private readonly IndexCreatorSettings _settings; - - protected LuceneIndexCreator(ITypeFinder typeFinder, IHostingEnvironment hostingEnvironment, IOptions settings) - { - _typeFinder = typeFinder; - _hostingEnvironment = hostingEnvironment; - _settings = settings.Value; - } - - public abstract IEnumerable Create(); - } -} diff --git a/src/Umbraco.Examine.Lucene/LuceneIndexDiagnostics.cs b/src/Umbraco.Examine.Lucene/LuceneIndexDiagnostics.cs index 1cba0767eb..6ad23b5992 100644 --- a/src/Umbraco.Examine.Lucene/LuceneIndexDiagnostics.cs +++ b/src/Umbraco.Examine.Lucene/LuceneIndexDiagnostics.cs @@ -1,8 +1,10 @@ -// Copyright (c) Umbraco. +// Copyright (c) Umbraco. // See LICENSE for more details. +using System; using System.Collections.Generic; -using Examine.LuceneEngine.Providers; +using System.Threading.Tasks; +using Examine.Lucene.Providers; using Lucene.Net.Store; using Microsoft.Extensions.Logging; using Umbraco.Cms.Core; @@ -25,37 +27,7 @@ namespace Umbraco.Cms.Infrastructure.Examine public LuceneIndex Index { get; } public ILogger Logger { get; } - public int DocumentCount - { - get - { - try - { - return Index.GetIndexDocumentCount(); - } - catch (AlreadyClosedException) - { - Logger.LogWarning("Cannot get GetIndexDocumentCount, the writer is already closed"); - return 0; - } - } - } - - public int FieldCount - { - get - { - try - { - return Index.GetIndexFieldCount(); - } - catch (AlreadyClosedException) - { - Logger.LogWarning("Cannot get GetIndexFieldCount, the writer is already closed"); - return 0; - } - } - } + public Attempt IsHealthy() { @@ -63,6 +35,10 @@ namespace Umbraco.Cms.Infrastructure.Examine return isHealthy ? Attempt.Succeed() : Attempt.Fail(indexError.Message); } + public long GetDocumentCount() => Index.GetDocumentCount(); + + public IEnumerable GetFieldNames() => Index.GetFieldNames(); + public virtual IReadOnlyDictionary Metadata { get diff --git a/src/Umbraco.Examine.Lucene/LuceneIndexDiagnosticsFactory.cs b/src/Umbraco.Examine.Lucene/LuceneIndexDiagnosticsFactory.cs index 322da710dc..bdfc299121 100644 --- a/src/Umbraco.Examine.Lucene/LuceneIndexDiagnosticsFactory.cs +++ b/src/Umbraco.Examine.Lucene/LuceneIndexDiagnosticsFactory.cs @@ -1,8 +1,8 @@ -// Copyright (c) Umbraco. +// Copyright (c) Umbraco. // See LICENSE for more details. using Examine; -using Examine.LuceneEngine.Providers; +using Examine.Lucene.Providers; using Microsoft.Extensions.Logging; using Umbraco.Cms.Core.Hosting; @@ -28,9 +28,13 @@ namespace Umbraco.Cms.Infrastructure.Examine if (!(index is IIndexDiagnostics indexDiag)) { if (index is LuceneIndex luceneIndex) + { indexDiag = new LuceneIndexDiagnostics(luceneIndex, _loggerFactory.CreateLogger(), _hostingEnvironment); + } else + { indexDiag = base.Create(index); + } } return indexDiag; } diff --git a/src/Umbraco.Examine.Lucene/LuceneRAMDirectoryFactory.cs b/src/Umbraco.Examine.Lucene/LuceneRAMDirectoryFactory.cs index 1f9802f072..2b25350f09 100644 --- a/src/Umbraco.Examine.Lucene/LuceneRAMDirectoryFactory.cs +++ b/src/Umbraco.Examine.Lucene/LuceneRAMDirectoryFactory.cs @@ -1,27 +1,26 @@ -// Copyright (c) Umbraco. +// Copyright (c) Umbraco. // See LICENSE for more details. using System; +using System.IO; +using Examine.Lucene.Directories; using Lucene.Net.Store; +using Directory = Lucene.Net.Store.Directory; namespace Umbraco.Cms.Infrastructure.Examine { - public class LuceneRAMDirectoryFactory : ILuceneDirectoryFactory + public class LuceneRAMDirectoryFactory : IDirectoryFactory { public LuceneRAMDirectoryFactory() { - } - public Lucene.Net.Store.Directory CreateDirectory(string indexName) => new RandomIdRAMDirectory(); + public Directory CreateDirectory(string indexName) => new RandomIdRAMDirectory(); private class RandomIdRAMDirectory : RAMDirectory { private readonly string _lockId = Guid.NewGuid().ToString(); - public override string GetLockId() - { - return _lockId; - } + public override string GetLockID() => _lockId; } } } diff --git a/src/Umbraco.Examine.Lucene/NoPrefixSimpleFsLockFactory.cs b/src/Umbraco.Examine.Lucene/NoPrefixSimpleFsLockFactory.cs index 38d704e681..ed6f47c882 100644 --- a/src/Umbraco.Examine.Lucene/NoPrefixSimpleFsLockFactory.cs +++ b/src/Umbraco.Examine.Lucene/NoPrefixSimpleFsLockFactory.cs @@ -6,6 +6,7 @@ using Lucene.Net.Store; namespace Umbraco.Cms.Infrastructure.Examine { + /// /// A custom that ensures a prefixless lock prefix /// diff --git a/src/Umbraco.Examine.Lucene/Umbraco.Examine.Lucene.csproj b/src/Umbraco.Examine.Lucene/Umbraco.Examine.Lucene.csproj index ef67c424d8..2417178d69 100644 --- a/src/Umbraco.Examine.Lucene/Umbraco.Examine.Lucene.csproj +++ b/src/Umbraco.Examine.Lucene/Umbraco.Examine.Lucene.csproj @@ -1,56 +1,42 @@ - - - net472 - Umbraco.Cms.Infrastructure.Examine - Umbraco CMS - Umbraco.Examine.Lucene - - - false - - Umbraco.Cms.Examine.Lucene - - - - true - bin\Release\Umbraco.Examine.Lucene.xml - - - - - - - - - - - - - - - - - - - - - - - 1.0.0 - runtime; build; native; contentfiles; analyzers; buildtransitive - all - - - - - 3.5.4 - runtime; build; native; contentfiles; analyzers - all - - - all - - - - + + netstandard2.0 + Umbraco.Cms.Infrastructure.Examine + Umbraco CMS + Umbraco.Examine.Lucene + + Umbraco.Cms.Examine.Lucene + + + true + bin\Release\Umbraco.Examine.Lucene.xml + + + + + + + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + 3.5.4 + runtime; build; native; contentfiles; analyzers + all + + + all + + + \ No newline at end of file diff --git a/src/Umbraco.Examine.Lucene/UmbracoApplicationRoot.cs b/src/Umbraco.Examine.Lucene/UmbracoApplicationRoot.cs new file mode 100644 index 0000000000..e99f986176 --- /dev/null +++ b/src/Umbraco.Examine.Lucene/UmbracoApplicationRoot.cs @@ -0,0 +1,23 @@ +using System.IO; +using Examine; +using Umbraco.Cms.Core.Hosting; + +namespace Umbraco.Cms.Infrastructure.Examine +{ + /// + /// Sets the Examine to be ExamineIndexes sub directory of the Umbraco TEMP folder + /// + public class UmbracoApplicationRoot : IApplicationRoot + { + private readonly IHostingEnvironment _hostingEnvironment; + + public UmbracoApplicationRoot(IHostingEnvironment hostingEnvironment) + => _hostingEnvironment = hostingEnvironment; + + public DirectoryInfo ApplicationRoot + => new DirectoryInfo( + Path.Combine( + _hostingEnvironment.MapPathContentRoot(Core.Constants.SystemDirectories.TempData), + "ExamineIndexes")); + } +} diff --git a/src/Umbraco.Examine.Lucene/UmbracoContentIndex.cs b/src/Umbraco.Examine.Lucene/UmbracoContentIndex.cs index 18b9945a6e..b3852254af 100644 --- a/src/Umbraco.Examine.Lucene/UmbracoContentIndex.cs +++ b/src/Umbraco.Examine.Lucene/UmbracoContentIndex.cs @@ -5,12 +5,10 @@ using System; using System.Collections.Generic; using System.Linq; using Examine; -using Examine.LuceneEngine; -using Lucene.Net.Analysis; -using Lucene.Net.Store; +using Examine.Lucene; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Umbraco.Cms.Core.Hosting; -using Umbraco.Cms.Core.Logging; using Umbraco.Cms.Core.Services; namespace Umbraco.Cms.Infrastructure.Examine @@ -21,49 +19,38 @@ namespace Umbraco.Cms.Infrastructure.Examine public class UmbracoContentIndex : UmbracoExamineIndex, IUmbracoContentIndex, IDisposable { private readonly ILogger _logger; - protected ILocalizationService LanguageService { get; } - #region Constructors - - /// - /// Create an index at runtime - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// public UmbracoContentIndex( - string name, - Directory luceneDirectory, - FieldDefinitionCollection fieldDefinitions, - Analyzer defaultAnalyzer, - IProfilingLogger profilingLogger, - ILogger logger, ILoggerFactory loggerFactory, + string name, + IOptionsSnapshot indexOptions, IHostingEnvironment hostingEnvironment, IRuntimeState runtimeState, - ILocalizationService languageService, - IContentValueSetValidator validator, - IReadOnlyDictionary indexValueTypes = null) - : base(name, luceneDirectory, fieldDefinitions, defaultAnalyzer, profilingLogger, logger, loggerFactory ,hostingEnvironment, runtimeState, validator, indexValueTypes) + ILocalizationService languageService = null) + : base(loggerFactory, name, indexOptions, hostingEnvironment, runtimeState) { - if (validator == null) throw new ArgumentNullException(nameof(validator)); - _logger = logger; - LanguageService = languageService ?? throw new ArgumentNullException(nameof(languageService)); + LanguageService = languageService; + _logger = loggerFactory.CreateLogger(); - if (validator is IContentValueSetValidator contentValueSetValidator) + LuceneDirectoryIndexOptions namedOptions = indexOptions.Get(name); + if (namedOptions == null) + { + throw new InvalidOperationException($"No named {typeof(LuceneDirectoryIndexOptions)} options with name {name}"); + } + + if (namedOptions.Validator is IContentValueSetValidator contentValueSetValidator) + { PublishedValuesOnly = contentValueSetValidator.PublishedValuesOnly; + } } - #endregion + protected ILocalizationService LanguageService { get; } + + /// + /// Explicitly override because we need to do validation differently than the underlying logic + /// + /// + void IIndex.IndexItems(IEnumerable values) => PerformIndexItems(values, OnIndexOperationComplete); /// /// Special check for invalid paths @@ -75,45 +62,48 @@ namespace Umbraco.Cms.Infrastructure.Examine // We don't want to re-enumerate this list, but we need to split it into 2x enumerables: invalid and valid items. // The Invalid items will be deleted, these are items that have invalid paths (i.e. moved to the recycle bin, etc...) // Then we'll index the Value group all together. - // We return 0 or 1 here so we can order the results and do the invalid first and then the valid. var invalidOrValid = values.GroupBy(v => { - if (!v.Values.TryGetValue("path", out var paths) || paths.Count <= 0 || paths[0] == null) - return 0; + if (!v.Values.TryGetValue("path", out List paths) || paths.Count <= 0 || paths[0] == null) + { + return ValueSetValidationResult.Failed; + } - //we know this is an IContentValueSetValidator - var validator = (IContentValueSetValidator)ValueSetValidator; - var path = paths[0].ToString(); + ValueSetValidationResult validationResult = ValueSetValidator.Validate(v); - return (!validator.ValidatePath(path, v.Category) - || !validator.ValidateRecycleBin(path, v.Category) - || !validator.ValidateProtectedContent(path, v.Category)) - ? 0 - : 1; + return validationResult; }).ToList(); var hasDeletes = false; var hasUpdates = false; - foreach (var group in invalidOrValid.OrderBy(x => x.Key)) - { - if (group.Key == 0) - { - hasDeletes = true; - //these are the invalid items so we'll delete them - //since the path is not valid we need to delete this item in case it exists in the index already and has now - //been moved to an invalid parent. - base.PerformDeleteFromIndex(group.Select(x => x.Id), args => { /*noop*/ }); - } - else + // ordering by descending so that Filtered/Failed processes first + foreach (IGrouping group in invalidOrValid.OrderByDescending(x => x.Key)) + { + switch (group.Key) { - hasUpdates = true; - //these are the valid ones, so just index them all at once - base.PerformIndexItems(group.ToList(), onComplete); + case ValueSetValidationResult.Valid: + hasUpdates = true; + + //these are the valid ones, so just index them all at once + base.PerformIndexItems(group.ToList(), onComplete); + break; + case ValueSetValidationResult.Failed: + // don't index anything that is invalid + break; + case ValueSetValidationResult.Filtered: + hasDeletes = true; + + // these are the invalid/filtered items so we'll delete them + // since the path is not valid we need to delete this item in + // case it exists in the index already and has now + // been moved to an invalid parent. + base.PerformDeleteFromIndex(group.Select(x => x.Id), null); + break; } } - if (hasDeletes && !hasUpdates || !hasDeletes && !hasUpdates) + if ((hasDeletes && !hasUpdates) || (!hasDeletes && !hasUpdates)) { //we need to manually call the completed method onComplete(new IndexOperationEventArgs(this, 0)); @@ -133,21 +123,27 @@ namespace Umbraco.Cms.Infrastructure.Examine protected override void PerformDeleteFromIndex(IEnumerable itemIds, Action onComplete) { var idsAsList = itemIds.ToList(); - foreach (var nodeId in idsAsList) + + for (int i = 0; i < idsAsList.Count; i++) { + string nodeId = idsAsList[i]; + //find all descendants based on path var descendantPath = $@"\-1\,*{nodeId}\,*"; var rawQuery = $"{UmbracoExamineFieldNames.IndexPathFieldName}:{descendantPath}"; - var searcher = GetSearcher(); - var c = searcher.CreateQuery(); + var c = Searcher.CreateQuery(); var filtered = c.NativeQuery(rawQuery); var results = filtered.Execute(); _logger. LogDebug("DeleteFromIndex with query: {Query} (found {TotalItems} results)", rawQuery, results.TotalItemCount); - //need to queue a delete item for each one found - QueueIndexOperation(results.Select(r => new IndexOperation(new ValueSet(r.Id), IndexOperationType.Delete))); + var toRemove = results.Select(x => x.Id).ToList(); + // delete those descendants (ensure base. is used here so we aren't calling ourselves!) + base.PerformDeleteFromIndex(toRemove, null); + + // remove any ids from our list that were part of the descendants + idsAsList.RemoveAll(x => toRemove.Contains(x)); } base.PerformDeleteFromIndex(idsAsList, onComplete); diff --git a/src/Umbraco.Examine.Lucene/UmbracoExamineIndex.cs b/src/Umbraco.Examine.Lucene/UmbracoExamineIndex.cs index 851dfbd152..5ebcb4877a 100644 --- a/src/Umbraco.Examine.Lucene/UmbracoExamineIndex.cs +++ b/src/Umbraco.Examine.Lucene/UmbracoExamineIndex.cs @@ -4,20 +4,17 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Threading; using Examine; -using Examine.LuceneEngine; -using Examine.LuceneEngine.Providers; -using Lucene.Net.Analysis; +using Examine.Lucene; +using Examine.Lucene.Providers; using Lucene.Net.Documents; using Lucene.Net.Index; using Lucene.Net.Store; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Umbraco.Cms.Core; using Umbraco.Cms.Core.Hosting; -using Umbraco.Cms.Core.Logging; using Umbraco.Cms.Core.Services; -using Directory = Lucene.Net.Store.Directory; namespace Umbraco.Cms.Infrastructure.Examine { @@ -27,61 +24,24 @@ namespace Umbraco.Cms.Infrastructure.Examine /// public abstract class UmbracoExamineIndex : LuceneIndex, IUmbracoIndex, IIndexDiagnostics { - private readonly ILogger _logger; - private readonly ILoggerFactory _loggerFactory; - + private readonly UmbracoExamineIndexDiagnostics _diagnostics; private readonly IRuntimeState _runtimeState; + private bool _hasLoggedInitLog = false; + private readonly ILogger _logger; - // note - // wrapping all operations that end up calling base.SafelyProcessQueueItems in a safe call - // context because they will fork a thread/task/whatever which should *not* capture our - // call context (and the database it can contain)! - // TODO: FIX Examine to not flow the ExecutionContext so callers don't need to worry about this! - - /// - /// Create a new - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// protected UmbracoExamineIndex( - string name, - Directory luceneDirectory, - FieldDefinitionCollection fieldDefinitions, - Analyzer defaultAnalyzer, - IProfilingLogger profilingLogger, - ILogger logger, ILoggerFactory loggerFactory, + string name, + IOptionsSnapshot indexOptions, IHostingEnvironment hostingEnvironment, - IRuntimeState runtimeState, - IValueSetValidator validator = null, - IReadOnlyDictionary indexValueTypes = null) - : base(name, luceneDirectory, fieldDefinitions, defaultAnalyzer, validator, indexValueTypes) + IRuntimeState runtimeState) + : base(loggerFactory, name, indexOptions) { - _logger = logger; - _loggerFactory = loggerFactory; _runtimeState = runtimeState; - ProfilingLogger = profilingLogger ?? throw new ArgumentNullException(nameof(profilingLogger)); - - //try to set the value of `LuceneIndexFolder` for diagnostic reasons - if (luceneDirectory is FSDirectory fsDir) - LuceneIndexFolder = fsDir.Directory; - - _diagnostics = new UmbracoExamineIndexDiagnostics(this, _loggerFactory.CreateLogger(), hostingEnvironment); + _diagnostics = new UmbracoExamineIndexDiagnostics(this, loggerFactory.CreateLogger(), hostingEnvironment); + _logger = loggerFactory.CreateLogger(); } - private readonly bool _configBased = false; - - protected IProfilingLogger ProfilingLogger { get; } - /// /// When set to true Umbraco will keep the index in sync with Umbraco data automatically /// @@ -89,14 +49,6 @@ namespace Umbraco.Cms.Infrastructure.Examine public bool PublishedValuesOnly { get; protected set; } = false; - /// - public IEnumerable GetFields() - { - //we know this is a LuceneSearcher - var searcher = (LuceneSearcher)GetSearcher(); - return searcher.GetAllIndexedFields(); - } - /// /// override to check if we can actually initialize. /// @@ -107,13 +59,7 @@ namespace Umbraco.Cms.Infrastructure.Examine { if (CanInitialize()) { - // Use ExecutionContext.SuppressFlow to prevent the current Execution Context (AsyncLocal) flow to child - // tasks executed in the base class so we don't leak Scopes. - // TODO: See notes at the top of this class - using (ExecutionContext.SuppressFlow()) - { - base.PerformDeleteFromIndex(itemIds, onComplete); - } + base.PerformDeleteFromIndex(itemIds, onComplete); } } @@ -121,13 +67,7 @@ namespace Umbraco.Cms.Infrastructure.Examine { if (CanInitialize()) { - // Use ExecutionContext.SuppressFlow to prevent the current Execution Context (AsyncLocal) flow to child - // tasks executed in the base class so we don't leak Scopes. - // TODO: See notes at the top of this class - using (ExecutionContext.SuppressFlow()) - { - base.PerformIndexItems(values, onComplete); - } + base.PerformIndexItems(values, onComplete); } } @@ -137,19 +77,15 @@ namespace Umbraco.Cms.Infrastructure.Examine /// protected bool CanInitialize() { - // only affects indexers that are config file based, if an index was created via code then - // this has no effect, it is assumed the index would not be created if it could not be initialized - return _configBased == false || _runtimeState.Level == RuntimeLevel.Run; - } + var canInit = _runtimeState.Level == RuntimeLevel.Run; - /// - /// overridden for logging - /// - /// - protected override void OnIndexingError(IndexingErrorEventArgs ex) - { - _logger.LogError(ex.InnerException, ex.Message); - base.OnIndexingError(ex); + if (!canInit && !_hasLoggedInitLog) + { + _hasLoggedInitLog = true; + _logger.LogWarning("Runtime state is not " + RuntimeLevel.Run + ", no indexing will occur"); + } + + return canInit; } /// @@ -167,31 +103,16 @@ namespace Umbraco.Cms.Infrastructure.Examine //remove the original value so we can store it the correct way d.RemoveField(f.Key); - d.Add(new Field( + d.Add(new StringField( f.Key, f.Value[0].ToString(), - Field.Store.YES, - Field.Index.NO, //don't index this field, we never want to search by it - Field.TermVector.NO)); + Field.Store.YES)); } } base.OnDocumentWriting(docArgs); } - /// - /// Overridden for logging. - /// - protected override void AddDocument(Document doc, ValueSet valueSet, IndexWriter writer) - { - _logger.LogDebug("Write lucene doc id:{DocumentId}, category:{DocumentCategory}, type:{DocumentItemType}", - valueSet.Id, - valueSet.Category, - valueSet.ItemType); - - base.AddDocument(doc, valueSet, writer); - } - protected override void OnTransformingIndexValues(IndexingItemEventArgs e) { base.OnTransformingIndexValues(e); @@ -210,15 +131,7 @@ namespace Umbraco.Cms.Infrastructure.Examine } } - #region IIndexDiagnostics - - private readonly UmbracoExamineIndexDiagnostics _diagnostics; - - public int DocumentCount => _diagnostics.DocumentCount; - public int FieldCount => _diagnostics.FieldCount; public Attempt IsHealthy() => _diagnostics.IsHealthy(); public virtual IReadOnlyDictionary Metadata => _diagnostics.Metadata; - - #endregion } } diff --git a/src/Umbraco.Examine.Lucene/UmbracoIndexesCreator.cs b/src/Umbraco.Examine.Lucene/UmbracoIndexesCreator.cs deleted file mode 100644 index aa7b30677f..0000000000 --- a/src/Umbraco.Examine.Lucene/UmbracoIndexesCreator.cs +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright (c) Umbraco. -// See LICENSE for more details. - -using System.Collections.Generic; -using Examine; -using Examine.LuceneEngine; -using Lucene.Net.Analysis.Standard; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using Umbraco.Cms.Core.Composing; -using Umbraco.Cms.Core.Configuration.Models; -using Umbraco.Cms.Core.Hosting; -using Umbraco.Cms.Core.Logging; -using Umbraco.Cms.Core.Services; -using Constants = Umbraco.Cms.Core.Constants; - -namespace Umbraco.Cms.Infrastructure.Examine -{ - /// - /// Creates the indexes used by Umbraco - /// - public class UmbracoIndexesCreator : LuceneIndexCreator, IUmbracoIndexesCreator - { - // TODO: we should inject the different IValueSetValidator so devs can just register them instead of overriding this class? - - public UmbracoIndexesCreator( - ITypeFinder typeFinder, - IProfilingLogger profilingLogger, - ILoggerFactory loggerFactory, - ILocalizationService languageService, - IPublicAccessService publicAccessService, - IMemberService memberService, - IUmbracoIndexConfig umbracoIndexConfig, - IHostingEnvironment hostingEnvironment, - IRuntimeState runtimeState, - IOptions settings, - ILuceneDirectoryFactory directoryFactory) : base(typeFinder, hostingEnvironment, settings) - { - ProfilingLogger = profilingLogger ?? throw new System.ArgumentNullException(nameof(profilingLogger)); - LoggerFactory = loggerFactory; - LanguageService = languageService ?? throw new System.ArgumentNullException(nameof(languageService)); - PublicAccessService = publicAccessService ?? throw new System.ArgumentNullException(nameof(publicAccessService)); - MemberService = memberService ?? throw new System.ArgumentNullException(nameof(memberService)); - UmbracoIndexConfig = umbracoIndexConfig; - HostingEnvironment = hostingEnvironment ?? throw new System.ArgumentNullException(nameof(hostingEnvironment)); - RuntimeState = runtimeState ?? throw new System.ArgumentNullException(nameof(runtimeState)); - DirectoryFactory = directoryFactory; - } - - protected IProfilingLogger ProfilingLogger { get; } - protected ILoggerFactory LoggerFactory { get; } - protected IHostingEnvironment HostingEnvironment { get; } - protected IRuntimeState RuntimeState { get; } - protected ILuceneDirectoryFactory DirectoryFactory { get; } - protected ILocalizationService LanguageService { get; } - protected IPublicAccessService PublicAccessService { get; } - protected IMemberService MemberService { get; } - protected IUmbracoIndexConfig UmbracoIndexConfig { get; } - - /// - /// Creates the Umbraco indexes - /// - /// - public override IEnumerable Create() - { - return new[] - { - CreateInternalIndex(), - CreateExternalIndex(), - CreateMemberIndex() - }; - } - - private IIndex CreateInternalIndex() - { - var index = new UmbracoContentIndex( - Constants.UmbracoIndexes.InternalIndexName, - DirectoryFactory.CreateDirectory(Constants.UmbracoIndexes.InternalIndexPath), - new UmbracoFieldDefinitionCollection(), - new CultureInvariantWhitespaceAnalyzer(), - ProfilingLogger, - LoggerFactory.CreateLogger(), - LoggerFactory, - HostingEnvironment, - RuntimeState, - LanguageService, - UmbracoIndexConfig.GetContentValueSetValidator() - ); - return index; - } - - private IIndex CreateExternalIndex() - { - var index = new UmbracoContentIndex( - Constants.UmbracoIndexes.ExternalIndexName, - DirectoryFactory.CreateDirectory(Constants.UmbracoIndexes.ExternalIndexPath), - new UmbracoFieldDefinitionCollection(), - new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30), - ProfilingLogger, - LoggerFactory.CreateLogger(), - LoggerFactory, - HostingEnvironment, - RuntimeState, - LanguageService, - UmbracoIndexConfig.GetPublishedContentValueSetValidator()); - return index; - } - - private IIndex CreateMemberIndex() - { - var index = new UmbracoMemberIndex( - Constants.UmbracoIndexes.MembersIndexName, - new UmbracoFieldDefinitionCollection(), - DirectoryFactory.CreateDirectory(Constants.UmbracoIndexes.MembersIndexPath), - new CultureInvariantWhitespaceAnalyzer(), - ProfilingLogger, - LoggerFactory.CreateLogger(), - LoggerFactory, - HostingEnvironment, - RuntimeState, - UmbracoIndexConfig.GetMemberValueSetValidator() - ); - return index; - } - } -} diff --git a/src/Umbraco.Examine.Lucene/UmbracoLockFactory.cs b/src/Umbraco.Examine.Lucene/UmbracoLockFactory.cs new file mode 100644 index 0000000000..89f61c1e53 --- /dev/null +++ b/src/Umbraco.Examine.Lucene/UmbracoLockFactory.cs @@ -0,0 +1,15 @@ +// Copyright (c) Umbraco. +// See LICENSE for more details. + +using System.IO; +using Examine.Lucene.Directories; +using Lucene.Net.Store; + +namespace Umbraco.Cms.Infrastructure.Examine +{ + public class UmbracoLockFactory : ILockFactory + { + public LockFactory GetLockFactory(DirectoryInfo directory) + => new NoPrefixSimpleFsLockFactory(directory); + } +} diff --git a/src/Umbraco.Examine.Lucene/UmbracoMemberIndex.cs b/src/Umbraco.Examine.Lucene/UmbracoMemberIndex.cs index 3889209fdb..0792dd8a6f 100644 --- a/src/Umbraco.Examine.Lucene/UmbracoMemberIndex.cs +++ b/src/Umbraco.Examine.Lucene/UmbracoMemberIndex.cs @@ -1,13 +1,11 @@ -// Copyright (c) Umbraco. +// Copyright (c) Umbraco. // See LICENSE for more details. -using Examine; -using Lucene.Net.Analysis; +using Examine.Lucene; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Umbraco.Cms.Core.Hosting; -using Umbraco.Cms.Core.Logging; using Umbraco.Cms.Core.Services; -using Directory = Lucene.Net.Store.Directory; namespace Umbraco.Cms.Infrastructure.Examine { @@ -16,31 +14,14 @@ namespace Umbraco.Cms.Infrastructure.Examine /// public class UmbracoMemberIndex : UmbracoExamineIndex, IUmbracoMemberIndex { - /// - /// Constructor to allow for creating an indexer at runtime - /// - /// - /// - /// - /// - /// - /// - /// - /// public UmbracoMemberIndex( - string name, - FieldDefinitionCollection fieldDefinitions, - Directory luceneDirectory, - Analyzer analyzer, - IProfilingLogger profilingLogger, - ILogger logger, ILoggerFactory loggerFactory, + string name, + IOptionsSnapshot indexOptions, IHostingEnvironment hostingEnvironment, - IRuntimeState runtimeState, - IValueSetValidator validator = null) : - base(name, luceneDirectory, fieldDefinitions, analyzer, profilingLogger, logger, loggerFactory, hostingEnvironment, runtimeState, validator) + IRuntimeState runtimeState) + : base(loggerFactory, name, indexOptions, hostingEnvironment, runtimeState) { } - } } diff --git a/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.CoreServices.cs b/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.CoreServices.cs index 6eb08bd4d5..14457e9687 100644 --- a/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.CoreServices.cs +++ b/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.CoreServices.cs @@ -154,8 +154,6 @@ namespace Umbraco.Cms.Infrastructure.DependencyInjection builder.Services.AddUnique(); - // Register noop versions for examine to be overridden by examine - builder.Services.AddUnique(); builder.Services.AddUnique(); builder.Services.AddUnique(); @@ -170,7 +168,6 @@ namespace Umbraco.Cms.Infrastructure.DependencyInjection // Services required to run background jobs (with out the handler) builder.Services.AddUnique(); - builder.Services.AddUnique(); return builder; } diff --git a/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.DistributedCache.cs b/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.DistributedCache.cs index 0757f2c725..05dba2cc0f 100644 --- a/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.DistributedCache.cs +++ b/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.DistributedCache.cs @@ -27,7 +27,8 @@ namespace Umbraco.Cms.Infrastructure.DependencyInjection /// public static IUmbracoBuilder AddDistributedCache(this IUmbracoBuilder builder) { - builder.SetDatabaseServerMessengerCallbacks(GetCallbacks); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); builder.SetServerMessenger(); builder.AddNotificationHandler(); builder.AddNotificationHandler(); @@ -59,24 +60,6 @@ namespace Umbraco.Cms.Infrastructure.DependencyInjection public static void SetServerRegistrar(this IUmbracoBuilder builder, IServerRoleAccessor registrar) => builder.Services.AddUnique(registrar); - /// - /// Sets the database server messenger options. - /// - /// The builder. - /// A function creating the options. - /// Use DatabaseServerRegistrarAndMessengerComposer.GetDefaultOptions to get the options that Umbraco would use by default. - public static void SetDatabaseServerMessengerCallbacks(this IUmbracoBuilder builder, Func factory) - => builder.Services.AddUnique(factory); - - /// - /// Sets the database server messenger options. - /// - /// The builder. - /// Options. - /// Use DatabaseServerRegistrarAndMessengerComposer.GetDefaultOptions to get the options that Umbraco would use by default. - public static void SetDatabaseServerMessengerOptions(this IUmbracoBuilder builder, DatabaseServerMessengerCallbacks options) - => builder.Services.AddUnique(options); - /// /// Sets the server messenger. /// @@ -101,36 +84,5 @@ namespace Umbraco.Cms.Infrastructure.DependencyInjection /// A server messenger. public static void SetServerMessenger(this IUmbracoBuilder builder, IServerMessenger registrar) => builder.Services.AddUnique(registrar); - - private static DatabaseServerMessengerCallbacks GetCallbacks(IServiceProvider factory) => new DatabaseServerMessengerCallbacks - { - // These callbacks will be executed if the server has not been synced - // (i.e. it is a new server or the lastsynced.txt file has been removed) - InitializingCallbacks = new Action[] - { - // rebuild the xml cache file if the server is not synced - () => - { - IPublishedSnapshotService publishedSnapshotService = factory.GetRequiredService(); - - // rebuild the published snapshot caches entirely, if the server is not synced - // this is equivalent to DistributedCache RefreshAll... but local only - // (we really should have a way to reuse RefreshAll... locally) - // note: refresh all content & media caches does refresh content types too - publishedSnapshotService.Notify(new[] { new DomainCacheRefresher.JsonPayload(0, DomainChangeTypes.RefreshAll) }); - publishedSnapshotService.Notify(new[] { new ContentCacheRefresher.JsonPayload(0, null, TreeChangeTypes.RefreshAll) }, out _, out _); - publishedSnapshotService.Notify(new[] { new MediaCacheRefresher.JsonPayload(0, null, TreeChangeTypes.RefreshAll) }, out _); - }, - - // rebuild indexes if the server is not synced - // NOTE: This will rebuild ALL indexes including the members, if developers want to target specific - // indexes then they can adjust this logic themselves. - () => - { - var indexRebuilder = factory.GetRequiredService(); - indexRebuilder.RebuildIndexes(false, TimeSpan.FromSeconds(5)); - } - } - }; } } diff --git a/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.Examine.cs b/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.Examine.cs index 103d7a198d..d061a4372c 100644 --- a/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.Examine.cs +++ b/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.Examine.cs @@ -28,7 +28,8 @@ namespace Umbraco.Cms.Infrastructure.DependencyInjection builder.Services.AddSingleton(); builder.Services.AddSingleton(); - builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); builder.Services.AddUnique(); builder.Services.AddUnique(); builder.Services.AddUnique(factory => @@ -49,14 +50,15 @@ namespace Umbraco.Cms.Infrastructure.DependencyInjection false)); builder.Services.AddUnique, MediaValueSetBuilder>(); builder.Services.AddUnique, MemberValueSetBuilder>(); - builder.Services.AddUnique(); + builder.Services.AddUnique(); - builder.AddNotificationHandler(); - builder.AddNotificationHandler(); - builder.AddNotificationHandler(); - builder.AddNotificationHandler(); - builder.AddNotificationHandler(); - builder.AddNotificationHandler(); + builder.AddNotificationHandler(); + builder.AddNotificationHandler(); + builder.AddNotificationHandler(); + builder.AddNotificationHandler(); + builder.AddNotificationHandler(); + + builder.AddNotificationHandler(); return builder; } diff --git a/src/Umbraco.Infrastructure/Examine/ContentIndexPopulator.cs b/src/Umbraco.Infrastructure/Examine/ContentIndexPopulator.cs index d9fd10f1d7..bd205e2009 100644 --- a/src/Umbraco.Infrastructure/Examine/ContentIndexPopulator.cs +++ b/src/Umbraco.Infrastructure/Examine/ContentIndexPopulator.cs @@ -1,7 +1,8 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using Examine; +using Microsoft.Extensions.Logging; using Umbraco.Cms.Core; using Umbraco.Cms.Core.Models; using Umbraco.Cms.Core.Persistence.Querying; @@ -27,6 +28,7 @@ namespace Umbraco.Cms.Infrastructure.Examine private readonly bool _publishedValuesOnly; private readonly int? _parentId; + private readonly ILogger _logger; /// /// Default constructor to lookup all content data @@ -34,20 +36,30 @@ namespace Umbraco.Cms.Infrastructure.Examine /// /// /// - public ContentIndexPopulator(IContentService contentService, IUmbracoDatabaseFactory umbracoDatabaseFactory, IContentValueSetBuilder contentValueSetBuilder) - : this(false, null, contentService, umbracoDatabaseFactory, contentValueSetBuilder) + public ContentIndexPopulator( + ILogger logger, + IContentService contentService, + IUmbracoDatabaseFactory umbracoDatabaseFactory, + IContentValueSetBuilder contentValueSetBuilder) + : this(logger, false, null, contentService, umbracoDatabaseFactory, contentValueSetBuilder) { } /// /// Optional constructor allowing specifying custom query parameters /// - public ContentIndexPopulator(bool publishedValuesOnly, int? parentId, IContentService contentService, IUmbracoDatabaseFactory umbracoDatabaseFactory, IValueSetBuilder contentValueSetBuilder) + public ContentIndexPopulator( + ILogger logger, + bool publishedValuesOnly, + int? parentId, + IContentService contentService, + IUmbracoDatabaseFactory umbracoDatabaseFactory, + IValueSetBuilder contentValueSetBuilder) { - if (umbracoDatabaseFactory == null) throw new ArgumentNullException(nameof(umbracoDatabaseFactory)); _contentService = contentService ?? throw new ArgumentNullException(nameof(contentService)); - _umbracoDatabaseFactory = umbracoDatabaseFactory; + _umbracoDatabaseFactory = umbracoDatabaseFactory ?? throw new ArgumentNullException(nameof(umbracoDatabaseFactory)); _contentValueSetBuilder = contentValueSetBuilder ?? throw new ArgumentNullException(nameof(contentValueSetBuilder)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _publishedValuesOnly = publishedValuesOnly; _parentId = parentId; } @@ -60,7 +72,11 @@ namespace Umbraco.Cms.Infrastructure.Examine protected override void PopulateIndexes(IReadOnlyList indexes) { - if (indexes.Count == 0) return; + if (indexes.Count == 0) + { + _logger.LogDebug($"{nameof(PopulateIndexes)} called with no indexes to populate. Typically means no index is registered with this populator."); + return; + } const int pageSize = 10000; var pageIndex = 0; @@ -144,9 +160,10 @@ namespace Umbraco.Cms.Infrastructure.Examine var valueSets = _contentValueSetBuilder.GetValueSets(indexableContent.ToArray()).ToList(); - // ReSharper disable once PossibleMultipleEnumeration - foreach (var index in indexes) + foreach (IIndex index in indexes) + { index.IndexItems(valueSets); + } } pageIndex++; diff --git a/src/Umbraco.Infrastructure/Examine/ContentValueSetValidator.cs b/src/Umbraco.Infrastructure/Examine/ContentValueSetValidator.cs index 463e8dee26..39d260a24d 100644 --- a/src/Umbraco.Infrastructure/Examine/ContentValueSetValidator.cs +++ b/src/Umbraco.Infrastructure/Examine/ContentValueSetValidator.cs @@ -1,4 +1,4 @@ -using System.Collections.Generic; +using System.Collections.Generic; using System.Linq; using Examine; using Umbraco.Cms.Core.Scoping; @@ -106,10 +106,14 @@ namespace Umbraco.Cms.Infrastructure.Examine if (valueSet.Category == IndexTypes.Content && PublishedValuesOnly) { if (!valueSet.Values.TryGetValue(UmbracoExamineFieldNames.PublishedFieldName, out var published)) + { return ValueSetValidationResult.Failed; + } if (!published[0].Equals("y")) + { return ValueSetValidationResult.Failed; + } //deal with variants, if there are unpublished variants than we need to remove them from the value set if (valueSet.Values.TryGetValue(UmbracoExamineFieldNames.VariesByCultureFieldName, out var variesByCulture) diff --git a/src/Umbraco.Infrastructure/Search/ExamineIndexModel.cs b/src/Umbraco.Infrastructure/Examine/ExamineIndexModel.cs similarity index 88% rename from src/Umbraco.Infrastructure/Search/ExamineIndexModel.cs rename to src/Umbraco.Infrastructure/Examine/ExamineIndexModel.cs index d14cef8ccf..ff9f499217 100644 --- a/src/Umbraco.Infrastructure/Search/ExamineIndexModel.cs +++ b/src/Umbraco.Infrastructure/Examine/ExamineIndexModel.cs @@ -1,7 +1,7 @@ -using System.Collections.Generic; +using System.Collections.Generic; using System.Runtime.Serialization; -namespace Umbraco.Cms.Infrastructure.Search +namespace Umbraco.Cms.Infrastructure.Examine { [DataContract(Name = "indexer", Namespace = "")] public class ExamineIndexModel diff --git a/src/Umbraco.Infrastructure/Examine/ExamineIndexRebuilder.cs b/src/Umbraco.Infrastructure/Examine/ExamineIndexRebuilder.cs new file mode 100644 index 0000000000..d7719cfd40 --- /dev/null +++ b/src/Umbraco.Infrastructure/Examine/ExamineIndexRebuilder.cs @@ -0,0 +1,207 @@ +// Copyright (c) Umbraco. +// See LICENSE for more details. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Examine; +using Microsoft.Extensions.Logging; +using Umbraco.Cms.Core; +using Umbraco.Cms.Core.Runtime; +using Umbraco.Cms.Core.Services; +using Umbraco.Cms.Infrastructure.HostedServices; + +namespace Umbraco.Cms.Infrastructure.Examine +{ + public class ExamineIndexRebuilder : IIndexRebuilder + { + private readonly IBackgroundTaskQueue _backgroundTaskQueue; + private readonly IMainDom _mainDom; + private readonly IRuntimeState _runtimeState; + private readonly ILogger _logger; + private readonly IExamineManager _examineManager; + private readonly IEnumerable _populators; + private readonly object _rebuildLocker = new(); + + /// + /// Initializes a new instance of the class. + /// + public ExamineIndexRebuilder( + IMainDom mainDom, + IRuntimeState runtimeState, + ILogger logger, + IExamineManager examineManager, + IEnumerable populators, + IBackgroundTaskQueue backgroundTaskQueue) + { + _mainDom = mainDom; + _runtimeState = runtimeState; + _logger = logger; + _examineManager = examineManager; + _populators = populators; + _backgroundTaskQueue = backgroundTaskQueue; + } + + public bool CanRebuild(string indexName) + { + if (!_examineManager.TryGetIndex(indexName, out IIndex index)) + { + throw new InvalidOperationException("No index found by name " + indexName); + } + + return _populators.Any(x => x.IsRegistered(index)); + } + + public virtual void RebuildIndex(string indexName, TimeSpan? delay = null, bool useBackgroundThread = true) + { + if (delay == null) + { + delay = TimeSpan.Zero; + } + + if (!CanRun()) + { + return; + } + + if (useBackgroundThread) + { + _logger.LogInformation($"Starting async background thread for rebuilding index {indexName}."); + + _backgroundTaskQueue.QueueBackgroundWorkItem( + cancellationToken => Task.Run(() => RebuildIndex(indexName, delay.Value, cancellationToken))); + } + else + { + RebuildIndex(indexName, delay.Value, CancellationToken.None); + } + } + + public virtual void RebuildIndexes(bool onlyEmptyIndexes, TimeSpan? delay = null, bool useBackgroundThread = true) + { + if (delay == null) + { + delay = TimeSpan.Zero; + } + + if (!CanRun()) + { + return; + } + + if (useBackgroundThread) + { + _logger.LogInformation($"Starting async background thread for {nameof(RebuildIndexes)}."); + + _backgroundTaskQueue.QueueBackgroundWorkItem( + cancellationToken => Task.Run(() => RebuildIndexes(onlyEmptyIndexes, delay.Value, cancellationToken))); + } + else + { + RebuildIndexes(onlyEmptyIndexes, delay.Value, CancellationToken.None); + } + } + + private bool CanRun() => _mainDom.IsMainDom && _runtimeState.Level >= RuntimeLevel.Run; + + private void RebuildIndex(string indexName, TimeSpan delay, CancellationToken cancellationToken) + { + if (delay > TimeSpan.Zero) + { + Thread.Sleep(delay); + } + + try + { + if (!Monitor.TryEnter(_rebuildLocker)) + { + _logger.LogWarning("Call was made to RebuildIndexes but the task runner for rebuilding is already running"); + } + else + { + if (!_examineManager.TryGetIndex(indexName, out IIndex index)) + { + throw new InvalidOperationException($"No index found with name {indexName}"); + } + + index.CreateIndex(); // clear the index + foreach (IIndexPopulator populator in _populators) + { + if (cancellationToken.IsCancellationRequested) + { + return; + } + + populator.Populate(index); + } + } + } + finally + { + if (Monitor.IsEntered(_rebuildLocker)) + { + Monitor.Exit(_rebuildLocker); + } + } + } + + private void RebuildIndexes(bool onlyEmptyIndexes, TimeSpan delay, CancellationToken cancellationToken) + { + if (delay > TimeSpan.Zero) + { + Thread.Sleep(delay); + } + + try + { + if (!Monitor.TryEnter(_rebuildLocker)) + { + _logger.LogWarning($"Call was made to {nameof(RebuildIndexes)} but the task runner for rebuilding is already running"); + } + else + { + IIndex[] indexes = (onlyEmptyIndexes + ? _examineManager.Indexes.Where(x => !x.IndexExists()) + : _examineManager.Indexes).ToArray(); + + if (indexes.Length == 0) + { + return; + } + + foreach (IIndex index in indexes) + { + index.CreateIndex(); // clear the index + } + + // run each populator over the indexes + foreach (IIndexPopulator populator in _populators) + { + if (cancellationToken.IsCancellationRequested) + { + return; + } + + try + { + populator.Populate(indexes); + } + catch (Exception e) + { + _logger.LogError(e, "Index populating failed for populator {Populator}", populator.GetType()); + } + } + } + } + finally + { + if (Monitor.IsEntered(_rebuildLocker)) + { + Monitor.Exit(_rebuildLocker); + } + } + } + } +} diff --git a/src/Umbraco.Infrastructure/Search/ExamineSearcherModel.cs b/src/Umbraco.Infrastructure/Examine/ExamineSearcherModel.cs similarity index 74% rename from src/Umbraco.Infrastructure/Search/ExamineSearcherModel.cs rename to src/Umbraco.Infrastructure/Examine/ExamineSearcherModel.cs index 8e6ea30c0c..c4b602e430 100644 --- a/src/Umbraco.Infrastructure/Search/ExamineSearcherModel.cs +++ b/src/Umbraco.Infrastructure/Examine/ExamineSearcherModel.cs @@ -1,6 +1,6 @@ -using System.Runtime.Serialization; +using System.Runtime.Serialization; -namespace Umbraco.Cms.Infrastructure.Search +namespace Umbraco.Cms.Infrastructure.Examine { [DataContract(Name = "searcher", Namespace = "")] public class ExamineSearcherModel diff --git a/src/Umbraco.Infrastructure/Examine/ExamineUmbracoIndexingHandler.cs b/src/Umbraco.Infrastructure/Examine/ExamineUmbracoIndexingHandler.cs new file mode 100644 index 0000000000..6c6d209e5a --- /dev/null +++ b/src/Umbraco.Infrastructure/Examine/ExamineUmbracoIndexingHandler.cs @@ -0,0 +1,394 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Threading.Tasks; +using Examine; +using Examine.Search; +using Microsoft.Extensions.Logging; +using Umbraco.Cms.Core.Logging; +using Umbraco.Cms.Core.Models; +using Umbraco.Cms.Core.Runtime; +using Umbraco.Cms.Core.Scoping; +using Umbraco.Cms.Infrastructure.HostedServices; +using Umbraco.Cms.Infrastructure.Search; +using Umbraco.Extensions; + +namespace Umbraco.Cms.Infrastructure.Examine +{ + /// + /// Indexing handler for Examine indexes + /// + internal class ExamineUmbracoIndexingHandler : IUmbracoIndexingHandler + { + // the default enlist priority is 100 + // enlist with a lower priority to ensure that anything "default" runs after us + // but greater that SafeXmlReaderWriter priority which is 60 + private const int EnlistPriority = 80; + private readonly IMainDom _mainDom; + private readonly ILogger _logger; + private readonly IProfilingLogger _profilingLogger; + private readonly IScopeProvider _scopeProvider; + private readonly IExamineManager _examineManager; + private readonly IBackgroundTaskQueue _backgroundTaskQueue; + private readonly IContentValueSetBuilder _contentValueSetBuilder; + private readonly IPublishedContentValueSetBuilder _publishedContentValueSetBuilder; + private readonly IValueSetBuilder _mediaValueSetBuilder; + private readonly IValueSetBuilder _memberValueSetBuilder; + private readonly Lazy _enabled; + + public ExamineUmbracoIndexingHandler( + IMainDom mainDom, + ILogger logger, + IProfilingLogger profilingLogger, + IScopeProvider scopeProvider, + IExamineManager examineManager, + IBackgroundTaskQueue backgroundTaskQueue, + IContentValueSetBuilder contentValueSetBuilder, + IPublishedContentValueSetBuilder publishedContentValueSetBuilder, + IValueSetBuilder mediaValueSetBuilder, + IValueSetBuilder memberValueSetBuilder) + { + _mainDom = mainDom; + _logger = logger; + _profilingLogger = profilingLogger; + _scopeProvider = scopeProvider; + _examineManager = examineManager; + _backgroundTaskQueue = backgroundTaskQueue; + _contentValueSetBuilder = contentValueSetBuilder; + _publishedContentValueSetBuilder = publishedContentValueSetBuilder; + _mediaValueSetBuilder = mediaValueSetBuilder; + _memberValueSetBuilder = memberValueSetBuilder; + _enabled = new Lazy(IsEnabled); + } + + /// + /// Used to lazily check if Examine Index handling is enabled + /// + /// + private bool IsEnabled() + { + //let's deal with shutting down Examine with MainDom + var examineShutdownRegistered = _mainDom.Register(release: () => + { + using (_profilingLogger.TraceDuration("Examine shutting down")) + { + _examineManager.Dispose(); + } + }); + + if (!examineShutdownRegistered) + { + _logger.LogInformation("Examine shutdown not registered, this AppDomain is not the MainDom, Examine will be disabled"); + + //if we could not register the shutdown examine ourselves, it means we are not maindom! in this case all of examine should be disabled! + Suspendable.ExamineEvents.SuspendIndexers(_logger); + return false; //exit, do not continue + } + + _logger.LogDebug("Examine shutdown registered with MainDom"); + + var registeredIndexers = _examineManager.Indexes.OfType().Count(x => x.EnableDefaultEventHandler); + + _logger.LogInformation("Adding examine event handlers for {RegisteredIndexers} index providers.", registeredIndexers); + + // don't bind event handlers if we're not suppose to listen + if (registeredIndexers == 0) + { + return false; + } + + return true; + } + + /// + public bool Enabled => _enabled.Value; + + /// + public void DeleteIndexForEntity(int entityId, bool keepIfUnpublished) + { + var actions = DeferedActions.Get(_scopeProvider); + if (actions != null) + { + actions.Add(new DeferedDeleteIndex(this, entityId, keepIfUnpublished)); + } + else + { + DeferedDeleteIndex.Execute(this, entityId, keepIfUnpublished); + } + } + + /// + public void ReIndexForContent(IContent sender, bool isPublished) + { + var actions = DeferedActions.Get(_scopeProvider); + if (actions != null) + { + actions.Add(new DeferedReIndexForContent(_backgroundTaskQueue, this, sender, isPublished)); + } + else + { + DeferedReIndexForContent.Execute(_backgroundTaskQueue, this, sender, isPublished); + } + } + + /// + public void ReIndexForMedia(IMedia sender, bool isPublished) + { + var actions = DeferedActions.Get(_scopeProvider); + if (actions != null) + { + actions.Add(new DeferedReIndexForMedia(_backgroundTaskQueue, this, sender, isPublished)); + } + else + { + DeferedReIndexForMedia.Execute(_backgroundTaskQueue, this, sender, isPublished); + } + } + + /// + public void ReIndexForMember(IMember member) + { + var actions = DeferedActions.Get(_scopeProvider); + if (actions != null) + { + actions.Add(new DeferedReIndexForMember(_backgroundTaskQueue, this, member)); + } + else + { + DeferedReIndexForMember.Execute(_backgroundTaskQueue, this, member); + } + } + + /// + public void DeleteDocumentsForContentTypes(IReadOnlyCollection removedContentTypes) + { + const int pageSize = 500; + + //Delete all content of this content/media/member type that is in any content indexer by looking up matched examine docs + foreach (var id in removedContentTypes) + { + foreach (var index in _examineManager.Indexes.OfType()) + { + var page = 0; + var total = long.MaxValue; + while (page * pageSize < total) + { + //paging with examine, see https://shazwazza.com/post/paging-with-examine/ + var results = index.Searcher + .CreateQuery() + .Field("nodeType", id.ToInvariantString()) + .Execute(QueryOptions.SkipTake(page * pageSize, pageSize)); + total = results.TotalItemCount; + var paged = results.Skip(page * pageSize); + + foreach (ISearchResult item in paged) + { + if (int.TryParse(item.Id, out int contentId)) + { + DeleteIndexForEntity(contentId, false); + } + } + + page++; + } + } + } + } + + #region Deferred Actions + private class DeferedActions + { + private readonly List _actions = new List(); + + public static DeferedActions Get(IScopeProvider scopeProvider) + { + IScopeContext scopeContext = scopeProvider.Context; + + return scopeContext?.Enlist("examineEvents", + () => new DeferedActions(), // creator + (completed, actions) => // action + { + if (completed) + { + actions.Execute(); + } + }, EnlistPriority); + } + + public void Add(DeferedAction action) => _actions.Add(action); + + private void Execute() + { + foreach (DeferedAction action in _actions) + { + action.Execute(); + } + } + } + + /// + /// An action that will execute at the end of the Scope being completed + /// + private abstract class DeferedAction + { + public virtual void Execute() + { } + } + + /// + /// Re-indexes an item on a background thread + /// + private class DeferedReIndexForContent : DeferedAction + { + private readonly IBackgroundTaskQueue _backgroundTaskQueue; + private readonly ExamineUmbracoIndexingHandler _ExamineUmbracoIndexingHandler; + private readonly IContent _content; + private readonly bool _isPublished; + + public DeferedReIndexForContent(IBackgroundTaskQueue backgroundTaskQueue, ExamineUmbracoIndexingHandler ExamineUmbracoIndexingHandler, IContent content, bool isPublished) + { + _backgroundTaskQueue = backgroundTaskQueue; + _ExamineUmbracoIndexingHandler = ExamineUmbracoIndexingHandler; + _content = content; + _isPublished = isPublished; + } + + public override void Execute() => Execute(_backgroundTaskQueue, _ExamineUmbracoIndexingHandler, _content, _isPublished); + + public static void Execute(IBackgroundTaskQueue backgroundTaskQueue, ExamineUmbracoIndexingHandler ExamineUmbracoIndexingHandler, IContent content, bool isPublished) + => backgroundTaskQueue.QueueBackgroundWorkItem(cancellationToken => + { + using IScope scope = ExamineUmbracoIndexingHandler._scopeProvider.CreateScope(autoComplete: true); + + // for content we have a different builder for published vs unpublished + // we don't want to build more value sets than is needed so we'll lazily build 2 one for published one for non-published + var builders = new Dictionary>> + { + [true] = new Lazy>(() => ExamineUmbracoIndexingHandler._publishedContentValueSetBuilder.GetValueSets(content).ToList()), + [false] = new Lazy>(() => ExamineUmbracoIndexingHandler._contentValueSetBuilder.GetValueSets(content).ToList()) + }; + + foreach (IUmbracoIndex index in ExamineUmbracoIndexingHandler._examineManager.Indexes.OfType() + //filter the indexers + .Where(x => isPublished || !x.PublishedValuesOnly) + .Where(x => x.EnableDefaultEventHandler)) + { + if (cancellationToken.IsCancellationRequested) + { + return Task.CompletedTask; + } + + List valueSet = builders[index.PublishedValuesOnly].Value; + index.IndexItems(valueSet); + } + + return Task.CompletedTask; + }); + } + + /// + /// Re-indexes an item on a background thread + /// + private class DeferedReIndexForMedia : DeferedAction + { + private readonly IBackgroundTaskQueue _backgroundTaskQueue; + private readonly ExamineUmbracoIndexingHandler _ExamineUmbracoIndexingHandler; + private readonly IMedia _media; + private readonly bool _isPublished; + + public DeferedReIndexForMedia(IBackgroundTaskQueue backgroundTaskQueue, ExamineUmbracoIndexingHandler ExamineUmbracoIndexingHandler, IMedia media, bool isPublished) + { + _backgroundTaskQueue = backgroundTaskQueue; + _ExamineUmbracoIndexingHandler = ExamineUmbracoIndexingHandler; + _media = media; + _isPublished = isPublished; + } + + public override void Execute() => Execute(_backgroundTaskQueue, _ExamineUmbracoIndexingHandler, _media, _isPublished); + + public static void Execute(IBackgroundTaskQueue backgroundTaskQueue, ExamineUmbracoIndexingHandler ExamineUmbracoIndexingHandler, IMedia media, bool isPublished) => + // perform the ValueSet lookup on a background thread + backgroundTaskQueue.QueueBackgroundWorkItem(cancellationToken => + { + using IScope scope = ExamineUmbracoIndexingHandler._scopeProvider.CreateScope(autoComplete: true); + + var valueSet = ExamineUmbracoIndexingHandler._mediaValueSetBuilder.GetValueSets(media).ToList(); + + foreach (IUmbracoIndex index in ExamineUmbracoIndexingHandler._examineManager.Indexes.OfType() + //filter the indexers + .Where(x => isPublished || !x.PublishedValuesOnly) + .Where(x => x.EnableDefaultEventHandler)) + { + index.IndexItems(valueSet); + } + + return Task.CompletedTask; + }); + } + + /// + /// Re-indexes an item on a background thread + /// + private class DeferedReIndexForMember : DeferedAction + { + private readonly ExamineUmbracoIndexingHandler _ExamineUmbracoIndexingHandler; + private readonly IMember _member; + private readonly IBackgroundTaskQueue _backgroundTaskQueue; + + public DeferedReIndexForMember(IBackgroundTaskQueue backgroundTaskQueue, ExamineUmbracoIndexingHandler ExamineUmbracoIndexingHandler, IMember member) + { + _ExamineUmbracoIndexingHandler = ExamineUmbracoIndexingHandler; + _member = member; + _backgroundTaskQueue = backgroundTaskQueue; + } + + public override void Execute() => Execute(_backgroundTaskQueue, _ExamineUmbracoIndexingHandler, _member); + + public static void Execute(IBackgroundTaskQueue backgroundTaskQueue, ExamineUmbracoIndexingHandler ExamineUmbracoIndexingHandler, IMember member) => + // perform the ValueSet lookup on a background thread + backgroundTaskQueue.QueueBackgroundWorkItem(cancellationToken => + { + using IScope scope = ExamineUmbracoIndexingHandler._scopeProvider.CreateScope(autoComplete: true); + + var valueSet = ExamineUmbracoIndexingHandler._memberValueSetBuilder.GetValueSets(member).ToList(); + foreach (IUmbracoIndex index in ExamineUmbracoIndexingHandler._examineManager.Indexes.OfType() + //filter the indexers + .Where(x => x.EnableDefaultEventHandler)) + { + index.IndexItems(valueSet); + } + + return Task.CompletedTask; + }); + } + + private class DeferedDeleteIndex : DeferedAction + { + private readonly ExamineUmbracoIndexingHandler _ExamineUmbracoIndexingHandler; + private readonly int _id; + private readonly bool _keepIfUnpublished; + + public DeferedDeleteIndex(ExamineUmbracoIndexingHandler ExamineUmbracoIndexingHandler, int id, bool keepIfUnpublished) + { + _ExamineUmbracoIndexingHandler = ExamineUmbracoIndexingHandler; + _id = id; + _keepIfUnpublished = keepIfUnpublished; + } + + public override void Execute() => Execute(_ExamineUmbracoIndexingHandler, _id, _keepIfUnpublished); + + public static void Execute(ExamineUmbracoIndexingHandler ExamineUmbracoIndexingHandler, int id, bool keepIfUnpublished) + { + var strId = id.ToString(CultureInfo.InvariantCulture); + foreach (var index in ExamineUmbracoIndexingHandler._examineManager.Indexes.OfType() + .Where(x => x.PublishedValuesOnly || !keepIfUnpublished) + .Where(x => x.EnableDefaultEventHandler)) + { + index.DeleteFromIndex(strId); + } + } + } + #endregion + } +} diff --git a/src/Umbraco.Infrastructure/Examine/GenericIndexDiagnostics.cs b/src/Umbraco.Infrastructure/Examine/GenericIndexDiagnostics.cs index 8c05926483..2ff01c51dc 100644 --- a/src/Umbraco.Infrastructure/Examine/GenericIndexDiagnostics.cs +++ b/src/Umbraco.Infrastructure/Examine/GenericIndexDiagnostics.cs @@ -1,6 +1,7 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; +using System.Threading.Tasks; using Examine; using Umbraco.Cms.Core; using Umbraco.Cms.Core.Composing; @@ -15,12 +16,9 @@ namespace Umbraco.Cms.Infrastructure.Examine public class GenericIndexDiagnostics : IIndexDiagnostics { private readonly IIndex _index; - private static readonly string[] IgnoreProperties = { "Description" }; + private static readonly string[] s_ignoreProperties = { "Description" }; - public GenericIndexDiagnostics(IIndex index) - { - _index = index; - } + public GenericIndexDiagnostics(IIndex index) => _index = index; public int DocumentCount => -1; //unknown @@ -33,8 +31,7 @@ namespace Umbraco.Cms.Infrastructure.Examine try { - var searcher = _index.GetSearcher(); - var result = searcher.Search("test"); + var result = _index.Searcher.Search("test"); return Attempt.Succeed(); //if we can search we'll assume it's healthy } catch (Exception e) @@ -43,6 +40,10 @@ namespace Umbraco.Cms.Infrastructure.Examine } } + public long GetDocumentCount() => -1L; + + public IEnumerable GetFieldNames() => Enumerable.Empty(); + public IReadOnlyDictionary Metadata { get @@ -50,7 +51,7 @@ namespace Umbraco.Cms.Infrastructure.Examine var result = new Dictionary(); var props = TypeHelper.CachedDiscoverableProperties(_index.GetType(), mustWrite: false) - .Where(x => IgnoreProperties.InvariantContains(x.Name) == false) + .Where(x => s_ignoreProperties.InvariantContains(x.Name) == false) .OrderBy(x => x.Name); foreach (var p in props) diff --git a/src/Umbraco.Infrastructure/Examine/IIndexCreator.cs b/src/Umbraco.Infrastructure/Examine/IIndexCreator.cs deleted file mode 100644 index aadaa00f46..0000000000 --- a/src/Umbraco.Infrastructure/Examine/IIndexCreator.cs +++ /dev/null @@ -1,13 +0,0 @@ -using System.Collections.Generic; -using Examine; - -namespace Umbraco.Cms.Infrastructure.Examine -{ - /// - /// Creates 's - /// - public interface IIndexCreator - { - IEnumerable Create(); - } -} diff --git a/src/Umbraco.Infrastructure/Examine/IIndexDiagnostics.cs b/src/Umbraco.Infrastructure/Examine/IIndexDiagnostics.cs index a4e1c0ca4f..716b7731eb 100644 --- a/src/Umbraco.Infrastructure/Examine/IIndexDiagnostics.cs +++ b/src/Umbraco.Infrastructure/Examine/IIndexDiagnostics.cs @@ -1,4 +1,6 @@ -using System.Collections.Generic; +using System.Collections.Generic; +using System.Threading.Tasks; +using Examine; using Umbraco.Cms.Core; namespace Umbraco.Cms.Infrastructure.Examine @@ -7,18 +9,8 @@ namespace Umbraco.Cms.Infrastructure.Examine /// /// Exposes diagnostic information about an index /// - public interface IIndexDiagnostics + public interface IIndexDiagnostics : IIndexStats { - /// - /// The number of documents in the index - /// - int DocumentCount { get; } - - /// - /// The number of fields in the index - /// - int FieldCount { get; } - /// /// If the index can be open/read /// diff --git a/src/Umbraco.Infrastructure/Examine/IIndexRebuilder.cs b/src/Umbraco.Infrastructure/Examine/IIndexRebuilder.cs new file mode 100644 index 0000000000..127a20d685 --- /dev/null +++ b/src/Umbraco.Infrastructure/Examine/IIndexRebuilder.cs @@ -0,0 +1,14 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Examine; + +namespace Umbraco.Cms.Infrastructure.Examine +{ + public interface IIndexRebuilder + { + bool CanRebuild(string indexName); + void RebuildIndex(string indexName, TimeSpan? delay = null, bool useBackgroundThread = true); + void RebuildIndexes(bool onlyEmptyIndexes, TimeSpan? delay = null, bool useBackgroundThread = true); + } +} diff --git a/src/Umbraco.Infrastructure/Examine/IUmbracoIndex.cs b/src/Umbraco.Infrastructure/Examine/IUmbracoIndex.cs index 8dfdf6d812..f2221e5c91 100644 --- a/src/Umbraco.Infrastructure/Examine/IUmbracoIndex.cs +++ b/src/Umbraco.Infrastructure/Examine/IUmbracoIndex.cs @@ -1,4 +1,4 @@ -using System.Collections.Generic; +using System.Collections.Generic; using Examine; namespace Umbraco.Cms.Infrastructure.Examine @@ -6,7 +6,7 @@ namespace Umbraco.Cms.Infrastructure.Examine /// /// A Marker interface for defining an Umbraco indexer /// - public interface IUmbracoIndex : IIndex + public interface IUmbracoIndex : IIndex, IIndexStats { /// /// When set to true Umbraco will keep the index in sync with Umbraco data automatically @@ -22,11 +22,5 @@ namespace Umbraco.Cms.Infrastructure.Examine /// * non-published Variants /// bool PublishedValuesOnly { get; } - - /// - /// Returns a list of all indexed fields - /// - /// - IEnumerable GetFields(); } } diff --git a/src/Umbraco.Infrastructure/Examine/IUmbracoIndexesCreator.cs b/src/Umbraco.Infrastructure/Examine/IUmbracoIndexesCreator.cs deleted file mode 100644 index df61901dba..0000000000 --- a/src/Umbraco.Infrastructure/Examine/IUmbracoIndexesCreator.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace Umbraco.Cms.Infrastructure.Examine -{ - /// - /// - /// Used to create the Umbraco indexes - /// - public interface IUmbracoIndexesCreator : IIndexCreator - { - } -} diff --git a/src/Umbraco.Infrastructure/Examine/IndexDiagnosticsFactory.cs b/src/Umbraco.Infrastructure/Examine/IndexDiagnosticsFactory.cs index ca2e732071..a60a373e65 100644 --- a/src/Umbraco.Infrastructure/Examine/IndexDiagnosticsFactory.cs +++ b/src/Umbraco.Infrastructure/Examine/IndexDiagnosticsFactory.cs @@ -1,4 +1,4 @@ -using Examine; +using Examine; namespace Umbraco.Cms.Infrastructure.Examine { @@ -9,8 +9,11 @@ namespace Umbraco.Cms.Infrastructure.Examine { public virtual IIndexDiagnostics Create(IIndex index) { - if (!(index is IIndexDiagnostics indexDiag)) + if (index is not IIndexDiagnostics indexDiag) + { indexDiag = new GenericIndexDiagnostics(index); + } + return indexDiag; } } diff --git a/src/Umbraco.Infrastructure/Examine/IndexPopulator.cs b/src/Umbraco.Infrastructure/Examine/IndexPopulator.cs index 2feac0710a..d32470d875 100644 --- a/src/Umbraco.Infrastructure/Examine/IndexPopulator.cs +++ b/src/Umbraco.Infrastructure/Examine/IndexPopulator.cs @@ -1,4 +1,4 @@ -using System.Collections.Generic; +using System.Collections.Generic; using System.Linq; using Examine; using Umbraco.Cms.Core.Collections; diff --git a/src/Umbraco.Infrastructure/Examine/IndexRebuilder.cs b/src/Umbraco.Infrastructure/Examine/IndexRebuilder.cs deleted file mode 100644 index 9e4fe6fed0..0000000000 --- a/src/Umbraco.Infrastructure/Examine/IndexRebuilder.cs +++ /dev/null @@ -1,81 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using Examine; -using Microsoft.Extensions.Logging; -using Umbraco.Cms.Core.Logging; - -namespace Umbraco.Cms.Infrastructure.Examine -{ - - /// - /// Utility to rebuild all indexes ensuring minimal data queries - /// - public class IndexRebuilder - { - private readonly IProfilingLogger _profilingLogger; - private readonly ILogger _logger; - private readonly IEnumerable _populators; - public IExamineManager ExamineManager { get; } - - public IndexRebuilder(IProfilingLogger profilingLogger , ILogger logger, IExamineManager examineManager, IEnumerable populators) - { - _profilingLogger = profilingLogger ; - _populators = populators; - _logger = logger; - ExamineManager = examineManager; - } - - public bool CanRebuild(IIndex index) - { - return _populators.Any(x => x.IsRegistered(index)); - } - - public void RebuildIndex(string indexName) - { - if (!ExamineManager.TryGetIndex(indexName, out var index)) - throw new InvalidOperationException($"No index found with name {indexName}"); - index.CreateIndex(); // clear the index - foreach (var populator in _populators) - { - populator.Populate(index); - } - } - - public void RebuildIndexes(bool onlyEmptyIndexes) - { - var indexes = (onlyEmptyIndexes - ? ExamineManager.Indexes.Where(x => !x.IndexExists()) - : ExamineManager.Indexes).ToArray(); - - if (indexes.Length == 0) return; - - OnRebuildingIndexes(new IndexRebuildingEventArgs(indexes)); - - foreach (var index in indexes) - { - index.CreateIndex(); // clear the index - } - - // run each populator over the indexes - foreach(var populator in _populators) - { - try - { - populator.Populate(indexes); - } - catch (Exception e) - { - _logger.LogError(e, "Index populating failed for populator {Populator}", populator.GetType()); - } - } - } - - /// - /// Event raised when indexes are being rebuilt - /// - public event EventHandler RebuildingIndexes; - - private void OnRebuildingIndexes(IndexRebuildingEventArgs args) => RebuildingIndexes?.Invoke(this, args); - } -} diff --git a/src/Umbraco.Infrastructure/Examine/IndexRebuildingEventArgs.cs b/src/Umbraco.Infrastructure/Examine/IndexRebuildingEventArgs.cs deleted file mode 100644 index fbe3dbcbe3..0000000000 --- a/src/Umbraco.Infrastructure/Examine/IndexRebuildingEventArgs.cs +++ /dev/null @@ -1,19 +0,0 @@ -using System; -using System.Collections.Generic; -using Examine; - -namespace Umbraco.Cms.Infrastructure.Examine -{ - public class IndexRebuildingEventArgs : EventArgs - { - public IndexRebuildingEventArgs(IEnumerable indexes) - { - Indexes = indexes; - } - - /// - /// The indexes being rebuilt - /// - public IEnumerable Indexes { get; } - } -} diff --git a/src/Umbraco.Infrastructure/Examine/MediaIndexPopulator.cs b/src/Umbraco.Infrastructure/Examine/MediaIndexPopulator.cs index 429285fa85..9f6e33f8dd 100644 --- a/src/Umbraco.Infrastructure/Examine/MediaIndexPopulator.cs +++ b/src/Umbraco.Infrastructure/Examine/MediaIndexPopulator.cs @@ -1,6 +1,7 @@ -using System.Collections.Generic; +using System.Collections.Generic; using System.Linq; using Examine; +using Microsoft.Extensions.Logging; using Umbraco.Cms.Core.Models; using Umbraco.Cms.Core.Services; @@ -11,6 +12,7 @@ namespace Umbraco.Cms.Infrastructure.Examine /// public class MediaIndexPopulator : IndexPopulator { + private readonly ILogger _logger; private readonly int? _parentId; private readonly IMediaService _mediaService; private readonly IValueSetBuilder _mediaValueSetBuilder; @@ -20,8 +22,8 @@ namespace Umbraco.Cms.Infrastructure.Examine /// /// /// - public MediaIndexPopulator(IMediaService mediaService, IValueSetBuilder mediaValueSetBuilder) - : this(null, mediaService, mediaValueSetBuilder) + public MediaIndexPopulator(ILogger logger, IMediaService mediaService, IValueSetBuilder mediaValueSetBuilder) + : this(logger, null, mediaService, mediaValueSetBuilder) { } @@ -31,8 +33,9 @@ namespace Umbraco.Cms.Infrastructure.Examine /// /// /// - public MediaIndexPopulator(int? parentId, IMediaService mediaService, IValueSetBuilder mediaValueSetBuilder) + public MediaIndexPopulator(ILogger logger, int? parentId, IMediaService mediaService, IValueSetBuilder mediaValueSetBuilder) { + _logger = logger; _parentId = parentId; _mediaService = mediaService; _mediaValueSetBuilder = mediaValueSetBuilder; @@ -40,7 +43,11 @@ namespace Umbraco.Cms.Infrastructure.Examine protected override void PopulateIndexes(IReadOnlyList indexes) { - if (indexes.Count == 0) return; + if (indexes.Count == 0) + { + _logger.LogDebug($"{nameof(PopulateIndexes)} called with no indexes to populate. Typically means no index is registered with this populator."); + return; + } const int pageSize = 10000; var pageIndex = 0; diff --git a/src/Umbraco.Infrastructure/Examine/NoopUmbracoIndexesCreator.cs b/src/Umbraco.Infrastructure/Examine/NoopUmbracoIndexesCreator.cs deleted file mode 100644 index e84fb96a74..0000000000 --- a/src/Umbraco.Infrastructure/Examine/NoopUmbracoIndexesCreator.cs +++ /dev/null @@ -1,14 +0,0 @@ -using System.Collections.Generic; -using System.Linq; -using Examine; - -namespace Umbraco.Cms.Infrastructure.Examine -{ - public class NoopUmbracoIndexesCreator : IUmbracoIndexesCreator - { - public IEnumerable Create() - { - return Enumerable.Empty(); - } - } -} diff --git a/src/Umbraco.Infrastructure/Examine/PublishedContentIndexPopulator.cs b/src/Umbraco.Infrastructure/Examine/PublishedContentIndexPopulator.cs index 4b55337670..f9ccaffdbc 100644 --- a/src/Umbraco.Infrastructure/Examine/PublishedContentIndexPopulator.cs +++ b/src/Umbraco.Infrastructure/Examine/PublishedContentIndexPopulator.cs @@ -1,4 +1,5 @@ -using Umbraco.Cms.Core.Services; +using Microsoft.Extensions.Logging; +using Umbraco.Cms.Core.Services; using Umbraco.Cms.Infrastructure.Persistence; namespace Umbraco.Cms.Infrastructure.Examine @@ -13,8 +14,8 @@ namespace Umbraco.Cms.Infrastructure.Examine /// public class PublishedContentIndexPopulator : ContentIndexPopulator { - public PublishedContentIndexPopulator(IContentService contentService, IUmbracoDatabaseFactory umbracoDatabaseFactory, IPublishedContentValueSetBuilder contentValueSetBuilder) : - base(true, null, contentService, umbracoDatabaseFactory, contentValueSetBuilder) + public PublishedContentIndexPopulator(ILogger logger, IContentService contentService, IUmbracoDatabaseFactory umbracoDatabaseFactory, IPublishedContentValueSetBuilder contentValueSetBuilder) : + base(logger, true, null, contentService, umbracoDatabaseFactory, contentValueSetBuilder) { } } diff --git a/src/Umbraco.Infrastructure/Examine/RebuildOnStartupHandler.cs b/src/Umbraco.Infrastructure/Examine/RebuildOnStartupHandler.cs new file mode 100644 index 0000000000..60f7478c3f --- /dev/null +++ b/src/Umbraco.Infrastructure/Examine/RebuildOnStartupHandler.cs @@ -0,0 +1,60 @@ +using System; +using System.Threading; +using Umbraco.Cms.Core.Events; +using Umbraco.Cms.Core.Notifications; +using Umbraco.Cms.Core.Sync; + +namespace Umbraco.Cms.Infrastructure.Examine +{ + /// + /// Handles how the indexes are rebuilt on startup + /// + /// + /// On the first HTTP request this will rebuild the Examine indexes if they are empty. + /// If it is a cold boot, they are all rebuilt. + /// + public sealed class RebuildOnStartupHandler : INotificationHandler + { + private readonly ISyncBootStateAccessor _syncBootStateAccessor; + private readonly ExamineIndexRebuilder _backgroundIndexRebuilder; + + // These must be static because notification handlers are transient. + // this does unfortunatley mean that one RebuildOnStartupHandler instance + // will be created for each front-end request even though we only use the first one. + // TODO: Is there a better way to acheive this without allocating? We cannot remove + // a handler from the notification system. It's not a huge deal but would be better + // with less objects. + private static bool _isReady; + private static bool _isReadSet; + private static object _isReadyLock; + + public RebuildOnStartupHandler( + ISyncBootStateAccessor syncBootStateAccessor, + ExamineIndexRebuilder backgroundIndexRebuilder) + { + _syncBootStateAccessor = syncBootStateAccessor; + _backgroundIndexRebuilder = backgroundIndexRebuilder; + } + + /// + /// On first http request schedule an index rebuild for any empty indexes (or all if it's a cold boot) + /// + /// + public void Handle(UmbracoRequestBeginNotification notification) + => LazyInitializer.EnsureInitialized( + ref _isReady, + ref _isReadSet, + ref _isReadyLock, + () => + { + SyncBootState bootState = _syncBootStateAccessor.GetSyncBootState(); + + _backgroundIndexRebuilder.RebuildIndexes( + // if it's not a cold boot, only rebuild empty ones + bootState != SyncBootState.ColdBoot, + TimeSpan.FromMinutes(1)); + + return true; + }); + } +} diff --git a/src/Umbraco.Infrastructure/Examine/UmbracoExamineExtensions.cs b/src/Umbraco.Infrastructure/Examine/UmbracoExamineExtensions.cs index 90f012f08a..0d341d1d9b 100644 --- a/src/Umbraco.Infrastructure/Examine/UmbracoExamineExtensions.cs +++ b/src/Umbraco.Infrastructure/Examine/UmbracoExamineExtensions.cs @@ -1,5 +1,6 @@ -using System.Collections.Generic; +using System.Collections.Generic; using System.Text.RegularExpressions; +using System.Threading.Tasks; using Examine; using Examine.Search; using Umbraco.Cms.Infrastructure.Examine; @@ -16,8 +17,6 @@ namespace Umbraco.Extensions /// internal static readonly Regex CultureIsoCodeFieldNameMatchExpression = new Regex("^([_\\w]+)_([a-z]{2}-[a-z0-9]{2,4})$", RegexOptions.Compiled); - - //TODO: We need a public method here to just match a field name against CultureIsoCodeFieldNameMatchExpression /// @@ -28,14 +27,19 @@ namespace Umbraco.Extensions /// public static IEnumerable GetCultureFields(this IUmbracoIndex index, string culture) { - var allFields = index.GetFields(); - // ReSharper disable once LoopCanBeConvertedToQuery + IEnumerable allFields = index.GetFieldNames(); + + var results = new List(); foreach (var field in allFields) { var match = CultureIsoCodeFieldNameMatchExpression.Match(field); if (match.Success && match.Groups.Count == 3 && culture.InvariantEquals(match.Groups[2].Value)) - yield return field; + { + results.Add(field); + } } + + return results; } /// @@ -46,8 +50,8 @@ namespace Umbraco.Extensions /// public static IEnumerable GetCultureAndInvariantFields(this IUmbracoIndex index, string culture) { - var allFields = index.GetFields(); - // ReSharper disable once LoopCanBeConvertedToQuery + IEnumerable allFields = index.GetFieldNames(); + foreach (var field in allFields) { var match = CultureIsoCodeFieldNameMatchExpression.Match(field); @@ -59,7 +63,6 @@ namespace Umbraco.Extensions { yield return field; //matches no culture field (invariant) } - } } diff --git a/src/Umbraco.Infrastructure/HostedServices/QueuedHostedService.cs b/src/Umbraco.Infrastructure/HostedServices/QueuedHostedService.cs index db933fec31..c15a37855e 100644 --- a/src/Umbraco.Infrastructure/HostedServices/QueuedHostedService.cs +++ b/src/Umbraco.Infrastructure/HostedServices/QueuedHostedService.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Hosting; @@ -35,8 +35,7 @@ namespace Umbraco.Cms.Infrastructure.HostedServices { while (!stoppingToken.IsCancellationRequested) { - var workItem = - await TaskQueue.DequeueAsync(stoppingToken); + Func workItem = await TaskQueue.DequeueAsync(stoppingToken); try { diff --git a/src/Umbraco.Infrastructure/HostedServices/RecurringHostedServiceBase.cs b/src/Umbraco.Infrastructure/HostedServices/RecurringHostedServiceBase.cs index 1ec13f334e..131b81322a 100644 --- a/src/Umbraco.Infrastructure/HostedServices/RecurringHostedServiceBase.cs +++ b/src/Umbraco.Infrastructure/HostedServices/RecurringHostedServiceBase.cs @@ -47,12 +47,25 @@ namespace Umbraco.Cms.Infrastructure.HostedServices /// Executes the task. /// /// The task state. - public async void ExecuteAsync(object state) => - // Delegate work to method returning a task, that can be called and asserted in a unit test. - // Without this there can be behaviour where tests pass, but an error within them causes the test - // running process to crash. - // Hat-tip: https://stackoverflow.com/a/14207615/489433 - await PerformExecuteAsync(state); + public async void ExecuteAsync(object state) + { + try + { + // First, stop the timer, we do not want tasks to execute in parallel + _timer?.Change(Timeout.Infinite, 0); + + // Delegate work to method returning a task, that can be called and asserted in a unit test. + // Without this there can be behaviour where tests pass, but an error within them causes the test + // running process to crash. + // Hat-tip: https://stackoverflow.com/a/14207615/489433 + await PerformExecuteAsync(state); + } + finally + { + // Resume now that the task is complete + _timer?.Change((int)_delay.TotalMilliseconds, (int)_period.TotalMilliseconds); + } + } public abstract Task PerformExecuteAsync(object state); diff --git a/src/Umbraco.Infrastructure/PublishedContentQuery.cs b/src/Umbraco.Infrastructure/PublishedContentQuery.cs index 1d13748aeb..47b98d8dc0 100644 --- a/src/Umbraco.Infrastructure/PublishedContentQuery.cs +++ b/src/Umbraco.Infrastructure/PublishedContentQuery.cs @@ -1,7 +1,8 @@ -using System; +using System; using System.Collections; using System.Collections.Generic; using System.Linq; +using System.Threading.Tasks; using System.Xml.XPath; using Examine; using Examine.Search; @@ -260,7 +261,7 @@ namespace Umbraco.Cms.Infrastructure $"No index found by name {indexName} or is not of type {typeof(IUmbracoIndex)}"); } - var query = umbIndex.GetSearcher().CreateQuery(IndexTypes.Content); + var query = umbIndex.Searcher.CreateQuery(IndexTypes.Content); IQueryExecutor queryExecutor; if (culture == "*") @@ -286,7 +287,7 @@ namespace Umbraco.Cms.Infrastructure var results = skip == 0 && take == 0 ? queryExecutor.Execute() - : queryExecutor.Execute(skip + take); + : queryExecutor.Execute(QueryOptions.SkipTake(skip, take)); totalRecords = results.TotalItemCount; @@ -316,7 +317,7 @@ namespace Umbraco.Cms.Infrastructure var results = skip == 0 && take == 0 ? query.Execute() - : query.Execute(skip + take); + : query.Execute(QueryOptions.SkipTake(skip, take)); totalRecords = results.TotalItemCount; diff --git a/src/Umbraco.Infrastructure/Runtime/CoreRuntime.cs b/src/Umbraco.Infrastructure/Runtime/CoreRuntime.cs index e004313ac3..e2b20ced8f 100644 --- a/src/Umbraco.Infrastructure/Runtime/CoreRuntime.cs +++ b/src/Umbraco.Infrastructure/Runtime/CoreRuntime.cs @@ -119,10 +119,10 @@ namespace Umbraco.Cms.Infrastructure.Runtime } - await _eventAggregator.PublishAsync(new UmbracoApplicationStartingNotification(State.Level), cancellationToken); - // create & initialize the components _components.Initialize(); + + await _eventAggregator.PublishAsync(new UmbracoApplicationStartingNotification(State.Level), cancellationToken); } private void DoUnattendedUpgrade() diff --git a/src/Umbraco.Infrastructure/Runtime/SqlMainDomLock.cs b/src/Umbraco.Infrastructure/Runtime/SqlMainDomLock.cs index c2c3262047..1c02898334 100644 --- a/src/Umbraco.Infrastructure/Runtime/SqlMainDomLock.cs +++ b/src/Umbraco.Infrastructure/Runtime/SqlMainDomLock.cs @@ -56,7 +56,7 @@ namespace Umbraco.Cms.Infrastructure.Runtime dbProviderFactoryCreator, databaseSchemaCreatorFactory); - MainDomKey = MainDomKeyPrefix + "-" + (NetworkHelper.MachineName + MainDom.GetMainDomId(_hostingEnvironment)).GenerateHash(); + MainDomKey = MainDomKeyPrefix + "-" + (Environment.MachineName + MainDom.GetMainDomId(_hostingEnvironment)).GenerateHash(); } public async Task AcquireLockAsync(int millisecondsTimeout) diff --git a/src/Umbraco.Infrastructure/Search/BackgroundIndexRebuilder.cs b/src/Umbraco.Infrastructure/Search/BackgroundIndexRebuilder.cs deleted file mode 100644 index 2fbceb2f9a..0000000000 --- a/src/Umbraco.Infrastructure/Search/BackgroundIndexRebuilder.cs +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright (c) Umbraco. -// See LICENSE for more details. - -using System; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Umbraco.Cms.Core.Runtime; -using Umbraco.Cms.Infrastructure.Examine; -using Umbraco.Cms.Infrastructure.HostedServices; - -namespace Umbraco.Cms.Infrastructure.Search -{ - /// - /// Utility to rebuild all indexes on a background thread - /// - public class BackgroundIndexRebuilder - { - private readonly IndexRebuilder _indexRebuilder; - private readonly IBackgroundTaskQueue _backgroundTaskQueue; - - private readonly IMainDom _mainDom; - private readonly ILogger _logger; - - private volatile bool _isRunning = false; - private static readonly object s_rebuildLocker = new object(); - - /// - /// Initializes a new instance of the class. - /// - public BackgroundIndexRebuilder( - IMainDom mainDom, - ILogger logger, - IndexRebuilder indexRebuilder, - IBackgroundTaskQueue backgroundTaskQueue) - { - _mainDom = mainDom; - _logger = logger; - _indexRebuilder = indexRebuilder; - _backgroundTaskQueue = backgroundTaskQueue; - } - - - /// - /// Called to rebuild empty indexes on startup - /// - public virtual void RebuildIndexes(bool onlyEmptyIndexes, TimeSpan? delay = null) - { - - lock (s_rebuildLocker) - { - if (_isRunning) - { - _logger.LogWarning("Call was made to RebuildIndexes but the task runner for rebuilding is already running"); - return; - } - - _logger.LogInformation("Starting initialize async background thread."); - - _backgroundTaskQueue.QueueBackgroundWorkItem(cancellationToken => RebuildIndexes(onlyEmptyIndexes, delay ?? TimeSpan.Zero, cancellationToken)); - - } - } - - private Task RebuildIndexes(bool onlyEmptyIndexes, TimeSpan delay, CancellationToken cancellationToken) - { - if (!_mainDom.IsMainDom) - { - return Task.CompletedTask; - } - - if (delay > TimeSpan.Zero) - { - Thread.Sleep(delay); - } - - _isRunning = true; - _indexRebuilder.RebuildIndexes(onlyEmptyIndexes); - _isRunning = false; - return Task.CompletedTask; - } - } -} diff --git a/src/Umbraco.Infrastructure/Search/ExamineNotificationHandler.cs b/src/Umbraco.Infrastructure/Search/ExamineNotificationHandler.cs deleted file mode 100644 index d0541cfd97..0000000000 --- a/src/Umbraco.Infrastructure/Search/ExamineNotificationHandler.cs +++ /dev/null @@ -1,838 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Globalization; -using System.Linq; -using System.Threading.Tasks; -using Examine; -using Microsoft.Extensions.Logging; -using Umbraco.Cms.Core; -using Umbraco.Cms.Core.Cache; -using Umbraco.Cms.Core.Events; -using Umbraco.Cms.Core.Logging; -using Umbraco.Cms.Core.Models; -using Umbraco.Cms.Core.Notifications; -using Umbraco.Cms.Core.Runtime; -using Umbraco.Cms.Core.Scoping; -using Umbraco.Cms.Core.Services; -using Umbraco.Cms.Core.Services.Changes; -using Umbraco.Cms.Core.Sync; -using Umbraco.Cms.Infrastructure.Examine; -using Umbraco.Extensions; - -namespace Umbraco.Cms.Infrastructure.Search -{ - public sealed class ExamineNotificationHandler : - INotificationHandler, - INotificationHandler, - INotificationHandler, - INotificationHandler, - INotificationHandler, - INotificationHandler - { - private readonly IExamineManager _examineManager; - private readonly IContentValueSetBuilder _contentValueSetBuilder; - private readonly IPublishedContentValueSetBuilder _publishedContentValueSetBuilder; - private readonly IValueSetBuilder _mediaValueSetBuilder; - private readonly IValueSetBuilder _memberValueSetBuilder; - private readonly BackgroundIndexRebuilder _backgroundIndexRebuilder; - private readonly TaskHelper _taskHelper; - private readonly IRuntimeState _runtimeState; - private readonly IScopeProvider _scopeProvider; - private readonly ServiceContext _services; - private readonly IMainDom _mainDom; - private readonly IProfilingLogger _profilingLogger; - private readonly ILogger _logger; - private readonly IUmbracoIndexesCreator _indexCreator; - private static bool s_deactivate_handlers; - - // the default enlist priority is 100 - // enlist with a lower priority to ensure that anything "default" runs after us - // but greater that SafeXmlReaderWriter priority which is 60 - private const int EnlistPriority = 80; - - public ExamineNotificationHandler(IMainDom mainDom, - IExamineManager examineManager, - IProfilingLogger profilingLogger, - ILogger logger, - IScopeProvider scopeProvider, - IUmbracoIndexesCreator indexCreator, - ServiceContext services, - IContentValueSetBuilder contentValueSetBuilder, - IPublishedContentValueSetBuilder publishedContentValueSetBuilder, - IValueSetBuilder mediaValueSetBuilder, - IValueSetBuilder memberValueSetBuilder, - BackgroundIndexRebuilder backgroundIndexRebuilder, - TaskHelper taskHelper, - IRuntimeState runtimeState) - { - _services = services; - _scopeProvider = scopeProvider; - _examineManager = examineManager; - _contentValueSetBuilder = contentValueSetBuilder; - _publishedContentValueSetBuilder = publishedContentValueSetBuilder; - _mediaValueSetBuilder = mediaValueSetBuilder; - _memberValueSetBuilder = memberValueSetBuilder; - _backgroundIndexRebuilder = backgroundIndexRebuilder; - _taskHelper = taskHelper; - _runtimeState = runtimeState; - _mainDom = mainDom; - _profilingLogger = profilingLogger; - _logger = logger; - _indexCreator = indexCreator; - } - public void Handle(UmbracoApplicationStartingNotification notification) - { - //let's deal with shutting down Examine with MainDom - var examineShutdownRegistered = _mainDom.Register(release: () => - { - using (_profilingLogger.TraceDuration("Examine shutting down")) - { - _examineManager.Dispose(); - } - }); - - if (!examineShutdownRegistered) - { - _logger.LogInformation("Examine shutdown not registered, this AppDomain is not the MainDom, Examine will be disabled"); - - //if we could not register the shutdown examine ourselves, it means we are not maindom! in this case all of examine should be disabled! - Suspendable.ExamineEvents.SuspendIndexers(_logger); - return; //exit, do not continue - } - - //create the indexes and register them with the manager - foreach (IIndex index in _indexCreator.Create()) - { - _examineManager.AddIndex(index); - } - - _logger.LogDebug("Examine shutdown registered with MainDom"); - - var registeredIndexers = _examineManager.Indexes.OfType().Count(x => x.EnableDefaultEventHandler); - - _logger.LogInformation("Adding examine event handlers for {RegisteredIndexers} index providers.", registeredIndexers); - - // don't bind event handlers if we're not suppose to listen - if (registeredIndexers == 0) - { - s_deactivate_handlers = true; - } - - if (_mainDom.IsMainDom && _runtimeState.Level >= RuntimeLevel.Run) - { - _backgroundIndexRebuilder.RebuildIndexes(true); - } - } - - - #region Cache refresher updated event handlers - - /// - /// Updates indexes based on content changes - /// - /// - /// - public void Handle(ContentCacheRefresherNotification args) - { - if (s_deactivate_handlers) - { - return; - } - if (Suspendable.ExamineEvents.CanIndex == false) - { - return; - } - - if (args.MessageType != MessageType.RefreshByPayload) - { - throw new NotSupportedException(); - } - - var contentService = _services.ContentService; - - foreach (var payload in (ContentCacheRefresher.JsonPayload[])args.MessageObject) - { - if (payload.ChangeTypes.HasType(TreeChangeTypes.Remove)) - { - // delete content entirely (with descendants) - // false: remove entirely from all indexes - DeleteIndexForEntity(payload.Id, false); - } - else if (payload.ChangeTypes.HasType(TreeChangeTypes.RefreshAll)) - { - // ExamineEvents does not support RefreshAll - // just ignore that payload - // so what?! - - // TODO: Rebuild the index at this point? - } - else // RefreshNode or RefreshBranch (maybe trashed) - { - // don't try to be too clever - refresh entirely - // there has to be race conditions in there ;-( - - var content = contentService.GetById(payload.Id); - if (content == null) - { - // gone fishing, remove entirely from all indexes (with descendants) - DeleteIndexForEntity(payload.Id, false); - continue; - } - - IContent published = null; - if (content.Published && contentService.IsPathPublished(content)) - { - published = content; - } - - if (published == null) - { - DeleteIndexForEntity(payload.Id, true); - } - - // just that content - ReIndexForContent(content, published != null); - - // branch - if (payload.ChangeTypes.HasType(TreeChangeTypes.RefreshBranch)) - { - var masked = published == null ? null : new List(); - const int pageSize = 500; - var page = 0; - var total = long.MaxValue; - while (page * pageSize < total) - { - var descendants = contentService.GetPagedDescendants(content.Id, page++, pageSize, out total, - //order by shallowest to deepest, this allows us to check it's published state without checking every item - ordering: Ordering.By("Path", Direction.Ascending)); - - foreach (var descendant in descendants) - { - published = null; - if (masked != null) // else everything is masked - { - if (masked.Contains(descendant.ParentId) || !descendant.Published) - { - masked.Add(descendant.Id); - } - else - { - published = descendant; - } - } - - ReIndexForContent(descendant, published != null); - } - } - } - } - - // NOTE - // - // DeleteIndexForEntity is handled by UmbracoContentIndexer.DeleteFromIndex() which takes - // care of also deleting the descendants - // - // ReIndexForContent is NOT taking care of descendants so we have to reload everything - // again in order to process the branch - we COULD improve that by just reloading the - // XML from database instead of reloading content & re-serializing! - // - // BUT ... pretty sure it is! see test "Index_Delete_Index_Item_Ensure_Heirarchy_Removed" - } - } - - public void Handle(MemberCacheRefresherNotification args) - { - if (s_deactivate_handlers) - { - return; - } - - if (Suspendable.ExamineEvents.CanIndex == false) - { - return; - } - - switch (args.MessageType) - { - case MessageType.RefreshById: - var c1 = _services.MemberService.GetById((int)args.MessageObject); - if (c1 != null) - { - ReIndexForMember(c1); - } - break; - case MessageType.RemoveById: - - // This is triggered when the item is permanently deleted - - DeleteIndexForEntity((int)args.MessageObject, false); - break; - case MessageType.RefreshByInstance: - if (args.MessageObject is IMember c3) - { - ReIndexForMember(c3); - } - break; - case MessageType.RemoveByInstance: - - // This is triggered when the item is permanently deleted - - if (args.MessageObject is IMember c4) - { - DeleteIndexForEntity(c4.Id, false); - } - break; - case MessageType.RefreshByPayload: - var payload = (MemberCacheRefresher.JsonPayload[])args.MessageObject; - foreach (var p in payload) - { - if (p.Removed) - { - DeleteIndexForEntity(p.Id, false); - } - else - { - var m = _services.MemberService.GetById(p.Id); - if (m != null) - { - ReIndexForMember(m); - } - } - } - break; - case MessageType.RefreshAll: - case MessageType.RefreshByJson: - default: - //We don't support these, these message types will not fire for unpublished content - break; - } - } - - public void Handle(MediaCacheRefresherNotification args) - { - if (s_deactivate_handlers) - { - return; - } - - if (Suspendable.ExamineEvents.CanIndex == false) - { - return; - } - - if (args.MessageType != MessageType.RefreshByPayload) - { - throw new NotSupportedException(); - } - - var mediaService = _services.MediaService; - - foreach (var payload in (MediaCacheRefresher.JsonPayload[])args.MessageObject) - { - if (payload.ChangeTypes.HasType(TreeChangeTypes.Remove)) - { - // remove from *all* indexes - DeleteIndexForEntity(payload.Id, false); - } - else if (payload.ChangeTypes.HasType(TreeChangeTypes.RefreshAll)) - { - // ExamineEvents does not support RefreshAll - // just ignore that payload - // so what?! - } - else // RefreshNode or RefreshBranch (maybe trashed) - { - var media = mediaService.GetById(payload.Id); - if (media == null) - { - // gone fishing, remove entirely - DeleteIndexForEntity(payload.Id, false); - continue; - } - - if (media.Trashed) - { - DeleteIndexForEntity(payload.Id, true); - } - - // just that media - ReIndexForMedia(media, !media.Trashed); - - // branch - if (payload.ChangeTypes.HasType(TreeChangeTypes.RefreshBranch)) - { - const int pageSize = 500; - var page = 0; - var total = long.MaxValue; - while (page * pageSize < total) - { - var descendants = mediaService.GetPagedDescendants(media.Id, page++, pageSize, out total); - foreach (var descendant in descendants) - { - ReIndexForMedia(descendant, !descendant.Trashed); - } - } - } - } - } - } - - public void Handle(LanguageCacheRefresherNotification args) - { - if (s_deactivate_handlers) - { - return; - } - - if (!(args.MessageObject is LanguageCacheRefresher.JsonPayload[] payloads)) - { - return; - } - - if (payloads.Length == 0) - { - return; - } - - var removedOrCultureChanged = payloads.Any(x => - x.ChangeType == LanguageCacheRefresher.JsonPayload.LanguageChangeType.ChangeCulture - || x.ChangeType == LanguageCacheRefresher.JsonPayload.LanguageChangeType.Remove); - - if (removedOrCultureChanged) - { - //if a lang is removed or it's culture has changed, we need to rebuild the indexes since - //field names and values in the index have a string culture value. - _backgroundIndexRebuilder.RebuildIndexes(false); - } - } - - /// - /// Updates indexes based on content type changes - /// - /// - /// - public void Handle(ContentTypeCacheRefresherNotification args) - { - if (s_deactivate_handlers) - { - return; - } - - if (Suspendable.ExamineEvents.CanIndex == false) - { - return; - } - - if (args.MessageType != MessageType.RefreshByPayload) - { - throw new NotSupportedException(); - } - - var changedIds = new Dictionary removedIds, List refreshedIds, List otherIds)>(); - - foreach (var payload in (ContentTypeCacheRefresher.JsonPayload[])args.MessageObject) - { - if (!changedIds.TryGetValue(payload.ItemType, out var idLists)) - { - idLists = (removedIds: new List(), refreshedIds: new List(), otherIds: new List()); - changedIds.Add(payload.ItemType, idLists); - } - - if (payload.ChangeTypes.HasType(ContentTypeChangeTypes.Remove)) - { - idLists.removedIds.Add(payload.Id); - } - else if (payload.ChangeTypes.HasType(ContentTypeChangeTypes.RefreshMain)) - { - idLists.refreshedIds.Add(payload.Id); - } - else if (payload.ChangeTypes.HasType(ContentTypeChangeTypes.RefreshOther)) - { - idLists.otherIds.Add(payload.Id); - } - } - - const int pageSize = 500; - - foreach (var ci in changedIds) - { - if (ci.Value.refreshedIds.Count > 0 || ci.Value.otherIds.Count > 0) - { - switch (ci.Key) - { - case var itemType when itemType == typeof(IContentType).Name: - RefreshContentOfContentTypes(ci.Value.refreshedIds.Concat(ci.Value.otherIds).Distinct().ToArray()); - break; - case var itemType when itemType == typeof(IMediaType).Name: - RefreshMediaOfMediaTypes(ci.Value.refreshedIds.Concat(ci.Value.otherIds).Distinct().ToArray()); - break; - case var itemType when itemType == typeof(IMemberType).Name: - RefreshMemberOfMemberTypes(ci.Value.refreshedIds.Concat(ci.Value.otherIds).Distinct().ToArray()); - break; - } - } - - //Delete all content of this content/media/member type that is in any content indexer by looking up matched examine docs - foreach (var id in ci.Value.removedIds) - { - foreach (var index in _examineManager.Indexes.OfType()) - { - var searcher = index.GetSearcher(); - - var page = 0; - var total = long.MaxValue; - while (page * pageSize < total) - { - //paging with examine, see https://shazwazza.com/post/paging-with-examine/ - var results = searcher.CreateQuery().Field("nodeType", id.ToInvariantString()).Execute(maxResults: pageSize * (page + 1)); - total = results.TotalItemCount; - var paged = results.Skip(page * pageSize); - - foreach (ISearchResult item in paged) - { - if (int.TryParse(item.Id, out int contentId)) - { - DeleteIndexForEntity(contentId, false); - } - } - - page++; - } - } - } - } - } - - private void RefreshMemberOfMemberTypes(int[] memberTypeIds) - { - const int pageSize = 500; - - IEnumerable memberTypes = _services.MemberTypeService.GetAll(memberTypeIds); - foreach (IMemberType memberType in memberTypes) - { - var page = 0; - var total = long.MaxValue; - while (page * pageSize < total) - { - IEnumerable memberToRefresh = _services.MemberService.GetAll( - page++, pageSize, out total, "LoginName", Direction.Ascending, - memberType.Alias); - - foreach (IMember c in memberToRefresh) - { - ReIndexForMember(c); - } - } - } - } - - private void RefreshMediaOfMediaTypes(int[] mediaTypeIds) - { - const int pageSize = 500; - var page = 0; - var total = long.MaxValue; - while (page * pageSize < total) - { - IEnumerable mediaToRefresh = _services.MediaService.GetPagedOfTypes( - //Re-index all content of these types - mediaTypeIds, - page++, pageSize, out total, null, - Ordering.By("Path", Direction.Ascending)); - - foreach (IMedia c in mediaToRefresh) - { - ReIndexForMedia(c, c.Trashed == false); - } - } - } - - private void RefreshContentOfContentTypes(int[] contentTypeIds) - { - const int pageSize = 500; - var page = 0; - var total = long.MaxValue; - while (page * pageSize < total) - { - IEnumerable contentToRefresh = _services.ContentService.GetPagedOfTypes( - //Re-index all content of these types - contentTypeIds, - page++, pageSize, out total, null, - //order by shallowest to deepest, this allows us to check it's published state without checking every item - Ordering.By("Path", Direction.Ascending)); - - //track which Ids have their paths are published - var publishChecked = new Dictionary(); - - foreach (IContent c in contentToRefresh) - { - var isPublished = false; - if (c.Published) - { - if (!publishChecked.TryGetValue(c.ParentId, out isPublished)) - { - //nothing by parent id, so query the service and cache the result for the next child to check against - isPublished = _services.ContentService.IsPathPublished(c); - publishChecked[c.Id] = isPublished; - } - } - - ReIndexForContent(c, isPublished); - } - } - } - - #endregion - - #region ReIndex/Delete for entity - private void ReIndexForContent(IContent sender, bool isPublished) - { - var actions = DeferedActions.Get(_scopeProvider); - if (actions != null) - { - actions.Add(new DeferedReIndexForContent(_taskHelper, this, sender, isPublished)); - } - else - { - DeferedReIndexForContent.Execute(_taskHelper, this, sender, isPublished); - } - } - - private void ReIndexForMember(IMember member) - { - var actions = DeferedActions.Get(_scopeProvider); - if (actions != null) - { - actions.Add(new DeferedReIndexForMember(_taskHelper, this, member)); - } - else - { - DeferedReIndexForMember.Execute(_taskHelper, this, member); - } - } - - private void ReIndexForMedia(IMedia sender, bool isPublished) - { - var actions = DeferedActions.Get(_scopeProvider); - if (actions != null) - { - actions.Add(new DeferedReIndexForMedia(_taskHelper, this, sender, isPublished)); - } - else - { - DeferedReIndexForMedia.Execute(_taskHelper, this, sender, isPublished); - } - } - - /// - /// Remove items from an index - /// - /// - /// - /// If true, indicates that we will only delete this item from indexes that don't support unpublished content. - /// If false it will delete this from all indexes regardless. - /// - private void DeleteIndexForEntity(int entityId, bool keepIfUnpublished) - { - var actions = DeferedActions.Get(_scopeProvider); - if (actions != null) - { - actions.Add(new DeferedDeleteIndex(this, entityId, keepIfUnpublished)); - } - else - { - DeferedDeleteIndex.Execute(this, entityId, keepIfUnpublished); - } - } - #endregion - - #region Deferred Actions - private class DeferedActions - { - private readonly List _actions = new List(); - - public static DeferedActions Get(IScopeProvider scopeProvider) - { - IScopeContext scopeContext = scopeProvider.Context; - - return scopeContext?.Enlist("examineEvents", - () => new DeferedActions(), // creator - (completed, actions) => // action - { - if (completed) - { - actions.Execute(); - } - }, EnlistPriority); - } - - public void Add(DeferedAction action) => _actions.Add(action); - - private void Execute() - { - foreach (DeferedAction action in _actions) - { - action.Execute(); - } - } - } - - /// - /// An action that will execute at the end of the Scope being completed - /// - private abstract class DeferedAction - { - public virtual void Execute() - { } - } - - /// - /// Re-indexes an item on a background thread - /// - private class DeferedReIndexForContent : DeferedAction - { - private readonly TaskHelper _taskHelper; - private readonly ExamineNotificationHandler _ExamineNotificationHandler; - private readonly IContent _content; - private readonly bool _isPublished; - - public DeferedReIndexForContent(TaskHelper taskHelper, ExamineNotificationHandler ExamineNotificationHandler, IContent content, bool isPublished) - { - _taskHelper = taskHelper; - _ExamineNotificationHandler = ExamineNotificationHandler; - _content = content; - _isPublished = isPublished; - } - - public override void Execute() => Execute(_taskHelper, _ExamineNotificationHandler, _content, _isPublished); - - public static void Execute(TaskHelper taskHelper, ExamineNotificationHandler ExamineNotificationHandler, IContent content, bool isPublished) - => taskHelper.RunBackgroundTask(() => - { - using IScope scope = ExamineNotificationHandler._scopeProvider.CreateScope(autoComplete: true); - - // for content we have a different builder for published vs unpublished - // we don't want to build more value sets than is needed so we'll lazily build 2 one for published one for non-published - var builders = new Dictionary>> - { - [true] = new Lazy>(() => ExamineNotificationHandler._publishedContentValueSetBuilder.GetValueSets(content).ToList()), - [false] = new Lazy>(() => ExamineNotificationHandler._contentValueSetBuilder.GetValueSets(content).ToList()) - }; - - foreach (IUmbracoIndex index in ExamineNotificationHandler._examineManager.Indexes.OfType() - //filter the indexers - .Where(x => isPublished || !x.PublishedValuesOnly) - .Where(x => x.EnableDefaultEventHandler)) - { - List valueSet = builders[index.PublishedValuesOnly].Value; - index.IndexItems(valueSet); - } - - return Task.CompletedTask; - }); - } - - /// - /// Re-indexes an item on a background thread - /// - private class DeferedReIndexForMedia : DeferedAction - { - private readonly TaskHelper _taskHelper; - private readonly ExamineNotificationHandler _ExamineNotificationHandler; - private readonly IMedia _media; - private readonly bool _isPublished; - - public DeferedReIndexForMedia(TaskHelper taskHelper, ExamineNotificationHandler ExamineNotificationHandler, IMedia media, bool isPublished) - { - _taskHelper = taskHelper; - _ExamineNotificationHandler = ExamineNotificationHandler; - _media = media; - _isPublished = isPublished; - } - - public override void Execute() => Execute(_taskHelper, _ExamineNotificationHandler, _media, _isPublished); - - public static void Execute(TaskHelper taskHelper, ExamineNotificationHandler ExamineNotificationHandler, IMedia media, bool isPublished) => - // perform the ValueSet lookup on a background thread - taskHelper.RunBackgroundTask(() => - { - using IScope scope = ExamineNotificationHandler._scopeProvider.CreateScope(autoComplete: true); - - var valueSet = ExamineNotificationHandler._mediaValueSetBuilder.GetValueSets(media).ToList(); - - foreach (IUmbracoIndex index in ExamineNotificationHandler._examineManager.Indexes.OfType() - //filter the indexers - .Where(x => isPublished || !x.PublishedValuesOnly) - .Where(x => x.EnableDefaultEventHandler)) - { - index.IndexItems(valueSet); - } - - return Task.CompletedTask; - }); - } - - /// - /// Re-indexes an item on a background thread - /// - private class DeferedReIndexForMember : DeferedAction - { - private readonly ExamineNotificationHandler _ExamineNotificationHandler; - private readonly IMember _member; - private readonly TaskHelper _taskHelper; - - public DeferedReIndexForMember(TaskHelper taskHelper, ExamineNotificationHandler ExamineNotificationHandler, IMember member) - { - _ExamineNotificationHandler = ExamineNotificationHandler; - _member = member; - _taskHelper = taskHelper; - } - - public override void Execute() => Execute(_taskHelper, _ExamineNotificationHandler, _member); - - public static void Execute(TaskHelper taskHelper, ExamineNotificationHandler ExamineNotificationHandler, IMember member) => - // perform the ValueSet lookup on a background thread - taskHelper.RunBackgroundTask(() => - { - using IScope scope = ExamineNotificationHandler._scopeProvider.CreateScope(autoComplete: true); - - var valueSet = ExamineNotificationHandler._memberValueSetBuilder.GetValueSets(member).ToList(); - foreach (IUmbracoIndex index in ExamineNotificationHandler._examineManager.Indexes.OfType() - //filter the indexers - .Where(x => x.EnableDefaultEventHandler)) - { - index.IndexItems(valueSet); - } - - return Task.CompletedTask; - }); - } - - private class DeferedDeleteIndex : DeferedAction - { - private readonly ExamineNotificationHandler _ExamineNotificationHandler; - private readonly int _id; - private readonly bool _keepIfUnpublished; - - public DeferedDeleteIndex(ExamineNotificationHandler ExamineNotificationHandler, int id, bool keepIfUnpublished) - { - _ExamineNotificationHandler = ExamineNotificationHandler; - _id = id; - _keepIfUnpublished = keepIfUnpublished; - } - - public override void Execute() => Execute(_ExamineNotificationHandler, _id, _keepIfUnpublished); - - public static void Execute(ExamineNotificationHandler ExamineNotificationHandler, int id, bool keepIfUnpublished) - { - var strId = id.ToString(CultureInfo.InvariantCulture); - foreach (var index in ExamineNotificationHandler._examineManager.Indexes.OfType() - .Where(x => x.PublishedValuesOnly || !keepIfUnpublished) - .Where(x => x.EnableDefaultEventHandler)) - { - index.DeleteFromIndex(strId); - } - } - } - #endregion - } -} diff --git a/src/Umbraco.Infrastructure/Search/ExamineUserComponent.cs b/src/Umbraco.Infrastructure/Search/ExamineUserComponent.cs deleted file mode 100644 index 6c39da44c7..0000000000 --- a/src/Umbraco.Infrastructure/Search/ExamineUserComponent.cs +++ /dev/null @@ -1,37 +0,0 @@ -using Umbraco.Cms.Core.Composing; -using Umbraco.Cms.Core.Runtime; - -namespace Umbraco.Cms.Infrastructure.Search -{ - /// - /// An abstract class for custom index authors to inherit from - /// - public abstract class ExamineUserComponent : IComponent - { - private readonly IMainDom _mainDom; - - public ExamineUserComponent(IMainDom mainDom) - { - _mainDom = mainDom; - } - - /// - /// Initialize the component, eagerly exits if ExamineComponent.ExamineEnabled == false - /// - public void Initialize() - { - if (!_mainDom.IsMainDom) return; - - InitializeComponent(); - } - - /// - /// Abstract method which executes to initialize this component if ExamineComponent.ExamineEnabled == true - /// - protected abstract void InitializeComponent(); - - public virtual void Terminate() - { - } - } -} diff --git a/src/Umbraco.Infrastructure/Search/IUmbracoIndexingHandler.cs b/src/Umbraco.Infrastructure/Search/IUmbracoIndexingHandler.cs new file mode 100644 index 0000000000..24c82c055d --- /dev/null +++ b/src/Umbraco.Infrastructure/Search/IUmbracoIndexingHandler.cs @@ -0,0 +1,37 @@ +using System.Collections.Generic; +using Umbraco.Cms.Core.Models; + +namespace Umbraco.Cms.Infrastructure.Search +{ + public interface IUmbracoIndexingHandler + { + /// + /// Returns true if the indexing handler is enabled + /// + /// + /// If this is false then there will be no data lookups executed to populate indexes + /// when service changes are made. + /// + bool Enabled { get; } + + void ReIndexForContent(IContent sender, bool isPublished); + void ReIndexForMember(IMember member); + void ReIndexForMedia(IMedia sender, bool isPublished); + + /// + /// Deletes all documents for the content type Ids + /// + /// + void DeleteDocumentsForContentTypes(IReadOnlyCollection removedContentTypes); + + /// + /// Remove items from an index + /// + /// + /// + /// If true, indicates that we will only delete this item from indexes that don't support unpublished content. + /// If false it will delete this from all indexes regardless. + /// + void DeleteIndexForEntity(int entityId, bool keepIfUnpublished); + } +} diff --git a/src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.Content.cs b/src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.Content.cs new file mode 100644 index 0000000000..ebebdb7f34 --- /dev/null +++ b/src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.Content.cs @@ -0,0 +1,137 @@ +using System; +using System.Collections.Generic; +using Umbraco.Cms.Core; +using Umbraco.Cms.Core.Cache; +using Umbraco.Cms.Core.Events; +using Umbraco.Cms.Core.Models; +using Umbraco.Cms.Core.Notifications; +using Umbraco.Cms.Core.Services; +using Umbraco.Cms.Core.Services.Changes; +using Umbraco.Cms.Core.Sync; +using Umbraco.Extensions; + +namespace Umbraco.Cms.Infrastructure.Search +{ + public sealed class ContentIndexingNotificationHandler : INotificationHandler + { + private readonly IUmbracoIndexingHandler _umbracoIndexingHandler; + private readonly IContentService _contentService; + + public ContentIndexingNotificationHandler(IUmbracoIndexingHandler umbracoIndexingHandler, IContentService contentService) + { + _umbracoIndexingHandler = umbracoIndexingHandler ?? throw new ArgumentNullException(nameof(umbracoIndexingHandler)); + _contentService = contentService ?? throw new ArgumentNullException(nameof(contentService)); + } + + /// + /// Updates indexes based on content changes + /// + /// + /// + public void Handle(ContentCacheRefresherNotification args) + { + if (!_umbracoIndexingHandler.Enabled) + { + return; + } + if (Suspendable.ExamineEvents.CanIndex == false) + { + return; + } + + if (args.MessageType != MessageType.RefreshByPayload) + { + throw new NotSupportedException(); + } + + foreach (var payload in (ContentCacheRefresher.JsonPayload[])args.MessageObject) + { + if (payload.ChangeTypes.HasType(TreeChangeTypes.Remove)) + { + // delete content entirely (with descendants) + // false: remove entirely from all indexes + _umbracoIndexingHandler.DeleteIndexForEntity(payload.Id, false); + } + else if (payload.ChangeTypes.HasType(TreeChangeTypes.RefreshAll)) + { + // ExamineEvents does not support RefreshAll + // just ignore that payload + // so what?! + + // TODO: Rebuild the index at this point? + } + else // RefreshNode or RefreshBranch (maybe trashed) + { + // don't try to be too clever - refresh entirely + // there has to be race conditions in there ;-( + + var content = _contentService.GetById(payload.Id); + if (content == null) + { + // gone fishing, remove entirely from all indexes (with descendants) + _umbracoIndexingHandler.DeleteIndexForEntity(payload.Id, false); + continue; + } + + IContent published = null; + if (content.Published && _contentService.IsPathPublished(content)) + { + published = content; + } + + if (published == null) + { + _umbracoIndexingHandler.DeleteIndexForEntity(payload.Id, true); + } + + // just that content + _umbracoIndexingHandler.ReIndexForContent(content, published != null); + + // branch + if (payload.ChangeTypes.HasType(TreeChangeTypes.RefreshBranch)) + { + var masked = published == null ? null : new List(); + const int pageSize = 500; + var page = 0; + var total = long.MaxValue; + while (page * pageSize < total) + { + var descendants = _contentService.GetPagedDescendants(content.Id, page++, pageSize, out total, + //order by shallowest to deepest, this allows us to check it's published state without checking every item + ordering: Ordering.By("Path", Direction.Ascending)); + + foreach (var descendant in descendants) + { + published = null; + if (masked != null) // else everything is masked + { + if (masked.Contains(descendant.ParentId) || !descendant.Published) + { + masked.Add(descendant.Id); + } + else + { + published = descendant; + } + } + + _umbracoIndexingHandler.ReIndexForContent(descendant, published != null); + } + } + } + } + + // NOTE + // + // DeleteIndexForEntity is handled by UmbracoContentIndexer.DeleteFromIndex() which takes + // care of also deleting the descendants + // + // ReIndexForContent is NOT taking care of descendants so we have to reload everything + // again in order to process the branch - we COULD improve that by just reloading the + // XML from database instead of reloading content & re-serializing! + // + // BUT ... pretty sure it is! see test "Index_Delete_Index_Item_Ensure_Heirarchy_Removed" + } + } + } +} diff --git a/src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.ContentType.cs b/src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.ContentType.cs new file mode 100644 index 0000000000..9bdc9fa3c4 --- /dev/null +++ b/src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.ContentType.cs @@ -0,0 +1,180 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Umbraco.Cms.Core; +using Umbraco.Cms.Core.Cache; +using Umbraco.Cms.Core.Events; +using Umbraco.Cms.Core.Models; +using Umbraco.Cms.Core.Notifications; +using Umbraco.Cms.Core.Services; +using Umbraco.Cms.Core.Services.Changes; +using Umbraco.Cms.Core.Sync; +using Umbraco.Extensions; + +namespace Umbraco.Cms.Infrastructure.Search +{ + public sealed class ContentTypeIndexingNotificationHandler : INotificationHandler + { + private readonly IUmbracoIndexingHandler _umbracoIndexingHandler; + private readonly IContentService _contentService; + private readonly IMemberService _memberService; + private readonly IMediaService _mediaService; + private readonly IMemberTypeService _memberTypeService; + + public ContentTypeIndexingNotificationHandler(IUmbracoIndexingHandler umbracoIndexingHandler, IContentService contentService, IMemberService memberService, IMediaService mediaService, IMemberTypeService memberTypeService) + { + _umbracoIndexingHandler = umbracoIndexingHandler ?? throw new ArgumentNullException(nameof(umbracoIndexingHandler)); + _contentService = contentService ?? throw new ArgumentNullException(nameof(contentService)); + _memberService = memberService ?? throw new ArgumentNullException(nameof(memberService)); + _mediaService = mediaService ?? throw new ArgumentNullException(nameof(mediaService)); + _memberTypeService = memberTypeService ?? throw new ArgumentNullException(nameof(memberTypeService)); + } + + /// + /// Updates indexes based on content type changes + /// + /// + /// + public void Handle(ContentTypeCacheRefresherNotification args) + { + if (!_umbracoIndexingHandler.Enabled) + { + return; + } + + if (Suspendable.ExamineEvents.CanIndex == false) + { + return; + } + + if (args.MessageType != MessageType.RefreshByPayload) + { + throw new NotSupportedException(); + } + + var changedIds = new Dictionary removedIds, List refreshedIds, List otherIds)>(); + + foreach (var payload in (ContentTypeCacheRefresher.JsonPayload[])args.MessageObject) + { + if (!changedIds.TryGetValue(payload.ItemType, out var idLists)) + { + idLists = (removedIds: new List(), refreshedIds: new List(), otherIds: new List()); + changedIds.Add(payload.ItemType, idLists); + } + + if (payload.ChangeTypes.HasType(ContentTypeChangeTypes.Remove)) + { + idLists.removedIds.Add(payload.Id); + } + else if (payload.ChangeTypes.HasType(ContentTypeChangeTypes.RefreshMain)) + { + idLists.refreshedIds.Add(payload.Id); + } + else if (payload.ChangeTypes.HasType(ContentTypeChangeTypes.RefreshOther)) + { + idLists.otherIds.Add(payload.Id); + } + } + + foreach (var ci in changedIds) + { + if (ci.Value.refreshedIds.Count > 0 || ci.Value.otherIds.Count > 0) + { + switch (ci.Key) + { + case var itemType when itemType == typeof(IContentType).Name: + RefreshContentOfContentTypes(ci.Value.refreshedIds.Concat(ci.Value.otherIds).Distinct().ToArray()); + break; + case var itemType when itemType == typeof(IMediaType).Name: + RefreshMediaOfMediaTypes(ci.Value.refreshedIds.Concat(ci.Value.otherIds).Distinct().ToArray()); + break; + case var itemType when itemType == typeof(IMemberType).Name: + RefreshMemberOfMemberTypes(ci.Value.refreshedIds.Concat(ci.Value.otherIds).Distinct().ToArray()); + break; + } + } + + //Delete all content of this content/media/member type that is in any content indexer by looking up matched examine docs + _umbracoIndexingHandler.DeleteDocumentsForContentTypes(ci.Value.removedIds); + } + } + + private void RefreshMemberOfMemberTypes(int[] memberTypeIds) + { + const int pageSize = 500; + + IEnumerable memberTypes = _memberTypeService.GetAll(memberTypeIds); + foreach (IMemberType memberType in memberTypes) + { + var page = 0; + var total = long.MaxValue; + while (page * pageSize < total) + { + IEnumerable memberToRefresh = _memberService.GetAll( + page++, pageSize, out total, "LoginName", Direction.Ascending, + memberType.Alias); + + foreach (IMember c in memberToRefresh) + { + _umbracoIndexingHandler.ReIndexForMember(c); + } + } + } + } + + private void RefreshMediaOfMediaTypes(int[] mediaTypeIds) + { + const int pageSize = 500; + var page = 0; + var total = long.MaxValue; + while (page * pageSize < total) + { + IEnumerable mediaToRefresh = _mediaService.GetPagedOfTypes( + //Re-index all content of these types + mediaTypeIds, + page++, pageSize, out total, null, + Ordering.By("Path", Direction.Ascending)); + + foreach (IMedia c in mediaToRefresh) + { + _umbracoIndexingHandler.ReIndexForMedia(c, c.Trashed == false); + } + } + } + + private void RefreshContentOfContentTypes(int[] contentTypeIds) + { + const int pageSize = 500; + var page = 0; + var total = long.MaxValue; + while (page * pageSize < total) + { + IEnumerable contentToRefresh = _contentService.GetPagedOfTypes( + //Re-index all content of these types + contentTypeIds, + page++, pageSize, out total, null, + //order by shallowest to deepest, this allows us to check it's published state without checking every item + Ordering.By("Path", Direction.Ascending)); + + //track which Ids have their paths are published + var publishChecked = new Dictionary(); + + foreach (IContent c in contentToRefresh) + { + var isPublished = false; + if (c.Published) + { + if (!publishChecked.TryGetValue(c.ParentId, out isPublished)) + { + //nothing by parent id, so query the service and cache the result for the next child to check against + isPublished = _contentService.IsPathPublished(c); + publishChecked[c.Id] = isPublished; + } + } + + _umbracoIndexingHandler.ReIndexForContent(c, isPublished); + } + } + } + } +} diff --git a/src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.Language.cs b/src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.Language.cs new file mode 100644 index 0000000000..2f7d5f66ca --- /dev/null +++ b/src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.Language.cs @@ -0,0 +1,50 @@ +using System; +using System.Linq; +using Umbraco.Cms.Core.Cache; +using Umbraco.Cms.Core.Events; +using Umbraco.Cms.Core.Notifications; +using Umbraco.Cms.Infrastructure.Examine; + +namespace Umbraco.Cms.Infrastructure.Search +{ + public sealed class LanguageIndexingNotificationHandler : INotificationHandler + { + private readonly IUmbracoIndexingHandler _umbracoIndexingHandler; + private readonly IIndexRebuilder _indexRebuilder; + + public LanguageIndexingNotificationHandler(IUmbracoIndexingHandler umbracoIndexingHandler, IIndexRebuilder indexRebuilder) + { + _umbracoIndexingHandler = umbracoIndexingHandler ?? throw new ArgumentNullException(nameof(umbracoIndexingHandler)); + _indexRebuilder = indexRebuilder ?? throw new ArgumentNullException(nameof(indexRebuilder)); + } + + public void Handle(LanguageCacheRefresherNotification args) + { + if (!_umbracoIndexingHandler.Enabled) + { + return; + } + + if (!(args.MessageObject is LanguageCacheRefresher.JsonPayload[] payloads)) + { + return; + } + + if (payloads.Length == 0) + { + return; + } + + var removedOrCultureChanged = payloads.Any(x => + x.ChangeType == LanguageCacheRefresher.JsonPayload.LanguageChangeType.ChangeCulture + || x.ChangeType == LanguageCacheRefresher.JsonPayload.LanguageChangeType.Remove); + + if (removedOrCultureChanged) + { + //if a lang is removed or it's culture has changed, we need to rebuild the indexes since + //field names and values in the index have a string culture value. + _indexRebuilder.RebuildIndexes(false); + } + } + } +} diff --git a/src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.Media.cs b/src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.Media.cs new file mode 100644 index 0000000000..8b37d047de --- /dev/null +++ b/src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.Media.cs @@ -0,0 +1,90 @@ +using System; +using Umbraco.Cms.Core.Cache; +using Umbraco.Cms.Core.Events; +using Umbraco.Cms.Core.Notifications; +using Umbraco.Cms.Core.Services; +using Umbraco.Cms.Core.Services.Changes; +using Umbraco.Cms.Core.Sync; +using Umbraco.Extensions; + +namespace Umbraco.Cms.Infrastructure.Search +{ + public sealed class MediaIndexingNotificationHandler : INotificationHandler + { + private readonly IUmbracoIndexingHandler _umbracoIndexingHandler; + private readonly IMediaService _mediaService; + + public MediaIndexingNotificationHandler(IUmbracoIndexingHandler umbracoIndexingHandler, IMediaService mediaService) + { + _umbracoIndexingHandler = umbracoIndexingHandler ?? throw new ArgumentNullException(nameof(umbracoIndexingHandler)); + _mediaService = mediaService ?? throw new ArgumentNullException(nameof(mediaService)); + } + + public void Handle(MediaCacheRefresherNotification args) + { + if (!_umbracoIndexingHandler.Enabled) + { + return; + } + + if (Suspendable.ExamineEvents.CanIndex == false) + { + return; + } + + if (args.MessageType != MessageType.RefreshByPayload) + { + throw new NotSupportedException(); + } + + foreach (var payload in (MediaCacheRefresher.JsonPayload[])args.MessageObject) + { + if (payload.ChangeTypes.HasType(TreeChangeTypes.Remove)) + { + // remove from *all* indexes + _umbracoIndexingHandler.DeleteIndexForEntity(payload.Id, false); + } + else if (payload.ChangeTypes.HasType(TreeChangeTypes.RefreshAll)) + { + // ExamineEvents does not support RefreshAll + // just ignore that payload + // so what?! + } + else // RefreshNode or RefreshBranch (maybe trashed) + { + var media = _mediaService.GetById(payload.Id); + if (media == null) + { + // gone fishing, remove entirely + _umbracoIndexingHandler.DeleteIndexForEntity(payload.Id, false); + continue; + } + + if (media.Trashed) + { + _umbracoIndexingHandler.DeleteIndexForEntity(payload.Id, true); + } + + // just that media + _umbracoIndexingHandler.ReIndexForMedia(media, !media.Trashed); + + // branch + if (payload.ChangeTypes.HasType(TreeChangeTypes.RefreshBranch)) + { + const int pageSize = 500; + var page = 0; + var total = long.MaxValue; + while (page * pageSize < total) + { + var descendants = _mediaService.GetPagedDescendants(media.Id, page++, pageSize, out total); + foreach (var descendant in descendants) + { + _umbracoIndexingHandler.ReIndexForMedia(descendant, !descendant.Trashed); + } + } + } + } + } + } + } +} diff --git a/src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.Member.cs b/src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.Member.cs new file mode 100644 index 0000000000..389b839c67 --- /dev/null +++ b/src/Umbraco.Infrastructure/Search/IndexingNotificationHandler.Member.cs @@ -0,0 +1,90 @@ +using System; +using Umbraco.Cms.Core.Cache; +using Umbraco.Cms.Core.Events; +using Umbraco.Cms.Core.Models; +using Umbraco.Cms.Core.Notifications; +using Umbraco.Cms.Core.Services; +using Umbraco.Cms.Core.Sync; + +namespace Umbraco.Cms.Infrastructure.Search +{ + public sealed class MemberIndexingNotificationHandler : INotificationHandler + { + private readonly IUmbracoIndexingHandler _umbracoIndexingHandler; + private readonly IMemberService _memberService; + + public MemberIndexingNotificationHandler(IUmbracoIndexingHandler umbracoIndexingHandler, IMemberService memberService) + { + _umbracoIndexingHandler = umbracoIndexingHandler ?? throw new ArgumentNullException(nameof(umbracoIndexingHandler)); + _memberService = memberService ?? throw new ArgumentNullException(nameof(memberService)); + } + + public void Handle(MemberCacheRefresherNotification args) + { + if (!_umbracoIndexingHandler.Enabled) + { + return; + } + + if (Suspendable.ExamineEvents.CanIndex == false) + { + return; + } + + switch (args.MessageType) + { + case MessageType.RefreshById: + var c1 = _memberService.GetById((int)args.MessageObject); + if (c1 != null) + { + _umbracoIndexingHandler.ReIndexForMember(c1); + } + break; + case MessageType.RemoveById: + + // This is triggered when the item is permanently deleted + + _umbracoIndexingHandler.DeleteIndexForEntity((int)args.MessageObject, false); + break; + case MessageType.RefreshByInstance: + if (args.MessageObject is IMember c3) + { + _umbracoIndexingHandler.ReIndexForMember(c3); + } + break; + case MessageType.RemoveByInstance: + + // This is triggered when the item is permanently deleted + + if (args.MessageObject is IMember c4) + { + _umbracoIndexingHandler.DeleteIndexForEntity(c4.Id, false); + } + break; + case MessageType.RefreshByPayload: + var payload = (MemberCacheRefresher.JsonPayload[])args.MessageObject; + foreach (var p in payload) + { + if (p.Removed) + { + _umbracoIndexingHandler.DeleteIndexForEntity(p.Id, false); + } + else + { + var m = _memberService.GetById(p.Id); + if (m != null) + { + _umbracoIndexingHandler.ReIndexForMember(m); + } + } + } + break; + case MessageType.RefreshAll: + case MessageType.RefreshByJson: + default: + //We don't support these, these message types will not fire for unpublished content + break; + } + } + } +} diff --git a/src/Umbraco.Infrastructure/Services/Implement/CacheInstructionService.cs b/src/Umbraco.Infrastructure/Services/Implement/CacheInstructionService.cs index 0a6e945a23..e0c0f56244 100644 --- a/src/Umbraco.Infrastructure/Services/Implement/CacheInstructionService.cs +++ b/src/Umbraco.Infrastructure/Services/Implement/CacheInstructionService.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Threading; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Newtonsoft.Json; @@ -22,8 +23,6 @@ namespace Umbraco.Cms.Core.Services.Implement /// public class CacheInstructionService : RepositoryService, ICacheInstructionService { - private readonly IServerRoleAccessor _serverRoleAccessor; - private readonly CacheRefresherCollection _cacheRefreshers; private readonly ICacheInstructionRepository _cacheInstructionRepository; private readonly IProfilingLogger _profilingLogger; private readonly ILogger _logger; @@ -36,16 +35,12 @@ namespace Umbraco.Cms.Core.Services.Implement IScopeProvider provider, ILoggerFactory loggerFactory, IEventMessagesFactory eventMessagesFactory, - IServerRoleAccessor serverRoleAccessor, - CacheRefresherCollection cacheRefreshers, ICacheInstructionRepository cacheInstructionRepository, IProfilingLogger profilingLogger, ILogger logger, IOptions globalSettings) : base(provider, loggerFactory, eventMessagesFactory) { - _serverRoleAccessor = serverRoleAccessor; - _cacheRefreshers = cacheRefreshers; _cacheInstructionRepository = cacheInstructionRepository; _profilingLogger = profilingLogger; _logger = logger; @@ -57,7 +52,7 @@ namespace Umbraco.Cms.Core.Services.Implement { using (IScope scope = ScopeProvider.CreateScope(autoComplete: true)) { - if (lastId == 0) + if (lastId <= 0) { var count = _cacheInstructionRepository.CountAll(); @@ -79,7 +74,6 @@ namespace Umbraco.Cms.Core.Services.Implement return false; } } - /// public bool IsInstructionCountOverLimit(int lastId, int limit, out int count) { @@ -133,22 +127,28 @@ namespace Umbraco.Cms.Core.Services.Implement new CacheInstruction(0, DateTime.UtcNow, JsonConvert.SerializeObject(instructions, Formatting.None), localIdentity, instructions.Sum(x => x.JsonIdCount)); /// - public CacheInstructionServiceProcessInstructionsResult ProcessInstructions(bool released, string localIdentity, DateTime lastPruned, int lastId) + public ProcessInstructionsResult ProcessInstructions( + CacheRefresherCollection cacheRefreshers, + ServerRole serverRole, + CancellationToken cancellationToken, + string localIdentity, + DateTime lastPruned, + int lastId) { using (_profilingLogger.DebugDuration("Syncing from database...")) using (IScope scope = ScopeProvider.CreateScope()) { - var numberOfInstructionsProcessed = ProcessDatabaseInstructions(released, localIdentity, ref lastId); + var numberOfInstructionsProcessed = ProcessDatabaseInstructions(cacheRefreshers, cancellationToken, localIdentity, ref lastId); // Check for pruning throttling. - if (released || (DateTime.UtcNow - lastPruned) <= _globalSettings.DatabaseServerMessenger.TimeBetweenPruneOperations) + if (cancellationToken.IsCancellationRequested || (DateTime.UtcNow - lastPruned) <= _globalSettings.DatabaseServerMessenger.TimeBetweenPruneOperations) { scope.Complete(); - return CacheInstructionServiceProcessInstructionsResult.AsCompleted(numberOfInstructionsProcessed, lastId); + return ProcessInstructionsResult.AsCompleted(numberOfInstructionsProcessed, lastId); } var instructionsWerePruned = false; - switch (_serverRoleAccessor.CurrentServerRole) + switch (serverRole) { case ServerRole.Single: case ServerRole.Master: @@ -160,8 +160,8 @@ namespace Umbraco.Cms.Core.Services.Implement scope.Complete(); return instructionsWerePruned - ? CacheInstructionServiceProcessInstructionsResult.AsCompletedAndPruned(numberOfInstructionsProcessed, lastId) - : CacheInstructionServiceProcessInstructionsResult.AsCompleted(numberOfInstructionsProcessed, lastId); + ? ProcessInstructionsResult.AsCompletedAndPruned(numberOfInstructionsProcessed, lastId) + : ProcessInstructionsResult.AsCompleted(numberOfInstructionsProcessed, lastId); } } @@ -172,7 +172,7 @@ namespace Umbraco.Cms.Core.Services.Implement /// Thread safety: this is NOT thread safe. Because it is NOT meant to run multi-threaded. /// /// Number of instructions processed. - private int ProcessDatabaseInstructions(bool released, string localIdentity, ref int lastId) + private int ProcessDatabaseInstructions(CacheRefresherCollection cacheRefreshers, CancellationToken cancellationToken, string localIdentity, ref int lastId) { // NOTE: // We 'could' recurse to ensure that no remaining instructions are pending in the table before proceeding but I don't think that @@ -205,7 +205,7 @@ namespace Umbraco.Cms.Core.Services.Implement { // If this flag gets set it means we're shutting down! In this case, we need to exit asap and cannot // continue processing anything otherwise we'll hold up the app domain shutdown. - if (released) + if (cancellationToken.IsCancellationRequested) { break; } @@ -227,7 +227,7 @@ namespace Umbraco.Cms.Core.Services.Implement List instructionBatch = GetAllInstructions(jsonInstructions); // Process as per-normal. - var success = ProcessDatabaseInstructions(instructionBatch, instruction, processed, released, ref lastId); + var success = ProcessDatabaseInstructions(cacheRefreshers, instructionBatch, instruction, processed, cancellationToken, ref lastId); // If they couldn't be all processed (i.e. we're shutting down) then exit. if (success == false) @@ -295,12 +295,18 @@ namespace Umbraco.Cms.Core.Services.Implement /// /// Returns true if all instructions in the batch were processed, otherwise false if they could not be due to the app being shut down /// - private bool ProcessDatabaseInstructions(IReadOnlyCollection instructionBatch, CacheInstruction instruction, HashSet processed, bool released, ref int lastId) + private bool ProcessDatabaseInstructions( + CacheRefresherCollection cacheRefreshers, + IReadOnlyCollection instructionBatch, + CacheInstruction instruction, + HashSet processed, + CancellationToken cancellationToken, + ref int lastId) { // Execute remote instructions & update lastId. try { - var result = NotifyRefreshers(instructionBatch, processed, released); + var result = NotifyRefreshers(cacheRefreshers, instructionBatch, processed, cancellationToken); if (result) { // If all instructions were processed, set the last id. @@ -330,12 +336,16 @@ namespace Umbraco.Cms.Core.Services.Implement /// /// Returns true if all instructions were processed, otherwise false if the processing was interrupted (i.e. by app shutdown). /// - private bool NotifyRefreshers(IEnumerable instructions, HashSet processed, bool released) + private bool NotifyRefreshers( + CacheRefresherCollection cacheRefreshers, + IEnumerable instructions, + HashSet processed, + CancellationToken cancellationToken) { foreach (RefreshInstruction instruction in instructions) { // Check if the app is shutting down, we need to exit if this happens. - if (released) + if (cancellationToken.IsCancellationRequested) { return false; } @@ -349,22 +359,22 @@ namespace Umbraco.Cms.Core.Services.Implement switch (instruction.RefreshType) { case RefreshMethodType.RefreshAll: - RefreshAll(instruction.RefresherId); + RefreshAll(cacheRefreshers, instruction.RefresherId); break; case RefreshMethodType.RefreshByGuid: - RefreshByGuid(instruction.RefresherId, instruction.GuidId); + RefreshByGuid(cacheRefreshers, instruction.RefresherId, instruction.GuidId); break; case RefreshMethodType.RefreshById: - RefreshById(instruction.RefresherId, instruction.IntId); + RefreshById(cacheRefreshers, instruction.RefresherId, instruction.IntId); break; case RefreshMethodType.RefreshByIds: - RefreshByIds(instruction.RefresherId, instruction.JsonIds); + RefreshByIds(cacheRefreshers, instruction.RefresherId, instruction.JsonIds); break; case RefreshMethodType.RefreshByJson: - RefreshByJson(instruction.RefresherId, instruction.JsonPayload); + RefreshByJson(cacheRefreshers, instruction.RefresherId, instruction.JsonPayload); break; case RefreshMethodType.RemoveById: - RemoveById(instruction.RefresherId, instruction.IntId); + RemoveById(cacheRefreshers, instruction.RefresherId, instruction.IntId); break; } @@ -374,48 +384,48 @@ namespace Umbraco.Cms.Core.Services.Implement return true; } - private void RefreshAll(Guid uniqueIdentifier) + private void RefreshAll(CacheRefresherCollection cacheRefreshers, Guid uniqueIdentifier) { - ICacheRefresher refresher = GetRefresher(uniqueIdentifier); + ICacheRefresher refresher = GetRefresher(cacheRefreshers, uniqueIdentifier); refresher.RefreshAll(); } - private void RefreshByGuid(Guid uniqueIdentifier, Guid id) + private void RefreshByGuid(CacheRefresherCollection cacheRefreshers, Guid uniqueIdentifier, Guid id) { - ICacheRefresher refresher = GetRefresher(uniqueIdentifier); + ICacheRefresher refresher = GetRefresher(cacheRefreshers, uniqueIdentifier); refresher.Refresh(id); } - private void RefreshById(Guid uniqueIdentifier, int id) + private void RefreshById(CacheRefresherCollection cacheRefreshers, Guid uniqueIdentifier, int id) { - ICacheRefresher refresher = GetRefresher(uniqueIdentifier); + ICacheRefresher refresher = GetRefresher(cacheRefreshers, uniqueIdentifier); refresher.Refresh(id); } - private void RefreshByIds(Guid uniqueIdentifier, string jsonIds) + private void RefreshByIds(CacheRefresherCollection cacheRefreshers, Guid uniqueIdentifier, string jsonIds) { - ICacheRefresher refresher = GetRefresher(uniqueIdentifier); + ICacheRefresher refresher = GetRefresher(cacheRefreshers, uniqueIdentifier); foreach (var id in JsonConvert.DeserializeObject(jsonIds)) { refresher.Refresh(id); } } - private void RefreshByJson(Guid uniqueIdentifier, string jsonPayload) + private void RefreshByJson(CacheRefresherCollection cacheRefreshers, Guid uniqueIdentifier, string jsonPayload) { - IJsonCacheRefresher refresher = GetJsonRefresher(uniqueIdentifier); + IJsonCacheRefresher refresher = GetJsonRefresher(cacheRefreshers, uniqueIdentifier); refresher.Refresh(jsonPayload); } - private void RemoveById(Guid uniqueIdentifier, int id) + private void RemoveById(CacheRefresherCollection cacheRefreshers, Guid uniqueIdentifier, int id) { - ICacheRefresher refresher = GetRefresher(uniqueIdentifier); + ICacheRefresher refresher = GetRefresher(cacheRefreshers, uniqueIdentifier); refresher.Remove(id); } - private ICacheRefresher GetRefresher(Guid id) + private ICacheRefresher GetRefresher(CacheRefresherCollection cacheRefreshers, Guid id) { - ICacheRefresher refresher = _cacheRefreshers[id]; + ICacheRefresher refresher = cacheRefreshers[id]; if (refresher == null) { throw new InvalidOperationException("Cache refresher with ID \"" + id + "\" does not exist."); @@ -424,7 +434,7 @@ namespace Umbraco.Cms.Core.Services.Implement return refresher; } - private IJsonCacheRefresher GetJsonRefresher(Guid id) => GetJsonRefresher(GetRefresher(id)); + private IJsonCacheRefresher GetJsonRefresher(CacheRefresherCollection cacheRefreshers, Guid id) => GetJsonRefresher(GetRefresher(cacheRefreshers, id)); private static IJsonCacheRefresher GetJsonRefresher(ICacheRefresher refresher) { diff --git a/src/Umbraco.Infrastructure/Services/Implement/ServerRegistrationService.cs b/src/Umbraco.Infrastructure/Services/Implement/ServerRegistrationService.cs index 9c03f9aabc..248de428a8 100644 --- a/src/Umbraco.Infrastructure/Services/Implement/ServerRegistrationService.cs +++ b/src/Umbraco.Infrastructure/Services/Implement/ServerRegistrationService.cs @@ -154,7 +154,7 @@ namespace Umbraco.Cms.Core.Services.Implement /// /// Gets the local server identity. /// - private string GetCurrentServerIdentity() => NetworkHelper.MachineName // eg DOMAIN\SERVER + private string GetCurrentServerIdentity() => Environment.MachineName // eg DOMAIN\SERVER + "/" + _hostingEnvironment.ApplicationId; // eg /LM/S3SVC/11/ROOT; } } diff --git a/src/Umbraco.Infrastructure/Suspendable.cs b/src/Umbraco.Infrastructure/Suspendable.cs index e96baa44e4..022a641094 100644 --- a/src/Umbraco.Infrastructure/Suspendable.cs +++ b/src/Umbraco.Infrastructure/Suspendable.cs @@ -2,7 +2,6 @@ using Microsoft.Extensions.Logging; using Umbraco.Cms.Core; using Umbraco.Cms.Core.Cache; using Umbraco.Cms.Infrastructure.Examine; -using Umbraco.Cms.Infrastructure.Search; namespace Umbraco.Cms.Infrastructure { @@ -81,7 +80,7 @@ namespace Umbraco.Cms.Infrastructure s_suspended = true; } - public static void ResumeIndexers(IndexRebuilder indexRebuilder, ILogger logger, BackgroundIndexRebuilder backgroundIndexRebuilder) + public static void ResumeIndexers(ExamineIndexRebuilder backgroundIndexRebuilder) { s_suspended = false; diff --git a/src/Umbraco.Infrastructure/Sync/BatchedDatabaseServerMessenger.cs b/src/Umbraco.Infrastructure/Sync/BatchedDatabaseServerMessenger.cs index 940ebfe0cd..9bae34cf3e 100644 --- a/src/Umbraco.Infrastructure/Sync/BatchedDatabaseServerMessenger.cs +++ b/src/Umbraco.Infrastructure/Sync/BatchedDatabaseServerMessenger.cs @@ -27,15 +27,18 @@ namespace Umbraco.Cms.Infrastructure.Sync /// public BatchedDatabaseServerMessenger( IMainDom mainDom, + CacheRefresherCollection cacheRefreshers, + IServerRoleAccessor serverRoleAccessor, ILogger logger, - DatabaseServerMessengerCallbacks callbacks, + ISyncBootStateAccessor syncBootStateAccessor, IHostingEnvironment hostingEnvironment, ICacheInstructionService cacheInstructionService, IJsonSerializer jsonSerializer, IRequestCache requestCache, IRequestAccessor requestAccessor, + LastSyncedFileManager lastSyncedFileManager, IOptions globalSettings) - : base(mainDom, logger, true, callbacks, hostingEnvironment, cacheInstructionService, jsonSerializer, globalSettings) + : base(mainDom, cacheRefreshers, serverRoleAccessor, logger, true, syncBootStateAccessor, hostingEnvironment, cacheInstructionService, jsonSerializer, lastSyncedFileManager, globalSettings) { _requestCache = requestCache; _requestAccessor = requestAccessor; diff --git a/src/Umbraco.Infrastructure/Sync/DatabaseServerMessenger.cs b/src/Umbraco.Infrastructure/Sync/DatabaseServerMessenger.cs index 0b2076a3a7..ee8793f5c9 100644 --- a/src/Umbraco.Infrastructure/Sync/DatabaseServerMessenger.cs +++ b/src/Umbraco.Infrastructure/Sync/DatabaseServerMessenger.cs @@ -1,8 +1,6 @@ using System; using System.Collections.Generic; using System.Diagnostics; -using System.Globalization; -using System.IO; using System.Linq; using System.Threading; using Microsoft.Extensions.Logging; @@ -15,7 +13,6 @@ using Umbraco.Cms.Core.Runtime; using Umbraco.Cms.Core.Serialization; using Umbraco.Cms.Core.Services; using Umbraco.Cms.Core.Sync; -using Umbraco.Extensions; namespace Umbraco.Cms.Infrastructure.Sync { @@ -31,57 +28,61 @@ namespace Umbraco.Cms.Infrastructure.Sync */ private readonly IMainDom _mainDom; + private readonly CacheRefresherCollection _cacheRefreshers; + private readonly IServerRoleAccessor _serverRoleAccessor; + private readonly ISyncBootStateAccessor _syncBootStateAccessor; private readonly ManualResetEvent _syncIdle; private readonly object _locko = new object(); private readonly IHostingEnvironment _hostingEnvironment; - - private readonly Lazy _distCacheFilePath; - private int _lastId = -1; + private readonly LastSyncedFileManager _lastSyncedFileManager; private DateTime _lastSync; private DateTime _lastPruned; - private readonly Lazy _initialized; + private readonly Lazy _initialized; private bool _syncing; - private bool _released; + private readonly CancellationTokenSource _cancellationTokenSource = new CancellationTokenSource(); + private readonly CancellationToken _cancellationToken; /// /// Initializes a new instance of the class. /// protected DatabaseServerMessenger( IMainDom mainDom, + CacheRefresherCollection cacheRefreshers, + IServerRoleAccessor serverRoleAccessor, ILogger logger, bool distributedEnabled, - DatabaseServerMessengerCallbacks callbacks, + ISyncBootStateAccessor syncBootStateAccessor, IHostingEnvironment hostingEnvironment, ICacheInstructionService cacheInstructionService, IJsonSerializer jsonSerializer, + LastSyncedFileManager lastSyncedFileManager, IOptions globalSettings) : base(distributedEnabled) { + _cancellationToken = _cancellationTokenSource.Token; _mainDom = mainDom; + _cacheRefreshers = cacheRefreshers; + _serverRoleAccessor = serverRoleAccessor; _hostingEnvironment = hostingEnvironment; Logger = logger; - Callbacks = callbacks ?? throw new ArgumentNullException(nameof(callbacks)); + _syncBootStateAccessor = syncBootStateAccessor; CacheInstructionService = cacheInstructionService; JsonSerializer = jsonSerializer; + _lastSyncedFileManager = lastSyncedFileManager; GlobalSettings = globalSettings.Value; _lastPruned = _lastSync = DateTime.UtcNow; _syncIdle = new ManualResetEvent(true); - _distCacheFilePath = new Lazy(() => GetDistCacheFilePath(hostingEnvironment)); // See notes on _localIdentity - LocalIdentity = NetworkHelper.MachineName // eg DOMAIN\SERVER + LocalIdentity = Environment.MachineName // eg DOMAIN\SERVER + "/" + hostingEnvironment.ApplicationId // eg /LM/S3SVC/11/ROOT + " [P" + Process.GetCurrentProcess().Id // eg 1234 + "/D" + AppDomain.CurrentDomain.Id // eg 22 + "] " + Guid.NewGuid().ToString("N").ToUpper(); // make it truly unique - _initialized = new Lazy(EnsureInitialized); + _initialized = new Lazy(InitializeWithMainDom); } - private string DistCacheFilePath => _distCacheFilePath.Value; - - public DatabaseServerMessengerCallbacks Callbacks { get; } - public GlobalSettings GlobalSettings { get; } protected ILogger Logger { get; } @@ -102,12 +103,17 @@ namespace Umbraco.Cms.Infrastructure.Sync /// protected string LocalIdentity { get; } + /// + /// Returns true if initialization was successfull (i.e. Is MainDom) + /// + protected bool EnsureInitialized() => _initialized.Value.HasValue; + #region Messenger // we don't care if there are servers listed or not, // if distributed call is enabled we will make the call protected override bool RequiresDistributed(ICacheRefresher refresher, MessageType dispatchType) - => _initialized.Value && DistributedEnabled; + => EnsureInitialized() && DistributedEnabled; protected override void DeliverRemote( ICacheRefresher refresher, @@ -134,7 +140,7 @@ namespace Umbraco.Cms.Infrastructure.Sync /// /// Boots the messenger. /// - private bool EnsureInitialized() + private SyncBootState? InitializeWithMainDom() { // weight:10, must release *before* the published snapshot service, because once released // the service will *not* be able to properly handle our notifications anymore. @@ -145,15 +151,15 @@ namespace Umbraco.Cms.Infrastructure.Sync { lock (_locko) { - _released = true; // no more syncs + _cancellationTokenSource.Cancel(); // no more syncs } - // Wait a max of 5 seconds and then return, so that we don't block + // Wait a max of 3 seconds and then return, so that we don't block // the entire MainDom callbacks chain and prevent the AppDomain from // properly releasing MainDom - a timeout here means that one refresher // is taking too much time processing, however when it's done we will // not update lastId and stop everything. - var idle = _syncIdle.WaitOne(5000); + var idle = _syncIdle.WaitOne(3000); if (idle == false) { Logger.LogWarning("The wait lock timed out, application is shutting down. The current instruction batch will be re-processed."); @@ -163,17 +169,11 @@ namespace Umbraco.Cms.Infrastructure.Sync if (registered == false) { - return false; + // return null if we cannot initialize + return null; } - ReadLastSynced(); // get _lastId - - if (CacheInstructionService.IsColdBootRequired(_lastId)) - { - _lastId = -1; // reset _lastId if instructions are missing - } - - return Initialize(); // boot + return InitializeColdBootState(); } // @@ -183,70 +183,32 @@ namespace Umbraco.Cms.Infrastructure.Sync /// Thread safety: this is NOT thread safe. Because it is NOT meant to run multi-threaded. /// Callers MUST ensure thread-safety. /// - private bool Initialize() + private SyncBootState InitializeColdBootState() { lock (_locko) { - if (_released) + if (_cancellationToken.IsCancellationRequested) { - return false; + return SyncBootState.Unknown; } - var coldboot = false; + SyncBootState syncState = _syncBootStateAccessor.GetSyncBootState(); - // Never synced before. - if (_lastId < 0) - { - // We haven't synced - in this case we aren't going to sync the whole thing, we will assume this is a new - // server and it will need to rebuild it's own caches, e.g. Lucene or the XML cache file. - Logger.LogWarning("No last synced Id found, this generally means this is a new server/install." - + " The server will build its caches and indexes, and then adjust its last synced Id to the latest found in" - + " the database and maintain cache updates based on that Id."); - - coldboot = true; - } - else - { - // Check for how many instructions there are to process, each row contains a count of the number of instructions contained in each - // row so we will sum these numbers to get the actual count. - var limit = GlobalSettings.DatabaseServerMessenger.MaxProcessingInstructionCount; - if (CacheInstructionService.IsInstructionCountOverLimit(_lastId, limit, out int count)) - { - // Too many instructions, proceed to cold boot. - Logger.LogWarning( - "The instruction count ({InstructionCount}) exceeds the specified MaxProcessingInstructionCount ({MaxProcessingInstructionCount})." - + " The server will skip existing instructions, rebuild its caches and indexes entirely, adjust its last synced Id" - + " to the latest found in the database and maintain cache updates based on that Id.", - count, limit); - - coldboot = true; - } - } - - if (coldboot) + if (syncState == SyncBootState.ColdBoot) { // Get the last id in the db and store it. // Note: Do it BEFORE initializing otherwise some instructions might get lost // when doing it before. Some instructions might run twice but this is not an issue. var maxId = CacheInstructionService.GetMaxInstructionId(); - // If there is a max currently, or if we've never synced. - if (maxId > 0 || _lastId < 0) + // if there is a max currently, or if we've never synced + if (maxId > 0 || _lastSyncedFileManager.LastSyncedId < 0) { - SaveLastSynced(maxId); - } - - // Execute initializing callbacks. - if (Callbacks.InitializingCallbacks != null) - { - foreach (Action callback in Callbacks.InitializingCallbacks) - { - callback(); - } + _lastSyncedFileManager.SaveLastSyncedId(maxId); } } - return true; + return syncState; } } @@ -255,7 +217,7 @@ namespace Umbraco.Cms.Infrastructure.Sync /// public override void Sync() { - if (!_initialized.Value) + if (!EnsureInitialized()) { return; } @@ -268,7 +230,7 @@ namespace Umbraco.Cms.Infrastructure.Sync } // Don't continue if we are released - if (_released) + if (_cancellationToken.IsCancellationRequested) { return; } @@ -286,7 +248,14 @@ namespace Umbraco.Cms.Infrastructure.Sync try { - CacheInstructionServiceProcessInstructionsResult result = CacheInstructionService.ProcessInstructions(_released, LocalIdentity, _lastPruned, _lastId); + ProcessInstructionsResult result = CacheInstructionService.ProcessInstructions( + _cacheRefreshers, + _serverRoleAccessor.CurrentServerRole, + _cancellationToken, + LocalIdentity, + _lastPruned, + _lastSyncedFileManager.LastSyncedId); + if (result.InstructionsWerePruned) { _lastPruned = _lastSync; @@ -294,7 +263,7 @@ namespace Umbraco.Cms.Infrastructure.Sync if (result.LastId > 0) { - SaveLastSynced(result.LastId); + _lastSyncedFileManager.SaveLastSyncedId(result.LastId); } } finally @@ -309,60 +278,6 @@ namespace Umbraco.Cms.Infrastructure.Sync } } - /// - /// Reads the last-synced id from file into memory. - /// - /// - /// Thread safety: this is NOT thread safe. Because it is NOT meant to run multi-threaded. - /// - private void ReadLastSynced() - { - if (File.Exists(DistCacheFilePath) == false) - { - return; - } - - var content = File.ReadAllText(DistCacheFilePath); - if (int.TryParse(content, out var last)) - { - _lastId = last; - } - } - - /// - /// Updates the in-memory last-synced id and persists it to file. - /// - /// The id. - /// - /// Thread safety: this is NOT thread safe. Because it is NOT meant to run multi-threaded. - /// - private void SaveLastSynced(int id) - { - File.WriteAllText(DistCacheFilePath, id.ToString(CultureInfo.InvariantCulture)); - _lastId = id; - } - - private string GetDistCacheFilePath(IHostingEnvironment hostingEnvironment) - { - var fileName = _hostingEnvironment.ApplicationId.ReplaceNonAlphanumericChars(string.Empty) + "-lastsynced.txt"; - - var distCacheFilePath = Path.Combine(hostingEnvironment.LocalTempPath, "DistCache", fileName); - - //ensure the folder exists - var folder = Path.GetDirectoryName(distCacheFilePath); - if (folder == null) - { - throw new InvalidOperationException("The folder could not be determined for the file " + distCacheFilePath); - } - - if (Directory.Exists(folder) == false) - { - Directory.CreateDirectory(folder); - } - - return distCacheFilePath; - } - #endregion } } diff --git a/src/Umbraco.Infrastructure/Sync/LastSyncedFileManager.cs b/src/Umbraco.Infrastructure/Sync/LastSyncedFileManager.cs new file mode 100644 index 0000000000..3b3351fd93 --- /dev/null +++ b/src/Umbraco.Infrastructure/Sync/LastSyncedFileManager.cs @@ -0,0 +1,89 @@ +using System; +using System.Globalization; +using System.IO; +using System.Threading; +using Umbraco.Cms.Core.Hosting; +using Umbraco.Extensions; + +namespace Umbraco.Cms.Infrastructure.Sync +{ + public sealed class LastSyncedFileManager + { + private string _distCacheFile; + private bool _lastIdReady; + private object _lastIdLock; + private int _lastId; + private readonly IHostingEnvironment _hostingEnvironment; + + public LastSyncedFileManager(IHostingEnvironment hostingEnvironment) + => _hostingEnvironment = hostingEnvironment; + + /// + /// Persists the last-synced id to file. + /// + /// The id. + public void SaveLastSyncedId(int id) + { + lock (_lastIdLock) + { + if (!_lastIdReady) + { + throw new InvalidOperationException("Cannot save the last synced id before it is read"); + } + + File.WriteAllText(DistCacheFilePath, id.ToString(CultureInfo.InvariantCulture)); + _lastId = id; + } + } + + /// + /// Returns the last-synced id. + /// + public int LastSyncedId => LazyInitializer.EnsureInitialized( + ref _lastId, + ref _lastIdReady, + ref _lastIdLock, + () => + { + // On first load, read from file, else it will return the in-memory _lastId value + + var distCacheFilePath = DistCacheFilePath; + + if (File.Exists(distCacheFilePath)) + { + var content = File.ReadAllText(distCacheFilePath); + if (int.TryParse(content, out var last)) + { + return last; + } + } + + return -1; + }); + + /// + /// Gets the dist cache file path (once). + /// + /// + public string DistCacheFilePath => LazyInitializer.EnsureInitialized(ref _distCacheFile, () => + { + var fileName = (Environment.MachineName + _hostingEnvironment.ApplicationId).GenerateHash() + "-lastsynced.txt"; + + var distCacheFilePath = Path.Combine(_hostingEnvironment.LocalTempPath, "DistCache", fileName); + + //ensure the folder exists + var folder = Path.GetDirectoryName(distCacheFilePath); + if (folder == null) + { + throw new InvalidOperationException("The folder could not be determined for the file " + distCacheFilePath); + } + + if (Directory.Exists(folder) == false) + { + Directory.CreateDirectory(folder); + } + + return distCacheFilePath; + }); + } +} diff --git a/src/Umbraco.Infrastructure/Sync/SyncBootStateAccessor.cs b/src/Umbraco.Infrastructure/Sync/SyncBootStateAccessor.cs new file mode 100644 index 0000000000..9a77c57965 --- /dev/null +++ b/src/Umbraco.Infrastructure/Sync/SyncBootStateAccessor.cs @@ -0,0 +1,84 @@ +using System.Threading; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Umbraco.Cms.Core.Configuration.Models; +using Umbraco.Cms.Core.Services; +using Umbraco.Cms.Core.Sync; + +namespace Umbraco.Cms.Infrastructure.Sync +{ + public class SyncBootStateAccessor : ISyncBootStateAccessor + { + private readonly ILogger _logger; + private readonly LastSyncedFileManager _lastSyncedFileManager; + private readonly GlobalSettings _globalSettings; + private readonly ICacheInstructionService _cacheInstructionService; + + private SyncBootState _syncBootState; + private bool _syncBootStateReady; + private object _syncBootStateLock; + + public SyncBootStateAccessor( + ILogger logger, + LastSyncedFileManager lastSyncedFileManager, + IOptions globalSettings, + ICacheInstructionService cacheInstructionService) + { + _logger = logger; + _lastSyncedFileManager = lastSyncedFileManager; + _globalSettings = globalSettings.Value; + _cacheInstructionService = cacheInstructionService; + } + + public SyncBootState GetSyncBootState() + => LazyInitializer.EnsureInitialized( + ref _syncBootState, + ref _syncBootStateReady, + ref _syncBootStateLock, + () => InitializeColdBootState(_lastSyncedFileManager.LastSyncedId)); + + private SyncBootState InitializeColdBootState(int lastId) + { + var coldboot = false; + + // Never synced before. + if (lastId < 0) + { + // We haven't synced - in this case we aren't going to sync the whole thing, we will assume this is a new + // server and it will need to rebuild it's own caches, e.g. Lucene or the XML cache file. + _logger.LogWarning("No last synced Id found, this generally means this is a new server/install. " + + "A cold boot will be triggered."); + + coldboot = true; + } + else + { + if (_cacheInstructionService.IsColdBootRequired(lastId)) + { + _logger.LogWarning("Last synced Id found {LastSyncedId} but was not found in the database. This generally means this server/install " + + " has been idle for too long and the instructions in the database have been pruned. A cold boot will be triggered.", lastId); + + coldboot = true; + } + else + { + // Check for how many instructions there are to process, each row contains a count of the number of instructions contained in each + // row so we will sum these numbers to get the actual count. + var limit = _globalSettings.DatabaseServerMessenger.MaxProcessingInstructionCount; + if (_cacheInstructionService.IsInstructionCountOverLimit(lastId, limit, out int count)) + { + // Too many instructions, proceed to cold boot. + _logger.LogWarning( + "The instruction count ({InstructionCount}) exceeds the specified MaxProcessingInstructionCount ({MaxProcessingInstructionCount}). " + + "A cold boot will be triggered.", + count, limit); + + coldboot = true; + } + } + } + + return coldboot ? SyncBootState.ColdBoot : SyncBootState.WarmBoot; + } + } +} diff --git a/src/Umbraco.Infrastructure/Umbraco.Infrastructure.csproj b/src/Umbraco.Infrastructure/Umbraco.Infrastructure.csproj index 9dbacfa267..54970e58a9 100644 --- a/src/Umbraco.Infrastructure/Umbraco.Infrastructure.csproj +++ b/src/Umbraco.Infrastructure/Umbraco.Infrastructure.csproj @@ -49,7 +49,7 @@ - + all diff --git a/src/Umbraco.PublishedCache.NuCache/DependencyInjection/UmbracoBuilderExtensions.cs b/src/Umbraco.PublishedCache.NuCache/DependencyInjection/UmbracoBuilderExtensions.cs index 82e62b2328..113a2245d8 100644 --- a/src/Umbraco.PublishedCache.NuCache/DependencyInjection/UmbracoBuilderExtensions.cs +++ b/src/Umbraco.PublishedCache.NuCache/DependencyInjection/UmbracoBuilderExtensions.cs @@ -29,9 +29,6 @@ namespace Umbraco.Extensions // must register default options, required in the service ctor builder.Services.TryAddTransient(factory => new PublishedSnapshotServiceOptions()); builder.SetPublishedSnapshotService(); - - // Add as itself - builder.Services.TryAddSingleton(); builder.Services.TryAddSingleton(); // replace this service since we want to improve the content/media diff --git a/src/Umbraco.PublishedCache.NuCache/PublishedSnapshot.cs b/src/Umbraco.PublishedCache.NuCache/PublishedSnapshot.cs index ead279a199..fc4c64d552 100644 --- a/src/Umbraco.PublishedCache.NuCache/PublishedSnapshot.cs +++ b/src/Umbraco.PublishedCache.NuCache/PublishedSnapshot.cs @@ -14,9 +14,9 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache #region Constructors - public PublishedSnapshot(PublishedSnapshotService service, bool defaultPreview) + public PublishedSnapshot(IPublishedSnapshotService service, bool defaultPreview) { - _service = service; + _service = service as PublishedSnapshotService; _defaultPreview = defaultPreview; } @@ -38,7 +38,17 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache } } - private PublishedSnapshotElements Elements => _elements ?? (_elements = _service.GetElements(_defaultPreview)); + private PublishedSnapshotElements Elements + { + get + { + if (_service == null) + { + throw new InvalidOperationException($"The {typeof(PublishedSnapshot)} cannot be used when the {typeof(IPublishedSnapshotService)} is not the default type {typeof(PublishedSnapshotService)}"); + } + return _elements ??= _service.GetElements(_defaultPreview); + } + } public void Resync() { diff --git a/src/Umbraco.PublishedCache.NuCache/PublishedSnapshotService.cs b/src/Umbraco.PublishedCache.NuCache/PublishedSnapshotService.cs index 916fb2da5e..9c08a2fc5a 100644 --- a/src/Umbraco.PublishedCache.NuCache/PublishedSnapshotService.cs +++ b/src/Umbraco.PublishedCache.NuCache/PublishedSnapshotService.cs @@ -19,6 +19,7 @@ using Umbraco.Cms.Core.Runtime; using Umbraco.Cms.Core.Scoping; using Umbraco.Cms.Core.Services; using Umbraco.Cms.Core.Services.Changes; +using Umbraco.Cms.Core.Sync; using Umbraco.Cms.Infrastructure.PublishedCache.DataSource; using Umbraco.Cms.Infrastructure.PublishedCache.Persistence; using Umbraco.Extensions; @@ -29,6 +30,9 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache { internal class PublishedSnapshotService : IPublishedSnapshotService { + private readonly PublishedSnapshotServiceOptions _options; + private readonly ISyncBootStateAccessor _syncBootStateAccessor; + private readonly IMainDom _mainDom; private readonly ServiceContext _serviceContext; private readonly IPublishedContentTypeFactory _publishedContentTypeFactory; private readonly IPublishedSnapshotAccessor _publishedSnapshotAccessor; @@ -39,7 +43,6 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache private readonly ILogger _logger; private readonly ILoggerFactory _loggerFactory; private readonly GlobalSettings _globalSettings; - private readonly IEntityXmlSerializer _entitySerializer; private readonly IPublishedModelFactory _publishedModelFactory; private readonly IDefaultCultureAccessor _defaultCultureAccessor; private readonly IHostingEnvironment _hostingEnvironment; @@ -49,9 +52,9 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache private bool _isReadSet; private object _isReadyLock; - private readonly ContentStore _contentStore; - private readonly ContentStore _mediaStore; - private readonly SnapDictionary _domainStore; + private ContentStore _contentStore; + private ContentStore _mediaStore; + private SnapDictionary _domainStore; private readonly object _storesLock = new object(); private readonly object _elementsLock = new object(); @@ -73,6 +76,7 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache public PublishedSnapshotService( PublishedSnapshotServiceOptions options, + ISyncBootStateAccessor syncBootStateAccessor, IMainDom mainDom, ServiceContext serviceContext, IPublishedContentTypeFactory publishedContentTypeFactory, @@ -84,11 +88,13 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache INuCacheContentService publishedContentService, IDefaultCultureAccessor defaultCultureAccessor, IOptions globalSettings, - IEntityXmlSerializer entitySerializer, IPublishedModelFactory publishedModelFactory, IHostingEnvironment hostingEnvironment, IOptions config) { + _options = options; + _syncBootStateAccessor = syncBootStateAccessor; + _mainDom = mainDom; _serviceContext = serviceContext; _publishedContentTypeFactory = publishedContentTypeFactory; _publishedSnapshotAccessor = publishedSnapshotAccessor; @@ -102,41 +108,7 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache _globalSettings = globalSettings.Value; _hostingEnvironment = hostingEnvironment; _config = config.Value; - - // we need an Xml serializer here so that the member cache can support XPath, - // for members this is done by navigating the serialized-to-xml member - _entitySerializer = entitySerializer; _publishedModelFactory = publishedModelFactory; - - // lock this entire call, we only want a single thread to be accessing the stores at once and within - // the call below to mainDom.Register, a callback may occur on a threadpool thread to MainDomRelease - // at the same time as we are trying to write to the stores. MainDomRelease also locks on _storesLock so - // it will not be able to close the stores until we are done populating (if the store is empty) - lock (_storesLock) - { - if (!options.IgnoreLocalDb) - { - mainDom.Register(MainDomRegister, MainDomRelease); - - // stores are created with a db so they can write to it, but they do not read from it, - // stores need to be populated, happens in OnResolutionFrozen which uses _localDbExists to - // figure out whether it can read the databases or it should populate them from sql - - _logger.LogInformation("Creating the content store, localContentDbExists? {LocalContentDbExists}", _localContentDbExists); - _contentStore = new ContentStore(_publishedSnapshotAccessor, _variationContextAccessor, _loggerFactory.CreateLogger("ContentStore"), _loggerFactory, _publishedModelFactory, _localContentDb); - _logger.LogInformation("Creating the media store, localMediaDbExists? {LocalMediaDbExists}", _localMediaDbExists); - _mediaStore = new ContentStore(_publishedSnapshotAccessor, _variationContextAccessor, _loggerFactory.CreateLogger("ContentStore"), _loggerFactory, _publishedModelFactory, _localMediaDb); - } - else - { - _logger.LogInformation("Creating the content store (local db ignored)"); - _contentStore = new ContentStore(_publishedSnapshotAccessor, _variationContextAccessor, _loggerFactory.CreateLogger("ContentStore"), _loggerFactory, _publishedModelFactory); - _logger.LogInformation("Creating the media store (local db ignored)"); - _mediaStore = new ContentStore(_publishedSnapshotAccessor, _variationContextAccessor, _loggerFactory.CreateLogger("ContentStore"), _loggerFactory, _publishedModelFactory); - } - - _domainStore = new SnapDictionary(); - } } protected PublishedSnapshot CurrentPublishedSnapshot => (PublishedSnapshot)_publishedSnapshotAccessor.PublishedSnapshot; @@ -144,13 +116,29 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache // NOTE: These aren't used within this object but are made available internally to improve the IdKey lookup performance // when nucache is enabled. // TODO: Does this need to be here? - internal int GetDocumentId(Guid udi) => GetId(_contentStore, udi); + internal int GetDocumentId(Guid udi) + { + EnsureCaches(); + return GetId(_contentStore, udi); + } - internal int GetMediaId(Guid udi) => GetId(_mediaStore, udi); + internal int GetMediaId(Guid udi) + { + EnsureCaches(); + return GetId(_mediaStore, udi); + } - internal Guid GetDocumentUid(int id) => GetUid(_contentStore, id); + internal Guid GetDocumentUid(int id) + { + EnsureCaches(); + return GetUid(_contentStore, id); + } - internal Guid GetMediaUid(int id) => GetUid(_mediaStore, id); + internal Guid GetMediaUid(int id) + { + EnsureCaches(); + return GetUid(_mediaStore, id); + } private int GetId(ContentStore store, Guid uid) => store.LiveSnapshot.Get(uid)?.Id ?? 0; @@ -249,7 +237,7 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache } /// - /// Populates the stores + /// Lazily populates the stores only when they are first requested /// internal void EnsureCaches() => LazyInitializer.EnsureInitialized( ref _isReady, @@ -257,15 +245,43 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache ref _isReadyLock, () => { - // even though we are ready locked here we want to ensure that the stores lock is also locked + // lock this entire call, we only want a single thread to be accessing the stores at once and within + // the call below to mainDom.Register, a callback may occur on a threadpool thread to MainDomRelease + // at the same time as we are trying to write to the stores. MainDomRelease also locks on _storesLock so + // it will not be able to close the stores until we are done populating (if the store is empty) lock (_storesLock) { + if (!_options.IgnoreLocalDb) + { + _mainDom.Register(MainDomRegister, MainDomRelease); + + // stores are created with a db so they can write to it, but they do not read from it, + // stores need to be populated, happens in OnResolutionFrozen which uses _localDbExists to + // figure out whether it can read the databases or it should populate them from sql + + _logger.LogInformation("Creating the content store, localContentDbExists? {LocalContentDbExists}", _localContentDbExists); + _contentStore = new ContentStore(_publishedSnapshotAccessor, _variationContextAccessor, _loggerFactory.CreateLogger("ContentStore"), _loggerFactory, _publishedModelFactory, _localContentDb); + _logger.LogInformation("Creating the media store, localMediaDbExists? {LocalMediaDbExists}", _localMediaDbExists); + _mediaStore = new ContentStore(_publishedSnapshotAccessor, _variationContextAccessor, _loggerFactory.CreateLogger("ContentStore"), _loggerFactory, _publishedModelFactory, _localMediaDb); + } + else + { + _logger.LogInformation("Creating the content store (local db ignored)"); + _contentStore = new ContentStore(_publishedSnapshotAccessor, _variationContextAccessor, _loggerFactory.CreateLogger("ContentStore"), _loggerFactory, _publishedModelFactory); + _logger.LogInformation("Creating the media store (local db ignored)"); + _mediaStore = new ContentStore(_publishedSnapshotAccessor, _variationContextAccessor, _loggerFactory.CreateLogger("ContentStore"), _loggerFactory, _publishedModelFactory); + } + + _domainStore = new SnapDictionary(); + var okContent = false; var okMedia = false; + SyncBootState bootState = _syncBootStateAccessor.GetSyncBootState(); + try { - if (_localContentDbExists) + if (bootState != SyncBootState.ColdBoot && _localContentDbExists) { okContent = LockAndLoadContent(() => LoadContentFromLocalDbLocked(true)); if (!okContent) @@ -274,7 +290,7 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache } } - if (_localMediaDbExists) + if (bootState != SyncBootState.ColdBoot && _localMediaDbExists) { okMedia = LockAndLoadMedia(() => LoadMediaFromLocalDbLocked(true)); if (!okMedia) diff --git a/src/Umbraco.PublishedCache.NuCache/PublishedSnapshotStatus.cs b/src/Umbraco.PublishedCache.NuCache/PublishedSnapshotStatus.cs index df4f803006..6a75e3e021 100644 --- a/src/Umbraco.PublishedCache.NuCache/PublishedSnapshotStatus.cs +++ b/src/Umbraco.PublishedCache.NuCache/PublishedSnapshotStatus.cs @@ -11,9 +11,9 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache private readonly PublishedSnapshotService _service; private readonly INuCacheContentService _publishedContentService; - public PublishedSnapshotStatus(PublishedSnapshotService service, INuCacheContentService publishedContentService) + public PublishedSnapshotStatus(IPublishedSnapshotService service, INuCacheContentService publishedContentService) { - _service = service; + _service = service as PublishedSnapshotService; _publishedContentService = publishedContentService; } @@ -23,6 +23,12 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache /// public string GetStatus() { + if (_service == null) + { + return $"The current {typeof(IPublishedSnapshotService)} is not the default type. A status cannot be determined."; + } + + // TODO: This should be private _service.EnsureCaches(); var dbCacheIsOk = _publishedContentService.VerifyContentDbCache() diff --git a/src/Umbraco.Core/TaskHelper.cs b/src/Umbraco.Tests.Common/TaskHelper.cs similarity index 95% rename from src/Umbraco.Core/TaskHelper.cs rename to src/Umbraco.Tests.Common/TaskHelper.cs index ba9f865eba..8b22f7b47d 100644 --- a/src/Umbraco.Core/TaskHelper.cs +++ b/src/Umbraco.Tests.Common/TaskHelper.cs @@ -7,7 +7,7 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -namespace Umbraco.Cms.Core +namespace Umbraco.Cms.Tests.Common { /// /// Helper class to not repeat common patterns with Task. @@ -24,7 +24,7 @@ namespace Umbraco.Cms.Core public void RunBackgroundTask(Func fn) => ExecuteBackgroundTask(fn); // for tests, returning the Task as a public API indicates it can be awaited that is not what we want to do - internal Task ExecuteBackgroundTask(Func fn) + public Task ExecuteBackgroundTask(Func fn) { // it is also possible to use UnsafeQueueUserWorkItem which does not flow the execution context, // however that seems more difficult to use for async operations. @@ -45,7 +45,7 @@ namespace Umbraco.Cms.Core public void RunLongRunningBackgroundTask(Func fn) => ExecuteLongRunningBackgroundTask(fn); // for tests, returning the Task as a public API indicates it can be awaited that is not what we want to do - internal Task ExecuteLongRunningBackgroundTask(Func fn) + public Task ExecuteLongRunningBackgroundTask(Func fn) { // it is also possible to use UnsafeQueueUserWorkItem which does not flow the execution context, // however that seems more difficult to use for async operations. diff --git a/src/Umbraco.Tests.Integration/Implementations/TestHostingEnvironment.cs b/src/Umbraco.Tests.Common/Testing/TestHostingEnvironment.cs similarity index 62% rename from src/Umbraco.Tests.Integration/Implementations/TestHostingEnvironment.cs rename to src/Umbraco.Tests.Common/Testing/TestHostingEnvironment.cs index 8980a91cff..e34161a3c2 100644 --- a/src/Umbraco.Tests.Integration/Implementations/TestHostingEnvironment.cs +++ b/src/Umbraco.Tests.Common/Testing/TestHostingEnvironment.cs @@ -7,15 +7,22 @@ using Umbraco.Cms.Core.Configuration.Models; using Umbraco.Cms.Web.Common.AspNetCore; using IHostingEnvironment = Umbraco.Cms.Core.Hosting.IHostingEnvironment; -namespace Umbraco.Cms.Tests.Integration.Implementations +namespace Umbraco.Cms.Tests.Common.Testing { - public class TestHostingEnvironment : AspNetCoreHostingEnvironment, Cms.Core.Hosting.IHostingEnvironment + public class TestHostingEnvironment : AspNetCoreHostingEnvironment, IHostingEnvironment { - public TestHostingEnvironment(IOptionsMonitor hostingSettings,IOptionsMonitor webRoutingSettings, IWebHostEnvironment webHostEnvironment) - : base(hostingSettings,webRoutingSettings, webHostEnvironment) + public TestHostingEnvironment( + IOptionsMonitor hostingSettings, + IOptionsMonitor webRoutingSettings, + IWebHostEnvironment webHostEnvironment) + : base(null, hostingSettings, webRoutingSettings, webHostEnvironment) { } + // override + string IHostingEnvironment.ApplicationId { get; } = "TestApplication"; + + /// /// Gets a value indicating whether we are hosted. /// diff --git a/src/Umbraco.Tests.Common/Umbraco.Tests.Common.csproj b/src/Umbraco.Tests.Common/Umbraco.Tests.Common.csproj index 373c319218..58527c4cfa 100644 --- a/src/Umbraco.Tests.Common/Umbraco.Tests.Common.csproj +++ b/src/Umbraco.Tests.Common/Umbraco.Tests.Common.csproj @@ -1,7 +1,7 @@ - + - netstandard2.0 + net5.0 Umbraco.Cms.Tests.Common Umbraco.Cms.Tests Umbraco CMS Test Tools @@ -24,5 +24,6 @@ + diff --git a/src/Umbraco.Tests.Integration/ComponentRuntimeTests.cs b/src/Umbraco.Tests.Integration/ComponentRuntimeTests.cs index ddac52872f..baa194b17e 100644 --- a/src/Umbraco.Tests.Integration/ComponentRuntimeTests.cs +++ b/src/Umbraco.Tests.Integration/ComponentRuntimeTests.cs @@ -10,6 +10,7 @@ using Umbraco.Cms.Core.Runtime; using Umbraco.Cms.Core.Services; using Umbraco.Cms.Tests.Common.Testing; using Umbraco.Cms.Tests.Integration.Testing; +using Umbraco.Extensions; namespace Umbraco.Cms.Tests.Integration { @@ -18,7 +19,11 @@ namespace Umbraco.Cms.Tests.Integration public class ComponentRuntimeTests : UmbracoIntegrationTest { // ensure composers are added - protected override void CustomTestSetup(IUmbracoBuilder builder) => builder.AddComposers(); + protected override void CustomTestSetup(IUmbracoBuilder builder) + { + builder.AddNuCache(); + builder.AddComposers(); + } /// /// This will boot up umbraco with components enabled to show they initialize and shutdown diff --git a/src/Umbraco.Tests.Integration/DependencyInjection/UmbracoBuilderExtensions.cs b/src/Umbraco.Tests.Integration/DependencyInjection/UmbracoBuilderExtensions.cs index 5cc94ce4c9..bb0da4d08a 100644 --- a/src/Umbraco.Tests.Integration/DependencyInjection/UmbracoBuilderExtensions.cs +++ b/src/Umbraco.Tests.Integration/DependencyInjection/UmbracoBuilderExtensions.cs @@ -2,8 +2,11 @@ // See LICENSE for more details. using System; +using System.Collections.Generic; using System.IO; using System.Linq; +using Examine; +using Examine.Lucene.Directories; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -21,7 +24,6 @@ using Umbraco.Cms.Core.WebAssets; using Umbraco.Cms.Infrastructure.Examine; using Umbraco.Cms.Infrastructure.HostedServices; using Umbraco.Cms.Infrastructure.PublishedCache; -using Umbraco.Cms.Infrastructure.Search; using Umbraco.Cms.Tests.Common.TestHelpers.Stubs; using Umbraco.Cms.Tests.Integration.Implementations; using Umbraco.Extensions; @@ -43,7 +45,7 @@ namespace Umbraco.Cms.Tests.Integration.DependencyInjection builder.Services.AddUnique(Mock.Of()); builder.Services.AddUnique(testHelper.MainDom); - builder.Services.AddUnique(); + builder.Services.AddUnique(); builder.Services.AddUnique(factory => Mock.Of()); // we don't want persisted nucache files in tests @@ -51,7 +53,7 @@ namespace Umbraco.Cms.Tests.Integration.DependencyInjection #if IS_WINDOWS // ensure all lucene indexes are using RAM directory (no file system) - builder.Services.AddUnique(); + builder.Services.AddUnique(); #endif // replace this service so that it can lookup the correct file locations @@ -97,18 +99,18 @@ namespace Umbraco.Cms.Tests.Integration.DependencyInjection } // replace the default so there is no background index rebuilder - private class TestBackgroundIndexRebuilder : BackgroundIndexRebuilder + private class TestBackgroundIndexRebuilder : ExamineIndexRebuilder { - public TestBackgroundIndexRebuilder( - IMainDom mainDom, - ILogger logger, - IndexRebuilder indexRebuilder, - IBackgroundTaskQueue backgroundTaskQueue) - : base(mainDom, logger, indexRebuilder, backgroundTaskQueue) + public TestBackgroundIndexRebuilder(IMainDom mainDom, IRuntimeState runtimeState, ILogger logger, IExamineManager examineManager, IEnumerable populators, IBackgroundTaskQueue backgroundTaskQueue) : base(mainDom, runtimeState, logger, examineManager, populators, backgroundTaskQueue) { } - public override void RebuildIndexes(bool onlyEmptyIndexes, TimeSpan? delay = null) + public override void RebuildIndex(string indexName, TimeSpan? delay = null, bool useBackgroundThread = true) + { + // noop + } + + public override void RebuildIndexes(bool onlyEmptyIndexes, TimeSpan? delay = null, bool useBackgroundThread = true) { // noop } diff --git a/src/Umbraco.Tests.Integration/Implementations/TestHelper.cs b/src/Umbraco.Tests.Integration/Implementations/TestHelper.cs index 8e897011d2..90c5e0eb02 100644 --- a/src/Umbraco.Tests.Integration/Implementations/TestHelper.cs +++ b/src/Umbraco.Tests.Integration/Implementations/TestHelper.cs @@ -12,8 +12,10 @@ using System.Reflection; using System.Threading; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.FileProviders; using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Hosting.Internal; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Moq; @@ -31,6 +33,7 @@ using Umbraco.Cms.Core.PropertyEditors; using Umbraco.Cms.Core.Runtime; using Umbraco.Cms.Infrastructure.Persistence; using Umbraco.Cms.Tests.Common; +using Umbraco.Cms.Tests.Common.Testing; using Umbraco.Cms.Web.Common.AspNetCore; using Umbraco.Extensions; using File = System.IO.File; diff --git a/src/Umbraco.Tests.Integration/TestServerTest/UmbracoTestServerTestBase.cs b/src/Umbraco.Tests.Integration/TestServerTest/UmbracoTestServerTestBase.cs index 635a17a2b1..b91a034420 100644 --- a/src/Umbraco.Tests.Integration/TestServerTest/UmbracoTestServerTestBase.cs +++ b/src/Umbraco.Tests.Integration/TestServerTest/UmbracoTestServerTestBase.cs @@ -150,6 +150,7 @@ namespace Umbraco.Cms.Tests.Integration.TestServerTest .AddConfiguration() .AddUmbracoCore() .AddWebComponents() + .AddNuCache() .AddRuntimeMinifier() .AddBackOfficeCore() .AddBackOfficeAuthentication() diff --git a/src/Umbraco.Tests.Integration/Testing/IntegrationTestComponent.cs b/src/Umbraco.Tests.Integration/Testing/IntegrationTestComponent.cs index 277510fc9e..c0b490e0e5 100644 --- a/src/Umbraco.Tests.Integration/Testing/IntegrationTestComponent.cs +++ b/src/Umbraco.Tests.Integration/Testing/IntegrationTestComponent.cs @@ -2,7 +2,7 @@ // See LICENSE for more details. using Examine; -using Examine.LuceneEngine.Providers; +using Examine.Lucene.Providers; using Umbraco.Cms.Core.Composing; namespace Umbraco.Cms.Tests.Integration.Testing @@ -31,7 +31,7 @@ namespace Umbraco.Cms.Tests.Integration.Testing { if (index is LuceneIndex luceneIndex) { - luceneIndex.ProcessNonAsync(); + luceneIndex.WithThreadingMode(IndexThreadingMode.Synchronous); } } } diff --git a/src/Umbraco.Tests.Integration/Testing/UmbracoIntegrationTest.cs b/src/Umbraco.Tests.Integration/Testing/UmbracoIntegrationTest.cs index dbf047cf48..f0eac637fd 100644 --- a/src/Umbraco.Tests.Integration/Testing/UmbracoIntegrationTest.cs +++ b/src/Umbraco.Tests.Integration/Testing/UmbracoIntegrationTest.cs @@ -138,12 +138,13 @@ namespace Umbraco.Cms.Tests.Integration.Testing Log.Logger = new LoggerConfiguration() .WriteTo.File(path, rollingInterval: RollingInterval.Day) + .MinimumLevel.Debug() .CreateLogger(); builder.AddSerilog(Log.Logger); }); case UmbracoTestOptions.Logger.Console: - return Microsoft.Extensions.Logging.LoggerFactory.Create(builder => builder.AddConsole()); + return Microsoft.Extensions.Logging.LoggerFactory.Create(builder => builder.AddConsole().SetMinimumLevel(LogLevel.Debug)); } } catch diff --git a/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/ExamineBaseTest.cs b/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/ExamineBaseTest.cs new file mode 100644 index 0000000000..8840988ac6 --- /dev/null +++ b/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/ExamineBaseTest.cs @@ -0,0 +1,135 @@ +using System; +using System.Data; +using Examine.Lucene.Providers; +using Examine.Search; +using Microsoft.Extensions.DependencyInjection; +using Moq; +using NPoco; +using NUnit.Framework; +using Umbraco.Cms.Core; +using Umbraco.Cms.Core.Events; +using Umbraco.Cms.Core.Hosting; +using Umbraco.Cms.Core.Models; +using Umbraco.Cms.Core.Persistence.Querying; +using Umbraco.Cms.Core.Scoping; +using Umbraco.Cms.Core.Services; +using Umbraco.Cms.Infrastructure.Examine; +using Umbraco.Cms.Infrastructure.Persistence; +using Umbraco.Cms.Infrastructure.Persistence.SqlSyntax; +using Umbraco.Cms.Tests.Integration.Testing; + +namespace Umbraco.Cms.Tests.Integration.Umbraco.Examine.Lucene.UmbracoExamine +{ + [TestFixture] + public abstract class ExamineBaseTest : UmbracoIntegrationTest + { + protected IndexInitializer IndexInitializer => Services.GetRequiredService(); + + protected IHostingEnvironment HostingEnvironment => Services.GetRequiredService(); + + protected IRuntimeState RunningRuntimeState { get; } = Mock.Of(x => x.Level == RuntimeLevel.Run); + + public override void ConfigureServices(IServiceCollection services) + { + base.ConfigureServices(services); + services.AddSingleton(); + } + + /// + /// Used to create and manage a testable index + /// + /// + /// + /// + /// + /// + /// + protected IDisposable GetSynchronousContentIndex( + bool publishedValuesOnly, + out UmbracoContentIndex index, + out ContentIndexPopulator contentRebuilder, + out ContentValueSetBuilder contentValueSetBuilder, + int? parentId = null, + IContentService contentService = null) + { + contentValueSetBuilder = IndexInitializer.GetContentValueSetBuilder(publishedValuesOnly); + + ISqlContext sqlContext = Mock.Of(x => x.Query() == Mock.Of>()); + IUmbracoDatabaseFactory dbFactory = Mock.Of(x => x.SqlContext == sqlContext); + + if (contentService == null) + { + contentService = IndexInitializer.GetMockContentService(); + } + + contentRebuilder = IndexInitializer.GetContentIndexRebuilder(contentService, publishedValuesOnly, dbFactory); + + var luceneDir = new RandomIdRAMDirectory(); + + ContentValueSetValidator validator; + + // if only published values then we'll change the validator for tests to + // ensure we don't support protected nodes and that we + // mock the public access service for the special protected node. + if (publishedValuesOnly) + { + var publicAccessServiceMock = new Mock(); + publicAccessServiceMock.Setup(x => x.IsProtected(It.IsAny())) + .Returns((string path) => + { + if (path.EndsWith("," + ExamineDemoDataContentService.ProtectedNode)) + { + return Attempt.Succeed(); + } + return Attempt.Fail(); + }); + + var scopeProviderMock = new Mock(); + scopeProviderMock.Setup(x => x.CreateScope( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(Mock.Of); + + validator = new ContentValueSetValidator( + publishedValuesOnly, + false, + publicAccessServiceMock.Object, + scopeProviderMock.Object, + parentId); + } + else + { + validator = new ContentValueSetValidator(publishedValuesOnly, parentId); + } + + index = IndexInitializer.GetUmbracoIndexer( + HostingEnvironment, + RunningRuntimeState, + luceneDir, + validator: validator); + + IDisposable syncMode = index.WithThreadingMode(IndexThreadingMode.Synchronous); + + return new DisposableWrapper(syncMode, index, luceneDir); + } + + private class DisposableWrapper : IDisposable + { + private readonly IDisposable[] _disposables; + + public DisposableWrapper(params IDisposable[] disposables) => _disposables = disposables; + + public void Dispose() + { + foreach (IDisposable d in _disposables) + { + d.Dispose(); + } + } + } + } +} diff --git a/src/Umbraco.Tests/UmbracoExamine/ExamineDemoDataContentService.cs b/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/ExamineDemoDataContentService.cs similarity index 94% rename from src/Umbraco.Tests/UmbracoExamine/ExamineDemoDataContentService.cs rename to src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/ExamineDemoDataContentService.cs index ca11680f68..8323acf9bf 100644 --- a/src/Umbraco.Tests/UmbracoExamine/ExamineDemoDataContentService.cs +++ b/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/ExamineDemoDataContentService.cs @@ -1,7 +1,7 @@ -using System.Xml.Linq; +using System.Xml.Linq; using System.Xml.XPath; -namespace Umbraco.Tests.UmbracoExamine +namespace Umbraco.Cms.Tests.Integration.Umbraco.Examine.Lucene.UmbracoExamine { // TODO: This is ultra hack and still left over from legacy but still works for testing atm public class ExamineDemoDataContentService diff --git a/src/Umbraco.Tests/UmbracoExamine/ExamineDemoDataMediaService.cs b/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/ExamineDemoDataMediaService.cs similarity index 88% rename from src/Umbraco.Tests/UmbracoExamine/ExamineDemoDataMediaService.cs rename to src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/ExamineDemoDataMediaService.cs index 035a31b240..a7172248db 100644 --- a/src/Umbraco.Tests/UmbracoExamine/ExamineDemoDataMediaService.cs +++ b/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/ExamineDemoDataMediaService.cs @@ -1,11 +1,11 @@ -using System; +using System; using System.IO; using System.Linq; using System.Text; using System.Xml.Linq; using System.Xml.XPath; -namespace Umbraco.Tests.UmbracoExamine +namespace Umbraco.Cms.Tests.Integration.Umbraco.Examine.Lucene.UmbracoExamine { // TODO: This is ultra hack and still left over from legacy but still works for testing atm internal class ExamineDemoDataMediaService diff --git a/src/Umbraco.Tests/UmbracoExamine/ExamineExtensions.cs b/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/ExamineExtensions.cs similarity index 98% rename from src/Umbraco.Tests/UmbracoExamine/ExamineExtensions.cs rename to src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/ExamineExtensions.cs index 9cca58719e..ee8aea385f 100644 --- a/src/Umbraco.Tests/UmbracoExamine/ExamineExtensions.cs +++ b/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/ExamineExtensions.cs @@ -1,11 +1,11 @@ -using Examine; +using Examine; using System; using System.Collections.Generic; using System.ComponentModel; using System.Linq; using System.Xml.Linq; -namespace Umbraco.Tests.UmbracoExamine +namespace Umbraco.Cms.Tests.Integration.Umbraco.Examine.Lucene.UmbracoExamine { /// /// LEGACY!! Static methods to help query umbraco xml diff --git a/src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs b/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/IndexInitializer.cs similarity index 53% rename from src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs rename to src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/IndexInitializer.cs index 6dfc0a39ce..b7aa9fafe1 100644 --- a/src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs +++ b/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/IndexInitializer.cs @@ -1,11 +1,14 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using Examine; +using Examine.Lucene; +using Examine.Lucene.Directories; using Lucene.Net.Analysis; using Lucene.Net.Analysis.Standard; using Lucene.Net.Store; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Moq; using Umbraco.Cms.Core.Hosting; using Umbraco.Cms.Core.Logging; @@ -14,48 +17,74 @@ using Umbraco.Cms.Core.Models.Membership; using Umbraco.Cms.Core.Persistence.Querying; using Umbraco.Cms.Core.PropertyEditors; using Umbraco.Cms.Core.Scoping; +using Umbraco.Cms.Core.Serialization; using Umbraco.Cms.Core.Services; using Umbraco.Cms.Core.Strings; using Umbraco.Cms.Infrastructure.Examine; using Umbraco.Cms.Infrastructure.Persistence; -using Umbraco.Tests.TestHelpers; using IContentService = Umbraco.Cms.Core.Services.IContentService; using IMediaService = Umbraco.Cms.Core.Services.IMediaService; -using Version = Lucene.Net.Util.Version; -namespace Umbraco.Tests.UmbracoExamine +namespace Umbraco.Cms.Tests.Integration.Umbraco.Examine.Lucene.UmbracoExamine { /// /// Used internally by test classes to initialize a new index from the template /// - internal static class IndexInitializer + public class IndexInitializer { - public static ContentValueSetBuilder GetContentValueSetBuilder(PropertyEditorCollection propertyEditors, IScopeProvider scopeProvider, bool publishedValuesOnly) + private readonly IShortStringHelper _shortStringHelper; + private readonly IJsonSerializer _jsonSerializer; + private readonly PropertyEditorCollection _propertyEditors; + private readonly IScopeProvider _scopeProvider; + private readonly ILoggerFactory _loggerFactory; + + public IndexInitializer( + IShortStringHelper shortStringHelper, + IJsonSerializer jsonSerializer, + PropertyEditorCollection propertyEditors, + IScopeProvider scopeProvider, + ILoggerFactory loggerFactory) + { + _shortStringHelper = shortStringHelper; + _jsonSerializer = jsonSerializer; + _propertyEditors = propertyEditors; + _scopeProvider = scopeProvider; + _loggerFactory = loggerFactory; + } + + public ContentValueSetBuilder GetContentValueSetBuilder(bool publishedValuesOnly) { var contentValueSetBuilder = new ContentValueSetBuilder( - propertyEditors, - new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider(TestHelper.ShortStringHelper) }), + _propertyEditors, + new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider(_shortStringHelper) }), GetMockUserService(), - TestHelper.ShortStringHelper, - scopeProvider, + _shortStringHelper, + _scopeProvider, publishedValuesOnly); return contentValueSetBuilder; } - public static ContentIndexPopulator GetContentIndexRebuilder(PropertyEditorCollection propertyEditors, IContentService contentService, IScopeProvider scopeProvider, IUmbracoDatabaseFactory umbracoDatabaseFactory, bool publishedValuesOnly) + public ContentIndexPopulator GetContentIndexRebuilder(IContentService contentService, bool publishedValuesOnly, IUmbracoDatabaseFactory umbracoDatabaseFactory) { - var contentValueSetBuilder = GetContentValueSetBuilder(propertyEditors, scopeProvider, publishedValuesOnly); - var contentIndexDataSource = new ContentIndexPopulator(publishedValuesOnly, null, contentService, umbracoDatabaseFactory, contentValueSetBuilder); + var contentValueSetBuilder = GetContentValueSetBuilder(publishedValuesOnly); + var contentIndexDataSource = new ContentIndexPopulator( + _loggerFactory.CreateLogger(), + publishedValuesOnly, + null, + contentService, + umbracoDatabaseFactory, + contentValueSetBuilder); return contentIndexDataSource; } - public static MediaIndexPopulator GetMediaIndexRebuilder(PropertyEditorCollection propertyEditors, IMediaService mediaService) + public MediaIndexPopulator GetMediaIndexRebuilder(IMediaService mediaService) { - var mediaValueSetBuilder = new MediaValueSetBuilder(propertyEditors, new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider(TestHelper.ShortStringHelper) }), GetMockUserService(), Mock.Of>(), TestHelper.ShortStringHelper, TestHelper.JsonSerializer); + var mediaValueSetBuilder = new MediaValueSetBuilder(_propertyEditors, new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider(_shortStringHelper) }), GetMockUserService(), Mock.Of>(), _shortStringHelper, _jsonSerializer); var mediaIndexDataSource = new MediaIndexPopulator(null, mediaService, mediaValueSetBuilder); return mediaIndexDataSource; } + public static IContentService GetMockContentService() { long longTotalRecs; @@ -64,23 +93,25 @@ namespace Umbraco.Tests.UmbracoExamine var allRecs = demoData.GetLatestContentByXPath("//*[@isDoc]") .Root .Elements() - .Select(x => Mock.Of( + .Select((xmlElement, index) => Mock.Of( m => - m.Id == (int)x.Attribute("id") && - m.ParentId == (int)x.Attribute("parentID") && - m.Level == (int)x.Attribute("level") && + m.Id == (int)xmlElement.Attribute("id") && + // have every second one published and include the special one + m.Published == ((ExamineDemoDataContentService.ProtectedNode == (int)xmlElement.Attribute("id")) || (index % 2 == 0 ? true : false)) && + m.ParentId == (int)xmlElement.Attribute("parentID") && + m.Level == (int)xmlElement.Attribute("level") && m.CreatorId == 0 && - m.SortOrder == (int)x.Attribute("sortOrder") && - m.CreateDate == (DateTime)x.Attribute("createDate") && - m.UpdateDate == (DateTime)x.Attribute("updateDate") && - m.Name == (string)x.Attribute(UmbracoExamineFieldNames.NodeNameFieldName) && - m.GetCultureName(It.IsAny()) == (string)x.Attribute(UmbracoExamineFieldNames.NodeNameFieldName) && - m.Path == (string)x.Attribute("path") && + m.SortOrder == (int)xmlElement.Attribute("sortOrder") && + m.CreateDate == (DateTime)xmlElement.Attribute("createDate") && + m.UpdateDate == (DateTime)xmlElement.Attribute("updateDate") && + m.Name == (string)xmlElement.Attribute(UmbracoExamineFieldNames.NodeNameFieldName) && + m.GetCultureName(It.IsAny()) == (string)xmlElement.Attribute(UmbracoExamineFieldNames.NodeNameFieldName) && + m.Path == (string)xmlElement.Attribute("path") && m.Properties == new PropertyCollection() && m.ContentType == Mock.Of(mt => mt.Icon == "test" && - mt.Alias == x.Name.LocalName && - mt.Id == (int)x.Attribute("nodeType")))) + mt.Alias == xmlElement.Name.LocalName && + mt.Id == (int)xmlElement.Attribute("nodeType")))) .ToArray(); @@ -90,10 +121,7 @@ namespace Umbraco.Tests.UmbracoExamine == allRecs); } - public static IUserService GetMockUserService() - { - return Mock.Of(x => x.GetProfileById(It.IsAny()) == Mock.Of(p => p.Id == 0 && p.Name == "admin")); - } + public IUserService GetMockUserService() => Mock.Of(x => x.GetProfileById(It.IsAny()) == Mock.Of(p => p.Id == 0 && p.Name == "admin")); public static IMediaService GetMockMediaService() { @@ -136,30 +164,23 @@ namespace Umbraco.Tests.UmbracoExamine return mediaServiceMock.Object; } - public static ILocalizationService GetMockLocalizationService() - { - return Mock.Of(x => x.GetAllLanguages() == Array.Empty()); - } + public ILocalizationService GetMockLocalizationService() => Mock.Of(x => x.GetAllLanguages() == Array.Empty()); - public static IMediaTypeService GetMockMediaTypeService() + public static IMediaTypeService GetMockMediaTypeService(IShortStringHelper shortStringHelper) { var mediaTypeServiceMock = new Mock(); mediaTypeServiceMock.Setup(x => x.GetAll()) .Returns(new List { - new MediaType(TestHelper.ShortStringHelper, -1) {Alias = "Folder", Name = "Folder", Id = 1031, Icon = "icon-folder"}, - new MediaType(TestHelper.ShortStringHelper, -1) {Alias = "Image", Name = "Image", Id = 1032, Icon = "icon-picture"} + new MediaType(shortStringHelper, -1) {Alias = "Folder", Name = "Folder", Id = 1031, Icon = "icon-folder"}, + new MediaType(shortStringHelper, -1) {Alias = "Image", Name = "Image", Id = 1032, Icon = "icon-picture"} }); return mediaTypeServiceMock.Object; } - public static IProfilingLogger GetMockProfilingLogger() - { - return new ProfilingLogger(Mock.Of>(), Mock.Of()); - } + public IProfilingLogger GetMockProfilingLogger() => new ProfilingLogger(Mock.Of>(), Mock.Of()); - public static UmbracoContentIndex GetUmbracoIndexer( - IProfilingLogger profilingLogger, + public UmbracoContentIndex GetUmbracoIndexer( IHostingEnvironment hostingEnvironment, IRuntimeState runtimeState, Directory luceneDir, @@ -171,40 +192,50 @@ namespace Umbraco.Tests.UmbracoExamine languageService = GetMockLocalizationService(); if (analyzer == null) - analyzer = new StandardAnalyzer(Version.LUCENE_30); + analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); if (validator == null) validator = new ContentValueSetValidator(true); - var i = new UmbracoContentIndex( + var options = GetOptions( "testIndexer", - luceneDir, - new UmbracoFieldDefinitionCollection(), - analyzer, - profilingLogger, - Mock.Of>(), - Mock.Of(), + new LuceneDirectoryIndexOptions + { + Analyzer = analyzer, + Validator = validator, + DirectoryFactory = new GenericDirectoryFactory(s => luceneDir), + FieldDefinitions = new UmbracoFieldDefinitionCollection() + }); + + var i = new UmbracoContentIndex( + _loggerFactory, + "testIndexer", + options, hostingEnvironment, runtimeState, - languageService, - validator); + languageService); i.IndexingError += IndexingError; + i.IndexOperationComplete += I_IndexOperationComplete; return i; } + private void I_IndexOperationComplete(object sender, IndexOperationEventArgs e) + { + + } + //public static MultiIndexSearcher GetMultiSearcher(Directory pdfDir, Directory simpleDir, Directory conventionDir, Directory cwsDir) //{ // var i = new MultiIndexSearcher("testSearcher", new[] { pdfDir, simpleDir, conventionDir, cwsDir }, new StandardAnalyzer(Version.LUCENE_29)); // return i; //} + public static IOptionsSnapshot GetOptions(string indexName, LuceneDirectoryIndexOptions options) + => Mock.Of>(x => x.Get(indexName) == options); - internal static void IndexingError(object sender, IndexingErrorEventArgs e) - { - throw new ApplicationException(e.Message, e.InnerException); - } + internal void IndexingError(object sender, IndexingErrorEventArgs e) => throw new ApplicationException(e.Message, e.Exception); } diff --git a/src/Umbraco.Tests/UmbracoExamine/IndexTest.cs b/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/IndexTest.cs similarity index 54% rename from src/Umbraco.Tests/UmbracoExamine/IndexTest.cs rename to src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/IndexTest.cs index 3daf185cd4..f6362a8156 100644 --- a/src/Umbraco.Tests/UmbracoExamine/IndexTest.cs +++ b/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/IndexTest.cs @@ -1,52 +1,67 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; -using System.Threading; using Examine; -using Examine.LuceneEngine.Providers; -using Lucene.Net.Index; -using Lucene.Net.Search; -using Microsoft.Extensions.DependencyInjection; using Newtonsoft.Json; using NUnit.Framework; using Umbraco.Cms.Core.Models; -using Umbraco.Cms.Core.PropertyEditors; using Umbraco.Cms.Infrastructure.Examine; +using Umbraco.Cms.Tests.Common.Builders; using Umbraco.Cms.Tests.Common.Testing; using Umbraco.Extensions; -using Umbraco.Tests.TestHelpers; -using Umbraco.Tests.TestHelpers.Entities; -namespace Umbraco.Tests.UmbracoExamine +namespace Umbraco.Cms.Tests.Integration.Umbraco.Examine.Lucene.UmbracoExamine { + /// /// Tests the standard indexing capabilities /// [TestFixture] - [Apartment(ApartmentState.STA)] - [UmbracoTest(Database = UmbracoTestOptions.Database.NewSchemaPerTest)] + [UmbracoTest(Database = UmbracoTestOptions.Database.None)] public class IndexTest : ExamineBaseTest { [Test] - public void Index_Property_Data_With_Value_Indexer() + public void GivenValidationParentNode_WhenContentIndexedUnderDifferentParent_DocumentIsNotIndexed() { - var contentValueSetBuilder = IndexInitializer.GetContentValueSetBuilder(Factory.GetRequiredService(), ScopeProvider, false); - - using (var luceneDir = new RandomIdRamDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, HostingEnvironment, RuntimeState, luceneDir, - validator: new ContentValueSetValidator(false))) - using (indexer.ProcessNonAsync()) + using (GetSynchronousContentIndex(false, out UmbracoContentIndex index, out _, out _, 999)) { - indexer.CreateIndex(); + var searcher = index.Searcher; - var contentType = MockedContentTypes.CreateBasicContentType(); + var contentService = new ExamineDemoDataContentService(); + //get a node from the data repo + var node = contentService.GetPublishedContentByXPath("//*[string-length(@id)>0 and number(@id)>0]") + .Root + .Elements() + .First(); + + ValueSet valueSet = node.ConvertToValueSet(IndexTypes.Content); + + // Ignored since the path isn't under 999 + index.IndexItems(new[] { valueSet }); + Assert.AreEqual(0, searcher.CreateQuery().Id(valueSet.Id).Execute().TotalItemCount); + + // Change so that it's under 999 and verify + valueSet.Values["path"] = new List { "-1,999," + valueSet.Id }; + index.IndexItems(new[] { valueSet }); + Assert.AreEqual(1, searcher.CreateQuery().Id(valueSet.Id).Execute().TotalItemCount); + } + } + + [Test] + public void GivenIndexingDocument_WhenGridPropertyData_ThenDataIndexedInSegregatedFields() + { + using (GetSynchronousContentIndex(false, out UmbracoContentIndex index, out _, out ContentValueSetBuilder contentValueSetBuilder, null)) + { + index.CreateIndex(); + + ContentType contentType = ContentTypeBuilder.CreateBasicContentType(); contentType.AddPropertyType(new PropertyType(TestHelper.ShortStringHelper, "test", ValueStorageType.Ntext) { Alias = "grid", Name = "Grid", PropertyEditorAlias = Cms.Core.Constants.PropertyEditors.Aliases.Grid }); - var content = MockedContent.CreateBasicContent(contentType); + Content content = ContentBuilder.CreateBasicContent(contentType); content.Id = 555; content.Path = "-1,555"; var gridVal = new GridValue @@ -100,15 +115,13 @@ namespace Umbraco.Tests.UmbracoExamine var json = JsonConvert.SerializeObject(gridVal); content.Properties["grid"].SetValue(json); - var valueSet = contentValueSetBuilder.GetValueSets(content); - indexer.IndexItems(valueSet); + IEnumerable valueSet = contentValueSetBuilder.GetValueSets(content); + index.IndexItems(valueSet); - var searcher = indexer.GetSearcher(); - - var results = searcher.CreateQuery().Id(555).Execute(); + ISearchResults results = index.Searcher.CreateQuery().Id(555).Execute(); Assert.AreEqual(1, results.TotalItemCount); - var result = results.First(); + ISearchResult result = results.First(); Assert.IsTrue(result.Values.ContainsKey("grid.row1")); Assert.AreEqual("value1", result.AllValues["grid.row1"][0]); Assert.AreEqual("value2", result.AllValues["grid.row1"][1]); @@ -120,95 +133,64 @@ namespace Umbraco.Tests.UmbracoExamine } [Test] - public void Rebuild_Index() - { - var contentRebuilder = IndexInitializer.GetContentIndexRebuilder(Factory.GetRequiredService(), IndexInitializer.GetMockContentService(), ScopeProvider, UmbracoDatabaseFactory,false); - var mediaRebuilder = IndexInitializer.GetMediaIndexRebuilder(Factory.GetRequiredService(), IndexInitializer.GetMockMediaService()); + public void GivenEmptyIndex_WhenUsingWithContentAndMediaPopulators_ThenIndexPopulated() + { + var mediaRebuilder = IndexInitializer.GetMediaIndexRebuilder(IndexInitializer.GetMockMediaService()); - using (var luceneDir = new RandomIdRamDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, HostingEnvironment, RuntimeState, luceneDir, - validator: new ContentValueSetValidator(false))) - using (indexer.ProcessNonAsync()) + using (GetSynchronousContentIndex(false, out UmbracoContentIndex index, out ContentIndexPopulator contentRebuilder, out _, null)) { - - var searcher = indexer.GetSearcher(); - //create the whole thing - contentRebuilder.Populate(indexer); - mediaRebuilder.Populate(indexer); + contentRebuilder.Populate(index); + mediaRebuilder.Populate(index); - var result = searcher.CreateQuery().All().Execute(); + var result = index.Searcher.CreateQuery().All().Execute(); Assert.AreEqual(29, result.TotalItemCount); } } - ///// /// /// Check that the node signalled as protected in the content service is not present in the index. /// [Test] - public void Index_Protected_Content_Not_Indexed() + public void GivenPublishedContentIndex_WhenProtectedContentIndexed_ThenItIsIgnored() { - var rebuilder = IndexInitializer.GetContentIndexRebuilder(Factory.GetRequiredService(), IndexInitializer.GetMockContentService(), ScopeProvider, UmbracoDatabaseFactory,false); - - - using (var luceneDir = new RandomIdRamDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, HostingEnvironment, RuntimeState, luceneDir)) - using (indexer.ProcessNonAsync()) - using (var searcher = ((LuceneSearcher)indexer.GetSearcher()).GetLuceneSearcher()) + using (GetSynchronousContentIndex(true, out UmbracoContentIndex index, out ContentIndexPopulator contentRebuilder, out _, null)) { //create the whole thing - rebuilder.Populate(indexer); + contentRebuilder.Populate(index); + Assert.Greater( + index.Searcher.CreateQuery().All().Execute().TotalItemCount, + 0); - var protectedQuery = new BooleanQuery(); - protectedQuery.Add( - new BooleanClause( - new TermQuery(new Term(ExamineFieldNames.CategoryFieldName, IndexTypes.Content)), - Occur.MUST)); - - protectedQuery.Add( - new BooleanClause( - new TermQuery(new Term(ExamineFieldNames.ItemIdFieldName, ExamineDemoDataContentService.ProtectedNode.ToString())), - Occur.MUST)); - - var collector = TopScoreDocCollector.Create(100, true); - - searcher.Search(protectedQuery, collector); - - Assert.AreEqual(0, collector.TotalHits, "Protected node should not be indexed"); + Assert.AreEqual( + 0, + index.Searcher.CreateQuery().Id(ExamineDemoDataContentService.ProtectedNode.ToString()).Execute().TotalItemCount); } - } [Test] - public void Index_Move_Media_From_Non_Indexable_To_Indexable_ParentID() + public void GivenMediaUnderNonIndexableParent_WhenMediaMovedUnderIndexableParent_ThenItIsIncludedInTheIndex() { // create a validator with // publishedValuesOnly false - // parentId 1116 (only content under that parent will be indexed) - var validator = new ContentValueSetValidator(false, 1116); - - using (var luceneDir = new RandomIdRamDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, HostingEnvironment, RuntimeState, luceneDir, validator: validator)) - using (indexer.ProcessNonAsync()) + // parentId 1116 (only content under that parent will be indexed) + using (GetSynchronousContentIndex(false, out UmbracoContentIndex index, out ContentIndexPopulator contentRebuilder, out _, 1116)) { - var searcher = indexer.GetSearcher(); - //get a node from the data repo (this one exists underneath 2222) var node = _mediaService.GetLatestMediaByXpath("//*[string-length(@id)>0 and number(@id)>0]") .Root.Elements() - .First(x => (int) x.Attribute("id") == 2112); + .First(x => (int)x.Attribute("id") == 2112); var currPath = (string)node.Attribute("path"); //should be : -1,1111,2222,2112 Assert.AreEqual("-1,1111,2222,2112", currPath); //ensure it's indexed - indexer.IndexItem(node.ConvertToValueSet(IndexTypes.Media)); + index.IndexItem(node.ConvertToValueSet(IndexTypes.Media)); //it will not exist because it exists under 2222 - var results = searcher.CreateQuery().Id(2112).Execute(); + var results = index.Searcher.CreateQuery().Id(2112).Execute(); Assert.AreEqual(0, results.Count()); //now mimic moving 2112 to 1116 @@ -217,38 +199,34 @@ namespace Umbraco.Tests.UmbracoExamine node.SetAttributeValue("parentID", "1116"); //now reindex the node, this should first delete it and then WILL add it because of the parent id constraint - indexer.IndexItems(new[] { node.ConvertToValueSet(IndexTypes.Media) }); + index.IndexItems(new[] { node.ConvertToValueSet(IndexTypes.Media) }); //now ensure it exists - results = searcher.CreateQuery().Id(2112).Execute(); + results = index.Searcher.CreateQuery().Id(2112).Execute(); Assert.AreEqual(1, results.Count()); } } [Test] - public void Index_Move_Media_To_Non_Indexable_ParentID() + public void GivenMediaUnderIndexableParent_WhenMediaMovedUnderNonIndexableParent_ThenItIsRemovedFromTheIndex() { // create a validator with // publishedValuesOnly false // parentId 2222 (only content under that parent will be indexed) - var validator = new ContentValueSetValidator(false, 2222); - - using (var luceneDir = new RandomIdRamDirectory()) - using (var indexer1 = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, HostingEnvironment, RuntimeState, luceneDir, validator: validator)) - using (indexer1.ProcessNonAsync()) + using (GetSynchronousContentIndex(false, out UmbracoContentIndex index, out ContentIndexPopulator contentRebuilder, out _, 2222)) { - var searcher = indexer1.GetSearcher(); + var searcher = index.Searcher; //get a node from the data repo (this one exists underneath 2222) var node = _mediaService.GetLatestMediaByXpath("//*[string-length(@id)>0 and number(@id)>0]") .Root.Elements() - .First(x => (int) x.Attribute("id") == 2112); + .First(x => (int)x.Attribute("id") == 2112); var currPath = (string)node.Attribute("path"); //should be : -1,1111,2222,2112 Assert.AreEqual("-1,1111,2222,2112", currPath); //ensure it's indexed - indexer1.IndexItem(node.ConvertToValueSet(IndexTypes.Media)); + index.IndexItem(node.ConvertToValueSet(IndexTypes.Media)); //it will exist because it exists under 2222 var results = searcher.CreateQuery().Id(2112).Execute(); @@ -259,7 +237,7 @@ namespace Umbraco.Tests.UmbracoExamine node.SetAttributeValue("parentID", "1116"); //now reindex the node, this should first delete it and then NOT add it because of the parent id constraint - indexer1.IndexItems(new[] { node.ConvertToValueSet(IndexTypes.Media) }); + index.IndexItems(new[] { node.ConvertToValueSet(IndexTypes.Media) }); //now ensure it's deleted results = searcher.CreateQuery().Id(2112).Execute(); @@ -273,38 +251,34 @@ namespace Umbraco.Tests.UmbracoExamine /// We then call the Examine method to re-index Content and do some comparisons to ensure that it worked correctly. /// [Test] - public void Index_Reindex_Content() + public void GivenEmptyIndex_WhenIndexedWithContentPopulator_ThenTheIndexIsPopulated() { - var rebuilder = IndexInitializer.GetContentIndexRebuilder(Factory.GetRequiredService(), IndexInitializer.GetMockContentService(), ScopeProvider, UmbracoDatabaseFactory,false); - using (var luceneDir = new RandomIdRamDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, HostingEnvironment, RuntimeState, luceneDir, - validator: new ContentValueSetValidator(false))) - using (indexer.ProcessNonAsync()) + using (GetSynchronousContentIndex(false, out UmbracoContentIndex index, out ContentIndexPopulator contentRebuilder, out _, null)) { - - var searcher = indexer.GetSearcher(); - //create the whole thing - rebuilder.Populate(indexer); + contentRebuilder.Populate(index); - var result = searcher.CreateQuery().Field(ExamineFieldNames.CategoryFieldName, IndexTypes.Content).Execute(); + var result = index.Searcher + .CreateQuery() + .Field(ExamineFieldNames.CategoryFieldName, IndexTypes.Content) + .Execute(); Assert.AreEqual(21, result.TotalItemCount); //delete all content - foreach (var r in result) - { - indexer.DeleteFromIndex(r.Id); - } - + index.DeleteFromIndex(result.Select(x => x.Id)); //ensure it's all gone - result = searcher.CreateQuery().Field(ExamineFieldNames.CategoryFieldName, IndexTypes.Content).Execute(); + result = index.Searcher.CreateQuery().Field(ExamineFieldNames.CategoryFieldName, IndexTypes.Content).Execute(); Assert.AreEqual(0, result.TotalItemCount); //call our indexing methods - rebuilder.Populate(indexer); + contentRebuilder.Populate(index); + + result = index.Searcher + .CreateQuery() + .Field(ExamineFieldNames.CategoryFieldName, IndexTypes.Content) + .Execute(); - result = searcher.CreateQuery().Field(ExamineFieldNames.CategoryFieldName, IndexTypes.Content).Execute(); Assert.AreEqual(21, result.TotalItemCount); } } @@ -313,26 +287,24 @@ namespace Umbraco.Tests.UmbracoExamine /// This will delete an item from the index and ensure that all children of the node are deleted too! /// [Test] - public void Index_Delete_Index_Item_Ensure_Heirarchy_Removed() + public void GivenPopulatedIndex_WhenDocumentDeleted_ThenItsHierarchyIsAlsoDeleted() { - - var rebuilder = IndexInitializer.GetContentIndexRebuilder(Factory.GetRequiredService(), IndexInitializer.GetMockContentService(), ScopeProvider, UmbracoDatabaseFactory,false); - - using (var luceneDir = new RandomIdRamDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, HostingEnvironment, RuntimeState, luceneDir)) - using (indexer.ProcessNonAsync()) + using (GetSynchronousContentIndex(false, out UmbracoContentIndex index, out ContentIndexPopulator contentRebuilder, out _, null)) { - var searcher = indexer.GetSearcher(); + var searcher = index.Searcher; //create the whole thing - rebuilder.Populate(indexer); + contentRebuilder.Populate(index); + + var results = searcher.CreateQuery().Id(1141).Execute(); + Assert.AreEqual(1, results.Count()); //now delete a node that has children - indexer.DeleteFromIndex(1140.ToString()); + index.DeleteFromIndex(1140.ToString()); //this node had children: 1141 & 1142, let's ensure they are also removed - var results = searcher.CreateQuery().Id(1141).Execute(); + results = searcher.CreateQuery().Id(1141).Execute(); Assert.AreEqual(0, results.Count()); results = searcher.CreateQuery().Id(1142).Execute(); diff --git a/src/Umbraco.Tests/Web/PublishedContentQueryTests.cs b/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/PublishedContentQueryTests.cs similarity index 62% rename from src/Umbraco.Tests/Web/PublishedContentQueryTests.cs rename to src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/PublishedContentQueryTests.cs index cc34cd4aba..f2269916a4 100644 --- a/src/Umbraco.Tests/Web/PublishedContentQueryTests.cs +++ b/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/PublishedContentQueryTests.cs @@ -1,33 +1,43 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using Examine; -using Examine.LuceneEngine.Providers; +using Examine.Lucene; +using Examine.Lucene.Directories; +using Examine.Lucene.Providers; using Lucene.Net.Store; +using Microsoft.Extensions.Logging; using Moq; using NUnit.Framework; using Umbraco.Cms.Core.Models.PublishedContent; using Umbraco.Cms.Core.PublishedCache; using Umbraco.Cms.Infrastructure; using Umbraco.Cms.Infrastructure.Examine; -using Umbraco.Tests.TestHelpers; -using Umbraco.Web; +using Umbraco.Cms.Tests.Common.Testing; -namespace Umbraco.Tests.Web +namespace Umbraco.Cms.Tests.Integration.Umbraco.Examine.Lucene.UmbracoExamine { [TestFixture] - public class PublishedContentQueryTests + [UmbracoTest(Database = UmbracoTestOptions.Database.None)] + public class PublishedContentQueryTests : ExamineBaseTest { private class TestIndex : LuceneIndex, IUmbracoIndex { private readonly string[] _fieldNames; - public TestIndex(string name, Directory luceneDirectory, string[] fieldNames) - : base(name, luceneDirectory, null, null, null, null) + public TestIndex(ILoggerFactory loggerFactory, string name, Directory luceneDirectory, string[] fieldNames) + : base( + loggerFactory, + name, + IndexInitializer.GetOptions(name, new LuceneDirectoryIndexOptions + { + DirectoryFactory = new GenericDirectoryFactory(s => luceneDirectory) + })) { _fieldNames = fieldNames; } + public bool EnableDefaultEventHandler => throw new NotImplementedException(); public bool PublishedValuesOnly => throw new NotImplementedException(); public IEnumerable GetFields() => _fieldNames; @@ -35,29 +45,29 @@ namespace Umbraco.Tests.Web private TestIndex CreateTestIndex(Directory luceneDirectory, string[] fieldNames) { - var indexer = new TestIndex("TestIndex", luceneDirectory, fieldNames); + var index = new TestIndex(LoggerFactory, "TestIndex", luceneDirectory, fieldNames); - using (indexer.ProcessNonAsync()) + using (index.WithThreadingMode(IndexThreadingMode.Synchronous)) { //populate with some test data - indexer.IndexItem(new ValueSet("1", "content", new Dictionary + index.IndexItem(new ValueSet("1", "content", new Dictionary { [fieldNames[0]] = "Hello world, there are products here", [UmbracoExamineFieldNames.VariesByCultureFieldName] = "n" })); - indexer.IndexItem(new ValueSet("2", "content", new Dictionary + index.IndexItem(new ValueSet("2", "content", new Dictionary { [fieldNames[1]] = "Hello world, there are products here", [UmbracoExamineFieldNames.VariesByCultureFieldName] = "y" })); - indexer.IndexItem(new ValueSet("3", "content", new Dictionary + index.IndexItem(new ValueSet("3", "content", new Dictionary { [fieldNames[2]] = "Hello world, there are products here", [UmbracoExamineFieldNames.VariesByCultureFieldName] = "y" })); } - return indexer; + return index; } private PublishedContentQuery CreatePublishedContentQuery(IIndex indexer) @@ -75,10 +85,10 @@ namespace Umbraco.Tests.Web return new PublishedContentQuery(snapshot, variationContextAccessor, examineManager.Object); } - [TestCase("fr-fr", ExpectedResult = "1, 3", TestName = "Search Culture: fr-fr. Must return both fr-fr and invariant results")] - [TestCase("en-us", ExpectedResult = "1, 2", TestName = "Search Culture: en-us. Must return both en-us and invariant results")] - [TestCase("*", ExpectedResult = "1, 2, 3", TestName = "Search Culture: *. Must return all cultures and all invariant results")] - [TestCase(null, ExpectedResult = "1", TestName = "Search Culture: null. Must return only invariant results")] + [TestCase("fr-fr", ExpectedResult = "1, 3", Description = "Search Culture: fr-fr. Must return both fr-fr and invariant results")] + [TestCase("en-us", ExpectedResult = "1, 2", Description = "Search Culture: en-us. Must return both en-us and invariant results")] + [TestCase("*", ExpectedResult = "1, 2, 3", Description = "Search Culture: *. Must return all cultures and all invariant results")] + [TestCase(null, ExpectedResult = "1", Description = "Search Culture: null. Must return only invariant results")] public string Search(string culture) { using (var luceneDir = new RandomIdRAMDirectory()) diff --git a/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/RandomIdRAMDirectory.cs b/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/RandomIdRAMDirectory.cs new file mode 100644 index 0000000000..3d8fc1f192 --- /dev/null +++ b/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/RandomIdRAMDirectory.cs @@ -0,0 +1,11 @@ +using System; +using Lucene.Net.Store; + +namespace Umbraco.Cms.Tests.Integration.Umbraco.Examine.Lucene.UmbracoExamine +{ + public class RandomIdRAMDirectory : RAMDirectory + { + private readonly string _lockId = Guid.NewGuid().ToString(); + public override string GetLockID() => _lockId; + } +} diff --git a/src/Umbraco.Tests/UmbracoExamine/SearchTests.cs b/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/SearchTests.cs similarity index 69% rename from src/Umbraco.Tests/UmbracoExamine/SearchTests.cs rename to src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/SearchTests.cs index c4698fcdf2..2aefc593db 100644 --- a/src/Umbraco.Tests/UmbracoExamine/SearchTests.cs +++ b/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/SearchTests.cs @@ -1,23 +1,22 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using Examine; +using Examine.Lucene.Providers; using Examine.Search; -using Microsoft.Extensions.DependencyInjection; using Moq; using NUnit.Framework; using Umbraco.Cms.Core.Models; using Umbraco.Cms.Core.Persistence.Querying; -using Umbraco.Cms.Core.PropertyEditors; using Umbraco.Cms.Core.Services; using Umbraco.Cms.Infrastructure.Examine; using Umbraco.Cms.Tests.Common.Testing; using Umbraco.Extensions; -namespace Umbraco.Tests.UmbracoExamine +namespace Umbraco.Cms.Tests.Integration.Umbraco.Examine.Lucene.UmbracoExamine { [TestFixture] - [UmbracoTest(Database = UmbracoTestOptions.Database.NewSchemaPerTest)] + [UmbracoTest(Database = UmbracoTestOptions.Database.NewSchemaPerTest, Logger = UmbracoTestOptions.Logger.Console)] public class SearchTests : ExamineBaseTest { @@ -54,17 +53,14 @@ namespace Umbraco.Tests.UmbracoExamine == allRecs); - var propertyEditors = Factory.GetRequiredService(); - var rebuilder = IndexInitializer.GetContentIndexRebuilder(propertyEditors, contentService, ScopeProvider, UmbracoDatabaseFactory,true); - - using (var luceneDir = new RandomIdRamDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, HostingEnvironment, RuntimeState, luceneDir)) - using (indexer.ProcessNonAsync()) + using (GetSynchronousContentIndex(false, out UmbracoContentIndex index, out ContentIndexPopulator contentRebuilder, out _, null, contentService)) { - indexer.CreateIndex(); - rebuilder.Populate(indexer); + index.CreateIndex(); + contentRebuilder.Populate(index); - var searcher = indexer.GetSearcher(); + var searcher = index.Searcher; + + Assert.Greater(searcher.CreateQuery().All().Execute().TotalItemCount, 0); var numberSortedCriteria = searcher.CreateQuery() .ParentId(1148) @@ -99,23 +95,5 @@ namespace Umbraco.Tests.UmbracoExamine return true; } - //[Test] - //public void Test_Index_Type_With_German_Analyzer() - //{ - // using (var luceneDir = new RandomIdRamDirectory()) - // { - // var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir, - // new GermanAnalyzer()); - // indexer.RebuildIndex(); - // var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir); - // } - //} - - //private readonly TestContentService _contentService = new TestContentService(); - //private readonly TestMediaService _mediaService = new TestMediaService(); - //private static UmbracoExamineSearcher _searcher; - //private static UmbracoContentIndexer _indexer; - //private Lucene.Net.Store.Directory _luceneDir; - } } diff --git a/src/Umbraco.Tests/UmbracoExamine/TestFiles.Designer.cs b/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/TestFiles.Designer.cs similarity index 93% rename from src/Umbraco.Tests/UmbracoExamine/TestFiles.Designer.cs rename to src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/TestFiles.Designer.cs index b60dc487de..166d329208 100644 --- a/src/Umbraco.Tests/UmbracoExamine/TestFiles.Designer.cs +++ b/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/TestFiles.Designer.cs @@ -8,7 +8,7 @@ // //------------------------------------------------------------------------------ -namespace Umbraco.Tests.UmbracoExamine { +namespace Umbraco.Cms.Tests.Integration.Umbraco.Examine.Lucene.UmbracoExamine { using System; @@ -19,7 +19,7 @@ namespace Umbraco.Tests.UmbracoExamine { // class via a tool like ResGen or Visual Studio. // To add or remove a member, edit your .ResX file then rerun ResGen // with the /str option, or rebuild your VS project. - [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "15.0.0.0")] + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "16.0.0.0")] [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] internal class TestFiles { @@ -39,7 +39,7 @@ namespace Umbraco.Tests.UmbracoExamine { internal static global::System.Resources.ResourceManager ResourceManager { get { if (object.ReferenceEquals(resourceMan, null)) { - global::System.Resources.ResourceManager temp = new global::System.Resources.ResourceManager("Umbraco.Tests.UmbracoExamine.TestFiles", typeof(TestFiles).Assembly); + global::System.Resources.ResourceManager temp = new global::System.Resources.ResourceManager("Umbraco.Cms.Tests.Integration.Umbraco.Examine.Lucene.UmbracoExamine.TestFiles", typeof(TestFiles).Assembly); resourceMan = temp; } return resourceMan; diff --git a/src/Umbraco.Tests/UmbracoExamine/TestFiles.resx b/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/TestFiles.resx similarity index 96% rename from src/Umbraco.Tests/UmbracoExamine/TestFiles.resx rename to src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/TestFiles.resx index e23540252a..b5ed853136 100644 --- a/src/Umbraco.Tests/UmbracoExamine/TestFiles.resx +++ b/src/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/TestFiles.resx @@ -1,4 +1,4 @@ - +