diff --git a/src/NuGet.Config b/src/NuGet.Config index 89e79db976..722f368692 100644 --- a/src/NuGet.Config +++ b/src/NuGet.Config @@ -3,5 +3,6 @@ + \ No newline at end of file diff --git a/src/Umbraco.Core/Constants-Examine.cs b/src/Umbraco.Core/Constants-Examine.cs index 1356e55775..2f52e289c6 100644 --- a/src/Umbraco.Core/Constants-Examine.cs +++ b/src/Umbraco.Core/Constants-Examine.cs @@ -18,21 +18,7 @@ /// The alias of the external content indexer /// public const string ExternalIndexer = "ExternalIndexer"; - - /// - /// The alias of the internal member searcher - /// - public const string InternalMemberSearcher = "InternalMemberSearcher"; - - /// - /// The alias of the internal content searcher - /// - public const string InternalSearcher = "InternalSearcher"; - - /// - /// The alias of the external content searcher - /// - public const string ExternalSearcher = "ExternalSearcher"; + } } } diff --git a/src/Umbraco.Core/IRuntimeState.cs b/src/Umbraco.Core/IRuntimeState.cs index 5c0c01ef3b..5fcfab1518 100644 --- a/src/Umbraco.Core/IRuntimeState.cs +++ b/src/Umbraco.Core/IRuntimeState.cs @@ -57,6 +57,16 @@ namespace Umbraco.Core /// RuntimeLevel Level { get; } + /// + /// Gets the current migration state. + /// + string CurrentMigrationState { get; } + + /// + /// Gets the final migration state. + /// + string FinalMigrationState { get; } + /// /// Gets the exception that caused the boot to fail. /// diff --git a/src/Umbraco.Core/Migrations/MigrationPlan.cs b/src/Umbraco.Core/Migrations/MigrationPlan.cs index f686ff283e..733ff281e5 100644 --- a/src/Umbraco.Core/Migrations/MigrationPlan.cs +++ b/src/Umbraco.Core/Migrations/MigrationPlan.cs @@ -7,6 +7,9 @@ using Umbraco.Core.Scoping; namespace Umbraco.Core.Migrations { + /// + /// Represents a migration plan. + /// public class MigrationPlan { private readonly IMigrationBuilder _migrationBuilder; @@ -16,31 +19,67 @@ namespace Umbraco.Core.Migrations private string _prevState; private string _finalState; - // initializes a non-executing plan + /// + /// Initializes a new instance of the class. + /// + /// The name of the plan. + /// The plan cannot be executed. Use this constructor e.g. when only validating the plan, + /// or trying to get its final state, without actually needing to execute it. public MigrationPlan(string name) { if (string.IsNullOrWhiteSpace(name)) throw new ArgumentNullOrEmptyException(nameof(name)); Name = name; + + // ReSharper disable once VirtualMemberCallInConstructor + // (accepted) + DefinePlan(); } - // initializes a plan + /// + /// Initializes a new instance of the class. + /// + /// The name of the plan. + /// A migration builder. + /// A logger. + /// The plan can be executed. public MigrationPlan(string name, IMigrationBuilder migrationBuilder, ILogger logger) { if (string.IsNullOrWhiteSpace(name)) throw new ArgumentNullOrEmptyException(nameof(name)); Name = name; _migrationBuilder = migrationBuilder ?? throw new ArgumentNullException(nameof(migrationBuilder)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + // ReSharper disable once VirtualMemberCallInConstructor + // (accepted) + DefinePlan(); } + /// + /// Defines the plan. + /// + protected virtual void DefinePlan() { } + + /// + /// Gets the name of the plan. + /// public string Name { get; } + /// + /// Adds an empty migration from source to target state. + /// public MigrationPlan Add(string sourceState, string targetState) => Add(sourceState, targetState); + /// + /// Adds a migration from source to target state. + /// public MigrationPlan Add(string sourceState, string targetState) where TMigration : IMigration => Add(sourceState, targetState, typeof(TMigration)); + /// + /// Adds a migration from source to target state. + /// public MigrationPlan Add(string sourceState, string targetState, Type migration) { if (sourceState == null) throw new ArgumentNullException(nameof(sourceState)); @@ -78,26 +117,50 @@ namespace Umbraco.Core.Migrations return this; } + /// + /// Chains an empty migration from chain to target state. + /// public MigrationPlan Chain(string targetState) => Chain(targetState); + /// + /// Chains a migration from chain to target state. + /// public MigrationPlan Chain(string targetState) where TMigration : IMigration => Chain(targetState, typeof(TMigration)); + /// + /// Chains a migration from chain to target state. + /// public MigrationPlan Chain(string targetState, Type migration) => Add(_prevState, targetState, migration); + /// + /// Sets the chain state. + /// public MigrationPlan From(string sourceState) { _prevState = sourceState ?? throw new ArgumentNullException(nameof(sourceState)); return this; } + /// + /// Gets the initial state. + /// + /// The initial state is the state when no entry for the plan + /// could be found in the database (i.e. the plan has never run). public virtual string InitialState => string.Empty; + /// + /// Gets the final state. + /// public string FinalState => _finalState ?? (_finalState = Validate()); + /// + /// Validates the plan. + /// + /// The plan's final state. public string Validate() { // quick check for dead ends - a dead end is a transition that has a target state @@ -134,6 +197,13 @@ namespace Umbraco.Core.Migrations return finalState; } + /// + /// Executes the plan. + /// + /// A scope. + /// The state to start execution at. + /// The final state. + /// The plan executes within the scope, which must then be completed. public string Execute(IScope scope, string fromState) { Validate(); @@ -142,7 +212,7 @@ namespace Umbraco.Core.Migrations throw new InvalidOperationException("Cannot execute a non-executing plan."); _logger.Info("Starting \"{0}\"...", () => Name); - var origState = fromState; //GetState(); + var origState = fromState ?? string.Empty; var info = "At " + (string.IsNullOrWhiteSpace(origState) ? "origin" : ("\"" + origState + "\"")) + "."; info = info.Replace("{", "{{").Replace("}", "}}"); // stupid log4net _logger.Info(info); @@ -159,7 +229,7 @@ namespace Umbraco.Core.Migrations var nextState = transition.TargetState; origState = nextState; - + _logger.Info("At \"{0}\".", origState); if (!_transitions.TryGetValue(origState, out transition)) diff --git a/src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs b/src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs index fb0b3fc670..f8cd4b0f78 100644 --- a/src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs +++ b/src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs @@ -18,18 +18,29 @@ namespace Umbraco.Core.Migrations.Upgrade /// public class UmbracoPlan : MigrationPlan { + /// + /// Initializes a new instance of the class. + /// public UmbracoPlan() : base(Constants.System.UmbracoUpgradePlanName) - { - DefinePlan(); - } + { } + /// + /// Initializes a new instance of the class. + /// public UmbracoPlan(IMigrationBuilder migrationBuilder, ILogger logger) : base(Constants.System.UmbracoUpgradePlanName, migrationBuilder, logger) - { - DefinePlan(); - } + { } + /// + /// + /// The default initial state in plans is string.Empty. + /// When upgrading from version 7, we want to use specific initial states + /// that are e.g. "{orig-7.9.3}", "{orig-7.11.1}", etc. so we can chain the proper + /// migrations. + /// This is also where we detect the current version, and reject invalid + /// upgrades (from a tool old version, or going back in time, etc). + /// public override string InitialState { get @@ -38,8 +49,8 @@ namespace Umbraco.Core.Migrations.Upgrade if (!SemVersion.TryParse(ConfigurationManager.AppSettings["umbracoConfigurationStatus"], out var currentVersion)) throw new InvalidOperationException("Could not get current version from web.config umbracoConfigurationStatus appSetting."); - // must be at least 7.?.? - fixme adjust when releasing - if (currentVersion < new SemVersion(7, 999)) + // must be at least 7.? - fixme adjust when releasing + if (currentVersion < new SemVersion(7, 9)) throw new InvalidOperationException($"Version {currentVersion} cannot be upgraded to {UmbracoVersion.SemanticVersion}."); // cannot go back in time @@ -62,64 +73,85 @@ namespace Umbraco.Core.Migrations.Upgrade } } - private void DefinePlan() + /// + protected override void DefinePlan() { + // NOTE: MODIFYING THE PLAN + // + // Please take great care when modifying the plan! + // + // * Creating a migration for version 8: + // Append the migration to the main version 8 chain, using a new guid. + // Update the final state (see end of file) to that guid. + // Append the migration to version 7 upgrade chains. + // * Porting a migration from version 7: + // Append the migration to the main version 8 chain, using a new guid. + // Update the final state (see end of file) to that guid. + // Update all init-7.x.y chains. + + // UPGRADE FROM 7 // - // when 8.0.0 is released, on the first upgrade, the state is automatically - // set to {init-7.x.y} where 7.x.y is the version (see above), and then we define upgrades. + // When upgrading from version 7, the state is automatically set to {init-7.x.y} where + // 7.x.y is the version. We need to define a chain starting at that state and taking + // us to version 8. And we need such a chain for each 7.x.y version that can be upgraded + // to version 8, bearing in mind that new releases of version 7 will probably be + // created *after* the first released of version 8. // - // then, as more v7 and v8 versions are released, new chains needs to be defined to - // support the upgrades (new v7 may backport some migrations and require their own - // upgrade paths, etc). - // fixme adjust when releasing + // fixme adjust when releasing the first public (alpha?) version - From("{init-7.8.0}"); - Chain("{7C447271-CA3F-4A6A-A913-5D77015655CB}"); // add more lock objects + // at the moment we don't support upgrading from versions older than 7.9.0 + // + From("{init-7.9.0}"); + Chain("{7C447271-CA3F-4A6A-A913-5D77015655CB}"); Chain("{CBFF58A2-7B50-4F75-8E98-249920DB0F37}"); Chain("{3D18920C-E84D-405C-A06A-B7CEE52FE5DD}"); + Chain("{FB0A5429-587E-4BD0-8A67-20F0E7E62FF7}"); Chain("{F0C42457-6A3B-4912-A7EA-F27ED85A2092}"); Chain("{8640C9E4-A1C0-4C59-99BB-609B4E604981}"); Chain("{DD1B99AF-8106-4E00-BAC7-A43003EA07F8}"); Chain("{9DF05B77-11D1-475C-A00A-B656AF7E0908}"); - Chain("{CA7DB949-3EF4-403D-8464-F9BA36A52E87}"); + Chain("{6FE3EF34-44A0-4992-B379-B40BC4EF1C4D}"); + Chain("{7F59355A-0EC9-4438-8157-EB517E6D2727}"); - // 7.8.1 = same as 7.8.0 - From("{init-7.8.1}"); - Chain("{init-7.8.0}"); + // must chain to v8 final state (see at end of file) + Chain("{8918450B-3DA0-4BB7-886A-6FA8B7E4186E}"); - // 7.9.0 = requires its own chain - From("{init-7.9.0}"); - // chain... - Chain("{82C4BA1D-7720-46B1-BBD7-07F3F73800E6}"); + // 7.9.1, .2, .3 = same as 7.9.0 + // 7.10.0 = same as 7.9.0 + From("{init-7.9.1}").Chain("{init-7.9.0}"); + From("{init-7.9.2}").Chain("{init-7.9.0}"); + From("{init-7.9.3}").Chain("{init-7.9.0}"); + From("{init-7.10.0}").Chain("{init-7.9.0}"); - // UPGRADE 8 + // VERSION 8 PLAN // - // starting from the original 8.0.0 final state, chain migrations to upgrade version 8, - // defining new final states as more migrations are added to the chain. + // this is the master Umbraco migration plan, starting from the very first version 8 + // release, which was a pre-pre-alpha, years ago. It contains migrations created + // for version 8, along with migrations ported from version 7 as version 7 evolves. + // It is therefore *normal* that some pure version 8 migrations are mixed with + // migrations merged from version 7. // - // before v8 is released, some sites may exist, and these "pre-8" versions require their - // own upgrade plan. in other words, this is also the plan for sites that were on v8 before - // v8 was released + // new migrations should always be *appended* to the *end* of the chain. // 8.0.0 - From("{init-origin}"); + From("{init-origin}"); // "origin" was 7.4.something Chain("{98347B5E-65BF-4DD7-BB43-A09CB7AF4FCA}"); Chain("{1E8165C4-942D-40DC-AC76-C5FF8831E400}"); Chain("{39E15568-7AAD-4D54-81D0-758CCFC529F8}"); Chain("{55C3F97D-BDA7-4FB1-A743-B0456B56EAA3}"); - // 7.5.0 + // merging from 7.5.0 Chain("{287F9E39-F673-42F7-908C-21659AB13B13}"); Chain("{2D08588A-AD90-479C-9F6E-A99B60BA7226}"); Chain("{2D917FF8-AC81-4C00-A407-1F4B1DF6089C}"); - // 7.5.5 + // merging from 7.5.5 Chain("{44484C32-EEB3-4A12-B1CB-11E02CE22AB2}"); - // 7.6.0 + // merging from 7.6.0 Chain("{3586E4E9-2922-49EB-8E2A-A530CE6DBDE0}"); Chain("{D4A5674F-654D-4CC7-85E5-CFDBC533A318}"); Chain("{7F828EDD-6622-4A8D-AD80-EEAF46C11680}"); @@ -130,7 +162,7 @@ namespace Umbraco.Core.Migrations.Upgrade Chain("{5496C6CC-3AE0-4789-AF49-5BB4E28FA424}"); Chain("{8995332B-085E-4C0C-849E-9A77E79F4293}"); - // 7.7.0 + // merging from 7.7.0 Chain("{74319856-7681-46B1-AA0D-F7E896FBE6A1}"); Chain("{0427B0A2-994A-4AB4-BFF3-31B20614F6C9}"); Chain("{F0D6F782-E432-46DE-A3A7-2AF06DB8853B}"); @@ -147,11 +179,7 @@ namespace Umbraco.Core.Migrations.Upgrade Chain("{CA7DB949-3EF4-403D-8464-F9BA36A52E87}"); Chain("{7F0BF916-F64E-4B25-864A-170D6E6B68E5}"); - // at this point of the chain, people started to work on v8, so whenever we - // merge stuff from v7, we have to chain the migrations here so they also - // run for v8. - - // mergin from 7.8.0 + // merging from 7.8.0 Chain("{FDCB727A-EFB6-49F3-89E4-A346503AB849}"); Chain("{2A796A08-4FE4-4783-A1A5-B8A6C8AA4A92}"); Chain("{1A46A98B-2AAB-4C8E-870F-A2D55A97FD1F}"); @@ -163,9 +191,9 @@ namespace Umbraco.Core.Migrations.Upgrade Chain("{FD8631BC-0388-425C-A451-5F58574F6F05}"); Chain("{2821F53E-C58B-4812-B184-9CD240F990D7}"); Chain("{8918450B-3DA0-4BB7-886A-6FA8B7E4186E}"); - Chain("FIXGUID NEW FINAL"); // FINAL STATE - MUST MATCH LAST ONE ABOVE ! + // whenever this changes, update all references in this file! Add(string.Empty, "{8918450B-3DA0-4BB7-886A-6FA8B7E4186E}"); } diff --git a/src/Umbraco.Core/Runtime/CoreRuntime.cs b/src/Umbraco.Core/Runtime/CoreRuntime.cs index 7d28ffcf57..44d7ad3f68 100644 --- a/src/Umbraco.Core/Runtime/CoreRuntime.cs +++ b/src/Umbraco.Core/Runtime/CoreRuntime.cs @@ -138,7 +138,7 @@ namespace Umbraco.Core.Runtime try { var dbfactory = container.GetInstance(); - SetRuntimeStateLevel(_state, dbfactory, Logger); + SetRuntimeStateLevel(dbfactory, Logger); Logger.Debug($"Runtime level: {_state.Level}"); } catch @@ -233,38 +233,38 @@ namespace Umbraco.Core.Runtime builder.AddCore(); } - private void SetRuntimeStateLevel(RuntimeState runtimeState, IUmbracoDatabaseFactory databaseFactory, ILogger logger) + private void SetRuntimeStateLevel(IUmbracoDatabaseFactory databaseFactory, ILogger logger) { var localVersion = UmbracoVersion.Local; // the local, files, version - var codeVersion = runtimeState.SemanticVersion; // the executing code version + var codeVersion = _state.SemanticVersion; // the executing code version var connect = false; // we don't know yet - runtimeState.Level = RuntimeLevel.Unknown; + _state.Level = RuntimeLevel.Unknown; if (localVersion == null) { // there is no local version, we are not installed logger.Debug("No local version, need to install Umbraco."); - runtimeState.Level = RuntimeLevel.Install; + _state.Level = RuntimeLevel.Install; } else if (localVersion != codeVersion) { // there *is* a local version, but it does not match the code version // need to upgrade logger.Debug($"Local version \"{localVersion}\" != code version \"{codeVersion}\", need to upgrade Umbraco."); - runtimeState.Level = RuntimeLevel.Upgrade; + _state.Level = RuntimeLevel.Upgrade; } else if (databaseFactory.Configured == false) { // local version *does* match code version, but the database is not configured // install (again? this is a weird situation...) logger.Debug("Database is not configured, need to install Umbraco."); - runtimeState.Level = RuntimeLevel.Install; + _state.Level = RuntimeLevel.Install; } // install? not going to test anything else - if (runtimeState.Level == RuntimeLevel.Install) + if (_state.Level == RuntimeLevel.Install) return; // else, keep going, @@ -284,14 +284,14 @@ namespace Umbraco.Core.Runtime { // cannot connect to configured database, this is bad, fail logger.Debug("Could not connect to database."); - runtimeState.Level = RuntimeLevel.BootFailed; + _state.Level = RuntimeLevel.BootFailed; // in fact, this is bad enough that we want to throw throw new BootFailedException("A connection string is configured but Umbraco could not connect to the database."); } // if we already know we want to upgrade, no need to look for migrations... - if (runtimeState.Level == RuntimeLevel.Upgrade) + if (_state.Level == RuntimeLevel.Upgrade) return; // else @@ -302,18 +302,19 @@ namespace Umbraco.Core.Runtime { exists = EnsureUmbracoUpgradeState(databaseFactory, logger); } - catch + catch (Exception e) { // can connect to the database but cannot access the migration table... need to install + logger.Warn(e, "Could not check the upgrade state."); logger.Debug("Could not check the upgrade state, need to install Umbraco."); - runtimeState.Level = RuntimeLevel.Install; + _state.Level = RuntimeLevel.Install; return; } if (exists) { // the database version matches the code & files version, all clear, can run - runtimeState.Level = RuntimeLevel.Run; + _state.Level = RuntimeLevel.Run; return; } @@ -323,7 +324,7 @@ namespace Umbraco.Core.Runtime // although the files version matches the code version, the database version does not // which means the local files have been upgraded but not the database - need to upgrade logger.Debug("Has not reached the final upgrade step, need to upgrade Umbraco."); - runtimeState.Level = RuntimeLevel.Upgrade; + _state.Level = RuntimeLevel.Upgrade; } protected virtual bool EnsureUmbracoUpgradeState(IUmbracoDatabaseFactory databaseFactory, ILogger logger) @@ -343,11 +344,12 @@ namespace Umbraco.Core.Runtime state = database.FirstOrDefault(sql)?.Value; } - var finalState = umbracoPlan.FinalState; + _state.CurrentMigrationState = state; + _state.FinalMigrationState = umbracoPlan.FinalState; - logger.Debug($"Final upgrade state is \"{finalState}\", database contains \"{state ?? ""}\"."); + logger.Debug($"Final upgrade state is \"{_state.FinalMigrationState}\", database contains \"{state ?? ""}\"."); - return state == finalState; + return state == _state.FinalMigrationState; } #region Locals diff --git a/src/Umbraco.Core/RuntimeState.cs b/src/Umbraco.Core/RuntimeState.cs index 3c3ebdbd84..d740352649 100644 --- a/src/Umbraco.Core/RuntimeState.cs +++ b/src/Umbraco.Core/RuntimeState.cs @@ -85,6 +85,12 @@ namespace Umbraco.Core /// This is either "/" or eg "/virtual". public string ApplicationVirtualPath { get; } = HttpRuntime.AppDomainAppVirtualPath; + /// + public string CurrentMigrationState { get; internal set; } + + /// + public string FinalMigrationState { get; internal set; } + /// /// Gets the runtime level of execution. /// diff --git a/src/Umbraco.Examine/BaseUmbracoIndexer.cs b/src/Umbraco.Examine/BaseUmbracoIndexer.cs deleted file mode 100644 index 24d3168240..0000000000 --- a/src/Umbraco.Examine/BaseUmbracoIndexer.cs +++ /dev/null @@ -1,459 +0,0 @@ -using System; -using System.Collections.Generic; -using System.ComponentModel; -using System.Linq; -using System.Text; -using Examine.LuceneEngine.Config; -using Examine.LuceneEngine.Providers; -using Examine.Providers; -using Lucene.Net.Analysis; -using Lucene.Net.Index; -using Umbraco.Core; -using Examine; -using System.IO; -using System.Xml; -using System.Xml.Linq; -using Examine.LuceneEngine; -using Examine.LuceneEngine.Faceting; -using Examine.LuceneEngine.Indexing; -using Lucene.Net.Documents; -using Lucene.Net.Store; -using Umbraco.Core.Composing; -using Umbraco.Core.Logging; -using Umbraco.Core.Xml; -using Umbraco.Examine.LocalStorage; -using Directory = Lucene.Net.Store.Directory; - -namespace Umbraco.Examine -{ - /// - /// An abstract provider containing the basic functionality to be able to query against - /// Umbraco data. - /// - public abstract class BaseUmbracoIndexer : LuceneIndexer - { - // note - // wrapping all operations that end up calling base.SafelyProcessQueueItems in a safe call - // context because they will fork a thread/task/whatever which should *not* capture our - // call context (and the database it can contain)! ideally we should be able to override - // SafelyProcessQueueItems but that's not possible in the current version of Examine. - - /// - /// Used to store the path of a content object - /// - public const string IndexPathFieldName = "__Path"; - public const string NodeKeyFieldName = "__Key"; - public const string IconFieldName = "__Icon"; - public const string PublishedFieldName = "__Published"; - /// - /// The prefix added to a field when it is duplicated in order to store the original raw value. - /// - public const string RawFieldPrefix = "__Raw_"; - - /// - /// Default constructor - /// - protected BaseUmbracoIndexer() - : base() - { - ProfilingLogger = Current.ProfilingLogger; - _configBased = true; - } - - protected BaseUmbracoIndexer( - IEnumerable fieldDefinitions, - Directory luceneDirectory, - Analyzer defaultAnalyzer, - ProfilingLogger profilingLogger, - IValueSetValidator validator = null, - FacetConfiguration facetConfiguration = null, IDictionary> indexValueTypes = null) - : base(fieldDefinitions, luceneDirectory, defaultAnalyzer, validator, facetConfiguration, indexValueTypes) - { - if (profilingLogger == null) throw new ArgumentNullException("profilingLogger"); - ProfilingLogger = profilingLogger; - } - - private readonly bool _configBased = false; - private LocalTempStorageIndexer _localTempStorageIndexer; - - /// - /// A type that defines the type of index for each Umbraco field (non user defined fields) - /// Alot of standard umbraco fields shouldn't be tokenized or even indexed, just stored into lucene - /// for retreival after searching. - /// - [Obsolete("IndexFieldPolicies is not really used apart for some legacy reasons - use FieldDefinition's instead")] - internal static readonly List IndexFieldPolicies - = new List - { - new StaticField("id", FieldIndexTypes.NOT_ANALYZED, false, string.Empty), - new StaticField("key", FieldIndexTypes.NOT_ANALYZED, false, string.Empty), - new StaticField( "version", FieldIndexTypes.NOT_ANALYZED, false, string.Empty), - new StaticField( "parentID", FieldIndexTypes.NOT_ANALYZED, false, string.Empty), - new StaticField( "level", FieldIndexTypes.NOT_ANALYZED, true, "NUMBER"), - new StaticField( "writerID", FieldIndexTypes.NOT_ANALYZED, false, string.Empty), - new StaticField( "creatorID", FieldIndexTypes.NOT_ANALYZED, false, string.Empty), - new StaticField( "nodeType", FieldIndexTypes.NOT_ANALYZED, false, string.Empty), - new StaticField( "template", FieldIndexTypes.NOT_ANALYZED, false, string.Empty), - new StaticField( "sortOrder", FieldIndexTypes.NOT_ANALYZED, true, "NUMBER"), - new StaticField( "createDate", FieldIndexTypes.NOT_ANALYZED, false, "DATETIME"), - new StaticField( "updateDate", FieldIndexTypes.NOT_ANALYZED, false, "DATETIME"), - new StaticField( "nodeName", FieldIndexTypes.ANALYZED, false, string.Empty), - new StaticField( "urlName", FieldIndexTypes.NOT_ANALYZED, false, string.Empty), - new StaticField( "writerName", FieldIndexTypes.ANALYZED, false, string.Empty), - new StaticField( "creatorName", FieldIndexTypes.ANALYZED, false, string.Empty), - new StaticField( "nodeTypeAlias", FieldIndexTypes.ANALYZED, false, string.Empty), - new StaticField( "path", FieldIndexTypes.NOT_ANALYZED, false, string.Empty) - }; - - protected ProfilingLogger ProfilingLogger { get; private set; } - - /// - /// Overridden to ensure that the umbraco system field definitions are in place - /// - /// - /// - protected override IEnumerable InitializeFieldDefinitions(IEnumerable originalDefinitions) - { - var fd = base.InitializeFieldDefinitions(originalDefinitions).ToList(); - fd.AddRange(new[] - { - new FieldDefinition("parentID", FieldDefinitionTypes.Integer), - new FieldDefinition("level", FieldDefinitionTypes.Integer), - new FieldDefinition("writerID", FieldDefinitionTypes.Integer), - new FieldDefinition("creatorID", FieldDefinitionTypes.Integer), - new FieldDefinition("sortOrder", FieldDefinitionTypes.Integer), - new FieldDefinition("template", FieldDefinitionTypes.Integer), - - new FieldDefinition("createDate", FieldDefinitionTypes.DateTime), - new FieldDefinition("updateDate", FieldDefinitionTypes.DateTime), - - new FieldDefinition("key", FieldDefinitionTypes.Raw), - new FieldDefinition("version", FieldDefinitionTypes.Raw), - new FieldDefinition("nodeType", FieldDefinitionTypes.Raw), - new FieldDefinition("template", FieldDefinitionTypes.Raw), - new FieldDefinition("urlName", FieldDefinitionTypes.Raw), - new FieldDefinition("path", FieldDefinitionTypes.Raw), - - new FieldDefinition(IndexPathFieldName, FieldDefinitionTypes.Raw), - new FieldDefinition(NodeTypeAliasFieldName, FieldDefinitionTypes.Raw), - new FieldDefinition(IconFieldName, FieldDefinitionTypes.Raw) - }); - return fd; - } - - public bool UseTempStorage - { - get { return _localTempStorageIndexer != null && _localTempStorageIndexer.LuceneDirectory != null; } - } - - public string TempStorageLocation - { - get - { - if (UseTempStorage == false) return string.Empty; - return _localTempStorageIndexer.TempPath; - } - } - - [Obsolete("This should not be used, it is used by the configuration based indexes but instead to disable Examine event handlers use the ExamineEvents class instead.")] - [EditorBrowsable(EditorBrowsableState.Never)] - public bool EnableDefaultEventHandler { get; protected set; } - - /// - /// the supported indexable types - /// - protected abstract IEnumerable SupportedTypes { get; } - - #region Initialize - - - /// - /// Setup the properties for the indexer from the provider settings - /// - /// - /// - /// - /// This is ONLY used for configuration based indexes - /// - public override void Initialize(string name, System.Collections.Specialized.NameValueCollection config) - { - EnableDefaultEventHandler = true; //set to true by default - bool enabled; - if (bool.TryParse(config["enableDefaultEventHandler"], out enabled)) - { - EnableDefaultEventHandler = enabled; - } - - ProfilingLogger.Logger.Debug(GetType(), "{0} indexer initializing", () => name); - - base.Initialize(name, config); - - //NOTES: useTempStorage is obsolete, tempStorageDirectory is obsolete, both have been superceded by Examine Core's IDirectoryFactory - // tempStorageDirectory never actually got finished in Umbraco Core but accidentally got shipped (it's only enabled on the searcher - // and not the indexer). So this whole block is just legacy - - //detect if a dir factory has been specified, if so then useTempStorage will not be used (deprecated) - - if (config["directoryFactory"] == null && config["useTempStorage"] != null) - { - throw new NotImplementedException("Fix how local temp storage works and is synced with Examine v2.0 - since a writer is always open we cannot snapshot it, we need to use the same logic in AzureDirectory"); - - _localTempStorageIndexer = new LocalTempStorageIndexer(); - - var fsDir = base.GetLuceneDirectory() as FSDirectory; - if (fsDir != null) - { - //Use the temp storage directory which will store the index in the local/codegen folder, this is useful - // for websites that are running from a remove file server and file IO latency becomes an issue - var attemptUseTempStorage = config["useTempStorage"].TryConvertTo(); - if (attemptUseTempStorage) - { - - var indexSet = IndexSets.Instance.Sets[IndexSetName]; - var configuredPath = indexSet.IndexPath; - - _localTempStorageIndexer.Initialize(config, configuredPath, fsDir, IndexingAnalyzer, attemptUseTempStorage.Result); - } - } - - } - } - - #endregion - - - public override Directory GetLuceneDirectory() - { - //if temp local storage is configured use that, otherwise return the default - if (UseTempStorage) - { - return _localTempStorageIndexer.LuceneDirectory; - } - - return base.GetLuceneDirectory(); - - } - - /// - /// override to check if we can actually initialize. - /// - /// - /// This check is required since the base examine lib will try to rebuild on startup - /// - public override void RebuildIndex() - { - if (CanInitialize()) - { - ProfilingLogger.Logger.Debug(GetType(), "Rebuilding index"); - using (new SafeCallContext()) - { - base.RebuildIndex(); - } - } - } - - /// - /// override to check if we can actually initialize. - /// - /// - /// This check is required since the base examine lib will try to rebuild on startup - /// - public override void IndexAll(string type) - { - if (CanInitialize()) - { - using (new SafeCallContext()) - { - base.IndexAll(type); - } - } - } - - public override void IndexItems(IEnumerable nodes) - { - if (CanInitialize()) - { - using (new SafeCallContext()) - { - base.IndexItems(nodes); - } - } - } - - [Obsolete("Use ValueSets with IndexItems instead")] - public override void ReIndexNode(XElement node, string type) - { - if (CanInitialize()) - { - if (SupportedTypes.Contains(type) == false) - return; - - if (node.Attribute("id") != null) - { - ProfilingLogger.Logger.Debug(GetType(), "ReIndexNode {0} with type {1}", () => node.Attribute("id"), () => type); - using (new SafeCallContext()) - { - base.ReIndexNode(node, type); - } - } - else - { - ProfilingLogger.Logger.Error(GetType(), "ReIndexNode cannot proceed, the format of the XElement is invalid", - new XmlException("XElement is invalid, the xml has not id attribute")); - } - } - } - - /// - /// override to check if we can actually initialize. - /// - /// - /// This check is required since the base examine lib will try to rebuild on startup - /// - public override void DeleteFromIndex(string nodeId) - { - if (CanInitialize()) - { - using (new SafeCallContext()) - { - base.DeleteFromIndex(nodeId); - } - } - } - - /// - /// Returns true if the Umbraco application is in a state that we can initialize the examine indexes - /// - protected bool CanInitialize() - { - // only affects indexers that are config file based, if an index was created via code then - // this has no effect, it is assumed the index would not be created if it could not be initialized - return _configBased == false || Current.RuntimeState.Level == RuntimeLevel.Run; - } - - /// - /// Reindexes all supported types - /// - protected override void PerformIndexRebuild() - { - foreach (var t in SupportedTypes) - { - IndexAll(t); - } - } - - /// - /// overridden for logging - /// - /// - protected override void OnIndexingError(IndexingErrorEventArgs e) - { - ProfilingLogger.Logger.Error(GetType(), e.Message, e.Exception); - base.OnIndexingError(e); - } - - /// - /// Override for logging - /// - /// - protected override void OnIgnoringIndexItem(IndexItemEventArgs e) - { - ProfilingLogger.Logger.Debug(GetType(), "OnIgnoringIndexItem {0} with type {1}", () => e.IndexItem.ValueSet.Id, () => e.IndexItem.ValueSet.IndexCategory); - base.OnIgnoringIndexItem(e); - } - - /// - /// This ensures that the special __Raw_ fields are indexed - /// - /// - protected override void OnDocumentWriting(DocumentWritingEventArgs docArgs) - { - var d = docArgs.Document; - - foreach (var f in docArgs.Values.Values.Where(x => x.Key.StartsWith(RawFieldPrefix))) - { - if (f.Value.Count > 0) - { - d.Add(new Field( - f.Key, - f.Value[0].ToString(), - Field.Store.YES, - Field.Index.NO, //don't index this field, we never want to search by it - Field.TermVector.NO)); - } - } - - ProfilingLogger.Logger.Debug(GetType(), "OnDocumentWriting {0} with type {1}", () => docArgs.Values.Id, () => docArgs.Values.ItemType); - - base.OnDocumentWriting(docArgs); - } - - protected override void OnItemIndexed(IndexItemEventArgs e) - { - ProfilingLogger.Logger.Debug(GetType(), "Index created for node {0}", () => e.IndexItem.Id); - base.OnItemIndexed(e); - } - - protected override void OnIndexDeleted(DeleteIndexEventArgs e) - { - ProfilingLogger.Logger.Debug(GetType(), "Index deleted for term: {0} with value {1}", () => e.DeletedTerm.Key, () => e.DeletedTerm.Value); - base.OnIndexDeleted(e); - } - - [Obsolete("This is no longer used, index optimization is no longer managed with the LuceneIndexer")] - protected override void OnIndexOptimizing(EventArgs e) - { - ProfilingLogger.Logger.Debug(GetType(), "Index is being optimized"); - base.OnIndexOptimizing(e); - } - - /// - /// Overridden for logging. - /// - /// - protected override void AddDocument(ValueSet values) - { - ProfilingLogger.Logger.Debug(GetType(), "AddDocument {0} with type {1}", () => values.Id, () => values.ItemType); - base.AddDocument(values); - } - - protected override void OnTransformingIndexValues(TransformingIndexDataEventArgs e) - { - base.OnTransformingIndexValues(e); - - //ensure special __Path field - if (e.OriginalValues.ContainsKey("path") && e.IndexItem.ValueSet.Values.ContainsKey(IndexPathFieldName) == false) - { - e.IndexItem.ValueSet.Values[IndexPathFieldName] = new List { e.OriginalValues["path"].First() }; - } - - //strip html of all users fields if we detect it has HTML in it. - //if that is the case, we'll create a duplicate 'raw' copy of it so that we can return - //the value of the field 'as-is'. - foreach (var originalValue in e.OriginalValues) - { - if (originalValue.Value.Any()) - { - var str = originalValue.Value.First() as string; - if (str != null) - { - if (XmlHelper.CouldItBeXml(str)) - { - //First save the raw value to a raw field, we will change the policy of this field by detecting the prefix later - e.IndexItem.ValueSet.Values[string.Concat(RawFieldPrefix, originalValue.Key)] = new List { str }; - //now replace the original value with the stripped html - //TODO: This should be done with an analzer?! - e.IndexItem.ValueSet.Values[originalValue.Key] = new List { str.StripHtml() }; - } - } - } - } - - //icon - if (e.OriginalValues.ContainsKey("icon") && e.IndexItem.ValueSet.Values.ContainsKey(IconFieldName) == false) - { - e.IndexItem.ValueSet.Values[IconFieldName] = new List { e.OriginalValues["icon"] }; - } - } - } -} diff --git a/src/Umbraco.Examine/Config/ConfigIndexCriteria.cs b/src/Umbraco.Examine/Config/ConfigIndexCriteria.cs new file mode 100644 index 0000000000..de2a5ced36 --- /dev/null +++ b/src/Umbraco.Examine/Config/ConfigIndexCriteria.cs @@ -0,0 +1,36 @@ +using System.Collections.Generic; +using System.Linq; +using Examine; + +namespace Umbraco.Examine.Config +{ + /// + /// a data structure for storing indexing/searching instructions based on config based indexers + /// + public class ConfigIndexCriteria + { + /// + /// Constructor + /// + /// + /// + /// + /// + /// + public ConfigIndexCriteria(IEnumerable standardFields, IEnumerable userFields, IEnumerable includeNodeTypes, IEnumerable excludeNodeTypes, int? parentNodeId) + { + UserFields = userFields.ToList(); + StandardFields = standardFields.ToList(); + IncludeItemTypes = includeNodeTypes; + ExcludeItemTypes = excludeNodeTypes; + ParentNodeId = parentNodeId; + } + + public IEnumerable StandardFields { get; internal set; } + public IEnumerable UserFields { get; internal set; } + + public IEnumerable IncludeItemTypes { get; internal set; } + public IEnumerable ExcludeItemTypes { get; internal set; } + public int? ParentNodeId { get; internal set; } + } +} \ No newline at end of file diff --git a/src/Umbraco.Examine/Config/ConfigIndexField.cs b/src/Umbraco.Examine/Config/ConfigIndexField.cs new file mode 100644 index 0000000000..ec9cbf797e --- /dev/null +++ b/src/Umbraco.Examine/Config/ConfigIndexField.cs @@ -0,0 +1,61 @@ +using System.Configuration; + +namespace Umbraco.Examine.Config +{ + /// + /// A configuration item representing a field to index + /// + public sealed class ConfigIndexField : ConfigurationElement + { + [ConfigurationProperty("Name", IsRequired = true)] + public string Name + { + get => (string)this["Name"]; + set => this["Name"] = value; + } + + [ConfigurationProperty("EnableSorting", IsRequired = false)] + public bool EnableSorting + { + get => (bool)this["EnableSorting"]; + set => this["EnableSorting"] = value; + } + + [ConfigurationProperty("Type", IsRequired = false, DefaultValue = "String")] + public string Type + { + get => (string)this["Type"]; + set => this["Type"] = value; + } + + public bool Equals(ConfigIndexField other) + { + if (ReferenceEquals(null, other)) return false; + if (ReferenceEquals(this, other)) return true; + return string.Equals(Name, other.Name); + } + + public override bool Equals(object obj) + { + if (ReferenceEquals(null, obj)) return false; + if (ReferenceEquals(this, obj)) return true; + if (obj.GetType() != this.GetType()) return false; + return Equals((ConfigIndexField)obj); + } + + public override int GetHashCode() + { + return Name.GetHashCode(); + } + + public static bool operator ==(ConfigIndexField left, ConfigIndexField right) + { + return Equals(left, right); + } + + public static bool operator !=(ConfigIndexField left, ConfigIndexField right) + { + return !Equals(left, right); + } + } +} \ No newline at end of file diff --git a/src/Umbraco.Examine/Config/IndexFieldCollectionExtensions.cs b/src/Umbraco.Examine/Config/IndexFieldCollectionExtensions.cs new file mode 100644 index 0000000000..eea117ce05 --- /dev/null +++ b/src/Umbraco.Examine/Config/IndexFieldCollectionExtensions.cs @@ -0,0 +1,15 @@ +using System.Collections.Generic; + +namespace Umbraco.Examine.Config +{ + public static class IndexFieldCollectionExtensions + { + public static List ToList(this IndexFieldCollection indexes) + { + List fields = new List(); + foreach (ConfigIndexField field in indexes) + fields.Add(field); + return fields; + } + } +} \ No newline at end of file diff --git a/src/Umbraco.Examine/Config/IndexSet.cs b/src/Umbraco.Examine/Config/IndexSet.cs new file mode 100644 index 0000000000..33b67412f2 --- /dev/null +++ b/src/Umbraco.Examine/Config/IndexSet.cs @@ -0,0 +1,122 @@ +using System.Configuration; +using System.IO; +using System.Web; +using System.Web.Hosting; + +namespace Umbraco.Examine.Config +{ + public sealed class IndexSet : ConfigurationElement + { + + [ConfigurationProperty("SetName", IsRequired = true, IsKey = true)] + public string SetName => (string)this["SetName"]; + + private string _indexPath = ""; + + /// + /// The folder path of where the lucene index is stored + /// + /// The index path. + /// + /// This can be set at runtime but will not be persisted to the configuration file + /// + [ConfigurationProperty("IndexPath", IsRequired = true, IsKey = false)] + public string IndexPath + { + get + { + if (string.IsNullOrEmpty(_indexPath)) + _indexPath = (string)this["IndexPath"]; + + return _indexPath; + } + set => _indexPath = value; + } + + /// + /// Returns the DirectoryInfo object for the index path. + /// + /// The index directory. + public DirectoryInfo IndexDirectory + { + get + { + //TODO: Get this out of the index set. We need to use the Indexer's DataService to lookup the folder so it can be unit tested. Probably need DataServices on the searcher then too + + //we need to de-couple the context + if (HttpContext.Current != null) + return new DirectoryInfo(HttpContext.Current.Server.MapPath(this.IndexPath)); + else if (HostingEnvironment.ApplicationID != null) + return new DirectoryInfo(HostingEnvironment.MapPath(this.IndexPath)); + else + return new DirectoryInfo(this.IndexPath); + } + } + + /// + /// When this property is set, the indexing will only index documents that are descendants of this node. + /// + [ConfigurationProperty("IndexParentId", IsRequired = false, IsKey = false)] + public int? IndexParentId + { + get + { + if (this["IndexParentId"] == null) + return null; + + return (int)this["IndexParentId"]; + } + } + + /// + /// The collection of node types to index, if not specified, all node types will be indexed (apart from the ones specified in the ExcludeNodeTypes collection). + /// + [ConfigurationCollection(typeof(IndexFieldCollection))] + [ConfigurationProperty("IncludeNodeTypes", IsDefaultCollection = false, IsRequired = false)] + public IndexFieldCollection IncludeNodeTypes => (IndexFieldCollection)base["IncludeNodeTypes"]; + + /// + /// The collection of node types to not index. If specified, these node types will not be indexed. + /// + [ConfigurationCollection(typeof(IndexFieldCollection))] + [ConfigurationProperty("ExcludeNodeTypes", IsDefaultCollection = false, IsRequired = false)] + public IndexFieldCollection ExcludeNodeTypes => (IndexFieldCollection)base["ExcludeNodeTypes"]; + + /// + /// A collection of user defined umbraco fields to index + /// + /// + /// If this property is not specified, or if it's an empty collection, the default user fields will be all user fields defined in Umbraco + /// + [ConfigurationCollection(typeof(IndexFieldCollection))] + [ConfigurationProperty("IndexUserFields", IsDefaultCollection = false, IsRequired = false)] + public IndexFieldCollection IndexUserFields => (IndexFieldCollection)base["IndexUserFields"]; + + /// + /// The fields umbraco values that will be indexed. i.e. id, nodeTypeAlias, writer, etc... + /// + /// + /// If this is not specified, or if it's an empty collection, the default optins will be specified: + /// - id + /// - version + /// - parentID + /// - level + /// - writerID + /// - creatorID + /// - nodeType + /// - template + /// - sortOrder + /// - createDate + /// - updateDate + /// - nodeName + /// - urlName + /// - writerName + /// - creatorName + /// - nodeTypeAlias + /// - path + /// + [ConfigurationCollection(typeof(IndexFieldCollection))] + [ConfigurationProperty("IndexAttributeFields", IsDefaultCollection = false, IsRequired = false)] + public IndexFieldCollection IndexAttributeFields => (IndexFieldCollection)base["IndexAttributeFields"]; + } +} diff --git a/src/Umbraco.Examine/Config/IndexSetCollection.cs b/src/Umbraco.Examine/Config/IndexSetCollection.cs new file mode 100644 index 0000000000..5ba2382563 --- /dev/null +++ b/src/Umbraco.Examine/Config/IndexSetCollection.cs @@ -0,0 +1,73 @@ +using System.Configuration; + + +namespace Umbraco.Examine.Config +{ + public sealed class IndexFieldCollection : ConfigurationElementCollection + { + #region Overridden methods to define collection + protected override ConfigurationElement CreateNewElement() + { + return new ConfigIndexField(); + } + protected override object GetElementKey(ConfigurationElement element) + { + ConfigIndexField field = (ConfigIndexField)element; + return field.Name; + } + + public override bool IsReadOnly() + { + return false; + } + #endregion + + /// + /// Adds an index field to the collection + /// + /// + public void Add(ConfigIndexField field) + { + BaseAdd(field, true); + } + + /// + /// Default property for accessing an IndexField definition + /// + /// Field Name + /// + public new ConfigIndexField this[string name] + { + get + { + return (ConfigIndexField)this.BaseGet(name); + } + } + + } + + + public sealed class IndexSetCollection : ConfigurationElementCollection + { + #region Overridden methods to define collection + protected override ConfigurationElement CreateNewElement() + { + return new IndexSet(); + } + protected override object GetElementKey(ConfigurationElement element) + { + return ((IndexSet)element).SetName; + } + public override ConfigurationElementCollectionType CollectionType => ConfigurationElementCollectionType.BasicMap; + protected override string ElementName => "IndexSet"; + + #endregion + + /// + /// Default property for accessing Image Sets + /// + /// + /// + public new IndexSet this[string setName] => (IndexSet)this.BaseGet(setName); + } +} \ No newline at end of file diff --git a/src/Umbraco.Examine/Config/IndexSets.cs b/src/Umbraco.Examine/Config/IndexSets.cs new file mode 100644 index 0000000000..4aa90b99d7 --- /dev/null +++ b/src/Umbraco.Examine/Config/IndexSets.cs @@ -0,0 +1,26 @@ +using System.Configuration; + +namespace Umbraco.Examine.Config +{ + public sealed class IndexSets : ConfigurationSection + { + + #region Singleton definition + + protected IndexSets() { } + static IndexSets() + { + Instance = ConfigurationManager.GetSection(SectionName) as IndexSets; + + } + public static IndexSets Instance { get; } + + #endregion + + private const string SectionName = "ExamineLuceneIndexSets"; + + [ConfigurationCollection(typeof(IndexSetCollection))] + [ConfigurationProperty("", IsDefaultCollection = true, IsRequired = true)] + public IndexSetCollection Sets => (IndexSetCollection)base[""]; + } +} \ No newline at end of file diff --git a/src/Umbraco.Examine/DataServices/UmbracoContentService.cs b/src/Umbraco.Examine/DataServices/UmbracoContentService.cs deleted file mode 100644 index 5f3fba6505..0000000000 --- a/src/Umbraco.Examine/DataServices/UmbracoContentService.cs +++ /dev/null @@ -1,115 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using Umbraco.Core; -using System.Xml.Linq; -using System.Collections; -using System.Xml.XPath; -using Examine.LuceneEngine; -using Umbraco.Core.Logging; -using Umbraco.Core.Scoping; -using Umbraco.Core.Services; -using Umbraco.Core.Models; - -namespace Umbraco.Examine.DataServices -{ - public class UmbracoContentService - { - private readonly IScopeProvider _scopeProvider; - private readonly ServiceContext _services; - private readonly ILogger _logger; - - public UmbracoContentService(IScopeProvider scopeProvider, ServiceContext services, ILogger logger) - { - _scopeProvider = scopeProvider; - _services = services; - _logger = logger; - } - - /// - /// removes html markup from a string - /// - /// - /// - public string StripHtml(string value) - { - return value.StripHtml(); - } - - /// - /// Gets published content by xpath - /// - /// - /// - public XDocument GetPublishedContentByXPath(string xpath) - { - //TODO: Remove the need for this, the best way would be to remove all requirements of examine based on Xml but that - // would take some time. Another way in the in-term would be to add a static delegate to this class which can be set - // on the WebBootManager to set how to get the XmlNodeByXPath but that is still ugly :( - return LegacyLibrary.GetXmlNodeByXPath(xpath).ToXDocument(); - } - - /// - /// This is quite an intensive operation... - /// get all root content, then get the XML structure for all children, - /// then run xpath against the navigator that's created - /// - /// - /// - [Obsolete("This should no longer be used, latest content will be indexed by using the IContentService directly")] - public XDocument GetLatestContentByXPath(string xpath) - { - var xmlContent = XDocument.Parse(""); - foreach (var c in _services.ContentService.GetRootContent()) - { - xmlContent.Root.Add(c.ToDeepXml(_services.PackagingService)); - } - var result = ((IEnumerable)xmlContent.XPathEvaluate(xpath)).Cast(); - return result.ToXDocument(); - } - - /// - /// Check if the node is protected - /// - /// - /// - /// - public bool IsProtected(int nodeId, string path) - { - return _services.PublicAccessService.IsProtected(path.EnsureEndsWith("," + nodeId)); - } - - /// - /// Returns a list of all of the user defined property names in Umbraco - /// - /// - - public IEnumerable GetAllUserPropertyNames() - { - using (var scope = _scopeProvider.CreateScope()) - { - try - { - var result = scope.Database.Fetch("select distinct alias from cmsPropertyType order by alias"); - scope.Complete(); - return result; - } - catch (Exception ex) - { - _logger.Error("EXCEPTION OCCURRED reading GetAllUserPropertyNames", ex); - scope.Complete(); - return Enumerable.Empty(); - } - } - } - - /// - /// Returns a list of all system field names in Umbraco - /// - /// - public IEnumerable GetAllSystemPropertyNames() - { - return UmbracoContentIndexer.IndexFieldPolicies.Select(x => x.Name); - } - } -} diff --git a/src/Umbraco.Examine/DataServices/UmbracoMediaService.cs b/src/Umbraco.Examine/DataServices/UmbracoMediaService.cs deleted file mode 100644 index 436cd99b9e..0000000000 --- a/src/Umbraco.Examine/DataServices/UmbracoMediaService.cs +++ /dev/null @@ -1,45 +0,0 @@ -using System; -using System.Collections; -using System.Linq; -using System.Xml.Linq; -using System.Xml.XPath; -using Examine.LuceneEngine; -using Umbraco.Core.Services; -using Umbraco.Core.Models; - -namespace Umbraco.Examine.DataServices -{ - - /// - /// Data service used to query for media - /// - [Obsolete("This should no longer be used, latest content will be indexed by using the IMediaService directly")] - public class UmbracoMediaService - { - private readonly ServiceContext _services; - - public UmbracoMediaService(ServiceContext services) - { - _services = services; - } - - /// - /// This is quite an intensive operation... - /// get all root media, then get the XML structure for all children, - /// then run xpath against the navigator that's created - /// - /// - /// - [Obsolete("This should no longer be used, latest content will be indexed by using the IMediaService directly")] - public XDocument GetLatestMediaByXpath(string xpath) - { - var xmlMedia = XDocument.Parse(""); - foreach (var m in _services.MediaService.GetRootMedia()) - { - xmlMedia.Root.Add(m.ToDeepXml(_services.PackagingService)); - } - var result = ((IEnumerable)xmlMedia.XPathEvaluate(xpath)).Cast(); - return result.ToXDocument(); - } - } -} diff --git a/src/Umbraco.Examine/DeletePolicyTracker.cs b/src/Umbraco.Examine/DeletePolicyTracker.cs deleted file mode 100644 index 2c851ed39b..0000000000 --- a/src/Umbraco.Examine/DeletePolicyTracker.cs +++ /dev/null @@ -1,23 +0,0 @@ -using System.Collections.Concurrent; -using System.IO; -using Lucene.Net.Index; - -namespace Umbraco.Examine -{ - internal sealed class DeletePolicyTracker - { - private static readonly DeletePolicyTracker Instance = new DeletePolicyTracker(); - private readonly ConcurrentDictionary _directories = new ConcurrentDictionary(); - - public static DeletePolicyTracker Current - { - get { return Instance; } - } - - public IndexDeletionPolicy GetPolicy(Lucene.Net.Store.Directory directory) - { - var resolved = _directories.GetOrAdd(directory.GetLockId(), s => new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy())); - return resolved; - } - } -} diff --git a/src/Umbraco.Examine/ExamineIndexCollectionAccessor.cs b/src/Umbraco.Examine/ExamineIndexCollectionAccessor.cs deleted file mode 100644 index 806cbeadc7..0000000000 --- a/src/Umbraco.Examine/ExamineIndexCollectionAccessor.cs +++ /dev/null @@ -1,13 +0,0 @@ -using System.Collections.Generic; -using Examine; - -namespace Umbraco.Examine -{ - /// - /// Default implementation of IExamineIndexCollectionAccessor to return indexes from Examinemanager - /// - public class ExamineIndexCollectionAccessor : IExamineIndexCollectionAccessor - { - public IReadOnlyDictionary Indexes => ExamineManager.Instance.IndexProviders; - } -} diff --git a/src/Umbraco.Examine/IExamineIndexCollectionAccessor.cs b/src/Umbraco.Examine/IExamineIndexCollectionAccessor.cs deleted file mode 100644 index 4c55abc789..0000000000 --- a/src/Umbraco.Examine/IExamineIndexCollectionAccessor.cs +++ /dev/null @@ -1,13 +0,0 @@ -using System.Collections.Generic; -using Examine; - -namespace Umbraco.Examine -{ - /// - /// Returns a collection of IExamineIndexer - /// - public interface IExamineIndexCollectionAccessor - { - IReadOnlyDictionary Indexes { get; } - } -} diff --git a/src/Umbraco.Examine/LegacyExtensions.cs b/src/Umbraco.Examine/LegacyExtensions.cs deleted file mode 100644 index cf1eebb5e4..0000000000 --- a/src/Umbraco.Examine/LegacyExtensions.cs +++ /dev/null @@ -1,50 +0,0 @@ -using System.Linq; -using Examine; -using Examine.LuceneEngine.Config; -using Umbraco.Core.Services; - -namespace Umbraco.Examine -{ - internal static class LegacyExtensions - { - - private static readonly object Locker = new object(); - - public static IIndexCriteria ToIndexCriteria(this IndexSet set, IContentTypeService contentTypeService) - { - if (set.IndexUserFields.Count == 0) - { - lock (Locker) - { - //we need to add all user fields to the collection if it is empty (this is the default if none are specified) - var userFields = contentTypeService.GetAllPropertyTypeAliases(); - foreach (var u in userFields) - { - set.IndexUserFields.Add(new IndexField() { Name = u }); - } - } - } - - if (set.IndexAttributeFields.Count == 0) - { - lock (Locker) - { - //we need to add all system fields to the collection if it is empty (this is the default if none are specified) - var sysFields = BaseUmbracoIndexer.IndexFieldPolicies.Select(x => x.Name); - foreach (var s in sysFields) - { - set.IndexAttributeFields.Add(new IndexField() { Name = s }); - } - } - } - - return new IndexCriteria( - set.IndexAttributeFields.Cast().ToArray(), - set.IndexUserFields.Cast().ToArray(), - set.IncludeNodeTypes.ToList().Select(x => x.Name).ToArray(), - set.ExcludeNodeTypes.ToList().Select(x => x.Name).ToArray(), - set.IndexParentId); - } - - } -} diff --git a/src/Umbraco.Examine/LegacyLibrary.cs b/src/Umbraco.Examine/LegacyLibrary.cs deleted file mode 100644 index 5f3f296c9d..0000000000 --- a/src/Umbraco.Examine/LegacyLibrary.cs +++ /dev/null @@ -1,47 +0,0 @@ -using System; -using System.Reflection; -using System.Xml.XPath; - -namespace Umbraco.Examine -{ - /// - /// This is only used for backward compatibility to get access to the umbraco.library object but this needs to be done - /// via reflection because of the circular reference we have between Umbraco.Web and UmbracoExamine. - /// - internal static class LegacyLibrary - { - private static volatile Type _libraryType; - private static readonly object Locker = new object(); - private static Type LibraryType - { - get - { - if (_libraryType == null) - { - lock (Locker) - { - if (_libraryType == null) - { - var ass = Assembly.Load("Umbraco.Web"); - if (ass == null) - throw new InvalidOperationException("Could not load assembly Umbraco.Web.dll, the Umbraco.Web.dll needs to be loaded in the current app domain"); - var lib = ass.GetType("umbraco.library"); - if (lib == null) - throw new InvalidOperationException("Could not load type umbraco.library, the Umbraco.Web.dll needs to be loaded in the current app domain"); - _libraryType = lib; - } - } - } - return _libraryType; - } - } - - - internal static XPathNodeIterator GetXmlNodeByXPath(string xpathQuery) - { - var meth = LibraryType.GetMethod("GetXmlNodeByXPath", BindingFlags.Public | BindingFlags.Static); - return (XPathNodeIterator)meth.Invoke(null, new object[] { xpathQuery }); - } - - } -} diff --git a/src/Umbraco.Examine/LocalStorage/AzureLocalStorageDirectory.cs b/src/Umbraco.Examine/LocalStorage/AzureLocalStorageDirectory.cs deleted file mode 100644 index 1cdc64f879..0000000000 --- a/src/Umbraco.Examine/LocalStorage/AzureLocalStorageDirectory.cs +++ /dev/null @@ -1,33 +0,0 @@ -using System; -using System.Collections.Specialized; -using System.ComponentModel; -using System.IO; -using System.Web; -using Umbraco.Core; - -namespace Umbraco.Examine.LocalStorage -{ - /// - /// When running on Azure websites, we can use the local user's storage space - /// - [Obsolete("This has been superceded by IDirectoryFactory in Examine Core and should not be used")] - [EditorBrowsable(EditorBrowsableState.Never)] - public sealed class AzureLocalStorageDirectory : ILocalStorageDirectory - { - public DirectoryInfo GetLocalStorageDirectory(NameValueCollection config, string configuredPath) - { - var appDomainHash = HttpRuntime.AppDomainAppId.GenerateHash(); - var cachePath = Path.Combine(Environment.ExpandEnvironmentVariables("%temp%"), "LuceneDir", - //include the appdomain hash is just a safety check, for example if a website is moved from worker A to worker B and then back - // to worker A again, in theory the %temp% folder should already be empty but we really want to make sure that its not - // utilizing an old index - appDomainHash, - //ensure the temp path is consistent with the configured path location - configuredPath.TrimStart('~', '/').Replace("/", "\\")); - var azureDir = new DirectoryInfo(cachePath); - if (azureDir.Exists == false) - azureDir.Create(); - return azureDir; - } - } -} diff --git a/src/Umbraco.Examine/LocalStorage/CodeGenLocalStorageDirectory.cs b/src/Umbraco.Examine/LocalStorage/CodeGenLocalStorageDirectory.cs deleted file mode 100644 index 0678ca359e..0000000000 --- a/src/Umbraco.Examine/LocalStorage/CodeGenLocalStorageDirectory.cs +++ /dev/null @@ -1,29 +0,0 @@ -using System; -using System.Collections.Specialized; -using System.ComponentModel; -using System.IO; -using System.Web; - -namespace Umbraco.Examine.LocalStorage -{ - /// - /// Use the ASP.Net CodeGen folder to store the index files - /// - /// - /// This is the default implementation - but it comes with it's own limitations - the CodeGen folder is cleared whenever new - /// DLLs are changed in the /bin folder (among other circumstances) which means the index would be re-synced (or rebuilt) there. - /// - [Obsolete("This has been superceded by IDirectoryFactory in Examine Core and should not be used")] - [EditorBrowsable(EditorBrowsableState.Never)] - public sealed class CodeGenLocalStorageDirectory : ILocalStorageDirectory - { - public DirectoryInfo GetLocalStorageDirectory(NameValueCollection config, string configuredPath) - { - var codegenPath = HttpRuntime.CodegenDir; - var path = Path.Combine(codegenPath, - //ensure the temp path is consistent with the configured path location - configuredPath.TrimStart('~', '/').Replace("/", "\\")); - return new DirectoryInfo(path); - } - } -} diff --git a/src/Umbraco.Examine/LocalStorage/DirectoryTracker.cs b/src/Umbraco.Examine/LocalStorage/DirectoryTracker.cs deleted file mode 100644 index 5a6d4f6d2f..0000000000 --- a/src/Umbraco.Examine/LocalStorage/DirectoryTracker.cs +++ /dev/null @@ -1,26 +0,0 @@ -using System; -using System.Collections.Concurrent; -using System.IO; -using Lucene.Net.Store; - -namespace Umbraco.Examine.LocalStorage -{ - [Obsolete("This is used only for configuration based indexes")] - internal class DirectoryTracker - { - private static readonly DirectoryTracker Instance = new DirectoryTracker(); - - private readonly ConcurrentDictionary _directories = new ConcurrentDictionary(); - - public static DirectoryTracker Current - { - get { return Instance; } - } - - public Lucene.Net.Store.Directory GetDirectory(DirectoryInfo dir) - { - var resolved = _directories.GetOrAdd(dir.FullName, s => new SimpleFSDirectory(dir)); - return resolved; - } - } -} diff --git a/src/Umbraco.Examine/LocalStorage/ILocalStorageDirectory.cs b/src/Umbraco.Examine/LocalStorage/ILocalStorageDirectory.cs deleted file mode 100644 index aec007ec60..0000000000 --- a/src/Umbraco.Examine/LocalStorage/ILocalStorageDirectory.cs +++ /dev/null @@ -1,13 +0,0 @@ -using System.Collections.Specialized; -using System.IO; - -namespace Umbraco.Examine.LocalStorage -{ - /// - /// Used to resolve the local storage folder - /// - public interface ILocalStorageDirectory - { - DirectoryInfo GetLocalStorageDirectory(NameValueCollection config, string configuredPath); - } -} diff --git a/src/Umbraco.Examine/LocalStorage/LocalStorageType.cs b/src/Umbraco.Examine/LocalStorage/LocalStorageType.cs deleted file mode 100644 index e1d2e0f943..0000000000 --- a/src/Umbraco.Examine/LocalStorage/LocalStorageType.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace Umbraco.Examine.LocalStorage -{ - public enum LocalStorageType - { - Sync, - LocalOnly - } -} diff --git a/src/Umbraco.Examine/LocalStorage/LocalTempStorageDirectory.cs b/src/Umbraco.Examine/LocalStorage/LocalTempStorageDirectory.cs deleted file mode 100644 index 6319dc8878..0000000000 --- a/src/Umbraco.Examine/LocalStorage/LocalTempStorageDirectory.cs +++ /dev/null @@ -1,207 +0,0 @@ -using System; -using System.IO; -using System.Linq; -using Lucene.Net.Store; -using Directory = Lucene.Net.Store.Directory; - -namespace Umbraco.Examine.LocalStorage -{ - /// - /// Used to read data from local temp storage and write to both local storage and main storage - /// - public class LocalTempStorageDirectory : Directory - { - private readonly FSDirectory _tempStorageDir; - private readonly FSDirectory _realDirectory; - private LockFactory _lockFactory; - - public LocalTempStorageDirectory( - DirectoryInfo tempStorageDir, - FSDirectory realDirectory) - { - if (tempStorageDir == null) throw new ArgumentNullException("tempStorageDir"); - if (realDirectory == null) throw new ArgumentNullException("realDirectory"); - - _tempStorageDir = new SimpleFSDirectory(tempStorageDir); - _realDirectory = realDirectory; - _lockFactory = new MultiIndexLockFactory(_realDirectory, _tempStorageDir); - - Enabled = true; - - } - - /// - /// If initialization fails, it will be disabled and then this will just wrap the 'real directory' - /// - internal bool Enabled { get; set; } - - public override string[] ListAll() - { - //always from the real dir - return _realDirectory.ListAll(); - } - - /// Returns true if a file with the given name exists. - public override bool FileExists(string name) - { - //always from the real dir - return _realDirectory.FileExists(name); - } - - /// Returns the time the named file was last modified. - public override long FileModified(string name) - { - //always from the real dir - return _realDirectory.FileModified(name); - } - - /// Set the modified time of an existing file to now. - public override void TouchFile(string name) - { - //always from the real dir - _realDirectory.TouchFile(name); - - //perform on both dirs - if (Enabled) - { - _tempStorageDir.TouchFile(name); - } - } - - /// Removes an existing file in the directory. - public override void DeleteFile(string name) - { - _realDirectory.DeleteFile(name); - - //perform on both dirs - if (Enabled) - { - _tempStorageDir.DeleteFile(name); - } - } - - - /// Returns the length of a file in the directory. - public override long FileLength(string name) - { - //always from the real dir - return _realDirectory.FileLength(name); - } - - /// - /// Creates a new, empty file in the directory with the given name. - /// Returns a stream writing this file. - /// - public override IndexOutput CreateOutput(string name) - { - //write to both indexes - if (Enabled) - { - return new MultiIndexOutput( - _tempStorageDir.CreateOutput(name), - _realDirectory.CreateOutput(name)); - } - - return _realDirectory.CreateOutput(name); - } - - /// - /// Returns a stream reading an existing file. - /// - public override IndexInput OpenInput(string name) - { - if (Enabled) - { - //return the reader from the cache, not the real dir - return _tempStorageDir.OpenInput(name); - } - - return _realDirectory.OpenInput(name); - } - - /// - /// Creates an IndexInput for the file with the given name. - /// - public override IndexInput OpenInput(string name, int bufferSize) - { - if (Enabled) - { - //return the reader from the cache, not the real dir - return _tempStorageDir.OpenInput(name, bufferSize); - } - return _realDirectory.OpenInput(name, bufferSize); - } - - /// - /// Ensure that any writes to this file are moved to - /// stable storage. Lucene uses this to properly commit - /// changes to the index, to prevent a machine/OS crash - /// from corrupting the index. - /// - public override void Sync(string name) - { - _realDirectory.Sync(name); - _tempStorageDir.Sync(name); - base.Sync(name); - } - - public override Lock MakeLock(string name) - { - return _lockFactory.MakeLock(name); - } - - /// - /// Return a string identifier that uniquely differentiates - /// this Directory instance from other Directory instances. - /// This ID should be the same if two Directory instances - /// (even in different JVMs and/or on different machines) - /// are considered "the same index". This is how locking - /// "scopes" to the right index. - /// - /// - public override string GetLockId() - { - return string.Concat(_realDirectory.GetLockId(), _tempStorageDir.GetLockId()); - } - - public override LockFactory LockFactory - { - get - { - return _lockFactory; - //return _realDirectory.GetLockFactory(); - } - - } - - public override void ClearLock(string name) - { - _lockFactory.ClearLock(name); - - //_realDirectory.ClearLock(name); - - //if (Enabled) - //{ - // _tempStorageDir.ClearLock(name); - //} - } - - protected override void Dispose(bool disposing) - { - if (Enabled) - { - _tempStorageDir.Dispose(); - } - _realDirectory.Dispose(); - } - - public override void SetLockFactory(LockFactory lf) - { - _lockFactory = lf; - //_realDirectory.SetLockFactory(lf); - } - - - - } -} diff --git a/src/Umbraco.Examine/LocalStorage/LocalTempStorageDirectoryTracker.cs b/src/Umbraco.Examine/LocalStorage/LocalTempStorageDirectoryTracker.cs deleted file mode 100644 index d519925c07..0000000000 --- a/src/Umbraco.Examine/LocalStorage/LocalTempStorageDirectoryTracker.cs +++ /dev/null @@ -1,29 +0,0 @@ -using System.Collections.Concurrent; -using System.IO; -using Lucene.Net.Store; - -namespace Umbraco.Examine.LocalStorage -{ - internal class LocalTempStorageDirectoryTracker - { - private static readonly LocalTempStorageDirectoryTracker Instance = new LocalTempStorageDirectoryTracker(); - private readonly ConcurrentDictionary _directories = new ConcurrentDictionary(); - - public static LocalTempStorageDirectoryTracker Current - { - get { return Instance; } - } - - public LocalTempStorageDirectory GetDirectory(DirectoryInfo dir, FSDirectory realDir, bool disable = false) - { - var resolved = _directories.GetOrAdd(dir.FullName, s => new LocalTempStorageDirectory(dir, realDir)); - - if (disable) - { - resolved.Enabled = false; - } - - return resolved; - } - } -} diff --git a/src/Umbraco.Examine/LocalStorage/LocalTempStorageIndexer.cs b/src/Umbraco.Examine/LocalStorage/LocalTempStorageIndexer.cs deleted file mode 100644 index 93102578dd..0000000000 --- a/src/Umbraco.Examine/LocalStorage/LocalTempStorageIndexer.cs +++ /dev/null @@ -1,412 +0,0 @@ -using System; -using System.Collections.Specialized; -using System.IO; -using System.Linq; -using System.Threading; -using System.Web; -using System.Web.Compilation; -using Examine.LuceneEngine; -using Lucene.Net.Analysis; -using Lucene.Net.Index; -using Lucene.Net.Store; -using Umbraco.Core; -using Umbraco.Core.Composing; -using Umbraco.Core.IO; -using Umbraco.Core.Logging; -using Directory = System.IO.Directory; - -namespace Umbraco.Examine.LocalStorage -{ - internal enum InitializeDirectoryFlags - { - Success = 0, - SuccessNoIndexExists = 1, - - FailedCorrupt = 100, - FailedLocked = 101, - FailedFileSync = 102 - } - - internal class LocalTempStorageIndexer - { - public Lucene.Net.Store.Directory LuceneDirectory { get; private set; } - private readonly object _locker = new object(); - public SnapshotDeletionPolicy Snapshotter { get; private set; } - - public string TempPath { get; private set; } - - - public LocalTempStorageIndexer() - { - throw new NotImplementedException("Fix how local temp storage works and is synced with Examine v2.0 - since a writer is always open we cannot snapshot it, we need to use the same logic in AzureDirectory"); - - IndexDeletionPolicy policy = new KeepOnlyLastCommitDeletionPolicy(); - Snapshotter = new SnapshotDeletionPolicy(policy); - } - - public void Initialize(NameValueCollection config, string configuredPath, FSDirectory baseLuceneDirectory, Analyzer analyzer, LocalStorageType localStorageType) - { - //this is the default - ILocalStorageDirectory localStorageDir = new CodeGenLocalStorageDirectory(); - if (config["tempStorageDirectory"] != null) - { - //try to get the type - var dirType = BuildManager.GetType(config["tempStorageDirectory"], false); - if (dirType != null) - { - try - { - localStorageDir = (ILocalStorageDirectory)Activator.CreateInstance(dirType); - } - catch (Exception ex) - { - Current.Logger.Error( - string.Format("Could not create a temp storage location of type {0}, reverting to use the " + typeof (CodeGenLocalStorageDirectory).FullName, dirType), - ex); - } - } - } - - var tempPath = localStorageDir.GetLocalStorageDirectory(config, configuredPath); - if (tempPath == null) throw new InvalidOperationException("Could not resolve a temp location from the " + localStorageDir.GetType() + " specified"); - TempPath = tempPath.FullName; - - switch (localStorageType) - { - case LocalStorageType.Sync: - var success = InitializeLocalIndexAndDirectory(baseLuceneDirectory, analyzer, configuredPath); - - //create the custom lucene directory which will keep the main and temp FS's in sync - LuceneDirectory = LocalTempStorageDirectoryTracker.Current.GetDirectory( - new DirectoryInfo(TempPath), - baseLuceneDirectory, - //flag to disable the mirrored folder if not successful - (int)success >= 100); - - //If the master index simply doesn't exist, we don't continue to try to open anything since there will - // actually be nothing there. - if (success == InitializeDirectoryFlags.SuccessNoIndexExists) - { - return; - } - - //Try to open the reader, this will fail if the index is corrupt and we'll need to handle that - var result = DelegateExtensions.RetryUntilSuccessOrMaxAttempts(i => - { - try - { - using (IndexReader.Open( - LuceneDirectory, - DeletePolicyTracker.Current.GetPolicy(LuceneDirectory), - true)) - { - } - - return Attempt.Succeed(true); - } - catch (Exception ex) - { - Current.Logger.Warn( - ex, - string.Format("Could not open an index reader, local temp storage index is empty or corrupt... retrying... {0}", configuredPath)); - } - return Attempt.Fail(false); - }, 5, TimeSpan.FromSeconds(1)); - - if (result.Success == false) - { - Current.Logger.Warn( - string.Format("Could not open an index reader, local temp storage index is empty or corrupt... attempting to clear index files in local temp storage, will operate from main storage only {0}", configuredPath)); - - ClearFilesInPath(TempPath); - - //create the custom lucene directory which will keep the main and temp FS's in sync - LuceneDirectory = LocalTempStorageDirectoryTracker.Current.GetDirectory( - new DirectoryInfo(TempPath), - baseLuceneDirectory, - //Disable mirrored index, we're kind of screwed here only use master index - true); - } - - break; - case LocalStorageType.LocalOnly: - if (Directory.Exists(TempPath) == false) - { - Directory.CreateDirectory(TempPath); - } - LuceneDirectory = DirectoryTracker.Current.GetDirectory(new DirectoryInfo(TempPath)); - break; - default: - throw new ArgumentOutOfRangeException("localStorageType"); - } - } - - private void ClearFilesInPath(string path) - { - if (Directory.Exists(path)) - { - foreach (var file in Directory.GetFiles(path)) - { - try - { - File.Delete(file); - } - catch (Exception exInner) - { - Current.Logger.Error("Could not delete local temp storage index file", exInner); - } - } - } - } - - private bool ClearLuceneDirFiles(Lucene.Net.Store.Directory baseLuceneDirectory) - { - try - { - //unlock it! - IndexWriter.Unlock(baseLuceneDirectory); - - var fileLuceneDirectory = baseLuceneDirectory as FSDirectory; - if (fileLuceneDirectory != null) - { - foreach (var file in fileLuceneDirectory.ListAll()) - { - try - { - fileLuceneDirectory.DeleteFile(file); - } - catch (IOException) - { - if (file.InvariantEquals("write.lock")) - { - Current.Logger.Warn("The lock file could not be deleted but should be removed when the writer is disposed"); - } - - } - } - return true; - } - return false; - } - catch (Exception ex) - { - Current.Logger.Error("Could not clear corrupt index from main index folder, the index cannot be used", ex); - return false; - } - } - - private Attempt TryCreateWriter(Lucene.Net.Store.Directory baseLuceneDirectory, Analyzer analyzer) - { - try - { - var w = new IndexWriter( - //read from the underlying/default directory, not the temp codegen dir - baseLuceneDirectory, - analyzer, - Snapshotter, - IndexWriter.MaxFieldLength.UNLIMITED); - - //Done! - return Attempt.Succeed(w); - } - catch (Exception ex) - { - Current.Logger.Warn(ex, "Could not create index writer with snapshot policy for copying... retrying..."); - return Attempt.Fail(ex); - } - } - - /// - /// Attempts to create an index writer, it will retry on failure 5 times and on the last time will try to forcefully unlock the index files - /// - /// - /// - /// - private Attempt TryCreateWriterWithRetry(Lucene.Net.Store.Directory baseLuceneDirectory, Analyzer analyzer) - { - var maxTries = 5; - - var result = DelegateExtensions.RetryUntilSuccessOrMaxAttempts((currentTry) => - { - //last try... - if (currentTry == maxTries) - { - Current.Logger.Info("Could not acquire index lock, attempting to force unlock it..."); - //unlock it! - IndexWriter.Unlock(baseLuceneDirectory); - } - - var writerAttempt = TryCreateWriter(baseLuceneDirectory, analyzer); - if (writerAttempt) return writerAttempt; - Current.Logger.Info("Could not create writer on {0}, retrying ....", baseLuceneDirectory.ToString); - return Attempt.Fail(); - }, 5, TimeSpan.FromSeconds(1)); - - return result; - } - - private bool TryWaitForDirectoryUnlock(Lucene.Net.Store.Directory dir) - { - var maxTries = 5; - - var result = DelegateExtensions.RetryUntilSuccessOrMaxAttempts((currentTry) => - { - //last try... - if (currentTry == maxTries) - { - Current.Logger.Info("Could not acquire directory lock, attempting to force unlock it..."); - //unlock it! - IndexWriter.Unlock(dir); - } - - if (IndexWriter.IsLocked(dir) == false) return Attempt.Succeed(true); - Current.Logger.Info("Could not acquire directory lock for {0} writer, retrying ....", dir.ToString); - return Attempt.Fail(); - }, 5, TimeSpan.FromSeconds(1)); - - return result; - } - - private InitializeDirectoryFlags InitializeLocalIndexAndDirectory(Lucene.Net.Store.Directory baseLuceneDirectory, Analyzer analyzer, string configuredPath) - { - lock (_locker) - { - if (Directory.Exists(TempPath) == false) - { - Directory.CreateDirectory(TempPath); - } - - //copy index if it exists, don't do anything if it's not there - if (IndexReader.IndexExists(baseLuceneDirectory) == false) return InitializeDirectoryFlags.SuccessNoIndexExists; - - var writerAttempt = TryCreateWriterWithRetry(baseLuceneDirectory, analyzer); - - if (writerAttempt.Success == false) - { - Current.Logger.Error("Could not create index writer with snapshot policy for copying, the index cannot be used", writerAttempt.Exception); - return InitializeDirectoryFlags.FailedLocked; - } - - //Try to open the reader from the source, this will fail if the index is corrupt and we'll need to handle that - try - { - //NOTE: To date I've not seen this error occur - using (writerAttempt.Result.GetReader()) - { - } - } - catch (Exception ex) - { - writerAttempt.Result.Dispose(); - - Current.Logger.Error( - string.Format("Could not open an index reader, {0} is empty or corrupt... attempting to clear index files in master folder", configuredPath), - ex); - - if (ClearLuceneDirFiles(baseLuceneDirectory) == false) - { - //hrm, not much we can do in this situation, but this shouldn't happen - Current.Logger.Error("Could not open an index reader, index is corrupt.", ex); - return InitializeDirectoryFlags.FailedCorrupt; - } - - //the main index is now blank, we'll proceed as normal with a new empty index... - writerAttempt = TryCreateWriter(baseLuceneDirectory, analyzer); - if (writerAttempt.Success == false) - { - //ultra fail... - Current.Logger.Error("Could not create index writer with snapshot policy for copying, the index cannot be used", writerAttempt.Exception); - return InitializeDirectoryFlags.FailedLocked; - } - } - - using (writerAttempt.Result) - { - try - { - var basePath = IOHelper.MapPath(configuredPath); - - var commit = Snapshotter.Snapshot(); - var allSnapshotFiles = commit.FileNames - .Concat(new[] - { - commit.SegmentsFileName, - //we need to manually include the segments.gen file - "segments.gen" - }) - .Distinct() - .ToArray(); - - var tempDir = new DirectoryInfo(TempPath); - - //Get all files in the temp storage that don't exist in the snapshot collection, we want to remove these - var toRemove = tempDir.GetFiles() - .Select(x => x.Name) - .Except(allSnapshotFiles); - - using (var tempDirectory = new SimpleFSDirectory(tempDir)) - { - if (TryWaitForDirectoryUnlock(tempDirectory)) - { - foreach (var file in toRemove) - { - try - { - File.Delete(Path.Combine(TempPath, file)); - } - catch (IOException ex) - { - if (file.InvariantEquals("write.lock")) - { - //This might happen if the writer is open - Current.Logger.Warn("The lock file could not be deleted but should be removed when the writer is disposed"); - } - - Current.Logger.Debug("Could not delete non synced index file file, index sync will continue but old index files will remain - this shouldn't affect indexing/searching operations. {0}", () => ex.ToString()); - - } - } - } - else - { - //quit here, this shouldn't happen with all the checks above. - Current.Logger.Warn("Cannot sync index files from main storage, the temp file index is currently locked"); - return InitializeDirectoryFlags.FailedLocked; - } - - - foreach (var fileName in allSnapshotFiles.Where(f => f.IsNullOrWhiteSpace() == false)) - { - var destination = Path.Combine(TempPath, Path.GetFileName(fileName)); - - //don't copy if it's already there, lucene is 'write once' so this file is meant to be there already - if (File.Exists(destination)) continue; - - try - { - File.Copy( - Path.Combine(basePath, "Index", fileName), - destination); - } - catch (IOException ex) - { - Current.Logger.Error("Could not copy index file, could not sync from main storage", ex); - - //quit here - return InitializeDirectoryFlags.FailedFileSync; - } - } - } - } - finally - { - Snapshotter.Release(); - } - } - - Current.Logger.Info("Successfully sync'd main index to local temp storage for index: {0}", () => configuredPath); - return InitializeDirectoryFlags.Success; - } - } - } -} diff --git a/src/Umbraco.Examine/LocalStorage/MultiIndexLock.cs b/src/Umbraco.Examine/LocalStorage/MultiIndexLock.cs deleted file mode 100644 index 51abd23b2c..0000000000 --- a/src/Umbraco.Examine/LocalStorage/MultiIndexLock.cs +++ /dev/null @@ -1,54 +0,0 @@ -using Lucene.Net.Store; - -namespace Umbraco.Examine.LocalStorage -{ - /// - /// Lock that wraps multiple locks - /// - public class MultiIndexLock : Lock - { - private readonly Lock _dirMaster; - private readonly Lock _dirChild; - - public MultiIndexLock(Lock dirMaster, Lock dirChild) - { - _dirMaster = dirMaster; - _dirChild = dirChild; - } - - /// - /// Attempts to obtain exclusive access and immediately return - /// upon success or failure. - /// - /// - /// true iff exclusive access is obtained - /// - public override bool Obtain() - { - return _dirMaster.Obtain() && _dirChild.Obtain(); - } - - public override bool Obtain(long lockWaitTimeout) - { - return _dirMaster.Obtain(lockWaitTimeout) && _dirChild.Obtain(lockWaitTimeout); - } - - /// - /// Releases exclusive access. - /// - public override void Release() - { - _dirMaster.Release(); - _dirChild.Release(); - } - - /// - /// Returns true if the resource is currently locked. Note that one must - /// still call before using the resource. - /// - public override bool IsLocked() - { - return _dirMaster.IsLocked() || _dirChild.IsLocked(); - } - } -} diff --git a/src/Umbraco.Examine/LocalStorage/MultiIndexLockFactory.cs b/src/Umbraco.Examine/LocalStorage/MultiIndexLockFactory.cs deleted file mode 100644 index 085cf1580e..0000000000 --- a/src/Umbraco.Examine/LocalStorage/MultiIndexLockFactory.cs +++ /dev/null @@ -1,33 +0,0 @@ -using System; -using Lucene.Net.Store; - -namespace Umbraco.Examine.LocalStorage -{ - /// - /// Lock factory that wraps multiple factories - /// - public class MultiIndexLockFactory : LockFactory - { - private readonly FSDirectory _master; - private readonly FSDirectory _child; - - public MultiIndexLockFactory(FSDirectory master, FSDirectory child) - { - if (master == null) throw new ArgumentNullException("master"); - if (child == null) throw new ArgumentNullException("child"); - _master = master; - _child = child; - } - - public override Lock MakeLock(string lockName) - { - return new MultiIndexLock(_master.LockFactory.MakeLock(lockName), _child.LockFactory.MakeLock(lockName)); - } - - public override void ClearLock(string lockName) - { - _master.LockFactory.ClearLock(lockName); - _child.LockFactory.ClearLock(lockName); - } - } -} diff --git a/src/Umbraco.Examine/LocalStorage/MultiIndexOutput.cs b/src/Umbraco.Examine/LocalStorage/MultiIndexOutput.cs deleted file mode 100644 index bb6b061528..0000000000 --- a/src/Umbraco.Examine/LocalStorage/MultiIndexOutput.cs +++ /dev/null @@ -1,80 +0,0 @@ -using System; -using System.Linq; -using Lucene.Net.Store; - -namespace Umbraco.Examine.LocalStorage -{ - public class MultiIndexOutput : IndexOutput - { - private readonly IndexOutput[] _outputs; - - public MultiIndexOutput(params IndexOutput[] outputs) - { - if (outputs.Length < 1) - { - throw new InvalidOperationException("There must be at least one output specified"); - } - _outputs = outputs; - } - - public override void WriteByte(byte b) - { - foreach (var output in _outputs) - { - output.WriteByte(b); - } - } - - public override void WriteBytes(byte[] b, int offset, int length) - { - foreach (var output in _outputs) - { - output.WriteBytes(b, offset, length); - } - } - - public override void Flush() - { - foreach (var output in _outputs) - { - output.Flush(); - } - } - - protected override void Dispose(bool isDisposing) - { - foreach (var output in _outputs) - { - output.Dispose(); - } - } - - public override long FilePointer - { - get - { - //return the first - return _outputs[0].FilePointer; - } - - } - - public override void Seek(long pos) - { - foreach (var output in _outputs) - { - output.Seek(pos); - } - } - - public override long Length - { - get - { - //return the first - return _outputs[0].FilePointer; - } - - } - } -} diff --git a/src/Umbraco.Examine/NoPrefixSimpleFsLockFactory.cs b/src/Umbraco.Examine/NoPrefixSimpleFsLockFactory.cs new file mode 100644 index 0000000000..7c83223d0b --- /dev/null +++ b/src/Umbraco.Examine/NoPrefixSimpleFsLockFactory.cs @@ -0,0 +1,26 @@ +using System.IO; +using Lucene.Net.Store; + +namespace Umbraco.Examine +{ + /// + /// A custom that ensures a prefixless lock prefix + /// + /// + /// This is a work around for the Lucene APIs. By default Lucene will use a null prefix however when we set a custom + /// lock factory the null prefix is overwritten. + /// + public class NoPrefixSimpleFsLockFactory : SimpleFSLockFactory + { + public NoPrefixSimpleFsLockFactory(DirectoryInfo lockDir) : base(lockDir) + { + } + + public override string LockPrefix + { + get => base.LockPrefix; + set => base.LockPrefix = null; //always set to null + } + + } +} \ No newline at end of file diff --git a/src/Umbraco.Examine/StaticField.cs b/src/Umbraco.Examine/StaticField.cs deleted file mode 100644 index 237cbaaf26..0000000000 --- a/src/Umbraco.Examine/StaticField.cs +++ /dev/null @@ -1,21 +0,0 @@ -using Examine; -using Examine.LuceneEngine; - -namespace Umbraco.Examine -{ - internal class StaticField : IIndexField - { - public StaticField(string name, FieldIndexTypes indexType, bool enableSorting, string type) - { - Type = type; - EnableSorting = enableSorting; - IndexType = indexType; - Name = name; - } - - public string Name { get; set; } - public FieldIndexTypes IndexType { get; private set; } - public bool EnableSorting { get; set; } - public string Type { get; set; } - } -} diff --git a/src/Umbraco.Examine/StaticFieldCollection.cs b/src/Umbraco.Examine/StaticFieldCollection.cs deleted file mode 100644 index 2579299fc1..0000000000 --- a/src/Umbraco.Examine/StaticFieldCollection.cs +++ /dev/null @@ -1,28 +0,0 @@ -using System.Collections.ObjectModel; - -namespace Umbraco.Examine -{ - internal class StaticFieldCollection : KeyedCollection - { - protected override string GetKeyForItem(StaticField item) - { - return item.Name; - } - - /// - /// Implements TryGetValue using the underlying dictionary - /// - /// - /// - /// - public bool TryGetValue(string key, out StaticField field) - { - if (Dictionary == null) - { - field = null; - return false; - } - return Dictionary.TryGetValue(key, out field); - } - } -} diff --git a/src/Umbraco.Examine/Umbraco.Examine.csproj b/src/Umbraco.Examine/Umbraco.Examine.csproj index de401af610..2115afadbf 100644 --- a/src/Umbraco.Examine/Umbraco.Examine.csproj +++ b/src/Umbraco.Examine/Umbraco.Examine.csproj @@ -46,14 +46,30 @@ - - - + + 1.0.0-beta024 + - + + + + + + + + + + + + + + + + + Properties\SolutionInfo.cs diff --git a/src/Umbraco.Examine/UmbracoContentIndexer.cs b/src/Umbraco.Examine/UmbracoContentIndexer.cs index 3fddf4a762..2f177e2466 100644 --- a/src/Umbraco.Examine/UmbracoContentIndexer.cs +++ b/src/Umbraco.Examine/UmbracoContentIndexer.cs @@ -1,14 +1,13 @@ using System; using System.Collections.Generic; using System.Collections.Specialized; +using System.ComponentModel; using System.Linq; using Examine; using Umbraco.Core; using Umbraco.Core.Models; using Umbraco.Core.Services; using Umbraco.Core.Strings; -using Examine.LuceneEngine.Config; -using Examine.LuceneEngine.Faceting; using Examine.LuceneEngine.Indexing; using Examine.LuceneEngine.Providers; using Lucene.Net.Analysis; @@ -19,6 +18,7 @@ using Umbraco.Core.Persistence; using Umbraco.Core.Persistence.DatabaseModelDefinitions; using Umbraco.Core.Persistence.Querying; using Umbraco.Core.Scoping; +using Umbraco.Examine.Config; using IContentService = Umbraco.Core.Services.IContentService; using IMediaService = Umbraco.Core.Services.IMediaService; @@ -28,20 +28,21 @@ namespace Umbraco.Examine /// /// An indexer for Umbraco content and media /// - public class UmbracoContentIndexer : BaseUmbracoIndexer + public class UmbracoContentIndexer : UmbracoExamineIndexer { protected IContentService ContentService { get; } protected IMediaService MediaService { get; } protected IUserService UserService { get; } private readonly IEnumerable _urlSegmentProviders; - private readonly IScopeProvider _scopeProvider; private int? _parentId; #region Constructors - // default - bad, should inject instead - // usage: none + /// + /// Constructor for configuration providers + /// + [EditorBrowsable(EditorBrowsableState.Never)] public UmbracoContentIndexer() { ContentService = Current.Services.ContentService; @@ -49,13 +50,28 @@ namespace Umbraco.Examine UserService = Current.Services.UserService; _urlSegmentProviders = Current.UrlSegmentProviders; - _scopeProvider = Current.ScopeProvider; - InitializeQueries(); + InitializeQueries(Current.SqlContext); } - // usage: IndexInitializer (tests) + /// + /// Create an index at runtime + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// public UmbracoContentIndexer( + string name, IEnumerable fieldDefinitions, Directory luceneDirectory, Analyzer defaultAnalyzer, @@ -63,13 +79,12 @@ namespace Umbraco.Examine IContentService contentService, IMediaService mediaService, IUserService userService, + ISqlContext sqlContext, IEnumerable urlSegmentProviders, IValueSetValidator validator, UmbracoContentIndexerOptions options, - IScopeProvider scopeProvider, - FacetConfiguration facetConfiguration = null, - IDictionary> indexValueTypes = null) - : base(fieldDefinitions, luceneDirectory, defaultAnalyzer, profilingLogger, validator, facetConfiguration, indexValueTypes) + IReadOnlyDictionary> indexValueTypes = null) + : base(name, fieldDefinitions, luceneDirectory, defaultAnalyzer, profilingLogger, validator, indexValueTypes) { if (validator == null) throw new ArgumentNullException(nameof(validator)); if (options == null) throw new ArgumentNullException(nameof(options)); @@ -77,24 +92,20 @@ namespace Umbraco.Examine SupportProtectedContent = options.SupportProtectedContent; SupportUnpublishedContent = options.SupportUnpublishedContent; ParentId = options.ParentId; - //backward compat hack: - IndexerData = new IndexCriteria(Enumerable.Empty(), Enumerable.Empty(), Enumerable.Empty(), Enumerable.Empty(), - //hack to set the parent Id for backwards compat, when using this ctor the IndexerData will (should) always be null - options.ParentId); ContentService = contentService ?? throw new ArgumentNullException(nameof(contentService)); MediaService = mediaService ?? throw new ArgumentNullException(nameof(mediaService)); UserService = userService ?? throw new ArgumentNullException(nameof(userService)); _urlSegmentProviders = urlSegmentProviders ?? throw new ArgumentNullException(nameof(urlSegmentProviders)); - _scopeProvider = scopeProvider ?? throw new ArgumentNullException(nameof(scopeProvider)); - InitializeQueries(); + InitializeQueries(sqlContext); } - private void InitializeQueries() + private void InitializeQueries(ISqlContext sqlContext) { + if (sqlContext == null) throw new ArgumentNullException(nameof(sqlContext)); if (_publishedQuery == null) - _publishedQuery = Current.SqlContext.Query().Where(x => x.Published); + _publishedQuery = sqlContext.Query().Where(x => x.Published); } #endregion @@ -121,11 +132,9 @@ namespace Umbraco.Examine public override void Initialize(string name, NameValueCollection config) { - //check if there's a flag specifying to support unpublished content, //if not, set to false; - bool supportUnpublished; - if (config["supportUnpublished"] != null && bool.TryParse(config["supportUnpublished"], out supportUnpublished)) + if (config["supportUnpublished"] != null && bool.TryParse(config["supportUnpublished"], out var supportUnpublished)) SupportUnpublishedContent = supportUnpublished; else SupportUnpublishedContent = false; @@ -133,13 +142,25 @@ namespace Umbraco.Examine //check if there's a flag specifying to support protected content, //if not, set to false; - bool supportProtected; - if (config["supportProtected"] != null && bool.TryParse(config["supportProtected"], out supportProtected)) + if (config["supportProtected"] != null && bool.TryParse(config["supportProtected"], out var supportProtected)) SupportProtectedContent = supportProtected; else SupportProtectedContent = false; base.Initialize(name, config); + + //now we need to build up the indexer options so we can create our validator + int? parentId = null; + if (IndexSetName.IsNullOrWhiteSpace() == false) + { + var indexSet = IndexSets.Instance.Sets[IndexSetName]; + parentId = indexSet.IndexParentId; + } + ValueSetValidator = new UmbracoContentValueSetValidator( + new UmbracoContentIndexerOptions(SupportUnpublishedContent, SupportProtectedContent, parentId), + //Using a singleton here, we can't inject this when using config based providers and we don't use this + //anywhere else in this class + Current.Services.PublicAccessService); } #endregion @@ -163,12 +184,8 @@ namespace Umbraco.Examine /// public int? ParentId { - get - { - //fallback to the legacy data - return _parentId ?? (IndexerData == null ? (int?)null : IndexerData.ParentNodeId); - } - protected set { _parentId = value; } + get => _parentId ?? ConfigIndexCriteria?.ParentNodeId; + protected set => _parentId = value; } protected override IEnumerable SupportedTypes => new[] {IndexTypes.Content, IndexTypes.Media}; @@ -193,18 +210,14 @@ namespace Umbraco.Examine var searcher = GetSearcher(); var c = searcher.CreateCriteria(); var filtered = c.RawQuery(rawQuery); - var results = searcher.Find(filtered); + var results = searcher.Search(filtered); ProfilingLogger.Logger.Debug(GetType(), $"DeleteFromIndex with query: {rawQuery} (found {results.TotalItemCount} results)"); - //need to create a delete queue item for each one found + //need to queue a delete item for each one found foreach (var r in results) { - ProcessIndexOperation(new IndexOperation() - { - Operation = IndexOperationType.Delete, - Item = new IndexItem(new ValueSet(r.LongId, string.Empty)) - }); + QueueIndexOperation(new IndexOperation(IndexItem.ForId(r.Id), IndexOperationType.Delete)); } base.DeleteFromIndex(nodeId); @@ -250,16 +263,16 @@ namespace Umbraco.Examine //if specific types are declared we need to post filter them //TODO: Update the service layer to join the cmsContentType table so we can query by content type too - if (IndexerData != null && IndexerData.IncludeNodeTypes.Any()) + if (ConfigIndexCriteria != null && ConfigIndexCriteria.IncludeItemTypes.Any()) { - content = descendants.Where(x => IndexerData.IncludeNodeTypes.Contains(x.ContentType.Alias)).ToArray(); + content = descendants.Where(x => ConfigIndexCriteria.IncludeItemTypes.Contains(x.ContentType.Alias)).ToArray(); } else { content = descendants.ToArray(); } - IndexItems(GetValueSets(content)); + IndexItems(GetValueSets(_urlSegmentProviders, UserService, content)); pageIndex++; } while (content.Length == pageSize); @@ -286,16 +299,16 @@ namespace Umbraco.Examine //if specific types are declared we need to post filter them //TODO: Update the service layer to join the cmsContentType table so we can query by content type too - if (IndexerData != null && IndexerData.IncludeNodeTypes.Any()) + if (ConfigIndexCriteria != null && ConfigIndexCriteria.IncludeItemTypes.Any()) { - media = descendants.Where(x => IndexerData.IncludeNodeTypes.Contains(x.ContentType.Alias)).ToArray(); + media = descendants.Where(x => ConfigIndexCriteria.IncludeItemTypes.Contains(x.ContentType.Alias)).ToArray(); } else { media = descendants.ToArray(); } - IndexItems(GetValueSets(media)); + IndexItems(GetValueSets(_urlSegmentProviders, UserService, media)); pageIndex++; } while (media.Length == pageSize); @@ -304,11 +317,11 @@ namespace Umbraco.Examine } } - private IEnumerable GetValueSets(IEnumerable content) + public static IEnumerable GetValueSets(IEnumerable urlSegmentProviders, IUserService userService, params IContent[] content) { foreach (var c in content) { - var urlValue = c.GetUrlSegment(_urlSegmentProviders); + var urlValue = c.GetUrlSegment(urlSegmentProviders); var values = new Dictionary { {"icon", new object[] {c.ContentType.Icon}}, @@ -325,28 +338,40 @@ namespace Umbraco.Examine {"urlName", new object[] {urlValue}}, {"path", new object[] {c.Path}}, {"nodeType", new object[] {c.ContentType.Id}}, - {"creatorName", new object[] {c.GetCreatorProfile(UserService)?.Name ?? "??"}}, - {"writerName", new object[] {c.GetWriterProfile(UserService)?.Name ?? "??"}}, + {"creatorName", new object[] {c.GetCreatorProfile(userService)?.Name ?? "??"}}, + {"writerName", new object[] {c.GetWriterProfile(userService)?.Name ?? "??"}}, {"writerID", new object[] {c.WriterId}}, {"template", new object[] {c.Template?.Id ?? 0}} }; - foreach (var property in c.Properties.Where(p => p?.GetValue() != null && p.GetValue().ToString().IsNullOrWhiteSpace() == false)) + foreach (var property in c.Properties) { - values.Add(property.Alias, new[] {property.GetValue() }); + //only add the value if its not null or empty (we'll check for string explicitly here too) + var val = property.GetValue(); + switch (val) + { + case null: + continue; + case string strVal when strVal.IsNullOrWhiteSpace() == false: + values.Add(property.Alias, new[] { val }); + break; + default: + values.Add(property.Alias, new[] { val }); + break; + } } - var vs = new ValueSet(c.Id, IndexTypes.Content, c.ContentType.Alias, values); + var vs = new ValueSet(c.Id.ToInvariantString(), IndexTypes.Content, c.ContentType.Alias, values); yield return vs; } } - private IEnumerable GetValueSets(IEnumerable media) + public static IEnumerable GetValueSets(IEnumerable urlSegmentProviders, IUserService userService, params IMedia[] media) { foreach (var m in media) { - var urlValue = m.GetUrlSegment(_urlSegmentProviders); + var urlValue = m.GetUrlSegment(urlSegmentProviders); var values = new Dictionary { {"icon", new object[] {m.ContentType.Icon}}, @@ -362,40 +387,32 @@ namespace Umbraco.Examine {"urlName", new object[] {urlValue}}, {"path", new object[] {m.Path}}, {"nodeType", new object[] {m.ContentType.Id}}, - {"creatorName", new object[] {m.GetCreatorProfile(UserService).Name}} + {"creatorName", new object[] {m.GetCreatorProfile(userService).Name}} }; - foreach (var property in m.Properties.Where(p => p?.GetValue() != null && p.GetValue().ToString().IsNullOrWhiteSpace() == false)) + foreach (var property in m.Properties) { - values.Add(property.Alias, new[] { property.GetValue() }); + //only add the value if its not null or empty (we'll check for string explicitly here too) + var val = property.GetValue(); + switch (val) + { + case null: + continue; + case string strVal when strVal.IsNullOrWhiteSpace() == false: + values.Add(property.Alias, new[] { val }); + break; + default: + values.Add(property.Alias, new[] { val }); + break; + } } - var vs = new ValueSet(m.Id, IndexTypes.Media, m.ContentType.Alias, values); + var vs = new ValueSet(m.Id.ToInvariantString(), IndexTypes.Media, m.ContentType.Alias, values); yield return vs; } } - /// - /// Creates an IIndexCriteria object based on the indexSet passed in and our DataService - /// - /// - /// - /// - /// If we cannot initialize we will pass back empty indexer data since we cannot read from the database - /// - [Obsolete("IIndexCriteria is obsolete, this method is used only for configuration based indexes it is recommended to configure indexes on startup with code instead of config")] - protected override IIndexCriteria GetIndexerData(IndexSet indexSet) - { - if (CanInitialize()) - { - //NOTE: We are using a singleton here because: This is only ever used for configuration based scenarios, this is never - // used when the index is configured via code (the default), in which case IIndexCriteria is never used. When this is used - // the DI ctor is not used. - return indexSet.ToIndexCriteria(Current.Services.ContentTypeService); - } - return base.GetIndexerData(indexSet); - } #endregion } diff --git a/src/Umbraco.Examine/UmbracoContentIndexerOptions.cs b/src/Umbraco.Examine/UmbracoContentIndexerOptions.cs index 507556cdef..47b8a76c0f 100644 --- a/src/Umbraco.Examine/UmbracoContentIndexerOptions.cs +++ b/src/Umbraco.Examine/UmbracoContentIndexerOptions.cs @@ -1,7 +1,8 @@ using System; namespace Umbraco.Examine -{ +{ + /// /// Options used to configure the umbraco content indexer /// diff --git a/src/Umbraco.Examine/UmbracoContentValueSetValidator.cs b/src/Umbraco.Examine/UmbracoContentValueSetValidator.cs index 2d414cc38f..01056be6b9 100644 --- a/src/Umbraco.Examine/UmbracoContentValueSetValidator.cs +++ b/src/Umbraco.Examine/UmbracoContentValueSetValidator.cs @@ -24,10 +24,10 @@ namespace Umbraco.Examine public bool Validate(ValueSet valueSet) { //check for published content - if (valueSet.IndexCategory == IndexTypes.Content - && valueSet.Values.ContainsKey(BaseUmbracoIndexer.PublishedFieldName)) + if (valueSet.Category == IndexTypes.Content + && valueSet.Values.ContainsKey(UmbracoExamineIndexer.PublishedFieldName)) { - var published = valueSet.Values[BaseUmbracoIndexer.PublishedFieldName] != null && valueSet.Values[BaseUmbracoIndexer.PublishedFieldName][0].Equals(1); + var published = valueSet.Values[UmbracoExamineIndexer.PublishedFieldName] != null && valueSet.Values[UmbracoExamineIndexer.PublishedFieldName][0].Equals(1); //we don't support unpublished and the item is not published return false if (_options.SupportUnpublishedContent == false && published == false) { @@ -37,11 +37,11 @@ namespace Umbraco.Examine //must have a 'path' if (valueSet.Values.ContainsKey(PathKey) == false) return false; - var path = valueSet.Values[PathKey] == null ? string.Empty : valueSet.Values[PathKey][0].ToString(); + var path = valueSet.Values[PathKey]?[0].ToString() ?? string.Empty; // Test for access if we're only indexing published content // return nothing if we're not supporting protected content and it is protected, and we're not supporting unpublished content - if (valueSet.IndexCategory == IndexTypes.Content + if (valueSet.Category == IndexTypes.Content && _options.SupportUnpublishedContent == false && _options.SupportProtectedContent == false && _publicAccessService.IsProtected(path)) @@ -61,7 +61,7 @@ namespace Umbraco.Examine if (_options.SupportUnpublishedContent == false) { if (path.IsNullOrWhiteSpace()) return false; - var recycleBinId = valueSet.IndexCategory == IndexTypes.Content ? Constants.System.RecycleBinContent : Constants.System.RecycleBinMedia; + var recycleBinId = valueSet.Category == IndexTypes.Content ? Constants.System.RecycleBinContent : Constants.System.RecycleBinMedia; if (path.Contains(string.Concat(",", recycleBinId, ","))) return false; } diff --git a/src/Umbraco.Examine/UmbracoExamineExtensions.cs b/src/Umbraco.Examine/UmbracoExamineExtensions.cs new file mode 100644 index 0000000000..ab8819b276 --- /dev/null +++ b/src/Umbraco.Examine/UmbracoExamineExtensions.cs @@ -0,0 +1,92 @@ +using System; +using System.ComponentModel; +using System.Web; +using Examine.LuceneEngine.SearchCriteria; +using Examine.SearchCriteria; +using Umbraco.Core; +using Umbraco.Examine.Config; + +namespace Umbraco.Examine +{ + public static class UmbracoExamineExtensions + { + public static IBooleanOperation Id(this IQuery query, int id) + { + var fieldQuery = query.Id(id.ToInvariantString()); + return fieldQuery; + } + + /// + /// Query method to search on parent id + /// + /// + /// + /// + public static IBooleanOperation ParentId(this IQuery query, int id) + { + var fieldQuery = query.Field("parentID", id); + return fieldQuery; + } + + /// + /// Query method to search on node name + /// + /// + /// + /// + public static IBooleanOperation NodeName(this IQuery query, string nodeName) + { + var fieldQuery = query.Field("nodeName", (IExamineValue)new ExamineValue(Examineness.Explicit, nodeName)); + return fieldQuery; + } + + /// + /// Query method to search on node name + /// + /// + /// + /// + public static IBooleanOperation NodeName(this IQuery query, IExamineValue nodeName) + { + var fieldQuery = query.Field("nodeName", nodeName); + return fieldQuery; + } + + /// + /// Query method to search on node type alias + /// + /// + /// + /// + public static IBooleanOperation NodeTypeAlias(this IQuery query, string nodeTypeAlias) + { + var fieldQuery = query.Field("__NodeTypeAlias", (IExamineValue)new ExamineValue(Examineness.Explicit, nodeTypeAlias)); + return fieldQuery; + } + + /// + /// Query method to search on node type alias + /// + /// + /// + /// + public static IBooleanOperation NodeTypeAlias(this IQuery query, IExamineValue nodeTypeAlias) + { + var fieldQuery = query.Field("__NodeTypeAlias", nodeTypeAlias); + return fieldQuery; + } + + /// + /// Used to replace any available tokens in the index path before the lucene directory is assigned to the path + /// + /// + internal static void ReplaceTokensInIndexPath(this IndexSet indexSet) + { + if (indexSet == null) return; + indexSet.IndexPath = indexSet.IndexPath + .Replace("{machinename}", NetworkHelper.FileSafeMachineName) + .Replace("{appdomainappid}", (HttpRuntime.AppDomainAppId ?? string.Empty).ReplaceNonAlphanumericChars("")) + .EnsureEndsWith('/'); + } + } +} \ No newline at end of file diff --git a/src/Umbraco.Examine/UmbracoExamineIndexer.cs b/src/Umbraco.Examine/UmbracoExamineIndexer.cs new file mode 100644 index 0000000000..bec6415a2e --- /dev/null +++ b/src/Umbraco.Examine/UmbracoExamineIndexer.cs @@ -0,0 +1,425 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.IO; +using System.Linq; +using Examine.LuceneEngine.Providers; +using Lucene.Net.Analysis; +using Lucene.Net.Documents; +using Lucene.Net.Index; +using Umbraco.Core; +using Examine; +using Examine.LuceneEngine; +using Examine.LuceneEngine.Indexing; +using Umbraco.Core.Composing; +using Umbraco.Core.Logging; +using Umbraco.Core.Xml; +using Umbraco.Examine.Config; +using Directory = Lucene.Net.Store.Directory; + +namespace Umbraco.Examine +{ + /// + /// An abstract provider containing the basic functionality to be able to query against + /// Umbraco data. + /// + public abstract class UmbracoExamineIndexer : LuceneIndexer + { + // note + // wrapping all operations that end up calling base.SafelyProcessQueueItems in a safe call + // context because they will fork a thread/task/whatever which should *not* capture our + // call context (and the database it can contain)! ideally we should be able to override + // SafelyProcessQueueItems but that's not possible in the current version of Examine. + + /// + /// Used to store the path of a content object + /// + public const string IndexPathFieldName = "__Path"; + public const string NodeKeyFieldName = "__Key"; + public const string IconFieldName = "__Icon"; + public const string PublishedFieldName = "__Published"; + /// + /// The prefix added to a field when it is duplicated in order to store the original raw value. + /// + public const string RawFieldPrefix = "__Raw_"; + + /// + /// Constructor for config provider based indexes + /// + [EditorBrowsable(EditorBrowsableState.Never)] + protected UmbracoExamineIndexer() + : base() + { + ProfilingLogger = Current.ProfilingLogger; + _configBased = true; + + //This is using the config so we'll validate based on that + ValueSetValidator = new ValueSetValidatorDelegate(set => + { + + //check if this document is of a correct type of node type alias + if (ConfigIndexCriteria.IncludeItemTypes.Any()) + if (!ConfigIndexCriteria.IncludeItemTypes.Contains(set.ItemType)) + return false; + + //if this node type is part of our exclusion list, do not validate + if (ConfigIndexCriteria.ExcludeItemTypes.Any()) + if (ConfigIndexCriteria.ExcludeItemTypes.Contains(set.ItemType)) + return false; + + return true; + }); + } + + protected UmbracoExamineIndexer( + string name, + IEnumerable fieldDefinitions, + Directory luceneDirectory, + Analyzer defaultAnalyzer, + ProfilingLogger profilingLogger, + IValueSetValidator validator = null, + IReadOnlyDictionary> indexValueTypes = null) + : base(name, fieldDefinitions, luceneDirectory, defaultAnalyzer, validator, indexValueTypes) + { + ProfilingLogger = profilingLogger ?? throw new ArgumentNullException(nameof(profilingLogger)); + } + + private readonly bool _configBased = false; + + /// + /// A type that defines the type of index for each Umbraco field (non user defined fields) + /// Alot of standard umbraco fields shouldn't be tokenized or even indexed, just stored into lucene + /// for retreival after searching. + /// + internal static readonly FieldDefinition[] UmbracoIndexFields = + { + new FieldDefinition("parentID", FieldDefinitionTypes.Integer), + new FieldDefinition("level", FieldDefinitionTypes.Integer), + new FieldDefinition("writerID", FieldDefinitionTypes.Integer), + new FieldDefinition("creatorID", FieldDefinitionTypes.Integer), + new FieldDefinition("sortOrder", FieldDefinitionTypes.Integer), + new FieldDefinition("template", FieldDefinitionTypes.Integer), + + new FieldDefinition("createDate", FieldDefinitionTypes.DateTime), + new FieldDefinition("updateDate", FieldDefinitionTypes.DateTime), + + new FieldDefinition("key", FieldDefinitionTypes.InvariantCultureIgnoreCase), + new FieldDefinition("version", FieldDefinitionTypes.Raw), + new FieldDefinition("nodeType", FieldDefinitionTypes.InvariantCultureIgnoreCase), + new FieldDefinition("template", FieldDefinitionTypes.Raw), + new FieldDefinition("urlName", FieldDefinitionTypes.InvariantCultureIgnoreCase), + new FieldDefinition("path", FieldDefinitionTypes.Raw), + + new FieldDefinition(IndexPathFieldName, FieldDefinitionTypes.Raw), + new FieldDefinition(IconFieldName, FieldDefinitionTypes.Raw) + }; + + protected ProfilingLogger ProfilingLogger { get; } + + /// + /// Overridden to ensure that the umbraco system field definitions are in place + /// + /// + /// + /// + protected override FieldValueTypeCollection CreateFieldValueTypes(Directory x, IReadOnlyDictionary> indexValueTypesFactory = null) + { + foreach (var field in UmbracoIndexFields) + { + FieldDefinitionCollection.TryAdd(field.Name, field); + } + + return base.CreateFieldValueTypes(x, indexValueTypesFactory); + } + + /// + /// When set to true Umbraco will keep the index in sync with Umbraco data automatically + /// + public bool EnableDefaultEventHandler { get; set; } = true; + + /// + /// the supported indexable types + /// + protected abstract IEnumerable SupportedTypes { get; } + + protected ConfigIndexCriteria ConfigIndexCriteria { get; private set; } + + /// + /// The index set name which references an Examine + /// + public string IndexSetName { get; private set; } + + #region Initialize + + + /// + /// Setup the properties for the indexer from the provider settings + /// + /// + /// + /// + /// This is ONLY used for configuration based indexes + /// + public override void Initialize(string name, System.Collections.Specialized.NameValueCollection config) + { + ProfilingLogger.Logger.Debug(GetType(), "{0} indexer initializing", () => name); + + if (config["enableDefaultEventHandler"] != null && bool.TryParse(config["enableDefaultEventHandler"], out var enabled)) + { + EnableDefaultEventHandler = enabled; + } + + //Need to check if the index set or IndexerData is specified... + if (config["indexSet"] == null && (FieldDefinitionCollection.Count == 0)) + { + //if we don't have either, then we'll try to set the index set by naming conventions + var found = false; + if (name.EndsWith("Indexer")) + { + var setNameByConvension = name.Remove(name.LastIndexOf("Indexer")) + "IndexSet"; + //check if we can assign the index set by naming convention + var set = IndexSets.Instance.Sets.Cast().SingleOrDefault(x => x.SetName == setNameByConvension); + + if (set != null) + { + //we've found an index set by naming conventions :) + IndexSetName = set.SetName; + + var indexSet = IndexSets.Instance.Sets[IndexSetName]; + + //if tokens are declared in the path, then use them (i.e. {machinename} ) + indexSet.ReplaceTokensInIndexPath(); + + //get the index criteria and ensure folder + ConfigIndexCriteria = CreateFieldDefinitionsFromConfig(indexSet); + foreach (var fieldDefinition in ConfigIndexCriteria.StandardFields.Union(ConfigIndexCriteria.UserFields)) + { + FieldDefinitionCollection.TryAdd(fieldDefinition.Name, fieldDefinition); + } + + //now set the index folder + LuceneIndexFolder = new DirectoryInfo(Path.Combine(IndexSets.Instance.Sets[IndexSetName].IndexDirectory.FullName, "Index")); + + found = true; + } + } + + if (!found) + throw new ArgumentNullException("indexSet on LuceneExamineIndexer provider has not been set in configuration and/or the IndexerData property has not been explicitly set"); + + } + else if (config["indexSet"] != null) + { + //if an index set is specified, ensure it exists and initialize the indexer based on the set + + if (IndexSets.Instance.Sets[config["indexSet"]] == null) + { + throw new ArgumentException("The indexSet specified for the LuceneExamineIndexer provider does not exist"); + } + else + { + IndexSetName = config["indexSet"]; + + var indexSet = IndexSets.Instance.Sets[IndexSetName]; + + //if tokens are declared in the path, then use them (i.e. {machinename} ) + indexSet.ReplaceTokensInIndexPath(); + + //get the index criteria and ensure folder + ConfigIndexCriteria = CreateFieldDefinitionsFromConfig(indexSet); + foreach (var fieldDefinition in ConfigIndexCriteria.StandardFields.Union(ConfigIndexCriteria.UserFields)) + { + FieldDefinitionCollection.TryAdd(fieldDefinition.Name, fieldDefinition); + } + + //now set the index folder + LuceneIndexFolder = new DirectoryInfo(Path.Combine(IndexSets.Instance.Sets[IndexSetName].IndexDirectory.FullName, "Index")); + } + } + + base.Initialize(name, config); + } + + #endregion + + + /// + /// override to check if we can actually initialize. + /// + /// + /// This check is required since the base examine lib will try to rebuild on startup + /// + public override void RebuildIndex() + { + if (CanInitialize()) + { + ProfilingLogger.Logger.Debug(GetType(), "Rebuilding index"); + using (new SafeCallContext()) + { + base.RebuildIndex(); + } + } + } + + /// + /// override to check if we can actually initialize. + /// + /// + /// This check is required since the base examine lib will try to rebuild on startup + /// + public override void IndexAll(string type) + { + if (CanInitialize()) + { + using (new SafeCallContext()) + { + base.IndexAll(type); + } + } + } + + public override void IndexItems(IEnumerable nodes) + { + if (CanInitialize()) + { + using (new SafeCallContext()) + { + base.IndexItems(nodes); + } + } + } + + /// + /// override to check if we can actually initialize. + /// + /// + /// This check is required since the base examine lib will try to rebuild on startup + /// + public override void DeleteFromIndex(string nodeId) + { + if (CanInitialize()) + { + using (new SafeCallContext()) + { + base.DeleteFromIndex(nodeId); + } + } + } + + /// + /// Returns true if the Umbraco application is in a state that we can initialize the examine indexes + /// + /// + protected bool CanInitialize() + { + // only affects indexers that are config file based, if an index was created via code then + // this has no effect, it is assumed the index would not be created if it could not be initialized + return _configBased == false || Current.RuntimeState.Level == RuntimeLevel.Run; + } + + /// + /// Reindexes all supported types + /// + protected override void PerformIndexRebuild() + { + foreach (var t in SupportedTypes) + { + IndexAll(t); + } + } + + /// + /// overridden for logging + /// + /// + protected override void OnIndexingError(IndexingErrorEventArgs e) + { + ProfilingLogger.Logger.Error(GetType(), e.Message, e.InnerException); + base.OnIndexingError(e); + } + + /// + /// This ensures that the special __Raw_ fields are indexed + /// + /// + protected override void OnDocumentWriting(DocumentWritingEventArgs docArgs) + { + var d = docArgs.Document; + + foreach (var f in docArgs.ValueSet.Values.Where(x => x.Key.StartsWith(RawFieldPrefix))) + { + if (f.Value.Count > 0) + { + d.Add(new Field( + f.Key, + f.Value[0].ToString(), + Field.Store.YES, + Field.Index.NO, //don't index this field, we never want to search by it + Field.TermVector.NO)); + } + } + + ProfilingLogger.Logger.Debug(GetType(), "Write lucene doc id:{0}, category:{1}, type:{2}", docArgs.ValueSet.Id, docArgs.ValueSet.Category, docArgs.ValueSet.ItemType); + + base.OnDocumentWriting(docArgs); + } + + /// + /// Overridden for logging. + /// + protected override void AddDocument(Document doc, IndexItem item, IndexWriter writer) + { + ProfilingLogger.Logger.Debug(GetType(), "AddDocument {0} with type {1}", () => item.ValueSet.Id, () => item.ValueSet.ItemType); + base.AddDocument(doc, item, writer); + } + + protected override void OnTransformingIndexValues(IndexingItemEventArgs e) + { + base.OnTransformingIndexValues(e); + + //ensure special __Path field + if (e.IndexItem.ValueSet.Values.TryGetValue("path", out var path) && e.IndexItem.ValueSet.Values.ContainsKey(IndexPathFieldName) == false) + { + e.IndexItem.ValueSet.Values[IndexPathFieldName] = new List { path }; + } + + //strip html of all users fields if we detect it has HTML in it. + //if that is the case, we'll create a duplicate 'raw' copy of it so that we can return + //the value of the field 'as-is'. + foreach (var value in e.IndexItem.ValueSet.Values) + { + if (value.Value.Count > 0) + { + var str = value.Value.First() as string; + if (str != null) + { + if (XmlHelper.CouldItBeXml(str)) + { + //First save the raw value to a raw field, we will change the policy of this field by detecting the prefix later + e.IndexItem.ValueSet.Values[string.Concat(RawFieldPrefix, value.Key)] = new List { str }; + //now replace the original value with the stripped html + //TODO: This should be done with an analzer?! + e.IndexItem.ValueSet.Values[value.Key] = new List { str.StripHtml() }; + } + } + } + } + + //icon + if (e.IndexItem.ValueSet.Values.TryGetValue("icon", out var icon) && e.IndexItem.ValueSet.Values.ContainsKey(IconFieldName) == false) + { + e.IndexItem.ValueSet.Values[IconFieldName] = new List { icon }; + } + } + + private ConfigIndexCriteria CreateFieldDefinitionsFromConfig(IndexSet indexSet) + { + return new ConfigIndexCriteria( + indexSet.IndexAttributeFields.Cast().Select(x => new FieldDefinition(x.Name, x.Type)).ToArray(), + indexSet.IndexUserFields.Cast().Select(x => new FieldDefinition(x.Name, x.Type)).ToArray(), + indexSet.IncludeNodeTypes.ToList().Select(x => x.Name).ToArray(), + indexSet.ExcludeNodeTypes.ToList().Select(x => x.Name).ToArray(), + indexSet.IndexParentId); + } + } +} diff --git a/src/Umbraco.Examine/UmbracoExamineSearcher.cs b/src/Umbraco.Examine/UmbracoExamineSearcher.cs index c62c763cf8..4b5ac50af7 100644 --- a/src/Umbraco.Examine/UmbracoExamineSearcher.cs +++ b/src/Umbraco.Examine/UmbracoExamineSearcher.cs @@ -2,23 +2,12 @@ using System.ComponentModel; using System.IO; using System.Linq; -using System.Security; -using System.Web; -using System.Web.Compilation; -using Examine; -using Examine.LuceneEngine.Config; -using Examine.Providers; -using Examine.SearchCriteria; -using Lucene.Net.Index; -using Lucene.Net.Store; using Umbraco.Core; -using Examine.LuceneEngine; using Examine.LuceneEngine.Providers; -using Examine.LuceneEngine.SearchCriteria; using Lucene.Net.Analysis; +using Lucene.Net.Index; using Umbraco.Core.Composing; -using Umbraco.Core.Logging; -using Umbraco.Examine.LocalStorage; +using Umbraco.Examine.Config; using Directory = Lucene.Net.Store.Directory; @@ -29,10 +18,6 @@ namespace Umbraco.Examine /// public class UmbracoExamineSearcher : LuceneSearcher { - - private Lazy _localTempDirectory; - private LocalStorageType _localStorageType = LocalStorageType.Sync; - private string _name; private readonly bool _configBased = false; /// @@ -40,7 +25,6 @@ namespace Umbraco.Examine /// [EditorBrowsable(EditorBrowsableState.Never)] public UmbracoExamineSearcher() - : base() { _configBased = true; } @@ -48,10 +32,11 @@ namespace Umbraco.Examine /// /// Constructor to allow for creating an indexer at runtime /// + /// /// /// - public UmbracoExamineSearcher(DirectoryInfo indexPath, Analyzer analyzer) - : base(indexPath, analyzer) + public UmbracoExamineSearcher(string name, DirectoryInfo indexPath, Analyzer analyzer) + : base(name, indexPath, analyzer) { _configBased = false; } @@ -59,22 +44,25 @@ namespace Umbraco.Examine /// /// Constructor to allow for creating an indexer at runtime /// + /// /// /// - public UmbracoExamineSearcher(Directory luceneDirectory, Analyzer analyzer) - : base(luceneDirectory, analyzer) + public UmbracoExamineSearcher(string name, Directory luceneDirectory, Analyzer analyzer) + : base(name, luceneDirectory, analyzer) + { + _configBased = false; + } + + /// + public UmbracoExamineSearcher(string name, IndexWriter writer, Analyzer analyzer) : base(name, writer, analyzer) { _configBased = false; } /// - /// we override name because we need to manually set it if !CanInitialize() - /// since we cannot call base.Initialize in that case. + /// Name of the Lucene.NET index set /// - public override string Name - { - get { return _name; } - } + public string IndexSetName { get; private set; } /// /// Method used for initializing based on a configuration based searcher @@ -83,77 +71,61 @@ namespace Umbraco.Examine /// public override void Initialize(string name, System.Collections.Specialized.NameValueCollection config) { - if (name == null) throw new ArgumentNullException("name"); - - //ensure name is set - _name = name; - + if (string.IsNullOrWhiteSpace(name)) throw new ArgumentException("Value cannot be null or whitespace.", nameof(name)); + //We need to check if we actually can initialize, if not then don't continue if (CanInitialize() == false) { return; } + //need to check if the index set is specified, if it's not, we'll see if we can find one by convension + //if the folder is not null and the index set is null, we'll assume that this has been created at runtime. + //NOTE: Don't proceed if the _luceneDirectory is set since we already know where to look. + var luceneDirectory = GetLuceneDirectory(); + if (config["indexSet"] == null && (LuceneIndexFolder == null && luceneDirectory == null)) + { + //if we don't have either, then we'll try to set the index set by naming convensions + var found = false; + if (name.EndsWith("Searcher")) + { + var setNameByConvension = name.Remove(name.LastIndexOf("Searcher")) + "IndexSet"; + //check if we can assign the index set by naming convension + var set = IndexSets.Instance.Sets.Cast().SingleOrDefault(x => x.SetName == setNameByConvension); + + if (set != null) + { + set.ReplaceTokensInIndexPath(); + + //we've found an index set by naming convensions :) + IndexSetName = set.SetName; + found = true; + } + } + + if (!found) + throw new ArgumentNullException("indexSet on LuceneExamineIndexer provider has not been set in configuration"); + + //get the folder to index + LuceneIndexFolder = new DirectoryInfo(Path.Combine(IndexSets.Instance.Sets[IndexSetName].IndexDirectory.FullName, "Index")); + } + else if (config["indexSet"] != null && luceneDirectory == null) + { + if (IndexSets.Instance.Sets[config["indexSet"]] == null) + throw new ArgumentException("The indexSet specified for the LuceneExamineIndexer provider does not exist"); + + IndexSetName = config["indexSet"]; + + var indexSet = IndexSets.Instance.Sets[IndexSetName]; + + indexSet.ReplaceTokensInIndexPath(); + + //get the folder to index + LuceneIndexFolder = new DirectoryInfo(Path.Combine(indexSet.IndexDirectory.FullName, "Index")); + } + base.Initialize(name, config); - if (config != null && config["useTempStorage"] != null) - { - throw new NotImplementedException("Fix how local temp storage works and is synced with Examine v2.0 - since a writer is always open we cannot snapshot it, we need to use the same logic in AzureDirectory"); - - //Use the temp storage directory which will store the index in the local/codegen folder, this is useful - // for websites that are running from a remove file server and file IO latency becomes an issue - var attemptUseTempStorage = config["useTempStorage"].TryConvertTo(); - if (attemptUseTempStorage) - { - //this is the default - ILocalStorageDirectory localStorageDir = new CodeGenLocalStorageDirectory(); - if (config["tempStorageDirectory"] != null) - { - //try to get the type - var dirType = BuildManager.GetType(config["tempStorageDirectory"], false); - if (dirType != null) - { - try - { - localStorageDir = (ILocalStorageDirectory)Activator.CreateInstance(dirType); - } - catch (Exception ex) - { - Current.Logger.Error( - string.Format("Could not create a temp storage location of type {0}, reverting to use the " + typeof(CodeGenLocalStorageDirectory).FullName, dirType), - ex); - } - } - } - var indexSet = IndexSets.Instance.Sets[IndexSetName]; - var configuredPath = indexSet.IndexPath; - var tempPath = localStorageDir.GetLocalStorageDirectory(config, configuredPath); - if (tempPath == null) throw new InvalidOperationException("Could not resolve a temp location from the " + localStorageDir.GetType() + " specified"); - var localTempPath = tempPath.FullName; - _localStorageType = attemptUseTempStorage.Result; - - //initialize the lazy callback - _localTempDirectory = new Lazy(() => - { - switch (_localStorageType) - { - case LocalStorageType.Sync: - var fsDir = base.GetLuceneDirectory() as FSDirectory; - if (fsDir != null) - { - return LocalTempStorageDirectoryTracker.Current.GetDirectory( - new DirectoryInfo(localTempPath), - fsDir); - } - return base.GetLuceneDirectory(); - case LocalStorageType.LocalOnly: - return DirectoryTracker.Current.GetDirectory(new DirectoryInfo(localTempPath)); - default: - throw new ArgumentOutOfRangeException(); - } - }); - } - } } /// @@ -171,21 +143,14 @@ namespace Umbraco.Examine /// Returns a list of fields to search on, this will also exclude the IndexPathFieldName and node type alias /// /// - protected override string[] GetSearchFields() + public override string[] GetAllIndexedFields() { - var fields = base.GetSearchFields(); + var fields = base.GetAllIndexedFields(); return fields - .Where(x => x != BaseUmbracoIndexer.IndexPathFieldName) - .Where(x => x != LuceneIndexer.NodeTypeAliasFieldName) + .Where(x => x != UmbracoExamineIndexer.IndexPathFieldName) + .Where(x => x != LuceneIndexer.ItemTypeFieldName) .ToArray(); } - protected override Directory GetLuceneDirectory() - { - //local temp storage is not enabled, just return the default - return _localTempDirectory.IsValueCreated == false - ? base.GetLuceneDirectory() - : _localTempDirectory.Value; - } } } diff --git a/src/Umbraco.Examine/UmbracoMemberIndexer.cs b/src/Umbraco.Examine/UmbracoMemberIndexer.cs index 1f4b7fa79a..b7cbcc19bc 100644 --- a/src/Umbraco.Examine/UmbracoMemberIndexer.cs +++ b/src/Umbraco.Examine/UmbracoMemberIndexer.cs @@ -1,22 +1,18 @@ using System; -using System.Collections; using System.Linq; -using System.Xml.Linq; -using Examine.LuceneEngine.Config; using Umbraco.Core; using Umbraco.Core.Models; using Umbraco.Core.Persistence.DatabaseModelDefinitions; using Umbraco.Core.Services; -using Umbraco.Core.Strings; using System.Collections.Generic; +using System.ComponentModel; using Examine; -using System.IO; +using Examine.LuceneEngine; +using Examine.LuceneEngine.Indexing; using Examine.LuceneEngine.Providers; using Lucene.Net.Analysis; using Umbraco.Core.Composing; using Umbraco.Core.Logging; -using Umbraco.Core.Persistence; -using Umbraco.Core.Scoping; using Directory = Lucene.Net.Store.Directory; namespace Umbraco.Examine @@ -25,24 +21,23 @@ namespace Umbraco.Examine /// /// Custom indexer for members /// - public class UmbracoMemberIndexer : BaseUmbracoIndexer + public class UmbracoMemberIndexer : UmbracoExamineIndexer { private readonly IMemberService _memberService; - private readonly IScopeProvider _scopeProvider; /// - /// Default constructor + /// Constructor for config/provider based indexes /// + [EditorBrowsable(EditorBrowsableState.Never)] public UmbracoMemberIndexer() - : base() { _memberService = Current.Services.MemberService; - _scopeProvider = Current.ScopeProvider; } /// /// Constructor to allow for creating an indexer at runtime /// + /// /// /// /// @@ -50,76 +45,35 @@ namespace Umbraco.Examine /// /// public UmbracoMemberIndexer( + string name, IEnumerable fieldDefinitions, Directory luceneDirectory, Analyzer analyzer, ProfilingLogger profilingLogger, IValueSetValidator validator, IMemberService memberService) : - base(fieldDefinitions, luceneDirectory, analyzer, profilingLogger, validator) + base(name, fieldDefinitions, luceneDirectory, analyzer, profilingLogger, validator) { - if (memberService == null) throw new ArgumentNullException("memberService"); - _memberService = memberService; + _memberService = memberService ?? throw new ArgumentNullException(nameof(memberService)); } - /// - /// Ensures that the'_searchEmail' is added to the user fields so that it is indexed - without having to modify the config - /// - /// - /// - [Obsolete("IIndexCriteria is obsolete, this method is used only for configuration based indexes it is recommended to configure indexes on startup with code instead of config")] - protected override IIndexCriteria GetIndexerData(IndexSet indexSet) - { - //TODO: This is only required for config based index delcaration - We need to change this! - - var indexerData = base.GetIndexerData(indexSet); - - if (CanInitialize()) - { - //If the fields are missing a custom _searchEmail, then add it - - if (indexerData.UserFields.Any(x => x.Name == "_searchEmail") == false) - { - var field = new IndexField { Name = "_searchEmail" }; - var policy = IndexFieldPolicies.FirstOrDefault(x => x.Name == "_searchEmail"); - if (policy != null) - { - field.Type = policy.Type; - field.EnableSorting = policy.EnableSorting; - } - - return new IndexCriteria( - indexerData.StandardFields, - indexerData.UserFields.Concat(new[] { field }), - indexerData.IncludeNodeTypes, - indexerData.ExcludeNodeTypes, - indexerData.ParentNodeId - ); - } - } - - return indexerData; - } /// /// Overridden to ensure that the umbraco system field definitions are in place /// - /// + /// + /// /// - protected override IEnumerable InitializeFieldDefinitions(IEnumerable originalDefinitions) + protected override FieldValueTypeCollection CreateFieldValueTypes(Directory x, IReadOnlyDictionary> indexValueTypesFactory = null) { - var result = base.InitializeFieldDefinitions(originalDefinitions).ToList(); - result.Add(new FieldDefinition("__key", FieldDefinitionTypes.Raw)); - return result; + var keyDef = new FieldDefinition("__key", FieldDefinitionTypes.Raw); + FieldDefinitionCollection.TryAdd(keyDef.Name, keyDef); + + return base.CreateFieldValueTypes(x, indexValueTypesFactory); } - /// - /// The supported types for this indexer - /// - protected override IEnumerable SupportedTypes - { - get { return new[] {IndexTypes.Member}; } - } + /// + protected override IEnumerable SupportedTypes => new[] {IndexTypes.Member}; /// /// Reindex all members @@ -136,44 +90,38 @@ namespace Umbraco.Examine IMember[] members; - using (var scope = _scopeProvider.CreateScope()) + if (ConfigIndexCriteria != null && ConfigIndexCriteria.IncludeItemTypes.Any()) { - if (IndexerData != null && IndexerData.IncludeNodeTypes.Any()) + //if there are specific node types then just index those + foreach (var nodeType in ConfigIndexCriteria.IncludeItemTypes) { - //if there are specific node types then just index those - foreach (var nodeType in IndexerData.IncludeNodeTypes) - { - do - { - long total; - members = _memberService.GetAll(pageIndex, pageSize, out total, "LoginName", Direction.Ascending, true, null, nodeType).ToArray(); - - IndexItems(GetValueSets(members)); - - pageIndex++; - } while (members.Length == pageSize); - } - } - else - { - //no node types specified, do all members do { - long total; - members = _memberService.GetAll(pageIndex, pageSize, out total).ToArray(); + members = _memberService.GetAll(pageIndex, pageSize, out _, "LoginName", Direction.Ascending, true, null, nodeType).ToArray(); IndexItems(GetValueSets(members)); pageIndex++; } while (members.Length == pageSize); } - scope.Complete(); + } + else + { + //no node types specified, do all members + do + { + members = _memberService.GetAll(pageIndex, pageSize, out _).ToArray(); + + IndexItems(GetValueSets(members)); + + pageIndex++; + } while (members.Length == pageSize); } } - private IEnumerable GetValueSets(IEnumerable member) + public static IEnumerable GetValueSets(params IMember[] members) { - foreach (var m in member) + foreach (var m in members) { var values = new Dictionary { @@ -193,30 +141,48 @@ namespace Umbraco.Examine {"email", new object[] {m.Email}}, }; - foreach (var property in m.Properties.Where(p => p != null && p.GetValue() != null && p.GetValue().ToString().IsNullOrWhiteSpace() == false)) + foreach (var property in m.Properties) { - values.Add(property.Alias, new[] { property.GetValue() }); + //only add the value if its not null or empty (we'll check for string explicitly here too) + var val = property.GetValue(); + switch (val) + { + case null: + continue; + case string strVal when strVal.IsNullOrWhiteSpace() == false: + values.Add(property.Alias, new[] { val }); + break; + default: + values.Add(property.Alias, new[] { val }); + break; + } } - var vs = new ValueSet(m.Id, IndexTypes.Content, m.ContentType.Alias, values); + var vs = new ValueSet(m.Id.ToInvariantString(), IndexTypes.Content, m.ContentType.Alias, values); yield return vs; } } - protected override void OnTransformingIndexValues(TransformingIndexDataEventArgs e) + /// + /// Ensure some custom values are added to the index + /// + /// + protected override void OnTransformingIndexValues(IndexingItemEventArgs e) { base.OnTransformingIndexValues(e); - if (e.OriginalValues.ContainsKey("key") && e.IndexItem.ValueSet.Values.ContainsKey("__key") == false) + if (e.IndexItem.ValueSet.Values.TryGetValue("key", out var key) && e.IndexItem.ValueSet.Values.ContainsKey("__key") == false) { - e.IndexItem.ValueSet.Values["__key"] = new List {e.OriginalValues["key"]}; - } - if (e.OriginalValues.ContainsKey("email") && e.IndexItem.ValueSet.Values.ContainsKey("_searchEmail") == false) - { - e.IndexItem.ValueSet.Values["_searchEmail"] = new List { e.OriginalValues["email"].ToString().Replace(".", " ").Replace("@", " ") }; + //double __ prefix means it will be indexed as culture invariant + e.IndexItem.ValueSet.Values["__key"] = new List { key }; } + if (e.IndexItem.ValueSet.Values.TryGetValue("email", out var email) && e.IndexItem.ValueSet.Values.ContainsKey("_searchEmail") == false) + { + //will be indexed as full text (the default anaylyzer) + e.IndexItem.ValueSet.Values["_searchEmail"] = new List { email?.ToString().Replace(".", " ").Replace("@", " ") }; + } } } diff --git a/src/Umbraco.Examine/XsltExtensions.cs b/src/Umbraco.Examine/XsltExtensions.cs deleted file mode 100644 index 63bb0a480a..0000000000 --- a/src/Umbraco.Examine/XsltExtensions.cs +++ /dev/null @@ -1,255 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Security; -using System.Xml.Linq; -using System.Xml.XPath; -using Examine; -using Examine.LuceneEngine; -using Examine.LuceneEngine.Providers; -using Examine.LuceneEngine.SearchCriteria; -using Examine.SearchCriteria; -using Examine.Providers; -using Umbraco.Core.Macros; - -namespace Umbraco.Examine -{ - /// - /// Methods to support Umbraco XSLT extensions. - /// - /// - /// XSLT extensions will ONLY work for provider that have a base class of BaseUmbracoIndexer - /// - [XsltExtension("Examine")] - - public class XsltExtensions - { - /// - /// Uses the provider specified to search, returning an XPathNodeIterator - /// - /// - /// - /// - /// - /// - internal static XPathNodeIterator Search(string searchText, bool useWildcards, LuceneSearcher provider, string indexType) - { - if (provider == null) throw new ArgumentNullException("provider"); - - var results = provider.Search(searchText, useWildcards, indexType); - return GetResultsAsXml(results); - } - - /// - /// Uses the provider specified to search, returning an XPathNodeIterator - /// - /// The search text. - /// if set to true [use wildcards]. - /// Name of the provider. - /// Type of the index. - /// - public static XPathNodeIterator Search(string searchText, bool useWildcards, string providerName, string indexType) - { - var provider = (LuceneSearcher)ExamineManager.Instance.SearchProviderCollection[providerName]; - - return Search(searchText, useWildcards, provider, indexType); - } - - /// - /// Uses the provider specified to search, returning an XPathNodeIterator - /// - /// - /// - /// - /// - public static XPathNodeIterator Search(string searchText, bool useWildcards, string providerName) - { - return Search(searchText, useWildcards, providerName, string.Empty); - } - - /// - /// Uses the default provider specified to search, returning an XPathNodeIterator - /// - /// The search query - /// Enable a wildcard search query - /// A node-set of the search results - public static XPathNodeIterator Search(string searchText, bool useWildcards) - { - return Search(searchText, useWildcards, ExamineManager.Instance.DefaultSearchProvider.Name); - } - - /// - /// Uses the default provider specified to search, returning an XPathNodeIterator - /// - /// The search query - /// A node-set of the search results - public static XPathNodeIterator Search(string searchText) - { - return Search(searchText, true); - } - - /// - /// Will perform a search against the media index type only - /// - /// - /// - /// - /// - public static XPathNodeIterator SearchMediaOnly(string searchText, bool useWildcards, string providerName) - { - return Search(searchText, useWildcards, providerName, IndexTypes.Media); - } - - /// - /// Will perform a search against the media index type only - /// - /// - /// - /// - public static XPathNodeIterator SearchMediaOnly(string searchText, bool useWildcards) - { - return SearchMediaOnly(searchText, useWildcards, ExamineManager.Instance.DefaultSearchProvider.Name); - } - - /// - /// Will perform a search against the media index type only - /// - /// - /// - public static XPathNodeIterator SearchMediaOnly(string searchText) - { - return SearchMediaOnly(searchText, true); - } - - /// - /// Searches the member only. - /// - /// The search text. - /// if set to true [use wildcards]. - /// Name of the provider. - /// - public static XPathNodeIterator SearchMemberOnly(string searchText, bool useWildcards, string providerName) - { - return Search(searchText, useWildcards, providerName, IndexTypes.Member); - } - - /// - /// Searches the member only. - /// - /// The search text. - /// if set to true [use wildcards]. - /// - public static XPathNodeIterator SearchMemberOnly(string searchText, bool useWildcards) - { - return SearchMemberOnly(searchText, useWildcards, ExamineManager.Instance.DefaultSearchProvider.Name); - } - - /// - /// Searches the member only. - /// - /// The search text. - /// - public static XPathNodeIterator SearchMemberOnly(string searchText) - { - return SearchMemberOnly(searchText, true); - } - - /// - /// Will perform a search against the content index type only - /// - /// - /// - /// - /// - public static XPathNodeIterator SearchContentOnly(string searchText, bool useWildcards, string providerName) - { - return Search(searchText, useWildcards, providerName, IndexTypes.Content); - } - - - /// - /// Will perform a search against the content index type only - /// - /// - /// - /// - public static XPathNodeIterator SearchContentOnly(string searchText, bool useWildcards) - { - return SearchContentOnly(searchText, useWildcards, ExamineManager.Instance.DefaultSearchProvider.Name); - } - - /// - /// Will perform a search against the content index type only - /// - /// - /// - public static XPathNodeIterator SearchContentOnly(string searchText) - { - return SearchContentOnly(searchText, true); - } - - /// - /// Gets the results as XML. - /// - /// The results. - /// - private static XPathNodeIterator GetResultsAsXml(ILuceneSearchResults results) - { - // create the XDocument - XDocument doc = new XDocument(); - - // check there are any search results - if (results.TotalItemCount > 0) - { - // create the root element - XElement root = new XElement("nodes"); - - // iterate through the search results - foreach (SearchResult result in results) - { - // create a new element - XElement node = new XElement("node"); - - // create the @id attribute - XAttribute nodeId = new XAttribute("id", result.Id); - - // create the @score attribute - XAttribute nodeScore = new XAttribute("score", result.Score); - - // add the content - node.Add(nodeId, nodeScore); - - foreach (KeyValuePair field in result.Fields) - { - // create a new element - XElement data = new XElement("data"); - - // create the @alias attribute - XAttribute alias = new XAttribute("alias", field.Key); - - // assign the value to a CDATA section - XCData value = new XCData(field.Value); - - // append the content - data.Add(alias, value); - - // append the element - node.Add(data); - } - - // add the node - root.Add(node); - } - - // add the root node - doc.Add(root); - } - else - { - doc.Add(new XElement("error", "There were no search results.")); - } - - return doc.CreateNavigator().Select("/"); - } - } -} diff --git a/src/Umbraco.Tests/Cache/PublishedCache/PublishedMediaCacheTests.cs b/src/Umbraco.Tests/Cache/PublishedCache/PublishedMediaCacheTests.cs index 136929d4cf..34def77faa 100644 --- a/src/Umbraco.Tests/Cache/PublishedCache/PublishedMediaCacheTests.cs +++ b/src/Umbraco.Tests/Cache/PublishedCache/PublishedMediaCacheTests.cs @@ -1,4 +1,5 @@ using System; +using System.Collections; using System.Collections.Generic; using System.Linq; using System.Xml; @@ -161,31 +162,31 @@ namespace Umbraco.Tests.Cache.PublishedCache var ctx = GetUmbracoContext("/test"); var key = Guid.NewGuid(); - var result = new SearchResult() - { - LongId = 1234, - Score = 1 - }; - result.Fields.Add("__IndexType", "media"); - result.Fields.Add("__NodeId", "1234"); - result.Fields.Add("__NodeTypeAlias", Constants.Conventions.MediaTypes.Image); - result.Fields.Add("__Path", "-1,1234"); - result.Fields.Add("__nodeName", "Test"); - result.Fields.Add("id", "1234"); - result.Fields.Add("key", key.ToString()); - result.Fields.Add("urlName", "/media/test.jpg"); - result.Fields.Add("nodeType", "0"); - result.Fields.Add("sortOrder", "0"); - result.Fields.Add("level", "2"); - result.Fields.Add("nodeName", "Test"); - result.Fields.Add("nodeTypeAlias", Constants.Conventions.MediaTypes.Image); - result.Fields.Add("parentID", "-1"); - result.Fields.Add("path", "-1,1234"); - result.Fields.Add("updateDate", DateTime.Parse("2012-07-16T10:34:09").Ticks.ToString()); - result.Fields.Add("createDate", DateTime.Parse("2012-07-17T10:34:09").Ticks.ToString()); - result.Fields.Add("creatorID", "0"); - result.Fields.Add("creatorName", "Shannon"); + var fields = new Dictionary + { + {"__IndexType", "media"}, + {"__NodeId", "1234"}, + {"__NodeTypeAlias", Constants.Conventions.MediaTypes.Image}, + {"__Path", "-1,1234"}, + {"__nodeName", "Test"}, + {"id", "1234"}, + {"key", key.ToString()}, + {"urlName", "/media/test.jpg"}, + {"nodeType", "0"}, + {"sortOrder", "0"}, + {"level", "2"}, + {"nodeName", "Test"}, + {"nodeTypeAlias", Constants.Conventions.MediaTypes.Image}, + {"parentID", "-1"}, + {"path", "-1,1234"}, + {"updateDate", DateTime.Parse("2012-07-16T10:34:09").Ticks.ToString()}, + {"createDate", DateTime.Parse("2012-07-17T10:34:09").Ticks.ToString()}, + {"creatorID", "0"}, + {"creatorName", "Shannon"} + }; + var result = new SearchResult("1234", 1, 1, () => fields.ToDictionary(x => x.Key, x => new List { x.Value })); + var store = new PublishedMediaCache(new XmlStore((XmlDocument)null, null, null, null), ServiceContext.MediaService, ServiceContext.UserService, new StaticCacheProvider(), ContentTypesCache); var doc = store.CreateFromCacheValues(store.ConvertFromSearchResult(result)); diff --git a/src/Umbraco.Tests/Composing/TypeFinderTests.cs b/src/Umbraco.Tests/Composing/TypeFinderTests.cs index 3bb5a7dc0e..29458deefb 100644 --- a/src/Umbraco.Tests/Composing/TypeFinderTests.cs +++ b/src/Umbraco.Tests/Composing/TypeFinderTests.cs @@ -49,7 +49,7 @@ namespace Umbraco.Tests.Composing //typeof(TabPage).Assembly, typeof(System.Web.Mvc.ActionResult).Assembly, typeof(TypeFinder).Assembly, - typeof(global::Umbraco.Examine.BaseUmbracoIndexer).Assembly + typeof(global::Umbraco.Examine.UmbracoExamineIndexer).Assembly }; } diff --git a/src/Umbraco.Tests/PublishedContent/PublishedMediaTests.cs b/src/Umbraco.Tests/PublishedContent/PublishedMediaTests.cs index 8e5436cce1..1e3303fa3e 100644 --- a/src/Umbraco.Tests/PublishedContent/PublishedMediaTests.cs +++ b/src/Umbraco.Tests/PublishedContent/PublishedMediaTests.cs @@ -12,7 +12,7 @@ using Umbraco.Web; using Umbraco.Web.PublishedCache.XmlPublishedCache; using System.Linq; using System.Xml; -using Examine.Session; +using Examine; using Umbraco.Core.Cache; using Umbraco.Core.Models.PublishedContent; using Umbraco.Core.Strings; @@ -109,13 +109,12 @@ namespace Umbraco.Tests.PublishedContent [Test] public void Ensure_Children_Sorted_With_Examine() { - using (var luceneDir = new RAMDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, options: new UmbracoContentIndexerOptions(true, false, null))) - using (var session = new ThreadScopedIndexSession(indexer.SearcherContext)) + using (var luceneDir = new RandomIdRamDirectory()) + using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, options: new UmbracoContentIndexerOptions(true, false, null))) { - indexer.RebuildIndex(); - session.WaitForChanges(); + indexer.RebuildIndex(); + var searcher = indexer.GetSearcher(); var ctx = GetUmbracoContext("/test"); var cache = new PublishedMediaCache(ServiceContext.MediaService, ServiceContext.UserService, searcher, indexer, new StaticCacheProvider(), ContentTypesCache); @@ -136,12 +135,12 @@ namespace Umbraco.Tests.PublishedContent [Test] public void Do_Not_Find_In_Recycle_Bin() { - using (var luceneDir = new RAMDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, options: new UmbracoContentIndexerOptions(true, false, null))) - using (var session = new ThreadScopedIndexSession(indexer.SearcherContext)) + using (var luceneDir = new RandomIdRamDirectory()) + using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, options: new UmbracoContentIndexerOptions(true, false, null))) + using (indexer.ProcessNonAsync()) { indexer.RebuildIndex(); - session.WaitForChanges(); + var searcher = indexer.GetSearcher(); var ctx = GetUmbracoContext("/test"); @@ -159,11 +158,11 @@ namespace Umbraco.Tests.PublishedContent 10726 jpg "); - indexer.ReIndexNode(newXml, "media"); - session.WaitForChanges(); + indexer.IndexItems(new[]{ newXml.ConvertToValueSet("media") }); + //ensure it still exists in the index (raw examine search) - var criteria = searcher.CreateSearchCriteria(); + var criteria = searcher.CreateCriteria(); var filter = criteria.Id(3113); var found = searcher.Search(filter.Compile()); Assert.IsNotNull(found); @@ -180,12 +179,12 @@ namespace Umbraco.Tests.PublishedContent [Test] public void Children_With_Examine() { - using (var luceneDir = new RAMDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, options: new UmbracoContentIndexerOptions(true, false, null))) - using (var session = new ThreadScopedIndexSession(indexer.SearcherContext)) + using (var luceneDir = new RandomIdRamDirectory()) + using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, options: new UmbracoContentIndexerOptions(true, false, null))) + using (indexer.ProcessNonAsync()) { indexer.RebuildIndex(); - session.WaitForChanges(); + var searcher = indexer.GetSearcher(); var ctx = GetUmbracoContext("/test"); @@ -205,12 +204,12 @@ namespace Umbraco.Tests.PublishedContent [Test] public void Descendants_With_Examine() { - using (var luceneDir = new RAMDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, options: new UmbracoContentIndexerOptions(true, false, null))) - using (var session = new ThreadScopedIndexSession(indexer.SearcherContext)) + using (var luceneDir = new RandomIdRamDirectory()) + using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, options: new UmbracoContentIndexerOptions(true, false, null))) + using (indexer.ProcessNonAsync()) { indexer.RebuildIndex(); - session.WaitForChanges(); + var searcher = indexer.GetSearcher(); var ctx = GetUmbracoContext("/test"); @@ -230,12 +229,12 @@ namespace Umbraco.Tests.PublishedContent [Test] public void DescendantsOrSelf_With_Examine() { - using (var luceneDir = new RAMDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, options: new UmbracoContentIndexerOptions(true, false, null))) - using (var session = new ThreadScopedIndexSession(indexer.SearcherContext)) + using (var luceneDir = new RandomIdRamDirectory()) + using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, options: new UmbracoContentIndexerOptions(true, false, null))) + using (indexer.ProcessNonAsync()) { indexer.RebuildIndex(); - session.WaitForChanges(); + var searcher = indexer.GetSearcher(); var ctx = GetUmbracoContext("/test"); @@ -255,12 +254,12 @@ namespace Umbraco.Tests.PublishedContent [Test] public void Ancestors_With_Examine() { - using (var luceneDir = new RAMDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, options: new UmbracoContentIndexerOptions(true, false, null))) - using (var session = new ThreadScopedIndexSession(indexer.SearcherContext)) + using (var luceneDir = new RandomIdRamDirectory()) + using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, options: new UmbracoContentIndexerOptions(true, false, null))) + using (indexer.ProcessNonAsync()) { indexer.RebuildIndex(); - session.WaitForChanges(); + var ctx = GetUmbracoContext("/test"); var searcher = indexer.GetSearcher(); @@ -277,12 +276,12 @@ namespace Umbraco.Tests.PublishedContent [Test] public void AncestorsOrSelf_With_Examine() { - using (var luceneDir = new RAMDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, options: new UmbracoContentIndexerOptions(true, false, null))) - using (var session = new ThreadScopedIndexSession(indexer.SearcherContext)) + using (var luceneDir = new RandomIdRamDirectory()) + using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, options: new UmbracoContentIndexerOptions(true, false, null))) + using (indexer.ProcessNonAsync()) { indexer.RebuildIndex(); - session.WaitForChanges(); + var ctx = GetUmbracoContext("/test"); var searcher = indexer.GetSearcher(); diff --git a/src/Umbraco.Tests/Runtimes/CoreRuntimeTests.cs b/src/Umbraco.Tests/Runtimes/CoreRuntimeTests.cs index b73f6e6c56..3288800bac 100644 --- a/src/Umbraco.Tests/Runtimes/CoreRuntimeTests.cs +++ b/src/Umbraco.Tests/Runtimes/CoreRuntimeTests.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Web.Hosting; +using Examine; using LightInject; using Moq; using NUnit.Framework; @@ -163,7 +164,7 @@ namespace Umbraco.Tests.Runtimes base.Compose(composition); composition.Container.Register(factory => SettingsForTests.GetDefault()); - composition.Container.RegisterSingleton(); + composition.Container.RegisterSingleton(); Composed = true; } diff --git a/src/Umbraco.Tests/TestHelpers/Stubs/TestExamineManager.cs b/src/Umbraco.Tests/TestHelpers/Stubs/TestExamineManager.cs new file mode 100644 index 0000000000..14c6a08c8b --- /dev/null +++ b/src/Umbraco.Tests/TestHelpers/Stubs/TestExamineManager.cs @@ -0,0 +1,69 @@ +using System.Collections.Concurrent; +using System.Collections.Generic; +using Examine; + +namespace Umbraco.Tests.TestHelpers.Stubs +{ + internal class TestExamineManager : IExamineManager + { + private readonly ConcurrentDictionary _indexers = new ConcurrentDictionary(); + private readonly ConcurrentDictionary _searchers = new ConcurrentDictionary(); + + public void AddIndexer(string name, IIndexer indexer) + { + _indexers.TryAdd(name, indexer); + } + + public void AddSearcher(string name, ISearcher searcher) + { + _searchers.TryAdd(name, searcher); + } + + public void DeleteFromIndexes(string nodeId) + { + //noop + } + + public void DeleteFromIndexes(string nodeId, IEnumerable providers) + { + //noop + } + + public void Dispose() + { + //noop + } + + public IIndexer GetIndexer(string indexerName) + { + return _indexers.TryGetValue(indexerName, out var indexer) ? indexer : null; + } + + public ISearcher GetRegisteredSearcher(string searcherName) + { + return _searchers.TryGetValue(searcherName, out var indexer) ? indexer : null; + } + + public void IndexAll(string indexCategory) + { + //noop + } + + public void IndexItems(ValueSet[] nodes) + { + //noop + } + + public void IndexItems(ValueSet[] nodes, IEnumerable providers) + { + //noop + } + + public void RebuildIndexes() + { + //noop + } + + public IReadOnlyDictionary IndexProviders => _indexers; + } +} diff --git a/src/Umbraco.Tests/TestHelpers/Stubs/TestIndexCollectionAccessor.cs b/src/Umbraco.Tests/TestHelpers/Stubs/TestIndexCollectionAccessor.cs deleted file mode 100644 index 550bb6e613..0000000000 --- a/src/Umbraco.Tests/TestHelpers/Stubs/TestIndexCollectionAccessor.cs +++ /dev/null @@ -1,11 +0,0 @@ -using System.Collections.Generic; -using Examine; -using Umbraco.Examine; - -namespace Umbraco.Tests.TestHelpers.Stubs -{ - public class TestIndexCollectionAccessor : IExamineIndexCollectionAccessor - { - public IReadOnlyDictionary Indexes => new Dictionary(); - } -} diff --git a/src/Umbraco.Tests/Testing/UmbracoTestBase.cs b/src/Umbraco.Tests/Testing/UmbracoTestBase.cs index 7dd8061bf7..7a627a0558 100644 --- a/src/Umbraco.Tests/Testing/UmbracoTestBase.cs +++ b/src/Umbraco.Tests/Testing/UmbracoTestBase.cs @@ -3,6 +3,7 @@ using System.IO; using System.Linq; using System.Reflection; using AutoMapper; +using Examine; using LightInject; using Moq; using NUnit.Framework; @@ -258,7 +259,7 @@ namespace Umbraco.Tests.Testing Container.RegisterSingleton(factory => settings.Content); Container.RegisterSingleton(factory => settings.Templates); Container.Register(factory => new MediaFileSystem(Mock.Of())); - Container.RegisterSingleton(); + Container.RegisterSingleton(factory => ExamineManager.Instance); // replace some stuff Container.RegisterSingleton(factory => Mock.Of(), "ScriptFileSystem"); diff --git a/src/Umbraco.Tests/Umbraco.Tests.csproj b/src/Umbraco.Tests/Umbraco.Tests.csproj index 71b2304ee0..03e06799da 100644 --- a/src/Umbraco.Tests/Umbraco.Tests.csproj +++ b/src/Umbraco.Tests/Umbraco.Tests.csproj @@ -59,8 +59,8 @@ ..\packages\Castle.Core.4.1.1\lib\net45\Castle.Core.dll - - ..\packages\Examine.2.0.0-beta2\lib\net45\Examine.dll + + ..\packages\Examine.1.0.0-beta024\lib\net45\Examine.dll ..\packages\SharpZipLib.0.86.0\lib\20\ICSharpCode.SharpZipLib.dll @@ -272,6 +272,7 @@ + @@ -288,11 +289,11 @@ - + @@ -577,7 +578,6 @@ True TestFiles.resx - diff --git a/src/Umbraco.Tests/UmbracoExamine/EventsTest.cs b/src/Umbraco.Tests/UmbracoExamine/EventsTest.cs index d480cd9b1e..f22ae6ef3f 100644 --- a/src/Umbraco.Tests/UmbracoExamine/EventsTest.cs +++ b/src/Umbraco.Tests/UmbracoExamine/EventsTest.cs @@ -1,10 +1,8 @@ using System; using System.Linq; using Examine; -using Examine.Session; using Lucene.Net.Store; using NUnit.Framework; -using Umbraco.Tests.TestHelpers; using Umbraco.Tests.Testing; using Umbraco.Examine; @@ -17,23 +15,14 @@ namespace Umbraco.Tests.UmbracoExamine [Test] public void Events_Ignoring_Node() { - using (var luceneDir = new RAMDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, + using (var luceneDir = new RandomIdRamDirectory()) + using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, //make parent id 999 so all are ignored options: new UmbracoContentIndexerOptions(false, false, 999))) - using (var session = new ThreadScopedIndexSession(indexer.SearcherContext)) + using (indexer.ProcessNonAsync()) { var searcher = indexer.GetSearcher(); - - var isIgnored = false; - - EventHandler ignoringNode = (s, e) => - { - isIgnored = true; - }; - - indexer.IgnoringNode += ignoringNode; - + var contentService = new ExamineDemoDataContentService(); //get a node from the data repo var node = contentService.GetPublishedContentByXPath("//*[string-length(@id)>0 and number(@id)>0]") @@ -41,10 +30,12 @@ namespace Umbraco.Tests.UmbracoExamine .Elements() .First(); - indexer.ReIndexNode(node, IndexTypes.Content); + var valueSet = node.ConvertToValueSet(IndexTypes.Content); + indexer.IndexItems(new[] {valueSet}); + var found = searcher.Search(searcher.CreateCriteria().Id((string) node.Attribute("id")).Compile()); - Assert.IsTrue(isIgnored); + Assert.AreEqual(0, found.TotalItemCount); } diff --git a/src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs b/src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs index 6e82f7d21e..30bcc3766d 100644 --- a/src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs +++ b/src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs @@ -10,6 +10,7 @@ using Moq; using Umbraco.Core.Logging; using Umbraco.Core.Models; using Umbraco.Core.Models.Membership; +using Umbraco.Core.Persistence; using Umbraco.Core.Persistence.DatabaseModelDefinitions; using Umbraco.Core.Persistence.Querying; using Umbraco.Core.Scoping; @@ -30,6 +31,7 @@ namespace Umbraco.Tests.UmbracoExamine public static UmbracoContentIndexer GetUmbracoIndexer( ProfilingLogger profilingLogger, Directory luceneDir, + ISqlContext sqlContext, Analyzer analyzer = null, IContentService contentService = null, IMediaService mediaService = null, @@ -80,6 +82,7 @@ namespace Umbraco.Tests.UmbracoExamine { userService = Mock.Of(x => x.GetProfileById(It.IsAny()) == Mock.Of(p => p.Id == 0 && p.Name == "admin")); } + if (mediaService == null) { long totalRecs; @@ -169,23 +172,24 @@ namespace Umbraco.Tests.UmbracoExamine //query // .Setup(x => x.GetWhereClauses()) // .Returns(new List> { new Tuple($"{Constants.DatabaseSchema.Tables.Document}.published", new object[] { 1 }) }); - var scopeProvider = new Mock(); + //scopeProvider // .Setup(x => x.Query()) // .Returns(query.Object); var i = new UmbracoContentIndexer( - new[] { new FieldDefinition("", FieldDefinitionTypes.FullText) }, + "testIndexer", + Enumerable.Empty(), luceneDir, analyzer, profilingLogger, contentService, mediaService, userService, + sqlContext, new[] {new DefaultUrlSegmentProvider()}, new UmbracoContentValueSetValidator(options, Mock.Of()), - options, - scopeProvider.Object); + options); i.IndexingError += IndexingError; @@ -194,19 +198,19 @@ namespace Umbraco.Tests.UmbracoExamine public static LuceneSearcher GetLuceneSearcher(Directory luceneDir) { - return new LuceneSearcher(luceneDir, new StandardAnalyzer(Version.LUCENE_29)); + return new LuceneSearcher("testSearcher", luceneDir, new StandardAnalyzer(Version.LUCENE_29)); } public static MultiIndexSearcher GetMultiSearcher(Directory pdfDir, Directory simpleDir, Directory conventionDir, Directory cwsDir) { - var i = new MultiIndexSearcher(new[] { pdfDir, simpleDir, conventionDir, cwsDir }, new StandardAnalyzer(Version.LUCENE_29)); + var i = new MultiIndexSearcher("testSearcher", new[] { pdfDir, simpleDir, conventionDir, cwsDir }, new StandardAnalyzer(Version.LUCENE_29)); return i; } internal static void IndexingError(object sender, IndexingErrorEventArgs e) { - throw new ApplicationException(e.Message, e.Exception); + throw new ApplicationException(e.Message, e.InnerException); } diff --git a/src/Umbraco.Tests/UmbracoExamine/IndexTest.cs b/src/Umbraco.Tests/UmbracoExamine/IndexTest.cs index 8964a6e15d..3d82aa6839 100644 --- a/src/Umbraco.Tests/UmbracoExamine/IndexTest.cs +++ b/src/Umbraco.Tests/UmbracoExamine/IndexTest.cs @@ -1,17 +1,15 @@ using System.Linq; +using Examine; using Examine.LuceneEngine.Providers; -using Examine.Session; using Lucene.Net.Index; using Lucene.Net.Search; using Lucene.Net.Store; using NUnit.Framework; -using Umbraco.Tests.TestHelpers; using Umbraco.Tests.Testing; using Umbraco.Examine; namespace Umbraco.Tests.UmbracoExamine { - /// /// Tests the standard indexing capabilities /// @@ -24,17 +22,16 @@ namespace Umbraco.Tests.UmbracoExamine public void Rebuild_Index() { - using (var luceneDir = new RAMDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, options: new UmbracoContentIndexerOptions(true, false, null))) - using (var session = new ThreadScopedIndexSession(indexer.SearcherContext)) + using (var luceneDir = new RandomIdRamDirectory()) + using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, options: new UmbracoContentIndexerOptions(true, false, null))) + using (indexer.ProcessNonAsync()) { var searcher = indexer.GetSearcher(); //create the whole thing indexer.RebuildIndex(); - session.WaitForChanges(); - var result = searcher.Find(searcher.CreateCriteria().All().Compile()); + var result = searcher.Search(searcher.CreateCriteria().All().Compile()); Assert.AreEqual(29, result.TotalItemCount); } @@ -48,24 +45,23 @@ namespace Umbraco.Tests.UmbracoExamine public void Index_Protected_Content_Not_Indexed() { - using (var luceneDir = new RAMDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir)) - using (var session = new ThreadScopedIndexSession(indexer.SearcherContext)) - using (var searcher = indexer.GetSearcher().GetSearcher()) + using (var luceneDir = new RandomIdRamDirectory()) + using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext)) + using (indexer.ProcessNonAsync()) + using (var searcher = ((LuceneSearcher)indexer.GetSearcher()).GetLuceneSearcher()) { //create the whole thing indexer.RebuildIndex(); - session.WaitForChanges(); var protectedQuery = new BooleanQuery(); protectedQuery.Add( new BooleanClause( - new TermQuery(new Term(LuceneIndexer.IndexTypeFieldName, IndexTypes.Content)), + new TermQuery(new Term(LuceneIndexer.CategoryFieldName, IndexTypes.Content)), Occur.MUST)); protectedQuery.Add( new BooleanClause( - new TermQuery(new Term(LuceneIndexer.IndexNodeIdFieldName, ExamineDemoDataContentService.ProtectedNode.ToString())), + new TermQuery(new Term(LuceneIndexer.ItemIdFieldName, ExamineDemoDataContentService.ProtectedNode.ToString())), Occur.MUST)); var collector = TopScoreDocCollector.Create(100, true); @@ -80,11 +76,11 @@ namespace Umbraco.Tests.UmbracoExamine [Test] public void Index_Move_Media_From_Non_Indexable_To_Indexable_ParentID() { - using (var luceneDir = new RAMDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, + using (var luceneDir = new RandomIdRamDirectory()) + using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, //make parent id 1116 options: new UmbracoContentIndexerOptions(false, false, 1116))) - using (var session = new ThreadScopedIndexSession(indexer.SearcherContext)) + using (indexer.ProcessNonAsync()) { var searcher = indexer.GetSearcher(); @@ -99,12 +95,10 @@ namespace Umbraco.Tests.UmbracoExamine Assert.AreEqual("-1,1111,2222,2112", currPath); //ensure it's indexed - indexer.ReIndexNode(node, IndexTypes.Media); - - session.WaitForChanges(); + indexer.IndexItems(new []{ node.ConvertToValueSet(IndexTypes.Media) }); //it will not exist because it exists under 2222 - var results = searcher.Search(searcher.CreateSearchCriteria().Id(2112).Compile()); + var results = searcher.Search(searcher.CreateCriteria().Id(2112).Compile()); Assert.AreEqual(0, results.Count()); //now mimic moving 2112 to 1116 @@ -113,12 +107,10 @@ namespace Umbraco.Tests.UmbracoExamine node.SetAttributeValue("parentID", "1116"); //now reindex the node, this should first delete it and then WILL add it because of the parent id constraint - indexer.ReIndexNode(node, IndexTypes.Media); - - session.WaitForChanges(); + indexer.IndexItems(new[] { node.ConvertToValueSet(IndexTypes.Media) }); //now ensure it exists - results = searcher.Search(searcher.CreateSearchCriteria().Id(2112).Compile()); + results = searcher.Search(searcher.CreateCriteria().Id(2112).Compile()); Assert.AreEqual(1, results.Count()); } } @@ -126,11 +118,11 @@ namespace Umbraco.Tests.UmbracoExamine [Test] public void Index_Move_Media_To_Non_Indexable_ParentID() { - using (var luceneDir = new RAMDirectory()) - using (var indexer1 = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, + using (var luceneDir = new RandomIdRamDirectory()) + using (var indexer1 = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, //make parent id 2222 options: new UmbracoContentIndexerOptions(false, false, 2222))) - using (var session = new ThreadScopedIndexSession(indexer1.SearcherContext)) + using (indexer1.ProcessNonAsync()) { var searcher = indexer1.GetSearcher(); @@ -145,12 +137,12 @@ namespace Umbraco.Tests.UmbracoExamine Assert.AreEqual("-1,1111,2222,2112", currPath); //ensure it's indexed - indexer1.ReIndexNode(node, IndexTypes.Media); + indexer1.IndexItems(new[] { node.ConvertToValueSet(IndexTypes.Media) }); + - session.WaitForChanges(); //it will exist because it exists under 2222 - var results = searcher.Search(searcher.CreateSearchCriteria().Id(2112).Compile()); + var results = searcher.Search(searcher.CreateCriteria().Id(2112).Compile()); Assert.AreEqual(1, results.Count()); //now mimic moving the node underneath 1116 instead of 2222 @@ -158,12 +150,12 @@ namespace Umbraco.Tests.UmbracoExamine node.SetAttributeValue("parentID", "1116"); //now reindex the node, this should first delete it and then NOT add it because of the parent id constraint - indexer1.ReIndexNode(node, IndexTypes.Media); + indexer1.IndexItems(new[] { node.ConvertToValueSet(IndexTypes.Media) }); + - session.WaitForChanges(); //now ensure it's deleted - results = searcher.Search(searcher.CreateSearchCriteria().Id(2112).Compile()); + results = searcher.Search(searcher.CreateCriteria().Id(2112).Compile()); Assert.AreEqual(0, results.Count()); } } @@ -176,36 +168,36 @@ namespace Umbraco.Tests.UmbracoExamine [Test] public void Index_Reindex_Content() { - using (var luceneDir = new RAMDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, options: new UmbracoContentIndexerOptions(true, false, null))) - using (var session = new ThreadScopedIndexSession(indexer.SearcherContext)) + using (var luceneDir = new RandomIdRamDirectory()) + using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, options: new UmbracoContentIndexerOptions(true, false, null))) + using (indexer.ProcessNonAsync()) { var searcher = indexer.GetSearcher(); //create the whole thing indexer.RebuildIndex(); - session.WaitForChanges(); + - var result = searcher.Find(searcher.CreateCriteria().Field(LuceneIndexer.IndexTypeFieldName, IndexTypes.Content).Compile()); + var result = searcher.Search(searcher.CreateCriteria().Field(LuceneIndexer.CategoryFieldName, IndexTypes.Content).Compile()); Assert.AreEqual(21, result.TotalItemCount); //delete all content foreach (var r in result) { - indexer.DeleteFromIndex(r.LongId); + indexer.DeleteFromIndex(r.Id); } - session.WaitForChanges(); + //ensure it's all gone - result = searcher.Find(searcher.CreateCriteria().Field(LuceneIndexer.IndexTypeFieldName, IndexTypes.Content).Compile()); + result = searcher.Search(searcher.CreateCriteria().Field(LuceneIndexer.CategoryFieldName, IndexTypes.Content).Compile()); Assert.AreEqual(0, result.TotalItemCount); //call our indexing methods indexer.IndexAll(IndexTypes.Content); - session.WaitForChanges(); + - result = searcher.Find(searcher.CreateCriteria().Field(LuceneIndexer.IndexTypeFieldName, IndexTypes.Content).Compile()); + result = searcher.Search(searcher.CreateCriteria().Field(LuceneIndexer.CategoryFieldName, IndexTypes.Content).Compile()); Assert.AreEqual(21, result.TotalItemCount); } @@ -219,27 +211,27 @@ namespace Umbraco.Tests.UmbracoExamine [Test] public void Index_Delete_Index_Item_Ensure_Heirarchy_Removed() { - using (var luceneDir = new RAMDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir)) - using (var session = new ThreadScopedIndexSession(indexer.SearcherContext)) + using (var luceneDir = new RandomIdRamDirectory()) + using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext)) + using (indexer.ProcessNonAsync()) { var searcher = indexer.GetSearcher(); //create the whole thing indexer.RebuildIndex(); - session.WaitForChanges(); + //now delete a node that has children indexer.DeleteFromIndex(1140.ToString()); //this node had children: 1141 & 1142, let's ensure they are also removed - session.WaitForChanges(); + - var results = searcher.Search(searcher.CreateSearchCriteria().Id(1141).Compile()); + var results = searcher.Search(searcher.CreateCriteria().Id(1141).Compile()); Assert.AreEqual(0, results.Count()); - results = searcher.Search(searcher.CreateSearchCriteria().Id(1142).Compile()); + results = searcher.Search(searcher.CreateCriteria().Id(1142).Compile()); Assert.AreEqual(0, results.Count()); } diff --git a/src/Umbraco.Tests/UmbracoExamine/RandomIdRamDirectory.cs b/src/Umbraco.Tests/UmbracoExamine/RandomIdRamDirectory.cs new file mode 100644 index 0000000000..95df0dfc3d --- /dev/null +++ b/src/Umbraco.Tests/UmbracoExamine/RandomIdRamDirectory.cs @@ -0,0 +1,17 @@ +using System; +using Lucene.Net.Store; + +namespace Umbraco.Tests.UmbracoExamine +{ + /// + /// This is needed for all tests, else the lockid collides with directories during testing + /// + public class RandomIdRamDirectory : RAMDirectory + { + private readonly string _lockId = Guid.NewGuid().ToString(); + public override string GetLockId() + { + return _lockId; + } + } +} \ No newline at end of file diff --git a/src/Umbraco.Tests/UmbracoExamine/SearchTests.cs b/src/Umbraco.Tests/UmbracoExamine/SearchTests.cs index d1ebf68871..c71dc6f242 100644 --- a/src/Umbraco.Tests/UmbracoExamine/SearchTests.cs +++ b/src/Umbraco.Tests/UmbracoExamine/SearchTests.cs @@ -5,13 +5,12 @@ using Examine; using Lucene.Net.Store; using NUnit.Framework; using Examine.LuceneEngine.SearchCriteria; -using Examine.Session; using Moq; using Umbraco.Core.Models; using Umbraco.Core.Persistence.DatabaseModelDefinitions; using Umbraco.Core.Persistence.Querying; using Umbraco.Core.Services; -using Umbraco.Tests.TestHelpers; +using Umbraco.Examine; using Umbraco.Tests.Testing; namespace Umbraco.Tests.UmbracoExamine @@ -54,21 +53,21 @@ namespace Umbraco.Tests.UmbracoExamine == allRecs); - using (var luceneDir = new RAMDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, contentService: contentService)) - using (var session = new ThreadScopedIndexSession(indexer.SearcherContext)) + using (var luceneDir = new RandomIdRamDirectory()) + using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, contentService: contentService)) + using (indexer.ProcessNonAsync()) { indexer.RebuildIndex(); - session.WaitForChanges(); + var searcher = indexer.GetSearcher(); - var numberSortedCriteria = searcher.CreateSearchCriteria() + var numberSortedCriteria = searcher.CreateCriteria() .ParentId(1148).And() .OrderBy(new SortableField("sortOrder", SortType.Int)); var numberSortedResult = searcher.Search(numberSortedCriteria.Compile()); - var stringSortedCriteria = searcher.CreateSearchCriteria() + var stringSortedCriteria = searcher.CreateCriteria() .ParentId(1148).And() .OrderBy("sortOrder"); //will default to string var stringSortedResult = searcher.Search(stringSortedCriteria.Compile()); @@ -99,7 +98,7 @@ namespace Umbraco.Tests.UmbracoExamine //[Test] //public void Test_Index_Type_With_German_Analyzer() //{ - // using (var luceneDir = new RAMDirectory()) + // using (var luceneDir = new RandomIdRamDirectory()) // { // var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir, // new GermanAnalyzer()); diff --git a/src/Umbraco.Tests/UmbracoExamine/TestIndexField.cs b/src/Umbraco.Tests/UmbracoExamine/TestIndexField.cs deleted file mode 100644 index b2e2d7d823..0000000000 --- a/src/Umbraco.Tests/UmbracoExamine/TestIndexField.cs +++ /dev/null @@ -1,11 +0,0 @@ -using Examine; - -namespace Umbraco.Tests.UmbracoExamine -{ - public class TestIndexField : IIndexField - { - public string Name { get; set; } - public bool EnableSorting { get; set; } - public string Type { get; set; } - } -} diff --git a/src/Umbraco.Tests/packages.config b/src/Umbraco.Tests/packages.config index b75ca5912b..f1b5109576 100644 --- a/src/Umbraco.Tests/packages.config +++ b/src/Umbraco.Tests/packages.config @@ -2,7 +2,7 @@ - + diff --git a/src/Umbraco.Web.UI.Client/src/installer/steps/upgrade.html b/src/Umbraco.Web.UI.Client/src/installer/steps/upgrade.html index 4edf2eda25..472ceb7135 100644 --- a/src/Umbraco.Web.UI.Client/src/installer/steps/upgrade.html +++ b/src/Umbraco.Web.UI.Client/src/installer/steps/upgrade.html @@ -1,18 +1,19 @@ 

Upgrading Umbraco

- Welcome to the Umbraco installer. You see this screen because your Umbraco installation needs a quick upgrade of its database and files, which will ensure your website is kept as fast, secure and up to date as possible. + Welcome to the Umbraco installer. You see this screen because your Umbraco installation needs a quick upgrade + of its database and files, which will ensure your website is kept as fast, secure and up to date as possible.

-

- To read a report of changes between your current version {{installer.current.model.currentVersion}} and this version you're upgrading to {{installer.current.model.newVersion}} + Detected current version {{installer.current.model.currentVersion}} ({{installer.current.model.currentState}}), + which needs to be upgraded to {{installer.current.model.newVersion}} ({{installer.current.model.newState}}). + To compare versions and read a report of changes between versions, use the View Report button below.

View Report

-

- Simply click continue below to be guided through the rest of the upgrade + Simply click continue below to be guided through the rest of the upgrade.

diff --git a/src/Umbraco.Web.UI.Client/src/views/dashboard/developer/examinemanagement.html b/src/Umbraco.Web.UI.Client/src/views/dashboard/developer/examinemanagement.html index 1c166c54ab..645b783c65 100644 --- a/src/Umbraco.Web.UI.Client/src/views/dashboard/developer/examinemanagement.html +++ b/src/Umbraco.Web.UI.Client/src/views/dashboard/developer/examinemanagement.html @@ -76,21 +76,7 @@ Fields in index {{indexer.fieldCount}} - - - Has deletions? - - {{indexer.deletionCount > 0}} - ({{indexer.deletionCount}}) - - - - Optimized? - - {{indexer.isOptimized}} - - - +

diff --git a/src/Umbraco.Web.UI.Client/src/views/dashboard/developer/examinemgmt.controller.js b/src/Umbraco.Web.UI.Client/src/views/dashboard/developer/examinemgmt.controller.js index 5f6bb23001..f4e78a9dc3 100644 --- a/src/Umbraco.Web.UI.Client/src/views/dashboard/developer/examinemgmt.controller.js +++ b/src/Umbraco.Web.UI.Client/src/views/dashboard/developer/examinemgmt.controller.js @@ -91,30 +91,6 @@ function ExamineMgmtController($scope, umbRequestHelper, $log, $http, $q, $timeo } } - $scope.optimizeIndex = function(indexer) { - if (confirm("This will cause the index to be optimized which will improve its performance. " + - "It is not recommended to optimize an index during times of high website traffic " + - "or when editors are editing content.")) { - indexer.isProcessing = true; - - umbRequestHelper.resourcePromise( - $http.post(umbRequestHelper.getApiUrl("examineMgmtBaseUrl", - "PostOptimizeIndex", - { indexerName: indexer.name })), - 'Failed to optimize index') - .then(function() { - - //optimizing has started, nothing is returned accept a 200 status code. - //lets poll to see if it is done. - $timeout(function() { - checkProcessing(indexer, "PostCheckOptimizeIndex"); - }, - 1000); - - }); - } - } - $scope.closeSearch = function(searcher) { searcher.isSearching = true; } diff --git a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj index c2c923b52a..0bf9e597ca 100644 --- a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj +++ b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj @@ -57,6 +57,9 @@ ..\packages\ClientDependency.1.9.6\lib\net45\ClientDependency.Core.dll + + ..\packages\Examine.1.0.0-beta024\lib\net45\Examine.dll + ..\packages\ImageProcessor.Web.4.8.4\lib\net45\ImageProcessor.Web.dll @@ -273,10 +276,6 @@ ../packages/ClientDependency-Mvc5.1.8.0.0/lib/net45/ClientDependency.Core.Mvc.dll True - - ../packages/Examine.2.0.0-beta2/lib/net45/Examine.dll - True - ../packages/SharpZipLib.0.86.0/lib/20/ICSharpCode.SharpZipLib.dll True diff --git a/src/Umbraco.Web.UI/config/ExamineSettings.Release.config b/src/Umbraco.Web.UI/config/ExamineSettings.Release.config index 49287b21c3..ac0f43ce1a 100644 --- a/src/Umbraco.Web.UI/config/ExamineSettings.Release.config +++ b/src/Umbraco.Web.UI/config/ExamineSettings.Release.config @@ -12,12 +12,12 @@ More information and documentation can be found on GitHub: https://github.com/Sh + analyzer="Examine.LuceneEngine.CultureInvariantWhitespaceAnalyzer, Examine"/> + analyzer="Examine.LuceneEngine.CultureInvariantStandardAnalyzer, Examine"/> @@ -25,7 +25,7 @@ More information and documentation can be found on GitHub: https://github.com/Sh - + @@ -33,7 +33,7 @@ More information and documentation can be found on GitHub: https://github.com/Sh + analyzer="Examine.LuceneEngine.CultureInvariantStandardAnalyzer, Examine" /> diff --git a/src/Umbraco.Web.UI/config/ExamineSettings.config b/src/Umbraco.Web.UI/config/ExamineSettings.config index 40586ee854..68208427cf 100644 --- a/src/Umbraco.Web.UI/config/ExamineSettings.config +++ b/src/Umbraco.Web.UI/config/ExamineSettings.config @@ -12,12 +12,12 @@ More information and documentation can be found on GitHub: https://github.com/Sh + analyzer="Examine.LuceneEngine.CultureInvariantWhitespaceAnalyzer, Examine"/> + analyzer="Examine.LuceneEngine.CultureInvariantStandardAnalyzer, Examine"/> @@ -25,7 +25,7 @@ More information and documentation can be found on GitHub: https://github.com/Sh - + @@ -33,8 +33,7 @@ More information and documentation can be found on GitHub: https://github.com/Sh + analyzer="Examine.LuceneEngine.CultureInvariantStandardAnalyzer, Examine" /> diff --git a/src/Umbraco.Web.UI/packages.config b/src/Umbraco.Web.UI/packages.config index 1b519c30ad..98b3d56f79 100644 --- a/src/Umbraco.Web.UI/packages.config +++ b/src/Umbraco.Web.UI/packages.config @@ -4,7 +4,7 @@ - + diff --git a/src/Umbraco.Web.UI/web.Template.config b/src/Umbraco.Web.UI/web.Template.config index cf3761d1b0..905ea3b5e5 100644 --- a/src/Umbraco.Web.UI/web.Template.config +++ b/src/Umbraco.Web.UI/web.Template.config @@ -11,7 +11,7 @@
-
+
diff --git a/src/Umbraco.Web/Components/DatabaseServerRegistrarAndMessengerComponent.cs b/src/Umbraco.Web/Components/DatabaseServerRegistrarAndMessengerComponent.cs index 2048ba9e58..bd97c483b7 100644 --- a/src/Umbraco.Web/Components/DatabaseServerRegistrarAndMessengerComponent.cs +++ b/src/Umbraco.Web/Components/DatabaseServerRegistrarAndMessengerComponent.cs @@ -17,6 +17,8 @@ using Umbraco.Web.Cache; using Umbraco.Web.Composing; using Umbraco.Web.Routing; using Umbraco.Web.Scheduling; +using Umbraco.Examine; +using Umbraco.Web.Search; namespace Umbraco.Web.Components { @@ -44,6 +46,7 @@ namespace Umbraco.Web.Components private BackgroundTaskRunner _processTaskRunner; private bool _started; private IBackgroundTask[] _tasks; + private IExamineManager _examineManager; public override void Compose(Composition composition) { @@ -76,32 +79,21 @@ namespace Umbraco.Web.Components // (we really should have a way to reuse RefreshAll... locally) // note: refresh all content & media caches does refresh content types too var svc = Current.PublishedSnapshotService; - bool ignored1, ignored2; svc.Notify(new[] { new DomainCacheRefresher.JsonPayload(0, DomainChangeTypes.RefreshAll) }); - svc.Notify(new[] { new ContentCacheRefresher.JsonPayload(0, TreeChangeTypes.RefreshAll) }, out ignored1, out ignored2); - svc.Notify(new[] { new MediaCacheRefresher.JsonPayload(0, TreeChangeTypes.RefreshAll) }, out ignored1); + svc.Notify(new[] { new ContentCacheRefresher.JsonPayload(0, TreeChangeTypes.RefreshAll) }, out _, out _); + svc.Notify(new[] { new MediaCacheRefresher.JsonPayload(0, TreeChangeTypes.RefreshAll) }, out _); }, //rebuild indexes if the server is not synced // NOTE: This will rebuild ALL indexes including the members, if developers want to target specific // indexes then they can adjust this logic themselves. - () => RebuildIndexes(false) + () => ExamineComponent.RebuildIndexes(false, _examineManager, _logger) } }); }); } - // fixme - this should move to something else, we should not depend on Examine here! - private static void RebuildIndexes(bool onlyEmptyIndexes) - { - var indexers = (IEnumerable>) ExamineManager.Instance.IndexProviders; - if (onlyEmptyIndexes) - indexers = indexers.Where(x => x.Value.IsIndexNew()); - foreach (var indexer in indexers) - indexer.Value.RebuildIndex(); - } - - public void Initialize(IRuntimeState runtime, IServerRegistrar serverRegistrar, IServerMessenger serverMessenger, IServerRegistrationService registrationService, ILogger logger) + public void Initialize(IRuntimeState runtime, IServerRegistrar serverRegistrar, IServerMessenger serverMessenger, IServerRegistrationService registrationService, ILogger logger, IExamineManager examineManager) { if (UmbracoConfig.For.UmbracoSettings().DistributedCall.Enabled) return; @@ -114,6 +106,7 @@ namespace Umbraco.Web.Components _runtime = runtime; _logger = logger; _registrationService = registrationService; + _examineManager = examineManager; _touchTaskRunner = new BackgroundTaskRunner("ServerRegistration", new BackgroundTaskRunnerOptions { AutoStart = true }, logger); diff --git a/src/Umbraco.Web/ExamineExtensions.cs b/src/Umbraco.Web/ExamineExtensions.cs index 455a857879..00471a44e1 100644 --- a/src/Umbraco.Web/ExamineExtensions.cs +++ b/src/Umbraco.Web/ExamineExtensions.cs @@ -18,10 +18,12 @@ namespace Umbraco.Web foreach (var result in results.OrderByDescending(x => x.Score)) { - var content = cache.GetById(result.Id); + if (!int.TryParse(result.Id, out var intId)) continue; //invalid + var content = cache.GetById(intId); if (content == null) continue; // skip if this doesn't exist in the cache list.Add(new PublishedSearchResult(content, result.Score)); + } return list; diff --git a/src/Umbraco.Web/IPublishedContentQuery.cs b/src/Umbraco.Web/IPublishedContentQuery.cs index ddaae9922d..055dcbe181 100644 --- a/src/Umbraco.Web/IPublishedContentQuery.cs +++ b/src/Umbraco.Web/IPublishedContentQuery.cs @@ -34,21 +34,21 @@ namespace Umbraco.Web /// /// Searches content. /// - IEnumerable Search(string term, bool useWildCards = true, string searchProvider = null); + IEnumerable Search(string term, bool useWildCards = true, string indexName = null); /// /// Searches content. /// - IEnumerable Search(int skip, int take, out int totalRecords, string term, bool useWildCards = true, string searchProvider = null); + IEnumerable Search(int skip, int take, out int totalRecords, string term, bool useWildCards = true, string indexName = null); /// /// Searches content. /// - IEnumerable Search(Examine.SearchCriteria.ISearchCriteria criteria, Examine.Providers.BaseSearchProvider searchProvider = null); + IEnumerable Search(Examine.SearchCriteria.ISearchCriteria criteria, Examine.ISearcher searchProvider = null); /// /// Searches content. /// - IEnumerable Search(int skip, int take, out int totalrecords, Examine.SearchCriteria.ISearchCriteria criteria, Examine.Providers.BaseSearchProvider searchProvider = null); + IEnumerable Search(int skip, int take, out int totalRecords, Examine.SearchCriteria.ISearchCriteria criteria, Examine.ISearcher searchProvider = null); } } diff --git a/src/Umbraco.Web/Install/InstallSteps/UpgradeStep.cs b/src/Umbraco.Web/Install/InstallSteps/UpgradeStep.cs index 75b027f69f..1d36fe9e12 100644 --- a/src/Umbraco.Web/Install/InstallSteps/UpgradeStep.cs +++ b/src/Umbraco.Web/Install/InstallSteps/UpgradeStep.cs @@ -1,4 +1,6 @@ -using Umbraco.Core.Configuration; +using System; +using Umbraco.Core.Composing; +using Umbraco.Core.Configuration; using Umbraco.Web.Install.Models; namespace Umbraco.Web.Install.InstallSteps @@ -9,26 +11,28 @@ namespace Umbraco.Web.Install.InstallSteps [InstallSetupStep(InstallationType.Upgrade, "Upgrade", "upgrade", 1, "Upgrading Umbraco to the latest and greatest version.")] internal class UpgradeStep : InstallSetupStep { - public override bool RequiresExecution(object model) - { - return true; - } + public override bool RequiresExecution(object model) => true; - public override InstallSetupResult Execute(object model) - { - return null; - } + public override InstallSetupResult Execute(object model) => null; public override object ViewModel { get { - // fixme where is the "detected current version"? var currentVersion = UmbracoVersion.Local.ToString(); - var newVersion = UmbracoVersion.Current.ToString(); + var newVersion = UmbracoVersion.SemanticVersion.ToString(); + + var state = Current.RuntimeState; // fixme inject + var currentState = state.CurrentMigrationState; + if (string.IsNullOrWhiteSpace(currentState)) currentState = "unknown"; + var newState = state.FinalMigrationState?.Trim('{', '}'); + if (string.IsNullOrWhiteSpace(newState)) newState = "unknown"; + else if (Guid.TryParse(newState, out _)) + newState = newState.Substring(0, 8); + var reportUrl = $"https://our.umbraco.org/contribute/releases/compare?from={currentVersion}&to={newVersion}¬es=1"; - return new { currentVersion, newVersion, reportUrl }; + return new { currentVersion, newVersion, currentState, newState, reportUrl }; } } } diff --git a/src/Umbraco.Web/Models/Mapping/EntityMapperProfile.cs b/src/Umbraco.Web/Models/Mapping/EntityMapperProfile.cs index 38440c0b74..917f61a429 100644 --- a/src/Umbraco.Web/Models/Mapping/EntityMapperProfile.cs +++ b/src/Umbraco.Web/Models/Mapping/EntityMapperProfile.cs @@ -122,22 +122,22 @@ namespace Umbraco.Web.Models.Mapping .AfterMap((src, dest) => { //get the icon if there is one - dest.Icon = src.Fields.ContainsKey(BaseUmbracoIndexer.IconFieldName) - ? src.Fields[BaseUmbracoIndexer.IconFieldName] + dest.Icon = src.Fields.ContainsKey(UmbracoExamineIndexer.IconFieldName) + ? src.Fields[UmbracoExamineIndexer.IconFieldName] : "icon-document"; dest.Name = src.Fields.ContainsKey("nodeName") ? src.Fields["nodeName"] : "[no name]"; - if (src.Fields.ContainsKey(UmbracoContentIndexer.NodeKeyFieldName)) + if (src.Fields.ContainsKey(UmbracoExamineIndexer.NodeKeyFieldName)) { Guid key; - if (Guid.TryParse(src.Fields[UmbracoContentIndexer.NodeKeyFieldName], out key)) + if (Guid.TryParse(src.Fields[UmbracoExamineIndexer.NodeKeyFieldName], out key)) { dest.Key = key; //need to set the UDI - if (src.Fields.ContainsKey(LuceneIndexer.IndexTypeFieldName)) + if (src.Fields.ContainsKey(LuceneIndexer.CategoryFieldName)) { - switch (src.Fields[LuceneIndexer.IndexTypeFieldName]) + switch (src.Fields[LuceneIndexer.CategoryFieldName]) { case IndexTypes.Member: dest.Udi = new GuidUdi(Constants.UdiEntityType.Member, dest.Key); @@ -165,15 +165,15 @@ namespace Umbraco.Web.Models.Mapping dest.ParentId = -1; } } - dest.Path = src.Fields.ContainsKey(UmbracoContentIndexer.IndexPathFieldName) ? src.Fields[UmbracoContentIndexer.IndexPathFieldName] : ""; + dest.Path = src.Fields.ContainsKey(UmbracoExamineIndexer.IndexPathFieldName) ? src.Fields[UmbracoExamineIndexer.IndexPathFieldName] : ""; - if (src.Fields.ContainsKey(LuceneIndexer.NodeTypeAliasFieldName)) + if (src.Fields.ContainsKey(LuceneIndexer.ItemTypeFieldName)) { - dest.AdditionalData.Add("contentType", src.Fields[LuceneIndexer.NodeTypeAliasFieldName]); + dest.AdditionalData.Add("contentType", src.Fields[LuceneIndexer.ItemTypeFieldName]); } }); - CreateMap>() + CreateMap>() .ConvertUsing(results => results.Select(Mapper.Map).ToList()); CreateMap, IEnumerable>() diff --git a/src/Umbraco.Web/PropertyEditors/GridPropertyEditor.cs b/src/Umbraco.Web/PropertyEditors/GridPropertyEditor.cs index 6256f49c67..0f3211b743 100644 --- a/src/Umbraco.Web/PropertyEditors/GridPropertyEditor.cs +++ b/src/Umbraco.Web/PropertyEditors/GridPropertyEditor.cs @@ -27,84 +27,80 @@ namespace Umbraco.Web.PropertyEditors internal void DocumentWriting(object sender, Examine.LuceneEngine.DocumentWritingEventArgs e) { - var indexer = (BaseUmbracoIndexer)sender; - foreach (var field in indexer.IndexerData.UserFields) + foreach (var value in e.ValueSet.Values) { - if (e.Fields.ContainsKey(field.Name)) + //if there is a value, it's a string and it's detected as json + if (value.Value.Count > 0 && value.Value[0] != null && (value.Value[0] is string firstVal) && firstVal.DetectIsJson()) { - if (e.Fields[field.Name].DetectIsJson()) + try { - try + //TODO: We should deserialize this to Umbraco.Core.Models.GridValue instead of doing the below + var json = JsonConvert.DeserializeObject(firstVal); + + //check if this is formatted for grid json + if (json.HasValues && json.TryGetValue("name", out _) && json.TryGetValue("sections", out _)) { - //TODO: We should deserialize this to Umbraco.Core.Models.GridValue instead of doing the below - var json = JsonConvert.DeserializeObject(e.Fields[field.Name]); - - //check if this is formatted for grid json - JToken name; - JToken sections; - if (json.HasValues && json.TryGetValue("name", out name) && json.TryGetValue("sections", out sections)) + //get all values and put them into a single field (using JsonPath) + var sb = new StringBuilder(); + foreach (var row in json.SelectTokens("$.sections[*].rows[*]")) { - //get all values and put them into a single field (using JsonPath) - var sb = new StringBuilder(); - foreach (var row in json.SelectTokens("$.sections[*].rows[*]")) - { - var rowName = row["name"].Value(); - var areaVals = row.SelectTokens("$.areas[*].controls[*].value"); + var rowName = row["name"].Value(); + var areaVals = row.SelectTokens("$.areas[*].controls[*].value"); - foreach (var areaVal in areaVals) + foreach (var areaVal in areaVals) + { + //TODO: If it's not a string, then it's a json formatted value - + // we cannot really index this in a smart way since it could be 'anything' + if (areaVal.Type == JTokenType.String) { - //TODO: If it's not a string, then it's a json formatted value - - // we cannot really index this in a smart way since it could be 'anything' - if (areaVal.Type == JTokenType.String) - { - var str = areaVal.Value(); - str = XmlHelper.CouldItBeXml(str) ? str.StripHtml() : str; - sb.Append(str); - sb.Append(" "); - - //add the row name as an individual field - e.Document.Add( - new Field( - string.Format("{0}.{1}", field.Name, rowName), str, Field.Store.YES, Field.Index.ANALYZED)); - } + var str = areaVal.Value(); + str = XmlHelper.CouldItBeXml(str) ? str.StripHtml() : str; + sb.Append(str); + sb.Append(" "); + //add the row name as an individual field + e.Document.Add( + new Field( + $"{value.Key}.{rowName}", str, Field.Store.YES, Field.Index.ANALYZED)); } - } - if (sb.Length > 0) - { - //First save the raw value to a raw field - e.Document.Add( - new Field( - string.Format("{0}{1}", UmbracoContentIndexer.RawFieldPrefix, field.Name), - e.Fields[field.Name], Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS, Field.TermVector.NO)); - - //now replace the original value with the combined/cleaned value - e.Document.RemoveField(field.Name); - e.Document.Add( - new Field( - field.Name, - sb.ToString(), Field.Store.YES, Field.Index.ANALYZED)); } } - } - catch (InvalidCastException) - { - //swallow...on purpose, there's a chance that this isn't the json format we are looking for - // and we don't want that to affect the website. - } - catch (JsonException) - { - //swallow...on purpose, there's a chance that this isn't json and we don't want that to affect - // the website. - } - catch (ArgumentException) - { - //swallow on purpose to prevent this error: - // Can not add Newtonsoft.Json.Linq.JValue to Newtonsoft.Json.Linq.JObject. + + if (sb.Length > 0) + { + //First save the raw value to a raw field + e.Document.Add( + new Field( + $"{UmbracoExamineIndexer.RawFieldPrefix}{value.Key}", + firstVal, Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS, Field.TermVector.NO)); + + //now replace the original value with the combined/cleaned value + e.Document.RemoveField(value.Key); + e.Document.Add( + new Field( + value.Key, + sb.ToString(), Field.Store.YES, Field.Index.ANALYZED)); + } } } + catch (InvalidCastException) + { + //swallow...on purpose, there's a chance that this isn't the json format we are looking for + // and we don't want that to affect the website. + } + catch (JsonException) + { + //swallow...on purpose, there's a chance that this isn't json and we don't want that to affect + // the website. + } + catch (ArgumentException) + { + //swallow on purpose to prevent this error: + // Can not add Newtonsoft.Json.Linq.JValue to Newtonsoft.Json.Linq.JObject. + } } + } } diff --git a/src/Umbraco.Web/PropertyEditors/PropertyEditorsComponent.cs b/src/Umbraco.Web/PropertyEditors/PropertyEditorsComponent.cs index f8b8d89a1c..6c0233a7e3 100644 --- a/src/Umbraco.Web/PropertyEditors/PropertyEditorsComponent.cs +++ b/src/Umbraco.Web/PropertyEditors/PropertyEditorsComponent.cs @@ -1,4 +1,5 @@ using System.Linq; +using Examine; using Umbraco.Core; using Umbraco.Core.Components; using Umbraco.Core.Models; @@ -11,7 +12,7 @@ namespace Umbraco.Web.PropertyEditors [RuntimeLevel(MinLevel = RuntimeLevel.Run)] internal class PropertyEditorsComponent : UmbracoComponentBase, IUmbracoCoreComponent { - public void Initialize(IRuntimeState runtime, PropertyEditorCollection propertyEditors, IExamineIndexCollectionAccessor indexCollection) + public void Initialize(IRuntimeState runtime, PropertyEditorCollection propertyEditors) { var fileUpload = propertyEditors.OfType().FirstOrDefault(); if (fileUpload != null) Initialize(fileUpload); diff --git a/src/Umbraco.Web/PublishedCache/XmlPublishedCache/PublishedMediaCache.cs b/src/Umbraco.Web/PublishedCache/XmlPublishedCache/PublishedMediaCache.cs index df4b7990ad..5391c890a3 100644 --- a/src/Umbraco.Web/PublishedCache/XmlPublishedCache/PublishedMediaCache.cs +++ b/src/Umbraco.Web/PublishedCache/XmlPublishedCache/PublishedMediaCache.cs @@ -40,8 +40,8 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache // when they are null the cache derives them from the ExamineManager, see // method GetExamineManagerSafe(). // - private readonly ILuceneSearcher _searchProvider; - private readonly BaseIndexProvider _indexProvider; + private readonly ISearcher _searchProvider; + private readonly IIndexer _indexProvider; private readonly XmlStore _xmlStore; private readonly PublishedContentTypeCache _contentTypeCache; @@ -51,10 +51,8 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache public PublishedMediaCache(XmlStore xmlStore, IMediaService mediaService, IUserService userService, ICacheProvider cacheProvider, PublishedContentTypeCache contentTypeCache) : base(false) { - if (mediaService == null) throw new ArgumentNullException(nameof(mediaService)); - if (userService == null) throw new ArgumentNullException(nameof(userService)); - _mediaService = mediaService; - _userService = userService; + _mediaService = mediaService ?? throw new ArgumentNullException(nameof(mediaService)); + _userService = userService ?? throw new ArgumentNullException(nameof(userService)); _cacheProvider = cacheProvider; _xmlStore = xmlStore; @@ -70,18 +68,13 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache /// /// /// - internal PublishedMediaCache(IMediaService mediaService, IUserService userService, ILuceneSearcher searchProvider, BaseIndexProvider indexProvider, ICacheProvider cacheProvider, PublishedContentTypeCache contentTypeCache) + internal PublishedMediaCache(IMediaService mediaService, IUserService userService, ISearcher searchProvider, BaseIndexProvider indexProvider, ICacheProvider cacheProvider, PublishedContentTypeCache contentTypeCache) : base(false) { - if (mediaService == null) throw new ArgumentNullException(nameof(mediaService)); - if (userService == null) throw new ArgumentNullException(nameof(userService)); - if (searchProvider == null) throw new ArgumentNullException(nameof(searchProvider)); - if (indexProvider == null) throw new ArgumentNullException(nameof(indexProvider)); - - _mediaService = mediaService; - _userService = userService; - _searchProvider = searchProvider; - _indexProvider = indexProvider; + _mediaService = mediaService ?? throw new ArgumentNullException(nameof(mediaService)); + _userService = userService ?? throw new ArgumentNullException(nameof(userService)); + _searchProvider = searchProvider ?? throw new ArgumentNullException(nameof(searchProvider)); + _indexProvider = indexProvider ?? throw new ArgumentNullException(nameof(indexProvider)); _cacheProvider = cacheProvider; _contentTypeCache = contentTypeCache; } @@ -117,8 +110,8 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache // first check in Examine for the cache values // +(+parentID:-1) +__IndexType:media - var criteria = searchProvider.CreateSearchCriteria("media"); - var filter = criteria.ParentId(-1).Not().Field(UmbracoContentIndexer.IndexPathFieldName, "-1,-21,".MultipleCharacterWildcard()); + var criteria = searchProvider.CreateCriteria("media"); + var filter = criteria.ParentId(-1).Not().Field(UmbracoExamineIndexer.IndexPathFieldName, "-1,-21,".MultipleCharacterWildcard()); var result = searchProvider.Search(filter.Compile()); if (result != null) @@ -228,7 +221,7 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache public override bool HasContent(bool preview) { throw new NotImplementedException(); } - private static ExamineManager GetExamineManagerSafe() + private static IExamineManager GetExamineManagerSafe() { try { @@ -239,34 +232,8 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache return null; } } - - private BaseIndexProvider GetIndexProviderSafe() - { - if (_indexProvider != null) - return _indexProvider; - - var eMgr = GetExamineManagerSafe(); - if (eMgr == null) return null; - - try - { - //by default use the InternalSearcher - var indexer = eMgr.IndexProviderCollection[Constants.Examine.InternalIndexer]; - if (indexer.IndexerData.IncludeNodeTypes.Any() || indexer.IndexerData.ExcludeNodeTypes.Any()) - { - Current.Logger.Warn("The InternalIndexer for examine is configured incorrectly, it should not list any include/exclude node types or field names, it should simply be configured as: " + ""); - } - return indexer; - } - catch (Exception ex) - { - Current.Logger.Error("Could not retrieve the InternalIndexer", ex); - //something didn't work, continue returning null. - } - return null; - } - - private ILuceneSearcher GetSearchProviderSafe() + + private ISearcher GetSearchProviderSafe() { if (_searchProvider != null) return _searchProvider; @@ -276,7 +243,7 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache try { - //by default use the InternalSearcher + //by default use the internal index return eMgr.GetSearcher(Constants.Examine.InternalIndexer); } catch (FileNotFoundException) @@ -328,8 +295,8 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache // // note that since the use of the wildcard, it automatically escapes it in Lucene. - var criteria = searchProvider.CreateSearchCriteria("media"); - var filter = criteria.Id(id).Not().Field(BaseUmbracoIndexer.IndexPathFieldName, "-1,-21,".MultipleCharacterWildcard()); + var criteria = searchProvider.CreateCriteria("media"); + var filter = criteria.Id(id.ToInvariantString()).Not().Field(UmbracoExamineIndexer.IndexPathFieldName, "-1,-21,".MultipleCharacterWildcard()); var result = searchProvider.Search(filter.Compile()).FirstOrDefault(); if (result != null) return ConvertFromSearchResult(result); @@ -395,11 +362,10 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache internal CacheValues ConvertFromSearchResult(SearchResult searchResult) { // note: fixing fields in 7.x, removed by Shan for 8.0 - var values = new Dictionary(searchResult.Fields); - + return new CacheValues { - Values = values, + Values = searchResult.Fields, FromExamine = true }; } @@ -515,7 +481,7 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache { //We are going to check for a special field however, that is because in some cases we store a 'Raw' //value in the index such as for xml/html. - var rawValue = dd.Properties.FirstOrDefault(x => x.Alias.InvariantEquals(BaseUmbracoIndexer.RawFieldPrefix + alias)); + var rawValue = dd.Properties.FirstOrDefault(x => x.Alias.InvariantEquals(UmbracoExamineIndexer.RawFieldPrefix + alias)); return rawValue ?? dd.Properties.FirstOrDefault(x => x.Alias.InvariantEquals(alias)); } @@ -545,12 +511,12 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache //first check in Examine as this is WAY faster var criteria = searchProvider.CreateCriteria("media"); - var filter = criteria.ParentId(parentId).Not().Field(BaseUmbracoIndexer.IndexPathFieldName, "-1,-21,".MultipleCharacterWildcard()); + var filter = criteria.ParentId(parentId).Not().Field(UmbracoExamineIndexer.IndexPathFieldName, "-1,-21,".MultipleCharacterWildcard()); //the above filter will create a query like this, NOTE: That since the use of the wildcard, it automatically escapes it in Lucene. //+(+parentId:3113 -__Path:-1,-21,*) +__IndexType:media // sort with the Sort field (updated for 8.0) - var results = searchProvider.Find( + var results = searchProvider.Search( filter.And().OrderBy(new SortableField("sortOrder", SortType.Int)).Compile()); if (results.Any()) @@ -661,7 +627,7 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache private static readonly string[] IgnoredKeys = { "version", "isDoc" }; public DictionaryPublishedContent( - IDictionary valueDictionary, + IReadOnlyDictionary valueDictionary, Func getParent, Func> getChildren, Func getProperty, @@ -687,7 +653,7 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache ValidateAndSetProperty(valueDictionary, val => _sortOrder = int.Parse(val), "sortOrder"); ValidateAndSetProperty(valueDictionary, val => _name = val, "nodeName", "__nodeName"); ValidateAndSetProperty(valueDictionary, val => _urlName = val, "urlName"); - ValidateAndSetProperty(valueDictionary, val => _documentTypeAlias = val, "nodeTypeAlias", LuceneIndexer.NodeTypeAliasFieldName); + ValidateAndSetProperty(valueDictionary, val => _documentTypeAlias = val, "nodeTypeAlias", LuceneIndexer.ItemTypeFieldName); ValidateAndSetProperty(valueDictionary, val => _documentTypeId = int.Parse(val), "nodeType"); //ValidateAndSetProperty(valueDictionary, val => _writerName = val, "writerName"); ValidateAndSetProperty(valueDictionary, val => _creatorName = val, "creatorName", "writerName"); //this is a bit of a hack fix for: U4-1132 @@ -852,7 +818,7 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache private readonly ICollection _properties; private readonly PublishedContentType _contentType; - private void ValidateAndSetProperty(IDictionary valueDictionary, Action setProperty, params string[] potentialKeys) + private void ValidateAndSetProperty(IReadOnlyDictionary valueDictionary, Action setProperty, params string[] potentialKeys) { var key = potentialKeys.FirstOrDefault(x => valueDictionary.ContainsKey(x) && valueDictionary[x] != null); if (key == null) @@ -901,7 +867,7 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache internal class CacheValues { - public IDictionary Values { get; set; } + public IReadOnlyDictionary Values { get; set; } public XPathNavigator XPath { get; set; } public bool FromExamine { get; set; } } @@ -976,7 +942,7 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache GetValuesValue(v.Values, "path", "__Path").Contains(fid)); } - private static string GetValuesValue(IDictionary d, params string[] keys) + private static string GetValuesValue(IReadOnlyDictionary d, params string[] keys) { string value = null; var ignored = keys.Any(x => d.TryGetValue(x, out value)); diff --git a/src/Umbraco.Web/PublishedContentExtensions.cs b/src/Umbraco.Web/PublishedContentExtensions.cs index be51c36931..7bdd196f20 100644 --- a/src/Umbraco.Web/PublishedContentExtensions.cs +++ b/src/Umbraco.Web/PublishedContentExtensions.cs @@ -4,6 +4,7 @@ using System.Data; using System.Globalization; using System.Linq; using System.Web; +using Examine; using Examine.LuceneEngine.SearchCriteria; using Umbraco.Core.Models.PublishedContent; using Umbraco.Core; @@ -220,49 +221,59 @@ namespace Umbraco.Web #region Search - public static IEnumerable Search(this IPublishedContent content, string term, bool useWildCards = true, string searchProvider = null) - { - var searcher = Examine.ExamineManager.Instance.DefaultSearchProvider; - if (string.IsNullOrEmpty(searchProvider) == false) - searcher = Examine.ExamineManager.Instance.SearchProviderCollection[searchProvider]; + public static IEnumerable Search(this IPublishedContent content, string term, bool useWildCards = true, string indexName = null) + { + //TODO: we should pass in the IExamineManager? + + var searcher = string.IsNullOrEmpty(indexName) + ? Examine.ExamineManager.Instance.GetSearcher(Constants.Examine.ExternalIndexer) + : Examine.ExamineManager.Instance.GetSearcher(indexName); + + if (searcher == null) + throw new InvalidOperationException("No searcher found for index " + indexName); var t = term.Escape().Value; if (useWildCards) t = term.MultipleCharacterWildcard().Value; var luceneQuery = "+__Path:(" + content.Path.Replace("-", "\\-") + "*) +" + t; - var crit = searcher.CreateSearchCriteria().RawQuery(luceneQuery); + var crit = searcher.CreateCriteria().RawQuery(luceneQuery); return content.Search(crit, searcher); } - public static IEnumerable SearchDescendants(this IPublishedContent content, string term, bool useWildCards = true, string searchProvider = null) + public static IEnumerable SearchDescendants(this IPublishedContent content, string term, bool useWildCards = true, string indexName = null) { - return content.Search(term, useWildCards, searchProvider); + return content.Search(term, useWildCards, indexName); } - public static IEnumerable SearchChildren(this IPublishedContent content, string term, bool useWildCards = true, string searchProvider = null) + public static IEnumerable SearchChildren(this IPublishedContent content, string term, bool useWildCards = true, string indexName = null) { - var searcher = Examine.ExamineManager.Instance.DefaultSearchProvider; - if (string.IsNullOrEmpty(searchProvider) == false) - searcher = Examine.ExamineManager.Instance.SearchProviderCollection[searchProvider]; + //TODO: we should pass in the IExamineManager? + + var searcher = string.IsNullOrEmpty(indexName) + ? Examine.ExamineManager.Instance.GetSearcher(Constants.Examine.ExternalIndexer) + : Examine.ExamineManager.Instance.GetSearcher(indexName); + + if (searcher == null) + throw new InvalidOperationException("No searcher found for index " + indexName); var t = term.Escape().Value; if (useWildCards) t = term.MultipleCharacterWildcard().Value; var luceneQuery = "+parentID:" + content.Id + " +" + t; - var crit = searcher.CreateSearchCriteria().RawQuery(luceneQuery); + var crit = searcher.CreateCriteria().RawQuery(luceneQuery); return content.Search(crit, searcher); } - public static IEnumerable Search(this IPublishedContent content, Examine.SearchCriteria.ISearchCriteria criteria, Examine.Providers.BaseSearchProvider searchProvider = null) + public static IEnumerable Search(this IPublishedContent content, Examine.SearchCriteria.ISearchCriteria criteria, Examine.ISearcher searchProvider = null) { - var s = Examine.ExamineManager.Instance.DefaultSearchProvider; - if (searchProvider != null) - s = searchProvider; + //TODO: we should pass in the IExamineManager? + var s = searchProvider ?? Examine.ExamineManager.Instance.GetSearcher(Constants.Examine.ExternalIndexer); + var results = s.Search(criteria); return results.ToPublishedSearchResults(UmbracoContext.Current.ContentCache); } diff --git a/src/Umbraco.Web/PublishedContentQuery.cs b/src/Umbraco.Web/PublishedContentQuery.cs index fef1f55e0f..6bb649f8cd 100644 --- a/src/Umbraco.Web/PublishedContentQuery.cs +++ b/src/Umbraco.Web/PublishedContentQuery.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Linq; using System.Reflection; using System.Xml.XPath; +using Examine; using Examine.LuceneEngine.Providers; using Examine.LuceneEngine.SearchCriteria; using Examine.SearchCriteria; @@ -220,19 +221,25 @@ namespace Umbraco.Web #region Search /// - public IEnumerable Search(string term, bool useWildCards = true, string searchProvider = null) + public IEnumerable Search(string term, bool useWildCards = true, string indexName = null) { - return Search(0, 0, out _, term, useWildCards, searchProvider); + return Search(0, 0, out _, term, useWildCards, indexName); } /// - public IEnumerable Search(int skip, int take, out int totalRecords, string term, bool useWildCards = true, string searchProvider = null) + public IEnumerable Search(int skip, int take, out int totalRecords, string term, bool useWildCards = true, string indexName = null) { - if (_query != null) return _query.Search(skip, take, out totalRecords, term, useWildCards, searchProvider); + //TODO: Can we inject IExamineManager? - var searcher = string.IsNullOrWhiteSpace(searchProvider) - ? Examine.ExamineManager.Instance.DefaultSearchProvider - : Examine.ExamineManager.Instance.SearchProviderCollection[searchProvider]; + if (_query != null) return _query.Search(skip, take, out totalRecords, term, useWildCards, indexName); + + var indexer = string.IsNullOrEmpty(indexName) + ? Examine.ExamineManager.Instance.GetIndexer(Constants.Examine.ExternalIndexer) + : Examine.ExamineManager.Instance.GetIndexer(indexName); + + if (indexer == null) throw new InvalidOperationException("No index found by name " + indexName); + + var searcher = indexer.GetSearcher(); if (skip == 0 && take == 0) { @@ -241,30 +248,24 @@ namespace Umbraco.Web return results.ToPublishedSearchResults(_contentCache); } - if (!(searcher is BaseLuceneSearcher luceneSearcher)) - { - var results = searcher.Search(term, useWildCards); - totalRecords = results.TotalItemCount; - // Examine skip, Linq take - return results.Skip(skip).ToPublishedSearchResults(_contentCache).Take(take); - } - - var criteria = SearchAllFields(term, useWildCards, luceneSearcher); + var criteria = SearchAllFields(term, useWildCards, searcher, indexer); return Search(skip, take, out totalRecords, criteria, searcher); } /// - public IEnumerable Search(Examine.SearchCriteria.ISearchCriteria criteria, Examine.Providers.BaseSearchProvider searchProvider = null) + public IEnumerable Search(ISearchCriteria criteria, Examine.ISearcher searchProvider = null) { return Search(0, 0, out _, criteria, searchProvider); } /// - public IEnumerable Search(int skip, int take, out int totalRecords, Examine.SearchCriteria.ISearchCriteria criteria, Examine.Providers.BaseSearchProvider searchProvider = null) + public IEnumerable Search(int skip, int take, out int totalRecords, ISearchCriteria criteria, Examine.ISearcher searchProvider = null) { if (_query != null) return _query.Search(skip, take, out totalRecords, criteria, searchProvider); - var searcher = searchProvider ?? Examine.ExamineManager.Instance.DefaultSearchProvider; + //TODO: Can we inject IExamineManager? + + var searcher = searchProvider ?? Examine.ExamineManager.Instance.GetSearcher(Constants.Examine.ExternalIndexer); var results = skip == 0 && take == 0 ? searcher.Search(criteria) @@ -277,45 +278,25 @@ namespace Umbraco.Web /// /// Creates an ISearchCriteria for searching all fields in a . /// - /// - /// This is here because some of this stuff is internal in Examine. - /// - private ISearchCriteria SearchAllFields(string searchText, bool useWildcards, BaseLuceneSearcher searcher) + private ISearchCriteria SearchAllFields(string searchText, bool useWildcards, Examine.ISearcher searcher, Examine.IIndexer indexer) { - var sc = searcher.CreateSearchCriteria(); + var sc = searcher.CreateCriteria(); - if (_examineGetSearchFields == null) - { - //get the GetSearchFields method from BaseLuceneSearcher - _examineGetSearchFields = typeof(BaseLuceneSearcher).GetMethod("GetSearchFields", BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Static); - } - - //get the results of searcher.BaseLuceneSearcher() using ugly reflection since it's not public - var searchFields = (IEnumerable) _examineGetSearchFields.Invoke(searcher, null); + //if we're dealing with a lucene searcher, we can get all of it's indexed fields, + //else we can get the defined fields for the index. + var searchFields = (searcher is BaseLuceneSearcher luceneSearcher) + ? luceneSearcher.GetAllIndexedFields() + : indexer.FieldDefinitionCollection.Keys; //this is what Examine does internally to create ISearchCriteria for searching all fields var strArray = searchText.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries); + sc = useWildcards == false ? sc.GroupedOr(searchFields, strArray).Compile() - : sc.GroupedOr(searchFields, strArray.Select(x => new CustomExamineValue(Examineness.ComplexWildcard, x.MultipleCharacterWildcard().Value)).ToArray()).Compile(); + : sc.GroupedOr(searchFields, strArray.Select(x => (IExamineValue)new ExamineValue(Examineness.ComplexWildcard, x.MultipleCharacterWildcard().Value)).ToArray()).Compile(); return sc; - } + } - private static MethodInfo _examineGetSearchFields; - - //support class since Examine doesn't expose it's own ExamineValue class publicly - private class CustomExamineValue : IExamineValue - { - public CustomExamineValue(Examineness vagueness, string value) - { - this.Examineness = vagueness; - this.Value = value; - this.Level = 1f; - } - public Examineness Examineness { get; private set; } - public string Value { get; private set; } - public float Level { get; private set; } - } #endregion } diff --git a/src/Umbraco.Web/Runtime/WebRuntimeComponent.cs b/src/Umbraco.Web/Runtime/WebRuntimeComponent.cs index 519e356627..2913986500 100644 --- a/src/Umbraco.Web/Runtime/WebRuntimeComponent.cs +++ b/src/Umbraco.Web/Runtime/WebRuntimeComponent.cs @@ -12,6 +12,7 @@ using System.Web.Mvc; using System.Web.Routing; using ClientDependency.Core.CompositeFiles.Providers; using ClientDependency.Core.Config; +using Examine; using LightInject; using Microsoft.AspNet.SignalR; using Umbraco.Core; @@ -99,7 +100,7 @@ namespace Umbraco.Web.Runtime composition.Container.RegisterSingleton(); composition.Container.RegisterSingleton(); - composition.Container.RegisterSingleton(); + composition.Container.RegisterSingleton(factory => ExamineManager.Instance); // IoC setup for LightInject for MVC/WebApi // see comments on MixedLightInjectScopeManagerProvider for explainations of what we are doing here diff --git a/src/Umbraco.Web/Search/ExamineComponent.cs b/src/Umbraco.Web/Search/ExamineComponent.cs index 50940ecf4c..278f124344 100644 --- a/src/Umbraco.Web/Search/ExamineComponent.cs +++ b/src/Umbraco.Web/Search/ExamineComponent.cs @@ -1,4 +1,5 @@ using System; +using System.Collections; using System.Collections.Generic; using System.Globalization; using System.Linq; @@ -6,8 +7,9 @@ using System.Threading; using System.Xml.Linq; using Examine; using Examine.LuceneEngine; -using Examine.Session; +using Examine.LuceneEngine.Providers; using Lucene.Net.Documents; +using Lucene.Net.Index; using Umbraco.Core; using Umbraco.Core.Cache; using Umbraco.Core.Components; @@ -18,6 +20,7 @@ using Umbraco.Core.Scoping; using Umbraco.Core.Services; using Umbraco.Core.Services.Changes; using Umbraco.Core.Services.Implement; +using Umbraco.Core.Strings; using Umbraco.Core.Sync; using Umbraco.Web.Cache; using Umbraco.Web.Composing; @@ -32,64 +35,70 @@ namespace Umbraco.Web.Search [RuntimeLevel(MinLevel = RuntimeLevel.Run)] public sealed class ExamineComponent : UmbracoComponentBase, IUmbracoCoreComponent { + private IExamineManager _examineManager; + private static bool _disableExamineIndexing = false; + private static volatile bool _isConfigured = false; + private static readonly object IsConfiguredLocker = new object(); private IScopeProvider _scopeProvider; + private UrlSegmentProviderCollection _urlSegmentProviders; + private ServiceContext _services; // the default enlist priority is 100 // enlist with a lower priority to ensure that anything "default" runs after us // but greater that SafeXmlReaderWriter priority which is 60 private const int EnlistPriority = 80; - public void Initialize(IRuntimeState runtime, PropertyEditorCollection propertyEditors, IExamineIndexCollectionAccessor indexCollection, IScopeProvider scopeProvider, ILogger logger) + internal void Initialize(IRuntimeState runtime, MainDom mainDom, PropertyEditorCollection propertyEditors, IExamineManager examineManager, ProfilingLogger profilingLogger, IScopeProvider scopeProvider, UrlSegmentProviderCollection urlSegmentProviderCollection, ServiceContext services) { + _services = services; + _urlSegmentProviders = urlSegmentProviderCollection; _scopeProvider = scopeProvider; + _examineManager = examineManager; - logger.Info("Starting initialize async background thread."); + //We want to manage Examine's appdomain shutdown sequence ourselves so first we'll disable Examine's default behavior + //and then we'll use MainDom to control Examine's shutdown + ExamineManager.DisableDefaultHostingEnvironmentRegistration(); - // make it async in order not to slow down the boot - // fixme - should be a proper background task else we cannot stop it! - var bg = new Thread(() => + //we want to tell examine to use a different fs lock instead of the default NativeFSFileLock which could cause problems if the appdomain + //terminates and in some rare cases would only allow unlocking of the file if IIS is forcefully terminated. Instead we'll rely on the simplefslock + //which simply checks the existence of the lock file + DirectoryTracker.DefaultLockFactory = d => { - try + var simpleFsLockFactory = new NoPrefixSimpleFsLockFactory(d); + return simpleFsLockFactory; + }; + + //let's deal with shutting down Examine with MainDom + var examineShutdownRegistered = mainDom.Register(() => + { + using (profilingLogger.TraceDuration("Examine shutting down")) { - // from WebRuntimeComponent - // rebuilds any empty indexes - RebuildIndexes(true); - } - catch (Exception e) - { - logger.Error("Failed to rebuild empty indexes.", e); - } - - try - { - // from PropertyEditorsComponent - var grid = propertyEditors.OfType().FirstOrDefault(); - if (grid != null) BindGridToExamine(grid, indexCollection); - } - catch (Exception e) - { - logger.Error("Failed to bind grid property editor.", e); + ExamineManager.Instance.Dispose(); } }); - bg.Start(); - // the rest is the original Examine event handler + if (!examineShutdownRegistered) + { + profilingLogger.Logger.Debug("Examine shutdown not registered, this appdomain is not the MainDom, Examine will be disabled"); - logger.Info("Initialize and bind to business logic events."); + //if we could not register the shutdown examine ourselves, it means we are not maindom! in this case all of examine should be disabled! + Suspendable.ExamineEvents.SuspendIndexers(); + _disableExamineIndexing = true; + return; //exit, do not continue + } - //TODO: For now we'll make this true, it means that indexes will be near real time - // we'll see about what implications this may have - should be great in most scenarios - DefaultExamineSession.RequireImmediateConsistency = true; + profilingLogger.Logger.Debug("Examine shutdown registered with MainDom"); - var registeredProviders = ExamineManager.Instance.IndexProviderCollection - .OfType().Count(x => x.EnableDefaultEventHandler); + var registeredIndexers = examineManager.IndexProviders.Values.OfType().Count(x => x.EnableDefaultEventHandler); - logger.Info($"Adding examine event handlers for {registeredProviders} index providers."); + profilingLogger.Logger.Info($"Adding examine event handlers for {registeredIndexers} index providers."); // don't bind event handlers if we're not suppose to listen - if (registeredProviders == 0) + if (registeredIndexers == 0) return; + BindGridToExamine(profilingLogger.Logger, examineManager, propertyEditors); + // bind to distributed cache events - this ensures that this logic occurs on ALL servers // that are taking part in a load balanced environment. ContentCacheRefresher.CacheUpdated += ContentCacheRefresherUpdated; @@ -100,36 +109,123 @@ namespace Umbraco.Web.Search // events handling removed in ef013f9d3b945d0a48a306ff1afbd49c10c3fff8 // because, could not make sense of it? - var contentIndexer = ExamineManager.Instance.IndexProviderCollection[Constants.Examine.InternalIndexer] as UmbracoContentIndexer; - if (contentIndexer != null) - { - contentIndexer.DocumentWriting += IndexerDocumentWriting; - } - var memberIndexer = ExamineManager.Instance.IndexProviderCollection[Constants.Examine.InternalMemberIndexer] as UmbracoMemberIndexer; - if (memberIndexer != null) - { - memberIndexer.DocumentWriting += IndexerDocumentWriting; - } + EnsureUnlocked(profilingLogger.Logger, examineManager); + + RebuildIndexesOnStartup(profilingLogger.Logger); } - private static void RebuildIndexes(bool onlyEmptyIndexes) + /// + /// Called to rebuild empty indexes on startup + /// + /// + private void RebuildIndexesOnStartup(ILogger logger) { - var indexers = (IEnumerable>)ExamineManager.Instance.IndexProviders; + //TODO: need a way to disable rebuilding on startup + + logger.Info("Starting initialize async background thread."); + + // make it async in order not to slow down the boot + // fixme - should be a proper background task else we cannot stop it! + var bg = new Thread(() => + { + try + { + // rebuilds any empty indexes + RebuildIndexes(true, _examineManager, logger); + } + catch (Exception e) + { + logger.Error("Failed to rebuild empty indexes.", e); + } + }); + bg.Start(); + } + + /// + /// Used to rebuild indexes on startup or cold boot + /// + /// + /// + /// + internal static void RebuildIndexes(bool onlyEmptyIndexes, IExamineManager examineManager, ILogger logger) + { + //do not attempt to do this if this has been disabled since we are not the main dom. + //this can be called during a cold boot + if (_disableExamineIndexing) return; + + EnsureUnlocked(logger, examineManager); + if (onlyEmptyIndexes) - indexers = indexers.Where(x => x.Value.IsIndexNew()); - foreach (var indexer in indexers) - indexer.Value.RebuildIndex(); + { + foreach (var indexer in examineManager.IndexProviders.Values.Where(x => x.IsIndexNew())) + { + indexer.RebuildIndex(); + } + } + else + { + //do all of them + ExamineManager.Instance.RebuildIndexes(); + } } - private static void BindGridToExamine(GridPropertyEditor grid, IExamineIndexCollectionAccessor indexCollection) + /// + /// Must be called to each index is unlocked before any indexing occurs + /// + /// + /// Indexing rebuilding can occur on a normal boot if the indexes are empty or on a cold boot by the database server messenger. Before + /// either of these happens, we need to configure the indexes. + /// + private static void EnsureUnlocked(ILogger logger, IExamineManager examineManager) { - var indexes = indexCollection.Indexes; - if (indexes == null) return; - foreach (var i in indexes.Values.OfType()) - i.DocumentWriting += grid.DocumentWriting; + if (_disableExamineIndexing) return; + if (_isConfigured) return; + + lock (IsConfiguredLocker) + { + //double chekc + if (_isConfigured) return; + + _isConfigured = true; + + foreach (var luceneIndexer in examineManager.IndexProviders.Values.OfType()) + { + //We now need to disable waiting for indexing for Examine so that the appdomain is shutdown immediately and doesn't wait for pending + //indexing operations. We used to wait for indexing operations to complete but this can cause more problems than that is worth because + //that could end up halting shutdown for a very long time causing overlapping appdomains and many other problems. + luceneIndexer.WaitForIndexQueueOnShutdown = false; + + //we should check if the index is locked ... it shouldn't be! We are using simple fs lock now and we are also ensuring that + //the indexes are not operational unless MainDom is true + var dir = luceneIndexer.GetLuceneDirectory(); + if (IndexWriter.IsLocked(dir)) + { + logger.Info("Forcing index " + luceneIndexer.Name + " to be unlocked since it was left in a locked state"); + IndexWriter.Unlock(dir); + } + } + } } - void MemberCacheRefresherUpdated(MemberCacheRefresher sender, CacheRefresherEventArgs args) + private static void BindGridToExamine(ILogger logger, IExamineManager examineManager, IEnumerable propertyEditors) + { + //bind the grid property editors - this is a hack until http://issues.umbraco.org/issue/U4-8437 + try + { + var grid = propertyEditors.OfType().FirstOrDefault(); + if (grid != null) + { + foreach (var i in examineManager.IndexProviders.Values.OfType()) + i.DocumentWriting += grid.DocumentWriting; + } + } + catch (Exception e) + { + logger.Error("Failed to bind grid property editor.", e); + } + } + + private void MemberCacheRefresherUpdated(MemberCacheRefresher sender, CacheRefresherEventArgs args) { if (Suspendable.ExamineEvents.CanIndex == false) return; @@ -137,7 +233,7 @@ namespace Umbraco.Web.Search switch (args.MessageType) { case MessageType.RefreshById: - var c1 = Current.Services.MemberService.GetById((int)args.MessageObject); + var c1 = _services.MemberService.GetById((int)args.MessageObject); if (c1 != null) { ReIndexForMember(c1); @@ -150,8 +246,7 @@ namespace Umbraco.Web.Search DeleteIndexForEntity((int)args.MessageObject, false); break; case MessageType.RefreshByInstance: - var c3 = args.MessageObject as IMember; - if (c3 != null) + if (args.MessageObject is IMember c3) { ReIndexForMember(c3); } @@ -160,8 +255,7 @@ namespace Umbraco.Web.Search // This is triggered when the item is permanently deleted - var c4 = args.MessageObject as IMember; - if (c4 != null) + if (args.MessageObject is IMember c4) { DeleteIndexForEntity(c4.Id, false); } @@ -174,7 +268,7 @@ namespace Umbraco.Web.Search } } - void MediaCacheRefresherUpdated(MediaCacheRefresher sender, CacheRefresherEventArgs args) + private void MediaCacheRefresherUpdated(MediaCacheRefresher sender, CacheRefresherEventArgs args) { if (Suspendable.ExamineEvents.CanIndex == false) return; @@ -182,7 +276,7 @@ namespace Umbraco.Web.Search if (args.MessageType != MessageType.RefreshByPayload) throw new NotSupportedException(); - var mediaService = Current.Services.MediaService; + var mediaService = _services.MediaService; foreach (var payload in (MediaCacheRefresher.JsonPayload[]) args.MessageObject) { @@ -223,7 +317,7 @@ namespace Umbraco.Web.Search } } - void ContentCacheRefresherUpdated(ContentCacheRefresher sender, CacheRefresherEventArgs args) + private void ContentCacheRefresherUpdated(ContentCacheRefresher sender, CacheRefresherEventArgs args) { if (Suspendable.ExamineEvents.CanIndex == false) return; @@ -231,7 +325,7 @@ namespace Umbraco.Web.Search if (args.MessageType != MessageType.RefreshByPayload) throw new NotSupportedException(); - var contentService = Current.Services.ContentService; + var contentService = _services.ContentService; foreach (var payload in (ContentCacheRefresher.JsonPayload[]) args.MessageObject) { @@ -296,6 +390,8 @@ namespace Umbraco.Web.Search // ReIndexForContent is NOT taking care of descendants so we have to reload everything // again in order to process the branch - we COULD improve that by just reloading the // XML from database instead of reloading content & re-serializing! + // + // BUT ... pretty sure it is! see test "Index_Delete_Index_Item_Ensure_Heirarchy_Removed" } } @@ -325,26 +421,27 @@ namespace Umbraco.Web.Search { var actions = DeferedActions.Get(_scopeProvider); if (actions != null) - actions.Add(new DeferedReIndexForContent(sender, supportUnpublished)); + actions.Add(new DeferedReIndexForContent(this, sender, supportUnpublished)); else - DeferedReIndexForContent.Execute(sender, supportUnpublished); + DeferedReIndexForContent.Execute(this, sender, supportUnpublished); } private void ReIndexForMember(IMember member) { var actions = DeferedActions.Get(_scopeProvider); if (actions != null) - actions.Add(new DeferedReIndexForMember(member)); + actions.Add(new DeferedReIndexForMember(this, member)); else - DeferedReIndexForMember.Execute(member); } + DeferedReIndexForMember.Execute(this, member); + } private void ReIndexForMedia(IMedia sender, bool isMediaPublished) { var actions = DeferedActions.Get(_scopeProvider); if (actions != null) - actions.Add(new DeferedReIndexForMedia(sender, isMediaPublished)); + actions.Add(new DeferedReIndexForMedia(this, sender, isMediaPublished)); else - DeferedReIndexForMedia.Execute(sender, isMediaPublished); + DeferedReIndexForMedia.Execute(this, sender, isMediaPublished); } /// @@ -359,189 +456,165 @@ namespace Umbraco.Web.Search { var actions = DeferedActions.Get(_scopeProvider); if (actions != null) - actions.Add(new DeferedDeleteIndex(entityId, keepIfUnpublished)); + actions.Add(new DeferedDeleteIndex(this, entityId, keepIfUnpublished)); else - DeferedDeleteIndex.Execute(entityId, keepIfUnpublished); - } - - /// - /// Event handler to create a lower cased version of the node name, this is so we can support case-insensitive searching and still - /// use the Whitespace Analyzer - /// - /// - /// - - private static void IndexerDocumentWriting(object sender, DocumentWritingEventArgs e) - { - if (e.Fields.Keys.Contains("nodeName")) - { - //TODO: This logic should really be put into the content indexer instead of hidden here!! - - //add the lower cased version - e.Document.Add(new Field("__nodeName", - e.Fields["nodeName"].ToLower(), - Field.Store.YES, - Field.Index.ANALYZED, - Field.TermVector.NO - )); - } + DeferedDeleteIndex.Execute(this, entityId, keepIfUnpublished); } private class DeferedActions - { - private readonly List _actions = new List(); + { + private readonly List _actions = new List(); - public static DeferedActions Get(IScopeProvider scopeProvider) - { - var scopeContext = scopeProvider.Context; - if (scopeContext == null) return null; + public static DeferedActions Get(IScopeProvider scopeProvider) + { + var scopeContext = scopeProvider.Context; - return scopeContext.Enlist("examineEvents", - () => new DeferedActions(), // creator - (completed, actions) => // action - { - if (completed) actions.Execute(); - }, EnlistPriority); - } + return scopeContext?.Enlist("examineEvents", + () => new DeferedActions(), // creator + (completed, actions) => // action + { + if (completed) actions.Execute(); + }, EnlistPriority); + } - public void Add(DeferedAction action) - { + public void Add(DeferedAction action) + { _actions.Add(action); - } + } - private void Execute() - { - foreach (var action in _actions) - action.Execute(); - } - } + private void Execute() + { + foreach (var action in _actions) + action.Execute(); + } + } - private abstract class DeferedAction - { + private abstract class DeferedAction + { public virtual void Execute() { } - } + } - private class DeferedReIndexForContent : DeferedAction - { - private readonly IContent _content; - private readonly bool? _supportUnpublished; + private class DeferedReIndexForContent : DeferedAction + { + private readonly ExamineComponent _examineComponent; + private readonly IContent _content; + private readonly bool? _supportUnpublished; - public DeferedReIndexForContent(IContent content, bool? supportUnpublished) - { - _content = content; - _supportUnpublished = supportUnpublished; - } + public DeferedReIndexForContent(ExamineComponent examineComponent, IContent content, bool? supportUnpublished) + { + _examineComponent = examineComponent; + _content = content; + _supportUnpublished = supportUnpublished; + } - public override void Execute() - { - Execute(_content, _supportUnpublished); - } + public override void Execute() + { + Execute(_examineComponent, _content, _supportUnpublished); + } - public static void Execute(IContent content, bool? supportUnpublished) - { - var xml = content.ToXml(); - //add an icon attribute to get indexed - xml.Add(new XAttribute("icon", content.ContentType.Icon)); + public static void Execute(ExamineComponent examineComponent, IContent content, bool? supportUnpublished) + { + var valueSet = UmbracoContentIndexer.GetValueSets(examineComponent._urlSegmentProviders, examineComponent._services.UserService, content); - ExamineManager.Instance.ReIndexNode( - xml, IndexTypes.Content, - ExamineManager.Instance.IndexProviderCollection.OfType() + ExamineManager.Instance.IndexItems( + valueSet.ToArray(), + examineComponent._examineManager.IndexProviders.Values.OfType() + // only for the specified indexers + .Where(x => supportUnpublished.HasValue == false || supportUnpublished.Value == x.SupportUnpublishedContent) + .Where(x => x.EnableDefaultEventHandler)); + } + } - //Index this item for all indexers if the content is published, otherwise if the item is not published - // then only index this for indexers supporting unpublished content + private class DeferedReIndexForMedia : DeferedAction + { + private readonly ExamineComponent _examineComponent; + private readonly IMedia _media; + private readonly bool _isPublished; - .Where(x => supportUnpublished.HasValue == false || supportUnpublished.Value == x.SupportUnpublishedContent) - .Where(x => x.EnableDefaultEventHandler)); - } - } + public DeferedReIndexForMedia(ExamineComponent examineComponent, IMedia media, bool isPublished) + { + _examineComponent = examineComponent; + _media = media; + _isPublished = isPublished; + } - private class DeferedReIndexForMedia : DeferedAction - { - private readonly IMedia _media; - private readonly bool _isPublished; + public override void Execute() + { + Execute(_examineComponent, _media, _isPublished); + } - public DeferedReIndexForMedia(IMedia media, bool isPublished) - { - _media = media; - _isPublished = isPublished; - } + public static void Execute(ExamineComponent examineComponent, IMedia media, bool isPublished) + { + var valueSet = UmbracoContentIndexer.GetValueSets(examineComponent._urlSegmentProviders, examineComponent._services.UserService, media); - public override void Execute() - { - Execute(_media, _isPublished); - } + ExamineManager.Instance.IndexItems( + valueSet.ToArray(), + examineComponent._examineManager.IndexProviders.Values.OfType() + // index this item for all indexers if the media is not trashed, otherwise if the item is trashed + // then only index this for indexers supporting unpublished media + .Where(x => isPublished || (x.SupportUnpublishedContent)) + .Where(x => x.EnableDefaultEventHandler)); + } + } - public static void Execute(IMedia media, bool isPublished) - { - var xml = media.ToXml(); - //add an icon attribute to get indexed - xml.Add(new XAttribute("icon", media.ContentType.Icon)); + private class DeferedReIndexForMember : DeferedAction + { + private readonly ExamineComponent _examineComponent; + private readonly IMember _member; - ExamineManager.Instance.ReIndexNode( - xml, IndexTypes.Media, - ExamineManager.Instance.IndexProviderCollection.OfType() + public DeferedReIndexForMember(ExamineComponent examineComponent, IMember member) + { + _examineComponent = examineComponent; + _member = member; + } - //Index this item for all indexers if the media is not trashed, otherwise if the item is trashed - // then only index this for indexers supporting unpublished media + public override void Execute() + { + Execute(_examineComponent, _member); + } - .Where(x => isPublished || (x.SupportUnpublishedContent)) - .Where(x => x.EnableDefaultEventHandler)); - } - } + public static void Execute(ExamineComponent examineComponent, IMember member) + { + var valueSet = UmbracoMemberIndexer.GetValueSets(member); - private class DeferedReIndexForMember : DeferedAction - { - private readonly IMember _member; + ExamineManager.Instance.IndexItems( + valueSet.ToArray(), + examineComponent._examineManager.IndexProviders.Values.OfType() + //ensure that only the providers are flagged to listen execute + .Where(x => x.EnableDefaultEventHandler)); + } + } - public DeferedReIndexForMember(IMember member) - { - _member = member; - } + private class DeferedDeleteIndex : DeferedAction + { + private readonly ExamineComponent _examineComponent; + private readonly int _id; + private readonly bool _keepIfUnpublished; - public override void Execute() - { - Execute(_member); - } + public DeferedDeleteIndex(ExamineComponent examineComponent, int id, bool keepIfUnpublished) + { + _examineComponent = examineComponent; + _id = id; + _keepIfUnpublished = keepIfUnpublished; + } - public static void Execute(IMember member) - { - ExamineManager.Instance.ReIndexNode( - member.ToXml(), IndexTypes.Member, - ExamineManager.Instance.IndexProviderCollection.OfType() - //ensure that only the providers are flagged to listen execute - .Where(x => x.EnableDefaultEventHandler)); - } - } + public override void Execute() + { + Execute(_examineComponent, _id, _keepIfUnpublished); + } - private class DeferedDeleteIndex : DeferedAction - { - private readonly int _id; - private readonly bool _keepIfUnpublished; + public static void Execute(ExamineComponent examineComponent, int id, bool keepIfUnpublished) + { + ExamineManager.Instance.DeleteFromIndexes( + id.ToString(CultureInfo.InvariantCulture), + examineComponent._examineManager.IndexProviders.Values.OfType() + // if keepIfUnpublished == true then only delete this item from indexes not supporting unpublished content, + // otherwise if keepIfUnpublished == false then remove from all indexes + .Where(x => keepIfUnpublished == false || (x is UmbracoContentIndexer && ((UmbracoContentIndexer)x).SupportUnpublishedContent == false)) + .Where(x => x.EnableDefaultEventHandler)); + } + } - public DeferedDeleteIndex(int id, bool keepIfUnpublished) - { - _id = id; - _keepIfUnpublished = keepIfUnpublished; - } - - public override void Execute() - { - Execute(_id, _keepIfUnpublished); - } - - public static void Execute(int id, bool keepIfUnpublished) - { - ExamineManager.Instance.DeleteFromIndex( - id.ToString(CultureInfo.InvariantCulture), - ExamineManager.Instance.IndexProviderCollection.OfType() - - //if keepIfUnpublished == true then only delete this item from indexes not supporting unpublished content, - // otherwise if keepIfUnpublished == false then remove from all indexes - - .Where(x => keepIfUnpublished == false || x.SupportUnpublishedContent == false) - .Where(x => x.EnableDefaultEventHandler)); - } - } } } diff --git a/src/Umbraco.Web/Search/ExamineIndexerModel.cs b/src/Umbraco.Web/Search/ExamineIndexerModel.cs index c8cf2ff622..372f50764a 100644 --- a/src/Umbraco.Web/Search/ExamineIndexerModel.cs +++ b/src/Umbraco.Web/Search/ExamineIndexerModel.cs @@ -22,19 +22,7 @@ namespace Umbraco.Web.Search /// [DataMember(Name = "fieldCount")] public int FieldCount { get; set; } - - /// - /// The number of documents flagged for deletion in the index - /// - [DataMember(Name = "deletionCount")] - public int DeletionCount { get; set; } - - /// - /// Whether or not the indexed is optimized - /// - [DataMember(Name = "isOptimized")] - public bool IsOptimized{ get; set; } - + /// /// Generally will always be true unless someone has created a new non-lucene index /// diff --git a/src/Umbraco.Web/Search/LuceneIndexerExtensions.cs b/src/Umbraco.Web/Search/LuceneIndexerExtensions.cs index f333eca267..58f230b2cd 100644 --- a/src/Umbraco.Web/Search/LuceneIndexerExtensions.cs +++ b/src/Umbraco.Web/Search/LuceneIndexerExtensions.cs @@ -48,8 +48,9 @@ namespace Umbraco.Web.Search { try { - var searcher = indexer.GetSearcher().GetSearcher() as IndexSearcher; - if (searcher == null) return 0; + if (!((indexer.GetSearcher() as LuceneSearcher)?.GetLuceneSearcher() is IndexSearcher searcher)) + return 0; + using (searcher) using (var reader = searcher.IndexReader) { @@ -70,12 +71,11 @@ namespace Umbraco.Web.Search /// public static int GetIndexFieldCount(this LuceneIndexer indexer) { - //TODO: check for closing! and AlreadyClosedException - try { - var searcher = indexer.GetSearcher().GetSearcher() as IndexSearcher; - if (searcher == null) return 0; + if (!((indexer.GetSearcher() as LuceneSearcher)?.GetLuceneSearcher() is IndexSearcher searcher)) + return 0; + using (searcher) using (var reader = searcher.IndexReader) { @@ -89,30 +89,6 @@ namespace Umbraco.Web.Search } } - /// - /// Returns true if the index is optimized or not - /// - /// - /// - public static bool IsIndexOptimized(this LuceneIndexer indexer) - { - try - { - var searcher = indexer.GetSearcher().GetSearcher() as IndexSearcher; - if (searcher == null) return true; - using (searcher) - using (var reader = searcher.IndexReader) - { - return reader.IsOptimized(); - } - } - catch (AlreadyClosedException) - { - Current.Logger.Warn(typeof(ExamineExtensions), "Cannot get IsIndexOptimized, the writer is already closed"); - return false; - } - } - /// /// Check if the index is locked /// @@ -136,8 +112,9 @@ namespace Umbraco.Web.Search { try { - var searcher = indexer.GetSearcher().GetSearcher() as IndexSearcher; - if (searcher == null) return 0; + if (!((indexer.GetSearcher() as LuceneSearcher)?.GetLuceneSearcher() is IndexSearcher searcher)) + return 0; + using (searcher) using (var reader = searcher.IndexReader) { diff --git a/src/Umbraco.Web/Search/UmbracoTreeSearcher.cs b/src/Umbraco.Web/Search/UmbracoTreeSearcher.cs index 477590f2c9..66a7115dc7 100644 --- a/src/Umbraco.Web/Search/UmbracoTreeSearcher.cs +++ b/src/Umbraco.Web/Search/UmbracoTreeSearcher.cs @@ -39,7 +39,7 @@ namespace Umbraco.Web.Search var sb = new StringBuilder(); string type; - var searcher = Constants.Examine.InternalSearcher; + var indexer = Constants.Examine.InternalIndexer; var fields = new[] { "id", "__NodeId" }; var umbracoContext = umbracoHelper.UmbracoContext; @@ -48,7 +48,7 @@ namespace Umbraco.Web.Search switch (entityType) { case UmbracoEntityTypes.Member: - searcher = Constants.Examine.InternalMemberSearcher; + indexer = Constants.Examine.InternalMemberIndexer; type = "member"; fields = new[] { "id", "__NodeId", "email", "loginName" }; if (searchFrom != null && searchFrom != Constants.Conventions.MemberTypes.AllMembersListId && searchFrom.Trim() != "-1") @@ -72,7 +72,7 @@ namespace Umbraco.Web.Search throw new NotSupportedException("The " + typeof(UmbracoTreeSearcher) + " currently does not support searching against object type " + entityType); } - var internalSearcher = ExamineManager.Instance.SearchProviderCollection[searcher]; + var internalSearcher = ExamineManager.Instance.GetSearcher(indexer); //build a lucene query: // the __nodeName will be boosted 10x without wildcards @@ -180,7 +180,7 @@ namespace Umbraco.Web.Search sb.Append("+__IndexType:"); sb.Append(type); - var raw = internalSearcher.CreateSearchCriteria().RawQuery(sb.ToString()); + var raw = internalSearcher.CreateCriteria().RawQuery(sb.ToString()); var result = internalSearcher //only return the number of items specified to read up to the amount of records to fill from 0 -> the number of items on the page requested @@ -277,10 +277,10 @@ namespace Umbraco.Web.Search m.Icon = "icon-user"; } - var searchResult = results.First(x => x.Id.ToInvariantString() == m.Id.ToString()); + var searchResult = results.First(x => x.Id == m.Id.ToString()); if (searchResult.Fields.ContainsKey("email") && searchResult.Fields["email"] != null) { - m.AdditionalData["Email"] = results.First(x => x.Id.ToInvariantString() == m.Id.ToString()).Fields["email"]; + m.AdditionalData["Email"] = results.First(x => x.Id == m.Id.ToString()).Fields["email"]; } if (searchResult.Fields.ContainsKey("__key") && searchResult.Fields["__key"] != null) { diff --git a/src/Umbraco.Web/Suspendable.cs b/src/Umbraco.Web/Suspendable.cs index 877c9797e5..9e15b2419e 100644 --- a/src/Umbraco.Web/Suspendable.cs +++ b/src/Umbraco.Web/Suspendable.cs @@ -2,7 +2,9 @@ using Examine; using Examine.Providers; using Umbraco.Core.Composing; +using Umbraco.Examine; using Umbraco.Web.Cache; +using Umbraco.Web.Search; namespace Umbraco.Web { @@ -77,10 +79,9 @@ namespace Umbraco.Web _tried = false; // fixme - could we fork this on a background thread? - foreach (BaseIndexProvider indexer in ExamineManager.Instance.IndexProviderCollection) - { - indexer.RebuildIndex(); - } + //TODO: when resuming do we always want a full rebuild of all indexes? + // fixme - can we inject IExamineManager somehow? + ExamineComponent.RebuildIndexes(false, ExamineManager.Instance, Current.Logger); } } diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index 84d9e7a38f..2163a2e078 100644 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -72,7 +72,7 @@ - + diff --git a/src/Umbraco.Web/WebServices/ExamineManagementApiController.cs b/src/Umbraco.Web/WebServices/ExamineManagementApiController.cs index 69e88f2c7b..34ffd1426b 100644 --- a/src/Umbraco.Web/WebServices/ExamineManagementApiController.cs +++ b/src/Umbraco.Web/WebServices/ExamineManagementApiController.cs @@ -1,21 +1,22 @@ using System; using System.Collections.Generic; +using System.ComponentModel; using System.Linq; using System.Net; using System.Net.Http; +using System.Reflection; using System.Web.Http; using Examine; using Examine.LuceneEngine; using Examine.LuceneEngine.Providers; using Examine.Providers; -using Lucene.Net.Index; using Lucene.Net.Search; -using Lucene.Net.Store; using Umbraco.Core; using Umbraco.Core.Cache; using Umbraco.Core.Logging; using Umbraco.Core.Composing; using Umbraco.Core.Services; +using Umbraco.Examine; using Umbraco.Web.Search; using Umbraco.Web.WebApi; using Umbraco.Web.WebApi.Filters; @@ -25,10 +26,16 @@ namespace Umbraco.Web.WebServices [ValidateAngularAntiForgeryToken] public class ExamineManagementApiController : UmbracoAuthorizedApiController { - //TODO: Fix all of this for searchers/indexers that are not configured via code (i.e. the core ones once we do that) - // We will need to be able to search an index directly without having to go through all of the searchers + public ExamineManagementApiController(IExamineManager examineManager, ILogger logger, IRuntimeCacheProvider runtimeCacheProvider) + { + _examineManager = examineManager; + _logger = logger; + _runtimeCacheProvider = runtimeCacheProvider; + } - private ExamineManager _examineManager; + private readonly IExamineManager _examineManager; + private readonly ILogger _logger; + private readonly IRuntimeCacheProvider _runtimeCacheProvider; /// /// Checks if the member internal index is consistent with the data stored in the database @@ -40,7 +47,7 @@ namespace Umbraco.Web.WebServices var total = Services.MemberService.Count(); var searcher = _examineManager.GetSearcher(Constants.Examine.InternalMemberIndexer); - var criteria = searcher.CreateSearchCriteria().RawQuery("__IndexType:member"); + var criteria = searcher.CreateCriteria().RawQuery("__IndexType:member"); var totalIndexed = searcher.Search(criteria); return total == totalIndexed.TotalItemCount; } @@ -55,7 +62,7 @@ namespace Umbraco.Web.WebServices var total = Services.MediaService.Count(); var searcher = _examineManager.GetSearcher(Constants.Examine.InternalIndexer); - var criteria = searcher.CreateSearchCriteria().RawQuery("__IndexType:media"); + var criteria = searcher.CreateCriteria().RawQuery("__IndexType:media"); var totalIndexed = searcher.Search(criteria); return total == totalIndexed.TotalItemCount; } @@ -70,7 +77,7 @@ namespace Umbraco.Web.WebServices var total = Services.ContentService.Count(); var searcher = _examineManager.GetSearcher(Constants.Examine.InternalIndexer); - var criteria = searcher.CreateSearchCriteria().RawQuery("__IndexType:content"); + var criteria = searcher.CreateCriteria().RawQuery("__IndexType:content"); var totalIndexed = searcher.Search(criteria); return total == totalIndexed.TotalItemCount; } @@ -81,7 +88,7 @@ namespace Umbraco.Web.WebServices /// public IEnumerable GetIndexerDetails() { - return ExamineManager.Instance.IndexProviders.Select(CreateModel).OrderBy(x => + return _examineManager.IndexProviders.Select(CreateModel).OrderBy(x => { //order by name , but strip the "Indexer" from the end if it exists return x.Name.TrimEnd("Indexer"); @@ -95,19 +102,23 @@ namespace Umbraco.Web.WebServices public IEnumerable GetSearcherDetails() { var model = new List( - ExamineManager.Instance.SearchProviderCollection.Cast().Select(searcher => + _examineManager.IndexProviders.Select(indexer => { + var searcher = indexer.Value.GetSearcher(); + var searcherName = (searcher as BaseLuceneSearcher)?.Name ?? string.Concat(indexer.Key, "Searcher"); + var indexerModel = new ExamineSearcherModel() { - Name = searcher.Name + Name = searcherName }; var props = TypeHelper.CachedDiscoverableProperties(searcher.GetType(), mustWrite: false) //ignore these properties - .Where(x => new[] {"Description"}.InvariantContains(x.Name) == false) - .OrderBy(x => x.Name); + .Where(x => new[] {"Description"}.InvariantContains(x.Name) == false) + .Where(x => x.GetCustomAttribute()?.State != EditorBrowsableState.Never) + .OrderBy(x => x.Name); foreach (var p in props) { - indexerModel.ProviderProperties.Add(p.Name, p.GetValue(searcher, null).ToString()); + indexerModel.ProviderProperties.Add(p.Name, p.GetValue(searcher, null)?.ToString()); } return indexerModel; }).OrderBy(x => @@ -118,7 +129,7 @@ namespace Umbraco.Web.WebServices return model; } - public ILuceneSearchResults GetSearchResults(string searcherName, string query, string queryType) + public ISearchResults GetSearchResults(string searcherName, string query, string queryType) { if (queryType == null) { @@ -129,48 +140,22 @@ namespace Umbraco.Web.WebServices if (query.IsNullOrWhiteSpace()) return LuceneSearchResults.Empty(); - LuceneSearcher searcher; - var msg = ValidateLuceneSearcher(searcherName, out searcher); + var msg = ValidateLuceneSearcher(searcherName, out var searcher); if (msg.IsSuccessStatusCode) { if (queryType.InvariantEquals("text")) { - return searcher.Find(query, false); + return searcher.Search(query, false); } if (queryType.InvariantEquals("lucene")) { - return searcher.Find(searcher.CreateCriteria().RawQuery(query)); + return searcher.Search(searcher.CreateCriteria().RawQuery(query)); } throw new HttpResponseException(Request.CreateResponse(HttpStatusCode.NotFound)); } throw new HttpResponseException(msg); } - /// - /// Optimizes an index - /// - public HttpResponseMessage PostOptimizeIndex(string indexerName) - { - IExamineIndexer indexer; - var msg = ValidateLuceneIndexer(indexerName, out indexer); - var luceneIndexer = indexer as LuceneIndexer; - if (luceneIndexer != null && msg.IsSuccessStatusCode) - { - try - { - luceneIndexer.OptimizeIndex(); - } - catch (Exception ex) - { - var response = Request.CreateResponse(HttpStatusCode.Conflict); - response.Content = new StringContent(string.Format("The index could not be optimized, most likely there is another thread currently writing to the index. Error: {0}", ex)); - response.ReasonPhrase = "Could Not Optimize"; - return response; - } - } - return msg; - } - /// /// Rebuilds the index /// @@ -178,11 +163,10 @@ namespace Umbraco.Web.WebServices /// public HttpResponseMessage PostRebuildIndex(string indexerName) { - LuceneIndexer indexer; - var msg = ValidateLuceneIndexer(indexerName, out indexer); + var msg = ValidateLuceneIndexer(indexerName, out LuceneIndexer indexer); if (msg.IsSuccessStatusCode) { - Current.Logger.Info(string.Format("Rebuilding index '{0}'", indexerName)); + _logger.Info($"Rebuilding index '{indexerName}'"); //remove it in case there's a handler there alraedy indexer.IndexOperationComplete -= Indexer_IndexOperationComplete; @@ -196,54 +180,33 @@ namespace Umbraco.Web.WebServices try { indexer.RebuildIndex(); - } - // fixme - Examine issues - // cannot enable that piece of code from v7 as Index.Write.Unlock does not seem to exist anymore? - //catch (LockObtainFailedException) - //{ - // //this will occur if the index is locked (which it should defo not be!) so in this case we'll forcibly unlock it and try again - - // try - // { - // IndexWriter.Unlock(indexer.GetLuceneDirectory()); - // indexer.RebuildIndex(); - // } - // catch (Exception e) - // { - // return HandleException(e, indexer); - // } - //} + } catch (Exception ex) { - return HandleException(ex, indexer); + //ensure it's not listening + indexer.IndexOperationComplete -= Indexer_IndexOperationComplete; + Logger.Error("An error occurred rebuilding index", ex); + var response = Request.CreateResponse(HttpStatusCode.Conflict); + response.Content = new StringContent($"The index could not be rebuilt at this time, most likely there is another thread currently writing to the index. Error: {ex}"); + response.ReasonPhrase = "Could Not Rebuild"; + return response; } } return msg; } - private HttpResponseMessage HandleException(Exception ex, LuceneIndexer indexer) - { - //ensure it's not listening - indexer.IndexOperationComplete -= Indexer_IndexOperationComplete; - Logger.Error("An error occurred rebuilding index", ex); - var response = Request.CreateResponse(HttpStatusCode.Conflict); - response.Content = new StringContent(string.Format("The index could not be rebuilt at this time, most likely there is another thread currently writing to the index. Error: {0}", ex)); - response.ReasonPhrase = "Could Not Rebuild"; - return response; - } - //static listener so it's not GC'd - private static void Indexer_IndexOperationComplete(object sender, EventArgs e) + private void Indexer_IndexOperationComplete(object sender, EventArgs e) { var indexer = (LuceneIndexer) sender; //ensure it's not listening anymore indexer.IndexOperationComplete -= Indexer_IndexOperationComplete; - Current.Logger.Info($"Rebuilding index '{indexer.Name}' done, {indexer.CommitCount} items committed (can differ from the number of items in the index)"); + _logger.Info($"Rebuilding index '{indexer.Name}' done, {indexer.CommitCount} items committed (can differ from the number of items in the index)"); var cacheKey = "temp_indexing_op_" + indexer.Name; - Current.ApplicationCache.RuntimeCache.ClearCacheItem(cacheKey); + _runtimeCacheProvider.ClearCacheItem(cacheKey); } /// @@ -257,8 +220,7 @@ namespace Umbraco.Web.WebServices /// public ExamineIndexerModel PostCheckRebuildIndex(string indexerName) { - LuceneIndexer indexer; - var msg = ValidateLuceneIndexer(indexerName, out indexer); + var msg = ValidateLuceneIndexer(indexerName, out LuceneIndexer indexer); if (msg.IsSuccessStatusCode) { var cacheKey = "temp_indexing_op_" + indexerName; @@ -266,42 +228,25 @@ namespace Umbraco.Web.WebServices //if its still there then it's not done return found != null ? null - : CreateModel(new KeyValuePair(indexerName, indexer)); + : CreateModel(new KeyValuePair(indexerName, indexer)); } throw new HttpResponseException(msg); } - /// - /// Checks if the index is optimized - /// - /// - /// - public ExamineIndexerModel PostCheckOptimizeIndex(string indexerName) - { - LuceneIndexer indexer; - var msg = ValidateLuceneIndexer(indexerName, out indexer); - if (msg.IsSuccessStatusCode) - { - var isOptimized = indexer.IsIndexOptimized(); - return isOptimized == false - ? null - : CreateModel(new KeyValuePair(indexerName, indexer)); - } - throw new HttpResponseException(msg); - } - - private ExamineIndexerModel CreateModel(KeyValuePair indexer) + private ExamineIndexerModel CreateModel(KeyValuePair indexerKeyVal) { + var indexer = indexerKeyVal.Value; + var indexName = indexerKeyVal.Key; var indexerModel = new ExamineIndexerModel() { - FieldDefinitions = indexer.Value.FieldDefinitions, - Name = indexer.Key + FieldDefinitions = indexer.FieldDefinitionCollection, + Name = indexName }; var props = TypeHelper.CachedDiscoverableProperties(indexer.GetType(), mustWrite: false) //ignore these properties - .Where(x => new[] {"IndexerData", "Description", "WorkingFolder"}.InvariantContains(x.Name) == false) - .OrderBy(x => x.Name); + .Where(x => new[] {"IndexerData", "Description", "WorkingFolder"}.InvariantContains(x.Name) == false) + .OrderBy(x => x.Name); foreach (var p in props) { @@ -310,22 +255,20 @@ namespace Umbraco.Web.WebServices { // Do not warn for new new attribute that is optional if(string.Equals(p.Name, "DirectoryFactory", StringComparison.InvariantCultureIgnoreCase) == false) - Logger.Warn("Property value was null when setting up property on indexer: " + indexer.Key + " property: " + p.Name); + Logger.Warn("Property value was null when setting up property on indexer: " + indexName + " property: " + p.Name); val = string.Empty; } indexerModel.ProviderProperties.Add(p.Name, val.ToString()); } - var luceneIndexer = indexer.Value as LuceneIndexer; - if (luceneIndexer != null) + if (indexer is LuceneIndexer luceneIndexer) { indexerModel.IsLuceneIndex = true; if (luceneIndexer.IndexExists()) { - Exception indexError; - indexerModel.IsHealthy = luceneIndexer.IsHealthy(out indexError); + indexerModel.IsHealthy = luceneIndexer.IsHealthy(out var indexError); if (indexerModel.IsHealthy == false) { @@ -336,15 +279,11 @@ namespace Umbraco.Web.WebServices indexerModel.DocumentCount = luceneIndexer.GetIndexDocumentCount(); indexerModel.FieldCount = luceneIndexer.GetIndexFieldCount(); - indexerModel.IsOptimized = luceneIndexer.IsIndexOptimized(); - indexerModel.DeletionCount = luceneIndexer.GetDeletedDocumentsCount(); } else { indexerModel.DocumentCount = 0; indexerModel.FieldCount = 0; - indexerModel.IsOptimized = true; - indexerModel.DeletionCount = 0; } } return indexerModel; @@ -352,41 +291,46 @@ namespace Umbraco.Web.WebServices private HttpResponseMessage ValidateLuceneSearcher(string searcherName, out LuceneSearcher searcher) { - if (ExamineManager.Instance.SearchProviderCollection.Cast().Any(x => x.Name == searcherName)) + foreach (var indexer in _examineManager.IndexProviders) { - searcher = ExamineManager.Instance.SearchProviderCollection[searcherName] as LuceneSearcher; - if (searcher == null) - { - var response = Request.CreateResponse(HttpStatusCode.BadRequest); - response.Content = new StringContent(string.Format("The searcher {0} is not of type {1}", searcherName, typeof(LuceneSearcher))); - response.ReasonPhrase = "Wrong Searcher Type"; - return response; - } - //return Ok! - return Request.CreateResponse(HttpStatusCode.OK); + var s = indexer.Value.GetSearcher(); + var sName = (s as BaseLuceneSearcher)?.Name ?? string.Concat(indexer.Key, "Searcher"); + if (sName != searcherName) continue; + searcher = s as LuceneSearcher; + + //Found it, return OK + if (searcher != null) return Request.CreateResponse(HttpStatusCode.OK); + + //Return an error since it's not the right type + var response = Request.CreateResponse(HttpStatusCode.BadRequest); + response.Content = new StringContent($"The searcher {searcherName} is not of type {typeof(LuceneSearcher)}"); + response.ReasonPhrase = "Wrong Searcher Type"; + return response; } searcher = null; var response1 = Request.CreateResponse(HttpStatusCode.BadRequest); - response1.Content = new StringContent(string.Format("No searcher found with name = {0}", searcherName)); + response1.Content = new StringContent($"No searcher found with name = {searcherName}"); response1.ReasonPhrase = "Searcher Not Found"; return response1; } private HttpResponseMessage ValidateLuceneIndexer(string indexerName, out T indexer) - where T : class, IExamineIndexer + where T : class, IIndexer { indexer = null; - if (ExamineManager.Instance.IndexProviders.ContainsKey(indexerName)) + if (_examineManager.IndexProviders.ContainsKey(indexerName) + && _examineManager.IndexProviders[indexerName] is T casted) { //return Ok! + indexer = casted; return Request.CreateResponse(HttpStatusCode.OK); } var response = Request.CreateResponse(HttpStatusCode.BadRequest); - response.Content = new StringContent(string.Format("No indexer found with name = {0}", indexerName)); + response.Content = new StringContent($"No indexer found with name = {indexerName} of type {typeof(T)}"); response.ReasonPhrase = "Indexer Not Found"; return response; } diff --git a/src/Umbraco.Web/umbraco.presentation/umbraco/dialogs/search.aspx.cs b/src/Umbraco.Web/umbraco.presentation/umbraco/dialogs/search.aspx.cs index 20a4e2ba54..d570b32d85 100644 --- a/src/Umbraco.Web/umbraco.presentation/umbraco/dialogs/search.aspx.cs +++ b/src/Umbraco.Web/umbraco.presentation/umbraco/dialogs/search.aspx.cs @@ -16,6 +16,7 @@ using Umbraco.Core.Xml; namespace umbraco.presentation.dialogs { + //fixme - is this even used anymore? public partial class search : Umbraco.Web.UI.Pages.UmbracoEnsuredPage { @@ -59,11 +60,11 @@ namespace umbraco.presentation.dialogs //if it doesn't start with "*", then search only nodeName and nodeId var internalSearcher = (CurrentApp == Constants.Applications.Members) - ? ExamineManager.Instance.SearchProviderCollection["InternalMemberSearcher"] - : ExamineManager.Instance.SearchProviderCollection["InternalSearcher"]; + ? ExamineManager.Instance.GetSearcher(Constants.Examine.InternalMemberIndexer) + : ExamineManager.Instance.GetSearcher(Constants.Examine.InternalIndexer); //create some search criteria, make everything combined to be 'And' and only search the current app - var criteria = internalSearcher.CreateSearchCriteria(CurrentApp, Examine.SearchCriteria.BooleanOperation.And); + var criteria = internalSearcher.CreateCriteria(CurrentApp, Examine.SearchCriteria.BooleanOperation.And); IEnumerable results; if (txt.StartsWith("*"))