yv01p/Umbraco-CMS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'dev-v8' of https://github.com/umbraco/Umbraco-CMS into dev-v8

Browse Source 
# Conflicts:
#	src/Umbraco.Web/PublishedContentQuery.cs
This commit is contained in:
Shannon
2018-03-29 22:35:46 +11:00
parent 83e90969b3 23e8632df6
commit 7fd9caf889
86 changed files with 2141 additions and 3238 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/NuGet.Config
View File

@@ -3,5 +3,6 @@
<packageSources>
<add key="nuget.org" value="https://www.nuget.org/api/v2/" />
<add key="umbracocore" value="https://www.myget.org/F/umbracocore/api/v3/index.json" />
<add key="examineAppVeyor" value="https://ci.appveyor.com/nuget/examine-f73l6qv0oqfh/" />
</packageSources>
</configuration>

16
src/Umbraco.Core/Constants-Examine.cs
View File

@@ -18,21 +18,7 @@
/// The alias of the external content indexer
/// </summary>
public const string ExternalIndexer = "ExternalIndexer";
/// <summary>
/// The alias of the internal member searcher
/// </summary>
public const string InternalMemberSearcher = "InternalMemberSearcher";
/// <summary>
/// The alias of the internal content searcher
/// </summary>
public const string InternalSearcher = "InternalSearcher";
/// <summary>
/// The alias of the external content searcher
/// </summary>
public const string ExternalSearcher = "ExternalSearcher";
}
}
}

10
src/Umbraco.Core/IRuntimeState.cs
View File

@@ -57,6 +57,16 @@ namespace Umbraco.Core
/// </summary>
RuntimeLevel Level { get; }
/// <summary>
/// Gets the current migration state.
/// </summary>
string CurrentMigrationState { get; }
/// <summary>
/// Gets the final migration state.
/// </summary>
string FinalMigrationState { get; }
/// <summary>
/// Gets the exception that caused the boot to fail.
/// </summary>

78
src/Umbraco.Core/Migrations/MigrationPlan.cs
View File

@@ -7,6 +7,9 @@ using Umbraco.Core.Scoping;
namespace Umbraco.Core.Migrations
{
/// <summary>
/// Represents a migration plan.
/// </summary>
public class MigrationPlan
{
private readonly IMigrationBuilder _migrationBuilder;
@@ -16,31 +19,67 @@ namespace Umbraco.Core.Migrations
private string _prevState;
private string _finalState;
// initializes a non-executing plan
/// <summary>
/// Initializes a new instance of the <see cref="MigrationPlan"/> class.
/// </summary>
/// <param name="name">The name of the plan.</param>
/// <remarks>The plan cannot be executed. Use this constructor e.g. when only validating the plan,
/// or trying to get its final state, without actually needing to execute it.</remarks>
public MigrationPlan(string name)
{
if (string.IsNullOrWhiteSpace(name)) throw new ArgumentNullOrEmptyException(nameof(name));
Name = name;
// ReSharper disable once VirtualMemberCallInConstructor
// (accepted)
DefinePlan();
}
// initializes a plan
/// <summary>
/// Initializes a new instance of the <see cref="MigrationPlan"/> class.
/// </summary>
/// <param name="name">The name of the plan.</param>
/// <param name="migrationBuilder">A migration builder.</param>
/// <param name="logger">A logger.</param>
/// <remarks>The plan can be executed.</remarks>
public MigrationPlan(string name, IMigrationBuilder migrationBuilder, ILogger logger)
{
if (string.IsNullOrWhiteSpace(name)) throw new ArgumentNullOrEmptyException(nameof(name));
Name = name;
_migrationBuilder = migrationBuilder ?? throw new ArgumentNullException(nameof(migrationBuilder));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
// ReSharper disable once VirtualMemberCallInConstructor
// (accepted)
DefinePlan();
}
/// <summary>
/// Defines the plan.
/// </summary>
protected virtual void DefinePlan() { }
/// <summary>
/// Gets the name of the plan.
/// </summary>
public string Name { get; }
/// <summary>
/// Adds an empty migration from source to target state.
/// </summary>
public MigrationPlan Add(string sourceState, string targetState)
=> Add<NoopMigration>(sourceState, targetState);
/// <summary>
/// Adds a migration from source to target state.
/// </summary>
public MigrationPlan Add<TMigration>(string sourceState, string targetState)
where TMigration : IMigration
=> Add(sourceState, targetState, typeof(TMigration));
/// <summary>
/// Adds a migration from source to target state.
/// </summary>
public MigrationPlan Add(string sourceState, string targetState, Type migration)
{
if (sourceState == null) throw new ArgumentNullException(nameof(sourceState));
@@ -78,26 +117,50 @@ namespace Umbraco.Core.Migrations
return this;
}
/// <summary>
/// Chains an empty migration from chain to target state.
/// </summary>
public MigrationPlan Chain(string targetState)
=> Chain<NoopMigration>(targetState);
/// <summary>
/// Chains a migration from chain to target state.
/// </summary>
public MigrationPlan Chain<TMigration>(string targetState)
where TMigration : IMigration
=> Chain(targetState, typeof(TMigration));
/// <summary>
/// Chains a migration from chain to target state.
/// </summary>
public MigrationPlan Chain(string targetState, Type migration)
=> Add(_prevState, targetState, migration);
/// <summary>
/// Sets the chain state.
/// </summary>
public MigrationPlan From(string sourceState)
{
_prevState = sourceState ?? throw new ArgumentNullException(nameof(sourceState));
return this;
}
/// <summary>
/// Gets the initial state.
/// </summary>
/// <remarks>The initial state is the state when no entry for the plan
/// could be found in the database (i.e. the plan has never run).</remarks>
public virtual string InitialState => string.Empty;
/// <summary>
/// Gets the final state.
/// </summary>
public string FinalState => _finalState ?? (_finalState = Validate());
/// <summary>
/// Validates the plan.
/// </summary>
/// <returns>The plan's final state.</returns>
public string Validate()
{
// quick check for dead ends - a dead end is a transition that has a target state
@@ -134,6 +197,13 @@ namespace Umbraco.Core.Migrations
return finalState;
}
/// <summary>
/// Executes the plan.
/// </summary>
/// <param name="scope">A scope.</param>
/// <param name="fromState">The state to start execution at.</param>
/// <returns>The final state.</returns>
/// <remarks>The plan executes within the scope, which must then be completed.</remarks>
public string Execute(IScope scope, string fromState)
{
Validate();
@@ -142,7 +212,7 @@ namespace Umbraco.Core.Migrations
throw new InvalidOperationException("Cannot execute a non-executing plan.");
_logger.Info<MigrationPlan>("Starting \"{0}\"...", () => Name);
var origState = fromState; //GetState();
var origState = fromState ?? string.Empty;
var info = "At " + (string.IsNullOrWhiteSpace(origState) ? "origin" : ("\"" + origState + "\"")) + ".";
info = info.Replace("{", "{{").Replace("}", "}}"); // stupid log4net
_logger.Info<MigrationPlan>(info);
@@ -159,7 +229,7 @@ namespace Umbraco.Core.Migrations
var nextState = transition.TargetState;
origState = nextState;
_logger.Info<MigrationPlan>("At \"{0}\".", origState);
if (!_transitions.TryGetValue(origState, out transition))

112
src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs
View File

@@ -18,18 +18,29 @@ namespace Umbraco.Core.Migrations.Upgrade
/// </summary>
public class UmbracoPlan : MigrationPlan
{
/// <summary>
/// Initializes a new instance of the <see cref="UmbracoPlan"/> class.
/// </summary>
public UmbracoPlan()
: base(Constants.System.UmbracoUpgradePlanName)
{
DefinePlan();
}
{ }
/// <summary>
/// Initializes a new instance of the <see cref="UmbracoPlan"/> class.
/// </summary>
public UmbracoPlan(IMigrationBuilder migrationBuilder, ILogger logger)
: base(Constants.System.UmbracoUpgradePlanName, migrationBuilder, logger)
{
DefinePlan();
}
{ }
/// <inheritdoc />
/// <remarks>
/// <para>The default initial state in plans is string.Empty.</para>
/// <para>When upgrading from version 7, we want to use specific initial states
/// that are e.g. "{orig-7.9.3}", "{orig-7.11.1}", etc. so we can chain the proper
/// migrations.</para>
/// <para>This is also where we detect the current version, and reject invalid
/// upgrades (from a tool old version, or going back in time, etc).</para>
/// </remarks>
public override string InitialState
{
get
@@ -38,8 +49,8 @@ namespace Umbraco.Core.Migrations.Upgrade
if (!SemVersion.TryParse(ConfigurationManager.AppSettings["umbracoConfigurationStatus"], out var currentVersion))
throw new InvalidOperationException("Could not get current version from web.config umbracoConfigurationStatus appSetting.");
// must be at least 7.?.? - fixme adjust when releasing
if (currentVersion < new SemVersion(7, 999))
// must be at least 7.? - fixme adjust when releasing
if (currentVersion < new SemVersion(7, 9))
throw new InvalidOperationException($"Version {currentVersion} cannot be upgraded to {UmbracoVersion.SemanticVersion}.");
// cannot go back in time
@@ -62,64 +73,85 @@ namespace Umbraco.Core.Migrations.Upgrade
}
}
private void DefinePlan()
/// <inheritdoc />
protected override void DefinePlan()
{
// NOTE: MODIFYING THE PLAN
//
// Please take great care when modifying the plan!
//
// * Creating a migration for version 8:
// Append the migration to the main version 8 chain, using a new guid.
// Update the final state (see end of file) to that guid.
// Append the migration to version 7 upgrade chains.
// * Porting a migration from version 7:
// Append the migration to the main version 8 chain, using a new guid.
// Update the final state (see end of file) to that guid.
// Update all init-7.x.y chains.
// UPGRADE FROM 7
//
// when 8.0.0 is released, on the first upgrade, the state is automatically
// set to {init-7.x.y} where 7.x.y is the version (see above), and then we define upgrades.
// When upgrading from version 7, the state is automatically set to {init-7.x.y} where
// 7.x.y is the version. We need to define a chain starting at that state and taking
// us to version 8. And we need such a chain for each 7.x.y version that can be upgraded
// to version 8, bearing in mind that new releases of version 7 will probably be
// created *after* the first released of version 8.
//
// then, as more v7 and v8 versions are released, new chains needs to be defined to
// support the upgrades (new v7 may backport some migrations and require their own
// upgrade paths, etc).
// fixme adjust when releasing
// fixme adjust when releasing the first public (alpha?) version
From("{init-7.8.0}");
Chain<V_8_0_0.AddLockObjects>("{7C447271-CA3F-4A6A-A913-5D77015655CB}"); // add more lock objects
// at the moment we don't support upgrading from versions older than 7.9.0
//
From("{init-7.9.0}");
Chain<V_8_0_0.AddLockObjects>("{7C447271-CA3F-4A6A-A913-5D77015655CB}");
Chain<AddContentNuTable>("{CBFF58A2-7B50-4F75-8E98-249920DB0F37}");
Chain<RefactorXmlColumns>("{3D18920C-E84D-405C-A06A-B7CEE52FE5DD}");
Chain<VariantsMigration>("{FB0A5429-587E-4BD0-8A67-20F0E7E62FF7}");
Chain<DropMigrationsTable>("{F0C42457-6A3B-4912-A7EA-F27ED85A2092}");
Chain<DataTypeMigration>("{8640C9E4-A1C0-4C59-99BB-609B4E604981}");
Chain<TagsMigration>("{DD1B99AF-8106-4E00-BAC7-A43003EA07F8}");
Chain<SuperZero>("{9DF05B77-11D1-475C-A00A-B656AF7E0908}");
Chain<PropertyEditorsMigration>("{CA7DB949-3EF4-403D-8464-F9BA36A52E87}");
Chain<PropertyEditorsMigration>("{6FE3EF34-44A0-4992-B379-B40BC4EF1C4D}");
Chain<LanguageColumns>("{7F59355A-0EC9-4438-8157-EB517E6D2727}");
// 7.8.1 = same as 7.8.0
From("{init-7.8.1}");
Chain("{init-7.8.0}");
// must chain to v8 final state (see at end of file)
Chain("{8918450B-3DA0-4BB7-886A-6FA8B7E4186E}");
// 7.9.0 = requires its own chain
From("{init-7.9.0}");
// chain...
Chain("{82C4BA1D-7720-46B1-BBD7-07F3F73800E6}");
// 7.9.1, .2, .3 = same as 7.9.0
// 7.10.0 = same as 7.9.0
From("{init-7.9.1}").Chain("{init-7.9.0}");
From("{init-7.9.2}").Chain("{init-7.9.0}");
From("{init-7.9.3}").Chain("{init-7.9.0}");
From("{init-7.10.0}").Chain("{init-7.9.0}");
// UPGRADE 8
// VERSION 8 PLAN
//
// starting from the original 8.0.0 final state, chain migrations to upgrade version 8,
// defining new final states as more migrations are added to the chain.
// this is the master Umbraco migration plan, starting from the very first version 8
// release, which was a pre-pre-alpha, years ago. It contains migrations created
// for version 8, along with migrations ported from version 7 as version 7 evolves.
// It is therefore *normal* that some pure version 8 migrations are mixed with
// migrations merged from version 7.
//
// before v8 is released, some sites may exist, and these "pre-8" versions require their
// own upgrade plan. in other words, this is also the plan for sites that were on v8 before
// v8 was released
// new migrations should always be *appended* to the *end* of the chain.
// 8.0.0
From("{init-origin}");
From("{init-origin}"); // "origin" was 7.4.something
Chain<V_8_0_0.AddLockTable>("{98347B5E-65BF-4DD7-BB43-A09CB7AF4FCA}");
Chain<V_8_0_0.AddLockObjects>("{1E8165C4-942D-40DC-AC76-C5FF8831E400}");
Chain<AddContentNuTable>("{39E15568-7AAD-4D54-81D0-758CCFC529F8}");
Chain<RefactorXmlColumns>("{55C3F97D-BDA7-4FB1-A743-B0456B56EAA3}");
// 7.5.0
// merging from 7.5.0
Chain<RemoveStylesheetDataAndTablesAgain>("{287F9E39-F673-42F7-908C-21659AB13B13}");
Chain<V_7_5_0.UpdateUniqueIndexOnPropertyData>("{2D08588A-AD90-479C-9F6E-A99B60BA7226}");
Chain<AddRedirectUrlTable>("{2D917FF8-AC81-4C00-A407-1F4B1DF6089C}");
// 7.5.5
// merging from 7.5.5
Chain<UpdateAllowedMediaTypesAtRoot>("{44484C32-EEB3-4A12-B1CB-11E02CE22AB2}");
// 7.6.0
// merging from 7.6.0
Chain<AddIndexesToUmbracoRelationTables>("{3586E4E9-2922-49EB-8E2A-A530CE6DBDE0}");
Chain<AddIndexToCmsMemberLoginName>("{D4A5674F-654D-4CC7-85E5-CFDBC533A318}");
Chain<AddIndexToUmbracoNodePath>("{7F828EDD-6622-4A8D-AD80-EEAF46C11680}");
@@ -130,7 +162,7 @@ namespace Umbraco.Core.Migrations.Upgrade
Chain<V_7_6_0.UpdateUniqueIndexOnPropertyData>("{5496C6CC-3AE0-4789-AF49-5BB4E28FA424}");
Chain<RemoveUmbracoDeployTables>("{8995332B-085E-4C0C-849E-9A77E79F4293}");
// 7.7.0
// merging from 7.7.0
Chain<AddIndexToDictionaryKeyColumn>("{74319856-7681-46B1-AA0D-F7E896FBE6A1}");
Chain<AddUserGroupTables>("{0427B0A2-994A-4AB4-BFF3-31B20614F6C9}");
Chain<AddUserStartNodeTable>("{F0D6F782-E432-46DE-A3A7-2AF06DB8853B}");
@@ -147,11 +179,7 @@ namespace Umbraco.Core.Migrations.Upgrade
Chain<PropertyEditorsMigration>("{CA7DB949-3EF4-403D-8464-F9BA36A52E87}");
Chain<LanguageColumns>("{7F0BF916-F64E-4B25-864A-170D6E6B68E5}");
// at this point of the chain, people started to work on v8, so whenever we
// merge stuff from v7, we have to chain the migrations here so they also
// run for v8.
// mergin from 7.8.0
// merging from 7.8.0
Chain<AddUserLoginTable>("{FDCB727A-EFB6-49F3-89E4-A346503AB849}");
Chain<AddTourDataUserColumn>("{2A796A08-4FE4-4783-A1A5-B8A6C8AA4A92}");
Chain<AddMediaVersionTable>("{1A46A98B-2AAB-4C8E-870F-A2D55A97FD1F}");
@@ -163,9 +191,9 @@ namespace Umbraco.Core.Migrations.Upgrade
Chain<AddUmbracoAuditTable>("{FD8631BC-0388-425C-A451-5F58574F6F05}");
Chain<AddUmbracoConsentTable>("{2821F53E-C58B-4812-B184-9CD240F990D7}");
Chain<CreateSensitiveDataUserGroup>("{8918450B-3DA0-4BB7-886A-6FA8B7E4186E}");
Chain<LanguageColumns>("FIXGUID NEW FINAL");
// FINAL STATE - MUST MATCH LAST ONE ABOVE !
// whenever this changes, update all references in this file!
Add(string.Empty, "{8918450B-3DA0-4BB7-886A-6FA8B7E4186E}");
}

36
src/Umbraco.Core/Runtime/CoreRuntime.cs
View File

@@ -138,7 +138,7 @@ namespace Umbraco.Core.Runtime
try
{
var dbfactory = container.GetInstance<IUmbracoDatabaseFactory>();
SetRuntimeStateLevel(_state, dbfactory, Logger);
SetRuntimeStateLevel(dbfactory, Logger);
Logger.Debug<CoreRuntime>($"Runtime level: {_state.Level}");
}
catch
@@ -233,38 +233,38 @@ namespace Umbraco.Core.Runtime
builder.AddCore();
}
private void SetRuntimeStateLevel(RuntimeState runtimeState, IUmbracoDatabaseFactory databaseFactory, ILogger logger)
private void SetRuntimeStateLevel(IUmbracoDatabaseFactory databaseFactory, ILogger logger)
{
var localVersion = UmbracoVersion.Local; // the local, files, version
var codeVersion = runtimeState.SemanticVersion; // the executing code version
var codeVersion = _state.SemanticVersion; // the executing code version
var connect = false;
// we don't know yet
runtimeState.Level = RuntimeLevel.Unknown;
_state.Level = RuntimeLevel.Unknown;
if (localVersion == null)
{
// there is no local version, we are not installed
logger.Debug<CoreRuntime>("No local version, need to install Umbraco.");
runtimeState.Level = RuntimeLevel.Install;
_state.Level = RuntimeLevel.Install;
}
else if (localVersion != codeVersion)
{
// there *is* a local version, but it does not match the code version
// need to upgrade
logger.Debug<CoreRuntime>($"Local version \"{localVersion}\" != code version \"{codeVersion}\", need to upgrade Umbraco.");
runtimeState.Level = RuntimeLevel.Upgrade;
_state.Level = RuntimeLevel.Upgrade;
}
else if (databaseFactory.Configured == false)
{
// local version *does* match code version, but the database is not configured
// install (again? this is a weird situation...)
logger.Debug<CoreRuntime>("Database is not configured, need to install Umbraco.");
runtimeState.Level = RuntimeLevel.Install;
_state.Level = RuntimeLevel.Install;
}
// install? not going to test anything else
if (runtimeState.Level == RuntimeLevel.Install)
if (_state.Level == RuntimeLevel.Install)
return;
// else, keep going,
@@ -284,14 +284,14 @@ namespace Umbraco.Core.Runtime
{
// cannot connect to configured database, this is bad, fail
logger.Debug<CoreRuntime>("Could not connect to database.");
runtimeState.Level = RuntimeLevel.BootFailed;
_state.Level = RuntimeLevel.BootFailed;
// in fact, this is bad enough that we want to throw
throw new BootFailedException("A connection string is configured but Umbraco could not connect to the database.");
}
// if we already know we want to upgrade, no need to look for migrations...
if (runtimeState.Level == RuntimeLevel.Upgrade)
if (_state.Level == RuntimeLevel.Upgrade)
return;
// else
@@ -302,18 +302,19 @@ namespace Umbraco.Core.Runtime
{
exists = EnsureUmbracoUpgradeState(databaseFactory, logger);
}
catch
catch (Exception e)
{
// can connect to the database but cannot access the migration table... need to install
logger.Warn<CoreRuntime>(e, "Could not check the upgrade state.");
logger.Debug<CoreRuntime>("Could not check the upgrade state, need to install Umbraco.");
runtimeState.Level = RuntimeLevel.Install;
_state.Level = RuntimeLevel.Install;
return;
}
if (exists)
{
// the database version matches the code & files version, all clear, can run
runtimeState.Level = RuntimeLevel.Run;
_state.Level = RuntimeLevel.Run;
return;
}
@@ -323,7 +324,7 @@ namespace Umbraco.Core.Runtime
// although the files version matches the code version, the database version does not
// which means the local files have been upgraded but not the database - need to upgrade
logger.Debug<CoreRuntime>("Has not reached the final upgrade step, need to upgrade Umbraco.");
runtimeState.Level = RuntimeLevel.Upgrade;
_state.Level = RuntimeLevel.Upgrade;
}
protected virtual bool EnsureUmbracoUpgradeState(IUmbracoDatabaseFactory databaseFactory, ILogger logger)
@@ -343,11 +344,12 @@ namespace Umbraco.Core.Runtime
state = database.FirstOrDefault<KeyValueDto>(sql)?.Value;
}
var finalState = umbracoPlan.FinalState;
_state.CurrentMigrationState = state;
_state.FinalMigrationState = umbracoPlan.FinalState;
logger.Debug<CoreRuntime>($"Final upgrade state is \"{finalState}\", database contains \"{state ?? "<null>"}\".");
logger.Debug<CoreRuntime>($"Final upgrade state is \"{_state.FinalMigrationState}\", database contains \"{state ?? "<null>"}\".");
return state == finalState;
return state == _state.FinalMigrationState;
}
#region Locals

6
src/Umbraco.Core/RuntimeState.cs
View File

@@ -85,6 +85,12 @@ namespace Umbraco.Core
/// <remarks>This is either "/" or eg "/virtual".</remarks>
public string ApplicationVirtualPath { get; } = HttpRuntime.AppDomainAppVirtualPath;
/// <inheritdoc />
public string CurrentMigrationState { get; internal set; }
/// <inheritdoc />
public string FinalMigrationState { get; internal set; }
/// <summary>
/// Gets the runtime level of execution.
/// </summary>

459
src/Umbraco.Examine/BaseUmbracoIndexer.cs
View File

@@ -1,459 +0,0 @@
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Linq;
using System.Text;
using Examine.LuceneEngine.Config;
using Examine.LuceneEngine.Providers;
using Examine.Providers;
using Lucene.Net.Analysis;
using Lucene.Net.Index;
using Umbraco.Core;
using Examine;
using System.IO;
using System.Xml;
using System.Xml.Linq;
using Examine.LuceneEngine;
using Examine.LuceneEngine.Faceting;
using Examine.LuceneEngine.Indexing;
using Lucene.Net.Documents;
using Lucene.Net.Store;
using Umbraco.Core.Composing;
using Umbraco.Core.Logging;
using Umbraco.Core.Xml;
using Umbraco.Examine.LocalStorage;
using Directory = Lucene.Net.Store.Directory;
namespace Umbraco.Examine
{
/// <summary>
/// An abstract provider containing the basic functionality to be able to query against
/// Umbraco data.
/// </summary>
public abstract class BaseUmbracoIndexer : LuceneIndexer
{
// note
// wrapping all operations that end up calling base.SafelyProcessQueueItems in a safe call
// context because they will fork a thread/task/whatever which should *not* capture our
// call context (and the database it can contain)! ideally we should be able to override
// SafelyProcessQueueItems but that's not possible in the current version of Examine.
/// <summary>
/// Used to store the path of a content object
/// </summary>
public const string IndexPathFieldName = "__Path";
public const string NodeKeyFieldName = "__Key";
public const string IconFieldName = "__Icon";
public const string PublishedFieldName = "__Published";
/// <summary>
/// The prefix added to a field when it is duplicated in order to store the original raw value.
/// </summary>
public const string RawFieldPrefix = "__Raw_";
/// <summary>
/// Default constructor
/// </summary>
protected BaseUmbracoIndexer()
: base()
{
ProfilingLogger = Current.ProfilingLogger;
_configBased = true;
}
protected BaseUmbracoIndexer(
IEnumerable<FieldDefinition> fieldDefinitions,
Directory luceneDirectory,
Analyzer defaultAnalyzer,
ProfilingLogger profilingLogger,
IValueSetValidator validator = null,
FacetConfiguration facetConfiguration = null, IDictionary<string, Func<string, IIndexValueType>> indexValueTypes = null)
: base(fieldDefinitions, luceneDirectory, defaultAnalyzer, validator, facetConfiguration, indexValueTypes)
{
if (profilingLogger == null) throw new ArgumentNullException("profilingLogger");
ProfilingLogger = profilingLogger;
}
private readonly bool _configBased = false;
private LocalTempStorageIndexer _localTempStorageIndexer;
/// <summary>
/// A type that defines the type of index for each Umbraco field (non user defined fields)
/// Alot of standard umbraco fields shouldn't be tokenized or even indexed, just stored into lucene
/// for retreival after searching.
/// </summary>
[Obsolete("IndexFieldPolicies is not really used apart for some legacy reasons - use FieldDefinition's instead")]
internal static readonly List<StaticField> IndexFieldPolicies
= new List<StaticField>
{
new StaticField("id", FieldIndexTypes.NOT_ANALYZED, false, string.Empty),
new StaticField("key", FieldIndexTypes.NOT_ANALYZED, false, string.Empty),
new StaticField( "version", FieldIndexTypes.NOT_ANALYZED, false, string.Empty),
new StaticField( "parentID", FieldIndexTypes.NOT_ANALYZED, false, string.Empty),
new StaticField( "level", FieldIndexTypes.NOT_ANALYZED, true, "NUMBER"),
new StaticField( "writerID", FieldIndexTypes.NOT_ANALYZED, false, string.Empty),
new StaticField( "creatorID", FieldIndexTypes.NOT_ANALYZED, false, string.Empty),
new StaticField( "nodeType", FieldIndexTypes.NOT_ANALYZED, false, string.Empty),
new StaticField( "template", FieldIndexTypes.NOT_ANALYZED, false, string.Empty),
new StaticField( "sortOrder", FieldIndexTypes.NOT_ANALYZED, true, "NUMBER"),
new StaticField( "createDate", FieldIndexTypes.NOT_ANALYZED, false, "DATETIME"),
new StaticField( "updateDate", FieldIndexTypes.NOT_ANALYZED, false, "DATETIME"),
new StaticField( "nodeName", FieldIndexTypes.ANALYZED, false, string.Empty),
new StaticField( "urlName", FieldIndexTypes.NOT_ANALYZED, false, string.Empty),
new StaticField( "writerName", FieldIndexTypes.ANALYZED, false, string.Empty),
new StaticField( "creatorName", FieldIndexTypes.ANALYZED, false, string.Empty),
new StaticField( "nodeTypeAlias", FieldIndexTypes.ANALYZED, false, string.Empty),
new StaticField( "path", FieldIndexTypes.NOT_ANALYZED, false, string.Empty)
};
protected ProfilingLogger ProfilingLogger { get; private set; }
/// <summary>
/// Overridden to ensure that the umbraco system field definitions are in place
/// </summary>
/// <param name="originalDefinitions"></param>
/// <returns></returns>
protected override IEnumerable<FieldDefinition> InitializeFieldDefinitions(IEnumerable<FieldDefinition> originalDefinitions)
{
var fd = base.InitializeFieldDefinitions(originalDefinitions).ToList();
fd.AddRange(new[]
{
new FieldDefinition("parentID", FieldDefinitionTypes.Integer),
new FieldDefinition("level", FieldDefinitionTypes.Integer),
new FieldDefinition("writerID", FieldDefinitionTypes.Integer),
new FieldDefinition("creatorID", FieldDefinitionTypes.Integer),
new FieldDefinition("sortOrder", FieldDefinitionTypes.Integer),
new FieldDefinition("template", FieldDefinitionTypes.Integer),
new FieldDefinition("createDate", FieldDefinitionTypes.DateTime),
new FieldDefinition("updateDate", FieldDefinitionTypes.DateTime),
new FieldDefinition("key", FieldDefinitionTypes.Raw),
new FieldDefinition("version", FieldDefinitionTypes.Raw),
new FieldDefinition("nodeType", FieldDefinitionTypes.Raw),
new FieldDefinition("template", FieldDefinitionTypes.Raw),
new FieldDefinition("urlName", FieldDefinitionTypes.Raw),
new FieldDefinition("path", FieldDefinitionTypes.Raw),
new FieldDefinition(IndexPathFieldName, FieldDefinitionTypes.Raw),
new FieldDefinition(NodeTypeAliasFieldName, FieldDefinitionTypes.Raw),
new FieldDefinition(IconFieldName, FieldDefinitionTypes.Raw)
});
return fd;
}
public bool UseTempStorage
{
get { return _localTempStorageIndexer != null && _localTempStorageIndexer.LuceneDirectory != null; }
}
public string TempStorageLocation
{
get
{
if (UseTempStorage == false) return string.Empty;
return _localTempStorageIndexer.TempPath;
}
}
[Obsolete("This should not be used, it is used by the configuration based indexes but instead to disable Examine event handlers use the ExamineEvents class instead.")]
[EditorBrowsable(EditorBrowsableState.Never)]
public bool EnableDefaultEventHandler { get; protected set; }
/// <summary>
/// the supported indexable types
/// </summary>
protected abstract IEnumerable<string> SupportedTypes { get; }
#region Initialize
/// <summary>
/// Setup the properties for the indexer from the provider settings
/// </summary>
/// <param name="name"></param>
/// <param name="config"></param>
/// <remarks>
/// This is ONLY used for configuration based indexes
/// </remarks>
public override void Initialize(string name, System.Collections.Specialized.NameValueCollection config)
{
EnableDefaultEventHandler = true; //set to true by default
bool enabled;
if (bool.TryParse(config["enableDefaultEventHandler"], out enabled))
{
EnableDefaultEventHandler = enabled;
}
ProfilingLogger.Logger.Debug(GetType(), "{0} indexer initializing", () => name);
base.Initialize(name, config);
//NOTES: useTempStorage is obsolete, tempStorageDirectory is obsolete, both have been superceded by Examine Core's IDirectoryFactory
// tempStorageDirectory never actually got finished in Umbraco Core but accidentally got shipped (it's only enabled on the searcher
// and not the indexer). So this whole block is just legacy
//detect if a dir factory has been specified, if so then useTempStorage will not be used (deprecated)
if (config["directoryFactory"] == null && config["useTempStorage"] != null)
{
throw new NotImplementedException("Fix how local temp storage works and is synced with Examine v2.0 - since a writer is always open we cannot snapshot it, we need to use the same logic in AzureDirectory");
_localTempStorageIndexer = new LocalTempStorageIndexer();
var fsDir = base.GetLuceneDirectory() as FSDirectory;
if (fsDir != null)
{
//Use the temp storage directory which will store the index in the local/codegen folder, this is useful
// for websites that are running from a remove file server and file IO latency becomes an issue
var attemptUseTempStorage = config["useTempStorage"].TryConvertTo<LocalStorageType>();
if (attemptUseTempStorage)
{
var indexSet = IndexSets.Instance.Sets[IndexSetName];
var configuredPath = indexSet.IndexPath;
_localTempStorageIndexer.Initialize(config, configuredPath, fsDir, IndexingAnalyzer, attemptUseTempStorage.Result);
}
}
}
}
#endregion
public override Directory GetLuceneDirectory()
{
//if temp local storage is configured use that, otherwise return the default
if (UseTempStorage)
{
return _localTempStorageIndexer.LuceneDirectory;
}
return base.GetLuceneDirectory();
}
/// <summary>
/// override to check if we can actually initialize.
/// </summary>
/// <remarks>
/// This check is required since the base examine lib will try to rebuild on startup
/// </remarks>
public override void RebuildIndex()
{
if (CanInitialize())
{
ProfilingLogger.Logger.Debug(GetType(), "Rebuilding index");
using (new SafeCallContext())
{
base.RebuildIndex();
}
}
}
/// <summary>
/// override to check if we can actually initialize.
/// </summary>
/// <remarks>
/// This check is required since the base examine lib will try to rebuild on startup
/// </remarks>
public override void IndexAll(string type)
{
if (CanInitialize())
{
using (new SafeCallContext())
{
base.IndexAll(type);
}
}
}
public override void IndexItems(IEnumerable<ValueSet> nodes)
{
if (CanInitialize())
{
using (new SafeCallContext())
{
base.IndexItems(nodes);
}
}
}
[Obsolete("Use ValueSets with IndexItems instead")]
public override void ReIndexNode(XElement node, string type)
{
if (CanInitialize())
{
if (SupportedTypes.Contains(type) == false)
return;
if (node.Attribute("id") != null)
{
ProfilingLogger.Logger.Debug(GetType(), "ReIndexNode {0} with type {1}", () => node.Attribute("id"), () => type);
using (new SafeCallContext())
{
base.ReIndexNode(node, type);
}
}
else
{
ProfilingLogger.Logger.Error(GetType(), "ReIndexNode cannot proceed, the format of the XElement is invalid",
new XmlException("XElement is invalid, the xml has not id attribute"));
}
}
}
/// <summary>
/// override to check if we can actually initialize.
/// </summary>
/// <remarks>
/// This check is required since the base examine lib will try to rebuild on startup
/// </remarks>
public override void DeleteFromIndex(string nodeId)
{
if (CanInitialize())
{
using (new SafeCallContext())
{
base.DeleteFromIndex(nodeId);
}
}
}
/// <summary>
/// Returns true if the Umbraco application is in a state that we can initialize the examine indexes
/// </summary>
protected bool CanInitialize()
{
// only affects indexers that are config file based, if an index was created via code then
// this has no effect, it is assumed the index would not be created if it could not be initialized
return _configBased == false || Current.RuntimeState.Level == RuntimeLevel.Run;
}
/// <summary>
/// Reindexes all supported types
/// </summary>
protected override void PerformIndexRebuild()
{
foreach (var t in SupportedTypes)
{
IndexAll(t);
}
}
/// <summary>
/// overridden for logging
/// </summary>
/// <param name="e"></param>
protected override void OnIndexingError(IndexingErrorEventArgs e)
{
ProfilingLogger.Logger.Error(GetType(), e.Message, e.Exception);
base.OnIndexingError(e);
}
/// <summary>
/// Override for logging
/// </summary>
/// <param name="e"></param>
protected override void OnIgnoringIndexItem(IndexItemEventArgs e)
{
ProfilingLogger.Logger.Debug(GetType(), "OnIgnoringIndexItem {0} with type {1}", () => e.IndexItem.ValueSet.Id, () => e.IndexItem.ValueSet.IndexCategory);
base.OnIgnoringIndexItem(e);
}
/// <summary>
/// This ensures that the special __Raw_ fields are indexed
/// </summary>
/// <param name="docArgs"></param>
protected override void OnDocumentWriting(DocumentWritingEventArgs docArgs)
{
var d = docArgs.Document;
foreach (var f in docArgs.Values.Values.Where(x => x.Key.StartsWith(RawFieldPrefix)))
{
if (f.Value.Count > 0)
{
d.Add(new Field(
f.Key,
f.Value[0].ToString(),
Field.Store.YES,
Field.Index.NO, //don't index this field, we never want to search by it
Field.TermVector.NO));
}
}
ProfilingLogger.Logger.Debug(GetType(), "OnDocumentWriting {0} with type {1}", () => docArgs.Values.Id, () => docArgs.Values.ItemType);
base.OnDocumentWriting(docArgs);
}
protected override void OnItemIndexed(IndexItemEventArgs e)
{
ProfilingLogger.Logger.Debug(GetType(), "Index created for node {0}", () => e.IndexItem.Id);
base.OnItemIndexed(e);
}
protected override void OnIndexDeleted(DeleteIndexEventArgs e)
{
ProfilingLogger.Logger.Debug(GetType(), "Index deleted for term: {0} with value {1}", () => e.DeletedTerm.Key, () => e.DeletedTerm.Value);
base.OnIndexDeleted(e);
}
[Obsolete("This is no longer used, index optimization is no longer managed with the LuceneIndexer")]
protected override void OnIndexOptimizing(EventArgs e)
{
ProfilingLogger.Logger.Debug(GetType(), "Index is being optimized");
base.OnIndexOptimizing(e);
}
/// <summary>
/// Overridden for logging.
/// </summary>
/// <param name="values"></param>
protected override void AddDocument(ValueSet values)
{
ProfilingLogger.Logger.Debug(GetType(), "AddDocument {0} with type {1}", () => values.Id, () => values.ItemType);
base.AddDocument(values);
}
protected override void OnTransformingIndexValues(TransformingIndexDataEventArgs e)
{
base.OnTransformingIndexValues(e);
//ensure special __Path field
if (e.OriginalValues.ContainsKey("path") && e.IndexItem.ValueSet.Values.ContainsKey(IndexPathFieldName) == false)
{
e.IndexItem.ValueSet.Values[IndexPathFieldName] = new List<object> { e.OriginalValues["path"].First() };
}
//strip html of all users fields if we detect it has HTML in it.
//if that is the case, we'll create a duplicate 'raw' copy of it so that we can return
//the value of the field 'as-is'.
foreach (var originalValue in e.OriginalValues)
{
if (originalValue.Value.Any())
{
var str = originalValue.Value.First() as string;
if (str != null)
{
if (XmlHelper.CouldItBeXml(str))
{
//First save the raw value to a raw field, we will change the policy of this field by detecting the prefix later
e.IndexItem.ValueSet.Values[string.Concat(RawFieldPrefix, originalValue.Key)] = new List<object> { str };
//now replace the original value with the stripped html
//TODO: This should be done with an analzer?!
e.IndexItem.ValueSet.Values[originalValue.Key] = new List<object> { str.StripHtml() };
}
}
}
}
//icon
if (e.OriginalValues.ContainsKey("icon") && e.IndexItem.ValueSet.Values.ContainsKey(IconFieldName) == false)
{
e.IndexItem.ValueSet.Values[IconFieldName] = new List<object> { e.OriginalValues["icon"] };
}
}
}
}

36
src/Umbraco.Examine/Config/ConfigIndexCriteria.cs Normal file
View File

@@ -0,0 +1,36 @@
using System.Collections.Generic;
using System.Linq;
using Examine;
namespace Umbraco.Examine.Config
{
/// <summary>
/// a data structure for storing indexing/searching instructions based on config based indexers
/// </summary>
public class ConfigIndexCriteria
{
///<summary>
/// Constructor
///</summary>
///<param name="standardFields"></param>
///<param name="userFields"></param>
///<param name="includeNodeTypes"></param>
///<param name="excludeNodeTypes"></param>
///<param name="parentNodeId"></param>
public ConfigIndexCriteria(IEnumerable<FieldDefinition> standardFields, IEnumerable<FieldDefinition> userFields, IEnumerable<string> includeNodeTypes, IEnumerable<string> excludeNodeTypes, int? parentNodeId)
{
UserFields = userFields.ToList();
StandardFields = standardFields.ToList();
IncludeItemTypes = includeNodeTypes;
ExcludeItemTypes = excludeNodeTypes;
ParentNodeId = parentNodeId;
}
public IEnumerable<FieldDefinition> StandardFields { get; internal set; }
public IEnumerable<FieldDefinition> UserFields { get; internal set; }
public IEnumerable<string> IncludeItemTypes { get; internal set; }
public IEnumerable<string> ExcludeItemTypes { get; internal set; }
public int? ParentNodeId { get; internal set; }
}
}

61
src/Umbraco.Examine/Config/ConfigIndexField.cs Normal file
View File

@@ -0,0 +1,61 @@
using System.Configuration;
namespace Umbraco.Examine.Config
{
///<summary>
/// A configuration item representing a field to index
///</summary>
public sealed class ConfigIndexField : ConfigurationElement
{
[ConfigurationProperty("Name", IsRequired = true)]
public string Name
{
get => (string)this["Name"];
set => this["Name"] = value;
}
[ConfigurationProperty("EnableSorting", IsRequired = false)]
public bool EnableSorting
{
get => (bool)this["EnableSorting"];
set => this["EnableSorting"] = value;
}
[ConfigurationProperty("Type", IsRequired = false, DefaultValue = "String")]
public string Type
{
get => (string)this["Type"];
set => this["Type"] = value;
}
public bool Equals(ConfigIndexField other)
{
if (ReferenceEquals(null, other)) return false;
if (ReferenceEquals(this, other)) return true;
return string.Equals(Name, other.Name);
}
public override bool Equals(object obj)
{
if (ReferenceEquals(null, obj)) return false;
if (ReferenceEquals(this, obj)) return true;
if (obj.GetType() != this.GetType()) return false;
return Equals((ConfigIndexField)obj);
}
public override int GetHashCode()
{
return Name.GetHashCode();
}
public static bool operator ==(ConfigIndexField left, ConfigIndexField right)
{
return Equals(left, right);
}
public static bool operator !=(ConfigIndexField left, ConfigIndexField right)
{
return !Equals(left, right);
}
}
}

15
src/Umbraco.Examine/Config/IndexFieldCollectionExtensions.cs Normal file
View File

@@ -0,0 +1,15 @@
using System.Collections.Generic;
namespace Umbraco.Examine.Config
{
public static class IndexFieldCollectionExtensions
{
public static List<ConfigIndexField> ToList(this IndexFieldCollection indexes)
{
List<ConfigIndexField> fields = new List<ConfigIndexField>();
foreach (ConfigIndexField field in indexes)
fields.Add(field);
return fields;
}
}
}

122
src/Umbraco.Examine/Config/IndexSet.cs Normal file
View File

@@ -0,0 +1,122 @@
using System.Configuration;
using System.IO;
using System.Web;
using System.Web.Hosting;
namespace Umbraco.Examine.Config
{
public sealed class IndexSet : ConfigurationElement
{
[ConfigurationProperty("SetName", IsRequired = true, IsKey = true)]
public string SetName => (string)this["SetName"];
private string _indexPath = "";
/// <summary>
/// The folder path of where the lucene index is stored
/// </summary>
/// <value>The index path.</value>
/// <remarks>
/// This can be set at runtime but will not be persisted to the configuration file
/// </remarks>
[ConfigurationProperty("IndexPath", IsRequired = true, IsKey = false)]
public string IndexPath
{
get
{
if (string.IsNullOrEmpty(_indexPath))
_indexPath = (string)this["IndexPath"];
return _indexPath;
}
set => _indexPath = value;
}
/// <summary>
/// Returns the DirectoryInfo object for the index path.
/// </summary>
/// <value>The index directory.</value>
public DirectoryInfo IndexDirectory
{
get
{
//TODO: Get this out of the index set. We need to use the Indexer's DataService to lookup the folder so it can be unit tested. Probably need DataServices on the searcher then too
//we need to de-couple the context
if (HttpContext.Current != null)
return new DirectoryInfo(HttpContext.Current.Server.MapPath(this.IndexPath));
else if (HostingEnvironment.ApplicationID != null)
return new DirectoryInfo(HostingEnvironment.MapPath(this.IndexPath));
else
return new DirectoryInfo(this.IndexPath);
}
}
/// <summary>
/// When this property is set, the indexing will only index documents that are descendants of this node.
/// </summary>
[ConfigurationProperty("IndexParentId", IsRequired = false, IsKey = false)]
public int? IndexParentId
{
get
{
if (this["IndexParentId"] == null)
return null;
return (int)this["IndexParentId"];
}
}
/// <summary>
/// The collection of node types to index, if not specified, all node types will be indexed (apart from the ones specified in the ExcludeNodeTypes collection).
/// </summary>
[ConfigurationCollection(typeof(IndexFieldCollection))]
[ConfigurationProperty("IncludeNodeTypes", IsDefaultCollection = false, IsRequired = false)]
public IndexFieldCollection IncludeNodeTypes => (IndexFieldCollection)base["IncludeNodeTypes"];
/// <summary>
/// The collection of node types to not index. If specified, these node types will not be indexed.
/// </summary>
[ConfigurationCollection(typeof(IndexFieldCollection))]
[ConfigurationProperty("ExcludeNodeTypes", IsDefaultCollection = false, IsRequired = false)]
public IndexFieldCollection ExcludeNodeTypes => (IndexFieldCollection)base["ExcludeNodeTypes"];
/// <summary>
/// A collection of user defined umbraco fields to index
/// </summary>
/// <remarks>
/// If this property is not specified, or if it's an empty collection, the default user fields will be all user fields defined in Umbraco
/// </remarks>
[ConfigurationCollection(typeof(IndexFieldCollection))]
[ConfigurationProperty("IndexUserFields", IsDefaultCollection = false, IsRequired = false)]
public IndexFieldCollection IndexUserFields => (IndexFieldCollection)base["IndexUserFields"];
/// <summary>
/// The fields umbraco values that will be indexed. i.e. id, nodeTypeAlias, writer, etc...
/// </summary>
/// <remarks>
/// If this is not specified, or if it's an empty collection, the default optins will be specified:
/// - id
/// - version
/// - parentID
/// - level
/// - writerID
/// - creatorID
/// - nodeType
/// - template
/// - sortOrder
/// - createDate
/// - updateDate
/// - nodeName
/// - urlName
/// - writerName
/// - creatorName
/// - nodeTypeAlias
/// - path
/// </remarks>
[ConfigurationCollection(typeof(IndexFieldCollection))]
[ConfigurationProperty("IndexAttributeFields", IsDefaultCollection = false, IsRequired = false)]
public IndexFieldCollection IndexAttributeFields => (IndexFieldCollection)base["IndexAttributeFields"];
}
}

73
src/Umbraco.Examine/Config/IndexSetCollection.cs Normal file
View File

@@ -0,0 +1,73 @@
using System.Configuration;
namespace Umbraco.Examine.Config
{
public sealed class IndexFieldCollection : ConfigurationElementCollection
{
#region Overridden methods to define collection
protected override ConfigurationElement CreateNewElement()
{
return new ConfigIndexField();
}
protected override object GetElementKey(ConfigurationElement element)
{
ConfigIndexField field = (ConfigIndexField)element;
return field.Name;
}
public override bool IsReadOnly()
{
return false;
}
#endregion
/// <summary>
/// Adds an index field to the collection
/// </summary>
/// <param name="field"></param>
public void Add(ConfigIndexField field)
{
BaseAdd(field, true);
}
/// <summary>
/// Default property for accessing an IndexField definition
/// </summary>
/// <value>Field Name</value>
/// <returns></returns>
public new ConfigIndexField this[string name]
{
get
{
return (ConfigIndexField)this.BaseGet(name);
}
}
}
public sealed class IndexSetCollection : ConfigurationElementCollection
{
#region Overridden methods to define collection
protected override ConfigurationElement CreateNewElement()
{
return new IndexSet();
}
protected override object GetElementKey(ConfigurationElement element)
{
return ((IndexSet)element).SetName;
}
public override ConfigurationElementCollectionType CollectionType => ConfigurationElementCollectionType.BasicMap;
protected override string ElementName => "IndexSet";
#endregion
/// <summary>
/// Default property for accessing Image Sets
/// </summary>
/// <param name="setName"></param>
/// <returns></returns>
public new IndexSet this[string setName] => (IndexSet)this.BaseGet(setName);
}
}

26
src/Umbraco.Examine/Config/IndexSets.cs Normal file
View File

@@ -0,0 +1,26 @@
using System.Configuration;
namespace Umbraco.Examine.Config
{
public sealed class IndexSets : ConfigurationSection
{
#region Singleton definition
protected IndexSets() { }
static IndexSets()
{
Instance = ConfigurationManager.GetSection(SectionName) as IndexSets;
}
public static IndexSets Instance { get; }
#endregion
private const string SectionName = "ExamineLuceneIndexSets";
[ConfigurationCollection(typeof(IndexSetCollection))]
[ConfigurationProperty("", IsDefaultCollection = true, IsRequired = true)]
public IndexSetCollection Sets => (IndexSetCollection)base[""];
}
}

115
src/Umbraco.Examine/DataServices/UmbracoContentService.cs
View File

@@ -1,115 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using Umbraco.Core;
using System.Xml.Linq;
using System.Collections;
using System.Xml.XPath;
using Examine.LuceneEngine;
using Umbraco.Core.Logging;
using Umbraco.Core.Scoping;
using Umbraco.Core.Services;
using Umbraco.Core.Models;
namespace Umbraco.Examine.DataServices
{
public class UmbracoContentService
{
private readonly IScopeProvider _scopeProvider;
private readonly ServiceContext _services;
private readonly ILogger _logger;
public UmbracoContentService(IScopeProvider scopeProvider, ServiceContext services, ILogger logger)
{
_scopeProvider = scopeProvider;
_services = services;
_logger = logger;
}
/// <summary>
/// removes html markup from a string
/// </summary>
/// <param name="value"></param>
/// <returns></returns>
public string StripHtml(string value)
{
return value.StripHtml();
}
/// <summary>
/// Gets published content by xpath
/// </summary>
/// <param name="xpath"></param>
/// <returns></returns>
public XDocument GetPublishedContentByXPath(string xpath)
{
//TODO: Remove the need for this, the best way would be to remove all requirements of examine based on Xml but that
// would take some time. Another way in the in-term would be to add a static delegate to this class which can be set
// on the WebBootManager to set how to get the XmlNodeByXPath but that is still ugly :(
return LegacyLibrary.GetXmlNodeByXPath(xpath).ToXDocument();
}
/// <summary>
/// This is quite an intensive operation...
/// get all root content, then get the XML structure for all children,
/// then run xpath against the navigator that's created
/// </summary>
/// <param name="xpath"></param>
/// <returns></returns>
[Obsolete("This should no longer be used, latest content will be indexed by using the IContentService directly")]
public XDocument GetLatestContentByXPath(string xpath)
{
var xmlContent = XDocument.Parse("<content></content>");
foreach (var c in _services.ContentService.GetRootContent())
{
xmlContent.Root.Add(c.ToDeepXml(_services.PackagingService));
}
var result = ((IEnumerable)xmlContent.XPathEvaluate(xpath)).Cast<XElement>();
return result.ToXDocument();
}
/// <summary>
/// Check if the node is protected
/// </summary>
/// <param name="nodeId"></param>
/// <param name="path"></param>
/// <returns></returns>
public bool IsProtected(int nodeId, string path)
{
return _services.PublicAccessService.IsProtected(path.EnsureEndsWith("," + nodeId));
}
/// <summary>
/// Returns a list of all of the user defined property names in Umbraco
/// </summary>
/// <returns></returns>
public IEnumerable<string> GetAllUserPropertyNames()
{
using (var scope = _scopeProvider.CreateScope())
{
try
{
var result = scope.Database.Fetch<string>("select distinct alias from cmsPropertyType order by alias");
scope.Complete();
return result;
}
catch (Exception ex)
{
_logger.Error<UmbracoContentService>("EXCEPTION OCCURRED reading GetAllUserPropertyNames", ex);
scope.Complete();
return Enumerable.Empty<string>();
}
}
}
/// <summary>
/// Returns a list of all system field names in Umbraco
/// </summary>
/// <returns></returns>
public IEnumerable<string> GetAllSystemPropertyNames()
{
return UmbracoContentIndexer.IndexFieldPolicies.Select(x => x.Name);
}
}
}

45
src/Umbraco.Examine/DataServices/UmbracoMediaService.cs
View File

@@ -1,45 +0,0 @@
using System;
using System.Collections;
using System.Linq;
using System.Xml.Linq;
using System.Xml.XPath;
using Examine.LuceneEngine;
using Umbraco.Core.Services;
using Umbraco.Core.Models;
namespace Umbraco.Examine.DataServices
{
/// <summary>
/// Data service used to query for media
/// </summary>
[Obsolete("This should no longer be used, latest content will be indexed by using the IMediaService directly")]
public class UmbracoMediaService
{
private readonly ServiceContext _services;
public UmbracoMediaService(ServiceContext services)
{
_services = services;
}
/// <summary>
/// This is quite an intensive operation...
/// get all root media, then get the XML structure for all children,
/// then run xpath against the navigator that's created
/// </summary>
/// <param name="xpath"></param>
/// <returns></returns>
[Obsolete("This should no longer be used, latest content will be indexed by using the IMediaService directly")]
public XDocument GetLatestMediaByXpath(string xpath)
{
var xmlMedia = XDocument.Parse("<media></media>");
foreach (var m in _services.MediaService.GetRootMedia())
{
xmlMedia.Root.Add(m.ToDeepXml(_services.PackagingService));
}
var result = ((IEnumerable)xmlMedia.XPathEvaluate(xpath)).Cast<XElement>();
return result.ToXDocument();
}
}
}

23
src/Umbraco.Examine/DeletePolicyTracker.cs
View File

@@ -1,23 +0,0 @@
using System.Collections.Concurrent;
using System.IO;
using Lucene.Net.Index;
namespace Umbraco.Examine
{
internal sealed class DeletePolicyTracker
{
private static readonly DeletePolicyTracker Instance = new DeletePolicyTracker();
private readonly ConcurrentDictionary<string, IndexDeletionPolicy> _directories = new ConcurrentDictionary<string, IndexDeletionPolicy>();
public static DeletePolicyTracker Current
{
get { return Instance; }
}
public IndexDeletionPolicy GetPolicy(Lucene.Net.Store.Directory directory)
{
var resolved = _directories.GetOrAdd(directory.GetLockId(), s => new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()));
return resolved;
}
}
}

13
src/Umbraco.Examine/ExamineIndexCollectionAccessor.cs
View File

@@ -1,13 +0,0 @@
using System.Collections.Generic;
using Examine;
namespace Umbraco.Examine
{
/// <summary>
/// Default implementation of IExamineIndexCollectionAccessor to return indexes from Examinemanager
/// </summary>
public class ExamineIndexCollectionAccessor : IExamineIndexCollectionAccessor
{
public IReadOnlyDictionary<string, IExamineIndexer> Indexes => ExamineManager.Instance.IndexProviders;
}
}

13
src/Umbraco.Examine/IExamineIndexCollectionAccessor.cs
View File

@@ -1,13 +0,0 @@
using System.Collections.Generic;
using Examine;
namespace Umbraco.Examine
{
/// <summary>
/// Returns a collection of IExamineIndexer
/// </summary>
public interface IExamineIndexCollectionAccessor
{
IReadOnlyDictionary<string, IExamineIndexer> Indexes { get; }
}
}

50
src/Umbraco.Examine/LegacyExtensions.cs
View File

@@ -1,50 +0,0 @@
using System.Linq;
using Examine;
using Examine.LuceneEngine.Config;
using Umbraco.Core.Services;
namespace Umbraco.Examine
{
internal static class LegacyExtensions
{
private static readonly object Locker = new object();
public static IIndexCriteria ToIndexCriteria(this IndexSet set, IContentTypeService contentTypeService)
{
if (set.IndexUserFields.Count == 0)
{
lock (Locker)
{
//we need to add all user fields to the collection if it is empty (this is the default if none are specified)
var userFields = contentTypeService.GetAllPropertyTypeAliases();
foreach (var u in userFields)
{
set.IndexUserFields.Add(new IndexField() { Name = u });
}
}
}
if (set.IndexAttributeFields.Count == 0)
{
lock (Locker)
{
//we need to add all system fields to the collection if it is empty (this is the default if none are specified)
var sysFields = BaseUmbracoIndexer.IndexFieldPolicies.Select(x => x.Name);
foreach (var s in sysFields)
{
set.IndexAttributeFields.Add(new IndexField() { Name = s });
}
}
}
return new IndexCriteria(
set.IndexAttributeFields.Cast<IIndexField>().ToArray(),
set.IndexUserFields.Cast<IIndexField>().ToArray(),
set.IncludeNodeTypes.ToList().Select(x => x.Name).ToArray(),
set.ExcludeNodeTypes.ToList().Select(x => x.Name).ToArray(),
set.IndexParentId);
}
}
}

47
src/Umbraco.Examine/LegacyLibrary.cs
View File

@@ -1,47 +0,0 @@
using System;
using System.Reflection;
using System.Xml.XPath;
namespace Umbraco.Examine
{
/// <summary>
/// This is only used for backward compatibility to get access to the umbraco.library object but this needs to be done
/// via reflection because of the circular reference we have between Umbraco.Web and UmbracoExamine.
/// </summary>
internal static class LegacyLibrary
{
private static volatile Type _libraryType;
private static readonly object Locker = new object();
private static Type LibraryType
{
get
{
if (_libraryType == null)
{
lock (Locker)
{
if (_libraryType == null)
{
var ass = Assembly.Load("Umbraco.Web");
if (ass == null)
throw new InvalidOperationException("Could not load assembly Umbraco.Web.dll, the Umbraco.Web.dll needs to be loaded in the current app domain");
var lib = ass.GetType("umbraco.library");
if (lib == null)
throw new InvalidOperationException("Could not load type umbraco.library, the Umbraco.Web.dll needs to be loaded in the current app domain");
_libraryType = lib;
}
}
}
return _libraryType;
}
}
internal static XPathNodeIterator GetXmlNodeByXPath(string xpathQuery)
{
var meth = LibraryType.GetMethod("GetXmlNodeByXPath", BindingFlags.Public | BindingFlags.Static);
return (XPathNodeIterator)meth.Invoke(null, new object[] { xpathQuery });
}
}
}

33
src/Umbraco.Examine/LocalStorage/AzureLocalStorageDirectory.cs
View File

@@ -1,33 +0,0 @@
using System;
using System.Collections.Specialized;
using System.ComponentModel;
using System.IO;
using System.Web;
using Umbraco.Core;
namespace Umbraco.Examine.LocalStorage
{
/// <summary>
/// When running on Azure websites, we can use the local user's storage space
/// </summary>
[Obsolete("This has been superceded by IDirectoryFactory in Examine Core and should not be used")]
[EditorBrowsable(EditorBrowsableState.Never)]
public sealed class AzureLocalStorageDirectory : ILocalStorageDirectory
{
public DirectoryInfo GetLocalStorageDirectory(NameValueCollection config, string configuredPath)
{
var appDomainHash = HttpRuntime.AppDomainAppId.GenerateHash();
var cachePath = Path.Combine(Environment.ExpandEnvironmentVariables("%temp%"), "LuceneDir",
//include the appdomain hash is just a safety check, for example if a website is moved from worker A to worker B and then back
// to worker A again, in theory the %temp% folder should already be empty but we really want to make sure that its not
// utilizing an old index
appDomainHash,
//ensure the temp path is consistent with the configured path location
configuredPath.TrimStart('~', '/').Replace("/", "\\"));
var azureDir = new DirectoryInfo(cachePath);
if (azureDir.Exists == false)
azureDir.Create();
return azureDir;
}
}
}

29
src/Umbraco.Examine/LocalStorage/CodeGenLocalStorageDirectory.cs
View File

@@ -1,29 +0,0 @@
using System;
using System.Collections.Specialized;
using System.ComponentModel;
using System.IO;
using System.Web;
namespace Umbraco.Examine.LocalStorage
{
/// <summary>
/// Use the ASP.Net CodeGen folder to store the index files
/// </summary>
/// <remarks>
/// This is the default implementation - but it comes with it's own limitations - the CodeGen folder is cleared whenever new
/// DLLs are changed in the /bin folder (among other circumstances) which means the index would be re-synced (or rebuilt) there.
/// </remarks>
[Obsolete("This has been superceded by IDirectoryFactory in Examine Core and should not be used")]
[EditorBrowsable(EditorBrowsableState.Never)]
public sealed class CodeGenLocalStorageDirectory : ILocalStorageDirectory
{
public DirectoryInfo GetLocalStorageDirectory(NameValueCollection config, string configuredPath)
{
var codegenPath = HttpRuntime.CodegenDir;
var path = Path.Combine(codegenPath,
//ensure the temp path is consistent with the configured path location
configuredPath.TrimStart('~', '/').Replace("/", "\\"));
return new DirectoryInfo(path);
}
}
}

26
src/Umbraco.Examine/LocalStorage/DirectoryTracker.cs
View File

@@ -1,26 +0,0 @@
using System;
using System.Collections.Concurrent;
using System.IO;
using Lucene.Net.Store;
namespace Umbraco.Examine.LocalStorage
{
[Obsolete("This is used only for configuration based indexes")]
internal class DirectoryTracker
{
private static readonly DirectoryTracker Instance = new DirectoryTracker();
private readonly ConcurrentDictionary<string, Lucene.Net.Store.Directory> _directories = new ConcurrentDictionary<string, Lucene.Net.Store.Directory>();
public static DirectoryTracker Current
{
get { return Instance; }
}
public Lucene.Net.Store.Directory GetDirectory(DirectoryInfo dir)
{
var resolved = _directories.GetOrAdd(dir.FullName, s => new SimpleFSDirectory(dir));
return resolved;
}
}
}

13
src/Umbraco.Examine/LocalStorage/ILocalStorageDirectory.cs
View File

@@ -1,13 +0,0 @@
using System.Collections.Specialized;
using System.IO;
namespace Umbraco.Examine.LocalStorage
{
/// <summary>
/// Used to resolve the local storage folder
/// </summary>
public interface ILocalStorageDirectory
{
DirectoryInfo GetLocalStorageDirectory(NameValueCollection config, string configuredPath);
}
}

8
src/Umbraco.Examine/LocalStorage/LocalStorageType.cs
View File

@@ -1,8 +0,0 @@
namespace Umbraco.Examine.LocalStorage
{
public enum LocalStorageType
{
Sync,
LocalOnly
}
}

207
src/Umbraco.Examine/LocalStorage/LocalTempStorageDirectory.cs
View File

@@ -1,207 +0,0 @@
using System;
using System.IO;
using System.Linq;
using Lucene.Net.Store;
using Directory = Lucene.Net.Store.Directory;
namespace Umbraco.Examine.LocalStorage
{
/// <summary>
/// Used to read data from local temp storage and write to both local storage and main storage
/// </summary>
public class LocalTempStorageDirectory : Directory
{
private readonly FSDirectory _tempStorageDir;
private readonly FSDirectory _realDirectory;
private LockFactory _lockFactory;
public LocalTempStorageDirectory(
DirectoryInfo tempStorageDir,
FSDirectory realDirectory)
{
if (tempStorageDir == null) throw new ArgumentNullException("tempStorageDir");
if (realDirectory == null) throw new ArgumentNullException("realDirectory");
_tempStorageDir = new SimpleFSDirectory(tempStorageDir);
_realDirectory = realDirectory;
_lockFactory = new MultiIndexLockFactory(_realDirectory, _tempStorageDir);
Enabled = true;
}
/// <summary>
/// If initialization fails, it will be disabled and then this will just wrap the 'real directory'
/// </summary>
internal bool Enabled { get; set; }
public override string[] ListAll()
{
//always from the real dir
return _realDirectory.ListAll();
}
/// <summary>Returns true if a file with the given name exists. </summary>
public override bool FileExists(string name)
{
//always from the real dir
return _realDirectory.FileExists(name);
}
/// <summary>Returns the time the named file was last modified. </summary>
public override long FileModified(string name)
{
//always from the real dir
return _realDirectory.FileModified(name);
}
/// <summary>Set the modified time of an existing file to now. </summary>
public override void TouchFile(string name)
{
//always from the real dir
_realDirectory.TouchFile(name);
//perform on both dirs
if (Enabled)
{
_tempStorageDir.TouchFile(name);
}
}
/// <summary>Removes an existing file in the directory. </summary>
public override void DeleteFile(string name)
{
_realDirectory.DeleteFile(name);
//perform on both dirs
if (Enabled)
{
_tempStorageDir.DeleteFile(name);
}
}
/// <summary>Returns the length of a file in the directory. </summary>
public override long FileLength(string name)
{
//always from the real dir
return _realDirectory.FileLength(name);
}
/// <summary>
/// Creates a new, empty file in the directory with the given name.
/// Returns a stream writing this file.
/// </summary>
public override IndexOutput CreateOutput(string name)
{
//write to both indexes
if (Enabled)
{
return new MultiIndexOutput(
_tempStorageDir.CreateOutput(name),
_realDirectory.CreateOutput(name));
}
return _realDirectory.CreateOutput(name);
}
/// <summary>
/// Returns a stream reading an existing file.
/// </summary>
public override IndexInput OpenInput(string name)
{
if (Enabled)
{
//return the reader from the cache, not the real dir
return _tempStorageDir.OpenInput(name);
}
return _realDirectory.OpenInput(name);
}
/// <summary>
/// Creates an IndexInput for the file with the given name.
/// </summary>
public override IndexInput OpenInput(string name, int bufferSize)
{
if (Enabled)
{
//return the reader from the cache, not the real dir
return _tempStorageDir.OpenInput(name, bufferSize);
}
return _realDirectory.OpenInput(name, bufferSize);
}
/// <summary>
/// Ensure that any writes to this file are moved to
/// stable storage. Lucene uses this to properly commit
/// changes to the index, to prevent a machine/OS crash
/// from corrupting the index.
/// </summary>
public override void Sync(string name)
{
_realDirectory.Sync(name);
_tempStorageDir.Sync(name);
base.Sync(name);
}
public override Lock MakeLock(string name)
{
return _lockFactory.MakeLock(name);
}
/// <summary>
/// Return a string identifier that uniquely differentiates
/// this Directory instance from other Directory instances.
/// This ID should be the same if two Directory instances
/// (even in different JVMs and/or on different machines)
/// are considered "the same index". This is how locking
/// "scopes" to the right index.
///
/// </summary>
public override string GetLockId()
{
return string.Concat(_realDirectory.GetLockId(), _tempStorageDir.GetLockId());
}
public override LockFactory LockFactory
{
get
{
return _lockFactory;
//return _realDirectory.GetLockFactory();
}
}
public override void ClearLock(string name)
{
_lockFactory.ClearLock(name);
//_realDirectory.ClearLock(name);
//if (Enabled)
//{
// _tempStorageDir.ClearLock(name);
//}
}
protected override void Dispose(bool disposing)
{
if (Enabled)
{
_tempStorageDir.Dispose();
}
_realDirectory.Dispose();
}
public override void SetLockFactory(LockFactory lf)
{
_lockFactory = lf;
//_realDirectory.SetLockFactory(lf);
}
}
}

29
src/Umbraco.Examine/LocalStorage/LocalTempStorageDirectoryTracker.cs
View File

@@ -1,29 +0,0 @@
using System.Collections.Concurrent;
using System.IO;
using Lucene.Net.Store;
namespace Umbraco.Examine.LocalStorage
{
internal class LocalTempStorageDirectoryTracker
{
private static readonly LocalTempStorageDirectoryTracker Instance = new LocalTempStorageDirectoryTracker();
private readonly ConcurrentDictionary<string, LocalTempStorageDirectory> _directories = new ConcurrentDictionary<string, LocalTempStorageDirectory>();
public static LocalTempStorageDirectoryTracker Current
{
get { return Instance; }
}
public LocalTempStorageDirectory GetDirectory(DirectoryInfo dir, FSDirectory realDir, bool disable = false)
{
var resolved = _directories.GetOrAdd(dir.FullName, s => new LocalTempStorageDirectory(dir, realDir));
if (disable)
{
resolved.Enabled = false;
}
return resolved;
}
}
}

412
src/Umbraco.Examine/LocalStorage/LocalTempStorageIndexer.cs
View File

@@ -1,412 +0,0 @@
using System;
using System.Collections.Specialized;
using System.IO;
using System.Linq;
using System.Threading;
using System.Web;
using System.Web.Compilation;
using Examine.LuceneEngine;
using Lucene.Net.Analysis;
using Lucene.Net.Index;
using Lucene.Net.Store;
using Umbraco.Core;
using Umbraco.Core.Composing;
using Umbraco.Core.IO;
using Umbraco.Core.Logging;
using Directory = System.IO.Directory;
namespace Umbraco.Examine.LocalStorage
{
internal enum InitializeDirectoryFlags
{
Success = 0,
SuccessNoIndexExists = 1,
FailedCorrupt = 100,
FailedLocked = 101,
FailedFileSync = 102
}
internal class LocalTempStorageIndexer
{
public Lucene.Net.Store.Directory LuceneDirectory { get; private set; }
private readonly object _locker = new object();
public SnapshotDeletionPolicy Snapshotter { get; private set; }
public string TempPath { get; private set; }
public LocalTempStorageIndexer()
{
throw new NotImplementedException("Fix how local temp storage works and is synced with Examine v2.0 - since a writer is always open we cannot snapshot it, we need to use the same logic in AzureDirectory");
IndexDeletionPolicy policy = new KeepOnlyLastCommitDeletionPolicy();
Snapshotter = new SnapshotDeletionPolicy(policy);
}
public void Initialize(NameValueCollection config, string configuredPath, FSDirectory baseLuceneDirectory, Analyzer analyzer, LocalStorageType localStorageType)
{
//this is the default
ILocalStorageDirectory localStorageDir = new CodeGenLocalStorageDirectory();
if (config["tempStorageDirectory"] != null)
{
//try to get the type
var dirType = BuildManager.GetType(config["tempStorageDirectory"], false);
if (dirType != null)
{
try
{
localStorageDir = (ILocalStorageDirectory)Activator.CreateInstance(dirType);
}
catch (Exception ex)
{
Current.Logger.Error<LocalTempStorageIndexer>(
string.Format("Could not create a temp storage location of type {0}, reverting to use the " + typeof (CodeGenLocalStorageDirectory).FullName, dirType),
ex);
}
}
}
var tempPath = localStorageDir.GetLocalStorageDirectory(config, configuredPath);
if (tempPath == null) throw new InvalidOperationException("Could not resolve a temp location from the " + localStorageDir.GetType() + " specified");
TempPath = tempPath.FullName;
switch (localStorageType)
{
case LocalStorageType.Sync:
var success = InitializeLocalIndexAndDirectory(baseLuceneDirectory, analyzer, configuredPath);
//create the custom lucene directory which will keep the main and temp FS's in sync
LuceneDirectory = LocalTempStorageDirectoryTracker.Current.GetDirectory(
new DirectoryInfo(TempPath),
baseLuceneDirectory,
//flag to disable the mirrored folder if not successful
(int)success >= 100);
//If the master index simply doesn't exist, we don't continue to try to open anything since there will
// actually be nothing there.
if (success == InitializeDirectoryFlags.SuccessNoIndexExists)
{
return;
}
//Try to open the reader, this will fail if the index is corrupt and we'll need to handle that
var result = DelegateExtensions.RetryUntilSuccessOrMaxAttempts(i =>
{
try
{
using (IndexReader.Open(
LuceneDirectory,
DeletePolicyTracker.Current.GetPolicy(LuceneDirectory),
true))
{
}
return Attempt.Succeed(true);
}
catch (Exception ex)
{
Current.Logger.Warn<LocalTempStorageIndexer>(
ex,
string.Format("Could not open an index reader, local temp storage index is empty or corrupt... retrying... {0}", configuredPath));
}
return Attempt.Fail(false);
}, 5, TimeSpan.FromSeconds(1));
if (result.Success == false)
{
Current.Logger.Warn<LocalTempStorageIndexer>(
string.Format("Could not open an index reader, local temp storage index is empty or corrupt... attempting to clear index files in local temp storage, will operate from main storage only {0}", configuredPath));
ClearFilesInPath(TempPath);
//create the custom lucene directory which will keep the main and temp FS's in sync
LuceneDirectory = LocalTempStorageDirectoryTracker.Current.GetDirectory(
new DirectoryInfo(TempPath),
baseLuceneDirectory,
//Disable mirrored index, we're kind of screwed here only use master index
true);
}
break;
case LocalStorageType.LocalOnly:
if (Directory.Exists(TempPath) == false)
{
Directory.CreateDirectory(TempPath);
}
LuceneDirectory = DirectoryTracker.Current.GetDirectory(new DirectoryInfo(TempPath));
break;
default:
throw new ArgumentOutOfRangeException("localStorageType");
}
}
private void ClearFilesInPath(string path)
{
if (Directory.Exists(path))
{
foreach (var file in Directory.GetFiles(path))
{
try
{
File.Delete(file);
}
catch (Exception exInner)
{
Current.Logger.Error<LocalTempStorageIndexer>("Could not delete local temp storage index file", exInner);
}
}
}
}
private bool ClearLuceneDirFiles(Lucene.Net.Store.Directory baseLuceneDirectory)
{
try
{
//unlock it!
IndexWriter.Unlock(baseLuceneDirectory);
var fileLuceneDirectory = baseLuceneDirectory as FSDirectory;
if (fileLuceneDirectory != null)
{
foreach (var file in fileLuceneDirectory.ListAll())
{
try
{
fileLuceneDirectory.DeleteFile(file);
}
catch (IOException)
{
if (file.InvariantEquals("write.lock"))
{
Current.Logger.Warn<LocalTempStorageIndexer>("The lock file could not be deleted but should be removed when the writer is disposed");
}
}
}
return true;
}
return false;
}
catch (Exception ex)
{
Current.Logger.Error<LocalTempStorageIndexer>("Could not clear corrupt index from main index folder, the index cannot be used", ex);
return false;
}
}
private Attempt<IndexWriter> TryCreateWriter(Lucene.Net.Store.Directory baseLuceneDirectory, Analyzer analyzer)
{
try
{
var w = new IndexWriter(
//read from the underlying/default directory, not the temp codegen dir
baseLuceneDirectory,
analyzer,
Snapshotter,
IndexWriter.MaxFieldLength.UNLIMITED);
//Done!
return Attempt.Succeed(w);
}
catch (Exception ex)
{
Current.Logger.Warn<LocalTempStorageIndexer>(ex, "Could not create index writer with snapshot policy for copying... retrying...");
return Attempt<IndexWriter>.Fail(ex);
}
}
/// <summary>
/// Attempts to create an index writer, it will retry on failure 5 times and on the last time will try to forcefully unlock the index files
/// </summary>
/// <param name="baseLuceneDirectory"></param>
/// <param name="analyzer"></param>
/// <returns></returns>
private Attempt<IndexWriter> TryCreateWriterWithRetry(Lucene.Net.Store.Directory baseLuceneDirectory, Analyzer analyzer)
{
var maxTries = 5;
var result = DelegateExtensions.RetryUntilSuccessOrMaxAttempts((currentTry) =>
{
//last try...
if (currentTry == maxTries)
{
Current.Logger.Info<LocalTempStorageIndexer>("Could not acquire index lock, attempting to force unlock it...");
//unlock it!
IndexWriter.Unlock(baseLuceneDirectory);
}
var writerAttempt = TryCreateWriter(baseLuceneDirectory, analyzer);
if (writerAttempt) return writerAttempt;
Current.Logger.Info<LocalTempStorageIndexer>("Could not create writer on {0}, retrying ....", baseLuceneDirectory.ToString);
return Attempt<IndexWriter>.Fail();
}, 5, TimeSpan.FromSeconds(1));
return result;
}
private bool TryWaitForDirectoryUnlock(Lucene.Net.Store.Directory dir)
{
var maxTries = 5;
var result = DelegateExtensions.RetryUntilSuccessOrMaxAttempts((currentTry) =>
{
//last try...
if (currentTry == maxTries)
{
Current.Logger.Info<LocalTempStorageIndexer>("Could not acquire directory lock, attempting to force unlock it...");
//unlock it!
IndexWriter.Unlock(dir);
}
if (IndexWriter.IsLocked(dir) == false) return Attempt.Succeed(true);
Current.Logger.Info<LocalTempStorageIndexer>("Could not acquire directory lock for {0} writer, retrying ....", dir.ToString);
return Attempt<bool>.Fail();
}, 5, TimeSpan.FromSeconds(1));
return result;
}
private InitializeDirectoryFlags InitializeLocalIndexAndDirectory(Lucene.Net.Store.Directory baseLuceneDirectory, Analyzer analyzer, string configuredPath)
{
lock (_locker)
{
if (Directory.Exists(TempPath) == false)
{
Directory.CreateDirectory(TempPath);
}
//copy index if it exists, don't do anything if it's not there
if (IndexReader.IndexExists(baseLuceneDirectory) == false) return InitializeDirectoryFlags.SuccessNoIndexExists;
var writerAttempt = TryCreateWriterWithRetry(baseLuceneDirectory, analyzer);
if (writerAttempt.Success == false)
{
Current.Logger.Error<LocalTempStorageIndexer>("Could not create index writer with snapshot policy for copying, the index cannot be used", writerAttempt.Exception);
return InitializeDirectoryFlags.FailedLocked;
}
//Try to open the reader from the source, this will fail if the index is corrupt and we'll need to handle that
try
{
//NOTE: To date I've not seen this error occur
using (writerAttempt.Result.GetReader())
{
}
}
catch (Exception ex)
{
writerAttempt.Result.Dispose();
Current.Logger.Error<LocalTempStorageIndexer>(
string.Format("Could not open an index reader, {0} is empty or corrupt... attempting to clear index files in master folder", configuredPath),
ex);
if (ClearLuceneDirFiles(baseLuceneDirectory) == false)
{
//hrm, not much we can do in this situation, but this shouldn't happen
Current.Logger.Error<LocalTempStorageIndexer>("Could not open an index reader, index is corrupt.", ex);
return InitializeDirectoryFlags.FailedCorrupt;
}
//the main index is now blank, we'll proceed as normal with a new empty index...
writerAttempt = TryCreateWriter(baseLuceneDirectory, analyzer);
if (writerAttempt.Success == false)
{
//ultra fail...
Current.Logger.Error<LocalTempStorageIndexer>("Could not create index writer with snapshot policy for copying, the index cannot be used", writerAttempt.Exception);
return InitializeDirectoryFlags.FailedLocked;
}
}
using (writerAttempt.Result)
{
try
{
var basePath = IOHelper.MapPath(configuredPath);
var commit = Snapshotter.Snapshot();
var allSnapshotFiles = commit.FileNames
.Concat(new[]
{
commit.SegmentsFileName,
//we need to manually include the segments.gen file
"segments.gen"
})
.Distinct()
.ToArray();
var tempDir = new DirectoryInfo(TempPath);
//Get all files in the temp storage that don't exist in the snapshot collection, we want to remove these
var toRemove = tempDir.GetFiles()
.Select(x => x.Name)
.Except(allSnapshotFiles);
using (var tempDirectory = new SimpleFSDirectory(tempDir))
{
if (TryWaitForDirectoryUnlock(tempDirectory))
{
foreach (var file in toRemove)
{
try
{
File.Delete(Path.Combine(TempPath, file));
}
catch (IOException ex)
{
if (file.InvariantEquals("write.lock"))
{
//This might happen if the writer is open
Current.Logger.Warn<LocalTempStorageIndexer>("The lock file could not be deleted but should be removed when the writer is disposed");
}
Current.Logger.Debug<LocalTempStorageIndexer>("Could not delete non synced index file file, index sync will continue but old index files will remain - this shouldn't affect indexing/searching operations. {0}", () => ex.ToString());
}
}
}
else
{
//quit here, this shouldn't happen with all the checks above.
Current.Logger.Warn<LocalTempStorageIndexer>("Cannot sync index files from main storage, the temp file index is currently locked");
return InitializeDirectoryFlags.FailedLocked;
}
foreach (var fileName in allSnapshotFiles.Where(f => f.IsNullOrWhiteSpace() == false))
{
var destination = Path.Combine(TempPath, Path.GetFileName(fileName));
//don't copy if it's already there, lucene is 'write once' so this file is meant to be there already
if (File.Exists(destination)) continue;
try
{
File.Copy(
Path.Combine(basePath, "Index", fileName),
destination);
}
catch (IOException ex)
{
Current.Logger.Error<LocalTempStorageIndexer>("Could not copy index file, could not sync from main storage", ex);
//quit here
return InitializeDirectoryFlags.FailedFileSync;
}
}
}
}
finally
{
Snapshotter.Release();
}
}
Current.Logger.Info<LocalTempStorageIndexer>("Successfully sync'd main index to local temp storage for index: {0}", () => configuredPath);
return InitializeDirectoryFlags.Success;
}
}
}
}

54
src/Umbraco.Examine/LocalStorage/MultiIndexLock.cs
View File

@@ -1,54 +0,0 @@
using Lucene.Net.Store;
namespace Umbraco.Examine.LocalStorage
{
/// <summary>
/// Lock that wraps multiple locks
/// </summary>
public class MultiIndexLock : Lock
{
private readonly Lock _dirMaster;
private readonly Lock _dirChild;
public MultiIndexLock(Lock dirMaster, Lock dirChild)
{
_dirMaster = dirMaster;
_dirChild = dirChild;
}
/// <summary>
/// Attempts to obtain exclusive access and immediately return
/// upon success or failure.
/// </summary>
/// <returns>
/// true iff exclusive access is obtained
/// </returns>
public override bool Obtain()
{
return _dirMaster.Obtain() && _dirChild.Obtain();
}
public override bool Obtain(long lockWaitTimeout)
{
return _dirMaster.Obtain(lockWaitTimeout) && _dirChild.Obtain(lockWaitTimeout);
}
/// <summary>
/// Releases exclusive access.
/// </summary>
public override void Release()
{
_dirMaster.Release();
_dirChild.Release();
}
/// <summary>
/// Returns true if the resource is currently locked. Note that one must
/// still call <see cref="M:Lucene.Net.Store.Lock.Obtain"/> before using the resource.
/// </summary>
public override bool IsLocked()
{
return _dirMaster.IsLocked() || _dirChild.IsLocked();
}
}
}

33
src/Umbraco.Examine/LocalStorage/MultiIndexLockFactory.cs
View File

@@ -1,33 +0,0 @@
using System;
using Lucene.Net.Store;
namespace Umbraco.Examine.LocalStorage
{
/// <summary>
/// Lock factory that wraps multiple factories
/// </summary>
public class MultiIndexLockFactory : LockFactory
{
private readonly FSDirectory _master;
private readonly FSDirectory _child;
public MultiIndexLockFactory(FSDirectory master, FSDirectory child)
{
if (master == null) throw new ArgumentNullException("master");
if (child == null) throw new ArgumentNullException("child");
_master = master;
_child = child;
}
public override Lock MakeLock(string lockName)
{
return new MultiIndexLock(_master.LockFactory.MakeLock(lockName), _child.LockFactory.MakeLock(lockName));
}
public override void ClearLock(string lockName)
{
_master.LockFactory.ClearLock(lockName);
_child.LockFactory.ClearLock(lockName);
}
}
}

80
src/Umbraco.Examine/LocalStorage/MultiIndexOutput.cs
View File

@@ -1,80 +0,0 @@
using System;
using System.Linq;
using Lucene.Net.Store;
namespace Umbraco.Examine.LocalStorage
{
public class MultiIndexOutput : IndexOutput
{
private readonly IndexOutput[] _outputs;
public MultiIndexOutput(params IndexOutput[] outputs)
{
if (outputs.Length < 1)
{
throw new InvalidOperationException("There must be at least one output specified");
}
_outputs = outputs;
}
public override void WriteByte(byte b)
{
foreach (var output in _outputs)
{
output.WriteByte(b);
}
}
public override void WriteBytes(byte[] b, int offset, int length)
{
foreach (var output in _outputs)
{
output.WriteBytes(b, offset, length);
}
}
public override void Flush()
{
foreach (var output in _outputs)
{
output.Flush();
}
}
protected override void Dispose(bool isDisposing)
{
foreach (var output in _outputs)
{
output.Dispose();
}
}
public override long FilePointer
{
get
{
//return the first
return _outputs[0].FilePointer;
}
}
public override void Seek(long pos)
{
foreach (var output in _outputs)
{
output.Seek(pos);
}
}
public override long Length
{
get
{
//return the first
return _outputs[0].FilePointer;
}
}
}
}

26
src/Umbraco.Examine/NoPrefixSimpleFsLockFactory.cs Normal file
View File

@@ -0,0 +1,26 @@
using System.IO;
using Lucene.Net.Store;
namespace Umbraco.Examine
{
/// <summary>
/// A custom <see cref="SimpleFSLockFactory"/> that ensures a prefixless lock prefix
/// </summary>
/// <remarks>
/// This is a work around for the Lucene APIs. By default Lucene will use a null prefix however when we set a custom
/// lock factory the null prefix is overwritten.
/// </remarks>
public class NoPrefixSimpleFsLockFactory : SimpleFSLockFactory
{
public NoPrefixSimpleFsLockFactory(DirectoryInfo lockDir) : base(lockDir)
{
}
public override string LockPrefix
{
get => base.LockPrefix;
set => base.LockPrefix = null; //always set to null
}
}
}

21
src/Umbraco.Examine/StaticField.cs
View File

@@ -1,21 +0,0 @@
using Examine;
using Examine.LuceneEngine;
namespace Umbraco.Examine
{
internal class StaticField : IIndexField
{
public StaticField(string name, FieldIndexTypes indexType, bool enableSorting, string type)
{
Type = type;
EnableSorting = enableSorting;
IndexType = indexType;
Name = name;
}
public string Name { get; set; }
public FieldIndexTypes IndexType { get; private set; }
public bool EnableSorting { get; set; }
public string Type { get; set; }
}
}

28
src/Umbraco.Examine/StaticFieldCollection.cs
View File

@@ -1,28 +0,0 @@
using System.Collections.ObjectModel;
namespace Umbraco.Examine
{
internal class StaticFieldCollection : KeyedCollection<string, StaticField>
{
protected override string GetKeyForItem(StaticField item)
{
return item.Name;
}
/// <summary>
/// Implements TryGetValue using the underlying dictionary
/// </summary>
/// <param name="key"></param>
/// <param name="field"></param>
/// <returns></returns>
public bool TryGetValue(string key, out StaticField field)
{
if (Dictionary == null)
{
field = null;
return false;
}
return Dictionary.TryGetValue(key, out field);
}
}
}

24
src/Umbraco.Examine/Umbraco.Examine.csproj
View File

@@ -46,14 +46,30 @@
</ItemGroup>
<ItemGroup>
<!-- note: NuGet deals with transitive references now -->
<PackageReference Include="Examine" Version="2.0.0-beta2" />
<PackageReference Include="Lucene.Net" Version="3.0.3" />
<PackageReference Include="Lucene.Net.Contrib" Version="3.0.3" />
<PackageReference Include="Examine">
<Version>1.0.0-beta024</Version>
</PackageReference>
<PackageReference Include="Newtonsoft.Json" Version="10.0.3" />
<PackageReference Include="NPoco" Version="3.6.1" />
</ItemGroup>
<ItemGroup>
<Compile Include="**\*.cs" Exclude="obj\**\*.cs" />
<Compile Include="Config\ConfigIndexCriteria.cs" />
<Compile Include="Config\ConfigIndexField.cs" />
<Compile Include="Config\IndexFieldCollectionExtensions.cs" />
<Compile Include="Config\IndexSet.cs" />
<Compile Include="Config\IndexSetCollection.cs" />
<Compile Include="Config\IndexSets.cs" />
<Compile Include="UmbracoExamineExtensions.cs" />
<Compile Include="IndexTypes.cs" />
<Compile Include="LoggingLevel.cs" />
<Compile Include="NoPrefixSimpleFsLockFactory.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="UmbracoContentIndexer.cs" />
<Compile Include="UmbracoContentIndexerOptions.cs" />
<Compile Include="UmbracoContentValueSetValidator.cs" />
<Compile Include="UmbracoExamineIndexer.cs" />
<Compile Include="UmbracoExamineSearcher.cs" />
<Compile Include="UmbracoMemberIndexer.cs" />
<Compile Include="..\SolutionInfo.cs">
<Link>Properties\SolutionInfo.cs</Link>
</Compile>

173
src/Umbraco.Examine/UmbracoContentIndexer.cs
View File

@@ -1,14 +1,13 @@
using System;
using System.Collections.Generic;
using System.Collections.Specialized;
using System.ComponentModel;
using System.Linq;
using Examine;
using Umbraco.Core;
using Umbraco.Core.Models;
using Umbraco.Core.Services;
using Umbraco.Core.Strings;
using Examine.LuceneEngine.Config;
using Examine.LuceneEngine.Faceting;
using Examine.LuceneEngine.Indexing;
using Examine.LuceneEngine.Providers;
using Lucene.Net.Analysis;
@@ -19,6 +18,7 @@ using Umbraco.Core.Persistence;
using Umbraco.Core.Persistence.DatabaseModelDefinitions;
using Umbraco.Core.Persistence.Querying;
using Umbraco.Core.Scoping;
using Umbraco.Examine.Config;
using IContentService = Umbraco.Core.Services.IContentService;
using IMediaService = Umbraco.Core.Services.IMediaService;
@@ -28,20 +28,21 @@ namespace Umbraco.Examine
/// <summary>
/// An indexer for Umbraco content and media
/// </summary>
public class UmbracoContentIndexer : BaseUmbracoIndexer
public class UmbracoContentIndexer : UmbracoExamineIndexer
{
protected IContentService ContentService { get; }
protected IMediaService MediaService { get; }
protected IUserService UserService { get; }
private readonly IEnumerable<IUrlSegmentProvider> _urlSegmentProviders;
private readonly IScopeProvider _scopeProvider;
private int? _parentId;
#region Constructors
// default - bad, should inject instead
// usage: none
/// <summary>
/// Constructor for configuration providers
/// </summary>
[EditorBrowsable(EditorBrowsableState.Never)]
public UmbracoContentIndexer()
{
ContentService = Current.Services.ContentService;
@@ -49,13 +50,28 @@ namespace Umbraco.Examine
UserService = Current.Services.UserService;
_urlSegmentProviders = Current.UrlSegmentProviders;
_scopeProvider = Current.ScopeProvider;
InitializeQueries();
InitializeQueries(Current.SqlContext);
}
// usage: IndexInitializer (tests)
/// <summary>
/// Create an index at runtime
/// </summary>
/// <param name="name"></param>
/// <param name="fieldDefinitions"></param>
/// <param name="luceneDirectory"></param>
/// <param name="defaultAnalyzer"></param>
/// <param name="profilingLogger"></param>
/// <param name="contentService"></param>
/// <param name="mediaService"></param>
/// <param name="userService"></param>
/// <param name="sqlContext"></param>
/// <param name="urlSegmentProviders"></param>
/// <param name="validator"></param>
/// <param name="options"></param>
/// <param name="indexValueTypes"></param>
public UmbracoContentIndexer(
string name,
IEnumerable<FieldDefinition> fieldDefinitions,
Directory luceneDirectory,
Analyzer defaultAnalyzer,
@@ -63,13 +79,12 @@ namespace Umbraco.Examine
IContentService contentService,
IMediaService mediaService,
IUserService userService,
ISqlContext sqlContext,
IEnumerable<IUrlSegmentProvider> urlSegmentProviders,
IValueSetValidator validator,
UmbracoContentIndexerOptions options,
IScopeProvider scopeProvider,
FacetConfiguration facetConfiguration = null,
IDictionary<string, Func<string, IIndexValueType>> indexValueTypes = null)
: base(fieldDefinitions, luceneDirectory, defaultAnalyzer, profilingLogger, validator, facetConfiguration, indexValueTypes)
IReadOnlyDictionary<string, Func<string, IIndexValueType>> indexValueTypes = null)
: base(name, fieldDefinitions, luceneDirectory, defaultAnalyzer, profilingLogger, validator, indexValueTypes)
{
if (validator == null) throw new ArgumentNullException(nameof(validator));
if (options == null) throw new ArgumentNullException(nameof(options));
@@ -77,24 +92,20 @@ namespace Umbraco.Examine
SupportProtectedContent = options.SupportProtectedContent;
SupportUnpublishedContent = options.SupportUnpublishedContent;
ParentId = options.ParentId;
//backward compat hack:
IndexerData = new IndexCriteria(Enumerable.Empty<IIndexField>(), Enumerable.Empty<IIndexField>(), Enumerable.Empty<string>(), Enumerable.Empty<string>(),
//hack to set the parent Id for backwards compat, when using this ctor the IndexerData will (should) always be null
options.ParentId);
ContentService = contentService ?? throw new ArgumentNullException(nameof(contentService));
MediaService = mediaService ?? throw new ArgumentNullException(nameof(mediaService));
UserService = userService ?? throw new ArgumentNullException(nameof(userService));
_urlSegmentProviders = urlSegmentProviders ?? throw new ArgumentNullException(nameof(urlSegmentProviders));
_scopeProvider = scopeProvider ?? throw new ArgumentNullException(nameof(scopeProvider));
InitializeQueries();
InitializeQueries(sqlContext);
}
private void InitializeQueries()
private void InitializeQueries(ISqlContext sqlContext)
{
if (sqlContext == null) throw new ArgumentNullException(nameof(sqlContext));
if (_publishedQuery == null)
_publishedQuery = Current.SqlContext.Query<IContent>().Where(x => x.Published);
_publishedQuery = sqlContext.Query<IContent>().Where(x => x.Published);
}
#endregion
@@ -121,11 +132,9 @@ namespace Umbraco.Examine
public override void Initialize(string name, NameValueCollection config)
{
//check if there's a flag specifying to support unpublished content,
//if not, set to false;
bool supportUnpublished;
if (config["supportUnpublished"] != null && bool.TryParse(config["supportUnpublished"], out supportUnpublished))
if (config["supportUnpublished"] != null && bool.TryParse(config["supportUnpublished"], out var supportUnpublished))
SupportUnpublishedContent = supportUnpublished;
else
SupportUnpublishedContent = false;
@@ -133,13 +142,25 @@ namespace Umbraco.Examine
//check if there's a flag specifying to support protected content,
//if not, set to false;
bool supportProtected;
if (config["supportProtected"] != null && bool.TryParse(config["supportProtected"], out supportProtected))
if (config["supportProtected"] != null && bool.TryParse(config["supportProtected"], out var supportProtected))
SupportProtectedContent = supportProtected;
else
SupportProtectedContent = false;
base.Initialize(name, config);
//now we need to build up the indexer options so we can create our validator
int? parentId = null;
if (IndexSetName.IsNullOrWhiteSpace() == false)
{
var indexSet = IndexSets.Instance.Sets[IndexSetName];
parentId = indexSet.IndexParentId;
}
ValueSetValidator = new UmbracoContentValueSetValidator(
new UmbracoContentIndexerOptions(SupportUnpublishedContent, SupportProtectedContent, parentId),
//Using a singleton here, we can't inject this when using config based providers and we don't use this
//anywhere else in this class
Current.Services.PublicAccessService);
}
#endregion
@@ -163,12 +184,8 @@ namespace Umbraco.Examine
/// </summary>
public int? ParentId
{
get
{
//fallback to the legacy data
return _parentId ?? (IndexerData == null ? (int?)null : IndexerData.ParentNodeId);
}
protected set { _parentId = value; }
get => _parentId ?? ConfigIndexCriteria?.ParentNodeId;
protected set => _parentId = value;
}
protected override IEnumerable<string> SupportedTypes => new[] {IndexTypes.Content, IndexTypes.Media};
@@ -193,18 +210,14 @@ namespace Umbraco.Examine
var searcher = GetSearcher();
var c = searcher.CreateCriteria();
var filtered = c.RawQuery(rawQuery);
var results = searcher.Find(filtered);
var results = searcher.Search(filtered);
ProfilingLogger.Logger.Debug(GetType(), $"DeleteFromIndex with query: {rawQuery} (found {results.TotalItemCount} results)");
//need to create a delete queue item for each one found
//need to queue a delete item for each one found
foreach (var r in results)
{
ProcessIndexOperation(new IndexOperation()
{
Operation = IndexOperationType.Delete,
Item = new IndexItem(new ValueSet(r.LongId, string.Empty))
});
QueueIndexOperation(new IndexOperation(IndexItem.ForId(r.Id), IndexOperationType.Delete));
}
base.DeleteFromIndex(nodeId);
@@ -250,16 +263,16 @@ namespace Umbraco.Examine
//if specific types are declared we need to post filter them
//TODO: Update the service layer to join the cmsContentType table so we can query by content type too
if (IndexerData != null && IndexerData.IncludeNodeTypes.Any())
if (ConfigIndexCriteria != null && ConfigIndexCriteria.IncludeItemTypes.Any())
{
content = descendants.Where(x => IndexerData.IncludeNodeTypes.Contains(x.ContentType.Alias)).ToArray();
content = descendants.Where(x => ConfigIndexCriteria.IncludeItemTypes.Contains(x.ContentType.Alias)).ToArray();
}
else
{
content = descendants.ToArray();
}
IndexItems(GetValueSets(content));
IndexItems(GetValueSets(_urlSegmentProviders, UserService, content));
pageIndex++;
} while (content.Length == pageSize);
@@ -286,16 +299,16 @@ namespace Umbraco.Examine
//if specific types are declared we need to post filter them
//TODO: Update the service layer to join the cmsContentType table so we can query by content type too
if (IndexerData != null && IndexerData.IncludeNodeTypes.Any())
if (ConfigIndexCriteria != null && ConfigIndexCriteria.IncludeItemTypes.Any())
{
media = descendants.Where(x => IndexerData.IncludeNodeTypes.Contains(x.ContentType.Alias)).ToArray();
media = descendants.Where(x => ConfigIndexCriteria.IncludeItemTypes.Contains(x.ContentType.Alias)).ToArray();
}
else
{
media = descendants.ToArray();
}
IndexItems(GetValueSets(media));
IndexItems(GetValueSets(_urlSegmentProviders, UserService, media));
pageIndex++;
} while (media.Length == pageSize);
@@ -304,11 +317,11 @@ namespace Umbraco.Examine
}
}
private IEnumerable<ValueSet> GetValueSets(IEnumerable<IContent> content)
public static IEnumerable<ValueSet> GetValueSets(IEnumerable<IUrlSegmentProvider> urlSegmentProviders, IUserService userService, params IContent[] content)
{
foreach (var c in content)
{
var urlValue = c.GetUrlSegment(_urlSegmentProviders);
var urlValue = c.GetUrlSegment(urlSegmentProviders);
var values = new Dictionary<string, object[]>
{
{"icon", new object[] {c.ContentType.Icon}},
@@ -325,28 +338,40 @@ namespace Umbraco.Examine
{"urlName", new object[] {urlValue}},
{"path", new object[] {c.Path}},
{"nodeType", new object[] {c.ContentType.Id}},
{"creatorName", new object[] {c.GetCreatorProfile(UserService)?.Name ?? "??"}},
{"writerName", new object[] {c.GetWriterProfile(UserService)?.Name ?? "??"}},
{"creatorName", new object[] {c.GetCreatorProfile(userService)?.Name ?? "??"}},
{"writerName", new object[] {c.GetWriterProfile(userService)?.Name ?? "??"}},
{"writerID", new object[] {c.WriterId}},
{"template", new object[] {c.Template?.Id ?? 0}}
};
foreach (var property in c.Properties.Where(p => p?.GetValue() != null && p.GetValue().ToString().IsNullOrWhiteSpace() == false))
foreach (var property in c.Properties)
{
values.Add(property.Alias, new[] {property.GetValue() });
//only add the value if its not null or empty (we'll check for string explicitly here too)
var val = property.GetValue();
switch (val)
{
case null:
continue;
case string strVal when strVal.IsNullOrWhiteSpace() == false:
values.Add(property.Alias, new[] { val });
break;
default:
values.Add(property.Alias, new[] { val });
break;
}
}
var vs = new ValueSet(c.Id, IndexTypes.Content, c.ContentType.Alias, values);
var vs = new ValueSet(c.Id.ToInvariantString(), IndexTypes.Content, c.ContentType.Alias, values);
yield return vs;
}
}
private IEnumerable<ValueSet> GetValueSets(IEnumerable<IMedia> media)
public static IEnumerable<ValueSet> GetValueSets(IEnumerable<IUrlSegmentProvider> urlSegmentProviders, IUserService userService, params IMedia[] media)
{
foreach (var m in media)
{
var urlValue = m.GetUrlSegment(_urlSegmentProviders);
var urlValue = m.GetUrlSegment(urlSegmentProviders);
var values = new Dictionary<string, object[]>
{
{"icon", new object[] {m.ContentType.Icon}},
@@ -362,40 +387,32 @@ namespace Umbraco.Examine
{"urlName", new object[] {urlValue}},
{"path", new object[] {m.Path}},
{"nodeType", new object[] {m.ContentType.Id}},
{"creatorName", new object[] {m.GetCreatorProfile(UserService).Name}}
{"creatorName", new object[] {m.GetCreatorProfile(userService).Name}}
};
foreach (var property in m.Properties.Where(p => p?.GetValue() != null && p.GetValue().ToString().IsNullOrWhiteSpace() == false))
foreach (var property in m.Properties)
{
values.Add(property.Alias, new[] { property.GetValue() });
//only add the value if its not null or empty (we'll check for string explicitly here too)
var val = property.GetValue();
switch (val)
{
case null:
continue;
case string strVal when strVal.IsNullOrWhiteSpace() == false:
values.Add(property.Alias, new[] { val });
break;
default:
values.Add(property.Alias, new[] { val });
break;
}
}
var vs = new ValueSet(m.Id, IndexTypes.Media, m.ContentType.Alias, values);
var vs = new ValueSet(m.Id.ToInvariantString(), IndexTypes.Media, m.ContentType.Alias, values);
yield return vs;
}
}
/// <summary>
/// Creates an IIndexCriteria object based on the indexSet passed in and our DataService
/// </summary>
/// <param name="indexSet"></param>
/// <returns></returns>
/// <remarks>
/// If we cannot initialize we will pass back empty indexer data since we cannot read from the database
/// </remarks>
[Obsolete("IIndexCriteria is obsolete, this method is used only for configuration based indexes it is recommended to configure indexes on startup with code instead of config")]
protected override IIndexCriteria GetIndexerData(IndexSet indexSet)
{
if (CanInitialize())
{
//NOTE: We are using a singleton here because: This is only ever used for configuration based scenarios, this is never
// used when the index is configured via code (the default), in which case IIndexCriteria is never used. When this is used
// the DI ctor is not used.
return indexSet.ToIndexCriteria(Current.Services.ContentTypeService);
}
return base.GetIndexerData(indexSet);
}
#endregion
}

3
src/Umbraco.Examine/UmbracoContentIndexerOptions.cs
View File

@@ -1,7 +1,8 @@
using System;
namespace Umbraco.Examine
{
{
/// <summary>
/// Options used to configure the umbraco content indexer
/// </summary>

12
src/Umbraco.Examine/UmbracoContentValueSetValidator.cs
View File

@@ -24,10 +24,10 @@ namespace Umbraco.Examine
public bool Validate(ValueSet valueSet)
{
//check for published content
if (valueSet.IndexCategory == IndexTypes.Content
&& valueSet.Values.ContainsKey(BaseUmbracoIndexer.PublishedFieldName))
if (valueSet.Category == IndexTypes.Content
&& valueSet.Values.ContainsKey(UmbracoExamineIndexer.PublishedFieldName))
{
var published = valueSet.Values[BaseUmbracoIndexer.PublishedFieldName] != null && valueSet.Values[BaseUmbracoIndexer.PublishedFieldName][0].Equals(1);
var published = valueSet.Values[UmbracoExamineIndexer.PublishedFieldName] != null && valueSet.Values[UmbracoExamineIndexer.PublishedFieldName][0].Equals(1);
//we don't support unpublished and the item is not published return false
if (_options.SupportUnpublishedContent == false && published == false)
{
@@ -37,11 +37,11 @@ namespace Umbraco.Examine
//must have a 'path'
if (valueSet.Values.ContainsKey(PathKey) == false) return false;
var path = valueSet.Values[PathKey] == null ? string.Empty : valueSet.Values[PathKey][0].ToString();
var path = valueSet.Values[PathKey]?[0].ToString() ?? string.Empty;
// Test for access if we're only indexing published content
// return nothing if we're not supporting protected content and it is protected, and we're not supporting unpublished content
if (valueSet.IndexCategory == IndexTypes.Content
if (valueSet.Category == IndexTypes.Content
&& _options.SupportUnpublishedContent == false
&& _options.SupportProtectedContent == false
&& _publicAccessService.IsProtected(path))
@@ -61,7 +61,7 @@ namespace Umbraco.Examine
if (_options.SupportUnpublishedContent == false)
{
if (path.IsNullOrWhiteSpace()) return false;
var recycleBinId = valueSet.IndexCategory == IndexTypes.Content ? Constants.System.RecycleBinContent : Constants.System.RecycleBinMedia;
var recycleBinId = valueSet.Category == IndexTypes.Content ? Constants.System.RecycleBinContent : Constants.System.RecycleBinMedia;
if (path.Contains(string.Concat(",", recycleBinId, ",")))
return false;
}

92
src/Umbraco.Examine/UmbracoExamineExtensions.cs Normal file
View File

@@ -0,0 +1,92 @@
using System;
using System.ComponentModel;
using System.Web;
using Examine.LuceneEngine.SearchCriteria;
using Examine.SearchCriteria;
using Umbraco.Core;
using Umbraco.Examine.Config;
namespace Umbraco.Examine
{
public static class UmbracoExamineExtensions
{
public static IBooleanOperation Id(this IQuery query, int id)
{
var fieldQuery = query.Id(id.ToInvariantString());
return fieldQuery;
}
/// <summary>
/// Query method to search on parent id
/// </summary>
/// <param name="query"></param>
/// <param name="id"></param>
/// <returns></returns>
public static IBooleanOperation ParentId(this IQuery query, int id)
{
var fieldQuery = query.Field("parentID", id);
return fieldQuery;
}
/// <summary>
/// Query method to search on node name
/// </summary>
/// <param name="query"></param>
/// <param name="nodeName"></param>
/// <returns></returns>
public static IBooleanOperation NodeName(this IQuery query, string nodeName)
{
var fieldQuery = query.Field("nodeName", (IExamineValue)new ExamineValue(Examineness.Explicit, nodeName));
return fieldQuery;
}
/// <summary>
/// Query method to search on node name
/// </summary>
/// <param name="query"></param>
/// <param name="nodeName"></param>
/// <returns></returns>
public static IBooleanOperation NodeName(this IQuery query, IExamineValue nodeName)
{
var fieldQuery = query.Field("nodeName", nodeName);
return fieldQuery;
}
/// <summary>
/// Query method to search on node type alias
/// </summary>
/// <param name="query"></param>
/// <param name="nodeTypeAlias"></param>
/// <returns></returns>
public static IBooleanOperation NodeTypeAlias(this IQuery query, string nodeTypeAlias)
{
var fieldQuery = query.Field("__NodeTypeAlias", (IExamineValue)new ExamineValue(Examineness.Explicit, nodeTypeAlias));
return fieldQuery;
}
/// <summary>
/// Query method to search on node type alias
/// </summary>
/// <param name="query"></param>
/// <param name="nodeTypeAlias"></param>
/// <returns></returns>
public static IBooleanOperation NodeTypeAlias(this IQuery query, IExamineValue nodeTypeAlias)
{
var fieldQuery = query.Field("__NodeTypeAlias", nodeTypeAlias);
return fieldQuery;
}
/// <summary>
/// Used to replace any available tokens in the index path before the lucene directory is assigned to the path
/// </summary>
/// <param name="indexSet"></param>
internal static void ReplaceTokensInIndexPath(this IndexSet indexSet)
{
if (indexSet == null) return;
indexSet.IndexPath = indexSet.IndexPath
.Replace("{machinename}", NetworkHelper.FileSafeMachineName)
.Replace("{appdomainappid}", (HttpRuntime.AppDomainAppId ?? string.Empty).ReplaceNonAlphanumericChars(""))
.EnsureEndsWith('/');
}
}
}

425
src/Umbraco.Examine/UmbracoExamineIndexer.cs Normal file
View File

@@ -0,0 +1,425 @@
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.IO;
using System.Linq;
using Examine.LuceneEngine.Providers;
using Lucene.Net.Analysis;
using Lucene.Net.Documents;
using Lucene.Net.Index;
using Umbraco.Core;
using Examine;
using Examine.LuceneEngine;
using Examine.LuceneEngine.Indexing;
using Umbraco.Core.Composing;
using Umbraco.Core.Logging;
using Umbraco.Core.Xml;
using Umbraco.Examine.Config;
using Directory = Lucene.Net.Store.Directory;
namespace Umbraco.Examine
{
/// <summary>
/// An abstract provider containing the basic functionality to be able to query against
/// Umbraco data.
/// </summary>
public abstract class UmbracoExamineIndexer : LuceneIndexer
{
// note
// wrapping all operations that end up calling base.SafelyProcessQueueItems in a safe call
// context because they will fork a thread/task/whatever which should *not* capture our
// call context (and the database it can contain)! ideally we should be able to override
// SafelyProcessQueueItems but that's not possible in the current version of Examine.
/// <summary>
/// Used to store the path of a content object
/// </summary>
public const string IndexPathFieldName = "__Path";
public const string NodeKeyFieldName = "__Key";
public const string IconFieldName = "__Icon";
public const string PublishedFieldName = "__Published";
/// <summary>
/// The prefix added to a field when it is duplicated in order to store the original raw value.
/// </summary>
public const string RawFieldPrefix = "__Raw_";
/// <summary>
/// Constructor for config provider based indexes
/// </summary>
[EditorBrowsable(EditorBrowsableState.Never)]
protected UmbracoExamineIndexer()
: base()
{
ProfilingLogger = Current.ProfilingLogger;
_configBased = true;
//This is using the config so we'll validate based on that
ValueSetValidator = new ValueSetValidatorDelegate(set =>
{
//check if this document is of a correct type of node type alias
if (ConfigIndexCriteria.IncludeItemTypes.Any())
if (!ConfigIndexCriteria.IncludeItemTypes.Contains(set.ItemType))
return false;
//if this node type is part of our exclusion list, do not validate
if (ConfigIndexCriteria.ExcludeItemTypes.Any())
if (ConfigIndexCriteria.ExcludeItemTypes.Contains(set.ItemType))
return false;
return true;
});
}
protected UmbracoExamineIndexer(
string name,
IEnumerable<FieldDefinition> fieldDefinitions,
Directory luceneDirectory,
Analyzer defaultAnalyzer,
ProfilingLogger profilingLogger,
IValueSetValidator validator = null,
IReadOnlyDictionary<string, Func<string, IIndexValueType>> indexValueTypes = null)
: base(name, fieldDefinitions, luceneDirectory, defaultAnalyzer, validator, indexValueTypes)
{
ProfilingLogger = profilingLogger ?? throw new ArgumentNullException(nameof(profilingLogger));
}
private readonly bool _configBased = false;
/// <summary>
/// A type that defines the type of index for each Umbraco field (non user defined fields)
/// Alot of standard umbraco fields shouldn't be tokenized or even indexed, just stored into lucene
/// for retreival after searching.
/// </summary>
internal static readonly FieldDefinition[] UmbracoIndexFields =
{
new FieldDefinition("parentID", FieldDefinitionTypes.Integer),
new FieldDefinition("level", FieldDefinitionTypes.Integer),
new FieldDefinition("writerID", FieldDefinitionTypes.Integer),
new FieldDefinition("creatorID", FieldDefinitionTypes.Integer),
new FieldDefinition("sortOrder", FieldDefinitionTypes.Integer),
new FieldDefinition("template", FieldDefinitionTypes.Integer),
new FieldDefinition("createDate", FieldDefinitionTypes.DateTime),
new FieldDefinition("updateDate", FieldDefinitionTypes.DateTime),
new FieldDefinition("key", FieldDefinitionTypes.InvariantCultureIgnoreCase),
new FieldDefinition("version", FieldDefinitionTypes.Raw),
new FieldDefinition("nodeType", FieldDefinitionTypes.InvariantCultureIgnoreCase),
new FieldDefinition("template", FieldDefinitionTypes.Raw),
new FieldDefinition("urlName", FieldDefinitionTypes.InvariantCultureIgnoreCase),
new FieldDefinition("path", FieldDefinitionTypes.Raw),
new FieldDefinition(IndexPathFieldName, FieldDefinitionTypes.Raw),
new FieldDefinition(IconFieldName, FieldDefinitionTypes.Raw)
};
protected ProfilingLogger ProfilingLogger { get; }
/// <summary>
/// Overridden to ensure that the umbraco system field definitions are in place
/// </summary>
/// <param name="x"></param>
/// <param name="indexValueTypesFactory"></param>
/// <returns></returns>
protected override FieldValueTypeCollection CreateFieldValueTypes(Directory x, IReadOnlyDictionary<string, Func<string, IIndexValueType>> indexValueTypesFactory = null)
{
foreach (var field in UmbracoIndexFields)
{
FieldDefinitionCollection.TryAdd(field.Name, field);
}
return base.CreateFieldValueTypes(x, indexValueTypesFactory);
}
/// <summary>
/// When set to true Umbraco will keep the index in sync with Umbraco data automatically
/// </summary>
public bool EnableDefaultEventHandler { get; set; } = true;
/// <summary>
/// the supported indexable types
/// </summary>
protected abstract IEnumerable<string> SupportedTypes { get; }
protected ConfigIndexCriteria ConfigIndexCriteria { get; private set; }
/// <summary>
/// The index set name which references an Examine <see cref="IndexSet"/>
/// </summary>
public string IndexSetName { get; private set; }
#region Initialize
/// <summary>
/// Setup the properties for the indexer from the provider settings
/// </summary>
/// <param name="name"></param>
/// <param name="config"></param>
/// <remarks>
/// This is ONLY used for configuration based indexes
/// </remarks>
public override void Initialize(string name, System.Collections.Specialized.NameValueCollection config)
{
ProfilingLogger.Logger.Debug(GetType(), "{0} indexer initializing", () => name);
if (config["enableDefaultEventHandler"] != null && bool.TryParse(config["enableDefaultEventHandler"], out var enabled))
{
EnableDefaultEventHandler = enabled;
}
//Need to check if the index set or IndexerData is specified...
if (config["indexSet"] == null && (FieldDefinitionCollection.Count == 0))
{
//if we don't have either, then we'll try to set the index set by naming conventions
var found = false;
if (name.EndsWith("Indexer"))
{
var setNameByConvension = name.Remove(name.LastIndexOf("Indexer")) + "IndexSet";
//check if we can assign the index set by naming convention
var set = IndexSets.Instance.Sets.Cast<IndexSet>().SingleOrDefault(x => x.SetName == setNameByConvension);
if (set != null)
{
//we've found an index set by naming conventions :)
IndexSetName = set.SetName;
var indexSet = IndexSets.Instance.Sets[IndexSetName];
//if tokens are declared in the path, then use them (i.e. {machinename} )
indexSet.ReplaceTokensInIndexPath();
//get the index criteria and ensure folder
ConfigIndexCriteria = CreateFieldDefinitionsFromConfig(indexSet);
foreach (var fieldDefinition in ConfigIndexCriteria.StandardFields.Union(ConfigIndexCriteria.UserFields))
{
FieldDefinitionCollection.TryAdd(fieldDefinition.Name, fieldDefinition);
}
//now set the index folder
LuceneIndexFolder = new DirectoryInfo(Path.Combine(IndexSets.Instance.Sets[IndexSetName].IndexDirectory.FullName, "Index"));
found = true;
}
}
if (!found)
throw new ArgumentNullException("indexSet on LuceneExamineIndexer provider has not been set in configuration and/or the IndexerData property has not been explicitly set");
}
else if (config["indexSet"] != null)
{
//if an index set is specified, ensure it exists and initialize the indexer based on the set
if (IndexSets.Instance.Sets[config["indexSet"]] == null)
{
throw new ArgumentException("The indexSet specified for the LuceneExamineIndexer provider does not exist");
}
else
{
IndexSetName = config["indexSet"];
var indexSet = IndexSets.Instance.Sets[IndexSetName];
//if tokens are declared in the path, then use them (i.e. {machinename} )
indexSet.ReplaceTokensInIndexPath();
//get the index criteria and ensure folder
ConfigIndexCriteria = CreateFieldDefinitionsFromConfig(indexSet);
foreach (var fieldDefinition in ConfigIndexCriteria.StandardFields.Union(ConfigIndexCriteria.UserFields))
{
FieldDefinitionCollection.TryAdd(fieldDefinition.Name, fieldDefinition);
}
//now set the index folder
LuceneIndexFolder = new DirectoryInfo(Path.Combine(IndexSets.Instance.Sets[IndexSetName].IndexDirectory.FullName, "Index"));
}
}
base.Initialize(name, config);
}
#endregion
/// <summary>
/// override to check if we can actually initialize.
/// </summary>
/// <remarks>
/// This check is required since the base examine lib will try to rebuild on startup
/// </remarks>
public override void RebuildIndex()
{
if (CanInitialize())
{
ProfilingLogger.Logger.Debug(GetType(), "Rebuilding index");
using (new SafeCallContext())
{
base.RebuildIndex();
}
}
}
/// <summary>
/// override to check if we can actually initialize.
/// </summary>
/// <remarks>
/// This check is required since the base examine lib will try to rebuild on startup
/// </remarks>
public override void IndexAll(string type)
{
if (CanInitialize())
{
using (new SafeCallContext())
{
base.IndexAll(type);
}
}
}
public override void IndexItems(IEnumerable<ValueSet> nodes)
{
if (CanInitialize())
{
using (new SafeCallContext())
{
base.IndexItems(nodes);
}
}
}
/// <summary>
/// override to check if we can actually initialize.
/// </summary>
/// <remarks>
/// This check is required since the base examine lib will try to rebuild on startup
/// </remarks>
public override void DeleteFromIndex(string nodeId)
{
if (CanInitialize())
{
using (new SafeCallContext())
{
base.DeleteFromIndex(nodeId);
}
}
}
/// <summary>
/// Returns true if the Umbraco application is in a state that we can initialize the examine indexes
/// </summary>
/// <returns></returns>
protected bool CanInitialize()
{
// only affects indexers that are config file based, if an index was created via code then
// this has no effect, it is assumed the index would not be created if it could not be initialized
return _configBased == false || Current.RuntimeState.Level == RuntimeLevel.Run;
}
/// <summary>
/// Reindexes all supported types
/// </summary>
protected override void PerformIndexRebuild()
{
foreach (var t in SupportedTypes)
{
IndexAll(t);
}
}
/// <summary>
/// overridden for logging
/// </summary>
/// <param name="e"></param>
protected override void OnIndexingError(IndexingErrorEventArgs e)
{
ProfilingLogger.Logger.Error(GetType(), e.Message, e.InnerException);
base.OnIndexingError(e);
}
/// <summary>
/// This ensures that the special __Raw_ fields are indexed
/// </summary>
/// <param name="docArgs"></param>
protected override void OnDocumentWriting(DocumentWritingEventArgs docArgs)
{
var d = docArgs.Document;
foreach (var f in docArgs.ValueSet.Values.Where(x => x.Key.StartsWith(RawFieldPrefix)))
{
if (f.Value.Count > 0)
{
d.Add(new Field(
f.Key,
f.Value[0].ToString(),
Field.Store.YES,
Field.Index.NO, //don't index this field, we never want to search by it
Field.TermVector.NO));
}
}
ProfilingLogger.Logger.Debug(GetType(), "Write lucene doc id:{0}, category:{1}, type:{2}", docArgs.ValueSet.Id, docArgs.ValueSet.Category, docArgs.ValueSet.ItemType);
base.OnDocumentWriting(docArgs);
}
/// <summary>
/// Overridden for logging.
/// </summary>
protected override void AddDocument(Document doc, IndexItem item, IndexWriter writer)
{
ProfilingLogger.Logger.Debug(GetType(), "AddDocument {0} with type {1}", () => item.ValueSet.Id, () => item.ValueSet.ItemType);
base.AddDocument(doc, item, writer);
}
protected override void OnTransformingIndexValues(IndexingItemEventArgs e)
{
base.OnTransformingIndexValues(e);
//ensure special __Path field
if (e.IndexItem.ValueSet.Values.TryGetValue("path", out var path) && e.IndexItem.ValueSet.Values.ContainsKey(IndexPathFieldName) == false)
{
e.IndexItem.ValueSet.Values[IndexPathFieldName] = new List<object> { path };
}
//strip html of all users fields if we detect it has HTML in it.
//if that is the case, we'll create a duplicate 'raw' copy of it so that we can return
//the value of the field 'as-is'.
foreach (var value in e.IndexItem.ValueSet.Values)
{
if (value.Value.Count > 0)
{
var str = value.Value.First() as string;
if (str != null)
{
if (XmlHelper.CouldItBeXml(str))
{
//First save the raw value to a raw field, we will change the policy of this field by detecting the prefix later
e.IndexItem.ValueSet.Values[string.Concat(RawFieldPrefix, value.Key)] = new List<object> { str };
//now replace the original value with the stripped html
//TODO: This should be done with an analzer?!
e.IndexItem.ValueSet.Values[value.Key] = new List<object> { str.StripHtml() };
}
}
}
}
//icon
if (e.IndexItem.ValueSet.Values.TryGetValue("icon", out var icon) && e.IndexItem.ValueSet.Values.ContainsKey(IconFieldName) == false)
{
e.IndexItem.ValueSet.Values[IconFieldName] = new List<object> { icon };
}
}
private ConfigIndexCriteria CreateFieldDefinitionsFromConfig(IndexSet indexSet)
{
return new ConfigIndexCriteria(
indexSet.IndexAttributeFields.Cast<ConfigIndexField>().Select(x => new FieldDefinition(x.Name, x.Type)).ToArray(),
indexSet.IndexUserFields.Cast<ConfigIndexField>().Select(x => new FieldDefinition(x.Name, x.Type)).ToArray(),
indexSet.IncludeNodeTypes.ToList().Select(x => x.Name).ToArray(),
indexSet.ExcludeNodeTypes.ToList().Select(x => x.Name).ToArray(),
indexSet.IndexParentId);
}
}
}

169
src/Umbraco.Examine/UmbracoExamineSearcher.cs
View File

@@ -2,23 +2,12 @@
using System.ComponentModel;
using System.IO;
using System.Linq;
using System.Security;
using System.Web;
using System.Web.Compilation;
using Examine;
using Examine.LuceneEngine.Config;
using Examine.Providers;
using Examine.SearchCriteria;
using Lucene.Net.Index;
using Lucene.Net.Store;
using Umbraco.Core;
using Examine.LuceneEngine;
using Examine.LuceneEngine.Providers;
using Examine.LuceneEngine.SearchCriteria;
using Lucene.Net.Analysis;
using Lucene.Net.Index;
using Umbraco.Core.Composing;
using Umbraco.Core.Logging;
using Umbraco.Examine.LocalStorage;
using Umbraco.Examine.Config;
using Directory = Lucene.Net.Store.Directory;
@@ -29,10 +18,6 @@ namespace Umbraco.Examine
/// </summary>
public class UmbracoExamineSearcher : LuceneSearcher
{
private Lazy<Directory> _localTempDirectory;
private LocalStorageType _localStorageType = LocalStorageType.Sync;
private string _name;
private readonly bool _configBased = false;
/// <summary>
@@ -40,7 +25,6 @@ namespace Umbraco.Examine
/// </summary>
[EditorBrowsable(EditorBrowsableState.Never)]
public UmbracoExamineSearcher()
: base()
{
_configBased = true;
}
@@ -48,10 +32,11 @@ namespace Umbraco.Examine
/// <summary>
/// Constructor to allow for creating an indexer at runtime
/// </summary>
/// <param name="name"></param>
/// <param name="indexPath"></param>
/// <param name="analyzer"></param>
public UmbracoExamineSearcher(DirectoryInfo indexPath, Analyzer analyzer)
: base(indexPath, analyzer)
public UmbracoExamineSearcher(string name, DirectoryInfo indexPath, Analyzer analyzer)
: base(name, indexPath, analyzer)
{
_configBased = false;
}
@@ -59,22 +44,25 @@ namespace Umbraco.Examine
/// <summary>
/// Constructor to allow for creating an indexer at runtime
/// </summary>
/// <param name="name"></param>
/// <param name="luceneDirectory"></param>
/// <param name="analyzer"></param>
public UmbracoExamineSearcher(Directory luceneDirectory, Analyzer analyzer)
: base(luceneDirectory, analyzer)
public UmbracoExamineSearcher(string name, Directory luceneDirectory, Analyzer analyzer)
: base(name, luceneDirectory, analyzer)
{
_configBased = false;
}
/// <inheritdoc />
public UmbracoExamineSearcher(string name, IndexWriter writer, Analyzer analyzer) : base(name, writer, analyzer)
{
_configBased = false;
}
/// <summary>
/// we override name because we need to manually set it if !CanInitialize()
/// since we cannot call base.Initialize in that case.
/// Name of the Lucene.NET index set
/// </summary>
public override string Name
{
get { return _name; }
}
public string IndexSetName { get; private set; }
/// <summary>
/// Method used for initializing based on a configuration based searcher
@@ -83,77 +71,61 @@ namespace Umbraco.Examine
/// <param name="config"></param>
public override void Initialize(string name, System.Collections.Specialized.NameValueCollection config)
{
if (name == null) throw new ArgumentNullException("name");
//ensure name is set
_name = name;
if (string.IsNullOrWhiteSpace(name)) throw new ArgumentException("Value cannot be null or whitespace.", nameof(name));
//We need to check if we actually can initialize, if not then don't continue
if (CanInitialize() == false)
{
return;
}
//need to check if the index set is specified, if it's not, we'll see if we can find one by convension
//if the folder is not null and the index set is null, we'll assume that this has been created at runtime.
//NOTE: Don't proceed if the _luceneDirectory is set since we already know where to look.
var luceneDirectory = GetLuceneDirectory();
if (config["indexSet"] == null && (LuceneIndexFolder == null && luceneDirectory == null))
{
//if we don't have either, then we'll try to set the index set by naming convensions
var found = false;
if (name.EndsWith("Searcher"))
{
var setNameByConvension = name.Remove(name.LastIndexOf("Searcher")) + "IndexSet";
//check if we can assign the index set by naming convension
var set = IndexSets.Instance.Sets.Cast<IndexSet>().SingleOrDefault(x => x.SetName == setNameByConvension);
if (set != null)
{
set.ReplaceTokensInIndexPath();
//we've found an index set by naming convensions :)
IndexSetName = set.SetName;
found = true;
}
}
if (!found)
throw new ArgumentNullException("indexSet on LuceneExamineIndexer provider has not been set in configuration");
//get the folder to index
LuceneIndexFolder = new DirectoryInfo(Path.Combine(IndexSets.Instance.Sets[IndexSetName].IndexDirectory.FullName, "Index"));
}
else if (config["indexSet"] != null && luceneDirectory == null)
{
if (IndexSets.Instance.Sets[config["indexSet"]] == null)
throw new ArgumentException("The indexSet specified for the LuceneExamineIndexer provider does not exist");
IndexSetName = config["indexSet"];
var indexSet = IndexSets.Instance.Sets[IndexSetName];
indexSet.ReplaceTokensInIndexPath();
//get the folder to index
LuceneIndexFolder = new DirectoryInfo(Path.Combine(indexSet.IndexDirectory.FullName, "Index"));
}
base.Initialize(name, config);
if (config != null && config["useTempStorage"] != null)
{
throw new NotImplementedException("Fix how local temp storage works and is synced with Examine v2.0 - since a writer is always open we cannot snapshot it, we need to use the same logic in AzureDirectory");
//Use the temp storage directory which will store the index in the local/codegen folder, this is useful
// for websites that are running from a remove file server and file IO latency becomes an issue
var attemptUseTempStorage = config["useTempStorage"].TryConvertTo<LocalStorageType>();
if (attemptUseTempStorage)
{
//this is the default
ILocalStorageDirectory localStorageDir = new CodeGenLocalStorageDirectory();
if (config["tempStorageDirectory"] != null)
{
//try to get the type
var dirType = BuildManager.GetType(config["tempStorageDirectory"], false);
if (dirType != null)
{
try
{
localStorageDir = (ILocalStorageDirectory)Activator.CreateInstance(dirType);
}
catch (Exception ex)
{
Current.Logger.Error<UmbracoExamineSearcher>(
string.Format("Could not create a temp storage location of type {0}, reverting to use the " + typeof(CodeGenLocalStorageDirectory).FullName, dirType),
ex);
}
}
}
var indexSet = IndexSets.Instance.Sets[IndexSetName];
var configuredPath = indexSet.IndexPath;
var tempPath = localStorageDir.GetLocalStorageDirectory(config, configuredPath);
if (tempPath == null) throw new InvalidOperationException("Could not resolve a temp location from the " + localStorageDir.GetType() + " specified");
var localTempPath = tempPath.FullName;
_localStorageType = attemptUseTempStorage.Result;
//initialize the lazy callback
_localTempDirectory = new Lazy<Directory>(() =>
{
switch (_localStorageType)
{
case LocalStorageType.Sync:
var fsDir = base.GetLuceneDirectory() as FSDirectory;
if (fsDir != null)
{
return LocalTempStorageDirectoryTracker.Current.GetDirectory(
new DirectoryInfo(localTempPath),
fsDir);
}
return base.GetLuceneDirectory();
case LocalStorageType.LocalOnly:
return DirectoryTracker.Current.GetDirectory(new DirectoryInfo(localTempPath));
default:
throw new ArgumentOutOfRangeException();
}
});
}
}
}
/// <summary>
@@ -171,21 +143,14 @@ namespace Umbraco.Examine
/// Returns a list of fields to search on, this will also exclude the IndexPathFieldName and node type alias
/// </summary>
/// <returns></returns>
protected override string[] GetSearchFields()
public override string[] GetAllIndexedFields()
{
var fields = base.GetSearchFields();
var fields = base.GetAllIndexedFields();
return fields
.Where(x => x != BaseUmbracoIndexer.IndexPathFieldName)
.Where(x => x != LuceneIndexer.NodeTypeAliasFieldName)
.Where(x => x != UmbracoExamineIndexer.IndexPathFieldName)
.Where(x => x != LuceneIndexer.ItemTypeFieldName)
.ToArray();
}
protected override Directory GetLuceneDirectory()
{
//local temp storage is not enabled, just return the default
return _localTempDirectory.IsValueCreated == false
? base.GetLuceneDirectory()
: _localTempDirectory.Value;
}
}
}

164
src/Umbraco.Examine/UmbracoMemberIndexer.cs
View File

@@ -1,22 +1,18 @@
using System;
using System.Collections;
using System.Linq;
using System.Xml.Linq;
using Examine.LuceneEngine.Config;
using Umbraco.Core;
using Umbraco.Core.Models;
using Umbraco.Core.Persistence.DatabaseModelDefinitions;
using Umbraco.Core.Services;
using Umbraco.Core.Strings;
using System.Collections.Generic;
using System.ComponentModel;
using Examine;
using System.IO;
using Examine.LuceneEngine;
using Examine.LuceneEngine.Indexing;
using Examine.LuceneEngine.Providers;
using Lucene.Net.Analysis;
using Umbraco.Core.Composing;
using Umbraco.Core.Logging;
using Umbraco.Core.Persistence;
using Umbraco.Core.Scoping;
using Directory = Lucene.Net.Store.Directory;
namespace Umbraco.Examine
@@ -25,24 +21,23 @@ namespace Umbraco.Examine
/// <summary>
/// Custom indexer for members
/// </summary>
public class UmbracoMemberIndexer : BaseUmbracoIndexer
public class UmbracoMemberIndexer : UmbracoExamineIndexer
{
private readonly IMemberService _memberService;
private readonly IScopeProvider _scopeProvider;
/// <summary>
/// Default constructor
/// Constructor for config/provider based indexes
/// </summary>
[EditorBrowsable(EditorBrowsableState.Never)]
public UmbracoMemberIndexer()
: base()
{
_memberService = Current.Services.MemberService;
_scopeProvider = Current.ScopeProvider;
}
/// <summary>
/// Constructor to allow for creating an indexer at runtime
/// </summary>
/// <param name="name"></param>
/// <param name="fieldDefinitions"></param>
/// <param name="luceneDirectory"></param>
/// <param name="profilingLogger"></param>
@@ -50,76 +45,35 @@ namespace Umbraco.Examine
/// <param name="memberService"></param>
/// <param name="analyzer"></param>
public UmbracoMemberIndexer(
string name,
IEnumerable<FieldDefinition> fieldDefinitions,
Directory luceneDirectory,
Analyzer analyzer,
ProfilingLogger profilingLogger,
IValueSetValidator validator,
IMemberService memberService) :
base(fieldDefinitions, luceneDirectory, analyzer, profilingLogger, validator)
base(name, fieldDefinitions, luceneDirectory, analyzer, profilingLogger, validator)
{
if (memberService == null) throw new ArgumentNullException("memberService");
_memberService = memberService;
_memberService = memberService ?? throw new ArgumentNullException(nameof(memberService));
}
/// <summary>
/// Ensures that the'_searchEmail' is added to the user fields so that it is indexed - without having to modify the config
/// </summary>
/// <param name="indexSet"></param>
/// <returns></returns>
[Obsolete("IIndexCriteria is obsolete, this method is used only for configuration based indexes it is recommended to configure indexes on startup with code instead of config")]
protected override IIndexCriteria GetIndexerData(IndexSet indexSet)
{
//TODO: This is only required for config based index delcaration - We need to change this!
var indexerData = base.GetIndexerData(indexSet);
if (CanInitialize())
{
//If the fields are missing a custom _searchEmail, then add it
if (indexerData.UserFields.Any(x => x.Name == "_searchEmail") == false)
{
var field = new IndexField { Name = "_searchEmail" };
var policy = IndexFieldPolicies.FirstOrDefault(x => x.Name == "_searchEmail");
if (policy != null)
{
field.Type = policy.Type;
field.EnableSorting = policy.EnableSorting;
}
return new IndexCriteria(
indexerData.StandardFields,
indexerData.UserFields.Concat(new[] { field }),
indexerData.IncludeNodeTypes,
indexerData.ExcludeNodeTypes,
indexerData.ParentNodeId
);
}
}
return indexerData;
}
/// <summary>
/// Overridden to ensure that the umbraco system field definitions are in place
/// </summary>
/// <param name="originalDefinitions"></param>
/// <param name="x"></param>
/// <param name="indexValueTypesFactory"></param>
/// <returns></returns>
protected override IEnumerable<FieldDefinition> InitializeFieldDefinitions(IEnumerable<FieldDefinition> originalDefinitions)
protected override FieldValueTypeCollection CreateFieldValueTypes(Directory x, IReadOnlyDictionary<string, Func<string, IIndexValueType>> indexValueTypesFactory = null)
{
var result = base.InitializeFieldDefinitions(originalDefinitions).ToList();
result.Add(new FieldDefinition("__key", FieldDefinitionTypes.Raw));
return result;
var keyDef = new FieldDefinition("__key", FieldDefinitionTypes.Raw);
FieldDefinitionCollection.TryAdd(keyDef.Name, keyDef);
return base.CreateFieldValueTypes(x, indexValueTypesFactory);
}
/// <summary>
/// The supported types for this indexer
/// </summary>
protected override IEnumerable<string> SupportedTypes
{
get { return new[] {IndexTypes.Member}; }
}
/// <inheritdoc />
protected override IEnumerable<string> SupportedTypes => new[] {IndexTypes.Member};
/// <summary>
/// Reindex all members
@@ -136,44 +90,38 @@ namespace Umbraco.Examine
IMember[] members;
using (var scope = _scopeProvider.CreateScope())
if (ConfigIndexCriteria != null && ConfigIndexCriteria.IncludeItemTypes.Any())
{
if (IndexerData != null && IndexerData.IncludeNodeTypes.Any())
//if there are specific node types then just index those
foreach (var nodeType in ConfigIndexCriteria.IncludeItemTypes)
{
//if there are specific node types then just index those
foreach (var nodeType in IndexerData.IncludeNodeTypes)
{
do
{
long total;
members = _memberService.GetAll(pageIndex, pageSize, out total, "LoginName", Direction.Ascending, true, null, nodeType).ToArray();
IndexItems(GetValueSets(members));
pageIndex++;
} while (members.Length == pageSize);
}
}
else
{
//no node types specified, do all members
do
{
long total;
members = _memberService.GetAll(pageIndex, pageSize, out total).ToArray();
members = _memberService.GetAll(pageIndex, pageSize, out _, "LoginName", Direction.Ascending, true, null, nodeType).ToArray();
IndexItems(GetValueSets(members));
pageIndex++;
} while (members.Length == pageSize);
}
scope.Complete();
}
else
{
//no node types specified, do all members
do
{
members = _memberService.GetAll(pageIndex, pageSize, out _).ToArray();
IndexItems(GetValueSets(members));
pageIndex++;
} while (members.Length == pageSize);
}
}
private IEnumerable<ValueSet> GetValueSets(IEnumerable<IMember> member)
public static IEnumerable<ValueSet> GetValueSets(params IMember[] members)
{
foreach (var m in member)
foreach (var m in members)
{
var values = new Dictionary<string, object[]>
{
@@ -193,30 +141,48 @@ namespace Umbraco.Examine
{"email", new object[] {m.Email}},
};
foreach (var property in m.Properties.Where(p => p != null && p.GetValue() != null && p.GetValue().ToString().IsNullOrWhiteSpace() == false))
foreach (var property in m.Properties)
{
values.Add(property.Alias, new[] { property.GetValue() });
//only add the value if its not null or empty (we'll check for string explicitly here too)
var val = property.GetValue();
switch (val)
{
case null:
continue;
case string strVal when strVal.IsNullOrWhiteSpace() == false:
values.Add(property.Alias, new[] { val });
break;
default:
values.Add(property.Alias, new[] { val });
break;
}
}
var vs = new ValueSet(m.Id, IndexTypes.Content, m.ContentType.Alias, values);
var vs = new ValueSet(m.Id.ToInvariantString(), IndexTypes.Content, m.ContentType.Alias, values);
yield return vs;
}
}
protected override void OnTransformingIndexValues(TransformingIndexDataEventArgs e)
/// <summary>
/// Ensure some custom values are added to the index
/// </summary>
/// <param name="e"></param>
protected override void OnTransformingIndexValues(IndexingItemEventArgs e)
{
base.OnTransformingIndexValues(e);
if (e.OriginalValues.ContainsKey("key") && e.IndexItem.ValueSet.Values.ContainsKey("__key") == false)
if (e.IndexItem.ValueSet.Values.TryGetValue("key", out var key) && e.IndexItem.ValueSet.Values.ContainsKey("__key") == false)
{
e.IndexItem.ValueSet.Values["__key"] = new List<object> {e.OriginalValues["key"]};
}
if (e.OriginalValues.ContainsKey("email") && e.IndexItem.ValueSet.Values.ContainsKey("_searchEmail") == false)
{
e.IndexItem.ValueSet.Values["_searchEmail"] = new List<object> { e.OriginalValues["email"].ToString().Replace(".", " ").Replace("@", " ") };
//double __ prefix means it will be indexed as culture invariant
e.IndexItem.ValueSet.Values["__key"] = new List<object> { key };
}
if (e.IndexItem.ValueSet.Values.TryGetValue("email", out var email) && e.IndexItem.ValueSet.Values.ContainsKey("_searchEmail") == false)
{
//will be indexed as full text (the default anaylyzer)
e.IndexItem.ValueSet.Values["_searchEmail"] = new List<object> { email?.ToString().Replace(".", " ").Replace("@", " ") };
}
}
}

255
src/Umbraco.Examine/XsltExtensions.cs
View File

@@ -1,255 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Security;
using System.Xml.Linq;
using System.Xml.XPath;
using Examine;
using Examine.LuceneEngine;
using Examine.LuceneEngine.Providers;
using Examine.LuceneEngine.SearchCriteria;
using Examine.SearchCriteria;
using Examine.Providers;
using Umbraco.Core.Macros;
namespace Umbraco.Examine
{
/// <summary>
/// Methods to support Umbraco XSLT extensions.
/// </summary>
/// <remarks>
/// XSLT extensions will ONLY work for provider that have a base class of BaseUmbracoIndexer
/// </remarks>
[XsltExtension("Examine")]
public class XsltExtensions
{
///<summary>
/// Uses the provider specified to search, returning an XPathNodeIterator
///</summary>
///<param name="searchText"></param>
///<param name="useWildcards"></param>
///<param name="provider"></param>
///<param name="indexType"></param>
///<returns></returns>
internal static XPathNodeIterator Search(string searchText, bool useWildcards, LuceneSearcher provider, string indexType)
{
if (provider == null) throw new ArgumentNullException("provider");
var results = provider.Search(searchText, useWildcards, indexType);
return GetResultsAsXml(results);
}
/// <summary>
/// Uses the provider specified to search, returning an XPathNodeIterator
/// </summary>
/// <param name="searchText">The search text.</param>
/// <param name="useWildcards">if set to <c>true</c> [use wildcards].</param>
/// <param name="providerName">Name of the provider.</param>
/// <param name="indexType">Type of the index.</param>
/// <returns></returns>
public static XPathNodeIterator Search(string searchText, bool useWildcards, string providerName, string indexType)
{
var provider = (LuceneSearcher)ExamineManager.Instance.SearchProviderCollection[providerName];
return Search(searchText, useWildcards, provider, indexType);
}
/// <summary>
/// Uses the provider specified to search, returning an XPathNodeIterator
/// </summary>
/// <param name="searchText"></param>
/// <param name="useWildcards"></param>
/// <param name="providerName"></param>
/// <returns></returns>
public static XPathNodeIterator Search(string searchText, bool useWildcards, string providerName)
{
return Search(searchText, useWildcards, providerName, string.Empty);
}
/// <summary>
/// Uses the default provider specified to search, returning an XPathNodeIterator
/// </summary>
/// <param name="searchText">The search query</param>
/// <param name="useWildcards">Enable a wildcard search query</param>
/// <returns>A node-set of the search results</returns>
public static XPathNodeIterator Search(string searchText, bool useWildcards)
{
return Search(searchText, useWildcards, ExamineManager.Instance.DefaultSearchProvider.Name);
}
/// <summary>
/// Uses the default provider specified to search, returning an XPathNodeIterator
/// </summary>
/// <param name="searchText">The search query</param>
/// <returns>A node-set of the search results</returns>
public static XPathNodeIterator Search(string searchText)
{
return Search(searchText, true);
}
/// <summary>
/// Will perform a search against the media index type only
/// </summary>
/// <param name="searchText"></param>
/// <param name="useWildcards"></param>
/// <param name="providerName"></param>
/// <returns></returns>
public static XPathNodeIterator SearchMediaOnly(string searchText, bool useWildcards, string providerName)
{
return Search(searchText, useWildcards, providerName, IndexTypes.Media);
}
/// <summary>
/// Will perform a search against the media index type only
/// </summary>
/// <param name="searchText"></param>
/// <param name="useWildcards"></param>
/// <returns></returns>
public static XPathNodeIterator SearchMediaOnly(string searchText, bool useWildcards)
{
return SearchMediaOnly(searchText, useWildcards, ExamineManager.Instance.DefaultSearchProvider.Name);
}
/// <summary>
/// Will perform a search against the media index type only
/// </summary>
/// <param name="searchText"></param>
/// <returns></returns>
public static XPathNodeIterator SearchMediaOnly(string searchText)
{
return SearchMediaOnly(searchText, true);
}
/// <summary>
/// Searches the member only.
/// </summary>
/// <param name="searchText">The search text.</param>
/// <param name="useWildcards">if set to <c>true</c> [use wildcards].</param>
/// <param name="providerName">Name of the provider.</param>
/// <returns></returns>
public static XPathNodeIterator SearchMemberOnly(string searchText, bool useWildcards, string providerName)
{
return Search(searchText, useWildcards, providerName, IndexTypes.Member);
}
/// <summary>
/// Searches the member only.
/// </summary>
/// <param name="searchText">The search text.</param>
/// <param name="useWildcards">if set to <c>true</c> [use wildcards].</param>
/// <returns></returns>
public static XPathNodeIterator SearchMemberOnly(string searchText, bool useWildcards)
{
return SearchMemberOnly(searchText, useWildcards, ExamineManager.Instance.DefaultSearchProvider.Name);
}
/// <summary>
/// Searches the member only.
/// </summary>
/// <param name="searchText">The search text.</param>
/// <returns></returns>
public static XPathNodeIterator SearchMemberOnly(string searchText)
{
return SearchMemberOnly(searchText, true);
}
/// <summary>
/// Will perform a search against the content index type only
/// </summary>
/// <param name="searchText"></param>
/// <param name="useWildcards"></param>
/// <param name="providerName"></param>
/// <returns></returns>
public static XPathNodeIterator SearchContentOnly(string searchText, bool useWildcards, string providerName)
{
return Search(searchText, useWildcards, providerName, IndexTypes.Content);
}
/// <summary>
/// Will perform a search against the content index type only
/// </summary>
/// <param name="searchText"></param>
/// <param name="useWildcards"></param>
/// <returns></returns>
public static XPathNodeIterator SearchContentOnly(string searchText, bool useWildcards)
{
return SearchContentOnly(searchText, useWildcards, ExamineManager.Instance.DefaultSearchProvider.Name);
}
/// <summary>
/// Will perform a search against the content index type only
/// </summary>
/// <param name="searchText"></param>
/// <returns></returns>
public static XPathNodeIterator SearchContentOnly(string searchText)
{
return SearchContentOnly(searchText, true);
}
/// <summary>
/// Gets the results as XML.
/// </summary>
/// <param name="results">The results.</param>
/// <returns></returns>
private static XPathNodeIterator GetResultsAsXml(ILuceneSearchResults results)
{
// create the XDocument
XDocument doc = new XDocument();
// check there are any search results
if (results.TotalItemCount > 0)
{
// create the root element
XElement root = new XElement("nodes");
// iterate through the search results
foreach (SearchResult result in results)
{
// create a new <node> element
XElement node = new XElement("node");
// create the @id attribute
XAttribute nodeId = new XAttribute("id", result.Id);
// create the @score attribute
XAttribute nodeScore = new XAttribute("score", result.Score);
// add the content
node.Add(nodeId, nodeScore);
foreach (KeyValuePair<String, String> field in result.Fields)
{
// create a new <data> element
XElement data = new XElement("data");
// create the @alias attribute
XAttribute alias = new XAttribute("alias", field.Key);
// assign the value to a CDATA section
XCData value = new XCData(field.Value);
// append the content
data.Add(alias, value);
// append the <data> element
node.Add(data);
}
// add the node
root.Add(node);
}
// add the root node
doc.Add(root);
}
else
{
doc.Add(new XElement("error", "There were no search results."));
}
return doc.CreateNavigator().Select("/");
}
}
}

49
src/Umbraco.Tests/Cache/PublishedCache/PublishedMediaCacheTests.cs
View File

@@ -1,4 +1,5 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Xml;
@@ -161,31 +162,31 @@ namespace Umbraco.Tests.Cache.PublishedCache
var ctx = GetUmbracoContext("/test");
var key = Guid.NewGuid();
var result = new SearchResult()
{
LongId = 1234,
Score = 1
};
result.Fields.Add("__IndexType", "media");
result.Fields.Add("__NodeId", "1234");
result.Fields.Add("__NodeTypeAlias", Constants.Conventions.MediaTypes.Image);
result.Fields.Add("__Path", "-1,1234");
result.Fields.Add("__nodeName", "Test");
result.Fields.Add("id", "1234");
result.Fields.Add("key", key.ToString());
result.Fields.Add("urlName", "/media/test.jpg");
result.Fields.Add("nodeType", "0");
result.Fields.Add("sortOrder", "0");
result.Fields.Add("level", "2");
result.Fields.Add("nodeName", "Test");
result.Fields.Add("nodeTypeAlias", Constants.Conventions.MediaTypes.Image);
result.Fields.Add("parentID", "-1");
result.Fields.Add("path", "-1,1234");
result.Fields.Add("updateDate", DateTime.Parse("2012-07-16T10:34:09").Ticks.ToString());
result.Fields.Add("createDate", DateTime.Parse("2012-07-17T10:34:09").Ticks.ToString());
result.Fields.Add("creatorID", "0");
result.Fields.Add("creatorName", "Shannon");
var fields = new Dictionary<string, string>
{
{"__IndexType", "media"},
{"__NodeId", "1234"},
{"__NodeTypeAlias", Constants.Conventions.MediaTypes.Image},
{"__Path", "-1,1234"},
{"__nodeName", "Test"},
{"id", "1234"},
{"key", key.ToString()},
{"urlName", "/media/test.jpg"},
{"nodeType", "0"},
{"sortOrder", "0"},
{"level", "2"},
{"nodeName", "Test"},
{"nodeTypeAlias", Constants.Conventions.MediaTypes.Image},
{"parentID", "-1"},
{"path", "-1,1234"},
{"updateDate", DateTime.Parse("2012-07-16T10:34:09").Ticks.ToString()},
{"createDate", DateTime.Parse("2012-07-17T10:34:09").Ticks.ToString()},
{"creatorID", "0"},
{"creatorName", "Shannon"}
};
var result = new SearchResult("1234", 1, 1, () => fields.ToDictionary(x => x.Key, x => new List<string> { x.Value }));
var store = new PublishedMediaCache(new XmlStore((XmlDocument)null, null, null, null), ServiceContext.MediaService, ServiceContext.UserService, new StaticCacheProvider(), ContentTypesCache);
var doc = store.CreateFromCacheValues(store.ConvertFromSearchResult(result));

2
src/Umbraco.Tests/Composing/TypeFinderTests.cs
View File

@@ -49,7 +49,7 @@ namespace Umbraco.Tests.Composing
//typeof(TabPage).Assembly,
typeof(System.Web.Mvc.ActionResult).Assembly,
typeof(TypeFinder).Assembly,
typeof(global::Umbraco.Examine.BaseUmbracoIndexer).Assembly
typeof(global::Umbraco.Examine.UmbracoExamineIndexer).Assembly
};
}

65
src/Umbraco.Tests/PublishedContent/PublishedMediaTests.cs
View File

@@ -12,7 +12,7 @@ using Umbraco.Web;
using Umbraco.Web.PublishedCache.XmlPublishedCache;
using System.Linq;
using System.Xml;
using Examine.Session;
using Examine;
using Umbraco.Core.Cache;
using Umbraco.Core.Models.PublishedContent;
using Umbraco.Core.Strings;
@@ -109,13 +109,12 @@ namespace Umbraco.Tests.PublishedContent
[Test]
public void Ensure_Children_Sorted_With_Examine()
{
using (var luceneDir = new RAMDirectory())
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, options: new UmbracoContentIndexerOptions(true, false, null)))
using (var session = new ThreadScopedIndexSession(indexer.SearcherContext))
using (var luceneDir = new RandomIdRamDirectory())
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, options: new UmbracoContentIndexerOptions(true, false, null)))
{
indexer.RebuildIndex();
session.WaitForChanges();
indexer.RebuildIndex();
var searcher = indexer.GetSearcher();
var ctx = GetUmbracoContext("/test");
var cache = new PublishedMediaCache(ServiceContext.MediaService, ServiceContext.UserService, searcher, indexer, new StaticCacheProvider(), ContentTypesCache);
@@ -136,12 +135,12 @@ namespace Umbraco.Tests.PublishedContent
[Test]
public void Do_Not_Find_In_Recycle_Bin()
{
using (var luceneDir = new RAMDirectory())
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, options: new UmbracoContentIndexerOptions(true, false, null)))
using (var session = new ThreadScopedIndexSession(indexer.SearcherContext))
using (var luceneDir = new RandomIdRamDirectory())
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, options: new UmbracoContentIndexerOptions(true, false, null)))
using (indexer.ProcessNonAsync())
{
indexer.RebuildIndex();
session.WaitForChanges();
var searcher = indexer.GetSearcher();
var ctx = GetUmbracoContext("/test");
@@ -159,11 +158,11 @@ namespace Umbraco.Tests.PublishedContent
<data alias='umbracoBytes'>10726</data>
<data alias='umbracoExtension'>jpg</data>
</node>");
indexer.ReIndexNode(newXml, "media");
session.WaitForChanges();
indexer.IndexItems(new[]{ newXml.ConvertToValueSet("media") });
//ensure it still exists in the index (raw examine search)
var criteria = searcher.CreateSearchCriteria();
var criteria = searcher.CreateCriteria();
var filter = criteria.Id(3113);
var found = searcher.Search(filter.Compile());
Assert.IsNotNull(found);
@@ -180,12 +179,12 @@ namespace Umbraco.Tests.PublishedContent
[Test]
public void Children_With_Examine()
{
using (var luceneDir = new RAMDirectory())
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, options: new UmbracoContentIndexerOptions(true, false, null)))
using (var session = new ThreadScopedIndexSession(indexer.SearcherContext))
using (var luceneDir = new RandomIdRamDirectory())
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, options: new UmbracoContentIndexerOptions(true, false, null)))
using (indexer.ProcessNonAsync())
{
indexer.RebuildIndex();
session.WaitForChanges();
var searcher = indexer.GetSearcher();
var ctx = GetUmbracoContext("/test");
@@ -205,12 +204,12 @@ namespace Umbraco.Tests.PublishedContent
[Test]
public void Descendants_With_Examine()
{
using (var luceneDir = new RAMDirectory())
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, options: new UmbracoContentIndexerOptions(true, false, null)))
using (var session = new ThreadScopedIndexSession(indexer.SearcherContext))
using (var luceneDir = new RandomIdRamDirectory())
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, options: new UmbracoContentIndexerOptions(true, false, null)))
using (indexer.ProcessNonAsync())
{
indexer.RebuildIndex();
session.WaitForChanges();
var searcher = indexer.GetSearcher();
var ctx = GetUmbracoContext("/test");
@@ -230,12 +229,12 @@ namespace Umbraco.Tests.PublishedContent
[Test]
public void DescendantsOrSelf_With_Examine()
{
using (var luceneDir = new RAMDirectory())
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, options: new UmbracoContentIndexerOptions(true, false, null)))
using (var session = new ThreadScopedIndexSession(indexer.SearcherContext))
using (var luceneDir = new RandomIdRamDirectory())
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, options: new UmbracoContentIndexerOptions(true, false, null)))
using (indexer.ProcessNonAsync())
{
indexer.RebuildIndex();
session.WaitForChanges();
var searcher = indexer.GetSearcher();
var ctx = GetUmbracoContext("/test");
@@ -255,12 +254,12 @@ namespace Umbraco.Tests.PublishedContent
[Test]
public void Ancestors_With_Examine()
{
using (var luceneDir = new RAMDirectory())
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, options: new UmbracoContentIndexerOptions(true, false, null)))
using (var session = new ThreadScopedIndexSession(indexer.SearcherContext))
using (var luceneDir = new RandomIdRamDirectory())
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, options: new UmbracoContentIndexerOptions(true, false, null)))
using (indexer.ProcessNonAsync())
{
indexer.RebuildIndex();
session.WaitForChanges();
var ctx = GetUmbracoContext("/test");
var searcher = indexer.GetSearcher();
@@ -277,12 +276,12 @@ namespace Umbraco.Tests.PublishedContent
[Test]
public void AncestorsOrSelf_With_Examine()
{
using (var luceneDir = new RAMDirectory())
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, options: new UmbracoContentIndexerOptions(true, false, null)))
using (var session = new ThreadScopedIndexSession(indexer.SearcherContext))
using (var luceneDir = new RandomIdRamDirectory())
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, options: new UmbracoContentIndexerOptions(true, false, null)))
using (indexer.ProcessNonAsync())
{
indexer.RebuildIndex();
session.WaitForChanges();
var ctx = GetUmbracoContext("/test");
var searcher = indexer.GetSearcher();

3
src/Umbraco.Tests/Runtimes/CoreRuntimeTests.cs
View File

@@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Web.Hosting;
using Examine;
using LightInject;
using Moq;
using NUnit.Framework;
@@ -163,7 +164,7 @@ namespace Umbraco.Tests.Runtimes
base.Compose(composition);
composition.Container.Register(factory => SettingsForTests.GetDefault());
composition.Container.RegisterSingleton<IExamineIndexCollectionAccessor, TestIndexCollectionAccessor>();
composition.Container.RegisterSingleton<IExamineManager, TestExamineManager>();
Composed = true;
}

69
src/Umbraco.Tests/TestHelpers/Stubs/TestExamineManager.cs Normal file
View File

@@ -0,0 +1,69 @@
using System.Collections.Concurrent;
using System.Collections.Generic;
using Examine;
namespace Umbraco.Tests.TestHelpers.Stubs
{
internal class TestExamineManager : IExamineManager
{
private readonly ConcurrentDictionary<string, IIndexer> _indexers = new ConcurrentDictionary<string, IIndexer>();
private readonly ConcurrentDictionary<string, ISearcher> _searchers = new ConcurrentDictionary<string, ISearcher>();
public void AddIndexer(string name, IIndexer indexer)
{
_indexers.TryAdd(name, indexer);
}
public void AddSearcher(string name, ISearcher searcher)
{
_searchers.TryAdd(name, searcher);
}
public void DeleteFromIndexes(string nodeId)
{
//noop
}
public void DeleteFromIndexes(string nodeId, IEnumerable<IIndexer> providers)
{
//noop
}
public void Dispose()
{
//noop
}
public IIndexer GetIndexer(string indexerName)
{
return _indexers.TryGetValue(indexerName, out var indexer) ? indexer : null;
}
public ISearcher GetRegisteredSearcher(string searcherName)
{
return _searchers.TryGetValue(searcherName, out var indexer) ? indexer : null;
}
public void IndexAll(string indexCategory)
{
//noop
}
public void IndexItems(ValueSet[] nodes)
{
//noop
}
public void IndexItems(ValueSet[] nodes, IEnumerable<IIndexer> providers)
{
//noop
}
public void RebuildIndexes()
{
//noop
}
public IReadOnlyDictionary<string, IIndexer> IndexProviders => _indexers;
}
}

11
src/Umbraco.Tests/TestHelpers/Stubs/TestIndexCollectionAccessor.cs
View File

@@ -1,11 +0,0 @@
using System.Collections.Generic;
using Examine;
using Umbraco.Examine;
namespace Umbraco.Tests.TestHelpers.Stubs
{
public class TestIndexCollectionAccessor : IExamineIndexCollectionAccessor
{
public IReadOnlyDictionary<string, IExamineIndexer> Indexes => new Dictionary<string, IExamineIndexer>();
}
}

3
src/Umbraco.Tests/Testing/UmbracoTestBase.cs
View File

@@ -3,6 +3,7 @@ using System.IO;
using System.Linq;
using System.Reflection;
using AutoMapper;
using Examine;
using LightInject;
using Moq;
using NUnit.Framework;
@@ -258,7 +259,7 @@ namespace Umbraco.Tests.Testing
Container.RegisterSingleton(factory => settings.Content);
Container.RegisterSingleton(factory => settings.Templates);
Container.Register(factory => new MediaFileSystem(Mock.Of<IFileSystem>()));
Container.RegisterSingleton<IExamineIndexCollectionAccessor, TestIndexCollectionAccessor>();
Container.RegisterSingleton<IExamineManager>(factory => ExamineManager.Instance);
// replace some stuff
Container.RegisterSingleton(factory => Mock.Of<IFileSystem>(), "ScriptFileSystem");

8
src/Umbraco.Tests/Umbraco.Tests.csproj
View File

@@ -59,8 +59,8 @@
<Reference Include="Castle.Core, Version=4.1.1.0, Culture=neutral, PublicKeyToken=407dd0808d44fbdc, processorArchitecture=MSIL">
<HintPath>..\packages\Castle.Core.4.1.1\lib\net45\Castle.Core.dll</HintPath>
</Reference>
<Reference Include="Examine, Version=2.0.0.0, Culture=neutral, processorArchitecture=MSIL">
<HintPath>..\packages\Examine.2.0.0-beta2\lib\net45\Examine.dll</HintPath>
<Reference Include="Examine, Version=1.0.0.0, Culture=neutral, processorArchitecture=MSIL">
<HintPath>..\packages\Examine.1.0.0-beta024\lib\net45\Examine.dll</HintPath>
</Reference>
<Reference Include="ICSharpCode.SharpZipLib, Version=0.86.0.518, Culture=neutral, PublicKeyToken=1b03e6acf1164f73, processorArchitecture=MSIL">
<HintPath>..\packages\SharpZipLib.0.86.0\lib\20\ICSharpCode.SharpZipLib.dll</HintPath>
@@ -272,6 +272,7 @@
<Compile Include="TestHelpers\ControllerTesting\TraceExceptionLogger.cs" />
<Compile Include="TestHelpers\Entities\MockedUserGroup.cs" />
<Compile Include="TestHelpers\NoHttpContextAccessor.cs" />
<Compile Include="TestHelpers\Stubs\TestExamineManager.cs" />
<Compile Include="Testing\ContentBaseExtensions.cs" />
<Compile Include="Testing\TestDatabase.cs" />
<Compile Include="Testing\TestingTests\NUnitTests.cs" />
@@ -288,11 +289,11 @@
<Compile Include="Testing\UmbracoTestOptions.cs" />
<Compile Include="CoreThings\TryConvertToTests.cs" />
<Compile Include="TestHelpers\Stubs\TestPublishedSnapshotAccessor.cs" />
<Compile Include="TestHelpers\Stubs\TestIndexCollectionAccessor.cs" />
<Compile Include="TestHelpers\TestObjects-Mocks.cs" />
<Compile Include="TestHelpers\TestObjects.cs" />
<Compile Include="TestHelpers\Stubs\TestUmbracoContextAccessor.cs" />
<Compile Include="CoreThings\UdiTests.cs" />
<Compile Include="UmbracoExamine\RandomIdRamDirectory.cs" />
<Compile Include="Web\AngularIntegration\AngularAntiForgeryTests.cs" />
<Compile Include="Web\AngularIntegration\ContentModelSerializationTests.cs" />
<Compile Include="Web\AngularIntegration\JsInitializationTests.cs" />
@@ -577,7 +578,6 @@
<DesignTime>True</DesignTime>
<DependentUpon>TestFiles.resx</DependentUpon>
</Compile>
<Compile Include="UmbracoExamine\TestIndexField.cs" />
<Compile Include="UI\LegacyDialogTests.cs" />
<Compile Include="UmbracoExamine\ExamineDemoDataMediaService.cs" />
<Compile Include="UmbracoExamine\ExamineDemoDataContentService.cs" />

25
src/Umbraco.Tests/UmbracoExamine/EventsTest.cs
View File

@@ -1,10 +1,8 @@
using System;
using System.Linq;
using Examine;
using Examine.Session;
using Lucene.Net.Store;
using NUnit.Framework;
using Umbraco.Tests.TestHelpers;
using Umbraco.Tests.Testing;
using Umbraco.Examine;
@@ -17,23 +15,14 @@ namespace Umbraco.Tests.UmbracoExamine
[Test]
public void Events_Ignoring_Node()
{
using (var luceneDir = new RAMDirectory())
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir,
using (var luceneDir = new RandomIdRamDirectory())
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext,
//make parent id 999 so all are ignored
options: new UmbracoContentIndexerOptions(false, false, 999)))
using (var session = new ThreadScopedIndexSession(indexer.SearcherContext))
using (indexer.ProcessNonAsync())
{
var searcher = indexer.GetSearcher();
var isIgnored = false;
EventHandler<IndexingNodeDataEventArgs> ignoringNode = (s, e) =>
{
isIgnored = true;
};
indexer.IgnoringNode += ignoringNode;
var contentService = new ExamineDemoDataContentService();
//get a node from the data repo
var node = contentService.GetPublishedContentByXPath("//*[string-length(@id)>0 and number(@id)>0]")
@@ -41,10 +30,12 @@ namespace Umbraco.Tests.UmbracoExamine
.Elements()
.First();
indexer.ReIndexNode(node, IndexTypes.Content);
var valueSet = node.ConvertToValueSet(IndexTypes.Content);
indexer.IndexItems(new[] {valueSet});
var found = searcher.Search(searcher.CreateCriteria().Id((string) node.Attribute("id")).Compile());
Assert.IsTrue(isIgnored);
Assert.AreEqual(0, found.TotalItemCount);
}

18
src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs
View File

@@ -10,6 +10,7 @@ using Moq;
using Umbraco.Core.Logging;
using Umbraco.Core.Models;
using Umbraco.Core.Models.Membership;
using Umbraco.Core.Persistence;
using Umbraco.Core.Persistence.DatabaseModelDefinitions;
using Umbraco.Core.Persistence.Querying;
using Umbraco.Core.Scoping;
@@ -30,6 +31,7 @@ namespace Umbraco.Tests.UmbracoExamine
public static UmbracoContentIndexer GetUmbracoIndexer(
ProfilingLogger profilingLogger,
Directory luceneDir,
ISqlContext sqlContext,
Analyzer analyzer = null,
IContentService contentService = null,
IMediaService mediaService = null,
@@ -80,6 +82,7 @@ namespace Umbraco.Tests.UmbracoExamine
{
userService = Mock.Of<IUserService>(x => x.GetProfileById(It.IsAny<int>()) == Mock.Of<IProfile>(p => p.Id == 0 && p.Name == "admin"));
}
if (mediaService == null)
{
long totalRecs;
@@ -169,23 +172,24 @@ namespace Umbraco.Tests.UmbracoExamine
//query
// .Setup(x => x.GetWhereClauses())
// .Returns(new List<Tuple<string, object[]>> { new Tuple<string, object[]>($"{Constants.DatabaseSchema.Tables.Document}.published", new object[] { 1 }) });
var scopeProvider = new Mock<IScopeProvider>();
//scopeProvider
// .Setup(x => x.Query<IContent>())
// .Returns(query.Object);
var i = new UmbracoContentIndexer(
new[] { new FieldDefinition("", FieldDefinitionTypes.FullText) },
"testIndexer",
Enumerable.Empty<FieldDefinition>(),
luceneDir,
analyzer,
profilingLogger,
contentService,
mediaService,
userService,
sqlContext,
new[] {new DefaultUrlSegmentProvider()},
new UmbracoContentValueSetValidator(options, Mock.Of<IPublicAccessService>()),
options,
scopeProvider.Object);
options);
i.IndexingError += IndexingError;
@@ -194,19 +198,19 @@ namespace Umbraco.Tests.UmbracoExamine
public static LuceneSearcher GetLuceneSearcher(Directory luceneDir)
{
return new LuceneSearcher(luceneDir, new StandardAnalyzer(Version.LUCENE_29));
return new LuceneSearcher("testSearcher", luceneDir, new StandardAnalyzer(Version.LUCENE_29));
}
public static MultiIndexSearcher GetMultiSearcher(Directory pdfDir, Directory simpleDir, Directory conventionDir, Directory cwsDir)
{
var i = new MultiIndexSearcher(new[] { pdfDir, simpleDir, conventionDir, cwsDir }, new StandardAnalyzer(Version.LUCENE_29));
var i = new MultiIndexSearcher("testSearcher", new[] { pdfDir, simpleDir, conventionDir, cwsDir }, new StandardAnalyzer(Version.LUCENE_29));
return i;
}
internal static void IndexingError(object sender, IndexingErrorEventArgs e)
{
throw new ApplicationException(e.Message, e.Exception);
throw new ApplicationException(e.Message, e.InnerException);
}

96
src/Umbraco.Tests/UmbracoExamine/IndexTest.cs
View File

@@ -1,17 +1,15 @@
using System.Linq;
using Examine;
using Examine.LuceneEngine.Providers;
using Examine.Session;
using Lucene.Net.Index;
using Lucene.Net.Search;
using Lucene.Net.Store;
using NUnit.Framework;
using Umbraco.Tests.TestHelpers;
using Umbraco.Tests.Testing;
using Umbraco.Examine;
namespace Umbraco.Tests.UmbracoExamine
{
/// <summary>
/// Tests the standard indexing capabilities
/// </summary>
@@ -24,17 +22,16 @@ namespace Umbraco.Tests.UmbracoExamine
public void Rebuild_Index()
{
using (var luceneDir = new RAMDirectory())
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, options: new UmbracoContentIndexerOptions(true, false, null)))
using (var session = new ThreadScopedIndexSession(indexer.SearcherContext))
using (var luceneDir = new RandomIdRamDirectory())
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, options: new UmbracoContentIndexerOptions(true, false, null)))
using (indexer.ProcessNonAsync())
{
var searcher = indexer.GetSearcher();
//create the whole thing
indexer.RebuildIndex();
session.WaitForChanges();
var result = searcher.Find(searcher.CreateCriteria().All().Compile());
var result = searcher.Search(searcher.CreateCriteria().All().Compile());
Assert.AreEqual(29, result.TotalItemCount);
}
@@ -48,24 +45,23 @@ namespace Umbraco.Tests.UmbracoExamine
public void Index_Protected_Content_Not_Indexed()
{
using (var luceneDir = new RAMDirectory())
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir))
using (var session = new ThreadScopedIndexSession(indexer.SearcherContext))
using (var searcher = indexer.GetSearcher().GetSearcher())
using (var luceneDir = new RandomIdRamDirectory())
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext))
using (indexer.ProcessNonAsync())
using (var searcher = ((LuceneSearcher)indexer.GetSearcher()).GetLuceneSearcher())
{
//create the whole thing
indexer.RebuildIndex();
session.WaitForChanges();
var protectedQuery = new BooleanQuery();
protectedQuery.Add(
new BooleanClause(
new TermQuery(new Term(LuceneIndexer.IndexTypeFieldName, IndexTypes.Content)),
new TermQuery(new Term(LuceneIndexer.CategoryFieldName, IndexTypes.Content)),
Occur.MUST));
protectedQuery.Add(
new BooleanClause(
new TermQuery(new Term(LuceneIndexer.IndexNodeIdFieldName, ExamineDemoDataContentService.ProtectedNode.ToString())),
new TermQuery(new Term(LuceneIndexer.ItemIdFieldName, ExamineDemoDataContentService.ProtectedNode.ToString())),
Occur.MUST));
var collector = TopScoreDocCollector.Create(100, true);
@@ -80,11 +76,11 @@ namespace Umbraco.Tests.UmbracoExamine
[Test]
public void Index_Move_Media_From_Non_Indexable_To_Indexable_ParentID()
{
using (var luceneDir = new RAMDirectory())
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir,
using (var luceneDir = new RandomIdRamDirectory())
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext,
//make parent id 1116
options: new UmbracoContentIndexerOptions(false, false, 1116)))
using (var session = new ThreadScopedIndexSession(indexer.SearcherContext))
using (indexer.ProcessNonAsync())
{
var searcher = indexer.GetSearcher();
@@ -99,12 +95,10 @@ namespace Umbraco.Tests.UmbracoExamine
Assert.AreEqual("-1,1111,2222,2112", currPath);
//ensure it's indexed
indexer.ReIndexNode(node, IndexTypes.Media);
session.WaitForChanges();
indexer.IndexItems(new []{ node.ConvertToValueSet(IndexTypes.Media) });
//it will not exist because it exists under 2222
var results = searcher.Search(searcher.CreateSearchCriteria().Id(2112).Compile());
var results = searcher.Search(searcher.CreateCriteria().Id(2112).Compile());
Assert.AreEqual(0, results.Count());
//now mimic moving 2112 to 1116
@@ -113,12 +107,10 @@ namespace Umbraco.Tests.UmbracoExamine
node.SetAttributeValue("parentID", "1116");
//now reindex the node, this should first delete it and then WILL add it because of the parent id constraint
indexer.ReIndexNode(node, IndexTypes.Media);
session.WaitForChanges();
indexer.IndexItems(new[] { node.ConvertToValueSet(IndexTypes.Media) });
//now ensure it exists
results = searcher.Search(searcher.CreateSearchCriteria().Id(2112).Compile());
results = searcher.Search(searcher.CreateCriteria().Id(2112).Compile());
Assert.AreEqual(1, results.Count());
}
}
@@ -126,11 +118,11 @@ namespace Umbraco.Tests.UmbracoExamine
[Test]
public void Index_Move_Media_To_Non_Indexable_ParentID()
{
using (var luceneDir = new RAMDirectory())
using (var indexer1 = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir,
using (var luceneDir = new RandomIdRamDirectory())
using (var indexer1 = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext,
//make parent id 2222
options: new UmbracoContentIndexerOptions(false, false, 2222)))
using (var session = new ThreadScopedIndexSession(indexer1.SearcherContext))
using (indexer1.ProcessNonAsync())
{
var searcher = indexer1.GetSearcher();
@@ -145,12 +137,12 @@ namespace Umbraco.Tests.UmbracoExamine
Assert.AreEqual("-1,1111,2222,2112", currPath);
//ensure it's indexed
indexer1.ReIndexNode(node, IndexTypes.Media);
indexer1.IndexItems(new[] { node.ConvertToValueSet(IndexTypes.Media) });
session.WaitForChanges();
//it will exist because it exists under 2222
var results = searcher.Search(searcher.CreateSearchCriteria().Id(2112).Compile());
var results = searcher.Search(searcher.CreateCriteria().Id(2112).Compile());
Assert.AreEqual(1, results.Count());
//now mimic moving the node underneath 1116 instead of 2222
@@ -158,12 +150,12 @@ namespace Umbraco.Tests.UmbracoExamine
node.SetAttributeValue("parentID", "1116");
//now reindex the node, this should first delete it and then NOT add it because of the parent id constraint
indexer1.ReIndexNode(node, IndexTypes.Media);
indexer1.IndexItems(new[] { node.ConvertToValueSet(IndexTypes.Media) });
session.WaitForChanges();
//now ensure it's deleted
results = searcher.Search(searcher.CreateSearchCriteria().Id(2112).Compile());
results = searcher.Search(searcher.CreateCriteria().Id(2112).Compile());
Assert.AreEqual(0, results.Count());
}
}
@@ -176,36 +168,36 @@ namespace Umbraco.Tests.UmbracoExamine
[Test]
public void Index_Reindex_Content()
{
using (var luceneDir = new RAMDirectory())
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, options: new UmbracoContentIndexerOptions(true, false, null)))
using (var session = new ThreadScopedIndexSession(indexer.SearcherContext))
using (var luceneDir = new RandomIdRamDirectory())
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, options: new UmbracoContentIndexerOptions(true, false, null)))
using (indexer.ProcessNonAsync())
{
var searcher = indexer.GetSearcher();
//create the whole thing
indexer.RebuildIndex();
session.WaitForChanges();
var result = searcher.Find(searcher.CreateCriteria().Field(LuceneIndexer.IndexTypeFieldName, IndexTypes.Content).Compile());
var result = searcher.Search(searcher.CreateCriteria().Field(LuceneIndexer.CategoryFieldName, IndexTypes.Content).Compile());
Assert.AreEqual(21, result.TotalItemCount);
//delete all content
foreach (var r in result)
{
indexer.DeleteFromIndex(r.LongId);
indexer.DeleteFromIndex(r.Id);
}
session.WaitForChanges();
//ensure it's all gone
result = searcher.Find(searcher.CreateCriteria().Field(LuceneIndexer.IndexTypeFieldName, IndexTypes.Content).Compile());
result = searcher.Search(searcher.CreateCriteria().Field(LuceneIndexer.CategoryFieldName, IndexTypes.Content).Compile());
Assert.AreEqual(0, result.TotalItemCount);
//call our indexing methods
indexer.IndexAll(IndexTypes.Content);
session.WaitForChanges();
result = searcher.Find(searcher.CreateCriteria().Field(LuceneIndexer.IndexTypeFieldName, IndexTypes.Content).Compile());
result = searcher.Search(searcher.CreateCriteria().Field(LuceneIndexer.CategoryFieldName, IndexTypes.Content).Compile());
Assert.AreEqual(21, result.TotalItemCount);
}
@@ -219,27 +211,27 @@ namespace Umbraco.Tests.UmbracoExamine
[Test]
public void Index_Delete_Index_Item_Ensure_Heirarchy_Removed()
{
using (var luceneDir = new RAMDirectory())
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir))
using (var session = new ThreadScopedIndexSession(indexer.SearcherContext))
using (var luceneDir = new RandomIdRamDirectory())
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext))
using (indexer.ProcessNonAsync())
{
var searcher = indexer.GetSearcher();
//create the whole thing
indexer.RebuildIndex();
session.WaitForChanges();
//now delete a node that has children
indexer.DeleteFromIndex(1140.ToString());
//this node had children: 1141 & 1142, let's ensure they are also removed
session.WaitForChanges();
var results = searcher.Search(searcher.CreateSearchCriteria().Id(1141).Compile());
var results = searcher.Search(searcher.CreateCriteria().Id(1141).Compile());
Assert.AreEqual(0, results.Count());
results = searcher.Search(searcher.CreateSearchCriteria().Id(1142).Compile());
results = searcher.Search(searcher.CreateCriteria().Id(1142).Compile());
Assert.AreEqual(0, results.Count());
}

17
src/Umbraco.Tests/UmbracoExamine/RandomIdRamDirectory.cs Normal file
View File

@@ -0,0 +1,17 @@
using System;
using Lucene.Net.Store;
namespace Umbraco.Tests.UmbracoExamine
{
/// <summary>
/// This is needed for all tests, else the lockid collides with directories during testing
/// </summary>
public class RandomIdRamDirectory : RAMDirectory
{
private readonly string _lockId = Guid.NewGuid().ToString();
public override string GetLockId()
{
return _lockId;
}
}
}

17
src/Umbraco.Tests/UmbracoExamine/SearchTests.cs
View File

@@ -5,13 +5,12 @@ using Examine;
using Lucene.Net.Store;
using NUnit.Framework;
using Examine.LuceneEngine.SearchCriteria;
using Examine.Session;
using Moq;
using Umbraco.Core.Models;
using Umbraco.Core.Persistence.DatabaseModelDefinitions;
using Umbraco.Core.Persistence.Querying;
using Umbraco.Core.Services;
using Umbraco.Tests.TestHelpers;
using Umbraco.Examine;
using Umbraco.Tests.Testing;
namespace Umbraco.Tests.UmbracoExamine
@@ -54,21 +53,21 @@ namespace Umbraco.Tests.UmbracoExamine
==
allRecs);
using (var luceneDir = new RAMDirectory())
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, contentService: contentService))
using (var session = new ThreadScopedIndexSession(indexer.SearcherContext))
using (var luceneDir = new RandomIdRamDirectory())
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, contentService: contentService))
using (indexer.ProcessNonAsync())
{
indexer.RebuildIndex();
session.WaitForChanges();
var searcher = indexer.GetSearcher();
var numberSortedCriteria = searcher.CreateSearchCriteria()
var numberSortedCriteria = searcher.CreateCriteria()
.ParentId(1148).And()
.OrderBy(new SortableField("sortOrder", SortType.Int));
var numberSortedResult = searcher.Search(numberSortedCriteria.Compile());
var stringSortedCriteria = searcher.CreateSearchCriteria()
var stringSortedCriteria = searcher.CreateCriteria()
.ParentId(1148).And()
.OrderBy("sortOrder"); //will default to string
var stringSortedResult = searcher.Search(stringSortedCriteria.Compile());
@@ -99,7 +98,7 @@ namespace Umbraco.Tests.UmbracoExamine
//[Test]
//public void Test_Index_Type_With_German_Analyzer()
//{
// using (var luceneDir = new RAMDirectory())
// using (var luceneDir = new RandomIdRamDirectory())
// {
// var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir,
// new GermanAnalyzer());

11
src/Umbraco.Tests/UmbracoExamine/TestIndexField.cs
View File

@@ -1,11 +0,0 @@
using Examine;
namespace Umbraco.Tests.UmbracoExamine
{
public class TestIndexField : IIndexField
{
public string Name { get; set; }
public bool EnableSorting { get; set; }
public string Type { get; set; }
}
}

2
src/Umbraco.Tests/packages.config
View File

@@ -2,7 +2,7 @@
<packages>
<package id="AutoMapper" version="6.1.1" targetFramework="net461" />
<package id="Castle.Core" version="4.1.1" targetFramework="net461" />
<package id="Examine" version="2.0.0-beta2" targetFramework="net461" />
<package id="Examine" version="1.0.0-beta024" targetFramework="net461" />
<package id="LightInject" version="5.0.3" targetFramework="net461" />
<package id="LightInject.Annotation" version="1.1.0" targetFramework="net461" />
<package id="log4net" version="2.0.8" targetFramework="net461" />

11
src/Umbraco.Web.UI.Client/src/installer/steps/upgrade.html
View File

@@ -1,18 +1,19 @@
<div>
<h1>Upgrading Umbraco</h1>
<p>
Welcome to the Umbraco installer. You see this screen because your Umbraco installation needs a quick upgrade of its database and files, which will ensure your website is kept as fast, secure and up to date as possible.
Welcome to the Umbraco installer. You see this screen because your Umbraco installation needs a quick upgrade
of its database and files, which will ensure your website is kept as fast, secure and up to date as possible.
</p>
<p>
To read a report of changes between your current version <strong>{{installer.current.model.currentVersion}}</strong> and this version you're upgrading to <strong>{{installer.current.model.newVersion}}</strong>
Detected current version <strong>{{installer.current.model.currentVersion}}</strong> ({{installer.current.model.currentState}}),
which needs to be upgraded to <strong>{{installer.current.model.newVersion}}</strong> ({{installer.current.model.newState}}).
To compare versions and read a report of changes between versions, use the <em>View Report</em> button below.
</p>
<p>
<a ng-href="{{installer.current.model.reportUrl}}" target="_blank" class="btn btn-info">View Report</a>
</p>
<p>
Simply click <strong>continue</strong> below to be guided through the rest of the upgrade
Simply click <strong>continue</strong> below to be guided through the rest of the upgrade.
</p>
<p>
<button class="btn btn-success" ng-click="install()">Continue</button>

16
src/Umbraco.Web.UI.Client/src/views/dashboard/developer/examinemanagement.html
View File

@@ -76,21 +76,7 @@
<tr>
<th>Fields in index</th>
<td>{{indexer.fieldCount}}</td>
</tr>
<tr>
<th>Has deletions?</th>
<td>
<span>{{indexer.deletionCount > 0}}</span>
(<span>{{indexer.deletionCount}}</span>)
</td>
</tr>
<tr>
<th>Optimized?</th>
<td>
<span>{{indexer.isOptimized}}</span>
</td>
</tr>
</tr>
</table>
</div>
</li>

24
src/Umbraco.Web.UI.Client/src/views/dashboard/developer/examinemgmt.controller.js
View File

@@ -91,30 +91,6 @@ function ExamineMgmtController($scope, umbRequestHelper, $log, $http, $q, $timeo
}
}
$scope.optimizeIndex = function(indexer) {
if (confirm("This will cause the index to be optimized which will improve its performance. " +
"It is not recommended to optimize an index during times of high website traffic " +
"or when editors are editing content.")) {
indexer.isProcessing = true;
umbRequestHelper.resourcePromise(
$http.post(umbRequestHelper.getApiUrl("examineMgmtBaseUrl",
"PostOptimizeIndex",
{ indexerName: indexer.name })),
'Failed to optimize index')
.then(function() {
//optimizing has started, nothing is returned accept a 200 status code.
//lets poll to see if it is done.
$timeout(function() {
checkProcessing(indexer, "PostCheckOptimizeIndex");
},
1000);
});
}
}
$scope.closeSearch = function(searcher) {
searcher.isSearching = true;
}

7
src/Umbraco.Web.UI/Umbraco.Web.UI.csproj
View File

@@ -57,6 +57,9 @@
<Reference Include="ClientDependency.Core, Version=1.9.6.0, Culture=neutral, processorArchitecture=MSIL">
<HintPath>..\packages\ClientDependency.1.9.6\lib\net45\ClientDependency.Core.dll</HintPath>
</Reference>
<Reference Include="Examine, Version=1.0.0.0, Culture=neutral, processorArchitecture=MSIL">
<HintPath>..\packages\Examine.1.0.0-beta024\lib\net45\Examine.dll</HintPath>
</Reference>
<Reference Include="ImageProcessor.Web, Version=4.8.4.0, Culture=neutral, processorArchitecture=MSIL">
<HintPath>..\packages\ImageProcessor.Web.4.8.4\lib\net45\ImageProcessor.Web.dll</HintPath>
</Reference>
@@ -273,10 +276,6 @@
<HintPath>../packages/ClientDependency-Mvc5.1.8.0.0/lib/net45/ClientDependency.Core.Mvc.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Examine">
<HintPath>../packages/Examine.2.0.0-beta2/lib/net45/Examine.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="ICSharpCode.SharpZipLib">
<HintPath>../packages/SharpZipLib.0.86.0/lib/20/ICSharpCode.SharpZipLib.dll</HintPath>
<Private>True</Private>

8
src/Umbraco.Web.UI/config/ExamineSettings.Release.config
View File

@@ -12,12 +12,12 @@ More information and documentation can be found on GitHub: https://github.com/Sh
<add name="InternalIndexer" type="Umbraco.Examine.UmbracoContentIndexer, Umbraco.Examine"
supportUnpublished="true"
supportProtected="true"
analyzer="Lucene.Net.Analysis.WhitespaceAnalyzer, Lucene.Net"/>
analyzer="Examine.LuceneEngine.CultureInvariantWhitespaceAnalyzer, Examine"/>
<add name="InternalMemberIndexer" type="Umbraco.Examine.UmbracoMemberIndexer, Umbraco.Examine"
supportUnpublished="true"
supportProtected="true"
analyzer="Lucene.Net.Analysis.Standard.StandardAnalyzer, Lucene.Net"/>
analyzer="Examine.LuceneEngine.CultureInvariantStandardAnalyzer, Examine"/>
<!-- default external indexer, which excludes protected and unpublished pages-->
<add name="ExternalIndexer" type="Umbraco.Examine.UmbracoContentIndexer, Umbraco.Examine"/>
@@ -25,7 +25,7 @@ More information and documentation can be found on GitHub: https://github.com/Sh
</providers>
</ExamineIndexProviders>
<ExamineSearchProviders defaultProvider="ExternalSearcher">
<ExamineSearchProviders>
<providers>
<add name="InternalSearcher" type="Umbraco.Examine.UmbracoExamineSearcher, Umbraco.Examine"
analyzer="Lucene.Net.Analysis.WhitespaceAnalyzer, Lucene.Net"/>
@@ -33,7 +33,7 @@ More information and documentation can be found on GitHub: https://github.com/Sh
<add name="ExternalSearcher" type="Umbraco.Examine.UmbracoExamineSearcher, Umbraco.Examine" />
<add name="InternalMemberSearcher" type="Umbraco.Examine.UmbracoExamineSearcher, Umbraco.Examine"
analyzer="Lucene.Net.Analysis.Standard.StandardAnalyzer, Lucene.Net" enableLeadingWildcard="true"/>
analyzer="Examine.LuceneEngine.CultureInvariantStandardAnalyzer, Examine" />
</providers>
</ExamineSearchProviders>

9
src/Umbraco.Web.UI/config/ExamineSettings.config
View File

@@ -12,12 +12,12 @@ More information and documentation can be found on GitHub: https://github.com/Sh
<add name="InternalIndexer" type="Umbraco.Examine.UmbracoContentIndexer, Umbraco.Examine"
supportUnpublished="true"
supportProtected="true"
analyzer="Lucene.Net.Analysis.WhitespaceAnalyzer, Lucene.Net"/>
analyzer="Examine.LuceneEngine.CultureInvariantWhitespaceAnalyzer, Examine"/>
<add name="InternalMemberIndexer" type="Umbraco.Examine.UmbracoMemberIndexer, Umbraco.Examine"
supportUnpublished="true"
supportProtected="true"
analyzer="Lucene.Net.Analysis.Standard.StandardAnalyzer, Lucene.Net"/>
analyzer="Examine.LuceneEngine.CultureInvariantStandardAnalyzer, Examine"/>
<!-- default external indexer, which excludes protected and unpublished pages-->
<add name="ExternalIndexer" type="Umbraco.Examine.UmbracoContentIndexer, Umbraco.Examine"/>
@@ -25,7 +25,7 @@ More information and documentation can be found on GitHub: https://github.com/Sh
</providers>
</ExamineIndexProviders>
<ExamineSearchProviders defaultProvider="ExternalSearcher">
<ExamineSearchProviders>
<providers>
<add name="InternalSearcher" type="Umbraco.Examine.UmbracoExamineSearcher, Umbraco.Examine"
analyzer="Lucene.Net.Analysis.WhitespaceAnalyzer, Lucene.Net"/>
@@ -33,8 +33,7 @@ More information and documentation can be found on GitHub: https://github.com/Sh
<add name="ExternalSearcher" type="Umbraco.Examine.UmbracoExamineSearcher, Umbraco.Examine"/>
<add name="InternalMemberSearcher" type="Umbraco.Examine.UmbracoExamineSearcher, Umbraco.Examine"
analyzer="Lucene.Net.Analysis.Standard.StandardAnalyzer, Lucene.Net"
enableLeadingWildcard="true"/>
analyzer="Examine.LuceneEngine.CultureInvariantStandardAnalyzer, Examine" />
</providers>
</ExamineSearchProviders>

2
src/Umbraco.Web.UI/packages.config
View File

@@ -4,7 +4,7 @@
<package id="ClientDependency" version="1.9.6" targetFramework="net461" />
<package id="ClientDependency-Mvc5" version="1.8.0.0" targetFramework="net461" />
<package id="CSharpTest.Net.Collections" version="14.906.1403.1082" targetFramework="net461" />
<package id="Examine" version="2.0.0-beta2" targetFramework="net461" />
<package id="Examine" version="1.0.0-beta024" targetFramework="net461" />
<package id="ImageProcessor" version="2.5.4" targetFramework="net461" />
<package id="ImageProcessor.Web" version="4.8.4" targetFramework="net461" />
<package id="ImageProcessor.Web.Config" version="2.3.0.0" targetFramework="net461" />

2
src/Umbraco.Web.UI/web.Template.config
View File

@@ -11,7 +11,7 @@
<section name="microsoft.scripting" type="Microsoft.Scripting.Hosting.Configuration.Section, Microsoft.Scripting, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35" requirePermission="false" />
<section name="clientDependency" type="ClientDependency.Core.Config.ClientDependencySection, ClientDependency.Core" requirePermission="false" />
<section name="Examine" type="Examine.Config.ExamineSettings, Examine" requirePermission="false" />
<section name="ExamineLuceneIndexSets" type="Examine.LuceneEngine.Config.IndexSets, Examine" requirePermission="false" />
<section name="ExamineLuceneIndexSets" type="Umbraco.Examine.Config.IndexSets, Umbraco.Examine" requirePermission="false" />
<section name="log4net" type="log4net.Config.Log4NetConfigurationSectionHandler, log4net" requirePermission="false" />
<sectionGroup name="umbracoConfiguration">

23
src/Umbraco.Web/Components/DatabaseServerRegistrarAndMessengerComponent.cs
View File

@@ -17,6 +17,8 @@ using Umbraco.Web.Cache;
using Umbraco.Web.Composing;
using Umbraco.Web.Routing;
using Umbraco.Web.Scheduling;
using Umbraco.Examine;
using Umbraco.Web.Search;
namespace Umbraco.Web.Components
{
@@ -44,6 +46,7 @@ namespace Umbraco.Web.Components
private BackgroundTaskRunner<IBackgroundTask> _processTaskRunner;
private bool _started;
private IBackgroundTask[] _tasks;
private IExamineManager _examineManager;
public override void Compose(Composition composition)
{
@@ -76,32 +79,21 @@ namespace Umbraco.Web.Components
// (we really should have a way to reuse RefreshAll... locally)
// note: refresh all content & media caches does refresh content types too
var svc = Current.PublishedSnapshotService;
bool ignored1, ignored2;
svc.Notify(new[] { new DomainCacheRefresher.JsonPayload(0, DomainChangeTypes.RefreshAll) });
svc.Notify(new[] { new ContentCacheRefresher.JsonPayload(0, TreeChangeTypes.RefreshAll) }, out ignored1, out ignored2);
svc.Notify(new[] { new MediaCacheRefresher.JsonPayload(0, TreeChangeTypes.RefreshAll) }, out ignored1);
svc.Notify(new[] { new ContentCacheRefresher.JsonPayload(0, TreeChangeTypes.RefreshAll) }, out _, out _);
svc.Notify(new[] { new MediaCacheRefresher.JsonPayload(0, TreeChangeTypes.RefreshAll) }, out _);
},
//rebuild indexes if the server is not synced
// NOTE: This will rebuild ALL indexes including the members, if developers want to target specific
// indexes then they can adjust this logic themselves.
() => RebuildIndexes(false)
() => ExamineComponent.RebuildIndexes(false, _examineManager, _logger)
}
});
});
}
// fixme - this should move to something else, we should not depend on Examine here!
private static void RebuildIndexes(bool onlyEmptyIndexes)
{
var indexers = (IEnumerable<KeyValuePair<string, IExamineIndexer>>) ExamineManager.Instance.IndexProviders;
if (onlyEmptyIndexes)
indexers = indexers.Where(x => x.Value.IsIndexNew());
foreach (var indexer in indexers)
indexer.Value.RebuildIndex();
}
public void Initialize(IRuntimeState runtime, IServerRegistrar serverRegistrar, IServerMessenger serverMessenger, IServerRegistrationService registrationService, ILogger logger)
public void Initialize(IRuntimeState runtime, IServerRegistrar serverRegistrar, IServerMessenger serverMessenger, IServerRegistrationService registrationService, ILogger logger, IExamineManager examineManager)
{
if (UmbracoConfig.For.UmbracoSettings().DistributedCall.Enabled) return;
@@ -114,6 +106,7 @@ namespace Umbraco.Web.Components
_runtime = runtime;
_logger = logger;
_registrationService = registrationService;
_examineManager = examineManager;
_touchTaskRunner = new BackgroundTaskRunner<IBackgroundTask>("ServerRegistration",
new BackgroundTaskRunnerOptions { AutoStart = true }, logger);

4
src/Umbraco.Web/ExamineExtensions.cs
View File

@@ -18,10 +18,12 @@ namespace Umbraco.Web
foreach (var result in results.OrderByDescending(x => x.Score))
{
var content = cache.GetById(result.Id);
if (!int.TryParse(result.Id, out var intId)) continue; //invalid
var content = cache.GetById(intId);
if (content == null) continue; // skip if this doesn't exist in the cache
list.Add(new PublishedSearchResult(content, result.Score));
}
return list;

8
src/Umbraco.Web/IPublishedContentQuery.cs
View File

@@ -34,21 +34,21 @@ namespace Umbraco.Web
/// <summary>
/// Searches content.
/// </summary>
IEnumerable<PublishedSearchResult> Search(string term, bool useWildCards = true, string searchProvider = null);
IEnumerable<PublishedSearchResult> Search(string term, bool useWildCards = true, string indexName = null);
/// <summary>
/// Searches content.
/// </summary>
IEnumerable<PublishedSearchResult> Search(int skip, int take, out int totalRecords, string term, bool useWildCards = true, string searchProvider = null);
IEnumerable<PublishedSearchResult> Search(int skip, int take, out int totalRecords, string term, bool useWildCards = true, string indexName = null);
/// <summary>
/// Searches content.
/// </summary>
IEnumerable<PublishedSearchResult> Search(Examine.SearchCriteria.ISearchCriteria criteria, Examine.Providers.BaseSearchProvider searchProvider = null);
IEnumerable<PublishedSearchResult> Search(Examine.SearchCriteria.ISearchCriteria criteria, Examine.ISearcher searchProvider = null);
/// <summary>
/// Searches content.
/// </summary>
IEnumerable<PublishedSearchResult> Search(int skip, int take, out int totalrecords, Examine.SearchCriteria.ISearchCriteria criteria, Examine.Providers.BaseSearchProvider searchProvider = null);
IEnumerable<PublishedSearchResult> Search(int skip, int take, out int totalRecords, Examine.SearchCriteria.ISearchCriteria criteria, Examine.ISearcher searchProvider = null);
}
}

28
src/Umbraco.Web/Install/InstallSteps/UpgradeStep.cs
View File

@@ -1,4 +1,6 @@
using Umbraco.Core.Configuration;
using System;
using Umbraco.Core.Composing;
using Umbraco.Core.Configuration;
using Umbraco.Web.Install.Models;
namespace Umbraco.Web.Install.InstallSteps
@@ -9,26 +11,28 @@ namespace Umbraco.Web.Install.InstallSteps
[InstallSetupStep(InstallationType.Upgrade, "Upgrade", "upgrade", 1, "Upgrading Umbraco to the latest and greatest version.")]
internal class UpgradeStep : InstallSetupStep<object>
{
public override bool RequiresExecution(object model)
{
return true;
}
public override bool RequiresExecution(object model) => true;
public override InstallSetupResult Execute(object model)
{
return null;
}
public override InstallSetupResult Execute(object model) => null;
public override object ViewModel
{
get
{
// fixme where is the "detected current version"?
var currentVersion = UmbracoVersion.Local.ToString();
var newVersion = UmbracoVersion.Current.ToString();
var newVersion = UmbracoVersion.SemanticVersion.ToString();
var state = Current.RuntimeState; // fixme inject
var currentState = state.CurrentMigrationState;
if (string.IsNullOrWhiteSpace(currentState)) currentState = "unknown";
var newState = state.FinalMigrationState?.Trim('{', '}');
if (string.IsNullOrWhiteSpace(newState)) newState = "unknown";
else if (Guid.TryParse(newState, out _))
newState = newState.Substring(0, 8);
var reportUrl = $"https://our.umbraco.org/contribute/releases/compare?from={currentVersion}&to={newVersion}&notes=1";
return new { currentVersion, newVersion, reportUrl };
return new { currentVersion, newVersion, currentState, newState, reportUrl };
}
}
}

20
src/Umbraco.Web/Models/Mapping/EntityMapperProfile.cs
View File

@@ -122,22 +122,22 @@ namespace Umbraco.Web.Models.Mapping
.AfterMap((src, dest) =>
{
//get the icon if there is one
dest.Icon = src.Fields.ContainsKey(BaseUmbracoIndexer.IconFieldName)
? src.Fields[BaseUmbracoIndexer.IconFieldName]
dest.Icon = src.Fields.ContainsKey(UmbracoExamineIndexer.IconFieldName)
? src.Fields[UmbracoExamineIndexer.IconFieldName]
: "icon-document";
dest.Name = src.Fields.ContainsKey("nodeName") ? src.Fields["nodeName"] : "[no name]";
if (src.Fields.ContainsKey(UmbracoContentIndexer.NodeKeyFieldName))
if (src.Fields.ContainsKey(UmbracoExamineIndexer.NodeKeyFieldName))
{
Guid key;
if (Guid.TryParse(src.Fields[UmbracoContentIndexer.NodeKeyFieldName], out key))
if (Guid.TryParse(src.Fields[UmbracoExamineIndexer.NodeKeyFieldName], out key))
{
dest.Key = key;
//need to set the UDI
if (src.Fields.ContainsKey(LuceneIndexer.IndexTypeFieldName))
if (src.Fields.ContainsKey(LuceneIndexer.CategoryFieldName))
{
switch (src.Fields[LuceneIndexer.IndexTypeFieldName])
switch (src.Fields[LuceneIndexer.CategoryFieldName])
{
case IndexTypes.Member:
dest.Udi = new GuidUdi(Constants.UdiEntityType.Member, dest.Key);
@@ -165,15 +165,15 @@ namespace Umbraco.Web.Models.Mapping
dest.ParentId = -1;
}
}
dest.Path = src.Fields.ContainsKey(UmbracoContentIndexer.IndexPathFieldName) ? src.Fields[UmbracoContentIndexer.IndexPathFieldName] : "";
dest.Path = src.Fields.ContainsKey(UmbracoExamineIndexer.IndexPathFieldName) ? src.Fields[UmbracoExamineIndexer.IndexPathFieldName] : "";
if (src.Fields.ContainsKey(LuceneIndexer.NodeTypeAliasFieldName))
if (src.Fields.ContainsKey(LuceneIndexer.ItemTypeFieldName))
{
dest.AdditionalData.Add("contentType", src.Fields[LuceneIndexer.NodeTypeAliasFieldName]);
dest.AdditionalData.Add("contentType", src.Fields[LuceneIndexer.ItemTypeFieldName]);
}
});
CreateMap<ILuceneSearchResults, IEnumerable<SearchResultItem>>()
CreateMap<ISearchResults, IEnumerable<SearchResultItem>>()
.ConvertUsing(results => results.Select(Mapper.Map<SearchResultItem>).ToList());
CreateMap<IEnumerable<SearchResult>, IEnumerable<SearchResultItem>>()

122
src/Umbraco.Web/PropertyEditors/GridPropertyEditor.cs
View File

@@ -27,84 +27,80 @@ namespace Umbraco.Web.PropertyEditors
internal void DocumentWriting(object sender, Examine.LuceneEngine.DocumentWritingEventArgs e)
{
var indexer = (BaseUmbracoIndexer)sender;
foreach (var field in indexer.IndexerData.UserFields)
foreach (var value in e.ValueSet.Values)
{
if (e.Fields.ContainsKey(field.Name))
//if there is a value, it's a string and it's detected as json
if (value.Value.Count > 0 && value.Value[0] != null && (value.Value[0] is string firstVal) && firstVal.DetectIsJson())
{
if (e.Fields[field.Name].DetectIsJson())
try
{
try
//TODO: We should deserialize this to Umbraco.Core.Models.GridValue instead of doing the below
var json = JsonConvert.DeserializeObject<JObject>(firstVal);
//check if this is formatted for grid json
if (json.HasValues && json.TryGetValue("name", out _) && json.TryGetValue("sections", out _))
{
//TODO: We should deserialize this to Umbraco.Core.Models.GridValue instead of doing the below
var json = JsonConvert.DeserializeObject<JObject>(e.Fields[field.Name]);
//check if this is formatted for grid json
JToken name;
JToken sections;
if (json.HasValues && json.TryGetValue("name", out name) && json.TryGetValue("sections", out sections))
//get all values and put them into a single field (using JsonPath)
var sb = new StringBuilder();
foreach (var row in json.SelectTokens("$.sections[*].rows[*]"))
{
//get all values and put them into a single field (using JsonPath)
var sb = new StringBuilder();
foreach (var row in json.SelectTokens("$.sections[*].rows[*]"))
{
var rowName = row["name"].Value<string>();
var areaVals = row.SelectTokens("$.areas[*].controls[*].value");
var rowName = row["name"].Value<string>();
var areaVals = row.SelectTokens("$.areas[*].controls[*].value");
foreach (var areaVal in areaVals)
foreach (var areaVal in areaVals)
{
//TODO: If it's not a string, then it's a json formatted value -
// we cannot really index this in a smart way since it could be 'anything'
if (areaVal.Type == JTokenType.String)
{
//TODO: If it's not a string, then it's a json formatted value -
// we cannot really index this in a smart way since it could be 'anything'
if (areaVal.Type == JTokenType.String)
{
var str = areaVal.Value<string>();
str = XmlHelper.CouldItBeXml(str) ? str.StripHtml() : str;
sb.Append(str);
sb.Append(" ");
//add the row name as an individual field
e.Document.Add(
new Field(
string.Format("{0}.{1}", field.Name, rowName), str, Field.Store.YES, Field.Index.ANALYZED));
}
var str = areaVal.Value<string>();
str = XmlHelper.CouldItBeXml(str) ? str.StripHtml() : str;
sb.Append(str);
sb.Append(" ");
//add the row name as an individual field
e.Document.Add(
new Field(
$"{value.Key}.{rowName}", str, Field.Store.YES, Field.Index.ANALYZED));
}
}
if (sb.Length > 0)
{
//First save the raw value to a raw field
e.Document.Add(
new Field(
string.Format("{0}{1}", UmbracoContentIndexer.RawFieldPrefix, field.Name),
e.Fields[field.Name], Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS, Field.TermVector.NO));
//now replace the original value with the combined/cleaned value
e.Document.RemoveField(field.Name);
e.Document.Add(
new Field(
field.Name,
sb.ToString(), Field.Store.YES, Field.Index.ANALYZED));
}
}
}
catch (InvalidCastException)
{
//swallow...on purpose, there's a chance that this isn't the json format we are looking for
// and we don't want that to affect the website.
}
catch (JsonException)
{
//swallow...on purpose, there's a chance that this isn't json and we don't want that to affect
// the website.
}
catch (ArgumentException)
{
//swallow on purpose to prevent this error:
// Can not add Newtonsoft.Json.Linq.JValue to Newtonsoft.Json.Linq.JObject.
if (sb.Length > 0)
{
//First save the raw value to a raw field
e.Document.Add(
new Field(
$"{UmbracoExamineIndexer.RawFieldPrefix}{value.Key}",
firstVal, Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS, Field.TermVector.NO));
//now replace the original value with the combined/cleaned value
e.Document.RemoveField(value.Key);
e.Document.Add(
new Field(
value.Key,
sb.ToString(), Field.Store.YES, Field.Index.ANALYZED));
}
}
}
catch (InvalidCastException)
{
//swallow...on purpose, there's a chance that this isn't the json format we are looking for
// and we don't want that to affect the website.
}
catch (JsonException)
{
//swallow...on purpose, there's a chance that this isn't json and we don't want that to affect
// the website.
}
catch (ArgumentException)
{
//swallow on purpose to prevent this error:
// Can not add Newtonsoft.Json.Linq.JValue to Newtonsoft.Json.Linq.JObject.
}
}
}
}

3
src/Umbraco.Web/PropertyEditors/PropertyEditorsComponent.cs
View File

@@ -1,4 +1,5 @@
using System.Linq;
using Examine;
using Umbraco.Core;
using Umbraco.Core.Components;
using Umbraco.Core.Models;
@@ -11,7 +12,7 @@ namespace Umbraco.Web.PropertyEditors
[RuntimeLevel(MinLevel = RuntimeLevel.Run)]
internal class PropertyEditorsComponent : UmbracoComponentBase, IUmbracoCoreComponent
{
public void Initialize(IRuntimeState runtime, PropertyEditorCollection propertyEditors, IExamineIndexCollectionAccessor indexCollection)
public void Initialize(IRuntimeState runtime, PropertyEditorCollection propertyEditors)
{
var fileUpload = propertyEditors.OfType<FileUploadPropertyEditor>().FirstOrDefault();
if (fileUpload != null) Initialize(fileUpload);

88
src/Umbraco.Web/PublishedCache/XmlPublishedCache/PublishedMediaCache.cs
View File

@@ -40,8 +40,8 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache
// when they are null the cache derives them from the ExamineManager, see
// method GetExamineManagerSafe().
//
private readonly ILuceneSearcher _searchProvider;
private readonly BaseIndexProvider _indexProvider;
private readonly ISearcher _searchProvider;
private readonly IIndexer _indexProvider;
private readonly XmlStore _xmlStore;
private readonly PublishedContentTypeCache _contentTypeCache;
@@ -51,10 +51,8 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache
public PublishedMediaCache(XmlStore xmlStore, IMediaService mediaService, IUserService userService, ICacheProvider cacheProvider, PublishedContentTypeCache contentTypeCache)
: base(false)
{
if (mediaService == null) throw new ArgumentNullException(nameof(mediaService));
if (userService == null) throw new ArgumentNullException(nameof(userService));
_mediaService = mediaService;
_userService = userService;
_mediaService = mediaService ?? throw new ArgumentNullException(nameof(mediaService));
_userService = userService ?? throw new ArgumentNullException(nameof(userService));
_cacheProvider = cacheProvider;
_xmlStore = xmlStore;
@@ -70,18 +68,13 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache
/// <param name="indexProvider"></param>
/// <param name="cacheProvider"></param>
/// <param name="contentTypeCache"></param>
internal PublishedMediaCache(IMediaService mediaService, IUserService userService, ILuceneSearcher searchProvider, BaseIndexProvider indexProvider, ICacheProvider cacheProvider, PublishedContentTypeCache contentTypeCache)
internal PublishedMediaCache(IMediaService mediaService, IUserService userService, ISearcher searchProvider, BaseIndexProvider indexProvider, ICacheProvider cacheProvider, PublishedContentTypeCache contentTypeCache)
: base(false)
{
if (mediaService == null) throw new ArgumentNullException(nameof(mediaService));
if (userService == null) throw new ArgumentNullException(nameof(userService));
if (searchProvider == null) throw new ArgumentNullException(nameof(searchProvider));
if (indexProvider == null) throw new ArgumentNullException(nameof(indexProvider));
_mediaService = mediaService;
_userService = userService;
_searchProvider = searchProvider;
_indexProvider = indexProvider;
_mediaService = mediaService ?? throw new ArgumentNullException(nameof(mediaService));
_userService = userService ?? throw new ArgumentNullException(nameof(userService));
_searchProvider = searchProvider ?? throw new ArgumentNullException(nameof(searchProvider));
_indexProvider = indexProvider ?? throw new ArgumentNullException(nameof(indexProvider));
_cacheProvider = cacheProvider;
_contentTypeCache = contentTypeCache;
}
@@ -117,8 +110,8 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache
// first check in Examine for the cache values
// +(+parentID:-1) +__IndexType:media
var criteria = searchProvider.CreateSearchCriteria("media");
var filter = criteria.ParentId(-1).Not().Field(UmbracoContentIndexer.IndexPathFieldName, "-1,-21,".MultipleCharacterWildcard());
var criteria = searchProvider.CreateCriteria("media");
var filter = criteria.ParentId(-1).Not().Field(UmbracoExamineIndexer.IndexPathFieldName, "-1,-21,".MultipleCharacterWildcard());
var result = searchProvider.Search(filter.Compile());
if (result != null)
@@ -228,7 +221,7 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache
public override bool HasContent(bool preview) { throw new NotImplementedException(); }
private static ExamineManager GetExamineManagerSafe()
private static IExamineManager GetExamineManagerSafe()
{
try
{
@@ -239,34 +232,8 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache
return null;
}
}
private BaseIndexProvider GetIndexProviderSafe()
{
if (_indexProvider != null)
return _indexProvider;
var eMgr = GetExamineManagerSafe();
if (eMgr == null) return null;
try
{
//by default use the InternalSearcher
var indexer = eMgr.IndexProviderCollection[Constants.Examine.InternalIndexer];
if (indexer.IndexerData.IncludeNodeTypes.Any() || indexer.IndexerData.ExcludeNodeTypes.Any())
{
Current.Logger.Warn<PublishedMediaCache>("The InternalIndexer for examine is configured incorrectly, it should not list any include/exclude node types or field names, it should simply be configured as: " + "<IndexSet SetName=\"InternalIndexSet\" IndexPath=\"~/App_Data/TEMP/ExamineIndexes/Internal/\" />");
}
return indexer;
}
catch (Exception ex)
{
Current.Logger.Error<PublishedMediaCache>("Could not retrieve the InternalIndexer", ex);
//something didn't work, continue returning null.
}
return null;
}
private ILuceneSearcher GetSearchProviderSafe()
private ISearcher GetSearchProviderSafe()
{
if (_searchProvider != null)
return _searchProvider;
@@ -276,7 +243,7 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache
try
{
//by default use the InternalSearcher
//by default use the internal index
return eMgr.GetSearcher(Constants.Examine.InternalIndexer);
}
catch (FileNotFoundException)
@@ -328,8 +295,8 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache
//
// note that since the use of the wildcard, it automatically escapes it in Lucene.
var criteria = searchProvider.CreateSearchCriteria("media");
var filter = criteria.Id(id).Not().Field(BaseUmbracoIndexer.IndexPathFieldName, "-1,-21,".MultipleCharacterWildcard());
var criteria = searchProvider.CreateCriteria("media");
var filter = criteria.Id(id.ToInvariantString()).Not().Field(UmbracoExamineIndexer.IndexPathFieldName, "-1,-21,".MultipleCharacterWildcard());
var result = searchProvider.Search(filter.Compile()).FirstOrDefault();
if (result != null) return ConvertFromSearchResult(result);
@@ -395,11 +362,10 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache
internal CacheValues ConvertFromSearchResult(SearchResult searchResult)
{
// note: fixing fields in 7.x, removed by Shan for 8.0
var values = new Dictionary<string, string>(searchResult.Fields);
return new CacheValues
{
Values = values,
Values = searchResult.Fields,
FromExamine = true
};
}
@@ -515,7 +481,7 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache
{
//We are going to check for a special field however, that is because in some cases we store a 'Raw'
//value in the index such as for xml/html.
var rawValue = dd.Properties.FirstOrDefault(x => x.Alias.InvariantEquals(BaseUmbracoIndexer.RawFieldPrefix + alias));
var rawValue = dd.Properties.FirstOrDefault(x => x.Alias.InvariantEquals(UmbracoExamineIndexer.RawFieldPrefix + alias));
return rawValue
?? dd.Properties.FirstOrDefault(x => x.Alias.InvariantEquals(alias));
}
@@ -545,12 +511,12 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache
//first check in Examine as this is WAY faster
var criteria = searchProvider.CreateCriteria("media");
var filter = criteria.ParentId(parentId).Not().Field(BaseUmbracoIndexer.IndexPathFieldName, "-1,-21,".MultipleCharacterWildcard());
var filter = criteria.ParentId(parentId).Not().Field(UmbracoExamineIndexer.IndexPathFieldName, "-1,-21,".MultipleCharacterWildcard());
//the above filter will create a query like this, NOTE: That since the use of the wildcard, it automatically escapes it in Lucene.
//+(+parentId:3113 -__Path:-1,-21,*) +__IndexType:media
// sort with the Sort field (updated for 8.0)
var results = searchProvider.Find(
var results = searchProvider.Search(
filter.And().OrderBy(new SortableField("sortOrder", SortType.Int)).Compile());
if (results.Any())
@@ -661,7 +627,7 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache
private static readonly string[] IgnoredKeys = { "version", "isDoc" };
public DictionaryPublishedContent(
IDictionary<string, string> valueDictionary,
IReadOnlyDictionary<string, string> valueDictionary,
Func<int, IPublishedContent> getParent,
Func<int, XPathNavigator, IEnumerable<IPublishedContent>> getChildren,
Func<DictionaryPublishedContent, string, IPublishedProperty> getProperty,
@@ -687,7 +653,7 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache
ValidateAndSetProperty(valueDictionary, val => _sortOrder = int.Parse(val), "sortOrder");
ValidateAndSetProperty(valueDictionary, val => _name = val, "nodeName", "__nodeName");
ValidateAndSetProperty(valueDictionary, val => _urlName = val, "urlName");
ValidateAndSetProperty(valueDictionary, val => _documentTypeAlias = val, "nodeTypeAlias", LuceneIndexer.NodeTypeAliasFieldName);
ValidateAndSetProperty(valueDictionary, val => _documentTypeAlias = val, "nodeTypeAlias", LuceneIndexer.ItemTypeFieldName);
ValidateAndSetProperty(valueDictionary, val => _documentTypeId = int.Parse(val), "nodeType");
//ValidateAndSetProperty(valueDictionary, val => _writerName = val, "writerName");
ValidateAndSetProperty(valueDictionary, val => _creatorName = val, "creatorName", "writerName"); //this is a bit of a hack fix for: U4-1132
@@ -852,7 +818,7 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache
private readonly ICollection<IPublishedProperty> _properties;
private readonly PublishedContentType _contentType;
private void ValidateAndSetProperty(IDictionary<string, string> valueDictionary, Action<string> setProperty, params string[] potentialKeys)
private void ValidateAndSetProperty(IReadOnlyDictionary<string, string> valueDictionary, Action<string> setProperty, params string[] potentialKeys)
{
var key = potentialKeys.FirstOrDefault(x => valueDictionary.ContainsKey(x) && valueDictionary[x] != null);
if (key == null)
@@ -901,7 +867,7 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache
internal class CacheValues
{
public IDictionary<string, string> Values { get; set; }
public IReadOnlyDictionary<string, string> Values { get; set; }
public XPathNavigator XPath { get; set; }
public bool FromExamine { get; set; }
}
@@ -976,7 +942,7 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache
GetValuesValue(v.Values, "path", "__Path").Contains(fid));
}
private static string GetValuesValue(IDictionary<string, string> d, params string[] keys)
private static string GetValuesValue(IReadOnlyDictionary<string, string> d, params string[] keys)
{
string value = null;
var ignored = keys.Any(x => d.TryGetValue(x, out value));

45
src/Umbraco.Web/PublishedContentExtensions.cs
View File

@@ -4,6 +4,7 @@ using System.Data;
using System.Globalization;
using System.Linq;
using System.Web;
using Examine;
using Examine.LuceneEngine.SearchCriteria;
using Umbraco.Core.Models.PublishedContent;
using Umbraco.Core;
@@ -220,49 +221,59 @@ namespace Umbraco.Web
#region Search
public static IEnumerable<PublishedSearchResult> Search(this IPublishedContent content, string term, bool useWildCards = true, string searchProvider = null)
{
var searcher = Examine.ExamineManager.Instance.DefaultSearchProvider;
if (string.IsNullOrEmpty(searchProvider) == false)
searcher = Examine.ExamineManager.Instance.SearchProviderCollection[searchProvider];
public static IEnumerable<PublishedSearchResult> Search(this IPublishedContent content, string term, bool useWildCards = true, string indexName = null)
{
//TODO: we should pass in the IExamineManager?
var searcher = string.IsNullOrEmpty(indexName)
? Examine.ExamineManager.Instance.GetSearcher(Constants.Examine.ExternalIndexer)
: Examine.ExamineManager.Instance.GetSearcher(indexName);
if (searcher == null)
throw new InvalidOperationException("No searcher found for index " + indexName);
var t = term.Escape().Value;
if (useWildCards)
t = term.MultipleCharacterWildcard().Value;
var luceneQuery = "+__Path:(" + content.Path.Replace("-", "\\-") + "*) +" + t;
var crit = searcher.CreateSearchCriteria().RawQuery(luceneQuery);
var crit = searcher.CreateCriteria().RawQuery(luceneQuery);
return content.Search(crit, searcher);
}
public static IEnumerable<PublishedSearchResult> SearchDescendants(this IPublishedContent content, string term, bool useWildCards = true, string searchProvider = null)
public static IEnumerable<PublishedSearchResult> SearchDescendants(this IPublishedContent content, string term, bool useWildCards = true, string indexName = null)
{
return content.Search(term, useWildCards, searchProvider);
return content.Search(term, useWildCards, indexName);
}
public static IEnumerable<PublishedSearchResult> SearchChildren(this IPublishedContent content, string term, bool useWildCards = true, string searchProvider = null)
public static IEnumerable<PublishedSearchResult> SearchChildren(this IPublishedContent content, string term, bool useWildCards = true, string indexName = null)
{
var searcher = Examine.ExamineManager.Instance.DefaultSearchProvider;
if (string.IsNullOrEmpty(searchProvider) == false)
searcher = Examine.ExamineManager.Instance.SearchProviderCollection[searchProvider];
//TODO: we should pass in the IExamineManager?
var searcher = string.IsNullOrEmpty(indexName)
? Examine.ExamineManager.Instance.GetSearcher(Constants.Examine.ExternalIndexer)
: Examine.ExamineManager.Instance.GetSearcher(indexName);
if (searcher == null)
throw new InvalidOperationException("No searcher found for index " + indexName);
var t = term.Escape().Value;
if (useWildCards)
t = term.MultipleCharacterWildcard().Value;
var luceneQuery = "+parentID:" + content.Id + " +" + t;
var crit = searcher.CreateSearchCriteria().RawQuery(luceneQuery);
var crit = searcher.CreateCriteria().RawQuery(luceneQuery);
return content.Search(crit, searcher);
}
public static IEnumerable<PublishedSearchResult> Search(this IPublishedContent content, Examine.SearchCriteria.ISearchCriteria criteria, Examine.Providers.BaseSearchProvider searchProvider = null)
public static IEnumerable<PublishedSearchResult> Search(this IPublishedContent content, Examine.SearchCriteria.ISearchCriteria criteria, Examine.ISearcher searchProvider = null)
{
var s = Examine.ExamineManager.Instance.DefaultSearchProvider;
if (searchProvider != null)
s = searchProvider;
//TODO: we should pass in the IExamineManager?
var s = searchProvider ?? Examine.ExamineManager.Instance.GetSearcher(Constants.Examine.ExternalIndexer);
var results = s.Search(criteria);
return results.ToPublishedSearchResults(UmbracoContext.Current.ContentCache);
}

79
src/Umbraco.Web/PublishedContentQuery.cs
View File

@@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.Linq;
using System.Reflection;
using System.Xml.XPath;
using Examine;
using Examine.LuceneEngine.Providers;
using Examine.LuceneEngine.SearchCriteria;
using Examine.SearchCriteria;
@@ -220,19 +221,25 @@ namespace Umbraco.Web
#region Search
/// <inheritdoc />
public IEnumerable<PublishedSearchResult> Search(string term, bool useWildCards = true, string searchProvider = null)
public IEnumerable<PublishedSearchResult> Search(string term, bool useWildCards = true, string indexName = null)
{
return Search(0, 0, out _, term, useWildCards, searchProvider);
return Search(0, 0, out _, term, useWildCards, indexName);
}
/// <inheritdoc />
public IEnumerable<PublishedSearchResult> Search(int skip, int take, out int totalRecords, string term, bool useWildCards = true, string searchProvider = null)
public IEnumerable<PublishedSearchResult> Search(int skip, int take, out int totalRecords, string term, bool useWildCards = true, string indexName = null)
{
if (_query != null) return _query.Search(skip, take, out totalRecords, term, useWildCards, searchProvider);
//TODO: Can we inject IExamineManager?
var searcher = string.IsNullOrWhiteSpace(searchProvider)
? Examine.ExamineManager.Instance.DefaultSearchProvider
: Examine.ExamineManager.Instance.SearchProviderCollection[searchProvider];
if (_query != null) return _query.Search(skip, take, out totalRecords, term, useWildCards, indexName);
var indexer = string.IsNullOrEmpty(indexName)
? Examine.ExamineManager.Instance.GetIndexer(Constants.Examine.ExternalIndexer)
: Examine.ExamineManager.Instance.GetIndexer(indexName);
if (indexer == null) throw new InvalidOperationException("No index found by name " + indexName);
var searcher = indexer.GetSearcher();
if (skip == 0 && take == 0)
{
@@ -241,30 +248,24 @@ namespace Umbraco.Web
return results.ToPublishedSearchResults(_contentCache);
}
if (!(searcher is BaseLuceneSearcher luceneSearcher))
{
var results = searcher.Search(term, useWildCards);
totalRecords = results.TotalItemCount;
// Examine skip, Linq take
return results.Skip(skip).ToPublishedSearchResults(_contentCache).Take(take);
}
var criteria = SearchAllFields(term, useWildCards, luceneSearcher);
var criteria = SearchAllFields(term, useWildCards, searcher, indexer);
return Search(skip, take, out totalRecords, criteria, searcher);
}
/// <inheritdoc />
public IEnumerable<PublishedSearchResult> Search(Examine.SearchCriteria.ISearchCriteria criteria, Examine.Providers.BaseSearchProvider searchProvider = null)
public IEnumerable<PublishedSearchResult> Search(ISearchCriteria criteria, Examine.ISearcher searchProvider = null)
{
return Search(0, 0, out _, criteria, searchProvider);
}
/// <inheritdoc />
public IEnumerable<PublishedSearchResult> Search(int skip, int take, out int totalRecords, Examine.SearchCriteria.ISearchCriteria criteria, Examine.Providers.BaseSearchProvider searchProvider = null)
public IEnumerable<PublishedSearchResult> Search(int skip, int take, out int totalRecords, ISearchCriteria criteria, Examine.ISearcher searchProvider = null)
{
if (_query != null) return _query.Search(skip, take, out totalRecords, criteria, searchProvider);
var searcher = searchProvider ?? Examine.ExamineManager.Instance.DefaultSearchProvider;
//TODO: Can we inject IExamineManager?
var searcher = searchProvider ?? Examine.ExamineManager.Instance.GetSearcher(Constants.Examine.ExternalIndexer);
var results = skip == 0 && take == 0
? searcher.Search(criteria)
@@ -277,45 +278,25 @@ namespace Umbraco.Web
/// <summary>
/// Creates an ISearchCriteria for searching all fields in a <see cref="BaseLuceneSearcher"/>.
/// </summary>
/// <remarks>
/// This is here because some of this stuff is internal in Examine.
/// </remarks>
private ISearchCriteria SearchAllFields(string searchText, bool useWildcards, BaseLuceneSearcher searcher)
private ISearchCriteria SearchAllFields(string searchText, bool useWildcards, Examine.ISearcher searcher, Examine.IIndexer indexer)
{
var sc = searcher.CreateSearchCriteria();
var sc = searcher.CreateCriteria();
if (_examineGetSearchFields == null)
{
//get the GetSearchFields method from BaseLuceneSearcher
_examineGetSearchFields = typeof(BaseLuceneSearcher).GetMethod("GetSearchFields", BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Static);
}
//get the results of searcher.BaseLuceneSearcher() using ugly reflection since it's not public
var searchFields = (IEnumerable<string>) _examineGetSearchFields.Invoke(searcher, null);
//if we're dealing with a lucene searcher, we can get all of it's indexed fields,
//else we can get the defined fields for the index.
var searchFields = (searcher is BaseLuceneSearcher luceneSearcher)
? luceneSearcher.GetAllIndexedFields()
: indexer.FieldDefinitionCollection.Keys;
//this is what Examine does internally to create ISearchCriteria for searching all fields
var strArray = searchText.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
sc = useWildcards == false
? sc.GroupedOr(searchFields, strArray).Compile()
: sc.GroupedOr(searchFields, strArray.Select(x => new CustomExamineValue(Examineness.ComplexWildcard, x.MultipleCharacterWildcard().Value)).ToArray<IExamineValue>()).Compile();
: sc.GroupedOr(searchFields, strArray.Select(x => (IExamineValue)new ExamineValue(Examineness.ComplexWildcard, x.MultipleCharacterWildcard().Value)).ToArray()).Compile();
return sc;
}
}
private static MethodInfo _examineGetSearchFields;
//support class since Examine doesn't expose it's own ExamineValue class publicly
private class CustomExamineValue : IExamineValue
{
public CustomExamineValue(Examineness vagueness, string value)
{
this.Examineness = vagueness;
this.Value = value;
this.Level = 1f;
}
public Examineness Examineness { get; private set; }
public string Value { get; private set; }
public float Level { get; private set; }
}
#endregion
}

3
src/Umbraco.Web/Runtime/WebRuntimeComponent.cs
View File

@@ -12,6 +12,7 @@ using System.Web.Mvc;
using System.Web.Routing;
using ClientDependency.Core.CompositeFiles.Providers;
using ClientDependency.Core.Config;
using Examine;
using LightInject;
using Microsoft.AspNet.SignalR;
using Umbraco.Core;
@@ -99,7 +100,7 @@ namespace Umbraco.Web.Runtime
composition.Container.RegisterSingleton<IApplicationTreeService, ApplicationTreeService>();
composition.Container.RegisterSingleton<ISectionService, SectionService>();
composition.Container.RegisterSingleton<IExamineIndexCollectionAccessor, ExamineIndexCollectionAccessor>();
composition.Container.RegisterSingleton<IExamineManager>(factory => ExamineManager.Instance);
// IoC setup for LightInject for MVC/WebApi
// see comments on MixedLightInjectScopeManagerProvider for explainations of what we are doing here

519
src/Umbraco.Web/Search/ExamineComponent.cs
View File

@@ -1,4 +1,5 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
@@ -6,8 +7,9 @@ using System.Threading;
using System.Xml.Linq;
using Examine;
using Examine.LuceneEngine;
using Examine.Session;
using Examine.LuceneEngine.Providers;
using Lucene.Net.Documents;
using Lucene.Net.Index;
using Umbraco.Core;
using Umbraco.Core.Cache;
using Umbraco.Core.Components;
@@ -18,6 +20,7 @@ using Umbraco.Core.Scoping;
using Umbraco.Core.Services;
using Umbraco.Core.Services.Changes;
using Umbraco.Core.Services.Implement;
using Umbraco.Core.Strings;
using Umbraco.Core.Sync;
using Umbraco.Web.Cache;
using Umbraco.Web.Composing;
@@ -32,64 +35,70 @@ namespace Umbraco.Web.Search
[RuntimeLevel(MinLevel = RuntimeLevel.Run)]
public sealed class ExamineComponent : UmbracoComponentBase, IUmbracoCoreComponent
{
private IExamineManager _examineManager;
private static bool _disableExamineIndexing = false;
private static volatile bool _isConfigured = false;
private static readonly object IsConfiguredLocker = new object();
private IScopeProvider _scopeProvider;
private UrlSegmentProviderCollection _urlSegmentProviders;
private ServiceContext _services;
// the default enlist priority is 100
// enlist with a lower priority to ensure that anything "default" runs after us
// but greater that SafeXmlReaderWriter priority which is 60
private const int EnlistPriority = 80;
public void Initialize(IRuntimeState runtime, PropertyEditorCollection propertyEditors, IExamineIndexCollectionAccessor indexCollection, IScopeProvider scopeProvider, ILogger logger)
internal void Initialize(IRuntimeState runtime, MainDom mainDom, PropertyEditorCollection propertyEditors, IExamineManager examineManager, ProfilingLogger profilingLogger, IScopeProvider scopeProvider, UrlSegmentProviderCollection urlSegmentProviderCollection, ServiceContext services)
{
_services = services;
_urlSegmentProviders = urlSegmentProviderCollection;
_scopeProvider = scopeProvider;
_examineManager = examineManager;
logger.Info<ExamineComponent>("Starting initialize async background thread.");
//We want to manage Examine's appdomain shutdown sequence ourselves so first we'll disable Examine's default behavior
//and then we'll use MainDom to control Examine's shutdown
ExamineManager.DisableDefaultHostingEnvironmentRegistration();
// make it async in order not to slow down the boot
// fixme - should be a proper background task else we cannot stop it!
var bg = new Thread(() =>
//we want to tell examine to use a different fs lock instead of the default NativeFSFileLock which could cause problems if the appdomain
//terminates and in some rare cases would only allow unlocking of the file if IIS is forcefully terminated. Instead we'll rely on the simplefslock
//which simply checks the existence of the lock file
DirectoryTracker.DefaultLockFactory = d =>
{
try
var simpleFsLockFactory = new NoPrefixSimpleFsLockFactory(d);
return simpleFsLockFactory;
};
//let's deal with shutting down Examine with MainDom
var examineShutdownRegistered = mainDom.Register(() =>
{
using (profilingLogger.TraceDuration<ExamineComponent>("Examine shutting down"))
{
// from WebRuntimeComponent
// rebuilds any empty indexes
RebuildIndexes(true);
}
catch (Exception e)
{
logger.Error<ExamineComponent>("Failed to rebuild empty indexes.", e);
}
try
{
// from PropertyEditorsComponent
var grid = propertyEditors.OfType<GridPropertyEditor>().FirstOrDefault();
if (grid != null) BindGridToExamine(grid, indexCollection);
}
catch (Exception e)
{
logger.Error<ExamineComponent>("Failed to bind grid property editor.", e);
ExamineManager.Instance.Dispose();
}
});
bg.Start();
// the rest is the original Examine event handler
if (!examineShutdownRegistered)
{
profilingLogger.Logger.Debug<ExamineComponent>("Examine shutdown not registered, this appdomain is not the MainDom, Examine will be disabled");
logger.Info<ExamineComponent>("Initialize and bind to business logic events.");
//if we could not register the shutdown examine ourselves, it means we are not maindom! in this case all of examine should be disabled!
Suspendable.ExamineEvents.SuspendIndexers();
_disableExamineIndexing = true;
return; //exit, do not continue
}
//TODO: For now we'll make this true, it means that indexes will be near real time
// we'll see about what implications this may have - should be great in most scenarios
DefaultExamineSession.RequireImmediateConsistency = true;
profilingLogger.Logger.Debug<ExamineComponent>("Examine shutdown registered with MainDom");
var registeredProviders = ExamineManager.Instance.IndexProviderCollection
.OfType<BaseUmbracoIndexer>().Count(x => x.EnableDefaultEventHandler);
var registeredIndexers = examineManager.IndexProviders.Values.OfType<UmbracoExamineIndexer>().Count(x => x.EnableDefaultEventHandler);
logger.Info<ExamineComponent>($"Adding examine event handlers for {registeredProviders} index providers.");
profilingLogger.Logger.Info<ExamineComponent>($"Adding examine event handlers for {registeredIndexers} index providers.");
// don't bind event handlers if we're not suppose to listen
if (registeredProviders == 0)
if (registeredIndexers == 0)
return;
BindGridToExamine(profilingLogger.Logger, examineManager, propertyEditors);
// bind to distributed cache events - this ensures that this logic occurs on ALL servers
// that are taking part in a load balanced environment.
ContentCacheRefresher.CacheUpdated += ContentCacheRefresherUpdated;
@@ -100,36 +109,123 @@ namespace Umbraco.Web.Search
// events handling removed in ef013f9d3b945d0a48a306ff1afbd49c10c3fff8
// because, could not make sense of it?
var contentIndexer = ExamineManager.Instance.IndexProviderCollection[Constants.Examine.InternalIndexer] as UmbracoContentIndexer;
if (contentIndexer != null)
{
contentIndexer.DocumentWriting += IndexerDocumentWriting;
}
var memberIndexer = ExamineManager.Instance.IndexProviderCollection[Constants.Examine.InternalMemberIndexer] as UmbracoMemberIndexer;
if (memberIndexer != null)
{
memberIndexer.DocumentWriting += IndexerDocumentWriting;
}
EnsureUnlocked(profilingLogger.Logger, examineManager);
RebuildIndexesOnStartup(profilingLogger.Logger);
}
private static void RebuildIndexes(bool onlyEmptyIndexes)
/// <summary>
/// Called to rebuild empty indexes on startup
/// </summary>
/// <param name="logger"></param>
private void RebuildIndexesOnStartup(ILogger logger)
{
var indexers = (IEnumerable<KeyValuePair<string, IExamineIndexer>>)ExamineManager.Instance.IndexProviders;
//TODO: need a way to disable rebuilding on startup
logger.Info<ExamineComponent>("Starting initialize async background thread.");
// make it async in order not to slow down the boot
// fixme - should be a proper background task else we cannot stop it!
var bg = new Thread(() =>
{
try
{
// rebuilds any empty indexes
RebuildIndexes(true, _examineManager, logger);
}
catch (Exception e)
{
logger.Error<ExamineComponent>("Failed to rebuild empty indexes.", e);
}
});
bg.Start();
}
/// <summary>
/// Used to rebuild indexes on startup or cold boot
/// </summary>
/// <param name="onlyEmptyIndexes"></param>
/// <param name="examineManager"></param>
/// <param name="logger"></param>
internal static void RebuildIndexes(bool onlyEmptyIndexes, IExamineManager examineManager, ILogger logger)
{
//do not attempt to do this if this has been disabled since we are not the main dom.
//this can be called during a cold boot
if (_disableExamineIndexing) return;
EnsureUnlocked(logger, examineManager);
if (onlyEmptyIndexes)
indexers = indexers.Where(x => x.Value.IsIndexNew());
foreach (var indexer in indexers)
indexer.Value.RebuildIndex();
{
foreach (var indexer in examineManager.IndexProviders.Values.Where(x => x.IsIndexNew()))
{
indexer.RebuildIndex();
}
}
else
{
//do all of them
ExamineManager.Instance.RebuildIndexes();
}
}
private static void BindGridToExamine(GridPropertyEditor grid, IExamineIndexCollectionAccessor indexCollection)
/// <summary>
/// Must be called to each index is unlocked before any indexing occurs
/// </summary>
/// <remarks>
/// Indexing rebuilding can occur on a normal boot if the indexes are empty or on a cold boot by the database server messenger. Before
/// either of these happens, we need to configure the indexes.
/// </remarks>
private static void EnsureUnlocked(ILogger logger, IExamineManager examineManager)
{
var indexes = indexCollection.Indexes;
if (indexes == null) return;
foreach (var i in indexes.Values.OfType<BaseUmbracoIndexer>())
i.DocumentWriting += grid.DocumentWriting;
if (_disableExamineIndexing) return;
if (_isConfigured) return;
lock (IsConfiguredLocker)
{
//double chekc
if (_isConfigured) return;
_isConfigured = true;
foreach (var luceneIndexer in examineManager.IndexProviders.Values.OfType<LuceneIndexer>())
{
//We now need to disable waiting for indexing for Examine so that the appdomain is shutdown immediately and doesn't wait for pending
//indexing operations. We used to wait for indexing operations to complete but this can cause more problems than that is worth because
//that could end up halting shutdown for a very long time causing overlapping appdomains and many other problems.
luceneIndexer.WaitForIndexQueueOnShutdown = false;
//we should check if the index is locked ... it shouldn't be! We are using simple fs lock now and we are also ensuring that
//the indexes are not operational unless MainDom is true
var dir = luceneIndexer.GetLuceneDirectory();
if (IndexWriter.IsLocked(dir))
{
logger.Info<ExamineComponent>("Forcing index " + luceneIndexer.Name + " to be unlocked since it was left in a locked state");
IndexWriter.Unlock(dir);
}
}
}
}
void MemberCacheRefresherUpdated(MemberCacheRefresher sender, CacheRefresherEventArgs args)
private static void BindGridToExamine(ILogger logger, IExamineManager examineManager, IEnumerable propertyEditors)
{
//bind the grid property editors - this is a hack until http://issues.umbraco.org/issue/U4-8437
try
{
var grid = propertyEditors.OfType<GridPropertyEditor>().FirstOrDefault();
if (grid != null)
{
foreach (var i in examineManager.IndexProviders.Values.OfType<UmbracoExamineIndexer>())
i.DocumentWriting += grid.DocumentWriting;
}
}
catch (Exception e)
{
logger.Error<ExamineComponent>("Failed to bind grid property editor.", e);
}
}
private void MemberCacheRefresherUpdated(MemberCacheRefresher sender, CacheRefresherEventArgs args)
{
if (Suspendable.ExamineEvents.CanIndex == false)
return;
@@ -137,7 +233,7 @@ namespace Umbraco.Web.Search
switch (args.MessageType)
{
case MessageType.RefreshById:
var c1 = Current.Services.MemberService.GetById((int)args.MessageObject);
var c1 = _services.MemberService.GetById((int)args.MessageObject);
if (c1 != null)
{
ReIndexForMember(c1);
@@ -150,8 +246,7 @@ namespace Umbraco.Web.Search
DeleteIndexForEntity((int)args.MessageObject, false);
break;
case MessageType.RefreshByInstance:
var c3 = args.MessageObject as IMember;
if (c3 != null)
if (args.MessageObject is IMember c3)
{
ReIndexForMember(c3);
}
@@ -160,8 +255,7 @@ namespace Umbraco.Web.Search
// This is triggered when the item is permanently deleted
var c4 = args.MessageObject as IMember;
if (c4 != null)
if (args.MessageObject is IMember c4)
{
DeleteIndexForEntity(c4.Id, false);
}
@@ -174,7 +268,7 @@ namespace Umbraco.Web.Search
}
}
void MediaCacheRefresherUpdated(MediaCacheRefresher sender, CacheRefresherEventArgs args)
private void MediaCacheRefresherUpdated(MediaCacheRefresher sender, CacheRefresherEventArgs args)
{
if (Suspendable.ExamineEvents.CanIndex == false)
return;
@@ -182,7 +276,7 @@ namespace Umbraco.Web.Search
if (args.MessageType != MessageType.RefreshByPayload)
throw new NotSupportedException();
var mediaService = Current.Services.MediaService;
var mediaService = _services.MediaService;
foreach (var payload in (MediaCacheRefresher.JsonPayload[]) args.MessageObject)
{
@@ -223,7 +317,7 @@ namespace Umbraco.Web.Search
}
}
void ContentCacheRefresherUpdated(ContentCacheRefresher sender, CacheRefresherEventArgs args)
private void ContentCacheRefresherUpdated(ContentCacheRefresher sender, CacheRefresherEventArgs args)
{
if (Suspendable.ExamineEvents.CanIndex == false)
return;
@@ -231,7 +325,7 @@ namespace Umbraco.Web.Search
if (args.MessageType != MessageType.RefreshByPayload)
throw new NotSupportedException();
var contentService = Current.Services.ContentService;
var contentService = _services.ContentService;
foreach (var payload in (ContentCacheRefresher.JsonPayload[]) args.MessageObject)
{
@@ -296,6 +390,8 @@ namespace Umbraco.Web.Search
// ReIndexForContent is NOT taking care of descendants so we have to reload everything
// again in order to process the branch - we COULD improve that by just reloading the
// XML from database instead of reloading content & re-serializing!
//
// BUT ... pretty sure it is! see test "Index_Delete_Index_Item_Ensure_Heirarchy_Removed"
}
}
@@ -325,26 +421,27 @@ namespace Umbraco.Web.Search
{
var actions = DeferedActions.Get(_scopeProvider);
if (actions != null)
actions.Add(new DeferedReIndexForContent(sender, supportUnpublished));
actions.Add(new DeferedReIndexForContent(this, sender, supportUnpublished));
else
DeferedReIndexForContent.Execute(sender, supportUnpublished);
DeferedReIndexForContent.Execute(this, sender, supportUnpublished);
}
private void ReIndexForMember(IMember member)
{
var actions = DeferedActions.Get(_scopeProvider);
if (actions != null)
actions.Add(new DeferedReIndexForMember(member));
actions.Add(new DeferedReIndexForMember(this, member));
else
DeferedReIndexForMember.Execute(member); }
DeferedReIndexForMember.Execute(this, member);
}
private void ReIndexForMedia(IMedia sender, bool isMediaPublished)
{
var actions = DeferedActions.Get(_scopeProvider);
if (actions != null)
actions.Add(new DeferedReIndexForMedia(sender, isMediaPublished));
actions.Add(new DeferedReIndexForMedia(this, sender, isMediaPublished));
else
DeferedReIndexForMedia.Execute(sender, isMediaPublished);
DeferedReIndexForMedia.Execute(this, sender, isMediaPublished);
}
/// <summary>
@@ -359,189 +456,165 @@ namespace Umbraco.Web.Search
{
var actions = DeferedActions.Get(_scopeProvider);
if (actions != null)
actions.Add(new DeferedDeleteIndex(entityId, keepIfUnpublished));
actions.Add(new DeferedDeleteIndex(this, entityId, keepIfUnpublished));
else
DeferedDeleteIndex.Execute(entityId, keepIfUnpublished);
}
/// <summary>
/// Event handler to create a lower cased version of the node name, this is so we can support case-insensitive searching and still
/// use the Whitespace Analyzer
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private static void IndexerDocumentWriting(object sender, DocumentWritingEventArgs e)
{
if (e.Fields.Keys.Contains("nodeName"))
{
//TODO: This logic should really be put into the content indexer instead of hidden here!!
//add the lower cased version
e.Document.Add(new Field("__nodeName",
e.Fields["nodeName"].ToLower(),
Field.Store.YES,
Field.Index.ANALYZED,
Field.TermVector.NO
));
}
DeferedDeleteIndex.Execute(this, entityId, keepIfUnpublished);
}
private class DeferedActions
{
private readonly List<DeferedAction> _actions = new List<DeferedAction>();
{
private readonly List<DeferedAction> _actions = new List<DeferedAction>();
public static DeferedActions Get(IScopeProvider scopeProvider)
{
var scopeContext = scopeProvider.Context;
if (scopeContext == null) return null;
public static DeferedActions Get(IScopeProvider scopeProvider)
{
var scopeContext = scopeProvider.Context;
return scopeContext.Enlist("examineEvents",
() => new DeferedActions(), // creator
(completed, actions) => // action
{
if (completed) actions.Execute();
}, EnlistPriority);
}
return scopeContext?.Enlist("examineEvents",
() => new DeferedActions(), // creator
(completed, actions) => // action
{
if (completed) actions.Execute();
}, EnlistPriority);
}
public void Add(DeferedAction action)
{
public void Add(DeferedAction action)
{
_actions.Add(action);
}
}
private void Execute()
{
foreach (var action in _actions)
action.Execute();
}
}
private void Execute()
{
foreach (var action in _actions)
action.Execute();
}
}
private abstract class DeferedAction
{
private abstract class DeferedAction
{
public virtual void Execute()
{ }
}
}
private class DeferedReIndexForContent : DeferedAction
{
private readonly IContent _content;
private readonly bool? _supportUnpublished;
private class DeferedReIndexForContent : DeferedAction
{
private readonly ExamineComponent _examineComponent;
private readonly IContent _content;
private readonly bool? _supportUnpublished;
public DeferedReIndexForContent(IContent content, bool? supportUnpublished)
{
_content = content;
_supportUnpublished = supportUnpublished;
}
public DeferedReIndexForContent(ExamineComponent examineComponent, IContent content, bool? supportUnpublished)
{
_examineComponent = examineComponent;
_content = content;
_supportUnpublished = supportUnpublished;
}
public override void Execute()
{
Execute(_content, _supportUnpublished);
}
public override void Execute()
{
Execute(_examineComponent, _content, _supportUnpublished);
}
public static void Execute(IContent content, bool? supportUnpublished)
{
var xml = content.ToXml();
//add an icon attribute to get indexed
xml.Add(new XAttribute("icon", content.ContentType.Icon));
public static void Execute(ExamineComponent examineComponent, IContent content, bool? supportUnpublished)
{
var valueSet = UmbracoContentIndexer.GetValueSets(examineComponent._urlSegmentProviders, examineComponent._services.UserService, content);
ExamineManager.Instance.ReIndexNode(
xml, IndexTypes.Content,
ExamineManager.Instance.IndexProviderCollection.OfType<UmbracoContentIndexer>()
ExamineManager.Instance.IndexItems(
valueSet.ToArray(),
examineComponent._examineManager.IndexProviders.Values.OfType<UmbracoContentIndexer>()
// only for the specified indexers
.Where(x => supportUnpublished.HasValue == false || supportUnpublished.Value == x.SupportUnpublishedContent)
.Where(x => x.EnableDefaultEventHandler));
}
}
//Index this item for all indexers if the content is published, otherwise if the item is not published
// then only index this for indexers supporting unpublished content
private class DeferedReIndexForMedia : DeferedAction
{
private readonly ExamineComponent _examineComponent;
private readonly IMedia _media;
private readonly bool _isPublished;
.Where(x => supportUnpublished.HasValue == false || supportUnpublished.Value == x.SupportUnpublishedContent)
.Where(x => x.EnableDefaultEventHandler));
}
}
public DeferedReIndexForMedia(ExamineComponent examineComponent, IMedia media, bool isPublished)
{
_examineComponent = examineComponent;
_media = media;
_isPublished = isPublished;
}
private class DeferedReIndexForMedia : DeferedAction
{
private readonly IMedia _media;
private readonly bool _isPublished;
public override void Execute()
{
Execute(_examineComponent, _media, _isPublished);
}
public DeferedReIndexForMedia(IMedia media, bool isPublished)
{
_media = media;
_isPublished = isPublished;
}
public static void Execute(ExamineComponent examineComponent, IMedia media, bool isPublished)
{
var valueSet = UmbracoContentIndexer.GetValueSets(examineComponent._urlSegmentProviders, examineComponent._services.UserService, media);
public override void Execute()
{
Execute(_media, _isPublished);
}
ExamineManager.Instance.IndexItems(
valueSet.ToArray(),
examineComponent._examineManager.IndexProviders.Values.OfType<UmbracoContentIndexer>()
// index this item for all indexers if the media is not trashed, otherwise if the item is trashed
// then only index this for indexers supporting unpublished media
.Where(x => isPublished || (x.SupportUnpublishedContent))
.Where(x => x.EnableDefaultEventHandler));
}
}
public static void Execute(IMedia media, bool isPublished)
{
var xml = media.ToXml();
//add an icon attribute to get indexed
xml.Add(new XAttribute("icon", media.ContentType.Icon));
private class DeferedReIndexForMember : DeferedAction
{
private readonly ExamineComponent _examineComponent;
private readonly IMember _member;
ExamineManager.Instance.ReIndexNode(
xml, IndexTypes.Media,
ExamineManager.Instance.IndexProviderCollection.OfType<UmbracoContentIndexer>()
public DeferedReIndexForMember(ExamineComponent examineComponent, IMember member)
{
_examineComponent = examineComponent;
_member = member;
}
//Index this item for all indexers if the media is not trashed, otherwise if the item is trashed
// then only index this for indexers supporting unpublished media
public override void Execute()
{
Execute(_examineComponent, _member);
}
.Where(x => isPublished || (x.SupportUnpublishedContent))
.Where(x => x.EnableDefaultEventHandler));
}
}
public static void Execute(ExamineComponent examineComponent, IMember member)
{
var valueSet = UmbracoMemberIndexer.GetValueSets(member);
private class DeferedReIndexForMember : DeferedAction
{
private readonly IMember _member;
ExamineManager.Instance.IndexItems(
valueSet.ToArray(),
examineComponent._examineManager.IndexProviders.Values.OfType<UmbracoExamineIndexer>()
//ensure that only the providers are flagged to listen execute
.Where(x => x.EnableDefaultEventHandler));
}
}
public DeferedReIndexForMember(IMember member)
{
_member = member;
}
private class DeferedDeleteIndex : DeferedAction
{
private readonly ExamineComponent _examineComponent;
private readonly int _id;
private readonly bool _keepIfUnpublished;
public override void Execute()
{
Execute(_member);
}
public DeferedDeleteIndex(ExamineComponent examineComponent, int id, bool keepIfUnpublished)
{
_examineComponent = examineComponent;
_id = id;
_keepIfUnpublished = keepIfUnpublished;
}
public static void Execute(IMember member)
{
ExamineManager.Instance.ReIndexNode(
member.ToXml(), IndexTypes.Member,
ExamineManager.Instance.IndexProviderCollection.OfType<BaseUmbracoIndexer>()
//ensure that only the providers are flagged to listen execute
.Where(x => x.EnableDefaultEventHandler));
}
}
public override void Execute()
{
Execute(_examineComponent, _id, _keepIfUnpublished);
}
private class DeferedDeleteIndex : DeferedAction
{
private readonly int _id;
private readonly bool _keepIfUnpublished;
public static void Execute(ExamineComponent examineComponent, int id, bool keepIfUnpublished)
{
ExamineManager.Instance.DeleteFromIndexes(
id.ToString(CultureInfo.InvariantCulture),
examineComponent._examineManager.IndexProviders.Values.OfType<UmbracoExamineIndexer>()
// if keepIfUnpublished == true then only delete this item from indexes not supporting unpublished content,
// otherwise if keepIfUnpublished == false then remove from all indexes
.Where(x => keepIfUnpublished == false || (x is UmbracoContentIndexer && ((UmbracoContentIndexer)x).SupportUnpublishedContent == false))
.Where(x => x.EnableDefaultEventHandler));
}
}
public DeferedDeleteIndex(int id, bool keepIfUnpublished)
{
_id = id;
_keepIfUnpublished = keepIfUnpublished;
}
public override void Execute()
{
Execute(_id, _keepIfUnpublished);
}
public static void Execute(int id, bool keepIfUnpublished)
{
ExamineManager.Instance.DeleteFromIndex(
id.ToString(CultureInfo.InvariantCulture),
ExamineManager.Instance.IndexProviderCollection.OfType<UmbracoContentIndexer>()
//if keepIfUnpublished == true then only delete this item from indexes not supporting unpublished content,
// otherwise if keepIfUnpublished == false then remove from all indexes
.Where(x => keepIfUnpublished == false || x.SupportUnpublishedContent == false)
.Where(x => x.EnableDefaultEventHandler));
}
}
}
}

14
src/Umbraco.Web/Search/ExamineIndexerModel.cs
View File

@@ -22,19 +22,7 @@ namespace Umbraco.Web.Search
/// </summary>
[DataMember(Name = "fieldCount")]
public int FieldCount { get; set; }
/// <summary>
/// The number of documents flagged for deletion in the index
/// </summary>
[DataMember(Name = "deletionCount")]
public int DeletionCount { get; set; }
/// <summary>
/// Whether or not the indexed is optimized
/// </summary>
[DataMember(Name = "isOptimized")]
public bool IsOptimized{ get; set; }
/// <summary>
/// Generally will always be true unless someone has created a new non-lucene index
/// </summary>

41
src/Umbraco.Web/Search/LuceneIndexerExtensions.cs
View File

@@ -48,8 +48,9 @@ namespace Umbraco.Web.Search
{
try
{
var searcher = indexer.GetSearcher().GetSearcher() as IndexSearcher;
if (searcher == null) return 0;
if (!((indexer.GetSearcher() as LuceneSearcher)?.GetLuceneSearcher() is IndexSearcher searcher))
return 0;
using (searcher)
using (var reader = searcher.IndexReader)
{
@@ -70,12 +71,11 @@ namespace Umbraco.Web.Search
/// <returns></returns>
public static int GetIndexFieldCount(this LuceneIndexer indexer)
{
//TODO: check for closing! and AlreadyClosedException
try
{
var searcher = indexer.GetSearcher().GetSearcher() as IndexSearcher;
if (searcher == null) return 0;
if (!((indexer.GetSearcher() as LuceneSearcher)?.GetLuceneSearcher() is IndexSearcher searcher))
return 0;
using (searcher)
using (var reader = searcher.IndexReader)
{
@@ -89,30 +89,6 @@ namespace Umbraco.Web.Search
}
}
/// <summary>
/// Returns true if the index is optimized or not
/// </summary>
/// <param name="indexer"></param>
/// <returns></returns>
public static bool IsIndexOptimized(this LuceneIndexer indexer)
{
try
{
var searcher = indexer.GetSearcher().GetSearcher() as IndexSearcher;
if (searcher == null) return true;
using (searcher)
using (var reader = searcher.IndexReader)
{
return reader.IsOptimized();
}
}
catch (AlreadyClosedException)
{
Current.Logger.Warn(typeof(ExamineExtensions), "Cannot get IsIndexOptimized, the writer is already closed");
return false;
}
}
/// <summary>
/// Check if the index is locked
/// </summary>
@@ -136,8 +112,9 @@ namespace Umbraco.Web.Search
{
try
{
var searcher = indexer.GetSearcher().GetSearcher() as IndexSearcher;
if (searcher == null) return 0;
if (!((indexer.GetSearcher() as LuceneSearcher)?.GetLuceneSearcher() is IndexSearcher searcher))
return 0;
using (searcher)
using (var reader = searcher.IndexReader)
{

12
src/Umbraco.Web/Search/UmbracoTreeSearcher.cs
View File

@@ -39,7 +39,7 @@ namespace Umbraco.Web.Search
var sb = new StringBuilder();
string type;
var searcher = Constants.Examine.InternalSearcher;
var indexer = Constants.Examine.InternalIndexer;
var fields = new[] { "id", "__NodeId" };
var umbracoContext = umbracoHelper.UmbracoContext;
@@ -48,7 +48,7 @@ namespace Umbraco.Web.Search
switch (entityType)
{
case UmbracoEntityTypes.Member:
searcher = Constants.Examine.InternalMemberSearcher;
indexer = Constants.Examine.InternalMemberIndexer;
type = "member";
fields = new[] { "id", "__NodeId", "email", "loginName" };
if (searchFrom != null && searchFrom != Constants.Conventions.MemberTypes.AllMembersListId && searchFrom.Trim() != "-1")
@@ -72,7 +72,7 @@ namespace Umbraco.Web.Search
throw new NotSupportedException("The " + typeof(UmbracoTreeSearcher) + " currently does not support searching against object type " + entityType);
}
var internalSearcher = ExamineManager.Instance.SearchProviderCollection[searcher];
var internalSearcher = ExamineManager.Instance.GetSearcher(indexer);
//build a lucene query:
// the __nodeName will be boosted 10x without wildcards
@@ -180,7 +180,7 @@ namespace Umbraco.Web.Search
sb.Append("+__IndexType:");
sb.Append(type);
var raw = internalSearcher.CreateSearchCriteria().RawQuery(sb.ToString());
var raw = internalSearcher.CreateCriteria().RawQuery(sb.ToString());
var result = internalSearcher
//only return the number of items specified to read up to the amount of records to fill from 0 -> the number of items on the page requested
@@ -277,10 +277,10 @@ namespace Umbraco.Web.Search
m.Icon = "icon-user";
}
var searchResult = results.First(x => x.Id.ToInvariantString() == m.Id.ToString());
var searchResult = results.First(x => x.Id == m.Id.ToString());
if (searchResult.Fields.ContainsKey("email") && searchResult.Fields["email"] != null)
{
m.AdditionalData["Email"] = results.First(x => x.Id.ToInvariantString() == m.Id.ToString()).Fields["email"];
m.AdditionalData["Email"] = results.First(x => x.Id == m.Id.ToString()).Fields["email"];
}
if (searchResult.Fields.ContainsKey("__key") && searchResult.Fields["__key"] != null)
{

9
src/Umbraco.Web/Suspendable.cs
View File

@@ -2,7 +2,9 @@
using Examine;
using Examine.Providers;
using Umbraco.Core.Composing;
using Umbraco.Examine;
using Umbraco.Web.Cache;
using Umbraco.Web.Search;
namespace Umbraco.Web
{
@@ -77,10 +79,9 @@ namespace Umbraco.Web
_tried = false;
// fixme - could we fork this on a background thread?
foreach (BaseIndexProvider indexer in ExamineManager.Instance.IndexProviderCollection)
{
indexer.RebuildIndex();
}
//TODO: when resuming do we always want a full rebuild of all indexes?
// fixme - can we inject IExamineManager somehow?
ExamineComponent.RebuildIndexes(false, ExamineManager.Instance, Current.Logger);
}
}

2
src/Umbraco.Web/Umbraco.Web.csproj
View File

@@ -72,7 +72,7 @@
<PackageReference Include="ClientDependency" Version="1.9.6" />
<PackageReference Include="CSharpTest.Net.Collections" Version="14.906.1403.1082" />
<PackageReference Include="dotless" Version="1.5.2" />
<PackageReference Include="Examine" Version="2.0.0-beta2" />
<PackageReference Include="Examine" Version="1.0.0-beta024" />
<PackageReference Include="HtmlAgilityPack" Version="1.5.1" />
<PackageReference Include="LightInject" Version="5.0.3" />
<PackageReference Include="LightInject.Annotation" Version="1.1.0" />

204
src/Umbraco.Web/WebServices/ExamineManagementApiController.cs
View File

@@ -1,21 +1,22 @@
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Reflection;
using System.Web.Http;
using Examine;
using Examine.LuceneEngine;
using Examine.LuceneEngine.Providers;
using Examine.Providers;
using Lucene.Net.Index;
using Lucene.Net.Search;
using Lucene.Net.Store;
using Umbraco.Core;
using Umbraco.Core.Cache;
using Umbraco.Core.Logging;
using Umbraco.Core.Composing;
using Umbraco.Core.Services;
using Umbraco.Examine;
using Umbraco.Web.Search;
using Umbraco.Web.WebApi;
using Umbraco.Web.WebApi.Filters;
@@ -25,10 +26,16 @@ namespace Umbraco.Web.WebServices
[ValidateAngularAntiForgeryToken]
public class ExamineManagementApiController : UmbracoAuthorizedApiController
{
//TODO: Fix all of this for searchers/indexers that are not configured via code (i.e. the core ones once we do that)
// We will need to be able to search an index directly without having to go through all of the searchers
public ExamineManagementApiController(IExamineManager examineManager, ILogger logger, IRuntimeCacheProvider runtimeCacheProvider)
{
_examineManager = examineManager;
_logger = logger;
_runtimeCacheProvider = runtimeCacheProvider;
}
private ExamineManager _examineManager;
private readonly IExamineManager _examineManager;
private readonly ILogger _logger;
private readonly IRuntimeCacheProvider _runtimeCacheProvider;
/// <summary>
/// Checks if the member internal index is consistent with the data stored in the database
@@ -40,7 +47,7 @@ namespace Umbraco.Web.WebServices
var total = Services.MemberService.Count();
var searcher = _examineManager.GetSearcher(Constants.Examine.InternalMemberIndexer);
var criteria = searcher.CreateSearchCriteria().RawQuery("__IndexType:member");
var criteria = searcher.CreateCriteria().RawQuery("__IndexType:member");
var totalIndexed = searcher.Search(criteria);
return total == totalIndexed.TotalItemCount;
}
@@ -55,7 +62,7 @@ namespace Umbraco.Web.WebServices
var total = Services.MediaService.Count();
var searcher = _examineManager.GetSearcher(Constants.Examine.InternalIndexer);
var criteria = searcher.CreateSearchCriteria().RawQuery("__IndexType:media");
var criteria = searcher.CreateCriteria().RawQuery("__IndexType:media");
var totalIndexed = searcher.Search(criteria);
return total == totalIndexed.TotalItemCount;
}
@@ -70,7 +77,7 @@ namespace Umbraco.Web.WebServices
var total = Services.ContentService.Count();
var searcher = _examineManager.GetSearcher(Constants.Examine.InternalIndexer);
var criteria = searcher.CreateSearchCriteria().RawQuery("__IndexType:content");
var criteria = searcher.CreateCriteria().RawQuery("__IndexType:content");
var totalIndexed = searcher.Search(criteria);
return total == totalIndexed.TotalItemCount;
}
@@ -81,7 +88,7 @@ namespace Umbraco.Web.WebServices
/// <returns></returns>
public IEnumerable<ExamineIndexerModel> GetIndexerDetails()
{
return ExamineManager.Instance.IndexProviders.Select(CreateModel).OrderBy(x =>
return _examineManager.IndexProviders.Select(CreateModel).OrderBy(x =>
{
//order by name , but strip the "Indexer" from the end if it exists
return x.Name.TrimEnd("Indexer");
@@ -95,19 +102,23 @@ namespace Umbraco.Web.WebServices
public IEnumerable<ExamineSearcherModel> GetSearcherDetails()
{
var model = new List<ExamineSearcherModel>(
ExamineManager.Instance.SearchProviderCollection.Cast<BaseSearchProvider>().Select(searcher =>
_examineManager.IndexProviders.Select(indexer =>
{
var searcher = indexer.Value.GetSearcher();
var searcherName = (searcher as BaseLuceneSearcher)?.Name ?? string.Concat(indexer.Key, "Searcher");
var indexerModel = new ExamineSearcherModel()
{
Name = searcher.Name
Name = searcherName
};
var props = TypeHelper.CachedDiscoverableProperties(searcher.GetType(), mustWrite: false)
//ignore these properties
.Where(x => new[] {"Description"}.InvariantContains(x.Name) == false)
.OrderBy(x => x.Name);
.Where(x => new[] {"Description"}.InvariantContains(x.Name) == false)
.Where(x => x.GetCustomAttribute<EditorBrowsableAttribute>()?.State != EditorBrowsableState.Never)
.OrderBy(x => x.Name);
foreach (var p in props)
{
indexerModel.ProviderProperties.Add(p.Name, p.GetValue(searcher, null).ToString());
indexerModel.ProviderProperties.Add(p.Name, p.GetValue(searcher, null)?.ToString());
}
return indexerModel;
}).OrderBy(x =>
@@ -118,7 +129,7 @@ namespace Umbraco.Web.WebServices
return model;
}
public ILuceneSearchResults GetSearchResults(string searcherName, string query, string queryType)
public ISearchResults GetSearchResults(string searcherName, string query, string queryType)
{
if (queryType == null)
{
@@ -129,48 +140,22 @@ namespace Umbraco.Web.WebServices
if (query.IsNullOrWhiteSpace())
return LuceneSearchResults.Empty();
LuceneSearcher searcher;
var msg = ValidateLuceneSearcher(searcherName, out searcher);
var msg = ValidateLuceneSearcher(searcherName, out var searcher);
if (msg.IsSuccessStatusCode)
{
if (queryType.InvariantEquals("text"))
{
return searcher.Find(query, false);
return searcher.Search(query, false);
}
if (queryType.InvariantEquals("lucene"))
{
return searcher.Find(searcher.CreateCriteria().RawQuery(query));
return searcher.Search(searcher.CreateCriteria().RawQuery(query));
}
throw new HttpResponseException(Request.CreateResponse(HttpStatusCode.NotFound));
}
throw new HttpResponseException(msg);
}
/// <summary>
/// Optimizes an index
/// </summary>
public HttpResponseMessage PostOptimizeIndex(string indexerName)
{
IExamineIndexer indexer;
var msg = ValidateLuceneIndexer(indexerName, out indexer);
var luceneIndexer = indexer as LuceneIndexer;
if (luceneIndexer != null && msg.IsSuccessStatusCode)
{
try
{
luceneIndexer.OptimizeIndex();
}
catch (Exception ex)
{
var response = Request.CreateResponse(HttpStatusCode.Conflict);
response.Content = new StringContent(string.Format("The index could not be optimized, most likely there is another thread currently writing to the index. Error: {0}", ex));
response.ReasonPhrase = "Could Not Optimize";
return response;
}
}
return msg;
}
/// <summary>
/// Rebuilds the index
/// </summary>
@@ -178,11 +163,10 @@ namespace Umbraco.Web.WebServices
/// <returns></returns>
public HttpResponseMessage PostRebuildIndex(string indexerName)
{
LuceneIndexer indexer;
var msg = ValidateLuceneIndexer(indexerName, out indexer);
var msg = ValidateLuceneIndexer(indexerName, out LuceneIndexer indexer);
if (msg.IsSuccessStatusCode)
{
Current.Logger.Info<ExamineManagementApiController>(string.Format("Rebuilding index '{0}'", indexerName));
_logger.Info<ExamineManagementApiController>($"Rebuilding index '{indexerName}'");
//remove it in case there's a handler there alraedy
indexer.IndexOperationComplete -= Indexer_IndexOperationComplete;
@@ -196,54 +180,33 @@ namespace Umbraco.Web.WebServices
try
{
indexer.RebuildIndex();
}
// fixme - Examine issues
// cannot enable that piece of code from v7 as Index.Write.Unlock does not seem to exist anymore?
//catch (LockObtainFailedException)
//{
// //this will occur if the index is locked (which it should defo not be!) so in this case we'll forcibly unlock it and try again
// try
// {
// IndexWriter.Unlock(indexer.GetLuceneDirectory());
// indexer.RebuildIndex();
// }
// catch (Exception e)
// {
// return HandleException(e, indexer);
// }
//}
}
catch (Exception ex)
{
return HandleException(ex, indexer);
//ensure it's not listening
indexer.IndexOperationComplete -= Indexer_IndexOperationComplete;
Logger.Error<ExamineManagementApiController>("An error occurred rebuilding index", ex);
var response = Request.CreateResponse(HttpStatusCode.Conflict);
response.Content = new StringContent($"The index could not be rebuilt at this time, most likely there is another thread currently writing to the index. Error: {ex}");
response.ReasonPhrase = "Could Not Rebuild";
return response;
}
}
return msg;
}
private HttpResponseMessage HandleException(Exception ex, LuceneIndexer indexer)
{
//ensure it's not listening
indexer.IndexOperationComplete -= Indexer_IndexOperationComplete;
Logger.Error<ExamineManagementApiController>("An error occurred rebuilding index", ex);
var response = Request.CreateResponse(HttpStatusCode.Conflict);
response.Content = new StringContent(string.Format("The index could not be rebuilt at this time, most likely there is another thread currently writing to the index. Error: {0}", ex));
response.ReasonPhrase = "Could Not Rebuild";
return response;
}
//static listener so it's not GC'd
private static void Indexer_IndexOperationComplete(object sender, EventArgs e)
private void Indexer_IndexOperationComplete(object sender, EventArgs e)
{
var indexer = (LuceneIndexer) sender;
//ensure it's not listening anymore
indexer.IndexOperationComplete -= Indexer_IndexOperationComplete;
Current.Logger.Info<ExamineManagementApiController>($"Rebuilding index '{indexer.Name}' done, {indexer.CommitCount} items committed (can differ from the number of items in the index)");
_logger.Info<ExamineManagementApiController>($"Rebuilding index '{indexer.Name}' done, {indexer.CommitCount} items committed (can differ from the number of items in the index)");
var cacheKey = "temp_indexing_op_" + indexer.Name;
Current.ApplicationCache.RuntimeCache.ClearCacheItem(cacheKey);
_runtimeCacheProvider.ClearCacheItem(cacheKey);
}
/// <summary>
@@ -257,8 +220,7 @@ namespace Umbraco.Web.WebServices
/// </remarks>
public ExamineIndexerModel PostCheckRebuildIndex(string indexerName)
{
LuceneIndexer indexer;
var msg = ValidateLuceneIndexer(indexerName, out indexer);
var msg = ValidateLuceneIndexer(indexerName, out LuceneIndexer indexer);
if (msg.IsSuccessStatusCode)
{
var cacheKey = "temp_indexing_op_" + indexerName;
@@ -266,42 +228,25 @@ namespace Umbraco.Web.WebServices
//if its still there then it's not done
return found != null
? null
: CreateModel(new KeyValuePair<string, IExamineIndexer>(indexerName, indexer));
: CreateModel(new KeyValuePair<string, IIndexer>(indexerName, indexer));
}
throw new HttpResponseException(msg);
}
/// <summary>
/// Checks if the index is optimized
/// </summary>
/// <param name="indexerName"></param>
/// <returns></returns>
public ExamineIndexerModel PostCheckOptimizeIndex(string indexerName)
{
LuceneIndexer indexer;
var msg = ValidateLuceneIndexer(indexerName, out indexer);
if (msg.IsSuccessStatusCode)
{
var isOptimized = indexer.IsIndexOptimized();
return isOptimized == false
? null
: CreateModel(new KeyValuePair<string, IExamineIndexer>(indexerName, indexer));
}
throw new HttpResponseException(msg);
}
private ExamineIndexerModel CreateModel(KeyValuePair<string, IExamineIndexer> indexer)
private ExamineIndexerModel CreateModel(KeyValuePair<string, IIndexer> indexerKeyVal)
{
var indexer = indexerKeyVal.Value;
var indexName = indexerKeyVal.Key;
var indexerModel = new ExamineIndexerModel()
{
FieldDefinitions = indexer.Value.FieldDefinitions,
Name = indexer.Key
FieldDefinitions = indexer.FieldDefinitionCollection,
Name = indexName
};
var props = TypeHelper.CachedDiscoverableProperties(indexer.GetType(), mustWrite: false)
//ignore these properties
.Where(x => new[] {"IndexerData", "Description", "WorkingFolder"}.InvariantContains(x.Name) == false)
.OrderBy(x => x.Name);
.Where(x => new[] {"IndexerData", "Description", "WorkingFolder"}.InvariantContains(x.Name) == false)
.OrderBy(x => x.Name);
foreach (var p in props)
{
@@ -310,22 +255,20 @@ namespace Umbraco.Web.WebServices
{
// Do not warn for new new attribute that is optional
if(string.Equals(p.Name, "DirectoryFactory", StringComparison.InvariantCultureIgnoreCase) == false)
Logger.Warn<ExamineManagementApiController>("Property value was null when setting up property on indexer: " + indexer.Key + " property: " + p.Name);
Logger.Warn<ExamineManagementApiController>("Property value was null when setting up property on indexer: " + indexName + " property: " + p.Name);
val = string.Empty;
}
indexerModel.ProviderProperties.Add(p.Name, val.ToString());
}
var luceneIndexer = indexer.Value as LuceneIndexer;
if (luceneIndexer != null)
if (indexer is LuceneIndexer luceneIndexer)
{
indexerModel.IsLuceneIndex = true;
if (luceneIndexer.IndexExists())
{
Exception indexError;
indexerModel.IsHealthy = luceneIndexer.IsHealthy(out indexError);
indexerModel.IsHealthy = luceneIndexer.IsHealthy(out var indexError);
if (indexerModel.IsHealthy == false)
{
@@ -336,15 +279,11 @@ namespace Umbraco.Web.WebServices
indexerModel.DocumentCount = luceneIndexer.GetIndexDocumentCount();
indexerModel.FieldCount = luceneIndexer.GetIndexFieldCount();
indexerModel.IsOptimized = luceneIndexer.IsIndexOptimized();
indexerModel.DeletionCount = luceneIndexer.GetDeletedDocumentsCount();
}
else
{
indexerModel.DocumentCount = 0;
indexerModel.FieldCount = 0;
indexerModel.IsOptimized = true;
indexerModel.DeletionCount = 0;
}
}
return indexerModel;
@@ -352,41 +291,46 @@ namespace Umbraco.Web.WebServices
private HttpResponseMessage ValidateLuceneSearcher(string searcherName, out LuceneSearcher searcher)
{
if (ExamineManager.Instance.SearchProviderCollection.Cast<BaseSearchProvider>().Any(x => x.Name == searcherName))
foreach (var indexer in _examineManager.IndexProviders)
{
searcher = ExamineManager.Instance.SearchProviderCollection[searcherName] as LuceneSearcher;
if (searcher == null)
{
var response = Request.CreateResponse(HttpStatusCode.BadRequest);
response.Content = new StringContent(string.Format("The searcher {0} is not of type {1}", searcherName, typeof(LuceneSearcher)));
response.ReasonPhrase = "Wrong Searcher Type";
return response;
}
//return Ok!
return Request.CreateResponse(HttpStatusCode.OK);
var s = indexer.Value.GetSearcher();
var sName = (s as BaseLuceneSearcher)?.Name ?? string.Concat(indexer.Key, "Searcher");
if (sName != searcherName) continue;
searcher = s as LuceneSearcher;
//Found it, return OK
if (searcher != null) return Request.CreateResponse(HttpStatusCode.OK);
//Return an error since it's not the right type
var response = Request.CreateResponse(HttpStatusCode.BadRequest);
response.Content = new StringContent($"The searcher {searcherName} is not of type {typeof(LuceneSearcher)}");
response.ReasonPhrase = "Wrong Searcher Type";
return response;
}
searcher = null;
var response1 = Request.CreateResponse(HttpStatusCode.BadRequest);
response1.Content = new StringContent(string.Format("No searcher found with name = {0}", searcherName));
response1.Content = new StringContent($"No searcher found with name = {searcherName}");
response1.ReasonPhrase = "Searcher Not Found";
return response1;
}
private HttpResponseMessage ValidateLuceneIndexer<T>(string indexerName, out T indexer)
where T : class, IExamineIndexer
where T : class, IIndexer
{
indexer = null;
if (ExamineManager.Instance.IndexProviders.ContainsKey(indexerName))
if (_examineManager.IndexProviders.ContainsKey(indexerName)
&& _examineManager.IndexProviders[indexerName] is T casted)
{
//return Ok!
indexer = casted;
return Request.CreateResponse(HttpStatusCode.OK);
}
var response = Request.CreateResponse(HttpStatusCode.BadRequest);
response.Content = new StringContent(string.Format("No indexer found with name = {0}", indexerName));
response.Content = new StringContent($"No indexer found with name = {indexerName} of type {typeof(T)}");
response.ReasonPhrase = "Indexer Not Found";
return response;
}

7
src/Umbraco.Web/umbraco.presentation/umbraco/dialogs/search.aspx.cs
View File

@@ -16,6 +16,7 @@ using Umbraco.Core.Xml;
namespace umbraco.presentation.dialogs
{
//fixme - is this even used anymore?
public partial class search : Umbraco.Web.UI.Pages.UmbracoEnsuredPage
{
@@ -59,11 +60,11 @@ namespace umbraco.presentation.dialogs
//if it doesn't start with "*", then search only nodeName and nodeId
var internalSearcher = (CurrentApp == Constants.Applications.Members)
? ExamineManager.Instance.SearchProviderCollection["InternalMemberSearcher"]
: ExamineManager.Instance.SearchProviderCollection["InternalSearcher"];
? ExamineManager.Instance.GetSearcher(Constants.Examine.InternalMemberIndexer)
: ExamineManager.Instance.GetSearcher(Constants.Examine.InternalIndexer);
//create some search criteria, make everything combined to be 'And' and only search the current app
var criteria = internalSearcher.CreateSearchCriteria(CurrentApp, Examine.SearchCriteria.BooleanOperation.And);
var criteria = internalSearcher.CreateCriteria(CurrentApp, Examine.SearchCriteria.BooleanOperation.And);
IEnumerable<SearchResult> results;
if (txt.StartsWith("*"))