yv01p/Umbraco-CMS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Gets examine working with proper events, tests are working, etc...

Browse Source
This commit is contained in:
Shannon
2018-03-28 16:12:43 +11:00
parent f0a342d84e
commit 52aa02668f
18 changed files with 335 additions and 183 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/Umbraco.Examine/Umbraco.Examine.csproj
View File

@@ -47,7 +47,7 @@
<ItemGroup>
<!-- note: NuGet deals with transitive references now -->
<PackageReference Include="Examine">
<Version>1.0.0-beta012</Version>
<Version>1.0.0-beta017</Version>
</PackageReference>
<PackageReference Include="Newtonsoft.Json" Version="10.0.3" />
<PackageReference Include="NPoco" Version="3.6.1" />

36
src/Umbraco.Examine/UmbracoContentIndexer.cs
View File

@@ -57,6 +57,7 @@ namespace Umbraco.Examine
/// <summary>
/// Create an index at runtime
/// </summary>
/// <param name="name"></param>
/// <param name="fieldDefinitions"></param>
/// <param name="luceneDirectory"></param>
/// <param name="defaultAnalyzer"></param>
@@ -70,6 +71,7 @@ namespace Umbraco.Examine
/// <param name="options"></param>
/// <param name="indexValueTypes"></param>
public UmbracoContentIndexer(
string name,
IEnumerable<FieldDefinition> fieldDefinitions,
Directory luceneDirectory,
Analyzer defaultAnalyzer,
@@ -82,7 +84,7 @@ namespace Umbraco.Examine
IValueSetValidator validator,
UmbracoContentIndexerOptions options,
IReadOnlyDictionary<string, Func<string, IIndexValueType>> indexValueTypes = null)
: base(fieldDefinitions, luceneDirectory, defaultAnalyzer, profilingLogger, validator, indexValueTypes)
: base(name, fieldDefinitions, luceneDirectory, defaultAnalyzer, profilingLogger, validator, indexValueTypes)
{
if (validator == null) throw new ArgumentNullException(nameof(validator));
if (options == null) throw new ArgumentNullException(nameof(options));
@@ -342,9 +344,21 @@ namespace Umbraco.Examine
{"template", new object[] {c.Template?.Id ?? 0}}
};
foreach (var property in c.Properties.Where(p => p?.GetValue() != null && p.GetValue().ToString().IsNullOrWhiteSpace() == false))
foreach (var property in c.Properties)
{
values.Add(property.Alias, new[] {property.GetValue() });
//only add the value if its not null or empty (we'll check for string explicitly here too)
var val = property.GetValue();
switch (val)
{
case null:
continue;
case string strVal when strVal.IsNullOrWhiteSpace() == false:
values.Add(property.Alias, new[] { val });
break;
default:
values.Add(property.Alias, new[] { val });
break;
}
}
var vs = new ValueSet(c.Id.ToInvariantString(), IndexTypes.Content, c.ContentType.Alias, values);
@@ -376,9 +390,21 @@ namespace Umbraco.Examine
{"creatorName", new object[] {m.GetCreatorProfile(userService).Name}}
};
foreach (var property in m.Properties.Where(p => p?.GetValue() != null && p.GetValue().ToString().IsNullOrWhiteSpace() == false))
foreach (var property in m.Properties)
{
values.Add(property.Alias, new[] { property.GetValue() });
//only add the value if its not null or empty (we'll check for string explicitly here too)
var val = property.GetValue();
switch (val)
{
case null:
continue;
case string strVal when strVal.IsNullOrWhiteSpace() == false:
values.Add(property.Alias, new[] { val });
break;
default:
values.Add(property.Alias, new[] { val });
break;
}
}
var vs = new ValueSet(m.Id.ToInvariantString(), IndexTypes.Media, m.ContentType.Alias, values);

15
src/Umbraco.Examine/UmbracoExamineIndexer.cs
View File

@@ -72,13 +72,14 @@ namespace Umbraco.Examine
}
protected UmbracoExamineIndexer(
string name,
IEnumerable<FieldDefinition> fieldDefinitions,
Directory luceneDirectory,
Analyzer defaultAnalyzer,
ProfilingLogger profilingLogger,
IValueSetValidator validator = null,
IReadOnlyDictionary<string, Func<string, IIndexValueType>> indexValueTypes = null)
: base(fieldDefinitions, luceneDirectory, defaultAnalyzer, validator, indexValueTypes)
: base(name, fieldDefinitions, luceneDirectory, defaultAnalyzer, validator, indexValueTypes)
{
ProfilingLogger = profilingLogger ?? throw new ArgumentNullException(nameof(profilingLogger));
}
@@ -131,10 +132,10 @@ namespace Umbraco.Examine
return base.CreateFieldValueTypes(x, indexValueTypesFactory);
}
//TODO: Remove this?
[Obsolete("This should not be used, it is used by the configuration based indexes but instead to disable Examine event handlers use the ExamineEvents class instead.")]
[EditorBrowsable(EditorBrowsableState.Never)]
public bool EnableDefaultEventHandler { get; protected set; }
/// <summary>
/// When set to true Umbraco will keep the index in sync with Umbraco data automatically
/// </summary>
public bool EnableDefaultEventHandler { get; set; } = true;
/// <summary>
/// the supported indexable types
@@ -163,9 +164,7 @@ namespace Umbraco.Examine
{
ProfilingLogger.Logger.Debug(GetType(), "{0} indexer initializing", () => name);
EnableDefaultEventHandler = true; //set to true by default
bool enabled;
if (bool.TryParse(config["enableDefaultEventHandler"], out enabled))
if (config["enableDefaultEventHandler"] != null && bool.TryParse(config["enableDefaultEventHandler"], out var enabled))
{
EnableDefaultEventHandler = enabled;
}

112
src/Umbraco.Examine/UmbracoExamineMultiIndexSearcher.cs
View File

@@ -1,61 +1,65 @@
using System;
using System.Collections.Generic;
using System.Collections.Specialized;
using System.IO;
using System.Linq;
using Examine.LuceneEngine.Providers;
using Lucene.Net.Analysis;
using Umbraco.Examine.Config;
using Directory = Lucene.Net.Store.Directory;
//using System;
//using System.Collections.Generic;
//using System.Collections.Specialized;
//using System.IO;
//using System.Linq;
//using Examine.LuceneEngine.Providers;
//using Lucene.Net.Analysis;
//using Umbraco.Examine.Config;
//using Directory = Lucene.Net.Store.Directory;
namespace Umbraco.Examine
{
public class UmbracoExamineMultiIndexSearcher : MultiIndexSearcher
{
public UmbracoExamineMultiIndexSearcher()
{
}
//namespace Umbraco.Examine
//{
// public class UmbracoExamineMultiIndexSearcher : MultiIndexSearcher
// {
// public UmbracoExamineMultiIndexSearcher()
// {
// }
public UmbracoExamineMultiIndexSearcher(IEnumerable<DirectoryInfo> indexPath, Analyzer analyzer) : base(indexPath, analyzer)
{
}
// public UmbracoExamineMultiIndexSearcher(string name, IEnumerable<DirectoryInfo> indexPath, Analyzer analyzer)
// : base(name, indexPath, analyzer)
// {
// }
public UmbracoExamineMultiIndexSearcher(IEnumerable<Directory> luceneDirs, Analyzer analyzer) : base(luceneDirs, analyzer)
{
}
// public UmbracoExamineMultiIndexSearcher(string name, IEnumerable<Directory> luceneDirs, Analyzer analyzer)
// : base(name, luceneDirs, analyzer)
// {
// }
public override void Initialize(string name, NameValueCollection config)
{
base.Initialize(name, config);
// public override void Initialize(string name, NameValueCollection config)
// {
// base.Initialize(name, config);
//need to check if the index set is specified, if it's not, we'll see if we can find one by convension
//if the folder is not null and the index set is null, we'll assume that this has been created at runtime.
if (config["indexSets"] == null)
{
throw new ArgumentNullException("indexSets on MultiIndexSearcher provider has not been set in configuration");
}
// //need to check if the index set is specified, if it's not, we'll see if we can find one by convension
// //if the folder is not null and the index set is null, we'll assume that this has been created at runtime.
// if (config["indexSets"] == null)
// {
// throw new ArgumentNullException("indexSets on MultiIndexSearcher provider has not been set in configuration");
// }
var toSearch = new List<IndexSet>();
var sets = IndexSets.Instance.Sets.Cast<IndexSet>();
foreach (var i in config["indexSets"].Split(','))
{
var s = sets.SingleOrDefault(x => x.SetName == i);
if (s == null)
{
throw new ArgumentException("The index set " + i + " does not exist");
}
toSearch.Add(s);
}
// var toSearch = new List<IndexSet>();
// var sets = IndexSets.Instance.Sets.Cast<IndexSet>();
// foreach (var i in config["indexSets"].Split(','))
// {
// var s = sets.SingleOrDefault(x => x.SetName == i);
// if (s == null)
// {
// throw new ArgumentException("The index set " + i + " does not exist");
// }
// toSearch.Add(s);
// }
//create the searchers
var analyzer = DefaultLuceneAnalyzer;
var searchers = new List<LuceneSearcher>();
//DO NOT PUT THIS INTO LINQ BECAUSE THE SECURITY ACCCESS SHIT WONT WORK
foreach (var s in toSearch)
{
searchers.Add(new LuceneSearcher(s.IndexDirectory, analyzer));
}
Searchers = searchers;
}
}
}
// //create the searchers
// var analyzer = LuceneAnalyzer;
// var searchers = new List<LuceneSearcher>();
// var n = 0;
// foreach (var s in toSearch)
// {
// searchers.Add(new LuceneSearcher(Name + "_" + n, s.IndexDirectory, analyzer));
// n++;
// }
// Searchers = searchers;
// }
// }
//}

31
src/Umbraco.Examine/UmbracoExamineSearcher.cs
View File

@@ -5,6 +5,7 @@ using System.Linq;
using Umbraco.Core;
using Examine.LuceneEngine.Providers;
using Lucene.Net.Analysis;
using Lucene.Net.Index;
using Umbraco.Core.Composing;
using Umbraco.Examine.Config;
using Directory = Lucene.Net.Store.Directory;
@@ -17,8 +18,6 @@ namespace Umbraco.Examine
/// </summary>
public class UmbracoExamineSearcher : LuceneSearcher
{
private string _name;
private readonly bool _configBased = false;
/// <summary>
@@ -26,7 +25,6 @@ namespace Umbraco.Examine
/// </summary>
[EditorBrowsable(EditorBrowsableState.Never)]
public UmbracoExamineSearcher()
: base()
{
_configBased = true;
}
@@ -34,10 +32,11 @@ namespace Umbraco.Examine
/// <summary>
/// Constructor to allow for creating an indexer at runtime
/// </summary>
/// <param name="name"></param>
/// <param name="indexPath"></param>
/// <param name="analyzer"></param>
public UmbracoExamineSearcher(DirectoryInfo indexPath, Analyzer analyzer)
: base(indexPath, analyzer)
public UmbracoExamineSearcher(string name, DirectoryInfo indexPath, Analyzer analyzer)
: base(name, indexPath, analyzer)
{
_configBased = false;
}
@@ -45,21 +44,20 @@ namespace Umbraco.Examine
/// <summary>
/// Constructor to allow for creating an indexer at runtime
/// </summary>
/// <param name="name"></param>
/// <param name="luceneDirectory"></param>
/// <param name="analyzer"></param>
public UmbracoExamineSearcher(Directory luceneDirectory, Analyzer analyzer)
: base(luceneDirectory, analyzer)
public UmbracoExamineSearcher(string name, Directory luceneDirectory, Analyzer analyzer)
: base(name, luceneDirectory, analyzer)
{
_configBased = false;
}
//TODO: What about the NRT ctor?
/// <summary>
/// we override name because we need to manually set it if !CanInitialize()
/// since we cannot call base.Initialize in that case.
/// </summary>
public override string Name => _name;
/// <inheritdoc />
public UmbracoExamineSearcher(string name, IndexWriter writer, Analyzer analyzer) : base(name, writer, analyzer)
{
_configBased = false;
}
/// <summary>
/// Name of the Lucene.NET index set
@@ -73,9 +71,8 @@ namespace Umbraco.Examine
/// <param name="config"></param>
public override void Initialize(string name, System.Collections.Specialized.NameValueCollection config)
{
//ensure name is set
_name = name ?? throw new ArgumentNullException(nameof(name));
if (string.IsNullOrWhiteSpace(name)) throw new ArgumentException("Value cannot be null or whitespace.", nameof(name));
//We need to check if we actually can initialize, if not then don't continue
if (CanInitialize() == false)
{

21
src/Umbraco.Examine/UmbracoMemberIndexer.cs
View File

@@ -28,7 +28,6 @@ namespace Umbraco.Examine
/// <summary>
/// Constructor for config/provider based indexes
/// </summary>
///
[EditorBrowsable(EditorBrowsableState.Never)]
public UmbracoMemberIndexer()
{
@@ -38,6 +37,7 @@ namespace Umbraco.Examine
/// <summary>
/// Constructor to allow for creating an indexer at runtime
/// </summary>
/// <param name="name"></param>
/// <param name="fieldDefinitions"></param>
/// <param name="luceneDirectory"></param>
/// <param name="profilingLogger"></param>
@@ -45,13 +45,14 @@ namespace Umbraco.Examine
/// <param name="memberService"></param>
/// <param name="analyzer"></param>
public UmbracoMemberIndexer(
string name,
IEnumerable<FieldDefinition> fieldDefinitions,
Directory luceneDirectory,
Analyzer analyzer,
ProfilingLogger profilingLogger,
IValueSetValidator validator,
IMemberService memberService) :
base(fieldDefinitions, luceneDirectory, analyzer, profilingLogger, validator)
base(name, fieldDefinitions, luceneDirectory, analyzer, profilingLogger, validator)
{
_memberService = memberService ?? throw new ArgumentNullException(nameof(memberService));
}
@@ -140,9 +141,21 @@ namespace Umbraco.Examine
{"email", new object[] {m.Email}},
};
foreach (var property in m.Properties.Where(p => p?.GetValue() != null && p.GetValue().ToString().IsNullOrWhiteSpace() == false))
foreach (var property in m.Properties)
{
values.Add(property.Alias, new[] { property.GetValue() });
//only add the value if its not null or empty (we'll check for string explicitly here too)
var val = property.GetValue();
switch (val)
{
case null:
continue;
case string strVal when strVal.IsNullOrWhiteSpace() == false:
values.Add(property.Alias, new[] { val });
break;
default:
values.Add(property.Alias, new[] { val });
break;
}
}
var vs = new ValueSet(m.Id.ToInvariantString(), IndexTypes.Content, m.ContentType.Alias, values);

2
src/Umbraco.Tests/Umbraco.Tests.csproj
View File

@@ -60,7 +60,7 @@
<HintPath>..\packages\Castle.Core.4.1.1\lib\net45\Castle.Core.dll</HintPath>
</Reference>
<Reference Include="Examine, Version=1.0.0.0, Culture=neutral, processorArchitecture=MSIL">
<HintPath>..\packages\Examine.1.0.0-beta012\lib\net45\Examine.dll</HintPath>
<HintPath>..\packages\Examine.1.0.0-beta017\lib\net45\Examine.dll</HintPath>
</Reference>
<Reference Include="ICSharpCode.SharpZipLib, Version=0.86.0.518, Culture=neutral, PublicKeyToken=1b03e6acf1164f73, processorArchitecture=MSIL">
<HintPath>..\packages\SharpZipLib.0.86.0\lib\20\ICSharpCode.SharpZipLib.dll</HintPath>

5
src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs
View File

@@ -178,6 +178,7 @@ namespace Umbraco.Tests.UmbracoExamine
// .Returns(query.Object);
var i = new UmbracoContentIndexer(
"testIndexer",
Enumerable.Empty<FieldDefinition>(),
luceneDir,
analyzer,
@@ -197,12 +198,12 @@ namespace Umbraco.Tests.UmbracoExamine
public static LuceneSearcher GetLuceneSearcher(Directory luceneDir)
{
return new LuceneSearcher(luceneDir, new StandardAnalyzer(Version.LUCENE_29));
return new LuceneSearcher("testSearcher", luceneDir, new StandardAnalyzer(Version.LUCENE_29));
}
public static MultiIndexSearcher GetMultiSearcher(Directory pdfDir, Directory simpleDir, Directory conventionDir, Directory cwsDir)
{
var i = new MultiIndexSearcher(new[] { pdfDir, simpleDir, conventionDir, cwsDir }, new StandardAnalyzer(Version.LUCENE_29));
var i = new MultiIndexSearcher("testSearcher", new[] { pdfDir, simpleDir, conventionDir, cwsDir }, new StandardAnalyzer(Version.LUCENE_29));
return i;
}

2
src/Umbraco.Tests/packages.config
View File

@@ -2,7 +2,7 @@
<packages>
<package id="AutoMapper" version="6.1.1" targetFramework="net461" />
<package id="Castle.Core" version="4.1.1" targetFramework="net461" />
<package id="Examine" version="1.0.0-beta012" targetFramework="net461" />
<package id="Examine" version="1.0.0-beta017" targetFramework="net461" />
<package id="LightInject" version="5.0.3" targetFramework="net461" />
<package id="LightInject.Annotation" version="1.1.0" targetFramework="net461" />
<package id="log4net" version="2.0.8" targetFramework="net461" />

2
src/Umbraco.Web.UI/Umbraco.Web.UI.csproj
View File

@@ -55,7 +55,7 @@
<HintPath>..\packages\AutoMapper.6.1.1\lib\net45\AutoMapper.dll</HintPath>
</Reference>
<Reference Include="Examine, Version=1.0.0.0, Culture=neutral, processorArchitecture=MSIL">
<HintPath>..\packages\Examine.1.0.0-beta012\lib\net45\Examine.dll</HintPath>
<HintPath>..\packages\Examine.1.0.0-beta017\lib\net45\Examine.dll</HintPath>
</Reference>
<Reference Include="ImageProcessor.Web, Version=4.8.4.0, Culture=neutral, processorArchitecture=MSIL">
<HintPath>..\packages\ImageProcessor.Web.4.8.4\lib\net45\ImageProcessor.Web.dll</HintPath>

8
src/Umbraco.Web.UI/config/ExamineSettings.Release.config
View File

@@ -12,12 +12,12 @@ More information and documentation can be found on GitHub: https://github.com/Sh
<add name="InternalIndexer" type="Umbraco.Examine.UmbracoContentIndexer, Umbraco.Examine"
supportUnpublished="true"
supportProtected="true"
analyzer="Lucene.Net.Analysis.WhitespaceAnalyzer, Lucene.Net"/>
analyzer="Examine.LuceneEngine.CultureInvariantWhitespaceAnalyzer, Examine"/>
<add name="InternalMemberIndexer" type="Umbraco.Examine.UmbracoMemberIndexer, Umbraco.Examine"
supportUnpublished="true"
supportProtected="true"
analyzer="Lucene.Net.Analysis.Standard.StandardAnalyzer, Lucene.Net"/>
analyzer="Examine.LuceneEngine.CultureInvariantStandardAnalyzer, Examine"/>
<!-- default external indexer, which excludes protected and unpublished pages-->
<add name="ExternalIndexer" type="Umbraco.Examine.UmbracoContentIndexer, Umbraco.Examine"/>
@@ -25,7 +25,7 @@ More information and documentation can be found on GitHub: https://github.com/Sh
</providers>
</ExamineIndexProviders>
<ExamineSearchProviders defaultProvider="ExternalSearcher">
<ExamineSearchProviders>
<providers>
<add name="InternalSearcher" type="Umbraco.Examine.UmbracoExamineSearcher, Umbraco.Examine"
analyzer="Lucene.Net.Analysis.WhitespaceAnalyzer, Lucene.Net"/>
@@ -33,7 +33,7 @@ More information and documentation can be found on GitHub: https://github.com/Sh
<add name="ExternalSearcher" type="Umbraco.Examine.UmbracoExamineSearcher, Umbraco.Examine" />
<add name="InternalMemberSearcher" type="Umbraco.Examine.UmbracoExamineSearcher, Umbraco.Examine"
analyzer="Lucene.Net.Analysis.Standard.StandardAnalyzer, Lucene.Net" enableLeadingWildcard="true"/>
analyzer="Examine.LuceneEngine.CultureInvariantStandardAnalyzer, Examine" />
</providers>
</ExamineSearchProviders>

9
src/Umbraco.Web.UI/config/ExamineSettings.config
View File

@@ -12,12 +12,12 @@ More information and documentation can be found on GitHub: https://github.com/Sh
<add name="InternalIndexer" type="Umbraco.Examine.UmbracoContentIndexer, Umbraco.Examine"
supportUnpublished="true"
supportProtected="true"
analyzer="Lucene.Net.Analysis.WhitespaceAnalyzer, Lucene.Net"/>
analyzer="Examine.LuceneEngine.CultureInvariantWhitespaceAnalyzer, Examine"/>
<add name="InternalMemberIndexer" type="Umbraco.Examine.UmbracoMemberIndexer, Umbraco.Examine"
supportUnpublished="true"
supportProtected="true"
analyzer="Lucene.Net.Analysis.Standard.StandardAnalyzer, Lucene.Net"/>
analyzer="Examine.LuceneEngine.CultureInvariantStandardAnalyzer, Examine"/>
<!-- default external indexer, which excludes protected and unpublished pages-->
<add name="ExternalIndexer" type="Umbraco.Examine.UmbracoContentIndexer, Umbraco.Examine"/>
@@ -25,7 +25,7 @@ More information and documentation can be found on GitHub: https://github.com/Sh
</providers>
</ExamineIndexProviders>
<ExamineSearchProviders defaultProvider="ExternalSearcher">
<ExamineSearchProviders>
<providers>
<add name="InternalSearcher" type="Umbraco.Examine.UmbracoExamineSearcher, Umbraco.Examine"
analyzer="Lucene.Net.Analysis.WhitespaceAnalyzer, Lucene.Net"/>
@@ -33,8 +33,7 @@ More information and documentation can be found on GitHub: https://github.com/Sh
<add name="ExternalSearcher" type="Umbraco.Examine.UmbracoExamineSearcher, Umbraco.Examine"/>
<add name="InternalMemberSearcher" type="Umbraco.Examine.UmbracoExamineSearcher, Umbraco.Examine"
analyzer="Lucene.Net.Analysis.Standard.StandardAnalyzer, Lucene.Net"
enableLeadingWildcard="true"/>
analyzer="Examine.LuceneEngine.CultureInvariantStandardAnalyzer, Examine" />
</providers>
</ExamineSearchProviders>

2
src/Umbraco.Web.UI/packages.config
View File

@@ -4,7 +4,7 @@
<package id="ClientDependency" version="1.9.2" targetFramework="net461" />
<package id="ClientDependency-Mvc5" version="1.8.0.0" targetFramework="net461" />
<package id="CSharpTest.Net.Collections" version="14.906.1403.1082" targetFramework="net461" />
<package id="Examine" version="1.0.0-beta012" targetFramework="net461" />
<package id="Examine" version="1.0.0-beta017" targetFramework="net461" />
<package id="ImageProcessor" version="2.5.4" targetFramework="net461" />
<package id="ImageProcessor.Web" version="4.8.4" targetFramework="net461" />
<package id="ImageProcessor.Web.Config" version="2.3.0.0" targetFramework="net461" />

230
src/Umbraco.Web/Search/ExamineComponent.cs
View File

@@ -1,4 +1,5 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
@@ -6,7 +7,9 @@ using System.Threading;
using System.Xml.Linq;
using Examine;
using Examine.LuceneEngine;
using Examine.LuceneEngine.Providers;
using Lucene.Net.Documents;
using Lucene.Net.Index;
using Umbraco.Core;
using Umbraco.Core.Cache;
using Umbraco.Core.Components;
@@ -29,51 +32,68 @@ namespace Umbraco.Web.Search
[RuntimeLevel(MinLevel = RuntimeLevel.Run)]
public sealed class ExamineComponent : UmbracoComponentBase, IUmbracoCoreComponent
{
public void Initialize(IRuntimeState runtime, PropertyEditorCollection propertyEditors, IExamineIndexCollectionAccessor indexCollection, ILogger logger)
//fixme - we are injecting this which is nice, but we still use ExamineManager everywhere, we could instead interface IExamineManager?
private IExamineIndexCollectionAccessor _indexCollection;
private static bool _disableExamineIndexing = false;
private static volatile bool __isConfigured = false;
private static readonly object IsConfiguredLocker = new object();
public void Initialize(IRuntimeState runtime, PropertyEditorCollection propertyEditors, IExamineIndexCollectionAccessor indexCollection, ProfilingLogger profilingLogger)
{
logger.Info<ExamineComponent>("Starting initialize async background thread.");
_indexCollection = indexCollection;
// make it async in order not to slow down the boot
// fixme - should be a proper background task else we cannot stop it!
var bg = new Thread(() =>
//fixme we cannot inject MainDom since it's internal, so thsi is the only way we can get it, alternatively we can add the container to the container and resolve
//directly from the container but that's not nice either
if (!(runtime is RuntimeState coreRuntime))
throw new NotSupportedException($"Unsupported IRuntimeState implementation {runtime.GetType().FullName}, expecting {typeof(RuntimeState).FullName}.");
//We want to manage Examine's appdomain shutdown sequence ourselves so first we'll disable Examine's default behavior
//and then we'll use MainDom to control Examine's shutdown
ExamineManager.DisableDefaultHostingEnvironmentRegistration();
//we want to tell examine to use a different fs lock instead of the default NativeFSFileLock which could cause problems if the appdomain
//terminates and in some rare cases would only allow unlocking of the file if IIS is forcefully terminated. Instead we'll rely on the simplefslock
//which simply checks the existence of the lock file
DirectoryTracker.DefaultLockFactory = d =>
{
try
var simpleFsLockFactory = new NoPrefixSimpleFsLockFactory(d);
return simpleFsLockFactory;
};
//let's deal with shutting down Examine with MainDom
var examineShutdownRegistered = coreRuntime.MainDom.Register(() =>
{
using (profilingLogger.TraceDuration<ExamineComponent>("Examine shutting down"))
{
// from WebRuntimeComponent
// rebuilds any empty indexes
RebuildIndexes(true);
}
catch (Exception e)
{
logger.Error<ExamineComponent>("Failed to rebuild empty indexes.", e);
}
try
{
// from PropertyEditorsComponent
var grid = propertyEditors.OfType<GridPropertyEditor>().FirstOrDefault();
if (grid != null) BindGridToExamine(grid, indexCollection);
}
catch (Exception e)
{
logger.Error<ExamineComponent>("Failed to bind grid property editor.", e);
//Due to the way Examine's own IRegisteredObject works, we'll first run it with immediate=false and then true so that
//it's correct subroutines are executed (otherwise we'd have to run this logic manually ourselves)
ExamineManager.Instance.Stop(false);
ExamineManager.Instance.Stop(true);
}
});
bg.Start();
// the rest is the original Examine event handler
if (!examineShutdownRegistered)
{
profilingLogger.Logger.Debug<ExamineComponent>("Examine shutdown not registered, this appdomain is not the MainDom, Examine will be disabled");
logger.Info<ExamineComponent>("Initialize and bind to business logic events.");
var registeredProviders = ExamineManager.Instance.IndexProviders
.OfType<UmbracoExamineIndexer>().Count(x => x.EnableDefaultEventHandler);
//if we could not register the shutdown examine ourselves, it means we are not maindom! in this case all of examine should be disabled!
Suspendable.ExamineEvents.SuspendIndexers();
_disableExamineIndexing = true;
return; //exit, do not continue
}
logger.Info<ExamineComponent>($"Adding examine event handlers for {registeredProviders} index providers.");
profilingLogger.Logger.Debug<ExamineComponent>("Examine shutdown registered with MainDom");
var registeredIndexers = indexCollection.Indexes.Values.OfType<UmbracoExamineIndexer>().Count(x => x.EnableDefaultEventHandler);
profilingLogger.Logger.Info<ExamineComponent>($"Adding examine event handlers for {registeredIndexers} index providers.");
// don't bind event handlers if we're not suppose to listen
if (registeredProviders == 0)
if (registeredIndexers == 0)
return;
BindGridToExamine(profilingLogger.Logger, indexCollection, propertyEditors);
// bind to distributed cache events - this ensures that this logic occurs on ALL servers
// that are taking part in a load balanced environment.
ContentCacheRefresher.CacheUpdated += ContentCacheRefresherUpdated;
@@ -83,26 +103,124 @@ namespace Umbraco.Web.Search
// fixme - content type?
// events handling removed in ef013f9d3b945d0a48a306ff1afbd49c10c3fff8
// because, could not make sense of it?
EnsureUnlocked(profilingLogger.Logger, indexCollection);
RebuildIndexesOnStartup(profilingLogger.Logger);
}
private static void RebuildIndexes(bool onlyEmptyIndexes)
/// <summary>
/// Called to rebuild empty indexes on startup
/// </summary>
/// <param name="logger"></param>
private void RebuildIndexesOnStartup(ILogger logger)
{
var indexers = ExamineManager.Instance.IndexProviders;
//TODO: need a way to disable rebuilding on startup
logger.Info<ExamineComponent>("Starting initialize async background thread.");
// make it async in order not to slow down the boot
// fixme - should be a proper background task else we cannot stop it!
var bg = new Thread(() =>
{
try
{
// rebuilds any empty indexes
RebuildIndexes(true, _indexCollection, logger);
}
catch (Exception e)
{
logger.Error<ExamineComponent>("Failed to rebuild empty indexes.", e);
}
});
bg.Start();
}
/// <summary>
/// Used to rebuild indexes on startup or cold boot
/// </summary>
/// <param name="onlyEmptyIndexes"></param>
/// <param name="indexCollection"></param>
/// <param name="logger"></param>
internal static void RebuildIndexes(bool onlyEmptyIndexes, IExamineIndexCollectionAccessor indexCollection, ILogger logger)
{
//do not attempt to do this if this has been disabled since we are not the main dom.
//this can be called during a cold boot
if (_disableExamineIndexing) return;
EnsureUnlocked(logger, indexCollection);
if (onlyEmptyIndexes)
indexers = indexers.Where(x => x.Value.IsIndexNew()).ToDictionary(x => x.Key, x => x.Value);
foreach (var indexer in indexers)
indexer.Value.RebuildIndex();
{
foreach (var indexer in indexCollection.Indexes.Values.Where(x => x.IsIndexNew()))
{
indexer.RebuildIndex();
}
}
else
{
//do all of them
ExamineManager.Instance.RebuildIndexes();
}
}
private static void BindGridToExamine(GridPropertyEditor grid, IExamineIndexCollectionAccessor indexCollection)
/// <summary>
/// Must be called to each index is unlocked before any indexing occurs
/// </summary>
/// <remarks>
/// Indexing rebuilding can occur on a normal boot if the indexes are empty or on a cold boot by the database server messenger. Before
/// either of these happens, we need to configure the indexes.
/// </remarks>
private static void EnsureUnlocked(ILogger logger, IExamineIndexCollectionAccessor indexCollection)
{
var indexes = indexCollection.Indexes;
if (indexes == null) return;
foreach (var i in indexes.Values.OfType<UmbracoExamineIndexer>())
i.DocumentWriting += grid.DocumentWriting;
if (_disableExamineIndexing) return;
if (__isConfigured) return;
lock (IsConfiguredLocker)
{
//double chekc
if (__isConfigured) return;
__isConfigured = true;
foreach (var luceneIndexer in indexCollection.Indexes.Values.OfType<LuceneIndexer>())
{
//We now need to disable waiting for indexing for Examine so that the appdomain is shutdown immediately and doesn't wait for pending
//indexing operations. We used to wait for indexing operations to complete but this can cause more problems than that is worth because
//that could end up halting shutdown for a very long time causing overlapping appdomains and many other problems.
luceneIndexer.WaitForIndexQueueOnShutdown = false;
//we should check if the index is locked ... it shouldn't be! We are using simple fs lock now and we are also ensuring that
//the indexes are not operational unless MainDom is true
var dir = luceneIndexer.GetLuceneDirectory();
if (IndexWriter.IsLocked(dir))
{
logger.Info<ExamineComponent>("Forcing index " + luceneIndexer.Name + " to be unlocked since it was left in a locked state");
IndexWriter.Unlock(dir);
}
}
}
}
static void MemberCacheRefresherUpdated(MemberCacheRefresher sender, CacheRefresherEventArgs args)
private static void BindGridToExamine(ILogger logger, IExamineIndexCollectionAccessor indexCollection, IEnumerable propertyEditors)
{
//bind the grid property editors - this is a hack until http://issues.umbraco.org/issue/U4-8437
try
{
var grid = propertyEditors.OfType<GridPropertyEditor>().FirstOrDefault();
if (grid != null)
{
foreach (var i in indexCollection.Indexes.Values.OfType<UmbracoExamineIndexer>())
i.DocumentWriting += grid.DocumentWriting;
}
}
catch (Exception e)
{
logger.Error<ExamineComponent>("Failed to bind grid property editor.", e);
}
}
private void MemberCacheRefresherUpdated(MemberCacheRefresher sender, CacheRefresherEventArgs args)
{
if (Suspendable.ExamineEvents.CanIndex == false)
return;
@@ -145,7 +263,7 @@ namespace Umbraco.Web.Search
}
}
static void MediaCacheRefresherUpdated(MediaCacheRefresher sender, CacheRefresherEventArgs args)
private void MediaCacheRefresherUpdated(MediaCacheRefresher sender, CacheRefresherEventArgs args)
{
if (Suspendable.ExamineEvents.CanIndex == false)
return;
@@ -194,7 +312,7 @@ namespace Umbraco.Web.Search
}
}
static void ContentCacheRefresherUpdated(ContentCacheRefresher sender, CacheRefresherEventArgs args)
private void ContentCacheRefresherUpdated(ContentCacheRefresher sender, CacheRefresherEventArgs args)
{
if (Suspendable.ExamineEvents.CanIndex == false)
return;
@@ -270,7 +388,7 @@ namespace Umbraco.Web.Search
}
}
private static void ReIndexForContent(IContent content, IContent published)
private void ReIndexForContent(IContent content, IContent published)
{
if (published != null && content.VersionId == published.VersionId)
{
@@ -292,36 +410,36 @@ namespace Umbraco.Web.Search
}
}
private static void ReIndexForContent(IContent sender, bool? supportUnpublished = null)
private void ReIndexForContent(IContent sender, bool? supportUnpublished = null)
{
var valueSet = UmbracoContentIndexer.GetValueSets(Current.UrlSegmentProviders, Current.Services.UserService, sender);
ExamineManager.Instance.IndexItems(
valueSet.ToArray(),
ExamineManager.Instance.IndexProviders.OfType<UmbracoContentIndexer>()
valueSet.ToArray(),
_indexCollection.Indexes.Values.OfType<UmbracoContentIndexer>()
// only for the specified indexers
.Where(x => supportUnpublished.HasValue == false || supportUnpublished.Value == x.SupportUnpublishedContent)
.Where(x => x.EnableDefaultEventHandler));
}
private static void ReIndexForMember(IMember member)
private void ReIndexForMember(IMember member)
{
var valueSet = UmbracoMemberIndexer.GetValueSets(member);
ExamineManager.Instance.IndexItems(
valueSet.ToArray(),
ExamineManager.Instance.IndexProviders.OfType<UmbracoExamineIndexer>()
_indexCollection.Indexes.Values.OfType<UmbracoExamineIndexer>()
//ensure that only the providers are flagged to listen execute
.Where(x => x.EnableDefaultEventHandler));
}
private static void ReIndexForMedia(IMedia sender, bool isMediaPublished)
private void ReIndexForMedia(IMedia sender, bool isMediaPublished)
{
var valueSet = UmbracoContentIndexer.GetValueSets(Current.UrlSegmentProviders, Current.Services.UserService, sender);
ExamineManager.Instance.IndexItems(
valueSet.ToArray(),
ExamineManager.Instance.IndexProviders.OfType<UmbracoContentIndexer>()
_indexCollection.Indexes.Values.OfType<UmbracoContentIndexer>()
// index this item for all indexers if the media is not trashed, otherwise if the item is trashed
// then only index this for indexers supporting unpublished media
.Where(x => isMediaPublished || (x.SupportUnpublishedContent))
@@ -336,11 +454,11 @@ namespace Umbraco.Web.Search
/// If true, indicates that we will only delete this item from indexes that don't support unpublished content.
/// If false it will delete this from all indexes regardless.
/// </param>
private static void DeleteIndexForEntity(int entityId, bool keepIfUnpublished)
private void DeleteIndexForEntity(int entityId, bool keepIfUnpublished)
{
ExamineManager.Instance.DeleteFromIndex(
ExamineManager.Instance.DeleteFromIndexes(
entityId.ToString(CultureInfo.InvariantCulture),
ExamineManager.Instance.IndexProviders.OfType<UmbracoExamineIndexer>()
_indexCollection.Indexes.Values.OfType<UmbracoExamineIndexer>()
// if keepIfUnpublished == true then only delete this item from indexes not supporting unpublished content,
// otherwise if keepIfUnpublished == false then remove from all indexes
.Where(x => keepIfUnpublished == false || (x is UmbracoContentIndexer && ((UmbracoContentIndexer)x).SupportUnpublishedContent == false))

23
src/Umbraco.Web/Strategies/DatabaseServerRegistrarAndMessengerComponent.cs
View File

@@ -18,7 +18,9 @@ using Umbraco.Web.Routing;
using Umbraco.Web.Scheduling;
using LightInject;
using Umbraco.Core.Exceptions;
using Umbraco.Examine;
using Umbraco.Web.Composing;
using Umbraco.Web.Search;
namespace Umbraco.Web.Strategies
{
@@ -45,6 +47,7 @@ namespace Umbraco.Web.Strategies
private bool _started;
private TouchServerTask _task;
private IUmbracoDatabaseFactory _databaseFactory;
private IExamineIndexCollectionAccessor _indexCollection;
public override void Compose(Composition composition)
{
@@ -77,38 +80,28 @@ namespace Umbraco.Web.Strategies
// (we really should have a way to reuse RefreshAll... locally)
// note: refresh all content & media caches does refresh content types too
var svc = Current.PublishedSnapshotService;
bool ignored1, ignored2;
svc.Notify(new[] { new DomainCacheRefresher.JsonPayload(0, DomainChangeTypes.RefreshAll) });
svc.Notify(new[] { new ContentCacheRefresher.JsonPayload(0, TreeChangeTypes.RefreshAll) }, out ignored1, out ignored2);
svc.Notify(new[] { new MediaCacheRefresher.JsonPayload(0, TreeChangeTypes.RefreshAll) }, out ignored1);
svc.Notify(new[] { new ContentCacheRefresher.JsonPayload(0, TreeChangeTypes.RefreshAll) }, out _, out _);
svc.Notify(new[] { new MediaCacheRefresher.JsonPayload(0, TreeChangeTypes.RefreshAll) }, out _);
},
//rebuild indexes if the server is not synced
// NOTE: This will rebuild ALL indexes including the members, if developers want to target specific
// indexes then they can adjust this logic themselves.
() => RebuildIndexes(false)
() => ExamineComponent.RebuildIndexes(false, _indexCollection, _logger)
}
});
});
}
// fixme - this should move to something else, we should not depend on Examine here!
private static void RebuildIndexes(bool onlyEmptyIndexes)
{
var indexers = (IEnumerable<KeyValuePair<string, IIndexer>>) ExamineManager.Instance.IndexProviders;
if (onlyEmptyIndexes)
indexers = indexers.Where(x => x.Value.IsIndexNew());
foreach (var indexer in indexers)
indexer.Value.RebuildIndex();
}
public void Initialize(IRuntimeState runtime, IServerRegistrar serverRegistrar, IServerRegistrationService registrationService, IUmbracoDatabaseFactory databaseFactory, ILogger logger)
public void Initialize(IRuntimeState runtime, IServerRegistrar serverRegistrar, IServerRegistrationService registrationService, IUmbracoDatabaseFactory databaseFactory, ILogger logger, IExamineIndexCollectionAccessor indexCollection)
{
if (UmbracoConfig.For.UmbracoSettings().DistributedCall.Enabled) return;
_registrar = serverRegistrar as DatabaseServerRegistrar;
if (_registrar == null) throw new Exception("panic: registar.");
_indexCollection = indexCollection;
_runtime = runtime;
_databaseFactory = databaseFactory;
_logger = logger;

9
src/Umbraco.Web/Suspendable.cs
View File

@@ -2,7 +2,9 @@
using Examine;
using Examine.Providers;
using Umbraco.Core.Composing;
using Umbraco.Examine;
using Umbraco.Web.Cache;
using Umbraco.Web.Search;
namespace Umbraco.Web
{
@@ -77,10 +79,9 @@ namespace Umbraco.Web
_tried = false;
// fixme - could we fork this on a background thread?
foreach (BaseIndexProvider indexer in ExamineManager.Instance.IndexProviderCollection)
{
indexer.RebuildIndex();
}
//TODO: when resuming do we always want a full rebuild of all indexes?
//We are calling into the ExamineComponent since this rebuilds only when MainDom is active
ExamineComponent.RebuildIndexes(false, new ExamineIndexCollectionAccessor(), Current.Logger);
}
}

2
src/Umbraco.Web/Umbraco.Web.csproj
View File

@@ -66,7 +66,7 @@
<PackageReference Include="ClientDependency" Version="1.9.2" />
<PackageReference Include="CSharpTest.Net.Collections" Version="14.906.1403.1082" />
<PackageReference Include="dotless" Version="1.5.2" />
<PackageReference Include="Examine" Version="1.0.0-beta012" />
<PackageReference Include="Examine" Version="1.0.0-beta017" />
<PackageReference Include="HtmlAgilityPack" Version="1.5.1" />
<PackageReference Include="LightInject" Version="5.0.3" />
<PackageReference Include="LightInject.Annotation" Version="1.1.0" />

7
src/Umbraco.Web/WebServices/ExamineManagementApiController.cs
View File

@@ -23,9 +23,10 @@ namespace Umbraco.Web.WebServices
[ValidateAngularAntiForgeryToken]
public class ExamineManagementApiController : UmbracoAuthorizedApiController
{
public ExamineManagementApiController(ExamineManager examineManager, ILogger logger, IRuntimeCacheProvider runtimeCacheProvider)
public ExamineManagementApiController(ILogger logger, IRuntimeCacheProvider runtimeCacheProvider)
{
_examineManager = examineManager;
//fixme can we inject this? we'll need an IExamineManager
_examineManager = ExamineManager.Instance;
_logger = logger;
_runtimeCacheProvider = runtimeCacheProvider;
}
@@ -114,7 +115,7 @@ namespace Umbraco.Web.WebServices
.OrderBy(x => x.Name);
foreach (var p in props)
{
indexerModel.ProviderProperties.Add(p.Name, p.GetValue(searcher, null).ToString());
indexerModel.ProviderProperties.Add(p.Name, p.GetValue(searcher, null)?.ToString());
}
return indexerModel;
}).OrderBy(x =>