yv01p/Umbraco-CMS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

decouples data lookup from indexers and rebuilding logic

Browse Source
This commit is contained in:
Shannon
2018-11-28 01:23:02 +11:00
parent ff3af8b7a7
commit 3fd0aef18e
30 changed files with 670 additions and 697 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
src/Umbraco.Examine/Config/IndexFieldCollection.cs Normal file
View File

@@ -0,0 +1,47 @@
using System.Configuration;
namespace Umbraco.Examine.Config
{
public sealed class IndexFieldCollection : ConfigurationElementCollection
{
#region Overridden methods to define collection
protected override ConfigurationElement CreateNewElement()
{
return new ConfigIndexField();
}
protected override object GetElementKey(ConfigurationElement element)
{
ConfigIndexField field = (ConfigIndexField)element;
return field.Name;
}
public override bool IsReadOnly()
{
return false;
}
#endregion
/// <summary>
/// Adds an index field to the collection
/// </summary>
/// <param name="field"></param>
public void Add(ConfigIndexField field)
{
BaseAdd(field, true);
}
/// <summary>
/// Default property for accessing an IndexField definition
/// </summary>
/// <value>Field Name</value>
/// <returns></returns>
public new ConfigIndexField this[string name]
{
get
{
return (ConfigIndexField)this.BaseGet(name);
}
}
}
}

44
src/Umbraco.Examine/Config/IndexSetCollection.cs
View File

@@ -3,50 +3,6 @@
namespace Umbraco.Examine.Config
{
public sealed class IndexFieldCollection : ConfigurationElementCollection
{
#region Overridden methods to define collection
protected override ConfigurationElement CreateNewElement()
{
return new ConfigIndexField();
}
protected override object GetElementKey(ConfigurationElement element)
{
ConfigIndexField field = (ConfigIndexField)element;
return field.Name;
}
public override bool IsReadOnly()
{
return false;
}
#endregion
/// <summary>
/// Adds an index field to the collection
/// </summary>
/// <param name="field"></param>
public void Add(ConfigIndexField field)
{
BaseAdd(field, true);
}
/// <summary>
/// Default property for accessing an IndexField definition
/// </summary>
/// <value>Field Name</value>
/// <returns></returns>
public new ConfigIndexField this[string name]
{
get
{
return (ConfigIndexField)this.BaseGet(name);
}
}
}
public sealed class IndexSetCollection : ConfigurationElementCollection
{
#region Overridden methods to define collection

97
src/Umbraco.Examine/ContentIndexPopulator.cs Normal file
View File

@@ -0,0 +1,97 @@
using System;
using System.Linq;
using Examine;
using Umbraco.Core.Models;
using Umbraco.Core.Services;
using Umbraco.Core.Persistence;
using Umbraco.Core.Persistence.DatabaseModelDefinitions;
using Umbraco.Core.Persistence.Querying;
namespace Umbraco.Examine
{
/// <summary>
/// Performs the data lookups required to rebuild a content index
/// </summary>
public class ContentIndexPopulator : IIndexPopulator
{
private readonly IContentService _contentService;
private readonly IValueSetBuilder<IContent> _contentValueSetBuilder;
/// <summary>
/// This is a static query, it's parameters don't change so store statically
/// </summary>
private static IQuery<IContent> _publishedQuery;
private readonly bool _supportUnpublishedContent;
private readonly int? _parentId;
/// <summary>
/// Default constructor to lookup all content data
/// </summary>
/// <param name="contentService"></param>
/// <param name="sqlContext"></param>
/// <param name="contentValueSetBuilder"></param>
public ContentIndexPopulator(IContentService contentService, ISqlContext sqlContext, IValueSetBuilder<IContent> contentValueSetBuilder)
: this(true, null, contentService, sqlContext, contentValueSetBuilder)
{
}
/// <summary>
/// Optional constructor allowing specifying custom query parameters
/// </summary>
/// <param name="supportUnpublishedContent"></param>
/// <param name="parentId"></param>
/// <param name="contentService"></param>
/// <param name="sqlContext"></param>
/// <param name="contentValueSetBuilder"></param>
public ContentIndexPopulator(bool supportUnpublishedContent, int? parentId, IContentService contentService, ISqlContext sqlContext, IValueSetBuilder<IContent> contentValueSetBuilder)
{
if (sqlContext == null) throw new ArgumentNullException(nameof(sqlContext));
_contentService = contentService ?? throw new ArgumentNullException(nameof(contentService));
_contentValueSetBuilder = contentValueSetBuilder ?? throw new ArgumentNullException(nameof(contentValueSetBuilder));
if (_publishedQuery != null)
_publishedQuery = sqlContext.Query<IContent>().Where(x => x.Published);
_supportUnpublishedContent = supportUnpublishedContent;
_parentId = parentId;
}
public void Populate(params IIndexer[] indexes)
{
const int pageSize = 10000;
var pageIndex = 0;
var contentParentId = -1;
if (_parentId.HasValue && _parentId.Value > 0)
{
contentParentId = _parentId.Value;
}
IContent[] content;
do
{
long total;
if (_supportUnpublishedContent)
{
content = _contentService.GetPagedDescendants(contentParentId, pageIndex, pageSize, out total).ToArray();
}
else
{
//add the published filter
//note: We will filter for published variants in the validator
content = _contentService.GetPagedDescendants(contentParentId, pageIndex, pageSize, out total,
_publishedQuery, Ordering.By("Path", Direction.Ascending)).ToArray();
}
if (content.Length > 0)
{
foreach (var index in indexes)
index.IndexItems(_contentValueSetBuilder.GetValueSets(content));
}
pageIndex++;
} while (content.Length == pageSize);
}
}
}

16
src/Umbraco.Examine/IIndexPopulator.cs Normal file
View File

@@ -0,0 +1,16 @@
using Examine;
namespace Umbraco.Examine
{
/// <summary>
/// Populates indexes with data
/// </summary>
public interface IIndexPopulator
{
/// <summary>
/// Populates indexes with data
/// </summary>
/// <param name="indexes"></param>
void Populate(params IIndexer[] indexes);
}
}

12
src/Umbraco.Examine/IUmbracoContentIndexer.cs Normal file
View File

@@ -0,0 +1,12 @@
using Examine;
namespace Umbraco.Examine
{
//TODO: Rethink this, need a better way of rebuilding
/// <summary>
/// A Marker interface for defining an Umbraco content indexer
/// </summary>
public interface IUmbracoContentIndexer : IIndexer
{
}
}

1
src/Umbraco.Examine/IUmbracoIndexer.cs
View File

@@ -2,6 +2,7 @@
namespace Umbraco.Examine
{
/// <summary>
/// A Marker interface for defining an Umbraco indexer
/// </summary>

12
src/Umbraco.Examine/IUmbracoMediaIndexer.cs Normal file
View File

@@ -0,0 +1,12 @@
using Examine;
namespace Umbraco.Examine
{
//TODO: Rethink this, need a better way of rebuilding
/// <summary>
/// A Marker interface for defining an Umbraco media indexer
/// </summary>
public interface IUmbracoMediaIndexer : IIndexer
{
}
}

48
src/Umbraco.Examine/IndexRebuilder.cs Normal file
View File

@@ -0,0 +1,48 @@
using System.Linq;
using Examine;
namespace Umbraco.Examine
{
/// <summary>
/// Utility to rebuild all indexes ensuring minimal data queries
/// </summary>
public class IndexRebuilder
{
public IExamineManager ExamineManager { get; }
private readonly ContentIndexPopulator _contentIndexPopulator;
private readonly MediaIndexPopulator _mediaIndexPopulator;
public IndexRebuilder(IExamineManager examineManager, ContentIndexPopulator contentIndexPopulator, MediaIndexPopulator mediaIndexPopulator)
{
ExamineManager = examineManager;
_contentIndexPopulator = contentIndexPopulator;
_mediaIndexPopulator = mediaIndexPopulator;
}
public void RebuildIndexes(bool onlyEmptyIndexes)
{
var indexes = (onlyEmptyIndexes
? ExamineManager.IndexProviders.Values.Where(x => x.IndexExists())
: ExamineManager.IndexProviders.Values).ToList();
var contentIndexes = indexes.Where(x => x is IUmbracoContentIndexer).ToArray();
var mediaIndexes = indexes.Where(x => x is IUmbracoContentIndexer).ToArray();
var nonUmbracoIndexes = indexes.Except(contentIndexes).Except(mediaIndexes).ToArray();
foreach(var index in indexes)
index.CreateIndex(); // clear the index
//reindex all content/media indexes with the same data source/lookup
_contentIndexPopulator.Populate(contentIndexes);
_mediaIndexPopulator.Populate(mediaIndexes);
//then do the rest
foreach (var index in nonUmbracoIndexes)
{
index.CreateIndex();
//TODO: How to rebuild?
}
}
}
}

68
src/Umbraco.Examine/MediaIndexPopulator.cs Normal file
View File

@@ -0,0 +1,68 @@
using System.Linq;
using Examine;
using Umbraco.Core.Models;
using Umbraco.Core.Services;
namespace Umbraco.Examine
{
/// <summary>
/// Performs the data lookups required to rebuild a media index
/// </summary>
public class MediaIndexPopulator : IIndexPopulator
{
private readonly int? _parentId;
private readonly IMediaService _mediaService;
private readonly IValueSetBuilder<IMedia> _mediaValueSetBuilder;
/// <summary>
/// Default constructor to lookup all content data
/// </summary>
/// <param name="mediaService"></param>
/// <param name="mediaValueSetBuilder"></param>
public MediaIndexPopulator(IMediaService mediaService, IValueSetBuilder<IMedia> mediaValueSetBuilder)
: this(null, mediaService, mediaValueSetBuilder)
{
}
/// <summary>
/// Optional constructor allowing specifying custom query parameters
/// </summary>
/// <param name="parentId"></param>
/// <param name="mediaService"></param>
/// <param name="mediaValueSetBuilder"></param>
public MediaIndexPopulator(int? parentId, IMediaService mediaService, IValueSetBuilder<IMedia> mediaValueSetBuilder)
{
_parentId = parentId;
_mediaService = mediaService;
_mediaValueSetBuilder = mediaValueSetBuilder;
}
public void Populate(params IIndexer[] indexes)
{
const int pageSize = 10000;
var pageIndex = 0;
var mediaParentId = -1;
if (_parentId.HasValue && _parentId.Value > 0)
{
mediaParentId = _parentId.Value;
}
IMedia[] media;
do
{
media = _mediaService.GetPagedDescendants(mediaParentId, pageIndex, pageSize, out var total).ToArray();
if (media.Length > 0)
{
foreach (var index in indexes)
index.IndexItems(_mediaValueSetBuilder.GetValueSets(media));
}
pageIndex++;
} while (media.Length == pageSize);
}
}
}

42
src/Umbraco.Examine/MemberIndexPopulator.cs Normal file
View File

@@ -0,0 +1,42 @@
using System.Linq;
using Examine;
using Umbraco.Core.Models;
using Umbraco.Core.Services;
namespace Umbraco.Examine
{
public class MemberIndexPopulator: IIndexPopulator
{
private readonly IMemberService _memberService;
private readonly IValueSetBuilder<IMember> _valueSetBuilder;
public MemberIndexPopulator(IMemberService memberService, IValueSetBuilder<IMember> valueSetBuilder)
{
_memberService = memberService;
_valueSetBuilder = valueSetBuilder;
}
public void Populate(params IIndexer[] indexes)
{
const int pageSize = 1000;
var pageIndex = 0;
IMember[] members;
//TODO: Add validators for member indexers for ConfigIndexCriteria.IncludeItemTypes
//no node types specified, do all members
do
{
members = _memberService.GetAll(pageIndex, pageSize, out _).ToArray();
if (members.Length > 0)
{
foreach (var index in indexes)
index.IndexItems(_valueSetBuilder.GetValueSets(members));
}
pageIndex++;
} while (members.Length == pageSize);
}
}
}

22
src/Umbraco.Examine/PublishedContentIndexPopulator.cs Normal file
View File

@@ -0,0 +1,22 @@
using Umbraco.Core.Models;
using Umbraco.Core.Services;
using Umbraco.Core.Persistence;
namespace Umbraco.Examine
{
/// <summary>
/// Performs the data lookups required to rebuild a content index containing only published content
/// </summary>
/// <remarks>
/// The published (external) index will still rebuild just fine using the default <see cref="ContentIndexPopulator"/> which is what
/// is used when rebuilding all indexes, but this will be used when the single index is rebuilt and will go a little bit faster
/// since the data query is more specific.
/// </remarks>
public class PublishedContentIndexPopulator : ContentIndexPopulator
{
public PublishedContentIndexPopulator(IContentService contentService, ISqlContext sqlContext, IValueSetBuilder<IContent> contentValueSetBuilder) :
base(false, null, contentService, sqlContext, contentValueSetBuilder)
{
}
}
}

11
src/Umbraco.Examine/Umbraco.Examine.csproj
View File

@@ -48,7 +48,7 @@
</ItemGroup>
<ItemGroup>
<!-- note: NuGet deals with transitive references now -->
<PackageReference Include="Examine" Version="1.0.0-beta032" />
<PackageReference Include="Examine" Version="1.0.0-beta034" />
<PackageReference Include="Newtonsoft.Json" Version="11.0.2" />
<PackageReference Include="NPoco" Version="3.9.4" />
</ItemGroup>
@@ -56,15 +56,24 @@
<Compile Include="BaseValueSetBuilder.cs" />
<Compile Include="Config\ConfigIndexCriteria.cs" />
<Compile Include="Config\ConfigIndexField.cs" />
<Compile Include="Config\IndexFieldCollection.cs" />
<Compile Include="Config\IndexFieldCollectionExtensions.cs" />
<Compile Include="Config\IndexSet.cs" />
<Compile Include="Config\IndexSetCollection.cs" />
<Compile Include="Config\IndexSets.cs" />
<Compile Include="ContentIndexPopulator.cs" />
<Compile Include="ContentValueSetBuilder.cs" />
<Compile Include="IIndexPopulator.cs" />
<Compile Include="IndexRebuilder.cs" />
<Compile Include="IUmbracoContentIndexer.cs" />
<Compile Include="IUmbracoIndexer.cs" />
<Compile Include="IUmbracoMediaIndexer.cs" />
<Compile Include="IValueSetBuilder.cs" />
<Compile Include="MediaIndexPopulator.cs" />
<Compile Include="MediaValueSetBuilder.cs" />
<Compile Include="MemberIndexPopulator.cs" />
<Compile Include="MemberValueSetBuilder.cs" />
<Compile Include="PublishedContentIndexPopulator.cs" />
<Compile Include="UmbracoExamineExtensions.cs" />
<Compile Include="IndexTypes.cs" />
<Compile Include="NoPrefixSimpleFsLockFactory.cs" />

215
src/Umbraco.Examine/UmbracoContentIndexer.cs
View File

@@ -2,10 +2,8 @@
using System.Collections.Generic;
using System.Collections.Specialized;
using System.ComponentModel;
using System.Linq;
using Examine;
using Umbraco.Core;
using Umbraco.Core.Models;
using Umbraco.Core.Services;
using Umbraco.Core.Strings;
using Examine.LuceneEngine.Indexing;
@@ -14,29 +12,17 @@ using Lucene.Net.Analysis;
using Lucene.Net.Store;
using Umbraco.Core.Composing;
using Umbraco.Core.Logging;
using Umbraco.Core.Persistence;
using Umbraco.Core.Persistence.DatabaseModelDefinitions;
using Umbraco.Core.Persistence.Querying;
using Umbraco.Core.Scoping;
using Umbraco.Examine.Config;
using IContentService = Umbraco.Core.Services.IContentService;
using IMediaService = Umbraco.Core.Services.IMediaService;
using Examine.LuceneEngine;
using Umbraco.Core.PropertyEditors;
namespace Umbraco.Examine
{
/// <summary>
/// An indexer for Umbraco content and media
/// </summary>
public class UmbracoContentIndexer : UmbracoExamineIndexer
public class UmbracoContentIndexer : UmbracoExamineIndexer, IUmbracoContentIndexer, IUmbracoMediaIndexer
{
public const string VariesByCultureFieldName = UmbracoExamineIndexer.SpecialFieldPrefix + "VariesByCulture";
public IValueSetBuilder<IMedia> MediaValueSetBuilder { get; }
public IValueSetBuilder<IContent> ContentValueSetBuilder { get; }
protected IContentService ContentService { get; }
protected IMediaService MediaService { get; }
protected ILocalizationService LanguageService { get; }
private int? _parentId;
@@ -49,14 +35,7 @@ namespace Umbraco.Examine
[EditorBrowsable(EditorBrowsableState.Never)]
public UmbracoContentIndexer()
{
ContentService = Current.Services.ContentService;
MediaService = Current.Services.MediaService;
LanguageService = Current.Services.LocalizationService;
ContentValueSetBuilder = new ContentValueSetBuilder(Current.PropertyEditors, Current.UrlSegmentProviders, Current.Services.UserService);
MediaValueSetBuilder = new MediaValueSetBuilder(Current.PropertyEditors, Current.UrlSegmentProviders, Current.Services.UserService);
InitializeQueries(Current.SqlContext);
}
/// <summary>
@@ -67,9 +46,6 @@ namespace Umbraco.Examine
/// <param name="luceneDirectory"></param>
/// <param name="defaultAnalyzer"></param>
/// <param name="profilingLogger"></param>
/// <param name="contentService"></param>
/// <param name="mediaService"></param>
/// <param name="sqlContext"></param>
/// <param name="validator"></param>
/// <param name="options"></param>
/// <param name="indexValueTypes"></param>
@@ -79,12 +55,7 @@ namespace Umbraco.Examine
Directory luceneDirectory,
Analyzer defaultAnalyzer,
ProfilingLogger profilingLogger,
IValueSetBuilder<IContent> contentValueSetBuilder,
IValueSetBuilder<IMedia> mediaValueSetBuilder,
IContentService contentService,
IMediaService mediaService,
ILocalizationService languageService,
ISqlContext sqlContext,
IValueSetValidator validator,
UmbracoContentIndexerOptions options,
IReadOnlyDictionary<string, Func<string, IIndexValueType>> indexValueTypes = null)
@@ -96,21 +67,9 @@ namespace Umbraco.Examine
SupportProtectedContent = options.SupportProtectedContent;
SupportUnpublishedContent = options.SupportUnpublishedContent;
ParentId = options.ParentId;
ContentValueSetBuilder = contentValueSetBuilder ?? throw new ArgumentNullException(nameof(contentValueSetBuilder));
MediaValueSetBuilder = mediaValueSetBuilder ?? throw new ArgumentNullException(nameof(mediaValueSetBuilder));
ContentService = contentService ?? throw new ArgumentNullException(nameof(contentService));
MediaService = mediaService ?? throw new ArgumentNullException(nameof(mediaService));
LanguageService = languageService ?? throw new ArgumentNullException(nameof(languageService));
InitializeQueries(sqlContext);
}
private void InitializeQueries(ISqlContext sqlContext)
{
if (sqlContext == null) throw new ArgumentNullException(nameof(sqlContext));
if (_publishedQuery == null)
_publishedQuery = sqlContext.Query<IContent>().Where(x => x.Published);
}
#endregion
@@ -133,9 +92,11 @@ namespace Umbraco.Examine
/// <exception cref="T:System.InvalidOperationException">
/// An attempt is made to call <see cref="M:System.Configuration.Provider.ProviderBase.Initialize(System.String,System.Collections.Specialized.NameValueCollection)"/> on a provider after the provider has already been initialized.
/// </exception>
[EditorBrowsable(EditorBrowsableState.Never)]
public override void Initialize(string name, NameValueCollection config)
{
base.Initialize(name, config);
//check if there's a flag specifying to support unpublished content,
//if not, set to false;
if (config["supportUnpublished"] != null && bool.TryParse(config["supportUnpublished"], out var supportUnpublished))
@@ -143,7 +104,6 @@ namespace Umbraco.Examine
else
SupportUnpublishedContent = false;
//check if there's a flag specifying to support protected content,
//if not, set to false;
if (config["supportProtected"] != null && bool.TryParse(config["supportProtected"], out var supportProtected))
@@ -151,8 +111,6 @@ namespace Umbraco.Examine
else
SupportProtectedContent = false;
base.Initialize(name, config);
//now we need to build up the indexer options so we can create our validator
int? parentId = null;
if (IndexSetName.IsNullOrWhiteSpace() == false)
@@ -160,11 +118,15 @@ namespace Umbraco.Examine
var indexSet = IndexSets.Instance.Sets[IndexSetName];
parentId = indexSet.IndexParentId;
}
ValueSetValidator = new UmbracoContentValueSetValidator(
new UmbracoContentIndexerOptions(SupportUnpublishedContent, SupportProtectedContent, parentId),
new UmbracoContentIndexerOptions(
SupportUnpublishedContent, SupportProtectedContent, parentId,
ConfigIndexCriteria.IncludeItemTypes, ConfigIndexCriteria.ExcludeItemTypes),
//Using a singleton here, we can't inject this when using config based providers and we don't use this
//anywhere else in this class
Current.Services.PublicAccessService);
}
#endregion
@@ -186,8 +148,6 @@ namespace Umbraco.Examine
protected set => _parentId = value;
}
protected override IEnumerable<string> SupportedTypes => new[] {IndexTypes.Content, IndexTypes.Media};
#endregion
#region Public methods
@@ -245,164 +205,7 @@ namespace Umbraco.Examine
return base.CreateFieldValueTypes(indexValueTypesFactory);
}
/// <summary>
/// This is a static query, it's parameters don't change so store statically
/// </summary>
private static IQuery<IContent> _publishedQuery;
protected override void PerformIndexAll(string type)
{
const int pageSize = 10000;
var pageIndex = 0;
switch (type)
{
case IndexTypes.Content:
var contentParentId = -1;
if (ParentId.HasValue && ParentId.Value > 0)
{
contentParentId = ParentId.Value;
}
IContent[] content;
do
{
long total;
IEnumerable<IContent> descendants;
if (SupportUnpublishedContent)
{
descendants = ContentService.GetPagedDescendants(contentParentId, pageIndex, pageSize, out total);
}
else
{
//add the published filter
descendants = ContentService.GetPagedDescendants(contentParentId, pageIndex, pageSize, out total,
_publishedQuery, Ordering.By("Path", Direction.Ascending));
}
//if specific types are declared we need to post filter them
//TODO: Update the service layer to join the cmsContentType table so we can query by content type too
if (ConfigIndexCriteria != null && ConfigIndexCriteria.IncludeItemTypes.Any())
{
content = descendants.Where(x => ConfigIndexCriteria.IncludeItemTypes.Contains(x.ContentType.Alias)).ToArray();
}
else
{
content = descendants.ToArray();
}
IndexItems(ContentValueSetBuilder.GetValueSets(content));
pageIndex++;
} while (content.Length == pageSize);
break;
case IndexTypes.Media:
var mediaParentId = -1;
if (ParentId.HasValue && ParentId.Value > 0)
{
mediaParentId = ParentId.Value;
}
IMedia[] media;
do
{
var descendants = MediaService.GetPagedDescendants(mediaParentId, pageIndex, pageSize, out _);
//if specific types are declared we need to post filter them
//TODO: Update the service layer to join the cmsContentType table so we can query by content type too
if (ConfigIndexCriteria != null && ConfigIndexCriteria.IncludeItemTypes.Any())
{
media = descendants.Where(x => ConfigIndexCriteria.IncludeItemTypes.Contains(x.ContentType.Alias)).ToArray();
}
else
{
media = descendants.ToArray();
}
IndexItems(MediaValueSetBuilder.GetValueSets(media));
pageIndex++;
} while (media.Length == pageSize);
break;
}
}
//TODO: We want to make a public method that iterates a data set, potentially with callbacks so that we can iterate
// a single data set once but populate multiple indexes with it. This is for Startup performance when no indexes exist.
// This could be used for any indexer of type UmbracoContentIndexer - BUT that means we need to make another interface
// for content indexers since UmbracoContentIndexer is strongly tied to lucene, so maybe we have a more generic interface
// or add to the current IUmbracoIndexer interface
#endregion
}
public class ContentIndexDataSource
{
public ContentIndexDataSource(bool supportUnpublishedContent, int? parentId,
IContentService contentService, ISqlContext sqlContext,
IValueSetBuilder<IContent> contentValueSetBuilder)
{
SupportUnpublishedContent = supportUnpublishedContent;
ParentId = parentId;
ContentService = contentService;
_contentValueSetBuilder = contentValueSetBuilder;
if (sqlContext == null) throw new ArgumentNullException(nameof(sqlContext));
_publishedQuery = sqlContext.Query<IContent>().Where(x => x.Published);
}
public bool SupportUnpublishedContent { get; }
public int? ParentId { get; }
public IContentService ContentService { get; }
/// <summary>
/// This is a static query, it's parameters don't change so store statically
/// </summary>
private static IQuery<IContent> _publishedQuery;
private readonly IValueSetBuilder<IContent> _contentValueSetBuilder;
public void Index(params IIndexer[] indexes)
{
const int pageSize = 10000;
var pageIndex = 0;
var contentParentId = -1;
if (ParentId.HasValue && ParentId.Value > 0)
{
contentParentId = ParentId.Value;
}
IContent[] content;
do
{
long total;
IEnumerable<IContent> descendants;
if (SupportUnpublishedContent)
{
descendants = ContentService.GetPagedDescendants(contentParentId, pageIndex, pageSize, out total);
}
else
{
//add the published filter
//note: We will filter for published variants in the validator
descendants = ContentService.GetPagedDescendants(contentParentId, pageIndex, pageSize, out total,
_publishedQuery, Ordering.By("Path", Direction.Ascending));
}
content = descendants.ToArray();
foreach(var index in indexes)
index.IndexItems(_contentValueSetBuilder.GetValueSets(content));
pageIndex++;
} while (content.Length == pageSize);
}
}
}

63
src/Umbraco.Examine/UmbracoExamineIndexer.cs
View File

@@ -166,11 +166,6 @@ namespace Umbraco.Examine
/// </remarks>
public bool SupportUnpublishedContent { get; protected set; } = false;
/// <summary>
/// the supported indexable types
/// </summary>
protected abstract IEnumerable<string> SupportedTypes { get; }
protected ConfigIndexCriteria ConfigIndexCriteria { get; private set; }
/// <summary>
@@ -271,53 +266,6 @@ namespace Umbraco.Examine
#endregion
/// <summary>
/// override to check if we can actually initialize.
/// </summary>
/// <remarks>
/// This check is required since the base examine lib will try to rebuild on startup
/// </remarks>
public override void RebuildIndex()
{
if (CanInitialize())
{
ProfilingLogger.Logger.Debug(GetType(), "Rebuilding index");
using (new SafeCallContext())
{
base.RebuildIndex();
}
}
}
/// <summary>
/// override to check if we can actually initialize.
/// </summary>
/// <remarks>
/// This check is required since the base examine lib will try to rebuild on startup
/// </remarks>
public override void IndexAll(string type)
{
if (CanInitialize())
{
using (new SafeCallContext())
{
base.IndexAll(type);
}
}
}
public override void IndexItems(IEnumerable<ValueSet> nodes)
{
if (CanInitialize())
{
using (new SafeCallContext())
{
base.IndexItems(nodes);
}
}
}
/// <summary>
/// override to check if we can actually initialize.
/// </summary>
@@ -346,17 +294,6 @@ namespace Umbraco.Examine
return _configBased == false || Current.RuntimeState.Level == RuntimeLevel.Run;
}
/// <summary>
/// Reindexes all supported types
/// </summary>
protected override void PerformIndexRebuild()
{
foreach (var t in SupportedTypes)
{
IndexAll(t);
}
}
/// <summary>
/// overridden for logging
/// </summary>

59
src/Umbraco.Examine/UmbracoMemberIndexer.cs
View File

@@ -23,17 +23,12 @@ namespace Umbraco.Examine
/// </summary>
public class UmbracoMemberIndexer : UmbracoExamineIndexer
{
private readonly IValueSetBuilder<IMember> _valueSetBuilder;
private readonly IMemberService _memberService;
/// <summary>
/// Constructor for config/provider based indexes
/// </summary>
[EditorBrowsable(EditorBrowsableState.Never)]
public UmbracoMemberIndexer()
{
_memberService = Current.Services.MemberService;
_valueSetBuilder = new MemberValueSetBuilder(Current.PropertyEditors);
}
/// <summary>
@@ -44,7 +39,6 @@ namespace Umbraco.Examine
/// <param name="luceneDirectory"></param>
/// <param name="profilingLogger"></param>
/// <param name="validator"></param>
/// <param name="memberService"></param>
/// <param name="analyzer"></param>
public UmbracoMemberIndexer(
string name,
@@ -52,13 +46,9 @@ namespace Umbraco.Examine
Directory luceneDirectory,
Analyzer analyzer,
ProfilingLogger profilingLogger,
IValueSetBuilder<IMember> valueSetBuilder,
IMemberService memberService,
IValueSetValidator validator = null) :
base(name, fieldDefinitions, luceneDirectory, analyzer, profilingLogger, validator)
{
_valueSetBuilder = valueSetBuilder ?? throw new ArgumentNullException(nameof(valueSetBuilder));
_memberService = memberService ?? throw new ArgumentNullException(nameof(memberService));
}
@@ -76,55 +66,6 @@ namespace Umbraco.Examine
return base.CreateFieldValueTypes(indexValueTypesFactory);
}
/// <inheritdoc />
protected override IEnumerable<string> SupportedTypes => new[] {IndexTypes.Member};
/// <summary>
/// Reindex all members
/// </summary>
/// <param name="type"></param>
protected override void PerformIndexAll(string type)
{
//This only supports members
if (SupportedTypes.Contains(type) == false)
return;
const int pageSize = 1000;
var pageIndex = 0;
IMember[] members;
if (ConfigIndexCriteria != null && ConfigIndexCriteria.IncludeItemTypes.Any())
{
//if there are specific node types then just index those
foreach (var nodeType in ConfigIndexCriteria.IncludeItemTypes)
{
do
{
members = _memberService.GetAll(pageIndex, pageSize, out _, "LoginName", Direction.Ascending, true, null, nodeType).ToArray();
IndexItems(_valueSetBuilder.GetValueSets(members));
pageIndex++;
} while (members.Length == pageSize);
}
}
else
{
//no node types specified, do all members
do
{
members = _memberService.GetAll(pageIndex, pageSize, out _).ToArray();
IndexItems(_valueSetBuilder.GetValueSets(members));
pageIndex++;
} while (members.Length == pageSize);
}
}
/// <summary>
/// Ensure some custom values are added to the index
/// </summary>

49
src/Umbraco.Tests/PublishedContent/PublishedMediaTests.cs
View File

@@ -113,12 +113,13 @@ namespace Umbraco.Tests.PublishedContent
[Test]
public void Ensure_Children_Sorted_With_Examine()
{
var rebuilder = IndexInitializer.GetMediaIndexRebuilder(Container.GetInstance<PropertyEditorCollection>(), IndexInitializer.GetMockMediaService());
using (var luceneDir = new RandomIdRamDirectory())
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, Container.GetInstance<PropertyEditorCollection>(),
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir,
options: new UmbracoContentIndexerOptions(true, false, null)))
{
indexer.RebuildIndex();
rebuilder.Populate(indexer);
var searcher = indexer.GetSearcher();
var ctx = GetUmbracoContext("/test");
@@ -139,14 +140,15 @@ namespace Umbraco.Tests.PublishedContent
[Test]
public void Do_Not_Find_In_Recycle_Bin()
{
var rebuilder = IndexInitializer.GetMediaIndexRebuilder(Container.GetInstance<PropertyEditorCollection>(), IndexInitializer.GetMockMediaService());
using (var luceneDir = new RandomIdRamDirectory())
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, Container.GetInstance<PropertyEditorCollection>(),
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir,
//include unpublished content since this uses the 'internal' indexer, it's up to the media cache to filter
options: new UmbracoContentIndexerOptions(true, false, null)))
using (indexer.ProcessNonAsync())
{
indexer.RebuildIndex();
rebuilder.Populate(indexer);
var searcher = indexer.GetSearcher();
var ctx = GetUmbracoContext("/test");
@@ -185,13 +187,14 @@ namespace Umbraco.Tests.PublishedContent
[Test]
public void Children_With_Examine()
{
var rebuilder = IndexInitializer.GetMediaIndexRebuilder(Container.GetInstance<PropertyEditorCollection>(), IndexInitializer.GetMockMediaService());
using (var luceneDir = new RandomIdRamDirectory())
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, Container.GetInstance<PropertyEditorCollection>(),
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir,
options: new UmbracoContentIndexerOptions(true, false, null)))
using (indexer.ProcessNonAsync())
{
indexer.RebuildIndex();
rebuilder.Populate(indexer);
var searcher = indexer.GetSearcher();
var ctx = GetUmbracoContext("/test");
@@ -211,13 +214,14 @@ namespace Umbraco.Tests.PublishedContent
[Test]
public void Descendants_With_Examine()
{
var rebuilder = IndexInitializer.GetMediaIndexRebuilder(Container.GetInstance<PropertyEditorCollection>(), IndexInitializer.GetMockMediaService());
using (var luceneDir = new RandomIdRamDirectory())
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, Container.GetInstance<PropertyEditorCollection>(),
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir,
options: new UmbracoContentIndexerOptions(true, false, null)))
using (indexer.ProcessNonAsync())
{
indexer.RebuildIndex();
rebuilder.Populate(indexer);
var searcher = indexer.GetSearcher();
var ctx = GetUmbracoContext("/test");
@@ -237,13 +241,14 @@ namespace Umbraco.Tests.PublishedContent
[Test]
public void DescendantsOrSelf_With_Examine()
{
var rebuilder = IndexInitializer.GetMediaIndexRebuilder(Container.GetInstance<PropertyEditorCollection>(), IndexInitializer.GetMockMediaService());
using (var luceneDir = new RandomIdRamDirectory())
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, Container.GetInstance<PropertyEditorCollection>(),
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir,
options: new UmbracoContentIndexerOptions(true, false, null)))
using (indexer.ProcessNonAsync())
{
indexer.RebuildIndex();
rebuilder.Populate(indexer);
var searcher = indexer.GetSearcher();
var ctx = GetUmbracoContext("/test");
@@ -263,13 +268,15 @@ namespace Umbraco.Tests.PublishedContent
[Test]
public void Ancestors_With_Examine()
{
var rebuilder = IndexInitializer.GetMediaIndexRebuilder(Container.GetInstance<PropertyEditorCollection>(), IndexInitializer.GetMockMediaService());
using (var luceneDir = new RandomIdRamDirectory())
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, Container.GetInstance<PropertyEditorCollection>(),
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir,
options: new UmbracoContentIndexerOptions(true, false, null)))
using (indexer.ProcessNonAsync())
{
indexer.RebuildIndex();
rebuilder.Populate(indexer);
var ctx = GetUmbracoContext("/test");
var searcher = indexer.GetSearcher();
@@ -286,12 +293,14 @@ namespace Umbraco.Tests.PublishedContent
[Test]
public void AncestorsOrSelf_With_Examine()
{
var rebuilder = IndexInitializer.GetMediaIndexRebuilder(Container.GetInstance<PropertyEditorCollection>(), IndexInitializer.GetMockMediaService());
using (var luceneDir = new RandomIdRamDirectory())
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, Container.GetInstance<PropertyEditorCollection>(),
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir,
options: new UmbracoContentIndexerOptions(true, false, null)))
using (indexer.ProcessNonAsync())
{
indexer.RebuildIndex();
rebuilder.Populate(indexer);
var ctx = GetUmbracoContext("/test");

42
src/Umbraco.Tests/TestHelpers/Stubs/TestExamineManager.cs
View File

@@ -9,26 +9,16 @@ namespace Umbraco.Tests.TestHelpers.Stubs
private readonly ConcurrentDictionary<string, IIndexer> _indexers = new ConcurrentDictionary<string, IIndexer>();
private readonly ConcurrentDictionary<string, ISearcher> _searchers = new ConcurrentDictionary<string, ISearcher>();
public void AddIndexer(string name, IIndexer indexer)
public void AddIndexer(IIndexer indexer)
{
_indexers.TryAdd(name, indexer);
_indexers.TryAdd(indexer.Name, indexer);
}
public void AddSearcher(string name, ISearcher searcher)
public void AddSearcher(ISearcher searcher)
{
_searchers.TryAdd(name, searcher);
_searchers.TryAdd(searcher.Name, searcher);
}
public void DeleteFromIndexes(string nodeId)
{
//noop
}
public void DeleteFromIndexes(string nodeId, IEnumerable<IIndexer> providers)
{
//noop
}
public void Dispose()
{
//noop
@@ -39,31 +29,11 @@ namespace Umbraco.Tests.TestHelpers.Stubs
return _indexers.TryGetValue(indexerName, out var indexer) ? indexer : null;
}
public ISearcher GetRegisteredSearcher(string searcherName)
public ISearcher GetSearcher(string searcherName)
{
return _searchers.TryGetValue(searcherName, out var indexer) ? indexer : null;
}
public void IndexAll(string indexCategory)
{
//noop
}
public void IndexItems(ValueSet[] nodes)
{
//noop
}
public void IndexItems(ValueSet[] nodes, IEnumerable<IIndexer> providers)
{
//noop
}
public void RebuildIndexes()
{
//noop
}
public IReadOnlyDictionary<string, IIndexer> IndexProviders => _indexers;
}
}

2
src/Umbraco.Tests/Umbraco.Tests.csproj
View File

@@ -77,7 +77,7 @@
<ItemGroup>
<PackageReference Include="AutoMapper" Version="7.0.1" />
<PackageReference Include="Castle.Core" Version="4.2.1" />
<PackageReference Include="Examine" Version="1.0.0-beta032" />
<PackageReference Include="Examine" Version="1.0.0-beta034" />
<PackageReference Include="HtmlAgilityPack">
<Version>1.8.9</Version>
</PackageReference>

2
src/Umbraco.Tests/UmbracoExamine/EventsTest.cs
View File

@@ -19,7 +19,7 @@ namespace Umbraco.Tests.UmbracoExamine
public void Events_Ignoring_Node()
{
using (var luceneDir = new RandomIdRamDirectory())
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, Container.GetInstance<PropertyEditorCollection>(),
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir,
//make parent id 999 so all are ignored
options: new UmbracoContentIndexerOptions(false, false, 999)))
using (indexer.ProcessNonAsync())

258
src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs
View File

@@ -29,168 +29,146 @@ namespace Umbraco.Tests.UmbracoExamine
/// </summary>
internal static class IndexInitializer
{
public static ContentValueSetBuilder GetContentValueSetBuilder(PropertyEditorCollection propertyEditors)
{
var contentValueSetBuilder = new ContentValueSetBuilder(propertyEditors, new[] { new DefaultUrlSegmentProvider() }, IndexInitializer.GetMockUserService());
return contentValueSetBuilder;
}
public static ContentIndexPopulator GetContentIndexRebuilder(PropertyEditorCollection propertyEditors, IContentService contentService, ISqlContext sqlContext)
{
var contentValueSetBuilder = GetContentValueSetBuilder(propertyEditors);
var contentIndexDataSource = new ContentIndexPopulator(true, null, contentService, sqlContext, contentValueSetBuilder);
return contentIndexDataSource;
}
public static MediaIndexPopulator GetMediaIndexRebuilder(PropertyEditorCollection propertyEditors, IMediaService mediaService)
{
var mediaValueSetBuilder = new MediaValueSetBuilder(propertyEditors, new[] { new DefaultUrlSegmentProvider() }, IndexInitializer.GetMockUserService());
var mediaIndexDataSource = new MediaIndexPopulator(null, mediaService, mediaValueSetBuilder);
return mediaIndexDataSource;
}
public static IContentService GetMockContentService()
{
long longTotalRecs;
var demoData = new ExamineDemoDataContentService();
var allRecs = demoData.GetLatestContentByXPath("//*[@isDoc]")
.Root
.Elements()
.Select(x => Mock.Of<IContent>(
m =>
m.Id == (int)x.Attribute("id") &&
m.ParentId == (int)x.Attribute("parentID") &&
m.Level == (int)x.Attribute("level") &&
m.CreatorId == 0 &&
m.SortOrder == (int)x.Attribute("sortOrder") &&
m.CreateDate == (DateTime)x.Attribute("createDate") &&
m.UpdateDate == (DateTime)x.Attribute("updateDate") &&
m.Name == (string)x.Attribute("nodeName") &&
m.GetCultureName(It.IsAny<string>()) == (string)x.Attribute("nodeName") &&
m.Path == (string)x.Attribute("path") &&
m.Properties == new PropertyCollection() &&
m.ContentType == Mock.Of<IContentType>(mt =>
mt.Icon == "test" &&
mt.Alias == x.Name.LocalName &&
mt.Id == (int)x.Attribute("nodeType"))))
.ToArray();
return Mock.Of<IContentService>(
x => x.GetPagedDescendants(
It.IsAny<int>(), It.IsAny<long>(), It.IsAny<int>(), out longTotalRecs, It.IsAny<IQuery<IContent>>(), It.IsAny<Ordering>())
== allRecs);
}
public static IUserService GetMockUserService()
{
return Mock.Of<IUserService>(x => x.GetProfileById(It.IsAny<int>()) == Mock.Of<IProfile>(p => p.Id == 0 && p.Name == "admin"));
}
public static IMediaService GetMockMediaService()
{
long totalRecs;
var demoData = new ExamineDemoDataMediaService();
var allRecs = demoData.GetLatestMediaByXpath("//node")
.Root
.Elements()
.Select(x => Mock.Of<IMedia>(
m =>
m.Id == (int)x.Attribute("id") &&
m.ParentId == (int)x.Attribute("parentID") &&
m.Level == (int)x.Attribute("level") &&
m.CreatorId == 0 &&
m.SortOrder == (int)x.Attribute("sortOrder") &&
m.CreateDate == (DateTime)x.Attribute("createDate") &&
m.UpdateDate == (DateTime)x.Attribute("updateDate") &&
m.Name == (string)x.Attribute("nodeName") &&
m.GetCultureName(It.IsAny<string>()) == (string)x.Attribute("nodeName") &&
m.Path == (string)x.Attribute("path") &&
m.Properties == new PropertyCollection() &&
m.ContentType == Mock.Of<IMediaType>(mt =>
mt.Alias == (string)x.Attribute("nodeTypeAlias") &&
mt.Id == (int)x.Attribute("nodeType"))))
.ToArray();
// MOCK!
var mediaServiceMock = new Mock<IMediaService>();
mediaServiceMock
.Setup(x => x.GetPagedDescendants(
It.IsAny<int>(), It.IsAny<long>(), It.IsAny<int>(), out totalRecs, It.IsAny<IQuery<IMedia>>(), It.IsAny<Ordering>())
).Returns(() => allRecs);
//mediaServiceMock.Setup(service => service.GetPagedXmlEntries(It.IsAny<string>(), It.IsAny<long>(), It.IsAny<int>(), out longTotalRecs))
// .Returns(() => allRecs.Select(x => x.ToXml()));
return mediaServiceMock.Object;
}
public static ILocalizationService GetMockLocalizationService()
{
return Mock.Of<ILocalizationService>(x => x.GetAllLanguages() == Array.Empty<ILanguage>());
}
public static IMediaTypeService GetMockMediaTypeService()
{
var mediaTypeServiceMock = new Mock<IMediaTypeService>();
mediaTypeServiceMock.Setup(x => x.GetAll())
.Returns(new List<IMediaType>
{
new MediaType(-1) {Alias = "Folder", Name = "Folder", Id = 1031, Icon = "icon-folder"},
new MediaType(-1) {Alias = "Image", Name = "Image", Id = 1032, Icon = "icon-picture"}
});
return mediaTypeServiceMock.Object;
}
public static UmbracoContentIndexer GetUmbracoIndexer(
ProfilingLogger profilingLogger,
Directory luceneDir,
ISqlContext sqlContext,
PropertyEditorCollection propertyEditors,
Analyzer analyzer = null,
IContentService contentService = null,
IMediaService mediaService = null,
IMemberService memberService = null,
IUserService userService = null,
ILocalizationService languageService = null,
IContentTypeService contentTypeService = null,
IMediaTypeService mediaTypeService = null,
UmbracoContentIndexerOptions options = null)
{
if (languageService == null)
{
languageService = Mock.Of<ILocalizationService>(
x => x.GetAllLanguages() == Array.Empty<ILanguage>());
}
if (contentService == null)
{
long longTotalRecs;
var demoData = new ExamineDemoDataContentService();
var allRecs = demoData.GetLatestContentByXPath("//*[@isDoc]")
.Root
.Elements()
.Select(x => Mock.Of<IContent>(
m =>
m.Id == (int)x.Attribute("id") &&
m.ParentId == (int)x.Attribute("parentID") &&
m.Level == (int)x.Attribute("level") &&
m.CreatorId == 0 &&
m.SortOrder == (int)x.Attribute("sortOrder") &&
m.CreateDate == (DateTime)x.Attribute("createDate") &&
m.UpdateDate == (DateTime)x.Attribute("updateDate") &&
m.Name == (string)x.Attribute("nodeName") &&
m.GetCultureName(It.IsAny<string>()) == (string)x.Attribute("nodeName") &&
m.Path == (string)x.Attribute("path") &&
m.Properties == new PropertyCollection() &&
m.ContentType == Mock.Of<IContentType>(mt =>
mt.Icon == "test" &&
mt.Alias == x.Name.LocalName &&
mt.Id == (int)x.Attribute("nodeType"))))
.ToArray();
contentService = Mock.Of<IContentService>(
x => x.GetPagedDescendants(
It.IsAny<int>(), It.IsAny<long>(), It.IsAny<int>(), out longTotalRecs, It.IsAny<IQuery<IContent>>(), It.IsAny<Ordering>())
==
allRecs);
}
if (userService == null)
{
userService = Mock.Of<IUserService>(x => x.GetProfileById(It.IsAny<int>()) == Mock.Of<IProfile>(p => p.Id == 0 && p.Name == "admin"));
}
if (mediaService == null)
{
long totalRecs;
var demoData = new ExamineDemoDataMediaService();
var allRecs = demoData.GetLatestMediaByXpath("//node")
.Root
.Elements()
.Select(x => Mock.Of<IMedia>(
m =>
m.Id == (int) x.Attribute("id") &&
m.ParentId == (int) x.Attribute("parentID") &&
m.Level == (int) x.Attribute("level") &&
m.CreatorId == 0 &&
m.SortOrder == (int) x.Attribute("sortOrder") &&
m.CreateDate == (DateTime) x.Attribute("createDate") &&
m.UpdateDate == (DateTime) x.Attribute("updateDate") &&
m.Name == (string) x.Attribute("nodeName") &&
m.GetCultureName(It.IsAny<string>()) == (string)x.Attribute("nodeName") &&
m.Path == (string) x.Attribute("path") &&
m.Properties == new PropertyCollection() &&
m.ContentType == Mock.Of<IMediaType>(mt =>
mt.Alias == (string) x.Attribute("nodeTypeAlias") &&
mt.Id == (int) x.Attribute("nodeType"))))
.ToArray();
// MOCK!
var mediaServiceMock = new Mock<IMediaService>();
mediaServiceMock
.Setup(x => x.GetPagedDescendants(
It.IsAny<int>(), It.IsAny<long>(), It.IsAny<int>(), out totalRecs, It.IsAny<IQuery<IMedia>>(), It.IsAny<Ordering>())
).Returns(() => allRecs);
//mediaServiceMock.Setup(service => service.GetPagedXmlEntries(It.IsAny<string>(), It.IsAny<long>(), It.IsAny<int>(), out longTotalRecs))
// .Returns(() => allRecs.Select(x => x.ToXml()));
mediaService = mediaServiceMock.Object;
}
languageService = GetMockLocalizationService();
if (analyzer == null)
{
analyzer = new StandardAnalyzer(Version.LUCENE_30);
}
//var indexSet = new IndexSet();
// var indexCriteria = indexSet.ToIndexCriteria(dataService, UmbracoContentIndexer.IndexFieldPolicies);
//var i = new UmbracoContentIndexer(indexCriteria,
// luceneDir, //custom lucene directory
// dataService,
// contentService,
// mediaService,
// dataTypeService,
// userService,
// new[] { new DefaultUrlSegmentProvider() },
// analyzer,
// false);
//i.IndexSecondsInterval = 1;
if (options == null)
{
options = new UmbracoContentIndexerOptions(false, false, null);
}
if (mediaTypeService == null)
{
var mediaTypeServiceMock = new Mock<IMediaTypeService>();
mediaTypeServiceMock.Setup(x => x.GetAll())
.Returns(new List<IMediaType>
{
new MediaType(-1) {Alias = "Folder", Name = "Folder", Id = 1031, Icon = "icon-folder"},
new MediaType(-1) {Alias = "Image", Name = "Image", Id = 1032, Icon = "icon-picture"}
});
mediaTypeService = mediaTypeServiceMock.Object;
}
// fixme oops?!
//var query = new Mock<IQuery<IContent>>();
//query
// .Setup(x => x.GetWhereClauses())
// .Returns(new List<Tuple<string, object[]>> { new Tuple<string, object[]>($"{Constants.DatabaseSchema.Tables.Document}.published", new object[] { 1 }) });
//scopeProvider
// .Setup(x => x.Query<IContent>())
// .Returns(query.Object);
var i = new UmbracoContentIndexer(
"testIndexer",
UmbracoExamineIndexer.UmbracoIndexFieldDefinitions,
luceneDir,
analyzer,
profilingLogger,
new ContentValueSetBuilder(propertyEditors, new[] { new DefaultUrlSegmentProvider() }, userService),
new MediaValueSetBuilder(propertyEditors, new[] { new DefaultUrlSegmentProvider() }, userService),
contentService,
mediaService,
languageService,
sqlContext,
new UmbracoContentValueSetValidator(options, Mock.Of<IPublicAccessService>()),
options);

50
src/Umbraco.Tests/UmbracoExamine/IndexTest.cs
View File

@@ -30,13 +30,14 @@ namespace Umbraco.Tests.UmbracoExamine
[Test]
public void Index_Property_Data_With_Value_Indexer()
{
var contentValueSetBuilder = IndexInitializer.GetContentValueSetBuilder(Container.GetInstance<PropertyEditorCollection>());
using (var luceneDir = new RandomIdRamDirectory())
using (var indexer = IndexInitializer.GetUmbracoIndexer(
ProfilingLogger, luceneDir, ScopeProvider.SqlContext, Container.GetInstance<PropertyEditorCollection>(),
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir,
options: new UmbracoContentIndexerOptions(true, false, null)))
using (indexer.ProcessNonAsync())
{
indexer.EnsureIndex(true);
indexer.CreateIndex();
var contentType = MockedContentTypes.CreateBasicContentType();
contentType.AddPropertyType(new PropertyType("test", ValueStorageType.Ntext)
@@ -99,8 +100,7 @@ namespace Umbraco.Tests.UmbracoExamine
var json = JsonConvert.SerializeObject(gridVal);
content.Properties["grid"].SetValue(json);
var valueSet = indexer.ContentValueSetBuilder.GetValueSets(content);
var valueSet = contentValueSetBuilder.GetValueSets(content);
indexer.IndexItems(valueSet);
var searcher = indexer.GetSearcher();
@@ -122,15 +122,19 @@ namespace Umbraco.Tests.UmbracoExamine
[Test]
public void Rebuild_Index()
{
var contentRebuilder = IndexInitializer.GetContentIndexRebuilder(Container.GetInstance<PropertyEditorCollection>(), IndexInitializer.GetMockContentService(), ScopeProvider.SqlContext);
var mediaRebuilder = IndexInitializer.GetMediaIndexRebuilder(Container.GetInstance<PropertyEditorCollection>(), IndexInitializer.GetMockMediaService());
using (var luceneDir = new RandomIdRamDirectory())
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, Container.GetInstance<PropertyEditorCollection>(),
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir,
options: new UmbracoContentIndexerOptions(true, false, null)))
using (indexer.ProcessNonAsync())
{
var searcher = indexer.GetSearcher();
//create the whole thing
indexer.RebuildIndex();
contentRebuilder.Populate(indexer);
mediaRebuilder.Populate(indexer);
var result = searcher.Search(searcher.CreateCriteria().All().Compile());
@@ -145,13 +149,17 @@ namespace Umbraco.Tests.UmbracoExamine
[Test]
public void Index_Protected_Content_Not_Indexed()
{
var rebuilder = IndexInitializer.GetContentIndexRebuilder(Container.GetInstance<PropertyEditorCollection>(), IndexInitializer.GetMockContentService(), ScopeProvider.SqlContext);
using (var luceneDir = new RandomIdRamDirectory())
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, Container.GetInstance<PropertyEditorCollection>()))
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir))
using (indexer.ProcessNonAsync())
using (var searcher = ((LuceneSearcher)indexer.GetSearcher()).GetLuceneSearcher())
{
//create the whole thing
indexer.RebuildIndex();
rebuilder.Populate(indexer);
var protectedQuery = new BooleanQuery();
protectedQuery.Add(
@@ -176,8 +184,9 @@ namespace Umbraco.Tests.UmbracoExamine
[Test]
public void Index_Move_Media_From_Non_Indexable_To_Indexable_ParentID()
{
using (var luceneDir = new RandomIdRamDirectory())
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, Container.GetInstance<PropertyEditorCollection>(),
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir,
//make parent id 1116
options: new UmbracoContentIndexerOptions(false, false, 1116)))
using (indexer.ProcessNonAsync())
@@ -219,7 +228,7 @@ namespace Umbraco.Tests.UmbracoExamine
public void Index_Move_Media_To_Non_Indexable_ParentID()
{
using (var luceneDir = new RandomIdRamDirectory())
using (var indexer1 = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, Container.GetInstance<PropertyEditorCollection>(),
using (var indexer1 = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir,
//make parent id 2222
options: new UmbracoContentIndexerOptions(false, false, 2222)))
using (indexer1.ProcessNonAsync())
@@ -268,16 +277,17 @@ namespace Umbraco.Tests.UmbracoExamine
[Test]
public void Index_Reindex_Content()
{
var rebuilder = IndexInitializer.GetContentIndexRebuilder(Container.GetInstance<PropertyEditorCollection>(), IndexInitializer.GetMockContentService(), ScopeProvider.SqlContext);
using (var luceneDir = new RandomIdRamDirectory())
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, Container.GetInstance<PropertyEditorCollection>(),
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir,
options: new UmbracoContentIndexerOptions(true, false, null)))
using (indexer.ProcessNonAsync())
{
var searcher = indexer.GetSearcher();
//create the whole thing
indexer.RebuildIndex();
rebuilder.Populate(indexer);
var result = searcher.Search(searcher.CreateCriteria().Field(LuceneIndexer.CategoryFieldName, IndexTypes.Content).Compile());
Assert.AreEqual(21, result.TotalItemCount);
@@ -294,9 +304,7 @@ namespace Umbraco.Tests.UmbracoExamine
Assert.AreEqual(0, result.TotalItemCount);
//call our indexing methods
indexer.IndexAll(IndexTypes.Content);
rebuilder.Populate(indexer);
result = searcher.Search(searcher.CreateCriteria().Field(LuceneIndexer.CategoryFieldName, IndexTypes.Content).Compile());
Assert.AreEqual(21, result.TotalItemCount);
@@ -309,15 +317,17 @@ namespace Umbraco.Tests.UmbracoExamine
[Test]
public void Index_Delete_Index_Item_Ensure_Heirarchy_Removed()
{
var rebuilder = IndexInitializer.GetContentIndexRebuilder(Container.GetInstance<PropertyEditorCollection>(), IndexInitializer.GetMockContentService(), ScopeProvider.SqlContext);
using (var luceneDir = new RandomIdRamDirectory())
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext, Container.GetInstance<PropertyEditorCollection>()))
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir))
using (indexer.ProcessNonAsync())
{
var searcher = indexer.GetSearcher();
//create the whole thing
indexer.RebuildIndex();
rebuilder.Populate(indexer);
//now delete a node that has children

13
src/Umbraco.Tests/UmbracoExamine/SearchTests.cs
View File

@@ -7,11 +7,13 @@ using NUnit.Framework;
using Examine.LuceneEngine.SearchCriteria;
using Moq;
using Umbraco.Core.Models;
using Umbraco.Core.Persistence;
using Umbraco.Core.Persistence.Querying;
using Umbraco.Core.Services;
using Umbraco.Examine;
using Umbraco.Tests.Testing;
using Umbraco.Core.PropertyEditors;
using Umbraco.Core.Strings;
namespace Umbraco.Tests.UmbracoExamine
{
@@ -53,15 +55,16 @@ namespace Umbraco.Tests.UmbracoExamine
==
allRecs);
var propertyEditors = Container.GetInstance<PropertyEditorCollection>();
var rebuilder = IndexInitializer.GetContentIndexRebuilder(propertyEditors, contentService, ScopeProvider.SqlContext);
using (var luceneDir = new RandomIdRamDirectory())
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, ScopeProvider.SqlContext,
Container.GetInstance<PropertyEditorCollection>(),
contentService: contentService))
using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir))
using (indexer.ProcessNonAsync())
{
indexer.RebuildIndex();
indexer.CreateIndex();
rebuilder.Populate(indexer);
var searcher = indexer.GetSearcher();
var numberSortedCriteria = searcher.CreateCriteria()

2
src/Umbraco.Web.UI/Umbraco.Web.UI.csproj
View File

@@ -88,7 +88,7 @@
<PackageReference Include="CSharpTest.Net.Collections" Version="14.906.1403.1082" />
<PackageReference Include="ClientDependency" Version="1.9.7" />
<PackageReference Include="ClientDependency-Mvc5" Version="1.8.0.0" />
<PackageReference Include="Examine" Version="1.0.0-beta032" />
<PackageReference Include="Examine" Version="1.0.0-beta034" />
<PackageReference Include="ImageProcessor.Web" Version="4.9.3.25" />
<PackageReference Include="ImageProcessor.Web.Config" Version="2.4.1.19" />
<PackageReference Include="Lucene.Net.Contrib" Version="3.0.3" />

9
src/Umbraco.Web/Components/DatabaseServerRegistrarAndMessengerComponent.cs
View File

@@ -11,6 +11,7 @@ using Umbraco.Core.Scoping;
using Umbraco.Core.Services;
using Umbraco.Core.Services.Changes;
using Umbraco.Core.Sync;
using Umbraco.Examine;
using Umbraco.Web.Cache;
using Umbraco.Web.Composing;
using Umbraco.Web.Routing;
@@ -48,7 +49,7 @@ namespace Umbraco.Web.Components
private BackgroundTaskRunner<IBackgroundTask> _processTaskRunner;
private bool _started;
private IBackgroundTask[] _tasks;
private IExamineManager _examineManager;
private IndexRebuilder _indexRebuilder;
public override void Compose(Composition composition)
{
@@ -87,13 +88,13 @@ namespace Umbraco.Web.Components
//rebuild indexes if the server is not synced
// NOTE: This will rebuild ALL indexes including the members, if developers want to target specific
// indexes then they can adjust this logic themselves.
() => ExamineComponent.RebuildIndexes(_examineManager, _logger, false, 5000)
() => ExamineComponent.RebuildIndexes(_indexRebuilder, _logger, false, 5000)
}
});
});
}
public void Initialize(IRuntimeState runtime, IServerRegistrar serverRegistrar, IServerMessenger serverMessenger, IServerRegistrationService registrationService, ILogger logger, IExamineManager examineManager)
public void Initialize(IRuntimeState runtime, IServerRegistrar serverRegistrar, IServerMessenger serverMessenger, IServerRegistrationService registrationService, ILogger logger, IndexRebuilder indexRebuilder)
{
_registrar = serverRegistrar as DatabaseServerRegistrar;
if (_registrar == null) throw new Exception("panic: registar.");
@@ -106,7 +107,7 @@ namespace Umbraco.Web.Components
_runtime = runtime;
_logger = logger;
_registrationService = registrationService;
_examineManager = examineManager;
_indexRebuilder = indexRebuilder;
_touchTaskRunner = new BackgroundTaskRunner<IBackgroundTask>("ServerRegistration",
new BackgroundTaskRunnerOptions { AutoStart = true }, logger);

4
src/Umbraco.Web/Editors/ExamineManagementController.cs
View File

@@ -165,7 +165,9 @@ namespace Umbraco.Web.Editors
try
{
indexer.RebuildIndex();
//TODO: Rebuilding isn't build directly into an index, we need a new IRebuildIndex or similar interface that can be registered
throw new NotImplementedException("Implement rebuilding!");
//indexer.RebuildIndex();
}
catch (Exception ex)
{

117
src/Umbraco.Web/Search/ExamineComponent.cs
View File

@@ -42,6 +42,8 @@ namespace Umbraco.Web.Search
private IValueSetBuilder<IContent> _contentValueSetBuilder;
private IValueSetBuilder<IMedia> _mediaValueSetBuilder;
private IValueSetBuilder<IMember> _memberValueSetBuilder;
private ContentIndexPopulator _contentIndexPopulator;
private MediaIndexPopulator _mediaIndexPopulator;
private static bool _disableExamineIndexing = false;
private static volatile bool _isConfigured = false;
private static readonly object IsConfiguredLocker = new object();
@@ -49,7 +51,6 @@ namespace Umbraco.Web.Search
private ServiceContext _services;
private static BackgroundTaskRunner<IBackgroundTask> _rebuildOnStartupRunner;
private static readonly object _rebuildLocker = new object();
private IEnumerable<IUrlSegmentProvider> _urlSegmentProviders;
// the default enlist priority is 100
// enlist with a lower priority to ensure that anything "default" runs after us
@@ -60,8 +61,11 @@ namespace Umbraco.Web.Search
{
base.Compose(composition);
composition.Container.RegisterSingleton<ContentIndexPopulator>();
composition.Container.RegisterSingleton<PublishedContentIndexPopulator>();
composition.Container.RegisterSingleton<MediaIndexPopulator>();
composition.Container.RegisterSingleton<IndexRebuilder>();
composition.Container.RegisterSingleton<IUmbracoIndexesBuilder, UmbracoIndexesBuilder>();
composition.Container.RegisterSingleton<ContentValueSetBuilder>();
composition.Container.RegisterSingleton<IValueSetBuilder<IContent>, ContentValueSetBuilder>();
composition.Container.RegisterSingleton<IValueSetBuilder<IMedia>, MediaValueSetBuilder>();
composition.Container.RegisterSingleton<IValueSetBuilder<IMember>, MemberValueSetBuilder>();
@@ -69,17 +73,19 @@ namespace Umbraco.Web.Search
internal void Initialize(IRuntimeState runtime, MainDom mainDom, PropertyEditorCollection propertyEditors,
IExamineManager examineManager, ProfilingLogger profilingLogger,
IScopeProvider scopeProvider, IUmbracoIndexesBuilder indexBuilder, ServiceContext services,
IEnumerable<IUrlSegmentProvider> urlSegmentProviders,
IScopeProvider scopeProvider, IUmbracoIndexesBuilder indexBuilder,
IndexRebuilder indexRebuilder, ServiceContext services,
IValueSetBuilder<IContent> contentValueSetBuilder,
IValueSetBuilder<IMedia> mediaValueSetBuilder,
IValueSetBuilder<IMember> memberValueSetBuilder)
IValueSetBuilder<IMember> memberValueSetBuilder,
ContentIndexPopulator contentIndexPopulator, PublishedContentIndexPopulator publishedContentIndexPopulator,
MediaIndexPopulator mediaIndexPopulator)
{
_services = services;
_scopeProvider = scopeProvider;
_examineManager = examineManager;
_urlSegmentProviders = urlSegmentProviders;
_contentIndexPopulator = contentIndexPopulator;
_mediaIndexPopulator = mediaIndexPopulator;
_contentValueSetBuilder = contentValueSetBuilder;
_mediaValueSetBuilder = mediaValueSetBuilder;
_memberValueSetBuilder = memberValueSetBuilder;
@@ -111,7 +117,7 @@ namespace Umbraco.Web.Search
profilingLogger.Logger.Debug<ExamineComponent>("Examine shutdown not registered, this appdomain is not the MainDom, Examine will be disabled");
//if we could not register the shutdown examine ourselves, it means we are not maindom! in this case all of examine should be disabled!
Suspendable.ExamineEvents.SuspendIndexers();
Suspendable.ExamineEvents.SuspendIndexers(profilingLogger.Logger);
_disableExamineIndexing = true;
return; //exit, do not continue
}
@@ -119,7 +125,7 @@ namespace Umbraco.Web.Search
//create the indexes and register them with the manager
foreach(var index in indexBuilder.Create())
{
_examineManager.AddIndexer(index.Key, index.Value);
_examineManager.AddIndexer(index);
}
profilingLogger.Logger.Debug<ExamineComponent>("Examine shutdown registered with MainDom");
@@ -141,7 +147,7 @@ namespace Umbraco.Web.Search
EnsureUnlocked(profilingLogger.Logger, examineManager);
RebuildIndexes(examineManager, profilingLogger.Logger, true, 5000);
RebuildIndexes(indexRebuilder, profilingLogger.Logger, true, 5000);
}
@@ -149,7 +155,7 @@ namespace Umbraco.Web.Search
/// Called to rebuild empty indexes on startup
/// </summary>
/// <param name="logger"></param>
public static void RebuildIndexes(IExamineManager examineManager, ILogger logger, bool onlyEmptyIndexes, int waitMilliseconds = 0)
public static void RebuildIndexes(IndexRebuilder indexRebuilder, ILogger logger, bool onlyEmptyIndexes, int waitMilliseconds = 0)
{
//TODO: need a way to disable rebuilding on startup
@@ -163,7 +169,7 @@ namespace Umbraco.Web.Search
logger.Info<ExamineComponent>("Starting initialize async background thread.");
//do the rebuild on a managed background thread
var task = new RebuildOnStartupTask(examineManager, logger, onlyEmptyIndexes, waitMilliseconds);
var task = new RebuildOnStartupTask(indexRebuilder, logger, onlyEmptyIndexes, waitMilliseconds);
_rebuildOnStartupRunner = new BackgroundTaskRunner<IBackgroundTask>(
"RebuildIndexesOnStartup",
@@ -695,14 +701,15 @@ namespace Umbraco.Web.Search
public static void Execute(ExamineComponent examineComponent, IContent content, bool? supportUnpublished)
{
var valueSet = examineComponent._contentValueSetBuilder.GetValueSets(content);
var valueSet = examineComponent._contentValueSetBuilder.GetValueSets(content).ToList();
examineComponent._examineManager.IndexItems(
valueSet.ToArray(),
examineComponent._examineManager.IndexProviders.Values.OfType<UmbracoContentIndexer>()
// only for the specified indexers
.Where(x => supportUnpublished.HasValue == false || supportUnpublished.Value == x.SupportUnpublishedContent)
.Where(x => x.EnableDefaultEventHandler));
foreach (var index in examineComponent._examineManager.IndexProviders.Values.OfType<UmbracoContentIndexer>()
// only for the specified indexers
.Where(x => supportUnpublished.HasValue == false || supportUnpublished.Value == x.SupportUnpublishedContent)
.Where(x => x.EnableDefaultEventHandler))
{
index.IndexItems(valueSet);
}
}
}
@@ -726,15 +733,16 @@ namespace Umbraco.Web.Search
public static void Execute(ExamineComponent examineComponent, IMedia media, bool isPublished)
{
var valueSet = examineComponent._mediaValueSetBuilder.GetValueSets(media);
var valueSet = examineComponent._mediaValueSetBuilder.GetValueSets(media).ToList();
examineComponent._examineManager.IndexItems(
valueSet.ToArray(),
examineComponent._examineManager.IndexProviders.Values.OfType<UmbracoContentIndexer>()
// index this item for all indexers if the media is not trashed, otherwise if the item is trashed
// then only index this for indexers supporting unpublished media
.Where(x => isPublished || (x.SupportUnpublishedContent))
.Where(x => x.EnableDefaultEventHandler));
foreach (var index in examineComponent._examineManager.IndexProviders.Values.OfType<UmbracoContentIndexer>()
// index this item for all indexers if the media is not trashed, otherwise if the item is trashed
// then only index this for indexers supporting unpublished media
.Where(x => isPublished || (x.SupportUnpublishedContent))
.Where(x => x.EnableDefaultEventHandler))
{
index.IndexItems(valueSet);
}
}
}
@@ -756,13 +764,13 @@ namespace Umbraco.Web.Search
public static void Execute(ExamineComponent examineComponent, IMember member)
{
var valueSet = examineComponent._memberValueSetBuilder.GetValueSets(member);
examineComponent._examineManager.IndexItems(
valueSet.ToArray(),
examineComponent._examineManager.IndexProviders.Values.OfType<IUmbracoIndexer>()
//ensure that only the providers are flagged to listen execute
.Where(x => x.EnableDefaultEventHandler));
var valueSet = examineComponent._memberValueSetBuilder.GetValueSets(member).ToList();
foreach (var index in examineComponent._examineManager.IndexProviders.Values.OfType<IUmbracoIndexer>()
//ensure that only the providers are flagged to listen execute
.Where(x => x.EnableDefaultEventHandler))
{
index.IndexItems(valueSet);
}
}
}
@@ -786,13 +794,15 @@ namespace Umbraco.Web.Search
public static void Execute(ExamineComponent examineComponent, int id, bool keepIfUnpublished)
{
examineComponent._examineManager.DeleteFromIndexes(
id.ToString(CultureInfo.InvariantCulture),
examineComponent._examineManager.IndexProviders.Values.OfType<IUmbracoIndexer>()
// if keepIfUnpublished == true then only delete this item from indexes not supporting unpublished content,
// otherwise if keepIfUnpublished == false then remove from all indexes
.Where(x => keepIfUnpublished == false || x.SupportUnpublishedContent == false)
.Where(x => x.EnableDefaultEventHandler));
var strId = id.ToString(CultureInfo.InvariantCulture);
foreach (var index in examineComponent._examineManager.IndexProviders.Values.OfType<IUmbracoIndexer>()
// if keepIfUnpublished == true then only delete this item from indexes not supporting unpublished content,
// otherwise if keepIfUnpublished == false then remove from all indexes
.Where(x => keepIfUnpublished == false || x.SupportUnpublishedContent == false)
.Where(x => x.EnableDefaultEventHandler))
{
index.DeleteFromIndex(strId);
}
}
}
#endregion
@@ -802,15 +812,15 @@ namespace Umbraco.Web.Search
/// </summary>
private class RebuildOnStartupTask : IBackgroundTask
{
private readonly IExamineManager _examineManager;
private readonly IndexRebuilder _indexRebuilder;
private readonly ILogger _logger;
private readonly bool _onlyEmptyIndexes;
private readonly int _waitMilliseconds;
public RebuildOnStartupTask(IExamineManager examineManager, ILogger logger, bool onlyEmptyIndexes, int waitMilliseconds = 0)
public RebuildOnStartupTask(IndexRebuilder indexRebuilder, ILogger logger, bool onlyEmptyIndexes, int waitMilliseconds = 0)
{
_examineManager = examineManager;
_logger = logger;
_indexRebuilder = indexRebuilder ?? throw new ArgumentNullException(nameof(indexRebuilder));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_onlyEmptyIndexes = onlyEmptyIndexes;
_waitMilliseconds = waitMilliseconds;
}
@@ -843,7 +853,6 @@ namespace Umbraco.Web.Search
/// <summary>
/// Used to rebuild indexes on startup or cold boot
/// </summary>
/// <param name="onlyEmptyIndexes"></param>
private void RebuildIndexes()
{
//do not attempt to do this if this has been disabled since we are not the main dom.
@@ -853,20 +862,8 @@ namespace Umbraco.Web.Search
if (_waitMilliseconds > 0)
Thread.Sleep(_waitMilliseconds);
EnsureUnlocked(_logger, _examineManager);
if (_onlyEmptyIndexes)
{
foreach (var indexer in _examineManager.IndexProviders.Values.Where(x => x.IsIndexNew()))
{
indexer.RebuildIndex();
}
}
else
{
//do all of them
_examineManager.RebuildIndexes();
}
EnsureUnlocked(_logger, _indexRebuilder.ExamineManager);
_indexRebuilder.RebuildIndexes(_onlyEmptyIndexes);
}
}
}

2
src/Umbraco.Web/Search/IUmbracoIndexesBuilder.cs
View File

@@ -5,6 +5,6 @@ namespace Umbraco.Web.Search
{
internal interface IUmbracoIndexesBuilder
{
IReadOnlyDictionary<string, IIndexer> Create();
IEnumerable<IIndexer> Create();
}
}

46
src/Umbraco.Web/Search/UmbracoIndexesBuilder.cs
View File

@@ -26,39 +26,33 @@ namespace Umbraco.Web.Search
//TODO: we should inject the different IValueSetValidator so devs can just register them instead of overriding this class?
public UmbracoIndexesBuilder(ProfilingLogger profilingLogger,
IValueSetBuilder<IContent> contentValueSetBuilder,
IValueSetBuilder<IMedia> mediaValueSetBuilder,
IValueSetBuilder<IMember> memberValueSetBuilder,
IContentService contentService,
IMediaService mediaService,
ContentIndexPopulator contentIndexPopulator,
PublishedContentIndexPopulator publishedContentIndexPopulator,
MediaIndexPopulator mediaIndexPopulator,
ILocalizationService languageService,
IPublicAccessService publicAccessService,
IMemberService memberService,
ISqlContext sqlContext)
IMemberService memberService)
{
ProfilingLogger = profilingLogger ?? throw new System.ArgumentNullException(nameof(profilingLogger));
ContentValueSetBuilder = contentValueSetBuilder ?? throw new System.ArgumentNullException(nameof(contentValueSetBuilder));
MediaValueSetBuilder = mediaValueSetBuilder ?? throw new System.ArgumentNullException(nameof(mediaValueSetBuilder));
MemberValueSetBuilder = memberValueSetBuilder ?? throw new System.ArgumentNullException(nameof(memberValueSetBuilder));
ContentService = contentService ?? throw new System.ArgumentNullException(nameof(contentService));
MediaService = mediaService ?? throw new System.ArgumentNullException(nameof(mediaService));
ContentIndexPopulator = contentIndexPopulator;
PublishedContentIndexPopulator = publishedContentIndexPopulator;
MediaIndexPopulator = mediaIndexPopulator;
LanguageService = languageService ?? throw new System.ArgumentNullException(nameof(languageService));
PublicAccessService = publicAccessService ?? throw new System.ArgumentNullException(nameof(publicAccessService));
MemberService = memberService ?? throw new System.ArgumentNullException(nameof(memberService));
SqlContext = sqlContext ?? throw new System.ArgumentNullException(nameof(sqlContext));
}
protected ProfilingLogger ProfilingLogger { get; }
protected IValueSetBuilder<IContent> ContentValueSetBuilder { get; }
protected IValueSetBuilder<IMedia> MediaValueSetBuilder { get; }
protected IValueSetBuilder<IMember> MemberValueSetBuilder { get; }
protected IContentService ContentService { get; }
protected IMediaService MediaService { get; }
protected ContentIndexPopulator ContentIndexPopulator { get; }
protected PublishedContentIndexPopulator PublishedContentIndexPopulator { get; }
protected MediaIndexPopulator MediaIndexPopulator { get; }
protected ILocalizationService LanguageService { get; }
protected IPublicAccessService PublicAccessService { get; }
protected IMemberService MemberService { get; }
protected ISqlContext SqlContext { get; }
public const string InternalIndexPath = "Internal";
public const string ExternalIndexPath = "External";
public const string MembersIndexPath = "Members";
@@ -72,27 +66,26 @@ namespace Umbraco.Web.Search
/// Creates the Umbraco indexes
/// </summary>
/// <returns></returns>
public IReadOnlyDictionary<string, IIndexer> Create()
public IEnumerable<IIndexer> Create()
{
return new Dictionary<string, IIndexer>
return new []
{
[InternalIndexPath] = CreateContentIndex(InternalIndexPath, new UmbracoContentIndexerOptions(true, true, null), new CultureInvariantWhitespaceAnalyzer()),
[ExternalIndexPath] = CreateContentIndex(ExternalIndexPath, new UmbracoContentIndexerOptions(false, false, null), new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30)),
[MembersIndexPath] = CreateMemberIndex()
CreateContentIndex(InternalIndexPath, new UmbracoContentIndexerOptions(true, true, null), new CultureInvariantWhitespaceAnalyzer()),
CreateContentIndex(ExternalIndexPath, new UmbracoContentIndexerOptions(false, false, null), new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30)),
CreateMemberIndex()
};
}
private IIndexer CreateContentIndex(string name, UmbracoContentIndexerOptions options, Analyzer analyzer)
{
var index = new UmbracoContentIndexer(
$"{name}Indexer",
//fixme - how to deal with languages like in UmbracoContentIndexer.CreateFieldValueTypes
UmbracoExamineIndexer.UmbracoIndexFieldDefinitions,
GetFileSystemLuceneDirectory(name),
analyzer,
ProfilingLogger, ContentValueSetBuilder, MediaValueSetBuilder,
ContentService, MediaService, LanguageService, SqlContext,
ProfilingLogger,
LanguageService,
GetContentValueSetValidator(options),
options);
return index;
@@ -100,14 +93,13 @@ namespace Umbraco.Web.Search
private IIndexer CreateMemberIndex()
{
var appData = Path.Combine(IOHelper.MapPath(SystemDirectories.Data), "TEMP", "ExamineIndexes", MembersIndexPath);
var index = new UmbracoMemberIndexer(
$"{MembersIndexPath}Indexer",
//fixme - how to deal with languages like in UmbracoContentIndexer.CreateFieldValueTypes
UmbracoExamineIndexer.UmbracoIndexFieldDefinitions,
GetFileSystemLuceneDirectory(MembersIndexPath),
new CultureInvariantWhitespaceAnalyzer(),
ProfilingLogger, MemberValueSetBuilder, MemberService,
ProfilingLogger,
GetMemberValueSetValidator());
return index;
}

12
src/Umbraco.Web/Suspendable.cs
View File

@@ -1,6 +1,7 @@
using System;
using Examine;
using Examine.Providers;
using Umbraco.Core.Logging;
using Umbraco.Core.Composing;
using Umbraco.Examine;
using Umbraco.Web.Cache;
@@ -63,24 +64,23 @@ namespace Umbraco.Web
}
}
public static void SuspendIndexers()
public static void SuspendIndexers(ILogger logger)
{
Current.ProfilingLogger.Logger.Info(typeof (ExamineEvents), "Suspend indexers.");
logger.Info(typeof (ExamineEvents), "Suspend indexers.");
_suspended = true;
}
public static void ResumeIndexers()
public static void ResumeIndexers(IndexRebuilder indexRebuilder, ILogger logger)
{
_suspended = false;
Current.ProfilingLogger.Logger.Info(typeof (ExamineEvents), "Resume indexers (rebuild:{Tried}).", _tried);
logger.Info(typeof (ExamineEvents), "Resume indexers (rebuild:{Tried}).", _tried);
if (_tried == false) return;
_tried = false;
//TODO: when resuming do we always want a full rebuild of all indexes?
// fixme - can we inject these somehow?
ExamineComponent.RebuildIndexes(ExamineManager.Instance, Current.Logger, false);
ExamineComponent.RebuildIndexes(indexRebuilder, logger, false);
}
}

2
src/Umbraco.Web/Umbraco.Web.csproj
View File

@@ -62,7 +62,7 @@
<PackageReference Include="AutoMapper" Version="7.0.1" />
<PackageReference Include="ClientDependency" Version="1.9.7" />
<PackageReference Include="CSharpTest.Net.Collections" Version="14.906.1403.1082" />
<PackageReference Include="Examine" Version="1.0.0-beta032" />
<PackageReference Include="Examine" Version="1.0.0-beta034" />
<PackageReference Include="HtmlAgilityPack" Version="1.8.9" />
<PackageReference Include="ImageProcessor">
<Version>2.6.2.25</Version>