yv01p/Umbraco-CMS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Uses IIndexPopulator to populate/rebuild indexes

Browse Source
This commit is contained in:
Shannon
2018-11-28 14:17:11 +11:00
parent 3fd0aef18e
commit e3548efed4
11 changed files with 105 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
src/Umbraco.Core/Constants-Indexes.cs Normal file
View File

@@ -0,0 +1,21 @@
using System;
using System.ComponentModel;
namespace Umbraco.Core
{
public static partial class Constants
{
public static class UmbracoIndexes
{
public const string InternalIndexName = InternalIndexPath + "Indexer";
public const string ExternalIndexName = ExternalIndexPath + "Indexer";
public const string MembersIndexName = MembersIndexPath + "Indexer";
public const string InternalIndexPath = "Internal";
public const string ExternalIndexPath = "External";
public const string MembersIndexPath = "Members";
}
}
}

1
src/Umbraco.Core/Umbraco.Core.csproj
View File

@@ -313,6 +313,7 @@
<Compile Include="Constants-PropertyTypeGroups.cs" />
<Compile Include="Constants-Security.cs" />
<Compile Include="Constants-System.cs" />
<Compile Include="Constants-Indexes.cs" />
<Compile Include="Constants-Web.cs" />
<Compile Include="Constants.cs" />
<Compile Include="ContentVariationExtensions.cs" />

16
src/Umbraco.Examine/ContentIndexPopulator.cs
View File

@@ -1,6 +1,8 @@
using System;
using System.Collections.Generic;
using System.Linq;
using Examine;
using Umbraco.Core;
using Umbraco.Core.Models;
using Umbraco.Core.Services;
using Umbraco.Core.Persistence;
@@ -13,7 +15,7 @@ namespace Umbraco.Examine
/// <summary>
/// Performs the data lookups required to rebuild a content index
/// </summary>
public class ContentIndexPopulator : IIndexPopulator
public class ContentIndexPopulator : IndexPopulator
{
private readonly IContentService _contentService;
private readonly IValueSetBuilder<IContent> _contentValueSetBuilder;
@@ -54,9 +56,12 @@ namespace Umbraco.Examine
_publishedQuery = sqlContext.Query<IContent>().Where(x => x.Published);
_supportUnpublishedContent = supportUnpublishedContent;
_parentId = parentId;
RegisterIndex(Constants.UmbracoIndexes.InternalIndexName);
RegisterIndex(Constants.UmbracoIndexes.ExternalIndexName);
}
public void Populate(params IIndexer[] indexes)
protected override void PopulateIndexes(IEnumerable<IIndexer> indexes)
{
const int pageSize = 10000;
var pageIndex = 0;
@@ -70,22 +75,21 @@ namespace Umbraco.Examine
do
{
long total;
if (_supportUnpublishedContent)
{
content = _contentService.GetPagedDescendants(contentParentId, pageIndex, pageSize, out total).ToArray();
content = _contentService.GetPagedDescendants(contentParentId, pageIndex, pageSize, out _).ToArray();
}
else
{
//add the published filter
//note: We will filter for published variants in the validator
content = _contentService.GetPagedDescendants(contentParentId, pageIndex, pageSize, out total,
content = _contentService.GetPagedDescendants(contentParentId, pageIndex, pageSize, out _,
_publishedQuery, Ordering.By("Path", Direction.Ascending)).ToArray();
}
if (content.Length > 0)
{
// ReSharper disable once PossibleMultipleEnumeration
foreach (var index in indexes)
index.IndexItems(_contentValueSetBuilder.GetValueSets(content));
}

9
src/Umbraco.Examine/IIndexPopulator.cs
View File

@@ -1,16 +1,15 @@
using Examine;
using System.Collections.Generic;
using Examine;
namespace Umbraco.Examine
{
/// <summary>
/// Populates indexes with data
/// </summary>
public interface IIndexPopulator
{
/// <summary>
/// Populates indexes with data
/// Populate indexers
/// </summary>
/// <param name="indexes"></param>
void Populate(params IIndexer[] indexes);
}
}

32
src/Umbraco.Examine/IndexPopulator.cs Normal file
View File

@@ -0,0 +1,32 @@
using System.Collections.Generic;
using System.Linq;
using Examine;
namespace Umbraco.Examine
{
public abstract class IndexPopulator : IIndexPopulator
{
private readonly HashSet<string> _registeredIndexes = new HashSet<string>();
/// <summary>
/// Registers an index for this populator
/// </summary>
/// <param name="indexName"></param>
public void RegisterIndex(string indexName)
{
_registeredIndexes.Add(indexName);
}
/// <summary>
/// Returns a list of index names that his populate is associated with
/// </summary>
public IEnumerable<string> RegisteredIndexes => _registeredIndexes;
public void Populate(params IIndexer[] indexes)
{
PopulateIndexes(indexes.Where(x => RegisteredIndexes.Contains(x.Name)));
}
protected abstract void PopulateIndexes(IEnumerable<IIndexer> indexes);
}
}

30
src/Umbraco.Examine/IndexRebuilder.cs
View File

@@ -1,4 +1,5 @@
using System.Linq;
using System.Collections.Generic;
using System.Linq;
using Examine;
namespace Umbraco.Examine
@@ -8,41 +9,28 @@ namespace Umbraco.Examine
/// </summary>
public class IndexRebuilder
{
private readonly IEnumerable<IIndexPopulator> _populators;
public IExamineManager ExamineManager { get; }
private readonly ContentIndexPopulator _contentIndexPopulator;
private readonly MediaIndexPopulator _mediaIndexPopulator;
public IndexRebuilder(IExamineManager examineManager, ContentIndexPopulator contentIndexPopulator, MediaIndexPopulator mediaIndexPopulator)
public IndexRebuilder(IExamineManager examineManager, IEnumerable<IIndexPopulator> populators)
{
_populators = populators;
ExamineManager = examineManager;
_contentIndexPopulator = contentIndexPopulator;
_mediaIndexPopulator = mediaIndexPopulator;
}
public void RebuildIndexes(bool onlyEmptyIndexes)
{
var indexes = (onlyEmptyIndexes
? ExamineManager.IndexProviders.Values.Where(x => x.IndexExists())
: ExamineManager.IndexProviders.Values).ToList();
var contentIndexes = indexes.Where(x => x is IUmbracoContentIndexer).ToArray();
var mediaIndexes = indexes.Where(x => x is IUmbracoContentIndexer).ToArray();
var nonUmbracoIndexes = indexes.Except(contentIndexes).Except(mediaIndexes).ToArray();
: ExamineManager.IndexProviders.Values).ToArray();
foreach(var index in indexes)
index.CreateIndex(); // clear the index
//reindex all content/media indexes with the same data source/lookup
_contentIndexPopulator.Populate(contentIndexes);
_mediaIndexPopulator.Populate(mediaIndexes);
//then do the rest
foreach (var index in nonUmbracoIndexes)
foreach (var populator in _populators)
{
index.CreateIndex();
//TODO: How to rebuild?
}
populator.Populate(indexes);
}
}
}
}

13
src/Umbraco.Examine/MediaIndexPopulator.cs
View File

@@ -1,5 +1,7 @@
using System.Linq;
using System.Collections.Generic;
using System.Linq;
using Examine;
using Umbraco.Core;
using Umbraco.Core.Models;
using Umbraco.Core.Services;
@@ -8,7 +10,7 @@ namespace Umbraco.Examine
/// <summary>
/// Performs the data lookups required to rebuild a media index
/// </summary>
public class MediaIndexPopulator : IIndexPopulator
public class MediaIndexPopulator : IndexPopulator
{
private readonly int? _parentId;
private readonly IMediaService _mediaService;
@@ -35,9 +37,12 @@ namespace Umbraco.Examine
_parentId = parentId;
_mediaService = mediaService;
_mediaValueSetBuilder = mediaValueSetBuilder;
RegisterIndex(Constants.UmbracoIndexes.InternalIndexName);
RegisterIndex(Constants.UmbracoIndexes.ExternalIndexName);
}
public void Populate(params IIndexer[] indexes)
protected override void PopulateIndexes(IEnumerable<IIndexer> indexes)
{
const int pageSize = 10000;
var pageIndex = 0;
@@ -57,6 +62,7 @@ namespace Umbraco.Examine
if (media.Length > 0)
{
// ReSharper disable once PossibleMultipleEnumeration
foreach (var index in indexes)
index.IndexItems(_mediaValueSetBuilder.GetValueSets(media));
}
@@ -64,5 +70,6 @@ namespace Umbraco.Examine
pageIndex++;
} while (media.Length == pageSize);
}
}
}

15
src/Umbraco.Examine/MemberIndexPopulator.cs
View File

@@ -1,11 +1,13 @@
using System.Linq;
using System.Collections.Generic;
using System.Linq;
using Examine;
using Lucene.Net.Util;
using Umbraco.Core.Models;
using Umbraco.Core.Services;
namespace Umbraco.Examine
{
public class MemberIndexPopulator: IIndexPopulator
public class MemberIndexPopulator : IndexPopulator
{
private readonly IMemberService _memberService;
private readonly IValueSetBuilder<IMember> _valueSetBuilder;
@@ -14,8 +16,10 @@ namespace Umbraco.Examine
{
_memberService = memberService;
_valueSetBuilder = valueSetBuilder;
RegisterIndex(Core.Constants.UmbracoIndexes.MembersIndexName);
}
public void Populate(params IIndexer[] indexes)
protected override void PopulateIndexes(IEnumerable<IIndexer> indexes)
{
const int pageSize = 1000;
var pageIndex = 0;
@@ -31,10 +35,11 @@ namespace Umbraco.Examine
if (members.Length > 0)
{
// ReSharper disable once PossibleMultipleEnumeration
foreach (var index in indexes)
index.IndexItems(_valueSetBuilder.GetValueSets(members));
}
}
pageIndex++;
} while (members.Length == pageSize);
}

1
src/Umbraco.Examine/Umbraco.Examine.csproj
View File

@@ -64,6 +64,7 @@
<Compile Include="ContentIndexPopulator.cs" />
<Compile Include="ContentValueSetBuilder.cs" />
<Compile Include="IIndexPopulator.cs" />
<Compile Include="IndexPopulator.cs" />
<Compile Include="IndexRebuilder.cs" />
<Compile Include="IUmbracoContentIndexer.cs" />
<Compile Include="IUmbracoIndexer.cs" />

1
src/Umbraco.Web/Search/ExamineComponent.cs
View File

@@ -829,7 +829,6 @@ namespace Umbraco.Web.Search
public void Dispose()
{
throw new NotImplementedException();
}
public void Run()

30
src/Umbraco.Web/Search/UmbracoIndexesBuilder.cs
View File

@@ -26,36 +26,20 @@ namespace Umbraco.Web.Search
//TODO: we should inject the different IValueSetValidator so devs can just register them instead of overriding this class?
public UmbracoIndexesBuilder(ProfilingLogger profilingLogger,
IValueSetBuilder<IMember> memberValueSetBuilder,
ContentIndexPopulator contentIndexPopulator,
PublishedContentIndexPopulator publishedContentIndexPopulator,
MediaIndexPopulator mediaIndexPopulator,
ILocalizationService languageService,
IPublicAccessService publicAccessService,
IMemberService memberService)
{
ProfilingLogger = profilingLogger ?? throw new System.ArgumentNullException(nameof(profilingLogger));
MemberValueSetBuilder = memberValueSetBuilder ?? throw new System.ArgumentNullException(nameof(memberValueSetBuilder));
ContentIndexPopulator = contentIndexPopulator;
PublishedContentIndexPopulator = publishedContentIndexPopulator;
MediaIndexPopulator = mediaIndexPopulator;
LanguageService = languageService ?? throw new System.ArgumentNullException(nameof(languageService));
PublicAccessService = publicAccessService ?? throw new System.ArgumentNullException(nameof(publicAccessService));
MemberService = memberService ?? throw new System.ArgumentNullException(nameof(memberService));
}
protected ProfilingLogger ProfilingLogger { get; }
protected IValueSetBuilder<IMember> MemberValueSetBuilder { get; }
protected ContentIndexPopulator ContentIndexPopulator { get; }
protected PublishedContentIndexPopulator PublishedContentIndexPopulator { get; }
protected MediaIndexPopulator MediaIndexPopulator { get; }
protected ILocalizationService LanguageService { get; }
protected IPublicAccessService PublicAccessService { get; }
protected IMemberService MemberService { get; }
public const string InternalIndexPath = "Internal";
public const string ExternalIndexPath = "External";
public const string MembersIndexPath = "Members";
/// <summary>
/// By default these are the member fields we index
@@ -70,19 +54,19 @@ namespace Umbraco.Web.Search
{
return new []
{
CreateContentIndex(InternalIndexPath, new UmbracoContentIndexerOptions(true, true, null), new CultureInvariantWhitespaceAnalyzer()),
CreateContentIndex(ExternalIndexPath, new UmbracoContentIndexerOptions(false, false, null), new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30)),
CreateContentIndex(Constants.UmbracoIndexes.InternalIndexName, Constants.UmbracoIndexes.InternalIndexPath, new UmbracoContentIndexerOptions(true, true, null), new CultureInvariantWhitespaceAnalyzer()),
CreateContentIndex(Constants.UmbracoIndexes.ExternalIndexName, Constants.UmbracoIndexes.ExternalIndexPath, new UmbracoContentIndexerOptions(false, false, null), new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30)),
CreateMemberIndex()
};
}
private IIndexer CreateContentIndex(string name, UmbracoContentIndexerOptions options, Analyzer analyzer)
private IIndexer CreateContentIndex(string name, string path, UmbracoContentIndexerOptions options, Analyzer analyzer)
{
var index = new UmbracoContentIndexer(
$"{name}Indexer",
name,
//fixme - how to deal with languages like in UmbracoContentIndexer.CreateFieldValueTypes
UmbracoExamineIndexer.UmbracoIndexFieldDefinitions,
GetFileSystemLuceneDirectory(name),
GetFileSystemLuceneDirectory(path),
analyzer,
ProfilingLogger,
LanguageService,
@@ -94,10 +78,10 @@ namespace Umbraco.Web.Search
private IIndexer CreateMemberIndex()
{
var index = new UmbracoMemberIndexer(
$"{MembersIndexPath}Indexer",
Constants.UmbracoIndexes.MembersIndexName,
//fixme - how to deal with languages like in UmbracoContentIndexer.CreateFieldValueTypes
UmbracoExamineIndexer.UmbracoIndexFieldDefinitions,
GetFileSystemLuceneDirectory(MembersIndexPath),
GetFileSystemLuceneDirectory(Constants.UmbracoIndexes.MembersIndexPath),
new CultureInvariantWhitespaceAnalyzer(),
ProfilingLogger,
GetMemberValueSetValidator());