Files
Umbraco-CMS/tests/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/IndexInitializer.cs

266 lines
11 KiB
C#
Raw Normal View History

2018-06-29 19:52:40 +02:00
using Examine;
Examine 2.0 integration (#10241) * Init commit for examine 2.0 work, most old umb examine tests working, probably a lot that doesn't * Gets Umbraco Examine tests passing and makes some sense out of them, fixes some underlying issues. * Large refactor, remove TaskHelper, rename Notifications to be consistent, Gets all examine/lucene indexes building and startup ordered in the correct way, removes old files, creates new IUmbracoIndexingHandler for abstracting out all index operations for umbraco data, abstracts out IIndexRebuilder, Fixes Stack overflow with LiveModelsProvider and loading assemblies, ports some changes from v8 for startup handling with cold boots, refactors out LastSyncedFileManager * fix up issues with rebuilding and management dashboard. * removes old files, removes NetworkHelper, fixes LastSyncedFileManager implementation to ensure the machine name is used, fix up logging with cold boot state. * Makes MainDom safer to use and makes PublishedSnapshotService lazily register with MainDom * lazily acquire application id (fix unit tests) * Fixes resource casing and missing test file * Ensures caches when requiring internal services for PublishedSnapshotService, UseNuCache is a separate call, shouldn't be buried in AddWebComponents, was also causing issues in integration tests since nucache was being used for the Id2Key service. * For UmbracoTestServerTestBase enable nucache services * Fixing tests * Fix another test * Fixes tests, use TestHostingEnvironment, make Tests.Common use net5, remove old Lucene.Net.Contrib ref. * Fixes up some review notes * Fixes issue with doubly registering PublishedSnapshotService meanig there could be 2x instances of it * Checks for parseexception when executing the query * Use application root instead of duplicating functionality. * Added Examine project to netcore only solution file * Fixed casing issue with LazyLoad, that is not lowercase. * uses cancellationToken instead of bool flag, fixes always reading lastId from the LastSyncedFileManager, fixes RecurringHostedServiceBase so that there isn't an overlapping thread for the same task type * Fix tests * remove legacy test project from solution file * Fix test Co-authored-by: Bjarke Berg <mail@bergmania.dk>
2021-05-18 18:31:38 +10:00
using Examine.Lucene;
using Examine.Lucene.Directories;
2018-06-29 19:52:40 +02:00
using Lucene.Net.Analysis;
using Lucene.Net.Analysis.Standard;
2020-09-15 15:14:44 +02:00
using Microsoft.Extensions.Logging;
Examine 2.0 integration (#10241) * Init commit for examine 2.0 work, most old umb examine tests working, probably a lot that doesn't * Gets Umbraco Examine tests passing and makes some sense out of them, fixes some underlying issues. * Large refactor, remove TaskHelper, rename Notifications to be consistent, Gets all examine/lucene indexes building and startup ordered in the correct way, removes old files, creates new IUmbracoIndexingHandler for abstracting out all index operations for umbraco data, abstracts out IIndexRebuilder, Fixes Stack overflow with LiveModelsProvider and loading assemblies, ports some changes from v8 for startup handling with cold boots, refactors out LastSyncedFileManager * fix up issues with rebuilding and management dashboard. * removes old files, removes NetworkHelper, fixes LastSyncedFileManager implementation to ensure the machine name is used, fix up logging with cold boot state. * Makes MainDom safer to use and makes PublishedSnapshotService lazily register with MainDom * lazily acquire application id (fix unit tests) * Fixes resource casing and missing test file * Ensures caches when requiring internal services for PublishedSnapshotService, UseNuCache is a separate call, shouldn't be buried in AddWebComponents, was also causing issues in integration tests since nucache was being used for the Id2Key service. * For UmbracoTestServerTestBase enable nucache services * Fixing tests * Fix another test * Fixes tests, use TestHostingEnvironment, make Tests.Common use net5, remove old Lucene.Net.Contrib ref. * Fixes up some review notes * Fixes issue with doubly registering PublishedSnapshotService meanig there could be 2x instances of it * Checks for parseexception when executing the query * Use application root instead of duplicating functionality. * Added Examine project to netcore only solution file * Fixed casing issue with LazyLoad, that is not lowercase. * uses cancellationToken instead of bool flag, fixes always reading lastId from the LastSyncedFileManager, fixes RecurringHostedServiceBase so that there isn't an overlapping thread for the same task type * Fix tests * remove legacy test project from solution file * Fix test Co-authored-by: Bjarke Berg <mail@bergmania.dk>
2021-05-18 18:31:38 +10:00
using Microsoft.Extensions.Options;
2018-06-29 19:52:40 +02:00
using Moq;
using Umbraco.Cms.Core.Configuration.Models;
using Umbraco.Cms.Core.Hosting;
using Umbraco.Cms.Core.Logging;
using Umbraco.Cms.Core.Models;
using Umbraco.Cms.Core.Models.Membership;
using Umbraco.Cms.Core.Persistence.Querying;
using Umbraco.Cms.Core.PropertyEditors;
using Umbraco.Cms.Core.Scoping;
using Umbraco.Cms.Core.Services;
using Umbraco.Cms.Core.Strings;
2021-02-12 10:57:50 +01:00
using Umbraco.Cms.Infrastructure.Examine;
using Umbraco.Cms.Infrastructure.Persistence;
using Directory = Lucene.Net.Store.Directory;
2018-06-29 19:52:40 +02:00
namespace Umbraco.Cms.Tests.Integration.Umbraco.Examine.Lucene.UmbracoExamine;
/// <summary>
/// Used internally by test classes to initialize a new index from the template
/// </summary>
public class IndexInitializer
2018-06-29 19:52:40 +02:00
{
private readonly IOptions<ContentSettings> _contentSettings;
private readonly ILoggerFactory _loggerFactory;
private readonly MediaUrlGeneratorCollection _mediaUrlGenerators;
private readonly PropertyEditorCollection _propertyEditors;
private readonly IScopeProvider _scopeProvider;
private readonly IShortStringHelper _shortStringHelper;
public IndexInitializer(
IShortStringHelper shortStringHelper,
PropertyEditorCollection propertyEditors,
MediaUrlGeneratorCollection mediaUrlGenerators,
IScopeProvider scopeProvider,
ILoggerFactory loggerFactory,
IOptions<ContentSettings> contentSettings)
2018-06-29 19:52:40 +02:00
{
_shortStringHelper = shortStringHelper;
_propertyEditors = propertyEditors;
_mediaUrlGenerators = mediaUrlGenerators;
_scopeProvider = scopeProvider;
_loggerFactory = loggerFactory;
_contentSettings = contentSettings;
}
Examine 2.0 integration (#10241) * Init commit for examine 2.0 work, most old umb examine tests working, probably a lot that doesn't * Gets Umbraco Examine tests passing and makes some sense out of them, fixes some underlying issues. * Large refactor, remove TaskHelper, rename Notifications to be consistent, Gets all examine/lucene indexes building and startup ordered in the correct way, removes old files, creates new IUmbracoIndexingHandler for abstracting out all index operations for umbraco data, abstracts out IIndexRebuilder, Fixes Stack overflow with LiveModelsProvider and loading assemblies, ports some changes from v8 for startup handling with cold boots, refactors out LastSyncedFileManager * fix up issues with rebuilding and management dashboard. * removes old files, removes NetworkHelper, fixes LastSyncedFileManager implementation to ensure the machine name is used, fix up logging with cold boot state. * Makes MainDom safer to use and makes PublishedSnapshotService lazily register with MainDom * lazily acquire application id (fix unit tests) * Fixes resource casing and missing test file * Ensures caches when requiring internal services for PublishedSnapshotService, UseNuCache is a separate call, shouldn't be buried in AddWebComponents, was also causing issues in integration tests since nucache was being used for the Id2Key service. * For UmbracoTestServerTestBase enable nucache services * Fixing tests * Fix another test * Fixes tests, use TestHostingEnvironment, make Tests.Common use net5, remove old Lucene.Net.Contrib ref. * Fixes up some review notes * Fixes issue with doubly registering PublishedSnapshotService meanig there could be 2x instances of it * Checks for parseexception when executing the query * Use application root instead of duplicating functionality. * Added Examine project to netcore only solution file * Fixed casing issue with LazyLoad, that is not lowercase. * uses cancellationToken instead of bool flag, fixes always reading lastId from the LastSyncedFileManager, fixes RecurringHostedServiceBase so that there isn't an overlapping thread for the same task type * Fix tests * remove legacy test project from solution file * Fix test Co-authored-by: Bjarke Berg <mail@bergmania.dk>
2021-05-18 18:31:38 +10:00
public ContentValueSetBuilder GetContentValueSetBuilder(bool publishedValuesOnly)
{
var contentValueSetBuilder = new ContentValueSetBuilder(
_propertyEditors,
new UrlSegmentProviderCollection(() => new[] { new DefaultUrlSegmentProvider(_shortStringHelper) }),
GetMockUserService(),
_shortStringHelper,
_scopeProvider,
publishedValuesOnly);
return contentValueSetBuilder;
}
public ContentIndexPopulator GetContentIndexRebuilder(IContentService contentService, bool publishedValuesOnly, IUmbracoDatabaseFactory umbracoDatabaseFactory)
{
var contentValueSetBuilder = GetContentValueSetBuilder(publishedValuesOnly);
var contentIndexDataSource = new ContentIndexPopulator(
_loggerFactory.CreateLogger<ContentIndexPopulator>(),
publishedValuesOnly,
null,
contentService,
umbracoDatabaseFactory,
contentValueSetBuilder);
return contentIndexDataSource;
}
public MediaIndexPopulator GetMediaIndexRebuilder(IMediaService mediaService)
{
var mediaValueSetBuilder = new MediaValueSetBuilder(
_propertyEditors,
new UrlSegmentProviderCollection(() => new[] { new DefaultUrlSegmentProvider(_shortStringHelper) }),
_mediaUrlGenerators,
GetMockUserService(),
_shortStringHelper,
_contentSettings);
var mediaIndexDataSource = new MediaIndexPopulator(null, mediaService, mediaValueSetBuilder);
return mediaIndexDataSource;
}
Examine 2.0 integration (#10241) * Init commit for examine 2.0 work, most old umb examine tests working, probably a lot that doesn't * Gets Umbraco Examine tests passing and makes some sense out of them, fixes some underlying issues. * Large refactor, remove TaskHelper, rename Notifications to be consistent, Gets all examine/lucene indexes building and startup ordered in the correct way, removes old files, creates new IUmbracoIndexingHandler for abstracting out all index operations for umbraco data, abstracts out IIndexRebuilder, Fixes Stack overflow with LiveModelsProvider and loading assemblies, ports some changes from v8 for startup handling with cold boots, refactors out LastSyncedFileManager * fix up issues with rebuilding and management dashboard. * removes old files, removes NetworkHelper, fixes LastSyncedFileManager implementation to ensure the machine name is used, fix up logging with cold boot state. * Makes MainDom safer to use and makes PublishedSnapshotService lazily register with MainDom * lazily acquire application id (fix unit tests) * Fixes resource casing and missing test file * Ensures caches when requiring internal services for PublishedSnapshotService, UseNuCache is a separate call, shouldn't be buried in AddWebComponents, was also causing issues in integration tests since nucache was being used for the Id2Key service. * For UmbracoTestServerTestBase enable nucache services * Fixing tests * Fix another test * Fixes tests, use TestHostingEnvironment, make Tests.Common use net5, remove old Lucene.Net.Contrib ref. * Fixes up some review notes * Fixes issue with doubly registering PublishedSnapshotService meanig there could be 2x instances of it * Checks for parseexception when executing the query * Use application root instead of duplicating functionality. * Added Examine project to netcore only solution file * Fixed casing issue with LazyLoad, that is not lowercase. * uses cancellationToken instead of bool flag, fixes always reading lastId from the LastSyncedFileManager, fixes RecurringHostedServiceBase so that there isn't an overlapping thread for the same task type * Fix tests * remove legacy test project from solution file * Fix test Co-authored-by: Bjarke Berg <mail@bergmania.dk>
2021-05-18 18:31:38 +10:00
public static IContentService GetMockContentService()
{
long longTotalRecs;
var demoData = new ExamineDemoDataContentService();
var allRecs = demoData.GetLatestContentByXPath("//*[@isDoc]")
.Root
.Elements()
.Select((xmlElement, index) => Mock.Of<IContent>(
m =>
m.Id == (int)xmlElement.Attribute("id") &&
// have every second one published and include the special one
m.Published == (ExamineDemoDataContentService.ProtectedNode == (int)xmlElement.Attribute("id") ||
(index % 2 == 0 ? true : false)) &&
m.ParentId == (int)xmlElement.Attribute("parentID") &&
m.Level == (int)xmlElement.Attribute("level") &&
m.CreatorId == 0 &&
m.SortOrder == (int)xmlElement.Attribute("sortOrder") &&
m.CreateDate == (DateTime)xmlElement.Attribute("createDate") &&
m.UpdateDate == (DateTime)xmlElement.Attribute("updateDate") &&
m.Name == (string)xmlElement.Attribute(UmbracoExamineFieldNames.NodeNameFieldName) &&
m.GetCultureName(It.IsAny<string>()) ==
(string)xmlElement.Attribute(UmbracoExamineFieldNames.NodeNameFieldName) &&
m.Path == (string)xmlElement.Attribute("path") &&
m.Properties == new PropertyCollection() &&
m.ContentType == Mock.Of<ISimpleContentType>(mt =>
mt.Icon == "test" &&
mt.Alias == xmlElement.Name.LocalName &&
mt.Id == (int)xmlElement.Attribute("nodeType"))))
.ToArray();
return Mock.Of<IContentService>(
x => x.GetPagedDescendants(It.IsAny<int>(), It.IsAny<long>(), It.IsAny<int>(), out longTotalRecs, It.IsAny<IQuery<IContent>>(), It.IsAny<Ordering>())
== allRecs);
}
public IUserService GetMockUserService() => Mock.Of<IUserService>(x =>
x.GetProfileById(It.IsAny<int>()) == Mock.Of<IProfile>(p => p.Id == 0 && p.Name == "admin"));
public static IMediaService GetMockMediaService()
{
long totalRecs;
var demoData = new ExamineDemoDataMediaService();
var allRecs = demoData.GetLatestMediaByXpath("//node")
.Root
.Elements()
.Select(x => Mock.Of<IMedia>(
m =>
m.Id == (int)x.Attribute("id") &&
m.ParentId == (int)x.Attribute("parentID") &&
m.Level == (int)x.Attribute("level") &&
m.CreatorId == 0 &&
m.SortOrder == (int)x.Attribute("sortOrder") &&
m.CreateDate == (DateTime)x.Attribute("createDate") &&
m.UpdateDate == (DateTime)x.Attribute("updateDate") &&
m.Name == (string)x.Attribute(UmbracoExamineFieldNames.NodeNameFieldName) &&
m.GetCultureName(It.IsAny<string>()) ==
(string)x.Attribute(UmbracoExamineFieldNames.NodeNameFieldName) &&
m.Path == (string)x.Attribute("path") &&
m.Properties == new PropertyCollection() &&
m.ContentType == Mock.Of<ISimpleContentType>(mt =>
mt.Alias == (string)x.Attribute("nodeTypeAlias") &&
mt.Id == (int)x.Attribute("nodeType"))))
.ToArray();
// MOCK!
var mediaServiceMock = new Mock<IMediaService>();
mediaServiceMock
.Setup(x => x.GetPagedDescendants(
It.IsAny<int>(),
It.IsAny<long>(),
It.IsAny<int>(),
out totalRecs,
It.IsAny<IQuery<IMedia>>(),
It.IsAny<Ordering>()))
.Returns(() => allRecs);
//mediaServiceMock.Setup(service => service.GetPagedXmlEntries(It.IsAny<string>(), It.IsAny<long>(), It.IsAny<int>(), out longTotalRecs))
// .Returns(() => allRecs.Select(x => x.ToXml()));
return mediaServiceMock.Object;
}
public ILocalizationService GetMockLocalizationService() =>
Mock.Of<ILocalizationService>(x => x.GetAllLanguages() == Array.Empty<ILanguage>());
public static IMediaTypeService GetMockMediaTypeService(IShortStringHelper shortStringHelper)
{
var mediaTypeServiceMock = new Mock<IMediaTypeService>();
mediaTypeServiceMock.Setup(x => x.GetAll())
.Returns(new List<IMediaType>
{
new MediaType(shortStringHelper, -1)
{
Alias = "Folder", Name = "Folder", Id = 1031, Icon = "icon-folder"
},
new MediaType(shortStringHelper, -1)
{
Alias = "Image", Name = "Image", Id = 1032, Icon = "icon-picture"
}
});
return mediaTypeServiceMock.Object;
}
public IProfilingLogger GetMockProfilingLogger() =>
new ProfilingLogger(Mock.Of<ILogger<ProfilingLogger>>(), Mock.Of<IProfiler>());
public UmbracoContentIndex GetUmbracoIndexer(
IHostingEnvironment hostingEnvironment,
IRuntimeState runtimeState,
Directory luceneDir,
Analyzer analyzer = null,
ILocalizationService languageService = null,
IContentValueSetValidator validator = null)
{
if (languageService == null)
2018-06-29 19:52:40 +02:00
{
languageService = GetMockLocalizationService();
}
if (analyzer == null)
{
analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion);
2018-06-29 19:52:40 +02:00
}
if (validator == null)
Examine 2.0 integration (#10241) * Init commit for examine 2.0 work, most old umb examine tests working, probably a lot that doesn't * Gets Umbraco Examine tests passing and makes some sense out of them, fixes some underlying issues. * Large refactor, remove TaskHelper, rename Notifications to be consistent, Gets all examine/lucene indexes building and startup ordered in the correct way, removes old files, creates new IUmbracoIndexingHandler for abstracting out all index operations for umbraco data, abstracts out IIndexRebuilder, Fixes Stack overflow with LiveModelsProvider and loading assemblies, ports some changes from v8 for startup handling with cold boots, refactors out LastSyncedFileManager * fix up issues with rebuilding and management dashboard. * removes old files, removes NetworkHelper, fixes LastSyncedFileManager implementation to ensure the machine name is used, fix up logging with cold boot state. * Makes MainDom safer to use and makes PublishedSnapshotService lazily register with MainDom * lazily acquire application id (fix unit tests) * Fixes resource casing and missing test file * Ensures caches when requiring internal services for PublishedSnapshotService, UseNuCache is a separate call, shouldn't be buried in AddWebComponents, was also causing issues in integration tests since nucache was being used for the Id2Key service. * For UmbracoTestServerTestBase enable nucache services * Fixing tests * Fix another test * Fixes tests, use TestHostingEnvironment, make Tests.Common use net5, remove old Lucene.Net.Contrib ref. * Fixes up some review notes * Fixes issue with doubly registering PublishedSnapshotService meanig there could be 2x instances of it * Checks for parseexception when executing the query * Use application root instead of duplicating functionality. * Added Examine project to netcore only solution file * Fixed casing issue with LazyLoad, that is not lowercase. * uses cancellationToken instead of bool flag, fixes always reading lastId from the LastSyncedFileManager, fixes RecurringHostedServiceBase so that there isn't an overlapping thread for the same task type * Fix tests * remove legacy test project from solution file * Fix test Co-authored-by: Bjarke Berg <mail@bergmania.dk>
2021-05-18 18:31:38 +10:00
{
validator = new ContentValueSetValidator(true);
Examine 2.0 integration (#10241) * Init commit for examine 2.0 work, most old umb examine tests working, probably a lot that doesn't * Gets Umbraco Examine tests passing and makes some sense out of them, fixes some underlying issues. * Large refactor, remove TaskHelper, rename Notifications to be consistent, Gets all examine/lucene indexes building and startup ordered in the correct way, removes old files, creates new IUmbracoIndexingHandler for abstracting out all index operations for umbraco data, abstracts out IIndexRebuilder, Fixes Stack overflow with LiveModelsProvider and loading assemblies, ports some changes from v8 for startup handling with cold boots, refactors out LastSyncedFileManager * fix up issues with rebuilding and management dashboard. * removes old files, removes NetworkHelper, fixes LastSyncedFileManager implementation to ensure the machine name is used, fix up logging with cold boot state. * Makes MainDom safer to use and makes PublishedSnapshotService lazily register with MainDom * lazily acquire application id (fix unit tests) * Fixes resource casing and missing test file * Ensures caches when requiring internal services for PublishedSnapshotService, UseNuCache is a separate call, shouldn't be buried in AddWebComponents, was also causing issues in integration tests since nucache was being used for the Id2Key service. * For UmbracoTestServerTestBase enable nucache services * Fixing tests * Fix another test * Fixes tests, use TestHostingEnvironment, make Tests.Common use net5, remove old Lucene.Net.Contrib ref. * Fixes up some review notes * Fixes issue with doubly registering PublishedSnapshotService meanig there could be 2x instances of it * Checks for parseexception when executing the query * Use application root instead of duplicating functionality. * Added Examine project to netcore only solution file * Fixed casing issue with LazyLoad, that is not lowercase. * uses cancellationToken instead of bool flag, fixes always reading lastId from the LastSyncedFileManager, fixes RecurringHostedServiceBase so that there isn't an overlapping thread for the same task type * Fix tests * remove legacy test project from solution file * Fix test Co-authored-by: Bjarke Berg <mail@bergmania.dk>
2021-05-18 18:31:38 +10:00
}
var options = GetOptions(
"testIndexer",
new LuceneDirectoryIndexOptions
{
Analyzer = analyzer,
Validator = validator,
DirectoryFactory = new GenericDirectoryFactory(s => luceneDir),
FieldDefinitions = new UmbracoFieldDefinitionCollection()
});
var i = new UmbracoContentIndex(
_loggerFactory,
"testIndexer",
options,
hostingEnvironment,
runtimeState,
languageService);
i.IndexingError += IndexingError;
i.IndexOperationComplete += I_IndexOperationComplete;
return i;
}
2018-06-29 19:52:40 +02:00
private void I_IndexOperationComplete(object sender, IndexOperationEventArgs e)
{
}
2018-06-29 19:52:40 +02:00
//public static MultiIndexSearcher GetMultiSearcher(Directory pdfDir, Directory simpleDir, Directory conventionDir, Directory cwsDir)
//{
// var i = new MultiIndexSearcher("testSearcher", new[] { pdfDir, simpleDir, conventionDir, cwsDir }, new StandardAnalyzer(Version.LUCENE_29));
// return i;
//}
2018-06-29 19:52:40 +02:00
public static IOptionsMonitor<LuceneDirectoryIndexOptions> GetOptions(string indexName, LuceneDirectoryIndexOptions options)
=> Mock.Of<IOptionsMonitor<LuceneDirectoryIndexOptions>>(x => x.Get(indexName) == options);
2018-06-29 19:52:40 +02:00
internal void IndexingError(object sender, IndexingErrorEventArgs e) =>
throw new ApplicationException(e.Message, e.Exception);
2018-06-29 19:52:40 +02:00
}