V12 Integration test for InternalIndex and ExternalIndex Search (#15167)

* Added the possibility for us to search through the external index

* Added ForceInstantExecution

* Added InternalIndex integration tests

* Added External Index integration tests

* Removed comments

* Remove unused internal field

* Add mechanism to wait for index rebuilding

* Remove some of the usages of sleep

* Remove sleeps

* Added comments from review

* Added a method for getting the IndexPath

---------

Co-authored-by: Nikolaj <nikolajlauridsen@protonmail.ch>
This commit is contained in:
Andreas Zerbst
2023-12-04 10:00:26 +01:00
committed by GitHub
parent 0a852aeb07
commit 2a56d4e376
5 changed files with 1229 additions and 0 deletions

View File

@@ -0,0 +1,611 @@
using Examine;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Identity;
using Microsoft.Extensions.DependencyInjection;
using Moq;
using NUnit.Framework;
using Umbraco.Cms.Core;
using Umbraco.Cms.Core.Cache;
using Umbraco.Cms.Core.Models;
using Umbraco.Cms.Core.Models.ContentEditing;
using Umbraco.Cms.Core.Notifications;
using Umbraco.Cms.Core.Security;
using Umbraco.Cms.Core.Services;
using Umbraco.Cms.Core.Sync;
using Umbraco.Cms.Infrastructure.Examine;
using Umbraco.Cms.Infrastructure.Examine.DependencyInjection;
using Umbraco.Cms.Infrastructure.HostedServices;
using Umbraco.Cms.Infrastructure.Search;
using Umbraco.Cms.Tests.Common.Builders;
using Umbraco.Cms.Tests.Common.Builders.Extensions;
using Umbraco.Cms.Tests.Common.Testing;
using Umbraco.Cms.Tests.Integration.Umbraco.Infrastructure.Services;
using Umbraco.Cms.Web.BackOffice.Security;
namespace Umbraco.Cms.Tests.Integration.Umbraco.Examine.Lucene.UmbracoExamine;
[TestFixture]
[UmbracoTest(Database = UmbracoTestOptions.Database.NewSchemaPerTest)]
public class BackOfficeExamineSearcherTests : ExamineBaseTest
{
[SetUp]
public void Setup()
{
TestHelper.DeleteDirectory(GetIndexPath(Constants.UmbracoIndexes.InternalIndexName));
TestHelper.DeleteDirectory(GetIndexPath(Constants.UmbracoIndexes.ExternalIndexName));
var httpContext = new DefaultHttpContext();
httpContext.RequestServices = Services;
Mock.Get(TestHelper.GetHttpContextAccessor()).Setup(x => x.HttpContext).Returns(httpContext);
}
[TearDown]
public void TearDown()
{
// When disposing examine, it does a final write, which ends up locking the file if the indexing is not done yet. So we have this wait to circumvent that.
Thread.Sleep(1500);
// Sometimes we do not dispose all services in time and the test fails because the log file is locked. Resulting in all other tests failing as well
Services.DisposeIfDisposable();
}
private IBackOfficeExamineSearcher BackOfficeExamineSearcher => GetRequiredService<IBackOfficeExamineSearcher>();
private IContentTypeService ContentTypeService => GetRequiredService<IContentTypeService>();
private ILocalizationService LocalizationService => GetRequiredService<ILocalizationService>();
private ContentService ContentService => (ContentService)GetRequiredService<IContentService>();
private IUserStore<BackOfficeIdentityUser> BackOfficeUserStore =>
GetRequiredService<IUserStore<BackOfficeIdentityUser>>();
private IBackOfficeSignInManager BackOfficeSignInManager => GetRequiredService<IBackOfficeSignInManager>();
protected override void CustomTestSetup(IUmbracoBuilder builder)
{
builder.Services.AddUnique<IBackOfficeExamineSearcher, BackOfficeExamineSearcher>();
builder.Services.AddUnique<IServerMessenger, ContentEventsTests.LocalServerMessenger>();
builder
.AddNotificationHandler<ContentTreeChangeNotification,
ContentTreeChangeDistributedCacheNotificationHandler>();
builder.AddNotificationHandler<ContentCacheRefresherNotification, ContentIndexingNotificationHandler>();
builder.AddExamineIndexes();
builder.AddBackOfficeIdentity();
builder.Services.AddHostedService<QueuedHostedService>();
}
private IEnumerable<ISearchResult> BackOfficeExamineSearch(string query, int pageSize = 20, int pageIndex = 0) =>
BackOfficeExamineSearcher.Search(query, UmbracoEntityTypes.Document,
pageSize, pageIndex, out _, ignoreUserStartNodes: true);
private async Task SetupUserIdentity(string userId)
{
var identity =
await BackOfficeUserStore.FindByIdAsync(userId, CancellationToken.None);
await BackOfficeSignInManager.SignInAsync(identity, false);
}
private async Task<PublishResult> CreateDefaultPublishedContent(string contentName)
{
var contentType = new ContentTypeBuilder()
.WithId(0)
.Build();
await ExecuteAndWaitForIndexing(() => ContentTypeService.Save(contentType), Constants.UmbracoIndexes.InternalIndexName);
var content = new ContentBuilder()
.WithId(0)
.WithName(contentName)
.WithContentType(contentType)
.Build();
var createdContent = await ExecuteAndWaitForIndexing(() => ContentService.SaveAndPublish(content), Constants.UmbracoIndexes.InternalIndexName);
return createdContent;
}
private async Task<PublishResult> CreateDefaultPublishedContentWithTwoLanguages(string englishNodeName, string danishNodeName)
{
const string usIso = "en-US";
const string dkIso = "da";
var langDa = new LanguageBuilder()
.WithCultureInfo(dkIso)
.Build();
LocalizationService.Save(langDa);
var contentType = new ContentTypeBuilder()
.WithId(0)
.WithContentVariation(ContentVariation.Culture)
.Build();
await ExecuteAndWaitForIndexing(() => ContentTypeService.Save(contentType), Constants.UmbracoIndexes.InternalIndexName);
var content = new ContentBuilder()
.WithId(0)
.WithCultureName(usIso, englishNodeName)
.WithCultureName(dkIso, danishNodeName)
.WithContentType(contentType)
.Build();
var createdContent = await ExecuteAndWaitForIndexing(() => ContentService.SaveAndPublish(content), Constants.UmbracoIndexes.InternalIndexName);
return createdContent;
}
[Test]
public async Task Search_Published_Content_With_Empty_Query()
{
await SetupUserIdentity(Constants.Security.SuperUserIdAsString);
// Arrange
const string contentName = "TestContent";
await CreateDefaultPublishedContent(contentName);
string query = string.Empty;
// Act
IEnumerable<ISearchResult> actual = BackOfficeExamineSearch(query);
// Assert
Assert.AreEqual(0, actual.Count());
}
[Test]
public async Task Search_Published_Content_With_Query_By_Content_Name()
{
// Arrange
await SetupUserIdentity(Constants.Security.SuperUserIdAsString);
const string contentName = "TestContent";
await CreateDefaultPublishedContent(contentName);
string query = contentName;
// Act
IEnumerable<ISearchResult> actual = BackOfficeExamineSearch(query);
// Assert
IEnumerable<ISearchResult> searchResults = actual.ToArray();
Assert.AreEqual(1, searchResults.Count());
Assert.AreEqual(searchResults.First().Values["nodeName"], contentName);
}
[Test]
public async Task Search_Published_Content_With_Query_By_Non_Existing_Content_Name()
{
// Arrange
await SetupUserIdentity(Constants.Security.SuperUserIdAsString);
const string contentName = "TestContent";
await CreateDefaultPublishedContent(contentName);
string query = "ContentTest";
// Act
IEnumerable<ISearchResult> actual = BackOfficeExamineSearch(query);
// Assert
Assert.AreEqual(0, actual.Count());
}
[Test]
public async Task Search_Published_Content_With_Query_By_Content_Id()
{
// Arrange
await SetupUserIdentity(Constants.Security.SuperUserIdAsString);
const string contentName = "RandomContentName";
PublishResult createdContent = await CreateDefaultPublishedContent(contentName);
string contentId = createdContent.Content.Id.ToString();
string query = contentId;
// Act
IEnumerable<ISearchResult> actual = BackOfficeExamineSearch(query);
// Assert
IEnumerable<ISearchResult> searchResults = actual.ToArray();
Assert.AreEqual(1, searchResults.Count());
Assert.AreEqual(searchResults.First().Values["nodeName"], contentName);
Assert.AreEqual(searchResults.First().Id, contentId);
}
[Test]
public async Task Search_Two_Published_Content_With_Similar_Names_By_Name()
{
// Arrange
await SetupUserIdentity(Constants.Security.SuperUserIdAsString);
const string contentName = "TestName Original";
const string secondContentName = "TestName Copy";
var contentType = new ContentTypeBuilder()
.WithId(0)
.Build();
await ExecuteAndWaitForIndexing(() => ContentTypeService.Save(contentType), Constants.UmbracoIndexes.InternalIndexName);
var firstContent = new ContentBuilder()
.WithId(0)
.WithName(contentName)
.WithContentType(contentType)
.Build();
await ExecuteAndWaitForIndexing(() => ContentService.SaveAndPublish(firstContent), Constants.UmbracoIndexes.InternalIndexName);
var secondContent = new ContentBuilder()
.WithId(0)
.WithName(secondContentName)
.WithContentType(contentType)
.Build();
await ExecuteAndWaitForIndexing(() => ContentService.SaveAndPublish(secondContent), Constants.UmbracoIndexes.InternalIndexName);
string query = contentName;
// Act
IEnumerable<ISearchResult> actual = BackOfficeExamineSearch(query);
// Assert
IEnumerable<ISearchResult> searchResults = actual.ToArray();
Assert.AreEqual(2, searchResults.Count());
// Checks if the first content in the search is the original content
Assert.AreEqual(searchResults.First().Id, firstContent.Id.ToString());
// Checks if the score for the original name is higher than the score for the copy
Assert.Greater(searchResults.First().Score, searchResults.Last().Score);
}
[Test]
public async Task Search_For_Child_Published_Content_With_Query_By_Content_Name()
{
// Arrange
await SetupUserIdentity(Constants.Security.SuperUserIdAsString);
const string contentName = "ParentTestContent";
const string childContentName = "ChildTestContent";
var contentType = new ContentTypeBuilder()
.WithName("Document")
.Build();
await ExecuteAndWaitForIndexing(() => ContentTypeService.Save(contentType), Constants.UmbracoIndexes.InternalIndexName);
var content = new ContentBuilder()
.WithName(contentName)
.WithContentType(contentType)
.Build();
await ExecuteAndWaitForIndexing(() => ContentService.SaveAndPublish(content), Constants.UmbracoIndexes.InternalIndexName);
var childContent = new ContentBuilder()
.WithName(childContentName)
.WithContentType(contentType)
.WithParentId(content.Id)
.Build();
await ExecuteAndWaitForIndexing(() => ContentService.SaveAndPublish(childContent), Constants.UmbracoIndexes.InternalIndexName);
string parentQuery = content.Id.ToString();
string childQuery = childContent.Id.ToString();
// Act
IEnumerable<ISearchResult> parentContentActual = BackOfficeExamineSearch(parentQuery);
IEnumerable<ISearchResult> childContentActual = BackOfficeExamineSearch(childQuery);
// Assert
IEnumerable<ISearchResult> contentActual = parentContentActual.ToArray();
IEnumerable<ISearchResult> searchResults = childContentActual.ToArray();
Assert.AreEqual(1, contentActual.Count());
Assert.AreEqual(1, searchResults.Count());
Assert.AreEqual(contentActual.First().Values["nodeName"], contentName);
Assert.AreEqual(searchResults.First().Values["nodeName"], childContentName);
}
[Test]
public async Task Search_For_Child_In_Child_Published_Content_With_Query_By_Content_Name()
{
// Arrange
await SetupUserIdentity(Constants.Security.SuperUserIdAsString);
const string contentName = "ParentTestContent";
const string childContentName = "ChildTestContent";
const string childChildContentName = "ChildChildTestContent";
var contentType = new ContentTypeBuilder()
.WithName("Document")
.Build();
ContentTypeService.Save(contentType);
var content = new ContentBuilder()
.WithName(contentName)
.WithContentType(contentType)
.Build();
await ExecuteAndWaitForIndexing(() => ContentService.SaveAndPublish(content), Constants.UmbracoIndexes.InternalIndexName);
var childContent = new ContentBuilder()
.WithName(childContentName)
.WithContentType(contentType)
.WithParentId(content.Id)
.Build();
await ExecuteAndWaitForIndexing(() => ContentService.SaveAndPublish(childContent), Constants.UmbracoIndexes.InternalIndexName);
var childChildContent = new ContentBuilder()
.WithName(childChildContentName)
.WithContentType(contentType)
.WithParentId(childContent.Id)
.Build();
await ExecuteAndWaitForIndexing(() => ContentService.SaveAndPublish(childChildContent), Constants.UmbracoIndexes.InternalIndexName);
string parentQuery = content.Id.ToString();
string childQuery = childContent.Id.ToString();
string childChildQuery = childChildContent.Id.ToString();
// Act
IEnumerable<ISearchResult> parentContentActual = BackOfficeExamineSearch(parentQuery);
IEnumerable<ISearchResult> childContentActual = BackOfficeExamineSearch(childQuery);
IEnumerable<ISearchResult> childChildContentActual = BackOfficeExamineSearch(childChildQuery);
IEnumerable<ISearchResult> parentSearchResults = parentContentActual.ToArray();
IEnumerable<ISearchResult> childSearchResults = childContentActual.ToArray();
IEnumerable<ISearchResult> childChildSearchResults = childChildContentActual.ToArray();
// Assert
Assert.AreEqual(1, parentSearchResults.Count());
Assert.AreEqual(1, childSearchResults.Count());
Assert.AreEqual(1, childChildSearchResults.Count());
Assert.AreEqual(parentSearchResults.First().Values["nodeName"], contentName);
Assert.AreEqual(childSearchResults.First().Values["nodeName"], childContentName);
Assert.AreEqual(childChildSearchResults.First().Values["nodeName"], childChildContentName);
}
[Test]
public async Task Search_Published_Content_With_Query_With_Content_Name_No_User_Logged_In()
{
// Arrange
const string contentName = "TestContent";
PublishResult createdContent = await CreateDefaultPublishedContent(contentName);
string query = createdContent.Content.Id.ToString();
// Act
IEnumerable<ISearchResult> actual = BackOfficeExamineSearch(query);
// Assert
Assert.AreEqual(0, actual.Count());
}
// Multiple Languages
[Test]
public async Task Search_Published_Content_By_Content_Name_With_Two_Languages()
{
// Arrange
await SetupUserIdentity(Constants.Security.SuperUserIdAsString);
const string usIso = "en-US";
const string dkIso = "da";
const string englishNodeName = "EnglishNode";
const string danishNodeName = "DanishNode";
var langDa = new LanguageBuilder()
.WithCultureInfo(dkIso)
.Build();
LocalizationService.Save(langDa);
var contentType = new ContentTypeBuilder()
.WithId(0)
.WithContentVariation(ContentVariation.Culture)
.Build();
await ExecuteAndWaitForIndexing(() => ContentTypeService.Save(contentType), Constants.UmbracoIndexes.InternalIndexName);
var content = new ContentBuilder()
.WithId(0)
.WithCultureName(usIso, englishNodeName)
.WithCultureName(dkIso, danishNodeName)
.WithContentType(contentType)
.Build();
PublishResult createdContent = await ExecuteAndWaitForIndexing(() => ContentService.SaveAndPublish(content), Constants.UmbracoIndexes.InternalIndexName);
string query = createdContent.Content.Id.ToString();
// Act
IEnumerable<ISearchResult> actual = BackOfficeExamineSearch(query);
// Assert
IEnumerable<ISearchResult> searchResults = actual.ToArray();
Assert.AreEqual(1, searchResults.Count());
var nodeNameEn = searchResults.First().Values["nodeName_en-us"];
var nodeNameDa = searchResults.First().Values["nodeName_da"];
Assert.AreEqual(englishNodeName, nodeNameEn);
Assert.AreEqual(danishNodeName, nodeNameDa);
}
[Test]
public async Task Search_For_Published_Content_Name_With_Two_Languages_By_Default_Language_Content_Name()
{
// Arrange
await SetupUserIdentity(Constants.Security.SuperUserIdAsString);
const string englishNodeName = "EnglishNode";
const string danishNodeName = "DanishNode";
await CreateDefaultPublishedContentWithTwoLanguages(englishNodeName, danishNodeName);
string query = englishNodeName;
// Act
IEnumerable<ISearchResult> actual = BackOfficeExamineSearch(query);
// Assert
IEnumerable<ISearchResult> searchResults = actual.ToArray();
Assert.AreEqual(1, searchResults.Count());
var nodeNameEn = searchResults.First().Values["nodeName_en-us"];
var nodeNameDa = searchResults.First().Values["nodeName_da"];
Assert.AreEqual(englishNodeName, nodeNameEn);
Assert.AreEqual(danishNodeName, nodeNameDa);
}
[Test]
public async Task Search_For_Published_Content_Name_With_Two_Languages_By_Non_Default_Language_Content_Name()
{
// Arrange
await SetupUserIdentity(Constants.Security.SuperUserIdAsString);
const string englishNodeName = "EnglishNode";
const string danishNodeName = "DanishNode";
await CreateDefaultPublishedContentWithTwoLanguages(englishNodeName, danishNodeName);
string query = danishNodeName;
// Act
IEnumerable<ISearchResult> actual = BackOfficeExamineSearch(query);
// Assert
IEnumerable<ISearchResult> searchResults = actual.ToArray();
Assert.AreEqual(1, searchResults.Count());
var nodeNameDa = searchResults.First().Values["nodeName_da"];
var nodeNameEn = searchResults.First().Values["nodeName_en-us"];
Assert.AreEqual(englishNodeName, nodeNameEn);
Assert.AreEqual(danishNodeName, nodeNameDa);
}
[Test]
public async Task Search_Published_Content_With_Two_Languages_By_Id()
{
// Arrange
await SetupUserIdentity(Constants.Security.SuperUserIdAsString);
const string englishNodeName = "EnglishNode";
const string danishNodeName = "DanishNode";
var contentNode = await CreateDefaultPublishedContentWithTwoLanguages(englishNodeName, danishNodeName);
string query = contentNode.Content.Id.ToString();
// Act
IEnumerable<ISearchResult> actual = BackOfficeExamineSearch(query);
// Assert
IEnumerable<ISearchResult> searchResults = actual.ToArray();
Assert.AreEqual(1, searchResults.Count());
var nodeNameDa = searchResults.First().Values["nodeName_da"];
var nodeNameEn = searchResults.First().Values["nodeName_en-us"];
Assert.AreEqual(englishNodeName, nodeNameEn);
Assert.AreEqual(danishNodeName, nodeNameDa);
}
// Check All Indexed Values
[Test]
public async Task Check_All_Indexed_Values_For_Published_Content_With_No_Properties()
{
// Arrange
await SetupUserIdentity(Constants.Security.SuperUserIdAsString);
const string contentName = "TestContent";
var contentType = new ContentTypeBuilder()
.WithId(0)
.Build();
ContentTypeService.Save(contentType);
var contentNode = new ContentBuilder()
.WithId(0)
.WithName(contentName)
.WithContentType(contentType)
.Build();
await ExecuteAndWaitForIndexing(() => ContentService.SaveAndPublish(contentNode), Constants.UmbracoIndexes.InternalIndexName);
string query = contentName;
// Act
IEnumerable<ISearchResult> actual = BackOfficeExamineSearch(query);
// Assert
IEnumerable<ISearchResult> searchResults = actual.ToArray();
Assert.AreEqual(1, searchResults.Count());
string contentNodePublish = string.Empty;
if (contentNode.Published)
{
contentNodePublish = "y";
}
string contentTypeCultureVariations = string.Empty;
if (contentType.Variations == ContentVariation.Nothing)
{
contentTypeCultureVariations = "n";
}
Assert.Multiple(() =>
{
Assert.AreEqual(searchResults.First().Values["__NodeId"], contentNode.Id.ToString());
Assert.AreEqual(searchResults.First().Values["__IndexType"], "content");
Assert.AreEqual(searchResults.First().Values["__NodeTypeAlias"], contentNode.ContentType.Alias);
Assert.AreEqual(searchResults.First().Values["__Published"], contentNodePublish);
Assert.AreEqual(searchResults.First().Values["id"], contentNode.Id.ToString());
Assert.AreEqual(searchResults.First().Values["__Key"], contentNode.Key.ToString());
Assert.AreEqual(searchResults.First().Values["parentID"], contentNode.ParentId.ToString());
Assert.AreEqual(searchResults.First().Values["nodeName"], contentNode.Name);
Assert.AreEqual(searchResults.First().Values["__VariesByCulture"], contentTypeCultureVariations);
Assert.AreEqual(searchResults.First().Values["__Icon"], contentNode.ContentType.Icon);
});
}
[Test]
public async Task Check_All_Indexed_Values_For_Published_Content_With_Properties()
{
// Arrange
await SetupUserIdentity(Constants.Security.SuperUserIdAsString);
const string contentName = "TestContent";
const string propertyEditorName = "TestBox";
var contentType = new ContentTypeBuilder()
.WithId(0)
.AddPropertyType()
.WithPropertyEditorAlias(Constants.PropertyEditors.Aliases.TextBox)
.WithName(propertyEditorName)
.WithAlias("testBox")
.Done()
.Build();
ContentTypeService.Save(contentType);
var contentNode = new ContentBuilder()
.WithId(0)
.WithName(contentName)
.WithContentType(contentType)
.WithPropertyValues(new { testBox = "TestValue" })
.Build();
await ExecuteAndWaitForIndexing(() => ContentService.SaveAndPublish(contentNode), Constants.UmbracoIndexes.InternalIndexName);
string query = contentName;
// Act
IEnumerable<ISearchResult> actual = BackOfficeExamineSearch(query);
// Assert
IEnumerable<ISearchResult> searchResults = actual.ToArray();
Assert.AreEqual(1, searchResults.Count());
string contentNodePublish = string.Empty;
string contentTypeCultureVariations = string.Empty;
if (contentNode.Published)
{
contentNodePublish = "y";
}
if (contentType.Variations == ContentVariation.Nothing)
{
contentTypeCultureVariations = "n";
}
Assert.Multiple(() =>
{
Assert.AreEqual(searchResults.First().Values["__NodeId"], contentNode.Id.ToString());
Assert.AreEqual(searchResults.First().Values["__IndexType"], "content");
Assert.AreEqual(searchResults.First().Values["__NodeTypeAlias"], contentNode.ContentType.Alias);
Assert.AreEqual(searchResults.First().Values["__Published"], contentNodePublish);
Assert.AreEqual(searchResults.First().Values["id"], contentNode.Id.ToString());
Assert.AreEqual(searchResults.First().Values["__Key"], contentNode.Key.ToString());
Assert.AreEqual(searchResults.First().Values["parentID"], contentNode.ParentId.ToString());
Assert.AreEqual(searchResults.First().Values["nodeName"], contentNode.Name);
Assert.AreEqual(searchResults.First().Values["__VariesByCulture"], contentTypeCultureVariations);
Assert.AreEqual(searchResults.First().Values["__Icon"], contentNode.ContentType.Icon);
});
}
}

View File

@@ -1,4 +1,5 @@
using System.Data;
using Examine;
using Examine.Lucene.Providers;
using Microsoft.Extensions.DependencyInjection;
using Moq;
@@ -25,6 +26,8 @@ public abstract class ExamineBaseTest : UmbracoIntegrationTest
protected IRuntimeState RunningRuntimeState { get; } = Mock.Of<IRuntimeState>(x => x.Level == RuntimeLevel.Run);
protected IExamineManager ExamineManager => GetRequiredService<IExamineManager>();
protected override void ConfigureTestServices(IServiceCollection services)
=> services.AddSingleton<IndexInitializer>();
@@ -113,6 +116,43 @@ public abstract class ExamineBaseTest : UmbracoIntegrationTest
return new DisposableWrapper(syncMode, index, luceneDir);
}
private AutoResetEvent indexingHandle = new(false);
protected async Task ExecuteAndWaitForIndexing(Action indexUpdatingAction, string indexName) =>
await ExecuteAndWaitForIndexing<int?>(
() =>
{
indexUpdatingAction();
return null;
}, indexName);
/// <summary>
/// Performs and action and waits for the specified index to be done indexing.
/// </summary>
/// <param name="indexUpdatingAction">The action that causes the index to be updated.</param>
/// <param name="indexName">The name of the index to wait for rebuild.</param>
/// <typeparam name="T">The type returned from the action.</typeparam>
/// <returns>The result of the action.</returns>
protected async Task<T> ExecuteAndWaitForIndexing<T> (Func<T> indexUpdatingAction, string indexName)
{
// Set up an action to release the handle when the index is populated.
if (ExamineManager.TryGetIndex(indexName, out IIndex index) is false)
{
throw new InvalidOperationException($"Could not find index: {indexName}");
}
index.IndexOperationComplete += (_, _) =>
{
indexingHandle.Set();
};
// Perform the action, and wait for the handle to be freed, meaning the index is done populating.
var result = indexUpdatingAction();
await indexingHandle.WaitOneAsync();
return result;
}
private class DisposableWrapper : IDisposable
{
private readonly IDisposable[] _disposables;
@@ -127,4 +167,10 @@ public abstract class ExamineBaseTest : UmbracoIntegrationTest
}
}
}
protected string GetIndexPath(string indexName)
{
var root = TestContext.CurrentContext.TestDirectory.Split("Umbraco.Tests.Integration")[0];
return Path.Combine(root, "Umbraco.Tests.Integration", "umbraco", "Data", "TEMP", "ExamineIndexes", indexName);
}
}

View File

@@ -0,0 +1,417 @@
// Copyright (c) Umbraco.
// See LICENSE for more details.
using System.Globalization;
using System.Text;
using System.Text.RegularExpressions;
using Examine;
using Examine.Search;
using Lucene.Net.QueryParsers.Classic;
using Umbraco.Cms.Core;
using Umbraco.Cms.Core.Cache;
using Umbraco.Cms.Core.Mapping;
using Umbraco.Cms.Core.Models;
using Umbraco.Cms.Core.Models.ContentEditing;
using Umbraco.Cms.Core.Models.Entities;
using Umbraco.Cms.Core.Models.Membership;
using Umbraco.Cms.Core.Routing;
using Umbraco.Cms.Core.Security;
using Umbraco.Cms.Core.Services;
using Umbraco.Cms.Infrastructure.Examine;
namespace Umbraco.Cms.Tests.Integration.Umbraco.Examine.Lucene.UmbracoExamine;
public class ExamineExternalIndexSearcherTest : IExamineExternalIndexSearcherTest
{
private readonly AppCaches _appCaches;
private readonly IBackOfficeSecurityAccessor _backOfficeSecurityAccessor;
private readonly IEntityService _entityService;
private readonly IExamineManager _examineManager;
private readonly ILocalizationService _languageService;
private readonly IPublishedUrlProvider _publishedUrlProvider;
private readonly IUmbracoTreeSearcherFields _treeSearcherFields;
private readonly IUmbracoMapper _umbracoMapper;
public ExamineExternalIndexSearcherTest(
IExamineManager examineManager,
ILocalizationService languageService,
IBackOfficeSecurityAccessor backOfficeSecurityAccessor,
IEntityService entityService,
IUmbracoTreeSearcherFields treeSearcherFields,
AppCaches appCaches,
IUmbracoMapper umbracoMapper,
IPublishedUrlProvider publishedUrlProvider)
{
_examineManager = examineManager;
_languageService = languageService;
_backOfficeSecurityAccessor = backOfficeSecurityAccessor;
_entityService = entityService;
_treeSearcherFields = treeSearcherFields;
_appCaches = appCaches;
_umbracoMapper = umbracoMapper;
_publishedUrlProvider = publishedUrlProvider;
}
public IEnumerable<ISearchResult> Search(
string query,
UmbracoEntityTypes entityType,
int pageSize,
long pageIndex,
out long totalFound,
string? searchFrom = null,
bool ignoreUserStartNodes = false)
{
var sb = new StringBuilder();
string type;
var indexName = Constants.UmbracoIndexes.ExternalIndexName;
var fields = _treeSearcherFields.GetBackOfficeFields().ToList();
ISet<string> fieldsToLoad = new HashSet<string>(_treeSearcherFields.GetBackOfficeFieldsToLoad());
// TODO: WE should try to allow passing in a lucene raw query, however we will still need to do some manual string
// manipulation for things like start paths, member types, etc...
//if (Examine.ExamineExtensions.TryParseLuceneQuery(query))
//{
//}
//special GUID check since if a user searches on one specifically we need to escape it
if (Guid.TryParse(query, out Guid g))
{
query = "\"" + g + "\"";
}
IUser? currentUser = _backOfficeSecurityAccessor?.BackOfficeSecurity?.CurrentUser;
switch (entityType)
{
case UmbracoEntityTypes.Member:
indexName = Constants.UmbracoIndexes.MembersIndexName;
type = "member";
fields.AddRange(_treeSearcherFields.GetBackOfficeMembersFields());
foreach (var field in _treeSearcherFields.GetBackOfficeMembersFieldsToLoad())
{
fieldsToLoad.Add(field);
}
if (searchFrom != null && searchFrom != Constants.Conventions.MemberTypes.AllMembersListId &&
searchFrom.Trim() != "-1")
{
sb.Append("+__NodeTypeAlias:");
sb.Append(searchFrom);
sb.Append(" ");
}
break;
case UmbracoEntityTypes.Media:
type = "media";
fields.AddRange(_treeSearcherFields.GetBackOfficeMediaFields());
foreach (var field in _treeSearcherFields.GetBackOfficeMediaFieldsToLoad())
{
fieldsToLoad.Add(field);
}
var allMediaStartNodes = currentUser != null
? currentUser.CalculateMediaStartNodeIds(_entityService, _appCaches)
: Array.Empty<int>();
AppendPath(sb, UmbracoObjectTypes.Media, allMediaStartNodes, searchFrom, ignoreUserStartNodes, _entityService);
break;
case UmbracoEntityTypes.Document:
type = "content";
fields.AddRange(_treeSearcherFields.GetBackOfficeDocumentFields());
foreach (var field in _treeSearcherFields.GetBackOfficeDocumentFieldsToLoad())
{
fieldsToLoad.Add(field);
}
var allContentStartNodes = currentUser != null
? currentUser.CalculateContentStartNodeIds(_entityService, _appCaches)
: Array.Empty<int>();
AppendPath(sb, UmbracoObjectTypes.Document, allContentStartNodes, searchFrom, ignoreUserStartNodes, _entityService);
break;
default:
throw new NotSupportedException("The " + typeof(BackOfficeExamineSearcher) +
" currently does not support searching against object type " +
entityType);
}
if (!_examineManager.TryGetIndex(indexName, out IIndex? index))
{
throw new InvalidOperationException("No index found by name " + indexName);
}
if (!BuildQuery(sb, query, searchFrom, fields, type))
{
totalFound = 0;
return Enumerable.Empty<ISearchResult>();
}
ISearchResults? result = index.Searcher
.CreateQuery()
.NativeQuery(sb.ToString())
.SelectFields(fieldsToLoad)
//only return the number of items specified to read up to the amount of records to fill from 0 -> the number of items on the page requested
.Execute(QueryOptions.SkipTake(Convert.ToInt32(pageSize * pageIndex), pageSize));
totalFound = result.TotalItemCount;
return result;
}
private bool BuildQuery(StringBuilder sb, string query, string? searchFrom, List<string> fields, string type)
{
//build a lucene query:
// the nodeName will be boosted 10x without wildcards
// then nodeName will be matched normally with wildcards
// the rest will be normal without wildcards
var allLangs = _languageService.GetAllLanguages().Select(x => x.IsoCode.ToLowerInvariant()).ToList();
// the chars [*-_] in the query will mess everything up so let's remove those
// However we cannot just remove - and _ since these signify a space, so we instead replace them with that.
query = Regex.Replace(query, "[\\*]", string.Empty);
query = Regex.Replace(query, "[\\-_]", " ");
//check if text is surrounded by single or double quotes, if so, then exact match
var surroundedByQuotes = Regex.IsMatch(query, "^\".*?\"$")
|| Regex.IsMatch(query, "^\'.*?\'$");
if (surroundedByQuotes)
{
//strip quotes, escape string, the replace again
query = query.Trim(Constants.CharArrays.DoubleQuoteSingleQuote);
query = QueryParserBase.Escape(query);
//nothing to search
if (searchFrom.IsNullOrWhiteSpace() && query.IsNullOrWhiteSpace())
{
return false;
}
//update the query with the query term
if (query.IsNullOrWhiteSpace() == false)
{
//add back the surrounding quotes
query = string.Format("{0}{1}{0}", "\"", query);
sb.Append("+(");
AppendNodeNamePhraseWithBoost(sb, query, allLangs);
foreach (var f in fields)
{
//additional fields normally
sb.Append(f);
sb.Append(": (");
sb.Append(query);
sb.Append(") ");
}
sb.Append(") ");
}
}
else
{
var trimmed = query.Trim(Constants.CharArrays.DoubleQuoteSingleQuote);
//nothing to search
if (searchFrom.IsNullOrWhiteSpace() && trimmed.IsNullOrWhiteSpace())
{
return false;
}
//update the query with the query term
if (trimmed.IsNullOrWhiteSpace() == false)
{
query = QueryParserBase.Escape(query);
var querywords = query.Split(Constants.CharArrays.Space, StringSplitOptions.RemoveEmptyEntries);
sb.Append("+(");
AppendNodeNameExactWithBoost(sb, query, allLangs);
AppendNodeNameWithWildcards(sb, querywords, allLangs);
foreach (var f in fields)
{
var queryWordsReplaced = new string[querywords.Length];
// when searching file names containing hyphens we need to replace the hyphens with spaces
if (f.Equals(UmbracoExamineFieldNames.UmbracoFileFieldName))
{
for (var index = 0; index < querywords.Length; index++)
{
queryWordsReplaced[index] =
querywords[index].Replace("\\-", " ").Replace("_", " ").Trim(" ");
}
}
else
{
queryWordsReplaced = querywords;
}
//additional fields normally
sb.Append(f);
sb.Append(":");
sb.Append("(");
foreach (var w in queryWordsReplaced)
{
sb.Append(w.ToLower());
sb.Append("* ");
}
sb.Append(")");
sb.Append(" ");
}
sb.Append(") ");
}
}
//must match index type
sb.Append("+__IndexType:");
sb.Append(type);
return true;
}
private void AppendNodeNamePhraseWithBoost(StringBuilder sb, string query, IEnumerable<string> allLangs)
{
//node name exactly boost x 10
sb.Append("nodeName: (");
sb.Append(query.ToLower());
sb.Append(")^10.0 ");
//also search on all variant node names
foreach (var lang in allLangs)
{
//node name exactly boost x 10
sb.Append($"nodeName_{lang}: (");
sb.Append(query.ToLower());
sb.Append(")^10.0 ");
}
}
private void AppendNodeNameExactWithBoost(StringBuilder sb, string query, IEnumerable<string> allLangs)
{
//node name exactly boost x 10
sb.Append("nodeName:");
sb.Append("\"");
sb.Append(query.ToLower());
sb.Append("\"");
sb.Append("^10.0 ");
//also search on all variant node names
foreach (var lang in allLangs)
{
//node name exactly boost x 10
sb.Append($"nodeName_{lang}:");
sb.Append("\"");
sb.Append(query.ToLower());
sb.Append("\"");
sb.Append("^10.0 ");
}
}
private void AppendNodeNameWithWildcards(StringBuilder sb, string[] querywords, IEnumerable<string> allLangs)
{
//node name normally with wildcards
sb.Append("nodeName:");
sb.Append("(");
foreach (var w in querywords)
{
sb.Append(w.ToLower());
sb.Append("* ");
}
sb.Append(") ");
//also search on all variant node names
foreach (var lang in allLangs)
{
//node name normally with wildcards
sb.Append($"nodeName_{lang}:");
sb.Append("(");
foreach (var w in querywords)
{
sb.Append(w.ToLower());
sb.Append("* ");
}
sb.Append(") ");
}
}
private void AppendPath(StringBuilder sb, UmbracoObjectTypes objectType, int[]? startNodeIds, string? searchFrom, bool ignoreUserStartNodes, IEntityService entityService)
{
if (sb == null)
{
throw new ArgumentNullException(nameof(sb));
}
if (entityService == null)
{
throw new ArgumentNullException(nameof(entityService));
}
UdiParser.TryParse(searchFrom, true, out Udi? udi);
searchFrom = udi == null ? searchFrom : entityService.GetId(udi).Result.ToString();
TreeEntityPath? entityPath =
int.TryParse(searchFrom, NumberStyles.Integer, CultureInfo.InvariantCulture, out var searchFromId) &&
searchFromId > 0
? entityService.GetAllPaths(objectType, searchFromId).FirstOrDefault()
: null;
if (entityPath != null)
{
// find... only what's underneath
sb.Append("+__Path:");
AppendPath(sb, entityPath.Path, false);
sb.Append(" ");
}
else if (startNodeIds?.Length == 0)
{
// make sure we don't find anything
sb.Append("+__Path:none ");
}
else if (startNodeIds?.Contains(-1) == false && ignoreUserStartNodes == false) // -1 = no restriction
{
IEnumerable<TreeEntityPath> entityPaths = entityService.GetAllPaths(objectType, startNodeIds);
// for each start node, find the start node, and what's underneath
// +__Path:(-1*,1234 -1*,1234,* -1*,5678 -1*,5678,* ...)
sb.Append("+__Path:(");
var first = true;
foreach (TreeEntityPath ep in entityPaths)
{
if (first)
{
first = false;
}
else
{
sb.Append(" ");
}
AppendPath(sb, ep.Path, true);
}
sb.Append(") ");
}
}
private void AppendPath(StringBuilder sb, string path, bool includeThisNode)
{
path = path.Replace("-", "\\-").Replace(",", "\\,");
if (includeThisNode)
{
sb.Append(path);
sb.Append(" ");
}
sb.Append(path);
sb.Append("\\,*");
}
}

View File

@@ -0,0 +1,137 @@
using Examine;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Identity;
using Microsoft.Extensions.DependencyInjection;
using Moq;
using NUnit.Framework;
using Umbraco.Cms.Core;
using Umbraco.Cms.Core.Cache;
using Umbraco.Cms.Core.Models.ContentEditing;
using Umbraco.Cms.Core.Notifications;
using Umbraco.Cms.Core.Security;
using Umbraco.Cms.Core.Services;
using Umbraco.Cms.Core.Sync;
using Umbraco.Cms.Infrastructure.Examine.DependencyInjection;
using Umbraco.Cms.Infrastructure.HostedServices;
using Umbraco.Cms.Infrastructure.Search;
using Umbraco.Cms.Tests.Common.Builders;
using Umbraco.Cms.Tests.Common.Builders.Extensions;
using Umbraco.Cms.Tests.Common.Testing;
using Umbraco.Cms.Tests.Integration.Umbraco.Infrastructure.Services;
using Umbraco.Cms.Web.BackOffice.Security;
namespace Umbraco.Cms.Tests.Integration.Umbraco.Examine.Lucene.UmbracoExamine;
[TestFixture]
[UmbracoTest(Database = UmbracoTestOptions.Database.NewSchemaPerTest)]
public class ExamineExternalIndexTests : ExamineBaseTest
{
private const string ContentName = "TestContent";
[SetUp]
public void Setup()
{
TestHelper.DeleteDirectory(GetIndexPath(Constants.UmbracoIndexes.InternalIndexName));
TestHelper.DeleteDirectory(GetIndexPath(Constants.UmbracoIndexes.ExternalIndexName));
var httpContext = new DefaultHttpContext();
httpContext.RequestServices = Services;
Mock.Get(TestHelper.GetHttpContextAccessor()).Setup(x => x.HttpContext).Returns(httpContext);
}
[TearDown]
public void TearDown()
{
// When disposing examine, it does a final write, which ends up locking the file if the indexing is not done yet. So we have this wait to circumvent that.
Thread.Sleep(1500);
// Sometimes we do not dispose all services in time and the test fails because the log file is locked. Resulting in all other tests failing as well
Services.DisposeIfDisposable();
}
private IExamineExternalIndexSearcherTest ExamineExternalIndexSearcher =>
GetRequiredService<IExamineExternalIndexSearcherTest>();
private IContentTypeService ContentTypeService => GetRequiredService<IContentTypeService>();
private ContentService ContentService => (ContentService)GetRequiredService<IContentService>();
private IUserStore<BackOfficeIdentityUser> BackOfficeUserStore =>
GetRequiredService<IUserStore<BackOfficeIdentityUser>>();
private IBackOfficeSignInManager BackOfficeSignInManager => GetRequiredService<IBackOfficeSignInManager>();
protected override void CustomTestSetup(IUmbracoBuilder builder)
{
builder.Services.AddUnique<IExamineExternalIndexSearcherTest, ExamineExternalIndexSearcherTest>();
builder.Services.AddUnique<IServerMessenger, ContentEventsTests.LocalServerMessenger>();
builder
.AddNotificationHandler<ContentTreeChangeNotification,
ContentTreeChangeDistributedCacheNotificationHandler>();
builder.AddNotificationHandler<ContentCacheRefresherNotification, ContentIndexingNotificationHandler>();
builder.AddExamineIndexes();
builder.AddBackOfficeIdentity();
builder.Services.AddHostedService<QueuedHostedService>();
}
private IEnumerable<ISearchResult> ExamineExternalIndexSearch(string query, int pageSize = 20, int pageIndex = 0) =>
ExamineExternalIndexSearcher.Search(query, UmbracoEntityTypes.Document,
pageSize, pageIndex, out _, ignoreUserStartNodes: true);
private async Task SetupUserIdentity(string userId)
{
var identity =
await BackOfficeUserStore.FindByIdAsync(userId, CancellationToken.None);
await BackOfficeSignInManager.SignInAsync(identity, false);
}
[Test]
public async Task Search_Published_Content_With_Query_By_Content_Name()
{
// Arrange
await SetupUserIdentity(Constants.Security.SuperUserIdAsString);
var contentType = new ContentTypeBuilder()
.WithId(0)
.Build();
ContentTypeService.Save(contentType);
var content = new ContentBuilder()
.WithId(0)
.WithName(ContentName)
.WithContentType(contentType)
.Build();
await ExecuteAndWaitForIndexing(() => ContentService.SaveAndPublish(content), Constants.UmbracoIndexes.ExternalIndexName);
// Act
IEnumerable<ISearchResult> actual = ExamineExternalIndexSearch(ContentName);
// Assert
IEnumerable<ISearchResult> searchResults = actual.ToArray();
Assert.AreEqual(1, searchResults.Count());
Assert.AreEqual(searchResults.First().Values["nodeName"], ContentName);
}
[Test]
public async Task Search_Unpublished_Content_With_Query_By_Content_Name()
{
// Arrange
await SetupUserIdentity(Constants.Security.SuperUserIdAsString);
var contentType = new ContentTypeBuilder()
.WithId(0)
.Build();
ContentTypeService.Save(contentType);
var content = new ContentBuilder()
.WithId(0)
.WithName(ContentName)
.WithContentType(contentType)
.Build();
await ExecuteAndWaitForIndexing(() => ContentService.Save(content), Constants.UmbracoIndexes.ExternalIndexName);
// Act
IEnumerable<ISearchResult> actual = ExamineExternalIndexSearch(ContentName);
// Assert
Assert.AreEqual(0, actual.Count());
}
}

View File

@@ -0,0 +1,18 @@
using Examine;
using Umbraco.Cms.Core.Models.ContentEditing;
namespace Umbraco.Cms.Tests.Integration.Umbraco.Examine.Lucene.UmbracoExamine;
public interface IExamineExternalIndexSearcherTest
{
IEnumerable<ISearchResult> Search(
string query,
UmbracoEntityTypes entityType,
int pageSize,
long pageIndex,
out long totalFound,
string? searchFrom = null,
bool ignoreUserStartNodes = false);
}