yv01p/Umbraco-CMS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

All Lucene based logic is part of Umbraco.Examine.Lucene, no other part of Umbraco now knows about Lucene at all.

Browse Source
This commit is contained in:
Shannon
2020-01-29 12:24:57 +11:00
parent 4183514b52
commit 5b5aa4c499
29 changed files with 602 additions and 375 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/Umbraco.Web/Search/IUmbracoTreeSearcherFields.cs → src/Umbraco.Abstractions/Search/IUmbracoTreeSearcherFields.cs
View File

@@ -1,7 +1,7 @@
using System.Collections.Generic;
namespace Umbraco.Web.Search
{
{
/// <summary>
/// Used to propagate hardcoded internal Field lists
/// </summary>

346
src/Umbraco.Examine.Lucene/BackOfficeExamineSearcher.cs Normal file
View File

@@ -0,0 +1,346 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using Examine;
using Umbraco.Core;
using Umbraco.Core.Models;
using Umbraco.Core.Models.Identity;
using Umbraco.Core.Persistence;
using Umbraco.Core.Services;
using Umbraco.Web.Models.ContentEditing;
using Umbraco.Web.Search;
namespace Umbraco.Examine
{
public class BackOfficeExamineSearcher : IBackOfficeExamineSearcher
{
private readonly IExamineManager _examineManager;
private readonly ILocalizationService _languageService;
private readonly ICurrentUserAccessor _currentUserAccessor;
private readonly IEntityService _entityService;
private readonly IUmbracoTreeSearcherFields _treeSearcherFields;
public BackOfficeExamineSearcher(IExamineManager examineManager,
ILocalizationService languageService,
ICurrentUserAccessor currentUserAccessor,
IEntityService entityService,
IUmbracoTreeSearcherFields treeSearcherFields)
{
_examineManager = examineManager;
_languageService = languageService;
_currentUserAccessor = currentUserAccessor;
_entityService = entityService;
_treeSearcherFields = treeSearcherFields;
}
public IEnumerable<ISearchResult> Search(string query, UmbracoEntityTypes entityType, int pageSize, long pageIndex, out long totalFound, string searchFrom = null, bool ignoreUserStartNodes = false)
{
var sb = new StringBuilder();
string type;
var indexName = Constants.UmbracoIndexes.InternalIndexName;
var fields = _treeSearcherFields.GetBackOfficeFields().ToList();
// TODO: WE should try to allow passing in a lucene raw query, however we will still need to do some manual string
// manipulation for things like start paths, member types, etc...
//if (Examine.ExamineExtensions.TryParseLuceneQuery(query))
//{
//}
//special GUID check since if a user searches on one specifically we need to escape it
if (Guid.TryParse(query, out var g))
{
query = "\"" + g.ToString() + "\"";
}
var currentUser = _currentUserAccessor.TryGetCurrentUser();
switch (entityType)
{
case UmbracoEntityTypes.Member:
indexName = Constants.UmbracoIndexes.MembersIndexName;
type = "member";
fields.AddRange(_treeSearcherFields.GetBackOfficeMembersFields());
if (searchFrom != null && searchFrom != Constants.Conventions.MemberTypes.AllMembersListId && searchFrom.Trim() != "-1")
{
sb.Append("+__NodeTypeAlias:");
sb.Append(searchFrom);
sb.Append(" ");
}
break;
case UmbracoEntityTypes.Media:
type = "media";
fields.AddRange(_treeSearcherFields.GetBackOfficeMediaFields());
var allMediaStartNodes = currentUser != null
? currentUser.CalculateMediaStartNodeIds(_entityService)
: Array.Empty<int>();
AppendPath(sb, UmbracoObjectTypes.Media, allMediaStartNodes, searchFrom, ignoreUserStartNodes, _entityService);
break;
case UmbracoEntityTypes.Document:
type = "content";
fields.AddRange(_treeSearcherFields.GetBackOfficeDocumentFields());
var allContentStartNodes = currentUser != null
? currentUser.CalculateContentStartNodeIds(_entityService)
: Array.Empty<int>();
AppendPath(sb, UmbracoObjectTypes.Document, allContentStartNodes, searchFrom, ignoreUserStartNodes, _entityService);
break;
default:
throw new NotSupportedException("The " + typeof(BackOfficeExamineSearcher) + " currently does not support searching against object type " + entityType);
}
if (!_examineManager.TryGetIndex(indexName, out var index))
throw new InvalidOperationException("No index found by name " + indexName);
var internalSearcher = index.GetSearcher();
if (!BuildQuery(sb, query, searchFrom, fields, type))
{
totalFound = 0;
return Enumerable.Empty<ISearchResult>();
}
var result = internalSearcher.CreateQuery().NativeQuery(sb.ToString())
//only return the number of items specified to read up to the amount of records to fill from 0 -> the number of items on the page requested
.Execute(Convert.ToInt32(pageSize * (pageIndex + 1)));
totalFound = result.TotalItemCount;
var pagedResult = result.Skip(Convert.ToInt32(pageIndex));
return pagedResult;
}
private bool BuildQuery(StringBuilder sb, string query, string searchFrom, List<string> fields, string type)
{
//build a lucene query:
// the nodeName will be boosted 10x without wildcards
// then nodeName will be matched normally with wildcards
// the rest will be normal without wildcards
var allLangs = _languageService.GetAllLanguages().Select(x => x.IsoCode.ToLowerInvariant()).ToList();
//check if text is surrounded by single or double quotes, if so, then exact match
var surroundedByQuotes = Regex.IsMatch(query, "^\".*?\"$")
|| Regex.IsMatch(query, "^\'.*?\'$");
if (surroundedByQuotes)
{
//strip quotes, escape string, the replace again
query = query.Trim('\"', '\'');
query = Lucene.Net.QueryParsers.QueryParser.Escape(query);
//nothing to search
if (searchFrom.IsNullOrWhiteSpace() && query.IsNullOrWhiteSpace())
{
return false;
}
//update the query with the query term
if (query.IsNullOrWhiteSpace() == false)
{
//add back the surrounding quotes
query = string.Format("{0}{1}{0}", "\"", query);
sb.Append("+(");
AppendNodeNamePhraseWithBoost(sb, query, allLangs);
foreach (var f in fields)
{
//additional fields normally
sb.Append(f);
sb.Append(": (");
sb.Append(query);
sb.Append(") ");
}
sb.Append(") ");
}
}
else
{
var trimmed = query.Trim(new[] { '\"', '\'' });
//nothing to search
if (searchFrom.IsNullOrWhiteSpace() && trimmed.IsNullOrWhiteSpace())
{
return false;
}
//update the query with the query term
if (trimmed.IsNullOrWhiteSpace() == false)
{
query = Lucene.Net.QueryParsers.QueryParser.Escape(query);
var querywords = query.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
sb.Append("+(");
AppendNodeNameExactWithBoost(sb, query, allLangs);
AppendNodeNameWithWildcards(sb, querywords, allLangs);
foreach (var f in fields)
{
var queryWordsReplaced = new string[querywords.Length];
// when searching file names containing hyphens we need to replace the hyphens with spaces
if (f.Equals(UmbracoExamineFieldNames.UmbracoFileFieldName))
{
for (var index = 0; index < querywords.Length; index++)
{
queryWordsReplaced[index] = querywords[index].Replace("\\-", " ").Replace("_", " ").Trim(" ");
}
}
else
{
queryWordsReplaced = querywords;
}
//additional fields normally
sb.Append(f);
sb.Append(":");
sb.Append("(");
foreach (var w in queryWordsReplaced)
{
sb.Append(w.ToLower());
sb.Append("* ");
}
sb.Append(")");
sb.Append(" ");
}
sb.Append(") ");
}
}
//must match index type
sb.Append("+__IndexType:");
sb.Append(type);
return true;
}
private void AppendNodeNamePhraseWithBoost(StringBuilder sb, string query, IEnumerable<string> allLangs)
{
//node name exactly boost x 10
sb.Append("nodeName: (");
sb.Append(query.ToLower());
sb.Append(")^10.0 ");
//also search on all variant node names
foreach (var lang in allLangs)
{
//node name exactly boost x 10
sb.Append($"nodeName_{lang}: (");
sb.Append(query.ToLower());
sb.Append(")^10.0 ");
}
}
private void AppendNodeNameExactWithBoost(StringBuilder sb, string query, IEnumerable<string> allLangs)
{
//node name exactly boost x 10
sb.Append("nodeName:");
sb.Append("\"");
sb.Append(query.ToLower());
sb.Append("\"");
sb.Append("^10.0 ");
//also search on all variant node names
foreach (var lang in allLangs)
{
//node name exactly boost x 10
sb.Append($"nodeName_{lang}:");
sb.Append("\"");
sb.Append(query.ToLower());
sb.Append("\"");
sb.Append("^10.0 ");
}
}
private void AppendNodeNameWithWildcards(StringBuilder sb, string[] querywords, IEnumerable<string> allLangs)
{
//node name normally with wildcards
sb.Append("nodeName:");
sb.Append("(");
foreach (var w in querywords)
{
sb.Append(w.ToLower());
sb.Append("* ");
}
sb.Append(") ");
//also search on all variant node names
foreach (var lang in allLangs)
{
//node name normally with wildcards
sb.Append($"nodeName_{lang}:");
sb.Append("(");
foreach (var w in querywords)
{
sb.Append(w.ToLower());
sb.Append("* ");
}
sb.Append(") ");
}
}
private void AppendPath(StringBuilder sb, UmbracoObjectTypes objectType, int[] startNodeIds, string searchFrom, bool ignoreUserStartNodes, IEntityService entityService)
{
if (sb == null) throw new ArgumentNullException(nameof(sb));
if (entityService == null) throw new ArgumentNullException(nameof(entityService));
UdiParser.TryParse(searchFrom, true, out var udi);
searchFrom = udi == null ? searchFrom : entityService.GetId(udi).Result.ToString();
var entityPath = int.TryParse(searchFrom, out var searchFromId) && searchFromId > 0
? entityService.GetAllPaths(objectType, searchFromId).FirstOrDefault()
: null;
if (entityPath != null)
{
// find... only what's underneath
sb.Append("+__Path:");
AppendPath(sb, entityPath.Path, false);
sb.Append(" ");
}
else if (startNodeIds.Length == 0)
{
// make sure we don't find anything
sb.Append("+__Path:none ");
}
else if (startNodeIds.Contains(-1) == false && ignoreUserStartNodes == false) // -1 = no restriction
{
var entityPaths = entityService.GetAllPaths(objectType, startNodeIds);
// for each start node, find the start node, and what's underneath
// +__Path:(-1*,1234 -1*,1234,* -1*,5678 -1*,5678,* ...)
sb.Append("+__Path:(");
var first = true;
foreach (var ep in entityPaths)
{
if (first)
first = false;
else
sb.Append(" ");
AppendPath(sb, ep.Path, true);
}
sb.Append(") ");
}
}
private void AppendPath(StringBuilder sb, string path, bool includeThisNode)
{
path = path.Replace("-", "\\-").Replace(",", "\\,");
if (includeThisNode)
{
sb.Append(path);
sb.Append(" ");
}
sb.Append(path);
sb.Append("\\,*");
}
}
}

7
src/Umbraco.Examine.Lucene/ExamineExtensions.cs
View File

@@ -1,7 +1,5 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using Examine;
using Examine.LuceneEngine.Providers;
using Lucene.Net.Analysis;
@@ -15,13 +13,12 @@ using System.Threading;
namespace Umbraco.Examine
{
/// <summary>
/// Extension methods for the LuceneIndex
/// </summary>
public static class ExamineExtensions
{
private static bool _isConfigured = false;
private static object _configuredInit = null;
private static object _isConfiguredLocker = new object();
@@ -74,7 +71,7 @@ namespace Umbraco.Examine
/// <remarks>
/// This is not thread safe, use with care
/// </remarks>
internal static void ConfigureLuceneIndexes(this IExamineManager examineManager, ILogger logger, bool disableExamineIndexing)
private static void ConfigureLuceneIndexes(this IExamineManager examineManager, ILogger logger, bool disableExamineIndexing)
{
foreach (var luceneIndexer in examineManager.Indexes.OfType<LuceneIndex>())
{

50
src/Umbraco.Examine.Lucene/ExamineLuceneComponent.cs Normal file
View File

@@ -0,0 +1,50 @@
using Examine;
using Examine.LuceneEngine.Directories;
using Umbraco.Core;
using Umbraco.Core.Composing;
using Umbraco.Core.Logging;
namespace Umbraco.Examine
{
public sealed class ExamineLuceneComponent : IComponent
{
private readonly IndexRebuilder _indexRebuilder;
private readonly IExamineManager _examineManager;
private readonly IMainDom _mainDom;
private readonly ILogger _logger;
public ExamineLuceneComponent(IndexRebuilder indexRebuilder, IExamineManager examineManager, IMainDom mainDom, ILogger logger)
{
_indexRebuilder = indexRebuilder;
_examineManager = examineManager;
_mainDom = mainDom;
_logger = logger;
}
public void Initialize()
{
//we want to tell examine to use a different fs lock instead of the default NativeFSFileLock which could cause problems if the AppDomain
//terminates and in some rare cases would only allow unlocking of the file if IIS is forcefully terminated. Instead we'll rely on the simplefslock
//which simply checks the existence of the lock file
DirectoryFactory.DefaultLockFactory = d =>
{
var simpleFsLockFactory = new NoPrefixSimpleFsLockFactory(d);
return simpleFsLockFactory;
};
_indexRebuilder.RebuildingIndexes += IndexRebuilder_RebuildingIndexes;
}
/// <summary>
/// Handles event to ensure that all lucene based indexes are properly configured before rebuilding
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void IndexRebuilder_RebuildingIndexes(object sender, IndexRebuildingEventArgs e) => _examineManager.ConfigureIndexes(_mainDom, _logger);
public void Terminate()
{
}
}
}

21
src/Umbraco.Examine.Lucene/ExamineLuceneComposer.cs Normal file
View File

@@ -0,0 +1,21 @@
using Umbraco.Core;
using Umbraco.Core.Composing;
using Umbraco.Web.Search;
namespace Umbraco.Examine
{
// We want to run after core composers since we are replacing some items
[ComposeAfter(typeof(ICoreComposer))]
[RuntimeLevel(MinLevel = RuntimeLevel.Run)]
public sealed class ExamineLuceneComposer : ComponentComposer<ExamineLuceneComponent>
{
public override void Compose(Composition composition)
{
base.Compose(composition);
composition.RegisterUnique<IBackOfficeExamineSearcher, BackOfficeExamineSearcher>();
composition.RegisterUnique<IUmbracoIndexesCreator, UmbracoIndexesCreator>();
composition.RegisterUnique<IIndexDiagnosticsFactory, LuceneIndexDiagnosticsFactory>();
}
}
}

33
src/Umbraco.Examine.Lucene/ExamineLuceneFinalComponent.cs Normal file
View File

@@ -0,0 +1,33 @@
using Examine;
using Umbraco.Core;
using Umbraco.Core.Composing;
using Umbraco.Core.Logging;
namespace Umbraco.Examine
{
public class ExamineLuceneFinalComponent : IComponent
{
private readonly IProfilingLogger _logger;
private readonly IExamineManager _examineManager;
private readonly IMainDom _mainDom;
public ExamineLuceneFinalComponent(IProfilingLogger logger, IExamineManager examineManager, IMainDom mainDom)
{
_logger = logger;
_examineManager = examineManager;
_mainDom = mainDom;
}
public void Initialize()
{
if (!_mainDom.IsMainDom) return;
// Ensures all lucene based indexes are unlocked and ready to go
_examineManager.ConfigureIndexes(_mainDom, _logger);
}
public void Terminate()
{
}
}
}

13
src/Umbraco.Examine.Lucene/ExamineLuceneFinalComposer.cs Normal file
View File

@@ -0,0 +1,13 @@
using Umbraco.Core;
using Umbraco.Core.Composing;
namespace Umbraco.Examine
{
// examine's Lucene final composer composes after all user composers
// and *also* after ICoreComposer (in case IUserComposer is disabled)
[ComposeAfter(typeof(IUserComposer))]
[ComposeAfter(typeof(ICoreComposer))]
[RuntimeLevel(MinLevel = RuntimeLevel.Run)]
public class ExamineLuceneFinalComposer : ComponentComposer<ExamineLuceneFinalComponent>
{ }
}

35
src/Umbraco.Examine.Lucene/LuceneIndexDiagnosticsFactory.cs Normal file
View File

@@ -0,0 +1,35 @@
using Examine;
using Examine.LuceneEngine.Providers;
using Umbraco.Core.Logging;
using Umbraco.Core.IO;
namespace Umbraco.Examine
{
/// <summary>
/// Implementation of <see cref="IIndexDiagnosticsFactory"/> which returns <see cref="LuceneIndexDiagnostics"/>
/// for lucene based indexes that don't have an implementation else fallsback to the default <see cref="IndexDiagnosticsFactory"/> implementation.
/// </summary>
public class LuceneIndexDiagnosticsFactory : IndexDiagnosticsFactory
{
private readonly ILogger _logger;
private readonly IIOHelper _ioHelper;
public LuceneIndexDiagnosticsFactory(ILogger logger, IIOHelper ioHelper)
{
_logger = logger;
_ioHelper = ioHelper;
}
public override IIndexDiagnostics Create(IIndex index)
{
if (!(index is IIndexDiagnostics indexDiag))
{
if (index is LuceneIndex luceneIndex)
indexDiag = new LuceneIndexDiagnostics(luceneIndex, _logger, _ioHelper);
else
indexDiag = base.Create(index);
}
return indexDiag;
}
}
}

7
src/Umbraco.Examine.Lucene/Umbraco.Examine.Lucene.csproj
View File

@@ -66,8 +66,14 @@
</PackageReference>
</ItemGroup>
<ItemGroup>
<Compile Include="BackOfficeExamineSearcher.cs" />
<Compile Include="ExamineExtensions.cs" />
<Compile Include="ExamineLuceneComponent.cs" />
<Compile Include="ExamineLuceneComposer.cs" />
<Compile Include="ExamineLuceneFinalComponent.cs" />
<Compile Include="ExamineLuceneFinalComposer.cs" />
<Compile Include="LuceneIndexDiagnostics.cs" />
<Compile Include="LuceneIndexDiagnosticsFactory.cs" />
<Compile Include="UmbracoExamineExtensions.cs" />
<Compile Include="NoPrefixSimpleFsLockFactory.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
@@ -75,6 +81,7 @@
<Compile Include="UmbracoExamineIndexDiagnostics.cs" />
<Compile Include="UmbracoExamineIndex.cs" />
<Compile Include="LuceneIndexCreator.cs" />
<Compile Include="UmbracoIndexesCreator.cs" />
<Compile Include="UmbracoMemberIndex.cs" />
<Compile Include="..\SolutionInfo.cs">
<Link>Properties\SolutionInfo.cs</Link>

4
src/Umbraco.Web/Search/UmbracoIndexesCreator.cs → src/Umbraco.Examine.Lucene/UmbracoIndexesCreator.cs
View File

@@ -1,5 +1,4 @@
using System;
using System.Collections.Generic;
using System.Collections.Generic;
using Umbraco.Core.Logging;
using Umbraco.Core.Services;
using Umbraco.Examine;
@@ -9,7 +8,6 @@ using Examine;
using Umbraco.Core;
using Umbraco.Core.Composing;
using Umbraco.Core.IO;
using Umbraco.Web.PublishedCache.NuCache;
namespace Umbraco.Web.Search
{

4
src/Umbraco.Examine.Lucene/UmbracoMemberIndex.cs
View File

@@ -1,6 +1,4 @@
using System.Collections.Generic;
using Examine;
using Examine.LuceneEngine;
using Examine;
using Lucene.Net.Analysis;
using Umbraco.Core;
using Umbraco.Core.IO;

3
src/Umbraco.Web/Search/GenericIndexDiagnostics.cs → src/Umbraco.Examine/GenericIndexDiagnostics.cs
View File

@@ -4,9 +4,8 @@ using System.Linq;
using Examine;
using Umbraco.Core;
using Umbraco.Core.Composing;
using Umbraco.Examine;
namespace Umbraco.Web.Search
namespace Umbraco.Examine
{
/// <summary>

17
src/Umbraco.Examine/IBackOfficeExamineSearcher.cs Normal file
View File

@@ -0,0 +1,17 @@
using Examine;
using System.Collections.Generic;
using Umbraco.Web.Models.ContentEditing;
namespace Umbraco.Examine
{
/// <summary>
/// Used to search the back office for Examine indexed entities (Documents, Media and Members)
/// </summary>
public interface IBackOfficeExamineSearcher
{
IEnumerable<ISearchResult> Search(string query,
UmbracoEntityTypes entityType,
int pageSize,
long pageIndex, out long totalFound, string searchFrom = null, bool ignoreUserStartNodes = false);
}
}

1
src/Umbraco.Examine/IIndexDiagnostics.cs
View File

@@ -4,6 +4,7 @@ using Umbraco.Core;
namespace Umbraco.Examine
{
/// <summary>
/// Exposes diagnostic information about an index
/// </summary>

13
src/Umbraco.Examine/IIndexDiagnosticsFactory.cs Normal file
View File

@@ -0,0 +1,13 @@
using Examine;
namespace Umbraco.Examine
{
/// <summary>
/// Creates <see cref="IIndexDiagnostics"/> for an index if it doesn't implement <see cref="IIndexDiagnostics"/>
/// </summary>
public interface IIndexDiagnosticsFactory
{
IIndexDiagnostics Create(IIndex index);
}
}

6
src/Umbraco.Web/Search/IUmbracoIndexesCreator.cs → src/Umbraco.Examine/IUmbracoIndexesCreator.cs
View File

@@ -1,8 +1,4 @@
using System.Collections.Generic;
using Examine;
using Umbraco.Examine;
namespace Umbraco.Web.Search
namespace Umbraco.Examine
{
/// <inheritdoc />
/// <summary>

17
src/Umbraco.Examine/IndexDiagnosticsFactory.cs Normal file
View File

@@ -0,0 +1,17 @@
using Examine;
namespace Umbraco.Examine
{
/// <summary>
/// Default implementation of <see cref="IIndexDiagnosticsFactory"/> which returns <see cref="GenericIndexDiagnostics"/> for indexes that don't have an implementation
/// </summary>
public class IndexDiagnosticsFactory : IIndexDiagnosticsFactory
{
public virtual IIndexDiagnostics Create(IIndex index)
{
if (!(index is IIndexDiagnostics indexDiag))
indexDiag = new GenericIndexDiagnostics(index);
return indexDiag;
}
}
}

10
src/Umbraco.Examine/IndexRebuilder.cs
View File

@@ -5,7 +5,7 @@ using System.Threading.Tasks;
using Examine;
namespace Umbraco.Examine
{
{
/// <summary>
/// Utility to rebuild all indexes ensuring minimal data queries
@@ -45,6 +45,8 @@ namespace Umbraco.Examine
if (indexes.Length == 0) return;
OnRebuildingIndexes(new IndexRebuildingEventArgs(indexes));
foreach (var index in indexes)
{
index.CreateIndex(); // clear the index
@@ -54,5 +56,11 @@ namespace Umbraco.Examine
Parallel.ForEach(_populators, populator => populator.Populate(indexes));
}
/// <summary>
/// Event raised when indexes are being rebuilt
/// </summary>
public event EventHandler<IndexRebuildingEventArgs> RebuildingIndexes;
private void OnRebuildingIndexes(IndexRebuildingEventArgs args) => RebuildingIndexes?.Invoke(this, args);
}
}

19
src/Umbraco.Examine/IndexRebuildingEventArgs.cs Normal file
View File

@@ -0,0 +1,19 @@
using System;
using System.Collections.Generic;
using Examine;
namespace Umbraco.Examine
{
public class IndexRebuildingEventArgs : EventArgs
{
public IndexRebuildingEventArgs(IEnumerable<IIndex> indexes)
{
Indexes = indexes;
}
/// <summary>
/// The indexes being rebuilt
/// </summary>
public IEnumerable<IIndex> Indexes { get; }
}
}

20
src/Umbraco.Web/Editors/ExamineManagementController.cs
View File

@@ -5,7 +5,6 @@ using System.Net;
using System.Net.Http;
using System.Web.Http;
using Examine;
using Examine.LuceneEngine.Providers;
using Umbraco.Core;
using Umbraco.Core.Cache;
using Umbraco.Core.IO;
@@ -24,17 +23,19 @@ namespace Umbraco.Web.Editors
private readonly IExamineManager _examineManager;
private readonly ILogger _logger;
private readonly IIOHelper _ioHelper;
private readonly IIndexDiagnosticsFactory _indexDiagnosticsFactory;
private readonly IAppPolicyCache _runtimeCache;
private readonly IndexRebuilder _indexRebuilder;
public ExamineManagementController(IExamineManager examineManager, ILogger logger, IIOHelper ioHelper,
public ExamineManagementController(IExamineManager examineManager, ILogger logger, IIOHelper ioHelper, IIndexDiagnosticsFactory indexDiagnosticsFactory,
AppCaches appCaches,
IndexRebuilder indexRebuilder)
{
_examineManager = examineManager;
_logger = logger;
_ioHelper = ioHelper;
_indexDiagnosticsFactory = indexDiagnosticsFactory;
_runtimeCache = appCaches.RuntimeCache;
_indexRebuilder = indexRebuilder;
}
@@ -69,9 +70,8 @@ namespace Umbraco.Web.Editors
if (!msg.IsSuccessStatusCode)
throw new HttpResponseException(msg);
var results = global::Umbraco.Examine.ExamineExtensions.TryParseLuceneQuery(query)
? searcher.CreateQuery().NativeQuery(query).Execute(maxResults: pageSize * (pageIndex + 1))
: searcher.Search(query, maxResults: pageSize * (pageIndex + 1));
// NativeQuery will work for a single word/phrase too (but depends on the implementation) the lucene one will work.
var results = searcher.CreateQuery().NativeQuery(query).Execute(maxResults: pageSize * (pageIndex + 1));
var pagedResults = results.Skip(pageIndex * pageSize);
@@ -173,19 +173,11 @@ namespace Umbraco.Web.Editors
}
}
private ExamineIndexModel CreateModel(IIndex index)
{
var indexName = index.Name;
if (!(index is IIndexDiagnostics indexDiag))
{
if (index is LuceneIndex luceneIndex)
indexDiag = new LuceneIndexDiagnostics(luceneIndex, Logger, _ioHelper);
else
indexDiag = new GenericIndexDiagnostics(index);
}
var indexDiag = _indexDiagnosticsFactory.Create(index);
var isHealth = indexDiag.IsHealthy();
var properties = new Dictionary<string, object>

2
src/Umbraco.Web/ExamineExtensions.cs
View File

@@ -1,8 +1,6 @@
using System;
using System.Collections.Generic;
using System.Linq;
using Examine;
using Examine.LuceneEngine.Providers;
using Umbraco.Core.Models.PublishedContent;
using Umbraco.Examine;
using Umbraco.Web.PublishedCache;

2
src/Umbraco.Web/Models/Mapping/EntityMapDefinition.cs
View File

@@ -1,8 +1,6 @@
using System;
using System.Collections.Generic;
using System.Linq;
using Examine;
using Examine.LuceneEngine.Providers;
using Umbraco.Core;
using Umbraco.Core.Mapping;
using Umbraco.Core.Models;

1
src/Umbraco.Web/Runtime/WebInitialComposer.cs
View File

@@ -44,6 +44,7 @@ using Umbraco.Web.Trees;
using Umbraco.Web.WebApi;
using Current = Umbraco.Web.Composing.Current;
using Umbraco.Web.PropertyEditors;
using Umbraco.Examine;
namespace Umbraco.Web.Runtime
{

1
src/Umbraco.Web/Search/BackgroundIndexRebuilder.cs
View File

@@ -118,7 +118,6 @@ namespace Umbraco.Web.Search
if (_waitMilliseconds > 0)
Thread.Sleep(_waitMilliseconds);
_indexRebuilder.ExamineManager.ConfigureIndexes(_mainDom, _logger);
_indexRebuilder.RebuildIndexes(_onlyEmptyIndexes);
}
}

12
src/Umbraco.Web/Search/ExamineComponent.cs
View File

@@ -13,9 +13,6 @@ using Umbraco.Core.Services.Changes;
using Umbraco.Core.Sync;
using Umbraco.Web.Cache;
using Umbraco.Examine;
using Examine.LuceneEngine.Directories;
using Umbraco.Web.Composing;
using System.ComponentModel;
namespace Umbraco.Web.Search
{
@@ -65,15 +62,6 @@ namespace Umbraco.Web.Search
public void Initialize()
{
//we want to tell examine to use a different fs lock instead of the default NativeFSFileLock which could cause problems if the AppDomain
//terminates and in some rare cases would only allow unlocking of the file if IIS is forcefully terminated. Instead we'll rely on the simplefslock
//which simply checks the existence of the lock file
DirectoryFactory.DefaultLockFactory = d =>
{
var simpleFsLockFactory = new NoPrefixSimpleFsLockFactory(d);
return simpleFsLockFactory;
};
//let's deal with shutting down Examine with MainDom
var examineShutdownRegistered = _mainDom.Register(() =>
{

2
src/Umbraco.Web/Search/ExamineComposer.cs
View File

@@ -30,7 +30,7 @@ namespace Umbraco.Web.Search
composition.Register<IndexRebuilder>(Lifetime.Singleton);
composition.RegisterUnique<IUmbracoIndexConfig, UmbracoIndexConfig>();
composition.RegisterUnique<IUmbracoIndexesCreator, UmbracoIndexesCreator>();
composition.RegisterUnique<IIndexDiagnosticsFactory, IndexDiagnosticsFactory>();
composition.RegisterUnique<IPublishedContentValueSetBuilder>(factory =>
new ContentValueSetBuilder(
factory.GetInstance<PropertyEditorCollection>(),

8
src/Umbraco.Web/Search/ExamineFinalComponent.cs
View File

@@ -12,15 +12,11 @@ namespace Umbraco.Web.Search
/// </summary>
public sealed class ExamineFinalComponent : IComponent
{
private readonly IProfilingLogger _logger;
private readonly IExamineManager _examineManager;
BackgroundIndexRebuilder _indexRebuilder;
private readonly IMainDom _mainDom;
public ExamineFinalComponent(IProfilingLogger logger, IExamineManager examineManager, BackgroundIndexRebuilder indexRebuilder, IMainDom mainDom)
public ExamineFinalComponent(BackgroundIndexRebuilder indexRebuilder, IMainDom mainDom)
{
_logger = logger;
_examineManager = examineManager;
_indexRebuilder = indexRebuilder;
_mainDom = mainDom;
}
@@ -29,8 +25,6 @@ namespace Umbraco.Web.Search
{
if (!_mainDom.IsMainDom) return;
_examineManager.ConfigureIndexes(_mainDom, _logger);
// TODO: Instead of waiting 5000 ms, we could add an event handler on to fulfilling the first request, then start?
_indexRebuilder.RebuildIndexes(true, 5000);
}

310
src/Umbraco.Web/Search/UmbracoTreeSearcher.cs
View File

@@ -1,8 +1,6 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using Examine;
using Umbraco.Core;
using Umbraco.Core.Mapping;
@@ -22,29 +20,26 @@ namespace Umbraco.Web.Search
/// </summary>
public class UmbracoTreeSearcher
{
private readonly IExamineManager _examineManager;
private readonly UmbracoContext _umbracoContext;
private readonly ILocalizationService _languageService;
private readonly IEntityService _entityService;
private readonly UmbracoMapper _mapper;
private readonly ISqlContext _sqlContext;
private readonly IUmbracoTreeSearcherFields _umbracoTreeSearcherFields;
private readonly IBackOfficeExamineSearcher _backOfficeExamineSearcher;
public UmbracoTreeSearcher(IExamineManager examineManager,
UmbracoContext umbracoContext,
public UmbracoTreeSearcher(UmbracoContext umbracoContext,
ILocalizationService languageService,
IEntityService entityService,
UmbracoMapper mapper,
ISqlContext sqlContext,IUmbracoTreeSearcherFields umbracoTreeSearcherFields)
ISqlContext sqlContext, IBackOfficeExamineSearcher backOfficeExamineSearcher)
{
_examineManager = examineManager ?? throw new ArgumentNullException(nameof(examineManager));
_umbracoContext = umbracoContext;
_languageService = languageService;
_entityService = entityService;
_mapper = mapper;
_sqlContext = sqlContext;
_umbracoTreeSearcherFields = umbracoTreeSearcherFields;
_backOfficeExamineSearcher = backOfficeExamineSearcher;
}
/// <summary>
@@ -66,72 +61,7 @@ namespace Umbraco.Web.Search
int pageSize,
long pageIndex, out long totalFound, string searchFrom = null, bool ignoreUserStartNodes = false)
{
var sb = new StringBuilder();
string type;
var indexName = Constants.UmbracoIndexes.InternalIndexName;
var fields = _umbracoTreeSearcherFields.GetBackOfficeFields().ToList();
// TODO: WE should try to allow passing in a lucene raw query, however we will still need to do some manual string
// manipulation for things like start paths, member types, etc...
//if (Examine.ExamineExtensions.TryParseLuceneQuery(query))
//{
//}
//special GUID check since if a user searches on one specifically we need to escape it
if (Guid.TryParse(query, out var g))
{
query = "\"" + g.ToString() + "\"";
}
switch (entityType)
{
case UmbracoEntityTypes.Member:
indexName = Constants.UmbracoIndexes.MembersIndexName;
type = "member";
fields.AddRange(_umbracoTreeSearcherFields.GetBackOfficeMembersFields());
if (searchFrom != null && searchFrom != Constants.Conventions.MemberTypes.AllMembersListId && searchFrom.Trim() != "-1")
{
sb.Append("+__NodeTypeAlias:");
sb.Append(searchFrom);
sb.Append(" ");
}
break;
case UmbracoEntityTypes.Media:
type = "media";
fields.AddRange(_umbracoTreeSearcherFields.GetBackOfficeMediaFields());
var allMediaStartNodes = _umbracoContext.Security.CurrentUser.CalculateMediaStartNodeIds(_entityService);
AppendPath(sb, UmbracoObjectTypes.Media, allMediaStartNodes, searchFrom, ignoreUserStartNodes, _entityService);
break;
case UmbracoEntityTypes.Document:
type = "content";
fields.AddRange(_umbracoTreeSearcherFields.GetBackOfficeDocumentFields());
var allContentStartNodes = _umbracoContext.Security.CurrentUser.CalculateContentStartNodeIds(_entityService);
AppendPath(sb, UmbracoObjectTypes.Document, allContentStartNodes, searchFrom, ignoreUserStartNodes, _entityService);
break;
default:
throw new NotSupportedException("The " + typeof(UmbracoTreeSearcher) + " currently does not support searching against object type " + entityType);
}
if (!_examineManager.TryGetIndex(indexName, out var index))
throw new InvalidOperationException("No index found by name " + indexName);
var internalSearcher = index.GetSearcher();
if (!BuildQuery(sb, query, searchFrom, fields, type))
{
totalFound = 0;
return Enumerable.Empty<SearchResultEntity>();
}
var result = internalSearcher.CreateQuery().NativeQuery(sb.ToString())
//only return the number of items specified to read up to the amount of records to fill from 0 -> the number of items on the page requested
.Execute(Convert.ToInt32(pageSize * (pageIndex + 1)));
totalFound = result.TotalItemCount;
var pagedResult = result.Skip(Convert.ToInt32(pageIndex));
var pagedResult = _backOfficeExamineSearcher.Search(query, entityType, pageSize, pageIndex, out totalFound, searchFrom, ignoreUserStartNodes);
switch (entityType)
{
@@ -166,236 +96,6 @@ namespace Umbraco.Web.Search
return _mapper.MapEnumerable<IEntitySlim, SearchResultEntity>(results);
}
private bool BuildQuery(StringBuilder sb, string query, string searchFrom, List<string> fields, string type)
{
//build a lucene query:
// the nodeName will be boosted 10x without wildcards
// then nodeName will be matched normally with wildcards
// the rest will be normal without wildcards
var allLangs = _languageService.GetAllLanguages().Select(x => x.IsoCode.ToLowerInvariant()).ToList();
//check if text is surrounded by single or double quotes, if so, then exact match
var surroundedByQuotes = Regex.IsMatch(query, "^\".*?\"$")
|| Regex.IsMatch(query, "^\'.*?\'$");
if (surroundedByQuotes)
{
//strip quotes, escape string, the replace again
query = query.Trim('\"', '\'');
query = Lucene.Net.QueryParsers.QueryParser.Escape(query);
//nothing to search
if (searchFrom.IsNullOrWhiteSpace() && query.IsNullOrWhiteSpace())
{
return false;
}
//update the query with the query term
if (query.IsNullOrWhiteSpace() == false)
{
//add back the surrounding quotes
query = string.Format("{0}{1}{0}", "\"", query);
sb.Append("+(");
AppendNodeNamePhraseWithBoost(sb, query, allLangs);
foreach (var f in fields)
{
//additional fields normally
sb.Append(f);
sb.Append(": (");
sb.Append(query);
sb.Append(") ");
}
sb.Append(") ");
}
}
else
{
var trimmed = query.Trim(new[] { '\"', '\'' });
//nothing to search
if (searchFrom.IsNullOrWhiteSpace() && trimmed.IsNullOrWhiteSpace())
{
return false;
}
//update the query with the query term
if (trimmed.IsNullOrWhiteSpace() == false)
{
query = Lucene.Net.QueryParsers.QueryParser.Escape(query);
var querywords = query.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
sb.Append("+(");
AppendNodeNameExactWithBoost(sb, query, allLangs);
AppendNodeNameWithWildcards(sb, querywords, allLangs);
foreach (var f in fields)
{
var queryWordsReplaced = new string[querywords.Length];
// when searching file names containing hyphens we need to replace the hyphens with spaces
if (f.Equals(UmbracoExamineFieldNames.UmbracoFileFieldName))
{
for (var index = 0; index < querywords.Length; index++)
{
queryWordsReplaced[index] = querywords[index].Replace("\\-", " ").Replace("_", " ").Trim(" ");
}
}
else
{
queryWordsReplaced = querywords;
}
//additional fields normally
sb.Append(f);
sb.Append(":");
sb.Append("(");
foreach (var w in queryWordsReplaced)
{
sb.Append(w.ToLower());
sb.Append("* ");
}
sb.Append(")");
sb.Append(" ");
}
sb.Append(") ");
}
}
//must match index type
sb.Append("+__IndexType:");
sb.Append(type);
return true;
}
private void AppendNodeNamePhraseWithBoost(StringBuilder sb, string query, IEnumerable<string> allLangs)
{
//node name exactly boost x 10
sb.Append("nodeName: (");
sb.Append(query.ToLower());
sb.Append(")^10.0 ");
//also search on all variant node names
foreach (var lang in allLangs)
{
//node name exactly boost x 10
sb.Append($"nodeName_{lang}: (");
sb.Append(query.ToLower());
sb.Append(")^10.0 ");
}
}
private void AppendNodeNameExactWithBoost(StringBuilder sb, string query, IEnumerable<string> allLangs)
{
//node name exactly boost x 10
sb.Append("nodeName:");
sb.Append("\"");
sb.Append(query.ToLower());
sb.Append("\"");
sb.Append("^10.0 ");
//also search on all variant node names
foreach (var lang in allLangs)
{
//node name exactly boost x 10
sb.Append($"nodeName_{lang}:");
sb.Append("\"");
sb.Append(query.ToLower());
sb.Append("\"");
sb.Append("^10.0 ");
}
}
private void AppendNodeNameWithWildcards(StringBuilder sb, string[] querywords, IEnumerable<string> allLangs)
{
//node name normally with wildcards
sb.Append("nodeName:");
sb.Append("(");
foreach (var w in querywords)
{
sb.Append(w.ToLower());
sb.Append("* ");
}
sb.Append(") ");
//also search on all variant node names
foreach (var lang in allLangs)
{
//node name normally with wildcards
sb.Append($"nodeName_{lang}:");
sb.Append("(");
foreach (var w in querywords)
{
sb.Append(w.ToLower());
sb.Append("* ");
}
sb.Append(") ");
}
}
private void AppendPath(StringBuilder sb, UmbracoObjectTypes objectType, int[] startNodeIds, string searchFrom, bool ignoreUserStartNodes, IEntityService entityService)
{
if (sb == null) throw new ArgumentNullException(nameof(sb));
if (entityService == null) throw new ArgumentNullException(nameof(entityService));
UdiParser.TryParse(searchFrom, true, out var udi);
searchFrom = udi == null ? searchFrom : entityService.GetId(udi).Result.ToString();
var entityPath = int.TryParse(searchFrom, out var searchFromId) && searchFromId > 0
? entityService.GetAllPaths(objectType, searchFromId).FirstOrDefault()
: null;
if (entityPath != null)
{
// find... only what's underneath
sb.Append("+__Path:");
AppendPath(sb, entityPath.Path, false);
sb.Append(" ");
}
else if (startNodeIds.Length == 0)
{
// make sure we don't find anything
sb.Append("+__Path:none ");
}
else if (startNodeIds.Contains(-1) == false && ignoreUserStartNodes == false) // -1 = no restriction
{
var entityPaths = entityService.GetAllPaths(objectType, startNodeIds);
// for each start node, find the start node, and what's underneath
// +__Path:(-1*,1234 -1*,1234,* -1*,5678 -1*,5678,* ...)
sb.Append("+__Path:(");
var first = true;
foreach (var ep in entityPaths)
{
if (first)
first = false;
else
sb.Append(" ");
AppendPath(sb, ep.Path, true);
}
sb.Append(") ");
}
}
private void AppendPath(StringBuilder sb, string path, bool includeThisNode)
{
path = path.Replace("-", "\\-").Replace(",", "\\,");
if (includeThisNode)
{
sb.Append(path);
sb.Append(" ");
}
sb.Append(path);
sb.Append("\\,*");
}
/// <summary>
/// Returns a collection of entities for media based on search results
/// </summary>

11
src/Umbraco.Web/Umbraco.Web.csproj
View File

@@ -65,9 +65,6 @@
<PackageReference Include="Examine.Core">
<Version>2.0.0-alpha.20200128.15</Version>
</PackageReference>
<PackageReference Include="Examine.Lucene">
<Version>2.0.0-alpha.20200128.15</Version>
</PackageReference>
<PackageReference Include="HtmlAgilityPack" Version="1.8.14" />
<PackageReference Include="ImageProcessor">
<Version>2.7.0.100</Version>
@@ -116,10 +113,6 @@
<Project>{f9b7fe05-0f93-4d0d-9c10-690b33ecbbd8}</Project>
<Name>Umbraco.Examine</Name>
</ProjectReference>
<ProjectReference Include="..\Umbraco.Examine.Lucene\Umbraco.Examine.Lucene.csproj">
<Project>{07fbc26b-2927-4a22-8d96-d644c667fecc}</Project>
<Name>Umbraco.Examine.Lucene</Name>
</ProjectReference>
<ProjectReference Include="..\Umbraco.Infrastructure\Umbraco.Infrastructure.csproj">
<Project>{3ae7bf57-966b-45a5-910a-954d7c554441}</Project>
<Name>Umbraco.Infrastructure</Name>
@@ -246,7 +239,6 @@
<Compile Include="Security\UmbracoEmailMessage.cs" />
<Compile Include="Security\UmbracoMembershipProviderBase.cs" />
<Compile Include="Security\UserAwarePasswordHasher.cs" />
<Compile Include="Search\IUmbracoTreeSearcherFields.cs" />
<Compile Include="Search\UmbracoTreeSearcherFields.cs" />
<Compile Include="StringExtensions.cs" />
<Compile Include="Templates\HtmlLocalLinkParser.cs" />
@@ -271,9 +263,6 @@
<Compile Include="Runtime\WebInitialComposer.cs" />
<Compile Include="Scheduling\SchedulerComposer.cs" />
<Compile Include="Search\ExamineComposer.cs" />
<Compile Include="Search\GenericIndexDiagnostics.cs" />
<Compile Include="Search\IUmbracoIndexesCreator.cs" />
<Compile Include="Search\UmbracoIndexesCreator.cs" />
<Compile Include="Security\ActiveDirectoryBackOfficeUserPasswordChecker.cs" />
<Compile Include="Security\BackOfficeClaimsIdentityFactory.cs" />
<Compile Include="Security\BackOfficeUserManagerMarker.cs" />