Merge branch 'temp8-backoffice-search-with-variants' into temp8

This commit is contained in:
Stephan
2018-12-19 16:02:54 +01:00
18 changed files with 253 additions and 157 deletions

View File

@@ -73,7 +73,7 @@ namespace Umbraco.Web.Editors
if (!msg.IsSuccessStatusCode)
throw new HttpResponseException(msg);
var results = TryParseLuceneQuery(query)
var results = Examine.ExamineExtensions.TryParseLuceneQuery(query)
? searcher.Search(searcher.CreateCriteria().RawQuery(query), maxResults: pageSize * (pageIndex + 1))
: searcher.Search(query, true, maxResults: pageSize * (pageIndex + 1));
@@ -92,28 +92,7 @@ namespace Umbraco.Web.Editors
};
}
private bool TryParseLuceneQuery(string query)
{
//TODO: I'd assume there would be a more strict way to parse the query but not that i can find yet, for now we'll
// also do this rudimentary check
if (!query.Contains(":"))
return false;
try
{
//This will pass with a plain old string without any fields, need to figure out a way to have it properly parse
var parsed = new QueryParser(Version.LUCENE_30, "nodeName", new KeywordAnalyzer()).Parse(query);
return true;
}
catch (ParseException)
{
return false;
}
catch (Exception)
{
return false;
}
}
/// <summary>
/// Check if the index has been rebuilt

View File

@@ -122,6 +122,8 @@ namespace Umbraco.Web.Models.Mapping
.ForMember(dest => dest.AdditionalData, opt => opt.Ignore())
.AfterMap((src, dest) =>
{
//TODO: Properly map this (not aftermap)
//get the icon if there is one
dest.Icon = src.Values.ContainsKey(UmbracoExamineIndex.IconFieldName)
? src.Values[UmbracoExamineIndex.IconFieldName]

View File

@@ -55,7 +55,7 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache
ValidateAndSetProperty(valueDictionary, val => _key = Guid.Parse(val), "key");
//ValidateAndSetProperty(valueDictionary, val => _templateId = int.Parse(val), "template", "templateId");
ValidateAndSetProperty(valueDictionary, val => _sortOrder = Int32.Parse(val), "sortOrder");
ValidateAndSetProperty(valueDictionary, val => _name = val, "nodeName", "__nodeName");
ValidateAndSetProperty(valueDictionary, val => _name = val, "nodeName");
ValidateAndSetProperty(valueDictionary, val => _urlName = val, "urlName");
ValidateAndSetProperty(valueDictionary, val => _documentTypeAlias = val, "nodeTypeAlias", LuceneIndex.ItemTypeFieldName);
ValidateAndSetProperty(valueDictionary, val => _documentTypeId = Int32.Parse(val), "nodeType");

View File

@@ -240,9 +240,9 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache
try
{
if (eMgr.TryGetIndex(Constants.Examine.InternalIndexer, out var index))
if (eMgr.TryGetIndex(Constants.UmbracoIndexes.InternalIndexName, out var index))
return index.GetSearcher();
throw new InvalidOperationException($"No index found by name {Constants.Examine.InternalIndexer}");
throw new InvalidOperationException($"No index found by name {Constants.UmbracoIndexes.InternalIndexName}");
}
catch (FileNotFoundException)
{

View File

@@ -270,7 +270,7 @@ namespace Umbraco.Web
{
//fixme: pass in the IExamineManager
indexName = string.IsNullOrEmpty(indexName) ? Constants.Examine.ExternalIndexer : indexName;
indexName = string.IsNullOrEmpty(indexName) ? Constants.UmbracoIndexes.ExternalIndexName : indexName;
if (!ExamineManager.Instance.TryGetIndex(indexName, out var index))
throw new InvalidOperationException("No index found with name " + indexName);
@@ -290,7 +290,7 @@ namespace Umbraco.Web
{
//fixme: pass in the IExamineManager
indexName = string.IsNullOrEmpty(indexName) ? Constants.Examine.ExternalIndexer : indexName;
indexName = string.IsNullOrEmpty(indexName) ? Constants.UmbracoIndexes.ExternalIndexName : indexName;
if (!ExamineManager.Instance.TryGetIndex(indexName, out var index))
throw new InvalidOperationException("No index found with name " + indexName);
@@ -312,8 +312,8 @@ namespace Umbraco.Web
if (searchProvider == null)
{
if (!ExamineManager.Instance.TryGetIndex(Constants.Examine.ExternalIndexer, out var index))
throw new InvalidOperationException("No index found with name " + Constants.Examine.ExternalIndexer);
if (!ExamineManager.Instance.TryGetIndex(Constants.UmbracoIndexes.ExternalIndexName, out var index))
throw new InvalidOperationException("No index found with name " + Constants.UmbracoIndexes.ExternalIndexName);
searchProvider = index.GetSearcher();
}
var results = searchProvider.Search(criteria);

View File

@@ -192,7 +192,7 @@ namespace Umbraco.Web
//fixme: inject IExamineManager
indexName = string.IsNullOrEmpty(indexName)
? Constants.Examine.ExternalIndexer
? Constants.UmbracoIndexes.ExternalIndexName
: indexName;
if (!ExamineManager.Instance.TryGetIndex(indexName, out var index))
@@ -220,8 +220,8 @@ namespace Umbraco.Web
//fixme: inject IExamineManager
if (searcher == null)
{
if (!ExamineManager.Instance.TryGetIndex(Constants.Examine.ExternalIndexer, out var index))
throw new InvalidOperationException($"No index found by name {Constants.Examine.ExternalIndexer}");
if (!ExamineManager.Instance.TryGetIndex(Constants.UmbracoIndexes.ExternalIndexName, out var index))
throw new InvalidOperationException($"No index found by name {Constants.UmbracoIndexes.ExternalIndexName}");
searcher = index.GetSearcher();
}

View File

@@ -128,7 +128,7 @@ namespace Umbraco.Web.Runtime
composition.Container.EnableWebApi(GlobalConfiguration.Configuration);
composition.Container.RegisterCollectionBuilder<SearchableTreeCollectionBuilder>()
.Add(() => typeLoader.GetTypes<ISearchableTree>()); // fixme which searchable trees?!
.Add(() => typeLoader.GetTypes<ISearchableTree>());
composition.Container.Register<UmbracoTreeSearcher>(new PerRequestLifeTime());

View File

@@ -26,6 +26,8 @@ using Examine.LuceneEngine.Directories;
using LightInject;
using Umbraco.Core.Composing;
using Umbraco.Core.Strings;
using Umbraco.Web.Models.ContentEditing;
using Umbraco.Web.Trees;
namespace Umbraco.Web.Search
{
@@ -53,6 +55,7 @@ namespace Umbraco.Web.Search
// but greater that SafeXmlReaderWriter priority which is 60
private const int EnlistPriority = 80;
public override void Compose(Composition composition)
{
base.Compose(composition);

View File

@@ -1,6 +1,8 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Linq;
using Umbraco.Core;
using Umbraco.Core.Composing;
using Umbraco.Core.Services;
using Umbraco.Web.Trees;
@@ -19,15 +21,17 @@ namespace Umbraco.Web.Search
private Dictionary<string, SearchableApplicationTree> CreateDictionary(IApplicationTreeService treeService)
{
var appTrees = treeService.GetAll().ToArray();
var dictionary = new Dictionary<string, SearchableApplicationTree>();
var appTrees = treeService.GetAll()
.OrderBy(x => x.SortOrder)
.ToArray();
var dictionary = new Dictionary<string, SearchableApplicationTree>(StringComparer.OrdinalIgnoreCase);
var searchableTrees = this.ToArray();
foreach (var searchableTree in searchableTrees)
foreach (var appTree in appTrees)
{
var found = appTrees.FirstOrDefault(x => x.Alias == searchableTree.TreeAlias);
var found = searchableTrees.FirstOrDefault(x => x.TreeAlias.InvariantEquals(appTree.Alias));
if (found != null)
{
dictionary[searchableTree.TreeAlias] = new SearchableApplicationTree(found.ApplicationAlias, found.Alias, searchableTree);
dictionary[found.TreeAlias] = new SearchableApplicationTree(appTree.ApplicationAlias, appTree.Alias, found);
}
}
return dictionary;

View File

@@ -21,5 +21,8 @@ namespace Umbraco.Web.Search
{
return new SearchableTreeCollection(CreateItems(), _treeService);
}
//per request because generally an instance of ISearchableTree is a controller
protected override ILifetime CollectionLifetime => new PerRequestLifeTime();
}
}

View File

@@ -10,6 +10,7 @@ using Umbraco.Core;
using Umbraco.Core.Composing;
using Umbraco.Core.Models;
using Umbraco.Core.Services;
using Umbraco.Examine;
using Umbraco.Web.Models.ContentEditing;
using Umbraco.Web.Trees;
using SearchResult = Examine.SearchResult;
@@ -23,11 +24,18 @@ namespace Umbraco.Web.Search
{
private readonly IExamineManager _examineManager;
private readonly UmbracoHelper _umbracoHelper;
private readonly ILocalizationService _languageService;
private readonly IEntityService _entityService;
public UmbracoTreeSearcher(IExamineManager examineManager, UmbracoHelper umbracoHelper)
public UmbracoTreeSearcher(IExamineManager examineManager,
UmbracoHelper umbracoHelper,
ILocalizationService languageService,
IEntityService entityService)
{
_examineManager = examineManager ?? throw new ArgumentNullException(nameof(examineManager));
_umbracoHelper = umbracoHelper ?? throw new ArgumentNullException(nameof(umbracoHelper));
_languageService = languageService;
_entityService = entityService;
}
/// <summary>
@@ -51,16 +59,22 @@ namespace Umbraco.Web.Search
var sb = new StringBuilder();
string type;
var indexName = Constants.Examine.InternalIndexer;
var indexName = Constants.UmbracoIndexes.InternalIndexName;
var fields = new[] { "id", "__NodeId" };
var umbracoContext = _umbracoHelper.UmbracoContext;
//TODO: WE should really just allow passing in a lucene raw query
//TODO: WE should try to allow passing in a lucene raw query, however we will still need to do some manual string
// manipulation for things like start paths, member types, etc...
//if (Examine.ExamineExtensions.TryParseLuceneQuery(query))
//{
//}
switch (entityType)
{
case UmbracoEntityTypes.Member:
indexName = Constants.Examine.InternalMemberIndexer;
indexName = Constants.UmbracoIndexes.MembersIndexName;
type = "member";
fields = new[] { "id", "__NodeId", "email", "loginName" };
if (searchFrom != null && searchFrom != Constants.Conventions.MemberTypes.AllMembersListId && searchFrom.Trim() != "-1")
@@ -72,13 +86,13 @@ namespace Umbraco.Web.Search
break;
case UmbracoEntityTypes.Media:
type = "media";
var allMediaStartNodes = umbracoContext.Security.CurrentUser.CalculateMediaStartNodeIds(Current.Services.EntityService);
AppendPath(sb, UmbracoObjectTypes.Media, allMediaStartNodes, searchFrom, Current.Services.EntityService);
var allMediaStartNodes = umbracoContext.Security.CurrentUser.CalculateMediaStartNodeIds(_entityService);
AppendPath(sb, UmbracoObjectTypes.Media, allMediaStartNodes, searchFrom, _entityService);
break;
case UmbracoEntityTypes.Document:
type = "content";
var allContentStartNodes = umbracoContext.Security.CurrentUser.CalculateContentStartNodeIds(Current.Services.EntityService);
AppendPath(sb, UmbracoObjectTypes.Document, allContentStartNodes, searchFrom, Current.Services.EntityService);
var allContentStartNodes = umbracoContext.Security.CurrentUser.CalculateContentStartNodeIds(_entityService);
AppendPath(sb, UmbracoObjectTypes.Document, allContentStartNodes, searchFrom, _entityService);
break;
default:
throw new NotSupportedException("The " + typeof(UmbracoTreeSearcher) + " currently does not support searching against object type " + entityType);
@@ -89,11 +103,43 @@ namespace Umbraco.Web.Search
var internalSearcher = index.GetSearcher();
if (!BuildQuery(sb, query, searchFrom, fields, type))
{
totalFound = 0;
return Enumerable.Empty<SearchResultEntity>();
}
var raw = internalSearcher.CreateCriteria().RawQuery(sb.ToString());
var result = internalSearcher
//only return the number of items specified to read up to the amount of records to fill from 0 -> the number of items on the page requested
.Search(raw, Convert.ToInt32(pageSize * (pageIndex + 1)));
totalFound = result.TotalItemCount;
var pagedResult = result.Skip(Convert.ToInt32(pageIndex));
switch (entityType)
{
case UmbracoEntityTypes.Member:
return MemberFromSearchResults(pagedResult.ToArray());
case UmbracoEntityTypes.Media:
return MediaFromSearchResults(pagedResult);
case UmbracoEntityTypes.Document:
return ContentFromSearchResults(pagedResult);
default:
throw new NotSupportedException("The " + typeof(UmbracoTreeSearcher) + " currently does not support searching against object type " + entityType);
}
}
private bool BuildQuery(StringBuilder sb, string query, string searchFrom, string[] fields, string type)
{
//build a lucene query:
// the __nodeName will be boosted 10x without wildcards
// then __nodeName will be matched normally with wildcards
// the nodeName will be boosted 10x without wildcards
// then nodeName will be matched normally with wildcards
// the rest will be normal without wildcards
var allLangs = _languageService.GetAllLanguages().Select(x => x.IsoCode.ToLowerInvariant()).ToList();
//check if text is surrounded by single or double quotes, if so, then exact match
var surroundedByQuotes = Regex.IsMatch(query, "^\".*?\"$")
@@ -102,15 +148,14 @@ namespace Umbraco.Web.Search
if (surroundedByQuotes)
{
//strip quotes, escape string, the replace again
query = query.Trim(new[] { '\"', '\'' });
query = query.Trim('\"', '\'');
query = Lucene.Net.QueryParsers.QueryParser.Escape(query);
//nothing to search
if (searchFrom.IsNullOrWhiteSpace() && query.IsNullOrWhiteSpace())
{
totalFound = 0;
return new List<SearchResultEntity>();
return false;
}
//update the query with the query term
@@ -119,10 +164,9 @@ namespace Umbraco.Web.Search
//add back the surrounding quotes
query = string.Format("{0}{1}{0}", "\"", query);
//node name exactly boost x 10
sb.Append("+(__nodeName: (");
sb.Append(query.ToLower());
sb.Append(")^10.0 ");
sb.Append("+(");
AppendNodeNamePhraseWithBoost(sb, query, allLangs);
foreach (var f in fields)
{
@@ -143,8 +187,7 @@ namespace Umbraco.Web.Search
//nothing to search
if (searchFrom.IsNullOrWhiteSpace() && trimmed.IsNullOrWhiteSpace())
{
totalFound = 0;
return new List<SearchResultEntity>();
return false;
}
//update the query with the query term
@@ -154,24 +197,12 @@ namespace Umbraco.Web.Search
var querywords = query.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
//node name exactly boost x 10
sb.Append("+(__nodeName:");
sb.Append("\"");
sb.Append(query.ToLower());
sb.Append("\"");
sb.Append("^10.0 ");
//node name normally with wildcards
sb.Append(" __nodeName:");
sb.Append("(");
foreach (var w in querywords)
{
sb.Append(w.ToLower());
sb.Append("* ");
}
sb.Append(") ");
sb.Append("+(");
AppendNodeNameExactWithBoost(sb, query, allLangs);
AppendNodeNameWithWildcards(sb, querywords, allLangs);
foreach (var f in fields)
{
//additional fields normally
@@ -195,26 +226,69 @@ namespace Umbraco.Web.Search
sb.Append("+__IndexType:");
sb.Append(type);
var raw = internalSearcher.CreateCriteria().RawQuery(sb.ToString());
return true;
}
var result = internalSearcher
//only return the number of items specified to read up to the amount of records to fill from 0 -> the number of items on the page requested
.Search(raw, Convert.ToInt32(pageSize * (pageIndex + 1)));
private void AppendNodeNamePhraseWithBoost(StringBuilder sb, string query, IEnumerable<string> allLangs)
{
//node name exactly boost x 10
sb.Append("nodeName: (");
sb.Append(query.ToLower());
sb.Append(")^10.0 ");
totalFound = result.TotalItemCount;
var pagedResult = result.Skip(Convert.ToInt32(pageIndex));
switch (entityType)
//also search on all variant node names
foreach (var lang in allLangs)
{
case UmbracoEntityTypes.Member:
return MemberFromSearchResults(pagedResult.ToArray());
case UmbracoEntityTypes.Media:
return MediaFromSearchResults(pagedResult);
case UmbracoEntityTypes.Document:
return ContentFromSearchResults(pagedResult);
default:
throw new NotSupportedException("The " + typeof(UmbracoTreeSearcher) + " currently does not support searching against object type " + entityType);
//node name exactly boost x 10
sb.Append($"nodeName_{lang}: (");
sb.Append(query.ToLower());
sb.Append(")^10.0 ");
}
}
private void AppendNodeNameExactWithBoost(StringBuilder sb, string query, IEnumerable<string> allLangs)
{
//node name exactly boost x 10
sb.Append("nodeName:");
sb.Append("\"");
sb.Append(query.ToLower());
sb.Append("\"");
sb.Append("^10.0 ");
//also search on all variant node names
foreach (var lang in allLangs)
{
//node name exactly boost x 10
sb.Append($"nodeName_{lang}:");
sb.Append("\"");
sb.Append(query.ToLower());
sb.Append("\"");
sb.Append("^10.0 ");
}
}
private void AppendNodeNameWithWildcards(StringBuilder sb, string[] querywords, IEnumerable<string> allLangs)
{
//node name normally with wildcards
sb.Append("nodeName:");
sb.Append("(");
foreach (var w in querywords)
{
sb.Append(w.ToLower());
sb.Append("* ");
}
sb.Append(") ");
//also search on all variant node names
foreach (var lang in allLangs)
{
//node name normally with wildcards
sb.Append($"nodeName_{lang}:");
sb.Append("(");
foreach (var w in querywords)
{
sb.Append(w.ToLower());
sb.Append("* ");
}
sb.Append(") ");
}
}
@@ -278,32 +352,33 @@ namespace Umbraco.Web.Search
/// </summary>
/// <param name="results"></param>
/// <returns></returns>
private IEnumerable<SearchResultEntity> MemberFromSearchResults(ISearchResult[] results)
private IEnumerable<SearchResultEntity> MemberFromSearchResults(IEnumerable<ISearchResult> results)
{
var mapped = Mapper.Map<IEnumerable<SearchResultEntity>>(results).ToArray();
//add additional data
foreach (var m in mapped)
foreach (var result in results)
{
var m = Mapper.Map<SearchResultEntity>(result);
//if no icon could be mapped, it will be set to document, so change it to picture
if (m.Icon == "icon-document")
{
m.Icon = "icon-user";
}
var searchResult = results.First(x => x.Id == m.Id.ToString());
if (searchResult.Values.ContainsKey("email") && searchResult.Values["email"] != null)
if (result.Values.ContainsKey("email") && result.Values["email"] != null)
{
m.AdditionalData["Email"] = results.First(x => x.Id == m.Id.ToString()).Values["email"];
m.AdditionalData["Email"] = result.Values["email"];
}
if (searchResult.Values.ContainsKey("__key") && searchResult.Values["__key"] != null)
if (result.Values.ContainsKey("__key") && result.Values["__key"] != null)
{
if (Guid.TryParse(searchResult.Values["__key"], out var key))
if (Guid.TryParse(result.Values["__key"], out var key))
{
m.Key = key;
}
}
yield return m;
}
return mapped;
}
/// <summary>
@@ -313,17 +388,17 @@ namespace Umbraco.Web.Search
/// <returns></returns>
private IEnumerable<SearchResultEntity> MediaFromSearchResults(IEnumerable<ISearchResult> results)
{
var mapped = Mapper.Map<IEnumerable<SearchResultEntity>>(results).ToArray();
//add additional data
foreach (var m in mapped)
foreach (var result in results)
{
var m = Mapper.Map<SearchResultEntity>(result);
//if no icon could be mapped, it will be set to document, so change it to picture
if (m.Icon == "icon-document")
{
m.Icon = "icon-picture";
}
yield return m;
}
return mapped;
}
/// <summary>
@@ -333,17 +408,28 @@ namespace Umbraco.Web.Search
/// <returns></returns>
private IEnumerable<SearchResultEntity> ContentFromSearchResults(IEnumerable<ISearchResult> results)
{
var mapped = Mapper.Map<IEnumerable<SearchResultEntity>>(results).ToArray();
//add additional data
foreach (var m in mapped)
var defaultLang = _languageService.GetDefaultLanguageIsoCode();
foreach (var result in results)
{
var intId = m.Id.TryConvertTo<int>();
var entity = Mapper.Map<SearchResultEntity>(result);
var intId = entity.Id.TryConvertTo<int>();
if (intId.Success)
{
m.AdditionalData["Url"] = _umbracoHelper.Url(intId.Result);
//if it varies by culture, return the default language URL
if (result.Values.TryGetValue(UmbracoContentIndex.VariesByCultureFieldName, out var varies) && varies == "1")
{
entity.AdditionalData["Url"] = _umbracoHelper.Url(intId.Result, defaultLang);
}
else
{
entity.AdditionalData["Url"] = _umbracoHelper.Url(intId.Result);
}
}
yield return entity;
}
return mapped;
}
}

View File

@@ -36,10 +36,12 @@ namespace Umbraco.Web.Trees
public class ContentTreeController : ContentTreeControllerBase, ISearchableTree
{
private readonly UmbracoTreeSearcher _treeSearcher;
private readonly ActionCollection _actions;
public ContentTreeController(UmbracoTreeSearcher treeSearcher)
public ContentTreeController(UmbracoTreeSearcher treeSearcher, ActionCollection actions)
{
_treeSearcher = treeSearcher;
_actions = actions;
}
protected override int RecycleBinId => Constants.System.RecycleBinContent;
@@ -127,7 +129,7 @@ namespace Umbraco.Web.Trees
// we need to get the default permissions as you can't set permissions on the very root node
var permission = Services.UserService.GetPermissions(Security.CurrentUser, Constants.System.Root).First();
var nodeActions = Current.Actions.FromEntityPermission(permission)
var nodeActions = _actions.FromEntityPermission(permission)
.Select(x => new MenuItem(x));
//these two are the standard items
@@ -313,8 +315,7 @@ namespace Umbraco.Web.Trees
private void AddActionNode<TAction>(IUmbracoEntity item, MenuItemCollection menu, bool hasSeparator = false, bool convert = false, bool opensDialog = false)
where TAction : IAction
{
//fixme: Inject
var menuItem = menu.Items.Add<TAction>(Services.TextService.Localize("actions", Current.Actions.GetAction<TAction>().Alias), hasSeparator);
var menuItem = menu.Items.Add<TAction>(Services.TextService.Localize("actions", _actions.GetAction<TAction>().Alias), hasSeparator);
if (convert) menuItem.ConvertLegacyMenuItem(item, "content", "content");
menuItem.OpensDialog = opensDialog;
}

View File

@@ -1,9 +1,10 @@
using System.Collections.Generic;
using Umbraco.Core.Composing;
using Umbraco.Web.Models.ContentEditing;
namespace Umbraco.Web.Trees
{
public interface ISearchableTree
public interface ISearchableTree : IDiscoverable
{
/// <summary>
/// The alias of the tree that the <see cref="ISearchableTree"/> belongs to