From eb3841fc0a5332f0bd73935cae4fffa5286a803c Mon Sep 17 00:00:00 2001 From: Shannon Date: Tue, 11 Dec 2018 17:51:47 +1100 Subject: [PATCH] Searches on variant nodeName, orders ISearchableTree results by the corresponding tree SortOrder, fixes assembly scanning ISearchableTree --- src/Umbraco.Examine/ExamineExtensions.cs | 26 +++ .../Properties/AssemblyInfo.cs | 1 + .../Editors/ExamineManagementController.cs | 25 +-- .../Runtime/WebRuntimeComponent.cs | 6 +- src/Umbraco.Web/Search/ExamineComponent.cs | 3 + .../Search/SearchableTreeCollection.cs | 10 +- src/Umbraco.Web/Search/UmbracoTreeSearcher.cs | 196 ++++++++++++------ 7 files changed, 177 insertions(+), 90 deletions(-) diff --git a/src/Umbraco.Examine/ExamineExtensions.cs b/src/Umbraco.Examine/ExamineExtensions.cs index 3681979267..9fdf5e9123 100644 --- a/src/Umbraco.Examine/ExamineExtensions.cs +++ b/src/Umbraco.Examine/ExamineExtensions.cs @@ -1,8 +1,11 @@ using System; using Examine.LuceneEngine.Providers; +using Lucene.Net.Analysis; using Lucene.Net.Index; +using Lucene.Net.QueryParsers; using Lucene.Net.Search; using Lucene.Net.Store; +using Version = Lucene.Net.Util.Version; namespace Umbraco.Examine { @@ -11,6 +14,29 @@ namespace Umbraco.Examine /// internal static class ExamineExtensions { + public static bool TryParseLuceneQuery(string query) + { + //TODO: I'd assume there would be a more strict way to parse the query but not that i can find yet, for now we'll + // also do this rudimentary check + if (!query.Contains(":")) + return false; + + try + { + //This will pass with a plain old string without any fields, need to figure out a way to have it properly parse + var parsed = new QueryParser(Version.LUCENE_30, "nodeName", new KeywordAnalyzer()).Parse(query); + return true; + } + catch (ParseException) + { + return false; + } + catch (Exception) + { + return false; + } + } + /// /// Checks if the index can be read/opened /// diff --git a/src/Umbraco.Examine/Properties/AssemblyInfo.cs b/src/Umbraco.Examine/Properties/AssemblyInfo.cs index 6713111968..5c42a236f4 100644 --- a/src/Umbraco.Examine/Properties/AssemblyInfo.cs +++ b/src/Umbraco.Examine/Properties/AssemblyInfo.cs @@ -8,6 +8,7 @@ using System.Runtime.CompilerServices; // Umbraco Cms [assembly: InternalsVisibleTo("Umbraco.Tests")] +[assembly: InternalsVisibleTo("Umbraco.Web")] // code analysis // IDE1006 is broken, wants _value syntax for consts, etc - and it's even confusing ppl at MS, kill it diff --git a/src/Umbraco.Web/Editors/ExamineManagementController.cs b/src/Umbraco.Web/Editors/ExamineManagementController.cs index 67209c91bd..2f96ee3d45 100644 --- a/src/Umbraco.Web/Editors/ExamineManagementController.cs +++ b/src/Umbraco.Web/Editors/ExamineManagementController.cs @@ -73,7 +73,7 @@ namespace Umbraco.Web.Editors if (!msg.IsSuccessStatusCode) throw new HttpResponseException(msg); - var results = TryParseLuceneQuery(query) + var results = Examine.ExamineExtensions.TryParseLuceneQuery(query) ? searcher.Search(searcher.CreateCriteria().RawQuery(query), maxResults: pageSize * (pageIndex + 1)) : searcher.Search(query, true, maxResults: pageSize * (pageIndex + 1)); @@ -92,28 +92,7 @@ namespace Umbraco.Web.Editors }; } - private bool TryParseLuceneQuery(string query) - { - //TODO: I'd assume there would be a more strict way to parse the query but not that i can find yet, for now we'll - // also do this rudimentary check - if (!query.Contains(":")) - return false; - - try - { - //This will pass with a plain old string without any fields, need to figure out a way to have it properly parse - var parsed = new QueryParser(Version.LUCENE_30, "nodeName", new KeywordAnalyzer()).Parse(query); - return true; - } - catch (ParseException) - { - return false; - } - catch (Exception) - { - return false; - } - } + /// /// Check if the index has been rebuilt diff --git a/src/Umbraco.Web/Runtime/WebRuntimeComponent.cs b/src/Umbraco.Web/Runtime/WebRuntimeComponent.cs index 2e1c934bc0..ccd9a7ef7d 100644 --- a/src/Umbraco.Web/Runtime/WebRuntimeComponent.cs +++ b/src/Umbraco.Web/Runtime/WebRuntimeComponent.cs @@ -127,8 +127,12 @@ namespace Umbraco.Web.Runtime composition.Container.RegisterUmbracoControllers(typeLoader, GetType().Assembly); composition.Container.EnableWebApi(GlobalConfiguration.Configuration); + //we aren't scanning for ISearchableTree since that is not IDiscoverable, instead we'll just filter what we've + //already scanned since all of our ISearchableTree is of type UmbracoApiController and in most cases any developers' + //own trees they want searched will also be of type UmbracoApiController. If a developer wants to replace one of ours + //then they will have to manually register/replace. composition.Container.RegisterCollectionBuilder() - .Add(() => typeLoader.GetTypes()); // fixme which searchable trees?! + .Add(() => typeLoader.GetTypes().Where(x => x.Implements())); composition.Container.Register(new PerRequestLifeTime()); diff --git a/src/Umbraco.Web/Search/ExamineComponent.cs b/src/Umbraco.Web/Search/ExamineComponent.cs index d8c1016c3e..65264d0308 100644 --- a/src/Umbraco.Web/Search/ExamineComponent.cs +++ b/src/Umbraco.Web/Search/ExamineComponent.cs @@ -26,6 +26,8 @@ using Examine.LuceneEngine.Directories; using LightInject; using Umbraco.Core.Composing; using Umbraco.Core.Strings; +using Umbraco.Web.Models.ContentEditing; +using Umbraco.Web.Trees; namespace Umbraco.Web.Search { @@ -53,6 +55,7 @@ namespace Umbraco.Web.Search // but greater that SafeXmlReaderWriter priority which is 60 private const int EnlistPriority = 80; + public override void Compose(Composition composition) { base.Compose(composition); diff --git a/src/Umbraco.Web/Search/SearchableTreeCollection.cs b/src/Umbraco.Web/Search/SearchableTreeCollection.cs index 86f4494353..81ee0a2898 100644 --- a/src/Umbraco.Web/Search/SearchableTreeCollection.cs +++ b/src/Umbraco.Web/Search/SearchableTreeCollection.cs @@ -19,15 +19,17 @@ namespace Umbraco.Web.Search private Dictionary CreateDictionary(IApplicationTreeService treeService) { - var appTrees = treeService.GetAll().ToArray(); + var appTrees = treeService.GetAll() + .OrderBy(x => x.SortOrder) + .ToArray(); var dictionary = new Dictionary(); var searchableTrees = this.ToArray(); - foreach (var searchableTree in searchableTrees) + foreach (var appTree in appTrees) { - var found = appTrees.FirstOrDefault(x => x.Alias == searchableTree.TreeAlias); + var found = searchableTrees.FirstOrDefault(x => x.TreeAlias == appTree.Alias); if (found != null) { - dictionary[searchableTree.TreeAlias] = new SearchableApplicationTree(found.ApplicationAlias, found.Alias, searchableTree); + dictionary[found.TreeAlias] = new SearchableApplicationTree(appTree.ApplicationAlias, appTree.Alias, found); } } return dictionary; diff --git a/src/Umbraco.Web/Search/UmbracoTreeSearcher.cs b/src/Umbraco.Web/Search/UmbracoTreeSearcher.cs index af2db4f7ac..8f977aab77 100644 --- a/src/Umbraco.Web/Search/UmbracoTreeSearcher.cs +++ b/src/Umbraco.Web/Search/UmbracoTreeSearcher.cs @@ -25,12 +25,17 @@ namespace Umbraco.Web.Search private readonly IExamineManager _examineManager; private readonly UmbracoHelper _umbracoHelper; private readonly ILocalizationService _languageService; + private readonly IEntityService _entityService; - public UmbracoTreeSearcher(IExamineManager examineManager, UmbracoHelper umbracoHelper, ILocalizationService languageService) + public UmbracoTreeSearcher(IExamineManager examineManager, + UmbracoHelper umbracoHelper, + ILocalizationService languageService, + IEntityService entityService) { _examineManager = examineManager ?? throw new ArgumentNullException(nameof(examineManager)); _umbracoHelper = umbracoHelper ?? throw new ArgumentNullException(nameof(umbracoHelper)); _languageService = languageService; + _entityService = entityService; } /// @@ -59,7 +64,13 @@ namespace Umbraco.Web.Search var umbracoContext = _umbracoHelper.UmbracoContext; - //TODO: WE should really just allow passing in a lucene raw query + //TODO: WE should try to allow passing in a lucene raw query, however we will still need to do some manual string + // manipulation for things like start paths, member types, etc... + //if (Examine.ExamineExtensions.TryParseLuceneQuery(query)) + //{ + + //} + switch (entityType) { case UmbracoEntityTypes.Member: @@ -75,13 +86,13 @@ namespace Umbraco.Web.Search break; case UmbracoEntityTypes.Media: type = "media"; - var allMediaStartNodes = umbracoContext.Security.CurrentUser.CalculateMediaStartNodeIds(Current.Services.EntityService); - AppendPath(sb, UmbracoObjectTypes.Media, allMediaStartNodes, searchFrom, Current.Services.EntityService); + var allMediaStartNodes = umbracoContext.Security.CurrentUser.CalculateMediaStartNodeIds(_entityService); + AppendPath(sb, UmbracoObjectTypes.Media, allMediaStartNodes, searchFrom, _entityService); break; case UmbracoEntityTypes.Document: type = "content"; - var allContentStartNodes = umbracoContext.Security.CurrentUser.CalculateContentStartNodeIds(Current.Services.EntityService); - AppendPath(sb, UmbracoObjectTypes.Document, allContentStartNodes, searchFrom, Current.Services.EntityService); + var allContentStartNodes = umbracoContext.Security.CurrentUser.CalculateContentStartNodeIds(_entityService); + AppendPath(sb, UmbracoObjectTypes.Document, allContentStartNodes, searchFrom, _entityService); break; default: throw new NotSupportedException("The " + typeof(UmbracoTreeSearcher) + " currently does not support searching against object type " + entityType); @@ -92,11 +103,43 @@ namespace Umbraco.Web.Search var internalSearcher = index.GetSearcher(); + if (!BuildQuery(sb, query, searchFrom, fields, type)) + { + totalFound = 0; + return Enumerable.Empty(); + } + + var raw = internalSearcher.CreateCriteria().RawQuery(sb.ToString()); + + var result = internalSearcher + //only return the number of items specified to read up to the amount of records to fill from 0 -> the number of items on the page requested + .Search(raw, Convert.ToInt32(pageSize * (pageIndex + 1))); + + totalFound = result.TotalItemCount; + + var pagedResult = result.Skip(Convert.ToInt32(pageIndex)); + + switch (entityType) + { + case UmbracoEntityTypes.Member: + return MemberFromSearchResults(pagedResult.ToArray()); + case UmbracoEntityTypes.Media: + return MediaFromSearchResults(pagedResult); + case UmbracoEntityTypes.Document: + return ContentFromSearchResults(pagedResult); + default: + throw new NotSupportedException("The " + typeof(UmbracoTreeSearcher) + " currently does not support searching against object type " + entityType); + } + } + + private bool BuildQuery(StringBuilder sb, string query, string searchFrom, string[] fields, string type) + { //build a lucene query: // the nodeName will be boosted 10x without wildcards // then nodeName will be matched normally with wildcards // the rest will be normal without wildcards + var allLangs = _languageService.GetAllLanguages().Select(x => x.IsoCode).ToList(); //check if text is surrounded by single or double quotes, if so, then exact match var surroundedByQuotes = Regex.IsMatch(query, "^\".*?\"$") @@ -105,15 +148,14 @@ namespace Umbraco.Web.Search if (surroundedByQuotes) { //strip quotes, escape string, the replace again - query = query.Trim(new[] { '\"', '\'' }); + query = query.Trim('\"', '\''); query = Lucene.Net.QueryParsers.QueryParser.Escape(query); //nothing to search if (searchFrom.IsNullOrWhiteSpace() && query.IsNullOrWhiteSpace()) { - totalFound = 0; - return new List(); + return false; } //update the query with the query term @@ -122,10 +164,9 @@ namespace Umbraco.Web.Search //add back the surrounding quotes query = string.Format("{0}{1}{0}", "\"", query); - //node name exactly boost x 10 - sb.Append("+(nodeName: ("); - sb.Append(query.ToLower()); - sb.Append(")^10.0 "); + sb.Append("+("); + + AppendNodeNamePhraseWithBoost(sb, query, allLangs); foreach (var f in fields) { @@ -146,8 +187,7 @@ namespace Umbraco.Web.Search //nothing to search if (searchFrom.IsNullOrWhiteSpace() && trimmed.IsNullOrWhiteSpace()) { - totalFound = 0; - return new List(); + return false; } //update the query with the query term @@ -157,24 +197,12 @@ namespace Umbraco.Web.Search var querywords = query.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries); - //node name exactly boost x 10 - sb.Append("+(nodeName:"); - sb.Append("\""); - sb.Append(query.ToLower()); - sb.Append("\""); - sb.Append("^10.0 "); - - //node name normally with wildcards - sb.Append(" nodeName:"); - sb.Append("("); - foreach (var w in querywords) - { - sb.Append(w.ToLower()); - sb.Append("* "); - } - sb.Append(") "); + sb.Append("+("); + AppendNodeNameExactWithBoost(sb, query, allLangs); + AppendNodeNameWithWildcards(sb, querywords, allLangs); + foreach (var f in fields) { //additional fields normally @@ -198,26 +226,69 @@ namespace Umbraco.Web.Search sb.Append("+__IndexType:"); sb.Append(type); - var raw = internalSearcher.CreateCriteria().RawQuery(sb.ToString()); + return true; + } - var result = internalSearcher - //only return the number of items specified to read up to the amount of records to fill from 0 -> the number of items on the page requested - .Search(raw, Convert.ToInt32(pageSize * (pageIndex + 1))); + private void AppendNodeNamePhraseWithBoost(StringBuilder sb, string query, IEnumerable allLangs) + { + //node name exactly boost x 10 + sb.Append("nodeName: ("); + sb.Append(query.ToLower()); + sb.Append(")^10.0 "); - totalFound = result.TotalItemCount; - - var pagedResult = result.Skip(Convert.ToInt32(pageIndex)); - - switch (entityType) + //also search on all variant node names + foreach (var lang in allLangs) { - case UmbracoEntityTypes.Member: - return MemberFromSearchResults(pagedResult.ToArray()); - case UmbracoEntityTypes.Media: - return MediaFromSearchResults(pagedResult); - case UmbracoEntityTypes.Document: - return ContentFromSearchResults(pagedResult); - default: - throw new NotSupportedException("The " + typeof(UmbracoTreeSearcher) + " currently does not support searching against object type " + entityType); + //node name exactly boost x 10 + sb.Append($"nodeName_{lang}: ("); + sb.Append(query.ToLower()); + sb.Append(")^10.0 "); + } + } + + private void AppendNodeNameExactWithBoost(StringBuilder sb, string query, IEnumerable allLangs) + { + //node name exactly boost x 10 + sb.Append("nodeName:"); + sb.Append("\""); + sb.Append(query.ToLower()); + sb.Append("\""); + sb.Append("^10.0 "); + //also search on all variant node names + foreach (var lang in allLangs) + { + //node name exactly boost x 10 + sb.Append($"nodeName_{lang}:"); + sb.Append("\""); + sb.Append(query.ToLower()); + sb.Append("\""); + sb.Append("^10.0 "); + } + } + + private void AppendNodeNameWithWildcards(StringBuilder sb, string[] querywords, IEnumerable allLangs) + { + //node name normally with wildcards + sb.Append("nodeName:"); + sb.Append("("); + foreach (var w in querywords) + { + sb.Append(w.ToLower()); + sb.Append("* "); + } + sb.Append(") "); + //also search on all variant node names + foreach (var lang in allLangs) + { + //node name normally with wildcards + sb.Append($"nodeName_{lang}:"); + sb.Append("("); + foreach (var w in querywords) + { + sb.Append(w.ToLower()); + sb.Append("* "); + } + sb.Append(") "); } } @@ -281,32 +352,33 @@ namespace Umbraco.Web.Search /// /// /// - private IEnumerable MemberFromSearchResults(ISearchResult[] results) + private IEnumerable MemberFromSearchResults(IEnumerable results) { - var mapped = Mapper.Map>(results).ToArray(); //add additional data - foreach (var m in mapped) + foreach (var result in results) { + var m = Mapper.Map(result); + //if no icon could be mapped, it will be set to document, so change it to picture if (m.Icon == "icon-document") { m.Icon = "icon-user"; } - - var searchResult = results.First(x => x.Id == m.Id.ToString()); - if (searchResult.Values.ContainsKey("email") && searchResult.Values["email"] != null) + + if (result.Values.ContainsKey("email") && result.Values["email"] != null) { - m.AdditionalData["Email"] = results.First(x => x.Id == m.Id.ToString()).Values["email"]; + m.AdditionalData["Email"] = result.Values["email"]; } - if (searchResult.Values.ContainsKey("__key") && searchResult.Values["__key"] != null) + if (result.Values.ContainsKey("__key") && result.Values["__key"] != null) { - if (Guid.TryParse(searchResult.Values["__key"], out var key)) + if (Guid.TryParse(result.Values["__key"], out var key)) { m.Key = key; } } + + yield return m; } - return mapped; } /// @@ -316,17 +388,17 @@ namespace Umbraco.Web.Search /// private IEnumerable MediaFromSearchResults(IEnumerable results) { - var mapped = Mapper.Map>(results).ToArray(); //add additional data - foreach (var m in mapped) + foreach (var result in results) { + var m = Mapper.Map(result); //if no icon could be mapped, it will be set to document, so change it to picture if (m.Icon == "icon-document") { m.Icon = "icon-picture"; } + yield return m; } - return mapped; } /// @@ -345,7 +417,7 @@ namespace Umbraco.Web.Search var intId = entity.Id.TryConvertTo(); if (intId.Success) { - //TODO: Here we need to figure out how to get the URL based on variant, etc... + //if it varies by culture, return the default language URL if (result.Values.TryGetValue(UmbracoContentIndex.VariesByCultureFieldName, out var varies) && varies == "1") { entity.AdditionalData["Url"] = _umbracoHelper.Url(intId.Result, defaultLang);