diff --git a/src/NuGet.Config b/src/NuGet.Config index dcccfdd5c1..2cb8d8dfbd 100644 --- a/src/NuGet.Config +++ b/src/NuGet.Config @@ -3,5 +3,6 @@ + \ No newline at end of file diff --git a/src/Umbraco.Examine/BaseUmbracoIndexer.cs b/src/Umbraco.Examine/BaseUmbracoIndexer.cs deleted file mode 100644 index 24d3168240..0000000000 --- a/src/Umbraco.Examine/BaseUmbracoIndexer.cs +++ /dev/null @@ -1,459 +0,0 @@ -using System; -using System.Collections.Generic; -using System.ComponentModel; -using System.Linq; -using System.Text; -using Examine.LuceneEngine.Config; -using Examine.LuceneEngine.Providers; -using Examine.Providers; -using Lucene.Net.Analysis; -using Lucene.Net.Index; -using Umbraco.Core; -using Examine; -using System.IO; -using System.Xml; -using System.Xml.Linq; -using Examine.LuceneEngine; -using Examine.LuceneEngine.Faceting; -using Examine.LuceneEngine.Indexing; -using Lucene.Net.Documents; -using Lucene.Net.Store; -using Umbraco.Core.Composing; -using Umbraco.Core.Logging; -using Umbraco.Core.Xml; -using Umbraco.Examine.LocalStorage; -using Directory = Lucene.Net.Store.Directory; - -namespace Umbraco.Examine -{ - /// - /// An abstract provider containing the basic functionality to be able to query against - /// Umbraco data. - /// - public abstract class BaseUmbracoIndexer : LuceneIndexer - { - // note - // wrapping all operations that end up calling base.SafelyProcessQueueItems in a safe call - // context because they will fork a thread/task/whatever which should *not* capture our - // call context (and the database it can contain)! ideally we should be able to override - // SafelyProcessQueueItems but that's not possible in the current version of Examine. - - /// - /// Used to store the path of a content object - /// - public const string IndexPathFieldName = "__Path"; - public const string NodeKeyFieldName = "__Key"; - public const string IconFieldName = "__Icon"; - public const string PublishedFieldName = "__Published"; - /// - /// The prefix added to a field when it is duplicated in order to store the original raw value. - /// - public const string RawFieldPrefix = "__Raw_"; - - /// - /// Default constructor - /// - protected BaseUmbracoIndexer() - : base() - { - ProfilingLogger = Current.ProfilingLogger; - _configBased = true; - } - - protected BaseUmbracoIndexer( - IEnumerable fieldDefinitions, - Directory luceneDirectory, - Analyzer defaultAnalyzer, - ProfilingLogger profilingLogger, - IValueSetValidator validator = null, - FacetConfiguration facetConfiguration = null, IDictionary> indexValueTypes = null) - : base(fieldDefinitions, luceneDirectory, defaultAnalyzer, validator, facetConfiguration, indexValueTypes) - { - if (profilingLogger == null) throw new ArgumentNullException("profilingLogger"); - ProfilingLogger = profilingLogger; - } - - private readonly bool _configBased = false; - private LocalTempStorageIndexer _localTempStorageIndexer; - - /// - /// A type that defines the type of index for each Umbraco field (non user defined fields) - /// Alot of standard umbraco fields shouldn't be tokenized or even indexed, just stored into lucene - /// for retreival after searching. - /// - [Obsolete("IndexFieldPolicies is not really used apart for some legacy reasons - use FieldDefinition's instead")] - internal static readonly List IndexFieldPolicies - = new List - { - new StaticField("id", FieldIndexTypes.NOT_ANALYZED, false, string.Empty), - new StaticField("key", FieldIndexTypes.NOT_ANALYZED, false, string.Empty), - new StaticField( "version", FieldIndexTypes.NOT_ANALYZED, false, string.Empty), - new StaticField( "parentID", FieldIndexTypes.NOT_ANALYZED, false, string.Empty), - new StaticField( "level", FieldIndexTypes.NOT_ANALYZED, true, "NUMBER"), - new StaticField( "writerID", FieldIndexTypes.NOT_ANALYZED, false, string.Empty), - new StaticField( "creatorID", FieldIndexTypes.NOT_ANALYZED, false, string.Empty), - new StaticField( "nodeType", FieldIndexTypes.NOT_ANALYZED, false, string.Empty), - new StaticField( "template", FieldIndexTypes.NOT_ANALYZED, false, string.Empty), - new StaticField( "sortOrder", FieldIndexTypes.NOT_ANALYZED, true, "NUMBER"), - new StaticField( "createDate", FieldIndexTypes.NOT_ANALYZED, false, "DATETIME"), - new StaticField( "updateDate", FieldIndexTypes.NOT_ANALYZED, false, "DATETIME"), - new StaticField( "nodeName", FieldIndexTypes.ANALYZED, false, string.Empty), - new StaticField( "urlName", FieldIndexTypes.NOT_ANALYZED, false, string.Empty), - new StaticField( "writerName", FieldIndexTypes.ANALYZED, false, string.Empty), - new StaticField( "creatorName", FieldIndexTypes.ANALYZED, false, string.Empty), - new StaticField( "nodeTypeAlias", FieldIndexTypes.ANALYZED, false, string.Empty), - new StaticField( "path", FieldIndexTypes.NOT_ANALYZED, false, string.Empty) - }; - - protected ProfilingLogger ProfilingLogger { get; private set; } - - /// - /// Overridden to ensure that the umbraco system field definitions are in place - /// - /// - /// - protected override IEnumerable InitializeFieldDefinitions(IEnumerable originalDefinitions) - { - var fd = base.InitializeFieldDefinitions(originalDefinitions).ToList(); - fd.AddRange(new[] - { - new FieldDefinition("parentID", FieldDefinitionTypes.Integer), - new FieldDefinition("level", FieldDefinitionTypes.Integer), - new FieldDefinition("writerID", FieldDefinitionTypes.Integer), - new FieldDefinition("creatorID", FieldDefinitionTypes.Integer), - new FieldDefinition("sortOrder", FieldDefinitionTypes.Integer), - new FieldDefinition("template", FieldDefinitionTypes.Integer), - - new FieldDefinition("createDate", FieldDefinitionTypes.DateTime), - new FieldDefinition("updateDate", FieldDefinitionTypes.DateTime), - - new FieldDefinition("key", FieldDefinitionTypes.Raw), - new FieldDefinition("version", FieldDefinitionTypes.Raw), - new FieldDefinition("nodeType", FieldDefinitionTypes.Raw), - new FieldDefinition("template", FieldDefinitionTypes.Raw), - new FieldDefinition("urlName", FieldDefinitionTypes.Raw), - new FieldDefinition("path", FieldDefinitionTypes.Raw), - - new FieldDefinition(IndexPathFieldName, FieldDefinitionTypes.Raw), - new FieldDefinition(NodeTypeAliasFieldName, FieldDefinitionTypes.Raw), - new FieldDefinition(IconFieldName, FieldDefinitionTypes.Raw) - }); - return fd; - } - - public bool UseTempStorage - { - get { return _localTempStorageIndexer != null && _localTempStorageIndexer.LuceneDirectory != null; } - } - - public string TempStorageLocation - { - get - { - if (UseTempStorage == false) return string.Empty; - return _localTempStorageIndexer.TempPath; - } - } - - [Obsolete("This should not be used, it is used by the configuration based indexes but instead to disable Examine event handlers use the ExamineEvents class instead.")] - [EditorBrowsable(EditorBrowsableState.Never)] - public bool EnableDefaultEventHandler { get; protected set; } - - /// - /// the supported indexable types - /// - protected abstract IEnumerable SupportedTypes { get; } - - #region Initialize - - - /// - /// Setup the properties for the indexer from the provider settings - /// - /// - /// - /// - /// This is ONLY used for configuration based indexes - /// - public override void Initialize(string name, System.Collections.Specialized.NameValueCollection config) - { - EnableDefaultEventHandler = true; //set to true by default - bool enabled; - if (bool.TryParse(config["enableDefaultEventHandler"], out enabled)) - { - EnableDefaultEventHandler = enabled; - } - - ProfilingLogger.Logger.Debug(GetType(), "{0} indexer initializing", () => name); - - base.Initialize(name, config); - - //NOTES: useTempStorage is obsolete, tempStorageDirectory is obsolete, both have been superceded by Examine Core's IDirectoryFactory - // tempStorageDirectory never actually got finished in Umbraco Core but accidentally got shipped (it's only enabled on the searcher - // and not the indexer). So this whole block is just legacy - - //detect if a dir factory has been specified, if so then useTempStorage will not be used (deprecated) - - if (config["directoryFactory"] == null && config["useTempStorage"] != null) - { - throw new NotImplementedException("Fix how local temp storage works and is synced with Examine v2.0 - since a writer is always open we cannot snapshot it, we need to use the same logic in AzureDirectory"); - - _localTempStorageIndexer = new LocalTempStorageIndexer(); - - var fsDir = base.GetLuceneDirectory() as FSDirectory; - if (fsDir != null) - { - //Use the temp storage directory which will store the index in the local/codegen folder, this is useful - // for websites that are running from a remove file server and file IO latency becomes an issue - var attemptUseTempStorage = config["useTempStorage"].TryConvertTo(); - if (attemptUseTempStorage) - { - - var indexSet = IndexSets.Instance.Sets[IndexSetName]; - var configuredPath = indexSet.IndexPath; - - _localTempStorageIndexer.Initialize(config, configuredPath, fsDir, IndexingAnalyzer, attemptUseTempStorage.Result); - } - } - - } - } - - #endregion - - - public override Directory GetLuceneDirectory() - { - //if temp local storage is configured use that, otherwise return the default - if (UseTempStorage) - { - return _localTempStorageIndexer.LuceneDirectory; - } - - return base.GetLuceneDirectory(); - - } - - /// - /// override to check if we can actually initialize. - /// - /// - /// This check is required since the base examine lib will try to rebuild on startup - /// - public override void RebuildIndex() - { - if (CanInitialize()) - { - ProfilingLogger.Logger.Debug(GetType(), "Rebuilding index"); - using (new SafeCallContext()) - { - base.RebuildIndex(); - } - } - } - - /// - /// override to check if we can actually initialize. - /// - /// - /// This check is required since the base examine lib will try to rebuild on startup - /// - public override void IndexAll(string type) - { - if (CanInitialize()) - { - using (new SafeCallContext()) - { - base.IndexAll(type); - } - } - } - - public override void IndexItems(IEnumerable nodes) - { - if (CanInitialize()) - { - using (new SafeCallContext()) - { - base.IndexItems(nodes); - } - } - } - - [Obsolete("Use ValueSets with IndexItems instead")] - public override void ReIndexNode(XElement node, string type) - { - if (CanInitialize()) - { - if (SupportedTypes.Contains(type) == false) - return; - - if (node.Attribute("id") != null) - { - ProfilingLogger.Logger.Debug(GetType(), "ReIndexNode {0} with type {1}", () => node.Attribute("id"), () => type); - using (new SafeCallContext()) - { - base.ReIndexNode(node, type); - } - } - else - { - ProfilingLogger.Logger.Error(GetType(), "ReIndexNode cannot proceed, the format of the XElement is invalid", - new XmlException("XElement is invalid, the xml has not id attribute")); - } - } - } - - /// - /// override to check if we can actually initialize. - /// - /// - /// This check is required since the base examine lib will try to rebuild on startup - /// - public override void DeleteFromIndex(string nodeId) - { - if (CanInitialize()) - { - using (new SafeCallContext()) - { - base.DeleteFromIndex(nodeId); - } - } - } - - /// - /// Returns true if the Umbraco application is in a state that we can initialize the examine indexes - /// - protected bool CanInitialize() - { - // only affects indexers that are config file based, if an index was created via code then - // this has no effect, it is assumed the index would not be created if it could not be initialized - return _configBased == false || Current.RuntimeState.Level == RuntimeLevel.Run; - } - - /// - /// Reindexes all supported types - /// - protected override void PerformIndexRebuild() - { - foreach (var t in SupportedTypes) - { - IndexAll(t); - } - } - - /// - /// overridden for logging - /// - /// - protected override void OnIndexingError(IndexingErrorEventArgs e) - { - ProfilingLogger.Logger.Error(GetType(), e.Message, e.Exception); - base.OnIndexingError(e); - } - - /// - /// Override for logging - /// - /// - protected override void OnIgnoringIndexItem(IndexItemEventArgs e) - { - ProfilingLogger.Logger.Debug(GetType(), "OnIgnoringIndexItem {0} with type {1}", () => e.IndexItem.ValueSet.Id, () => e.IndexItem.ValueSet.IndexCategory); - base.OnIgnoringIndexItem(e); - } - - /// - /// This ensures that the special __Raw_ fields are indexed - /// - /// - protected override void OnDocumentWriting(DocumentWritingEventArgs docArgs) - { - var d = docArgs.Document; - - foreach (var f in docArgs.Values.Values.Where(x => x.Key.StartsWith(RawFieldPrefix))) - { - if (f.Value.Count > 0) - { - d.Add(new Field( - f.Key, - f.Value[0].ToString(), - Field.Store.YES, - Field.Index.NO, //don't index this field, we never want to search by it - Field.TermVector.NO)); - } - } - - ProfilingLogger.Logger.Debug(GetType(), "OnDocumentWriting {0} with type {1}", () => docArgs.Values.Id, () => docArgs.Values.ItemType); - - base.OnDocumentWriting(docArgs); - } - - protected override void OnItemIndexed(IndexItemEventArgs e) - { - ProfilingLogger.Logger.Debug(GetType(), "Index created for node {0}", () => e.IndexItem.Id); - base.OnItemIndexed(e); - } - - protected override void OnIndexDeleted(DeleteIndexEventArgs e) - { - ProfilingLogger.Logger.Debug(GetType(), "Index deleted for term: {0} with value {1}", () => e.DeletedTerm.Key, () => e.DeletedTerm.Value); - base.OnIndexDeleted(e); - } - - [Obsolete("This is no longer used, index optimization is no longer managed with the LuceneIndexer")] - protected override void OnIndexOptimizing(EventArgs e) - { - ProfilingLogger.Logger.Debug(GetType(), "Index is being optimized"); - base.OnIndexOptimizing(e); - } - - /// - /// Overridden for logging. - /// - /// - protected override void AddDocument(ValueSet values) - { - ProfilingLogger.Logger.Debug(GetType(), "AddDocument {0} with type {1}", () => values.Id, () => values.ItemType); - base.AddDocument(values); - } - - protected override void OnTransformingIndexValues(TransformingIndexDataEventArgs e) - { - base.OnTransformingIndexValues(e); - - //ensure special __Path field - if (e.OriginalValues.ContainsKey("path") && e.IndexItem.ValueSet.Values.ContainsKey(IndexPathFieldName) == false) - { - e.IndexItem.ValueSet.Values[IndexPathFieldName] = new List { e.OriginalValues["path"].First() }; - } - - //strip html of all users fields if we detect it has HTML in it. - //if that is the case, we'll create a duplicate 'raw' copy of it so that we can return - //the value of the field 'as-is'. - foreach (var originalValue in e.OriginalValues) - { - if (originalValue.Value.Any()) - { - var str = originalValue.Value.First() as string; - if (str != null) - { - if (XmlHelper.CouldItBeXml(str)) - { - //First save the raw value to a raw field, we will change the policy of this field by detecting the prefix later - e.IndexItem.ValueSet.Values[string.Concat(RawFieldPrefix, originalValue.Key)] = new List { str }; - //now replace the original value with the stripped html - //TODO: This should be done with an analzer?! - e.IndexItem.ValueSet.Values[originalValue.Key] = new List { str.StripHtml() }; - } - } - } - } - - //icon - if (e.OriginalValues.ContainsKey("icon") && e.IndexItem.ValueSet.Values.ContainsKey(IconFieldName) == false) - { - e.IndexItem.ValueSet.Values[IconFieldName] = new List { e.OriginalValues["icon"] }; - } - } - } -} diff --git a/src/Umbraco.Examine/Config/ConfigIndexCriteria.cs b/src/Umbraco.Examine/Config/ConfigIndexCriteria.cs new file mode 100644 index 0000000000..de2a5ced36 --- /dev/null +++ b/src/Umbraco.Examine/Config/ConfigIndexCriteria.cs @@ -0,0 +1,36 @@ +using System.Collections.Generic; +using System.Linq; +using Examine; + +namespace Umbraco.Examine.Config +{ + /// + /// a data structure for storing indexing/searching instructions based on config based indexers + /// + public class ConfigIndexCriteria + { + /// + /// Constructor + /// + /// + /// + /// + /// + /// + public ConfigIndexCriteria(IEnumerable standardFields, IEnumerable userFields, IEnumerable includeNodeTypes, IEnumerable excludeNodeTypes, int? parentNodeId) + { + UserFields = userFields.ToList(); + StandardFields = standardFields.ToList(); + IncludeItemTypes = includeNodeTypes; + ExcludeItemTypes = excludeNodeTypes; + ParentNodeId = parentNodeId; + } + + public IEnumerable StandardFields { get; internal set; } + public IEnumerable UserFields { get; internal set; } + + public IEnumerable IncludeItemTypes { get; internal set; } + public IEnumerable ExcludeItemTypes { get; internal set; } + public int? ParentNodeId { get; internal set; } + } +} \ No newline at end of file diff --git a/src/Umbraco.Examine/Config/ConfigIndexField.cs b/src/Umbraco.Examine/Config/ConfigIndexField.cs new file mode 100644 index 0000000000..ec9cbf797e --- /dev/null +++ b/src/Umbraco.Examine/Config/ConfigIndexField.cs @@ -0,0 +1,61 @@ +using System.Configuration; + +namespace Umbraco.Examine.Config +{ + /// + /// A configuration item representing a field to index + /// + public sealed class ConfigIndexField : ConfigurationElement + { + [ConfigurationProperty("Name", IsRequired = true)] + public string Name + { + get => (string)this["Name"]; + set => this["Name"] = value; + } + + [ConfigurationProperty("EnableSorting", IsRequired = false)] + public bool EnableSorting + { + get => (bool)this["EnableSorting"]; + set => this["EnableSorting"] = value; + } + + [ConfigurationProperty("Type", IsRequired = false, DefaultValue = "String")] + public string Type + { + get => (string)this["Type"]; + set => this["Type"] = value; + } + + public bool Equals(ConfigIndexField other) + { + if (ReferenceEquals(null, other)) return false; + if (ReferenceEquals(this, other)) return true; + return string.Equals(Name, other.Name); + } + + public override bool Equals(object obj) + { + if (ReferenceEquals(null, obj)) return false; + if (ReferenceEquals(this, obj)) return true; + if (obj.GetType() != this.GetType()) return false; + return Equals((ConfigIndexField)obj); + } + + public override int GetHashCode() + { + return Name.GetHashCode(); + } + + public static bool operator ==(ConfigIndexField left, ConfigIndexField right) + { + return Equals(left, right); + } + + public static bool operator !=(ConfigIndexField left, ConfigIndexField right) + { + return !Equals(left, right); + } + } +} \ No newline at end of file diff --git a/src/Umbraco.Examine/Config/IndexFieldCollectionExtensions.cs b/src/Umbraco.Examine/Config/IndexFieldCollectionExtensions.cs new file mode 100644 index 0000000000..eea117ce05 --- /dev/null +++ b/src/Umbraco.Examine/Config/IndexFieldCollectionExtensions.cs @@ -0,0 +1,15 @@ +using System.Collections.Generic; + +namespace Umbraco.Examine.Config +{ + public static class IndexFieldCollectionExtensions + { + public static List ToList(this IndexFieldCollection indexes) + { + List fields = new List(); + foreach (ConfigIndexField field in indexes) + fields.Add(field); + return fields; + } + } +} \ No newline at end of file diff --git a/src/Umbraco.Examine/Config/IndexSet.cs b/src/Umbraco.Examine/Config/IndexSet.cs new file mode 100644 index 0000000000..33b67412f2 --- /dev/null +++ b/src/Umbraco.Examine/Config/IndexSet.cs @@ -0,0 +1,122 @@ +using System.Configuration; +using System.IO; +using System.Web; +using System.Web.Hosting; + +namespace Umbraco.Examine.Config +{ + public sealed class IndexSet : ConfigurationElement + { + + [ConfigurationProperty("SetName", IsRequired = true, IsKey = true)] + public string SetName => (string)this["SetName"]; + + private string _indexPath = ""; + + /// + /// The folder path of where the lucene index is stored + /// + /// The index path. + /// + /// This can be set at runtime but will not be persisted to the configuration file + /// + [ConfigurationProperty("IndexPath", IsRequired = true, IsKey = false)] + public string IndexPath + { + get + { + if (string.IsNullOrEmpty(_indexPath)) + _indexPath = (string)this["IndexPath"]; + + return _indexPath; + } + set => _indexPath = value; + } + + /// + /// Returns the DirectoryInfo object for the index path. + /// + /// The index directory. + public DirectoryInfo IndexDirectory + { + get + { + //TODO: Get this out of the index set. We need to use the Indexer's DataService to lookup the folder so it can be unit tested. Probably need DataServices on the searcher then too + + //we need to de-couple the context + if (HttpContext.Current != null) + return new DirectoryInfo(HttpContext.Current.Server.MapPath(this.IndexPath)); + else if (HostingEnvironment.ApplicationID != null) + return new DirectoryInfo(HostingEnvironment.MapPath(this.IndexPath)); + else + return new DirectoryInfo(this.IndexPath); + } + } + + /// + /// When this property is set, the indexing will only index documents that are descendants of this node. + /// + [ConfigurationProperty("IndexParentId", IsRequired = false, IsKey = false)] + public int? IndexParentId + { + get + { + if (this["IndexParentId"] == null) + return null; + + return (int)this["IndexParentId"]; + } + } + + /// + /// The collection of node types to index, if not specified, all node types will be indexed (apart from the ones specified in the ExcludeNodeTypes collection). + /// + [ConfigurationCollection(typeof(IndexFieldCollection))] + [ConfigurationProperty("IncludeNodeTypes", IsDefaultCollection = false, IsRequired = false)] + public IndexFieldCollection IncludeNodeTypes => (IndexFieldCollection)base["IncludeNodeTypes"]; + + /// + /// The collection of node types to not index. If specified, these node types will not be indexed. + /// + [ConfigurationCollection(typeof(IndexFieldCollection))] + [ConfigurationProperty("ExcludeNodeTypes", IsDefaultCollection = false, IsRequired = false)] + public IndexFieldCollection ExcludeNodeTypes => (IndexFieldCollection)base["ExcludeNodeTypes"]; + + /// + /// A collection of user defined umbraco fields to index + /// + /// + /// If this property is not specified, or if it's an empty collection, the default user fields will be all user fields defined in Umbraco + /// + [ConfigurationCollection(typeof(IndexFieldCollection))] + [ConfigurationProperty("IndexUserFields", IsDefaultCollection = false, IsRequired = false)] + public IndexFieldCollection IndexUserFields => (IndexFieldCollection)base["IndexUserFields"]; + + /// + /// The fields umbraco values that will be indexed. i.e. id, nodeTypeAlias, writer, etc... + /// + /// + /// If this is not specified, or if it's an empty collection, the default optins will be specified: + /// - id + /// - version + /// - parentID + /// - level + /// - writerID + /// - creatorID + /// - nodeType + /// - template + /// - sortOrder + /// - createDate + /// - updateDate + /// - nodeName + /// - urlName + /// - writerName + /// - creatorName + /// - nodeTypeAlias + /// - path + /// + [ConfigurationCollection(typeof(IndexFieldCollection))] + [ConfigurationProperty("IndexAttributeFields", IsDefaultCollection = false, IsRequired = false)] + public IndexFieldCollection IndexAttributeFields => (IndexFieldCollection)base["IndexAttributeFields"]; + } +} diff --git a/src/Umbraco.Examine/Config/IndexSetCollection.cs b/src/Umbraco.Examine/Config/IndexSetCollection.cs new file mode 100644 index 0000000000..5ba2382563 --- /dev/null +++ b/src/Umbraco.Examine/Config/IndexSetCollection.cs @@ -0,0 +1,73 @@ +using System.Configuration; + + +namespace Umbraco.Examine.Config +{ + public sealed class IndexFieldCollection : ConfigurationElementCollection + { + #region Overridden methods to define collection + protected override ConfigurationElement CreateNewElement() + { + return new ConfigIndexField(); + } + protected override object GetElementKey(ConfigurationElement element) + { + ConfigIndexField field = (ConfigIndexField)element; + return field.Name; + } + + public override bool IsReadOnly() + { + return false; + } + #endregion + + /// + /// Adds an index field to the collection + /// + /// + public void Add(ConfigIndexField field) + { + BaseAdd(field, true); + } + + /// + /// Default property for accessing an IndexField definition + /// + /// Field Name + /// + public new ConfigIndexField this[string name] + { + get + { + return (ConfigIndexField)this.BaseGet(name); + } + } + + } + + + public sealed class IndexSetCollection : ConfigurationElementCollection + { + #region Overridden methods to define collection + protected override ConfigurationElement CreateNewElement() + { + return new IndexSet(); + } + protected override object GetElementKey(ConfigurationElement element) + { + return ((IndexSet)element).SetName; + } + public override ConfigurationElementCollectionType CollectionType => ConfigurationElementCollectionType.BasicMap; + protected override string ElementName => "IndexSet"; + + #endregion + + /// + /// Default property for accessing Image Sets + /// + /// + /// + public new IndexSet this[string setName] => (IndexSet)this.BaseGet(setName); + } +} \ No newline at end of file diff --git a/src/Umbraco.Examine/Config/IndexSets.cs b/src/Umbraco.Examine/Config/IndexSets.cs new file mode 100644 index 0000000000..4aa90b99d7 --- /dev/null +++ b/src/Umbraco.Examine/Config/IndexSets.cs @@ -0,0 +1,26 @@ +using System.Configuration; + +namespace Umbraco.Examine.Config +{ + public sealed class IndexSets : ConfigurationSection + { + + #region Singleton definition + + protected IndexSets() { } + static IndexSets() + { + Instance = ConfigurationManager.GetSection(SectionName) as IndexSets; + + } + public static IndexSets Instance { get; } + + #endregion + + private const string SectionName = "ExamineLuceneIndexSets"; + + [ConfigurationCollection(typeof(IndexSetCollection))] + [ConfigurationProperty("", IsDefaultCollection = true, IsRequired = true)] + public IndexSetCollection Sets => (IndexSetCollection)base[""]; + } +} \ No newline at end of file diff --git a/src/Umbraco.Examine/DataServices/UmbracoContentService.cs b/src/Umbraco.Examine/DataServices/UmbracoContentService.cs deleted file mode 100644 index 5f3fba6505..0000000000 --- a/src/Umbraco.Examine/DataServices/UmbracoContentService.cs +++ /dev/null @@ -1,115 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using Umbraco.Core; -using System.Xml.Linq; -using System.Collections; -using System.Xml.XPath; -using Examine.LuceneEngine; -using Umbraco.Core.Logging; -using Umbraco.Core.Scoping; -using Umbraco.Core.Services; -using Umbraco.Core.Models; - -namespace Umbraco.Examine.DataServices -{ - public class UmbracoContentService - { - private readonly IScopeProvider _scopeProvider; - private readonly ServiceContext _services; - private readonly ILogger _logger; - - public UmbracoContentService(IScopeProvider scopeProvider, ServiceContext services, ILogger logger) - { - _scopeProvider = scopeProvider; - _services = services; - _logger = logger; - } - - /// - /// removes html markup from a string - /// - /// - /// - public string StripHtml(string value) - { - return value.StripHtml(); - } - - /// - /// Gets published content by xpath - /// - /// - /// - public XDocument GetPublishedContentByXPath(string xpath) - { - //TODO: Remove the need for this, the best way would be to remove all requirements of examine based on Xml but that - // would take some time. Another way in the in-term would be to add a static delegate to this class which can be set - // on the WebBootManager to set how to get the XmlNodeByXPath but that is still ugly :( - return LegacyLibrary.GetXmlNodeByXPath(xpath).ToXDocument(); - } - - /// - /// This is quite an intensive operation... - /// get all root content, then get the XML structure for all children, - /// then run xpath against the navigator that's created - /// - /// - /// - [Obsolete("This should no longer be used, latest content will be indexed by using the IContentService directly")] - public XDocument GetLatestContentByXPath(string xpath) - { - var xmlContent = XDocument.Parse(""); - foreach (var c in _services.ContentService.GetRootContent()) - { - xmlContent.Root.Add(c.ToDeepXml(_services.PackagingService)); - } - var result = ((IEnumerable)xmlContent.XPathEvaluate(xpath)).Cast(); - return result.ToXDocument(); - } - - /// - /// Check if the node is protected - /// - /// - /// - /// - public bool IsProtected(int nodeId, string path) - { - return _services.PublicAccessService.IsProtected(path.EnsureEndsWith("," + nodeId)); - } - - /// - /// Returns a list of all of the user defined property names in Umbraco - /// - /// - - public IEnumerable GetAllUserPropertyNames() - { - using (var scope = _scopeProvider.CreateScope()) - { - try - { - var result = scope.Database.Fetch("select distinct alias from cmsPropertyType order by alias"); - scope.Complete(); - return result; - } - catch (Exception ex) - { - _logger.Error("EXCEPTION OCCURRED reading GetAllUserPropertyNames", ex); - scope.Complete(); - return Enumerable.Empty(); - } - } - } - - /// - /// Returns a list of all system field names in Umbraco - /// - /// - public IEnumerable GetAllSystemPropertyNames() - { - return UmbracoContentIndexer.IndexFieldPolicies.Select(x => x.Name); - } - } -} diff --git a/src/Umbraco.Examine/DataServices/UmbracoMediaService.cs b/src/Umbraco.Examine/DataServices/UmbracoMediaService.cs deleted file mode 100644 index 436cd99b9e..0000000000 --- a/src/Umbraco.Examine/DataServices/UmbracoMediaService.cs +++ /dev/null @@ -1,45 +0,0 @@ -using System; -using System.Collections; -using System.Linq; -using System.Xml.Linq; -using System.Xml.XPath; -using Examine.LuceneEngine; -using Umbraco.Core.Services; -using Umbraco.Core.Models; - -namespace Umbraco.Examine.DataServices -{ - - /// - /// Data service used to query for media - /// - [Obsolete("This should no longer be used, latest content will be indexed by using the IMediaService directly")] - public class UmbracoMediaService - { - private readonly ServiceContext _services; - - public UmbracoMediaService(ServiceContext services) - { - _services = services; - } - - /// - /// This is quite an intensive operation... - /// get all root media, then get the XML structure for all children, - /// then run xpath against the navigator that's created - /// - /// - /// - [Obsolete("This should no longer be used, latest content will be indexed by using the IMediaService directly")] - public XDocument GetLatestMediaByXpath(string xpath) - { - var xmlMedia = XDocument.Parse(""); - foreach (var m in _services.MediaService.GetRootMedia()) - { - xmlMedia.Root.Add(m.ToDeepXml(_services.PackagingService)); - } - var result = ((IEnumerable)xmlMedia.XPathEvaluate(xpath)).Cast(); - return result.ToXDocument(); - } - } -} diff --git a/src/Umbraco.Examine/DeletePolicyTracker.cs b/src/Umbraco.Examine/DeletePolicyTracker.cs deleted file mode 100644 index 2c851ed39b..0000000000 --- a/src/Umbraco.Examine/DeletePolicyTracker.cs +++ /dev/null @@ -1,23 +0,0 @@ -using System.Collections.Concurrent; -using System.IO; -using Lucene.Net.Index; - -namespace Umbraco.Examine -{ - internal sealed class DeletePolicyTracker - { - private static readonly DeletePolicyTracker Instance = new DeletePolicyTracker(); - private readonly ConcurrentDictionary _directories = new ConcurrentDictionary(); - - public static DeletePolicyTracker Current - { - get { return Instance; } - } - - public IndexDeletionPolicy GetPolicy(Lucene.Net.Store.Directory directory) - { - var resolved = _directories.GetOrAdd(directory.GetLockId(), s => new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy())); - return resolved; - } - } -} diff --git a/src/Umbraco.Examine/ExamineIndexCollectionAccessor.cs b/src/Umbraco.Examine/ExamineIndexCollectionAccessor.cs index 806cbeadc7..7367069321 100644 --- a/src/Umbraco.Examine/ExamineIndexCollectionAccessor.cs +++ b/src/Umbraco.Examine/ExamineIndexCollectionAccessor.cs @@ -8,6 +8,6 @@ namespace Umbraco.Examine /// public class ExamineIndexCollectionAccessor : IExamineIndexCollectionAccessor { - public IReadOnlyDictionary Indexes => ExamineManager.Instance.IndexProviders; + public IReadOnlyDictionary Indexes => ExamineManager.Instance.IndexProviders; } } diff --git a/src/Umbraco.Examine/IExamineIndexCollectionAccessor.cs b/src/Umbraco.Examine/IExamineIndexCollectionAccessor.cs index 4c55abc789..674faebb68 100644 --- a/src/Umbraco.Examine/IExamineIndexCollectionAccessor.cs +++ b/src/Umbraco.Examine/IExamineIndexCollectionAccessor.cs @@ -4,10 +4,10 @@ using Examine; namespace Umbraco.Examine { /// - /// Returns a collection of IExamineIndexer + /// Returns a collection of /// public interface IExamineIndexCollectionAccessor { - IReadOnlyDictionary Indexes { get; } + IReadOnlyDictionary Indexes { get; } } } diff --git a/src/Umbraco.Examine/LegacyExtensions.cs b/src/Umbraco.Examine/LegacyExtensions.cs deleted file mode 100644 index cf1eebb5e4..0000000000 --- a/src/Umbraco.Examine/LegacyExtensions.cs +++ /dev/null @@ -1,50 +0,0 @@ -using System.Linq; -using Examine; -using Examine.LuceneEngine.Config; -using Umbraco.Core.Services; - -namespace Umbraco.Examine -{ - internal static class LegacyExtensions - { - - private static readonly object Locker = new object(); - - public static IIndexCriteria ToIndexCriteria(this IndexSet set, IContentTypeService contentTypeService) - { - if (set.IndexUserFields.Count == 0) - { - lock (Locker) - { - //we need to add all user fields to the collection if it is empty (this is the default if none are specified) - var userFields = contentTypeService.GetAllPropertyTypeAliases(); - foreach (var u in userFields) - { - set.IndexUserFields.Add(new IndexField() { Name = u }); - } - } - } - - if (set.IndexAttributeFields.Count == 0) - { - lock (Locker) - { - //we need to add all system fields to the collection if it is empty (this is the default if none are specified) - var sysFields = BaseUmbracoIndexer.IndexFieldPolicies.Select(x => x.Name); - foreach (var s in sysFields) - { - set.IndexAttributeFields.Add(new IndexField() { Name = s }); - } - } - } - - return new IndexCriteria( - set.IndexAttributeFields.Cast().ToArray(), - set.IndexUserFields.Cast().ToArray(), - set.IncludeNodeTypes.ToList().Select(x => x.Name).ToArray(), - set.ExcludeNodeTypes.ToList().Select(x => x.Name).ToArray(), - set.IndexParentId); - } - - } -} diff --git a/src/Umbraco.Examine/LegacyLibrary.cs b/src/Umbraco.Examine/LegacyLibrary.cs deleted file mode 100644 index 5f3f296c9d..0000000000 --- a/src/Umbraco.Examine/LegacyLibrary.cs +++ /dev/null @@ -1,47 +0,0 @@ -using System; -using System.Reflection; -using System.Xml.XPath; - -namespace Umbraco.Examine -{ - /// - /// This is only used for backward compatibility to get access to the umbraco.library object but this needs to be done - /// via reflection because of the circular reference we have between Umbraco.Web and UmbracoExamine. - /// - internal static class LegacyLibrary - { - private static volatile Type _libraryType; - private static readonly object Locker = new object(); - private static Type LibraryType - { - get - { - if (_libraryType == null) - { - lock (Locker) - { - if (_libraryType == null) - { - var ass = Assembly.Load("Umbraco.Web"); - if (ass == null) - throw new InvalidOperationException("Could not load assembly Umbraco.Web.dll, the Umbraco.Web.dll needs to be loaded in the current app domain"); - var lib = ass.GetType("umbraco.library"); - if (lib == null) - throw new InvalidOperationException("Could not load type umbraco.library, the Umbraco.Web.dll needs to be loaded in the current app domain"); - _libraryType = lib; - } - } - } - return _libraryType; - } - } - - - internal static XPathNodeIterator GetXmlNodeByXPath(string xpathQuery) - { - var meth = LibraryType.GetMethod("GetXmlNodeByXPath", BindingFlags.Public | BindingFlags.Static); - return (XPathNodeIterator)meth.Invoke(null, new object[] { xpathQuery }); - } - - } -} diff --git a/src/Umbraco.Examine/LocalStorage/AzureLocalStorageDirectory.cs b/src/Umbraco.Examine/LocalStorage/AzureLocalStorageDirectory.cs deleted file mode 100644 index 1cdc64f879..0000000000 --- a/src/Umbraco.Examine/LocalStorage/AzureLocalStorageDirectory.cs +++ /dev/null @@ -1,33 +0,0 @@ -using System; -using System.Collections.Specialized; -using System.ComponentModel; -using System.IO; -using System.Web; -using Umbraco.Core; - -namespace Umbraco.Examine.LocalStorage -{ - /// - /// When running on Azure websites, we can use the local user's storage space - /// - [Obsolete("This has been superceded by IDirectoryFactory in Examine Core and should not be used")] - [EditorBrowsable(EditorBrowsableState.Never)] - public sealed class AzureLocalStorageDirectory : ILocalStorageDirectory - { - public DirectoryInfo GetLocalStorageDirectory(NameValueCollection config, string configuredPath) - { - var appDomainHash = HttpRuntime.AppDomainAppId.GenerateHash(); - var cachePath = Path.Combine(Environment.ExpandEnvironmentVariables("%temp%"), "LuceneDir", - //include the appdomain hash is just a safety check, for example if a website is moved from worker A to worker B and then back - // to worker A again, in theory the %temp% folder should already be empty but we really want to make sure that its not - // utilizing an old index - appDomainHash, - //ensure the temp path is consistent with the configured path location - configuredPath.TrimStart('~', '/').Replace("/", "\\")); - var azureDir = new DirectoryInfo(cachePath); - if (azureDir.Exists == false) - azureDir.Create(); - return azureDir; - } - } -} diff --git a/src/Umbraco.Examine/LocalStorage/CodeGenLocalStorageDirectory.cs b/src/Umbraco.Examine/LocalStorage/CodeGenLocalStorageDirectory.cs deleted file mode 100644 index 0678ca359e..0000000000 --- a/src/Umbraco.Examine/LocalStorage/CodeGenLocalStorageDirectory.cs +++ /dev/null @@ -1,29 +0,0 @@ -using System; -using System.Collections.Specialized; -using System.ComponentModel; -using System.IO; -using System.Web; - -namespace Umbraco.Examine.LocalStorage -{ - /// - /// Use the ASP.Net CodeGen folder to store the index files - /// - /// - /// This is the default implementation - but it comes with it's own limitations - the CodeGen folder is cleared whenever new - /// DLLs are changed in the /bin folder (among other circumstances) which means the index would be re-synced (or rebuilt) there. - /// - [Obsolete("This has been superceded by IDirectoryFactory in Examine Core and should not be used")] - [EditorBrowsable(EditorBrowsableState.Never)] - public sealed class CodeGenLocalStorageDirectory : ILocalStorageDirectory - { - public DirectoryInfo GetLocalStorageDirectory(NameValueCollection config, string configuredPath) - { - var codegenPath = HttpRuntime.CodegenDir; - var path = Path.Combine(codegenPath, - //ensure the temp path is consistent with the configured path location - configuredPath.TrimStart('~', '/').Replace("/", "\\")); - return new DirectoryInfo(path); - } - } -} diff --git a/src/Umbraco.Examine/LocalStorage/DirectoryTracker.cs b/src/Umbraco.Examine/LocalStorage/DirectoryTracker.cs deleted file mode 100644 index 5a6d4f6d2f..0000000000 --- a/src/Umbraco.Examine/LocalStorage/DirectoryTracker.cs +++ /dev/null @@ -1,26 +0,0 @@ -using System; -using System.Collections.Concurrent; -using System.IO; -using Lucene.Net.Store; - -namespace Umbraco.Examine.LocalStorage -{ - [Obsolete("This is used only for configuration based indexes")] - internal class DirectoryTracker - { - private static readonly DirectoryTracker Instance = new DirectoryTracker(); - - private readonly ConcurrentDictionary _directories = new ConcurrentDictionary(); - - public static DirectoryTracker Current - { - get { return Instance; } - } - - public Lucene.Net.Store.Directory GetDirectory(DirectoryInfo dir) - { - var resolved = _directories.GetOrAdd(dir.FullName, s => new SimpleFSDirectory(dir)); - return resolved; - } - } -} diff --git a/src/Umbraco.Examine/LocalStorage/ILocalStorageDirectory.cs b/src/Umbraco.Examine/LocalStorage/ILocalStorageDirectory.cs deleted file mode 100644 index aec007ec60..0000000000 --- a/src/Umbraco.Examine/LocalStorage/ILocalStorageDirectory.cs +++ /dev/null @@ -1,13 +0,0 @@ -using System.Collections.Specialized; -using System.IO; - -namespace Umbraco.Examine.LocalStorage -{ - /// - /// Used to resolve the local storage folder - /// - public interface ILocalStorageDirectory - { - DirectoryInfo GetLocalStorageDirectory(NameValueCollection config, string configuredPath); - } -} diff --git a/src/Umbraco.Examine/LocalStorage/LocalStorageType.cs b/src/Umbraco.Examine/LocalStorage/LocalStorageType.cs deleted file mode 100644 index e1d2e0f943..0000000000 --- a/src/Umbraco.Examine/LocalStorage/LocalStorageType.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace Umbraco.Examine.LocalStorage -{ - public enum LocalStorageType - { - Sync, - LocalOnly - } -} diff --git a/src/Umbraco.Examine/LocalStorage/LocalTempStorageDirectory.cs b/src/Umbraco.Examine/LocalStorage/LocalTempStorageDirectory.cs deleted file mode 100644 index 6319dc8878..0000000000 --- a/src/Umbraco.Examine/LocalStorage/LocalTempStorageDirectory.cs +++ /dev/null @@ -1,207 +0,0 @@ -using System; -using System.IO; -using System.Linq; -using Lucene.Net.Store; -using Directory = Lucene.Net.Store.Directory; - -namespace Umbraco.Examine.LocalStorage -{ - /// - /// Used to read data from local temp storage and write to both local storage and main storage - /// - public class LocalTempStorageDirectory : Directory - { - private readonly FSDirectory _tempStorageDir; - private readonly FSDirectory _realDirectory; - private LockFactory _lockFactory; - - public LocalTempStorageDirectory( - DirectoryInfo tempStorageDir, - FSDirectory realDirectory) - { - if (tempStorageDir == null) throw new ArgumentNullException("tempStorageDir"); - if (realDirectory == null) throw new ArgumentNullException("realDirectory"); - - _tempStorageDir = new SimpleFSDirectory(tempStorageDir); - _realDirectory = realDirectory; - _lockFactory = new MultiIndexLockFactory(_realDirectory, _tempStorageDir); - - Enabled = true; - - } - - /// - /// If initialization fails, it will be disabled and then this will just wrap the 'real directory' - /// - internal bool Enabled { get; set; } - - public override string[] ListAll() - { - //always from the real dir - return _realDirectory.ListAll(); - } - - /// Returns true if a file with the given name exists. - public override bool FileExists(string name) - { - //always from the real dir - return _realDirectory.FileExists(name); - } - - /// Returns the time the named file was last modified. - public override long FileModified(string name) - { - //always from the real dir - return _realDirectory.FileModified(name); - } - - /// Set the modified time of an existing file to now. - public override void TouchFile(string name) - { - //always from the real dir - _realDirectory.TouchFile(name); - - //perform on both dirs - if (Enabled) - { - _tempStorageDir.TouchFile(name); - } - } - - /// Removes an existing file in the directory. - public override void DeleteFile(string name) - { - _realDirectory.DeleteFile(name); - - //perform on both dirs - if (Enabled) - { - _tempStorageDir.DeleteFile(name); - } - } - - - /// Returns the length of a file in the directory. - public override long FileLength(string name) - { - //always from the real dir - return _realDirectory.FileLength(name); - } - - /// - /// Creates a new, empty file in the directory with the given name. - /// Returns a stream writing this file. - /// - public override IndexOutput CreateOutput(string name) - { - //write to both indexes - if (Enabled) - { - return new MultiIndexOutput( - _tempStorageDir.CreateOutput(name), - _realDirectory.CreateOutput(name)); - } - - return _realDirectory.CreateOutput(name); - } - - /// - /// Returns a stream reading an existing file. - /// - public override IndexInput OpenInput(string name) - { - if (Enabled) - { - //return the reader from the cache, not the real dir - return _tempStorageDir.OpenInput(name); - } - - return _realDirectory.OpenInput(name); - } - - /// - /// Creates an IndexInput for the file with the given name. - /// - public override IndexInput OpenInput(string name, int bufferSize) - { - if (Enabled) - { - //return the reader from the cache, not the real dir - return _tempStorageDir.OpenInput(name, bufferSize); - } - return _realDirectory.OpenInput(name, bufferSize); - } - - /// - /// Ensure that any writes to this file are moved to - /// stable storage. Lucene uses this to properly commit - /// changes to the index, to prevent a machine/OS crash - /// from corrupting the index. - /// - public override void Sync(string name) - { - _realDirectory.Sync(name); - _tempStorageDir.Sync(name); - base.Sync(name); - } - - public override Lock MakeLock(string name) - { - return _lockFactory.MakeLock(name); - } - - /// - /// Return a string identifier that uniquely differentiates - /// this Directory instance from other Directory instances. - /// This ID should be the same if two Directory instances - /// (even in different JVMs and/or on different machines) - /// are considered "the same index". This is how locking - /// "scopes" to the right index. - /// - /// - public override string GetLockId() - { - return string.Concat(_realDirectory.GetLockId(), _tempStorageDir.GetLockId()); - } - - public override LockFactory LockFactory - { - get - { - return _lockFactory; - //return _realDirectory.GetLockFactory(); - } - - } - - public override void ClearLock(string name) - { - _lockFactory.ClearLock(name); - - //_realDirectory.ClearLock(name); - - //if (Enabled) - //{ - // _tempStorageDir.ClearLock(name); - //} - } - - protected override void Dispose(bool disposing) - { - if (Enabled) - { - _tempStorageDir.Dispose(); - } - _realDirectory.Dispose(); - } - - public override void SetLockFactory(LockFactory lf) - { - _lockFactory = lf; - //_realDirectory.SetLockFactory(lf); - } - - - - } -} diff --git a/src/Umbraco.Examine/LocalStorage/LocalTempStorageDirectoryTracker.cs b/src/Umbraco.Examine/LocalStorage/LocalTempStorageDirectoryTracker.cs deleted file mode 100644 index d519925c07..0000000000 --- a/src/Umbraco.Examine/LocalStorage/LocalTempStorageDirectoryTracker.cs +++ /dev/null @@ -1,29 +0,0 @@ -using System.Collections.Concurrent; -using System.IO; -using Lucene.Net.Store; - -namespace Umbraco.Examine.LocalStorage -{ - internal class LocalTempStorageDirectoryTracker - { - private static readonly LocalTempStorageDirectoryTracker Instance = new LocalTempStorageDirectoryTracker(); - private readonly ConcurrentDictionary _directories = new ConcurrentDictionary(); - - public static LocalTempStorageDirectoryTracker Current - { - get { return Instance; } - } - - public LocalTempStorageDirectory GetDirectory(DirectoryInfo dir, FSDirectory realDir, bool disable = false) - { - var resolved = _directories.GetOrAdd(dir.FullName, s => new LocalTempStorageDirectory(dir, realDir)); - - if (disable) - { - resolved.Enabled = false; - } - - return resolved; - } - } -} diff --git a/src/Umbraco.Examine/LocalStorage/LocalTempStorageIndexer.cs b/src/Umbraco.Examine/LocalStorage/LocalTempStorageIndexer.cs deleted file mode 100644 index 93102578dd..0000000000 --- a/src/Umbraco.Examine/LocalStorage/LocalTempStorageIndexer.cs +++ /dev/null @@ -1,412 +0,0 @@ -using System; -using System.Collections.Specialized; -using System.IO; -using System.Linq; -using System.Threading; -using System.Web; -using System.Web.Compilation; -using Examine.LuceneEngine; -using Lucene.Net.Analysis; -using Lucene.Net.Index; -using Lucene.Net.Store; -using Umbraco.Core; -using Umbraco.Core.Composing; -using Umbraco.Core.IO; -using Umbraco.Core.Logging; -using Directory = System.IO.Directory; - -namespace Umbraco.Examine.LocalStorage -{ - internal enum InitializeDirectoryFlags - { - Success = 0, - SuccessNoIndexExists = 1, - - FailedCorrupt = 100, - FailedLocked = 101, - FailedFileSync = 102 - } - - internal class LocalTempStorageIndexer - { - public Lucene.Net.Store.Directory LuceneDirectory { get; private set; } - private readonly object _locker = new object(); - public SnapshotDeletionPolicy Snapshotter { get; private set; } - - public string TempPath { get; private set; } - - - public LocalTempStorageIndexer() - { - throw new NotImplementedException("Fix how local temp storage works and is synced with Examine v2.0 - since a writer is always open we cannot snapshot it, we need to use the same logic in AzureDirectory"); - - IndexDeletionPolicy policy = new KeepOnlyLastCommitDeletionPolicy(); - Snapshotter = new SnapshotDeletionPolicy(policy); - } - - public void Initialize(NameValueCollection config, string configuredPath, FSDirectory baseLuceneDirectory, Analyzer analyzer, LocalStorageType localStorageType) - { - //this is the default - ILocalStorageDirectory localStorageDir = new CodeGenLocalStorageDirectory(); - if (config["tempStorageDirectory"] != null) - { - //try to get the type - var dirType = BuildManager.GetType(config["tempStorageDirectory"], false); - if (dirType != null) - { - try - { - localStorageDir = (ILocalStorageDirectory)Activator.CreateInstance(dirType); - } - catch (Exception ex) - { - Current.Logger.Error( - string.Format("Could not create a temp storage location of type {0}, reverting to use the " + typeof (CodeGenLocalStorageDirectory).FullName, dirType), - ex); - } - } - } - - var tempPath = localStorageDir.GetLocalStorageDirectory(config, configuredPath); - if (tempPath == null) throw new InvalidOperationException("Could not resolve a temp location from the " + localStorageDir.GetType() + " specified"); - TempPath = tempPath.FullName; - - switch (localStorageType) - { - case LocalStorageType.Sync: - var success = InitializeLocalIndexAndDirectory(baseLuceneDirectory, analyzer, configuredPath); - - //create the custom lucene directory which will keep the main and temp FS's in sync - LuceneDirectory = LocalTempStorageDirectoryTracker.Current.GetDirectory( - new DirectoryInfo(TempPath), - baseLuceneDirectory, - //flag to disable the mirrored folder if not successful - (int)success >= 100); - - //If the master index simply doesn't exist, we don't continue to try to open anything since there will - // actually be nothing there. - if (success == InitializeDirectoryFlags.SuccessNoIndexExists) - { - return; - } - - //Try to open the reader, this will fail if the index is corrupt and we'll need to handle that - var result = DelegateExtensions.RetryUntilSuccessOrMaxAttempts(i => - { - try - { - using (IndexReader.Open( - LuceneDirectory, - DeletePolicyTracker.Current.GetPolicy(LuceneDirectory), - true)) - { - } - - return Attempt.Succeed(true); - } - catch (Exception ex) - { - Current.Logger.Warn( - ex, - string.Format("Could not open an index reader, local temp storage index is empty or corrupt... retrying... {0}", configuredPath)); - } - return Attempt.Fail(false); - }, 5, TimeSpan.FromSeconds(1)); - - if (result.Success == false) - { - Current.Logger.Warn( - string.Format("Could not open an index reader, local temp storage index is empty or corrupt... attempting to clear index files in local temp storage, will operate from main storage only {0}", configuredPath)); - - ClearFilesInPath(TempPath); - - //create the custom lucene directory which will keep the main and temp FS's in sync - LuceneDirectory = LocalTempStorageDirectoryTracker.Current.GetDirectory( - new DirectoryInfo(TempPath), - baseLuceneDirectory, - //Disable mirrored index, we're kind of screwed here only use master index - true); - } - - break; - case LocalStorageType.LocalOnly: - if (Directory.Exists(TempPath) == false) - { - Directory.CreateDirectory(TempPath); - } - LuceneDirectory = DirectoryTracker.Current.GetDirectory(new DirectoryInfo(TempPath)); - break; - default: - throw new ArgumentOutOfRangeException("localStorageType"); - } - } - - private void ClearFilesInPath(string path) - { - if (Directory.Exists(path)) - { - foreach (var file in Directory.GetFiles(path)) - { - try - { - File.Delete(file); - } - catch (Exception exInner) - { - Current.Logger.Error("Could not delete local temp storage index file", exInner); - } - } - } - } - - private bool ClearLuceneDirFiles(Lucene.Net.Store.Directory baseLuceneDirectory) - { - try - { - //unlock it! - IndexWriter.Unlock(baseLuceneDirectory); - - var fileLuceneDirectory = baseLuceneDirectory as FSDirectory; - if (fileLuceneDirectory != null) - { - foreach (var file in fileLuceneDirectory.ListAll()) - { - try - { - fileLuceneDirectory.DeleteFile(file); - } - catch (IOException) - { - if (file.InvariantEquals("write.lock")) - { - Current.Logger.Warn("The lock file could not be deleted but should be removed when the writer is disposed"); - } - - } - } - return true; - } - return false; - } - catch (Exception ex) - { - Current.Logger.Error("Could not clear corrupt index from main index folder, the index cannot be used", ex); - return false; - } - } - - private Attempt TryCreateWriter(Lucene.Net.Store.Directory baseLuceneDirectory, Analyzer analyzer) - { - try - { - var w = new IndexWriter( - //read from the underlying/default directory, not the temp codegen dir - baseLuceneDirectory, - analyzer, - Snapshotter, - IndexWriter.MaxFieldLength.UNLIMITED); - - //Done! - return Attempt.Succeed(w); - } - catch (Exception ex) - { - Current.Logger.Warn(ex, "Could not create index writer with snapshot policy for copying... retrying..."); - return Attempt.Fail(ex); - } - } - - /// - /// Attempts to create an index writer, it will retry on failure 5 times and on the last time will try to forcefully unlock the index files - /// - /// - /// - /// - private Attempt TryCreateWriterWithRetry(Lucene.Net.Store.Directory baseLuceneDirectory, Analyzer analyzer) - { - var maxTries = 5; - - var result = DelegateExtensions.RetryUntilSuccessOrMaxAttempts((currentTry) => - { - //last try... - if (currentTry == maxTries) - { - Current.Logger.Info("Could not acquire index lock, attempting to force unlock it..."); - //unlock it! - IndexWriter.Unlock(baseLuceneDirectory); - } - - var writerAttempt = TryCreateWriter(baseLuceneDirectory, analyzer); - if (writerAttempt) return writerAttempt; - Current.Logger.Info("Could not create writer on {0}, retrying ....", baseLuceneDirectory.ToString); - return Attempt.Fail(); - }, 5, TimeSpan.FromSeconds(1)); - - return result; - } - - private bool TryWaitForDirectoryUnlock(Lucene.Net.Store.Directory dir) - { - var maxTries = 5; - - var result = DelegateExtensions.RetryUntilSuccessOrMaxAttempts((currentTry) => - { - //last try... - if (currentTry == maxTries) - { - Current.Logger.Info("Could not acquire directory lock, attempting to force unlock it..."); - //unlock it! - IndexWriter.Unlock(dir); - } - - if (IndexWriter.IsLocked(dir) == false) return Attempt.Succeed(true); - Current.Logger.Info("Could not acquire directory lock for {0} writer, retrying ....", dir.ToString); - return Attempt.Fail(); - }, 5, TimeSpan.FromSeconds(1)); - - return result; - } - - private InitializeDirectoryFlags InitializeLocalIndexAndDirectory(Lucene.Net.Store.Directory baseLuceneDirectory, Analyzer analyzer, string configuredPath) - { - lock (_locker) - { - if (Directory.Exists(TempPath) == false) - { - Directory.CreateDirectory(TempPath); - } - - //copy index if it exists, don't do anything if it's not there - if (IndexReader.IndexExists(baseLuceneDirectory) == false) return InitializeDirectoryFlags.SuccessNoIndexExists; - - var writerAttempt = TryCreateWriterWithRetry(baseLuceneDirectory, analyzer); - - if (writerAttempt.Success == false) - { - Current.Logger.Error("Could not create index writer with snapshot policy for copying, the index cannot be used", writerAttempt.Exception); - return InitializeDirectoryFlags.FailedLocked; - } - - //Try to open the reader from the source, this will fail if the index is corrupt and we'll need to handle that - try - { - //NOTE: To date I've not seen this error occur - using (writerAttempt.Result.GetReader()) - { - } - } - catch (Exception ex) - { - writerAttempt.Result.Dispose(); - - Current.Logger.Error( - string.Format("Could not open an index reader, {0} is empty or corrupt... attempting to clear index files in master folder", configuredPath), - ex); - - if (ClearLuceneDirFiles(baseLuceneDirectory) == false) - { - //hrm, not much we can do in this situation, but this shouldn't happen - Current.Logger.Error("Could not open an index reader, index is corrupt.", ex); - return InitializeDirectoryFlags.FailedCorrupt; - } - - //the main index is now blank, we'll proceed as normal with a new empty index... - writerAttempt = TryCreateWriter(baseLuceneDirectory, analyzer); - if (writerAttempt.Success == false) - { - //ultra fail... - Current.Logger.Error("Could not create index writer with snapshot policy for copying, the index cannot be used", writerAttempt.Exception); - return InitializeDirectoryFlags.FailedLocked; - } - } - - using (writerAttempt.Result) - { - try - { - var basePath = IOHelper.MapPath(configuredPath); - - var commit = Snapshotter.Snapshot(); - var allSnapshotFiles = commit.FileNames - .Concat(new[] - { - commit.SegmentsFileName, - //we need to manually include the segments.gen file - "segments.gen" - }) - .Distinct() - .ToArray(); - - var tempDir = new DirectoryInfo(TempPath); - - //Get all files in the temp storage that don't exist in the snapshot collection, we want to remove these - var toRemove = tempDir.GetFiles() - .Select(x => x.Name) - .Except(allSnapshotFiles); - - using (var tempDirectory = new SimpleFSDirectory(tempDir)) - { - if (TryWaitForDirectoryUnlock(tempDirectory)) - { - foreach (var file in toRemove) - { - try - { - File.Delete(Path.Combine(TempPath, file)); - } - catch (IOException ex) - { - if (file.InvariantEquals("write.lock")) - { - //This might happen if the writer is open - Current.Logger.Warn("The lock file could not be deleted but should be removed when the writer is disposed"); - } - - Current.Logger.Debug("Could not delete non synced index file file, index sync will continue but old index files will remain - this shouldn't affect indexing/searching operations. {0}", () => ex.ToString()); - - } - } - } - else - { - //quit here, this shouldn't happen with all the checks above. - Current.Logger.Warn("Cannot sync index files from main storage, the temp file index is currently locked"); - return InitializeDirectoryFlags.FailedLocked; - } - - - foreach (var fileName in allSnapshotFiles.Where(f => f.IsNullOrWhiteSpace() == false)) - { - var destination = Path.Combine(TempPath, Path.GetFileName(fileName)); - - //don't copy if it's already there, lucene is 'write once' so this file is meant to be there already - if (File.Exists(destination)) continue; - - try - { - File.Copy( - Path.Combine(basePath, "Index", fileName), - destination); - } - catch (IOException ex) - { - Current.Logger.Error("Could not copy index file, could not sync from main storage", ex); - - //quit here - return InitializeDirectoryFlags.FailedFileSync; - } - } - } - } - finally - { - Snapshotter.Release(); - } - } - - Current.Logger.Info("Successfully sync'd main index to local temp storage for index: {0}", () => configuredPath); - return InitializeDirectoryFlags.Success; - } - } - } -} diff --git a/src/Umbraco.Examine/LocalStorage/MultiIndexLock.cs b/src/Umbraco.Examine/LocalStorage/MultiIndexLock.cs deleted file mode 100644 index 51abd23b2c..0000000000 --- a/src/Umbraco.Examine/LocalStorage/MultiIndexLock.cs +++ /dev/null @@ -1,54 +0,0 @@ -using Lucene.Net.Store; - -namespace Umbraco.Examine.LocalStorage -{ - /// - /// Lock that wraps multiple locks - /// - public class MultiIndexLock : Lock - { - private readonly Lock _dirMaster; - private readonly Lock _dirChild; - - public MultiIndexLock(Lock dirMaster, Lock dirChild) - { - _dirMaster = dirMaster; - _dirChild = dirChild; - } - - /// - /// Attempts to obtain exclusive access and immediately return - /// upon success or failure. - /// - /// - /// true iff exclusive access is obtained - /// - public override bool Obtain() - { - return _dirMaster.Obtain() && _dirChild.Obtain(); - } - - public override bool Obtain(long lockWaitTimeout) - { - return _dirMaster.Obtain(lockWaitTimeout) && _dirChild.Obtain(lockWaitTimeout); - } - - /// - /// Releases exclusive access. - /// - public override void Release() - { - _dirMaster.Release(); - _dirChild.Release(); - } - - /// - /// Returns true if the resource is currently locked. Note that one must - /// still call before using the resource. - /// - public override bool IsLocked() - { - return _dirMaster.IsLocked() || _dirChild.IsLocked(); - } - } -} diff --git a/src/Umbraco.Examine/LocalStorage/MultiIndexLockFactory.cs b/src/Umbraco.Examine/LocalStorage/MultiIndexLockFactory.cs deleted file mode 100644 index 085cf1580e..0000000000 --- a/src/Umbraco.Examine/LocalStorage/MultiIndexLockFactory.cs +++ /dev/null @@ -1,33 +0,0 @@ -using System; -using Lucene.Net.Store; - -namespace Umbraco.Examine.LocalStorage -{ - /// - /// Lock factory that wraps multiple factories - /// - public class MultiIndexLockFactory : LockFactory - { - private readonly FSDirectory _master; - private readonly FSDirectory _child; - - public MultiIndexLockFactory(FSDirectory master, FSDirectory child) - { - if (master == null) throw new ArgumentNullException("master"); - if (child == null) throw new ArgumentNullException("child"); - _master = master; - _child = child; - } - - public override Lock MakeLock(string lockName) - { - return new MultiIndexLock(_master.LockFactory.MakeLock(lockName), _child.LockFactory.MakeLock(lockName)); - } - - public override void ClearLock(string lockName) - { - _master.LockFactory.ClearLock(lockName); - _child.LockFactory.ClearLock(lockName); - } - } -} diff --git a/src/Umbraco.Examine/LocalStorage/MultiIndexOutput.cs b/src/Umbraco.Examine/LocalStorage/MultiIndexOutput.cs deleted file mode 100644 index bb6b061528..0000000000 --- a/src/Umbraco.Examine/LocalStorage/MultiIndexOutput.cs +++ /dev/null @@ -1,80 +0,0 @@ -using System; -using System.Linq; -using Lucene.Net.Store; - -namespace Umbraco.Examine.LocalStorage -{ - public class MultiIndexOutput : IndexOutput - { - private readonly IndexOutput[] _outputs; - - public MultiIndexOutput(params IndexOutput[] outputs) - { - if (outputs.Length < 1) - { - throw new InvalidOperationException("There must be at least one output specified"); - } - _outputs = outputs; - } - - public override void WriteByte(byte b) - { - foreach (var output in _outputs) - { - output.WriteByte(b); - } - } - - public override void WriteBytes(byte[] b, int offset, int length) - { - foreach (var output in _outputs) - { - output.WriteBytes(b, offset, length); - } - } - - public override void Flush() - { - foreach (var output in _outputs) - { - output.Flush(); - } - } - - protected override void Dispose(bool isDisposing) - { - foreach (var output in _outputs) - { - output.Dispose(); - } - } - - public override long FilePointer - { - get - { - //return the first - return _outputs[0].FilePointer; - } - - } - - public override void Seek(long pos) - { - foreach (var output in _outputs) - { - output.Seek(pos); - } - } - - public override long Length - { - get - { - //return the first - return _outputs[0].FilePointer; - } - - } - } -} diff --git a/src/Umbraco.Examine/NoPrefixSimpleFsLockFactory.cs b/src/Umbraco.Examine/NoPrefixSimpleFsLockFactory.cs new file mode 100644 index 0000000000..7c83223d0b --- /dev/null +++ b/src/Umbraco.Examine/NoPrefixSimpleFsLockFactory.cs @@ -0,0 +1,26 @@ +using System.IO; +using Lucene.Net.Store; + +namespace Umbraco.Examine +{ + /// + /// A custom that ensures a prefixless lock prefix + /// + /// + /// This is a work around for the Lucene APIs. By default Lucene will use a null prefix however when we set a custom + /// lock factory the null prefix is overwritten. + /// + public class NoPrefixSimpleFsLockFactory : SimpleFSLockFactory + { + public NoPrefixSimpleFsLockFactory(DirectoryInfo lockDir) : base(lockDir) + { + } + + public override string LockPrefix + { + get => base.LockPrefix; + set => base.LockPrefix = null; //always set to null + } + + } +} \ No newline at end of file diff --git a/src/Umbraco.Examine/StaticField.cs b/src/Umbraco.Examine/StaticField.cs deleted file mode 100644 index 237cbaaf26..0000000000 --- a/src/Umbraco.Examine/StaticField.cs +++ /dev/null @@ -1,21 +0,0 @@ -using Examine; -using Examine.LuceneEngine; - -namespace Umbraco.Examine -{ - internal class StaticField : IIndexField - { - public StaticField(string name, FieldIndexTypes indexType, bool enableSorting, string type) - { - Type = type; - EnableSorting = enableSorting; - IndexType = indexType; - Name = name; - } - - public string Name { get; set; } - public FieldIndexTypes IndexType { get; private set; } - public bool EnableSorting { get; set; } - public string Type { get; set; } - } -} diff --git a/src/Umbraco.Examine/StaticFieldCollection.cs b/src/Umbraco.Examine/StaticFieldCollection.cs deleted file mode 100644 index 2579299fc1..0000000000 --- a/src/Umbraco.Examine/StaticFieldCollection.cs +++ /dev/null @@ -1,28 +0,0 @@ -using System.Collections.ObjectModel; - -namespace Umbraco.Examine -{ - internal class StaticFieldCollection : KeyedCollection - { - protected override string GetKeyForItem(StaticField item) - { - return item.Name; - } - - /// - /// Implements TryGetValue using the underlying dictionary - /// - /// - /// - /// - public bool TryGetValue(string key, out StaticField field) - { - if (Dictionary == null) - { - field = null; - return false; - } - return Dictionary.TryGetValue(key, out field); - } - } -} diff --git a/src/Umbraco.Examine/Umbraco.Examine.csproj b/src/Umbraco.Examine/Umbraco.Examine.csproj index de401af610..31f65e568d 100644 --- a/src/Umbraco.Examine/Umbraco.Examine.csproj +++ b/src/Umbraco.Examine/Umbraco.Examine.csproj @@ -46,14 +46,33 @@ - - - + + 1.0.0-beta011 + - + + + + + + + + + + + + + + + + + + + + Properties\SolutionInfo.cs diff --git a/src/Umbraco.Examine/UmbracoContentIndexer.cs b/src/Umbraco.Examine/UmbracoContentIndexer.cs index 3fddf4a762..918f33d79c 100644 --- a/src/Umbraco.Examine/UmbracoContentIndexer.cs +++ b/src/Umbraco.Examine/UmbracoContentIndexer.cs @@ -1,14 +1,13 @@ using System; using System.Collections.Generic; using System.Collections.Specialized; +using System.ComponentModel; using System.Linq; using Examine; using Umbraco.Core; using Umbraco.Core.Models; using Umbraco.Core.Services; using Umbraco.Core.Strings; -using Examine.LuceneEngine.Config; -using Examine.LuceneEngine.Faceting; using Examine.LuceneEngine.Indexing; using Examine.LuceneEngine.Providers; using Lucene.Net.Analysis; @@ -19,6 +18,7 @@ using Umbraco.Core.Persistence; using Umbraco.Core.Persistence.DatabaseModelDefinitions; using Umbraco.Core.Persistence.Querying; using Umbraco.Core.Scoping; +using Umbraco.Examine.Config; using IContentService = Umbraco.Core.Services.IContentService; using IMediaService = Umbraco.Core.Services.IMediaService; @@ -28,20 +28,21 @@ namespace Umbraco.Examine /// /// An indexer for Umbraco content and media /// - public class UmbracoContentIndexer : BaseUmbracoIndexer + public class UmbracoContentIndexer : UmbracoExamineIndexer { protected IContentService ContentService { get; } protected IMediaService MediaService { get; } protected IUserService UserService { get; } private readonly IEnumerable _urlSegmentProviders; - private readonly IScopeProvider _scopeProvider; private int? _parentId; #region Constructors - // default - bad, should inject instead - // usage: none + /// + /// Constructor for configuration providers + /// + [EditorBrowsable(EditorBrowsableState.Never)] public UmbracoContentIndexer() { ContentService = Current.Services.ContentService; @@ -49,12 +50,25 @@ namespace Umbraco.Examine UserService = Current.Services.UserService; _urlSegmentProviders = Current.UrlSegmentProviders; - _scopeProvider = Current.ScopeProvider; - InitializeQueries(); + InitializeQueries(Current.SqlContext); } - // usage: IndexInitializer (tests) + /// + /// Create an index at runtime + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// public UmbracoContentIndexer( IEnumerable fieldDefinitions, Directory luceneDirectory, @@ -63,13 +77,12 @@ namespace Umbraco.Examine IContentService contentService, IMediaService mediaService, IUserService userService, + ISqlContext sqlContext, IEnumerable urlSegmentProviders, IValueSetValidator validator, UmbracoContentIndexerOptions options, - IScopeProvider scopeProvider, - FacetConfiguration facetConfiguration = null, - IDictionary> indexValueTypes = null) - : base(fieldDefinitions, luceneDirectory, defaultAnalyzer, profilingLogger, validator, facetConfiguration, indexValueTypes) + IReadOnlyDictionary> indexValueTypes = null) + : base(fieldDefinitions, luceneDirectory, defaultAnalyzer, profilingLogger, validator, indexValueTypes) { if (validator == null) throw new ArgumentNullException(nameof(validator)); if (options == null) throw new ArgumentNullException(nameof(options)); @@ -77,24 +90,20 @@ namespace Umbraco.Examine SupportProtectedContent = options.SupportProtectedContent; SupportUnpublishedContent = options.SupportUnpublishedContent; ParentId = options.ParentId; - //backward compat hack: - IndexerData = new IndexCriteria(Enumerable.Empty(), Enumerable.Empty(), Enumerable.Empty(), Enumerable.Empty(), - //hack to set the parent Id for backwards compat, when using this ctor the IndexerData will (should) always be null - options.ParentId); ContentService = contentService ?? throw new ArgumentNullException(nameof(contentService)); MediaService = mediaService ?? throw new ArgumentNullException(nameof(mediaService)); UserService = userService ?? throw new ArgumentNullException(nameof(userService)); _urlSegmentProviders = urlSegmentProviders ?? throw new ArgumentNullException(nameof(urlSegmentProviders)); - _scopeProvider = scopeProvider ?? throw new ArgumentNullException(nameof(scopeProvider)); - InitializeQueries(); + InitializeQueries(sqlContext); } - private void InitializeQueries() + private void InitializeQueries(ISqlContext sqlContext) { + if (sqlContext == null) throw new ArgumentNullException(nameof(sqlContext)); if (_publishedQuery == null) - _publishedQuery = Current.SqlContext.Query().Where(x => x.Published); + _publishedQuery = sqlContext.Query().Where(x => x.Published); } #endregion @@ -121,11 +130,9 @@ namespace Umbraco.Examine public override void Initialize(string name, NameValueCollection config) { - //check if there's a flag specifying to support unpublished content, //if not, set to false; - bool supportUnpublished; - if (config["supportUnpublished"] != null && bool.TryParse(config["supportUnpublished"], out supportUnpublished)) + if (config["supportUnpublished"] != null && bool.TryParse(config["supportUnpublished"], out var supportUnpublished)) SupportUnpublishedContent = supportUnpublished; else SupportUnpublishedContent = false; @@ -133,13 +140,25 @@ namespace Umbraco.Examine //check if there's a flag specifying to support protected content, //if not, set to false; - bool supportProtected; - if (config["supportProtected"] != null && bool.TryParse(config["supportProtected"], out supportProtected)) + if (config["supportProtected"] != null && bool.TryParse(config["supportProtected"], out var supportProtected)) SupportProtectedContent = supportProtected; else SupportProtectedContent = false; base.Initialize(name, config); + + //now we need to build up the indexer options so we can create our validator + int? parentId = null; + if (IndexSetName.IsNullOrWhiteSpace() == false) + { + var indexSet = IndexSets.Instance.Sets[IndexSetName]; + parentId = indexSet.IndexParentId; + } + ValueSetValidator = new UmbracoContentValueSetValidator( + new UmbracoContentIndexerOptions(SupportUnpublishedContent, SupportProtectedContent, parentId), + //Using a singleton here, we can't inject this when using config based providers and we don't use this + //anywhere else in this class + Current.Services.PublicAccessService); } #endregion @@ -163,12 +182,8 @@ namespace Umbraco.Examine /// public int? ParentId { - get - { - //fallback to the legacy data - return _parentId ?? (IndexerData == null ? (int?)null : IndexerData.ParentNodeId); - } - protected set { _parentId = value; } + get => _parentId ?? ConfigIndexCriteria?.ParentNodeId; + protected set => _parentId = value; } protected override IEnumerable SupportedTypes => new[] {IndexTypes.Content, IndexTypes.Media}; @@ -193,18 +208,14 @@ namespace Umbraco.Examine var searcher = GetSearcher(); var c = searcher.CreateCriteria(); var filtered = c.RawQuery(rawQuery); - var results = searcher.Find(filtered); + var results = searcher.Search(filtered); ProfilingLogger.Logger.Debug(GetType(), $"DeleteFromIndex with query: {rawQuery} (found {results.TotalItemCount} results)"); - //need to create a delete queue item for each one found + //need to queue a delete item for each one found foreach (var r in results) { - ProcessIndexOperation(new IndexOperation() - { - Operation = IndexOperationType.Delete, - Item = new IndexItem(new ValueSet(r.LongId, string.Empty)) - }); + QueueIndexOperation(new IndexOperation(IndexItem.ForId(r.Id), IndexOperationType.Delete)); } base.DeleteFromIndex(nodeId); @@ -250,16 +261,16 @@ namespace Umbraco.Examine //if specific types are declared we need to post filter them //TODO: Update the service layer to join the cmsContentType table so we can query by content type too - if (IndexerData != null && IndexerData.IncludeNodeTypes.Any()) + if (ConfigIndexCriteria != null && ConfigIndexCriteria.IncludeItemTypes.Any()) { - content = descendants.Where(x => IndexerData.IncludeNodeTypes.Contains(x.ContentType.Alias)).ToArray(); + content = descendants.Where(x => ConfigIndexCriteria.IncludeItemTypes.Contains(x.ContentType.Alias)).ToArray(); } else { content = descendants.ToArray(); } - IndexItems(GetValueSets(content)); + IndexItems(GetValueSets(_urlSegmentProviders, UserService, content)); pageIndex++; } while (content.Length == pageSize); @@ -286,16 +297,16 @@ namespace Umbraco.Examine //if specific types are declared we need to post filter them //TODO: Update the service layer to join the cmsContentType table so we can query by content type too - if (IndexerData != null && IndexerData.IncludeNodeTypes.Any()) + if (ConfigIndexCriteria != null && ConfigIndexCriteria.IncludeItemTypes.Any()) { - media = descendants.Where(x => IndexerData.IncludeNodeTypes.Contains(x.ContentType.Alias)).ToArray(); + media = descendants.Where(x => ConfigIndexCriteria.IncludeItemTypes.Contains(x.ContentType.Alias)).ToArray(); } else { media = descendants.ToArray(); } - IndexItems(GetValueSets(media)); + IndexItems(GetValueSets(_urlSegmentProviders, UserService, media)); pageIndex++; } while (media.Length == pageSize); @@ -304,11 +315,11 @@ namespace Umbraco.Examine } } - private IEnumerable GetValueSets(IEnumerable content) + public static IEnumerable GetValueSets(IEnumerable urlSegmentProviders, IUserService userService, params IContent[] content) { foreach (var c in content) { - var urlValue = c.GetUrlSegment(_urlSegmentProviders); + var urlValue = c.GetUrlSegment(urlSegmentProviders); var values = new Dictionary { {"icon", new object[] {c.ContentType.Icon}}, @@ -325,8 +336,8 @@ namespace Umbraco.Examine {"urlName", new object[] {urlValue}}, {"path", new object[] {c.Path}}, {"nodeType", new object[] {c.ContentType.Id}}, - {"creatorName", new object[] {c.GetCreatorProfile(UserService)?.Name ?? "??"}}, - {"writerName", new object[] {c.GetWriterProfile(UserService)?.Name ?? "??"}}, + {"creatorName", new object[] {c.GetCreatorProfile(userService)?.Name ?? "??"}}, + {"writerName", new object[] {c.GetWriterProfile(userService)?.Name ?? "??"}}, {"writerID", new object[] {c.WriterId}}, {"template", new object[] {c.Template?.Id ?? 0}} }; @@ -336,17 +347,17 @@ namespace Umbraco.Examine values.Add(property.Alias, new[] {property.GetValue() }); } - var vs = new ValueSet(c.Id, IndexTypes.Content, c.ContentType.Alias, values); + var vs = new ValueSet(c.Id.ToInvariantString(), IndexTypes.Content, c.ContentType.Alias, values); yield return vs; } } - private IEnumerable GetValueSets(IEnumerable media) + public static IEnumerable GetValueSets(IEnumerable urlSegmentProviders, IUserService userService, params IMedia[] media) { foreach (var m in media) { - var urlValue = m.GetUrlSegment(_urlSegmentProviders); + var urlValue = m.GetUrlSegment(urlSegmentProviders); var values = new Dictionary { {"icon", new object[] {m.ContentType.Icon}}, @@ -362,7 +373,7 @@ namespace Umbraco.Examine {"urlName", new object[] {urlValue}}, {"path", new object[] {m.Path}}, {"nodeType", new object[] {m.ContentType.Id}}, - {"creatorName", new object[] {m.GetCreatorProfile(UserService).Name}} + {"creatorName", new object[] {m.GetCreatorProfile(userService).Name}} }; foreach (var property in m.Properties.Where(p => p?.GetValue() != null && p.GetValue().ToString().IsNullOrWhiteSpace() == false)) @@ -370,32 +381,12 @@ namespace Umbraco.Examine values.Add(property.Alias, new[] { property.GetValue() }); } - var vs = new ValueSet(m.Id, IndexTypes.Media, m.ContentType.Alias, values); + var vs = new ValueSet(m.Id.ToInvariantString(), IndexTypes.Media, m.ContentType.Alias, values); yield return vs; } } - /// - /// Creates an IIndexCriteria object based on the indexSet passed in and our DataService - /// - /// - /// - /// - /// If we cannot initialize we will pass back empty indexer data since we cannot read from the database - /// - [Obsolete("IIndexCriteria is obsolete, this method is used only for configuration based indexes it is recommended to configure indexes on startup with code instead of config")] - protected override IIndexCriteria GetIndexerData(IndexSet indexSet) - { - if (CanInitialize()) - { - //NOTE: We are using a singleton here because: This is only ever used for configuration based scenarios, this is never - // used when the index is configured via code (the default), in which case IIndexCriteria is never used. When this is used - // the DI ctor is not used. - return indexSet.ToIndexCriteria(Current.Services.ContentTypeService); - } - return base.GetIndexerData(indexSet); - } #endregion } diff --git a/src/Umbraco.Examine/UmbracoContentIndexerOptions.cs b/src/Umbraco.Examine/UmbracoContentIndexerOptions.cs index 507556cdef..47b8a76c0f 100644 --- a/src/Umbraco.Examine/UmbracoContentIndexerOptions.cs +++ b/src/Umbraco.Examine/UmbracoContentIndexerOptions.cs @@ -1,7 +1,8 @@ using System; namespace Umbraco.Examine -{ +{ + /// /// Options used to configure the umbraco content indexer /// diff --git a/src/Umbraco.Examine/UmbracoContentValueSetValidator.cs b/src/Umbraco.Examine/UmbracoContentValueSetValidator.cs index 2d414cc38f..01056be6b9 100644 --- a/src/Umbraco.Examine/UmbracoContentValueSetValidator.cs +++ b/src/Umbraco.Examine/UmbracoContentValueSetValidator.cs @@ -24,10 +24,10 @@ namespace Umbraco.Examine public bool Validate(ValueSet valueSet) { //check for published content - if (valueSet.IndexCategory == IndexTypes.Content - && valueSet.Values.ContainsKey(BaseUmbracoIndexer.PublishedFieldName)) + if (valueSet.Category == IndexTypes.Content + && valueSet.Values.ContainsKey(UmbracoExamineIndexer.PublishedFieldName)) { - var published = valueSet.Values[BaseUmbracoIndexer.PublishedFieldName] != null && valueSet.Values[BaseUmbracoIndexer.PublishedFieldName][0].Equals(1); + var published = valueSet.Values[UmbracoExamineIndexer.PublishedFieldName] != null && valueSet.Values[UmbracoExamineIndexer.PublishedFieldName][0].Equals(1); //we don't support unpublished and the item is not published return false if (_options.SupportUnpublishedContent == false && published == false) { @@ -37,11 +37,11 @@ namespace Umbraco.Examine //must have a 'path' if (valueSet.Values.ContainsKey(PathKey) == false) return false; - var path = valueSet.Values[PathKey] == null ? string.Empty : valueSet.Values[PathKey][0].ToString(); + var path = valueSet.Values[PathKey]?[0].ToString() ?? string.Empty; // Test for access if we're only indexing published content // return nothing if we're not supporting protected content and it is protected, and we're not supporting unpublished content - if (valueSet.IndexCategory == IndexTypes.Content + if (valueSet.Category == IndexTypes.Content && _options.SupportUnpublishedContent == false && _options.SupportProtectedContent == false && _publicAccessService.IsProtected(path)) @@ -61,7 +61,7 @@ namespace Umbraco.Examine if (_options.SupportUnpublishedContent == false) { if (path.IsNullOrWhiteSpace()) return false; - var recycleBinId = valueSet.IndexCategory == IndexTypes.Content ? Constants.System.RecycleBinContent : Constants.System.RecycleBinMedia; + var recycleBinId = valueSet.Category == IndexTypes.Content ? Constants.System.RecycleBinContent : Constants.System.RecycleBinMedia; if (path.Contains(string.Concat(",", recycleBinId, ","))) return false; } diff --git a/src/Umbraco.Examine/UmbracoExamineExtensions.cs b/src/Umbraco.Examine/UmbracoExamineExtensions.cs new file mode 100644 index 0000000000..ab8819b276 --- /dev/null +++ b/src/Umbraco.Examine/UmbracoExamineExtensions.cs @@ -0,0 +1,92 @@ +using System; +using System.ComponentModel; +using System.Web; +using Examine.LuceneEngine.SearchCriteria; +using Examine.SearchCriteria; +using Umbraco.Core; +using Umbraco.Examine.Config; + +namespace Umbraco.Examine +{ + public static class UmbracoExamineExtensions + { + public static IBooleanOperation Id(this IQuery query, int id) + { + var fieldQuery = query.Id(id.ToInvariantString()); + return fieldQuery; + } + + /// + /// Query method to search on parent id + /// + /// + /// + /// + public static IBooleanOperation ParentId(this IQuery query, int id) + { + var fieldQuery = query.Field("parentID", id); + return fieldQuery; + } + + /// + /// Query method to search on node name + /// + /// + /// + /// + public static IBooleanOperation NodeName(this IQuery query, string nodeName) + { + var fieldQuery = query.Field("nodeName", (IExamineValue)new ExamineValue(Examineness.Explicit, nodeName)); + return fieldQuery; + } + + /// + /// Query method to search on node name + /// + /// + /// + /// + public static IBooleanOperation NodeName(this IQuery query, IExamineValue nodeName) + { + var fieldQuery = query.Field("nodeName", nodeName); + return fieldQuery; + } + + /// + /// Query method to search on node type alias + /// + /// + /// + /// + public static IBooleanOperation NodeTypeAlias(this IQuery query, string nodeTypeAlias) + { + var fieldQuery = query.Field("__NodeTypeAlias", (IExamineValue)new ExamineValue(Examineness.Explicit, nodeTypeAlias)); + return fieldQuery; + } + + /// + /// Query method to search on node type alias + /// + /// + /// + /// + public static IBooleanOperation NodeTypeAlias(this IQuery query, IExamineValue nodeTypeAlias) + { + var fieldQuery = query.Field("__NodeTypeAlias", nodeTypeAlias); + return fieldQuery; + } + + /// + /// Used to replace any available tokens in the index path before the lucene directory is assigned to the path + /// + /// + internal static void ReplaceTokensInIndexPath(this IndexSet indexSet) + { + if (indexSet == null) return; + indexSet.IndexPath = indexSet.IndexPath + .Replace("{machinename}", NetworkHelper.FileSafeMachineName) + .Replace("{appdomainappid}", (HttpRuntime.AppDomainAppId ?? string.Empty).ReplaceNonAlphanumericChars("")) + .EnsureEndsWith('/'); + } + } +} \ No newline at end of file diff --git a/src/Umbraco.Examine/UmbracoExamineIndexer.cs b/src/Umbraco.Examine/UmbracoExamineIndexer.cs new file mode 100644 index 0000000000..3717b4cde2 --- /dev/null +++ b/src/Umbraco.Examine/UmbracoExamineIndexer.cs @@ -0,0 +1,426 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.IO; +using System.Linq; +using Examine.LuceneEngine.Providers; +using Lucene.Net.Analysis; +using Lucene.Net.Documents; +using Lucene.Net.Index; +using Umbraco.Core; +using Examine; +using Examine.LuceneEngine; +using Examine.LuceneEngine.Indexing; +using Umbraco.Core.Composing; +using Umbraco.Core.Logging; +using Umbraco.Core.Xml; +using Umbraco.Examine.Config; +using Directory = Lucene.Net.Store.Directory; + +namespace Umbraco.Examine +{ + /// + /// An abstract provider containing the basic functionality to be able to query against + /// Umbraco data. + /// + public abstract class UmbracoExamineIndexer : LuceneIndexer + { + // note + // wrapping all operations that end up calling base.SafelyProcessQueueItems in a safe call + // context because they will fork a thread/task/whatever which should *not* capture our + // call context (and the database it can contain)! ideally we should be able to override + // SafelyProcessQueueItems but that's not possible in the current version of Examine. + + /// + /// Used to store the path of a content object + /// + public const string IndexPathFieldName = "__Path"; + public const string NodeKeyFieldName = "__Key"; + public const string IconFieldName = "__Icon"; + public const string PublishedFieldName = "__Published"; + /// + /// The prefix added to a field when it is duplicated in order to store the original raw value. + /// + public const string RawFieldPrefix = "__Raw_"; + + /// + /// Constructor for config provider based indexes + /// + [EditorBrowsable(EditorBrowsableState.Never)] + protected UmbracoExamineIndexer() + : base() + { + ProfilingLogger = Current.ProfilingLogger; + _configBased = true; + + //This is using the config so we'll validate based on that + ValueSetValidator = new ValueSetValidatorDelegate(set => + { + + //check if this document is of a correct type of node type alias + if (ConfigIndexCriteria.IncludeItemTypes.Any()) + if (!ConfigIndexCriteria.IncludeItemTypes.Contains(set.ItemType)) + return false; + + //if this node type is part of our exclusion list, do not validate + if (ConfigIndexCriteria.ExcludeItemTypes.Any()) + if (ConfigIndexCriteria.ExcludeItemTypes.Contains(set.ItemType)) + return false; + + return true; + }); + } + + protected UmbracoExamineIndexer( + IEnumerable fieldDefinitions, + Directory luceneDirectory, + Analyzer defaultAnalyzer, + ProfilingLogger profilingLogger, + IValueSetValidator validator = null, + IReadOnlyDictionary> indexValueTypes = null) + : base(fieldDefinitions, luceneDirectory, defaultAnalyzer, validator, indexValueTypes) + { + ProfilingLogger = profilingLogger ?? throw new ArgumentNullException(nameof(profilingLogger)); + } + + private readonly bool _configBased = false; + + /// + /// A type that defines the type of index for each Umbraco field (non user defined fields) + /// Alot of standard umbraco fields shouldn't be tokenized or even indexed, just stored into lucene + /// for retreival after searching. + /// + internal static readonly FieldDefinition[] UmbracoIndexFields = + { + new FieldDefinition("parentID", FieldDefinitionTypes.Integer), + new FieldDefinition("level", FieldDefinitionTypes.Integer), + new FieldDefinition("writerID", FieldDefinitionTypes.Integer), + new FieldDefinition("creatorID", FieldDefinitionTypes.Integer), + new FieldDefinition("sortOrder", FieldDefinitionTypes.Integer), + new FieldDefinition("template", FieldDefinitionTypes.Integer), + + new FieldDefinition("createDate", FieldDefinitionTypes.DateTime), + new FieldDefinition("updateDate", FieldDefinitionTypes.DateTime), + + new FieldDefinition("key", FieldDefinitionTypes.InvariantCultureIgnoreCase), + new FieldDefinition("version", FieldDefinitionTypes.Raw), + new FieldDefinition("nodeType", FieldDefinitionTypes.InvariantCultureIgnoreCase), + new FieldDefinition("template", FieldDefinitionTypes.Raw), + new FieldDefinition("urlName", FieldDefinitionTypes.InvariantCultureIgnoreCase), + new FieldDefinition("path", FieldDefinitionTypes.Raw), + + new FieldDefinition(IndexPathFieldName, FieldDefinitionTypes.Raw), + new FieldDefinition(IconFieldName, FieldDefinitionTypes.Raw) + }; + + protected ProfilingLogger ProfilingLogger { get; } + + /// + /// Overridden to ensure that the umbraco system field definitions are in place + /// + /// + /// + /// + protected override FieldValueTypeCollection CreateFieldValueTypes(Directory x, IReadOnlyDictionary> indexValueTypesFactory = null) + { + foreach (var field in UmbracoIndexFields) + { + FieldDefinitionCollection.TryAdd(field.Name, field); + } + + return base.CreateFieldValueTypes(x, indexValueTypesFactory); + } + + //TODO: Remove this? + [Obsolete("This should not be used, it is used by the configuration based indexes but instead to disable Examine event handlers use the ExamineEvents class instead.")] + [EditorBrowsable(EditorBrowsableState.Never)] + public bool EnableDefaultEventHandler { get; protected set; } + + /// + /// the supported indexable types + /// + protected abstract IEnumerable SupportedTypes { get; } + + protected ConfigIndexCriteria ConfigIndexCriteria { get; private set; } + + /// + /// The index set name which references an Examine + /// + public string IndexSetName { get; private set; } + + #region Initialize + + + /// + /// Setup the properties for the indexer from the provider settings + /// + /// + /// + /// + /// This is ONLY used for configuration based indexes + /// + public override void Initialize(string name, System.Collections.Specialized.NameValueCollection config) + { + ProfilingLogger.Logger.Debug(GetType(), "{0} indexer initializing", () => name); + + EnableDefaultEventHandler = true; //set to true by default + bool enabled; + if (bool.TryParse(config["enableDefaultEventHandler"], out enabled)) + { + EnableDefaultEventHandler = enabled; + } + + //Need to check if the index set or IndexerData is specified... + if (config["indexSet"] == null && (FieldDefinitionCollection.Count == 0)) + { + //if we don't have either, then we'll try to set the index set by naming conventions + var found = false; + if (name.EndsWith("Indexer")) + { + var setNameByConvension = name.Remove(name.LastIndexOf("Indexer")) + "IndexSet"; + //check if we can assign the index set by naming convention + var set = IndexSets.Instance.Sets.Cast().SingleOrDefault(x => x.SetName == setNameByConvension); + + if (set != null) + { + //we've found an index set by naming conventions :) + IndexSetName = set.SetName; + + var indexSet = IndexSets.Instance.Sets[IndexSetName]; + + //if tokens are declared in the path, then use them (i.e. {machinename} ) + indexSet.ReplaceTokensInIndexPath(); + + //get the index criteria and ensure folder + ConfigIndexCriteria = CreateFieldDefinitionsFromConfig(indexSet); + foreach (var fieldDefinition in ConfigIndexCriteria.StandardFields.Union(ConfigIndexCriteria.UserFields)) + { + FieldDefinitionCollection.TryAdd(fieldDefinition.Name, fieldDefinition); + } + + //now set the index folder + LuceneIndexFolder = new DirectoryInfo(Path.Combine(IndexSets.Instance.Sets[IndexSetName].IndexDirectory.FullName, "Index")); + + found = true; + } + } + + if (!found) + throw new ArgumentNullException("indexSet on LuceneExamineIndexer provider has not been set in configuration and/or the IndexerData property has not been explicitly set"); + + } + else if (config["indexSet"] != null) + { + //if an index set is specified, ensure it exists and initialize the indexer based on the set + + if (IndexSets.Instance.Sets[config["indexSet"]] == null) + { + throw new ArgumentException("The indexSet specified for the LuceneExamineIndexer provider does not exist"); + } + else + { + IndexSetName = config["indexSet"]; + + var indexSet = IndexSets.Instance.Sets[IndexSetName]; + + //if tokens are declared in the path, then use them (i.e. {machinename} ) + indexSet.ReplaceTokensInIndexPath(); + + //get the index criteria and ensure folder + ConfigIndexCriteria = CreateFieldDefinitionsFromConfig(indexSet); + foreach (var fieldDefinition in ConfigIndexCriteria.StandardFields.Union(ConfigIndexCriteria.UserFields)) + { + FieldDefinitionCollection.TryAdd(fieldDefinition.Name, fieldDefinition); + } + + //now set the index folder + LuceneIndexFolder = new DirectoryInfo(Path.Combine(IndexSets.Instance.Sets[IndexSetName].IndexDirectory.FullName, "Index")); + } + } + + base.Initialize(name, config); + } + + #endregion + + + /// + /// override to check if we can actually initialize. + /// + /// + /// This check is required since the base examine lib will try to rebuild on startup + /// + public override void RebuildIndex() + { + if (CanInitialize()) + { + ProfilingLogger.Logger.Debug(GetType(), "Rebuilding index"); + using (new SafeCallContext()) + { + base.RebuildIndex(); + } + } + } + + /// + /// override to check if we can actually initialize. + /// + /// + /// This check is required since the base examine lib will try to rebuild on startup + /// + public override void IndexAll(string type) + { + if (CanInitialize()) + { + using (new SafeCallContext()) + { + base.IndexAll(type); + } + } + } + + public override void IndexItems(IEnumerable nodes) + { + if (CanInitialize()) + { + using (new SafeCallContext()) + { + base.IndexItems(nodes); + } + } + } + + /// + /// override to check if we can actually initialize. + /// + /// + /// This check is required since the base examine lib will try to rebuild on startup + /// + public override void DeleteFromIndex(string nodeId) + { + if (CanInitialize()) + { + using (new SafeCallContext()) + { + base.DeleteFromIndex(nodeId); + } + } + } + + /// + /// Returns true if the Umbraco application is in a state that we can initialize the examine indexes + /// + /// + protected bool CanInitialize() + { + // only affects indexers that are config file based, if an index was created via code then + // this has no effect, it is assumed the index would not be created if it could not be initialized + return _configBased == false || Current.RuntimeState.Level == RuntimeLevel.Run; + } + + /// + /// Reindexes all supported types + /// + protected override void PerformIndexRebuild() + { + foreach (var t in SupportedTypes) + { + IndexAll(t); + } + } + + /// + /// overridden for logging + /// + /// + protected override void OnIndexingError(IndexingErrorEventArgs e) + { + ProfilingLogger.Logger.Error(GetType(), e.Message, e.InnerException); + base.OnIndexingError(e); + } + + /// + /// This ensures that the special __Raw_ fields are indexed + /// + /// + protected override void OnDocumentWriting(DocumentWritingEventArgs docArgs) + { + var d = docArgs.Document; + + foreach (var f in docArgs.ValueSet.Values.Where(x => x.Key.StartsWith(RawFieldPrefix))) + { + if (f.Value.Count > 0) + { + d.Add(new Field( + f.Key, + f.Value[0].ToString(), + Field.Store.YES, + Field.Index.NO, //don't index this field, we never want to search by it + Field.TermVector.NO)); + } + } + + ProfilingLogger.Logger.Debug(GetType(), "Write lucene doc id:{0}, category:{1}, type:{2}", docArgs.ValueSet.Id, docArgs.ValueSet.Category, docArgs.ValueSet.ItemType); + + base.OnDocumentWriting(docArgs); + } + + /// + /// Overridden for logging. + /// + protected override void AddDocument(Document doc, IndexItem item, IndexWriter writer) + { + ProfilingLogger.Logger.Debug(GetType(), "AddDocument {0} with type {1}", () => item.ValueSet.Id, () => item.ValueSet.ItemType); + base.AddDocument(doc, item, writer); + } + + protected override void OnTransformingIndexValues(IndexingItemEventArgs e) + { + base.OnTransformingIndexValues(e); + + //ensure special __Path field + if (e.IndexItem.ValueSet.Values.TryGetValue("path", out var path) && e.IndexItem.ValueSet.Values.ContainsKey(IndexPathFieldName) == false) + { + e.IndexItem.ValueSet.Values[IndexPathFieldName] = new List { path }; + } + + //strip html of all users fields if we detect it has HTML in it. + //if that is the case, we'll create a duplicate 'raw' copy of it so that we can return + //the value of the field 'as-is'. + foreach (var value in e.IndexItem.ValueSet.Values) + { + if (value.Value.Count > 0) + { + var str = value.Value.First() as string; + if (str != null) + { + if (XmlHelper.CouldItBeXml(str)) + { + //First save the raw value to a raw field, we will change the policy of this field by detecting the prefix later + e.IndexItem.ValueSet.Values[string.Concat(RawFieldPrefix, value.Key)] = new List { str }; + //now replace the original value with the stripped html + //TODO: This should be done with an analzer?! + e.IndexItem.ValueSet.Values[value.Key] = new List { str.StripHtml() }; + } + } + } + } + + //icon + if (e.IndexItem.ValueSet.Values.TryGetValue("icon", out var icon) && e.IndexItem.ValueSet.Values.ContainsKey(IconFieldName) == false) + { + e.IndexItem.ValueSet.Values[IconFieldName] = new List { icon }; + } + } + + private ConfigIndexCriteria CreateFieldDefinitionsFromConfig(IndexSet indexSet) + { + return new ConfigIndexCriteria( + indexSet.IndexAttributeFields.Cast().Select(x => new FieldDefinition(x.Name, x.Type)).ToArray(), + indexSet.IndexUserFields.Cast().Select(x => new FieldDefinition(x.Name, x.Type)).ToArray(), + indexSet.IncludeNodeTypes.ToList().Select(x => x.Name).ToArray(), + indexSet.ExcludeNodeTypes.ToList().Select(x => x.Name).ToArray(), + indexSet.IndexParentId); + } + } +} diff --git a/src/Umbraco.Examine/UmbracoExamineMultiIndexSearcher.cs b/src/Umbraco.Examine/UmbracoExamineMultiIndexSearcher.cs new file mode 100644 index 0000000000..a5e63a1acd --- /dev/null +++ b/src/Umbraco.Examine/UmbracoExamineMultiIndexSearcher.cs @@ -0,0 +1,61 @@ +using System; +using System.Collections.Generic; +using System.Collections.Specialized; +using System.IO; +using System.Linq; +using Examine.LuceneEngine.Providers; +using Lucene.Net.Analysis; +using Umbraco.Examine.Config; +using Directory = Lucene.Net.Store.Directory; + +namespace Umbraco.Examine +{ + public class UmbracoExamineMultiIndexSearcher : MultiIndexSearcher + { + public UmbracoExamineMultiIndexSearcher() + { + } + + public UmbracoExamineMultiIndexSearcher(IEnumerable indexPath, Analyzer analyzer) : base(indexPath, analyzer) + { + } + + public UmbracoExamineMultiIndexSearcher(IEnumerable luceneDirs, Analyzer analyzer) : base(luceneDirs, analyzer) + { + } + + public override void Initialize(string name, NameValueCollection config) + { + base.Initialize(name, config); + + //need to check if the index set is specified, if it's not, we'll see if we can find one by convension + //if the folder is not null and the index set is null, we'll assume that this has been created at runtime. + if (config["indexSets"] == null) + { + throw new ArgumentNullException("indexSets on MultiIndexSearcher provider has not been set in configuration"); + } + + var toSearch = new List(); + var sets = IndexSets.Instance.Sets.Cast(); + foreach (var i in config["indexSets"].Split(',')) + { + var s = sets.SingleOrDefault(x => x.SetName == i); + if (s == null) + { + throw new ArgumentException("The index set " + i + " does not exist"); + } + toSearch.Add(s); + } + + //create the searchers + var analyzer = DefaultLuceneAnalyzer; + var searchers = new List(); + //DO NOT PUT THIS INTO LINQ BECAUSE THE SECURITY ACCCESS SHIT WONT WORK + foreach (var s in toSearch) + { + searchers.Add(new LuceneSearcher(s.IndexDirectory, analyzer)); + } + Searchers = searchers; + } + } +} \ No newline at end of file diff --git a/src/Umbraco.Examine/UmbracoExamineSearcher.cs b/src/Umbraco.Examine/UmbracoExamineSearcher.cs index c62c763cf8..f6c2bca963 100644 --- a/src/Umbraco.Examine/UmbracoExamineSearcher.cs +++ b/src/Umbraco.Examine/UmbracoExamineSearcher.cs @@ -2,23 +2,11 @@ using System.ComponentModel; using System.IO; using System.Linq; -using System.Security; -using System.Web; -using System.Web.Compilation; -using Examine; -using Examine.LuceneEngine.Config; -using Examine.Providers; -using Examine.SearchCriteria; -using Lucene.Net.Index; -using Lucene.Net.Store; using Umbraco.Core; -using Examine.LuceneEngine; using Examine.LuceneEngine.Providers; -using Examine.LuceneEngine.SearchCriteria; using Lucene.Net.Analysis; using Umbraco.Core.Composing; -using Umbraco.Core.Logging; -using Umbraco.Examine.LocalStorage; +using Umbraco.Examine.Config; using Directory = Lucene.Net.Store.Directory; @@ -30,8 +18,6 @@ namespace Umbraco.Examine public class UmbracoExamineSearcher : LuceneSearcher { - private Lazy _localTempDirectory; - private LocalStorageType _localStorageType = LocalStorageType.Sync; private string _name; private readonly bool _configBased = false; @@ -67,14 +53,18 @@ namespace Umbraco.Examine _configBased = false; } + //TODO: What about the NRT ctor? + /// /// we override name because we need to manually set it if !CanInitialize() /// since we cannot call base.Initialize in that case. /// - public override string Name - { - get { return _name; } - } + public override string Name => _name; + + /// + /// Name of the Lucene.NET index set + /// + public string IndexSetName { get; private set; } /// /// Method used for initializing based on a configuration based searcher @@ -83,10 +73,8 @@ namespace Umbraco.Examine /// public override void Initialize(string name, System.Collections.Specialized.NameValueCollection config) { - if (name == null) throw new ArgumentNullException("name"); - //ensure name is set - _name = name; + _name = name ?? throw new ArgumentNullException(nameof(name)); //We need to check if we actually can initialize, if not then don't continue if (CanInitialize() == false) @@ -94,66 +82,53 @@ namespace Umbraco.Examine return; } + //need to check if the index set is specified, if it's not, we'll see if we can find one by convension + //if the folder is not null and the index set is null, we'll assume that this has been created at runtime. + //NOTE: Don't proceed if the _luceneDirectory is set since we already know where to look. + var luceneDirectory = GetLuceneDirectory(); + if (config["indexSet"] == null && (LuceneIndexFolder == null && luceneDirectory == null)) + { + //if we don't have either, then we'll try to set the index set by naming convensions + var found = false; + if (name.EndsWith("Searcher")) + { + var setNameByConvension = name.Remove(name.LastIndexOf("Searcher")) + "IndexSet"; + //check if we can assign the index set by naming convension + var set = IndexSets.Instance.Sets.Cast().SingleOrDefault(x => x.SetName == setNameByConvension); + + if (set != null) + { + set.ReplaceTokensInIndexPath(); + + //we've found an index set by naming convensions :) + IndexSetName = set.SetName; + found = true; + } + } + + if (!found) + throw new ArgumentNullException("indexSet on LuceneExamineIndexer provider has not been set in configuration"); + + //get the folder to index + LuceneIndexFolder = new DirectoryInfo(Path.Combine(IndexSets.Instance.Sets[IndexSetName].IndexDirectory.FullName, "Index")); + } + else if (config["indexSet"] != null && luceneDirectory == null) + { + if (IndexSets.Instance.Sets[config["indexSet"]] == null) + throw new ArgumentException("The indexSet specified for the LuceneExamineIndexer provider does not exist"); + + IndexSetName = config["indexSet"]; + + var indexSet = IndexSets.Instance.Sets[IndexSetName]; + + indexSet.ReplaceTokensInIndexPath(); + + //get the folder to index + LuceneIndexFolder = new DirectoryInfo(Path.Combine(indexSet.IndexDirectory.FullName, "Index")); + } + base.Initialize(name, config); - if (config != null && config["useTempStorage"] != null) - { - throw new NotImplementedException("Fix how local temp storage works and is synced with Examine v2.0 - since a writer is always open we cannot snapshot it, we need to use the same logic in AzureDirectory"); - - //Use the temp storage directory which will store the index in the local/codegen folder, this is useful - // for websites that are running from a remove file server and file IO latency becomes an issue - var attemptUseTempStorage = config["useTempStorage"].TryConvertTo(); - if (attemptUseTempStorage) - { - //this is the default - ILocalStorageDirectory localStorageDir = new CodeGenLocalStorageDirectory(); - if (config["tempStorageDirectory"] != null) - { - //try to get the type - var dirType = BuildManager.GetType(config["tempStorageDirectory"], false); - if (dirType != null) - { - try - { - localStorageDir = (ILocalStorageDirectory)Activator.CreateInstance(dirType); - } - catch (Exception ex) - { - Current.Logger.Error( - string.Format("Could not create a temp storage location of type {0}, reverting to use the " + typeof(CodeGenLocalStorageDirectory).FullName, dirType), - ex); - } - } - } - var indexSet = IndexSets.Instance.Sets[IndexSetName]; - var configuredPath = indexSet.IndexPath; - var tempPath = localStorageDir.GetLocalStorageDirectory(config, configuredPath); - if (tempPath == null) throw new InvalidOperationException("Could not resolve a temp location from the " + localStorageDir.GetType() + " specified"); - var localTempPath = tempPath.FullName; - _localStorageType = attemptUseTempStorage.Result; - - //initialize the lazy callback - _localTempDirectory = new Lazy(() => - { - switch (_localStorageType) - { - case LocalStorageType.Sync: - var fsDir = base.GetLuceneDirectory() as FSDirectory; - if (fsDir != null) - { - return LocalTempStorageDirectoryTracker.Current.GetDirectory( - new DirectoryInfo(localTempPath), - fsDir); - } - return base.GetLuceneDirectory(); - case LocalStorageType.LocalOnly: - return DirectoryTracker.Current.GetDirectory(new DirectoryInfo(localTempPath)); - default: - throw new ArgumentOutOfRangeException(); - } - }); - } - } } /// @@ -175,17 +150,10 @@ namespace Umbraco.Examine { var fields = base.GetSearchFields(); return fields - .Where(x => x != BaseUmbracoIndexer.IndexPathFieldName) - .Where(x => x != LuceneIndexer.NodeTypeAliasFieldName) + .Where(x => x != UmbracoExamineIndexer.IndexPathFieldName) + .Where(x => x != LuceneIndexer.ItemTypeFieldName) .ToArray(); } - protected override Directory GetLuceneDirectory() - { - //local temp storage is not enabled, just return the default - return _localTempDirectory.IsValueCreated == false - ? base.GetLuceneDirectory() - : _localTempDirectory.Value; - } } } diff --git a/src/Umbraco.Examine/UmbracoMemberIndexer.cs b/src/Umbraco.Examine/UmbracoMemberIndexer.cs index 1f4b7fa79a..3b47be2d9d 100644 --- a/src/Umbraco.Examine/UmbracoMemberIndexer.cs +++ b/src/Umbraco.Examine/UmbracoMemberIndexer.cs @@ -1,22 +1,18 @@ using System; -using System.Collections; using System.Linq; -using System.Xml.Linq; -using Examine.LuceneEngine.Config; using Umbraco.Core; using Umbraco.Core.Models; using Umbraco.Core.Persistence.DatabaseModelDefinitions; using Umbraco.Core.Services; -using Umbraco.Core.Strings; using System.Collections.Generic; +using System.ComponentModel; using Examine; -using System.IO; +using Examine.LuceneEngine; +using Examine.LuceneEngine.Indexing; using Examine.LuceneEngine.Providers; using Lucene.Net.Analysis; using Umbraco.Core.Composing; using Umbraco.Core.Logging; -using Umbraco.Core.Persistence; -using Umbraco.Core.Scoping; using Directory = Lucene.Net.Store.Directory; namespace Umbraco.Examine @@ -25,19 +21,18 @@ namespace Umbraco.Examine /// /// Custom indexer for members /// - public class UmbracoMemberIndexer : BaseUmbracoIndexer + public class UmbracoMemberIndexer : UmbracoExamineIndexer { private readonly IMemberService _memberService; - private readonly IScopeProvider _scopeProvider; /// - /// Default constructor + /// Constructor for config/provider based indexes /// + /// + [EditorBrowsable(EditorBrowsableState.Never)] public UmbracoMemberIndexer() - : base() { _memberService = Current.Services.MemberService; - _scopeProvider = Current.ScopeProvider; } /// @@ -58,68 +53,26 @@ namespace Umbraco.Examine IMemberService memberService) : base(fieldDefinitions, luceneDirectory, analyzer, profilingLogger, validator) { - if (memberService == null) throw new ArgumentNullException("memberService"); - _memberService = memberService; + _memberService = memberService ?? throw new ArgumentNullException(nameof(memberService)); } - /// - /// Ensures that the'_searchEmail' is added to the user fields so that it is indexed - without having to modify the config - /// - /// - /// - [Obsolete("IIndexCriteria is obsolete, this method is used only for configuration based indexes it is recommended to configure indexes on startup with code instead of config")] - protected override IIndexCriteria GetIndexerData(IndexSet indexSet) - { - //TODO: This is only required for config based index delcaration - We need to change this! - - var indexerData = base.GetIndexerData(indexSet); - - if (CanInitialize()) - { - //If the fields are missing a custom _searchEmail, then add it - - if (indexerData.UserFields.Any(x => x.Name == "_searchEmail") == false) - { - var field = new IndexField { Name = "_searchEmail" }; - var policy = IndexFieldPolicies.FirstOrDefault(x => x.Name == "_searchEmail"); - if (policy != null) - { - field.Type = policy.Type; - field.EnableSorting = policy.EnableSorting; - } - - return new IndexCriteria( - indexerData.StandardFields, - indexerData.UserFields.Concat(new[] { field }), - indexerData.IncludeNodeTypes, - indexerData.ExcludeNodeTypes, - indexerData.ParentNodeId - ); - } - } - - return indexerData; - } /// /// Overridden to ensure that the umbraco system field definitions are in place /// - /// + /// + /// /// - protected override IEnumerable InitializeFieldDefinitions(IEnumerable originalDefinitions) + protected override FieldValueTypeCollection CreateFieldValueTypes(Directory x, IReadOnlyDictionary> indexValueTypesFactory = null) { - var result = base.InitializeFieldDefinitions(originalDefinitions).ToList(); - result.Add(new FieldDefinition("__key", FieldDefinitionTypes.Raw)); - return result; + var keyDef = new FieldDefinition("__key", FieldDefinitionTypes.Raw); + FieldDefinitionCollection.TryAdd(keyDef.Name, keyDef); + + return base.CreateFieldValueTypes(x, indexValueTypesFactory); } - /// - /// The supported types for this indexer - /// - protected override IEnumerable SupportedTypes - { - get { return new[] {IndexTypes.Member}; } - } + /// + protected override IEnumerable SupportedTypes => new[] {IndexTypes.Member}; /// /// Reindex all members @@ -136,44 +89,38 @@ namespace Umbraco.Examine IMember[] members; - using (var scope = _scopeProvider.CreateScope()) + if (ConfigIndexCriteria != null && ConfigIndexCriteria.IncludeItemTypes.Any()) { - if (IndexerData != null && IndexerData.IncludeNodeTypes.Any()) + //if there are specific node types then just index those + foreach (var nodeType in ConfigIndexCriteria.IncludeItemTypes) { - //if there are specific node types then just index those - foreach (var nodeType in IndexerData.IncludeNodeTypes) - { - do - { - long total; - members = _memberService.GetAll(pageIndex, pageSize, out total, "LoginName", Direction.Ascending, true, null, nodeType).ToArray(); - - IndexItems(GetValueSets(members)); - - pageIndex++; - } while (members.Length == pageSize); - } - } - else - { - //no node types specified, do all members do { - long total; - members = _memberService.GetAll(pageIndex, pageSize, out total).ToArray(); + members = _memberService.GetAll(pageIndex, pageSize, out _, "LoginName", Direction.Ascending, true, null, nodeType).ToArray(); IndexItems(GetValueSets(members)); pageIndex++; } while (members.Length == pageSize); } - scope.Complete(); + } + else + { + //no node types specified, do all members + do + { + members = _memberService.GetAll(pageIndex, pageSize, out _).ToArray(); + + IndexItems(GetValueSets(members)); + + pageIndex++; + } while (members.Length == pageSize); } } - private IEnumerable GetValueSets(IEnumerable member) + public static IEnumerable GetValueSets(params IMember[] members) { - foreach (var m in member) + foreach (var m in members) { var values = new Dictionary { @@ -193,30 +140,36 @@ namespace Umbraco.Examine {"email", new object[] {m.Email}}, }; - foreach (var property in m.Properties.Where(p => p != null && p.GetValue() != null && p.GetValue().ToString().IsNullOrWhiteSpace() == false)) + foreach (var property in m.Properties.Where(p => p?.GetValue() != null && p.GetValue().ToString().IsNullOrWhiteSpace() == false)) { values.Add(property.Alias, new[] { property.GetValue() }); } - var vs = new ValueSet(m.Id, IndexTypes.Content, m.ContentType.Alias, values); + var vs = new ValueSet(m.Id.ToInvariantString(), IndexTypes.Content, m.ContentType.Alias, values); yield return vs; } } - protected override void OnTransformingIndexValues(TransformingIndexDataEventArgs e) + /// + /// Ensure some custom values are added to the index + /// + /// + protected override void OnTransformingIndexValues(IndexingItemEventArgs e) { base.OnTransformingIndexValues(e); - if (e.OriginalValues.ContainsKey("key") && e.IndexItem.ValueSet.Values.ContainsKey("__key") == false) + if (e.IndexItem.ValueSet.Values.TryGetValue("key", out var key) && e.IndexItem.ValueSet.Values.ContainsKey("__key") == false) { - e.IndexItem.ValueSet.Values["__key"] = new List {e.OriginalValues["key"]}; - } - if (e.OriginalValues.ContainsKey("email") && e.IndexItem.ValueSet.Values.ContainsKey("_searchEmail") == false) - { - e.IndexItem.ValueSet.Values["_searchEmail"] = new List { e.OriginalValues["email"].ToString().Replace(".", " ").Replace("@", " ") }; + //double __ prefix means it will be indexed as culture invariant + e.IndexItem.ValueSet.Values["__key"] = new List { key }; } + if (e.IndexItem.ValueSet.Values.TryGetValue("email", out var email) && e.IndexItem.ValueSet.Values.ContainsKey("_searchEmail") == false) + { + //will be indexed as full text (the default anaylyzer) + e.IndexItem.ValueSet.Values["_searchEmail"] = new List { email?.ToString().Replace(".", " ").Replace("@", " ") }; + } } } diff --git a/src/Umbraco.Examine/XsltExtensions.cs b/src/Umbraco.Examine/XsltExtensions.cs deleted file mode 100644 index 63bb0a480a..0000000000 --- a/src/Umbraco.Examine/XsltExtensions.cs +++ /dev/null @@ -1,255 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Security; -using System.Xml.Linq; -using System.Xml.XPath; -using Examine; -using Examine.LuceneEngine; -using Examine.LuceneEngine.Providers; -using Examine.LuceneEngine.SearchCriteria; -using Examine.SearchCriteria; -using Examine.Providers; -using Umbraco.Core.Macros; - -namespace Umbraco.Examine -{ - /// - /// Methods to support Umbraco XSLT extensions. - /// - /// - /// XSLT extensions will ONLY work for provider that have a base class of BaseUmbracoIndexer - /// - [XsltExtension("Examine")] - - public class XsltExtensions - { - /// - /// Uses the provider specified to search, returning an XPathNodeIterator - /// - /// - /// - /// - /// - /// - internal static XPathNodeIterator Search(string searchText, bool useWildcards, LuceneSearcher provider, string indexType) - { - if (provider == null) throw new ArgumentNullException("provider"); - - var results = provider.Search(searchText, useWildcards, indexType); - return GetResultsAsXml(results); - } - - /// - /// Uses the provider specified to search, returning an XPathNodeIterator - /// - /// The search text. - /// if set to true [use wildcards]. - /// Name of the provider. - /// Type of the index. - /// - public static XPathNodeIterator Search(string searchText, bool useWildcards, string providerName, string indexType) - { - var provider = (LuceneSearcher)ExamineManager.Instance.SearchProviderCollection[providerName]; - - return Search(searchText, useWildcards, provider, indexType); - } - - /// - /// Uses the provider specified to search, returning an XPathNodeIterator - /// - /// - /// - /// - /// - public static XPathNodeIterator Search(string searchText, bool useWildcards, string providerName) - { - return Search(searchText, useWildcards, providerName, string.Empty); - } - - /// - /// Uses the default provider specified to search, returning an XPathNodeIterator - /// - /// The search query - /// Enable a wildcard search query - /// A node-set of the search results - public static XPathNodeIterator Search(string searchText, bool useWildcards) - { - return Search(searchText, useWildcards, ExamineManager.Instance.DefaultSearchProvider.Name); - } - - /// - /// Uses the default provider specified to search, returning an XPathNodeIterator - /// - /// The search query - /// A node-set of the search results - public static XPathNodeIterator Search(string searchText) - { - return Search(searchText, true); - } - - /// - /// Will perform a search against the media index type only - /// - /// - /// - /// - /// - public static XPathNodeIterator SearchMediaOnly(string searchText, bool useWildcards, string providerName) - { - return Search(searchText, useWildcards, providerName, IndexTypes.Media); - } - - /// - /// Will perform a search against the media index type only - /// - /// - /// - /// - public static XPathNodeIterator SearchMediaOnly(string searchText, bool useWildcards) - { - return SearchMediaOnly(searchText, useWildcards, ExamineManager.Instance.DefaultSearchProvider.Name); - } - - /// - /// Will perform a search against the media index type only - /// - /// - /// - public static XPathNodeIterator SearchMediaOnly(string searchText) - { - return SearchMediaOnly(searchText, true); - } - - /// - /// Searches the member only. - /// - /// The search text. - /// if set to true [use wildcards]. - /// Name of the provider. - /// - public static XPathNodeIterator SearchMemberOnly(string searchText, bool useWildcards, string providerName) - { - return Search(searchText, useWildcards, providerName, IndexTypes.Member); - } - - /// - /// Searches the member only. - /// - /// The search text. - /// if set to true [use wildcards]. - /// - public static XPathNodeIterator SearchMemberOnly(string searchText, bool useWildcards) - { - return SearchMemberOnly(searchText, useWildcards, ExamineManager.Instance.DefaultSearchProvider.Name); - } - - /// - /// Searches the member only. - /// - /// The search text. - /// - public static XPathNodeIterator SearchMemberOnly(string searchText) - { - return SearchMemberOnly(searchText, true); - } - - /// - /// Will perform a search against the content index type only - /// - /// - /// - /// - /// - public static XPathNodeIterator SearchContentOnly(string searchText, bool useWildcards, string providerName) - { - return Search(searchText, useWildcards, providerName, IndexTypes.Content); - } - - - /// - /// Will perform a search against the content index type only - /// - /// - /// - /// - public static XPathNodeIterator SearchContentOnly(string searchText, bool useWildcards) - { - return SearchContentOnly(searchText, useWildcards, ExamineManager.Instance.DefaultSearchProvider.Name); - } - - /// - /// Will perform a search against the content index type only - /// - /// - /// - public static XPathNodeIterator SearchContentOnly(string searchText) - { - return SearchContentOnly(searchText, true); - } - - /// - /// Gets the results as XML. - /// - /// The results. - /// - private static XPathNodeIterator GetResultsAsXml(ILuceneSearchResults results) - { - // create the XDocument - XDocument doc = new XDocument(); - - // check there are any search results - if (results.TotalItemCount > 0) - { - // create the root element - XElement root = new XElement("nodes"); - - // iterate through the search results - foreach (SearchResult result in results) - { - // create a new element - XElement node = new XElement("node"); - - // create the @id attribute - XAttribute nodeId = new XAttribute("id", result.Id); - - // create the @score attribute - XAttribute nodeScore = new XAttribute("score", result.Score); - - // add the content - node.Add(nodeId, nodeScore); - - foreach (KeyValuePair field in result.Fields) - { - // create a new element - XElement data = new XElement("data"); - - // create the @alias attribute - XAttribute alias = new XAttribute("alias", field.Key); - - // assign the value to a CDATA section - XCData value = new XCData(field.Value); - - // append the content - data.Add(alias, value); - - // append the element - node.Add(data); - } - - // add the node - root.Add(node); - } - - // add the root node - doc.Add(root); - } - else - { - doc.Add(new XElement("error", "There were no search results.")); - } - - return doc.CreateNavigator().Select("/"); - } - } -} diff --git a/src/Umbraco.Tests/Cache/PublishedCache/PublishedMediaCacheTests.cs b/src/Umbraco.Tests/Cache/PublishedCache/PublishedMediaCacheTests.cs index 136929d4cf..d78682c118 100644 --- a/src/Umbraco.Tests/Cache/PublishedCache/PublishedMediaCacheTests.cs +++ b/src/Umbraco.Tests/Cache/PublishedCache/PublishedMediaCacheTests.cs @@ -161,31 +161,31 @@ namespace Umbraco.Tests.Cache.PublishedCache var ctx = GetUmbracoContext("/test"); var key = Guid.NewGuid(); - var result = new SearchResult() - { - LongId = 1234, - Score = 1 - }; - result.Fields.Add("__IndexType", "media"); - result.Fields.Add("__NodeId", "1234"); - result.Fields.Add("__NodeTypeAlias", Constants.Conventions.MediaTypes.Image); - result.Fields.Add("__Path", "-1,1234"); - result.Fields.Add("__nodeName", "Test"); - result.Fields.Add("id", "1234"); - result.Fields.Add("key", key.ToString()); - result.Fields.Add("urlName", "/media/test.jpg"); - result.Fields.Add("nodeType", "0"); - result.Fields.Add("sortOrder", "0"); - result.Fields.Add("level", "2"); - result.Fields.Add("nodeName", "Test"); - result.Fields.Add("nodeTypeAlias", Constants.Conventions.MediaTypes.Image); - result.Fields.Add("parentID", "-1"); - result.Fields.Add("path", "-1,1234"); - result.Fields.Add("updateDate", DateTime.Parse("2012-07-16T10:34:09").Ticks.ToString()); - result.Fields.Add("createDate", DateTime.Parse("2012-07-17T10:34:09").Ticks.ToString()); - result.Fields.Add("creatorID", "0"); - result.Fields.Add("creatorName", "Shannon"); + var fields = new Dictionary + { + {"__IndexType", "media"}, + {"__NodeId", "1234"}, + {"__NodeTypeAlias", Constants.Conventions.MediaTypes.Image}, + {"__Path", "-1,1234"}, + {"__nodeName", "Test"}, + {"id", "1234"}, + {"key", key.ToString()}, + {"urlName", "/media/test.jpg"}, + {"nodeType", "0"}, + {"sortOrder", "0"}, + {"level", "2"}, + {"nodeName", "Test"}, + {"nodeTypeAlias", Constants.Conventions.MediaTypes.Image}, + {"parentID", "-1"}, + {"path", "-1,1234"}, + {"updateDate", DateTime.Parse("2012-07-16T10:34:09").Ticks.ToString()}, + {"createDate", DateTime.Parse("2012-07-17T10:34:09").Ticks.ToString()}, + {"creatorID", "0"}, + {"creatorName", "Shannon"} + }; + var result = new SearchResult("1234", 1, 1, fields.ToDictionary(x => x.Key, x => new[] {x.Value})); + var store = new PublishedMediaCache(new XmlStore((XmlDocument)null, null, null, null), ServiceContext.MediaService, ServiceContext.UserService, new StaticCacheProvider(), ContentTypesCache); var doc = store.CreateFromCacheValues(store.ConvertFromSearchResult(result)); diff --git a/src/Umbraco.Tests/Composing/TypeFinderTests.cs b/src/Umbraco.Tests/Composing/TypeFinderTests.cs index 3bb5a7dc0e..29458deefb 100644 --- a/src/Umbraco.Tests/Composing/TypeFinderTests.cs +++ b/src/Umbraco.Tests/Composing/TypeFinderTests.cs @@ -49,7 +49,7 @@ namespace Umbraco.Tests.Composing //typeof(TabPage).Assembly, typeof(System.Web.Mvc.ActionResult).Assembly, typeof(TypeFinder).Assembly, - typeof(global::Umbraco.Examine.BaseUmbracoIndexer).Assembly + typeof(global::Umbraco.Examine.UmbracoExamineIndexer).Assembly }; } diff --git a/src/Umbraco.Tests/PublishedContent/PublishedMediaTests.cs b/src/Umbraco.Tests/PublishedContent/PublishedMediaTests.cs index 8e5436cce1..21c656b2fb 100644 --- a/src/Umbraco.Tests/PublishedContent/PublishedMediaTests.cs +++ b/src/Umbraco.Tests/PublishedContent/PublishedMediaTests.cs @@ -12,7 +12,7 @@ using Umbraco.Web; using Umbraco.Web.PublishedCache.XmlPublishedCache; using System.Linq; using System.Xml; -using Examine.Session; +using Examine; using Umbraco.Core.Cache; using Umbraco.Core.Models.PublishedContent; using Umbraco.Core.Strings; @@ -111,11 +111,10 @@ namespace Umbraco.Tests.PublishedContent { using (var luceneDir = new RAMDirectory()) using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, options: new UmbracoContentIndexerOptions(true, false, null))) - using (var session = new ThreadScopedIndexSession(indexer.SearcherContext)) { - indexer.RebuildIndex(); - session.WaitForChanges(); + indexer.RebuildIndex(); + var searcher = indexer.GetSearcher(); var ctx = GetUmbracoContext("/test"); var cache = new PublishedMediaCache(ServiceContext.MediaService, ServiceContext.UserService, searcher, indexer, new StaticCacheProvider(), ContentTypesCache); @@ -138,10 +137,10 @@ namespace Umbraco.Tests.PublishedContent { using (var luceneDir = new RAMDirectory()) using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, options: new UmbracoContentIndexerOptions(true, false, null))) - using (var session = new ThreadScopedIndexSession(indexer.SearcherContext)) + using (indexer.ProcessNonAsync()) { indexer.RebuildIndex(); - session.WaitForChanges(); + var searcher = indexer.GetSearcher(); var ctx = GetUmbracoContext("/test"); @@ -159,11 +158,11 @@ namespace Umbraco.Tests.PublishedContent 10726 jpg "); - indexer.ReIndexNode(newXml, "media"); - session.WaitForChanges(); + indexer.IndexItems(new[]{ newXml.ConvertToValueSet("media") }); + //ensure it still exists in the index (raw examine search) - var criteria = searcher.CreateSearchCriteria(); + var criteria = searcher.CreateCriteria(); var filter = criteria.Id(3113); var found = searcher.Search(filter.Compile()); Assert.IsNotNull(found); @@ -182,10 +181,10 @@ namespace Umbraco.Tests.PublishedContent { using (var luceneDir = new RAMDirectory()) using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, options: new UmbracoContentIndexerOptions(true, false, null))) - using (var session = new ThreadScopedIndexSession(indexer.SearcherContext)) + using (indexer.ProcessNonAsync()) { indexer.RebuildIndex(); - session.WaitForChanges(); + var searcher = indexer.GetSearcher(); var ctx = GetUmbracoContext("/test"); @@ -207,10 +206,10 @@ namespace Umbraco.Tests.PublishedContent { using (var luceneDir = new RAMDirectory()) using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, options: new UmbracoContentIndexerOptions(true, false, null))) - using (var session = new ThreadScopedIndexSession(indexer.SearcherContext)) + using (indexer.ProcessNonAsync()) { indexer.RebuildIndex(); - session.WaitForChanges(); + var searcher = indexer.GetSearcher(); var ctx = GetUmbracoContext("/test"); @@ -232,10 +231,10 @@ namespace Umbraco.Tests.PublishedContent { using (var luceneDir = new RAMDirectory()) using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, options: new UmbracoContentIndexerOptions(true, false, null))) - using (var session = new ThreadScopedIndexSession(indexer.SearcherContext)) + using (indexer.ProcessNonAsync()) { indexer.RebuildIndex(); - session.WaitForChanges(); + var searcher = indexer.GetSearcher(); var ctx = GetUmbracoContext("/test"); @@ -257,10 +256,10 @@ namespace Umbraco.Tests.PublishedContent { using (var luceneDir = new RAMDirectory()) using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, options: new UmbracoContentIndexerOptions(true, false, null))) - using (var session = new ThreadScopedIndexSession(indexer.SearcherContext)) + using (indexer.ProcessNonAsync()) { indexer.RebuildIndex(); - session.WaitForChanges(); + var ctx = GetUmbracoContext("/test"); var searcher = indexer.GetSearcher(); @@ -279,10 +278,10 @@ namespace Umbraco.Tests.PublishedContent { using (var luceneDir = new RAMDirectory()) using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, options: new UmbracoContentIndexerOptions(true, false, null))) - using (var session = new ThreadScopedIndexSession(indexer.SearcherContext)) + using (indexer.ProcessNonAsync()) { indexer.RebuildIndex(); - session.WaitForChanges(); + var ctx = GetUmbracoContext("/test"); var searcher = indexer.GetSearcher(); diff --git a/src/Umbraco.Tests/TestHelpers/Stubs/TestIndexCollectionAccessor.cs b/src/Umbraco.Tests/TestHelpers/Stubs/TestIndexCollectionAccessor.cs index 550bb6e613..86428b4d4e 100644 --- a/src/Umbraco.Tests/TestHelpers/Stubs/TestIndexCollectionAccessor.cs +++ b/src/Umbraco.Tests/TestHelpers/Stubs/TestIndexCollectionAccessor.cs @@ -6,6 +6,6 @@ namespace Umbraco.Tests.TestHelpers.Stubs { public class TestIndexCollectionAccessor : IExamineIndexCollectionAccessor { - public IReadOnlyDictionary Indexes => new Dictionary(); + public IReadOnlyDictionary Indexes => new Dictionary(); } } diff --git a/src/Umbraco.Tests/Umbraco.Tests.csproj b/src/Umbraco.Tests/Umbraco.Tests.csproj index 7aa4fa6bf3..aa8182f15c 100644 --- a/src/Umbraco.Tests/Umbraco.Tests.csproj +++ b/src/Umbraco.Tests/Umbraco.Tests.csproj @@ -59,8 +59,8 @@ ..\packages\Castle.Core.4.1.1\lib\net45\Castle.Core.dll - - ..\packages\Examine.2.0.0-beta2\lib\net45\Examine.dll + + ..\packages\Examine.1.0.0-beta011\lib\net45\Examine.dll ..\packages\SharpZipLib.0.86.0\lib\20\ICSharpCode.SharpZipLib.dll @@ -575,7 +575,6 @@ True TestFiles.resx - diff --git a/src/Umbraco.Tests/UmbracoExamine/EventsTest.cs b/src/Umbraco.Tests/UmbracoExamine/EventsTest.cs index d480cd9b1e..95e338f66f 100644 --- a/src/Umbraco.Tests/UmbracoExamine/EventsTest.cs +++ b/src/Umbraco.Tests/UmbracoExamine/EventsTest.cs @@ -1,10 +1,8 @@ using System; using System.Linq; using Examine; -using Examine.Session; using Lucene.Net.Store; using NUnit.Framework; -using Umbraco.Tests.TestHelpers; using Umbraco.Tests.Testing; using Umbraco.Examine; @@ -21,19 +19,10 @@ namespace Umbraco.Tests.UmbracoExamine using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, //make parent id 999 so all are ignored options: new UmbracoContentIndexerOptions(false, false, 999))) - using (var session = new ThreadScopedIndexSession(indexer.SearcherContext)) + using (indexer.ProcessNonAsync()) { var searcher = indexer.GetSearcher(); - - var isIgnored = false; - - EventHandler ignoringNode = (s, e) => - { - isIgnored = true; - }; - - indexer.IgnoringNode += ignoringNode; - + var contentService = new ExamineDemoDataContentService(); //get a node from the data repo var node = contentService.GetPublishedContentByXPath("//*[string-length(@id)>0 and number(@id)>0]") @@ -41,10 +30,12 @@ namespace Umbraco.Tests.UmbracoExamine .Elements() .First(); - indexer.ReIndexNode(node, IndexTypes.Content); + var valueSet = node.ConvertToValueSet(IndexTypes.Content); + indexer.IndexItems(new[] {valueSet}); + var found = searcher.Search(searcher.CreateCriteria().Id((string) node.Attribute("id")).Compile()); - Assert.IsTrue(isIgnored); + Assert.IsNull(found); } diff --git a/src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs b/src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs index 6e82f7d21e..5ba2f2cdba 100644 --- a/src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs +++ b/src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs @@ -10,6 +10,7 @@ using Moq; using Umbraco.Core.Logging; using Umbraco.Core.Models; using Umbraco.Core.Models.Membership; +using Umbraco.Core.Persistence; using Umbraco.Core.Persistence.DatabaseModelDefinitions; using Umbraco.Core.Persistence.Querying; using Umbraco.Core.Scoping; @@ -37,6 +38,7 @@ namespace Umbraco.Tests.UmbracoExamine IUserService userService = null, IContentTypeService contentTypeService = null, IMediaTypeService mediaTypeService = null, + ISqlContext sqlContext = null, UmbracoContentIndexerOptions options = null) { if (contentService == null) @@ -80,6 +82,13 @@ namespace Umbraco.Tests.UmbracoExamine { userService = Mock.Of(x => x.GetProfileById(It.IsAny()) == Mock.Of(p => p.Id == 0 && p.Name == "admin")); } + + if (sqlContext == null) + { + //TODO: What do we need here? + sqlContext = Mock.Of(); + } + if (mediaService == null) { long totalRecs; @@ -182,10 +191,10 @@ namespace Umbraco.Tests.UmbracoExamine contentService, mediaService, userService, + sqlContext, new[] {new DefaultUrlSegmentProvider()}, new UmbracoContentValueSetValidator(options, Mock.Of()), - options, - scopeProvider.Object); + options); i.IndexingError += IndexingError; @@ -206,7 +215,7 @@ namespace Umbraco.Tests.UmbracoExamine internal static void IndexingError(object sender, IndexingErrorEventArgs e) { - throw new ApplicationException(e.Message, e.Exception); + throw new ApplicationException(e.Message, e.InnerException); } diff --git a/src/Umbraco.Tests/UmbracoExamine/IndexTest.cs b/src/Umbraco.Tests/UmbracoExamine/IndexTest.cs index 8964a6e15d..7757d4182d 100644 --- a/src/Umbraco.Tests/UmbracoExamine/IndexTest.cs +++ b/src/Umbraco.Tests/UmbracoExamine/IndexTest.cs @@ -1,11 +1,10 @@ using System.Linq; +using Examine; using Examine.LuceneEngine.Providers; -using Examine.Session; using Lucene.Net.Index; using Lucene.Net.Search; using Lucene.Net.Store; using NUnit.Framework; -using Umbraco.Tests.TestHelpers; using Umbraco.Tests.Testing; using Umbraco.Examine; @@ -26,15 +25,14 @@ namespace Umbraco.Tests.UmbracoExamine using (var luceneDir = new RAMDirectory()) using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, options: new UmbracoContentIndexerOptions(true, false, null))) - using (var session = new ThreadScopedIndexSession(indexer.SearcherContext)) + using (indexer.ProcessNonAsync()) { var searcher = indexer.GetSearcher(); //create the whole thing indexer.RebuildIndex(); - session.WaitForChanges(); - var result = searcher.Find(searcher.CreateCriteria().All().Compile()); + var result = searcher.Search(searcher.CreateCriteria().All().Compile()); Assert.AreEqual(29, result.TotalItemCount); } @@ -50,22 +48,21 @@ namespace Umbraco.Tests.UmbracoExamine using (var luceneDir = new RAMDirectory()) using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir)) - using (var session = new ThreadScopedIndexSession(indexer.SearcherContext)) - using (var searcher = indexer.GetSearcher().GetSearcher()) + using (indexer.ProcessNonAsync()) + using (var searcher = ((LuceneSearcher)indexer.GetSearcher()).GetLuceneSearcher()) { //create the whole thing indexer.RebuildIndex(); - session.WaitForChanges(); var protectedQuery = new BooleanQuery(); protectedQuery.Add( new BooleanClause( - new TermQuery(new Term(LuceneIndexer.IndexTypeFieldName, IndexTypes.Content)), + new TermQuery(new Term(LuceneIndexer.CategoryFieldName, IndexTypes.Content)), Occur.MUST)); protectedQuery.Add( new BooleanClause( - new TermQuery(new Term(LuceneIndexer.IndexNodeIdFieldName, ExamineDemoDataContentService.ProtectedNode.ToString())), + new TermQuery(new Term(LuceneIndexer.ItemIdFieldName, ExamineDemoDataContentService.ProtectedNode.ToString())), Occur.MUST)); var collector = TopScoreDocCollector.Create(100, true); @@ -84,7 +81,7 @@ namespace Umbraco.Tests.UmbracoExamine using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, //make parent id 1116 options: new UmbracoContentIndexerOptions(false, false, 1116))) - using (var session = new ThreadScopedIndexSession(indexer.SearcherContext)) + using (indexer.ProcessNonAsync()) { var searcher = indexer.GetSearcher(); @@ -99,12 +96,10 @@ namespace Umbraco.Tests.UmbracoExamine Assert.AreEqual("-1,1111,2222,2112", currPath); //ensure it's indexed - indexer.ReIndexNode(node, IndexTypes.Media); - - session.WaitForChanges(); + indexer.IndexItems(new []{ node.ConvertToValueSet(IndexTypes.Media) }); //it will not exist because it exists under 2222 - var results = searcher.Search(searcher.CreateSearchCriteria().Id(2112).Compile()); + var results = searcher.Search(searcher.CreateCriteria().Id(2112).Compile()); Assert.AreEqual(0, results.Count()); //now mimic moving 2112 to 1116 @@ -113,12 +108,10 @@ namespace Umbraco.Tests.UmbracoExamine node.SetAttributeValue("parentID", "1116"); //now reindex the node, this should first delete it and then WILL add it because of the parent id constraint - indexer.ReIndexNode(node, IndexTypes.Media); - - session.WaitForChanges(); + indexer.IndexItems(new[] { node.ConvertToValueSet(IndexTypes.Media) }); //now ensure it exists - results = searcher.Search(searcher.CreateSearchCriteria().Id(2112).Compile()); + results = searcher.Search(searcher.CreateCriteria().Id(2112).Compile()); Assert.AreEqual(1, results.Count()); } } @@ -130,7 +123,7 @@ namespace Umbraco.Tests.UmbracoExamine using (var indexer1 = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, //make parent id 2222 options: new UmbracoContentIndexerOptions(false, false, 2222))) - using (var session = new ThreadScopedIndexSession(indexer1.SearcherContext)) + using (indexer1.ProcessNonAsync()) { var searcher = indexer1.GetSearcher(); @@ -145,12 +138,12 @@ namespace Umbraco.Tests.UmbracoExamine Assert.AreEqual("-1,1111,2222,2112", currPath); //ensure it's indexed - indexer1.ReIndexNode(node, IndexTypes.Media); + indexer1.IndexItems(new[] { node.ConvertToValueSet(IndexTypes.Media) }); + - session.WaitForChanges(); //it will exist because it exists under 2222 - var results = searcher.Search(searcher.CreateSearchCriteria().Id(2112).Compile()); + var results = searcher.Search(searcher.CreateCriteria().Id(2112).Compile()); Assert.AreEqual(1, results.Count()); //now mimic moving the node underneath 1116 instead of 2222 @@ -158,12 +151,12 @@ namespace Umbraco.Tests.UmbracoExamine node.SetAttributeValue("parentID", "1116"); //now reindex the node, this should first delete it and then NOT add it because of the parent id constraint - indexer1.ReIndexNode(node, IndexTypes.Media); + indexer1.IndexItems(new[] { node.ConvertToValueSet(IndexTypes.Media) }); + - session.WaitForChanges(); //now ensure it's deleted - results = searcher.Search(searcher.CreateSearchCriteria().Id(2112).Compile()); + results = searcher.Search(searcher.CreateCriteria().Id(2112).Compile()); Assert.AreEqual(0, results.Count()); } } @@ -178,34 +171,34 @@ namespace Umbraco.Tests.UmbracoExamine { using (var luceneDir = new RAMDirectory()) using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, options: new UmbracoContentIndexerOptions(true, false, null))) - using (var session = new ThreadScopedIndexSession(indexer.SearcherContext)) + using (indexer.ProcessNonAsync()) { var searcher = indexer.GetSearcher(); //create the whole thing indexer.RebuildIndex(); - session.WaitForChanges(); + - var result = searcher.Find(searcher.CreateCriteria().Field(LuceneIndexer.IndexTypeFieldName, IndexTypes.Content).Compile()); + var result = searcher.Search(searcher.CreateCriteria().Field(LuceneIndexer.CategoryFieldName, IndexTypes.Content).Compile()); Assert.AreEqual(21, result.TotalItemCount); //delete all content foreach (var r in result) { - indexer.DeleteFromIndex(r.LongId); + indexer.DeleteFromIndex(r.Id); } - session.WaitForChanges(); + //ensure it's all gone - result = searcher.Find(searcher.CreateCriteria().Field(LuceneIndexer.IndexTypeFieldName, IndexTypes.Content).Compile()); + result = searcher.Search(searcher.CreateCriteria().Field(LuceneIndexer.CategoryFieldName, IndexTypes.Content).Compile()); Assert.AreEqual(0, result.TotalItemCount); //call our indexing methods indexer.IndexAll(IndexTypes.Content); - session.WaitForChanges(); + - result = searcher.Find(searcher.CreateCriteria().Field(LuceneIndexer.IndexTypeFieldName, IndexTypes.Content).Compile()); + result = searcher.Search(searcher.CreateCriteria().Field(LuceneIndexer.CategoryFieldName, IndexTypes.Content).Compile()); Assert.AreEqual(21, result.TotalItemCount); } @@ -221,25 +214,25 @@ namespace Umbraco.Tests.UmbracoExamine { using (var luceneDir = new RAMDirectory()) using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir)) - using (var session = new ThreadScopedIndexSession(indexer.SearcherContext)) + using (indexer.ProcessNonAsync()) { var searcher = indexer.GetSearcher(); //create the whole thing indexer.RebuildIndex(); - session.WaitForChanges(); + //now delete a node that has children indexer.DeleteFromIndex(1140.ToString()); //this node had children: 1141 & 1142, let's ensure they are also removed - session.WaitForChanges(); + - var results = searcher.Search(searcher.CreateSearchCriteria().Id(1141).Compile()); + var results = searcher.Search(searcher.CreateCriteria().Id(1141).Compile()); Assert.AreEqual(0, results.Count()); - results = searcher.Search(searcher.CreateSearchCriteria().Id(1142).Compile()); + results = searcher.Search(searcher.CreateCriteria().Id(1142).Compile()); Assert.AreEqual(0, results.Count()); } diff --git a/src/Umbraco.Tests/UmbracoExamine/SearchTests.cs b/src/Umbraco.Tests/UmbracoExamine/SearchTests.cs index d1ebf68871..de135c55bf 100644 --- a/src/Umbraco.Tests/UmbracoExamine/SearchTests.cs +++ b/src/Umbraco.Tests/UmbracoExamine/SearchTests.cs @@ -5,13 +5,12 @@ using Examine; using Lucene.Net.Store; using NUnit.Framework; using Examine.LuceneEngine.SearchCriteria; -using Examine.Session; using Moq; using Umbraco.Core.Models; using Umbraco.Core.Persistence.DatabaseModelDefinitions; using Umbraco.Core.Persistence.Querying; using Umbraco.Core.Services; -using Umbraco.Tests.TestHelpers; +using Umbraco.Examine; using Umbraco.Tests.Testing; namespace Umbraco.Tests.UmbracoExamine @@ -56,19 +55,19 @@ namespace Umbraco.Tests.UmbracoExamine using (var luceneDir = new RAMDirectory()) using (var indexer = IndexInitializer.GetUmbracoIndexer(ProfilingLogger, luceneDir, contentService: contentService)) - using (var session = new ThreadScopedIndexSession(indexer.SearcherContext)) + using (indexer.ProcessNonAsync()) { indexer.RebuildIndex(); - session.WaitForChanges(); + var searcher = indexer.GetSearcher(); - var numberSortedCriteria = searcher.CreateSearchCriteria() + var numberSortedCriteria = searcher.CreateCriteria() .ParentId(1148).And() .OrderBy(new SortableField("sortOrder", SortType.Int)); var numberSortedResult = searcher.Search(numberSortedCriteria.Compile()); - var stringSortedCriteria = searcher.CreateSearchCriteria() + var stringSortedCriteria = searcher.CreateCriteria() .ParentId(1148).And() .OrderBy("sortOrder"); //will default to string var stringSortedResult = searcher.Search(stringSortedCriteria.Compile()); diff --git a/src/Umbraco.Tests/UmbracoExamine/TestIndexField.cs b/src/Umbraco.Tests/UmbracoExamine/TestIndexField.cs deleted file mode 100644 index b2e2d7d823..0000000000 --- a/src/Umbraco.Tests/UmbracoExamine/TestIndexField.cs +++ /dev/null @@ -1,11 +0,0 @@ -using Examine; - -namespace Umbraco.Tests.UmbracoExamine -{ - public class TestIndexField : IIndexField - { - public string Name { get; set; } - public bool EnableSorting { get; set; } - public string Type { get; set; } - } -} diff --git a/src/Umbraco.Tests/packages.config b/src/Umbraco.Tests/packages.config index b75ca5912b..1e5c1ef082 100644 --- a/src/Umbraco.Tests/packages.config +++ b/src/Umbraco.Tests/packages.config @@ -2,7 +2,7 @@ - + diff --git a/src/Umbraco.Web.UI.Client/src/views/dashboard/developer/examinemanagement.html b/src/Umbraco.Web.UI.Client/src/views/dashboard/developer/examinemanagement.html index e80e7775da..72e7bf9eed 100644 --- a/src/Umbraco.Web.UI.Client/src/views/dashboard/developer/examinemanagement.html +++ b/src/Umbraco.Web.UI.Client/src/views/dashboard/developer/examinemanagement.html @@ -76,21 +76,7 @@ Fields in index {{indexer.fieldCount}} - - - Has deletions? - - {{indexer.deletionCount > 0}} - ({{indexer.deletionCount}}) - - - - Optimized? - - {{indexer.isOptimized}} - - - + diff --git a/src/Umbraco.Web.UI.Client/src/views/dashboard/developer/examinemgmt.controller.js b/src/Umbraco.Web.UI.Client/src/views/dashboard/developer/examinemgmt.controller.js index 5f6bb23001..f4e78a9dc3 100644 --- a/src/Umbraco.Web.UI.Client/src/views/dashboard/developer/examinemgmt.controller.js +++ b/src/Umbraco.Web.UI.Client/src/views/dashboard/developer/examinemgmt.controller.js @@ -91,30 +91,6 @@ function ExamineMgmtController($scope, umbRequestHelper, $log, $http, $q, $timeo } } - $scope.optimizeIndex = function(indexer) { - if (confirm("This will cause the index to be optimized which will improve its performance. " + - "It is not recommended to optimize an index during times of high website traffic " + - "or when editors are editing content.")) { - indexer.isProcessing = true; - - umbRequestHelper.resourcePromise( - $http.post(umbRequestHelper.getApiUrl("examineMgmtBaseUrl", - "PostOptimizeIndex", - { indexerName: indexer.name })), - 'Failed to optimize index') - .then(function() { - - //optimizing has started, nothing is returned accept a 200 status code. - //lets poll to see if it is done. - $timeout(function() { - checkProcessing(indexer, "PostCheckOptimizeIndex"); - }, - 1000); - - }); - } - } - $scope.closeSearch = function(searcher) { searcher.isSearching = true; } diff --git a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj index 4c8df3cc87..d7faff68be 100644 --- a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj +++ b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj @@ -54,6 +54,9 @@ ..\packages\AutoMapper.6.1.1\lib\net45\AutoMapper.dll + + ..\packages\Examine.1.0.0-beta011\lib\net45\Examine.dll + ..\packages\ImageProcessor.Web.4.8.4\lib\net45\ImageProcessor.Web.dll @@ -274,10 +277,6 @@ ../packages/ClientDependency-Mvc5.1.8.0.0/lib/net45/ClientDependency.Core.Mvc.dll True - - ../packages/Examine.2.0.0-beta2/lib/net45/Examine.dll - True - ../packages/SharpZipLib.0.86.0/lib/20/ICSharpCode.SharpZipLib.dll True diff --git a/src/Umbraco.Web.UI/packages.config b/src/Umbraco.Web.UI/packages.config index 37bc562410..23dd19a812 100644 --- a/src/Umbraco.Web.UI/packages.config +++ b/src/Umbraco.Web.UI/packages.config @@ -4,7 +4,7 @@ - + diff --git a/src/Umbraco.Web/ExamineExtensions.cs b/src/Umbraco.Web/ExamineExtensions.cs index 455a857879..00471a44e1 100644 --- a/src/Umbraco.Web/ExamineExtensions.cs +++ b/src/Umbraco.Web/ExamineExtensions.cs @@ -18,10 +18,12 @@ namespace Umbraco.Web foreach (var result in results.OrderByDescending(x => x.Score)) { - var content = cache.GetById(result.Id); + if (!int.TryParse(result.Id, out var intId)) continue; //invalid + var content = cache.GetById(intId); if (content == null) continue; // skip if this doesn't exist in the cache list.Add(new PublishedSearchResult(content, result.Score)); + } return list; diff --git a/src/Umbraco.Web/IPublishedContentQuery.cs b/src/Umbraco.Web/IPublishedContentQuery.cs index 2938fa36d9..7e7a57b047 100644 --- a/src/Umbraco.Web/IPublishedContentQuery.cs +++ b/src/Umbraco.Web/IPublishedContentQuery.cs @@ -36,9 +36,9 @@ namespace Umbraco.Web /// /// /// - /// + /// The index to search /// - IEnumerable Search(string term, bool useWildCards = true, string searchProvider = null); + IEnumerable Search(string term, bool useWildCards = true, string indexName = null); /// /// Searhes content @@ -46,6 +46,6 @@ namespace Umbraco.Web /// /// /// - IEnumerable Search(Examine.SearchCriteria.ISearchCriteria criteria, Examine.Providers.BaseSearchProvider searchProvider = null); + IEnumerable Search(Examine.SearchCriteria.ISearchCriteria criteria, Examine.ISearcher searchProvider = null); } } diff --git a/src/Umbraco.Web/Models/Mapping/EntityMapperProfile.cs b/src/Umbraco.Web/Models/Mapping/EntityMapperProfile.cs index 27116e8cdc..6b90698823 100644 --- a/src/Umbraco.Web/Models/Mapping/EntityMapperProfile.cs +++ b/src/Umbraco.Web/Models/Mapping/EntityMapperProfile.cs @@ -122,22 +122,22 @@ namespace Umbraco.Web.Models.Mapping .AfterMap((src, dest) => { //get the icon if there is one - dest.Icon = src.Fields.ContainsKey(BaseUmbracoIndexer.IconFieldName) - ? src.Fields[BaseUmbracoIndexer.IconFieldName] + dest.Icon = src.Fields.ContainsKey(UmbracoExamineIndexer.IconFieldName) + ? src.Fields[UmbracoExamineIndexer.IconFieldName] : "icon-document"; dest.Name = src.Fields.ContainsKey("nodeName") ? src.Fields["nodeName"] : "[no name]"; - if (src.Fields.ContainsKey(UmbracoContentIndexer.NodeKeyFieldName)) + if (src.Fields.ContainsKey(UmbracoExamineIndexer.NodeKeyFieldName)) { Guid key; - if (Guid.TryParse(src.Fields[UmbracoContentIndexer.NodeKeyFieldName], out key)) + if (Guid.TryParse(src.Fields[UmbracoExamineIndexer.NodeKeyFieldName], out key)) { dest.Key = key; //need to set the UDI - if (src.Fields.ContainsKey(LuceneIndexer.IndexTypeFieldName)) + if (src.Fields.ContainsKey(LuceneIndexer.CategoryFieldName)) { - switch (src.Fields[LuceneIndexer.IndexTypeFieldName]) + switch (src.Fields[LuceneIndexer.CategoryFieldName]) { case IndexTypes.Member: dest.Udi = new GuidUdi(Constants.UdiEntityType.Member, dest.Key); @@ -165,15 +165,15 @@ namespace Umbraco.Web.Models.Mapping dest.ParentId = -1; } } - dest.Path = src.Fields.ContainsKey(UmbracoContentIndexer.IndexPathFieldName) ? src.Fields[UmbracoContentIndexer.IndexPathFieldName] : ""; + dest.Path = src.Fields.ContainsKey(UmbracoExamineIndexer.IndexPathFieldName) ? src.Fields[UmbracoExamineIndexer.IndexPathFieldName] : ""; - if (src.Fields.ContainsKey(LuceneIndexer.NodeTypeAliasFieldName)) + if (src.Fields.ContainsKey(LuceneIndexer.ItemTypeFieldName)) { - dest.AdditionalData.Add("contentType", src.Fields[LuceneIndexer.NodeTypeAliasFieldName]); + dest.AdditionalData.Add("contentType", src.Fields[LuceneIndexer.ItemTypeFieldName]); } }); - CreateMap>() + CreateMap>() .ConvertUsing(results => results.Select(Mapper.Map).ToList()); CreateMap, IEnumerable>() diff --git a/src/Umbraco.Web/PropertyEditors/GridPropertyEditor.cs b/src/Umbraco.Web/PropertyEditors/GridPropertyEditor.cs index 6256f49c67..0f3211b743 100644 --- a/src/Umbraco.Web/PropertyEditors/GridPropertyEditor.cs +++ b/src/Umbraco.Web/PropertyEditors/GridPropertyEditor.cs @@ -27,84 +27,80 @@ namespace Umbraco.Web.PropertyEditors internal void DocumentWriting(object sender, Examine.LuceneEngine.DocumentWritingEventArgs e) { - var indexer = (BaseUmbracoIndexer)sender; - foreach (var field in indexer.IndexerData.UserFields) + foreach (var value in e.ValueSet.Values) { - if (e.Fields.ContainsKey(field.Name)) + //if there is a value, it's a string and it's detected as json + if (value.Value.Count > 0 && value.Value[0] != null && (value.Value[0] is string firstVal) && firstVal.DetectIsJson()) { - if (e.Fields[field.Name].DetectIsJson()) + try { - try + //TODO: We should deserialize this to Umbraco.Core.Models.GridValue instead of doing the below + var json = JsonConvert.DeserializeObject(firstVal); + + //check if this is formatted for grid json + if (json.HasValues && json.TryGetValue("name", out _) && json.TryGetValue("sections", out _)) { - //TODO: We should deserialize this to Umbraco.Core.Models.GridValue instead of doing the below - var json = JsonConvert.DeserializeObject(e.Fields[field.Name]); - - //check if this is formatted for grid json - JToken name; - JToken sections; - if (json.HasValues && json.TryGetValue("name", out name) && json.TryGetValue("sections", out sections)) + //get all values and put them into a single field (using JsonPath) + var sb = new StringBuilder(); + foreach (var row in json.SelectTokens("$.sections[*].rows[*]")) { - //get all values and put them into a single field (using JsonPath) - var sb = new StringBuilder(); - foreach (var row in json.SelectTokens("$.sections[*].rows[*]")) - { - var rowName = row["name"].Value(); - var areaVals = row.SelectTokens("$.areas[*].controls[*].value"); + var rowName = row["name"].Value(); + var areaVals = row.SelectTokens("$.areas[*].controls[*].value"); - foreach (var areaVal in areaVals) + foreach (var areaVal in areaVals) + { + //TODO: If it's not a string, then it's a json formatted value - + // we cannot really index this in a smart way since it could be 'anything' + if (areaVal.Type == JTokenType.String) { - //TODO: If it's not a string, then it's a json formatted value - - // we cannot really index this in a smart way since it could be 'anything' - if (areaVal.Type == JTokenType.String) - { - var str = areaVal.Value(); - str = XmlHelper.CouldItBeXml(str) ? str.StripHtml() : str; - sb.Append(str); - sb.Append(" "); - - //add the row name as an individual field - e.Document.Add( - new Field( - string.Format("{0}.{1}", field.Name, rowName), str, Field.Store.YES, Field.Index.ANALYZED)); - } + var str = areaVal.Value(); + str = XmlHelper.CouldItBeXml(str) ? str.StripHtml() : str; + sb.Append(str); + sb.Append(" "); + //add the row name as an individual field + e.Document.Add( + new Field( + $"{value.Key}.{rowName}", str, Field.Store.YES, Field.Index.ANALYZED)); } - } - if (sb.Length > 0) - { - //First save the raw value to a raw field - e.Document.Add( - new Field( - string.Format("{0}{1}", UmbracoContentIndexer.RawFieldPrefix, field.Name), - e.Fields[field.Name], Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS, Field.TermVector.NO)); - - //now replace the original value with the combined/cleaned value - e.Document.RemoveField(field.Name); - e.Document.Add( - new Field( - field.Name, - sb.ToString(), Field.Store.YES, Field.Index.ANALYZED)); } } - } - catch (InvalidCastException) - { - //swallow...on purpose, there's a chance that this isn't the json format we are looking for - // and we don't want that to affect the website. - } - catch (JsonException) - { - //swallow...on purpose, there's a chance that this isn't json and we don't want that to affect - // the website. - } - catch (ArgumentException) - { - //swallow on purpose to prevent this error: - // Can not add Newtonsoft.Json.Linq.JValue to Newtonsoft.Json.Linq.JObject. + + if (sb.Length > 0) + { + //First save the raw value to a raw field + e.Document.Add( + new Field( + $"{UmbracoExamineIndexer.RawFieldPrefix}{value.Key}", + firstVal, Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS, Field.TermVector.NO)); + + //now replace the original value with the combined/cleaned value + e.Document.RemoveField(value.Key); + e.Document.Add( + new Field( + value.Key, + sb.ToString(), Field.Store.YES, Field.Index.ANALYZED)); + } } } + catch (InvalidCastException) + { + //swallow...on purpose, there's a chance that this isn't the json format we are looking for + // and we don't want that to affect the website. + } + catch (JsonException) + { + //swallow...on purpose, there's a chance that this isn't json and we don't want that to affect + // the website. + } + catch (ArgumentException) + { + //swallow on purpose to prevent this error: + // Can not add Newtonsoft.Json.Linq.JValue to Newtonsoft.Json.Linq.JObject. + } } + } } diff --git a/src/Umbraco.Web/PublishedCache/XmlPublishedCache/PublishedMediaCache.cs b/src/Umbraco.Web/PublishedCache/XmlPublishedCache/PublishedMediaCache.cs index 690ae7e5b7..9e2f8ebad1 100644 --- a/src/Umbraco.Web/PublishedCache/XmlPublishedCache/PublishedMediaCache.cs +++ b/src/Umbraco.Web/PublishedCache/XmlPublishedCache/PublishedMediaCache.cs @@ -40,8 +40,8 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache // when they are null the cache derives them from the ExamineManager, see // method GetExamineManagerSafe(). // - private readonly ILuceneSearcher _searchProvider; - private readonly BaseIndexProvider _indexProvider; + private readonly ISearcher _searchProvider; + private readonly IIndexer _indexProvider; private readonly XmlStore _xmlStore; private readonly PublishedContentTypeCache _contentTypeCache; @@ -70,7 +70,7 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache /// /// /// - internal PublishedMediaCache(IMediaService mediaService, IUserService userService, ILuceneSearcher searchProvider, BaseIndexProvider indexProvider, ICacheProvider cacheProvider, PublishedContentTypeCache contentTypeCache) + internal PublishedMediaCache(IMediaService mediaService, IUserService userService, ISearcher searchProvider, BaseIndexProvider indexProvider, ICacheProvider cacheProvider, PublishedContentTypeCache contentTypeCache) : base(false) { if (mediaService == null) throw new ArgumentNullException(nameof(mediaService)); @@ -203,34 +203,8 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache return null; } } - - private BaseIndexProvider GetIndexProviderSafe() - { - if (_indexProvider != null) - return _indexProvider; - - var eMgr = GetExamineManagerSafe(); - if (eMgr == null) return null; - - try - { - //by default use the InternalSearcher - var indexer = eMgr.IndexProviderCollection[Constants.Examine.InternalIndexer]; - if (indexer.IndexerData.IncludeNodeTypes.Any() || indexer.IndexerData.ExcludeNodeTypes.Any()) - { - Current.Logger.Warn("The InternalIndexer for examine is configured incorrectly, it should not list any include/exclude node types or field names, it should simply be configured as: " + ""); - } - return indexer; - } - catch (Exception ex) - { - Current.Logger.Error("Could not retrieve the InternalIndexer", ex); - //something didn't work, continue returning null. - } - return null; - } - - private ILuceneSearcher GetSearchProviderSafe() + + private ISearcher GetSearchProviderSafe() { if (_searchProvider != null) return _searchProvider; @@ -241,7 +215,7 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache try { //by default use the InternalSearcher - return eMgr.GetSearcher(Constants.Examine.InternalIndexer); + return eMgr.GetIndexSearcher(Constants.Examine.InternalIndexer); } catch (FileNotFoundException) { @@ -292,8 +266,8 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache // // note that since the use of the wildcard, it automatically escapes it in Lucene. - var criteria = searchProvider.CreateSearchCriteria("media"); - var filter = criteria.Id(id).Not().Field(BaseUmbracoIndexer.IndexPathFieldName, "-1,-21,".MultipleCharacterWildcard()); + var criteria = searchProvider.CreateCriteria("media"); + var filter = criteria.Id(id.ToInvariantString()).Not().Field(UmbracoExamineIndexer.IndexPathFieldName, "-1,-21,".MultipleCharacterWildcard()); var result = searchProvider.Search(filter.Compile()).FirstOrDefault(); if (result != null) return ConvertFromSearchResult(result); @@ -359,11 +333,10 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache internal CacheValues ConvertFromSearchResult(SearchResult searchResult) { // note: fixing fields in 7.x, removed by Shan for 8.0 - var values = new Dictionary(searchResult.Fields); - + return new CacheValues { - Values = values, + Values = searchResult.Fields, FromExamine = true }; } @@ -479,7 +452,7 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache { //We are going to check for a special field however, that is because in some cases we store a 'Raw' //value in the index such as for xml/html. - var rawValue = dd.Properties.FirstOrDefault(x => x.Alias.InvariantEquals(BaseUmbracoIndexer.RawFieldPrefix + alias)); + var rawValue = dd.Properties.FirstOrDefault(x => x.Alias.InvariantEquals(UmbracoExamineIndexer.RawFieldPrefix + alias)); return rawValue ?? dd.Properties.FirstOrDefault(x => x.Alias.InvariantEquals(alias)); } @@ -509,12 +482,12 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache //first check in Examine as this is WAY faster var criteria = searchProvider.CreateCriteria("media"); - var filter = criteria.ParentId(parentId).Not().Field(BaseUmbracoIndexer.IndexPathFieldName, "-1,-21,".MultipleCharacterWildcard()); + var filter = criteria.ParentId(parentId).Not().Field(UmbracoExamineIndexer.IndexPathFieldName, "-1,-21,".MultipleCharacterWildcard()); //the above filter will create a query like this, NOTE: That since the use of the wildcard, it automatically escapes it in Lucene. //+(+parentId:3113 -__Path:-1,-21,*) +__IndexType:media // sort with the Sort field (updated for 8.0) - var results = searchProvider.Find( + var results = searchProvider.Search( filter.And().OrderBy(new SortableField("sortOrder", SortType.Int)).Compile()); if (results.Any()) @@ -625,7 +598,7 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache private static readonly string[] IgnoredKeys = { "version", "isDoc" }; public DictionaryPublishedContent( - IDictionary valueDictionary, + IReadOnlyDictionary valueDictionary, Func getParent, Func> getChildren, Func getProperty, @@ -651,7 +624,7 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache ValidateAndSetProperty(valueDictionary, val => _sortOrder = int.Parse(val), "sortOrder"); ValidateAndSetProperty(valueDictionary, val => _name = val, "nodeName", "__nodeName"); ValidateAndSetProperty(valueDictionary, val => _urlName = val, "urlName"); - ValidateAndSetProperty(valueDictionary, val => _documentTypeAlias = val, "nodeTypeAlias", LuceneIndexer.NodeTypeAliasFieldName); + ValidateAndSetProperty(valueDictionary, val => _documentTypeAlias = val, "nodeTypeAlias", LuceneIndexer.ItemTypeFieldName); ValidateAndSetProperty(valueDictionary, val => _documentTypeId = int.Parse(val), "nodeType"); //ValidateAndSetProperty(valueDictionary, val => _writerName = val, "writerName"); ValidateAndSetProperty(valueDictionary, val => _creatorName = val, "creatorName", "writerName"); //this is a bit of a hack fix for: U4-1132 @@ -816,7 +789,7 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache private readonly ICollection _properties; private readonly PublishedContentType _contentType; - private void ValidateAndSetProperty(IDictionary valueDictionary, Action setProperty, params string[] potentialKeys) + private void ValidateAndSetProperty(IReadOnlyDictionary valueDictionary, Action setProperty, params string[] potentialKeys) { var key = potentialKeys.FirstOrDefault(x => valueDictionary.ContainsKey(x) && valueDictionary[x] != null); if (key == null) @@ -865,7 +838,7 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache internal class CacheValues { - public IDictionary Values { get; set; } + public IReadOnlyDictionary Values { get; set; } public XPathNavigator XPath { get; set; } public bool FromExamine { get; set; } } @@ -940,7 +913,7 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache GetValuesValue(v.Values, "path", "__Path").Contains(fid)); } - private static string GetValuesValue(IDictionary d, params string[] keys) + private static string GetValuesValue(IReadOnlyDictionary d, params string[] keys) { string value = null; var ignored = keys.Any(x => d.TryGetValue(x, out value)); diff --git a/src/Umbraco.Web/PublishedContentExtensions.cs b/src/Umbraco.Web/PublishedContentExtensions.cs index be51c36931..ba9a354145 100644 --- a/src/Umbraco.Web/PublishedContentExtensions.cs +++ b/src/Umbraco.Web/PublishedContentExtensions.cs @@ -220,49 +220,53 @@ namespace Umbraco.Web #region Search - public static IEnumerable Search(this IPublishedContent content, string term, bool useWildCards = true, string searchProvider = null) + public static IEnumerable Search(this IPublishedContent content, string term, bool useWildCards = true, string indexName = null) { - var searcher = Examine.ExamineManager.Instance.DefaultSearchProvider; - if (string.IsNullOrEmpty(searchProvider) == false) - searcher = Examine.ExamineManager.Instance.SearchProviderCollection[searchProvider]; + var searcher = string.IsNullOrEmpty(indexName) + ? Examine.ExamineManager.Instance.GetIndexSearcher(Constants.Examine.ExternalIndexer) + : Examine.ExamineManager.Instance.GetIndexSearcher(indexName); + + if (searcher == null) + throw new InvalidOperationException("No searcher found for index " + indexName); var t = term.Escape().Value; if (useWildCards) t = term.MultipleCharacterWildcard().Value; var luceneQuery = "+__Path:(" + content.Path.Replace("-", "\\-") + "*) +" + t; - var crit = searcher.CreateSearchCriteria().RawQuery(luceneQuery); + var crit = searcher.CreateCriteria().RawQuery(luceneQuery); return content.Search(crit, searcher); } - public static IEnumerable SearchDescendants(this IPublishedContent content, string term, bool useWildCards = true, string searchProvider = null) + public static IEnumerable SearchDescendants(this IPublishedContent content, string term, bool useWildCards = true, string indexName = null) { - return content.Search(term, useWildCards, searchProvider); + return content.Search(term, useWildCards, indexName); } - public static IEnumerable SearchChildren(this IPublishedContent content, string term, bool useWildCards = true, string searchProvider = null) + public static IEnumerable SearchChildren(this IPublishedContent content, string term, bool useWildCards = true, string indexName = null) { - var searcher = Examine.ExamineManager.Instance.DefaultSearchProvider; - if (string.IsNullOrEmpty(searchProvider) == false) - searcher = Examine.ExamineManager.Instance.SearchProviderCollection[searchProvider]; + var searcher = string.IsNullOrEmpty(indexName) + ? Examine.ExamineManager.Instance.GetIndexSearcher(Constants.Examine.ExternalIndexer) + : Examine.ExamineManager.Instance.GetIndexSearcher(indexName); + + if (searcher == null) + throw new InvalidOperationException("No searcher found for index " + indexName); var t = term.Escape().Value; if (useWildCards) t = term.MultipleCharacterWildcard().Value; var luceneQuery = "+parentID:" + content.Id + " +" + t; - var crit = searcher.CreateSearchCriteria().RawQuery(luceneQuery); + var crit = searcher.CreateCriteria().RawQuery(luceneQuery); return content.Search(crit, searcher); } - public static IEnumerable Search(this IPublishedContent content, Examine.SearchCriteria.ISearchCriteria criteria, Examine.Providers.BaseSearchProvider searchProvider = null) + public static IEnumerable Search(this IPublishedContent content, Examine.SearchCriteria.ISearchCriteria criteria, Examine.ISearcher searchProvider = null) { - var s = Examine.ExamineManager.Instance.DefaultSearchProvider; - if (searchProvider != null) - s = searchProvider; - + var s = searchProvider ?? Examine.ExamineManager.Instance.GetIndexSearcher(Constants.Examine.ExternalIndexer); + var results = s.Search(criteria); return results.ToPublishedSearchResults(UmbracoContext.Current.ContentCache); } diff --git a/src/Umbraco.Web/PublishedContentQuery.cs b/src/Umbraco.Web/PublishedContentQuery.cs index 2e48bf1c55..2adf51c3e1 100644 --- a/src/Umbraco.Web/PublishedContentQuery.cs +++ b/src/Umbraco.Web/PublishedContentQuery.cs @@ -222,16 +222,19 @@ namespace Umbraco.Web /// /// /// - /// + /// The index to search /// - public IEnumerable Search(string term, bool useWildCards = true, string searchProvider = null) + public IEnumerable Search(string term, bool useWildCards = true, string indexName = null) { - if (_query != null) return _query.Search(term, useWildCards, searchProvider); + if (_query != null) return _query.Search(term, useWildCards, indexName); - var searcher = Examine.ExamineManager.Instance.DefaultSearchProvider; - if (string.IsNullOrEmpty(searchProvider) == false) - searcher = Examine.ExamineManager.Instance.SearchProviderCollection[searchProvider]; + var searcher = string.IsNullOrEmpty(indexName) + ? Examine.ExamineManager.Instance.GetIndexSearcher(Constants.Examine.ExternalIndexer) + : Examine.ExamineManager.Instance.GetIndexSearcher(indexName); + if (searcher == null) + throw new InvalidOperationException("No searcher found for index " + indexName); + var results = searcher.Search(term, useWildCards); return results.ToPublishedSearchResults(_contentCache); } @@ -242,13 +245,11 @@ namespace Umbraco.Web /// /// /// - public IEnumerable Search(Examine.SearchCriteria.ISearchCriteria criteria, Examine.Providers.BaseSearchProvider searchProvider = null) + public IEnumerable Search(Examine.SearchCriteria.ISearchCriteria criteria, Examine.ISearcher searchProvider = null) { if (_query != null) return _query.Search(criteria, searchProvider); - var s = Examine.ExamineManager.Instance.DefaultSearchProvider; - if (searchProvider != null) - s = searchProvider; + var s = searchProvider ?? Examine.ExamineManager.Instance.GetIndexSearcher(Constants.Examine.ExternalIndexer); var results = s.Search(criteria); return results.ToPublishedSearchResults(_contentCache); diff --git a/src/Umbraco.Web/Search/ExamineComponent.cs b/src/Umbraco.Web/Search/ExamineComponent.cs index 411f77fb3f..69358e59eb 100644 --- a/src/Umbraco.Web/Search/ExamineComponent.cs +++ b/src/Umbraco.Web/Search/ExamineComponent.cs @@ -6,7 +6,6 @@ using System.Threading; using System.Xml.Linq; using Examine; using Examine.LuceneEngine; -using Examine.Session; using Lucene.Net.Documents; using Umbraco.Core; using Umbraco.Core.Cache; @@ -14,7 +13,6 @@ using Umbraco.Core.Components; using Umbraco.Core.Logging; using Umbraco.Core.Models; using Umbraco.Core.PropertyEditors; -using Umbraco.Core.Services; using Umbraco.Core.Services.Changes; using Umbraco.Core.Services.Implement; using Umbraco.Core.Sync; @@ -66,13 +64,9 @@ namespace Umbraco.Web.Search // the rest is the original Examine event handler logger.Info("Initialize and bind to business logic events."); - - //TODO: For now we'll make this true, it means that indexes will be near real time - // we'll see about what implications this may have - should be great in most scenarios - DefaultExamineSession.RequireImmediateConsistency = true; - - var registeredProviders = ExamineManager.Instance.IndexProviderCollection - .OfType().Count(x => x.EnableDefaultEventHandler); + + var registeredProviders = ExamineManager.Instance.IndexProviders + .OfType().Count(x => x.EnableDefaultEventHandler); logger.Info($"Adding examine event handlers for {registeredProviders} index providers."); @@ -89,24 +83,13 @@ namespace Umbraco.Web.Search // fixme - content type? // events handling removed in ef013f9d3b945d0a48a306ff1afbd49c10c3fff8 // because, could not make sense of it? - - var contentIndexer = ExamineManager.Instance.IndexProviderCollection[Constants.Examine.InternalIndexer] as UmbracoContentIndexer; - if (contentIndexer != null) - { - contentIndexer.DocumentWriting += IndexerDocumentWriting; - } - var memberIndexer = ExamineManager.Instance.IndexProviderCollection[Constants.Examine.InternalMemberIndexer] as UmbracoMemberIndexer; - if (memberIndexer != null) - { - memberIndexer.DocumentWriting += IndexerDocumentWriting; - } } private static void RebuildIndexes(bool onlyEmptyIndexes) { - var indexers = (IEnumerable>)ExamineManager.Instance.IndexProviders; + var indexers = ExamineManager.Instance.IndexProviders; if (onlyEmptyIndexes) - indexers = indexers.Where(x => x.Value.IsIndexNew()); + indexers = indexers.Where(x => x.Value.IsIndexNew()).ToDictionary(x => x.Key, x => x.Value); foreach (var indexer in indexers) indexer.Value.RebuildIndex(); } @@ -115,7 +98,7 @@ namespace Umbraco.Web.Search { var indexes = indexCollection.Indexes; if (indexes == null) return; - foreach (var i in indexes.Values.OfType()) + foreach (var i in indexes.Values.OfType()) i.DocumentWriting += grid.DocumentWriting; } @@ -140,8 +123,7 @@ namespace Umbraco.Web.Search DeleteIndexForEntity((int)args.MessageObject, false); break; case MessageType.RefreshByInstance: - var c3 = args.MessageObject as IMember; - if (c3 != null) + if (args.MessageObject is IMember c3) { ReIndexForMember(c3); } @@ -150,8 +132,7 @@ namespace Umbraco.Web.Search // This is triggered when the item is permanently deleted - var c4 = args.MessageObject as IMember; - if (c4 != null) + if (args.MessageObject is IMember c4) { DeleteIndexForEntity(c4.Id, false); } @@ -313,13 +294,11 @@ namespace Umbraco.Web.Search private static void ReIndexForContent(IContent sender, bool? supportUnpublished = null) { - var xml = sender.ToXml(); - //add an icon attribute to get indexed - xml.Add(new XAttribute("icon", sender.ContentType.Icon)); + var valueSet = UmbracoContentIndexer.GetValueSets(Current.UrlSegmentProviders, Current.Services.UserService, sender); - ExamineManager.Instance.ReIndexNode( - xml, IndexTypes.Content, - ExamineManager.Instance.IndexProviderCollection.OfType() + ExamineManager.Instance.IndexItems( + valueSet.ToArray(), + ExamineManager.Instance.IndexProviders.OfType() // only for the specified indexers .Where(x => supportUnpublished.HasValue == false || supportUnpublished.Value == x.SupportUnpublishedContent) .Where(x => x.EnableDefaultEventHandler)); @@ -327,22 +306,22 @@ namespace Umbraco.Web.Search private static void ReIndexForMember(IMember member) { - ExamineManager.Instance.ReIndexNode( - member.ToXml(), IndexTypes.Member, - ExamineManager.Instance.IndexProviderCollection.OfType() + var valueSet = UmbracoMemberIndexer.GetValueSets(member); + + ExamineManager.Instance.IndexItems( + valueSet.ToArray(), + ExamineManager.Instance.IndexProviders.OfType() //ensure that only the providers are flagged to listen execute .Where(x => x.EnableDefaultEventHandler)); } private static void ReIndexForMedia(IMedia sender, bool isMediaPublished) { - var xml = sender.ToXml(); - //add an icon attribute to get indexed - xml.Add(new XAttribute("icon", sender.ContentType.Icon)); + var valueSet = UmbracoContentIndexer.GetValueSets(Current.UrlSegmentProviders, Current.Services.UserService, sender); - ExamineManager.Instance.ReIndexNode( - xml, IndexTypes.Media, - ExamineManager.Instance.IndexProviderCollection.OfType() + ExamineManager.Instance.IndexItems( + valueSet.ToArray(), + ExamineManager.Instance.IndexProviders.OfType() // index this item for all indexers if the media is not trashed, otherwise if the item is trashed // then only index this for indexers supporting unpublished media .Where(x => isMediaPublished || (x.SupportUnpublishedContent)) @@ -361,34 +340,12 @@ namespace Umbraco.Web.Search { ExamineManager.Instance.DeleteFromIndex( entityId.ToString(CultureInfo.InvariantCulture), - ExamineManager.Instance.IndexProviderCollection.OfType() + ExamineManager.Instance.IndexProviders.OfType() // if keepIfUnpublished == true then only delete this item from indexes not supporting unpublished content, // otherwise if keepIfUnpublished == false then remove from all indexes .Where(x => keepIfUnpublished == false || (x is UmbracoContentIndexer && ((UmbracoContentIndexer)x).SupportUnpublishedContent == false)) .Where(x => x.EnableDefaultEventHandler)); } - /// - /// Event handler to create a lower cased version of the node name, this is so we can support case-insensitive searching and still - /// use the Whitespace Analyzer - /// - /// - /// - - private static void IndexerDocumentWriting(object sender, DocumentWritingEventArgs e) - { - if (e.Fields.Keys.Contains("nodeName")) - { - //TODO: This logic should really be put into the content indexer instead of hidden here!! - - //add the lower cased version - e.Document.Add(new Field("__nodeName", - e.Fields["nodeName"].ToLower(), - Field.Store.YES, - Field.Index.ANALYZED, - Field.TermVector.NO - )); - } - } } } diff --git a/src/Umbraco.Web/Search/ExamineIndexerModel.cs b/src/Umbraco.Web/Search/ExamineIndexerModel.cs index c8cf2ff622..372f50764a 100644 --- a/src/Umbraco.Web/Search/ExamineIndexerModel.cs +++ b/src/Umbraco.Web/Search/ExamineIndexerModel.cs @@ -22,19 +22,7 @@ namespace Umbraco.Web.Search /// [DataMember(Name = "fieldCount")] public int FieldCount { get; set; } - - /// - /// The number of documents flagged for deletion in the index - /// - [DataMember(Name = "deletionCount")] - public int DeletionCount { get; set; } - - /// - /// Whether or not the indexed is optimized - /// - [DataMember(Name = "isOptimized")] - public bool IsOptimized{ get; set; } - + /// /// Generally will always be true unless someone has created a new non-lucene index /// diff --git a/src/Umbraco.Web/Search/LuceneIndexerExtensions.cs b/src/Umbraco.Web/Search/LuceneIndexerExtensions.cs index f333eca267..58f230b2cd 100644 --- a/src/Umbraco.Web/Search/LuceneIndexerExtensions.cs +++ b/src/Umbraco.Web/Search/LuceneIndexerExtensions.cs @@ -48,8 +48,9 @@ namespace Umbraco.Web.Search { try { - var searcher = indexer.GetSearcher().GetSearcher() as IndexSearcher; - if (searcher == null) return 0; + if (!((indexer.GetSearcher() as LuceneSearcher)?.GetLuceneSearcher() is IndexSearcher searcher)) + return 0; + using (searcher) using (var reader = searcher.IndexReader) { @@ -70,12 +71,11 @@ namespace Umbraco.Web.Search /// public static int GetIndexFieldCount(this LuceneIndexer indexer) { - //TODO: check for closing! and AlreadyClosedException - try { - var searcher = indexer.GetSearcher().GetSearcher() as IndexSearcher; - if (searcher == null) return 0; + if (!((indexer.GetSearcher() as LuceneSearcher)?.GetLuceneSearcher() is IndexSearcher searcher)) + return 0; + using (searcher) using (var reader = searcher.IndexReader) { @@ -89,30 +89,6 @@ namespace Umbraco.Web.Search } } - /// - /// Returns true if the index is optimized or not - /// - /// - /// - public static bool IsIndexOptimized(this LuceneIndexer indexer) - { - try - { - var searcher = indexer.GetSearcher().GetSearcher() as IndexSearcher; - if (searcher == null) return true; - using (searcher) - using (var reader = searcher.IndexReader) - { - return reader.IsOptimized(); - } - } - catch (AlreadyClosedException) - { - Current.Logger.Warn(typeof(ExamineExtensions), "Cannot get IsIndexOptimized, the writer is already closed"); - return false; - } - } - /// /// Check if the index is locked /// @@ -136,8 +112,9 @@ namespace Umbraco.Web.Search { try { - var searcher = indexer.GetSearcher().GetSearcher() as IndexSearcher; - if (searcher == null) return 0; + if (!((indexer.GetSearcher() as LuceneSearcher)?.GetLuceneSearcher() is IndexSearcher searcher)) + return 0; + using (searcher) using (var reader = searcher.IndexReader) { diff --git a/src/Umbraco.Web/Search/UmbracoTreeSearcher.cs b/src/Umbraco.Web/Search/UmbracoTreeSearcher.cs index 917df00c4c..d82cdf8fe2 100644 --- a/src/Umbraco.Web/Search/UmbracoTreeSearcher.cs +++ b/src/Umbraco.Web/Search/UmbracoTreeSearcher.cs @@ -180,7 +180,7 @@ namespace Umbraco.Web.Search sb.Append("+__IndexType:"); sb.Append(type); - var raw = internalSearcher.CreateSearchCriteria().RawQuery(sb.ToString()); + var raw = internalSearcher.CreateCriteria().RawQuery(sb.ToString()); var result = internalSearcher //only return the number of items specified to read up to the amount of records to fill from 0 -> the number of items on the page requested @@ -273,10 +273,10 @@ namespace Umbraco.Web.Search m.Icon = "icon-user"; } - var searchResult = results.First(x => x.Id.ToInvariantString() == m.Id.ToString()); + var searchResult = results.First(x => x.Id == m.Id.ToString()); if (searchResult.Fields.ContainsKey("email") && searchResult.Fields["email"] != null) { - m.AdditionalData["Email"] = results.First(x => x.Id.ToInvariantString() == m.Id.ToString()).Fields["email"]; + m.AdditionalData["Email"] = results.First(x => x.Id == m.Id.ToString()).Fields["email"]; } if (searchResult.Fields.ContainsKey("__key") && searchResult.Fields["__key"] != null) { diff --git a/src/Umbraco.Web/Strategies/DatabaseServerRegistrarAndMessengerComponent.cs b/src/Umbraco.Web/Strategies/DatabaseServerRegistrarAndMessengerComponent.cs index 14f83c44eb..75ad252640 100644 --- a/src/Umbraco.Web/Strategies/DatabaseServerRegistrarAndMessengerComponent.cs +++ b/src/Umbraco.Web/Strategies/DatabaseServerRegistrarAndMessengerComponent.cs @@ -95,7 +95,7 @@ namespace Umbraco.Web.Strategies // fixme - this should move to something else, we should not depend on Examine here! private static void RebuildIndexes(bool onlyEmptyIndexes) { - var indexers = (IEnumerable>) ExamineManager.Instance.IndexProviders; + var indexers = (IEnumerable>) ExamineManager.Instance.IndexProviders; if (onlyEmptyIndexes) indexers = indexers.Where(x => x.Value.IsIndexNew()); foreach (var indexer in indexers) diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index 5551354d56..e44f2e715d 100644 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -66,7 +66,7 @@ - + diff --git a/src/Umbraco.Web/WebServices/ExamineManagementApiController.cs b/src/Umbraco.Web/WebServices/ExamineManagementApiController.cs index ab07429223..58d854524d 100644 --- a/src/Umbraco.Web/WebServices/ExamineManagementApiController.cs +++ b/src/Umbraco.Web/WebServices/ExamineManagementApiController.cs @@ -23,10 +23,16 @@ namespace Umbraco.Web.WebServices [ValidateAngularAntiForgeryToken] public class ExamineManagementApiController : UmbracoAuthorizedApiController { - //TODO: Fix all of this for searchers/indexers that are not configured via code (i.e. the core ones once we do that) - // We will need to be able to search an index directly without having to go through all of the searchers + public ExamineManagementApiController(ExamineManager examineManager, ILogger logger, IRuntimeCacheProvider runtimeCacheProvider) + { + _examineManager = examineManager; + _logger = logger; + _runtimeCacheProvider = runtimeCacheProvider; + } - private ExamineManager _examineManager; + private readonly ExamineManager _examineManager; + private readonly ILogger _logger; + private readonly IRuntimeCacheProvider _runtimeCacheProvider; /// /// Checks if the member internal index is consistent with the data stored in the database @@ -37,8 +43,8 @@ namespace Umbraco.Web.WebServices { var total = Services.MemberService.Count(); - var searcher = _examineManager.GetSearcher(Constants.Examine.InternalMemberIndexer); - var criteria = searcher.CreateSearchCriteria().RawQuery("__IndexType:member"); + var searcher = _examineManager.GetIndexSearcher(Constants.Examine.InternalMemberIndexer); + var criteria = searcher.CreateCriteria().RawQuery("__IndexType:member"); var totalIndexed = searcher.Search(criteria); return total == totalIndexed.TotalItemCount; } @@ -52,8 +58,8 @@ namespace Umbraco.Web.WebServices { var total = Services.MediaService.Count(); - var searcher = _examineManager.GetSearcher(Constants.Examine.InternalIndexer); - var criteria = searcher.CreateSearchCriteria().RawQuery("__IndexType:media"); + var searcher = _examineManager.GetIndexSearcher(Constants.Examine.InternalIndexer); + var criteria = searcher.CreateCriteria().RawQuery("__IndexType:media"); var totalIndexed = searcher.Search(criteria); return total == totalIndexed.TotalItemCount; } @@ -67,8 +73,8 @@ namespace Umbraco.Web.WebServices { var total = Services.ContentService.Count(); - var searcher = _examineManager.GetSearcher(Constants.Examine.InternalIndexer); - var criteria = searcher.CreateSearchCriteria().RawQuery("__IndexType:content"); + var searcher = _examineManager.GetIndexSearcher(Constants.Examine.InternalIndexer); + var criteria = searcher.CreateCriteria().RawQuery("__IndexType:content"); var totalIndexed = searcher.Search(criteria); return total == totalIndexed.TotalItemCount; } @@ -93,16 +99,19 @@ namespace Umbraco.Web.WebServices public IEnumerable GetSearcherDetails() { var model = new List( - ExamineManager.Instance.SearchProviderCollection.Cast().Select(searcher => + ExamineManager.Instance.IndexProviders.Select(indexer => { + var searcher = indexer.Value.GetSearcher(); + var searcherName = (searcher as BaseLuceneSearcher)?.Name ?? string.Concat(indexer.Key, "Searcher"); + var indexerModel = new ExamineSearcherModel() { - Name = searcher.Name + Name = searcherName }; var props = TypeHelper.CachedDiscoverableProperties(searcher.GetType(), mustWrite: false) //ignore these properties - .Where(x => new[] {"Description"}.InvariantContains(x.Name) == false) - .OrderBy(x => x.Name); + .Where(x => new[] {"Description"}.InvariantContains(x.Name) == false) + .OrderBy(x => x.Name); foreach (var p in props) { indexerModel.ProviderProperties.Add(p.Name, p.GetValue(searcher, null).ToString()); @@ -116,7 +125,7 @@ namespace Umbraco.Web.WebServices return model; } - public ILuceneSearchResults GetSearchResults(string searcherName, string query, string queryType) + public ISearchResults GetSearchResults(string searcherName, string query, string queryType) { if (queryType == null) { @@ -127,48 +136,22 @@ namespace Umbraco.Web.WebServices if (query.IsNullOrWhiteSpace()) return LuceneSearchResults.Empty(); - LuceneSearcher searcher; - var msg = ValidateLuceneSearcher(searcherName, out searcher); + var msg = ValidateLuceneSearcher(searcherName, out var searcher); if (msg.IsSuccessStatusCode) { if (queryType.InvariantEquals("text")) { - return searcher.Find(query, false); + return searcher.Search(query, false); } if (queryType.InvariantEquals("lucene")) { - return searcher.Find(searcher.CreateCriteria().RawQuery(query)); + return searcher.Search(searcher.CreateCriteria().RawQuery(query)); } throw new HttpResponseException(Request.CreateResponse(HttpStatusCode.NotFound)); } throw new HttpResponseException(msg); } - /// - /// Optimizes an index - /// - public HttpResponseMessage PostOptimizeIndex(string indexerName) - { - IExamineIndexer indexer; - var msg = ValidateLuceneIndexer(indexerName, out indexer); - var luceneIndexer = indexer as LuceneIndexer; - if (luceneIndexer != null && msg.IsSuccessStatusCode) - { - try - { - luceneIndexer.OptimizeIndex(); - } - catch (Exception ex) - { - var response = Request.CreateResponse(HttpStatusCode.Conflict); - response.Content = new StringContent(string.Format("The index could not be optimized, most likely there is another thread currently writing to the index. Error: {0}", ex)); - response.ReasonPhrase = "Could Not Optimize"; - return response; - } - } - return msg; - } - /// /// Rebuilds the index /// @@ -176,11 +159,10 @@ namespace Umbraco.Web.WebServices /// public HttpResponseMessage PostRebuildIndex(string indexerName) { - LuceneIndexer indexer; - var msg = ValidateLuceneIndexer(indexerName, out indexer); + var msg = ValidateLuceneIndexer(indexerName, out LuceneIndexer indexer); if (msg.IsSuccessStatusCode) { - Current.Logger.Info(string.Format("Rebuilding index '{0}'", indexerName)); + _logger.Info($"Rebuilding index '{indexerName}'"); //remove it in case there's a handler there alraedy indexer.IndexOperationComplete -= Indexer_IndexOperationComplete; @@ -201,7 +183,7 @@ namespace Umbraco.Web.WebServices indexer.IndexOperationComplete -= Indexer_IndexOperationComplete; Logger.Error("An error occurred rebuilding index", ex); var response = Request.CreateResponse(HttpStatusCode.Conflict); - response.Content = new StringContent(string.Format("The index could not be rebuilt at this time, most likely there is another thread currently writing to the index. Error: {0}", ex)); + response.Content = new StringContent($"The index could not be rebuilt at this time, most likely there is another thread currently writing to the index. Error: {ex}"); response.ReasonPhrase = "Could Not Rebuild"; return response; } @@ -210,17 +192,17 @@ namespace Umbraco.Web.WebServices } //static listener so it's not GC'd - private static void Indexer_IndexOperationComplete(object sender, EventArgs e) + private void Indexer_IndexOperationComplete(object sender, EventArgs e) { var indexer = (LuceneIndexer) sender; //ensure it's not listening anymore indexer.IndexOperationComplete -= Indexer_IndexOperationComplete; - Current.Logger.Info($"Rebuilding index '{indexer.Name}' done, {indexer.CommitCount} items committed (can differ from the number of items in the index)"); + _logger.Info($"Rebuilding index '{indexer.Name}' done, {indexer.CommitCount} items committed (can differ from the number of items in the index)"); var cacheKey = "temp_indexing_op_" + indexer.Name; - Current.ApplicationCache.RuntimeCache.ClearCacheItem(cacheKey); + _runtimeCacheProvider.ClearCacheItem(cacheKey); } /// @@ -234,8 +216,7 @@ namespace Umbraco.Web.WebServices /// public ExamineIndexerModel PostCheckRebuildIndex(string indexerName) { - LuceneIndexer indexer; - var msg = ValidateLuceneIndexer(indexerName, out indexer); + var msg = ValidateLuceneIndexer(indexerName, out LuceneIndexer indexer); if (msg.IsSuccessStatusCode) { var cacheKey = "temp_indexing_op_" + indexerName; @@ -243,42 +224,23 @@ namespace Umbraco.Web.WebServices //if its still there then it's not done return found != null ? null - : CreateModel(new KeyValuePair(indexerName, indexer)); + : CreateModel(new KeyValuePair(indexerName, indexer)); } throw new HttpResponseException(msg); } - /// - /// Checks if the index is optimized - /// - /// - /// - public ExamineIndexerModel PostCheckOptimizeIndex(string indexerName) - { - LuceneIndexer indexer; - var msg = ValidateLuceneIndexer(indexerName, out indexer); - if (msg.IsSuccessStatusCode) - { - var isOptimized = indexer.IsIndexOptimized(); - return isOptimized == false - ? null - : CreateModel(new KeyValuePair(indexerName, indexer)); - } - throw new HttpResponseException(msg); - } - - private ExamineIndexerModel CreateModel(KeyValuePair indexer) + private ExamineIndexerModel CreateModel(KeyValuePair indexer) { var indexerModel = new ExamineIndexerModel() { - FieldDefinitions = indexer.Value.FieldDefinitions, + FieldDefinitions = indexer.Value.FieldDefinitionCollection, Name = indexer.Key }; var props = TypeHelper.CachedDiscoverableProperties(indexer.GetType(), mustWrite: false) //ignore these properties - .Where(x => new[] {"IndexerData", "Description", "WorkingFolder"}.InvariantContains(x.Name) == false) - .OrderBy(x => x.Name); + .Where(x => new[] {"IndexerData", "Description", "WorkingFolder"}.InvariantContains(x.Name) == false) + .OrderBy(x => x.Name); foreach (var p in props) { @@ -294,15 +256,13 @@ namespace Umbraco.Web.WebServices indexerModel.ProviderProperties.Add(p.Name, val.ToString()); } - var luceneIndexer = indexer.Value as LuceneIndexer; - if (luceneIndexer != null) + if (indexer.Value is LuceneIndexer luceneIndexer) { indexerModel.IsLuceneIndex = true; if (luceneIndexer.IndexExists()) { - Exception indexError; - indexerModel.IsHealthy = luceneIndexer.IsHealthy(out indexError); + indexerModel.IsHealthy = luceneIndexer.IsHealthy(out var indexError); if (indexerModel.IsHealthy == false) { @@ -313,15 +273,11 @@ namespace Umbraco.Web.WebServices indexerModel.DocumentCount = luceneIndexer.GetIndexDocumentCount(); indexerModel.FieldCount = luceneIndexer.GetIndexFieldCount(); - indexerModel.IsOptimized = luceneIndexer.IsIndexOptimized(); - indexerModel.DeletionCount = luceneIndexer.GetDeletedDocumentsCount(); } else { indexerModel.DocumentCount = 0; indexerModel.FieldCount = 0; - indexerModel.IsOptimized = true; - indexerModel.DeletionCount = 0; } } return indexerModel; @@ -329,30 +285,33 @@ namespace Umbraco.Web.WebServices private HttpResponseMessage ValidateLuceneSearcher(string searcherName, out LuceneSearcher searcher) { - if (ExamineManager.Instance.SearchProviderCollection.Cast().Any(x => x.Name == searcherName)) + foreach (var indexer in ExamineManager.Instance.IndexProviders) { - searcher = ExamineManager.Instance.SearchProviderCollection[searcherName] as LuceneSearcher; - if (searcher == null) - { - var response = Request.CreateResponse(HttpStatusCode.BadRequest); - response.Content = new StringContent(string.Format("The searcher {0} is not of type {1}", searcherName, typeof(LuceneSearcher))); - response.ReasonPhrase = "Wrong Searcher Type"; - return response; - } - //return Ok! - return Request.CreateResponse(HttpStatusCode.OK); + var s = indexer.Value.GetSearcher(); + var sName = (s as BaseLuceneSearcher)?.Name ?? string.Concat(indexer.Key, "Searcher"); + if (sName != searcherName) continue; + searcher = s as LuceneSearcher; + + //Found it, return OK + if (searcher != null) return Request.CreateResponse(HttpStatusCode.OK); + + //Return an error since it's not the right type + var response = Request.CreateResponse(HttpStatusCode.BadRequest); + response.Content = new StringContent($"The searcher {searcherName} is not of type {typeof(LuceneSearcher)}"); + response.ReasonPhrase = "Wrong Searcher Type"; + return response; } searcher = null; var response1 = Request.CreateResponse(HttpStatusCode.BadRequest); - response1.Content = new StringContent(string.Format("No searcher found with name = {0}", searcherName)); + response1.Content = new StringContent($"No searcher found with name = {searcherName}"); response1.ReasonPhrase = "Searcher Not Found"; return response1; } private HttpResponseMessage ValidateLuceneIndexer(string indexerName, out T indexer) - where T : class, IExamineIndexer + where T : class, IIndexer { indexer = null; @@ -363,7 +322,7 @@ namespace Umbraco.Web.WebServices } var response = Request.CreateResponse(HttpStatusCode.BadRequest); - response.Content = new StringContent(string.Format("No indexer found with name = {0}", indexerName)); + response.Content = new StringContent($"No indexer found with name = {indexerName}"); response.ReasonPhrase = "Indexer Not Found"; return response; } diff --git a/src/Umbraco.Web/umbraco.presentation/umbraco/dialogs/search.aspx.cs b/src/Umbraco.Web/umbraco.presentation/umbraco/dialogs/search.aspx.cs index 20a4e2ba54..a307d8de9b 100644 --- a/src/Umbraco.Web/umbraco.presentation/umbraco/dialogs/search.aspx.cs +++ b/src/Umbraco.Web/umbraco.presentation/umbraco/dialogs/search.aspx.cs @@ -16,6 +16,7 @@ using Umbraco.Core.Xml; namespace umbraco.presentation.dialogs { + //fixme - is this even used anymore? public partial class search : Umbraco.Web.UI.Pages.UmbracoEnsuredPage { @@ -63,7 +64,7 @@ namespace umbraco.presentation.dialogs : ExamineManager.Instance.SearchProviderCollection["InternalSearcher"]; //create some search criteria, make everything combined to be 'And' and only search the current app - var criteria = internalSearcher.CreateSearchCriteria(CurrentApp, Examine.SearchCriteria.BooleanOperation.And); + var criteria = internalSearcher.CreateCriteria(CurrentApp, Examine.SearchCriteria.BooleanOperation.And); IEnumerable results; if (txt.StartsWith("*"))