diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index e165e4ee1a..8f828ade1f 100644 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -42,7 +42,7 @@ true - bin\ + bin\Debug\ false 285212672 false @@ -66,14 +66,14 @@ AllRules.ruleset - bin\ + bin\Release\ false 285212672 false TRACE - bin\umbraco.xml + bin\Release\umbraco.xml false 4096 false diff --git a/src/UmbracoExamine/BaseUmbracoIndexer.cs b/src/UmbracoExamine/BaseUmbracoIndexer.cs new file mode 100644 index 0000000000..542706a99b --- /dev/null +++ b/src/UmbracoExamine/BaseUmbracoIndexer.cs @@ -0,0 +1,349 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Security; +using System.Text; +using System.Threading; +using System.Web; +using Examine.LuceneEngine.Providers; +using Lucene.Net.Analysis; +using umbraco.BasePages; +using umbraco.BusinessLogic; +using UmbracoExamine.DataServices; +using Examine; +using System.IO; +using System.Xml.Linq; + +namespace UmbracoExamine +{ + + /// + /// An abstract provider containing the basic functionality to be able to query against + /// Umbraco data. + /// + public abstract class BaseUmbracoIndexer : LuceneIndexer + { + #region Constructors + + /// + /// Default constructor + /// + protected BaseUmbracoIndexer() + : base() + { + } + + /// + /// Constructor to allow for creating an indexer at runtime + /// + /// + /// + /// + /// + [SecuritySafeCritical] + protected BaseUmbracoIndexer(IIndexCriteria indexerData, DirectoryInfo indexPath, IDataService dataService, Analyzer analyzer, bool async) + : base(indexerData, indexPath, analyzer, async) + { + DataService = dataService; + } + + [SecuritySafeCritical] + protected BaseUmbracoIndexer(IIndexCriteria indexerData, Lucene.Net.Store.Directory luceneDirectory, IDataService dataService, Analyzer analyzer, bool async) + : base(indexerData, luceneDirectory, analyzer, async) + { + DataService = dataService; + } + + #endregion + + #region Properties + + /// + /// If true, the IndexingActionHandler will be run to keep the default index up to date. + /// + public bool EnableDefaultEventHandler { get; protected set; } + + /// + /// Determines if the manager will call the indexing methods when content is saved or deleted as + /// opposed to cache being updated. + /// + public bool SupportUnpublishedContent { get; protected set; } + + /// + /// The data service used for retreiving and submitting data to the cms + /// + public IDataService DataService { get; protected internal set; } + + /// + /// the supported indexable types + /// + protected abstract IEnumerable SupportedTypes { get; } + + #endregion + + #region Initialize + + + /// + /// Setup the properties for the indexer from the provider settings + /// + /// + /// + public override void Initialize(string name, System.Collections.Specialized.NameValueCollection config) + { + if (config["dataService"] != null && !string.IsNullOrEmpty(config["dataService"])) + { + //this should be a fully qualified type + var serviceType = Type.GetType(config["dataService"]); + DataService = (IDataService)Activator.CreateInstance(serviceType); + } + else if (DataService == null) + { + //By default, we will be using the UmbracoDataService + //generally this would only need to be set differently for unit testing + DataService = new UmbracoDataService(); + } + + DataService.LogService.LogLevel = LoggingLevel.Normal; + + if (config["logLevel"] != null && !string.IsNullOrEmpty(config["logLevel"])) + { + try + { + var logLevel = (LoggingLevel)Enum.Parse(typeof(LoggingLevel), config["logLevel"]); + DataService.LogService.LogLevel = logLevel; + } + catch (ArgumentException) + { + //FAILED + DataService.LogService.LogLevel = LoggingLevel.Normal; + } + } + + DataService.LogService.ProviderName = name; + + EnableDefaultEventHandler = true; //set to true by default + bool enabled; + if (bool.TryParse(config["enableDefaultEventHandler"], out enabled)) + { + EnableDefaultEventHandler = enabled; + } + + DataService.LogService.AddVerboseLog(-1, string.Format("{0} indexer initializing", name)); + + base.Initialize(name, config); + } + + #endregion + + //public override void RebuildIndex() + //{ + // //we can make the indexing rebuilding operation happen asynchronously in a web context by calling an http handler. + // //we should only do this when async='true', the current request is running in a web context and the current user is authenticated. + // if (RunAsync && HttpContext.Current != null) + // { + // if (UmbracoEnsuredPage.CurrentUser != null) + // { + // RebuildIndexAsync(); + // } + // else + // { + // //don't rebuild, user is not authenticated and if async is set then we shouldn't be generating the index files non-async either + // } + // } + // else + // { + // base.RebuildIndex(); + // } + //} + + #region Protected + + ///// + ///// Calls a web request in a worker thread to rebuild the indexes + ///// + //protected void RebuildIndexAsync() + //{ + // if (HttpContext.Current != null && UmbracoEnsuredPage.CurrentUser != null) + // { + // var handler = VirtualPathUtility.ToAbsolute(ExamineHandlerPath); + // var fullPath = HttpContext.Current.Request.Url.GetLeftPart(UriPartial.Authority) + handler + "?index=" + Name; + // var userContext = BasePage.umbracoUserContextID; + // var userContextCookie = HttpContext.Current.Request.Cookies["UserContext"]; + // var thread = new Thread(() => + // { + // var request = (HttpWebRequest)WebRequest.Create(fullPath); + // request.CookieContainer = new CookieContainer(); + // request.CookieContainer.Add(new Cookie("UserContext", userContext, userContextCookie.Path, + // string.IsNullOrEmpty(userContextCookie.Domain) ? "localhost" : userContextCookie.Domain)); + // request.Timeout = Timeout.Infinite; + // request.UseDefaultCredentials = true; + // request.Method = "GET"; + // request.Proxy = null; + + // HttpWebResponse response; + // try + // { + // response = (HttpWebResponse)request.GetResponse(); + + // if (response.StatusCode != HttpStatusCode.OK) + // { + // Log.Add(LogTypes.Custom, -1, "[UmbracoExamine] ExamineHandler request ended with an error: " + response.StatusDescription); + // } + // } + // catch (WebException ex) + // { + // Log.Add(LogTypes.Custom, -1, "[UmbracoExamine] ExamineHandler request threw an exception: " + ex.Message); + // } + + // }) { IsBackground = true, Name = "ExamineAsyncHandler" }; + + // thread.Start(); + // } + //} + + /// + /// Ensures that the node being indexed is of a correct type and is a descendent of the parent id specified. + /// + /// + /// + protected override bool ValidateDocument(XElement node) + { + //check if this document is a descendent of the parent + if (IndexerData.ParentNodeId.HasValue && IndexerData.ParentNodeId.Value > 0) + if (!((string)node.Attribute("path")).Contains("," + IndexerData.ParentNodeId.Value.ToString() + ",")) + return false; + + return base.ValidateDocument(node); + } + + /// + /// Reindexes all supported types + /// + protected override void PerformIndexRebuild() + { + foreach (var t in SupportedTypes) + { + IndexAll(t); + } + } + + public override void ReIndexNode(XElement node, string type) + { + if (!SupportedTypes.Contains(type)) + return; + + base.ReIndexNode(node, type); + } + + /// + /// Builds an xpath statement to query against Umbraco data for the index type specified, then + /// initiates the re-indexing of the data matched. + /// + /// + protected override void PerformIndexAll(string type) + { + if (!SupportedTypes.Contains(type)) + return; + + var xPath = "//*[(number(@id) > 0 and (@isDoc or @nodeTypeAlias)){0}]"; //we'll add more filters to this below if needed + + var sb = new StringBuilder(); + + //create the xpath statement to match node type aliases if specified + if (IndexerData.IncludeNodeTypes.Count() > 0) + { + sb.Append("("); + foreach (var field in IndexerData.IncludeNodeTypes) + { + //this can be used across both schemas + const string nodeTypeAlias = "(@nodeTypeAlias='{0}' or (count(@nodeTypeAlias)=0 and name()='{0}'))"; + + sb.Append(string.Format(nodeTypeAlias, field)); + sb.Append(" or "); + } + sb.Remove(sb.Length - 4, 4); //remove last " or " + sb.Append(")"); + } + + //create the xpath statement to match all children of the current node. + if (IndexerData.ParentNodeId.HasValue && IndexerData.ParentNodeId.Value > 0) + { + if (sb.Length > 0) + sb.Append(" and "); + sb.Append("("); + sb.Append("contains(@path, '," + IndexerData.ParentNodeId.Value + ",')"); //if the path contains comma - id - comma then the nodes must be a child + sb.Append(")"); + } + + //create the full xpath statement to match the appropriate nodes. If there is a filter + //then apply it, otherwise just select all nodes. + var filter = sb.ToString(); + xPath = string.Format(xPath, filter.Length > 0 ? " and " + filter : ""); + + //raise the event and set the xpath statement to the value returned + var args = new IndexingNodesEventArgs(IndexerData, xPath, type); + OnNodesIndexing(args); + if (args.Cancel) + { + return; + } + + xPath = args.XPath; + + DataService.LogService.AddVerboseLog(-1, string.Format("({0}) PerformIndexAll with XPATH: {1}", this.Name, xPath)); + + AddNodesToIndex(xPath, type); + } + + /// + /// Returns an XDocument for the entire tree stored for the IndexType specified. + /// + /// The xpath to the node. + /// The type of data to request from the data service. + /// Either the Content or Media xml. If the type is not of those specified null is returned + protected virtual XDocument GetXDocument(string xPath, string type) + { + if (type == IndexTypes.Content) + { + if (this.SupportUnpublishedContent) + { + return DataService.ContentService.GetLatestContentByXPath(xPath); + } + else + { + return DataService.ContentService.GetPublishedContentByXPath(xPath); + } + } + else if (type == IndexTypes.Media) + { + return DataService.MediaService.GetLatestMediaByXpath(xPath); + } + return null; + } + #endregion + + #region Private + /// + /// Adds all nodes with the given xPath root. + /// + /// The x path. + /// The type. + private void AddNodesToIndex(string xPath, string type) + { + // Get all the nodes of nodeTypeAlias == nodeTypeAlias + XDocument xDoc = GetXDocument(xPath, type); + if (xDoc != null) + { + XElement rootNode = xDoc.Root; + + IEnumerable children = rootNode.Elements(); + + AddNodesToIndex(children, type); + } + + } + #endregion + } +} diff --git a/src/UmbracoExamine/Config/IndexSetExtensions.cs b/src/UmbracoExamine/Config/IndexSetExtensions.cs new file mode 100644 index 0000000000..1a3fe68866 --- /dev/null +++ b/src/UmbracoExamine/Config/IndexSetExtensions.cs @@ -0,0 +1,64 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Examine; +using UmbracoExamine.DataServices; +using Examine.LuceneEngine.Config; + +namespace UmbracoExamine.Config +{ + /// + /// Extension methods for IndexSet + /// + public static class IndexSetExtensions + { + + private static readonly object Locker = new object(); + + /// + /// Convert the indexset to indexerdata. + /// This detects if there are no user/system fields specified and if not, uses the data service to look them + /// up and update the in memory IndexSet. + /// + /// + /// + /// + public static IIndexCriteria ToIndexCriteria(this IndexSet set, IDataService svc) + { + if (set.IndexUserFields.Count == 0) + { + lock (Locker) + { + //we need to add all user fields to the collection if it is empty (this is the default if none are specified) + var userFields = svc.ContentService.GetAllUserPropertyNames(); + foreach (var u in userFields) + { + set.IndexUserFields.Add(new IndexField() { Name = u }); + } + } + } + + if (set.IndexAttributeFields.Count == 0) + { + lock (Locker) + { + //we need to add all system fields to the collection if it is empty (this is the default if none are specified) + var sysFields = svc.ContentService.GetAllSystemPropertyNames(); + foreach (var s in sysFields) + { + set.IndexAttributeFields.Add(new IndexField() { Name = s }); + } + } + } + + return new IndexCriteria( + set.IndexAttributeFields.Cast().ToArray(), + set.IndexUserFields.Cast().ToArray(), + set.IncludeNodeTypes.ToList().Select(x => x.Name).ToArray(), + set.ExcludeNodeTypes.ToList().Select(x => x.Name).ToArray(), + set.IndexParentId); + } + + } +} diff --git a/src/UmbracoExamine/ContentExtensions.cs b/src/UmbracoExamine/ContentExtensions.cs new file mode 100644 index 0000000000..b2d82375ed --- /dev/null +++ b/src/UmbracoExamine/ContentExtensions.cs @@ -0,0 +1,72 @@ +using System.Collections.Generic; +using System.Linq; +using System.Security; +using System.Xml; +using System.Xml.Linq; +using System.Xml.XPath; +using umbraco; +using umbraco.cms.businesslogic; +using umbraco.cms.businesslogic.web; +using Examine.LuceneEngine; + +namespace UmbracoExamine +{ + /// + /// Static methods to help query umbraco xml + /// + public static class ContentExtensions + { + + /// + /// Converts a content node to XDocument + /// + /// + /// true if data is going to be returned from cache + /// + /// + /// If the type of node is not a Document, the cacheOnly has no effect, it will use the API to return + /// the xml. + /// + [SecuritySafeCritical] + public static XDocument ToXDocument(this Content node, bool cacheOnly) + { + if (cacheOnly && node.GetType().Equals(typeof(Document))) + { + var umbXml = library.GetXmlNodeById(node.Id.ToString()); + if (umbXml != null) + { + return umbXml.ToXDocument(); + } + } + + //this will also occur if umbraco hasn't cached content yet.... + + //if it's not a using cache and it's not cacheOnly, then retrieve the Xml using the API + return node.ToXDocument(); + } + + /// + /// Converts a content node to Xml + /// + /// + /// + [SecuritySafeCritical] + private static XDocument ToXDocument(this Content node) + { + var xDoc = new XmlDocument(); + var xNode = xDoc.CreateNode(XmlNodeType.Element, "node", ""); + node.XmlPopulate(xDoc, ref xNode, false); + + if (xNode.Attributes["nodeTypeAlias"] == null) + { + //we'll add the nodeTypeAlias ourselves + XmlAttribute d = xDoc.CreateAttribute("nodeTypeAlias"); + d.Value = node.ContentType.Alias; + xNode.Attributes.Append(d); + } + + return new XDocument(xNode.ToXElement()); + } + + } +} diff --git a/src/UmbracoExamine/DataServices/IContentService.cs b/src/UmbracoExamine/DataServices/IContentService.cs new file mode 100644 index 0000000000..502938fbd2 --- /dev/null +++ b/src/UmbracoExamine/DataServices/IContentService.cs @@ -0,0 +1,28 @@ +using System; +using System.Linq; +using System.Xml.Linq; +using System.Collections.Generic; + +namespace UmbracoExamine.DataServices +{ + public interface IContentService + { + XDocument GetLatestContentByXPath(string xpath); + XDocument GetPublishedContentByXPath(string xpath); + + /// + /// Returns a list of ALL properties names for all nodes defined in the data source + /// + /// + IEnumerable GetAllUserPropertyNames(); + + /// + /// Returns a list of ALL system property names for all nodes defined in the data source + /// + /// + IEnumerable GetAllSystemPropertyNames(); + + string StripHtml(string value); + bool IsProtected(int nodeId, string path); + } +} diff --git a/src/UmbracoExamine/DataServices/IDataService.cs b/src/UmbracoExamine/DataServices/IDataService.cs new file mode 100644 index 0000000000..190e195236 --- /dev/null +++ b/src/UmbracoExamine/DataServices/IDataService.cs @@ -0,0 +1,14 @@ +using System.Web; + + +namespace UmbracoExamine.DataServices +{ + public interface IDataService + { + IContentService ContentService { get; } + ILogService LogService { get; } + IMediaService MediaService { get; } + + string MapPath(string virtualPath); + } +} \ No newline at end of file diff --git a/src/UmbracoExamine/DataServices/ILogService.cs b/src/UmbracoExamine/DataServices/ILogService.cs new file mode 100644 index 0000000000..fb0e9dcc5d --- /dev/null +++ b/src/UmbracoExamine/DataServices/ILogService.cs @@ -0,0 +1,12 @@ +using System; +namespace UmbracoExamine.DataServices +{ + public interface ILogService + { + string ProviderName { get; set; } + void AddErrorLog(int nodeId, string msg); + void AddInfoLog(int nodeId, string msg); + void AddVerboseLog(int nodeId, string msg); + LoggingLevel LogLevel { get; set; } + } +} diff --git a/src/UmbracoExamine/DataServices/IMediaService.cs b/src/UmbracoExamine/DataServices/IMediaService.cs new file mode 100644 index 0000000000..88c07a281c --- /dev/null +++ b/src/UmbracoExamine/DataServices/IMediaService.cs @@ -0,0 +1,9 @@ +using System; +using System.Xml.Linq; +namespace UmbracoExamine.DataServices +{ + public interface IMediaService + { + XDocument GetLatestMediaByXpath(string xpath); + } +} diff --git a/src/UmbracoExamine/DataServices/UmbracoContentService.cs b/src/UmbracoExamine/DataServices/UmbracoContentService.cs new file mode 100644 index 0000000000..6ce1b3695e --- /dev/null +++ b/src/UmbracoExamine/DataServices/UmbracoContentService.cs @@ -0,0 +1,164 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Security; +using System.Text; +using umbraco; +using System.Xml.Linq; +using System.Xml; +using umbraco.cms.businesslogic.web; +using System.Collections; +using System.Xml.XPath; +using umbraco.DataLayer; +using umbraco.BusinessLogic; +using UmbracoExamine.Config; +using Examine.LuceneEngine; +using System.Data.SqlClient; +using System.Diagnostics; + +namespace UmbracoExamine.DataServices +{ + public class UmbracoContentService : UmbracoExamine.DataServices.IContentService + { + + /// + /// removes html markup from a string + /// + /// + /// + [SecuritySafeCritical] + public string StripHtml(string value) + { + return library.StripHtml(value); + } + + /// + /// Gets published content by xpath + /// + /// + /// + [SecuritySafeCritical] + public XDocument GetPublishedContentByXPath(string xpath) + { + return library.GetXmlNodeByXPath(xpath).ToXDocument(); + } + + /// + /// This is quite an intensive operation... + /// get all root content, then get the XML structure for all children, + /// then run xpath against the navigator that's created + /// + /// + /// + [SecuritySafeCritical] + public XDocument GetLatestContentByXPath(string xpath) + { + + var rootContent = Document.GetRootDocuments(); + var xmlContent = XDocument.Parse(""); + var xDoc = new XmlDocument(); + foreach (var c in rootContent) + { + var xNode = xDoc.CreateNode(XmlNodeType.Element, "node", ""); + c.XmlPopulate(xDoc, ref xNode, true); + + if (xNode.Attributes["nodeTypeAlias"] == null) + { + //we'll add the nodeTypeAlias ourselves + XmlAttribute d = xDoc.CreateAttribute("nodeTypeAlias"); + d.Value = c.ContentType.Alias; + xNode.Attributes.Append(d); + } + + xmlContent.Root.Add(xNode.ToXElement()); + } + var result = ((IEnumerable)xmlContent.XPathEvaluate(xpath)).Cast(); + return result.ToXDocument(); + } + + /// + /// Unfortunately, we need to implement our own IsProtected method since + /// the Umbraco core code requires an HttpContext for this method and when we're running + /// async, there is no context + /// + /// + /// + [SecuritySafeCritical] + private XmlNode GetPage(int documentId) + { + XmlNode x = Access.AccessXml.SelectSingleNode("/access/page [@id=" + documentId.ToString() + "]"); + return x; + } + + /// + /// Unfortunately, we need to implement our own IsProtected method since + /// the Umbraco core code requires an HttpContext for this method and when we're running + /// async, there is no context + /// + /// + /// + /// + public bool IsProtected(int nodeId, string path) + { + foreach (string id in path.Split(',')) + { + if (GetPage(int.Parse(id)) != null) + { + return true; + } + } + return false; + } + + /// + /// Returns a list of all of the user defined property names in Umbraco + /// + /// + [SecuritySafeCritical] + public IEnumerable GetAllUserPropertyNames() + { + //this is how umb codebase 4.0 does this... booo, should be in the data layer, will fix in 4.1 + + var aliases = new List(); + var fieldSql = "select distinct alias from cmsPropertyType order by alias"; + try + { + using (var dr = Application.SqlHelper.ExecuteReader(fieldSql)) + { + while (dr.Read()) + { + aliases.Add(dr.GetString("alias")); + } + } + } + catch (Exception ex) + { + if (ex is SqlHelperException || ex is SqlException) + { + //if this happens, it could be due to wrong connection string, or something else. + //we don't want to crash the app because of this so we'll actually swallow this + //exception... Unfortunately logging probably won't work in this situation either :( + + Debug.WriteLine("EXCEPTION OCCURRED reading GetAllUserPropertyNames: " + ex.Message, "Error"); + Trace.WriteLine("EXCEPTION OCCURRED reading GetAllUserPropertyNames: " + ex.Message, "Error"); + } + else + { + throw ex; + } + } + + return aliases; + } + + /// + /// Returns a list of all system field names in Umbraco + /// + /// + public IEnumerable GetAllSystemPropertyNames() + { + return UmbracoContentIndexer.IndexFieldPolicies.Select(x => x.Key); + } + + } +} diff --git a/src/UmbracoExamine/DataServices/UmbracoDataService.cs b/src/UmbracoExamine/DataServices/UmbracoDataService.cs new file mode 100644 index 0000000000..9665dc05e3 --- /dev/null +++ b/src/UmbracoExamine/DataServices/UmbracoDataService.cs @@ -0,0 +1,25 @@ +using System.Web; +using System.Web.Hosting; + +namespace UmbracoExamine.DataServices +{ + public class UmbracoDataService : IDataService + { + public UmbracoDataService() + { + ContentService = new UmbracoContentService(); + MediaService = new UmbracoMediaService(); + LogService = new UmbracoLogService(); + } + + public IContentService ContentService { get; protected set; } + public IMediaService MediaService { get; protected set; } + public ILogService LogService { get; protected set; } + + public string MapPath(string virtualPath) + { + return HostingEnvironment.MapPath(virtualPath); + } + + } +} diff --git a/src/UmbracoExamine/DataServices/UmbracoLogService.cs b/src/UmbracoExamine/DataServices/UmbracoLogService.cs new file mode 100644 index 0000000000..d63af55fcf --- /dev/null +++ b/src/UmbracoExamine/DataServices/UmbracoLogService.cs @@ -0,0 +1,36 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Security; +using System.Text; +using umbraco.BusinessLogic; + +namespace UmbracoExamine.DataServices +{ + public class UmbracoLogService : UmbracoExamine.DataServices.ILogService + { + public string ProviderName { get; set; } + + [SecuritySafeCritical] + public void AddInfoLog(int nodeId, string msg) + { + Log.Add(LogTypes.Custom, nodeId, "[UmbracoExamine] (" + ProviderName + ")" + msg); + } + + [SecuritySafeCritical] + public void AddErrorLog(int nodeId, string msg) + { + Log.Add(LogTypes.Error, nodeId, "[UmbracoExamine] (" + ProviderName + ")" + msg); + } + + [SecuritySafeCritical] + public void AddVerboseLog(int nodeId, string msg) + { + if (LogLevel == LoggingLevel.Verbose) + Log.Add(LogTypes.Custom, nodeId, "[UmbracoExamine] (" + ProviderName + ")" + msg); + } + + public LoggingLevel LogLevel { get; set; } + + } +} diff --git a/src/UmbracoExamine/DataServices/UmbracoMediaService.cs b/src/UmbracoExamine/DataServices/UmbracoMediaService.cs new file mode 100644 index 0000000000..0daed53207 --- /dev/null +++ b/src/UmbracoExamine/DataServices/UmbracoMediaService.cs @@ -0,0 +1,51 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Security; +using System.Text; +using System.Xml.XPath; +using System.Xml.Linq; +using umbraco.cms.businesslogic.media; +using System.Collections; +using Examine.LuceneEngine; + +namespace UmbracoExamine.DataServices +{ + + /// + /// Data service used to query for media + /// + public class UmbracoMediaService : UmbracoExamine.DataServices.IMediaService + { + + /// + /// This is quite an intensive operation... + /// get all root media, then get the XML structure for all children, + /// then run xpath against the navigator that's created + /// + /// + /// + [SecuritySafeCritical] + public XDocument GetLatestMediaByXpath(string xpath) + { + + Media[] rootMedia = Media.GetRootMedias(); + var xmlMedia = XDocument.Parse(""); + foreach (Media media in rootMedia) + { + xmlMedia.Root.Add(GetMediaItem(media.Id)); + } + var result = ((IEnumerable)xmlMedia.XPathEvaluate(xpath)).Cast(); + return result.ToXDocument(); + } + + [SecuritySafeCritical] + private XElement GetMediaItem(int nodeId) + { + var nodes = umbraco.library.GetMedia(nodeId, true); + return XElement.Parse(nodes.Current.OuterXml); + } + + + } +} diff --git a/src/UmbracoExamine/IndexTypes.cs b/src/UmbracoExamine/IndexTypes.cs new file mode 100644 index 0000000000..6d941d21ae --- /dev/null +++ b/src/UmbracoExamine/IndexTypes.cs @@ -0,0 +1,38 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace UmbracoExamine +{ + /// + /// The index types stored in the Lucene Index + /// + public static class IndexTypes + { + + /// + /// The content index type + /// + /// + /// Is lower case because the Standard Analyzer requires lower case + /// + public const string Content = "content"; + + /// + /// The media index type + /// + /// + /// Is lower case because the Standard Analyzer requires lower case + /// + public const string Media = "media"; + + /// + /// The member index type + /// + /// + /// Is lower case because the Standard Analyzer requires lower case + /// + public const string Member = "member"; + } +} diff --git a/src/UmbracoExamine/LoggingLevel.cs b/src/UmbracoExamine/LoggingLevel.cs new file mode 100644 index 0000000000..36ece649af --- /dev/null +++ b/src/UmbracoExamine/LoggingLevel.cs @@ -0,0 +1,12 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace UmbracoExamine +{ + public enum LoggingLevel + { + Verbose, Normal + } +} diff --git a/src/UmbracoExamine/Properties/AssemblyInfo.cs b/src/UmbracoExamine/Properties/AssemblyInfo.cs new file mode 100644 index 0000000000..4997aae313 --- /dev/null +++ b/src/UmbracoExamine/Properties/AssemblyInfo.cs @@ -0,0 +1,36 @@ +using System.Reflection; +using System.Resources; +using System.Runtime.InteropServices; +using System.Runtime.CompilerServices; +using System.Security; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyCompany("umbraco")] +[assembly: AssemblyCopyright("Copyright © Umbraco 2012")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] +[assembly: AssemblyTitle("UmbracoExamine")] +[assembly: AssemblyDescription("Umbraco index & search providers based on the Examine model using Lucene.NET 2.9.2")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyProduct("UmbracoExamine")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("31c5b048-cfa8-49b4-8983-bdba0f99eef5")] + +[assembly: NeutralResourcesLanguage("en-US")] + +//NOTE: WE cannot make change the major version to be the same as Umbraco because of backwards compatibility, however we +// will make the minor version the same as the umbraco version +[assembly: AssemblyVersion("0.6.0.*")] +[assembly: AssemblyFileVersion("0.6.0.*")] + +[assembly: AllowPartiallyTrustedCallers] + +[assembly: InternalsVisibleTo("Umbraco.Tests")] diff --git a/src/UmbracoExamine/SearchCriteria/ExamineValue.cs b/src/UmbracoExamine/SearchCriteria/ExamineValue.cs new file mode 100644 index 0000000000..6a106269af --- /dev/null +++ b/src/UmbracoExamine/SearchCriteria/ExamineValue.cs @@ -0,0 +1,37 @@ +using Examine.SearchCriteria; + +namespace UmbracoExamine.SearchCriteria +{ + internal class ExamineValue : IExamineValue + { + public ExamineValue(Examineness vagueness, string value) : this(vagueness, value, 1) + { + } + + public ExamineValue(Examineness vagueness, string value, float level) + { + this.Examineness = vagueness; + this.Value = value; + this.Level = level; + } + + public Examineness Examineness + { + get; + private set; + } + + public string Value + { + get; + private set; + } + + public float Level + { + get; + private set; + } + + } +} diff --git a/src/UmbracoExamine/SearchCriteria/LuceneBooleanOperation.cs b/src/UmbracoExamine/SearchCriteria/LuceneBooleanOperation.cs new file mode 100644 index 0000000000..9845d9cf77 --- /dev/null +++ b/src/UmbracoExamine/SearchCriteria/LuceneBooleanOperation.cs @@ -0,0 +1,70 @@ +using Examine.SearchCriteria; +using Lucene.Net.Search; + +namespace UmbracoExamine.SearchCriteria +{ + /// + /// An implementation of the fluent API boolean operations + /// + public class LuceneBooleanOperation : IBooleanOperation + { + private LuceneSearchCriteria search; + + internal LuceneBooleanOperation(LuceneSearchCriteria search) + { + this.search = search; + } + + #region IBooleanOperation Members + + /// + /// Sets the next operation to be AND + /// + /// + public IQuery And() + { + return new LuceneQuery(this.search, BooleanClause.Occur.MUST); + } + + /// + /// Sets the next operation to be OR + /// + /// + public IQuery Or() + { + return new LuceneQuery(this.search, BooleanClause.Occur.SHOULD); + } + + /// + /// Sets the next operation to be NOT + /// + /// + public IQuery Not() + { + return new LuceneQuery(this.search, BooleanClause.Occur.MUST_NOT); + } + + /// + /// Compiles this instance for fluent API conclusion + /// + /// + public ISearchCriteria Compile() + { + if (!string.IsNullOrEmpty(this.search.SearchIndexType)) + { + var query = this.search.query; + + this.search.query = new BooleanQuery(); + this.search.query.Add(query, BooleanClause.Occur.MUST); + + //this.search.query.Add(this.search.queryParser.Parse("(" + query.ToString() + ")"), BooleanClause.Occur.MUST); + + this.search.FieldInternal(LuceneExamineIndexer.IndexTypeFieldName, new ExamineValue(Examineness.Explicit, this.search.SearchIndexType.ToString().ToLower()), BooleanClause.Occur.MUST); + } + + return this.search; + } + + #endregion + } +} diff --git a/src/UmbracoExamine/SearchCriteria/LuceneQuery.cs b/src/UmbracoExamine/SearchCriteria/LuceneQuery.cs new file mode 100644 index 0000000000..74d20f58a0 --- /dev/null +++ b/src/UmbracoExamine/SearchCriteria/LuceneQuery.cs @@ -0,0 +1,330 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Examine.SearchCriteria; +using Lucene.Net.Search; + +namespace UmbracoExamine.SearchCriteria +{ + public class LuceneQuery : IQuery + { + private LuceneSearchCriteria search; + private BooleanClause.Occur occurance; + + /// + /// Initializes a new instance of the class. + /// + /// The search. + /// The occurance. + internal LuceneQuery(LuceneSearchCriteria search, BooleanClause.Occur occurance) + { + this.search = search; + this.occurance = occurance; + } + + /// + /// Gets the boolean operation which this query method will be added as + /// + /// The boolean operation. + public BooleanOperation BooleanOperation + { + get { return occurance.ToBooleanOperation(); } + } + + + #region ISearch Members + + /// + /// Query on the id + /// + /// The id. + /// A new with the clause appended + public IBooleanOperation Id(int id) + { + return this.search.IdInternal(id, this.occurance); + } + + /// + /// Query on the NodeName + /// + /// Name of the node. + /// A new with the clause appended + public IBooleanOperation NodeName(string nodeName) + { + return this.search.NodeNameInternal(new ExamineValue(Examineness.Explicit, nodeName), occurance); + } + + /// + /// Query on the NodeTypeAlias + /// + /// The node type alias. + /// A new with the clause appended + public IBooleanOperation NodeTypeAlias(string nodeTypeAlias) + { + return this.search.NodeTypeAliasInternal(new ExamineValue(Examineness.Explicit, nodeTypeAlias), occurance); + } + + /// + /// Query on the Parent ID + /// + /// The id of the parent. + /// A new with the clause appended + public IBooleanOperation ParentId(int id) + { + return this.search.ParentIdInternal(id, occurance); + } + + /// + /// Query on the specified field + /// + /// Name of the field. + /// The field value. + /// A new with the clause appended + public IBooleanOperation Field(string fieldName, string fieldValue) + { + return this.search.FieldInternal(fieldName, new ExamineValue(Examineness.Explicit, fieldValue), occurance); + } + + /// + /// Ranges the specified field name. + /// + /// Name of the field. + /// The start. + /// The end. + /// A new with the clause appended + public IBooleanOperation Range(string fieldName, DateTime start, DateTime end) + { + return this.Range(fieldName, start, end, true, true); + } + + /// + /// Ranges the specified field name. + /// + /// Name of the field. + /// The start. + /// The end. + /// if set to true [include lower]. + /// if set to true [include upper]. + /// A new with the clause appended + public IBooleanOperation Range(string fieldName, DateTime start, DateTime end, bool includeLower, bool includeUpper) + { + return this.search.Range(fieldName, start, end, includeLower, includeUpper); + } + + /// + /// Ranges the specified field name. + /// + /// Name of the field. + /// The start. + /// The end. + /// A new with the clause appended + public IBooleanOperation Range(string fieldName, int start, int end) + { + return this.Range(fieldName, start, end, true, true); + } + + /// + /// Ranges the specified field name. + /// + /// Name of the field. + /// The start. + /// The end. + /// if set to true [include lower]. + /// if set to true [include upper]. + /// A new with the clause appended + public IBooleanOperation Range(string fieldName, int start, int end, bool includeLower, bool includeUpper) + { + return this.search.RangeInternal(fieldName, start, end, includeLower, includeUpper, occurance); + } + + /// + /// Ranges the specified field name. + /// + /// Name of the field. + /// The start. + /// The end. + /// A new with the clause appended + public IBooleanOperation Range(string fieldName, string start, string end) + { + return this.Range(fieldName, start, end, true, true); + } + + /// + /// Ranges the specified field name. + /// + /// Name of the field. + /// The start. + /// The end. + /// if set to true [include lower]. + /// if set to true [include upper]. + /// A new with the clause appended + public IBooleanOperation Range(string fieldName, string start, string end, bool includeLower, bool includeUpper) + { + return this.search.RangeInternal(fieldName, start, end, includeLower, includeUpper, occurance); + } + + /// + /// Query on the NodeName + /// + /// Name of the node. + /// A new with the clause appended + public IBooleanOperation NodeName(IExamineValue nodeName) + { + return this.search.NodeNameInternal(nodeName, occurance); + } + + /// + /// Query on the NodeTypeAlias + /// + /// The node type alias. + /// A new with the clause appended + public IBooleanOperation NodeTypeAlias(IExamineValue nodeTypeAlias) + { + return this.search.NodeTypeAliasInternal(nodeTypeAlias, occurance); + } + + /// + /// Query on the specified field + /// + /// Name of the field. + /// The field value. + /// A new with the clause appended + public IBooleanOperation Field(string fieldName, IExamineValue fieldValue) + { + return this.search.FieldInternal(fieldName, fieldValue, occurance); + } + + /// + /// Queries multiple fields with each being an And boolean operation + /// + /// The fields. + /// The query. + /// A new with the clause appended + public IBooleanOperation GroupedAnd(IEnumerable fields, params string[] query) + { + var fieldVals = new List(); + foreach (var f in query) + { + fieldVals.Add(new ExamineValue(Examineness.Explicit, f)); + } + return this.search.GroupedAndInternal(fields.ToArray(), fieldVals.ToArray(), this.occurance); + } + + /// + /// Queries multiple fields with each being an And boolean operation + /// + /// The fields. + /// The query. + /// A new with the clause appended + public IBooleanOperation GroupedAnd(IEnumerable fields, params IExamineValue[] query) + { + throw new NotImplementedException(); + } + + /// + /// Queries multiple fields with each being an Or boolean operation + /// + /// The fields. + /// The query. + /// A new with the clause appended + public IBooleanOperation GroupedOr(IEnumerable fields, params string[] query) + { + var fieldVals = new List(); + foreach (var f in query) + { + fieldVals.Add(new ExamineValue(Examineness.Explicit, f)); + } + return this.search.GroupedOrInternal(fields.ToArray(), fieldVals.ToArray(), this.occurance); + } + + /// + /// Queries multiple fields with each being an Or boolean operation + /// + /// The fields. + /// The query. + /// A new with the clause appended + public IBooleanOperation GroupedOr(IEnumerable fields, params IExamineValue[] query) + { + return this.search.GroupedOrInternal(fields.ToArray(), query, this.occurance); + } + + /// + /// Queries multiple fields with each being an Not boolean operation + /// + /// The fields. + /// The query. + /// A new with the clause appended + public IBooleanOperation GroupedNot(IEnumerable fields, params string[] query) + { + var fieldVals = new List(); + foreach (var f in query) + { + fieldVals.Add(new ExamineValue(Examineness.Explicit, f)); + } + return this.search.GroupedNotInternal(fields.ToArray(), fieldVals.ToArray(), this.occurance); + } + + /// + /// Queries multiple fields with each being an Not boolean operation + /// + /// The fields. + /// The query. + /// A new with the clause appended + public IBooleanOperation GroupedNot(IEnumerable fields, params IExamineValue[] query) + { + return this.search.GroupedNotInternal(fields.ToArray(), query, this.occurance); + } + + /// + /// Queries on multiple fields with their inclusions customly defined + /// + /// The fields. + /// The operations. + /// The query. + /// A new with the clause appended + public IBooleanOperation GroupedFlexible(IEnumerable fields, IEnumerable operations, params string[] query) + { + var fieldVals = new List(); + foreach (var f in query) + { + fieldVals.Add(new ExamineValue(Examineness.Explicit, f)); + } + return this.search.GroupedFlexibleInternal(fields.ToArray(), operations.ToArray(), fieldVals.ToArray(), occurance); + } + + /// + /// Queries on multiple fields with their inclusions customly defined + /// + /// The fields. + /// The operations. + /// The query. + /// A new with the clause appended + public IBooleanOperation GroupedFlexible(IEnumerable fields, IEnumerable operations, params IExamineValue[] query) + { + return this.search.GroupedFlexibleInternal(fields.ToArray(), operations.ToArray(), query, occurance); + } + + /// + /// Orders the results by the specified fields + /// + /// The field names. + /// A new with the clause appended + public IBooleanOperation OrderBy(params string[] fieldNames) + { + return this.search.OrderBy(fieldNames); + } + + /// + /// Orders the results by the specified fields in a descending order + /// + /// The field names. + /// A new with the clause appended + public IBooleanOperation OrderByDescending(params string[] fieldNames) + { + return this.search.OrderByDescending(fieldNames); + } + + #endregion + + } +} diff --git a/src/UmbracoExamine/SearchCriteria/LuceneSearchCriteria.cs b/src/UmbracoExamine/SearchCriteria/LuceneSearchCriteria.cs new file mode 100644 index 0000000000..e11c50b88f --- /dev/null +++ b/src/UmbracoExamine/SearchCriteria/LuceneSearchCriteria.cs @@ -0,0 +1,559 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using Examine; +using Examine.SearchCriteria; +using Lucene.Net.Analysis; +using Lucene.Net.QueryParsers; +using Lucene.Net.Search; +using Lucene.Net.Search.Spans; +using Lucene.Net.Index; +using Lucene.Net.Documents; + +namespace UmbracoExamine.SearchCriteria +{ + /// + /// This class is used to query against Lucene.Net + /// + public class LuceneSearchCriteria : ISearchCriteria + { + internal MultiFieldQueryParser queryParser; + internal BooleanQuery query; + internal List sortFields = new List(); + private readonly BooleanClause.Occur occurance; + private readonly Lucene.Net.Util.Version luceneVersion = Lucene.Net.Util.Version.LUCENE_29; + + internal LuceneSearchCriteria(string type, Analyzer analyzer, string[] fields, bool allowLeadingWildcards, BooleanOperation occurance) + { + Enforcer.ArgumentNotNull(fields, "fields"); + + SearchIndexType = type; + query = new BooleanQuery(); + this.BooleanOperation = occurance; + this.queryParser = new MultiFieldQueryParser(luceneVersion, fields, analyzer); + this.queryParser.SetAllowLeadingWildcard(allowLeadingWildcards); + this.occurance = occurance.ToLuceneOccurance(); + } + + /// + /// Gets the boolean operation which this query method will be added as + /// + /// The boolean operation. + public BooleanOperation BooleanOperation + { + get; + protected set; + } + + /// + /// Returns a that represents this instance. + /// + /// + /// A that represents this instance. + /// + public override string ToString() + { + return string.Format("{{ SearchIndexType: {0}, LuceneQuery: {1} }}", this.SearchIndexType, this.query.ToString()); + } + + private static void ValidateIExamineValue(IExamineValue v) + { + var ev = v as ExamineValue; + if (ev == null) + { + throw new ArgumentException("IExamineValue was not created from this provider. Ensure that it is created from the ISearchCriteria this provider exposes"); + } + } + + #region ISearchCriteria Members + + public string SearchIndexType + { + get; + protected set; + } + + public bool IncludeHitCount + { + get; + set; + } + + public int TotalHits + { + get; + internal protected set; + } + + #endregion + + #region ISearch Members + + /// + /// Query on the id + /// + /// The id. + /// A new with the clause appended + public IBooleanOperation Id(int id) + { + return IdInternal(id, occurance); + } + + internal protected IBooleanOperation IdInternal(int id, BooleanClause.Occur occurance) + { + //use a query parser (which uses the analyzer) to build up the field query which we want + query.Add(this.queryParser.GetFieldQuery(LuceneExamineIndexer.IndexNodeIdFieldName, id.ToString()), occurance); + + return new LuceneBooleanOperation(this); + } + + /// + /// Query on the NodeName + /// + /// Name of the node. + /// A new with the clause appended + public IBooleanOperation NodeName(string nodeName) + { + Enforcer.ArgumentNotNull(nodeName, "nodeName"); + return NodeName(new ExamineValue(Examineness.Explicit, nodeName)); + } + + /// + /// Query on the NodeName + /// + /// Name of the node. + /// A new with the clause appended + public IBooleanOperation NodeName(IExamineValue nodeName) + { + Enforcer.ArgumentNotNull(nodeName, "nodeName"); + return this.NodeNameInternal(nodeName, occurance); + } + + internal protected IBooleanOperation NodeNameInternal(IExamineValue examineValue, BooleanClause.Occur occurance) + { + return this.FieldInternal("nodeName", examineValue, occurance); + } + + /// + /// Query on the NodeTypeAlias + /// + /// The node type alias. + /// A new with the clause appended + public IBooleanOperation NodeTypeAlias(string nodeTypeAlias) + { + Enforcer.ArgumentNotNull(nodeTypeAlias, "nodeTypeAlias"); + return this.NodeTypeAlias(new ExamineValue(Examineness.Explicit, nodeTypeAlias)); + } + + /// + /// Query on the NodeTypeAlias + /// + /// The node type alias. + /// A new with the clause appended + public IBooleanOperation NodeTypeAlias(IExamineValue nodeTypeAlias) + { + Enforcer.ArgumentNotNull(nodeTypeAlias, "nodeTypeAlias"); + return this.NodeTypeAliasInternal(nodeTypeAlias, occurance); + } + + internal protected IBooleanOperation NodeTypeAliasInternal(IExamineValue examineValue, BooleanClause.Occur occurance) + { + return this.FieldInternal("nodeTypeAlias", examineValue, occurance); + } + + /// + /// Query on the Parent ID + /// + /// The id of the parent. + /// A new with the clause appended + public IBooleanOperation ParentId(int id) + { + return this.ParentIdInternal(id, occurance); + } + + internal protected IBooleanOperation ParentIdInternal(int id, BooleanClause.Occur occurance) + { + query.Add(this.queryParser.GetFieldQuery("parentID", id.ToString()), occurance); + + return new LuceneBooleanOperation(this); + } + + /// + /// Query on the specified field + /// + /// Name of the field. + /// The field value. + /// A new with the clause appended + public IBooleanOperation Field(string fieldName, string fieldValue) + { + Enforcer.ArgumentNotNull(fieldName, "fieldName"); + Enforcer.ArgumentNotNull(fieldValue, "fieldValue"); + return this.FieldInternal(fieldName, new ExamineValue(Examineness.Explicit, fieldValue), occurance); + } + + /// + /// Query on the specified field + /// + /// Name of the field. + /// The field value. + /// A new with the clause appended + public IBooleanOperation Field(string fieldName, IExamineValue fieldValue) + { + Enforcer.ArgumentNotNull(fieldName, "fieldName"); + Enforcer.ArgumentNotNull(fieldValue, "fieldValue"); + return this.FieldInternal(fieldName, fieldValue, occurance); + } + + /// + /// Returns the Lucene query object for a field given an IExamineValue + /// + /// + /// + /// A new with the clause appended + internal protected Query GetFieldInternalQuery(string fieldName, IExamineValue fieldValue) + { + Query queryToAdd; + + switch (fieldValue.Examineness) + { + case Examineness.Fuzzy: + queryToAdd = this.queryParser.GetFuzzyQuery(fieldName, fieldValue.Value, fieldValue.Level); + break; + case Examineness.SimpleWildcard: + case Examineness.ComplexWildcard: + queryToAdd = this.queryParser.GetWildcardQuery(fieldName, fieldValue.Value); + break; + case Examineness.Boosted: + queryToAdd = this.queryParser.GetFieldQuery(fieldName, fieldValue.Value); + queryToAdd.SetBoost(fieldValue.Level); + break; + case Examineness.Proximity: + //This is how you are supposed to do this based on this doc here: + //http://lucene.apache.org/java/2_4_1/api/org/apache/lucene/search/spans/package-summary.html#package_description + //but i think that lucene.net has an issue with it's internal parser since it parses to a very strange query + //we'll just manually make it instead below + + //var spans = new List(); + //foreach (var s in fieldValue.Value.Split(' ')) + //{ + // spans.Add(new SpanTermQuery(new Term(fieldName, s))); + //} + //queryToAdd = new SpanNearQuery(spans.ToArray(), Convert.ToInt32(fieldValue.Level), true); + + var proxQuery = fieldName + ":\"" + fieldValue.Value + "\"~" + Convert.ToInt32(fieldValue.Level).ToString(); + queryToAdd = queryParser.Parse(proxQuery); + + break; + case Examineness.Explicit: + default: + queryToAdd = this.queryParser.GetFieldQuery(fieldName, fieldValue.Value); + break; + } + return queryToAdd; + } + + internal protected IBooleanOperation FieldInternal(string fieldName, IExamineValue fieldValue, BooleanClause.Occur occurance) + { + Query queryToAdd = GetFieldInternalQuery(fieldName, fieldValue); + + if (queryToAdd != null) + query.Add(queryToAdd, occurance); + + return new LuceneBooleanOperation(this); + } + + public IBooleanOperation Range(string fieldName, DateTime start, DateTime end) + { + return this.Range(fieldName, start, end, true, true); + } + + public IBooleanOperation Range(string fieldName, DateTime start, DateTime end, bool includeLower, bool includeUpper) + { + //since lucene works on string's for all searching we need to flatten the date + return this.RangeInternal(fieldName, DateTools.DateToString(start, DateTools.Resolution.MILLISECOND), DateTools.DateToString(end, DateTools.Resolution.MILLISECOND), includeLower, includeUpper, occurance); + } + + public IBooleanOperation Range(string fieldName, int start, int end) + { + Enforcer.ArgumentNotNull(fieldName, "fieldName"); + return this.Range(fieldName, start, end, true, true); + } + + public IBooleanOperation Range(string fieldName, int start, int end, bool includeLower, bool includeUpper) + { + return this.RangeInternal(fieldName, start, end, includeLower, includeUpper, occurance); + } + + protected internal IBooleanOperation RangeInternal(string fieldName, int start, int end, bool includeLower, bool includeUpper, BooleanClause.Occur occurance) + { + query.Add(NumericRangeQuery.NewIntRange(fieldName, start, end, includeLower, includeUpper), occurance); + return new LuceneBooleanOperation(this); + } + + public IBooleanOperation Range(string fieldName, string start, string end) + { + Enforcer.ArgumentNotNull(fieldName, "fieldName"); + Enforcer.ArgumentNotNull(start, "start"); + Enforcer.ArgumentNotNull(end, "end"); + return this.Range(fieldName, start, end, true, true); + } + + public IBooleanOperation Range(string fieldName, string start, string end, bool includeLower, bool includeUpper) + { + Enforcer.ArgumentNotNull(fieldName, "fieldName"); + Enforcer.ArgumentNotNull(start, "start"); + Enforcer.ArgumentNotNull(end, "end"); + return this.RangeInternal(fieldName, start, end, includeLower, includeUpper, occurance); + } + + protected internal IBooleanOperation RangeInternal(string fieldName, string start, string end, bool includeLower, bool includeUpper, BooleanClause.Occur occurance) + { + query.Add(new TermRangeQuery(fieldName, start, end, includeLower, includeUpper), occurance); + + return new LuceneBooleanOperation(this); + } + + public IBooleanOperation GroupedAnd(IEnumerable fields, params string[] query) + { + Enforcer.ArgumentNotNull(fields, "fields"); + Enforcer.ArgumentNotNull(query, "query"); + + var fieldVals = new List(); + foreach (var f in query) + { + fieldVals.Add(new ExamineValue(Examineness.Explicit, f)); + } + return this.GroupedAnd(fields.ToArray(), fieldVals.ToArray()); + } + + public IBooleanOperation GroupedAnd(IEnumerable fields, IExamineValue[] fieldVals) + { + Enforcer.ArgumentNotNull(fields, "fields"); + Enforcer.ArgumentNotNull(query, "fieldVals"); + + return this.GroupedAndInternal(fields.ToArray(), fieldVals.ToArray(), occurance); + } + + protected internal IBooleanOperation GroupedAndInternal(string[] fields, IExamineValue[] fieldVals, BooleanClause.Occur occurance) + { + + //if there's only 1 query text we want to build up a string like this: + //(+field1:query +field2:query +field3:query) + //but Lucene will bork if you provide an array of length 1 (which is != to the field length) + + query.Add(GetMultiFieldQuery(fields, fieldVals, BooleanClause.Occur.MUST), occurance); + + return new LuceneBooleanOperation(this); + } + + public IBooleanOperation GroupedOr(IEnumerable fields, params string[] query) + { + Enforcer.ArgumentNotNull(fields, "fields"); + Enforcer.ArgumentNotNull(query, "query"); + + var fieldVals = new List(); + foreach (var f in query) + { + fieldVals.Add(new ExamineValue(Examineness.Explicit, f)); + } + + return this.GroupedOr(fields.ToArray(), fieldVals.ToArray()); + } + + public IBooleanOperation GroupedOr(IEnumerable fields, params IExamineValue[] fieldVals) + { + Enforcer.ArgumentNotNull(fields, "fields"); + Enforcer.ArgumentNotNull(query, "query"); + + return this.GroupedOrInternal(fields.ToArray(), fieldVals, occurance); + } + + protected internal IBooleanOperation GroupedOrInternal(string[] fields, IExamineValue[] fieldVals, BooleanClause.Occur occurance) + { + //if there's only 1 query text we want to build up a string like this: + //(field1:query field2:query field3:query) + //but Lucene will bork if you provide an array of length 1 (which is != to the field length) + + query.Add(GetMultiFieldQuery(fields, fieldVals, BooleanClause.Occur.SHOULD), occurance); + + return new LuceneBooleanOperation(this); + } + + public IBooleanOperation GroupedNot(IEnumerable fields, params string[] query) + { + Enforcer.ArgumentNotNull(fields, "fields"); + Enforcer.ArgumentNotNull(query, "query"); + + var fieldVals = new List(); + foreach (var f in query) + { + fieldVals.Add(new ExamineValue(Examineness.Explicit, f)); + } + + return this.GroupedNot(fields.ToArray(), fieldVals.ToArray()); + } + + public IBooleanOperation GroupedNot(IEnumerable fields, params IExamineValue[] query) + { + Enforcer.ArgumentNotNull(fields, "fields"); + Enforcer.ArgumentNotNull(query, "query"); + + return this.GroupedNotInternal(fields.ToArray(), query, occurance); + } + + protected internal IBooleanOperation GroupedNotInternal(string[] fields, IExamineValue[] fieldVals, BooleanClause.Occur occurance) + { + //if there's only 1 query text we want to build up a string like this: + //(!field1:query !field2:query !field3:query) + //but Lucene will bork if you provide an array of length 1 (which is != to the field length) + + query.Add(GetMultiFieldQuery(fields, fieldVals, BooleanClause.Occur.MUST_NOT), occurance); + + return new LuceneBooleanOperation(this); + } + + /// + /// Creates our own style 'multi field query' used internal for the grouped operations + /// + /// + /// + /// + /// A new with the clause appended + protected internal BooleanQuery GetMultiFieldQuery(string[] fields, IExamineValue[] fieldVals, BooleanClause.Occur occurance) + { + //if there's only 1 query text we want to build up a string like this: + //(!field1:query !field2:query !field3:query) + //but Lucene will bork if you provide an array of length 1 (which is != to the field length) + + var queryVals = new IExamineValue[fields.Length]; + if (fieldVals.Length == 1) + { + for (int i = 0; i < queryVals.Length; i++) + queryVals[i] = fieldVals[0]; + } + else + { + queryVals = fieldVals; + } + + var qry = new BooleanQuery(); + for (int i = 0; i < fields.Length; i++) + { + qry.Add(this.GetFieldInternalQuery(fields[i], queryVals[i]), occurance); + } + + return qry; + } + + public IBooleanOperation GroupedFlexible(IEnumerable fields, IEnumerable operations, params string[] query) + { + Enforcer.ArgumentNotNull(fields, "fields"); + Enforcer.ArgumentNotNull(query, "query"); + Enforcer.ArgumentNotNull(operations, "operations"); + + var fieldVals = new List(); + foreach (var f in query) + { + fieldVals.Add(new ExamineValue(Examineness.Explicit, f)); + } + + return this.GroupedFlexible(fields.ToArray(), operations.ToArray(), fieldVals.ToArray()); + } + + public IBooleanOperation GroupedFlexible(IEnumerable fields, IEnumerable operations, params IExamineValue[] fieldVals) + { + Enforcer.ArgumentNotNull(fields, "fields"); + Enforcer.ArgumentNotNull(query, "query"); + Enforcer.ArgumentNotNull(operations, "operations"); + + return this.GroupedFlexibleInternal(fields.ToArray(), operations.ToArray(), fieldVals, occurance); + } + + protected internal IBooleanOperation GroupedFlexibleInternal(string[] fields, BooleanOperation[] operations, IExamineValue[] fieldVals, BooleanClause.Occur occurance) + { + //if there's only 1 query text we want to build up a string like this: + //(field1:query field2:query field3:query) + //but Lucene will bork if you provide an array of length 1 (which is != to the field length) + + var flags = new BooleanClause.Occur[operations.Count()]; + for (int i = 0; i < flags.Length; i++) + flags[i] = operations.ElementAt(i).ToLuceneOccurance(); + + var queryVals = new IExamineValue[fields.Length]; + if (fieldVals.Length == 1) + { + for (int i = 0; i < queryVals.Length; i++) + queryVals[i] = fieldVals[0]; + } + else + { + queryVals = fieldVals; + } + + var qry = new BooleanQuery(); + for (int i = 0; i < fields.Length; i++) + { + qry.Add(this.GetFieldInternalQuery(fields[i], queryVals[i]), flags[i]); + } + + this.query.Add(qry, occurance); + + return new LuceneBooleanOperation(this); + } + + /// + /// Passes a raw search query to the provider to handle + /// + /// The query. + /// A new with the clause appended + public ISearchCriteria RawQuery(string query) + { + this.query.Add(this.queryParser.Parse(query), this.occurance); + + return this; + } + + /// + /// Orders the results by the specified fields + /// + /// The field names. + /// A new with the clause appended + public IBooleanOperation OrderBy(params string[] fieldNames) + { + Enforcer.ArgumentNotNull(fieldNames, "fieldNames"); + + return this.OrderByInternal(false, fieldNames); + } + + /// + /// Orders the results by the specified fields in a descending order + /// + /// The field names. + /// A new with the clause appended + public IBooleanOperation OrderByDescending(params string[] fieldNames) + { + Enforcer.ArgumentNotNull(fieldNames, "fieldNames"); + + return this.OrderByInternal(true, fieldNames); + } + + /// + /// Internal operation for adding the ordered results + /// + /// if set to true [descending]. + /// The field names. + /// A new with the clause appended + protected internal IBooleanOperation OrderByInternal(bool descending, params string[] fieldNames) + { + foreach (var fieldName in fieldNames) + { + this.sortFields.Add(new SortField(LuceneExamineIndexer.SortedFieldNamePrefix + fieldName, SortField.STRING, descending)); + } + + return new LuceneBooleanOperation(this); + } + + #endregion + } +} diff --git a/src/UmbracoExamine/SearchCriteria/LuceneSearchExtensions.cs b/src/UmbracoExamine/SearchCriteria/LuceneSearchExtensions.cs new file mode 100644 index 0000000000..3681c98e47 --- /dev/null +++ b/src/UmbracoExamine/SearchCriteria/LuceneSearchExtensions.cs @@ -0,0 +1,164 @@ +using Examine.SearchCriteria; +using Lucene.Net.Search; +using Lucene.Net.QueryParsers; +using System; + +namespace UmbracoExamine.SearchCriteria +{ + public static class LuceneSearchExtensions + { + /// + /// Adds a single character wildcard to the string for Lucene wildcard matching + /// + /// The string to wildcard. + /// An IExamineValue for the required operation + /// Thrown when the string is null or empty + public static IExamineValue SingleCharacterWildcard(this string s) + { + if (System.String.IsNullOrEmpty(s)) + throw new ArgumentException("Supplied string is null or empty.", "s"); + + return new ExamineValue(Examineness.SimpleWildcard, s + "?"); + } + + /// + /// Adds a multi-character wildcard to a string for Lucene wildcard matching + /// + /// The string to wildcard. + /// An IExamineValue for the required operation + /// Thrown when the string is null or empty + public static IExamineValue MultipleCharacterWildcard(this string s) + { + if (String.IsNullOrEmpty(s)) + throw new ArgumentException("Supplied string is null or empty.", "s"); + return new ExamineValue(Examineness.ComplexWildcard, s + "*"); + } + + /// + /// Configures the string for fuzzy matching in Lucene using the default fuzziness level + /// + /// The string to configure fuzzy matching on. + /// An IExamineValue for the required operation + /// Thrown when the string is null or empty + public static IExamineValue Fuzzy(this string s) + { + return Fuzzy(s, 0.5f); + } + + /// + /// Configures the string for fuzzy matching in Lucene using the supplied fuzziness level + /// + /// The string to configure fuzzy matching on. + /// The fuzzieness level. + /// + /// An IExamineValue for the required operation + /// + /// Thrown when the string is null or empty + public static IExamineValue Fuzzy(this string s, float fuzzieness) + { + if (String.IsNullOrEmpty(s)) + throw new ArgumentException("Supplied string is null or empty.", "s"); + return new ExamineValue(Examineness.Fuzzy, s, fuzzieness); + } + + /// + /// Configures the string for boosting in Lucene + /// + /// The string to wildcard. + /// The boost level. + /// + /// An IExamineValue for the required operation + /// + /// Thrown when the string is null or empty + public static IExamineValue Boost(this string s, float boost) + { + if (String.IsNullOrEmpty(s)) + throw new ArgumentException("Supplied string is null or empty.", "s"); + return new ExamineValue(Examineness.Boosted, s + "^", boost); + } + + /// + /// Configures the string for proximity matching + /// + /// The string to wildcard. + /// The proximity level. + /// + /// An IExamineValue for the required operation + /// + /// Thrown when the string is null or empty + public static IExamineValue Proximity(this string s, int proximity) + { + if (String.IsNullOrEmpty(s)) + throw new ArgumentException("Supplied string is null or empty.", "s"); + return new ExamineValue(Examineness.Proximity, s + "~", Convert.ToSingle(proximity)); + } + + /// + /// Escapes the string within Lucene + /// + /// The string to wildcard. + /// An IExamineValue for the required operation + /// Thrown when the string is null or empty + public static IExamineValue Escape(this string s) + { + if (String.IsNullOrEmpty(s)) + throw new ArgumentException("Supplied string is null or empty.", "s"); + return new ExamineValue(Examineness.Escaped, QueryParser.Escape(s)); + } + + /// + /// Sets up an for an additional Examiness + /// + /// The IExamineValue to continue working with. + /// The string to postfix. + /// Combined strings + public static string Then(this IExamineValue examineValue, string s) + { + if (examineValue == null) + throw new ArgumentNullException("examineValue", "examineValue is null."); + if (String.IsNullOrEmpty(s)) + throw new ArgumentException("Supplied string is null or empty.", "s"); + return examineValue.Value + s; + } + + /// + /// Converts an Examine boolean operation to a Lucene representation + /// + /// The operation. + /// The translated Boolean operation + public static BooleanClause.Occur ToLuceneOccurance(this BooleanOperation o) + { + switch (o) + { + case BooleanOperation.And: + return BooleanClause.Occur.MUST; + case BooleanOperation.Not: + return BooleanClause.Occur.MUST_NOT; + case BooleanOperation.Or: + default: + return BooleanClause.Occur.SHOULD; + } + } + + /// + /// Converts a Lucene boolean occurrence to an Examine representation + /// + /// The occurrence to translate. + /// The translated boolean occurrence + public static BooleanOperation ToBooleanOperation(this BooleanClause.Occur o) + { + if (o == BooleanClause.Occur.MUST) + { + return BooleanOperation.And; + } + else if (o == BooleanClause.Occur.MUST_NOT) + { + return BooleanOperation.Not; + } + else + { + return BooleanOperation.Or; + } + } + } +} diff --git a/src/UmbracoExamine/UmbracoContentIndexer.cs b/src/UmbracoExamine/UmbracoContentIndexer.cs new file mode 100644 index 0000000000..f58bec2dad --- /dev/null +++ b/src/UmbracoExamine/UmbracoContentIndexer.cs @@ -0,0 +1,375 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Collections.Specialized; +using System.IO; +using System.Linq; +using System.Security; +using System.Text; +using System.Web; +using System.Xml.Linq; +using Examine; +using Examine.Config; +using Examine.Providers; +using umbraco.cms.businesslogic; +using UmbracoExamine.DataServices; +using Examine.LuceneEngine; +using Examine.LuceneEngine.Config; +using UmbracoExamine.Config; +using Examine.LuceneEngine.Providers; +using Lucene.Net.Analysis; +using umbraco.BasePages; + + +namespace UmbracoExamine +{ + /// + /// + /// + public class UmbracoContentIndexer : BaseUmbracoIndexer + { + #region Constructors + + /// + /// Default constructor + /// + public UmbracoContentIndexer() + : base() { } + + /// + /// Constructor to allow for creating an indexer at runtime + /// + /// + /// + /// + /// + [SecuritySafeCritical] + public UmbracoContentIndexer(IIndexCriteria indexerData, DirectoryInfo indexPath, IDataService dataService, Analyzer analyzer, bool async) + : base(indexerData, indexPath, dataService, analyzer, async) { } + + /// + /// Constructor to allow for creating an indexer at runtime + /// + /// + /// + /// + /// + /// + [SecuritySafeCritical] + public UmbracoContentIndexer(IIndexCriteria indexerData, Lucene.Net.Store.Directory luceneDirectory, IDataService dataService, Analyzer analyzer, bool async) + : base(indexerData, luceneDirectory, dataService, analyzer, async) { } + + #endregion + + #region Constants & Fields + + /// + /// Used to store the path of a content object + /// + public const string IndexPathFieldName = "__Path"; + public const string NodeTypeAliasFieldName = "__NodeTypeAlias"; + + /// + /// A type that defines the type of index for each Umbraco field (non user defined fields) + /// Alot of standard umbraco fields shouldn't be tokenized or even indexed, just stored into lucene + /// for retreival after searching. + /// + internal static readonly Dictionary IndexFieldPolicies + = new Dictionary() + { + { "id", FieldIndexTypes.NOT_ANALYZED}, + { "version", FieldIndexTypes.NOT_ANALYZED}, + { "parentID", FieldIndexTypes.NOT_ANALYZED}, + { "level", FieldIndexTypes.NOT_ANALYZED}, + { "writerID", FieldIndexTypes.NOT_ANALYZED}, + { "creatorID", FieldIndexTypes.NOT_ANALYZED}, + { "nodeType", FieldIndexTypes.NOT_ANALYZED}, + { "template", FieldIndexTypes.NOT_ANALYZED}, + { "sortOrder", FieldIndexTypes.NOT_ANALYZED}, + { "createDate", FieldIndexTypes.NOT_ANALYZED}, + { "updateDate", FieldIndexTypes.NOT_ANALYZED}, + { "nodeName", FieldIndexTypes.ANALYZED}, + { "urlName", FieldIndexTypes.NOT_ANALYZED}, + { "writerName", FieldIndexTypes.ANALYZED}, + { "creatorName", FieldIndexTypes.ANALYZED}, + { "nodeTypeAlias", FieldIndexTypes.ANALYZED}, + { "path", FieldIndexTypes.NOT_ANALYZED} + }; + + #endregion + + #region Initialize + + /// + /// Set up all properties for the indexer based on configuration information specified. This will ensure that + /// all of the folders required by the indexer are created and exist. This will also create an instruction + /// file declaring the computer name that is part taking in the indexing. This file will then be used to + /// determine the master indexer machine in a load balanced environment (if one exists). + /// + /// The friendly name of the provider. + /// A collection of the name/value pairs representing the provider-specific attributes specified in the configuration for this provider. + /// + /// The name of the provider is null. + /// + /// + /// The name of the provider has a length of zero. + /// + /// + /// An attempt is made to call on a provider after the provider has already been initialized. + /// + public override void Initialize(string name, System.Collections.Specialized.NameValueCollection config) + { + + //check if there's a flag specifying to support unpublished content, + //if not, set to false; + bool supportUnpublished; + if (config["supportUnpublished"] != null && bool.TryParse(config["supportUnpublished"], out supportUnpublished)) + SupportUnpublishedContent = supportUnpublished; + else + SupportUnpublishedContent = false; + + + //check if there's a flag specifying to support protected content, + //if not, set to false; + bool supportProtected; + if (config["supportProtected"] != null && bool.TryParse(config["supportProtected"], out supportProtected)) + SupportProtectedContent = supportProtected; + else + SupportProtectedContent = false; + + + base.Initialize(name, config); + } + + #endregion + + #region Properties + + /// + /// By default this is false, if set to true then the indexer will include indexing content that is flagged as publicly protected. + /// This property is ignored if SupportUnpublishedContent is set to true. + /// + public bool SupportProtectedContent { get; protected internal set; } + + protected override IEnumerable SupportedTypes + { + get + { + return new string[] { IndexTypes.Content, IndexTypes.Media }; + } + } + + #endregion + + #region Event handlers + + protected override void OnIndexingError(IndexingErrorEventArgs e) + { + DataService.LogService.AddErrorLog(e.NodeId, string.Format("{0},{1}, IndexSet: {2}", e.Message, e.InnerException != null ? e.InnerException.Message : "", this.IndexSetName)); + base.OnIndexingError(e); + } + + //protected override void OnDocumentWriting(DocumentWritingEventArgs docArgs) + //{ + // DataService.LogService.AddVerboseLog(docArgs.NodeId, string.Format("({0}) DocumentWriting event for node ({1})", this.Name, LuceneIndexFolder.FullName)); + // base.OnDocumentWriting(docArgs); + //} + + protected override void OnNodeIndexed(IndexedNodeEventArgs e) + { + DataService.LogService.AddVerboseLog(e.NodeId, string.Format("Index created for node")); + base.OnNodeIndexed(e); + } + + protected override void OnIndexDeleted(DeleteIndexEventArgs e) + { + DataService.LogService.AddVerboseLog(-1, string.Format("Index deleted for term: {0} with value {1}", e.DeletedTerm.Key, e.DeletedTerm.Value)); + base.OnIndexDeleted(e); + } + + protected override void OnIndexOptimizing(EventArgs e) + { + DataService.LogService.AddInfoLog(-1, string.Format("Index is being optimized")); + base.OnIndexOptimizing(e); + } + + #endregion + + #region Public methods + + + /// + /// Overridden for logging + /// + /// + /// + public override void ReIndexNode(XElement node, string type) + { + if (!SupportedTypes.Contains(type)) + return; + + DataService.LogService.AddVerboseLog((int)node.Attribute("id"), string.Format("ReIndexNode with type: {0}", type)); + base.ReIndexNode(node, type); + } + + /// + /// Deletes a node from the index. + /// + /// + /// When a content node is deleted, we also need to delete it's children from the index so we need to perform a + /// custom Lucene search to find all decendents and create Delete item queues for them too. + /// + /// ID of the node to delete + public override void DeleteFromIndex(string nodeId) + { + //find all descendants based on path + var descendantPath = string.Format(@"\-1\,*{0}\,*", nodeId); + var rawQuery = string.Format("{0}:{1}", IndexPathFieldName, descendantPath); + var c = InternalSearcher.CreateSearchCriteria(); + var filtered = c.RawQuery(rawQuery); + var results = InternalSearcher.Search(filtered); + + DataService.LogService.AddVerboseLog(int.Parse(nodeId), string.Format("DeleteFromIndex with query: {0} (found {1} results)", rawQuery, results.Count())); + + //need to create a delete queue item for each one found + foreach (var r in results) + { + EnqueueIndexOperation(new IndexOperation() + { + Operation = IndexOperationType.Delete, + Item = new IndexItem(null, "", r.Id.ToString()) + }); + //SaveDeleteIndexQueueItem(new KeyValuePair(IndexNodeIdFieldName, r.Id.ToString())); + } + + base.DeleteFromIndex(nodeId); + } + #endregion + + #region Protected + + + + /// + /// Overridden for logging. + /// + /// + /// + protected override void AddSingleNodeToIndex(XElement node, string type) + { + DataService.LogService.AddVerboseLog((int)node.Attribute("id"), string.Format("AddSingleNodeToIndex with type: {0}", type)); + base.AddSingleNodeToIndex(node, type); + } + + public override void RebuildIndex() + { + DataService.LogService.AddVerboseLog(-1, "Rebuilding index"); + base.RebuildIndex(); + } + + /// + /// Override this method to strip all html from all user fields before raising the event, then after the event + /// ensure our special Path field is added to the collection + /// + /// + protected override void OnGatheringNodeData(IndexingNodeDataEventArgs e) + { + //strip html of all users fields + // Get all user data that we want to index and store into a dictionary + foreach (var field in IndexerData.UserFields) + { + if (e.Fields.ContainsKey(field.Name)) + { + e.Fields[field.Name] = DataService.ContentService.StripHtml(e.Fields[field.Name]); + } + } + + base.OnGatheringNodeData(e); + + //ensure the special path and node type alis fields is added to the dictionary to be saved to file + var path = e.Node.Attribute("path").Value; + if (!e.Fields.ContainsKey(IndexPathFieldName)) + e.Fields.Add(IndexPathFieldName, path); + + //this needs to support both schemas so get the nodeTypeAlias if it exists, otherwise the name + var nodeTypeAlias = e.Node.Attribute("nodeTypeAlias") == null ? e.Node.Name.LocalName : e.Node.Attribute("nodeTypeAlias").Value; + if (!e.Fields.ContainsKey(NodeTypeAliasFieldName)) + e.Fields.Add(NodeTypeAliasFieldName, nodeTypeAlias); + } + + /// + /// Called when a duplicate field is detected in the dictionary that is getting indexed. + /// + /// + /// + /// + protected override void OnDuplicateFieldWarning(int nodeId, string indexSetName, string fieldName) + { + base.OnDuplicateFieldWarning(nodeId, indexSetName, fieldName); + + this.DataService.LogService.AddInfoLog(nodeId, "Field \"" + fieldName + "\" is listed multiple times in the index set \"" + indexSetName + "\". Please ensure all names are unique"); + } + + /// + /// Overridden to add the path property to the special fields to index + /// + /// + /// + protected override Dictionary GetSpecialFieldsToIndex(Dictionary allValuesForIndexing) + { + var fields = base.GetSpecialFieldsToIndex(allValuesForIndexing); + + //adds the special path property to the index + fields.Add(IndexPathFieldName, allValuesForIndexing[IndexPathFieldName]); + + //adds the special node type alias property to the index + fields.Add(NodeTypeAliasFieldName, allValuesForIndexing[NodeTypeAliasFieldName]); + + return fields; + + } + + /// + /// Creates an IIndexCriteria object based on the indexSet passed in and our DataService + /// + /// + /// + protected override IIndexCriteria GetIndexerData(IndexSet indexSet) + { + return indexSet.ToIndexCriteria(DataService); + } + + /// + /// return the index policy for the field name passed in, if not found, return normal + /// + /// + /// + protected override FieldIndexTypes GetPolicy(string fieldName) + { + var def = IndexFieldPolicies.Where(x => x.Key == fieldName); + return (def.Count() == 0 ? FieldIndexTypes.ANALYZED : def.Single().Value); + } + + /// + /// Ensure that the content of this node is available for indexing (i.e. don't allow protected + /// content to be indexed when this is disabled). + /// + /// + protected override bool ValidateDocument(XElement node) + { + var nodeId = int.Parse(node.Attribute("id").Value); + // Test for access if we're only indexing published content + // return nothing if we're not supporting protected content and it is protected, and we're not supporting unpublished content + if (!SupportUnpublishedContent + && (!SupportProtectedContent + && DataService.ContentService.IsProtected(nodeId, node.Attribute("path").Value))) + { + return false; + } + + return base.ValidateDocument(node); + } + + #endregion + } +} diff --git a/src/UmbracoExamine/UmbracoEventManager.cs b/src/UmbracoExamine/UmbracoEventManager.cs new file mode 100644 index 0000000000..cebb837670 --- /dev/null +++ b/src/UmbracoExamine/UmbracoEventManager.cs @@ -0,0 +1,223 @@ +using System; +using System.Linq; +using System.Net; +using System.Security; +using System.Threading; +using System.Web; +using Examine.Config; +using umbraco; +using umbraco.BusinessLogic; +using umbraco.cms.businesslogic; +using umbraco.cms.businesslogic.media; +using umbraco.cms.businesslogic.web; +using Examine; +using Examine.Providers; +using umbraco.cms.businesslogic.member; +using Examine.LuceneEngine; +using Examine.LuceneEngine.Providers; +using Lucene.Net.Index; +using Lucene.Net.Store; + +namespace UmbracoExamine +{ + /// + /// An instance for wiring up Examine to the Umbraco events system + /// + public class UmbracoEventManager : ApplicationBase + { + /// + /// Creates a new instance of the class + /// + [SecuritySafeCritical] + public UmbracoEventManager() + { + var registeredProviders = ExamineManager.Instance.IndexProviderCollection.OfType() + .Where(x => x.EnableDefaultEventHandler) + .Count(); + + Log.Add(LogTypes.Custom, -1, "[UmbracoExamine] Adding examine event handlers for index providers: " + registeredProviders.ToString()); + + //don't bind event handlers if we're not suppose to listen + if (registeredProviders == 0) + return; + + Media.AfterSave += Media_AfterSave; + Media.AfterDelete += Media_AfterDelete; + CMSNode.AfterMove += Media_AfterMove; + + //These should only fire for providers that DONT have SupportUnpublishedContent set to true + content.AfterUpdateDocumentCache += content_AfterUpdateDocumentCache; + content.AfterClearDocumentCache += content_AfterClearDocumentCache; + + //These should only fire for providers that have SupportUnpublishedContent set to true + Document.AfterSave += Document_AfterSave; + Document.AfterDelete += Document_AfterDelete; + + Member.AfterSave += Member_AfterSave; + Member.AfterDelete += Member_AfterDelete; + } + + //This does work, however we need to lock down the httphandler and thats an issue... so i've removed this + //if people don't want to rebuild on app startup then they can disable it and reindex manually. + + [SecuritySafeCritical] + private void Member_AfterSave(Member sender, SaveEventArgs e) + { + //ensure that only the providers are flagged to listen execute + var xml = sender.ToXml(new System.Xml.XmlDocument(), false).ToXElement(); + var providers = ExamineManager.Instance.IndexProviderCollection.OfType() + .Where(x => x.EnableDefaultEventHandler); + ExamineManager.Instance.ReIndexNode(xml, IndexTypes.Member, providers); + } + + [SecuritySafeCritical] + private void Member_AfterDelete(Member sender, DeleteEventArgs e) + { + var nodeId = sender.Id.ToString(); + + //ensure that only the providers are flagged to listen execute + ExamineManager.Instance.DeleteFromIndex(nodeId, + ExamineManager.Instance.IndexProviderCollection.OfType() + .Where(x => x.EnableDefaultEventHandler)); + } + + /// + /// Only index using providers that SupportUnpublishedContent + /// + /// + /// + [SecuritySafeCritical] + private void Document_AfterSave(Document sender, SaveEventArgs e) + { + //ensure that only the providers that have unpublishing support enabled + //that are also flagged to listen + + //there's a bug in 4.0.x that fires the Document saving event handler for media when media is moved, + //therefore, we need to try to figure out if this is media or content. Currently one way to do this + //is by checking the creator ID property as it will be null if it is media. We then need to try to + //create the media object, see if it exists, and pass it to the media save event handler... yeah i know, + //pretty horrible but has to be done. + + try + { + var creator = sender.Creator; + if (creator != null) + { + //it's def a Document + ExamineManager.Instance.ReIndexNode(sender.ToXDocument(false).Root, IndexTypes.Content, + ExamineManager.Instance.IndexProviderCollection.OfType() + .Where(x => x.SupportUnpublishedContent + && x.EnableDefaultEventHandler)); + + return; //exit, we've indexed the content + } + } + catch (Exception) + { + //if we get this exception, it means it's most likely media, so well do our check next. + } + + //this is most likely media, not sure what kind of exception might get thrown in 4.0.x or 4.1 if accessing a null + //creator, so we catch everything. + + var m = new Media(sender.Id); + if (!string.IsNullOrEmpty(m.Path)) + { + //this is a media item, no exception occurred on access to path and it's not empty which means it was found + Media_AfterSave(m, e); + return; + } + + + } + + /// + /// Only remove indexes using providers that SupportUnpublishedContent + /// + /// + /// + [SecuritySafeCritical] + private void Document_AfterDelete(Document sender, DeleteEventArgs e) + { + var nodeId = sender.Id.ToString(); + + //ensure that only the providers that have unpublishing support enabled + //that are also flagged to listen + ExamineManager.Instance.DeleteFromIndex(nodeId, + ExamineManager.Instance.IndexProviderCollection.OfType() + .Where(x => x.SupportUnpublishedContent + && x.EnableDefaultEventHandler)); + } + + [SecuritySafeCritical] + private void Media_AfterDelete(Media sender, DeleteEventArgs e) + { + var nodeId = sender.Id.ToString(); + + //ensure that only the providers are flagged to listen execute + ExamineManager.Instance.DeleteFromIndex(nodeId, + ExamineManager.Instance.IndexProviderCollection.OfType() + .Where(x => x.EnableDefaultEventHandler)); + } + + private void Media_AfterSave(Media sender, umbraco.cms.businesslogic.SaveEventArgs e) + { + //ensure that only the providers are flagged to listen execute + IndexMedia(sender); + } + + private void IndexMedia(Media sender) + { + ExamineManager.Instance.ReIndexNode(sender.ToXDocument(true).Root, IndexTypes.Media, + ExamineManager.Instance.IndexProviderCollection.OfType() + .Where(x => x.EnableDefaultEventHandler)); + } + + /// + /// When media is moved, re-index + /// + /// + /// + private void Media_AfterMove(object sender, MoveEventArgs e) + { + if (sender is Media) + { + Media m = (Media)sender; + IndexMedia(m); + } + } + + /// + /// Only Update indexes for providers that dont SupportUnpublishedContent + /// + /// + /// + private void content_AfterUpdateDocumentCache(Document sender, umbraco.cms.businesslogic.DocumentCacheEventArgs e) + { + //ensure that only the providers that have DONT unpublishing support enabled + //that are also flagged to listen + ExamineManager.Instance.ReIndexNode(sender.ToXDocument(true).Root, IndexTypes.Content, + ExamineManager.Instance.IndexProviderCollection.OfType() + .Where(x => !x.SupportUnpublishedContent + && x.EnableDefaultEventHandler)); + } + + /// + /// Only update indexes for providers that don't SupportUnpublishedContnet + /// + /// + /// + [SecuritySafeCritical] + private void content_AfterClearDocumentCache(Document sender, DocumentCacheEventArgs e) + { + var nodeId = sender.Id.ToString(); + //ensure that only the providers that DONT have unpublishing support enabled + //that are also flagged to listen + ExamineManager.Instance.DeleteFromIndex(nodeId, + ExamineManager.Instance.IndexProviderCollection.OfType() + .Where(x => !x.SupportUnpublishedContent + && x.EnableDefaultEventHandler)); + } + + } +} diff --git a/src/UmbracoExamine/UmbracoExamine.csproj b/src/UmbracoExamine/UmbracoExamine.csproj new file mode 100644 index 0000000000..108e745f4f --- /dev/null +++ b/src/UmbracoExamine/UmbracoExamine.csproj @@ -0,0 +1,179 @@ + + + + Debug + AnyCPU + 9.0.21022 + 2.0 + {07FBC26B-2927-4A22-8D96-D644C667FECC} + Library + Properties + UmbracoExamine + UmbracoExamine + v4.0 + 512 + + + + + + + + + + + + + 3.5 + publish\ + true + Disk + false + Foreground + 7 + Days + false + false + true + 0 + 1.0.0.%2a + false + false + true + + ..\ + true + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + SecurityRules.ruleset + false + + + false + false + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + AllRules.ruleset + bin\Release\UmbracoExamine.XML + false + + + false + + + true + + + ..\Solution Items\TheFARM-Public.snk + + + + ..\packages\Examine.0.1.42.2941\lib\Examine.dll + + + False + ..\packages\SharpZipLib.0.86.0\lib\20\ICSharpCode.SharpZipLib.dll + + + False + ..\packages\Lucene.Net.2.9.4.1\lib\net40\Lucene.Net.dll + + + + + 3.5 + + + + + + 3.5 + + + + + + + + + + + + + + + + + + + + + + + + + + + False + .NET Framework 3.5 SP1 Client Profile + false + + + False + .NET Framework 3.5 SP1 + true + + + False + Windows Installer 3.1 + true + + + + + {e469a9ce-1bec-423f-ac44-713cd72457ea} + umbraco.businesslogic + + + {ccd75ec3-63db-4184-b49d-51c1dd337230} + umbraco.cms + + + {c7cb79f0-1c97-4b33-bfa7-00731b579ae2} + umbraco.datalayer + + + {511f6d8d-7717-440a-9a57-a507e9a8b27f} + umbraco.interfaces + + + {651e1350-91b6-44b7-bd60-7207006d7003} + Umbraco.Web + + + + + + + + + \ No newline at end of file diff --git a/src/UmbracoExamine/UmbracoExamineSearcher.cs b/src/UmbracoExamine/UmbracoExamineSearcher.cs new file mode 100644 index 0000000000..5fe5d5ba87 --- /dev/null +++ b/src/UmbracoExamine/UmbracoExamineSearcher.cs @@ -0,0 +1,83 @@ +using System; +using System.IO; +using System.Linq; +using System.Security; +using Examine; +using Examine.Providers; +using Examine.SearchCriteria; +using UmbracoExamine.Config; +using Examine.LuceneEngine; +using Examine.LuceneEngine.Providers; +using Examine.LuceneEngine.SearchCriteria; +using Lucene.Net.Analysis; + + +namespace UmbracoExamine +{ + /// + /// An Examine searcher which uses Lucene.Net as the + /// + public class UmbracoExamineSearcher : LuceneSearcher + { + + #region Constructors + + /// + /// Default constructor + /// + public UmbracoExamineSearcher() + : base() + { + } + + /// + /// Constructor to allow for creating an indexer at runtime + /// + /// + /// + [SecuritySafeCritical] + public UmbracoExamineSearcher(DirectoryInfo indexPath, Analyzer analyzer) + : base(indexPath, analyzer) + { + } + + /// + /// Constructor to allow for creating an indexer at runtime + /// + /// + /// + [SecuritySafeCritical] + public UmbracoExamineSearcher(Lucene.Net.Store.Directory luceneDirectory, Analyzer analyzer) + : base(luceneDirectory, analyzer) + { + } + + #endregion + + /// + /// Override in order to set the nodeTypeAlias field name of the underlying SearchCriteria to __NodeTypeAlias + /// + /// + /// + /// + public override ISearchCriteria CreateSearchCriteria(string type, BooleanOperation defaultOperation) + { + var criteria = base.CreateSearchCriteria(type, defaultOperation) as LuceneSearchCriteria; + criteria.NodeTypeAliasField = UmbracoContentIndexer.NodeTypeAliasFieldName; + return criteria; + } + + /// + /// Returns a list of fields to search on, this will also exclude the IndexPathFieldName and node type alias + /// + /// + protected internal override string[] GetSearchFields() + { + var fields = base.GetSearchFields(); + return fields + .Where(x => x != UmbracoContentIndexer.IndexPathFieldName) + .Where(x => x != UmbracoContentIndexer.NodeTypeAliasFieldName) + .ToArray(); + } + } +} diff --git a/src/UmbracoExamine/UmbracoMemberIndexer.cs b/src/UmbracoExamine/UmbracoMemberIndexer.cs new file mode 100644 index 0000000000..94a82b9818 --- /dev/null +++ b/src/UmbracoExamine/UmbracoMemberIndexer.cs @@ -0,0 +1,87 @@ +using System.Collections; +using System.Linq; +using System.Security; +using System.Xml.Linq; +using System.Xml.XPath; +using umbraco.cms.businesslogic.member; +using Examine.LuceneEngine; +using System.Collections.Generic; +using Examine; +using System.IO; +using UmbracoExamine.DataServices; +using Lucene.Net.Analysis; + +namespace UmbracoExamine +{ + /// + /// + /// + public class UmbracoMemberIndexer : UmbracoContentIndexer + { + + /// + /// Default constructor + /// + public UmbracoMemberIndexer() + : base() { } + + /// + /// Constructor to allow for creating an indexer at runtime + /// + /// + /// + /// + /// + [SecuritySafeCritical] + public UmbracoMemberIndexer(IIndexCriteria indexerData, DirectoryInfo indexPath, IDataService dataService, Analyzer analyzer, bool async) + : base(indexerData, indexPath, dataService, analyzer, async) { } + + /// + /// The supported types for this indexer + /// + protected override IEnumerable SupportedTypes + { + get + { + return new string[] { IndexTypes.Member }; + } + } + + [SecuritySafeCritical] + protected override XDocument GetXDocument(string xPath, string type) + { + if (type == IndexTypes.Member) + { + Member[] rootMembers = Member.GetAll; + var xmlMember = XDocument.Parse(""); + foreach (Member member in rootMembers) + { + xmlMember.Root.Add(GetMemberItem(member.Id)); + } + var result = ((IEnumerable)xmlMember.XPathEvaluate(xPath)).Cast(); + return result.ToXDocument(); + } + + return null; + } + + protected override System.Collections.Generic.Dictionary GetDataToIndex(XElement node, string type) + { + var data = base.GetDataToIndex(node, type); + + if (data.ContainsKey("email")) + { + data.Add("__email",data["email"].Replace("."," ").Replace("@"," ")); + } + + return data; + } + + [SecuritySafeCritical] + private XElement GetMemberItem(int nodeId) + { + var nodes = umbraco.library.GetMember(nodeId); + return XElement.Parse(nodes.Current.OuterXml); + } + } +} diff --git a/src/UmbracoExamine/XsltExtensions.cs b/src/UmbracoExamine/XsltExtensions.cs new file mode 100644 index 0000000000..7944db42da --- /dev/null +++ b/src/UmbracoExamine/XsltExtensions.cs @@ -0,0 +1,265 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Xml.Linq; +using System.Xml.XPath; +using Examine; +using Examine.LuceneEngine.Providers; +using Examine.LuceneEngine.SearchCriteria; +using Examine.SearchCriteria; +using Examine.Providers; +using UmbracoExamine.DataServices; + +namespace UmbracoExamine +{ + /// + /// Methods to support Umbraco XSLT extensions. + /// + /// + /// XSLT extensions will ONLY work for provider that have a base class of BaseUmbracoIndexer + /// + public class XsltExtensions + { + /// + /// Uses the provider specified to search, returning an XPathNodeIterator + /// + /// + /// + /// + /// + /// + internal static XPathNodeIterator Search(string searchText, bool useWildcards, LuceneSearcher provider, string indexType) + { + if (provider == null) throw new ArgumentNullException("provider"); + + var results = provider.Search(searchText, useWildcards, indexType); + return GetResultsAsXml(results); + } + + /// + /// Uses the provider specified to search, returning an XPathNodeIterator + /// + /// The search text. + /// if set to true [use wildcards]. + /// Name of the provider. + /// Type of the index. + /// + public static XPathNodeIterator Search(string searchText, bool useWildcards, string providerName, string indexType) + { + EnsureProvider(ExamineManager.Instance.SearchProviderCollection[providerName]); + + var provider = ExamineManager.Instance.SearchProviderCollection[providerName] as LuceneSearcher; + + return Search(searchText, useWildcards, provider, indexType); + } + + /// + /// Uses the provider specified to search, returning an XPathNodeIterator + /// + /// + /// + /// + /// + public static XPathNodeIterator Search(string searchText, bool useWildcards, string providerName) + { + return Search(searchText, useWildcards, providerName, string.Empty); + } + + /// + /// Uses the default provider specified to search, returning an XPathNodeIterator + /// + /// The search query + /// Enable a wildcard search query + /// A node-set of the search results + public static XPathNodeIterator Search(string searchText, bool useWildcards) + { + return Search(searchText, useWildcards, ExamineManager.Instance.DefaultSearchProvider.Name); + } + + /// + /// Uses the default provider specified to search, returning an XPathNodeIterator + /// + /// The search query + /// A node-set of the search results + public static XPathNodeIterator Search(string searchText) + { + return Search(searchText, true); + } + + /// + /// Will perform a search against the media index type only + /// + /// + /// + /// + /// + public static XPathNodeIterator SearchMediaOnly(string searchText, bool useWildcards, string providerName) + { + return Search(searchText, useWildcards, providerName, IndexTypes.Media); + } + + /// + /// Will perform a search against the media index type only + /// + /// + /// + /// + public static XPathNodeIterator SearchMediaOnly(string searchText, bool useWildcards) + { + return SearchMediaOnly(searchText, useWildcards, ExamineManager.Instance.DefaultSearchProvider.Name); + } + + /// + /// Will perform a search against the media index type only + /// + /// + /// + public static XPathNodeIterator SearchMediaOnly(string searchText) + { + return SearchMediaOnly(searchText, true); + } + + /// + /// Searches the member only. + /// + /// The search text. + /// if set to true [use wildcards]. + /// Name of the provider. + /// + public static XPathNodeIterator SearchMemberOnly(string searchText, bool useWildcards, string providerName) + { + return Search(searchText, useWildcards, providerName, IndexTypes.Member); + } + + /// + /// Searches the member only. + /// + /// The search text. + /// if set to true [use wildcards]. + /// + public static XPathNodeIterator SearchMemberOnly(string searchText, bool useWildcards) + { + return SearchMemberOnly(searchText, useWildcards, ExamineManager.Instance.DefaultSearchProvider.Name); + } + + /// + /// Searches the member only. + /// + /// The search text. + /// + public static XPathNodeIterator SearchMemberOnly(string searchText) + { + return SearchMemberOnly(searchText, true); + } + + /// + /// Will perform a search against the content index type only + /// + /// + /// + /// + /// + public static XPathNodeIterator SearchContentOnly(string searchText, bool useWildcards, string providerName) + { + return Search(searchText, useWildcards, providerName, IndexTypes.Content); + } + + + /// + /// Will perform a search against the content index type only + /// + /// + /// + /// + public static XPathNodeIterator SearchContentOnly(string searchText, bool useWildcards) + { + return SearchContentOnly(searchText, useWildcards, ExamineManager.Instance.DefaultSearchProvider.Name); + } + + /// + /// Will perform a search against the content index type only + /// + /// + /// + public static XPathNodeIterator SearchContentOnly(string searchText) + { + return SearchContentOnly(searchText, true); + } + + /// + /// Ensures the provider. + /// + /// The provider. + private static void EnsureProvider(BaseSearchProvider p) + { + if (!(p is LuceneSearcher)) + { + throw new NotSupportedException("XSLT Extensions are only support for providers of type LuceneSearcher"); + } + } + + /// + /// Gets the results as XML. + /// + /// The results. + /// + private static XPathNodeIterator GetResultsAsXml(ISearchResults results) + { + // create the XDocument + XDocument doc = new XDocument(); + + // check there are any search results + if (results.TotalItemCount > 0) + { + // create the root element + XElement root = new XElement("nodes"); + + // iterate through the search results + foreach (SearchResult result in results) + { + // create a new element + XElement node = new XElement("node"); + + // create the @id attribute + XAttribute nodeId = new XAttribute("id", result.Id); + + // create the @score attribute + XAttribute nodeScore = new XAttribute("score", result.Score); + + // add the content + node.Add(nodeId, nodeScore); + + foreach (KeyValuePair field in result.Fields) + { + // create a new element + XElement data = new XElement("data"); + + // create the @alias attribute + XAttribute alias = new XAttribute("alias", field.Key); + + // assign the value to a CDATA section + XCData value = new XCData(field.Value); + + // append the content + data.Add(alias, value); + + // append the element + node.Add(data); + } + + // add the node + root.Add(node); + } + + // add the root node + doc.Add(root); + } + else + { + doc.Add(new XElement("error", "There were no search results.")); + } + + return doc.CreateNavigator().Select("/"); + } + } +} \ No newline at end of file diff --git a/src/UmbracoExamine/packages.config b/src/UmbracoExamine/packages.config new file mode 100644 index 0000000000..0aa01cdb4a --- /dev/null +++ b/src/UmbracoExamine/packages.config @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/src/packages/repositories.config b/src/packages/repositories.config index f811520cac..dc0774dd5e 100644 --- a/src/packages/repositories.config +++ b/src/packages/repositories.config @@ -12,4 +12,5 @@ + \ No newline at end of file diff --git a/src/umbraco.sln b/src/umbraco.sln index 87674b9000..2e4bfa86e6 100644 --- a/src/umbraco.sln +++ b/src/umbraco.sln @@ -59,6 +59,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".nuget", ".nuget", "{66AB04 .nuget\NuGet.targets = .nuget\NuGet.targets EndProjectSection EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "UmbracoExamine", "UmbracoExamine\UmbracoExamine.csproj", "{07FBC26B-2927-4A22-8D96-D644C667FECC}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "UmbracoExamineLibs", "UmbracoExamineLibs", "{DD32977B-EF54-475B-9A1B-B97A502C6E58}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -125,11 +129,16 @@ Global {5D3B8245-ADA6-453F-A008-50ED04BFE770}.Debug|Any CPU.Build.0 = Debug|Any CPU {5D3B8245-ADA6-453F-A008-50ED04BFE770}.Release|Any CPU.ActiveCfg = Release|Any CPU {5D3B8245-ADA6-453F-A008-50ED04BFE770}.Release|Any CPU.Build.0 = Release|Any CPU + {07FBC26B-2927-4A22-8D96-D644C667FECC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {07FBC26B-2927-4A22-8D96-D644C667FECC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {07FBC26B-2927-4A22-8D96-D644C667FECC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {07FBC26B-2927-4A22-8D96-D644C667FECC}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection GlobalSection(NestedProjects) = preSolution {5D3B8245-ADA6-453F-A008-50ED04BFE770} = {B5BD12C1-A454-435E-8A46-FF4A364C0382} + {07FBC26B-2927-4A22-8D96-D644C667FECC} = {DD32977B-EF54-475B-9A1B-B97A502C6E58} EndGlobalSection EndGlobal