From 33036f80fea0d6e62fd6e0b48e5eeec6bb0379b9 Mon Sep 17 00:00:00 2001 From: Shannon Date: Thu, 13 Nov 2014 14:01:13 +1100 Subject: [PATCH] Fixes: U4-5789 Add native option to UmbracoExamine library to run the indexes locally --- src/UmbracoExamine/BaseUmbracoIndexer.cs | 13 +- src/UmbracoExamine/ContentExtensions.cs | 2 - .../DataServices/UmbracoContentService.cs | 10 +- .../DataServices/UmbracoLogService.cs | 6 +- .../DataServices/UmbracoMediaService.cs | 6 +- .../LocalStorage/LocalTempStorageDirectory.cs | 2 +- src/UmbracoExamine/Properties/AssemblyInfo.cs | 6 +- src/UmbracoExamine/UmbracoContentIndexer.cs | 12 +- src/UmbracoExamine/UmbracoExamineSearcher.cs | 8 +- src/UmbracoExamine/UmbracoMemberIndexer.cs | 128 ++++++------------ src/UmbracoExamine/XsltExtensions.cs | 2 +- src/umbraco.sln | 2 +- 12 files changed, 72 insertions(+), 125 deletions(-) diff --git a/src/UmbracoExamine/BaseUmbracoIndexer.cs b/src/UmbracoExamine/BaseUmbracoIndexer.cs index 8ccef8b22e..1ab0eae498 100644 --- a/src/UmbracoExamine/BaseUmbracoIndexer.cs +++ b/src/UmbracoExamine/BaseUmbracoIndexer.cs @@ -42,14 +42,12 @@ namespace UmbracoExamine /// /// /// - [SecuritySafeCritical] protected BaseUmbracoIndexer(IIndexCriteria indexerData, DirectoryInfo indexPath, IDataService dataService, Analyzer analyzer, bool async) : base(indexerData, indexPath, analyzer, async) { DataService = dataService; } - [SecuritySafeCritical] protected BaseUmbracoIndexer(IIndexCriteria indexerData, Lucene.Net.Store.Directory luceneDirectory, IDataService dataService, Analyzer analyzer, bool async) : base(indexerData, luceneDirectory, analyzer, async) { @@ -96,7 +94,6 @@ namespace UmbracoExamine /// /// /// - [SecuritySafeCritical] public override void Initialize(string name, System.Collections.Specialized.NameValueCollection config) { if (config["dataService"] != null && !string.IsNullOrEmpty(config["dataService"])) @@ -217,7 +214,6 @@ namespace UmbracoExamine /// Returns true if the Umbraco application is in a state that we can initialize the examine indexes /// /// - [SecuritySafeCritical] protected bool CanInitialize() { //check the DisableInitializationCheck and ensure that it is not set to true @@ -268,6 +264,9 @@ namespace UmbracoExamine /// protected override void PerformIndexAll(string type) { + //TODO: Fix all of this up, the whole xpath thing is horrible and was made sooooooooo long ago to only work with published content + // but not it's being used for all content types and is really bad for performance. + if (!SupportedTypes.Contains(type)) return; @@ -360,11 +359,9 @@ namespace UmbracoExamine XDocument xDoc = GetXDocument(xPath, type); if (xDoc != null) { - XElement rootNode = xDoc.Root; + var rootNode = xDoc.Root; - IEnumerable children = rootNode.Elements(); - - AddNodesToIndex(children, type); + AddNodesToIndex(rootNode.Elements(), type); } } diff --git a/src/UmbracoExamine/ContentExtensions.cs b/src/UmbracoExamine/ContentExtensions.cs index 541831c4cb..afe61bfdb4 100644 --- a/src/UmbracoExamine/ContentExtensions.cs +++ b/src/UmbracoExamine/ContentExtensions.cs @@ -28,7 +28,6 @@ namespace UmbracoExamine /// If the type of node is not a Document, the cacheOnly has no effect, it will use the API to return /// the xml. /// - [SecuritySafeCritical] [Obsolete("This method is no longer used and will be removed in future versions")] public static XDocument ToXDocument(this Content node, bool cacheOnly) { @@ -52,7 +51,6 @@ namespace UmbracoExamine /// /// /// - [SecuritySafeCritical] [Obsolete("This method is no longer used and will be removed in future versions")] private static XDocument ToXDocument(this Content node) { diff --git a/src/UmbracoExamine/DataServices/UmbracoContentService.cs b/src/UmbracoExamine/DataServices/UmbracoContentService.cs index 76789b19ee..ac3a693b50 100644 --- a/src/UmbracoExamine/DataServices/UmbracoContentService.cs +++ b/src/UmbracoExamine/DataServices/UmbracoContentService.cs @@ -29,14 +29,12 @@ namespace UmbracoExamine.DataServices private readonly ApplicationContext _applicationContext; - [SecuritySafeCritical] public UmbracoContentService() : this(ApplicationContext.Current) { } - [SecuritySafeCritical] public UmbracoContentService(ApplicationContext applicationContext) { _applicationContext = applicationContext; @@ -47,7 +45,6 @@ namespace UmbracoExamine.DataServices /// /// /// - [SecuritySafeCritical] public string StripHtml(string value) { return value.StripHtml(); @@ -58,7 +55,6 @@ namespace UmbracoExamine.DataServices /// /// /// - [SecuritySafeCritical] public XDocument GetPublishedContentByXPath(string xpath) { //TODO: Remove the need for this, the best way would be to remove all requirements of examine based on Xml but that @@ -74,7 +70,7 @@ namespace UmbracoExamine.DataServices /// /// /// - [SecuritySafeCritical] + public XDocument GetLatestContentByXPath(string xpath) { var xmlContent = XDocument.Parse(""); @@ -93,7 +89,7 @@ namespace UmbracoExamine.DataServices /// /// /// - [SecuritySafeCritical] + private static XmlNode GetPage(int documentId) { var xDoc = Access.GetXmlDocumentCopy(); @@ -118,7 +114,7 @@ namespace UmbracoExamine.DataServices /// Returns a list of all of the user defined property names in Umbraco /// /// - [SecuritySafeCritical] + public IEnumerable GetAllUserPropertyNames() { try diff --git a/src/UmbracoExamine/DataServices/UmbracoLogService.cs b/src/UmbracoExamine/DataServices/UmbracoLogService.cs index 7589e4ec2f..8768cded04 100644 --- a/src/UmbracoExamine/DataServices/UmbracoLogService.cs +++ b/src/UmbracoExamine/DataServices/UmbracoLogService.cs @@ -12,13 +12,13 @@ namespace UmbracoExamine.DataServices { public string ProviderName { get; set; } - [SecuritySafeCritical] + public void AddInfoLog(int nodeId, string msg) { LogHelper.Info("{0}, Provider={1}, NodeId={2}", () => msg, () => ProviderName, () => nodeId); } - [SecuritySafeCritical] + public void AddErrorLog(int nodeId, string msg) { //NOTE: not really the prettiest but since AddErrorLog is legacy code, we cannot change it now to accept a real Exception obj for @@ -28,7 +28,7 @@ namespace UmbracoExamine.DataServices new Exception(msg)); } - [SecuritySafeCritical] + public void AddVerboseLog(int nodeId, string msg) { LogHelper.Debug("{0}, Provider={1}, NodeId={2}", () => msg, () => ProviderName, () => nodeId); diff --git a/src/UmbracoExamine/DataServices/UmbracoMediaService.cs b/src/UmbracoExamine/DataServices/UmbracoMediaService.cs index cc505f732b..229169aa3d 100644 --- a/src/UmbracoExamine/DataServices/UmbracoMediaService.cs +++ b/src/UmbracoExamine/DataServices/UmbracoMediaService.cs @@ -19,14 +19,14 @@ namespace UmbracoExamine.DataServices { private readonly ServiceContext _services; - [SecuritySafeCritical] + public UmbracoMediaService() : this(ApplicationContext.Current.Services) { } - [SecuritySafeCritical] + public UmbracoMediaService(ServiceContext services) { _services = services; @@ -39,7 +39,7 @@ namespace UmbracoExamine.DataServices /// /// /// - [SecuritySafeCritical] + public XDocument GetLatestMediaByXpath(string xpath) { var xmlMedia = XDocument.Parse(""); diff --git a/src/UmbracoExamine/LocalStorage/LocalTempStorageDirectory.cs b/src/UmbracoExamine/LocalStorage/LocalTempStorageDirectory.cs index 8c903a850d..64d2765d00 100644 --- a/src/UmbracoExamine/LocalStorage/LocalTempStorageDirectory.cs +++ b/src/UmbracoExamine/LocalStorage/LocalTempStorageDirectory.cs @@ -49,8 +49,8 @@ namespace UmbracoExamine.LocalStorage public override void DeleteFile(string name) { //perform on both dirs - _realDirectory.DeleteFile(name); base.DeleteFile(name); + _realDirectory.DeleteFile(name); } /// Returns the length of a file in the directory. diff --git a/src/UmbracoExamine/Properties/AssemblyInfo.cs b/src/UmbracoExamine/Properties/AssemblyInfo.cs index 4997aae313..5a7b8f844d 100644 --- a/src/UmbracoExamine/Properties/AssemblyInfo.cs +++ b/src/UmbracoExamine/Properties/AssemblyInfo.cs @@ -28,9 +28,7 @@ using System.Security; //NOTE: WE cannot make change the major version to be the same as Umbraco because of backwards compatibility, however we // will make the minor version the same as the umbraco version -[assembly: AssemblyVersion("0.6.0.*")] -[assembly: AssemblyFileVersion("0.6.0.*")] - -[assembly: AllowPartiallyTrustedCallers] +[assembly: AssemblyVersion("0.7.0.*")] +[assembly: AssemblyFileVersion("0.7.0.*")] [assembly: InternalsVisibleTo("Umbraco.Tests")] diff --git a/src/UmbracoExamine/UmbracoContentIndexer.cs b/src/UmbracoExamine/UmbracoContentIndexer.cs index 921784cdd0..ce15753c64 100644 --- a/src/UmbracoExamine/UmbracoContentIndexer.cs +++ b/src/UmbracoExamine/UmbracoContentIndexer.cs @@ -47,7 +47,7 @@ namespace UmbracoExamine /// /// /// - [SecuritySafeCritical] + public UmbracoContentIndexer(IIndexCriteria indexerData, DirectoryInfo indexPath, IDataService dataService, Analyzer analyzer, bool async) : base(indexerData, indexPath, dataService, analyzer, async) { } @@ -59,7 +59,7 @@ namespace UmbracoExamine /// /// /// - [SecuritySafeCritical] + public UmbracoContentIndexer(IIndexCriteria indexerData, Lucene.Net.Store.Directory luceneDirectory, IDataService dataService, Analyzer analyzer, bool async) : base(indexerData, luceneDirectory, dataService, analyzer, async) { } @@ -128,8 +128,8 @@ namespace UmbracoExamine /// /// An attempt is made to call on a provider after the provider has already been initialized. /// - [SecuritySafeCritical] - public override void Initialize(string name, System.Collections.Specialized.NameValueCollection config) + + public override void Initialize(string name, NameValueCollection config) { //check if there's a flag specifying to support unpublished content, @@ -201,7 +201,7 @@ namespace UmbracoExamine /// This ensures that the special __Raw_ fields are indexed /// /// - [SecuritySafeCritical] + protected override void OnDocumentWriting(DocumentWritingEventArgs docArgs) { var d = docArgs.Document; @@ -363,7 +363,7 @@ namespace UmbracoExamine /// ensure our special Path field is added to the collection /// /// - [SecuritySafeCritical] + protected override void OnGatheringNodeData(IndexingNodeDataEventArgs e) { //strip html of all users fields if we detect it has HTML in it. diff --git a/src/UmbracoExamine/UmbracoExamineSearcher.cs b/src/UmbracoExamine/UmbracoExamineSearcher.cs index d295e09480..a8eb88466b 100644 --- a/src/UmbracoExamine/UmbracoExamineSearcher.cs +++ b/src/UmbracoExamine/UmbracoExamineSearcher.cs @@ -54,7 +54,7 @@ namespace UmbracoExamine } } - [SecuritySafeCritical] + public override void Initialize(string name, System.Collections.Specialized.NameValueCollection config) { if (name == null) throw new ArgumentNullException("name"); @@ -99,7 +99,7 @@ namespace UmbracoExamine /// /// /// - [SecuritySafeCritical] + public UmbracoExamineSearcher(DirectoryInfo indexPath, Analyzer analyzer) : base(indexPath, analyzer) { @@ -110,7 +110,7 @@ namespace UmbracoExamine /// /// /// - [SecuritySafeCritical] + public UmbracoExamineSearcher(Lucene.Net.Store.Directory luceneDirectory, Analyzer analyzer) : base(luceneDirectory, analyzer) { @@ -127,7 +127,7 @@ namespace UmbracoExamine /// Returns true if the Umbraco application is in a state that we can initialize the examine indexes /// /// - [SecuritySafeCritical] + protected bool CanInitialize() { //check the DisableInitializationCheck and ensure that it is not set to true diff --git a/src/UmbracoExamine/UmbracoMemberIndexer.cs b/src/UmbracoExamine/UmbracoMemberIndexer.cs index 2ac64cb712..35768af0d1 100644 --- a/src/UmbracoExamine/UmbracoMemberIndexer.cs +++ b/src/UmbracoExamine/UmbracoMemberIndexer.cs @@ -1,15 +1,13 @@ -using System.Collections; +using System; using System.Collections.Specialized; using System.Linq; -using System.Security; using System.Xml.Linq; -using System.Xml.XPath; using Examine.LuceneEngine.Config; using Lucene.Net.Index; using Umbraco.Core; +using Umbraco.Core.Models; +using Umbraco.Core.Services; using UmbracoExamine.Config; -using umbraco.cms.businesslogic.member; -using Examine.LuceneEngine; using System.Collections.Generic; using Examine; using System.IO; @@ -40,72 +38,11 @@ namespace UmbracoExamine /// /// /// - [SecuritySafeCritical] + public UmbracoMemberIndexer(IIndexCriteria indexerData, DirectoryInfo indexPath, IDataService dataService, Analyzer analyzer, bool async) : base(indexerData, indexPath, dataService, analyzer, async) { } - /// - /// Set up all properties for the indexer based on configuration information specified. This will ensure that - /// all of the folders required by the indexer are created and exist. This will also create an instruction - /// file declaring the computer name that is part taking in the indexing. This file will then be used to - /// determine the master indexer machine in a load balanced environment (if one exists). - /// - /// The friendly name of the provider. - /// A collection of the name/value pairs representing the provider-specific attributes specified in the configuration for this provider. - /// - /// The name of the provider is null. - /// - /// - /// The name of the provider has a length of zero. - /// - /// - /// An attempt is made to call on a provider after the provider has already been initialized. - /// - public override void Initialize(string name, NameValueCollection config) - { - base.Initialize(name, config); - - if (config != null && config["useTempStorage"] != null) - { - //Use the temp storage directory which will store the index in the local/codegen folder, this is useful - // for websites that are running from a remove file server and file IO latency becomes an issue - var attemptUseTempStorage = config["useTempStorage"].TryConvertTo(); - if (attemptUseTempStorage) - { - var indexSet = IndexSets.Instance.Sets[IndexSetName]; - var configuredPath = indexSet.IndexPath; - - _localTempStorageHelper.Initialize(config, configuredPath, base.GetLuceneDirectory(), IndexingAnalyzer); - } - } - } - - public override Lucene.Net.Store.Directory GetLuceneDirectory() - { - //if temp local storage is configured use that, otherwise return the default - if (_localTempStorageHelper.LuceneDirectory != null) - { - return _localTempStorageHelper.LuceneDirectory; - } - - return base.GetLuceneDirectory(); - - } - - public override IndexWriter GetIndexWriter() - { - //if temp local storage is configured use that, otherwise return the default - if (_localTempStorageHelper.LuceneDirectory != null) - { - return new IndexWriter(GetLuceneDirectory(), IndexingAnalyzer, - //create the writer with the snapshotter, though that won't make too much a difference because we are not keeping the writer open unless using nrt - // which we are not currently. - _localTempStorageHelper.Snapshotter, - IndexWriter.MaxFieldLength.UNLIMITED); - } - - return base.GetIndexWriter(); - } + /// /// Ensures that the'_searchEmail' is added to the user fields so that it is indexed - without having to modify the config @@ -141,23 +78,45 @@ namespace UmbracoExamine } } - [SecuritySafeCritical] - protected override XDocument GetXDocument(string xPath, string type) - { - if (type == IndexTypes.Member) - { - Member[] rootMembers = Member.GetAll; - var xmlMember = XDocument.Parse(""); - foreach (Member member in rootMembers) - { - xmlMember.Root.Add(GetMemberItem(member.Id)); - } - var result = ((IEnumerable)xmlMember.XPathEvaluate(xPath)).Cast(); - return result.ToXDocument(); - } + /// + /// Reindex all members + /// + /// + protected override void PerformIndexAll(string type) + { + //This only supports members + if (!SupportedTypes.Contains(type)) + return; - return null; - } + //Re-index all members in batches of 5000 + IEnumerable members; + const int pageSize = 5000; + var pageIndex = 0; + do + { + int total; + members = ApplicationContext.Current.Services.MemberService.GetAll(pageIndex, pageSize, out total); + + //AddNodesToIndex(GetSerializedMembers(members), type); + + pageIndex++; + + } while (members.Count() == pageSize); + } + + private IEnumerable GetSerializedMembers(IEnumerable members) + { + var serializer = new EntityXmlSerializer(); + foreach (var member in members) + { + yield return serializer.Serialize(ApplicationContext.Current.Services.DataTypeService, member); + } + } + + protected override XDocument GetXDocument(string xPath, string type) + { + throw new NotSupportedException(); + } protected override Dictionary GetSpecialFieldsToIndex(Dictionary allValuesForIndexing) { @@ -192,7 +151,6 @@ namespace UmbracoExamine } } - [SecuritySafeCritical] private static XElement GetMemberItem(int nodeId) { //TODO: Change this so that it is not using the LegacyLibrary, just serialize manually! diff --git a/src/UmbracoExamine/XsltExtensions.cs b/src/UmbracoExamine/XsltExtensions.cs index 567ac0b3cc..785a703b1c 100644 --- a/src/UmbracoExamine/XsltExtensions.cs +++ b/src/UmbracoExamine/XsltExtensions.cs @@ -21,7 +21,7 @@ namespace UmbracoExamine /// XSLT extensions will ONLY work for provider that have a base class of BaseUmbracoIndexer /// [XsltExtension("Examine")] - [SecuritySafeCritical] + public class XsltExtensions { /// diff --git a/src/umbraco.sln b/src/umbraco.sln index 7b71ba11d2..7b1c21f88c 100644 --- a/src/umbraco.sln +++ b/src/umbraco.sln @@ -1,6 +1,6 @@ Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio 2013 -VisualStudioVersion = 12.0.30723.0 +VisualStudioVersion = 12.0.31101.0 MinimumVisualStudioVersion = 10.0.40219.1 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Umbraco.Web.UI", "Umbraco.Web.UI\Umbraco.Web.UI.csproj", "{4C4C194C-B5E4-4991-8F87-4373E24CC19F}" EndProject