diff --git a/src/Umbraco.Tests/Umbraco.Tests.csproj b/src/Umbraco.Tests/Umbraco.Tests.csproj index ba7925a2e1..074bb19d71 100644 --- a/src/Umbraco.Tests/Umbraco.Tests.csproj +++ b/src/Umbraco.Tests/Umbraco.Tests.csproj @@ -58,9 +58,9 @@ ..\packages\AutoMapper.3.0.0\lib\net40\AutoMapper.Net4.dll - + False - ..\packages\Examine.0.1.58.2941\lib\Examine.dll + ..\packages\Examine.0.1.59.2941\lib\Examine.dll False diff --git a/src/Umbraco.Tests/packages.config b/src/Umbraco.Tests/packages.config index 3324f4f82e..d548c63204 100644 --- a/src/Umbraco.Tests/packages.config +++ b/src/Umbraco.Tests/packages.config @@ -2,7 +2,7 @@ - + diff --git a/src/Umbraco.Web.UI/Global.asax b/src/Umbraco.Web.UI/Global.asax index 8cb0d2d910..9036e4acd6 100644 --- a/src/Umbraco.Web.UI/Global.asax +++ b/src/Umbraco.Web.UI/Global.asax @@ -1 +1,2 @@ <%@ Application Inherits="Umbraco.Web.UmbracoApplication" Language="C#" %> + \ No newline at end of file diff --git a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj index 9b8b1c1ad5..c605138beb 100644 --- a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj +++ b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj @@ -127,9 +127,9 @@ False ..\packages\dotless.1.4.1.0\lib\dotless.Core.dll - + False - ..\packages\Examine.0.1.58.2941\lib\Examine.dll + ..\packages\Examine.0.1.59.2941\lib\Examine.dll False diff --git a/src/Umbraco.Web.UI/config/ExamineSettings.config b/src/Umbraco.Web.UI/config/ExamineSettings.config index 131867ac31..3fa388732b 100644 --- a/src/Umbraco.Web.UI/config/ExamineSettings.config +++ b/src/Umbraco.Web.UI/config/ExamineSettings.config @@ -11,16 +11,19 @@ More information and documentation can be found on CodePlex: http://umbracoexami - + @@ -28,12 +31,15 @@ More information and documentation can be found on CodePlex: http://umbracoexami + analyzer="Lucene.Net.Analysis.WhitespaceAnalyzer, Lucene.Net" + useTempStorage="Sync"/> - + + analyzer="Lucene.Net.Analysis.Standard.StandardAnalyzer, Lucene.Net" enableLeadingWildcard="true" + useTempStorage="Sync"/> diff --git a/src/Umbraco.Web.UI/packages.config b/src/Umbraco.Web.UI/packages.config index f16b109766..e192fea003 100644 --- a/src/Umbraco.Web.UI/packages.config +++ b/src/Umbraco.Web.UI/packages.config @@ -4,7 +4,7 @@ - + diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index 80c913e750..0a37f35967 100644 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -115,9 +115,9 @@ ..\packages\dotless.1.4.1.0\lib\dotless.Core.dll - + False - ..\packages\Examine.0.1.58.2941\lib\Examine.dll + ..\packages\Examine.0.1.59.2941\lib\Examine.dll False diff --git a/src/Umbraco.Web/packages.config b/src/Umbraco.Web/packages.config index 0a976f2457..18fe65b5be 100644 --- a/src/Umbraco.Web/packages.config +++ b/src/Umbraco.Web/packages.config @@ -3,7 +3,7 @@ - + diff --git a/src/UmbracoExamine.PDF/UmbracoExamine.PDF.csproj b/src/UmbracoExamine.PDF/UmbracoExamine.PDF.csproj index 699b8c4017..c1f220676e 100644 --- a/src/UmbracoExamine.PDF/UmbracoExamine.PDF.csproj +++ b/src/UmbracoExamine.PDF/UmbracoExamine.PDF.csproj @@ -46,9 +46,9 @@ false - + False - ..\packages\Examine.0.1.58.2941\lib\Examine.dll + ..\packages\Examine.0.1.59.2941\lib\Examine.dll False diff --git a/src/UmbracoExamine.PDF/packages.config b/src/UmbracoExamine.PDF/packages.config index 8f5fa8e4cc..b4149306f2 100644 --- a/src/UmbracoExamine.PDF/packages.config +++ b/src/UmbracoExamine.PDF/packages.config @@ -1,6 +1,6 @@  - + diff --git a/src/UmbracoExamine/BaseUmbracoIndexer.cs b/src/UmbracoExamine/BaseUmbracoIndexer.cs index 351c7be985..aa0a5c0b63 100644 --- a/src/UmbracoExamine/BaseUmbracoIndexer.cs +++ b/src/UmbracoExamine/BaseUmbracoIndexer.cs @@ -8,6 +8,7 @@ using System.Threading; using System.Web; using Examine.LuceneEngine.Config; using Examine.LuceneEngine.Providers; +using Examine.Providers; using Lucene.Net.Analysis; using Lucene.Net.Documents; using Lucene.Net.Index; @@ -65,6 +66,7 @@ namespace UmbracoExamine /// internal static bool? DisableInitializationCheck = null; private readonly LocalTempStorageIndexer _localTempStorageHelper = new LocalTempStorageIndexer(); + private BaseLuceneSearcher _internalTempStorageSearcher = null; #region Properties @@ -101,19 +103,6 @@ namespace UmbracoExamine /// public override void Initialize(string name, System.Collections.Specialized.NameValueCollection config) { - if (config != null && config["useTempStorage"] != null) - { - //Use the temp storage directory which will store the index in the local/codegen folder, this is useful - // for websites that are running from a remove file server and file IO latency becomes an issue - var attemptUseTempStorage = config["useTempStorage"].TryConvertTo(); - if (attemptUseTempStorage) - { - var indexSet = IndexSets.Instance.Sets[IndexSetName]; - var configuredPath = indexSet.IndexPath; - - _localTempStorageHelper.Initialize(config, configuredPath, base.GetLuceneDirectory(), IndexingAnalyzer); - } - } if (config["dataService"] != null && !string.IsNullOrEmpty(config["dataService"])) { @@ -161,10 +150,41 @@ namespace UmbracoExamine ExamineHelper.ReplaceTokensInIndexPath(name, config, "Indexer", () => IndexerData != null); base.Initialize(name, config); + + if (config["useTempStorage"] != null) + { + //Use the temp storage directory which will store the index in the local/codegen folder, this is useful + // for websites that are running from a remove file server and file IO latency becomes an issue + var attemptUseTempStorage = config["useTempStorage"].TryConvertTo(); + if (attemptUseTempStorage) + { + + var indexSet = IndexSets.Instance.Sets[IndexSetName]; + var configuredPath = indexSet.IndexPath; + + _localTempStorageHelper.Initialize(config, configuredPath, base.GetLuceneDirectory(), IndexingAnalyzer, attemptUseTempStorage.Result); + } + } } #endregion + protected override BaseSearchProvider InternalSearcher + { + get + { + //if temp local storage is configured use that, otherwise return the default + if (_localTempStorageHelper.LuceneDirectory != null) + { + //create one if one has not been created already + return _internalTempStorageSearcher + ?? (_internalTempStorageSearcher = new LuceneSearcher(_localTempStorageHelper.LuceneDirectory, IndexingAnalyzer)); + } + + return base.InternalSearcher; + } + } + public override Lucene.Net.Store.Directory GetLuceneDirectory() { //if temp local storage is configured use that, otherwise return the default @@ -177,19 +197,17 @@ namespace UmbracoExamine } - public override IndexWriter GetIndexWriter() + protected override IndexWriter CreateIndexWriter() { //if temp local storage is configured use that, otherwise return the default if (_localTempStorageHelper.LuceneDirectory != null) { return new IndexWriter(GetLuceneDirectory(), IndexingAnalyzer, - //create the writer with the snapshotter, though that won't make too much a difference because we are not keeping the writer open unless using nrt - // which we are not currently. - _localTempStorageHelper.Snapshotter, + DeletePolicyTracker.Current.GetPolicy(IndexSetName), IndexWriter.MaxFieldLength.UNLIMITED); } - return base.GetIndexWriter(); + return base.CreateIndexWriter(); } ///// diff --git a/src/UmbracoExamine/DeletePolicyTracker.cs b/src/UmbracoExamine/DeletePolicyTracker.cs new file mode 100644 index 0000000000..d002cc108d --- /dev/null +++ b/src/UmbracoExamine/DeletePolicyTracker.cs @@ -0,0 +1,23 @@ +using System.Collections.Concurrent; +using System.IO; +using Lucene.Net.Index; + +namespace UmbracoExamine +{ + internal sealed class DeletePolicyTracker + { + private static readonly DeletePolicyTracker Instance = new DeletePolicyTracker(); + private readonly ConcurrentDictionary _directories = new ConcurrentDictionary(); + + public static DeletePolicyTracker Current + { + get { return Instance; } + } + + public IndexDeletionPolicy GetPolicy(string indexSetName) + { + var resolved = _directories.GetOrAdd(indexSetName, s => new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy())); + return resolved; + } + } +} \ No newline at end of file diff --git a/src/UmbracoExamine/IndexTypes.cs b/src/UmbracoExamine/IndexTypes.cs index 6d941d21ae..22e25197e7 100644 --- a/src/UmbracoExamine/IndexTypes.cs +++ b/src/UmbracoExamine/IndexTypes.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Linq; using System.Text; +using Lucene.Net.Store; namespace UmbracoExamine { diff --git a/src/UmbracoExamine/LocalStorage/LocalStorageType.cs b/src/UmbracoExamine/LocalStorage/LocalStorageType.cs new file mode 100644 index 0000000000..457b211c64 --- /dev/null +++ b/src/UmbracoExamine/LocalStorage/LocalStorageType.cs @@ -0,0 +1,8 @@ +namespace UmbracoExamine.LocalStorage +{ + public enum LocalStorageType + { + Sync, + LocalOnly + } +} \ No newline at end of file diff --git a/src/UmbracoExamine/LocalStorage/LocalTempStorageDirectory.cs b/src/UmbracoExamine/LocalStorage/LocalTempStorageDirectory.cs index d945184056..4c133d5d41 100644 --- a/src/UmbracoExamine/LocalStorage/LocalTempStorageDirectory.cs +++ b/src/UmbracoExamine/LocalStorage/LocalTempStorageDirectory.cs @@ -6,6 +6,9 @@ using Lucene.Net.Store; namespace UmbracoExamine.LocalStorage { + /// + /// Used to read data from local temp storage and write to both local storage and main storage + /// public class LocalTempStorageDirectory : SimpleFSDirectory { private readonly Lucene.Net.Store.Directory _realDirectory; diff --git a/src/UmbracoExamine/LocalStorage/LocalTempStorageIndexer.cs b/src/UmbracoExamine/LocalStorage/LocalTempStorageIndexer.cs index 2e242c1284..911717a9ed 100644 --- a/src/UmbracoExamine/LocalStorage/LocalTempStorageIndexer.cs +++ b/src/UmbracoExamine/LocalStorage/LocalTempStorageIndexer.cs @@ -1,7 +1,9 @@ -using System.Collections.Specialized; +using System; +using System.Collections.Specialized; using System.IO; using System.Linq; using System.Web; +using Examine.LuceneEngine; using Lucene.Net.Analysis; using Lucene.Net.Index; using Lucene.Net.Store; @@ -18,27 +20,43 @@ namespace UmbracoExamine.LocalStorage public Lucene.Net.Store.Directory LuceneDirectory { get; private set; } private readonly object _locker = new object(); public SnapshotDeletionPolicy Snapshotter { get; private set; } - + public LocalTempStorageIndexer() { IndexDeletionPolicy policy = new KeepOnlyLastCommitDeletionPolicy(); Snapshotter = new SnapshotDeletionPolicy(policy); } - public void Initialize(NameValueCollection config, string configuredPath, Lucene.Net.Store.Directory baseLuceneDirectory, Analyzer analyzer) + public void Initialize(NameValueCollection config, string configuredPath, Lucene.Net.Store.Directory baseLuceneDirectory, Analyzer analyzer, LocalStorageType localStorageType) { var codegenPath = HttpRuntime.CodegenDir; _tempPath = Path.Combine(codegenPath, configuredPath.TrimStart('~', '/').Replace("/", "\\")); - var success = InitializeLocalIndexAndDirectory(baseLuceneDirectory, analyzer, configuredPath); + switch (localStorageType) + { + case LocalStorageType.Sync: + var success = InitializeLocalIndexAndDirectory(baseLuceneDirectory, analyzer, configuredPath); + + //create the custom lucene directory which will keep the main and temp FS's in sync + LuceneDirectory = LocalTempStorageDirectoryTracker.Current.GetDirectory( + new DirectoryInfo(_tempPath), + baseLuceneDirectory, + //flag to disable the mirrored folder if not successful + success == false); + break; + case LocalStorageType.LocalOnly: + if (Directory.Exists(_tempPath) == false) + { + Directory.CreateDirectory(_tempPath); + } + + LuceneDirectory = DirectoryTracker.Current.GetDirectory(new DirectoryInfo(_tempPath)); + break; + default: + throw new ArgumentOutOfRangeException("localStorageType"); + } - //create the custom lucene directory which will keep the main and temp FS's in sync - LuceneDirectory = LocalTempStorageDirectoryTracker.Current.GetDirectory( - new DirectoryInfo(_tempPath), - baseLuceneDirectory, - //flag to disable the mirrored folder if not successful - success == false); } private bool InitializeLocalIndexAndDirectory(Lucene.Net.Store.Directory baseLuceneDirectory, Analyzer analyzer, string configuredPath) @@ -64,7 +82,13 @@ namespace UmbracoExamine.LocalStorage var basePath = IOHelper.MapPath(configuredPath); var commit = Snapshotter.Snapshot(); - var allSnapshotFiles = commit.GetFileNames().Concat(new[] { commit.GetSegmentsFileName() }) + var allSnapshotFiles = commit.GetFileNames() + .Concat(new[] + { + commit.GetSegmentsFileName(), + //we need to manually include the segments.gen file + "segments.gen" + }) .Distinct() .ToArray(); @@ -75,39 +99,48 @@ namespace UmbracoExamine.LocalStorage .Select(x => x.Name) .Except(allSnapshotFiles); - using (var tempDirectory = new SimpleFSDirectory(tempDir)) + //using (var tempDirectory = new SimpleFSDirectory(tempDir)) + //{ + //TODO: We're ignoring if it is locked right now, it shouldn't be unless for some strange reason the + // last process hasn't fully shut down, in that case we're not going to worry about it. + + //if (IndexWriter.IsLocked(tempDirectory) == false) + //{ + foreach (var file in toRemove) { - if (IndexWriter.IsLocked(tempDirectory) == false) + try { - foreach (var file in toRemove) - { - try - { - File.Delete(Path.Combine(_tempPath, file)); - } - catch (IOException ex) - { - LogHelper.Error("Could not delete index file, could not sync from main storage", ex); - //quit here - return false; - } - } + File.Delete(Path.Combine(_tempPath, file)); } - else + catch (IOException ex) { - LogHelper.Warn("Cannot sync index files from main storage, the index is currently locked"); + LogHelper.WarnWithException("Could not delete non synced index file file, index sync will continue but old index files will remain - this shouldn't affect indexing/searching operations", ex); + + //TODO: we're ignoring this, as old files shouldn't affect the index/search operations, lucene files are 'write once' //quit here - return false; + //return false; } } + //} + //else + //{ + // LogHelper.Warn("Cannot sync index files from main storage, the index is currently locked"); + // //quit here + // return false; + //} foreach (var fileName in allSnapshotFiles.Where(f => f.IsNullOrWhiteSpace() == false)) { + var destination = Path.Combine(_tempPath, Path.GetFileName(fileName)); + + //don't copy if it's already there, lucene is 'write once' so this file is meant to be there already + if (File.Exists(destination)) continue; + try { File.Copy( Path.Combine(basePath, "Index", fileName), - Path.Combine(_tempPath, Path.GetFileName(fileName)), true); + destination); } catch (IOException ex) { @@ -118,11 +151,17 @@ namespace UmbracoExamine.LocalStorage } } + //} + + + } finally { Snapshotter.Release(); } + + } return true; diff --git a/src/UmbracoExamine/UmbracoExamine.csproj b/src/UmbracoExamine/UmbracoExamine.csproj index a697577a2e..de826ed659 100644 --- a/src/UmbracoExamine/UmbracoExamine.csproj +++ b/src/UmbracoExamine/UmbracoExamine.csproj @@ -82,9 +82,9 @@ ..\Solution Items\TheFARM-Public.snk - + False - ..\packages\Examine.0.1.57.2941\lib\Examine.dll + ..\packages\Examine.0.1.59.2941\lib\Examine.dll False @@ -119,9 +119,11 @@ + + diff --git a/src/UmbracoExamine/UmbracoExamineSearcher.cs b/src/UmbracoExamine/UmbracoExamineSearcher.cs index 2380db62f3..437e5c5220 100644 --- a/src/UmbracoExamine/UmbracoExamineSearcher.cs +++ b/src/UmbracoExamine/UmbracoExamineSearcher.cs @@ -7,6 +7,7 @@ using Examine; using Examine.LuceneEngine.Config; using Examine.Providers; using Examine.SearchCriteria; +using Lucene.Net.Index; using Lucene.Net.Store; using Umbraco.Core; using UmbracoExamine.Config; @@ -22,18 +23,19 @@ namespace UmbracoExamine /// /// An Examine searcher which uses Lucene.Net as the /// - public class UmbracoExamineSearcher : LuceneSearcher + public class UmbracoExamineSearcher : LuceneSearcher { private volatile Lucene.Net.Store.Directory _localTempDirectory; private static readonly object Locker = new object(); private string _localTempPath = null; + private LocalStorageType _localStorageType = LocalStorageType.Sync; #region Constructors - /// - /// Default constructor - /// + /// + /// Default constructor + /// public UmbracoExamineSearcher() : base() { @@ -53,7 +55,7 @@ namespace UmbracoExamine } } - + public override void Initialize(string name, System.Collections.Specialized.NameValueCollection config) { if (name == null) throw new ArgumentNullException("name"); @@ -78,13 +80,14 @@ namespace UmbracoExamine { //Use the temp storage directory which will store the index in the local/codegen folder, this is useful // for websites that are running from a remove file server and file IO latency becomes an issue - var attemptUseTempStorage = config["useTempStorage"].TryConvertTo(); + var attemptUseTempStorage = config["useTempStorage"].TryConvertTo(); if (attemptUseTempStorage) - { + { var indexSet = IndexSets.Instance.Sets[IndexSetName]; var configuredPath = indexSet.IndexPath; var codegenPath = HttpRuntime.CodegenDir; _localTempPath = Path.Combine(codegenPath, configuredPath.TrimStart('~', '/').Replace("/", "\\")); + _localStorageType = attemptUseTempStorage.Result; } } } @@ -94,24 +97,24 @@ namespace UmbracoExamine /// /// /// - - public UmbracoExamineSearcher(DirectoryInfo indexPath, Analyzer analyzer) + + public UmbracoExamineSearcher(DirectoryInfo indexPath, Analyzer analyzer) : base(indexPath, analyzer) { } - /// - /// Constructor to allow for creating an indexer at runtime - /// - /// - /// - - public UmbracoExamineSearcher(Lucene.Net.Store.Directory luceneDirectory, Analyzer analyzer) - : base(luceneDirectory, analyzer) - { - } + /// + /// Constructor to allow for creating an indexer at runtime + /// + /// + /// - #endregion + public UmbracoExamineSearcher(Lucene.Net.Store.Directory luceneDirectory, Analyzer analyzer) + : base(luceneDirectory, analyzer) + { + } + + #endregion /// /// Used for unit tests @@ -122,7 +125,7 @@ namespace UmbracoExamine /// Returns true if the Umbraco application is in a state that we can initialize the examine indexes /// /// - + protected bool CanInitialize() { //check the DisableInitializationCheck and ensure that it is not set to true @@ -135,7 +138,7 @@ namespace UmbracoExamine { return false; } - } + } return true; } @@ -165,6 +168,14 @@ namespace UmbracoExamine .ToArray(); } + protected override IndexReader OpenNewReader() + { + return IndexReader.Open( + GetLuceneDirectory(), + DeletePolicyTracker.Current.GetPolicy(IndexSetName), + true); + } + protected override Lucene.Net.Store.Directory GetLuceneDirectory() { //local temp storage is not enabled, just return the default @@ -177,9 +188,21 @@ namespace UmbracoExamine { if (_localTempDirectory == null) { - _localTempDirectory = LocalTempStorageDirectoryTracker.Current.GetDirectory( - new DirectoryInfo(_localTempPath), - base.GetLuceneDirectory()); + switch (_localStorageType) + { + case LocalStorageType.Sync: + _localTempDirectory = LocalTempStorageDirectoryTracker.Current.GetDirectory( + new DirectoryInfo(_localTempPath), + base.GetLuceneDirectory()); + break; + case LocalStorageType.LocalOnly: + _localTempDirectory = DirectoryTracker.Current.GetDirectory(new DirectoryInfo(_localTempPath)); + break; + default: + throw new ArgumentOutOfRangeException(); + } + + } } } diff --git a/src/UmbracoExamine/packages.config b/src/UmbracoExamine/packages.config index ec889e9ef6..ea08897b32 100644 --- a/src/UmbracoExamine/packages.config +++ b/src/UmbracoExamine/packages.config @@ -1,6 +1,6 @@  - + \ No newline at end of file diff --git a/src/umbraco.MacroEngines/packages.config b/src/umbraco.MacroEngines/packages.config index a45a7bffca..ca995d7c13 100644 --- a/src/umbraco.MacroEngines/packages.config +++ b/src/umbraco.MacroEngines/packages.config @@ -1,6 +1,6 @@  - + diff --git a/src/umbraco.MacroEngines/umbraco.MacroEngines.csproj b/src/umbraco.MacroEngines/umbraco.MacroEngines.csproj index 6980ac43ff..f6b611a6c8 100644 --- a/src/umbraco.MacroEngines/umbraco.MacroEngines.csproj +++ b/src/umbraco.MacroEngines/umbraco.MacroEngines.csproj @@ -45,9 +45,9 @@ false - + False - ..\packages\Examine.0.1.58.2941\lib\Examine.dll + ..\packages\Examine.0.1.59.2941\lib\Examine.dll False