yv01p/Umbraco-CMS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

updates the local storage sync, simplifies options and does more error checking in case sync cannot occur - which will default to normal storage.

Browse Source
This commit is contained in:
Shannon
2014-11-13 15:49:34 +11:00
parent 33036f80fe
commit 74ebf67aef
5 changed files with 133 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
src/UmbracoExamine/LocalStorage/LocalTempStorageDirectory.cs
View File

@@ -9,15 +9,21 @@ namespace UmbracoExamine.LocalStorage
public class LocalTempStorageDirectory : SimpleFSDirectory
{
private readonly Lucene.Net.Store.Directory _realDirectory;
public LocalTempStorageDirectory(
DirectoryInfo tempStorageDir,
Lucene.Net.Store.Directory realDirectory)
: base(tempStorageDir)
{
_realDirectory = realDirectory;
Enabled = true;
}
/// <summary>
/// If initialization fails, it will be disabled and then this will just wrap the 'real directory'
/// </summary>
internal bool Enabled { get; set; }
public override string[] ListAll()
{
//always from the real dir
@@ -49,7 +55,11 @@ namespace UmbracoExamine.LocalStorage
public override void DeleteFile(string name)
{
//perform on both dirs
base.DeleteFile(name);
if (Enabled)
{
base.DeleteFile(name);
}
_realDirectory.DeleteFile(name);
}
@@ -67,10 +77,14 @@ namespace UmbracoExamine.LocalStorage
public override IndexOutput CreateOutput(string name)
{
//write to both indexes
if (Enabled)
{
return new MultiIndexOutput(
base.CreateOutput(name),
_realDirectory.CreateOutput(name));
}
return new MultiIndexOutput(
base.CreateOutput(name),
_realDirectory.CreateOutput(name));
return _realDirectory.CreateOutput(name);
}
/// <summary>
@@ -78,13 +92,21 @@ namespace UmbracoExamine.LocalStorage
/// </summary>
public override IndexInput OpenInput(string name)
{
//return the reader from the cache, not the real dir
return base.OpenInput(name);
if (Enabled)
{
//return the reader from the cache, not the real dir
return base.OpenInput(name);
}
return _realDirectory.OpenInput(name);
}
public override void Dispose()
{
base.Dispose();
if (Enabled)
{
base.Dispose();
}
_realDirectory.Dispose();
}

29
src/UmbracoExamine/LocalStorage/LocalTempStorageDirectoryTracker.cs Normal file
View File

@@ -0,0 +1,29 @@
using System.Collections.Concurrent;
using System.IO;
using Directory = Lucene.Net.Store.Directory;
namespace UmbracoExamine.LocalStorage
{
internal class LocalTempStorageDirectoryTracker
{
private readonly static LocalTempStorageDirectoryTracker Instance = new LocalTempStorageDirectoryTracker();
private readonly ConcurrentDictionary<string, LocalTempStorageDirectory> _directories = new ConcurrentDictionary<string, LocalTempStorageDirectory>();
public static LocalTempStorageDirectoryTracker Current
{
get { return Instance; }
}
public LocalTempStorageDirectory GetDirectory(DirectoryInfo dir, Directory realDir, bool disable = false)
{
var resolved = _directories.GetOrAdd(dir.FullName, s => new LocalTempStorageDirectory(dir, realDir));
if (disable)
{
resolved.Enabled = false;
}
return resolved;
}
}
}

161
src/UmbracoExamine/LocalStorage/LocalTempStorageIndexer.cs
View File

@@ -16,10 +16,9 @@ namespace UmbracoExamine.LocalStorage
{
private string _tempPath;
public Lucene.Net.Store.Directory LuceneDirectory { get; private set; }
private static readonly object Locker = new object();
private readonly object _locker = new object();
public SnapshotDeletionPolicy Snapshotter { get; private set; }
private bool _syncStorage = false;
public LocalTempStorageIndexer()
{
IndexDeletionPolicy policy = new KeepOnlyLastCommitDeletionPolicy();
@@ -32,121 +31,101 @@ namespace UmbracoExamine.LocalStorage
_tempPath = Path.Combine(codegenPath, configuredPath.TrimStart('~', '/').Replace("/", "\\"));
if (config != null)
{
if (config["syncTempStorage"] != null)
{
var attempt = config["syncTempStorage"].TryConvertTo<bool>();
if (attempt)
{
_syncStorage = attempt.Result;
}
}
}
var success = InitializeLocalIndexAndDirectory(baseLuceneDirectory, analyzer, configuredPath);
InitializeLocalIndexAndDirectory(baseLuceneDirectory, analyzer, configuredPath);
//create the custom lucene directory which will keep the main and temp FS's in sync
LuceneDirectory = LocalTempStorageDirectoryTracker.Current.GetDirectory(
new DirectoryInfo(_tempPath),
baseLuceneDirectory,
//flag to disable the mirrored folder if not successful
success == false);
}
private void InitializeLocalIndexAndDirectory(Lucene.Net.Store.Directory baseLuceneDirectory, Analyzer analyzer, string configuredPath)
private bool InitializeLocalIndexAndDirectory(Lucene.Net.Store.Directory baseLuceneDirectory, Analyzer analyzer, string configuredPath)
{
lock (Locker)
lock (_locker)
{
if (Directory.Exists(_tempPath) == false)
{
Directory.CreateDirectory(_tempPath);
}
//if we are syncing storage to the main file system to temp files, then sync from the main FS to our temp FS
if (_syncStorage)
//copy index
using (new IndexWriter(
//read from the underlying/default directory, not the temp codegen dir
baseLuceneDirectory,
analyzer,
Snapshotter,
IndexWriter.MaxFieldLength.UNLIMITED))
{
//copy index
using (new IndexWriter(
//read from the underlying/default directory, not the temp codegen dir
baseLuceneDirectory,
analyzer,
Snapshotter,
IndexWriter.MaxFieldLength.UNLIMITED))
try
{
try
var basePath = IOHelper.MapPath(configuredPath);
var commit = Snapshotter.Snapshot();
var allSnapshotFiles = commit.GetFileNames().Concat(new[] { commit.GetSegmentsFileName() })
.Distinct()
.ToArray();
var tempDir = new DirectoryInfo(_tempPath);
//Get all files in the temp storage that don't exist in the snapshot collection, we want to remove these
var toRemove = tempDir.GetFiles()
.Select(x => x.Name)
.Except(allSnapshotFiles);
using (var tempDirectory = new SimpleFSDirectory(tempDir))
{
var basePath = IOHelper.MapPath(configuredPath);
var commit = Snapshotter.Snapshot();
var allSnapshotFiles = commit.GetFileNames().Concat(new[] {commit.GetSegmentsFileName()}).ToArray();
var tempDir = new DirectoryInfo(_tempPath);
//Get all files in the temp storage that don't exist in the snapshot collection, we want to remove these
var toRemove = tempDir.GetFiles()
.Select(x => x.Name)
.Except(allSnapshotFiles);
using (var tempDirectory = new SimpleFSDirectory(tempDir))
if (IndexWriter.IsLocked(tempDirectory) == false)
{
if (IndexWriter.IsLocked(tempDirectory) == false)
foreach (var file in toRemove)
{
foreach (var file in toRemove)
try
{
try
{
File.Delete(Path.Combine(_tempPath, file));
}
catch (IOException ex)
{
LogHelper.Error<LocalTempStorageIndexer>("Could not delete index file, could not sync from main storage", ex);
//quit here and do not assign the lucene directory, this means that the app will now just be working from normal storage
return;
}
File.Delete(Path.Combine(_tempPath, file));
}
catch (IOException ex)
{
LogHelper.Error<LocalTempStorageIndexer>("Could not delete index file, could not sync from main storage", ex);
//quit here
return false;
}
}
else
{
LogHelper.Warn<LocalTempStorageIndexer>("Cannot sync index files from main storage, the index is currently locked");
//quit here and do not assign the lucene directory, this means that the app will now just be working from normal storage
return;
}
}
foreach (var fileName in allSnapshotFiles.Where(f => f.IsNullOrWhiteSpace() == false))
else
{
try
{
File.Copy(
Path.Combine(basePath, "Index", fileName),
Path.Combine(_tempPath, Path.GetFileName(fileName)), true);
}
catch (IOException ex)
{
LogHelper.Error<LocalTempStorageIndexer>("Could not copy index file, could not sync from main storage", ex);
//quit here and do not assign the lucene directory, this means that the app will now just be working from normal storage
return;
}
LogHelper.Warn<LocalTempStorageIndexer>("Cannot sync index files from main storage, the index is currently locked");
//quit here
return false;
}
}
finally
foreach (var fileName in allSnapshotFiles.Where(f => f.IsNullOrWhiteSpace() == false))
{
Snapshotter.Release();
try
{
File.Copy(
Path.Combine(basePath, "Index", fileName),
Path.Combine(_tempPath, Path.GetFileName(fileName)), true);
}
catch (IOException ex)
{
LogHelper.Error<LocalTempStorageIndexer>("Could not copy index file, could not sync from main storage", ex);
//quit here
return false;
}
}
}
finally
{
Snapshotter.Release();
}
//create the custom lucene directory which will keep the main and temp FS's in sync
LuceneDirectory = new LocalTempStorageDirectory(
new DirectoryInfo(_tempPath),
baseLuceneDirectory);
}
else
{
//just return a normal lucene directory that uses the codegen folder
LuceneDirectory = FSDirectory.Open(new DirectoryInfo(_tempPath));
}
return true;
}
}
}

1
src/UmbracoExamine/UmbracoExamine.csproj
View File

@@ -122,6 +122,7 @@
<Compile Include="IndexTypes.cs" />
<Compile Include="LegacyLibrary.cs" />
<Compile Include="LocalStorage\LocalTempStorageDirectory.cs" />
<Compile Include="LocalStorage\LocalTempStorageDirectoryTracker.cs" />
<Compile Include="LocalStorage\LocalTempStorageIndexer.cs" />
<Compile Include="LocalStorage\MultiIndexOutput.cs" />
<Compile Include="LoggingLevel.cs" />

24
src/UmbracoExamine/UmbracoExamineSearcher.cs
View File

@@ -28,7 +28,6 @@ namespace UmbracoExamine
private volatile Lucene.Net.Store.Directory _localTempDirectory;
private static readonly object Locker = new object();
private string _localTempPath = null;
private bool _syncTempStorage = false;
#region Constructors
@@ -82,15 +81,6 @@ namespace UmbracoExamine
var codegenPath = HttpRuntime.CodegenDir;
_localTempPath = Path.Combine(codegenPath, configuredPath.TrimStart('~', '/').Replace("/", "\\"));
}
if (config["syncTempStorage"] != null)
{
var attemptSync = config["syncTempStorage"].TryConvertTo<bool>();
if (attemptSync)
{
_syncTempStorage = attemptSync.Result;
}
}
}
}
@@ -182,17 +172,9 @@ namespace UmbracoExamine
{
if (_localTempDirectory == null)
{
if (_syncTempStorage)
{
_localTempDirectory = new LocalTempStorageDirectory(
new DirectoryInfo(_localTempPath),
base.GetLuceneDirectory());
}
else
{
//not syncing just use a normal lucene directory
_localTempDirectory = FSDirectory.Open(new DirectoryInfo(_localTempPath));
}
_localTempDirectory = LocalTempStorageDirectoryTracker.Current.GetDirectory(
new DirectoryInfo(_localTempPath),
base.GetLuceneDirectory());
}
}
}