Added HashCodeCombiner and unit tests to support. Added ability to create a hashed file based on scanned assemblies in the

PluginManager and added a cached plugin xml file to the PluginManager. Now we don't actually scan any assemblies on app startup
during plugin finding if none of the DLLs have changed. Added trace statements to the UmbracoModule for better benchmarking for
how long a full request takes to load from start to finish.
This commit is contained in:
Shannon Deminick
2012-11-11 09:09:06 +05:00
parent c590c6a265
commit 083bab0528
8 changed files with 603 additions and 6 deletions

View File

@@ -0,0 +1,76 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Text;
namespace Umbraco.Core
{
/// <summary>
/// Used to create a hash code from multiple objects.
/// </summary>
/// <remarks>
/// .Net has a class the same as this: System.Web.Util.HashCodeCombiner and of course it works for all sorts of things
/// and is probably more stable in general, however we just need a quick easy class for this in order to create a unique
/// hash of plugins to see if they've changed.
/// </remarks>
internal class HashCodeCombiner
{
private int _combinedHash;
internal void AddInt(int i)
{
_combinedHash = ((_combinedHash << 5) + _combinedHash) ^ i;
}
internal void AddObject(object o)
{
AddInt(o.GetHashCode());
}
internal void AddDateTime(DateTime d)
{
AddInt(d.GetHashCode());
}
internal void AddCaseInsensitiveString(string s)
{
if (s != null)
AddInt((StringComparer.InvariantCultureIgnoreCase).GetHashCode(s));
}
internal void AddFile(FileInfo f)
{
AddCaseInsensitiveString(f.FullName);
AddDateTime(f.CreationTimeUtc);
AddDateTime(f.LastWriteTimeUtc);
AddInt(f.Length.GetHashCode());
}
internal void AddFolder(DirectoryInfo d)
{
AddCaseInsensitiveString(d.FullName);
AddDateTime(d.CreationTimeUtc);
AddDateTime(d.LastWriteTimeUtc);
foreach (var f in d.GetFiles())
{
AddFile(f);
}
foreach (var s in d.GetDirectories())
{
AddFolder(s);
}
}
/// <summary>
/// Returns the hex code of the combined hash code
/// </summary>
/// <returns></returns>
internal string GetCombinedHashCode()
{
return _combinedHash.ToString("x", CultureInfo.InvariantCulture);
}
}
}

View File

@@ -12,6 +12,7 @@ namespace Umbraco.Core.IO
//all paths has a starting but no trailing /
public class SystemDirectories
{
//TODO: Why on earth is this even configurable? You cannot change the /Bin folder in ASP.Net
public static string Bin
{
get

View File

@@ -1,10 +1,16 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Text;
using System.Threading;
using System.Xml;
using System.Xml.Linq;
using Umbraco.Core.Configuration;
using Umbraco.Core.IO;
using Umbraco.Core.Logging;
using Umbraco.Core.PropertyEditors;
using umbraco.interfaces;
@@ -21,16 +27,35 @@ namespace Umbraco.Core
///
/// This class can expose extension methods to resolve custom plugins
///
/// Before this class resolves any plugins it checks if the hash has changed for the DLLs in the /bin folder, if it hasn't
/// it will use the cached resolved plugins that it has already found which means that no assembly scanning is necessary. This leads
/// to much faster startup times.
/// </remarks>
internal class PluginManager
{
internal PluginManager()
{
_tempFolder = IOHelper.MapPath("~/App_Data/TEMP/PluginCache");
//create the folder if it doesn't exist
if (!Directory.Exists(_tempFolder))
{
Directory.CreateDirectory(_tempFolder);
}
//do the check if they've changed
HaveAssembliesChanged = CachedAssembliesHash != CurrentAssembliesHash;
//if they have changed, we need to write the new file
if (HaveAssembliesChanged)
{
WriteCachePluginsHash();
}
}
static PluginManager _resolver;
static readonly ReaderWriterLockSlim Lock = new ReaderWriterLockSlim();
private readonly string _tempFolder;
private long _cachedAssembliesHash = -1;
private long _currentAssembliesHash = -1;
/// <summary>
/// We will ensure that no matter what, only one of these is created, this is to ensure that caching always takes place
@@ -55,6 +80,204 @@ namespace Umbraco.Core
set { _resolver = value; }
}
#region Hash checking methods
/// <summary>
/// Returns a bool if the assemblies in the /bin have changed since they were last hashed.
/// </summary>
internal bool HaveAssembliesChanged { get; private set; }
/// <summary>
/// Returns the currently cached hash value of the scanned assemblies in the /bin folder. Returns 0
/// if no cache is found.
/// </summary>
/// <value> </value>
internal long CachedAssembliesHash
{
get
{
if (_cachedAssembliesHash != -1)
return _cachedAssembliesHash;
var filePath = Path.Combine(_tempFolder, "umbraco-plugins.hash");
if (!File.Exists(filePath))
return 0;
var hash = File.ReadAllText(filePath, Encoding.UTF8);
Int64 val;
if (Int64.TryParse(hash, out val))
{
_cachedAssembliesHash = val;
return _cachedAssembliesHash;
}
//it could not parse for some reason so we'll return 0.
return 0;
}
}
/// <summary>
/// Returns the current assemblies hash based on creating a hash from the assemblies in the /bin
/// </summary>
/// <value> </value>
internal long CurrentAssembliesHash
{
get
{
if (_currentAssembliesHash != -1)
return _currentAssembliesHash;
_currentAssembliesHash = GetAssembliesHash(new DirectoryInfo(IOHelper.MapPath(SystemDirectories.Bin)).GetFiles("*.dll"));
return _currentAssembliesHash;
}
}
/// <summary>
/// Writes the assembly hash file
/// </summary>
private void WriteCachePluginsHash()
{
var filePath = Path.Combine(_tempFolder, "umbraco-plugins.hash");
File.WriteAllText(filePath, CurrentAssembliesHash.ToString(), Encoding.UTF8);
}
/// <summary>
/// Returns a unique hash for the combination of FileInfo objects passed in
/// </summary>
/// <param name="plugins"></param>
/// <returns></returns>
internal static long GetAssembliesHash(IEnumerable<FileInfo> plugins)
{
using (DisposableTimer.TraceDuration<PluginManager>("Determining hash of plugins on disk", "Hash determined"))
{
var hashCombiner = new HashCodeCombiner();
//add each unique folder to the hash
foreach (var i in plugins.Select(x => x.Directory).DistinctBy(x => x.FullName))
{
hashCombiner.AddFolder(i);
}
return ConvertPluginsHashFromHex(hashCombiner.GetCombinedHashCode());
}
}
/// <summary>
/// Converts the hash value of current plugins to long from string
/// </summary>
/// <param name="val"></param>
/// <returns></returns>
internal static long ConvertPluginsHashFromHex(string val)
{
long outVal;
if (Int64.TryParse(val, NumberStyles.AllowHexSpecifier, CultureInfo.InvariantCulture, out outVal))
{
return outVal;
}
return 0;
}
/// <summary>
/// Attempts to resolve the list of plugin + assemblies found in the runtime for the base type 'T' passed in.
/// If the cache file doesn't exist, fails to load, is corrupt or the type 'T' element is not found then
/// a false attempt is returned.
/// </summary>
/// <typeparam name="T"></typeparam>
/// <returns></returns>
internal Attempt<IEnumerable<Tuple<string, string>>> TryGetCachedPluginsFromFile<T>()
{
var filePath = Path.Combine(_tempFolder, "umbraco-plugins.list");
if (!File.Exists(filePath))
return Attempt<IEnumerable<Tuple<string, string>>>.False;
try
{
var xml = XDocument.Load(filePath);
if (xml.Root == null)
return Attempt<IEnumerable<Tuple<string, string>>>.False;
var typeElement = xml.Root.Elements()
.SingleOrDefault(x =>
x.Name.LocalName == "baseType"
&& ((string) x.Attribute("type")) == typeof (T).FullName);
if (typeElement == null)
return Attempt<IEnumerable<Tuple<string, string>>>.False;
//return success
return new Attempt<IEnumerable<Tuple<string, string>>>(
true,
typeElement.Elements("add")
.Select(x => new Tuple<string, string>(
(string) x.Attribute("type"),
(string) x.Attribute("assembly"))));
}
catch (Exception)
{
//if the file is corrupted, etc... return false
return Attempt<IEnumerable<Tuple<string, string>>>.False;
}
}
/// <summary>
/// Adds/Updates the type list for the base type 'T' in the cached file
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="typesFound"></param>
/// <remarks>
/// THIS METHOD IS NOT THREAD SAFE
/// </remarks>
/// <example>
/// <![CDATA[
/// <plugins>
/// <baseType type="Test.Testing.Tester">
/// <add type="My.Assembly.MyTester" assembly="My.Assembly" />
/// <add type="Your.Assembly.YourTester" assembly="Your.Assembly" />
/// </baseType>
/// </plugins>
/// ]]>
/// </example>
internal void UpdateCachedPluginsFile<T>(IEnumerable<Type> typesFound)
{
var filePath = Path.Combine(_tempFolder, "umbraco-plugins.list");
XDocument xml;
try
{
xml = XDocument.Load(filePath);
}
catch
{
//if there's an exception loading then this is somehow corrupt, we'll just replace it.
File.Delete(filePath);
//create the document and the root
xml = new XDocument(new XElement("plugins"));
}
if (xml.Root == null)
{
//if for some reason there is no root, create it
xml.Add(new XElement("plugins"));
}
//find the type 'T' element to add or update
var typeElement = xml.Root.Elements()
.SingleOrDefault(x =>
x.Name.LocalName == "baseType"
&& ((string)x.Attribute("type")) == typeof(T).FullName);
if (typeElement == null)
{
//create the type element
typeElement = new XElement("baseType", new XAttribute("type", typeof(T).FullName));
//then add it to the root
xml.Root.Add(typeElement);
}
//now we have the type element, we need to clear any previous types as children and add/update it with new ones
typeElement.ReplaceNodes(typesFound
.Select(x =>
new XElement("add",
new XAttribute("type", x.FullName),
new XAttribute("assembly", x.Assembly.FullName))));
//save the xml file
xml.Save(filePath);
}
#endregion
private readonly ReaderWriterLockSlim _lock = new ReaderWriterLockSlim();
private readonly HashSet<TypeList> _types = new HashSet<TypeList>();
private IEnumerable<Assembly> _assemblies;
@@ -198,7 +421,7 @@ namespace Umbraco.Core
using (DisposableTimer.TraceDuration<PluginManager>(
String.Format("Starting resolution types of {0}", typeof(T).FullName),
String.Format("Completed resolution of types of {0}", typeof(T).FullName)))
{
{
//check if the TypeList already exists, if so return it, if not we'll create it
var typeList = _types.SingleOrDefault(x => x.IsTypeList<T>(resolutionType));
//if we're not caching the result then proceed, or if the type list doesn't exist then proceed
@@ -209,9 +432,41 @@ namespace Umbraco.Core
typeList = new TypeList<T>(resolutionType);
foreach (var t in finder())
//we first need to look into our cache file (this has nothing to do with the 'cacheResult' parameter which caches in memory).
if (!HaveAssembliesChanged)
{
typeList.AddType(t);
var fileCacheResult = TryGetCachedPluginsFromFile<T>();
if (fileCacheResult.Success)
{
var successfullyLoadedFromCache = true;
//we have a previous cache for this so we don't need to scan we just load what has been found in the file
foreach(var t in fileCacheResult.Result)
{
try
{
var type = Assembly.Load(t.Item2).GetType(t.Item1);
typeList.AddType(type);
}
catch (Exception ex)
{
//if there are any exceptions loading types, we have to exist, this should never happen so
//we will need to revert to scanning for types.
successfullyLoadedFromCache = false;
LogHelper.Error<PluginManager>("Could not load a cached plugin type: " + t.Item1 + " in assembly: " + t.Item2 + " now reverting to re-scanning assemblies for the base type: " + typeof (T).FullName, ex);
break;
}
}
if (!successfullyLoadedFromCache )
{
//we need to manually load by scanning if loading from the file was not successful.
LoadViaScanningAndUpdateCacheFile<T>(typeList, finder);
}
}
}
else
{
//we don't have a cache for this so proceed to look them up by scanning
LoadViaScanningAndUpdateCacheFile<T>(typeList, finder);
}
//only add the cache if we are to cache the results
@@ -226,6 +481,25 @@ namespace Umbraco.Core
}
}
/// <summary>
/// This method invokes the finder which scans the assemblies for the types and then loads the result into the type finder.
/// Once the results are loaded, we update the cached type xml file
/// </summary>
/// <param name="typeList"></param>
/// <param name="finder"></param>
/// <remarks>
/// THIS METHODS IS NOT THREAD SAFE
/// </remarks>
private void LoadViaScanningAndUpdateCacheFile<T>(TypeList typeList, Func<IEnumerable<Type>> finder)
{
//we don't have a cache for this so proceed to look them up by scanning
foreach (var t in finder())
{
typeList.AddType(t);
}
UpdateCachedPluginsFile<T>(typeList.GetTypes());
}
/// <summary>
/// Generic method to find the specified type and cache the result
/// </summary>

View File

@@ -66,6 +66,7 @@
<Compile Include="Dictionary\CultureDictionaryFactoryResolver.cs" />
<Compile Include="Dictionary\ICultureDictionaryFactory.cs" />
<Compile Include="Enum.cs" />
<Compile Include="HashCodeCombiner.cs" />
<Compile Include="IO\FileSystemWrapper.cs" />
<Compile Include="PublishedContentExtensions.cs" />
<Compile Include="Dictionary\ICultureDictionary.cs" />

View File

@@ -0,0 +1,154 @@
using System;
using System.IO;
using System.Reflection;
using NUnit.Framework;
using Umbraco.Core;
namespace Umbraco.Tests
{
[TestFixture]
public class HashCodeCombinerTests : PartialTrust.AbstractPartialTrustFixture<HashCodeCombinerTests>
{
private DirectoryInfo PrepareFolder()
{
var assDir = new FileInfo(Assembly.GetExecutingAssembly().Location).Directory;
var dir = Directory.CreateDirectory(Path.Combine(assDir.FullName, "HashCombiner", Guid.NewGuid().ToString("N")));
foreach (var f in dir.GetFiles())
{
f.Delete();
}
return dir;
}
[Test]
public void HashCombiner_Test_String()
{
var combiner1 = new HashCodeCombiner();
combiner1.AddCaseInsensitiveString("Hello");
var combiner2 = new HashCodeCombiner();
combiner2.AddCaseInsensitiveString("hello");
Assert.AreEqual(combiner1.GetCombinedHashCode(), combiner2.GetCombinedHashCode());
combiner2.AddCaseInsensitiveString("world");
Assert.AreNotEqual(combiner1.GetCombinedHashCode(), combiner2.GetCombinedHashCode());
}
[Test]
public void HashCombiner_Test_Int()
{
var combiner1 = new HashCodeCombiner();
combiner1.AddInt(1234);
var combiner2 = new HashCodeCombiner();
combiner2.AddInt(1234);
Assert.AreEqual(combiner1.GetCombinedHashCode(), combiner2.GetCombinedHashCode());
combiner2.AddInt(1);
Assert.AreNotEqual(combiner1.GetCombinedHashCode(), combiner2.GetCombinedHashCode());
}
[Test]
public void HashCombiner_Test_DateTime()
{
var dt = DateTime.Now;
var combiner1 = new HashCodeCombiner();
combiner1.AddDateTime(dt);
var combiner2 = new HashCodeCombiner();
combiner2.AddDateTime(dt);
Assert.AreEqual(combiner1.GetCombinedHashCode(), combiner2.GetCombinedHashCode());
combiner2.AddDateTime(DateTime.Now);
Assert.AreNotEqual(combiner1.GetCombinedHashCode(), combiner2.GetCombinedHashCode());
}
[Test]
public void HashCombiner_Test_File()
{
var dir = PrepareFolder();
var file1Path = Path.Combine(dir.FullName, "hastest1.txt");
File.Delete(file1Path);
using (var file1 = File.CreateText(Path.Combine(dir.FullName, "hastest1.txt")))
{
file1.WriteLine("hello");
}
var file2Path = Path.Combine(dir.FullName, "hastest2.txt");
File.Delete(file2Path);
using (var file2 = File.CreateText(Path.Combine(dir.FullName, "hastest2.txt")))
{
//even though files are the same, the dates are different
file2.WriteLine("hello");
}
var combiner1 = new HashCodeCombiner();
combiner1.AddFile(new FileInfo(file1Path));
var combiner2 = new HashCodeCombiner();
combiner2.AddFile(new FileInfo(file1Path));
var combiner3 = new HashCodeCombiner();
combiner3.AddFile(new FileInfo(file2Path));
Assert.AreEqual(combiner1.GetCombinedHashCode(), combiner2.GetCombinedHashCode());
Assert.AreNotEqual(combiner1.GetCombinedHashCode(), combiner3.GetCombinedHashCode());
combiner2.AddFile(new FileInfo(file2Path));
Assert.AreNotEqual(combiner1.GetCombinedHashCode(), combiner2.GetCombinedHashCode());
}
[Test]
public void HashCombiner_Test_Folder()
{
var dir = PrepareFolder();
var file1Path = Path.Combine(dir.FullName, "hastest1.txt");
File.Delete(file1Path);
using (var file1 = File.CreateText(Path.Combine(dir.FullName, "hastest1.txt")))
{
file1.WriteLine("hello");
}
//first test the whole folder
var combiner1 = new HashCodeCombiner();
combiner1.AddFolder(dir);
var combiner2 = new HashCodeCombiner();
combiner2.AddFolder(dir);
Assert.AreEqual(combiner1.GetCombinedHashCode(), combiner2.GetCombinedHashCode());
//now add a file to the folder
var file2Path = Path.Combine(dir.FullName, "hastest2.txt");
File.Delete(file2Path);
using (var file2 = File.CreateText(Path.Combine(dir.FullName, "hastest2.txt")))
{
//even though files are the same, the dates are different
file2.WriteLine("hello");
}
var combiner3 = new HashCodeCombiner();
combiner3.AddFolder(dir);
Assert.AreNotEqual(combiner1.GetCombinedHashCode(), combiner3.GetCombinedHashCode());
}
public override void TestSetup()
{
}
public override void TestTearDown()
{
}
}
}

View File

@@ -1,4 +1,8 @@
using System;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Reflection;
using NUnit.Framework;
using SqlCE4Umbraco;
using Umbraco.Core;
@@ -54,6 +58,82 @@ namespace Umbraco.Tests
};
}
private DirectoryInfo PrepareFolder()
{
var assDir = new FileInfo(Assembly.GetExecutingAssembly().Location).Directory;
var dir = Directory.CreateDirectory(Path.Combine(assDir.FullName, "PluginManager", Guid.NewGuid().ToString("N")));
foreach (var f in dir.GetFiles())
{
f.Delete();
}
return dir;
}
[Test]
public void Create_Cached_Plugin_File()
{
var types = new[] {typeof (PluginManager), typeof (PluginManagerTests), typeof (UmbracoContext)};
var manager = new PluginManager();
//yes this is silly, none of these types inherit from string, but this is just to test the xml file format
manager.UpdateCachedPluginsFile<string>(types);
var plugins = manager.TryGetCachedPluginsFromFile<string>();
Assert.IsTrue(plugins.Success);
Assert.AreEqual(3, plugins.Result.Count());
var shouldContain = types.Select(x => new System.Tuple<string, string>(x.FullName, x.Assembly.FullName));
//ensure they are all found
Assert.IsTrue(plugins.Result.ContainsAll(shouldContain));
}
[Test]
public void PluginHash_From_String()
{
var s = "hello my name is someone".GetHashCode().ToString("x", CultureInfo.InvariantCulture);
var output = PluginManager.ConvertPluginsHashFromHex(s);
Assert.AreNotEqual(0, output);
}
[Test]
public void Get_Plugins_Hash()
{
//Arrange
var dir = PrepareFolder();
var d1 = dir.CreateSubdirectory("1");
var d2 = dir.CreateSubdirectory("2");
var d3 = dir.CreateSubdirectory("3");
var d4 = dir.CreateSubdirectory("4");
var f1 = new FileInfo(Path.Combine(d1.FullName, "test1.dll"));
var f2 = new FileInfo(Path.Combine(d1.FullName, "test2.dll"));
var f3 = new FileInfo(Path.Combine(d2.FullName, "test1.dll"));
var f4 = new FileInfo(Path.Combine(d2.FullName, "test2.dll"));
var f5 = new FileInfo(Path.Combine(d3.FullName, "test1.dll"));
var f6 = new FileInfo(Path.Combine(d3.FullName, "test2.dll"));
var f7 = new FileInfo(Path.Combine(d4.FullName, "test1.dll"));
f1.CreateText().Close();
f2.CreateText().Close();
f3.CreateText().Close();
f4.CreateText().Close();
f5.CreateText().Close();
f6.CreateText().Close();
f7.CreateText().Close();
var list1 = new[] { f1, f2, f3, f4, f5, f6 };
var list2 = new[] { f1, f3, f5 };
var list3 = new[] { f1, f3, f5, f7 };
//Act
var hash1 = PluginManager.GetAssembliesHash(list1);
var hash2 = PluginManager.GetAssembliesHash(list2);
var hash3 = PluginManager.GetAssembliesHash(list3);
//Assert
//both should be the same since we only create the hash based on the unique folder of the list passed in, yet
//all files will exist in those folders still
Assert.AreEqual(hash1, hash2);
Assert.AreNotEqual(hash1, hash3);
}
[Test]
public void Ensure_Only_One_Type_List_Created()
{

View File

@@ -59,6 +59,7 @@
<Compile Include="DynamicDocument\PublishedContentDataTableTests.cs" />
<Compile Include="DynamicDocument\PublishedContentTests.cs" />
<Compile Include="DynamicDocument\StronglyTypedQueryTests.cs" />
<Compile Include="HashCodeCombinerTests.cs" />
<Compile Include="HtmlHelperExtensionMethodsTests.cs" />
<Compile Include="IO\IOHelperTest.cs" />
<Compile Include="LibraryTests.cs" />

View File

@@ -37,7 +37,10 @@ namespace Umbraco.Web
{
// do not process if client-side request
if (IsClientSideRequest(httpContext.Request.Url))
return;
return;
//write the trace output for diagnostics at the end of the request
httpContext.Trace.Write("UmbracoModule", "Umbraco request begins");
// ok, process
@@ -94,6 +97,8 @@ namespace Umbraco.Web
if (!EnsureUmbracoRoutablePage(umbracoContext, httpContext))
return;
httpContext.Trace.Write("UmbracoModule", "Umbraco request confirmed");
// ok, process
var uri = umbracoContext.OriginalRequestUrl;
@@ -400,7 +405,7 @@ namespace Umbraco.Web
{
app.BeginRequest += (sender, e) =>
{
var httpContext = ((HttpApplication)sender).Context;
var httpContext = ((HttpApplication)sender).Context;
BeginRequest(new HttpContextWrapper(httpContext));
};
@@ -417,7 +422,12 @@ namespace Umbraco.Web
PersistXmlCache(new HttpContextWrapper(httpContext));
};
// todo: initialize request errors handler
app.EndRequest += (sender, args) =>
{
var httpContext = ((HttpApplication)sender).Context;
//write the trace output for diagnostics at the end of the request
httpContext.Trace.Write("UmbracoModule", "Umbraco request completed");
};
}
public void Dispose()