yv01p/Umbraco-CMS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

U4-10301 Change the PluginManager hash to use more reliable and consistent hashes

Browse Source
This commit is contained in:
Shannon
2017-08-17 11:38:57 +10:00
parent 542406c402
commit f7097d571c
11 changed files with 405 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/Umbraco.Core/HashCodeCombiner.cs
View File

@@ -2,17 +2,19 @@
using System.Globalization;
using System.IO;
using System.Linq;
using System.Text;
namespace Umbraco.Core
{
/// <summary>
/// Used to create a hash code from multiple objects.
/// Used to create a .NET HashCode from multiple objects.
/// </summary>
/// <remarks>
/// .Net has a class the same as this: System.Web.Util.HashCodeCombiner and of course it works for all sorts of things
/// which we've not included here as we just need a quick easy class for this in order to create a unique
/// hash of directories/files to see if they have changed.
///
/// NOTE: It's probably best to not relying on the hashing result across AppDomains! If you need a constant/reliable hash value
/// between AppDomains use SHA1. This is perfect for hashing things in a very fast way for a single AppDomain.
/// </remarks>
internal class HashCodeCombiner
{

154
src/Umbraco.Core/HashGenerator.cs Normal file
View File

@@ -0,0 +1,154 @@
using System;
using System.Globalization;
using System.IO;
using System.Security.Cryptography;
using System.Text;
namespace Umbraco.Core
{
/// <summary>
/// Used to generate a string hash using crypto libraries over multiple objects
/// </summary>
/// <remarks>
/// This should be used to generate a reliable hash that survives AppDomain restarts.
/// This will use the crypto libs to generate the hash and will try to ensure that
/// strings, etc... are not re-allocated so it's not consuming much memory.
/// </remarks>
internal class HashGenerator : DisposableObject
{
public HashGenerator()
{
_writer = new StreamWriter(_ms, Encoding.Unicode, 1024, leaveOpen: true);
}
private readonly MemoryStream _ms = new MemoryStream();
private StreamWriter _writer;
internal void AddInt(int i)
{
_writer.Write(i);
}
internal void AddLong(long i)
{
_writer.Write(i);
}
internal void AddObject(object o)
{
_writer.Write(o);
}
internal void AddDateTime(DateTime d)
{
_writer.Write(d.Ticks);;
}
internal void AddString(string s)
{
if (s != null)
_writer.Write(s);
}
internal void AddCaseInsensitiveString(string s)
{
//I've tried to no allocate a new string with this which can be done if we use the CompareInfo.GetSortKey method which will create a new
//byte array that we can use to write to the output, however this also allocates new objects so i really don't think the performance
//would be much different. In any case, i'll leave this here for reference. We could write the bytes out based on the sort key,
//this is how we could deal with case insensitivity without allocating another string
//for reference see: https://stackoverflow.com/a/10452967/694494
//we could go a step further and s.Normalize() but we're not really dealing with crazy unicode with this class so far.
if (s != null)
_writer.Write(s.ToUpperInvariant());
}
internal void AddFileSystemItem(FileSystemInfo f)
{
//if it doesn't exist, don't proceed.
if (f.Exists == false)
return;
AddCaseInsensitiveString(f.FullName);
AddDateTime(f.CreationTimeUtc);
AddDateTime(f.LastWriteTimeUtc);
//check if it is a file or folder
var fileInfo = f as FileInfo;
if (fileInfo != null)
{
AddLong(fileInfo.Length);
}
var dirInfo = f as DirectoryInfo;
if (dirInfo != null)
{
foreach (var d in dirInfo.GetFiles())
{
AddFile(d);
}
foreach (var s in dirInfo.GetDirectories())
{
AddFolder(s);
}
}
}
internal void AddFile(FileInfo f)
{
AddFileSystemItem(f);
}
internal void AddFolder(DirectoryInfo d)
{
AddFileSystemItem(d);
}
/// <summary>
/// Returns the generated hash output of all added objects
/// </summary>
/// <returns></returns>
internal string GenerateHash()
{
//flush,close,dispose the writer,then create a new one since it's possible to keep adding after GenerateHash is called.
_writer.Flush();
_writer.Close();
_writer.Dispose();
_writer = new StreamWriter(_ms, Encoding.UTF8, 1024, leaveOpen: true);
var hashType = CryptoConfig.AllowOnlyFipsAlgorithms ? "SHA1" : "MD5";
//create an instance of the correct hashing provider based on the type passed in
var hasher = HashAlgorithm.Create(hashType);
if (hasher == null) throw new InvalidOperationException("No hashing type found by name " + hashType);
using (hasher)
{
var buffer = _ms.GetBuffer();
//get the hashed values created by our selected provider
var hashedByteArray = hasher.ComputeHash(buffer);
//create a StringBuilder object
var stringBuilder = new StringBuilder();
//loop to each each byte
foreach (var b in hashedByteArray)
{
//append it to our StringBuilder
stringBuilder.Append(b.ToString("x2"));
}
//return the hashed value
return stringBuilder.ToString();
}
}
protected override void DisposeResources()
{
_writer.Close();
_writer.Dispose();
_ms.Close();
_ms.Dispose();
}
}
}

115
src/Umbraco.Core/PluginManager.cs
View File

@@ -44,8 +44,8 @@ namespace Umbraco.Core
private readonly object _typesLock = new object();
private readonly Dictionary<TypeListKey, TypeList> _types = new Dictionary<TypeListKey, TypeList>();
private long _cachedAssembliesHash = -1;
private long _currentAssembliesHash = -1;
private string _cachedAssembliesHash = null;
private string _currentAssembliesHash = null;
private IEnumerable<Assembly> _assemblies;
private bool _reportedChange;
@@ -75,9 +75,9 @@ namespace Umbraco.Core
if (detectChanges)
{
//first check if the cached hash is 0, if it is then we ne
//first check if the cached hash is string.Empty, if it is then we need
//do the check if they've changed
RequiresRescanning = (CachedAssembliesHash != CurrentAssembliesHash) || CachedAssembliesHash == 0;
RequiresRescanning = (CachedAssembliesHash != CurrentAssembliesHash) || CachedAssembliesHash == string.Empty;
//if they have changed, we need to write the new file
if (RequiresRescanning)
{
@@ -180,23 +180,20 @@ namespace Umbraco.Core
/// <summary>
/// Gets the currently cached hash value of the scanned assemblies.
/// </summary>
/// <value>The cached hash value, or 0 if no cache is found.</value>
internal long CachedAssembliesHash
/// <value>The cached hash value, or string.Empty if no cache is found.</value>
internal string CachedAssembliesHash
{
get
{
if (_cachedAssembliesHash != -1)
if (_cachedAssembliesHash != null)
return _cachedAssembliesHash;
var filePath = GetPluginHashFilePath();
if (File.Exists(filePath) == false) return 0;
if (File.Exists(filePath) == false) return string.Empty;
var hash = File.ReadAllText(filePath, Encoding.UTF8);
long val;
if (long.TryParse(hash, out val) == false) return 0;
_cachedAssembliesHash = val;
_cachedAssembliesHash = hash;
return _cachedAssembliesHash;
}
}
@@ -205,11 +202,11 @@ namespace Umbraco.Core
/// Gets the current assemblies hash based on creating a hash from the assemblies in various places.
/// </summary>
/// <value>The current hash.</value>
internal long CurrentAssembliesHash
internal string CurrentAssembliesHash
{
get
{
if (_currentAssembliesHash != -1)
if (_currentAssembliesHash != null)
return _currentAssembliesHash;
_currentAssembliesHash = GetFileHash(new List<Tuple<FileSystemInfo, bool>>
@@ -245,41 +242,40 @@ namespace Umbraco.Core
/// <returns>The hash.</returns>
/// <remarks>Each file is a tuple containing the FileInfo object and a boolean which indicates whether to hash the
/// file properties (false) or the file contents (true).</remarks>
internal static long GetFileHash(IEnumerable<Tuple<FileSystemInfo, bool>> filesAndFolders, ProfilingLogger logger)
internal static string GetFileHash(IEnumerable<Tuple<FileSystemInfo, bool>> filesAndFolders, ProfilingLogger logger)
{
using (logger.TraceDuration<PluginManager>("Determining hash of code files on disk", "Hash determined"))
{
var hashCombiner = new HashCodeCombiner();
{
// get the distinct file infos to hash
var uniqInfos = new HashSet<string>();
var uniqContent = new HashSet<string>();
foreach (var fileOrFolder in filesAndFolders)
using (var generator = new HashGenerator())
{
var info = fileOrFolder.Item1;
if (fileOrFolder.Item2)
foreach (var fileOrFolder in filesAndFolders)
{
// add each unique file's contents to the hash
// normalize the content for cr/lf and case-sensitivity
if (uniqContent.Contains(info.FullName)) continue;
uniqContent.Add(info.FullName);
if (File.Exists(info.FullName) == false) continue;
var content = RemoveCrLf(File.ReadAllText(info.FullName));
hashCombiner.AddCaseInsensitiveString(content);
var info = fileOrFolder.Item1;
if (fileOrFolder.Item2)
{
// add each unique file's contents to the hash
// normalize the content for cr/lf and case-sensitivity
if (uniqContent.Add(info.FullName))
{
if (File.Exists(info.FullName) == false) continue;
var content = RemoveCrLf(File.ReadAllText(info.FullName));
generator.AddCaseInsensitiveString(content);
}
}
else
{
// add each unique folder/file to the hash
if (uniqInfos.Add(info.FullName))
{
generator.AddFileSystemItem(info);
}
}
}
else
{
// add each unique folder/file to the hash
if (uniqInfos.Contains(info.FullName)) continue;
uniqInfos.Add(info.FullName);
hashCombiner.AddFileSystemItem(info);
}
}
return ConvertHashToInt64(hashCombiner.GetCombinedHashCode());
return generator.GenerateHash();
}
}
}
@@ -303,34 +299,25 @@ namespace Umbraco.Core
/// <param name="filesAndFolders">A collection of files.</param>
/// <param name="logger">A profiling logger.</param>
/// <returns>The hash.</returns>
internal static long GetFileHash(IEnumerable<FileSystemInfo> filesAndFolders, ProfilingLogger logger)
internal static string GetFileHash(IEnumerable<FileSystemInfo> filesAndFolders, ProfilingLogger logger)
{
using (logger.TraceDuration<PluginManager>("Determining hash of code files on disk", "Hash determined"))
{
var hashCombiner = new HashCodeCombiner();
// get the distinct file infos to hash
var uniqInfos = new HashSet<string>();
foreach (var fileOrFolder in filesAndFolders)
using (var generator = new HashGenerator())
{
if (uniqInfos.Contains(fileOrFolder.FullName)) continue;
uniqInfos.Add(fileOrFolder.FullName);
hashCombiner.AddFileSystemItem(fileOrFolder);
}
// get the distinct file infos to hash
var uniqInfos = new HashSet<string>();
return ConvertHashToInt64(hashCombiner.GetCombinedHashCode());
foreach (var fileOrFolder in filesAndFolders)
{
if (uniqInfos.Contains(fileOrFolder.FullName)) continue;
uniqInfos.Add(fileOrFolder.FullName);
generator.AddFileSystemItem(fileOrFolder);
}
return generator.GenerateHash();
}
}
}
/// <summary>
/// Converts a string hash value into an Int64.
/// </summary>
internal static long ConvertHashToInt64(string val)
{
long outVal;
return long.TryParse(val, NumberStyles.AllowHexSpecifier, CultureInfo.InvariantCulture, out outVal) ? outVal : 0;
}
}
#endregion

2
src/Umbraco.Core/StringExtensions.cs
View File

@@ -759,7 +759,7 @@ namespace Umbraco.Core
foreach (var b in hashedByteArray)
{
//append it to our StringBuilder
stringBuilder.Append(b.ToString("x2").ToLower());
stringBuilder.Append(b.ToString("x2"));
}
//return the hashed value

10
src/Umbraco.Core/Sync/WebServiceServerMessenger.cs
View File

@@ -126,10 +126,12 @@ namespace Umbraco.Core.Sync
public static string GetServerHash(string machineName, string appDomainAppId)
{
var hasher = new HashCodeCombiner();
hasher.AddCaseInsensitiveString(appDomainAppId);
hasher.AddCaseInsensitiveString(machineName);
return hasher.GetCombinedHashCode();
using (var generator = new HashGenerator())
{
generator.AddString(machineName);
generator.AddString(appDomainAppId);
return generator.GenerateHash();
}
}
protected override bool RequiresDistributed(IEnumerable<IServerAddress> servers, ICacheRefresher refresher, MessageType messageType)

1
src/Umbraco.Core/Umbraco.Core.csproj
View File

@@ -342,6 +342,7 @@
<Compile Include="Events\SupersedeEventAttribute.cs" />
<Compile Include="Exceptions\ConnectionException.cs" />
<Compile Include="HashCodeHelper.cs" />
<Compile Include="HashGenerator.cs" />
<Compile Include="IHttpContextAccessor.cs" />
<Compile Include="Models\EntityBase\IDeletableEntity.cs" />
<Compile Include="Models\IUserControl.cs" />

2
src/Umbraco.Tests/HashCodeCombinerTests.cs
View File

@@ -6,7 +6,7 @@ using Umbraco.Core;
namespace Umbraco.Tests
{
[TestFixture]
[TestFixture]
public class HashCodeCombinerTests
{

184
src/Umbraco.Tests/HashGeneratorTests.cs Normal file
View File

@@ -0,0 +1,184 @@
using System;
using System.IO;
using System.Reflection;
using NUnit.Framework;
using Umbraco.Core;
namespace Umbraco.Tests
{
[TestFixture]
public class HashGeneratorTests
{
private string Generate(bool isCaseSensitive, params string[] strs)
{
using (var generator = new HashGenerator())
{
foreach (var str in strs)
{
if (isCaseSensitive)
generator.AddString(str);
else
generator.AddCaseInsensitiveString(str);
}
return generator.GenerateHash();
}
}
[Test]
public void Generate_Hash_Multiple_Strings_Case_Sensitive()
{
var hash1 = Generate(true, "hello", "world");
var hash2 = Generate(true, "hello", "world");
var hashFalse1 = Generate(true, "hello", "worlD");
var hashFalse2 = Generate(true, "hEllo", "world");
Assert.AreEqual(hash1, hash2);
Assert.AreNotEqual(hash1, hashFalse1);
Assert.AreNotEqual(hash1, hashFalse2);
}
[Test]
public void Generate_Hash_Multiple_Strings_Case_Insensitive()
{
var hash1 = Generate(false, "hello", "world");
var hash2 = Generate(false, "hello", "world");
var hashFalse1 = Generate(false, "hello", "worlD");
var hashFalse2 = Generate(false, "hEllo", "world");
Assert.AreEqual(hash1, hash2);
Assert.AreEqual(hash1, hashFalse1);
Assert.AreEqual(hash1, hashFalse2);
}
private DirectoryInfo PrepareFolder()
{
var assDir = new FileInfo(Assembly.GetExecutingAssembly().Location).Directory;
var dir = Directory.CreateDirectory(Path.Combine(assDir.FullName, "HashCombiner", Guid.NewGuid().ToString("N")));
foreach (var f in dir.GetFiles())
{
f.Delete();
}
return dir;
}
[Test]
public void HashCombiner_Test_String()
{
using (var combiner1 = new HashGenerator())
using (var combiner2 = new HashGenerator())
{
combiner1.AddCaseInsensitiveString("Hello");
combiner2.AddCaseInsensitiveString("hello");
Assert.AreEqual(combiner1.GenerateHash(), combiner2.GenerateHash());
combiner2.AddCaseInsensitiveString("world");
Assert.AreNotEqual(combiner1.GenerateHash(), combiner2.GenerateHash());
}
}
[Test]
public void HashCombiner_Test_Int()
{
using (var combiner1 = new HashGenerator())
using (var combiner2 = new HashGenerator())
{
combiner1.AddInt(1234);
combiner2.AddInt(1234);
Assert.AreEqual(combiner1.GenerateHash(), combiner2.GenerateHash());
combiner2.AddInt(1);
Assert.AreNotEqual(combiner1.GenerateHash(), combiner2.GenerateHash());
}
}
[Test]
public void HashCombiner_Test_DateTime()
{
using (var combiner1 = new HashGenerator())
using (var combiner2 = new HashGenerator())
{
var dt = DateTime.Now;
combiner1.AddDateTime(dt);
combiner2.AddDateTime(dt);
Assert.AreEqual(combiner1.GenerateHash(), combiner2.GenerateHash());
combiner2.AddDateTime(DateTime.Now);
Assert.AreNotEqual(combiner1.GenerateHash(), combiner2.GenerateHash());
}
}
[Test]
public void HashCombiner_Test_File()
{
using (var combiner1 = new HashGenerator())
using (var combiner2 = new HashGenerator())
using (var combiner3 = new HashGenerator())
{
var dir = PrepareFolder();
var file1Path = Path.Combine(dir.FullName, "hastest1.txt");
File.Delete(file1Path);
using (var file1 = File.CreateText(Path.Combine(dir.FullName, "hastest1.txt")))
{
file1.WriteLine("hello");
}
var file2Path = Path.Combine(dir.FullName, "hastest2.txt");
File.Delete(file2Path);
using (var file2 = File.CreateText(Path.Combine(dir.FullName, "hastest2.txt")))
{
//even though files are the same, the dates are different
file2.WriteLine("hello");
}
combiner1.AddFile(new FileInfo(file1Path));
combiner2.AddFile(new FileInfo(file1Path));
combiner3.AddFile(new FileInfo(file2Path));
Assert.AreEqual(combiner1.GenerateHash(), combiner2.GenerateHash());
Assert.AreNotEqual(combiner1.GenerateHash(), combiner3.GenerateHash());
combiner2.AddFile(new FileInfo(file2Path));
Assert.AreNotEqual(combiner1.GenerateHash(), combiner2.GenerateHash());
}
}
[Test]
public void HashCombiner_Test_Folder()
{
using (var combiner1 = new HashGenerator())
using (var combiner2 = new HashGenerator())
using (var combiner3 = new HashGenerator())
{
var dir = PrepareFolder();
var file1Path = Path.Combine(dir.FullName, "hastest1.txt");
File.Delete(file1Path);
using (var file1 = File.CreateText(Path.Combine(dir.FullName, "hastest1.txt")))
{
file1.WriteLine("hello");
}
//first test the whole folder
combiner1.AddFolder(dir);
combiner2.AddFolder(dir);
Assert.AreEqual(combiner1.GenerateHash(), combiner2.GenerateHash());
//now add a file to the folder
var file2Path = Path.Combine(dir.FullName, "hastest2.txt");
File.Delete(file2Path);
using (var file2 = File.CreateText(Path.Combine(dir.FullName, "hastest2.txt")))
{
//even though files are the same, the dates are different
file2.WriteLine("hello");
}
combiner3.AddFolder(dir);
Assert.AreNotEqual(combiner1.GenerateHash(), combiner3.GenerateHash());
}
}
}
}

10
src/Umbraco.Tests/Plugins/PluginManagerTests.cs
View File

@@ -218,15 +218,7 @@ AnotherContentFinder
//ensure they are all found
Assert.IsTrue(plugins.Result.ContainsAll(shouldContain));
}
[Test]
public void PluginHash_From_String()
{
var s = "hello my name is someone".GetHashCode().ToString("x", CultureInfo.InvariantCulture);
var output = PluginManager.ConvertHashToInt64(s);
Assert.AreNotEqual(0, output);
}
[Test]
public void Get_Plugins_Hash()
{

2
src/Umbraco.Tests/Strings/StringExtensionsTests.cs
View File

@@ -24,7 +24,7 @@ namespace Umbraco.Tests.Strings
{
ShortStringHelperResolver.Reset();
}
[TestCase("hello", "world", false)]
[TestCase("hello", "hello", true)]
[TestCase("hellohellohellohellohellohellohello", "hellohellohellohellohellohellohelloo", false)]

1
src/Umbraco.Tests/Umbraco.Tests.csproj
View File

@@ -165,6 +165,7 @@
<Compile Include="Cache\CacheRefresherEventHandlerTests.cs" />
<Compile Include="Dependencies\NuGet.cs" />
<Compile Include="CallContextTests.cs" />
<Compile Include="HashGeneratorTests.cs" />
<Compile Include="Issues\U9560.cs" />
<Compile Include="Collections\OrderedHashSetTests.cs" />
<Compile Include="IO\ShadowFileSystemTests.cs" />