From c1c189d47f73af319e473c08b38bb79b7ec0951b Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Thu, 9 Jul 2020 00:17:31 +1200 Subject: [PATCH 1/5] Wip support for compressing/decompressing nucache documents on a per property basis. Option for compressing the properties in sql/nucache.db. Option for immediate/lazy decompression of properties. Mapping support for shorter property alias. TODO: config file for property map TODO: HasValue and IsValue on propertyvalueconverterbase --- .../AppSettingsNucachePropertyMapFactory.cs | 47 +++++ .../DataSource/BTree.ContentDataSerializer.cs | 13 +- .../BTree.ContentNodeKitSerializer.cs | 20 +- ...Tree.DictionaryOfPropertyDataSerializer.cs | 2 +- .../NuCache/DataSource/BTree.cs | 4 +- .../IDictionaryOfPropertyDataSerializer.cs | 11 ++ .../INucachePropertyOptionsFactory.cs | 13 ++ .../DataSource/LazyCompressedString.cs | 31 +++ .../Lz4DictionaryOfPropertyDataSerializer.cs | 178 ++++++++++++++++++ .../MsgPackContentNestedDataSerializer.cs | 41 +++- .../DataSource/NucachePropertyOptions.cs | 20 ++ .../NuCache/DataSource/SerializerBase.cs | 21 ++- .../PublishedCache/NuCache/NuCacheComposer.cs | 22 ++- .../NuCache/PublishedSnapshotService.cs | 8 +- src/Umbraco.Web/Umbraco.Web.csproj | 9 + 15 files changed, 422 insertions(+), 18 deletions(-) create mode 100644 src/Umbraco.Web/PublishedCache/NuCache/DataSource/AppSettingsNucachePropertyMapFactory.cs create mode 100644 src/Umbraco.Web/PublishedCache/NuCache/DataSource/IDictionaryOfPropertyDataSerializer.cs create mode 100644 src/Umbraco.Web/PublishedCache/NuCache/DataSource/INucachePropertyOptionsFactory.cs create mode 100644 src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs create mode 100644 src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs create mode 100644 src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyOptions.cs diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/AppSettingsNucachePropertyMapFactory.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/AppSettingsNucachePropertyMapFactory.cs new file mode 100644 index 0000000000..ddc9bc9b1a --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/AppSettingsNucachePropertyMapFactory.cs @@ -0,0 +1,47 @@ +using System; +using System.Collections.Generic; +using System.Configuration; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + public class AppSettingsNucachePropertyMapFactory : INucachePropertyOptionsFactory + { + public NucachePropertyOptions GetNucachePropertyOptions() + { + NucachePropertyOptions options = new NucachePropertyOptions + { + PropertyMap = GetPropertyMap(), + LZ4CompressionLevel = K4os.Compression.LZ4.LZ4Level.L10_OPT, + MinimumCompressibleStringLength = null + }; + return options; + } + + public IReadOnlyDictionary GetPropertyMap() + { + var propertyMap = new Dictionary(); + // TODO: Use xml/json/c# to define map + var propertyDictionarySerializerMap = ConfigurationManager.AppSettings["Umbraco.Web.PublishedCache.NuCache.PropertySerializationMap"]; + if (!string.IsNullOrWhiteSpace(propertyDictionarySerializerMap)) + { + //propertyAlias,CompressionLevel,DecompressionLevel,mappedAlias; + propertyDictionarySerializerMap.Split(';') + .Select(x => + { + var y = x.Split(','); + (string alias, NucachePropertyCompressionLevel compressionLevel, NucachePropertyDecompressionLevel decompressionLevel, string mappedAlias) v = (y[0], + (NucachePropertyCompressionLevel)System.Enum.Parse(typeof(NucachePropertyCompressionLevel), y[1]), + (NucachePropertyDecompressionLevel)System.Enum.Parse(typeof(NucachePropertyDecompressionLevel), y[2]), + y[3] + ); + return v; + }) + .ToList().ForEach(x => propertyMap.Add(x.alias, (x.compressionLevel, x.decompressionLevel, x.mappedAlias))); + } + return propertyMap; + } + } +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.ContentDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.ContentDataSerializer.cs index d02af375c6..9cc5d3a701 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.ContentDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.ContentDataSerializer.cs @@ -5,8 +5,17 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource { class ContentDataSerializer : ISerializer { + public ContentDataSerializer(IDictionaryOfPropertyDataSerializer dictionaryOfPropertyDataSerializer = null) + { + _dictionaryOfPropertyDataSerializer = dictionaryOfPropertyDataSerializer; + if(_dictionaryOfPropertyDataSerializer == null) + { + _dictionaryOfPropertyDataSerializer = PropertiesSerializer; + } + } private static readonly DictionaryOfPropertyDataSerializer PropertiesSerializer = new DictionaryOfPropertyDataSerializer(); private static readonly DictionaryOfCultureVariationSerializer CultureVariationsSerializer = new DictionaryOfCultureVariationSerializer(); + private readonly IDictionaryOfPropertyDataSerializer _dictionaryOfPropertyDataSerializer; public ContentData ReadFrom(Stream stream) { @@ -19,7 +28,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource VersionDate = PrimitiveSerializer.DateTime.ReadFrom(stream), WriterId = PrimitiveSerializer.Int32.ReadFrom(stream), TemplateId = PrimitiveSerializer.Int32.ReadFrom(stream), - Properties = PropertiesSerializer.ReadFrom(stream), // TODO: We don't want to allocate empty arrays + Properties = _dictionaryOfPropertyDataSerializer.ReadFrom(stream), // TODO: We don't want to allocate empty arrays CultureInfos = CultureVariationsSerializer.ReadFrom(stream) // TODO: We don't want to allocate empty arrays }; } @@ -36,7 +45,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource { PrimitiveSerializer.Int32.WriteTo(value.TemplateId.Value, stream); } - PropertiesSerializer.WriteTo(value.Properties, stream); + _dictionaryOfPropertyDataSerializer.WriteTo(value.Properties, stream); CultureVariationsSerializer.WriteTo(value.CultureInfos, stream); } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.ContentNodeKitSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.ContentNodeKitSerializer.cs index f799869850..1d668ba4fd 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.ContentNodeKitSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.ContentNodeKitSerializer.cs @@ -5,7 +5,17 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource { internal class ContentNodeKitSerializer : ISerializer { - static readonly ContentDataSerializer DataSerializer = new ContentDataSerializer(); + public ContentNodeKitSerializer(ContentDataSerializer contentDataSerializer = null) + { + _contentDataSerializer = contentDataSerializer; + if(_contentDataSerializer == null) + { + _contentDataSerializer = DefaultDataSerializer; + } + } + static readonly ContentDataSerializer DefaultDataSerializer = new ContentDataSerializer(); + private readonly ContentDataSerializer _contentDataSerializer; + //static readonly ListOfIntSerializer ChildContentIdsSerializer = new ListOfIntSerializer(); public ContentNodeKit ReadFrom(Stream stream) @@ -26,10 +36,10 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource }; var hasDraft = PrimitiveSerializer.Boolean.ReadFrom(stream); if (hasDraft) - kit.DraftData = DataSerializer.ReadFrom(stream); + kit.DraftData = _contentDataSerializer.ReadFrom(stream); var hasPublished = PrimitiveSerializer.Boolean.ReadFrom(stream); if (hasPublished) - kit.PublishedData = DataSerializer.ReadFrom(stream); + kit.PublishedData = _contentDataSerializer.ReadFrom(stream); return kit; } @@ -47,11 +57,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource PrimitiveSerializer.Boolean.WriteTo(value.DraftData != null, stream); if (value.DraftData != null) - DataSerializer.WriteTo(value.DraftData, stream); + _contentDataSerializer.WriteTo(value.DraftData, stream); PrimitiveSerializer.Boolean.WriteTo(value.PublishedData != null, stream); if (value.PublishedData != null) - DataSerializer.WriteTo(value.PublishedData, stream); + _contentDataSerializer.WriteTo(value.PublishedData, stream); } } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs index e3fa6597e4..cb12164397 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs @@ -6,7 +6,7 @@ using Umbraco.Core; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { - internal class DictionaryOfPropertyDataSerializer : SerializerBase, ISerializer> + internal class DictionaryOfPropertyDataSerializer : SerializerBase, ISerializer>, IDictionaryOfPropertyDataSerializer { public IDictionary ReadFrom(Stream stream) { diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.cs index 910c0ca737..80d8488495 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.cs @@ -6,10 +6,10 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource { internal class BTree { - public static BPlusTree GetTree(string filepath, bool exists) + public static BPlusTree GetTree(string filepath, bool exists, ContentDataSerializer contentDataSerializer = null) { var keySerializer = new PrimitiveSerializer(); - var valueSerializer = new ContentNodeKitSerializer(); + var valueSerializer = new ContentNodeKitSerializer(contentDataSerializer); var options = new BPlusTree.OptionsV2(keySerializer, valueSerializer) { CreateFile = exists ? CreatePolicy.IfNeeded : CreatePolicy.Always, diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IDictionaryOfPropertyDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IDictionaryOfPropertyDataSerializer.cs new file mode 100644 index 0000000000..a086e3e2f3 --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IDictionaryOfPropertyDataSerializer.cs @@ -0,0 +1,11 @@ +using System.Collections.Generic; +using System.IO; + +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + internal interface IDictionaryOfPropertyDataSerializer + { + IDictionary ReadFrom(Stream stream); + void WriteTo(IDictionary value, Stream stream); + } +} \ No newline at end of file diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/INucachePropertyOptionsFactory.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/INucachePropertyOptionsFactory.cs new file mode 100644 index 0000000000..82df43a5bb --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/INucachePropertyOptionsFactory.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + public interface INucachePropertyOptionsFactory + { + NucachePropertyOptions GetNucachePropertyOptions(); + } +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs new file mode 100644 index 0000000000..99bc2f5859 --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs @@ -0,0 +1,31 @@ +using K4os.Compression.LZ4; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + public class LazyCompressedString + { + private byte[] _bytes; + private string _str; + + public LazyCompressedString(byte[] bytes) + { + _bytes = bytes; + } + public override string ToString() + { + return LazyInitializer.EnsureInitialized(ref _str, () => + { + var str = Encoding.UTF8.GetString(LZ4Pickler.Unpickle(_bytes)); + _bytes = null; + return str; + }); + } + } + +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs new file mode 100644 index 0000000000..7ec2fb17b6 --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs @@ -0,0 +1,178 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using CSharpTest.Net.Serialization; +using Umbraco.Core; +using System.Text; +using K4os.Compression.LZ4; + +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + /// + /// If/where to compress custom properties for nucache + /// + public enum NucachePropertyCompressionLevel + { + None = 0, + SQLDatabase = 1, + NucacheDatabase = 2 + } + /// + /// If/where to decompress custom properties for nucache + /// + public enum NucachePropertyDecompressionLevel + { + NotCompressed = 0, + Immediate = 1, + Lazy = 2 + } + + + internal class Lz4DictionaryOfPropertyDataSerializer : SerializerBase, ISerializer>, IDictionaryOfPropertyDataSerializer + { + private readonly IReadOnlyDictionary _compressProperties; + private readonly LZ4Level _compressionLevel; + private readonly long? _minimumStringLengthForCompression; + private readonly IReadOnlyDictionary _uncompressProperties; + + + public Lz4DictionaryOfPropertyDataSerializer(NucachePropertyOptions nucachePropertyOptions) + { + _compressProperties = nucachePropertyOptions.PropertyMap.ToList().ToDictionary(x => string.Intern(x.Value.mappedAlias), x => (x.Value.compress,x.Value.decompressionLevel, string.Intern(x.Value.mappedAlias))); + _minimumStringLengthForCompression = nucachePropertyOptions.MinimumCompressibleStringLength; + _uncompressProperties = _compressProperties.ToList().ToDictionary(x => x.Value.mappedAlias, x => (x.Value.compress, x.Value.decompressionLevel, x.Value.mappedAlias)); + + _compressionLevel = nucachePropertyOptions.LZ4CompressionLevel; + } + + + public IDictionary ReadFrom(Stream stream) + { + var dict = new Dictionary(StringComparer.InvariantCultureIgnoreCase); + + // read properties count + var pcount = PrimitiveSerializer.Int32.ReadFrom(stream); + + // read each property + for (var i = 0; i < pcount; i++) + { + + // read property alias + var alias = PrimitiveSerializer.String.ReadFrom(stream); + var map = GetDeSerializationMap(alias); + var key = string.Intern(map.MappedAlias ?? alias); + + // read values count + var vcount = PrimitiveSerializer.Int32.ReadFrom(stream); + + // create pdata and add to the dictionary + var pdatas = new List(); + + // for each value, read and add to pdata + for (var j = 0; j < vcount; j++) + { + var pdata = new PropertyData(); + pdatas.Add(pdata); + + // everything that can be null is read/written as object + // even though - culture and segment should never be null here, as 'null' represents + // the 'current' value, and string.Empty should be used to represent the invariant or + // neutral values - PropertyData throws when getting nulls, so falling back to + // string.Empty here - what else? + pdata.Culture = ReadStringObject(stream, true) ?? string.Empty; + pdata.Segment = ReadStringObject(stream, true) ?? string.Empty; + pdata.Value = ReadObject(stream); + + var decompressionLevel = NucachePropertyDecompressionLevel.Immediate; + if ((map.Compress.Equals(NucachePropertyCompressionLevel.NucacheDatabase) || map.Compress.Equals(NucachePropertyCompressionLevel.SQLDatabase)) + && pdata.Value != null && pdata.Value is byte[] byteArrayValue) + { + //Compressed string + switch (decompressionLevel) + { + case NucachePropertyDecompressionLevel.Lazy: + pdata.Value = new LazyCompressedString(byteArrayValue); + break; + case NucachePropertyDecompressionLevel.NotCompressed: + break;//Shouldn't be any not compressed + case NucachePropertyDecompressionLevel.Immediate: + default: + pdata.Value = Encoding.UTF8.GetString(LZ4Pickler.Unpickle(byteArrayValue)); + break; + } + } + } + + dict[key] = pdatas.ToArray(); + } + return dict; + } + + public void WriteTo(IDictionary value, Stream stream) + { + // write properties count + PrimitiveSerializer.Int32.WriteTo(value.Count, stream); + + // write each property + foreach (var (alias, values) in value) + { + var map = GetSerializationMap(alias); + + // write alias + PrimitiveSerializer.String.WriteTo(map.MappedAlias ?? alias, stream); + + // write values count + PrimitiveSerializer.Int32.WriteTo(values.Length, stream); + + // write each value + foreach (var pdata in values) + { + // everything that can be null is read/written as object + // even though - culture and segment should never be null here, + // see note in ReadFrom() method above + WriteObject(pdata.Culture ?? string.Empty, stream); + WriteObject(pdata.Segment ?? string.Empty, stream); + + //Only compress strings + if (pdata.Value is string stringValue && pdata.Value != null && map.Compress.Equals(NucachePropertyCompressionLevel.NucacheDatabase) + && (_minimumStringLengthForCompression == null + || !_minimumStringLengthForCompression.HasValue + || stringValue.Length > _minimumStringLengthForCompression.Value)) + { + var stringBytes = Encoding.UTF8.GetBytes(stringValue); + var compressedBytes = LZ4Pickler.Pickle(stringBytes, _compressionLevel); + WriteObject(compressedBytes, stream); + } + WriteObject(pdata.Value, stream); + } + } + } + private readonly (NucachePropertyCompressionLevel Compress, NucachePropertyDecompressionLevel decompressionLevel, string MappedAlias) DEFAULT_MAP =(NucachePropertyCompressionLevel.None, NucachePropertyDecompressionLevel.NotCompressed, null); + public (NucachePropertyCompressionLevel Compress, NucachePropertyDecompressionLevel decompressionLevel, string MappedAlias) GetSerializationMap(string propertyAlias) + { + if (_compressProperties == null) + { + return DEFAULT_MAP; + } + if (_compressProperties.TryGetValue(propertyAlias, out (NucachePropertyCompressionLevel compress, NucachePropertyDecompressionLevel decompressionLevel, string mappedAlias) map)) + { + return map; + } + + return DEFAULT_MAP; + } + public (NucachePropertyCompressionLevel Compress, NucachePropertyDecompressionLevel decompressionLevel, string MappedAlias) GetDeSerializationMap(string propertyAlias) + { + if (_uncompressProperties == null) + { + return DEFAULT_MAP; + } + if (_uncompressProperties.TryGetValue(propertyAlias, out (NucachePropertyCompressionLevel compress, NucachePropertyDecompressionLevel decompressionLevel, string mappedAlias) map)) + { + return map; + } + return DEFAULT_MAP; + } + } +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs index 1a093292b5..be4e8a9832 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs @@ -1,16 +1,20 @@ -using MessagePack; +using K4os.Compression.LZ4; +using MessagePack; using MessagePack.Formatters; using MessagePack.Resolvers; using System; using System.Collections.Generic; +using System.Linq; +using System.Text; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { internal class MsgPackContentNestedDataSerializer : IContentNestedDataByteSerializer { private MessagePackSerializerOptions _options; + private readonly NucachePropertyOptions _propertyOptions; - public MsgPackContentNestedDataSerializer() + public MsgPackContentNestedDataSerializer(NucachePropertyOptions propertyOptions = null) { var defaultOptions = ContractlessStandardResolver.Options; @@ -30,6 +34,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource _options = defaultOptions .WithResolver(resolver) .WithCompression(MessagePackCompression.Lz4BlockArray); + _propertyOptions = propertyOptions ?? new NucachePropertyOptions(); } public string ToJson(string serialized) @@ -48,10 +53,42 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource public string Serialize(ContentNestedData nestedData) { + Optimize(nestedData); + var bin = MessagePackSerializer.Serialize(nestedData, _options); return Convert.ToBase64String(bin); } + /// + /// Compress properties and map property names to shorter names + /// + /// + private void Optimize(ContentNestedData nestedData) + { + if (_propertyOptions.PropertyMap != null && _propertyOptions.PropertyMap.Any()) + { + foreach (var map in _propertyOptions.PropertyMap) + { + if (map.Value.compress.Equals(NucachePropertyCompressionLevel.SQLDatabase)) + { + if (nestedData.PropertyData.TryGetValue(map.Key, out PropertyData[] properties)) + { + foreach (var property in properties.Where(x => x.Value != null && x.Value is string)) + { + property.Value = LZ4Pickler.Pickle(Encoding.UTF8.GetBytes(property.Value as string), _propertyOptions.LZ4CompressionLevel); + } + } + } + if (map.Value.mappedAlias != null && !map.Key.Equals(map.Value.mappedAlias) + && nestedData.PropertyData.Remove(map.Key) && nestedData.PropertyData.TryGetValue(map.Key, out PropertyData[] properties2)) + { + nestedData.PropertyData.Remove(map.Key); + nestedData.PropertyData.Add(map.Value.mappedAlias, properties2); + } + } + } + } + public ContentNestedData DeserializeBytes(byte[] data) => MessagePackSerializer.Deserialize(data, _options); public byte[] SerializeBytes(ContentNestedData nestedData) => MessagePackSerializer.Serialize(nestedData, _options); diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyOptions.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyOptions.cs new file mode 100644 index 0000000000..f10787646a --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyOptions.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + public class NucachePropertyOptions + { + public IReadOnlyDictionary PropertyMap + { get; set; } = new Dictionary(); + + public K4os.Compression.LZ4.LZ4Level LZ4CompressionLevel { get; set; } = K4os.Compression.LZ4.LZ4Level.L00_FAST; + + public long? MinimumCompressibleStringLength { get; set; } + } +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs index afc0827ed3..885b5cf80e 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs @@ -16,6 +16,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource private const char PrefixDouble = 'B'; private const char PrefixDateTime = 'D'; private const char PrefixByte = 'O'; + private const char PrefixByteArray = 'A'; protected string ReadString(Stream stream) => PrimitiveSerializer.String.ReadFrom(stream); protected int ReadInt(Stream stream) => PrimitiveSerializer.Int32.ReadFrom(stream); @@ -23,6 +24,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource protected float ReadFloat(Stream stream) => PrimitiveSerializer.Float.ReadFrom(stream); protected double ReadDouble(Stream stream) => PrimitiveSerializer.Double.ReadFrom(stream); protected DateTime ReadDateTime(Stream stream) => PrimitiveSerializer.DateTime.ReadFrom(stream); + protected byte[] ReadByteArray(Stream stream) => PrimitiveSerializer.Bytes.ReadFrom(stream); private T? ReadObject(Stream stream, char t, Func read) where T : struct @@ -39,7 +41,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource var type = PrimitiveSerializer.Char.ReadFrom(stream); if (type == PrefixNull) return null; if (type != PrefixString) - throw new NotSupportedException($"Cannot deserialize type '{type}', expected 'S'."); + throw new NotSupportedException($"Cannot deserialize type '{type}', expected '{PrefixString}'."); return intern ? string.Intern(PrimitiveSerializer.String.ReadFrom(stream)) : PrimitiveSerializer.String.ReadFrom(stream); @@ -51,6 +53,16 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource protected double? ReadDoubleObject(Stream stream) => ReadObject(stream, PrefixDouble, ReadDouble); protected DateTime? ReadDateTimeObject(Stream stream) => ReadObject(stream, PrefixDateTime, ReadDateTime); + protected byte[] ReadByteArrayObject(Stream stream) // required 'cos byte[] is not a struct + { + var type = PrimitiveSerializer.Char.ReadFrom(stream); + if (type == PrefixNull) return null; + if (type != PrefixByteArray) + throw new NotSupportedException($"Cannot deserialize type '{type}', expected '{PrefixByteArray}'."); + return PrimitiveSerializer.Bytes.ReadFrom(stream); + } + + protected object ReadObject(Stream stream) => ReadObject(PrimitiveSerializer.Char.ReadFrom(stream), stream); @@ -81,6 +93,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource return PrimitiveSerializer.Double.ReadFrom(stream); case PrefixDateTime: return PrimitiveSerializer.DateTime.ReadFrom(stream); + case PrefixByteArray: + return PrimitiveSerializer.Bytes.ReadFrom(stream); default: throw new NotSupportedException($"Cannot deserialize unknown type '{type}'."); } @@ -137,6 +151,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource PrimitiveSerializer.Char.WriteTo(PrefixUInt32, stream); PrimitiveSerializer.UInt32.WriteTo(uInt32Value, stream); } + else if (value is byte[] byteArrayValue) + { + PrimitiveSerializer.Char.WriteTo(PrefixByteArray, stream); + PrimitiveSerializer.Bytes.WriteTo(byteArrayValue, stream); + } else throw new NotSupportedException("Value type " + value.GetType().FullName + " cannot be serialized."); } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs index 205ac55cdc..95ff91304d 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs @@ -1,4 +1,6 @@ -using System.Configuration; +using System.Collections.Generic; +using System.Configuration; +using System.Linq; using Umbraco.Core; using Umbraco.Core.Composing; using Umbraco.Web.PublishedCache.NuCache.DataSource; @@ -15,11 +17,26 @@ namespace Umbraco.Web.PublishedCache.NuCache if (serializer == "MsgPack") { - composition.Register(); + var propertyDictionarySerializer = ConfigurationManager.AppSettings["Umbraco.Web.PublishedCache.NuCache.DictionaryOfPropertiesSerializer"]; + if (propertyDictionarySerializer == "LZ4Map") + { + composition.Register(); + composition.Register(factory => + { + var lz4Serializer = factory.GetInstance(); + return new ContentDataSerializer(lz4Serializer); + }); + } + else + { + composition.Register(factory => new ContentDataSerializer(new DictionaryOfPropertyDataSerializer())); + } + composition.Register(); } else { composition.Register(); + composition.Register(factory => new ContentDataSerializer(new DictionaryOfPropertyDataSerializer())); } // register the NuCache database data source @@ -34,5 +51,6 @@ namespace Umbraco.Web.PublishedCache.NuCache // TODO: no NuCache health check yet //composition.HealthChecks().Add(); } + } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs index c87f035589..0135f204c7 100755 --- a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs @@ -50,6 +50,7 @@ namespace Umbraco.Web.PublishedCache.NuCache private readonly IDefaultCultureAccessor _defaultCultureAccessor; private readonly UrlSegmentProviderCollection _urlSegmentProviders; private readonly IContentNestedDataSerializer _contentNestedDataSerializer; + private readonly ContentDataSerializer _contentDataSerializer; // volatile because we read it with no lock private volatile bool _isReady; @@ -83,7 +84,7 @@ namespace Umbraco.Web.PublishedCache.NuCache IDataSource dataSource, IGlobalSettings globalSettings, IEntityXmlSerializer entitySerializer, IPublishedModelFactory publishedModelFactory, - UrlSegmentProviderCollection urlSegmentProviders, IContentNestedDataSerializer contentNestedDataSerializer) + UrlSegmentProviderCollection urlSegmentProviders, IContentNestedDataSerializer contentNestedDataSerializer, ContentDataSerializer contentDataSerializer = null) : base(publishedSnapshotAccessor, variationContextAccessor) { //if (Interlocked.Increment(ref _singletonCheck) > 1) @@ -101,6 +102,7 @@ namespace Umbraco.Web.PublishedCache.NuCache _globalSettings = globalSettings; _urlSegmentProviders = urlSegmentProviders; _contentNestedDataSerializer = contentNestedDataSerializer; + _contentDataSerializer = contentDataSerializer; // we need an Xml serializer here so that the member cache can support XPath, // for members this is done by navigating the serialized-to-xml member @@ -182,8 +184,8 @@ namespace Umbraco.Web.PublishedCache.NuCache _localMediaDbExists = File.Exists(localMediaDbPath); // if both local databases exist then GetTree will open them, else new databases will be created - _localContentDb = BTree.GetTree(localContentDbPath, _localContentDbExists); - _localMediaDb = BTree.GetTree(localMediaDbPath, _localMediaDbExists); + _localContentDb = BTree.GetTree(localContentDbPath, _localContentDbExists, _contentDataSerializer); + _localMediaDb = BTree.GetTree(localMediaDbPath, _localMediaDbExists, _contentDataSerializer); _logger.Info("Registered with MainDom, localContentDbExists? {LocalContentDbExists}, localMediaDbExists? {LocalMediaDbExists}", _localContentDbExists, _localMediaDbExists); } diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index 088fb0eeb3..6aae3f5ed9 100755 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -68,6 +68,9 @@ 2.7.0.100 + + 1.1.11 + @@ -248,9 +251,15 @@ + + + + + + From d1449a0f5cb38a94ddb836085446826dc3aae8b4 Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Thu, 9 Jul 2020 11:50:15 +1200 Subject: [PATCH 2/5] Fix key mapping --- .../DataSource/Lz4DictionaryOfPropertyDataSerializer.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs index 7ec2fb17b6..b1e913bdec 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs @@ -39,9 +39,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource public Lz4DictionaryOfPropertyDataSerializer(NucachePropertyOptions nucachePropertyOptions) { - _compressProperties = nucachePropertyOptions.PropertyMap.ToList().ToDictionary(x => string.Intern(x.Value.mappedAlias), x => (x.Value.compress,x.Value.decompressionLevel, string.Intern(x.Value.mappedAlias))); + _compressProperties = nucachePropertyOptions.PropertyMap.ToList().ToDictionary(x => string.Intern(x.Key), x => (x.Value.compress,x.Value.decompressionLevel, string.Intern(x.Value.mappedAlias))); _minimumStringLengthForCompression = nucachePropertyOptions.MinimumCompressibleStringLength; - _uncompressProperties = _compressProperties.ToList().ToDictionary(x => x.Value.mappedAlias, x => (x.Value.compress, x.Value.decompressionLevel, x.Value.mappedAlias)); + _uncompressProperties = _compressProperties.ToList().ToDictionary(x => x.Value.mappedAlias, x => (x.Value.compress, x.Value.decompressionLevel, x.Key)); _compressionLevel = nucachePropertyOptions.LZ4CompressionLevel; } From d4276dff58dd9ed69afbf1e3af34da775f80bde2 Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Tue, 4 Aug 2020 18:01:06 +1200 Subject: [PATCH 3/5] Fix composition. Store compression options in map --- .../AppSettingsNucachePropertyMapFactory.cs | 4 ++-- .../INucachePropertyOptionsFactory.cs | 4 ++-- .../DataSource/LazyCompressedString.cs | 10 +++++++- .../Lz4DictionaryOfPropertyDataSerializer.cs | 21 ++++++++-------- .../MsgPackContentNestedDataSerializer.cs | 4 ++-- .../PublishedCache/NuCache/NuCacheComposer.cs | 5 ++-- src/Umbraco.Web/Umbraco.Web.csproj | 2 +- src/umbraco.sln | 24 +++++++++++++++++-- 8 files changed, 51 insertions(+), 23 deletions(-) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/AppSettingsNucachePropertyMapFactory.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/AppSettingsNucachePropertyMapFactory.cs index ddc9bc9b1a..cf44290bd4 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/AppSettingsNucachePropertyMapFactory.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/AppSettingsNucachePropertyMapFactory.cs @@ -7,9 +7,9 @@ using System.Threading.Tasks; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { - public class AppSettingsNucachePropertyMapFactory : INucachePropertyOptionsFactory + public class AppSettingsNuCachePropertyMapFactory : INuCachePropertyOptionsFactory { - public NucachePropertyOptions GetNucachePropertyOptions() + public NucachePropertyOptions GetNuCachePropertyOptions() { NucachePropertyOptions options = new NucachePropertyOptions { diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/INucachePropertyOptionsFactory.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/INucachePropertyOptionsFactory.cs index 82df43a5bb..efc5e599ea 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/INucachePropertyOptionsFactory.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/INucachePropertyOptionsFactory.cs @@ -6,8 +6,8 @@ using System.Threading.Tasks; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { - public interface INucachePropertyOptionsFactory + public interface INuCachePropertyOptionsFactory { - NucachePropertyOptions GetNucachePropertyOptions(); + NucachePropertyOptions GetNuCachePropertyOptions(); } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs index 99bc2f5859..3d6e70c7b2 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs @@ -8,15 +8,23 @@ using System.Threading.Tasks; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { - public class LazyCompressedString + /// + /// Lazily decompresses a LZ4 Pickler compressed UTF8 string + /// + internal class LazyCompressedString { private byte[] _bytes; private string _str; + /// + /// Constructor + /// + /// LZ4 Pickle compressed UTF8 String public LazyCompressedString(byte[] bytes) { _bytes = bytes; } + public override string ToString() { return LazyInitializer.EnsureInitialized(ref _str, () => diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs index b1e913bdec..a5e3034872 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs @@ -32,18 +32,16 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource internal class Lz4DictionaryOfPropertyDataSerializer : SerializerBase, ISerializer>, IDictionaryOfPropertyDataSerializer { private readonly IReadOnlyDictionary _compressProperties; - private readonly LZ4Level _compressionLevel; - private readonly long? _minimumStringLengthForCompression; private readonly IReadOnlyDictionary _uncompressProperties; - public Lz4DictionaryOfPropertyDataSerializer(NucachePropertyOptions nucachePropertyOptions) + public Lz4DictionaryOfPropertyDataSerializer(INuCachePropertyOptionsFactory nucachePropertyOptionsFactory) { + var nucachePropertyOptions = nucachePropertyOptionsFactory.GetNuCachePropertyOptions(); _compressProperties = nucachePropertyOptions.PropertyMap.ToList().ToDictionary(x => string.Intern(x.Key), x => (x.Value.compress,x.Value.decompressionLevel, string.Intern(x.Value.mappedAlias))); - _minimumStringLengthForCompression = nucachePropertyOptions.MinimumCompressibleStringLength; _uncompressProperties = _compressProperties.ToList().ToDictionary(x => x.Value.mappedAlias, x => (x.Value.compress, x.Value.decompressionLevel, x.Key)); - _compressionLevel = nucachePropertyOptions.LZ4CompressionLevel; + _nucachePropertyOptions = nucachePropertyOptions; } @@ -84,12 +82,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource pdata.Segment = ReadStringObject(stream, true) ?? string.Empty; pdata.Value = ReadObject(stream); - var decompressionLevel = NucachePropertyDecompressionLevel.Immediate; if ((map.Compress.Equals(NucachePropertyCompressionLevel.NucacheDatabase) || map.Compress.Equals(NucachePropertyCompressionLevel.SQLDatabase)) && pdata.Value != null && pdata.Value is byte[] byteArrayValue) { //Compressed string - switch (decompressionLevel) + switch (map.decompressionLevel) { case NucachePropertyDecompressionLevel.Lazy: pdata.Value = new LazyCompressedString(byteArrayValue); @@ -136,12 +133,12 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource //Only compress strings if (pdata.Value is string stringValue && pdata.Value != null && map.Compress.Equals(NucachePropertyCompressionLevel.NucacheDatabase) - && (_minimumStringLengthForCompression == null - || !_minimumStringLengthForCompression.HasValue - || stringValue.Length > _minimumStringLengthForCompression.Value)) + && (_nucachePropertyOptions.MinimumCompressibleStringLength == null + || !_nucachePropertyOptions.MinimumCompressibleStringLength.HasValue + || stringValue.Length > _nucachePropertyOptions.MinimumCompressibleStringLength.Value)) { var stringBytes = Encoding.UTF8.GetBytes(stringValue); - var compressedBytes = LZ4Pickler.Pickle(stringBytes, _compressionLevel); + var compressedBytes = LZ4Pickler.Pickle(stringBytes, _nucachePropertyOptions.LZ4CompressionLevel); WriteObject(compressedBytes, stream); } WriteObject(pdata.Value, stream); @@ -149,6 +146,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } } private readonly (NucachePropertyCompressionLevel Compress, NucachePropertyDecompressionLevel decompressionLevel, string MappedAlias) DEFAULT_MAP =(NucachePropertyCompressionLevel.None, NucachePropertyDecompressionLevel.NotCompressed, null); + private readonly NucachePropertyOptions _nucachePropertyOptions; + public (NucachePropertyCompressionLevel Compress, NucachePropertyDecompressionLevel decompressionLevel, string MappedAlias) GetSerializationMap(string propertyAlias) { if (_compressProperties == null) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs index be4e8a9832..c83bc3e973 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs @@ -14,7 +14,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource private MessagePackSerializerOptions _options; private readonly NucachePropertyOptions _propertyOptions; - public MsgPackContentNestedDataSerializer(NucachePropertyOptions propertyOptions = null) + public MsgPackContentNestedDataSerializer(INuCachePropertyOptionsFactory propertyOptionsFactory = null) { var defaultOptions = ContractlessStandardResolver.Options; @@ -34,7 +34,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource _options = defaultOptions .WithResolver(resolver) .WithCompression(MessagePackCompression.Lz4BlockArray); - _propertyOptions = propertyOptions ?? new NucachePropertyOptions(); + _propertyOptions = propertyOptionsFactory?.GetNuCachePropertyOptions() ?? new NucachePropertyOptions(); } public string ToJson(string serialized) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs index 95ff91304d..1ee6b96eb1 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs @@ -14,13 +14,14 @@ namespace Umbraco.Web.PublishedCache.NuCache base.Compose(composition); var serializer = ConfigurationManager.AppSettings["Umbraco.Web.PublishedCache.NuCache.Serializer"]; - + composition.Register(); + composition.Register(); if (serializer == "MsgPack") { var propertyDictionarySerializer = ConfigurationManager.AppSettings["Umbraco.Web.PublishedCache.NuCache.DictionaryOfPropertiesSerializer"]; if (propertyDictionarySerializer == "LZ4Map") { - composition.Register(); + composition.Register(factory => { var lz4Serializer = factory.GetInstance(); diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index 6aae3f5ed9..549492603b 100755 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -251,7 +251,7 @@ - + diff --git a/src/umbraco.sln b/src/umbraco.sln index 63fb856b5d..da9d7ae557 100644 --- a/src/umbraco.sln +++ b/src/umbraco.sln @@ -58,8 +58,24 @@ Project("{E24C65DC-7377-472B-9ABA-BC803B73C61A}") = "Umbraco.Web.UI.Client", "ht StartServerOnDebug = "false" EndProjectSection EndProject -Project("{E24C65DC-7377-472B-9ABA-BC803B73C61A}") = "Umbraco.Tests.AcceptanceTest\", "Umbraco.Tests.AcceptanceTest\", "{9E4C8A12-FBE0-4673-8CE2-DF99D5D57817}" +Project("{E24C65DC-7377-472B-9ABA-BC803B73C61A}") = "Umbraco.Tests.AcceptanceTest", "Umbraco.Tests.AcceptanceTest\", "{9E4C8A12-FBE0-4673-8CE2-DF99D5D57817}" ProjectSection(WebsiteProperties) = preProject + TargetFrameworkMoniker = ".NETFramework,Version%3Dv4.0" + Debug.AspNetCompiler.VirtualPath = "/localhost_49800" + Debug.AspNetCompiler.PhysicalPath = "Umbraco.Tests.AcceptanceTest\" + Debug.AspNetCompiler.TargetPath = "PrecompiledWeb\localhost_49800\" + Debug.AspNetCompiler.Updateable = "true" + Debug.AspNetCompiler.ForceOverwrite = "true" + Debug.AspNetCompiler.FixedNames = "false" + Debug.AspNetCompiler.Debug = "True" + Release.AspNetCompiler.VirtualPath = "/localhost_49800" + Release.AspNetCompiler.PhysicalPath = "Umbraco.Tests.AcceptanceTest\" + Release.AspNetCompiler.TargetPath = "PrecompiledWeb\localhost_49800\" + Release.AspNetCompiler.Updateable = "true" + Release.AspNetCompiler.ForceOverwrite = "true" + Release.AspNetCompiler.FixedNames = "false" + Release.AspNetCompiler.Debug = "False" + VWDPort = "49800" SlnRelativePath = "Umbraco.Tests.AcceptanceTest\" EndProjectSection EndProject @@ -123,6 +139,10 @@ Global {4C4C194C-B5E4-4991-8F87-4373E24CC19F}.Release|Any CPU.Build.0 = Release|Any CPU {3819A550-DCEC-4153-91B4-8BA9F7F0B9B4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {3819A550-DCEC-4153-91B4-8BA9F7F0B9B4}.Release|Any CPU.ActiveCfg = Debug|Any CPU + {9E4C8A12-FBE0-4673-8CE2-DF99D5D57817}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9E4C8A12-FBE0-4673-8CE2-DF99D5D57817}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9E4C8A12-FBE0-4673-8CE2-DF99D5D57817}.Release|Any CPU.ActiveCfg = Debug|Any CPU + {9E4C8A12-FBE0-4673-8CE2-DF99D5D57817}.Release|Any CPU.Build.0 = Debug|Any CPU {651E1350-91B6-44B7-BD60-7207006D7003}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {651E1350-91B6-44B7-BD60-7207006D7003}.Debug|Any CPU.Build.0 = Debug|Any CPU {651E1350-91B6-44B7-BD60-7207006D7003}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -157,6 +177,7 @@ Global EndGlobalSection GlobalSection(NestedProjects) = preSolution {227C3B55-80E5-4E7E-A802-BE16C5128B9D} = {2849E9D4-3B4E-40A3-A309-F3CB4F0E125F} + {9E4C8A12-FBE0-4673-8CE2-DF99D5D57817} = {B5BD12C1-A454-435E-8A46-FF4A364C0382} {5D3B8245-ADA6-453F-A008-50ED04BFE770} = {B5BD12C1-A454-435E-8A46-FF4A364C0382} {E3F9F378-AFE1-40A5-90BD-82833375DBFE} = {227C3B55-80E5-4E7E-A802-BE16C5128B9D} {5B03EF4E-E0AC-4905-861B-8C3EC1A0D458} = {227C3B55-80E5-4E7E-A802-BE16C5128B9D} @@ -164,7 +185,6 @@ Global {3A33ADC9-C6C0-4DB1-A613-A9AF0210DF3D} = {B5BD12C1-A454-435E-8A46-FF4A364C0382} {C7311C00-2184-409B-B506-52A5FAEA8736} = {FD962632-184C-4005-A5F3-E705D92FC645} {FB5676ED-7A69-492C-B802-E7B24144C0FC} = {B5BD12C1-A454-435E-8A46-FF4A364C0382} - {9E4C8A12-FBE0-4673-8CE2-DF99D5D57817} = {B5BD12C1-A454-435E-8A46-FF4A364C0382} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {7A0F2E34-D2AF-4DAB-86A0-7D7764B3D0EC} From 451eacf7cbcee1c175fe676f5e1f9e237d406253 Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Tue, 4 Aug 2020 18:03:08 +1200 Subject: [PATCH 4/5] Fix NuCache spelling --- .../DataSource/AppSettingsNucachePropertyMapFactory.cs | 4 ++-- .../NuCache/DataSource/INucachePropertyOptionsFactory.cs | 2 +- .../DataSource/Lz4DictionaryOfPropertyDataSerializer.cs | 2 +- .../NuCache/DataSource/MsgPackContentNestedDataSerializer.cs | 4 ++-- .../NuCache/DataSource/NucachePropertyOptions.cs | 2 +- src/Umbraco.Web/Umbraco.Web.csproj | 4 ++-- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/AppSettingsNucachePropertyMapFactory.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/AppSettingsNucachePropertyMapFactory.cs index cf44290bd4..6ca4c3e666 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/AppSettingsNucachePropertyMapFactory.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/AppSettingsNucachePropertyMapFactory.cs @@ -9,9 +9,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource { public class AppSettingsNuCachePropertyMapFactory : INuCachePropertyOptionsFactory { - public NucachePropertyOptions GetNuCachePropertyOptions() + public NuCachePropertyOptions GetNuCachePropertyOptions() { - NucachePropertyOptions options = new NucachePropertyOptions + NuCachePropertyOptions options = new NuCachePropertyOptions { PropertyMap = GetPropertyMap(), LZ4CompressionLevel = K4os.Compression.LZ4.LZ4Level.L10_OPT, diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/INucachePropertyOptionsFactory.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/INucachePropertyOptionsFactory.cs index efc5e599ea..0cb694e1c1 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/INucachePropertyOptionsFactory.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/INucachePropertyOptionsFactory.cs @@ -8,6 +8,6 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource { public interface INuCachePropertyOptionsFactory { - NucachePropertyOptions GetNuCachePropertyOptions(); + NuCachePropertyOptions GetNuCachePropertyOptions(); } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs index a5e3034872..501ead8386 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs @@ -146,7 +146,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } } private readonly (NucachePropertyCompressionLevel Compress, NucachePropertyDecompressionLevel decompressionLevel, string MappedAlias) DEFAULT_MAP =(NucachePropertyCompressionLevel.None, NucachePropertyDecompressionLevel.NotCompressed, null); - private readonly NucachePropertyOptions _nucachePropertyOptions; + private readonly NuCachePropertyOptions _nucachePropertyOptions; public (NucachePropertyCompressionLevel Compress, NucachePropertyDecompressionLevel decompressionLevel, string MappedAlias) GetSerializationMap(string propertyAlias) { diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs index c83bc3e973..31421e7177 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs @@ -12,7 +12,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource internal class MsgPackContentNestedDataSerializer : IContentNestedDataByteSerializer { private MessagePackSerializerOptions _options; - private readonly NucachePropertyOptions _propertyOptions; + private readonly NuCachePropertyOptions _propertyOptions; public MsgPackContentNestedDataSerializer(INuCachePropertyOptionsFactory propertyOptionsFactory = null) { @@ -34,7 +34,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource _options = defaultOptions .WithResolver(resolver) .WithCompression(MessagePackCompression.Lz4BlockArray); - _propertyOptions = propertyOptionsFactory?.GetNuCachePropertyOptions() ?? new NucachePropertyOptions(); + _propertyOptions = propertyOptionsFactory?.GetNuCachePropertyOptions() ?? new NuCachePropertyOptions(); } public string ToJson(string serialized) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyOptions.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyOptions.cs index f10787646a..be0118f563 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyOptions.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyOptions.cs @@ -6,7 +6,7 @@ using System.Threading.Tasks; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { - public class NucachePropertyOptions + public class NuCachePropertyOptions { public IReadOnlyDictionary PropertyMap diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index 549492603b..a4e5640d36 100755 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -254,12 +254,12 @@ - + - + From e65f3a7e80a91e5296e7f5e8e1ad5d70a8545ef0 Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Tue, 4 Aug 2020 18:21:16 +1200 Subject: [PATCH 5/5] undo solution changes --- src/umbraco.sln | 24 ++---------------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/src/umbraco.sln b/src/umbraco.sln index da9d7ae557..63fb856b5d 100644 --- a/src/umbraco.sln +++ b/src/umbraco.sln @@ -58,24 +58,8 @@ Project("{E24C65DC-7377-472B-9ABA-BC803B73C61A}") = "Umbraco.Web.UI.Client", "ht StartServerOnDebug = "false" EndProjectSection EndProject -Project("{E24C65DC-7377-472B-9ABA-BC803B73C61A}") = "Umbraco.Tests.AcceptanceTest", "Umbraco.Tests.AcceptanceTest\", "{9E4C8A12-FBE0-4673-8CE2-DF99D5D57817}" +Project("{E24C65DC-7377-472B-9ABA-BC803B73C61A}") = "Umbraco.Tests.AcceptanceTest\", "Umbraco.Tests.AcceptanceTest\", "{9E4C8A12-FBE0-4673-8CE2-DF99D5D57817}" ProjectSection(WebsiteProperties) = preProject - TargetFrameworkMoniker = ".NETFramework,Version%3Dv4.0" - Debug.AspNetCompiler.VirtualPath = "/localhost_49800" - Debug.AspNetCompiler.PhysicalPath = "Umbraco.Tests.AcceptanceTest\" - Debug.AspNetCompiler.TargetPath = "PrecompiledWeb\localhost_49800\" - Debug.AspNetCompiler.Updateable = "true" - Debug.AspNetCompiler.ForceOverwrite = "true" - Debug.AspNetCompiler.FixedNames = "false" - Debug.AspNetCompiler.Debug = "True" - Release.AspNetCompiler.VirtualPath = "/localhost_49800" - Release.AspNetCompiler.PhysicalPath = "Umbraco.Tests.AcceptanceTest\" - Release.AspNetCompiler.TargetPath = "PrecompiledWeb\localhost_49800\" - Release.AspNetCompiler.Updateable = "true" - Release.AspNetCompiler.ForceOverwrite = "true" - Release.AspNetCompiler.FixedNames = "false" - Release.AspNetCompiler.Debug = "False" - VWDPort = "49800" SlnRelativePath = "Umbraco.Tests.AcceptanceTest\" EndProjectSection EndProject @@ -139,10 +123,6 @@ Global {4C4C194C-B5E4-4991-8F87-4373E24CC19F}.Release|Any CPU.Build.0 = Release|Any CPU {3819A550-DCEC-4153-91B4-8BA9F7F0B9B4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {3819A550-DCEC-4153-91B4-8BA9F7F0B9B4}.Release|Any CPU.ActiveCfg = Debug|Any CPU - {9E4C8A12-FBE0-4673-8CE2-DF99D5D57817}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {9E4C8A12-FBE0-4673-8CE2-DF99D5D57817}.Debug|Any CPU.Build.0 = Debug|Any CPU - {9E4C8A12-FBE0-4673-8CE2-DF99D5D57817}.Release|Any CPU.ActiveCfg = Debug|Any CPU - {9E4C8A12-FBE0-4673-8CE2-DF99D5D57817}.Release|Any CPU.Build.0 = Debug|Any CPU {651E1350-91B6-44B7-BD60-7207006D7003}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {651E1350-91B6-44B7-BD60-7207006D7003}.Debug|Any CPU.Build.0 = Debug|Any CPU {651E1350-91B6-44B7-BD60-7207006D7003}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -177,7 +157,6 @@ Global EndGlobalSection GlobalSection(NestedProjects) = preSolution {227C3B55-80E5-4E7E-A802-BE16C5128B9D} = {2849E9D4-3B4E-40A3-A309-F3CB4F0E125F} - {9E4C8A12-FBE0-4673-8CE2-DF99D5D57817} = {B5BD12C1-A454-435E-8A46-FF4A364C0382} {5D3B8245-ADA6-453F-A008-50ED04BFE770} = {B5BD12C1-A454-435E-8A46-FF4A364C0382} {E3F9F378-AFE1-40A5-90BD-82833375DBFE} = {227C3B55-80E5-4E7E-A802-BE16C5128B9D} {5B03EF4E-E0AC-4905-861B-8C3EC1A0D458} = {227C3B55-80E5-4E7E-A802-BE16C5128B9D} @@ -185,6 +164,7 @@ Global {3A33ADC9-C6C0-4DB1-A613-A9AF0210DF3D} = {B5BD12C1-A454-435E-8A46-FF4A364C0382} {C7311C00-2184-409B-B506-52A5FAEA8736} = {FD962632-184C-4005-A5F3-E705D92FC645} {FB5676ED-7A69-492C-B802-E7B24144C0FC} = {B5BD12C1-A454-435E-8A46-FF4A364C0382} + {9E4C8A12-FBE0-4673-8CE2-DF99D5D57817} = {B5BD12C1-A454-435E-8A46-FF4A364C0382} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {7A0F2E34-D2AF-4DAB-86A0-7D7764B3D0EC}