diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/AppSettingsNucachePropertyMapFactory.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/AppSettingsNucachePropertyMapFactory.cs new file mode 100644 index 0000000000..6ca4c3e666 --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/AppSettingsNucachePropertyMapFactory.cs @@ -0,0 +1,47 @@ +using System; +using System.Collections.Generic; +using System.Configuration; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + public class AppSettingsNuCachePropertyMapFactory : INuCachePropertyOptionsFactory + { + public NuCachePropertyOptions GetNuCachePropertyOptions() + { + NuCachePropertyOptions options = new NuCachePropertyOptions + { + PropertyMap = GetPropertyMap(), + LZ4CompressionLevel = K4os.Compression.LZ4.LZ4Level.L10_OPT, + MinimumCompressibleStringLength = null + }; + return options; + } + + public IReadOnlyDictionary GetPropertyMap() + { + var propertyMap = new Dictionary(); + // TODO: Use xml/json/c# to define map + var propertyDictionarySerializerMap = ConfigurationManager.AppSettings["Umbraco.Web.PublishedCache.NuCache.PropertySerializationMap"]; + if (!string.IsNullOrWhiteSpace(propertyDictionarySerializerMap)) + { + //propertyAlias,CompressionLevel,DecompressionLevel,mappedAlias; + propertyDictionarySerializerMap.Split(';') + .Select(x => + { + var y = x.Split(','); + (string alias, NucachePropertyCompressionLevel compressionLevel, NucachePropertyDecompressionLevel decompressionLevel, string mappedAlias) v = (y[0], + (NucachePropertyCompressionLevel)System.Enum.Parse(typeof(NucachePropertyCompressionLevel), y[1]), + (NucachePropertyDecompressionLevel)System.Enum.Parse(typeof(NucachePropertyDecompressionLevel), y[2]), + y[3] + ); + return v; + }) + .ToList().ForEach(x => propertyMap.Add(x.alias, (x.compressionLevel, x.decompressionLevel, x.mappedAlias))); + } + return propertyMap; + } + } +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.ContentDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.ContentDataSerializer.cs index d02af375c6..9cc5d3a701 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.ContentDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.ContentDataSerializer.cs @@ -5,8 +5,17 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource { class ContentDataSerializer : ISerializer { + public ContentDataSerializer(IDictionaryOfPropertyDataSerializer dictionaryOfPropertyDataSerializer = null) + { + _dictionaryOfPropertyDataSerializer = dictionaryOfPropertyDataSerializer; + if(_dictionaryOfPropertyDataSerializer == null) + { + _dictionaryOfPropertyDataSerializer = PropertiesSerializer; + } + } private static readonly DictionaryOfPropertyDataSerializer PropertiesSerializer = new DictionaryOfPropertyDataSerializer(); private static readonly DictionaryOfCultureVariationSerializer CultureVariationsSerializer = new DictionaryOfCultureVariationSerializer(); + private readonly IDictionaryOfPropertyDataSerializer _dictionaryOfPropertyDataSerializer; public ContentData ReadFrom(Stream stream) { @@ -19,7 +28,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource VersionDate = PrimitiveSerializer.DateTime.ReadFrom(stream), WriterId = PrimitiveSerializer.Int32.ReadFrom(stream), TemplateId = PrimitiveSerializer.Int32.ReadFrom(stream), - Properties = PropertiesSerializer.ReadFrom(stream), // TODO: We don't want to allocate empty arrays + Properties = _dictionaryOfPropertyDataSerializer.ReadFrom(stream), // TODO: We don't want to allocate empty arrays CultureInfos = CultureVariationsSerializer.ReadFrom(stream) // TODO: We don't want to allocate empty arrays }; } @@ -36,7 +45,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource { PrimitiveSerializer.Int32.WriteTo(value.TemplateId.Value, stream); } - PropertiesSerializer.WriteTo(value.Properties, stream); + _dictionaryOfPropertyDataSerializer.WriteTo(value.Properties, stream); CultureVariationsSerializer.WriteTo(value.CultureInfos, stream); } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.ContentNodeKitSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.ContentNodeKitSerializer.cs index f799869850..1d668ba4fd 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.ContentNodeKitSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.ContentNodeKitSerializer.cs @@ -5,7 +5,17 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource { internal class ContentNodeKitSerializer : ISerializer { - static readonly ContentDataSerializer DataSerializer = new ContentDataSerializer(); + public ContentNodeKitSerializer(ContentDataSerializer contentDataSerializer = null) + { + _contentDataSerializer = contentDataSerializer; + if(_contentDataSerializer == null) + { + _contentDataSerializer = DefaultDataSerializer; + } + } + static readonly ContentDataSerializer DefaultDataSerializer = new ContentDataSerializer(); + private readonly ContentDataSerializer _contentDataSerializer; + //static readonly ListOfIntSerializer ChildContentIdsSerializer = new ListOfIntSerializer(); public ContentNodeKit ReadFrom(Stream stream) @@ -26,10 +36,10 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource }; var hasDraft = PrimitiveSerializer.Boolean.ReadFrom(stream); if (hasDraft) - kit.DraftData = DataSerializer.ReadFrom(stream); + kit.DraftData = _contentDataSerializer.ReadFrom(stream); var hasPublished = PrimitiveSerializer.Boolean.ReadFrom(stream); if (hasPublished) - kit.PublishedData = DataSerializer.ReadFrom(stream); + kit.PublishedData = _contentDataSerializer.ReadFrom(stream); return kit; } @@ -47,11 +57,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource PrimitiveSerializer.Boolean.WriteTo(value.DraftData != null, stream); if (value.DraftData != null) - DataSerializer.WriteTo(value.DraftData, stream); + _contentDataSerializer.WriteTo(value.DraftData, stream); PrimitiveSerializer.Boolean.WriteTo(value.PublishedData != null, stream); if (value.PublishedData != null) - DataSerializer.WriteTo(value.PublishedData, stream); + _contentDataSerializer.WriteTo(value.PublishedData, stream); } } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs index e3fa6597e4..cb12164397 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs @@ -6,7 +6,7 @@ using Umbraco.Core; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { - internal class DictionaryOfPropertyDataSerializer : SerializerBase, ISerializer> + internal class DictionaryOfPropertyDataSerializer : SerializerBase, ISerializer>, IDictionaryOfPropertyDataSerializer { public IDictionary ReadFrom(Stream stream) { diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.cs index 910c0ca737..80d8488495 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.cs @@ -6,10 +6,10 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource { internal class BTree { - public static BPlusTree GetTree(string filepath, bool exists) + public static BPlusTree GetTree(string filepath, bool exists, ContentDataSerializer contentDataSerializer = null) { var keySerializer = new PrimitiveSerializer(); - var valueSerializer = new ContentNodeKitSerializer(); + var valueSerializer = new ContentNodeKitSerializer(contentDataSerializer); var options = new BPlusTree.OptionsV2(keySerializer, valueSerializer) { CreateFile = exists ? CreatePolicy.IfNeeded : CreatePolicy.Always, diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IDictionaryOfPropertyDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IDictionaryOfPropertyDataSerializer.cs new file mode 100644 index 0000000000..a086e3e2f3 --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IDictionaryOfPropertyDataSerializer.cs @@ -0,0 +1,11 @@ +using System.Collections.Generic; +using System.IO; + +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + internal interface IDictionaryOfPropertyDataSerializer + { + IDictionary ReadFrom(Stream stream); + void WriteTo(IDictionary value, Stream stream); + } +} \ No newline at end of file diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/INucachePropertyOptionsFactory.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/INucachePropertyOptionsFactory.cs new file mode 100644 index 0000000000..0cb694e1c1 --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/INucachePropertyOptionsFactory.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + public interface INuCachePropertyOptionsFactory + { + NuCachePropertyOptions GetNuCachePropertyOptions(); + } +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs new file mode 100644 index 0000000000..3d6e70c7b2 --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs @@ -0,0 +1,39 @@ +using K4os.Compression.LZ4; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + /// + /// Lazily decompresses a LZ4 Pickler compressed UTF8 string + /// + internal class LazyCompressedString + { + private byte[] _bytes; + private string _str; + + /// + /// Constructor + /// + /// LZ4 Pickle compressed UTF8 String + public LazyCompressedString(byte[] bytes) + { + _bytes = bytes; + } + + public override string ToString() + { + return LazyInitializer.EnsureInitialized(ref _str, () => + { + var str = Encoding.UTF8.GetString(LZ4Pickler.Unpickle(_bytes)); + _bytes = null; + return str; + }); + } + } + +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs new file mode 100644 index 0000000000..501ead8386 --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs @@ -0,0 +1,177 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using CSharpTest.Net.Serialization; +using Umbraco.Core; +using System.Text; +using K4os.Compression.LZ4; + +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + /// + /// If/where to compress custom properties for nucache + /// + public enum NucachePropertyCompressionLevel + { + None = 0, + SQLDatabase = 1, + NucacheDatabase = 2 + } + /// + /// If/where to decompress custom properties for nucache + /// + public enum NucachePropertyDecompressionLevel + { + NotCompressed = 0, + Immediate = 1, + Lazy = 2 + } + + + internal class Lz4DictionaryOfPropertyDataSerializer : SerializerBase, ISerializer>, IDictionaryOfPropertyDataSerializer + { + private readonly IReadOnlyDictionary _compressProperties; + private readonly IReadOnlyDictionary _uncompressProperties; + + + public Lz4DictionaryOfPropertyDataSerializer(INuCachePropertyOptionsFactory nucachePropertyOptionsFactory) + { + var nucachePropertyOptions = nucachePropertyOptionsFactory.GetNuCachePropertyOptions(); + _compressProperties = nucachePropertyOptions.PropertyMap.ToList().ToDictionary(x => string.Intern(x.Key), x => (x.Value.compress,x.Value.decompressionLevel, string.Intern(x.Value.mappedAlias))); + _uncompressProperties = _compressProperties.ToList().ToDictionary(x => x.Value.mappedAlias, x => (x.Value.compress, x.Value.decompressionLevel, x.Key)); + + _nucachePropertyOptions = nucachePropertyOptions; + } + + + public IDictionary ReadFrom(Stream stream) + { + var dict = new Dictionary(StringComparer.InvariantCultureIgnoreCase); + + // read properties count + var pcount = PrimitiveSerializer.Int32.ReadFrom(stream); + + // read each property + for (var i = 0; i < pcount; i++) + { + + // read property alias + var alias = PrimitiveSerializer.String.ReadFrom(stream); + var map = GetDeSerializationMap(alias); + var key = string.Intern(map.MappedAlias ?? alias); + + // read values count + var vcount = PrimitiveSerializer.Int32.ReadFrom(stream); + + // create pdata and add to the dictionary + var pdatas = new List(); + + // for each value, read and add to pdata + for (var j = 0; j < vcount; j++) + { + var pdata = new PropertyData(); + pdatas.Add(pdata); + + // everything that can be null is read/written as object + // even though - culture and segment should never be null here, as 'null' represents + // the 'current' value, and string.Empty should be used to represent the invariant or + // neutral values - PropertyData throws when getting nulls, so falling back to + // string.Empty here - what else? + pdata.Culture = ReadStringObject(stream, true) ?? string.Empty; + pdata.Segment = ReadStringObject(stream, true) ?? string.Empty; + pdata.Value = ReadObject(stream); + + if ((map.Compress.Equals(NucachePropertyCompressionLevel.NucacheDatabase) || map.Compress.Equals(NucachePropertyCompressionLevel.SQLDatabase)) + && pdata.Value != null && pdata.Value is byte[] byteArrayValue) + { + //Compressed string + switch (map.decompressionLevel) + { + case NucachePropertyDecompressionLevel.Lazy: + pdata.Value = new LazyCompressedString(byteArrayValue); + break; + case NucachePropertyDecompressionLevel.NotCompressed: + break;//Shouldn't be any not compressed + case NucachePropertyDecompressionLevel.Immediate: + default: + pdata.Value = Encoding.UTF8.GetString(LZ4Pickler.Unpickle(byteArrayValue)); + break; + } + } + } + + dict[key] = pdatas.ToArray(); + } + return dict; + } + + public void WriteTo(IDictionary value, Stream stream) + { + // write properties count + PrimitiveSerializer.Int32.WriteTo(value.Count, stream); + + // write each property + foreach (var (alias, values) in value) + { + var map = GetSerializationMap(alias); + + // write alias + PrimitiveSerializer.String.WriteTo(map.MappedAlias ?? alias, stream); + + // write values count + PrimitiveSerializer.Int32.WriteTo(values.Length, stream); + + // write each value + foreach (var pdata in values) + { + // everything that can be null is read/written as object + // even though - culture and segment should never be null here, + // see note in ReadFrom() method above + WriteObject(pdata.Culture ?? string.Empty, stream); + WriteObject(pdata.Segment ?? string.Empty, stream); + + //Only compress strings + if (pdata.Value is string stringValue && pdata.Value != null && map.Compress.Equals(NucachePropertyCompressionLevel.NucacheDatabase) + && (_nucachePropertyOptions.MinimumCompressibleStringLength == null + || !_nucachePropertyOptions.MinimumCompressibleStringLength.HasValue + || stringValue.Length > _nucachePropertyOptions.MinimumCompressibleStringLength.Value)) + { + var stringBytes = Encoding.UTF8.GetBytes(stringValue); + var compressedBytes = LZ4Pickler.Pickle(stringBytes, _nucachePropertyOptions.LZ4CompressionLevel); + WriteObject(compressedBytes, stream); + } + WriteObject(pdata.Value, stream); + } + } + } + private readonly (NucachePropertyCompressionLevel Compress, NucachePropertyDecompressionLevel decompressionLevel, string MappedAlias) DEFAULT_MAP =(NucachePropertyCompressionLevel.None, NucachePropertyDecompressionLevel.NotCompressed, null); + private readonly NuCachePropertyOptions _nucachePropertyOptions; + + public (NucachePropertyCompressionLevel Compress, NucachePropertyDecompressionLevel decompressionLevel, string MappedAlias) GetSerializationMap(string propertyAlias) + { + if (_compressProperties == null) + { + return DEFAULT_MAP; + } + if (_compressProperties.TryGetValue(propertyAlias, out (NucachePropertyCompressionLevel compress, NucachePropertyDecompressionLevel decompressionLevel, string mappedAlias) map)) + { + return map; + } + + return DEFAULT_MAP; + } + public (NucachePropertyCompressionLevel Compress, NucachePropertyDecompressionLevel decompressionLevel, string MappedAlias) GetDeSerializationMap(string propertyAlias) + { + if (_uncompressProperties == null) + { + return DEFAULT_MAP; + } + if (_uncompressProperties.TryGetValue(propertyAlias, out (NucachePropertyCompressionLevel compress, NucachePropertyDecompressionLevel decompressionLevel, string mappedAlias) map)) + { + return map; + } + return DEFAULT_MAP; + } + } +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs index 1a093292b5..31421e7177 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs @@ -1,16 +1,20 @@ -using MessagePack; +using K4os.Compression.LZ4; +using MessagePack; using MessagePack.Formatters; using MessagePack.Resolvers; using System; using System.Collections.Generic; +using System.Linq; +using System.Text; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { internal class MsgPackContentNestedDataSerializer : IContentNestedDataByteSerializer { private MessagePackSerializerOptions _options; + private readonly NuCachePropertyOptions _propertyOptions; - public MsgPackContentNestedDataSerializer() + public MsgPackContentNestedDataSerializer(INuCachePropertyOptionsFactory propertyOptionsFactory = null) { var defaultOptions = ContractlessStandardResolver.Options; @@ -30,6 +34,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource _options = defaultOptions .WithResolver(resolver) .WithCompression(MessagePackCompression.Lz4BlockArray); + _propertyOptions = propertyOptionsFactory?.GetNuCachePropertyOptions() ?? new NuCachePropertyOptions(); } public string ToJson(string serialized) @@ -48,10 +53,42 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource public string Serialize(ContentNestedData nestedData) { + Optimize(nestedData); + var bin = MessagePackSerializer.Serialize(nestedData, _options); return Convert.ToBase64String(bin); } + /// + /// Compress properties and map property names to shorter names + /// + /// + private void Optimize(ContentNestedData nestedData) + { + if (_propertyOptions.PropertyMap != null && _propertyOptions.PropertyMap.Any()) + { + foreach (var map in _propertyOptions.PropertyMap) + { + if (map.Value.compress.Equals(NucachePropertyCompressionLevel.SQLDatabase)) + { + if (nestedData.PropertyData.TryGetValue(map.Key, out PropertyData[] properties)) + { + foreach (var property in properties.Where(x => x.Value != null && x.Value is string)) + { + property.Value = LZ4Pickler.Pickle(Encoding.UTF8.GetBytes(property.Value as string), _propertyOptions.LZ4CompressionLevel); + } + } + } + if (map.Value.mappedAlias != null && !map.Key.Equals(map.Value.mappedAlias) + && nestedData.PropertyData.Remove(map.Key) && nestedData.PropertyData.TryGetValue(map.Key, out PropertyData[] properties2)) + { + nestedData.PropertyData.Remove(map.Key); + nestedData.PropertyData.Add(map.Value.mappedAlias, properties2); + } + } + } + } + public ContentNestedData DeserializeBytes(byte[] data) => MessagePackSerializer.Deserialize(data, _options); public byte[] SerializeBytes(ContentNestedData nestedData) => MessagePackSerializer.Serialize(nestedData, _options); diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyOptions.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyOptions.cs new file mode 100644 index 0000000000..be0118f563 --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyOptions.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + public class NuCachePropertyOptions + { + public IReadOnlyDictionary PropertyMap + { get; set; } = new Dictionary(); + + public K4os.Compression.LZ4.LZ4Level LZ4CompressionLevel { get; set; } = K4os.Compression.LZ4.LZ4Level.L00_FAST; + + public long? MinimumCompressibleStringLength { get; set; } + } +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs index afc0827ed3..885b5cf80e 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs @@ -16,6 +16,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource private const char PrefixDouble = 'B'; private const char PrefixDateTime = 'D'; private const char PrefixByte = 'O'; + private const char PrefixByteArray = 'A'; protected string ReadString(Stream stream) => PrimitiveSerializer.String.ReadFrom(stream); protected int ReadInt(Stream stream) => PrimitiveSerializer.Int32.ReadFrom(stream); @@ -23,6 +24,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource protected float ReadFloat(Stream stream) => PrimitiveSerializer.Float.ReadFrom(stream); protected double ReadDouble(Stream stream) => PrimitiveSerializer.Double.ReadFrom(stream); protected DateTime ReadDateTime(Stream stream) => PrimitiveSerializer.DateTime.ReadFrom(stream); + protected byte[] ReadByteArray(Stream stream) => PrimitiveSerializer.Bytes.ReadFrom(stream); private T? ReadObject(Stream stream, char t, Func read) where T : struct @@ -39,7 +41,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource var type = PrimitiveSerializer.Char.ReadFrom(stream); if (type == PrefixNull) return null; if (type != PrefixString) - throw new NotSupportedException($"Cannot deserialize type '{type}', expected 'S'."); + throw new NotSupportedException($"Cannot deserialize type '{type}', expected '{PrefixString}'."); return intern ? string.Intern(PrimitiveSerializer.String.ReadFrom(stream)) : PrimitiveSerializer.String.ReadFrom(stream); @@ -51,6 +53,16 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource protected double? ReadDoubleObject(Stream stream) => ReadObject(stream, PrefixDouble, ReadDouble); protected DateTime? ReadDateTimeObject(Stream stream) => ReadObject(stream, PrefixDateTime, ReadDateTime); + protected byte[] ReadByteArrayObject(Stream stream) // required 'cos byte[] is not a struct + { + var type = PrimitiveSerializer.Char.ReadFrom(stream); + if (type == PrefixNull) return null; + if (type != PrefixByteArray) + throw new NotSupportedException($"Cannot deserialize type '{type}', expected '{PrefixByteArray}'."); + return PrimitiveSerializer.Bytes.ReadFrom(stream); + } + + protected object ReadObject(Stream stream) => ReadObject(PrimitiveSerializer.Char.ReadFrom(stream), stream); @@ -81,6 +93,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource return PrimitiveSerializer.Double.ReadFrom(stream); case PrefixDateTime: return PrimitiveSerializer.DateTime.ReadFrom(stream); + case PrefixByteArray: + return PrimitiveSerializer.Bytes.ReadFrom(stream); default: throw new NotSupportedException($"Cannot deserialize unknown type '{type}'."); } @@ -137,6 +151,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource PrimitiveSerializer.Char.WriteTo(PrefixUInt32, stream); PrimitiveSerializer.UInt32.WriteTo(uInt32Value, stream); } + else if (value is byte[] byteArrayValue) + { + PrimitiveSerializer.Char.WriteTo(PrefixByteArray, stream); + PrimitiveSerializer.Bytes.WriteTo(byteArrayValue, stream); + } else throw new NotSupportedException("Value type " + value.GetType().FullName + " cannot be serialized."); } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs index 205ac55cdc..1ee6b96eb1 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs @@ -1,4 +1,6 @@ -using System.Configuration; +using System.Collections.Generic; +using System.Configuration; +using System.Linq; using Umbraco.Core; using Umbraco.Core.Composing; using Umbraco.Web.PublishedCache.NuCache.DataSource; @@ -12,14 +14,30 @@ namespace Umbraco.Web.PublishedCache.NuCache base.Compose(composition); var serializer = ConfigurationManager.AppSettings["Umbraco.Web.PublishedCache.NuCache.Serializer"]; - + composition.Register(); + composition.Register(); if (serializer == "MsgPack") { - composition.Register(); + var propertyDictionarySerializer = ConfigurationManager.AppSettings["Umbraco.Web.PublishedCache.NuCache.DictionaryOfPropertiesSerializer"]; + if (propertyDictionarySerializer == "LZ4Map") + { + + composition.Register(factory => + { + var lz4Serializer = factory.GetInstance(); + return new ContentDataSerializer(lz4Serializer); + }); + } + else + { + composition.Register(factory => new ContentDataSerializer(new DictionaryOfPropertyDataSerializer())); + } + composition.Register(); } else { composition.Register(); + composition.Register(factory => new ContentDataSerializer(new DictionaryOfPropertyDataSerializer())); } // register the NuCache database data source @@ -34,5 +52,6 @@ namespace Umbraco.Web.PublishedCache.NuCache // TODO: no NuCache health check yet //composition.HealthChecks().Add(); } + } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs index c87f035589..0135f204c7 100755 --- a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs @@ -50,6 +50,7 @@ namespace Umbraco.Web.PublishedCache.NuCache private readonly IDefaultCultureAccessor _defaultCultureAccessor; private readonly UrlSegmentProviderCollection _urlSegmentProviders; private readonly IContentNestedDataSerializer _contentNestedDataSerializer; + private readonly ContentDataSerializer _contentDataSerializer; // volatile because we read it with no lock private volatile bool _isReady; @@ -83,7 +84,7 @@ namespace Umbraco.Web.PublishedCache.NuCache IDataSource dataSource, IGlobalSettings globalSettings, IEntityXmlSerializer entitySerializer, IPublishedModelFactory publishedModelFactory, - UrlSegmentProviderCollection urlSegmentProviders, IContentNestedDataSerializer contentNestedDataSerializer) + UrlSegmentProviderCollection urlSegmentProviders, IContentNestedDataSerializer contentNestedDataSerializer, ContentDataSerializer contentDataSerializer = null) : base(publishedSnapshotAccessor, variationContextAccessor) { //if (Interlocked.Increment(ref _singletonCheck) > 1) @@ -101,6 +102,7 @@ namespace Umbraco.Web.PublishedCache.NuCache _globalSettings = globalSettings; _urlSegmentProviders = urlSegmentProviders; _contentNestedDataSerializer = contentNestedDataSerializer; + _contentDataSerializer = contentDataSerializer; // we need an Xml serializer here so that the member cache can support XPath, // for members this is done by navigating the serialized-to-xml member @@ -182,8 +184,8 @@ namespace Umbraco.Web.PublishedCache.NuCache _localMediaDbExists = File.Exists(localMediaDbPath); // if both local databases exist then GetTree will open them, else new databases will be created - _localContentDb = BTree.GetTree(localContentDbPath, _localContentDbExists); - _localMediaDb = BTree.GetTree(localMediaDbPath, _localMediaDbExists); + _localContentDb = BTree.GetTree(localContentDbPath, _localContentDbExists, _contentDataSerializer); + _localMediaDb = BTree.GetTree(localMediaDbPath, _localMediaDbExists, _contentDataSerializer); _logger.Info("Registered with MainDom, localContentDbExists? {LocalContentDbExists}, localMediaDbExists? {LocalMediaDbExists}", _localContentDbExists, _localMediaDbExists); } diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index 088fb0eeb3..a4e5640d36 100755 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -68,6 +68,9 @@ 2.7.0.100 + + 1.1.11 + @@ -248,9 +251,15 @@ + + + + + +