From ba8da3850ede43d0c2706fa556f9d52aa403ff83 Mon Sep 17 00:00:00 2001 From: Shannon Date: Wed, 26 Aug 2020 11:43:43 +1000 Subject: [PATCH] This gets things working with all compression levels - going to cleanup/simplify --- .../DataSource/LazyCompressedString.cs | 14 ++-- .../Lz4DictionaryOfPropertyDataSerializer.cs | 6 +- .../MsgPackContentNestedDataSerializer.cs | 76 ++++++++++++++++--- .../NuCachePropertyCompressionOptions.cs | 1 + .../NucachePropertyDecompressionLevel.cs | 3 + .../NuCache/DataSource/SerializerBase.cs | 9 ++- 6 files changed, 88 insertions(+), 21 deletions(-) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs index 55469ad791..9df40daf76 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs @@ -1,10 +1,7 @@ using K4os.Compression.LZ4; using System; -using System.Collections.Generic; -using System.Linq; using System.Text; using System.Threading; -using System.Threading.Tasks; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { @@ -13,10 +10,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource /// internal class LazyCompressedString { - // TODO: This could be a struct - - private byte[] _bytes; private string _str; + private byte[] _bytes; /// /// Constructor @@ -27,6 +22,13 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource _bytes = bytes; } + public byte[] GetBytes() + { + if (_bytes == null) + throw new InvalidOperationException("The bytes have already been expanded"); + return _bytes; + } + public override string ToString() { return LazyInitializer.EnsureInitialized(ref _str, () => diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs index b3d7f7dcbc..1a2d26f4b1 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs @@ -88,8 +88,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource break; } } - break; - } + break; + } } dict[key] = pdatas.ToArray(); @@ -122,7 +122,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource WriteObject(pdata.Culture ?? string.Empty, stream); WriteObject(pdata.Segment ?? string.Empty, stream); - //Only compress strings + //Only compress strings (if NucachePropertyCompressionLevel.SQLDatabase has been specified then the property value will already be compressed) switch (map.CompressLevel) { // If we're compressing into btree at the property level diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs index eb17bed858..0475810422 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs @@ -50,23 +50,36 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource public ContentNestedData Deserialize(string data) { var bin = Convert.FromBase64String(data); - var obj = MessagePackSerializer.Deserialize(bin, _options); - return obj; + var nestedData = MessagePackSerializer.Deserialize(bin, _options); + Expand(nestedData); + return nestedData; } public string Serialize(ContentNestedData nestedData) { - Optimize(nestedData); - + Compress(nestedData); var bin = MessagePackSerializer.Serialize(nestedData, _options); return Convert.ToBase64String(bin); } + public ContentNestedData DeserializeBytes(byte[] data) + { + var nestedData = MessagePackSerializer.Deserialize(data, _options); + Expand(nestedData); + return nestedData; + } + + public byte[] SerializeBytes(ContentNestedData nestedData) + { + Compress(nestedData); + return MessagePackSerializer.Serialize(nestedData, _options); + } + /// - /// Compress properties and map property names to shorter names + /// Used during serialization to compress properties and map property names to shorter names /// /// - private void Optimize(ContentNestedData nestedData) + private void Compress(ContentNestedData nestedData) { if (_propertyOptions.PropertyMap != null && _propertyOptions.PropertyMap.Count > 0) { @@ -95,13 +108,54 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } } - public ContentNestedData DeserializeBytes(byte[] data) => MessagePackSerializer.Deserialize(data, _options); - - public byte[] SerializeBytes(ContentNestedData nestedData) + /// + /// Used during deserialization to map the property data as lazy or expand the value and re-map back to the true property aliases + /// + /// + private void Expand(ContentNestedData nestedData) { - Optimize(nestedData); + if (_propertyOptions.PropertyMap != null && _propertyOptions.PropertyMap.Count > 0) + { + foreach (var map in _propertyOptions.PropertyMap) + { + if (map.Value.CompressLevel.Equals(NucachePropertyCompressionLevel.SQLDatabase)) + { + // if there is an alias map for this property then re-map to the real property alias + if (map.Value.MappedAlias != null && !map.Key.Equals(map.Value.MappedAlias) + && nestedData.PropertyData.TryGetValue(map.Value.MappedAlias, out PropertyData[] properties2)) + { + nestedData.PropertyData.Remove(map.Value.MappedAlias); + nestedData.PropertyData.Add(map.Key, properties2); + } - return MessagePackSerializer.Serialize(nestedData, _options); + if (nestedData.PropertyData.TryGetValue(map.Key, out PropertyData[] properties)) + { + foreach (var pdata in properties) + { + if (!(pdata.Value is null) && pdata.Value is byte[] byteArrayValue) + { + //Compressed string + switch (map.Value.DecompressLevel) + { + case NucachePropertyDecompressionLevel.Lazy: + pdata.Value = new LazyCompressedString(byteArrayValue); + break; + case NucachePropertyDecompressionLevel.NotCompressed: + //Shouldn't be any not compressed + throw new InvalidOperationException($"{NucachePropertyDecompressionLevel.NotCompressed} cannot be a decompression option for property {map.Key} since it's compresion option is {map.Value.CompressLevel}"); + case NucachePropertyDecompressionLevel.Immediate: + default: + pdata.Value = Encoding.UTF8.GetString(LZ4Pickler.Unpickle(byteArrayValue)); + break; + } + } + } + } + } + + + } + } } //private class ContentNestedDataResolver : IFormatterResolver diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NuCachePropertyCompressionOptions.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NuCachePropertyCompressionOptions.cs index 50fb20cadc..55ab813783 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NuCachePropertyCompressionOptions.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NuCachePropertyCompressionOptions.cs @@ -27,6 +27,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource public LZ4Level LZ4CompressionLevel { get; } = LZ4Level.L00_FAST; + // TODO: Unsure if we really want to keep this public long? MinimumCompressibleStringLength { get; } } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyDecompressionLevel.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyDecompressionLevel.cs index 335ae6ff4a..f4d485be71 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyDecompressionLevel.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyDecompressionLevel.cs @@ -6,7 +6,10 @@ public enum NucachePropertyDecompressionLevel { NotCompressed = 0, + + // TODO: I'm unsure if this will ever be necessary, lazy seems good and deserialization would only occur once Immediate = 1, + Lazy = 2 } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs index 4a11a8f0e5..b90c418750 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs @@ -98,7 +98,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource case PrefixDateTime: return PrimitiveSerializer.DateTime.ReadFrom(stream); case PrefixByteArray: - return PrimitiveSerializer.Bytes.ReadFrom(stream); + // When it's a byte array always return as a LazyCompressedString + // TODO: Else we need to make a different prefix for lazy vs eager loading + return new LazyCompressedString(PrimitiveSerializer.Bytes.ReadFrom(stream)); default: throw new NotSupportedException($"Cannot deserialize unknown type '{type}'."); } @@ -160,6 +162,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource PrimitiveSerializer.Char.WriteTo(PrefixByteArray, stream); PrimitiveSerializer.Bytes.WriteTo(byteArrayValue, stream); } + else if (value is LazyCompressedString lazyCompressedString) + { + PrimitiveSerializer.Char.WriteTo(PrefixByteArray, stream); + PrimitiveSerializer.Bytes.WriteTo(lazyCompressedString.GetBytes(), stream); + } else throw new NotSupportedException("Value type " + value.GetType().FullName + " cannot be serialized."); }