diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/AppSettingsNucachePropertyMapFactory.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/AppSettingsNucachePropertyMapFactory.cs index 271695d250..544231bd32 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/AppSettingsNucachePropertyMapFactory.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/AppSettingsNucachePropertyMapFactory.cs @@ -10,14 +10,13 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource // TODO: We'll remove this when the responsibility for compressing property data is at the property editor level internal class AppSettingsNuCachePropertyMapFactory : INuCachePropertyOptionsFactory { - public NuCachePropertyOptions GetNuCachePropertyOptions() + public NuCachePropertyCompressionOptions GetNuCachePropertyOptions() { - NuCachePropertyOptions options = new NuCachePropertyOptions - { - PropertyMap = GetPropertyMap(), - LZ4CompressionLevel = K4os.Compression.LZ4.LZ4Level.L10_OPT, - MinimumCompressibleStringLength = null - }; + var options = new NuCachePropertyCompressionOptions( + GetPropertyMap(), + K4os.Compression.LZ4.LZ4Level.L10_OPT, + null); + return options; } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs index 8b1b3e1496..32eb388bee 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs @@ -5,7 +5,7 @@ /// /// Serializes/Deserializes document to the SQL Database as bytes /// - public interface IContentNestedDataByteSerializer + public interface IContentNestedDataByteSerializer : IContentNestedDataSerializer { ContentNestedData DeserializeBytes(byte[] data); byte[] SerializeBytes(ContentNestedData nestedData); diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/INucachePropertyOptionsFactory.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/INucachePropertyOptionsFactory.cs index 0cb694e1c1..d423499744 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/INucachePropertyOptionsFactory.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/INucachePropertyOptionsFactory.cs @@ -8,6 +8,6 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource { public interface INuCachePropertyOptionsFactory { - NuCachePropertyOptions GetNuCachePropertyOptions(); + NuCachePropertyCompressionOptions GetNuCachePropertyOptions(); } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs index d71f0e2184..752cedc18b 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs @@ -68,6 +68,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource switch (map.CompressLevel) { + case NucachePropertyCompressionLevel.SQLDatabase: case NucachePropertyCompressionLevel.NuCacheDatabase: if (!(pdata.Value is null) && pdata.Value is byte[] byteArrayValue) { @@ -149,7 +150,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } } private static readonly NuCacheCompressionOptions DefaultMap = new NuCacheCompressionOptions(NucachePropertyCompressionLevel.None, NucachePropertyDecompressionLevel.NotCompressed, null); - private readonly NuCachePropertyOptions _nucachePropertyOptions; + private readonly NuCachePropertyCompressionOptions _nucachePropertyOptions; public NuCacheCompressionOptions GetSerializationMap(string propertyAlias) { diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs index 33181ebd5e..24e2bc7b27 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs @@ -1,17 +1,23 @@ -using MessagePack; +using K4os.Compression.LZ4; +using MessagePack; +using MessagePack.Formatters; using MessagePack.Resolvers; using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { /// /// Serializes/Deserializes document to the SQL Database as bytes using MessagePack /// - internal class MsgPackContentNestedDataSerializer : IContentNestedDataByteSerializer, IContentNestedDataSerializer + internal class MsgPackContentNestedDataSerializer : IContentNestedDataByteSerializer { - private readonly MessagePackSerializerOptions _options; + private MessagePackSerializerOptions _options; + private readonly NuCachePropertyCompressionOptions _propertyOptions; - public MsgPackContentNestedDataSerializer() + public MsgPackContentNestedDataSerializer(INuCachePropertyOptionsFactory propertyOptionsFactory = null) { var defaultOptions = ContractlessStandardResolver.Options; @@ -31,6 +37,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource _options = defaultOptions .WithResolver(resolver) .WithCompression(MessagePackCompression.Lz4BlockArray); + _propertyOptions = propertyOptionsFactory?.GetNuCachePropertyOptions() ?? NuCachePropertyCompressionOptions.Empty; } public string ToJson(string serialized) @@ -49,10 +56,45 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource public string Serialize(ContentNestedData nestedData) { + Optimize(nestedData); + var bin = MessagePackSerializer.Serialize(nestedData, _options); return Convert.ToBase64String(bin); } + /// + /// Compress properties and map property names to shorter names + /// + /// + private void Optimize(ContentNestedData nestedData) + { + if (_propertyOptions.PropertyMap != null && _propertyOptions.PropertyMap.Count > 0) + { + foreach (var map in _propertyOptions.PropertyMap) + { + if (map.Value.CompressLevel.Equals(NucachePropertyCompressionLevel.SQLDatabase)) + { + if (nestedData.PropertyData.TryGetValue(map.Key, out PropertyData[] properties)) + { + foreach (var property in properties.Where(x => x.Value != null && x.Value is string)) + { + property.Value = LZ4Pickler.Pickle(Encoding.UTF8.GetBytes(property.Value as string), _propertyOptions.LZ4CompressionLevel); + } + } + } + + // if there is an alias map for this property then use that instead of the real property alias + // (used to save memory, the mapped alias is normally a single char or at least a smaller string) + if (map.Value.MappedAlias != null && !map.Key.Equals(map.Value.MappedAlias) + && nestedData.PropertyData.Remove(map.Key) + && nestedData.PropertyData.TryGetValue(map.Key, out PropertyData[] properties2)) + { + nestedData.PropertyData.Add(map.Value.MappedAlias, properties2); + } + } + } + } + public ContentNestedData DeserializeBytes(byte[] data) => MessagePackSerializer.Deserialize(data, _options); public byte[] SerializeBytes(ContentNestedData nestedData) => MessagePackSerializer.Serialize(nestedData, _options); diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NuCachePropertyCompressionOptions.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NuCachePropertyCompressionOptions.cs new file mode 100644 index 0000000000..50fb20cadc --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NuCachePropertyCompressionOptions.cs @@ -0,0 +1,32 @@ +using K4os.Compression.LZ4; +using System; +using System.Collections.Generic; + +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + + public class NuCachePropertyCompressionOptions + { + /// + /// Returns empty options + /// + public static NuCachePropertyCompressionOptions Empty { get; } = new NuCachePropertyCompressionOptions(); + + private NuCachePropertyCompressionOptions() + { + } + + public NuCachePropertyCompressionOptions(IReadOnlyDictionary propertyMap, LZ4Level lZ4CompressionLevel, long? minimumCompressibleStringLength) + { + PropertyMap = propertyMap ?? throw new ArgumentNullException(nameof(propertyMap)); + LZ4CompressionLevel = lZ4CompressionLevel; + MinimumCompressibleStringLength = minimumCompressibleStringLength; + } + + public IReadOnlyDictionary PropertyMap { get; } = new Dictionary(); + + public LZ4Level LZ4CompressionLevel { get; } = LZ4Level.L00_FAST; + + public long? MinimumCompressibleStringLength { get; } + } +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyCompressionLevel.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyCompressionLevel.cs index 8a82dcb888..2f24a203ca 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyCompressionLevel.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyCompressionLevel.cs @@ -7,6 +7,14 @@ { None = 0, + /// + /// Compress property data at the nucache SQL DB table level + /// + /// + /// Only necessary if the document in the nucache SQL DB table isn't stored as compressed bytes + /// + SQLDatabase = 1, + /// /// Compress property data at the nucache BTree level /// diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyOptions.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyOptions.cs deleted file mode 100644 index f88f30ccd8..0000000000 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyOptions.cs +++ /dev/null @@ -1,17 +0,0 @@ -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace Umbraco.Web.PublishedCache.NuCache.DataSource -{ - - public class NuCachePropertyOptions - { - public IReadOnlyDictionary PropertyMap { get; set; } = new Dictionary(); - - public K4os.Compression.LZ4.LZ4Level LZ4CompressionLevel { get; set; } = K4os.Compression.LZ4.LZ4Level.L00_FAST; - - public long? MinimumCompressibleStringLength { get; set; } - } -} diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index 5715a5e060..fd54f550da 100755 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -262,7 +262,7 @@ - +