From 07de0876e776bca8ac1b43a46d3ea662d10125c2 Mon Sep 17 00:00:00 2001 From: Shannon Date: Thu, 13 Aug 2020 22:15:09 +1000 Subject: [PATCH] Cleans up some code, adds code comments --- .../DataSource/BTree.ContentDataSerializer.cs | 15 +-- ....DictionaryOfCultureVariationSerializer.cs | 3 + ...Tree.DictionaryOfPropertyDataSerializer.cs | 3 + .../NuCache/DataSource/ContentData.cs | 4 +- .../IContentNestedDataSerializer.cs | 18 ++-- .../DataSource/LazyCompressedString.cs | 2 + .../Lz4DictionaryOfPropertyDataSerializer.cs | 102 +++++++++--------- .../MsgPackContentNestedDataSerializer.cs | 12 ++- .../DataSource/NuCacheCompressionOptions.cs | 7 ++ .../NucachePropertyCompressionLevel.cs | 23 ++++ .../NucachePropertyDecompressionLevel.cs | 12 +++ src/Umbraco.Web/Umbraco.Web.csproj | 2 + 12 files changed, 135 insertions(+), 68 deletions(-) create mode 100644 src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyCompressionLevel.cs create mode 100644 src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyDecompressionLevel.cs diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.ContentDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.ContentDataSerializer.cs index 9cc5d3a701..1192c892d4 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.ContentDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.ContentDataSerializer.cs @@ -3,18 +3,21 @@ using CSharpTest.Net.Serialization; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { - class ContentDataSerializer : ISerializer + /// + /// Serializes/Deserializes data to BTree data source for + /// + internal class ContentDataSerializer : ISerializer { public ContentDataSerializer(IDictionaryOfPropertyDataSerializer dictionaryOfPropertyDataSerializer = null) { _dictionaryOfPropertyDataSerializer = dictionaryOfPropertyDataSerializer; if(_dictionaryOfPropertyDataSerializer == null) { - _dictionaryOfPropertyDataSerializer = PropertiesSerializer; + _dictionaryOfPropertyDataSerializer = DefaultPropertiesSerializer; } } - private static readonly DictionaryOfPropertyDataSerializer PropertiesSerializer = new DictionaryOfPropertyDataSerializer(); - private static readonly DictionaryOfCultureVariationSerializer CultureVariationsSerializer = new DictionaryOfCultureVariationSerializer(); + private static readonly DictionaryOfPropertyDataSerializer DefaultPropertiesSerializer = new DictionaryOfPropertyDataSerializer(); + private static readonly DictionaryOfCultureVariationSerializer DefaultCultureVariationsSerializer = new DictionaryOfCultureVariationSerializer(); private readonly IDictionaryOfPropertyDataSerializer _dictionaryOfPropertyDataSerializer; public ContentData ReadFrom(Stream stream) @@ -29,7 +32,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource WriterId = PrimitiveSerializer.Int32.ReadFrom(stream), TemplateId = PrimitiveSerializer.Int32.ReadFrom(stream), Properties = _dictionaryOfPropertyDataSerializer.ReadFrom(stream), // TODO: We don't want to allocate empty arrays - CultureInfos = CultureVariationsSerializer.ReadFrom(stream) // TODO: We don't want to allocate empty arrays + CultureInfos = DefaultCultureVariationsSerializer.ReadFrom(stream) // TODO: We don't want to allocate empty arrays }; } @@ -46,7 +49,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource PrimitiveSerializer.Int32.WriteTo(value.TemplateId.Value, stream); } _dictionaryOfPropertyDataSerializer.WriteTo(value.Properties, stream); - CultureVariationsSerializer.WriteTo(value.CultureInfos, stream); + DefaultCultureVariationsSerializer.WriteTo(value.CultureInfos, stream); } } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfCultureVariationSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfCultureVariationSerializer.cs index 1fcbdbdae7..b3a6ef2d05 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfCultureVariationSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfCultureVariationSerializer.cs @@ -6,6 +6,9 @@ using Umbraco.Core; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { + /// + /// Serializes/Deserializes culture variant data as a dictionary for BTree + /// internal class DictionaryOfCultureVariationSerializer : SerializerBase, ISerializer> { public IReadOnlyDictionary ReadFrom(Stream stream) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs index cb12164397..0b15c0ba4b 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs @@ -6,6 +6,9 @@ using Umbraco.Core; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { + /// + /// Serializes/Deserializes property data as a dictionary for BTree + /// internal class DictionaryOfPropertyDataSerializer : SerializerBase, ISerializer>, IDictionaryOfPropertyDataSerializer { public IDictionary ReadFrom(Stream stream) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentData.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentData.cs index 36586acd12..5f252591e0 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentData.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentData.cs @@ -3,7 +3,9 @@ using System.Collections.Generic; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { - // represents everything that is specific to edited or published version + /// + /// Represents everything that is specific to an edited or published content version + /// internal class ContentData { public string Name { get; set; } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs index edd061fbe4..32eb388bee 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs @@ -1,19 +1,19 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace Umbraco.Web.PublishedCache.NuCache.DataSource +namespace Umbraco.Web.PublishedCache.NuCache.DataSource { - // TODO: We need a better name, not sure why the class is called ContentNested in the first place + // TODO: We need better names if possible, not sure why the class is called ContentNested in the first place + + /// + /// Serializes/Deserializes document to the SQL Database as bytes + /// public interface IContentNestedDataByteSerializer : IContentNestedDataSerializer { ContentNestedData DeserializeBytes(byte[] data); byte[] SerializeBytes(ContentNestedData nestedData); } - // TODO: We need a better name, not sure why the class is called ContentNested in the first place + /// + /// Serializes/Deserializes document to the SQL Database as a string + /// public interface IContentNestedDataSerializer { ContentNestedData Deserialize(string data); diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs index 3d6e70c7b2..55469ad791 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs @@ -13,6 +13,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource /// internal class LazyCompressedString { + // TODO: This could be a struct + private byte[] _bytes; private string _str; diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs index c62af1293f..31f7763e04 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs @@ -9,26 +9,10 @@ using K4os.Compression.LZ4; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { - /// - /// If/where to compress custom properties for nucache - /// - public enum NucachePropertyCompressionLevel - { - None = 0, - SQLDatabase = 1, - NuCacheDatabase = 2 - } - /// - /// If/where to decompress custom properties for nucache - /// - public enum NucachePropertyDecompressionLevel - { - NotCompressed = 0, - Immediate = 1, - Lazy = 2 - } - + /// + /// Serializes/Deserializes property data as a dictionary for BTree with Lz4 compression options + /// internal class Lz4DictionaryOfPropertyDataSerializer : SerializerBase, ISerializer>, IDictionaryOfPropertyDataSerializer { private readonly IReadOnlyDictionary _compressProperties; @@ -58,7 +42,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource // read property alias var alias = PrimitiveSerializer.String.ReadFrom(stream); - var map = GetDeSerializationMap(alias); + var map = GetDeserializationMap(alias); var key = string.Intern(map.MappedAlias ?? alias); // read values count @@ -82,23 +66,30 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource pdata.Segment = ReadStringObject(stream, true) ?? string.Empty; pdata.Value = ReadObject(stream); - if ((map.CompressLevel.Equals(NucachePropertyCompressionLevel.NuCacheDatabase) || map.CompressLevel.Equals(NucachePropertyCompressionLevel.SQLDatabase)) - && pdata.Value != null && pdata.Value is byte[] byteArrayValue) - { - //Compressed string - switch (map.DecompressLevel) - { - case NucachePropertyDecompressionLevel.Lazy: - pdata.Value = new LazyCompressedString(byteArrayValue); - break; - case NucachePropertyDecompressionLevel.NotCompressed: - break;//Shouldn't be any not compressed - case NucachePropertyDecompressionLevel.Immediate: - default: - pdata.Value = Encoding.UTF8.GetString(LZ4Pickler.Unpickle(byteArrayValue)); - break; - } - } + switch (map.CompressLevel) + { + case NucachePropertyCompressionLevel.SQLDatabase: + case NucachePropertyCompressionLevel.NuCacheDatabase: + if (!(pdata.Value is null) && pdata.Value is byte[] byteArrayValue) + { + //Compressed string + switch (map.DecompressLevel) + { + case NucachePropertyDecompressionLevel.Lazy: + pdata.Value = new LazyCompressedString(byteArrayValue); + break; + case NucachePropertyDecompressionLevel.NotCompressed: + //Shouldn't be any not compressed + // TODO: Do we need to throw here? + break; + case NucachePropertyDecompressionLevel.Immediate: + default: + pdata.Value = Encoding.UTF8.GetString(LZ4Pickler.Unpickle(byteArrayValue)); + break; + } + } + break; + } } dict[key] = pdatas.ToArray(); @@ -131,17 +122,30 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource WriteObject(pdata.Culture ?? string.Empty, stream); WriteObject(pdata.Segment ?? string.Empty, stream); - //Only compress strings - if (pdata.Value is string stringValue && pdata.Value != null && map.CompressLevel.Equals(NucachePropertyCompressionLevel.NuCacheDatabase) - && (_nucachePropertyOptions.MinimumCompressibleStringLength == null - || !_nucachePropertyOptions.MinimumCompressibleStringLength.HasValue - || stringValue.Length > _nucachePropertyOptions.MinimumCompressibleStringLength.Value)) + //Only compress strings + switch (map.CompressLevel) { - var stringBytes = Encoding.UTF8.GetBytes(stringValue); - var compressedBytes = LZ4Pickler.Pickle(stringBytes, _nucachePropertyOptions.LZ4CompressionLevel); - WriteObject(compressedBytes, stream); + // If we're compressing into btree at the property level + case NucachePropertyCompressionLevel.NuCacheDatabase: + + if (pdata.Value is string stringValue && !(pdata.Value is null) + && (_nucachePropertyOptions.MinimumCompressibleStringLength is null + || !_nucachePropertyOptions.MinimumCompressibleStringLength.HasValue + || stringValue.Length > _nucachePropertyOptions.MinimumCompressibleStringLength.Value)) + { + var stringBytes = Encoding.UTF8.GetBytes(stringValue); + var compressedBytes = LZ4Pickler.Pickle(stringBytes, _nucachePropertyOptions.LZ4CompressionLevel); + WriteObject(compressedBytes, stream); + } + else + { + WriteObject(pdata.Value, stream); + } + break; + default: + WriteObject(pdata.Value, stream); + break; } - WriteObject(pdata.Value, stream); } } } @@ -150,7 +154,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource public NuCacheCompressionOptions GetSerializationMap(string propertyAlias) { - if (_compressProperties == null) + if (_compressProperties is null) { return DefaultMap; } @@ -161,9 +165,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource return DefaultMap; } - public NuCacheCompressionOptions GetDeSerializationMap(string propertyAlias) + public NuCacheCompressionOptions GetDeserializationMap(string propertyAlias) { - if (_uncompressProperties == null) + if (_uncompressProperties is null) { return DefaultMap; } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs index 133586382c..206051f839 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs @@ -9,6 +9,9 @@ using System.Text; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { + /// + /// Serializes/Deserializes document to the SQL Database as bytes using MessagePack + /// internal class MsgPackContentNestedDataSerializer : IContentNestedDataByteSerializer { private MessagePackSerializerOptions _options; @@ -65,7 +68,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource /// private void Optimize(ContentNestedData nestedData) { - if (_propertyOptions.PropertyMap != null && _propertyOptions.PropertyMap.Any()) + if (_propertyOptions.PropertyMap != null && _propertyOptions.PropertyMap.Count > 0) { foreach (var map in _propertyOptions.PropertyMap) { @@ -79,10 +82,13 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } } } + + // if there is an alias map for this property then use that instead of the real property alias + // (used to save memory, the mapped alias is normally a single char or at least a smaller string) if (map.Value.MappedAlias != null && !map.Key.Equals(map.Value.MappedAlias) - && nestedData.PropertyData.Remove(map.Key) && nestedData.PropertyData.TryGetValue(map.Key, out PropertyData[] properties2)) + && nestedData.PropertyData.Remove(map.Key) + && nestedData.PropertyData.TryGetValue(map.Key, out PropertyData[] properties2)) { - nestedData.PropertyData.Remove(map.Key); nestedData.PropertyData.Add(map.Value.MappedAlias, properties2); } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NuCacheCompressionOptions.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NuCacheCompressionOptions.cs index 64f1b675bc..36f1606008 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NuCacheCompressionOptions.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NuCacheCompressionOptions.cs @@ -14,6 +14,13 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource public NucachePropertyCompressionLevel CompressLevel { get; private set; } public NucachePropertyDecompressionLevel DecompressLevel { get; private set; } + + /// + /// Used to map a real property alias to a shorter moniker in memory + /// + /// + /// This is simply a memory saving mechanism + /// public string MappedAlias { get; private set; } public override bool Equals(object obj) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyCompressionLevel.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyCompressionLevel.cs new file mode 100644 index 0000000000..2f24a203ca --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyCompressionLevel.cs @@ -0,0 +1,23 @@ +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + /// + /// If/where to compress custom properties for nucache + /// + public enum NucachePropertyCompressionLevel + { + None = 0, + + /// + /// Compress property data at the nucache SQL DB table level + /// + /// + /// Only necessary if the document in the nucache SQL DB table isn't stored as compressed bytes + /// + SQLDatabase = 1, + + /// + /// Compress property data at the nucache BTree level + /// + NuCacheDatabase = 2 + } +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyDecompressionLevel.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyDecompressionLevel.cs new file mode 100644 index 0000000000..335ae6ff4a --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyDecompressionLevel.cs @@ -0,0 +1,12 @@ +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + /// + /// If/where to decompress custom properties for nucache + /// + public enum NucachePropertyDecompressionLevel + { + NotCompressed = 0, + Immediate = 1, + Lazy = 2 + } +} diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index 18c4a8ed93..5715a5e060 100755 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -260,6 +260,8 @@ + +