diff --git a/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs b/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs index debfddef98..c85973f4b0 100644 --- a/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs +++ b/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs @@ -55,14 +55,14 @@ namespace Umbraco.Tests.PublishedContent UrlSegment = "home" }; - var json = jsonSerializer.Serialize(content); - var msgPack = msgPackSerializer.Serialize(content); + var json = jsonSerializer.Serialize(1, content); + var msgPack = msgPackSerializer.Serialize(1, content); Console.WriteLine(json); Console.WriteLine(msgPackSerializer.ToJson(msgPack)); - var jsonContent = jsonSerializer.Deserialize(json); - var msgPackContent = msgPackSerializer.Deserialize(msgPack); + var jsonContent = jsonSerializer.Deserialize(1, json); + var msgPackContent = msgPackSerializer.Deserialize(1, msgPack); CollectionAssert.AreEqual(jsonContent.CultureData.Keys, msgPackContent.CultureData.Keys); diff --git a/src/Umbraco.Web/PropertyEditors/ComplexEditorPropertyCompressionOptions.cs b/src/Umbraco.Web/PropertyEditors/ComplexEditorPropertyCompressionOptions.cs new file mode 100644 index 0000000000..f4776f652d --- /dev/null +++ b/src/Umbraco.Web/PropertyEditors/ComplexEditorPropertyCompressionOptions.cs @@ -0,0 +1,48 @@ +using K4os.Compression.LZ4; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using Umbraco.Core.PropertyEditors; +using Umbraco.Core.Services; + +namespace Umbraco.Web.PropertyEditors +{ + + /// + /// Ensures all property types that have an editor storing a complex value are compressed + /// + public class ComplexEditorPropertyCompressionOptions : IPropertyCompressionOptions + { + private readonly IContentTypeService _contentTypeService; + private readonly PropertyEditorCollection _propertyEditors; + private readonly ConcurrentDictionary<(int, string), string> _editorValueTypes = new ConcurrentDictionary<(int, string), string>(); + + public ComplexEditorPropertyCompressionOptions(IContentTypeService contentTypeService, PropertyEditorCollection propertyEditors) + { + _contentTypeService = contentTypeService; + _propertyEditors = propertyEditors; + } + + public bool IsCompressed(int contentTypeId, string alias) + { + var valueType = _editorValueTypes.GetOrAdd((contentTypeId, alias), x => + { + var ct = _contentTypeService.Get(contentTypeId); + if (ct == null) return null; + + var propertyType = ct.CompositionPropertyTypes.FirstOrDefault(x => x.Alias == alias); + if (propertyType == null) return null; + + if (!_propertyEditors.TryGet(propertyType.PropertyEditorAlias, out var propertyEditor)) return null; + + var editor = propertyEditor.GetValueEditor(); + if (editor == null) return null; + + return editor.ValueType; + }); + + return valueType == ValueTypes.Json || valueType == ValueTypes.Xml || valueType == ValueTypes.Text; + } + } +} diff --git a/src/Umbraco.Web/PropertyEditors/IPropertyCompressionOptions.cs b/src/Umbraco.Web/PropertyEditors/IPropertyCompressionOptions.cs new file mode 100644 index 0000000000..70f3e7c6f0 --- /dev/null +++ b/src/Umbraco.Web/PropertyEditors/IPropertyCompressionOptions.cs @@ -0,0 +1,10 @@ +namespace Umbraco.Web.PropertyEditors +{ + /// + /// Determines if a property type's value should be compressed + /// + public interface IPropertyCompressionOptions + { + bool IsCompressed(int contentTypeId, string alias); + } +} diff --git a/src/Umbraco.Web/PropertyEditors/NoopPropertyCompressionOptions.cs b/src/Umbraco.Web/PropertyEditors/NoopPropertyCompressionOptions.cs new file mode 100644 index 0000000000..638306d7ca --- /dev/null +++ b/src/Umbraco.Web/PropertyEditors/NoopPropertyCompressionOptions.cs @@ -0,0 +1,10 @@ +namespace Umbraco.Web.PropertyEditors +{ + /// + /// Disables all compression for all properties + /// + internal class NoopPropertyCompressionOptions : IPropertyCompressionOptions + { + public bool IsCompressed(int contentTypeId, string alias) => false; + } +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/AppSettingsNucachePropertyMapFactory.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/AppSettingsNucachePropertyMapFactory.cs deleted file mode 100644 index 544231bd32..0000000000 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/AppSettingsNucachePropertyMapFactory.cs +++ /dev/null @@ -1,47 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Configuration; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace Umbraco.Web.PublishedCache.NuCache.DataSource -{ - // TODO: We'll remove this when the responsibility for compressing property data is at the property editor level - internal class AppSettingsNuCachePropertyMapFactory : INuCachePropertyOptionsFactory - { - public NuCachePropertyCompressionOptions GetNuCachePropertyOptions() - { - var options = new NuCachePropertyCompressionOptions( - GetPropertyMap(), - K4os.Compression.LZ4.LZ4Level.L10_OPT, - null); - - return options; - } - - public IReadOnlyDictionary GetPropertyMap() - { - var propertyMap = new Dictionary(); - // TODO: Use xml/json/c# to define map - var propertyDictionarySerializerMap = ConfigurationManager.AppSettings["Umbraco.Web.PublishedCache.NuCache.PropertySerializationMap"]; - if (!string.IsNullOrWhiteSpace(propertyDictionarySerializerMap)) - { - //propertyAlias,CompressionLevel,DecompressionLevel,mappedAlias; - propertyDictionarySerializerMap.Split(';') - .Select(x => - { - var y = x.Split(','); - (string alias, NucachePropertyCompressionLevel compressionLevel, NucachePropertyDecompressionLevel decompressionLevel, string mappedAlias) v = (y[0], - (NucachePropertyCompressionLevel)System.Enum.Parse(typeof(NucachePropertyCompressionLevel), y[1]), - (NucachePropertyDecompressionLevel)System.Enum.Parse(typeof(NucachePropertyDecompressionLevel), y[2]), - y[3] - ); - return v; - }) - .ToList().ForEach(x => propertyMap.Add(x.alias, new NuCacheCompressionOptions(x.compressionLevel, x.decompressionLevel, x.mappedAlias))); - } - return propertyMap; - } - } -} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs index 3cad7171be..a52fde2f0a 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs @@ -259,8 +259,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource else { var nested = _contentNestedDataSerializer is IContentNestedDataByteSerializer byteSerializer - ? byteSerializer.DeserializeBytes(dto.EditDataRaw) - : _contentNestedDataSerializer.Deserialize(dto.EditData); + ? byteSerializer.DeserializeBytes(dto.ContentTypeId, dto.EditDataRaw) + : _contentNestedDataSerializer.Deserialize(dto.ContentTypeId, dto.EditData); d = new ContentData { @@ -288,8 +288,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource else { var nested = _contentNestedDataSerializer is IContentNestedDataByteSerializer byteSerializer - ? byteSerializer.DeserializeBytes(dto.PubDataRaw) - : _contentNestedDataSerializer.Deserialize(dto.PubData); + ? byteSerializer.DeserializeBytes(dto.ContentTypeId, dto.PubDataRaw) + : _contentNestedDataSerializer.Deserialize(dto.ContentTypeId, dto.PubData); p = new ContentData { @@ -326,8 +326,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource throw new InvalidOperationException("No data for media " + dto.Id); var nested = _contentNestedDataSerializer is IContentNestedDataByteSerializer byteSerializer - ? byteSerializer.DeserializeBytes(dto.EditDataRaw) - : _contentNestedDataSerializer.Deserialize(dto.EditData); + ? byteSerializer.DeserializeBytes(dto.ContentTypeId, dto.EditDataRaw) + : _contentNestedDataSerializer.Deserialize(dto.ContentTypeId, dto.EditData); var p = new ContentData { diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs index 32eb388bee..09933d735d 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs @@ -7,8 +7,8 @@ /// public interface IContentNestedDataByteSerializer : IContentNestedDataSerializer { - ContentNestedData DeserializeBytes(byte[] data); - byte[] SerializeBytes(ContentNestedData nestedData); + ContentNestedData DeserializeBytes(int contentTypeId, byte[] data); + byte[] SerializeBytes(int contentTypeId, ContentNestedData nestedData); } /// @@ -16,7 +16,7 @@ /// public interface IContentNestedDataSerializer { - ContentNestedData Deserialize(string data); - string Serialize(ContentNestedData nestedData); + ContentNestedData Deserialize(int contentTypeId, string data); + string Serialize(int contentTypeId, ContentNestedData nestedData); } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/INucachePropertyOptionsFactory.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/INucachePropertyOptionsFactory.cs deleted file mode 100644 index d423499744..0000000000 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/INucachePropertyOptionsFactory.cs +++ /dev/null @@ -1,13 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace Umbraco.Web.PublishedCache.NuCache.DataSource -{ - public interface INuCachePropertyOptionsFactory - { - NuCachePropertyCompressionOptions GetNuCachePropertyOptions(); - } -} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs index 6635fa7090..d4f11591c1 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs @@ -11,7 +11,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource internal class JsonContentNestedDataSerializer : IContentNestedDataSerializer { - public ContentNestedData Deserialize(string data) + public ContentNestedData Deserialize(int contentTypeId, string data) { // by default JsonConvert will deserialize our numeric values as Int64 // which is bad, because they were Int32 in the database - take care @@ -30,7 +30,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource return JsonConvert.DeserializeObject(data, settings); } - public string Serialize(ContentNestedData nestedData) + public string Serialize(int contentTypeId, ContentNestedData nestedData) { // note that numeric values (which are Int32) are serialized without their // type (eg "value":1234) and JsonConvert by default deserializes them as Int64 diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs index 9df40daf76..3e0e796d36 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs @@ -1,17 +1,18 @@ using K4os.Compression.LZ4; using System; using System.Text; -using System.Threading; +using Umbraco.Core.Exceptions; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { /// /// Lazily decompresses a LZ4 Pickler compressed UTF8 string /// - internal class LazyCompressedString + internal struct LazyCompressedString { - private string _str; private byte[] _bytes; + private string _str; + private readonly object _locker; /// /// Constructor @@ -19,7 +20,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource /// LZ4 Pickle compressed UTF8 String public LazyCompressedString(byte[] bytes) { + _locker = new object(); _bytes = bytes; + _str = null; } public byte[] GetBytes() @@ -31,13 +34,18 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource public override string ToString() { - return LazyInitializer.EnsureInitialized(ref _str, () => + if (_str != null) return _str; + lock (_locker) { - var str = Encoding.UTF8.GetString(LZ4Pickler.Unpickle(_bytes)); + if (_str != null) return _str; // double check + if (_bytes == null) throw new PanicException("Bytes have already been cleared"); + _str = Encoding.UTF8.GetString(LZ4Pickler.Unpickle(_bytes)); _bytes = null; - return str; - }); + } + return _str; } + + public static implicit operator string(LazyCompressedString l) => l.ToString(); } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs deleted file mode 100644 index 1a2d26f4b1..0000000000 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs +++ /dev/null @@ -1,181 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using CSharpTest.Net.Serialization; -using Umbraco.Core; -using System.Text; -using K4os.Compression.LZ4; - -namespace Umbraco.Web.PublishedCache.NuCache.DataSource -{ - - /// - /// Serializes/Deserializes property data as a dictionary for BTree with Lz4 compression options - /// - internal class Lz4DictionaryOfPropertyDataSerializer : SerializerBase, ISerializer>, IDictionaryOfPropertyDataSerializer - { - private readonly IReadOnlyDictionary _compressProperties; - private readonly IReadOnlyDictionary _uncompressProperties; - - - public Lz4DictionaryOfPropertyDataSerializer(INuCachePropertyOptionsFactory nucachePropertyOptionsFactory) - { - var nucachePropertyOptions = nucachePropertyOptionsFactory.GetNuCachePropertyOptions(); - _compressProperties = nucachePropertyOptions.PropertyMap.ToList().ToDictionary(x => string.Intern(x.Key), x => new NuCacheCompressionOptions(x.Value.CompressLevel, x.Value.DecompressLevel, string.Intern(x.Value.MappedAlias))); - _uncompressProperties = _compressProperties.ToList().ToDictionary(x => x.Value.MappedAlias, x => new NuCacheCompressionOptions(x.Value.CompressLevel, x.Value.DecompressLevel, x.Key)); - - _nucachePropertyOptions = nucachePropertyOptions; - } - - - public IDictionary ReadFrom(Stream stream) - { - var dict = new Dictionary(StringComparer.InvariantCultureIgnoreCase); - - // read properties count - var pcount = PrimitiveSerializer.Int32.ReadFrom(stream); - - // read each property - for (var i = 0; i < pcount; i++) - { - - // read property alias - var alias = PrimitiveSerializer.String.ReadFrom(stream); - var map = GetDeserializationMap(alias); - var key = string.Intern(map.MappedAlias ?? alias); - - // read values count - var vcount = PrimitiveSerializer.Int32.ReadFrom(stream); - - // create pdata and add to the dictionary - var pdatas = new List(); - - // for each value, read and add to pdata - for (var j = 0; j < vcount; j++) - { - var pdata = new PropertyData(); - pdatas.Add(pdata); - - // everything that can be null is read/written as object - // even though - culture and segment should never be null here, as 'null' represents - // the 'current' value, and string.Empty should be used to represent the invariant or - // neutral values - PropertyData throws when getting nulls, so falling back to - // string.Empty here - what else? - pdata.Culture = ReadStringObject(stream, true) ?? string.Empty; - pdata.Segment = ReadStringObject(stream, true) ?? string.Empty; - pdata.Value = ReadObject(stream); - - switch (map.CompressLevel) - { - // If the property is compressed at either the DB or Nucache level, it means it's compressed here and we need to decompress - case NucachePropertyCompressionLevel.SQLDatabase: - case NucachePropertyCompressionLevel.NuCacheDatabase: - if (!(pdata.Value is null) && pdata.Value is byte[] byteArrayValue) - { - //Compressed string - switch (map.DecompressLevel) - { - case NucachePropertyDecompressionLevel.Lazy: - pdata.Value = new LazyCompressedString(byteArrayValue); - break; - case NucachePropertyDecompressionLevel.NotCompressed: - //Shouldn't be any not compressed - throw new InvalidOperationException($"{NucachePropertyDecompressionLevel.NotCompressed} cannot be a decompression option for property {alias} since it's compresion option is {map.CompressLevel}"); - case NucachePropertyDecompressionLevel.Immediate: - default: - pdata.Value = Encoding.UTF8.GetString(LZ4Pickler.Unpickle(byteArrayValue)); - break; - } - } - break; - } - } - - dict[key] = pdatas.ToArray(); - } - return dict; - } - - public void WriteTo(IDictionary value, Stream stream) - { - // write properties count - PrimitiveSerializer.Int32.WriteTo(value.Count, stream); - - // write each property - foreach (var (alias, values) in value) - { - var map = GetSerializationMap(alias); - - // write alias - PrimitiveSerializer.String.WriteTo(map.MappedAlias ?? alias, stream); - - // write values count - PrimitiveSerializer.Int32.WriteTo(values.Length, stream); - - // write each value - foreach (var pdata in values) - { - // everything that can be null is read/written as object - // even though - culture and segment should never be null here, - // see note in ReadFrom() method above - WriteObject(pdata.Culture ?? string.Empty, stream); - WriteObject(pdata.Segment ?? string.Empty, stream); - - //Only compress strings (if NucachePropertyCompressionLevel.SQLDatabase has been specified then the property value will already be compressed) - switch (map.CompressLevel) - { - // If we're compressing into btree at the property level - case NucachePropertyCompressionLevel.NuCacheDatabase: - - if (pdata.Value is string stringValue && !(pdata.Value is null) - && (_nucachePropertyOptions.MinimumCompressibleStringLength is null - || !_nucachePropertyOptions.MinimumCompressibleStringLength.HasValue - || stringValue.Length > _nucachePropertyOptions.MinimumCompressibleStringLength.Value)) - { - var stringBytes = Encoding.UTF8.GetBytes(stringValue); - var compressedBytes = LZ4Pickler.Pickle(stringBytes, _nucachePropertyOptions.LZ4CompressionLevel); - WriteObject(compressedBytes, stream); - } - else - { - WriteObject(pdata.Value, stream); - } - break; - default: - WriteObject(pdata.Value, stream); - break; - } - } - } - } - private static readonly NuCacheCompressionOptions DefaultMap = new NuCacheCompressionOptions(NucachePropertyCompressionLevel.None, NucachePropertyDecompressionLevel.NotCompressed, null); - private readonly NuCachePropertyCompressionOptions _nucachePropertyOptions; - - public NuCacheCompressionOptions GetSerializationMap(string propertyAlias) - { - if (_compressProperties is null) - { - return DefaultMap; - } - if (_compressProperties.TryGetValue(propertyAlias, out var map1)) - { - return map1; - } - - return DefaultMap; - } - public NuCacheCompressionOptions GetDeserializationMap(string propertyAlias) - { - if (_uncompressProperties is null) - { - return DefaultMap; - } - if (_uncompressProperties.TryGetValue(propertyAlias, out var map2)) - { - return map2; - } - return DefaultMap; - } - } -} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs index 0475810422..0ea2b96fbe 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs @@ -2,10 +2,12 @@ using MessagePack; using MessagePack.Formatters; using MessagePack.Resolvers; +using NPoco.FluentMappings; using System; using System.Collections.Generic; using System.Linq; using System.Text; +using Umbraco.Web.PropertyEditors; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { @@ -15,9 +17,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource internal class MsgPackContentNestedDataSerializer : IContentNestedDataByteSerializer { private MessagePackSerializerOptions _options; - private readonly NuCachePropertyCompressionOptions _propertyOptions; + private readonly IPropertyCompressionOptions _propertyOptions; - public MsgPackContentNestedDataSerializer(INuCachePropertyOptionsFactory propertyOptionsFactory = null) + public MsgPackContentNestedDataSerializer(IPropertyCompressionOptions propertyOptions = null) { var defaultOptions = ContractlessStandardResolver.Options; @@ -37,7 +39,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource _options = defaultOptions .WithResolver(resolver) .WithCompression(MessagePackCompression.Lz4BlockArray); - _propertyOptions = propertyOptionsFactory?.GetNuCachePropertyOptions() ?? NuCachePropertyCompressionOptions.Empty; + _propertyOptions = propertyOptions ?? new NoopPropertyCompressionOptions(); } public string ToJson(string serialized) @@ -47,113 +49,69 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource return json; } - public ContentNestedData Deserialize(string data) + public ContentNestedData Deserialize(int contentTypeId, string data) { var bin = Convert.FromBase64String(data); var nestedData = MessagePackSerializer.Deserialize(bin, _options); - Expand(nestedData); + Expand(contentTypeId, nestedData); return nestedData; } - public string Serialize(ContentNestedData nestedData) + public string Serialize(int contentTypeId, ContentNestedData nestedData) { - Compress(nestedData); + Compress(contentTypeId, nestedData); var bin = MessagePackSerializer.Serialize(nestedData, _options); return Convert.ToBase64String(bin); } - public ContentNestedData DeserializeBytes(byte[] data) + public ContentNestedData DeserializeBytes(int contentTypeId, byte[] data) { var nestedData = MessagePackSerializer.Deserialize(data, _options); - Expand(nestedData); + Expand(contentTypeId, nestedData); return nestedData; } - public byte[] SerializeBytes(ContentNestedData nestedData) + public byte[] SerializeBytes(int contentTypeId, ContentNestedData nestedData) { - Compress(nestedData); + Compress(contentTypeId, nestedData); return MessagePackSerializer.Serialize(nestedData, _options); } /// - /// Used during serialization to compress properties and map property names to shorter names + /// Used during serialization to compress properties /// /// - private void Compress(ContentNestedData nestedData) + private void Compress(int contentTypeId, ContentNestedData nestedData) { - if (_propertyOptions.PropertyMap != null && _propertyOptions.PropertyMap.Count > 0) + foreach(var propertyAliasToData in nestedData.PropertyData) { - foreach (var map in _propertyOptions.PropertyMap) + if (_propertyOptions.IsCompressed(contentTypeId, propertyAliasToData.Key)) { - if (map.Value.CompressLevel.Equals(NucachePropertyCompressionLevel.SQLDatabase)) + foreach(var property in propertyAliasToData.Value.Where(x => x.Value != null && x.Value is string)) { - if (nestedData.PropertyData.TryGetValue(map.Key, out PropertyData[] properties)) - { - foreach (var property in properties.Where(x => x.Value != null && x.Value is string)) - { - property.Value = LZ4Pickler.Pickle(Encoding.UTF8.GetBytes(property.Value as string), _propertyOptions.LZ4CompressionLevel); - } - } - } - - // if there is an alias map for this property then use that instead of the real property alias - // (used to save memory, the mapped alias is normally a single char or at least a smaller string) - if (map.Value.MappedAlias != null && !map.Key.Equals(map.Value.MappedAlias) - && nestedData.PropertyData.TryGetValue(map.Key, out PropertyData[] properties2)) - { - nestedData.PropertyData.Remove(map.Key); - nestedData.PropertyData.Add(map.Value.MappedAlias, properties2); + property.Value = LZ4Pickler.Pickle(Encoding.UTF8.GetBytes((string)property.Value), LZ4Level.L00_FAST); } } } } /// - /// Used during deserialization to map the property data as lazy or expand the value and re-map back to the true property aliases + /// Used during deserialization to map the property data as lazy or expand the value /// /// - private void Expand(ContentNestedData nestedData) + private void Expand(int contentTypeId, ContentNestedData nestedData) { - if (_propertyOptions.PropertyMap != null && _propertyOptions.PropertyMap.Count > 0) + foreach (var propertyAliasToData in nestedData.PropertyData) { - foreach (var map in _propertyOptions.PropertyMap) + if (_propertyOptions.IsCompressed(contentTypeId, propertyAliasToData.Key)) { - if (map.Value.CompressLevel.Equals(NucachePropertyCompressionLevel.SQLDatabase)) + foreach (var property in propertyAliasToData.Value.Where(x => x.Value != null)) { - // if there is an alias map for this property then re-map to the real property alias - if (map.Value.MappedAlias != null && !map.Key.Equals(map.Value.MappedAlias) - && nestedData.PropertyData.TryGetValue(map.Value.MappedAlias, out PropertyData[] properties2)) + if (property.Value is byte[] byteArrayValue) { - nestedData.PropertyData.Remove(map.Value.MappedAlias); - nestedData.PropertyData.Add(map.Key, properties2); - } - - if (nestedData.PropertyData.TryGetValue(map.Key, out PropertyData[] properties)) - { - foreach (var pdata in properties) - { - if (!(pdata.Value is null) && pdata.Value is byte[] byteArrayValue) - { - //Compressed string - switch (map.Value.DecompressLevel) - { - case NucachePropertyDecompressionLevel.Lazy: - pdata.Value = new LazyCompressedString(byteArrayValue); - break; - case NucachePropertyDecompressionLevel.NotCompressed: - //Shouldn't be any not compressed - throw new InvalidOperationException($"{NucachePropertyDecompressionLevel.NotCompressed} cannot be a decompression option for property {map.Key} since it's compresion option is {map.Value.CompressLevel}"); - case NucachePropertyDecompressionLevel.Immediate: - default: - pdata.Value = Encoding.UTF8.GetString(LZ4Pickler.Unpickle(byteArrayValue)); - break; - } - } - } + property.Value = new LazyCompressedString(byteArrayValue); } } - - } } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NuCacheCompressionOptions.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NuCacheCompressionOptions.cs deleted file mode 100644 index 36f1606008..0000000000 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NuCacheCompressionOptions.cs +++ /dev/null @@ -1,57 +0,0 @@ -using System; -using System.Collections.Generic; - -namespace Umbraco.Web.PublishedCache.NuCache.DataSource -{ - public struct NuCacheCompressionOptions : IEquatable - { - public NuCacheCompressionOptions(NucachePropertyCompressionLevel compressLevel, NucachePropertyDecompressionLevel decompressLevel, string mappedAlias) - { - CompressLevel = compressLevel; - DecompressLevel = decompressLevel; - MappedAlias = mappedAlias ?? throw new ArgumentNullException(nameof(mappedAlias)); - } - - public NucachePropertyCompressionLevel CompressLevel { get; private set; } - public NucachePropertyDecompressionLevel DecompressLevel { get; private set; } - - /// - /// Used to map a real property alias to a shorter moniker in memory - /// - /// - /// This is simply a memory saving mechanism - /// - public string MappedAlias { get; private set; } - - public override bool Equals(object obj) - { - return obj is NuCacheCompressionOptions options && Equals(options); - } - - public bool Equals(NuCacheCompressionOptions other) - { - return CompressLevel == other.CompressLevel && - DecompressLevel == other.DecompressLevel && - MappedAlias == other.MappedAlias; - } - - public override int GetHashCode() - { - var hashCode = 961370163; - hashCode = hashCode * -1521134295 + CompressLevel.GetHashCode(); - hashCode = hashCode * -1521134295 + DecompressLevel.GetHashCode(); - hashCode = hashCode * -1521134295 + EqualityComparer.Default.GetHashCode(MappedAlias); - return hashCode; - } - - public static bool operator ==(NuCacheCompressionOptions left, NuCacheCompressionOptions right) - { - return left.Equals(right); - } - - public static bool operator !=(NuCacheCompressionOptions left, NuCacheCompressionOptions right) - { - return !(left == right); - } - } -} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NuCachePropertyCompressionOptions.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NuCachePropertyCompressionOptions.cs deleted file mode 100644 index 55ab813783..0000000000 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NuCachePropertyCompressionOptions.cs +++ /dev/null @@ -1,33 +0,0 @@ -using K4os.Compression.LZ4; -using System; -using System.Collections.Generic; - -namespace Umbraco.Web.PublishedCache.NuCache.DataSource -{ - - public class NuCachePropertyCompressionOptions - { - /// - /// Returns empty options - /// - public static NuCachePropertyCompressionOptions Empty { get; } = new NuCachePropertyCompressionOptions(); - - private NuCachePropertyCompressionOptions() - { - } - - public NuCachePropertyCompressionOptions(IReadOnlyDictionary propertyMap, LZ4Level lZ4CompressionLevel, long? minimumCompressibleStringLength) - { - PropertyMap = propertyMap ?? throw new ArgumentNullException(nameof(propertyMap)); - LZ4CompressionLevel = lZ4CompressionLevel; - MinimumCompressibleStringLength = minimumCompressibleStringLength; - } - - public IReadOnlyDictionary PropertyMap { get; } = new Dictionary(); - - public LZ4Level LZ4CompressionLevel { get; } = LZ4Level.L00_FAST; - - // TODO: Unsure if we really want to keep this - public long? MinimumCompressibleStringLength { get; } - } -} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyCompressionLevel.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyCompressionLevel.cs deleted file mode 100644 index 23826fd722..0000000000 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyCompressionLevel.cs +++ /dev/null @@ -1,31 +0,0 @@ -namespace Umbraco.Web.PublishedCache.NuCache.DataSource -{ - /// - /// If/where to compress custom properties for nucache - /// - public enum NucachePropertyCompressionLevel - { - None = 0, - - /// - /// Compress property data at the nucache SQL DB table level - /// - /// - /// Idea being we only compress this once. - /// All the records in cmsContentNu need to be rebuilt when this gets enabled. - /// Good option as then we don't use up memory / cpu to compress at boot. - /// - SQLDatabase = 1, - - /// - /// Compress property data at the nucache BTree level - /// - /// - /// Compress the property when writing to nucache bplustree after reading from the database. - /// Idea being we compress this at rebuild / boot. - /// This option supports older items not being compressed already, at the expense of doing this compression at boot. - /// But it also means you can easily switch between None and NuCacheDatabase if performance is worse. - /// - NuCacheDatabase = 2 - } -} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyDecompressionLevel.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyDecompressionLevel.cs deleted file mode 100644 index f4d485be71..0000000000 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyDecompressionLevel.cs +++ /dev/null @@ -1,15 +0,0 @@ -namespace Umbraco.Web.PublishedCache.NuCache.DataSource -{ - /// - /// If/where to decompress custom properties for nucache - /// - public enum NucachePropertyDecompressionLevel - { - NotCompressed = 0, - - // TODO: I'm unsure if this will ever be necessary, lazy seems good and deserialization would only occur once - Immediate = 1, - - Lazy = 2 - } -} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs index b90c418750..8b02946fc2 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs @@ -16,11 +16,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource private const char PrefixDouble = 'B'; private const char PrefixDateTime = 'D'; private const char PrefixByte = 'O'; - - // TODO: It might make sense to have another prefix for an LZ4 compressed byte array. - // Would be an improvement for the SQLDatabase compression option because then you could mix compressed and decompressed properties with the same alias. - // For example, don't compress recent content, but compress older content. private const char PrefixByteArray = 'A'; + private const char PrefixCompressedStringByteArray = 'C'; protected string ReadString(Stream stream) => PrimitiveSerializer.String.ReadFrom(stream); protected int ReadInt(Stream stream) => PrimitiveSerializer.Int32.ReadFrom(stream); @@ -30,7 +27,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource protected DateTime ReadDateTime(Stream stream) => PrimitiveSerializer.DateTime.ReadFrom(stream); protected byte[] ReadByteArray(Stream stream) => PrimitiveSerializer.Bytes.ReadFrom(stream); - private T? ReadObject(Stream stream, char t, Func read) + private T? ReadStruct(Stream stream, char t, Func read) where T : struct { var type = PrimitiveSerializer.Char.ReadFrom(stream); @@ -51,29 +48,19 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource : PrimitiveSerializer.String.ReadFrom(stream); } - protected int? ReadIntObject(Stream stream) => ReadObject(stream, PrefixInt32, ReadInt); - protected long? ReadLongObject(Stream stream) => ReadObject(stream, PrefixLong, ReadLong); - protected float? ReadFloatObject(Stream stream) => ReadObject(stream, PrefixFloat, ReadFloat); - protected double? ReadDoubleObject(Stream stream) => ReadObject(stream, PrefixDouble, ReadDouble); - protected DateTime? ReadDateTimeObject(Stream stream) => ReadObject(stream, PrefixDateTime, ReadDateTime); - - protected byte[] ReadByteArrayObject(Stream stream) // required 'cos byte[] is not a struct - { - var type = PrimitiveSerializer.Char.ReadFrom(stream); - if (type == PrefixNull) return null; - if (type != PrefixByteArray) - throw new NotSupportedException($"Cannot deserialize type '{type}', expected '{PrefixByteArray}'."); - return PrimitiveSerializer.Bytes.ReadFrom(stream); - } - + protected int? ReadIntObject(Stream stream) => ReadStruct(stream, PrefixInt32, ReadInt); + protected long? ReadLongObject(Stream stream) => ReadStruct(stream, PrefixLong, ReadLong); + protected float? ReadFloatObject(Stream stream) => ReadStruct(stream, PrefixFloat, ReadFloat); + protected double? ReadDoubleObject(Stream stream) => ReadStruct(stream, PrefixDouble, ReadDouble); + protected DateTime? ReadDateTimeObject(Stream stream) => ReadStruct(stream, PrefixDateTime, ReadDateTime); protected object ReadObject(Stream stream) => ReadObject(PrimitiveSerializer.Char.ReadFrom(stream), stream); protected object ReadObject(char type, Stream stream) { - // NOTE: There is going to be a ton of boxing going on here, but i'm not sure we can avoid that because innevitably with our - // current model structure the value will need to end up being 'object' at some point anyways. + // NOTE: This method is only called when reading property data, some boxing may occur but all other reads for structs are + // done with ReadStruct to reduce all boxing. switch (type) { @@ -98,8 +85,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource case PrefixDateTime: return PrimitiveSerializer.DateTime.ReadFrom(stream); case PrefixByteArray: - // When it's a byte array always return as a LazyCompressedString - // TODO: Else we need to make a different prefix for lazy vs eager loading + return PrimitiveSerializer.Bytes.ReadFrom(stream); + case PrefixCompressedStringByteArray: return new LazyCompressedString(PrimitiveSerializer.Bytes.ReadFrom(stream)); default: throw new NotSupportedException($"Cannot deserialize unknown type '{type}'."); @@ -108,6 +95,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource protected void WriteObject(object value, Stream stream) { + // NOTE: This method is only currently used to write 'string' information, all other writes are done directly with the PrimitiveSerializer + // so no boxing occurs. Though potentially we should write everything via this class just like we do for reads. + if (value == null) { PrimitiveSerializer.Char.WriteTo(PrefixNull, stream); @@ -164,7 +154,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } else if (value is LazyCompressedString lazyCompressedString) { - PrimitiveSerializer.Char.WriteTo(PrefixByteArray, stream); + PrimitiveSerializer.Char.WriteTo(PrefixCompressedStringByteArray, stream); PrimitiveSerializer.Bytes.WriteTo(lazyCompressedString.GetBytes(), stream); } else diff --git a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs index c098b516a0..c5109db027 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs @@ -3,6 +3,7 @@ using System.Configuration; using System.Linq; using Umbraco.Core; using Umbraco.Core.Composing; +using Umbraco.Web.PropertyEditors; using Umbraco.Web.PublishedCache.NuCache.DataSource; namespace Umbraco.Web.PublishedCache.NuCache @@ -14,36 +15,22 @@ namespace Umbraco.Web.PublishedCache.NuCache base.Compose(composition); var serializer = ConfigurationManager.AppSettings["Umbraco.Web.PublishedCache.NuCache.Serializer"]; - composition.Register(); - composition.Register(); - - // TODO: Based on our findings it seems like this should not be configurable, we should just be using this because it's better - if (serializer == "MsgPack") + if (serializer != "MsgPack") { - var propertyDictionarySerializer = ConfigurationManager.AppSettings["Umbraco.Web.PublishedCache.NuCache.DictionaryOfPropertiesSerializer"]; - if (propertyDictionarySerializer == "LZ4Map") - { - - composition.Register(factory => - { - var lz4Serializer = factory.GetInstance(); - return new ContentDataSerializer(lz4Serializer); - }); - } - else - { - composition.Register(factory => new ContentDataSerializer(new DictionaryOfPropertyDataSerializer())); - } - composition.Register(); + // TODO: This allows people to revert to the legacy serializer, by default it will be MessagePack + composition.RegisterUnique(); + composition.RegisterUnique(); } else { - composition.Register(); - composition.Register(factory => new ContentDataSerializer(new DictionaryOfPropertyDataSerializer())); + composition.RegisterUnique(); + composition.RegisterUnique(); } + + composition.RegisterUnique(factory => new ContentDataSerializer(new DictionaryOfPropertyDataSerializer())); // register the NuCache database data source - composition.Register(); + composition.RegisterUnique(); // register the NuCache published snapshot service // must register default options, required in the service ctor diff --git a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs index 0135f204c7..e670cb75f6 100755 --- a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs @@ -1459,8 +1459,8 @@ namespace Umbraco.Web.PublishedCache.NuCache { NodeId = content.Id, Published = published, - Data = !(_contentNestedDataSerializer is IContentNestedDataByteSerializer) ? _contentNestedDataSerializer.Serialize(nestedData) : null, - RawData = (_contentNestedDataSerializer is IContentNestedDataByteSerializer byteSerializer) ? byteSerializer.SerializeBytes(nestedData) : null + Data = !(_contentNestedDataSerializer is IContentNestedDataByteSerializer) ? _contentNestedDataSerializer.Serialize(content.ContentTypeId, nestedData) : null, + RawData = (_contentNestedDataSerializer is IContentNestedDataByteSerializer byteSerializer) ? byteSerializer.SerializeBytes(content.ContentTypeId, nestedData) : null }; //Core.Composing.Current.Logger.Debug(dto.Data); diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index fac5a90b0d..cd689664a1 100644 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -256,20 +256,16 @@ + + - - - - - - - +