yv01p/Umbraco-CMS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

This gets things working with all compression levels - going to cleanup/simplify

Browse Source
This commit is contained in:
Shannon
2020-08-26 11:43:43 +10:00
parent 8ab5899db2
commit ba8da3850e
6 changed files with 88 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs
View File

@@ -1,10 +1,7 @@
using K4os.Compression.LZ4;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace Umbraco.Web.PublishedCache.NuCache.DataSource
{
@@ -13,10 +10,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
/// </summary>
internal class LazyCompressedString
{
// TODO: This could be a struct
private byte[] _bytes;
private string _str;
private byte[] _bytes;
/// <summary>
/// Constructor
@@ -27,6 +22,13 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
_bytes = bytes;
}
public byte[] GetBytes()
{
if (_bytes == null)
throw new InvalidOperationException("The bytes have already been expanded");
return _bytes;
}
public override string ToString()
{
return LazyInitializer.EnsureInitialized(ref _str, () =>

6
src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs
View File

@@ -88,8 +88,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
break;
}
}
break;
}
break;
}
}
dict[key] = pdatas.ToArray();
@@ -122,7 +122,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
WriteObject(pdata.Culture ?? string.Empty, stream);
WriteObject(pdata.Segment ?? string.Empty, stream);
//Only compress strings
//Only compress strings (if NucachePropertyCompressionLevel.SQLDatabase has been specified then the property value will already be compressed)
switch (map.CompressLevel)
{
// If we're compressing into btree at the property level

76
src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs
View File

@@ -50,23 +50,36 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
public ContentNestedData Deserialize(string data)
{
var bin = Convert.FromBase64String(data);
var obj = MessagePackSerializer.Deserialize<ContentNestedData>(bin, _options);
return obj;
var nestedData = MessagePackSerializer.Deserialize<ContentNestedData>(bin, _options);
Expand(nestedData);
return nestedData;
}
public string Serialize(ContentNestedData nestedData)
{
Optimize(nestedData);
Compress(nestedData);
var bin = MessagePackSerializer.Serialize(nestedData, _options);
return Convert.ToBase64String(bin);
}
public ContentNestedData DeserializeBytes(byte[] data)
{
var nestedData = MessagePackSerializer.Deserialize<ContentNestedData>(data, _options);
Expand(nestedData);
return nestedData;
}
public byte[] SerializeBytes(ContentNestedData nestedData)
{
Compress(nestedData);
return MessagePackSerializer.Serialize(nestedData, _options);
}
/// <summary>
/// Compress properties and map property names to shorter names
/// Used during serialization to compress properties and map property names to shorter names
/// </summary>
/// <param name="nestedData"></param>
private void Optimize(ContentNestedData nestedData)
private void Compress(ContentNestedData nestedData)
{
if (_propertyOptions.PropertyMap != null && _propertyOptions.PropertyMap.Count > 0)
{
@@ -95,13 +108,54 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
}
}
public ContentNestedData DeserializeBytes(byte[] data) => MessagePackSerializer.Deserialize<ContentNestedData>(data, _options);
public byte[] SerializeBytes(ContentNestedData nestedData)
/// <summary>
/// Used during deserialization to map the property data as lazy or expand the value and re-map back to the true property aliases
/// </summary>
/// <param name="nestedData"></param>
private void Expand(ContentNestedData nestedData)
{
Optimize(nestedData);
if (_propertyOptions.PropertyMap != null && _propertyOptions.PropertyMap.Count > 0)
{
foreach (var map in _propertyOptions.PropertyMap)
{
if (map.Value.CompressLevel.Equals(NucachePropertyCompressionLevel.SQLDatabase))
{
// if there is an alias map for this property then re-map to the real property alias
if (map.Value.MappedAlias != null && !map.Key.Equals(map.Value.MappedAlias)
&& nestedData.PropertyData.TryGetValue(map.Value.MappedAlias, out PropertyData[] properties2))
{
nestedData.PropertyData.Remove(map.Value.MappedAlias);
nestedData.PropertyData.Add(map.Key, properties2);
}
return MessagePackSerializer.Serialize(nestedData, _options);
if (nestedData.PropertyData.TryGetValue(map.Key, out PropertyData[] properties))
{
foreach (var pdata in properties)
{
if (!(pdata.Value is null) && pdata.Value is byte[] byteArrayValue)
{
//Compressed string
switch (map.Value.DecompressLevel)
{
case NucachePropertyDecompressionLevel.Lazy:
pdata.Value = new LazyCompressedString(byteArrayValue);
break;
case NucachePropertyDecompressionLevel.NotCompressed:
//Shouldn't be any not compressed
throw new InvalidOperationException($"{NucachePropertyDecompressionLevel.NotCompressed} cannot be a decompression option for property {map.Key} since it's compresion option is {map.Value.CompressLevel}");
case NucachePropertyDecompressionLevel.Immediate:
default:
pdata.Value = Encoding.UTF8.GetString(LZ4Pickler.Unpickle(byteArrayValue));
break;
}
}
}
}
}
}
}
}
//private class ContentNestedDataResolver : IFormatterResolver

1
src/Umbraco.Web/PublishedCache/NuCache/DataSource/NuCachePropertyCompressionOptions.cs
View File

@@ -27,6 +27,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
public LZ4Level LZ4CompressionLevel { get; } = LZ4Level.L00_FAST;
// TODO: Unsure if we really want to keep this
public long? MinimumCompressibleStringLength { get; }
}
}

3
src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyDecompressionLevel.cs
View File

@@ -6,7 +6,10 @@
public enum NucachePropertyDecompressionLevel
{
NotCompressed = 0,
// TODO: I'm unsure if this will ever be necessary, lazy seems good and deserialization would only occur once
Immediate = 1,
Lazy = 2
}
}

9
src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs
View File

@@ -98,7 +98,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
case PrefixDateTime:
return PrimitiveSerializer.DateTime.ReadFrom(stream);
case PrefixByteArray:
return PrimitiveSerializer.Bytes.ReadFrom(stream);
// When it's a byte array always return as a LazyCompressedString
// TODO: Else we need to make a different prefix for lazy vs eager loading
return new LazyCompressedString(PrimitiveSerializer.Bytes.ReadFrom(stream));
default:
throw new NotSupportedException($"Cannot deserialize unknown type '{type}'.");
}
@@ -160,6 +162,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
PrimitiveSerializer.Char.WriteTo(PrefixByteArray, stream);
PrimitiveSerializer.Bytes.WriteTo(byteArrayValue, stream);
}
else if (value is LazyCompressedString lazyCompressedString)
{
PrimitiveSerializer.Char.WriteTo(PrefixByteArray, stream);
PrimitiveSerializer.Bytes.WriteTo(lazyCompressedString.GetBytes(), stream);
}
else
throw new NotSupportedException("Value type " + value.GetType().FullName + " cannot be serialized.");
}