yv01p/Umbraco-CMS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Adds MessagePack serialization for nucache

Browse Source
This commit is contained in:
Shannon
2020-07-03 12:11:05 +10:00
parent e2ab2d2798
commit c63bfb866b
8 changed files with 215 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

103
src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs Normal file
View File

@@ -0,0 +1,103 @@
using NUnit.Framework;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Umbraco.Web.PublishedCache.NuCache.DataSource;
namespace Umbraco.Tests.PublishedContent
{
[TestFixture]
public class ContentSerializationTests
{
[Test]
public void Ensure_Same_Results()
{
var jsonSerializer = new JsonContentNestedDataSerializer();
var msgPackSerializer = new MsgPackContentNestedDataSerializer();
var now = DateTime.Now;
var content = new ContentNestedData
{
PropertyData = new Dictionary<string, PropertyData[]>
{
["propertyOne"] = new[]
{
new PropertyData
{
Culture = "en-US",
Segment = "test",
Value = "hello world"
}
},
["propertyTwo"] = new[]
{
new PropertyData
{
Culture = "en-US",
Segment = "test",
Value = "Lorem ipsum"
}
}
},
CultureData = new Dictionary<string, CultureVariation>
{
["en-US"] = new CultureVariation
{
Date = now,
IsDraft = false,
Name = "Home",
UrlSegment = "home"
}
},
UrlSegment = "home"
};
var json = jsonSerializer.Serialize(content);
var msgPack = msgPackSerializer.Serialize(content);
Console.WriteLine(json);
Console.WriteLine(msgPackSerializer.ToJson(msgPack));
var jsonContent = jsonSerializer.Deserialize(json);
var msgPackContent = msgPackSerializer.Deserialize(msgPack);
CollectionAssert.AreEqual(jsonContent.CultureData.Keys, msgPackContent.CultureData.Keys);
CollectionAssert.AreEqual(jsonContent.PropertyData.Keys, msgPackContent.PropertyData.Keys);
CollectionAssert.AreEqual(jsonContent.CultureData.Values, msgPackContent.CultureData.Values, new CultureVariationComparer());
CollectionAssert.AreEqual(jsonContent.PropertyData.Values, msgPackContent.PropertyData.Values, new PropertyDataComparer());
Assert.AreEqual(jsonContent.UrlSegment, msgPackContent.UrlSegment);
}
public class CultureVariationComparer : Comparer<CultureVariation>
{
public override int Compare(CultureVariation x, CultureVariation y)
{
if (x == null && y == null) return 0;
if (x == null && y != null) return -1;
if (x != null && y == null) return 1;
return x.Date.CompareTo(y.Date) | x.IsDraft.CompareTo(y.IsDraft) | x.Name.CompareTo(y.Name) | x.UrlSegment.CompareTo(y.UrlSegment);
}
}
public class PropertyDataComparer : Comparer<PropertyData>
{
public override int Compare(PropertyData x, PropertyData y)
{
if (x == null && y == null) return 0;
if (x == null && y != null) return -1;
if (x != null && y == null) return 1;
var xVal = x.Value?.ToString() ?? string.Empty;
var yVal = y.Value?.ToString() ?? string.Empty;
return x.Culture.CompareTo(y.Culture) | x.Segment.CompareTo(y.Segment) | xVal.CompareTo(yVal);
}
}
}
}

3
src/Umbraco.Tests/Umbraco.Tests.csproj
View File

@@ -109,7 +109,7 @@
<PackageReference Include="Selenium.WebDriver" Version="3.141.0" />
<PackageReference Include="Semver" Version="2.0.4" />
<PackageReference Include="Umbraco.SqlServerCE" Version="4.0.0.1" />
<PackageReference Include="System.Threading.Tasks.Extensions" Version="4.5.2" />
<PackageReference Include="System.Threading.Tasks.Extensions" Version="4.5.4" />
</ItemGroup>
<ItemGroup>
<Compile Include="Cache\DistributedCacheBinderTests.cs" />
@@ -149,6 +149,7 @@
<Compile Include="Persistence\Repositories\EntityRepositoryTest.cs" />
<Compile Include="PropertyEditors\DataValueReferenceFactoryCollectionTests.cs" />
<Compile Include="PropertyEditors\NestedContentPropertyComponentTests.cs" />
<Compile Include="PublishedContent\ContentSerializationTests.cs" />
<Compile Include="PublishedContent\NuCacheChildrenTests.cs" />
<Compile Include="PublishedContent\PublishedContentLanguageVariantTests.cs" />
<Compile Include="PublishedContent\PublishedContentSnapshotTestBase.cs" />

14
src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs
View File

@@ -1,13 +1,14 @@
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Reflection.Emit;
using System.Text;
using System.Threading.Tasks;
using Umbraco.Core.Serialization;
namespace Umbraco.Web.PublishedCache.NuCache.DataSource
{
internal class JsonContentNestedDataSerializer : IContentNestedDataSerializer
{
public ContentNestedData Deserialize(string data)
@@ -17,7 +18,13 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
var settings = new JsonSerializerSettings
{
Converters = new List<JsonConverter> { new ForceInt32Converter() }
Converters = new List<JsonConverter> { new ForceInt32Converter() },
// Explicitly specify date handling so that it's consistent and follows the same date handling as MessagePack
DateParseHandling = DateParseHandling.DateTime,
DateFormatHandling = DateFormatHandling.IsoDateFormat,
DateTimeZoneHandling = DateTimeZoneHandling.Utc,
DateFormatString = "o"
};
return JsonConvert.DeserializeObject<ContentNestedData>(data, settings);
@@ -25,6 +32,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
public string Serialize(ContentNestedData nestedData)
{
// note that numeric values (which are Int32) are serialized without their
// type (eg "value":1234) and JsonConvert by default deserializes them as Int64
return JsonConvert.SerializeObject(nestedData);
}
}

42
src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs Normal file
View File

@@ -0,0 +1,42 @@
using MessagePack;
using System;
namespace Umbraco.Web.PublishedCache.NuCache.DataSource
{
internal class MsgPackContentNestedDataSerializer : IContentNestedDataSerializer
{
private MessagePackSerializerOptions _options;
public MsgPackContentNestedDataSerializer()
{
_options = MessagePack.Resolvers.ContractlessStandardResolver.Options.WithCompression(MessagePackCompression.Lz4BlockArray);
}
public string ToJson(string serialized)
{
var bin = Convert.FromBase64String(serialized);
var json = MessagePackSerializer.ConvertToJson(bin, _options);
return json;
}
// TODO: Instead of returning base64 it would be more ideal to avoid that translation entirely and just store/retrieve raw bytes
// TODO: We need to write tests to serialize/deserialize between either of these serializers to ensure we end up with the same object
// i think this one is a bit quirky so far :)
public ContentNestedData Deserialize(string data)
{
var bin = Convert.FromBase64String(data);
var obj = MessagePackSerializer.Deserialize<ContentNestedData>(bin, _options);
return obj;
}
public string Serialize(ContentNestedData nestedData)
{
var bin = MessagePackSerializer.Serialize(
nestedData,
_options);
return Convert.ToBase64String(bin);
}
}
}

71
src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs
View File

@@ -6,6 +6,16 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
{
internal abstract class SerializerBase
{
private const char PrefixNull = 'N';
private const char PrefixString = 'S';
private const char PrefixInt32 = 'I';
private const char PrefixUInt16 = 'H';
private const char PrefixLong = 'L';
private const char PrefixFloat = 'F';
private const char PrefixDouble = 'B';
private const char PrefixDateTime = 'D';
private const char PrefixByte = 'O';
protected string ReadString(Stream stream) => PrimitiveSerializer.String.ReadFrom(stream);
protected int ReadInt(Stream stream) => PrimitiveSerializer.Int32.ReadFrom(stream);
protected long ReadLong(Stream stream) => PrimitiveSerializer.Int64.ReadFrom(stream);
@@ -17,7 +27,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
where T : struct
{
var type = PrimitiveSerializer.Char.ReadFrom(stream);
if (type == 'N') return null;
if (type == PrefixNull) return null;
if (type != t)
throw new NotSupportedException($"Cannot deserialize type '{type}', expected '{t}'.");
return read(stream);
@@ -26,40 +36,47 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
protected string ReadStringObject(Stream stream, bool intern = false) // required 'cos string is not a struct
{
var type = PrimitiveSerializer.Char.ReadFrom(stream);
if (type == 'N') return null;
if (type != 'S')
if (type == PrefixNull) return null;
if (type != PrefixString)
throw new NotSupportedException($"Cannot deserialize type '{type}', expected 'S'.");
return intern
? string.Intern(PrimitiveSerializer.String.ReadFrom(stream))
: PrimitiveSerializer.String.ReadFrom(stream);
}
protected int? ReadIntObject(Stream stream) => ReadObject(stream, 'I', ReadInt);
protected long? ReadLongObject(Stream stream) => ReadObject(stream, 'L', ReadLong);
protected float? ReadFloatObject(Stream stream) => ReadObject(stream, 'F', ReadFloat);
protected double? ReadDoubleObject(Stream stream) => ReadObject(stream, 'B', ReadDouble);
protected DateTime? ReadDateTimeObject(Stream stream) => ReadObject(stream, 'D', ReadDateTime);
protected int? ReadIntObject(Stream stream) => ReadObject(stream, PrefixInt32, ReadInt);
protected long? ReadLongObject(Stream stream) => ReadObject(stream, PrefixLong, ReadLong);
protected float? ReadFloatObject(Stream stream) => ReadObject(stream, PrefixFloat, ReadFloat);
protected double? ReadDoubleObject(Stream stream) => ReadObject(stream, PrefixDouble, ReadDouble);
protected DateTime? ReadDateTimeObject(Stream stream) => ReadObject(stream, PrefixDateTime, ReadDateTime);
protected object ReadObject(Stream stream)
=> ReadObject(PrimitiveSerializer.Char.ReadFrom(stream), stream);
protected object ReadObject(char type, Stream stream)
{
// NOTE: There is going to be a ton of boxing going on here, but i'm not sure we can avoid that because innevitably with our
// current model structure the value will need to end up being 'object' at some point anyways.
switch (type)
{
case 'N':
case PrefixNull:
return null;
case 'S':
case PrefixString:
return PrimitiveSerializer.String.ReadFrom(stream);
case 'I':
case PrefixInt32:
return PrimitiveSerializer.Int32.ReadFrom(stream);
case 'L':
case PrefixUInt16:
return PrimitiveSerializer.UInt16.ReadFrom(stream);
case PrefixByte:
return PrimitiveSerializer.Byte.ReadFrom(stream);
case PrefixLong:
return PrimitiveSerializer.Int64.ReadFrom(stream);
case 'F':
case PrefixFloat:
return PrimitiveSerializer.Float.ReadFrom(stream);
case 'B':
case PrefixDouble:
return PrimitiveSerializer.Double.ReadFrom(stream);
case 'D':
case PrefixDateTime:
return PrimitiveSerializer.DateTime.ReadFrom(stream);
default:
throw new NotSupportedException($"Cannot deserialize unknown type '{type}'.");
@@ -70,36 +87,46 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
{
if (value == null)
{
PrimitiveSerializer.Char.WriteTo('N', stream);
PrimitiveSerializer.Char.WriteTo(PrefixNull, stream);
}
else if (value is string stringValue)
{
PrimitiveSerializer.Char.WriteTo('S', stream);
PrimitiveSerializer.Char.WriteTo(PrefixString, stream);
PrimitiveSerializer.String.WriteTo(stringValue, stream);
}
else if (value is int intValue)
{
PrimitiveSerializer.Char.WriteTo('I', stream);
PrimitiveSerializer.Char.WriteTo(PrefixInt32, stream);
PrimitiveSerializer.Int32.WriteTo(intValue, stream);
}
else if (value is byte byteValue)
{
PrimitiveSerializer.Char.WriteTo(PrefixByte, stream);
PrimitiveSerializer.Byte.WriteTo(byteValue, stream);
}
else if (value is ushort ushortValue)
{
PrimitiveSerializer.Char.WriteTo(PrefixUInt16, stream);
PrimitiveSerializer.UInt16.WriteTo(ushortValue, stream);
}
else if (value is long longValue)
{
PrimitiveSerializer.Char.WriteTo('L', stream);
PrimitiveSerializer.Char.WriteTo(PrefixLong, stream);
PrimitiveSerializer.Int64.WriteTo(longValue, stream);
}
else if (value is float floatValue)
{
PrimitiveSerializer.Char.WriteTo('F', stream);
PrimitiveSerializer.Char.WriteTo(PrefixFloat, stream);
PrimitiveSerializer.Float.WriteTo(floatValue, stream);
}
else if (value is double doubleValue)
{
PrimitiveSerializer.Char.WriteTo('B', stream);
PrimitiveSerializer.Char.WriteTo(PrefixDouble, stream);
PrimitiveSerializer.Double.WriteTo(doubleValue, stream);
}
else if (value is DateTime dateValue)
{
PrimitiveSerializer.Char.WriteTo('D', stream);
PrimitiveSerializer.Char.WriteTo(PrefixDateTime, stream);
PrimitiveSerializer.DateTime.WriteTo(dateValue, stream);
}
else

3
src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs
View File

@@ -11,7 +11,8 @@ namespace Umbraco.Web.PublishedCache.NuCache
base.Compose(composition);
// register the NuCache NestedContentData serializer
composition.Register<IContentNestedDataSerializer, JsonContentNestedDataSerializer>();
//composition.Register<IContentNestedDataSerializer, JsonContentNestedDataSerializer>();
composition.Register<IContentNestedDataSerializer, MsgPackContentNestedDataSerializer>();
// register the NuCache database data source
composition.Register<IDataSource, DatabaseDataSource>();

4
src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs
View File

@@ -1456,10 +1456,6 @@ namespace Umbraco.Web.PublishedCache.NuCache
{
NodeId = content.Id,
Published = published,
// note that numeric values (which are Int32) are serialized without their
// type (eg "value":1234) and JsonConvert by default deserializes them as Int64
Data = _contentNestedDataSerializer.Serialize(nestedData)
};

6
src/Umbraco.Web/Umbraco.Web.csproj
View File

@@ -73,6 +73,9 @@
<PackageReference Include="LightInject.Mvc" Version="2.0.0" />
<PackageReference Include="LightInject.WebApi" Version="2.0.0" />
<PackageReference Include="Markdown" Version="2.2.1" />
<PackageReference Include="MessagePack">
<Version>2.1.152</Version>
</PackageReference>
<PackageReference Include="Microsoft.AspNet.Identity.Owin" Version="2.2.2" />
<PackageReference Include="Microsoft.AspNet.Mvc" Version="5.2.7" />
<PackageReference Include="Microsoft.AspNet.SignalR.Core" Version="2.4.0" />
@@ -247,6 +250,7 @@
<Compile Include="PropertyEditors\RichTextEditorPastedImages.cs" />
<Compile Include="PublishedCache\NuCache\DataSource\IContentNestedDataSerializer.cs" />
<Compile Include="PublishedCache\NuCache\DataSource\JsonContentNestedDataSerializer.cs" />
<Compile Include="PublishedCache\NuCache\DataSource\MsgPackContentNestedDataSerializer.cs" />
<Compile Include="PublishedCache\NuCache\PublishedSnapshotServiceOptions.cs" />
<Compile Include="PublishedCache\NuCache\Snap\GenObj.cs" />
<Compile Include="PublishedCache\NuCache\Snap\GenRef.cs" />
@@ -1266,7 +1270,7 @@
</PropertyGroup>
<ItemGroup>
<!-- we want to exclude all facade references ?! -->
<FixedReferencePath Include="@(ReferencePath)" Condition="'%(ReferencePath.FileName)' != 'System.ValueTuple' and '%(ReferencePath.FileName)' != 'System.Net.Http'" />
<FixedReferencePath Include="@(ReferencePath)" Condition="'%(ReferencePath.FileName)' != 'System.ValueTuple' and '%(ReferencePath.FileName)' != 'System.Net.Http' and '%(ReferencePath.FileName)' != 'Microsoft.Bcl.AsyncInterfaces' and '%(ReferencePath.FileName)' != 'System.Buffers' and '%(ReferencePath.FileName)' != 'System.Numerics.Vectors' and '%(ReferencePath.FileName)' != 'System.Runtime.CompilerServices.Unsafe'" />
</ItemGroup>
<Delete Files="$(TargetDir)$(TargetName).XmlSerializers.dll" ContinueOnError="true" />
<!--