diff --git a/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs b/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs new file mode 100644 index 0000000000..debfddef98 --- /dev/null +++ b/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs @@ -0,0 +1,103 @@ +using NUnit.Framework; +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Umbraco.Web.PublishedCache.NuCache.DataSource; + +namespace Umbraco.Tests.PublishedContent +{ + [TestFixture] + public class ContentSerializationTests + { + [Test] + public void Ensure_Same_Results() + { + var jsonSerializer = new JsonContentNestedDataSerializer(); + var msgPackSerializer = new MsgPackContentNestedDataSerializer(); + + var now = DateTime.Now; + var content = new ContentNestedData + { + PropertyData = new Dictionary + { + ["propertyOne"] = new[] + { + new PropertyData + { + Culture = "en-US", + Segment = "test", + Value = "hello world" + } + }, + ["propertyTwo"] = new[] + { + new PropertyData + { + Culture = "en-US", + Segment = "test", + Value = "Lorem ipsum" + } + } + }, + CultureData = new Dictionary + { + ["en-US"] = new CultureVariation + { + Date = now, + IsDraft = false, + Name = "Home", + UrlSegment = "home" + } + }, + UrlSegment = "home" + }; + + var json = jsonSerializer.Serialize(content); + var msgPack = msgPackSerializer.Serialize(content); + + Console.WriteLine(json); + Console.WriteLine(msgPackSerializer.ToJson(msgPack)); + + var jsonContent = jsonSerializer.Deserialize(json); + var msgPackContent = msgPackSerializer.Deserialize(msgPack); + + + CollectionAssert.AreEqual(jsonContent.CultureData.Keys, msgPackContent.CultureData.Keys); + CollectionAssert.AreEqual(jsonContent.PropertyData.Keys, msgPackContent.PropertyData.Keys); + CollectionAssert.AreEqual(jsonContent.CultureData.Values, msgPackContent.CultureData.Values, new CultureVariationComparer()); + CollectionAssert.AreEqual(jsonContent.PropertyData.Values, msgPackContent.PropertyData.Values, new PropertyDataComparer()); + Assert.AreEqual(jsonContent.UrlSegment, msgPackContent.UrlSegment); + } + + public class CultureVariationComparer : Comparer + { + public override int Compare(CultureVariation x, CultureVariation y) + { + if (x == null && y == null) return 0; + if (x == null && y != null) return -1; + if (x != null && y == null) return 1; + + return x.Date.CompareTo(y.Date) | x.IsDraft.CompareTo(y.IsDraft) | x.Name.CompareTo(y.Name) | x.UrlSegment.CompareTo(y.UrlSegment); + } + } + + public class PropertyDataComparer : Comparer + { + public override int Compare(PropertyData x, PropertyData y) + { + if (x == null && y == null) return 0; + if (x == null && y != null) return -1; + if (x != null && y == null) return 1; + + var xVal = x.Value?.ToString() ?? string.Empty; + var yVal = y.Value?.ToString() ?? string.Empty; + + return x.Culture.CompareTo(y.Culture) | x.Segment.CompareTo(y.Segment) | xVal.CompareTo(yVal); + } + } + + } +} diff --git a/src/Umbraco.Tests/Umbraco.Tests.csproj b/src/Umbraco.Tests/Umbraco.Tests.csproj index d2ee8e83df..c65110ab6b 100644 --- a/src/Umbraco.Tests/Umbraco.Tests.csproj +++ b/src/Umbraco.Tests/Umbraco.Tests.csproj @@ -109,7 +109,7 @@ - + @@ -149,6 +149,7 @@ + diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs index 4ef63c09fb..6635fa7090 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs @@ -1,13 +1,14 @@ using Newtonsoft.Json; -using System; using System.Collections.Generic; using System.Linq; +using System.Reflection.Emit; using System.Text; using System.Threading.Tasks; using Umbraco.Core.Serialization; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { + internal class JsonContentNestedDataSerializer : IContentNestedDataSerializer { public ContentNestedData Deserialize(string data) @@ -17,7 +18,13 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource var settings = new JsonSerializerSettings { - Converters = new List { new ForceInt32Converter() } + Converters = new List { new ForceInt32Converter() }, + + // Explicitly specify date handling so that it's consistent and follows the same date handling as MessagePack + DateParseHandling = DateParseHandling.DateTime, + DateFormatHandling = DateFormatHandling.IsoDateFormat, + DateTimeZoneHandling = DateTimeZoneHandling.Utc, + DateFormatString = "o" }; return JsonConvert.DeserializeObject(data, settings); @@ -25,6 +32,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource public string Serialize(ContentNestedData nestedData) { + // note that numeric values (which are Int32) are serialized without their + // type (eg "value":1234) and JsonConvert by default deserializes them as Int64 + return JsonConvert.SerializeObject(nestedData); } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs new file mode 100644 index 0000000000..8ff49a9544 --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs @@ -0,0 +1,42 @@ +using MessagePack; +using System; + +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + internal class MsgPackContentNestedDataSerializer : IContentNestedDataSerializer + { + private MessagePackSerializerOptions _options; + + public MsgPackContentNestedDataSerializer() + { + _options = MessagePack.Resolvers.ContractlessStandardResolver.Options.WithCompression(MessagePackCompression.Lz4BlockArray); + } + + public string ToJson(string serialized) + { + var bin = Convert.FromBase64String(serialized); + var json = MessagePackSerializer.ConvertToJson(bin, _options); + return json; + } + + // TODO: Instead of returning base64 it would be more ideal to avoid that translation entirely and just store/retrieve raw bytes + + // TODO: We need to write tests to serialize/deserialize between either of these serializers to ensure we end up with the same object + // i think this one is a bit quirky so far :) + + public ContentNestedData Deserialize(string data) + { + var bin = Convert.FromBase64String(data); + var obj = MessagePackSerializer.Deserialize(bin, _options); + return obj; + } + + public string Serialize(ContentNestedData nestedData) + { + var bin = MessagePackSerializer.Serialize( + nestedData, + _options); + return Convert.ToBase64String(bin); + } + } +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs index 6196be9a3a..0c89419b51 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs @@ -6,6 +6,16 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource { internal abstract class SerializerBase { + private const char PrefixNull = 'N'; + private const char PrefixString = 'S'; + private const char PrefixInt32 = 'I'; + private const char PrefixUInt16 = 'H'; + private const char PrefixLong = 'L'; + private const char PrefixFloat = 'F'; + private const char PrefixDouble = 'B'; + private const char PrefixDateTime = 'D'; + private const char PrefixByte = 'O'; + protected string ReadString(Stream stream) => PrimitiveSerializer.String.ReadFrom(stream); protected int ReadInt(Stream stream) => PrimitiveSerializer.Int32.ReadFrom(stream); protected long ReadLong(Stream stream) => PrimitiveSerializer.Int64.ReadFrom(stream); @@ -17,7 +27,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource where T : struct { var type = PrimitiveSerializer.Char.ReadFrom(stream); - if (type == 'N') return null; + if (type == PrefixNull) return null; if (type != t) throw new NotSupportedException($"Cannot deserialize type '{type}', expected '{t}'."); return read(stream); @@ -26,40 +36,47 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource protected string ReadStringObject(Stream stream, bool intern = false) // required 'cos string is not a struct { var type = PrimitiveSerializer.Char.ReadFrom(stream); - if (type == 'N') return null; - if (type != 'S') + if (type == PrefixNull) return null; + if (type != PrefixString) throw new NotSupportedException($"Cannot deserialize type '{type}', expected 'S'."); return intern ? string.Intern(PrimitiveSerializer.String.ReadFrom(stream)) : PrimitiveSerializer.String.ReadFrom(stream); } - protected int? ReadIntObject(Stream stream) => ReadObject(stream, 'I', ReadInt); - protected long? ReadLongObject(Stream stream) => ReadObject(stream, 'L', ReadLong); - protected float? ReadFloatObject(Stream stream) => ReadObject(stream, 'F', ReadFloat); - protected double? ReadDoubleObject(Stream stream) => ReadObject(stream, 'B', ReadDouble); - protected DateTime? ReadDateTimeObject(Stream stream) => ReadObject(stream, 'D', ReadDateTime); + protected int? ReadIntObject(Stream stream) => ReadObject(stream, PrefixInt32, ReadInt); + protected long? ReadLongObject(Stream stream) => ReadObject(stream, PrefixLong, ReadLong); + protected float? ReadFloatObject(Stream stream) => ReadObject(stream, PrefixFloat, ReadFloat); + protected double? ReadDoubleObject(Stream stream) => ReadObject(stream, PrefixDouble, ReadDouble); + protected DateTime? ReadDateTimeObject(Stream stream) => ReadObject(stream, PrefixDateTime, ReadDateTime); protected object ReadObject(Stream stream) => ReadObject(PrimitiveSerializer.Char.ReadFrom(stream), stream); protected object ReadObject(char type, Stream stream) { + // NOTE: There is going to be a ton of boxing going on here, but i'm not sure we can avoid that because innevitably with our + // current model structure the value will need to end up being 'object' at some point anyways. + switch (type) { - case 'N': + case PrefixNull: return null; - case 'S': + case PrefixString: return PrimitiveSerializer.String.ReadFrom(stream); - case 'I': + case PrefixInt32: return PrimitiveSerializer.Int32.ReadFrom(stream); - case 'L': + case PrefixUInt16: + return PrimitiveSerializer.UInt16.ReadFrom(stream); + case PrefixByte: + return PrimitiveSerializer.Byte.ReadFrom(stream); + case PrefixLong: return PrimitiveSerializer.Int64.ReadFrom(stream); - case 'F': + case PrefixFloat: return PrimitiveSerializer.Float.ReadFrom(stream); - case 'B': + case PrefixDouble: return PrimitiveSerializer.Double.ReadFrom(stream); - case 'D': + case PrefixDateTime: return PrimitiveSerializer.DateTime.ReadFrom(stream); default: throw new NotSupportedException($"Cannot deserialize unknown type '{type}'."); @@ -70,36 +87,46 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource { if (value == null) { - PrimitiveSerializer.Char.WriteTo('N', stream); + PrimitiveSerializer.Char.WriteTo(PrefixNull, stream); } else if (value is string stringValue) { - PrimitiveSerializer.Char.WriteTo('S', stream); + PrimitiveSerializer.Char.WriteTo(PrefixString, stream); PrimitiveSerializer.String.WriteTo(stringValue, stream); } else if (value is int intValue) { - PrimitiveSerializer.Char.WriteTo('I', stream); + PrimitiveSerializer.Char.WriteTo(PrefixInt32, stream); PrimitiveSerializer.Int32.WriteTo(intValue, stream); } + else if (value is byte byteValue) + { + PrimitiveSerializer.Char.WriteTo(PrefixByte, stream); + PrimitiveSerializer.Byte.WriteTo(byteValue, stream); + } + else if (value is ushort ushortValue) + { + PrimitiveSerializer.Char.WriteTo(PrefixUInt16, stream); + PrimitiveSerializer.UInt16.WriteTo(ushortValue, stream); + } else if (value is long longValue) { - PrimitiveSerializer.Char.WriteTo('L', stream); + PrimitiveSerializer.Char.WriteTo(PrefixLong, stream); PrimitiveSerializer.Int64.WriteTo(longValue, stream); } else if (value is float floatValue) { - PrimitiveSerializer.Char.WriteTo('F', stream); + PrimitiveSerializer.Char.WriteTo(PrefixFloat, stream); PrimitiveSerializer.Float.WriteTo(floatValue, stream); } else if (value is double doubleValue) { - PrimitiveSerializer.Char.WriteTo('B', stream); + PrimitiveSerializer.Char.WriteTo(PrefixDouble, stream); PrimitiveSerializer.Double.WriteTo(doubleValue, stream); } else if (value is DateTime dateValue) { - PrimitiveSerializer.Char.WriteTo('D', stream); + PrimitiveSerializer.Char.WriteTo(PrefixDateTime, stream); PrimitiveSerializer.DateTime.WriteTo(dateValue, stream); } else diff --git a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs index f67256bb6b..6d01b34a76 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs @@ -11,7 +11,8 @@ namespace Umbraco.Web.PublishedCache.NuCache base.Compose(composition); // register the NuCache NestedContentData serializer - composition.Register(); + //composition.Register(); + composition.Register(); // register the NuCache database data source composition.Register(); diff --git a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs index 5ce62271a7..6289667d3b 100755 --- a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs @@ -1456,10 +1456,6 @@ namespace Umbraco.Web.PublishedCache.NuCache { NodeId = content.Id, Published = published, - - // note that numeric values (which are Int32) are serialized without their - // type (eg "value":1234) and JsonConvert by default deserializes them as Int64 - Data = _contentNestedDataSerializer.Serialize(nestedData) }; diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index 01f81cfc83..088fb0eeb3 100755 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -73,6 +73,9 @@ + + 2.1.152 + @@ -247,6 +250,7 @@ + @@ -1266,7 +1270,7 @@ - +