diff --git a/src/Umbraco.Tests/Umbraco.Tests.csproj b/src/Umbraco.Tests/Umbraco.Tests.csproj index c65110ab6b..5518fb5678 100644 --- a/src/Umbraco.Tests/Umbraco.Tests.csproj +++ b/src/Umbraco.Tests/Umbraco.Tests.csproj @@ -100,7 +100,7 @@ - + diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.ContentDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.ContentDataSerializer.cs index 056eacd717..d02af375c6 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.ContentDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.ContentDataSerializer.cs @@ -19,8 +19,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource VersionDate = PrimitiveSerializer.DateTime.ReadFrom(stream), WriterId = PrimitiveSerializer.Int32.ReadFrom(stream), TemplateId = PrimitiveSerializer.Int32.ReadFrom(stream), - Properties = PropertiesSerializer.ReadFrom(stream), - CultureInfos = CultureVariationsSerializer.ReadFrom(stream) + Properties = PropertiesSerializer.ReadFrom(stream), // TODO: We don't want to allocate empty arrays + CultureInfos = CultureVariationsSerializer.ReadFrom(stream) // TODO: We don't want to allocate empty arrays }; } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentNestedData.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentNestedData.cs index 413ab4ab63..0ff24980fe 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentNestedData.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentNestedData.cs @@ -1,5 +1,7 @@ -using Newtonsoft.Json; +using MessagePack; +using Newtonsoft.Json; using System.Collections.Generic; +using System.Runtime.Serialization; using Umbraco.Core.Serialization; namespace Umbraco.Web.PublishedCache.NuCache.DataSource @@ -7,29 +9,37 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource /// /// The content item 1:M data that is serialized to JSON /// + [DataContract] // NOTE: Use DataContract annotations here to control how MessagePack serializes/deserializes the data to use INT keys public class ContentNestedData { + // TODO: We don't want to allocate empty arrays //dont serialize empty properties + [DataMember(Order = 0)] [JsonProperty("pd")] [JsonConverter(typeof(AutoInterningStringKeyCaseInsensitiveDictionaryConverter))] public Dictionary PropertyData { get; set; } + [DataMember(Order = 1)] [JsonProperty("cd")] [JsonConverter(typeof(AutoInterningStringKeyCaseInsensitiveDictionaryConverter))] public Dictionary CultureData { get; set; } + [DataMember(Order = 2)] [JsonProperty("us")] public string UrlSegment { get; set; } //Legacy properties used to deserialize existing nucache db entries + [DataMember(Order = 3)] [JsonProperty("properties")] [JsonConverter(typeof(CaseInsensitiveDictionaryConverter))] private Dictionary LegacyPropertyData { set { PropertyData = value; } } + [DataMember(Order = 4)] [JsonProperty("cultureData")] [JsonConverter(typeof(CaseInsensitiveDictionaryConverter))] private Dictionary LegacyCultureData { set { CultureData = value; } } + [DataMember(Order = 5)] [JsonProperty("urlSegment")] private string LegacyUrlSegment { set { UrlSegment = value; } } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/CultureVariation.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/CultureVariation.cs index b59e8c403c..dd3323fa0c 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/CultureVariation.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/CultureVariation.cs @@ -1,4 +1,5 @@ using System; +using System.Runtime.Serialization; using Newtonsoft.Json; namespace Umbraco.Web.PublishedCache.NuCache.DataSource @@ -6,30 +7,39 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource /// /// Represents the culture variation information on a content item /// + [DataContract] // NOTE: Use DataContract annotations here to control how MessagePack serializes/deserializes the data to use INT keys public class CultureVariation { + [DataMember(Order = 0)] [JsonProperty("nm")] public string Name { get; set; } + [DataMember(Order = 1)] [JsonProperty("us")] public string UrlSegment { get; set; } + [DataMember(Order = 2)] [JsonProperty("dt")] public DateTime Date { get; set; } + [DataMember(Order = 3)] [JsonProperty("isd")] public bool IsDraft { get; set; } //Legacy properties used to deserialize existing nucache db entries + [DataMember(Order = 4)] [JsonProperty("name")] private string LegacyName { set { Name = value; } } + [DataMember(Order = 5)] [JsonProperty("urlSegment")] private string LegacyUrlSegment { set { UrlSegment = value; } } + [DataMember(Order = 6)] [JsonProperty("date")] private DateTime LegacyDate { set { Date = value; } } + [DataMember(Order = 7)] [JsonProperty("isDraft")] private bool LegacyIsDraft { set { IsDraft = value; } } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs index 80cfabd470..e39f649eaf 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs @@ -20,12 +20,10 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource // provides efficient database access for NuCache internal class DatabaseDataSource : IDataSource { - - private const int PageSize = 500; private readonly IContentNestedDataSerializer _contentNestedDataSerializer; - internal DatabaseDataSource(IContentNestedDataSerializer contentNestedDataSerializer) + public DatabaseDataSource(IContentNestedDataSerializer contentNestedDataSerializer) { _contentNestedDataSerializer = contentNestedDataSerializer; } @@ -231,7 +229,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource VersionId = dto.VersionId, VersionDate = dto.EditVersionDate, WriterId = dto.EditWriterId, - Properties = nested.PropertyData, + Properties = nested.PropertyData, // TODO: We don't want to allocate empty arrays CultureInfos = nested.CultureData, UrlSegment = nested.UrlSegment }; @@ -259,7 +257,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource VersionId = dto.VersionId, VersionDate = dto.PubVersionDate, WriterId = dto.PubWriterId, - Properties = nested.PropertyData, + Properties = nested.PropertyData, // TODO: We don't want to allocate empty arrays CultureInfos = nested.CultureData }; } @@ -294,7 +292,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource VersionId = dto.VersionId, VersionDate = dto.EditVersionDate, WriterId = dto.CreatorId, // what-else? - Properties = nested.PropertyData, + Properties = nested.PropertyData, // TODO: We don't want to allocate empty arrays CultureInfos = nested.CultureData }; diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs index 8ff49a9544..99a306fecb 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs @@ -1,5 +1,8 @@ using MessagePack; +using MessagePack.Formatters; +using MessagePack.Resolvers; using System; +using System.Collections.Generic; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { @@ -9,7 +12,24 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource public MsgPackContentNestedDataSerializer() { - _options = MessagePack.Resolvers.ContractlessStandardResolver.Options.WithCompression(MessagePackCompression.Lz4BlockArray); + var defaultOptions = ContractlessStandardResolver.Options; + + var resolver = CompositeResolver.Create( + + // TODO: We want to be able to intern the strings for aliases when deserializing like we do for Newtonsoft but I'm unsure exactly how + // to do that but it would seem to be with a custom message pack resolver but I haven't quite figured out based on the docs how + // to do that since that is part of the int key -> string mapping operation, might have to see the source code to figure that one out. + + // resolver custom types first + // new ContentNestedDataResolver(), + + // finally use standard resolver + defaultOptions.Resolver + ); + + _options = defaultOptions + .WithResolver(resolver) + .WithCompression(MessagePackCompression.Lz4BlockArray); } public string ToJson(string serialized) @@ -21,9 +41,6 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource // TODO: Instead of returning base64 it would be more ideal to avoid that translation entirely and just store/retrieve raw bytes - // TODO: We need to write tests to serialize/deserialize between either of these serializers to ensure we end up with the same object - // i think this one is a bit quirky so far :) - public ContentNestedData Deserialize(string data) { var bin = Convert.FromBase64String(data); @@ -32,11 +49,89 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } public string Serialize(ContentNestedData nestedData) - { - var bin = MessagePackSerializer.Serialize( - nestedData, - _options); + { + var bin = MessagePackSerializer.Serialize(nestedData, _options); return Convert.ToBase64String(bin); } + + //private class ContentNestedDataResolver : IFormatterResolver + //{ + // // GetFormatter's get cost should be minimized so use type cache. + // public IMessagePackFormatter GetFormatter() => FormatterCache.Formatter; + + // private static class FormatterCache + // { + // public static readonly IMessagePackFormatter Formatter; + + // // generic's static constructor should be minimized for reduce type generation size! + // // use outer helper method. + // static FormatterCache() + // { + // Formatter = (IMessagePackFormatter)SampleCustomResolverGetFormatterHelper.GetFormatter(typeof(T)); + // } + // } + //} + + //internal static class SampleCustomResolverGetFormatterHelper + //{ + // // If type is concrete type, use type-formatter map + // static readonly Dictionary _formatterMap = new Dictionary() + // { + // {typeof(ContentNestedData), new ContentNestedDataFormatter()} + // // add more your own custom serializers. + // }; + + // internal static object GetFormatter(Type t) + // { + // object formatter; + // if (_formatterMap.TryGetValue(t, out formatter)) + // { + // return formatter; + // } + + // // If target type is generics, use MakeGenericType. + // if (t.IsGenericParameter && t.GetGenericTypeDefinition() == typeof(ValueTuple<,>)) + // { + // return Activator.CreateInstance(typeof(ValueTupleFormatter<,>).MakeGenericType(t.GenericTypeArguments)); + // } + + // // If type can not get, must return null for fallback mechanism. + // return null; + // } + //} + + //public class ContentNestedDataFormatter : IMessagePackFormatter + //{ + // public void Serialize(ref MessagePackWriter writer, ContentNestedData value, MessagePackSerializerOptions options) + // { + // if (value == null) + // { + // writer.WriteNil(); + // return; + // } + + // writer.WriteArrayHeader(3); + // writer.WriteString(value.UrlSegment); + // writer.WriteString(value.FullName); + // writer.WriteString(value.Age); + + // writer.WriteString(value.FullName); + // } + + // public ContentNestedData Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) + // { + // if (reader.TryReadNil()) + // { + // return null; + // } + + // options.Security.DepthStep(ref reader); + + // var path = reader.ReadString(); + + // reader.Depth--; + // return new FileInfo(path); + // } + //} } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/PropertyData.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/PropertyData.cs index d49a976b7a..b49a781e0c 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/PropertyData.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/PropertyData.cs @@ -1,15 +1,19 @@ using System; using System.ComponentModel; +using System.Runtime.Serialization; using Newtonsoft.Json; using Umbraco.Core.Serialization; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { + + [DataContract] // NOTE: Use DataContract annotations here to control how MessagePack serializes/deserializes the data to use INT keys public class PropertyData { private string _culture; private string _segment; + [DataMember(Order = 0)] [JsonConverter(typeof(AutoInterningStringConverter))] [DefaultValue("")] [JsonProperty(DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate, PropertyName = "c")] @@ -19,6 +23,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource set => _culture = value ?? throw new ArgumentNullException(nameof(value)); // TODO: or fallback to string.Empty? CANNOT be null } + [DataMember(Order = 1)] [JsonConverter(typeof(AutoInterningStringConverter))] [DefaultValue("")] [JsonProperty(DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate, PropertyName = "s")] @@ -28,22 +33,26 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource set => _segment = value ?? throw new ArgumentNullException(nameof(value)); // TODO: or fallback to string.Empty? CANNOT be null } + [DataMember(Order = 2)] [JsonProperty("v")] public object Value { get; set; } //Legacy properties used to deserialize existing nucache db entries + [DataMember(Order = 3)] [JsonProperty("culture")] private string LegacyCulture { set => Culture = value; } + [DataMember(Order = 4)] [JsonProperty("seg")] private string LegacySegment { set => Segment = value; } + [DataMember(Order = 5)] [JsonProperty("val")] private object LegacyValue { diff --git a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs index 6d01b34a76..205ac55cdc 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs @@ -1,4 +1,5 @@ -using Umbraco.Core; +using System.Configuration; +using Umbraco.Core; using Umbraco.Core.Composing; using Umbraco.Web.PublishedCache.NuCache.DataSource; @@ -10,9 +11,16 @@ namespace Umbraco.Web.PublishedCache.NuCache { base.Compose(composition); - // register the NuCache NestedContentData serializer - //composition.Register(); - composition.Register(); + var serializer = ConfigurationManager.AppSettings["Umbraco.Web.PublishedCache.NuCache.Serializer"]; + + if (serializer == "MsgPack") + { + composition.Register(); + } + else + { + composition.Register(); + } // register the NuCache database data source composition.Register();