Merge commit '94d525d88f713b36419f28bfda4d82ee68637d83' into v9/dev

# Conflicts:
#	build/NuSpecs/UmbracoCms.Web.nuspec
#	src/Umbraco.Core/Composing/Current.cs
#	src/Umbraco.Core/Persistence/NPocoDatabaseExtensions-Bulk.cs
#	src/Umbraco.Core/Runtime/CoreRuntime.cs
#	src/Umbraco.Infrastructure/Migrations/Upgrade/UmbracoPlan.cs
#	src/Umbraco.Infrastructure/Persistence/NPocoDatabaseExtensions.cs
#	src/Umbraco.Infrastructure/Persistence/UmbracoDatabase.cs
#	src/Umbraco.Persistence.SqlCe/SqlCeSyntaxProvider.cs
#	src/Umbraco.PublishedCache.NuCache/DataSource/BTree.cs
#	src/Umbraco.PublishedCache.NuCache/DataSource/ContentCacheDataModel.cs
#	src/Umbraco.PublishedCache.NuCache/DataSource/ContentCacheDataSerializationResult.cs
#	src/Umbraco.PublishedCache.NuCache/DataSource/ContentCacheDataSerializerEntityType.cs
#	src/Umbraco.PublishedCache.NuCache/DataSource/ContentData.cs
#	src/Umbraco.PublishedCache.NuCache/DataSource/ContentNestedData.cs
#	src/Umbraco.PublishedCache.NuCache/DataSource/CultureVariation.cs
#	src/Umbraco.PublishedCache.NuCache/DataSource/IContentCacheDataSerializer.cs
#	src/Umbraco.PublishedCache.NuCache/DataSource/IContentCacheDataSerializerFactory.cs
#	src/Umbraco.PublishedCache.NuCache/DataSource/IDictionaryOfPropertyDataSerializer.cs
#	src/Umbraco.PublishedCache.NuCache/DataSource/JsonContentNestedDataSerializer.cs
#	src/Umbraco.PublishedCache.NuCache/DataSource/JsonContentNestedDataSerializerFactory.cs
#	src/Umbraco.PublishedCache.NuCache/DataSource/LazyCompressedString.cs
#	src/Umbraco.PublishedCache.NuCache/DataSource/MsgPackContentNestedDataSerializer.cs
#	src/Umbraco.PublishedCache.NuCache/DataSource/MsgPackContentNestedDataSerializerFactory.cs
#	src/Umbraco.PublishedCache.NuCache/DataSource/PropertyData.cs
#	src/Umbraco.PublishedCache.NuCache/NuCacheSerializerComponent.cs
#	src/Umbraco.PublishedCache.NuCache/NuCacheSerializerComposer.cs
#	src/Umbraco.Tests.Integration/Umbraco.Infrastructure/Services/ContentTypeServiceVariantsTests.cs
#	src/Umbraco.Tests/App.config
#	src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs
#	src/Umbraco.Tests/PublishedContent/NuCacheTests.cs
#	src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs
#	src/Umbraco.Web.UI.NetCore/umbraco/config/lang/da.xml
#	src/Umbraco.Web.UI/web.Template.Debug.config
#	src/Umbraco.Web.UI/web.Template.config
#	src/Umbraco.Web/Composing/ModuleInjector.cs
#	src/Umbraco.Web/Editors/NuCacheStatusController.cs
#	src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentNestedData.cs
#	src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs
#	src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs
#	src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs
#	src/Umbraco.Web/Runtime/WebRuntime.cs
This commit is contained in:
Shannon
2021-06-24 09:43:57 -06:00
parent 4c89d036ac
commit 72671dbca8
139 changed files with 5650 additions and 1593 deletions

View File

@@ -3,10 +3,22 @@ using CSharpTest.Net.Serialization;
namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
{
class ContentDataSerializer : ISerializer<ContentData>
/// <summary>
/// Serializes/Deserializes data to BTree data source for <see cref="ContentData"/>
/// </summary>
internal class ContentDataSerializer : ISerializer<ContentData>
{
private static readonly DictionaryOfPropertyDataSerializer PropertiesSerializer = new DictionaryOfPropertyDataSerializer();
private static readonly DictionaryOfCultureVariationSerializer CultureVariationsSerializer = new DictionaryOfCultureVariationSerializer();
public ContentDataSerializer(IDictionaryOfPropertyDataSerializer dictionaryOfPropertyDataSerializer = null)
{
_dictionaryOfPropertyDataSerializer = dictionaryOfPropertyDataSerializer;
if(_dictionaryOfPropertyDataSerializer == null)
{
_dictionaryOfPropertyDataSerializer = DefaultPropertiesSerializer;
}
}
private static readonly DictionaryOfPropertyDataSerializer DefaultPropertiesSerializer = new DictionaryOfPropertyDataSerializer();
private static readonly DictionaryOfCultureVariationSerializer DefaultCultureVariationsSerializer = new DictionaryOfCultureVariationSerializer();
private readonly IDictionaryOfPropertyDataSerializer _dictionaryOfPropertyDataSerializer;
public ContentData ReadFrom(Stream stream)
{
@@ -19,8 +31,8 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
VersionDate = PrimitiveSerializer.DateTime.ReadFrom(stream),
WriterId = PrimitiveSerializer.Int32.ReadFrom(stream),
TemplateId = PrimitiveSerializer.Int32.ReadFrom(stream),
Properties = PropertiesSerializer.ReadFrom(stream),
CultureInfos = CultureVariationsSerializer.ReadFrom(stream)
Properties = _dictionaryOfPropertyDataSerializer.ReadFrom(stream), // TODO: We don't want to allocate empty arrays
CultureInfos = DefaultCultureVariationsSerializer.ReadFrom(stream) // TODO: We don't want to allocate empty arrays
};
}
@@ -36,8 +48,8 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
{
PrimitiveSerializer.Int32.WriteTo(value.TemplateId.Value, stream);
}
PropertiesSerializer.WriteTo(value.Properties, stream);
CultureVariationsSerializer.WriteTo(value.CultureInfos, stream);
_dictionaryOfPropertyDataSerializer.WriteTo(value.Properties, stream);
DefaultCultureVariationsSerializer.WriteTo(value.CultureInfos, stream);
}
}
}

View File

@@ -5,7 +5,17 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
{
internal class ContentNodeKitSerializer : ISerializer<ContentNodeKit>
{
static readonly ContentDataSerializer DataSerializer = new ContentDataSerializer();
public ContentNodeKitSerializer(ContentDataSerializer contentDataSerializer = null)
{
_contentDataSerializer = contentDataSerializer;
if(_contentDataSerializer == null)
{
_contentDataSerializer = DefaultDataSerializer;
}
}
static readonly ContentDataSerializer DefaultDataSerializer = new ContentDataSerializer();
private readonly ContentDataSerializer _contentDataSerializer;
//static readonly ListOfIntSerializer ChildContentIdsSerializer = new ListOfIntSerializer();
public ContentNodeKit ReadFrom(Stream stream)
@@ -26,10 +36,10 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
};
var hasDraft = PrimitiveSerializer.Boolean.ReadFrom(stream);
if (hasDraft)
kit.DraftData = DataSerializer.ReadFrom(stream);
kit.DraftData = _contentDataSerializer.ReadFrom(stream);
var hasPublished = PrimitiveSerializer.Boolean.ReadFrom(stream);
if (hasPublished)
kit.PublishedData = DataSerializer.ReadFrom(stream);
kit.PublishedData = _contentDataSerializer.ReadFrom(stream);
return kit;
}
@@ -47,11 +57,11 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
PrimitiveSerializer.Boolean.WriteTo(value.DraftData != null, stream);
if (value.DraftData != null)
DataSerializer.WriteTo(value.DraftData, stream);
_contentDataSerializer.WriteTo(value.DraftData, stream);
PrimitiveSerializer.Boolean.WriteTo(value.PublishedData != null, stream);
if (value.PublishedData != null)
DataSerializer.WriteTo(value.PublishedData, stream);
_contentDataSerializer.WriteTo(value.PublishedData, stream);
}
}
}

View File

@@ -6,6 +6,9 @@ using Umbraco.Extensions;
namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
{
/// <summary>
/// Serializes/Deserializes culture variant data as a dictionary for BTree
/// </summary>
internal class DictionaryOfCultureVariationSerializer : SerializerBase, ISerializer<IReadOnlyDictionary<string, CultureVariation>>
{
public IReadOnlyDictionary<string, CultureVariation> ReadFrom(Stream stream)
@@ -18,8 +21,13 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
var dict = new Dictionary<string, CultureVariation>(StringComparer.InvariantCultureIgnoreCase);
for (var i = 0; i < pcount; i++)
{
var languageId = PrimitiveSerializer.String.ReadFrom(stream);
var cultureVariation = new CultureVariation { Name = ReadStringObject(stream), UrlSegment = ReadStringObject(stream), Date = ReadDateTime(stream) };
var languageId = string.Intern(PrimitiveSerializer.String.ReadFrom(stream));
var cultureVariation = new CultureVariation
{
Name = ReadStringObject(stream),
UrlSegment = ReadStringObject(stream),
Date = ReadDateTime(stream)
};
dict[languageId] = cultureVariation;
}
return dict;

View File

@@ -6,44 +6,47 @@ using Umbraco.Extensions;
namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
{
internal class DictionaryOfPropertyDataSerializer : SerializerBase, ISerializer<IDictionary<string, PropertyData[]>>
/// <summary>
/// Serializes/Deserializes property data as a dictionary for BTree
/// </summary>
internal class DictionaryOfPropertyDataSerializer : SerializerBase, ISerializer<IDictionary<string, PropertyData[]>>, IDictionaryOfPropertyDataSerializer
{
public IDictionary<string, PropertyData[]> ReadFrom(Stream stream)
{
var dict = new Dictionary<string, PropertyData[]>(StringComparer.InvariantCultureIgnoreCase);
// read properties count
var pcount = PrimitiveSerializer.Int32.ReadFrom(stream);
var dict = new Dictionary<string, PropertyData[]>(pcount,StringComparer.InvariantCultureIgnoreCase);
// read each property
for (var i = 0; i < pcount; i++)
{
// read property alias
var key = PrimitiveSerializer.String.ReadFrom(stream);
var key = string.Intern(PrimitiveSerializer.String.ReadFrom(stream));
// read values count
var vcount = PrimitiveSerializer.Int32.ReadFrom(stream);
// create pdata and add to the dictionary
var pdatas = new List<PropertyData>();
var pdatas = new PropertyData[vcount];
// for each value, read and add to pdata
for (var j = 0; j < vcount; j++)
{
var pdata = new PropertyData();
pdatas.Add(pdata);
pdatas[j] =pdata;
// everything that can be null is read/written as object
// even though - culture and segment should never be null here, as 'null' represents
// the 'current' value, and string.Empty should be used to represent the invariant or
// neutral values - PropertyData throws when getting nulls, so falling back to
// string.Empty here - what else?
pdata.Culture = ReadStringObject(stream) ?? string.Empty;
pdata.Segment = ReadStringObject(stream) ?? string.Empty;
pdata.Culture = ReadStringObject(stream, true) ?? string.Empty;
pdata.Segment = ReadStringObject(stream, true) ?? string.Empty;
pdata.Value = ReadObject(stream);
}
dict[key] = pdatas.ToArray();
dict[key] = pdatas;
}
return dict;
}

View File

@@ -7,10 +7,10 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
{
internal class BTree
{
public static BPlusTree<int, ContentNodeKit> GetTree(string filepath, bool exists, NuCacheSettings settings)
public static BPlusTree<int, ContentNodeKit> GetTree(string filepath, bool exists, NuCacheSettings settings, ContentDataSerializer contentDataSerializer = null)
{
var keySerializer = new PrimitiveSerializer();
var valueSerializer = new ContentNodeKitSerializer();
var valueSerializer = new ContentNodeKitSerializer(contentDataSerializer);
var options = new BPlusTree<int, ContentNodeKit>.OptionsV2(keySerializer, valueSerializer)
{
CreateFile = exists ? CreatePolicy.IfNeeded : CreatePolicy.Always,
@@ -37,6 +37,7 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
//btree.
return tree;
}
private static int GetBlockSize(NuCacheSettings settings)

View File

@@ -0,0 +1,45 @@
using System.Collections.Generic;
using Newtonsoft.Json;
using Umbraco.Cms.Infrastructure.Serialization;
using System.Runtime.Serialization;
namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
{
/// <summary>
/// The content model stored in the content cache database table serialized as JSON
/// </summary>
[DataContract] // NOTE: Use DataContract annotations here to control how MessagePack serializes/deserializes the data to use INT keys
public class ContentCacheDataModel
{
// TODO: We don't want to allocate empty arrays
//dont serialize empty properties
[DataMember(Order = 0)]
[JsonProperty("pd")]
[JsonConverter(typeof(AutoInterningStringKeyCaseInsensitiveDictionaryConverter<PropertyData[]>))]
public Dictionary<string, PropertyData[]> PropertyData { get; set; }
[DataMember(Order = 1)]
[JsonProperty("cd")]
[JsonConverter(typeof(AutoInterningStringKeyCaseInsensitiveDictionaryConverter<CultureVariation>))]
public Dictionary<string, CultureVariation> CultureData { get; set; }
[DataMember(Order = 2)]
[JsonProperty("us")]
public string UrlSegment { get; set; }
//Legacy properties used to deserialize existing nucache db entries
[IgnoreDataMember]
[JsonProperty("properties")]
[JsonConverter(typeof(CaseInsensitiveDictionaryConverter<PropertyData[]>))]
private Dictionary<string, PropertyData[]> LegacyPropertyData { set => PropertyData = value; }
[IgnoreDataMember]
[JsonProperty("cultureData")]
[JsonConverter(typeof(CaseInsensitiveDictionaryConverter<CultureVariation>))]
private Dictionary<string, CultureVariation> LegacyCultureData { set => CultureData = value; }
[IgnoreDataMember]
[JsonProperty("urlSegment")]
private string LegacyUrlSegment { set => UrlSegment = value; }
}
}

View File

@@ -0,0 +1,43 @@
using System;
using System.Collections.Generic;
namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
{
/// <summary>
/// The serialization result from <see cref="IContentCacheDataSerializer"/> for which the serialized value
/// will be either a string or a byte[]
/// </summary>
public struct ContentCacheDataSerializationResult : IEquatable<ContentCacheDataSerializationResult>
{
public ContentCacheDataSerializationResult(string stringData, byte[] byteData)
{
StringData = stringData;
ByteData = byteData;
}
public string StringData { get; }
public byte[] ByteData { get; }
public override bool Equals(object obj)
=> obj is ContentCacheDataSerializationResult result && Equals(result);
public bool Equals(ContentCacheDataSerializationResult other)
=> StringData == other.StringData &&
EqualityComparer<byte[]>.Default.Equals(ByteData, other.ByteData);
public override int GetHashCode()
{
var hashCode = 1910544615;
hashCode = hashCode * -1521134295 + EqualityComparer<string>.Default.GetHashCode(StringData);
hashCode = hashCode * -1521134295 + EqualityComparer<byte[]>.Default.GetHashCode(ByteData);
return hashCode;
}
public static bool operator ==(ContentCacheDataSerializationResult left, ContentCacheDataSerializationResult right)
=> left.Equals(right);
public static bool operator !=(ContentCacheDataSerializationResult left, ContentCacheDataSerializationResult right)
=> !(left == right);
}
}

View File

@@ -0,0 +1,13 @@
using System;
namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
{
[Flags]
public enum ContentCacheDataSerializerEntityType
{
Document = 1,
Media = 2,
Member = 4
}
}

View File

@@ -3,7 +3,9 @@ using System.Collections.Generic;
namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
{
// represents everything that is specific to edited or published version
/// <summary>
/// Represents everything that is specific to an edited or published content version
/// </summary>
public class ContentData
{
public string Name { get; set; }

View File

@@ -1,36 +0,0 @@
using System.Collections.Generic;
using Newtonsoft.Json;
using Umbraco.Cms.Infrastructure.Serialization;
namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
{
/// <summary>
/// The content item 1:M data that is serialized to JSON
/// </summary>
internal class ContentNestedData
{
// dont serialize empty properties
[JsonProperty("pd")]
[JsonConverter(typeof(CaseInsensitiveDictionaryConverter<PropertyData[]>))]
public Dictionary<string, PropertyData[]> PropertyData { get; set; }
[JsonProperty("cd")]
[JsonConverter(typeof(CaseInsensitiveDictionaryConverter<CultureVariation>))]
public Dictionary<string, CultureVariation> CultureData { get; set; }
[JsonProperty("us")]
public string UrlSegment { get; set; }
// Legacy properties used to deserialize existing nucache db entries
[JsonProperty("properties")]
[JsonConverter(typeof(CaseInsensitiveDictionaryConverter<PropertyData[]>))]
private Dictionary<string, PropertyData[]> LegacyPropertyData { set { PropertyData = value; } }
[JsonProperty("cultureData")]
[JsonConverter(typeof(CaseInsensitiveDictionaryConverter<CultureVariation>))]
private Dictionary<string, CultureVariation> LegacyCultureData { set { CultureData = value; } }
[JsonProperty("urlSegment")]
private string LegacyUrlSegment { set { UrlSegment = value; } }
}
}

View File

@@ -1,12 +1,13 @@
using System;
using System;
using Umbraco.Cms.Core.Models;
namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
{
// read-only dto
internal class ContentSourceDto
internal class ContentSourceDto : IReadOnlyContentBase
{
public int Id { get; set; }
public Guid Uid { get; set; }
public Guid Key { get; set; }
public int ContentTypeId { get; set; }
public int Level { get; set; }
@@ -27,6 +28,7 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
public int EditWriterId { get; set; }
public int EditTemplateId { get; set; }
public string EditData { get; set; }
public byte[] EditDataRaw { get; set; }
// published data
public int PublishedVersionId { get; set; }
@@ -35,5 +37,11 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
public int PubWriterId { get; set; }
public int PubTemplateId { get; set; }
public string PubData { get; set; }
public byte[] PubDataRaw { get; set; }
// Explicit implementation
DateTime IReadOnlyContentBase.UpdateDate => EditVersionDate;
string IReadOnlyContentBase.Name => EditName;
int IReadOnlyContentBase.WriterId => EditWriterId;
}
}

View File

@@ -1,4 +1,5 @@
using System;
using System.Runtime.Serialization;
using Newtonsoft.Json;
namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
@@ -6,30 +7,39 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
/// <summary>
/// Represents the culture variation information on a content item
/// </summary>
[DataContract] // NOTE: Use DataContract annotations here to control how MessagePack serializes/deserializes the data to use INT keys
public class CultureVariation
{
[DataMember(Order = 0)]
[JsonProperty("nm")]
public string Name { get; set; }
[DataMember(Order = 1)]
[JsonProperty("us")]
public string UrlSegment { get; set; }
[DataMember(Order = 2)]
[JsonProperty("dt")]
public DateTime Date { get; set; }
[DataMember(Order = 3)]
[JsonProperty("isd")]
public bool IsDraft { get; set; }
//Legacy properties used to deserialize existing nucache db entries
[IgnoreDataMember]
[JsonProperty("name")]
private string LegacyName { set { Name = value; } }
[IgnoreDataMember]
[JsonProperty("urlSegment")]
private string LegacyUrlSegment { set { UrlSegment = value; } }
[IgnoreDataMember]
[JsonProperty("date")]
private DateTime LegacyDate { set { Date = value; } }
[IgnoreDataMember]
[JsonProperty("isDraft")]
private bool LegacyIsDraft { set { IsDraft = value; } }
}

View File

@@ -0,0 +1,25 @@
using Umbraco.Cms.Core.Models;
namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
{
/// <summary>
/// Serializes/Deserializes <see cref="ContentCacheDataModel"/> document to the SQL Database as a string
/// </summary>
/// <remarks>
/// Resolved from the <see cref="IContentCacheDataSerializerFactory"/>. This cannot be resolved from DI.
/// </remarks>
public interface IContentCacheDataSerializer
{
/// <summary>
/// Deserialize the data into a <see cref="ContentCacheDataModel"/>
/// </summary>
ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData);
/// <summary>
/// Serializes the <see cref="ContentCacheDataModel"/>
/// </summary>
ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model);
}
}

View File

@@ -0,0 +1,16 @@
namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
{
public interface IContentCacheDataSerializerFactory
{
/// <summary>
/// Gets or creates a new instance of <see cref="IContentCacheDataSerializer"/>
/// </summary>
/// <returns></returns>
/// <remarks>
/// This method may return the same instance, however this depends on the state of the application and if any underlying data has changed.
/// This method may also be used to initialize anything before a serialization/deserialization session occurs.
/// </remarks>
IContentCacheDataSerializer Create(ContentCacheDataSerializerEntityType types);
}
}

View File

@@ -0,0 +1,11 @@
using System.Collections.Generic;
using System.IO;
namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
{
internal interface IDictionaryOfPropertyDataSerializer
{
IDictionary<string, PropertyData[]> ReadFrom(Stream stream);
void WriteTo(IDictionary<string, PropertyData[]> value, Stream stream);
}
}

View File

@@ -0,0 +1,91 @@
using Newtonsoft.Json;
using System;
using System.Buffers;
using System.Collections.Generic;
using System.IO;
using Umbraco.Cms.Core.Models;
using Umbraco.Cms.Infrastructure.Serialization;
namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
{
public class JsonContentNestedDataSerializer : IContentCacheDataSerializer
{
// by default JsonConvert will deserialize our numeric values as Int64
// which is bad, because they were Int32 in the database - take care
private readonly JsonSerializerSettings _jsonSerializerSettings = new JsonSerializerSettings
{
Converters = new List<JsonConverter> { new ForceInt32Converter() },
// Explicitly specify date handling so that it's consistent and follows the same date handling as MessagePack
DateParseHandling = DateParseHandling.DateTime,
DateFormatHandling = DateFormatHandling.IsoDateFormat,
DateTimeZoneHandling = DateTimeZoneHandling.Utc,
DateFormatString = "o"
};
private readonly JsonNameTable _propertyNameTable = new DefaultJsonNameTable();
public ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData)
{
if (stringData == null && byteData != null)
throw new NotSupportedException($"{typeof(JsonContentNestedDataSerializer)} does not support byte[] serialization");
JsonSerializer serializer = JsonSerializer.Create(_jsonSerializerSettings);
using (JsonTextReader reader = new JsonTextReader(new StringReader(stringData)))
{
// reader will get buffer from array pool
reader.ArrayPool = JsonArrayPool.Instance;
reader.PropertyNameTable = _propertyNameTable;
return serializer.Deserialize<ContentCacheDataModel>(reader);
}
}
public ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model)
{
// note that numeric values (which are Int32) are serialized without their
// type (eg "value":1234) and JsonConvert by default deserializes them as Int64
var json = JsonConvert.SerializeObject(model);
return new ContentCacheDataSerializationResult(json, null);
}
}
public class JsonArrayPool : IArrayPool<char>
{
public static readonly JsonArrayPool Instance = new JsonArrayPool();
public char[] Rent(int minimumLength)
{
// get char array from System.Buffers shared pool
return ArrayPool<char>.Shared.Rent(minimumLength);
}
public void Return(char[] array)
{
// return char array to System.Buffers shared pool
ArrayPool<char>.Shared.Return(array);
}
}
public class AutomaticJsonNameTable : DefaultJsonNameTable
{
int nAutoAdded = 0;
int maxToAutoAdd;
public AutomaticJsonNameTable(int maxToAdd)
{
this.maxToAutoAdd = maxToAdd;
}
public override string Get(char[] key, int start, int length)
{
var s = base.Get(key, start, length);
if (s == null && nAutoAdded < maxToAutoAdd)
{
s = new string(key, start, length);
Add(s);
nAutoAdded++;
}
return s;
}
}
}

View File

@@ -0,0 +1,10 @@
using System;
namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
{
internal class JsonContentNestedDataSerializerFactory : IContentCacheDataSerializerFactory
{
private readonly Lazy<JsonContentNestedDataSerializer> _serializer = new Lazy<JsonContentNestedDataSerializer>();
public IContentCacheDataSerializer Create(ContentCacheDataSerializerEntityType types) => _serializer.Value;
}
}

View File

@@ -0,0 +1,109 @@
using K4os.Compression.LZ4;
using System;
using System.Diagnostics;
using System.Text;
using Umbraco.Cms.Core.Exceptions;
namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
{
/// <summary>
/// Lazily decompresses a LZ4 Pickler compressed UTF8 string
/// </summary>
[DebuggerDisplay("{Display}")]
internal struct LazyCompressedString
{
private byte[] _bytes;
private string _str;
private readonly object _locker;
/// <summary>
/// Constructor
/// </summary>
/// <param name="bytes">LZ4 Pickle compressed UTF8 String</param>
public LazyCompressedString(byte[] bytes)
{
_locker = new object();
_bytes = bytes;
_str = null;
}
public byte[] GetBytes()
{
if (_bytes == null)
{
throw new InvalidOperationException("The bytes have already been expanded");
}
return _bytes;
}
/// <summary>
/// Returns the decompressed string from the bytes. This methods can only be called once.
/// </summary>
/// <returns></returns>
/// <exception cref="InvalidOperationException">Throws if this is called more than once</exception>
public string DecompressString()
{
if (_str != null)
{
return _str;
}
lock (_locker)
{
if (_str != null)
{
// double check
return _str;
}
if (_bytes == null)
{
throw new InvalidOperationException("Bytes have already been cleared");
}
_str = Encoding.UTF8.GetString(LZ4Pickler.Unpickle(_bytes));
_bytes = null;
}
return _str;
}
/// <summary>
/// Used to display debugging output since ToString() can only be called once
/// </summary>
private string Display
{
get
{
if (_str != null)
{
return $"Decompressed: {_str}";
}
lock (_locker)
{
if (_str != null)
{
// double check
return $"Decompressed: {_str}";
}
if (_bytes == null)
{
// This shouldn't happen
throw new PanicException("Bytes have already been cleared");
}
else
{
return $"Compressed Bytes: {_bytes.Length}";
}
}
}
}
public override string ToString() => DecompressString();
public static implicit operator string(LazyCompressedString l) => l.ToString();
}
}

View File

@@ -0,0 +1,126 @@
using K4os.Compression.LZ4;
using MessagePack;
using MessagePack.Resolvers;
using System;
using System.Linq;
using System.Text;
using Umbraco.Cms.Core.Models;
using Umbraco.Cms.Core.PropertyEditors;
namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
{
/// <summary>
/// Serializes/Deserializes <see cref="ContentCacheDataModel"/> document to the SQL Database as bytes using MessagePack
/// </summary>
public class MsgPackContentNestedDataSerializer : IContentCacheDataSerializer
{
private readonly MessagePackSerializerOptions _options;
private readonly IPropertyCacheCompression _propertyOptions;
public MsgPackContentNestedDataSerializer(IPropertyCacheCompression propertyOptions)
{
_propertyOptions = propertyOptions ?? throw new ArgumentNullException(nameof(propertyOptions));
var defaultOptions = ContractlessStandardResolver.Options;
var resolver = CompositeResolver.Create(
// TODO: We want to be able to intern the strings for aliases when deserializing like we do for Newtonsoft but I'm unsure exactly how
// to do that but it would seem to be with a custom message pack resolver but I haven't quite figured out based on the docs how
// to do that since that is part of the int key -> string mapping operation, might have to see the source code to figure that one out.
// There are docs here on how to build one of these: https://github.com/neuecc/MessagePack-CSharp/blob/master/README.md#low-level-api-imessagepackformattert
// and there are a couple examples if you search on google for them but this will need to be a separate project.
// NOTE: resolver custom types first
// new ContentNestedDataResolver(),
// finally use standard resolver
defaultOptions.Resolver
);
_options = defaultOptions
.WithResolver(resolver)
.WithCompression(MessagePackCompression.Lz4BlockArray);
}
public string ToJson(byte[] bin)
{
var json = MessagePackSerializer.ConvertToJson(bin, _options);
return json;
}
public ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData)
{
if (byteData != null)
{
var cacheModel = MessagePackSerializer.Deserialize<ContentCacheDataModel>(byteData, _options);
Expand(content, cacheModel);
return cacheModel;
}
else if (stringData != null)
{
// NOTE: We don't really support strings but it's possible if manually used (i.e. tests)
var bin = Convert.FromBase64String(stringData);
var cacheModel = MessagePackSerializer.Deserialize<ContentCacheDataModel>(bin, _options);
Expand(content, cacheModel);
return cacheModel;
}
else
{
return null;
}
}
public ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model)
{
Compress(content, model);
var bytes = MessagePackSerializer.Serialize(model, _options);
return new ContentCacheDataSerializationResult(null, bytes);
}
/// <summary>
/// Used during serialization to compress properties
/// </summary>
/// <param name="model"></param>
/// <remarks>
/// This will essentially 'double compress' property data. The MsgPack data as a whole will already be compressed
/// but this will go a step further and double compress property data so that it is stored in the nucache file
/// as compressed bytes and therefore will exist in memory as compressed bytes. That is, until the bytes are
/// read/decompressed as a string to be displayed on the front-end. This allows for potentially a significant
/// memory savings but could also affect performance of first rendering pages while decompression occurs.
/// </remarks>
private void Compress(IReadOnlyContentBase content, ContentCacheDataModel model)
{
foreach(var propertyAliasToData in model.PropertyData)
{
if (_propertyOptions.IsCompressed(content, propertyAliasToData.Key))
{
foreach(var property in propertyAliasToData.Value.Where(x => x.Value != null && x.Value is string))
{
property.Value = LZ4Pickler.Pickle(Encoding.UTF8.GetBytes((string)property.Value), LZ4Level.L00_FAST);
}
}
}
}
/// <summary>
/// Used during deserialization to map the property data as lazy or expand the value
/// </summary>
/// <param name="nestedData"></param>
private void Expand(IReadOnlyContentBase content, ContentCacheDataModel nestedData)
{
foreach (var propertyAliasToData in nestedData.PropertyData)
{
if (_propertyOptions.IsCompressed(content, propertyAliasToData.Key))
{
foreach (var property in propertyAliasToData.Value.Where(x => x.Value != null))
{
if (property.Value is byte[] byteArrayValue)
{
property.Value = new LazyCompressedString(byteArrayValue);
}
}
}
}
}
}
}

View File

@@ -0,0 +1,69 @@
using System.Collections.Concurrent;
using System.Collections.Generic;
using Umbraco.Cms.Core.Models;
using Umbraco.Cms.Core.PropertyEditors;
using Umbraco.Cms.Core.Services;
namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
{
internal class MsgPackContentNestedDataSerializerFactory : IContentCacheDataSerializerFactory
{
private readonly IContentTypeService _contentTypeService;
private readonly IMediaTypeService _mediaTypeService;
private readonly IMemberTypeService _memberTypeService;
private readonly PropertyEditorCollection _propertyEditors;
private readonly IPropertyCacheCompressionOptions _compressionOptions;
private readonly ConcurrentDictionary<(int, string), bool> _isCompressedCache = new ConcurrentDictionary<(int, string), bool>();
public MsgPackContentNestedDataSerializerFactory(
IContentTypeService contentTypeService,
IMediaTypeService mediaTypeService,
IMemberTypeService memberTypeService,
PropertyEditorCollection propertyEditors,
IPropertyCacheCompressionOptions compressionOptions)
{
_contentTypeService = contentTypeService;
_mediaTypeService = mediaTypeService;
_memberTypeService = memberTypeService;
_propertyEditors = propertyEditors;
_compressionOptions = compressionOptions;
}
public IContentCacheDataSerializer Create(ContentCacheDataSerializerEntityType types)
{
// Depending on which entity types are being requested, we need to look up those content types
// to initialize the compression options.
// We need to initialize these options now so that any data lookups required are completed and are not done while the content cache
// is performing DB queries which will result in errors since we'll be trying to query with open readers.
// NOTE: The calls to GetAll() below should be cached if the data has not been changed.
var contentTypes = new Dictionary<int, IContentTypeComposition>();
if ((types & ContentCacheDataSerializerEntityType.Document) == ContentCacheDataSerializerEntityType.Document)
{
foreach(var ct in _contentTypeService.GetAll())
{
contentTypes[ct.Id] = ct;
}
}
if ((types & ContentCacheDataSerializerEntityType.Media) == ContentCacheDataSerializerEntityType.Media)
{
foreach (var ct in _mediaTypeService.GetAll())
{
contentTypes[ct.Id] = ct;
}
}
if ((types & ContentCacheDataSerializerEntityType.Member) == ContentCacheDataSerializerEntityType.Member)
{
foreach (var ct in _memberTypeService.GetAll())
{
contentTypes[ct.Id] = ct;
}
}
var compression = new PropertyCacheCompression(_compressionOptions, contentTypes, _propertyEditors, _isCompressedCache);
var serializer = new MsgPackContentNestedDataSerializer(compression);
return serializer;
}
}
}

View File

@@ -1,14 +1,20 @@
using System;
using System.ComponentModel;
using System.Runtime.Serialization;
using Newtonsoft.Json;
using Umbraco.Cms.Infrastructure.Serialization;
namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
{
[DataContract] // NOTE: Use DataContract annotations here to control how MessagePack serializes/deserializes the data to use INT keys
public class PropertyData
{
private string _culture;
private string _segment;
[DataMember(Order = 0)]
[JsonConverter(typeof(AutoInterningStringConverter))]
[DefaultValue("")]
[JsonProperty(DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate, PropertyName = "c")]
public string Culture
@@ -17,6 +23,8 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
set => _culture = value ?? throw new ArgumentNullException(nameof(value)); // TODO: or fallback to string.Empty? CANNOT be null
}
[DataMember(Order = 1)]
[JsonConverter(typeof(AutoInterningStringConverter))]
[DefaultValue("")]
[JsonProperty(DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate, PropertyName = "s")]
public string Segment
@@ -25,23 +33,26 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
set => _segment = value ?? throw new ArgumentNullException(nameof(value)); // TODO: or fallback to string.Empty? CANNOT be null
}
[DataMember(Order = 2)]
[JsonProperty("v")]
public object Value { get; set; }
// Legacy properties used to deserialize existing nucache db entries
[IgnoreDataMember]
[JsonProperty("culture")]
private string LegacyCulture
{
set => Culture = value;
}
[IgnoreDataMember]
[JsonProperty("seg")]
private string LegacySegment
{
set => Segment = value;
}
[IgnoreDataMember]
[JsonProperty("val")]
private object LegacyValue
{

View File

@@ -6,100 +6,218 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
{
internal abstract class SerializerBase
{
private const char PrefixNull = 'N';
private const char PrefixString = 'S';
private const char PrefixInt32 = 'I';
private const char PrefixUInt16 = 'H';
private const char PrefixUInt32 = 'J';
private const char PrefixLong = 'L';
private const char PrefixFloat = 'F';
private const char PrefixDouble = 'B';
private const char PrefixDateTime = 'D';
private const char PrefixByte = 'O';
private const char PrefixByteArray = 'A';
private const char PrefixCompressedStringByteArray = 'C';
private const char PrefixSignedByte = 'E';
private const char PrefixBool = 'M';
private const char PrefixGuid = 'G';
private const char PrefixTimeSpan = 'T';
private const char PrefixInt16 = 'Q';
private const char PrefixChar = 'R';
protected string ReadString(Stream stream) => PrimitiveSerializer.String.ReadFrom(stream);
protected int ReadInt(Stream stream) => PrimitiveSerializer.Int32.ReadFrom(stream);
protected long ReadLong(Stream stream) => PrimitiveSerializer.Int64.ReadFrom(stream);
protected float ReadFloat(Stream stream) => PrimitiveSerializer.Float.ReadFrom(stream);
protected double ReadDouble(Stream stream) => PrimitiveSerializer.Double.ReadFrom(stream);
protected DateTime ReadDateTime(Stream stream) => PrimitiveSerializer.DateTime.ReadFrom(stream);
protected byte[] ReadByteArray(Stream stream) => PrimitiveSerializer.Bytes.ReadFrom(stream);
private T? ReadObject<T>(Stream stream, char t, Func<Stream, T> read)
private T? ReadStruct<T>(Stream stream, char t, Func<Stream, T> read)
where T : struct
{
var type = PrimitiveSerializer.Char.ReadFrom(stream);
if (type == 'N') return null;
if (type == PrefixNull) return null;
if (type != t)
throw new NotSupportedException($"Cannot deserialize type '{type}', expected '{t}'.");
return read(stream);
}
protected string ReadStringObject(Stream stream) // required 'cos string is not a struct
protected string ReadStringObject(Stream stream, bool intern = false) // required 'cos string is not a struct
{
var type = PrimitiveSerializer.Char.ReadFrom(stream);
if (type == 'N') return null;
if (type != 'S')
throw new NotSupportedException($"Cannot deserialize type '{type}', expected 'S'.");
return PrimitiveSerializer.String.ReadFrom(stream);
if (type == PrefixNull) return null;
if (type != PrefixString)
throw new NotSupportedException($"Cannot deserialize type '{type}', expected '{PrefixString}'.");
return intern
? string.Intern(PrimitiveSerializer.String.ReadFrom(stream))
: PrimitiveSerializer.String.ReadFrom(stream);
}
protected int? ReadIntObject(Stream stream) => ReadObject(stream, 'I', ReadInt);
protected long? ReadLongObject(Stream stream) => ReadObject(stream, 'L', ReadLong);
protected float? ReadFloatObject(Stream stream) => ReadObject(stream, 'F', ReadFloat);
protected double? ReadDoubleObject(Stream stream) => ReadObject(stream, 'B', ReadDouble);
protected DateTime? ReadDateTimeObject(Stream stream) => ReadObject(stream, 'D', ReadDateTime);
protected int? ReadIntObject(Stream stream) => ReadStruct(stream, PrefixInt32, ReadInt);
protected long? ReadLongObject(Stream stream) => ReadStruct(stream, PrefixLong, ReadLong);
protected float? ReadFloatObject(Stream stream) => ReadStruct(stream, PrefixFloat, ReadFloat);
protected double? ReadDoubleObject(Stream stream) => ReadStruct(stream, PrefixDouble, ReadDouble);
protected DateTime? ReadDateTimeObject(Stream stream) => ReadStruct(stream, PrefixDateTime, ReadDateTime);
protected object ReadObject(Stream stream)
=> ReadObject(PrimitiveSerializer.Char.ReadFrom(stream), stream);
/// <summary>
/// Reads in a value based on its char type
/// </summary>
/// <param name="type"></param>
/// <param name="stream"></param>
/// <returns></returns>
/// <remarks>
/// This will incur boxing because the result is an object but in most cases the value will be a struct.
/// When the type is known use the specific methods like <see cref="ReadInt(Stream)"/> instead
/// </remarks>
protected object ReadObject(char type, Stream stream)
{
switch (type)
{
case 'N':
case PrefixNull:
return null;
case 'S':
case PrefixString:
return PrimitiveSerializer.String.ReadFrom(stream);
case 'I':
case PrefixInt32:
return PrimitiveSerializer.Int32.ReadFrom(stream);
case 'L':
case PrefixUInt16:
return PrimitiveSerializer.UInt16.ReadFrom(stream);
case PrefixUInt32:
return PrimitiveSerializer.UInt32.ReadFrom(stream);
case PrefixByte:
return PrimitiveSerializer.Byte.ReadFrom(stream);
case PrefixLong:
return PrimitiveSerializer.Int64.ReadFrom(stream);
case 'F':
case PrefixFloat:
return PrimitiveSerializer.Float.ReadFrom(stream);
case 'B':
case PrefixDouble:
return PrimitiveSerializer.Double.ReadFrom(stream);
case 'D':
case PrefixDateTime:
return PrimitiveSerializer.DateTime.ReadFrom(stream);
case PrefixByteArray:
return PrimitiveSerializer.Bytes.ReadFrom(stream);
case PrefixSignedByte:
return PrimitiveSerializer.SByte.ReadFrom(stream);
case PrefixBool:
return PrimitiveSerializer.Boolean.ReadFrom(stream);
case PrefixGuid:
return PrimitiveSerializer.Guid.ReadFrom(stream);
case PrefixTimeSpan:
return PrimitiveSerializer.TimeSpan.ReadFrom(stream);
case PrefixInt16:
return PrimitiveSerializer.Int16.ReadFrom(stream);
case PrefixChar:
return PrimitiveSerializer.Char.ReadFrom(stream);
case PrefixCompressedStringByteArray:
return new LazyCompressedString(PrimitiveSerializer.Bytes.ReadFrom(stream));
default:
throw new NotSupportedException($"Cannot deserialize unknown type '{type}'.");
}
}
/// <summary>
/// Writes a value to the stream ensuring it's char type is prefixed to the value for reading later
/// </summary>
/// <param name="value"></param>
/// <param name="stream"></param>
/// <remarks>
/// This method will incur boxing if the value is a struct. When the type is known use the <see cref="PrimitiveSerializer"/>
/// to write the value directly.
/// </remarks>
protected void WriteObject(object value, Stream stream)
{
if (value == null)
{
PrimitiveSerializer.Char.WriteTo('N', stream);
PrimitiveSerializer.Char.WriteTo(PrefixNull, stream);
}
else if (value is string stringValue)
{
PrimitiveSerializer.Char.WriteTo('S', stream);
PrimitiveSerializer.Char.WriteTo(PrefixString, stream);
PrimitiveSerializer.String.WriteTo(stringValue, stream);
}
else if (value is int intValue)
{
PrimitiveSerializer.Char.WriteTo('I', stream);
PrimitiveSerializer.Char.WriteTo(PrefixInt32, stream);
PrimitiveSerializer.Int32.WriteTo(intValue, stream);
}
else if (value is byte byteValue)
{
PrimitiveSerializer.Char.WriteTo(PrefixByte, stream);
PrimitiveSerializer.Byte.WriteTo(byteValue, stream);
}
else if (value is ushort ushortValue)
{
PrimitiveSerializer.Char.WriteTo(PrefixUInt16, stream);
PrimitiveSerializer.UInt16.WriteTo(ushortValue, stream);
}
else if (value is long longValue)
{
PrimitiveSerializer.Char.WriteTo('L', stream);
PrimitiveSerializer.Char.WriteTo(PrefixLong, stream);
PrimitiveSerializer.Int64.WriteTo(longValue, stream);
}
else if (value is float floatValue)
{
PrimitiveSerializer.Char.WriteTo('F', stream);
PrimitiveSerializer.Char.WriteTo(PrefixFloat, stream);
PrimitiveSerializer.Float.WriteTo(floatValue, stream);
}
else if (value is double doubleValue)
{
PrimitiveSerializer.Char.WriteTo('B', stream);
PrimitiveSerializer.Char.WriteTo(PrefixDouble, stream);
PrimitiveSerializer.Double.WriteTo(doubleValue, stream);
}
else if (value is DateTime dateValue)
{
PrimitiveSerializer.Char.WriteTo('D', stream);
PrimitiveSerializer.Char.WriteTo(PrefixDateTime, stream);
PrimitiveSerializer.DateTime.WriteTo(dateValue, stream);
}
else if (value is uint uInt32Value)
{
PrimitiveSerializer.Char.WriteTo(PrefixUInt32, stream);
PrimitiveSerializer.UInt32.WriteTo(uInt32Value, stream);
}
else if (value is byte[] byteArrayValue)
{
PrimitiveSerializer.Char.WriteTo(PrefixByteArray, stream);
PrimitiveSerializer.Bytes.WriteTo(byteArrayValue, stream);
}
else if (value is LazyCompressedString lazyCompressedString)
{
PrimitiveSerializer.Char.WriteTo(PrefixCompressedStringByteArray, stream);
PrimitiveSerializer.Bytes.WriteTo(lazyCompressedString.GetBytes(), stream);
}
else if (value is sbyte signedByteValue)
{
PrimitiveSerializer.Char.WriteTo(PrefixSignedByte, stream);
PrimitiveSerializer.SByte.WriteTo(signedByteValue, stream);
}
else if (value is bool boolValue)
{
PrimitiveSerializer.Char.WriteTo(PrefixBool, stream);
PrimitiveSerializer.Boolean.WriteTo(boolValue, stream);
}
else if (value is Guid guidValue)
{
PrimitiveSerializer.Char.WriteTo(PrefixGuid, stream);
PrimitiveSerializer.Guid.WriteTo(guidValue, stream);
}
else if (value is TimeSpan timespanValue)
{
PrimitiveSerializer.Char.WriteTo(PrefixTimeSpan, stream);
PrimitiveSerializer.TimeSpan.WriteTo(timespanValue, stream);
}
else if (value is short int16Value)
{
PrimitiveSerializer.Char.WriteTo(PrefixInt16, stream);
PrimitiveSerializer.Int16.WriteTo(int16Value, stream);
}
else if (value is char charValue)
{
PrimitiveSerializer.Char.WriteTo(PrefixChar, stream);
PrimitiveSerializer.Char.WriteTo(charValue, stream);
}
else
throw new NotSupportedException("Value type " + value.GetType().FullName + " cannot be serialized.");
}