yv01p/Umbraco-CMS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactors the serialization of the content data that is stored in the nucache table. This had to change because we need to resolve content type data in order to check if the property should be compressed and we cannot do that data lookup while the data is being processed since we get an open data reader exception. This is fixed now by using a serializer factory instead so the Create method can do any initialization needed prior to running any serialization operation. Renames a few things so we dont have ContentNested (whatever that meant )

Browse Source
This commit is contained in:
Shannon
2020-09-25 00:32:11 +10:00
parent 780b2e573b
commit 67a9b5bb97
30 changed files with 377 additions and 205 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/Umbraco.Web/PublishedCache/NuCache/ContentNodeKit.cs
View File

@@ -15,6 +15,7 @@ namespace Umbraco.Web.PublishedCache.NuCache
public bool IsNull => ContentTypeId < 0;
public static ContentNodeKit Empty { get; } = new ContentNodeKit();
public static ContentNodeKit Null { get; } = new ContentNodeKit { ContentTypeId = -1 };
public void Build(

4
src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentNestedData.cs → src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentCacheDataModel.cs
View File

@@ -7,10 +7,10 @@ using Umbraco.Core.Serialization;
namespace Umbraco.Web.PublishedCache.NuCache.DataSource
{
/// <summary>
/// The content item 1:M data that is serialized to JSON
/// The content model stored in the content cache database table serialized as JSON
/// </summary>
[DataContract] // NOTE: Use DataContract annotations here to control how MessagePack serializes/deserializes the data to use INT keys
public class ContentNestedData
public class ContentCacheDataModel
{
// TODO: We don't want to allocate empty arrays
//dont serialize empty properties

47
src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentCacheDataSerializationResult.cs Normal file
View File

@@ -0,0 +1,47 @@
using System;
using System.Collections.Generic;
namespace Umbraco.Web.PublishedCache.NuCache.DataSource
{
public struct ContentCacheDataSerializationResult : IEquatable<ContentCacheDataSerializationResult>
{
public ContentCacheDataSerializationResult(string stringData, byte[] byteData)
{
StringData = stringData;
ByteData = byteData;
}
public string StringData { get; }
public byte[] ByteData { get; }
public override bool Equals(object obj)
{
return obj is ContentCacheDataSerializationResult result && Equals(result);
}
public bool Equals(ContentCacheDataSerializationResult other)
{
return StringData == other.StringData &&
EqualityComparer<byte[]>.Default.Equals(ByteData, other.ByteData);
}
public override int GetHashCode()
{
var hashCode = 1910544615;
hashCode = hashCode * -1521134295 + EqualityComparer<string>.Default.GetHashCode(StringData);
hashCode = hashCode * -1521134295 + EqualityComparer<byte[]>.Default.GetHashCode(ByteData);
return hashCode;
}
public static bool operator ==(ContentCacheDataSerializationResult left, ContentCacheDataSerializationResult right)
{
return left.Equals(right);
}
public static bool operator !=(ContentCacheDataSerializationResult left, ContentCacheDataSerializationResult right)
{
return !(left == right);
}
}
}

13
src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentCacheDataSerializerEntityType.cs Normal file
View File

@@ -0,0 +1,13 @@
using System;
namespace Umbraco.Web.PublishedCache.NuCache.DataSource
{
[Flags]
public enum ContentCacheDataSerializerEntityType
{
Document = 1,
Media = 2,
Member = 4
}
}

82
src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs
View File

@@ -2,14 +2,12 @@
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using Newtonsoft.Json;
using NPoco;
using Umbraco.Core;
using Umbraco.Core.Logging;
using Umbraco.Core.Persistence;
using Umbraco.Core.Persistence.Dtos;
using Umbraco.Core.Scoping;
using Umbraco.Core.Serialization;
using Umbraco.Web.Composing;
using static Umbraco.Core.Persistence.NPocoSqlExtensions.Statics;
@@ -21,11 +19,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
internal class DatabaseDataSource : IDataSource
{
private const int PageSize = 500;
private readonly IContentNestedDataSerializer _contentNestedDataSerializer;
private readonly IContentCacheDataSerializerFactory _contentCacheDataSerializerFactory;
public DatabaseDataSource(IContentNestedDataSerializer contentNestedDataSerializer)
public DatabaseDataSource(IContentCacheDataSerializerFactory contentCacheDataSerializerFactory)
{
_contentNestedDataSerializer = contentNestedDataSerializer;
_contentCacheDataSerializerFactory = contentCacheDataSerializerFactory;
}
// we want arrays, we want them all loaded, not an enumerable
@@ -110,7 +108,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
.OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
var dto = scope.Database.Fetch<ContentSourceDto>(sql).FirstOrDefault();
return dto == null ? new ContentNodeKit() : CreateContentNodeKit(dto);
if (dto == null) return ContentNodeKit.Empty;
var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Document);
return CreateContentNodeKit(dto, serializer);
}
public IEnumerable<ContentNodeKit> GetAllContentSources(IScope scope)
@@ -125,12 +127,14 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
.Where<NodeDto>(x => x.NodeObjectType == Constants.ObjectTypes.Document && !x.Trashed);
var sqlCount = scope.SqlContext.Sql("SELECT COUNT(*) FROM (").Append(sqlCountQuery).Append(") npoco_tbl");
var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Document);
// We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout.
// We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that.
foreach (var row in scope.Database.QueryPaged<ContentSourceDto>(PageSize, sql, sqlCount))
{
yield return CreateContentNodeKit(row);
yield return CreateContentNodeKit(row, serializer);
}
}
@@ -143,12 +147,14 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
.Where<NodeDto>(x => x.NodeId == id, "x")
.OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Document);
// We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout.
// We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that.
foreach (var row in scope.Database.QueryPaged<ContentSourceDto>(PageSize, sql))
{
yield return CreateContentNodeKit(row);
yield return CreateContentNodeKit(row, serializer);
}
}
@@ -161,12 +167,14 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
.WhereIn<ContentDto>(x => x.ContentTypeId, ids)
.OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Document);
// We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout.
// We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that.
foreach (var row in scope.Database.QueryPaged<ContentSourceDto>(PageSize, sql))
{
yield return CreateContentNodeKit(row);
yield return CreateContentNodeKit(row, serializer);
}
}
@@ -201,7 +209,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
.OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
var dto = scope.Database.Fetch<ContentSourceDto>(sql).FirstOrDefault();
return dto == null ? new ContentNodeKit() : CreateMediaNodeKit(dto);
if (dto == null) return ContentNodeKit.Empty;
var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Media);
return CreateMediaNodeKit(dto, serializer);
}
public IEnumerable<ContentNodeKit> GetAllMediaSources(IScope scope)
@@ -210,11 +222,15 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
.Where<NodeDto>(x => x.NodeObjectType == Constants.ObjectTypes.Media && !x.Trashed)
.OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Media);
// We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout.
// We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that.
foreach (var row in scope.Database.QueryPaged<ContentSourceDto>(PageSize, sql))
yield return CreateMediaNodeKit(row);
{
yield return CreateMediaNodeKit(row, serializer);
}
}
public IEnumerable<ContentNodeKit> GetBranchMediaSources(IScope scope, int id)
@@ -226,11 +242,15 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
.Where<NodeDto>(x => x.NodeId == id, "x")
.OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Media);
// We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout.
// We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that.
foreach (var row in scope.Database.QueryPaged<ContentSourceDto>(PageSize, sql))
yield return CreateMediaNodeKit(row);
{
yield return CreateMediaNodeKit(row, serializer);
}
}
public IEnumerable<ContentNodeKit> GetTypeMediaSources(IScope scope, IEnumerable<int> ids)
@@ -242,14 +262,18 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
.WhereIn<ContentDto>(x => x.ContentTypeId, ids)
.OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Media);
// We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout.
// We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that.
foreach (var row in scope.Database.QueryPaged<ContentSourceDto>(PageSize, sql))
yield return CreateMediaNodeKit(row);
{
yield return CreateMediaNodeKit(row, serializer);
}
}
private ContentNodeKit CreateContentNodeKit(ContentSourceDto dto)
private ContentNodeKit CreateContentNodeKit(ContentSourceDto dto, IContentCacheDataSerializer serializer)
{
ContentData d = null;
ContentData p = null;
@@ -264,9 +288,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
}
else
{
var nested = _contentNestedDataSerializer is IContentNestedDataByteSerializer byteSerializer
? byteSerializer.DeserializeBytes(dto.ContentTypeId, dto.EditDataRaw)
: _contentNestedDataSerializer.Deserialize(dto.ContentTypeId, dto.EditData);
var deserializedContent = serializer.Deserialize(dto.ContentTypeId, dto.EditData, dto.EditDataRaw);
d = new ContentData
{
@@ -276,9 +298,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
VersionId = dto.VersionId,
VersionDate = dto.EditVersionDate,
WriterId = dto.EditWriterId,
Properties = nested.PropertyData, // TODO: We don't want to allocate empty arrays
CultureInfos = nested.CultureData,
UrlSegment = nested.UrlSegment
Properties = deserializedContent.PropertyData, // TODO: We don't want to allocate empty arrays
CultureInfos = deserializedContent.CultureData,
UrlSegment = deserializedContent.UrlSegment
};
}
}
@@ -293,21 +315,19 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
}
else
{
var nested = _contentNestedDataSerializer is IContentNestedDataByteSerializer byteSerializer
? byteSerializer.DeserializeBytes(dto.ContentTypeId, dto.PubDataRaw)
: _contentNestedDataSerializer.Deserialize(dto.ContentTypeId, dto.PubData);
var deserializedContent = serializer.Deserialize(dto.ContentTypeId, dto.PubData, dto.PubDataRaw);
p = new ContentData
{
Name = dto.PubName,
UrlSegment = nested.UrlSegment,
UrlSegment = deserializedContent.UrlSegment,
Published = true,
TemplateId = dto.PubTemplateId,
VersionId = dto.VersionId,
VersionDate = dto.PubVersionDate,
WriterId = dto.PubWriterId,
Properties = nested.PropertyData, // TODO: We don't want to allocate empty arrays
CultureInfos = nested.CultureData
Properties = deserializedContent.PropertyData, // TODO: We don't want to allocate empty arrays
CultureInfos = deserializedContent.CultureData
};
}
}
@@ -326,14 +346,12 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
return s;
}
private ContentNodeKit CreateMediaNodeKit(ContentSourceDto dto)
private ContentNodeKit CreateMediaNodeKit(ContentSourceDto dto, IContentCacheDataSerializer serializer)
{
if (dto.EditData == null && dto.EditDataRaw == null)
throw new InvalidOperationException("No data for media " + dto.Id);
var nested = _contentNestedDataSerializer is IContentNestedDataByteSerializer byteSerializer
? byteSerializer.DeserializeBytes(dto.ContentTypeId, dto.EditDataRaw)
: _contentNestedDataSerializer.Deserialize(dto.ContentTypeId, dto.EditData);
var deserializedMedia = serializer.Deserialize(dto.ContentTypeId, dto.EditData, dto.EditDataRaw);
var p = new ContentData
{
@@ -343,8 +361,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
VersionId = dto.VersionId,
VersionDate = dto.EditVersionDate,
WriterId = dto.CreatorId, // what-else?
Properties = nested.PropertyData, // TODO: We don't want to allocate empty arrays
CultureInfos = nested.CultureData
Properties = deserializedMedia.PropertyData, // TODO: We don't want to allocate empty arrays
CultureInfos = deserializedMedia.CultureData
};
var n = new ContentNode(dto.Id, dto.Uid,

18
src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializer.cs Normal file
View File

@@ -0,0 +1,18 @@
using Umbraco.Core.PropertyEditors;
namespace Umbraco.Web.PublishedCache.NuCache.DataSource
{
/// <summary>
/// Serializes/Deserializes <see cref="ContentCacheDataModel"/> document to the SQL Database as a string
/// </summary>
/// <remarks>
/// Resolved from the <see cref="IContentCacheDataSerializerFactory"/>. This cannot be resolved from DI.
/// </remarks>
public interface IContentCacheDataSerializer
{
ContentCacheDataModel Deserialize(int contentTypeId, string stringData, byte[] byteData);
ContentCacheDataSerializationResult Serialize(int contentTypeId, ContentCacheDataModel model);
}
}

16
src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializerFactory.cs Normal file
View File

@@ -0,0 +1,16 @@
namespace Umbraco.Web.PublishedCache.NuCache.DataSource
{
public interface IContentCacheDataSerializerFactory
{
/// <summary>
/// Gets or creates a new instance of <see cref="IContentCacheDataSerializer"/>
/// </summary>
/// <returns></returns>
/// <remarks>
/// This method may return the same instance, however this depends on the state of the application and if any underlying data has changed.
/// This method may also be used to initialize anything before a serialization/deserialization session occurs.
/// </remarks>
IContentCacheDataSerializer Create(ContentCacheDataSerializerEntityType types);
}
}

22
src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs
View File

@@ -1,22 +0,0 @@
namespace Umbraco.Web.PublishedCache.NuCache.DataSource
{
// TODO: We need better names if possible, not sure why the class is called ContentNested in the first place
/// <summary>
/// Serializes/Deserializes <see cref="ContentNestedData"/> document to the SQL Database as bytes
/// </summary>
public interface IContentNestedDataByteSerializer : IContentNestedDataSerializer
{
ContentNestedData DeserializeBytes(int contentTypeId, byte[] data);
byte[] SerializeBytes(int contentTypeId, ContentNestedData nestedData);
}
/// <summary>
/// Serializes/Deserializes <see cref="ContentNestedData"/> document to the SQL Database as a string
/// </summary>
public interface IContentNestedDataSerializer
{
ContentNestedData Deserialize(int contentTypeId, string data);
string Serialize(int contentTypeId, ContentNestedData nestedData);
}
}

21
src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs
View File

@@ -1,18 +1,18 @@
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Reflection.Emit;
using System.Text;
using System.Threading.Tasks;
using Umbraco.Core.Serialization;
namespace Umbraco.Web.PublishedCache.NuCache.DataSource
{
internal class JsonContentNestedDataSerializer : IContentNestedDataSerializer
public class JsonContentNestedDataSerializer : IContentCacheDataSerializer
{
public ContentNestedData Deserialize(int contentTypeId, string data)
public ContentCacheDataModel Deserialize(int contentTypeId, string stringData, byte[] byteData)
{
if (byteData != null)
throw new NotSupportedException($"{typeof(JsonContentNestedDataSerializer)} does not support byte[] serialization");
// by default JsonConvert will deserialize our numeric values as Int64
// which is bad, because they were Int32 in the database - take care
@@ -24,18 +24,19 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
DateParseHandling = DateParseHandling.DateTime,
DateFormatHandling = DateFormatHandling.IsoDateFormat,
DateTimeZoneHandling = DateTimeZoneHandling.Utc,
DateFormatString = "o"
DateFormatString = "o"
};
return JsonConvert.DeserializeObject<ContentNestedData>(data, settings);
return JsonConvert.DeserializeObject<ContentCacheDataModel>(stringData, settings);
}
public string Serialize(int contentTypeId, ContentNestedData nestedData)
public ContentCacheDataSerializationResult Serialize(int contentTypeId, ContentCacheDataModel model)
{
// note that numeric values (which are Int32) are serialized without their
// type (eg "value":1234) and JsonConvert by default deserializes them as Int64
return JsonConvert.SerializeObject(nestedData);
var json = JsonConvert.SerializeObject(model);
return new ContentCacheDataSerializationResult(json, null);
}
}
}

10
src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializerFactory.cs Normal file
View File

@@ -0,0 +1,10 @@
using System;
namespace Umbraco.Web.PublishedCache.NuCache.DataSource
{
internal class JsonContentNestedDataSerializerFactory : IContentCacheDataSerializerFactory
{
private Lazy<JsonContentNestedDataSerializer> _serializer = new Lazy<JsonContentNestedDataSerializer>();
public IContentCacheDataSerializer Create(ContentCacheDataSerializerEntityType types) => _serializer.Value;
}
}

77
src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs
View File

@@ -1,29 +1,27 @@
using K4os.Compression.LZ4;
using MessagePack;
using MessagePack.Formatters;
using MessagePack.Resolvers;
using NPoco.FluentMappings;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using Umbraco.Core.PropertyEditors;
using Umbraco.Web.PropertyEditors;
namespace Umbraco.Web.PublishedCache.NuCache.DataSource
{
/// <summary>
/// Serializes/Deserializes <see cref="ContentNestedData"/> document to the SQL Database as bytes using MessagePack
/// Serializes/Deserializes <see cref="ContentCacheDataModel"/> document to the SQL Database as bytes using MessagePack
/// </summary>
internal class MsgPackContentNestedDataSerializer : IContentNestedDataByteSerializer
public class MsgPackContentNestedDataSerializer : IContentCacheDataSerializer
{
private MessagePackSerializerOptions _options;
private readonly MessagePackSerializerOptions _options;
private readonly IPropertyCompressionOptions _propertyOptions;
public MsgPackContentNestedDataSerializer(IPropertyCompressionOptions propertyOptions = null)
public MsgPackContentNestedDataSerializer(IPropertyCompressionOptions propertyOptions)
{
var defaultOptions = ContractlessStandardResolver.Options;
_propertyOptions = propertyOptions ?? throw new ArgumentNullException(nameof(propertyOptions));
var defaultOptions = ContractlessStandardResolver.Options;
var resolver = CompositeResolver.Create(
// TODO: We want to be able to intern the strings for aliases when deserializing like we do for Newtonsoft but I'm unsure exactly how
@@ -39,52 +37,51 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
_options = defaultOptions
.WithResolver(resolver)
.WithCompression(MessagePackCompression.Lz4BlockArray);
_propertyOptions = propertyOptions ?? new NoopPropertyCompressionOptions();
.WithCompression(MessagePackCompression.Lz4BlockArray);
}
public string ToJson(string serialized)
public string ToJson(byte[] bin)
{
var bin = Convert.FromBase64String(serialized);
var json = MessagePackSerializer.ConvertToJson(bin, _options);
return json;
}
public ContentNestedData Deserialize(int contentTypeId, string data)
public ContentCacheDataModel Deserialize(int contentTypeId, string stringData, byte[] byteData)
{
var bin = Convert.FromBase64String(data);
var nestedData = MessagePackSerializer.Deserialize<ContentNestedData>(bin, _options);
Expand(contentTypeId, nestedData);
return nestedData;
if (stringData != null)
{
// NOTE: We don't really support strings but it's possible if manually used (i.e. tests)
var bin = Convert.FromBase64String(stringData);
var content = MessagePackSerializer.Deserialize<ContentCacheDataModel>(bin, _options);
Expand(contentTypeId, content);
return content;
}
else if (byteData != null)
{
var content = MessagePackSerializer.Deserialize<ContentCacheDataModel>(byteData, _options);
Expand(contentTypeId, content);
return content;
}
else
{
return null;
}
}
public string Serialize(int contentTypeId, ContentNestedData nestedData)
public ContentCacheDataSerializationResult Serialize(int contentTypeId, ContentCacheDataModel model)
{
Compress(contentTypeId, nestedData);
var bin = MessagePackSerializer.Serialize(nestedData, _options);
return Convert.ToBase64String(bin);
}
public ContentNestedData DeserializeBytes(int contentTypeId, byte[] data)
{
var nestedData = MessagePackSerializer.Deserialize<ContentNestedData>(data, _options);
Expand(contentTypeId, nestedData);
return nestedData;
}
public byte[] SerializeBytes(int contentTypeId, ContentNestedData nestedData)
{
Compress(contentTypeId, nestedData);
return MessagePackSerializer.Serialize(nestedData, _options);
Compress(contentTypeId, model);
var bytes = MessagePackSerializer.Serialize(model, _options);
return new ContentCacheDataSerializationResult(null, bytes);
}
/// <summary>
/// Used during serialization to compress properties
/// </summary>
/// <param name="nestedData"></param>
private void Compress(int contentTypeId, ContentNestedData nestedData)
/// <param name="model"></param>
private void Compress(int contentTypeId, ContentCacheDataModel model)
{
foreach(var propertyAliasToData in nestedData.PropertyData)
foreach(var propertyAliasToData in model.PropertyData)
{
if (_propertyOptions.IsCompressed(contentTypeId, propertyAliasToData.Key))
{
@@ -100,7 +97,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
/// Used during deserialization to map the property data as lazy or expand the value
/// </summary>
/// <param name="nestedData"></param>
private void Expand(int contentTypeId, ContentNestedData nestedData)
private void Expand(int contentTypeId, ContentCacheDataModel nestedData)
{
foreach (var propertyAliasToData in nestedData.PropertyData)
{
@@ -117,6 +114,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
}
}
//private class ContentNestedDataResolver : IFormatterResolver
//{
// // GetFormatter<T>'s get cost should be minimized so use type cache.

62
src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializerFactory.cs Normal file
View File

@@ -0,0 +1,62 @@
using System.Collections.Concurrent;
using System.Collections.Generic;
using Umbraco.Core.Models;
using Umbraco.Core.PropertyEditors;
using Umbraco.Core.Services;
namespace Umbraco.Web.PublishedCache.NuCache.DataSource
{
internal class MsgPackContentNestedDataSerializerFactory : IContentCacheDataSerializerFactory
{
private readonly IContentTypeService _contentTypeService;
private readonly IMediaTypeService _mediaTypeService;
private readonly IMemberTypeService _memberTypeService;
private readonly PropertyEditorCollection _propertyEditors;
private readonly ConcurrentDictionary<(int, string), CompressedStorageAttribute> _compressedStoragePropertyEditorCache = new ConcurrentDictionary<(int, string), CompressedStorageAttribute>();
public MsgPackContentNestedDataSerializerFactory(IContentTypeService contentTypeService, IMediaTypeService mediaTypeService, IMemberTypeService memberTypeService, PropertyEditorCollection propertyEditors)
{
_contentTypeService = contentTypeService;
_mediaTypeService = mediaTypeService;
_memberTypeService = memberTypeService;
_propertyEditors = propertyEditors;
}
public IContentCacheDataSerializer Create(ContentCacheDataSerializerEntityType types)
{
// Depending on which entity types are being requested, we need to look up those content types
// to initialize the compression options.
// We need to initialize these options now so that any data lookups required are completed and are not done while the content cache
// is performing DB queries which will result in errors since we'll be trying to query with open readers.
// NOTE: The calls to GetAll() below should be cached if the data has not been changed.
var contentTypes = new Dictionary<int, IContentTypeComposition>();
if ((types & ContentCacheDataSerializerEntityType.Document) == ContentCacheDataSerializerEntityType.Document)
{
foreach(var ct in _contentTypeService.GetAll())
{
contentTypes[ct.Id] = ct;
}
}
if ((types & ContentCacheDataSerializerEntityType.Media) == ContentCacheDataSerializerEntityType.Media)
{
foreach (var ct in _mediaTypeService.GetAll())
{
contentTypes[ct.Id] = ct;
}
}
if ((types & ContentCacheDataSerializerEntityType.Member) == ContentCacheDataSerializerEntityType.Member)
{
foreach (var ct in _memberTypeService.GetAll())
{
contentTypes[ct.Id] = ct;
}
}
var options = new CompressedStoragePropertyEditorCompressionOptions(contentTypes, _propertyEditors, _compressedStoragePropertyEditorCache);
var serializer = new MsgPackContentNestedDataSerializer(options);
return serializer;
}
}
}

11
src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs
View File

@@ -1,10 +1,7 @@
using System.Collections.Generic;
using System.Configuration;
using System.Linq;
using System.Configuration;
using Umbraco.Core;
using Umbraco.Core.Composing;
using Umbraco.Core.PropertyEditors;
using Umbraco.Web.PropertyEditors;
using Umbraco.Web.PublishedCache.NuCache.DataSource;
namespace Umbraco.Web.PublishedCache.NuCache
@@ -19,13 +16,11 @@ namespace Umbraco.Web.PublishedCache.NuCache
if (serializer != "MsgPack")
{
// TODO: This allows people to revert to the legacy serializer, by default it will be MessagePack
composition.RegisterUnique<IContentNestedDataSerializer, JsonContentNestedDataSerializer>();
composition.RegisterUnique<IPropertyCompressionOptions, NoopPropertyCompressionOptions>();
composition.RegisterUnique<IContentCacheDataSerializerFactory, JsonContentNestedDataSerializerFactory>();
}
else
{
composition.RegisterUnique<IContentNestedDataSerializer, MsgPackContentNestedDataSerializer>();
composition.RegisterUnique<IPropertyCompressionOptions, CompressedStoragePropertyEditorCompressionOptions>();
composition.RegisterUnique<IContentCacheDataSerializerFactory, MsgPackContentNestedDataSerializerFactory>();
}
composition.RegisterUnique(factory => new ContentDataSerializer(new DictionaryOfPropertyDataSerializer()));

68
src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs
View File

@@ -1,14 +1,10 @@
using System;
using System.Collections.Generic;
using System.Configuration;
using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using CSharpTest.Net.Collections;
using Newtonsoft.Json;
using Umbraco.Core;
using Umbraco.Core.Cache;
using Umbraco.Core.Configuration;
@@ -49,7 +45,7 @@ namespace Umbraco.Web.PublishedCache.NuCache
private readonly IPublishedModelFactory _publishedModelFactory;
private readonly IDefaultCultureAccessor _defaultCultureAccessor;
private readonly UrlSegmentProviderCollection _urlSegmentProviders;
private readonly IContentNestedDataSerializer _contentNestedDataSerializer;
private readonly IContentCacheDataSerializerFactory _contentCacheDataSerializerFactory;
private readonly ContentDataSerializer _contentDataSerializer;
// volatile because we read it with no lock
@@ -84,7 +80,7 @@ namespace Umbraco.Web.PublishedCache.NuCache
IDataSource dataSource, IGlobalSettings globalSettings,
IEntityXmlSerializer entitySerializer,
IPublishedModelFactory publishedModelFactory,
UrlSegmentProviderCollection urlSegmentProviders, IContentNestedDataSerializer contentNestedDataSerializer, ContentDataSerializer contentDataSerializer = null)
UrlSegmentProviderCollection urlSegmentProviders, IContentCacheDataSerializerFactory contentCacheDataSerializerFactory, ContentDataSerializer contentDataSerializer = null)
: base(publishedSnapshotAccessor, variationContextAccessor)
{
//if (Interlocked.Increment(ref _singletonCheck) > 1)
@@ -101,7 +97,7 @@ namespace Umbraco.Web.PublishedCache.NuCache
_defaultCultureAccessor = defaultCultureAccessor;
_globalSettings = globalSettings;
_urlSegmentProviders = urlSegmentProviders;
_contentNestedDataSerializer = contentNestedDataSerializer;
_contentCacheDataSerializerFactory = contentCacheDataSerializerFactory;
_contentDataSerializer = contentDataSerializer;
// we need an Xml serializer here so that the member cache can support XPath,
@@ -1286,8 +1282,10 @@ namespace Umbraco.Web.PublishedCache.NuCache
var db = args.Scope.Database;
var content = (Content)args.Entity;
var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Document);
// always refresh the edited data
OnRepositoryRefreshed(db, content, false);
OnRepositoryRefreshed(serializer, db, content, false);
// if unpublishing, remove published data from table
if (content.PublishedState == PublishedState.Unpublishing)
@@ -1295,33 +1293,37 @@ namespace Umbraco.Web.PublishedCache.NuCache
// if publishing, refresh the published data
else if (content.PublishedState == PublishedState.Publishing)
OnRepositoryRefreshed(db, content, true);
OnRepositoryRefreshed(serializer, db, content, true);
}
private void OnMediaRefreshedEntity(MediaRepository sender, MediaRepository.ScopedEntityEventArgs args)
{
var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Media);
var db = args.Scope.Database;
var media = args.Entity;
// refresh the edited data
OnRepositoryRefreshed(db, media, false);
OnRepositoryRefreshed(serializer, db, media, false);
}
private void OnMemberRefreshedEntity(MemberRepository sender, MemberRepository.ScopedEntityEventArgs args)
{
var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Member);
var db = args.Scope.Database;
var member = args.Entity;
// refresh the edited data
OnRepositoryRefreshed(db, member, false);
OnRepositoryRefreshed(serializer, db, member, false);
}
private void OnRepositoryRefreshed(IUmbracoDatabase db, IContentBase content, bool published)
private void OnRepositoryRefreshed(IContentCacheDataSerializer serializer, IUmbracoDatabase db, IContentBase content, bool published)
{
// use a custom SQL to update row version on each update
//db.InsertOrUpdate(dto);
var dto = GetDto(content, published);
var dto = GetDto(content, published, serializer);
db.InsertOrUpdate(dto,
"SET data=@data, dataRaw=@dataRaw, rv=rv+1 WHERE nodeId=@id AND published=@published",
new
@@ -1375,7 +1377,7 @@ namespace Umbraco.Web.PublishedCache.NuCache
}
}
private ContentNuDto GetDto(IContentBase content, bool published)
private ContentNuDto GetDto(IContentBase content, bool published, IContentCacheDataSerializer serializer)
{
// should inject these in ctor
// BUT for the time being we decide not to support ConvertDbToXml/String
@@ -1447,19 +1449,21 @@ namespace Umbraco.Web.PublishedCache.NuCache
}
//the dictionary that will be serialized
var nestedData = new ContentNestedData
var contentCacheData = new ContentCacheDataModel
{
PropertyData = propertyData,
CultureData = cultureData,
UrlSegment = content.GetUrlSegment(_urlSegmentProviders)
};
var serialized = serializer.Serialize(content.ContentTypeId, contentCacheData);
var dto = new ContentNuDto
{
NodeId = content.Id,
Published = published,
Data = !(_contentNestedDataSerializer is IContentNestedDataByteSerializer) ? _contentNestedDataSerializer.Serialize(content.ContentTypeId, nestedData) : null,
RawData = (_contentNestedDataSerializer is IContentNestedDataByteSerializer byteSerializer) ? byteSerializer.SerializeBytes(content.ContentTypeId, nestedData) : null
Data = serialized.StringData,
RawData = serialized.ByteData
};
//Core.Composing.Current.Logger.Debug<PublishedSnapshotService>(dto.Data);
@@ -1482,30 +1486,32 @@ namespace Umbraco.Web.PublishedCache.NuCache
public override void Rebuild()
{
_logger.Debug<PublishedSnapshotService>("Rebuilding...");
var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Document | ContentCacheDataSerializerEntityType.Media | ContentCacheDataSerializerEntityType.Member);
using (var scope = _scopeProvider.CreateScope(repositoryCacheMode: RepositoryCacheMode.Scoped))
{
scope.ReadLock(Constants.Locks.ContentTree);
scope.ReadLock(Constants.Locks.MediaTree);
scope.ReadLock(Constants.Locks.MemberTree);
RebuildContentDbCacheLocked(scope, GetSqlPagingSize(), null);
RebuildMediaDbCacheLocked(scope, GetSqlPagingSize(), null);
RebuildMemberDbCacheLocked(scope, GetSqlPagingSize(), null);
RebuildContentDbCacheLocked(serializer, scope, GetSqlPagingSize(), null);
RebuildMediaDbCacheLocked(serializer, scope, GetSqlPagingSize(), null);
RebuildMemberDbCacheLocked(serializer, scope, GetSqlPagingSize(), null);
scope.Complete();
}
}
public void RebuildContentDbCache(int groupSize = DefaultSqlPagingSize, IEnumerable<int> contentTypeIds = null)
{
var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Document);
using (var scope = _scopeProvider.CreateScope(repositoryCacheMode: RepositoryCacheMode.Scoped))
{
scope.ReadLock(Constants.Locks.ContentTree);
RebuildContentDbCacheLocked(scope, groupSize, contentTypeIds);
RebuildContentDbCacheLocked(serializer, scope, groupSize, contentTypeIds);
scope.Complete();
}
}
// assumes content tree lock
private void RebuildContentDbCacheLocked(IScope scope, int groupSize, IEnumerable<int> contentTypeIds)
private void RebuildContentDbCacheLocked(IContentCacheDataSerializer serializer, IScope scope, int groupSize, IEnumerable<int> contentTypeIds)
{
var contentTypeIdsA = contentTypeIds?.ToArray();
var contentObjectType = Constants.ObjectTypes.Document;
@@ -1552,11 +1558,11 @@ WHERE cmsContentNu.nodeId IN (
foreach (var c in descendants)
{
// always the edited version
items.Add(GetDto(c, false));
items.Add(GetDto(c, false, serializer));
// and also the published version if it makes any sense
if (c.Published)
items.Add(GetDto(c, true));
items.Add(GetDto(c, true, serializer));
count++;
}
@@ -1568,16 +1574,17 @@ WHERE cmsContentNu.nodeId IN (
public void RebuildMediaDbCache(int groupSize = DefaultSqlPagingSize, IEnumerable<int> contentTypeIds = null)
{
var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Media);
using (var scope = _scopeProvider.CreateScope(repositoryCacheMode: RepositoryCacheMode.Scoped))
{
scope.ReadLock(Constants.Locks.MediaTree);
RebuildMediaDbCacheLocked(scope, groupSize, contentTypeIds);
RebuildMediaDbCacheLocked(serializer, scope, groupSize, contentTypeIds);
scope.Complete();
}
}
// assumes media tree lock
public void RebuildMediaDbCacheLocked(IScope scope, int groupSize, IEnumerable<int> contentTypeIds)
public void RebuildMediaDbCacheLocked(IContentCacheDataSerializer serializer, IScope scope, int groupSize, IEnumerable<int> contentTypeIds)
{
var contentTypeIdsA = contentTypeIds?.ToArray();
var mediaObjectType = Constants.ObjectTypes.Media;
@@ -1619,7 +1626,7 @@ WHERE cmsContentNu.nodeId IN (
{
// the tree is locked, counting and comparing to total is safe
var descendants = _mediaRepository.GetPage(query, pageIndex++, groupSize, out total, null, Ordering.By("Path"));
var items = descendants.Select(m => GetDto(m, false)).ToList();
var items = descendants.Select(m => GetDto(m, false, serializer)).ToList();
db.BulkInsertRecords(items);
processed += items.Count;
} while (processed < total);
@@ -1627,16 +1634,17 @@ WHERE cmsContentNu.nodeId IN (
public void RebuildMemberDbCache(int groupSize = DefaultSqlPagingSize, IEnumerable<int> contentTypeIds = null)
{
var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Member);
using (var scope = _scopeProvider.CreateScope(repositoryCacheMode: RepositoryCacheMode.Scoped))
{
scope.ReadLock(Constants.Locks.MemberTree);
RebuildMemberDbCacheLocked(scope, groupSize, contentTypeIds);
RebuildMemberDbCacheLocked(serializer, scope, groupSize, contentTypeIds);
scope.Complete();
}
}
// assumes member tree lock
public void RebuildMemberDbCacheLocked(IScope scope, int groupSize, IEnumerable<int> contentTypeIds)
public void RebuildMemberDbCacheLocked(IContentCacheDataSerializer serializer, IScope scope, int groupSize, IEnumerable<int> contentTypeIds)
{
var contentTypeIdsA = contentTypeIds?.ToArray();
var memberObjectType = Constants.ObjectTypes.Member;
@@ -1677,7 +1685,7 @@ WHERE cmsContentNu.nodeId IN (
do
{
var descendants = _memberRepository.GetPage(query, pageIndex++, groupSize, out total, null, Ordering.By("Path"));
var items = descendants.Select(m => GetDto(m, false)).ToArray();
var items = descendants.Select(m => GetDto(m, false, serializer)).ToArray();
db.BulkInsertRecords(items);
processed += items.Length;
} while (processed < total);