yv01p/Umbraco-CMS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/v9/dev' into patch-separate-sqlce-from-integration-tests

Browse Source
This commit is contained in:
Bjarke Berg
2021-08-04 10:50:33 +02:00
parent 37066bb0e1 d1a004078f
commit e1dab63123
385 changed files with 4402 additions and 19726 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/Umbraco.PublishedCache.NuCache/DataSource/IContentCacheDataSerializer.cs
View File

@@ -14,12 +14,12 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
/// <summary>
/// Deserialize the data into a <see cref="ContentCacheDataModel"/>
/// </summary>
ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData);
ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData, bool published);
/// <summary>
/// Serializes the <see cref="ContentCacheDataModel"/>
/// Serializes the <see cref="ContentCacheDataModel"/>
/// </summary>
ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model);
ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model, bool published);
}
}

4
src/Umbraco.PublishedCache.NuCache/DataSource/JsonContentNestedDataSerializer.cs
View File

@@ -24,7 +24,7 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
DateFormatString = "o"
};
private readonly JsonNameTable _propertyNameTable = new DefaultJsonNameTable();
public ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData)
public ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData, bool published)
{
if (stringData == null && byteData != null)
throw new NotSupportedException($"{typeof(JsonContentNestedDataSerializer)} does not support byte[] serialization");
@@ -39,7 +39,7 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
}
}
public ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model)
public ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model, bool published)
{
// note that numeric values (which are Int32) are serialized without their
// type (eg "value":1234) and JsonConvert by default deserializes them as Int64

25
src/Umbraco.PublishedCache.NuCache/DataSource/MsgPackContentNestedDataSerializer.cs
View File

@@ -39,7 +39,8 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
_options = defaultOptions
.WithResolver(resolver)
.WithCompression(MessagePackCompression.Lz4BlockArray);
.WithCompression(MessagePackCompression.Lz4BlockArray)
.WithSecurity(MessagePackSecurity.UntrustedData);
}
public string ToJson(byte[] bin)
@@ -48,12 +49,12 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
return json;
}
public ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData)
public ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData, bool published)
{
if (byteData != null)
{
var cacheModel = MessagePackSerializer.Deserialize<ContentCacheDataModel>(byteData, _options);
Expand(content, cacheModel);
Expand(content, cacheModel, published);
return cacheModel;
}
else if (stringData != null)
@@ -61,7 +62,7 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
// NOTE: We don't really support strings but it's possible if manually used (i.e. tests)
var bin = Convert.FromBase64String(stringData);
var cacheModel = MessagePackSerializer.Deserialize<ContentCacheDataModel>(bin, _options);
Expand(content, cacheModel);
Expand(content, cacheModel, published);
return cacheModel;
}
else
@@ -70,9 +71,9 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
}
}
public ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model)
public ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model, bool published)
{
Compress(content, model);
Compress(content, model, published);
var bytes = MessagePackSerializer.Serialize(model, _options);
return new ContentCacheDataSerializationResult(null, bytes);
}
@@ -80,7 +81,9 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
/// <summary>
/// Used during serialization to compress properties
/// </summary>
/// <param name="content"></param>
/// <param name="model"></param>
/// <param name="published"></param>
/// <remarks>
/// This will essentially 'double compress' property data. The MsgPack data as a whole will already be compressed
/// but this will go a step further and double compress property data so that it is stored in the nucache file
@@ -88,11 +91,11 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
/// read/decompressed as a string to be displayed on the front-end. This allows for potentially a significant
/// memory savings but could also affect performance of first rendering pages while decompression occurs.
/// </remarks>
private void Compress(IReadOnlyContentBase content, ContentCacheDataModel model)
private void Compress(IReadOnlyContentBase content, ContentCacheDataModel model, bool published)
{
foreach(var propertyAliasToData in model.PropertyData)
{
if (_propertyOptions.IsCompressed(content, propertyAliasToData.Key))
if (_propertyOptions.IsCompressed(content, propertyAliasToData.Key, published))
{
foreach(var property in propertyAliasToData.Value.Where(x => x.Value != null && x.Value is string))
{
@@ -105,12 +108,14 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
/// <summary>
/// Used during deserialization to map the property data as lazy or expand the value
/// </summary>
/// <param name="content"></param>
/// <param name="nestedData"></param>
private void Expand(IReadOnlyContentBase content, ContentCacheDataModel nestedData)
/// <param name="published"></param>
private void Expand(IReadOnlyContentBase content, ContentCacheDataModel nestedData, bool published)
{
foreach (var propertyAliasToData in nestedData.PropertyData)
{
if (_propertyOptions.IsCompressed(content, propertyAliasToData.Key))
if (_propertyOptions.IsCompressed(content, propertyAliasToData.Key,published))
{
foreach (var property in propertyAliasToData.Value.Where(x => x.Value != null))
{

3
src/Umbraco.PublishedCache.NuCache/DataSource/MsgPackContentNestedDataSerializerFactory.cs
View File

@@ -3,7 +3,6 @@ using System.Collections.Generic;
using Umbraco.Cms.Core.Models;
using Umbraco.Cms.Core.PropertyEditors;
using Umbraco.Cms.Core.Services;
using Umbraco.Core.PropertyEditors;
namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
{
@@ -14,7 +13,7 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
private readonly IMemberTypeService _memberTypeService;
private readonly PropertyEditorCollection _propertyEditors;
private readonly IPropertyCacheCompressionOptions _compressionOptions;
private readonly ConcurrentDictionary<(int, string), bool> _isCompressedCache = new ConcurrentDictionary<(int, string), bool>();
private readonly ConcurrentDictionary<(int, string, bool), bool> _isCompressedCache = new ConcurrentDictionary<(int, string, bool), bool>();
public MsgPackContentNestedDataSerializerFactory(
IContentTypeService contentTypeService,

15
src/Umbraco.PublishedCache.NuCache/DependencyInjection/UmbracoBuilderExtensions.cs
View File

@@ -68,7 +68,20 @@ namespace Umbraco.Extensions
throw new IndexOutOfRangeException();
}
});
builder.Services.AddSingleton<IPropertyCacheCompressionOptions, NoopPropertyCacheCompressionOptions>();
builder.Services.AddSingleton<IPropertyCacheCompressionOptions>(s =>
{
IOptions<NuCacheSettings> options = s.GetRequiredService<IOptions<NuCacheSettings>>();
if (options.Value.NuCacheSerializerType == NuCacheSerializerType.MessagePack &&
options.Value.UnPublishedContentCompression)
{
return new UnPublishedContentPropertyCacheCompressionOptions();
}
return new NoopPropertyCacheCompressionOptions();
});
builder.Services.AddSingleton(s => new ContentDataSerializer(new DictionaryOfPropertyDataSerializer()));
// add the NuCache health check (hidden from type finder)

17
src/Umbraco.PublishedCache.NuCache/Persistence/NuCacheContentRepository.cs
View File

@@ -415,7 +415,7 @@ AND cmsContentNu.nodeId IS NULL
UrlSegment = content.GetUrlSegment(_shortStringHelper, _urlSegmentProviders)
};
var serialized = serializer.Serialize(ReadOnlyContentBaseAdapter.Create(content), contentCacheData);
var serialized = serializer.Serialize(ReadOnlyContentBaseAdapter.Create(content), contentCacheData, published);
var dto = new ContentNuDto
{
@@ -817,12 +817,13 @@ AND cmsContentNu.nodeId IS NULL
}
else
{
var deserializedContent = serializer.Deserialize(dto, dto.EditData, dto.EditDataRaw);
bool published = false;
var deserializedContent = serializer.Deserialize(dto, dto.EditData, dto.EditDataRaw, published);
d = new ContentData
{
Name = dto.EditName,
Published = false,
Published = published,
TemplateId = dto.EditTemplateId,
VersionId = dto.VersionId,
VersionDate = dto.EditVersionDate,
@@ -847,13 +848,14 @@ AND cmsContentNu.nodeId IS NULL
}
else
{
var deserializedContent = serializer.Deserialize(dto, dto.PubData, dto.PubDataRaw);
bool published = true;
var deserializedContent = serializer.Deserialize(dto, dto.PubData, dto.PubDataRaw, published);
p = new ContentData
{
Name = dto.PubName,
UrlSegment = deserializedContent.UrlSegment,
Published = true,
Published = published,
TemplateId = dto.PubTemplateId,
VersionId = dto.VersionId,
VersionDate = dto.PubVersionDate,
@@ -883,12 +885,13 @@ AND cmsContentNu.nodeId IS NULL
if (dto.EditData == null && dto.EditDataRaw == null)
throw new InvalidOperationException("No data for media " + dto.Id);
var deserializedMedia = serializer.Deserialize(dto, dto.EditData, dto.EditDataRaw);
bool published = true;
var deserializedMedia = serializer.Deserialize(dto, dto.EditData, dto.EditDataRaw, published);
var p = new ContentData
{
Name = dto.EditName,
Published = true,
Published = published,
TemplateId = -1,
VersionId = dto.VersionId,
VersionDate = dto.EditVersionDate,