yv01p/Umbraco-CMS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

merged v9/dev into this branch

Browse Source
This commit is contained in:
Zeegaan
2021-08-10 11:15:23 +02:00
parent 5d264fefdb 5c1712cba2
commit f3db706808
136 changed files with 1653 additions and 946 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/Umbraco.PublishedCache.NuCache/DataSource/BTree.ContentDataSerializer.cs
View File

@@ -1,4 +1,4 @@
using System.IO;
using System.IO;
using CSharpTest.Net.Serialization;
namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
@@ -6,7 +6,7 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
/// <summary>
/// Serializes/Deserializes data to BTree data source for <see cref="ContentData"/>
/// </summary>
internal class ContentDataSerializer : ISerializer<ContentData>
public class ContentDataSerializer : ISerializer<ContentData>
{
public ContentDataSerializer(IDictionaryOfPropertyDataSerializer dictionaryOfPropertyDataSerializer = null)
{

4
src/Umbraco.PublishedCache.NuCache/DataSource/BTree.cs
View File

@@ -1,11 +1,11 @@
using System.Configuration;
using System.Configuration;
using CSharpTest.Net.Collections;
using CSharpTest.Net.Serialization;
using Umbraco.Cms.Core.Configuration.Models;
namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
{
internal class BTree
public class BTree
{
public static BPlusTree<int, ContentNodeKit> GetTree(string filepath, bool exists, NuCacheSettings settings, ContentDataSerializer contentDataSerializer = null)
{

6
src/Umbraco.PublishedCache.NuCache/DataSource/IContentCacheDataSerializer.cs
View File

@@ -14,12 +14,12 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
/// <summary>
/// Deserialize the data into a <see cref="ContentCacheDataModel"/>
/// </summary>
ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData);
ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData, bool published);
/// <summary>
/// Serializes the <see cref="ContentCacheDataModel"/>
/// Serializes the <see cref="ContentCacheDataModel"/>
/// </summary>
ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model);
ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model, bool published);
}
}

2
src/Umbraco.PublishedCache.NuCache/DataSource/IDictionaryOfPropertyDataSerializer.cs
View File

@@ -3,7 +3,7 @@ using System.IO;
namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
{
internal interface IDictionaryOfPropertyDataSerializer
public interface IDictionaryOfPropertyDataSerializer
{
IDictionary<string, PropertyData[]> ReadFrom(Stream stream);
void WriteTo(IDictionary<string, PropertyData[]> value, Stream stream);

4
src/Umbraco.PublishedCache.NuCache/DataSource/JsonContentNestedDataSerializer.cs
View File

@@ -24,7 +24,7 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
DateFormatString = "o"
};
private readonly JsonNameTable _propertyNameTable = new DefaultJsonNameTable();
public ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData)
public ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData, bool published)
{
if (stringData == null && byteData != null)
throw new NotSupportedException($"{typeof(JsonContentNestedDataSerializer)} does not support byte[] serialization");
@@ -39,7 +39,7 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
}
}
public ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model)
public ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model, bool published)
{
// note that numeric values (which are Int32) are serialized without their
// type (eg "value":1234) and JsonConvert by default deserializes them as Int64

25
src/Umbraco.PublishedCache.NuCache/DataSource/MsgPackContentNestedDataSerializer.cs
View File

@@ -39,7 +39,8 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
_options = defaultOptions
.WithResolver(resolver)
.WithCompression(MessagePackCompression.Lz4BlockArray);
.WithCompression(MessagePackCompression.Lz4BlockArray)
.WithSecurity(MessagePackSecurity.UntrustedData);
}
public string ToJson(byte[] bin)
@@ -48,12 +49,12 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
return json;
}
public ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData)
public ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData, bool published)
{
if (byteData != null)
{
var cacheModel = MessagePackSerializer.Deserialize<ContentCacheDataModel>(byteData, _options);
Expand(content, cacheModel);
Expand(content, cacheModel, published);
return cacheModel;
}
else if (stringData != null)
@@ -61,7 +62,7 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
// NOTE: We don't really support strings but it's possible if manually used (i.e. tests)
var bin = Convert.FromBase64String(stringData);
var cacheModel = MessagePackSerializer.Deserialize<ContentCacheDataModel>(bin, _options);
Expand(content, cacheModel);
Expand(content, cacheModel, published);
return cacheModel;
}
else
@@ -70,9 +71,9 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
}
}
public ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model)
public ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model, bool published)
{
Compress(content, model);
Compress(content, model, published);
var bytes = MessagePackSerializer.Serialize(model, _options);
return new ContentCacheDataSerializationResult(null, bytes);
}
@@ -80,7 +81,9 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
/// <summary>
/// Used during serialization to compress properties
/// </summary>
/// <param name="content"></param>
/// <param name="model"></param>
/// <param name="published"></param>
/// <remarks>
/// This will essentially 'double compress' property data. The MsgPack data as a whole will already be compressed
/// but this will go a step further and double compress property data so that it is stored in the nucache file
@@ -88,11 +91,11 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
/// read/decompressed as a string to be displayed on the front-end. This allows for potentially a significant
/// memory savings but could also affect performance of first rendering pages while decompression occurs.
/// </remarks>
private void Compress(IReadOnlyContentBase content, ContentCacheDataModel model)
private void Compress(IReadOnlyContentBase content, ContentCacheDataModel model, bool published)
{
foreach(var propertyAliasToData in model.PropertyData)
{
if (_propertyOptions.IsCompressed(content, propertyAliasToData.Key))
if (_propertyOptions.IsCompressed(content, propertyAliasToData.Key, published))
{
foreach(var property in propertyAliasToData.Value.Where(x => x.Value != null && x.Value is string))
{
@@ -105,12 +108,14 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
/// <summary>
/// Used during deserialization to map the property data as lazy or expand the value
/// </summary>
/// <param name="content"></param>
/// <param name="nestedData"></param>
private void Expand(IReadOnlyContentBase content, ContentCacheDataModel nestedData)
/// <param name="published"></param>
private void Expand(IReadOnlyContentBase content, ContentCacheDataModel nestedData, bool published)
{
foreach (var propertyAliasToData in nestedData.PropertyData)
{
if (_propertyOptions.IsCompressed(content, propertyAliasToData.Key))
if (_propertyOptions.IsCompressed(content, propertyAliasToData.Key,published))
{
foreach (var property in propertyAliasToData.Value.Where(x => x.Value != null))
{

3
src/Umbraco.PublishedCache.NuCache/DataSource/MsgPackContentNestedDataSerializerFactory.cs
View File

@@ -3,7 +3,6 @@ using System.Collections.Generic;
using Umbraco.Cms.Core.Models;
using Umbraco.Cms.Core.PropertyEditors;
using Umbraco.Cms.Core.Services;
using Umbraco.Core.PropertyEditors;
namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
{
@@ -14,7 +13,7 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource
private readonly IMemberTypeService _memberTypeService;
private readonly PropertyEditorCollection _propertyEditors;
private readonly IPropertyCacheCompressionOptions _compressionOptions;
private readonly ConcurrentDictionary<(int, string), bool> _isCompressedCache = new ConcurrentDictionary<(int, string), bool>();
private readonly ConcurrentDictionary<(int, string, bool), bool> _isCompressedCache = new ConcurrentDictionary<(int, string, bool), bool>();
public MsgPackContentNestedDataSerializerFactory(
IContentTypeService contentTypeService,

15
src/Umbraco.PublishedCache.NuCache/DependencyInjection/UmbracoBuilderExtensions.cs
View File

@@ -68,7 +68,20 @@ namespace Umbraco.Extensions
throw new IndexOutOfRangeException();
}
});
builder.Services.AddSingleton<IPropertyCacheCompressionOptions, NoopPropertyCacheCompressionOptions>();
builder.Services.AddSingleton<IPropertyCacheCompressionOptions>(s =>
{
IOptions<NuCacheSettings> options = s.GetRequiredService<IOptions<NuCacheSettings>>();
if (options.Value.NuCacheSerializerType == NuCacheSerializerType.MessagePack &&
options.Value.UnPublishedContentCompression)
{
return new UnPublishedContentPropertyCacheCompressionOptions();
}
return new NoopPropertyCacheCompressionOptions();
});
builder.Services.AddSingleton(s => new ContentDataSerializer(new DictionaryOfPropertyDataSerializer()));
// add the NuCache health check (hidden from type finder)

17
src/Umbraco.PublishedCache.NuCache/Persistence/NuCacheContentRepository.cs
View File

@@ -415,7 +415,7 @@ AND cmsContentNu.nodeId IS NULL
UrlSegment = content.GetUrlSegment(_shortStringHelper, _urlSegmentProviders)
};
var serialized = serializer.Serialize(ReadOnlyContentBaseAdapter.Create(content), contentCacheData);
var serialized = serializer.Serialize(ReadOnlyContentBaseAdapter.Create(content), contentCacheData, published);
var dto = new ContentNuDto
{
@@ -817,12 +817,13 @@ AND cmsContentNu.nodeId IS NULL
}
else
{
var deserializedContent = serializer.Deserialize(dto, dto.EditData, dto.EditDataRaw);
bool published = false;
var deserializedContent = serializer.Deserialize(dto, dto.EditData, dto.EditDataRaw, published);
d = new ContentData
{
Name = dto.EditName,
Published = false,
Published = published,
TemplateId = dto.EditTemplateId,
VersionId = dto.VersionId,
VersionDate = dto.EditVersionDate,
@@ -847,13 +848,14 @@ AND cmsContentNu.nodeId IS NULL
}
else
{
var deserializedContent = serializer.Deserialize(dto, dto.PubData, dto.PubDataRaw);
bool published = true;
var deserializedContent = serializer.Deserialize(dto, dto.PubData, dto.PubDataRaw, published);
p = new ContentData
{
Name = dto.PubName,
UrlSegment = deserializedContent.UrlSegment,
Published = true,
Published = published,
TemplateId = dto.PubTemplateId,
VersionId = dto.VersionId,
VersionDate = dto.PubVersionDate,
@@ -883,12 +885,13 @@ AND cmsContentNu.nodeId IS NULL
if (dto.EditData == null && dto.EditDataRaw == null)
throw new InvalidOperationException("No data for media " + dto.Id);
var deserializedMedia = serializer.Deserialize(dto, dto.EditData, dto.EditDataRaw);
bool published = true;
var deserializedMedia = serializer.Deserialize(dto, dto.EditData, dto.EditDataRaw, published);
var p = new ContentData
{
Name = dto.EditName,
Published = true,
Published = published,
TemplateId = -1,
VersionId = dto.VersionId,
VersionDate = dto.EditVersionDate,

4
src/Umbraco.PublishedCache.NuCache/Umbraco.PublishedCache.NuCache.csproj
View File

@@ -19,8 +19,8 @@
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="MessagePack" Version="2.2.85" />
<PackageReference Include="K4os.Compression.LZ4" Version="1.2.6" />
<PackageReference Include="MessagePack" Version="2.3.75" />
<PackageReference Include="K4os.Compression.LZ4" Version="1.2.12" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
<PackageReference Include="Umbraco.Code" Version="1.1.1">
<PrivateAssets>all</PrivateAssets>