From f04e27ca197ae99975ed90aedaa4e53f95a2aee7 Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Tue, 22 Jun 2021 14:16:05 +0200 Subject: [PATCH 01/13] Bump version to 8.14.1 --- src/SolutionInfo.cs | 4 ++-- src/Umbraco.Web.UI/Umbraco.Web.UI.csproj | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/SolutionInfo.cs b/src/SolutionInfo.cs index 3619dc1371..3a43aec402 100644 --- a/src/SolutionInfo.cs +++ b/src/SolutionInfo.cs @@ -18,5 +18,5 @@ using System.Resources; [assembly: AssemblyVersion("8.0.0")] // these are FYI and changed automatically -[assembly: AssemblyFileVersion("8.14.0")] -[assembly: AssemblyInformationalVersion("8.14.0")] +[assembly: AssemblyFileVersion("8.14.1")] +[assembly: AssemblyInformationalVersion("8.14.1")] diff --git a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj index f3652a1273..f238984205 100644 --- a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj +++ b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj @@ -348,9 +348,9 @@ False True - 8140 + 8141 / - http://localhost:8140 + http://localhost:8141 8131 / http://localhost:8131 From 6816ae7ca41fb825b54dd23c9ffdfb1de32722a2 Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Fri, 2 Jul 2021 11:47:39 +0200 Subject: [PATCH 02/13] Fix dependencies to make 8.15 installable via NuGet (cherry picked from commit 3e208904356f1fe1262a7edee052dbf9663a6ad3) --- build/NuSpecs/UmbracoCms.Web.nuspec | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build/NuSpecs/UmbracoCms.Web.nuspec b/build/NuSpecs/UmbracoCms.Web.nuspec index e96a217f4e..7aebfae108 100644 --- a/build/NuSpecs/UmbracoCms.Web.nuspec +++ b/build/NuSpecs/UmbracoCms.Web.nuspec @@ -43,7 +43,8 @@ - + + From 4ac29927deb0183e9e27b42db32eaad661e1536d Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Wed, 7 Jul 2021 11:22:50 +0200 Subject: [PATCH 03/13] Bump version to 8.15.0 --- src/SolutionInfo.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SolutionInfo.cs b/src/SolutionInfo.cs index 6ef4d6ce85..201213cffa 100644 --- a/src/SolutionInfo.cs +++ b/src/SolutionInfo.cs @@ -19,4 +19,4 @@ using System.Resources; // these are FYI and changed automatically [assembly: AssemblyFileVersion("8.15.0")] -[assembly: AssemblyInformationalVersion("8.15.0-rc")] +[assembly: AssemblyInformationalVersion("8.15.0")] From 6b6525cc9891cc82d5be8701b91394bdb8ff5a9d Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Fri, 9 Jul 2021 19:33:42 +1200 Subject: [PATCH 04/13] Add whether the current content is published. Allowing for compression of just non published, non media content. --- .../IPropertyCacheCompression.cs | 10 +++++++-- .../IPropertyCacheCompressionOptions.cs | 9 +++++++- .../NoopPropertyCacheCompressionOptions.cs | 2 +- .../PropertyCacheCompression.cs | 10 ++++----- .../ContentSerializationTests.cs | 8 +++---- .../NuCache/DataSource/DatabaseDataSource.cs | 15 +++++++------ .../DataSource/IContentCacheDataSerializer.cs | 4 ++-- .../JsonContentNestedDataSerializer.cs | 4 ++-- .../MsgPackContentNestedDataSerializer.cs | 21 ++++++++++--------- ...gPackContentNestedDataSerializerFactory.cs | 2 +- .../NuCache/PublishedSnapshotService.cs | 2 +- 11 files changed, 52 insertions(+), 35 deletions(-) diff --git a/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompression.cs b/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompression.cs index 96a559630b..eb89173581 100644 --- a/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompression.cs +++ b/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompression.cs @@ -9,7 +9,13 @@ namespace Umbraco.Core.PropertyEditors /// /// public interface IPropertyCacheCompression - { - bool IsCompressed(IReadOnlyContentBase content, string propertyTypeAlias); + { + /// + /// Whether a property on the content is/should be compressed + /// + /// The content + /// The property to compress or not + /// Whether this content is the published version + bool IsCompressed(IReadOnlyContentBase content, string propertyTypeAlias,bool published); } } diff --git a/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompressionOptions.cs b/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompressionOptions.cs index 2fa0153f9e..71eb782ee2 100644 --- a/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompressionOptions.cs +++ b/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompressionOptions.cs @@ -4,6 +4,13 @@ namespace Umbraco.Core.PropertyEditors { public interface IPropertyCacheCompressionOptions { - bool IsCompressed(IReadOnlyContentBase content, PropertyType propertyType, IDataEditor dataEditor); + /// + /// Whether a property on the content is/should be compressed + /// + /// The content + /// The property to compress or not + /// The datatype of the property to compress or not + /// Whether this content is the published version + bool IsCompressed(IReadOnlyContentBase content, PropertyType propertyType, IDataEditor dataEditor,bool published); } } diff --git a/src/Umbraco.Core/PropertyEditors/NoopPropertyCacheCompressionOptions.cs b/src/Umbraco.Core/PropertyEditors/NoopPropertyCacheCompressionOptions.cs index 1f12d45769..f566172cd9 100644 --- a/src/Umbraco.Core/PropertyEditors/NoopPropertyCacheCompressionOptions.cs +++ b/src/Umbraco.Core/PropertyEditors/NoopPropertyCacheCompressionOptions.cs @@ -7,6 +7,6 @@ namespace Umbraco.Core.PropertyEditors /// internal class NoopPropertyCacheCompressionOptions : IPropertyCacheCompressionOptions { - public bool IsCompressed(IReadOnlyContentBase content, PropertyType propertyType, IDataEditor dataEditor) => false; + public bool IsCompressed(IReadOnlyContentBase content, PropertyType propertyType, IDataEditor dataEditor,bool published) => false; } } diff --git a/src/Umbraco.Core/PropertyEditors/PropertyCacheCompression.cs b/src/Umbraco.Core/PropertyEditors/PropertyCacheCompression.cs index 6be21fca7f..be0553ce65 100644 --- a/src/Umbraco.Core/PropertyEditors/PropertyCacheCompression.cs +++ b/src/Umbraco.Core/PropertyEditors/PropertyCacheCompression.cs @@ -14,13 +14,13 @@ namespace Umbraco.Core.PropertyEditors private readonly IPropertyCacheCompressionOptions _compressionOptions; private readonly IReadOnlyDictionary _contentTypes; private readonly PropertyEditorCollection _propertyEditors; - private readonly ConcurrentDictionary<(int contentTypeId, string propertyAlias), bool> _isCompressedCache; + private readonly ConcurrentDictionary<(int contentTypeId, string propertyAlias,bool published), bool> _isCompressedCache; public PropertyCacheCompression( IPropertyCacheCompressionOptions compressionOptions, IReadOnlyDictionary contentTypes, PropertyEditorCollection propertyEditors, - ConcurrentDictionary<(int, string), bool> compressedStoragePropertyEditorCache) + ConcurrentDictionary<(int, string,bool), bool> compressedStoragePropertyEditorCache) { _compressionOptions = compressionOptions; _contentTypes = contentTypes ?? throw new System.ArgumentNullException(nameof(contentTypes)); @@ -28,9 +28,9 @@ namespace Umbraco.Core.PropertyEditors _isCompressedCache = compressedStoragePropertyEditorCache; } - public bool IsCompressed(IReadOnlyContentBase content, string alias) + public bool IsCompressed(IReadOnlyContentBase content, string alias, bool published) { - var compressedStorage = _isCompressedCache.GetOrAdd((content.ContentTypeId, alias), x => + var compressedStorage = _isCompressedCache.GetOrAdd((content.ContentTypeId, alias, published), x => { if (!_contentTypes.TryGetValue(x.contentTypeId, out var ct)) return false; @@ -40,7 +40,7 @@ namespace Umbraco.Core.PropertyEditors if (!_propertyEditors.TryGet(propertyType.PropertyEditorAlias, out var propertyEditor)) return false; - return _compressionOptions.IsCompressed(content, propertyType, propertyEditor); + return _compressionOptions.IsCompressed(content, propertyType, propertyEditor, published); }); return compressedStorage; diff --git a/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs b/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs index b3543dad1a..9acad53364 100644 --- a/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs +++ b/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs @@ -56,14 +56,14 @@ namespace Umbraco.Tests.PublishedContent var content = Mock.Of(x => x.ContentTypeId == 1); - var json = jsonSerializer.Serialize(content, cacheModel).StringData; - var msgPack = msgPackSerializer.Serialize(content, cacheModel).ByteData; + var json = jsonSerializer.Serialize(content, cacheModel,false).StringData; + var msgPack = msgPackSerializer.Serialize(content, cacheModel, false).ByteData; Console.WriteLine(json); Console.WriteLine(msgPackSerializer.ToJson(msgPack)); - var jsonContent = jsonSerializer.Deserialize(content, json, null); - var msgPackContent = msgPackSerializer.Deserialize(content, null, msgPack); + var jsonContent = jsonSerializer.Deserialize(content, json, null,false); + var msgPackContent = msgPackSerializer.Deserialize(content, null, msgPack,false); CollectionAssert.AreEqual(jsonContent.CultureData.Keys, msgPackContent.CultureData.Keys); diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs index c112cc6efa..5ec51bffb8 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs @@ -393,12 +393,13 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } else { - var deserializedContent = serializer.Deserialize(dto, dto.EditData, dto.EditDataRaw); + bool published = false; + var deserializedContent = serializer.Deserialize(dto, dto.EditData, dto.EditDataRaw, published); d = new ContentData { Name = dto.EditName, - Published = false, + Published = published, TemplateId = dto.EditTemplateId, VersionId = dto.VersionId, VersionDate = dto.EditVersionDate, @@ -420,13 +421,14 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } else { - var deserializedContent = serializer.Deserialize(dto, dto.PubData, dto.PubDataRaw); + bool published = true; + var deserializedContent = serializer.Deserialize(dto, dto.PubData, dto.PubDataRaw, published); p = new ContentData { Name = dto.PubName, UrlSegment = deserializedContent.UrlSegment, - Published = true, + Published = published, TemplateId = dto.PubTemplateId, VersionId = dto.VersionId, VersionDate = dto.PubVersionDate, @@ -456,12 +458,13 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource if (dto.EditData == null && dto.EditDataRaw == null) throw new InvalidOperationException("No data for media " + dto.Id); - var deserializedMedia = serializer.Deserialize(dto, dto.EditData, dto.EditDataRaw); + bool published = true; + var deserializedMedia = serializer.Deserialize(dto, dto.EditData, dto.EditDataRaw, published); var p = new ContentData { Name = dto.EditName, - Published = true, + Published = published, TemplateId = -1, VersionId = dto.VersionId, VersionDate = dto.EditVersionDate, diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializer.cs index d1a83d8452..6c37046e8b 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializer.cs @@ -14,12 +14,12 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource /// /// Deserialize the data into a /// - ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData); + ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData, bool published); /// /// Serializes the /// - ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model); + ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model,bool published); } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs index 21cd0bf763..0cc10f5f98 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs @@ -24,7 +24,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource DateFormatString = "o" }; private readonly JsonNameTable _propertyNameTable = new DefaultJsonNameTable(); - public ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData) + public ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData,bool published) { if (stringData == null && byteData != null) throw new NotSupportedException($"{typeof(JsonContentNestedDataSerializer)} does not support byte[] serialization"); @@ -39,7 +39,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } } - public ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model) + public ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model,bool published) { // note that numeric values (which are Int32) are serialized without their // type (eg "value":1234) and JsonConvert by default deserializes them as Int64 diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs index 6ae872ef69..f69232aad3 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs @@ -39,7 +39,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource _options = defaultOptions .WithResolver(resolver) - .WithCompression(MessagePackCompression.Lz4BlockArray); + .WithCompression(MessagePackCompression.Lz4BlockArray) + .WithSecurity(MessagePackSecurity.UntrustedData); } public string ToJson(byte[] bin) @@ -48,12 +49,12 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource return json; } - public ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData) + public ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData,bool published) { if (byteData != null) { var cacheModel = MessagePackSerializer.Deserialize(byteData, _options); - Expand(content, cacheModel); + Expand(content, cacheModel, published); return cacheModel; } else if (stringData != null) @@ -61,7 +62,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource // NOTE: We don't really support strings but it's possible if manually used (i.e. tests) var bin = Convert.FromBase64String(stringData); var cacheModel = MessagePackSerializer.Deserialize(bin, _options); - Expand(content, cacheModel); + Expand(content, cacheModel,published); return cacheModel; } else @@ -70,9 +71,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } } - public ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model) + public ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model,bool published) { - Compress(content, model); + Compress(content, model, published); var bytes = MessagePackSerializer.Serialize(model, _options); return new ContentCacheDataSerializationResult(null, bytes); } @@ -88,11 +89,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource /// read/decompressed as a string to be displayed on the front-end. This allows for potentially a significant /// memory savings but could also affect performance of first rendering pages while decompression occurs. /// - private void Compress(IReadOnlyContentBase content, ContentCacheDataModel model) + private void Compress(IReadOnlyContentBase content, ContentCacheDataModel model,bool published) { foreach(var propertyAliasToData in model.PropertyData) { - if (_propertyOptions.IsCompressed(content, propertyAliasToData.Key)) + if (_propertyOptions.IsCompressed(content, propertyAliasToData.Key,published)) { foreach(var property in propertyAliasToData.Value.Where(x => x.Value != null && x.Value is string)) { @@ -106,11 +107,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource /// Used during deserialization to map the property data as lazy or expand the value /// /// - private void Expand(IReadOnlyContentBase content, ContentCacheDataModel nestedData) + private void Expand(IReadOnlyContentBase content, ContentCacheDataModel nestedData,bool published) { foreach (var propertyAliasToData in nestedData.PropertyData) { - if (_propertyOptions.IsCompressed(content, propertyAliasToData.Key)) + if (_propertyOptions.IsCompressed(content, propertyAliasToData.Key,published)) { foreach (var property in propertyAliasToData.Value.Where(x => x.Value != null)) { diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializerFactory.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializerFactory.cs index fcc3fa2bb8..29378caf0f 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializerFactory.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializerFactory.cs @@ -13,7 +13,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource private readonly IMemberTypeService _memberTypeService; private readonly PropertyEditorCollection _propertyEditors; private readonly IPropertyCacheCompressionOptions _compressionOptions; - private readonly ConcurrentDictionary<(int, string), bool> _isCompressedCache = new ConcurrentDictionary<(int, string), bool>(); + private readonly ConcurrentDictionary<(int, string,bool), bool> _isCompressedCache = new ConcurrentDictionary<(int, string,bool), bool>(); public MsgPackContentNestedDataSerializerFactory( IContentTypeService contentTypeService, diff --git a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs index f9c25b7b35..7ca425aad9 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs @@ -1491,7 +1491,7 @@ namespace Umbraco.Web.PublishedCache.NuCache UrlSegment = content.GetUrlSegment(_urlSegmentProviders) }; - var serialized = serializer.Serialize(ReadOnlyContentBaseAdapter.Create(content), contentCacheData); + var serialized = serializer.Serialize(ReadOnlyContentBaseAdapter.Create(content), contentCacheData,published); var dto = new ContentNuDto { From 044c900c2ba52428b7acecb8d15fc0769c5b9eaa Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Fri, 9 Jul 2021 20:14:35 +1200 Subject: [PATCH 05/13] Make IPropertyCacheCompressionOptions useful out of the box. key = "Umbraco.Web.PublishedCache.NuCache.CompressUnPublishedContent" value = "true" will compress all ntext properties on unpublished content --- ...dContentPropertyCacheCompressionOptions.cs | 29 +++++++++++++++++++ src/Umbraco.Core/Umbraco.Core.csproj | 1 + .../PublishedCache/NuCache/NuCacheComposer.cs | 12 ++++++-- .../NuCache/NuCacheSerializerComponent.cs | 1 + 4 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 src/Umbraco.Core/PropertyEditors/UnPublishedContentPropertyCacheCompressionOptions.cs diff --git a/src/Umbraco.Core/PropertyEditors/UnPublishedContentPropertyCacheCompressionOptions.cs b/src/Umbraco.Core/PropertyEditors/UnPublishedContentPropertyCacheCompressionOptions.cs new file mode 100644 index 0000000000..51433e476e --- /dev/null +++ b/src/Umbraco.Core/PropertyEditors/UnPublishedContentPropertyCacheCompressionOptions.cs @@ -0,0 +1,29 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Umbraco.Core.Models; + +namespace Umbraco.Core.PropertyEditors +{ + /// + /// Compress large, non published text properties + /// + internal class UnPublishedContentPropertyCacheCompressionOptions : IPropertyCacheCompressionOptions + { + public bool IsCompressed(IReadOnlyContentBase content, PropertyType propertyType, IDataEditor dataEditor, bool published) + { + if (published) + { + return false; + } + if (propertyType.SupportsPublishing && propertyType.ValueStorageType == ValueStorageType.Ntext) + { + //Only compress non published content that supports publishing and the property is text + return true; + } + return false; + } + } +} diff --git a/src/Umbraco.Core/Umbraco.Core.csproj b/src/Umbraco.Core/Umbraco.Core.csproj index 931cef070f..f6a523441e 100755 --- a/src/Umbraco.Core/Umbraco.Core.csproj +++ b/src/Umbraco.Core/Umbraco.Core.csproj @@ -171,6 +171,7 @@ + diff --git a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs index c6b214102b..4b3d7b20fa 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs @@ -26,8 +26,16 @@ namespace Umbraco.Web.PublishedCache.NuCache { composition.RegisterUnique(); } - - composition.RegisterUnique(); + var unPublishedContentCompression = ConfigurationManager.AppSettings[NuCacheSerializerComponent.Nucache_UnPublishedContentCompression_Key]; + if ("MsgPack" == serializer && "true" == unPublishedContentCompression) + { + composition.RegisterUnique(); + } + else + { + composition.RegisterUnique(); + } + composition.RegisterUnique(factory => new ContentDataSerializer(new DictionaryOfPropertyDataSerializer())); diff --git a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComponent.cs b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComponent.cs index a1d3ed2b12..c2d24b16ac 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComponent.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComponent.cs @@ -16,6 +16,7 @@ namespace Umbraco.Web.PublishedCache.NuCache public class NuCacheSerializerComponent : IComponent { internal const string Nucache_Serializer_Key = "Umbraco.Web.PublishedCache.NuCache.Serializer"; + internal const string Nucache_UnPublishedContentCompression_Key = "Umbraco.Web.PublishedCache.NuCache.CompressUnPublishedContent"; private const string JSON_SERIALIZER_VALUE = "JSON"; private readonly Lazy _service; private readonly IKeyValueService _keyValueService; From b119201480da1f99bf463776f71d2e73901f2aef Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Fri, 9 Jul 2021 20:55:03 +1200 Subject: [PATCH 06/13] fix serializer swap message --- .../PublishedCache/NuCache/NuCacheSerializerComponent.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComponent.cs b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComponent.cs index c2d24b16ac..41afb2b781 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComponent.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComponent.cs @@ -55,7 +55,7 @@ namespace Umbraco.Web.PublishedCache.NuCache { _profilingLogger.Warn($"Database NuCache was serialized using {currentSerializer}. Currently configured NuCache serializer {serializer}. Rebuilding Nucache"); - using (_profilingLogger.TraceDuration($"Rebuilding NuCache database with {currentSerializer} serializer")) + using (_profilingLogger.TraceDuration($"Rebuilding NuCache database with {serializer} serializer")) { _service.Value.Rebuild(); _keyValueService.SetValue(Nucache_Serializer_Key, serializer); From 6a8ed8012cc282e87b8d55c23f518f9991c0daba Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Fri, 9 Jul 2021 21:22:30 +1200 Subject: [PATCH 07/13] simplify --- .../UnPublishedContentPropertyCacheCompressionOptions.cs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/Umbraco.Core/PropertyEditors/UnPublishedContentPropertyCacheCompressionOptions.cs b/src/Umbraco.Core/PropertyEditors/UnPublishedContentPropertyCacheCompressionOptions.cs index 51433e476e..7a5212a985 100644 --- a/src/Umbraco.Core/PropertyEditors/UnPublishedContentPropertyCacheCompressionOptions.cs +++ b/src/Umbraco.Core/PropertyEditors/UnPublishedContentPropertyCacheCompressionOptions.cs @@ -14,11 +14,7 @@ namespace Umbraco.Core.PropertyEditors { public bool IsCompressed(IReadOnlyContentBase content, PropertyType propertyType, IDataEditor dataEditor, bool published) { - if (published) - { - return false; - } - if (propertyType.SupportsPublishing && propertyType.ValueStorageType == ValueStorageType.Ntext) + if (!published && propertyType.SupportsPublishing && propertyType.ValueStorageType == ValueStorageType.Ntext) { //Only compress non published content that supports publishing and the property is text return true; From 5e8b4572a0a322b8cfbe27fea3038367c76415fd Mon Sep 17 00:00:00 2001 From: Shannon Date: Fri, 9 Jul 2021 09:40:31 -0600 Subject: [PATCH 08/13] Ensures that the content nu data column is updated to support null during migration. --- .../Migrations/Upgrade/V_8_15_0/AddCmsContentNuByteColumn.cs | 3 +++ .../PublishedCache/NuCache/NuCacheSerializerComponent.cs | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Umbraco.Core/Migrations/Upgrade/V_8_15_0/AddCmsContentNuByteColumn.cs b/src/Umbraco.Core/Migrations/Upgrade/V_8_15_0/AddCmsContentNuByteColumn.cs index 3eab1a812e..9eeaa67189 100644 --- a/src/Umbraco.Core/Migrations/Upgrade/V_8_15_0/AddCmsContentNuByteColumn.cs +++ b/src/Umbraco.Core/Migrations/Upgrade/V_8_15_0/AddCmsContentNuByteColumn.cs @@ -16,6 +16,9 @@ namespace Umbraco.Core.Migrations.Upgrade.V_8_15_0 var columns = SqlSyntax.GetColumnsInSchema(Context.Database).ToList(); AddColumnIfNotExists(columns, "dataRaw"); + + // allow null + AlterColumn(Constants.DatabaseSchema.Tables.NodeData, "data"); } } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComponent.cs b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComponent.cs index a1d3ed2b12..fcbb5a3715 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComponent.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComponent.cs @@ -54,7 +54,7 @@ namespace Umbraco.Web.PublishedCache.NuCache { _profilingLogger.Warn($"Database NuCache was serialized using {currentSerializer}. Currently configured NuCache serializer {serializer}. Rebuilding Nucache"); - using (_profilingLogger.TraceDuration($"Rebuilding NuCache database with {currentSerializer} serializer")) + using (_profilingLogger.TraceDuration($"Rebuilding NuCache database with {serializer} serializer")) { _service.Value.Rebuild(); _keyValueService.SetValue(Nucache_Serializer_Key, serializer); From 7dc03256edf1bbe15b5b44c5d44a033369ce0e07 Mon Sep 17 00:00:00 2001 From: Shannon Date: Fri, 9 Jul 2021 12:05:19 -0600 Subject: [PATCH 09/13] Fixes SQLCE migration for nucache table. --- .../Create/Table/CreateTableOfDtoBuilder.cs | 2 + .../V_8_15_0/AddCmsContentNuByteColumn.cs | 47 +++++++++++++++++-- .../SqlSyntax/SqlSyntaxProviderBase.cs | 15 +++--- 3 files changed, 54 insertions(+), 10 deletions(-) diff --git a/src/Umbraco.Core/Migrations/Expressions/Create/Table/CreateTableOfDtoBuilder.cs b/src/Umbraco.Core/Migrations/Expressions/Create/Table/CreateTableOfDtoBuilder.cs index 4b73e9435d..a5e2fca1f7 100644 --- a/src/Umbraco.Core/Migrations/Expressions/Create/Table/CreateTableOfDtoBuilder.cs +++ b/src/Umbraco.Core/Migrations/Expressions/Create/Table/CreateTableOfDtoBuilder.cs @@ -9,6 +9,8 @@ namespace Umbraco.Core.Migrations.Expressions.Create.Table public class CreateTableOfDtoBuilder : IExecutableBuilder { private readonly IMigrationContext _context; + + // TODO: This doesn't do anything. private readonly DatabaseType[] _supportedDatabaseTypes; public CreateTableOfDtoBuilder(IMigrationContext context, params DatabaseType[] supportedDatabaseTypes) diff --git a/src/Umbraco.Core/Migrations/Upgrade/V_8_15_0/AddCmsContentNuByteColumn.cs b/src/Umbraco.Core/Migrations/Upgrade/V_8_15_0/AddCmsContentNuByteColumn.cs index 9eeaa67189..5217fc9870 100644 --- a/src/Umbraco.Core/Migrations/Upgrade/V_8_15_0/AddCmsContentNuByteColumn.cs +++ b/src/Umbraco.Core/Migrations/Upgrade/V_8_15_0/AddCmsContentNuByteColumn.cs @@ -1,5 +1,10 @@ -using System.Linq; +using NPoco; +using System.Data; +using System.Linq; +using Umbraco.Core.Persistence; +using Umbraco.Core.Persistence.DatabaseAnnotations; using Umbraco.Core.Persistence.Dtos; +using Umbraco.Core.PropertyEditors; namespace Umbraco.Core.Migrations.Upgrade.V_8_15_0 { @@ -13,12 +18,46 @@ namespace Umbraco.Core.Migrations.Upgrade.V_8_15_0 public override void Migrate() { + // allow null for the `data` field + if (DatabaseType.IsSqlCe()) + { + // SQLCE does not support altering NTEXT, so we have to jump through some hoops to do it + // All column ordering must remain the same as what is defined in the DTO so we need to create a temp table, + // drop orig and then re-create/copy. + Create.Table(withoutKeysAndIndexes: true).Do(); + Execute.Sql($"INSERT INTO [{TempTableName}] SELECT nodeId, published, data, rv FROM [{Constants.DatabaseSchema.Tables.NodeData}]").Do(); + Delete.Table(Constants.DatabaseSchema.Tables.NodeData).Do(); + Create.Table().Do(); + Execute.Sql($"INSERT INTO [{Constants.DatabaseSchema.Tables.NodeData}] SELECT nodeId, published, data, rv, NULL FROM [{TempTableName}]").Do(); + } + else + { + AlterColumn(Constants.DatabaseSchema.Tables.NodeData, "data"); + } + var columns = SqlSyntax.GetColumnsInSchema(Context.Database).ToList(); - AddColumnIfNotExists(columns, "dataRaw"); + } - // allow null - AlterColumn(Constants.DatabaseSchema.Tables.NodeData, "data"); + private const string TempTableName = Constants.DatabaseSchema.TableNamePrefix + "cms" + "ContentNuTEMP"; + + [TableName(TempTableName)] + [ExplicitColumns] + private class ContentNuDtoTemp + { + [Column("nodeId")] + public int NodeId { get; set; } + + [Column("published")] + public bool Published { get; set; } + + [Column("data")] + [SpecialDbType(SpecialDbTypes.NTEXT)] + [NullSetting(NullSetting = NullSettings.Null)] + public string Data { get; set; } + + [Column("rv")] + public long Rv { get; set; } } } } diff --git a/src/Umbraco.Core/Persistence/SqlSyntax/SqlSyntaxProviderBase.cs b/src/Umbraco.Core/Persistence/SqlSyntax/SqlSyntaxProviderBase.cs index 4d6b2eeea1..0d2d7aeb21 100644 --- a/src/Umbraco.Core/Persistence/SqlSyntax/SqlSyntaxProviderBase.cs +++ b/src/Umbraco.Core/Persistence/SqlSyntax/SqlSyntaxProviderBase.cs @@ -352,7 +352,7 @@ namespace Umbraco.Core.Persistence.SqlSyntax sql.Append(" "); sql.Append(FormatIdentity(column)); - var isNullable = column.IsNullable; + //var isNullable = column.IsNullable; //var constraint = FormatConstraint(column)?.TrimStart("CONSTRAINT "); //var hasConstraint = !string.IsNullOrWhiteSpace(constraint); @@ -360,11 +360,14 @@ namespace Umbraco.Core.Persistence.SqlSyntax //var defaultValue = FormatDefaultValue(column); //var hasDefaultValue = !string.IsNullOrWhiteSpace(defaultValue); - if (isNullable /*&& !hasConstraint && !hasDefaultValue*/) - { - sqls = Enumerable.Empty(); - return sql.ToString(); - } + // TODO: This used to exit if nullable but that means this would never work + // to return SQL if the column was nullable?!? I don't get it. This was here + // 4 years ago, I've removed it so that this works for nullable columns. + //if (isNullable /*&& !hasConstraint && !hasDefaultValue*/) + //{ + // sqls = Enumerable.Empty(); + // return sql.ToString(); + //} var msql = new List(); sqls = msql; From 93f5dd76317b437f14f37db49f1ac39ace300c39 Mon Sep 17 00:00:00 2001 From: Shannon Date: Fri, 9 Jul 2021 13:35:43 -0600 Subject: [PATCH 10/13] Changes bulk copy timeout to be infinite. Changes bulk copy to not allocate an arbitrary array. --- .../NPocoDatabaseExtensions-Bulk.cs | 35 ++++++++++++++----- .../Persistence/PocoDataDataReader.cs | 5 +-- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/src/Umbraco.Core/Persistence/NPocoDatabaseExtensions-Bulk.cs b/src/Umbraco.Core/Persistence/NPocoDatabaseExtensions-Bulk.cs index bff682d095..77cc0d6601 100644 --- a/src/Umbraco.Core/Persistence/NPocoDatabaseExtensions-Bulk.cs +++ b/src/Umbraco.Core/Persistence/NPocoDatabaseExtensions-Bulk.cs @@ -62,26 +62,33 @@ namespace Umbraco.Core.Persistence /// The number of records that were inserted. public static int BulkInsertRecords(this IUmbracoDatabase database, IEnumerable records, bool useNativeBulkInsert = true) { - var recordsA = records.ToArray(); - if (recordsA.Length == 0) return 0; + if (!records.Any()) return 0; var pocoData = database.PocoDataFactory.ForType(typeof(T)); if (pocoData == null) throw new InvalidOperationException("Could not find PocoData for " + typeof(T)); if (database.DatabaseType.IsSqlCe()) { - if (useNativeBulkInsert) return BulkInsertRecordsSqlCe(database, pocoData, recordsA); + if (useNativeBulkInsert) + { + return BulkInsertRecordsSqlCe(database, pocoData, records); + } + // else, no other choice - foreach (var record in recordsA) + var count = 0; + foreach (var record in records) + { database.Insert(record); - return recordsA.Length; + count++; + } + return count; } if (database.DatabaseType.IsSqlServer()) { return useNativeBulkInsert && database.DatabaseType.IsSqlServer2008OrLater() - ? BulkInsertRecordsSqlServer(database, pocoData, recordsA) - : BulkInsertRecordsWithCommands(database, recordsA); + ? BulkInsertRecordsSqlServer(database, pocoData, records) + : BulkInsertRecordsWithCommands(database, records.ToArray()); } throw new NotSupportedException(); } @@ -96,7 +103,9 @@ namespace Umbraco.Core.Persistence private static int BulkInsertRecordsWithCommands(IUmbracoDatabase database, T[] records) { foreach (var command in database.GenerateBulkInsertCommands(records)) + { command.ExecuteNonQuery(); + } return records.Length; // what else? } @@ -241,6 +250,10 @@ namespace Umbraco.Core.Persistence /// The number of records that were inserted. internal static int BulkInsertRecordsSqlServer(IUmbracoDatabase database, PocoData pocoData, IEnumerable records) { + // TODO: The main reason this exists is because the NPoco InsertBulk method doesn't return the number of items. + // It is worth investigating the performance of this vs NPoco's because we use a custom BulkDataReader + // which in theory should be more efficient than NPocos way of building up an in-memory DataTable. + // create command against the original database.Connection using (var command = database.CreateCommand(database.Connection, CommandType.Text, string.Empty)) { @@ -252,7 +265,13 @@ namespace Umbraco.Core.Persistence var syntax = database.SqlContext.SqlSyntax as SqlServerSyntaxProvider; if (syntax == null) throw new NotSupportedException("SqlSyntax must be SqlServerSyntaxProvider."); - using (var copy = new SqlBulkCopy(tConnection, SqlBulkCopyOptions.Default, tTransaction) { BulkCopyTimeout = 10000, DestinationTableName = tableName }) + using (var copy = new SqlBulkCopy(tConnection, SqlBulkCopyOptions.Default, tTransaction) + { + BulkCopyTimeout = 0, // 0 = no bulk copy timeout. If a timeout occurs it will be an connection/command timeout. + DestinationTableName = tableName, + // be consistent with NPoco: https://github.com/schotime/NPoco/blob/5117a55fde57547e928246c044fd40bd00b2d7d1/src/NPoco.SqlServer/SqlBulkCopyHelper.cs#L50 + BatchSize = 4096 + }) using (var bulkReader = new PocoDataDataReader(records, pocoData, syntax)) { //we need to add column mappings here because otherwise columns will be matched by their order and if the order of them are different in the DB compared diff --git a/src/Umbraco.Core/Persistence/PocoDataDataReader.cs b/src/Umbraco.Core/Persistence/PocoDataDataReader.cs index 460a4d3d90..397d903cc2 100644 --- a/src/Umbraco.Core/Persistence/PocoDataDataReader.cs +++ b/src/Umbraco.Core/Persistence/PocoDataDataReader.cs @@ -40,9 +40,10 @@ namespace Umbraco.Core.Persistence _tableDefinition = DefinitionFactory.GetTableDefinition(pd.Type, sqlSyntaxProvider); if (_tableDefinition == null) throw new InvalidOperationException("No table definition found for type " + pd.Type); - // only real columns, exclude result columns + // only real columns, exclude result/computed columns + // Like NPoco does: https://github.com/schotime/NPoco/blob/5117a55fde57547e928246c044fd40bd00b2d7d1/src/NPoco.SqlServer/SqlBulkCopyHelper.cs#L59 _readerColumns = pd.Columns - .Where(x => x.Value.ResultColumn == false) + .Where(x => x.Value.ResultColumn == false && x.Value.ComputedColumn == false) .Select(x => x.Value) .ToArray(); From a37f6c604f399b1c6f387adcc97fa71d0dc9abea Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Sat, 10 Jul 2021 14:25:26 +0200 Subject: [PATCH 11/13] Inconsistent formatting --- .../IPropertyCacheCompression.cs | 4 ++-- .../IPropertyCacheCompressionOptions.cs | 2 +- .../NoopPropertyCacheCompressionOptions.cs | 2 +- ...edContentPropertyCacheCompressionOptions.cs | 7 +------ .../ContentSerializationTests.cs | 6 +++--- .../DataSource/IContentCacheDataSerializer.cs | 4 ++-- .../JsonContentNestedDataSerializer.cs | 4 ++-- .../MsgPackContentNestedDataSerializer.cs | 18 +++++++++++------- ...sgPackContentNestedDataSerializerFactory.cs | 2 +- .../PublishedCache/NuCache/NuCacheComposer.cs | 8 ++++---- .../NuCache/PublishedSnapshotService.cs | 8 ++++---- 11 files changed, 32 insertions(+), 33 deletions(-) diff --git a/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompression.cs b/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompression.cs index eb89173581..69d10a2276 100644 --- a/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompression.cs +++ b/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompression.cs @@ -6,7 +6,7 @@ namespace Umbraco.Core.PropertyEditors /// Determines if a property type's value should be compressed in memory /// /// - /// + /// /// public interface IPropertyCacheCompression { @@ -16,6 +16,6 @@ namespace Umbraco.Core.PropertyEditors /// The content /// The property to compress or not /// Whether this content is the published version - bool IsCompressed(IReadOnlyContentBase content, string propertyTypeAlias,bool published); + bool IsCompressed(IReadOnlyContentBase content, string propertyTypeAlias, bool published); } } diff --git a/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompressionOptions.cs b/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompressionOptions.cs index 71eb782ee2..e4603d55e3 100644 --- a/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompressionOptions.cs +++ b/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompressionOptions.cs @@ -11,6 +11,6 @@ namespace Umbraco.Core.PropertyEditors /// The property to compress or not /// The datatype of the property to compress or not /// Whether this content is the published version - bool IsCompressed(IReadOnlyContentBase content, PropertyType propertyType, IDataEditor dataEditor,bool published); + bool IsCompressed(IReadOnlyContentBase content, PropertyType propertyType, IDataEditor dataEditor, bool published); } } diff --git a/src/Umbraco.Core/PropertyEditors/NoopPropertyCacheCompressionOptions.cs b/src/Umbraco.Core/PropertyEditors/NoopPropertyCacheCompressionOptions.cs index f566172cd9..ea3b8d8a2e 100644 --- a/src/Umbraco.Core/PropertyEditors/NoopPropertyCacheCompressionOptions.cs +++ b/src/Umbraco.Core/PropertyEditors/NoopPropertyCacheCompressionOptions.cs @@ -7,6 +7,6 @@ namespace Umbraco.Core.PropertyEditors /// internal class NoopPropertyCacheCompressionOptions : IPropertyCacheCompressionOptions { - public bool IsCompressed(IReadOnlyContentBase content, PropertyType propertyType, IDataEditor dataEditor,bool published) => false; + public bool IsCompressed(IReadOnlyContentBase content, PropertyType propertyType, IDataEditor dataEditor, bool published) => false; } } diff --git a/src/Umbraco.Core/PropertyEditors/UnPublishedContentPropertyCacheCompressionOptions.cs b/src/Umbraco.Core/PropertyEditors/UnPublishedContentPropertyCacheCompressionOptions.cs index 7a5212a985..ece25479cc 100644 --- a/src/Umbraco.Core/PropertyEditors/UnPublishedContentPropertyCacheCompressionOptions.cs +++ b/src/Umbraco.Core/PropertyEditors/UnPublishedContentPropertyCacheCompressionOptions.cs @@ -1,9 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Umbraco.Core.Models; +using Umbraco.Core.Models; namespace Umbraco.Core.PropertyEditors { diff --git a/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs b/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs index 9acad53364..9a44cf35f9 100644 --- a/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs +++ b/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs @@ -56,14 +56,14 @@ namespace Umbraco.Tests.PublishedContent var content = Mock.Of(x => x.ContentTypeId == 1); - var json = jsonSerializer.Serialize(content, cacheModel,false).StringData; + var json = jsonSerializer.Serialize(content, cacheModel, false).StringData; var msgPack = msgPackSerializer.Serialize(content, cacheModel, false).ByteData; Console.WriteLine(json); Console.WriteLine(msgPackSerializer.ToJson(msgPack)); - var jsonContent = jsonSerializer.Deserialize(content, json, null,false); - var msgPackContent = msgPackSerializer.Deserialize(content, null, msgPack,false); + var jsonContent = jsonSerializer.Deserialize(content, json, null, false); + var msgPackContent = msgPackSerializer.Deserialize(content, null, msgPack, false); CollectionAssert.AreEqual(jsonContent.CultureData.Keys, msgPackContent.CultureData.Keys); diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializer.cs index 6c37046e8b..4bdf7e9665 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializer.cs @@ -17,9 +17,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData, bool published); /// - /// Serializes the + /// Serializes the /// - ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model,bool published); + ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model, bool published); } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs index 0cc10f5f98..358561cabd 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs @@ -24,7 +24,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource DateFormatString = "o" }; private readonly JsonNameTable _propertyNameTable = new DefaultJsonNameTable(); - public ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData,bool published) + public ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData, bool published) { if (stringData == null && byteData != null) throw new NotSupportedException($"{typeof(JsonContentNestedDataSerializer)} does not support byte[] serialization"); @@ -39,7 +39,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } } - public ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model,bool published) + public ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model, bool published) { // note that numeric values (which are Int32) are serialized without their // type (eg "value":1234) and JsonConvert by default deserializes them as Int64 diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs index f69232aad3..f1400382e6 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs @@ -40,7 +40,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource _options = defaultOptions .WithResolver(resolver) .WithCompression(MessagePackCompression.Lz4BlockArray) - .WithSecurity(MessagePackSecurity.UntrustedData); + .WithSecurity(MessagePackSecurity.UntrustedData); } public string ToJson(byte[] bin) @@ -49,7 +49,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource return json; } - public ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData,bool published) + public ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData, bool published) { if (byteData != null) { @@ -62,7 +62,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource // NOTE: We don't really support strings but it's possible if manually used (i.e. tests) var bin = Convert.FromBase64String(stringData); var cacheModel = MessagePackSerializer.Deserialize(bin, _options); - Expand(content, cacheModel,published); + Expand(content, cacheModel, published); return cacheModel; } else @@ -71,7 +71,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } } - public ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model,bool published) + public ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model, bool published) { Compress(content, model, published); var bytes = MessagePackSerializer.Serialize(model, _options); @@ -81,7 +81,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource /// /// Used during serialization to compress properties /// + /// /// + /// /// /// This will essentially 'double compress' property data. The MsgPack data as a whole will already be compressed /// but this will go a step further and double compress property data so that it is stored in the nucache file @@ -89,11 +91,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource /// read/decompressed as a string to be displayed on the front-end. This allows for potentially a significant /// memory savings but could also affect performance of first rendering pages while decompression occurs. /// - private void Compress(IReadOnlyContentBase content, ContentCacheDataModel model,bool published) + private void Compress(IReadOnlyContentBase content, ContentCacheDataModel model, bool published) { foreach(var propertyAliasToData in model.PropertyData) { - if (_propertyOptions.IsCompressed(content, propertyAliasToData.Key,published)) + if (_propertyOptions.IsCompressed(content, propertyAliasToData.Key, published)) { foreach(var property in propertyAliasToData.Value.Where(x => x.Value != null && x.Value is string)) { @@ -106,8 +108,10 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource /// /// Used during deserialization to map the property data as lazy or expand the value /// + /// /// - private void Expand(IReadOnlyContentBase content, ContentCacheDataModel nestedData,bool published) + /// + private void Expand(IReadOnlyContentBase content, ContentCacheDataModel nestedData, bool published) { foreach (var propertyAliasToData in nestedData.PropertyData) { diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializerFactory.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializerFactory.cs index 29378caf0f..5245df8353 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializerFactory.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializerFactory.cs @@ -13,7 +13,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource private readonly IMemberTypeService _memberTypeService; private readonly PropertyEditorCollection _propertyEditors; private readonly IPropertyCacheCompressionOptions _compressionOptions; - private readonly ConcurrentDictionary<(int, string,bool), bool> _isCompressedCache = new ConcurrentDictionary<(int, string,bool), bool>(); + private readonly ConcurrentDictionary<(int, string,bool), bool> _isCompressedCache = new ConcurrentDictionary<(int, string, bool), bool>(); public MsgPackContentNestedDataSerializerFactory( IContentTypeService contentTypeService, diff --git a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs index 4b3d7b20fa..dd3907e254 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs @@ -24,10 +24,10 @@ namespace Umbraco.Web.PublishedCache.NuCache } else { - composition.RegisterUnique(); + composition.RegisterUnique(); } - var unPublishedContentCompression = ConfigurationManager.AppSettings[NuCacheSerializerComponent.Nucache_UnPublishedContentCompression_Key]; - if ("MsgPack" == serializer && "true" == unPublishedContentCompression) + var unPublishedContentCompression = ConfigurationManager.AppSettings[NuCacheSerializerComponent.Nucache_UnPublishedContentCompression_Key]; + if (serializer == "MsgPack" && unPublishedContentCompression == "true") { composition.RegisterUnique(); } @@ -35,7 +35,7 @@ namespace Umbraco.Web.PublishedCache.NuCache { composition.RegisterUnique(); } - + composition.RegisterUnique(factory => new ContentDataSerializer(new DictionaryOfPropertyDataSerializer())); diff --git a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs index 7ca425aad9..f3373dab6c 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs @@ -89,7 +89,7 @@ namespace Umbraco.Web.PublishedCache.NuCache IPublishedModelFactory publishedModelFactory, UrlSegmentProviderCollection urlSegmentProviders, ISyncBootStateAccessor syncBootStateAccessor, - IContentCacheDataSerializerFactory contentCacheDataSerializerFactory, + IContentCacheDataSerializerFactory contentCacheDataSerializerFactory, ContentDataSerializer contentDataSerializer = null) : base(publishedSnapshotAccessor, variationContextAccessor) { @@ -262,7 +262,7 @@ namespace Umbraco.Web.PublishedCache.NuCache if (!okMedia) _logger.Warn("Loading media from local db raised warnings, will reload from database."); } - + if (!okContent) LockAndLoadContent(scope => LoadContentFromDatabaseLocked(scope, true)); @@ -1168,7 +1168,7 @@ namespace Umbraco.Web.PublishedCache.NuCache if (Volatile.Read(ref _isReady) == false) { throw new InvalidOperationException("The published snapshot service has not properly initialized."); - } + } var preview = previewToken.IsNullOrWhiteSpace() == false; return new PublishedSnapshot(this, preview); @@ -1491,7 +1491,7 @@ namespace Umbraco.Web.PublishedCache.NuCache UrlSegment = content.GetUrlSegment(_urlSegmentProviders) }; - var serialized = serializer.Serialize(ReadOnlyContentBaseAdapter.Create(content), contentCacheData,published); + var serialized = serializer.Serialize(ReadOnlyContentBaseAdapter.Create(content), contentCacheData, published); var dto = new ContentNuDto { From 2a6bdf73300b358ef5be2b1e4b3a480d2e790ef8 Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Mon, 12 Jul 2021 10:48:21 +0200 Subject: [PATCH 12/13] Fixes umbraco/Umbraco-CMS#10402 We added an overload that would conflict for people with existing templates, that was a bit too premature. Disabled for now with a note to look into it for v9 again. --- src/Umbraco.Web/UrlHelperRenderExtensions.cs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/Umbraco.Web/UrlHelperRenderExtensions.cs b/src/Umbraco.Web/UrlHelperRenderExtensions.cs index 2c547c841e..33e657e7cf 100644 --- a/src/Umbraco.Web/UrlHelperRenderExtensions.cs +++ b/src/Umbraco.Web/UrlHelperRenderExtensions.cs @@ -281,14 +281,15 @@ namespace Umbraco.Web return CreateHtmlString(url, htmlEncode); } - public static IHtmlString GetCropUrl(this UrlHelper urlHelper, ImageCropperValue imageCropperValue, string cropAlias, bool htmlEncode = true) - { - if (imageCropperValue == null || string.IsNullOrEmpty(imageCropperValue.Src)) return EmptyHtmlString; - - var url = imageCropperValue.Src.GetCropUrl(imageCropperValue, cropAlias: cropAlias, useCropDimensions: true); - - return CreateHtmlString(url, htmlEncode); - } + // TODO: enable again in v9 and make sure to document that `@Url.GetCropUrl(Model.Property, cropAlias: "Featured")` needs to be updated - see https://github.com/umbraco/Umbraco-CMS/pull/10527 for alternatives + // public static IHtmlString GetCropUrl(this UrlHelper urlHelper, ImageCropperValue imageCropperValue, string cropAlias, bool htmlEncode = true) + // { + // if (imageCropperValue == null || string.IsNullOrEmpty(imageCropperValue.Src)) return EmptyHtmlString; + // + // var url = imageCropperValue.Src.GetCropUrl(imageCropperValue, cropAlias: cropAlias, useCropDimensions: true); + // + // return CreateHtmlString(url, htmlEncode); + // } public static IHtmlString GetCropUrl(this UrlHelper urlHelper, ImageCropperValue imageCropperValue, From 3def2ee2ad2897e1d1c6d9b1d077e4c4652d01d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niels=20Lyngs=C3=B8?= Date: Fri, 9 Jul 2021 11:29:01 +0200 Subject: [PATCH 13/13] add vm.model.onValueChanged callback (cherry picked from commit 8ebb37a5963c020bd49f1bda940efd6776f069fc) # Conflicts: # src/Umbraco.Web.UI.Client/src/views/propertyeditors/mediapicker3/umbMediaPicker3PropertyEditor.component.js --- .../umbMediaPicker3PropertyEditor.component.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/mediapicker3/umbMediaPicker3PropertyEditor.component.js b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/mediapicker3/umbMediaPicker3PropertyEditor.component.js index 675381d46e..3ad3309085 100644 --- a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/mediapicker3/umbMediaPicker3PropertyEditor.component.js +++ b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/mediapicker3/umbMediaPicker3PropertyEditor.component.js @@ -93,6 +93,10 @@ vm.model.value.forEach(mediaEntry => updateMediaEntryData(mediaEntry)); + // set the onValueChanged callback, this will tell us if the media picker model changed on the server + // once the data is submitted. If so we need to re-initialize + vm.model.onValueChanged = onServerValueChanged; + userService.getCurrentUser().then(function (userData) { if (!vm.model.config.startNodeId) { @@ -115,6 +119,15 @@ }; + function onServerValueChanged(newVal, oldVal) { + if(newVal === null || !Array.isArray(newVal)) { + newVal = []; + vm.model.value = newVal; + } + + vm.model.value.forEach(mediaEntry => updateMediaEntryData(mediaEntry)); + } + function setDirty() { if (vm.propertyForm) { vm.propertyForm.$setDirty();