From f04e27ca197ae99975ed90aedaa4e53f95a2aee7 Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Tue, 22 Jun 2021 14:16:05 +0200 Subject: [PATCH 01/24] Bump version to 8.14.1 --- src/SolutionInfo.cs | 4 ++-- src/Umbraco.Web.UI/Umbraco.Web.UI.csproj | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/SolutionInfo.cs b/src/SolutionInfo.cs index 3619dc1371..3a43aec402 100644 --- a/src/SolutionInfo.cs +++ b/src/SolutionInfo.cs @@ -18,5 +18,5 @@ using System.Resources; [assembly: AssemblyVersion("8.0.0")] // these are FYI and changed automatically -[assembly: AssemblyFileVersion("8.14.0")] -[assembly: AssemblyInformationalVersion("8.14.0")] +[assembly: AssemblyFileVersion("8.14.1")] +[assembly: AssemblyInformationalVersion("8.14.1")] diff --git a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj index f3652a1273..f238984205 100644 --- a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj +++ b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj @@ -348,9 +348,9 @@ False True - 8140 + 8141 / - http://localhost:8140 + http://localhost:8141 8131 / http://localhost:8131 From 6816ae7ca41fb825b54dd23c9ffdfb1de32722a2 Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Fri, 2 Jul 2021 11:47:39 +0200 Subject: [PATCH 02/24] Fix dependencies to make 8.15 installable via NuGet (cherry picked from commit 3e208904356f1fe1262a7edee052dbf9663a6ad3) --- build/NuSpecs/UmbracoCms.Web.nuspec | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build/NuSpecs/UmbracoCms.Web.nuspec b/build/NuSpecs/UmbracoCms.Web.nuspec index e96a217f4e..7aebfae108 100644 --- a/build/NuSpecs/UmbracoCms.Web.nuspec +++ b/build/NuSpecs/UmbracoCms.Web.nuspec @@ -43,7 +43,8 @@ - + + From 9b32c83182526b6532300fb6fbacea75a195c842 Mon Sep 17 00:00:00 2001 From: Mole Date: Tue, 6 Jul 2021 13:03:44 +0200 Subject: [PATCH 03/24] Update PropertyTypeGroupDto IdentitySeed to no longer collide with new media types --- src/Umbraco.Core/Persistence/Dtos/PropertyTypeGroupDto.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Umbraco.Core/Persistence/Dtos/PropertyTypeGroupDto.cs b/src/Umbraco.Core/Persistence/Dtos/PropertyTypeGroupDto.cs index c48a6697ef..3de376ee0a 100644 --- a/src/Umbraco.Core/Persistence/Dtos/PropertyTypeGroupDto.cs +++ b/src/Umbraco.Core/Persistence/Dtos/PropertyTypeGroupDto.cs @@ -12,7 +12,7 @@ namespace Umbraco.Core.Persistence.Dtos internal class PropertyTypeGroupDto { [Column("id")] - [PrimaryKeyColumn(IdentitySeed = 12)] + [PrimaryKeyColumn(IdentitySeed = 56)] public int Id { get; set; } [Column("contenttypeNodeId")] From dfa161adb485036236883bda0ba4b24ee7d1e7f3 Mon Sep 17 00:00:00 2001 From: Mole Date: Tue, 6 Jul 2021 13:08:20 +0200 Subject: [PATCH 04/24] Add migration --- .../Migrations/Upgrade/UmbracoPlan.cs | 1 + .../V_8_15_0/UpdateCmsPropertyGroupIdSeed.cs | 21 +++++++++++++++++++ src/Umbraco.Core/Umbraco.Core.csproj | 1 + 3 files changed, 23 insertions(+) create mode 100644 src/Umbraco.Core/Migrations/Upgrade/V_8_15_0/UpdateCmsPropertyGroupIdSeed.cs diff --git a/src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs b/src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs index d9a71a1cc2..15ae30d3b4 100644 --- a/src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs +++ b/src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs @@ -206,6 +206,7 @@ namespace Umbraco.Core.Migrations.Upgrade // to 8.15.0... To("{8DDDCD0B-D7D5-4C97-BD6A-6B38CA65752F}"); To("{4695D0C9-0729-4976-985B-048D503665D8}"); + To("{5C424554-A32D-4852-8ED1-A13508187901}"); //FINAL } diff --git a/src/Umbraco.Core/Migrations/Upgrade/V_8_15_0/UpdateCmsPropertyGroupIdSeed.cs b/src/Umbraco.Core/Migrations/Upgrade/V_8_15_0/UpdateCmsPropertyGroupIdSeed.cs new file mode 100644 index 0000000000..908b5cda60 --- /dev/null +++ b/src/Umbraco.Core/Migrations/Upgrade/V_8_15_0/UpdateCmsPropertyGroupIdSeed.cs @@ -0,0 +1,21 @@ +using NPoco; +using Umbraco.Core.Persistence; +using Umbraco.Core.Persistence.Dtos; + +namespace Umbraco.Core.Migrations.Upgrade.V_8_15_0 +{ + public class UpdateCmsPropertyGroupIdSeed : MigrationBase + { + public UpdateCmsPropertyGroupIdSeed(IMigrationContext context) : base(context) + { + } + + public override void Migrate() + { + if (DatabaseType.IsSqlCe()) + { + Database.Execute(Sql("ALTER TABLE [cmsPropertyTypeGroup] ALTER COLUMN [id] IDENTITY (56,1)")); + } + } + } +} diff --git a/src/Umbraco.Core/Umbraco.Core.csproj b/src/Umbraco.Core/Umbraco.Core.csproj index 931cef070f..d37907ae0f 100755 --- a/src/Umbraco.Core/Umbraco.Core.csproj +++ b/src/Umbraco.Core/Umbraco.Core.csproj @@ -141,6 +141,7 @@ + From 4ac29927deb0183e9e27b42db32eaad661e1536d Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Wed, 7 Jul 2021 11:22:50 +0200 Subject: [PATCH 05/24] Bump version to 8.15.0 --- src/SolutionInfo.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SolutionInfo.cs b/src/SolutionInfo.cs index 6ef4d6ce85..201213cffa 100644 --- a/src/SolutionInfo.cs +++ b/src/SolutionInfo.cs @@ -19,4 +19,4 @@ using System.Resources; // these are FYI and changed automatically [assembly: AssemblyFileVersion("8.15.0")] -[assembly: AssemblyInformationalVersion("8.15.0-rc")] +[assembly: AssemblyInformationalVersion("8.15.0")] From 6b6525cc9891cc82d5be8701b91394bdb8ff5a9d Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Fri, 9 Jul 2021 19:33:42 +1200 Subject: [PATCH 06/24] Add whether the current content is published. Allowing for compression of just non published, non media content. --- .../IPropertyCacheCompression.cs | 10 +++++++-- .../IPropertyCacheCompressionOptions.cs | 9 +++++++- .../NoopPropertyCacheCompressionOptions.cs | 2 +- .../PropertyCacheCompression.cs | 10 ++++----- .../ContentSerializationTests.cs | 8 +++---- .../NuCache/DataSource/DatabaseDataSource.cs | 15 +++++++------ .../DataSource/IContentCacheDataSerializer.cs | 4 ++-- .../JsonContentNestedDataSerializer.cs | 4 ++-- .../MsgPackContentNestedDataSerializer.cs | 21 ++++++++++--------- ...gPackContentNestedDataSerializerFactory.cs | 2 +- .../NuCache/PublishedSnapshotService.cs | 2 +- 11 files changed, 52 insertions(+), 35 deletions(-) diff --git a/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompression.cs b/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompression.cs index 96a559630b..eb89173581 100644 --- a/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompression.cs +++ b/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompression.cs @@ -9,7 +9,13 @@ namespace Umbraco.Core.PropertyEditors /// /// public interface IPropertyCacheCompression - { - bool IsCompressed(IReadOnlyContentBase content, string propertyTypeAlias); + { + /// + /// Whether a property on the content is/should be compressed + /// + /// The content + /// The property to compress or not + /// Whether this content is the published version + bool IsCompressed(IReadOnlyContentBase content, string propertyTypeAlias,bool published); } } diff --git a/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompressionOptions.cs b/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompressionOptions.cs index 2fa0153f9e..71eb782ee2 100644 --- a/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompressionOptions.cs +++ b/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompressionOptions.cs @@ -4,6 +4,13 @@ namespace Umbraco.Core.PropertyEditors { public interface IPropertyCacheCompressionOptions { - bool IsCompressed(IReadOnlyContentBase content, PropertyType propertyType, IDataEditor dataEditor); + /// + /// Whether a property on the content is/should be compressed + /// + /// The content + /// The property to compress or not + /// The datatype of the property to compress or not + /// Whether this content is the published version + bool IsCompressed(IReadOnlyContentBase content, PropertyType propertyType, IDataEditor dataEditor,bool published); } } diff --git a/src/Umbraco.Core/PropertyEditors/NoopPropertyCacheCompressionOptions.cs b/src/Umbraco.Core/PropertyEditors/NoopPropertyCacheCompressionOptions.cs index 1f12d45769..f566172cd9 100644 --- a/src/Umbraco.Core/PropertyEditors/NoopPropertyCacheCompressionOptions.cs +++ b/src/Umbraco.Core/PropertyEditors/NoopPropertyCacheCompressionOptions.cs @@ -7,6 +7,6 @@ namespace Umbraco.Core.PropertyEditors /// internal class NoopPropertyCacheCompressionOptions : IPropertyCacheCompressionOptions { - public bool IsCompressed(IReadOnlyContentBase content, PropertyType propertyType, IDataEditor dataEditor) => false; + public bool IsCompressed(IReadOnlyContentBase content, PropertyType propertyType, IDataEditor dataEditor,bool published) => false; } } diff --git a/src/Umbraco.Core/PropertyEditors/PropertyCacheCompression.cs b/src/Umbraco.Core/PropertyEditors/PropertyCacheCompression.cs index 6be21fca7f..be0553ce65 100644 --- a/src/Umbraco.Core/PropertyEditors/PropertyCacheCompression.cs +++ b/src/Umbraco.Core/PropertyEditors/PropertyCacheCompression.cs @@ -14,13 +14,13 @@ namespace Umbraco.Core.PropertyEditors private readonly IPropertyCacheCompressionOptions _compressionOptions; private readonly IReadOnlyDictionary _contentTypes; private readonly PropertyEditorCollection _propertyEditors; - private readonly ConcurrentDictionary<(int contentTypeId, string propertyAlias), bool> _isCompressedCache; + private readonly ConcurrentDictionary<(int contentTypeId, string propertyAlias,bool published), bool> _isCompressedCache; public PropertyCacheCompression( IPropertyCacheCompressionOptions compressionOptions, IReadOnlyDictionary contentTypes, PropertyEditorCollection propertyEditors, - ConcurrentDictionary<(int, string), bool> compressedStoragePropertyEditorCache) + ConcurrentDictionary<(int, string,bool), bool> compressedStoragePropertyEditorCache) { _compressionOptions = compressionOptions; _contentTypes = contentTypes ?? throw new System.ArgumentNullException(nameof(contentTypes)); @@ -28,9 +28,9 @@ namespace Umbraco.Core.PropertyEditors _isCompressedCache = compressedStoragePropertyEditorCache; } - public bool IsCompressed(IReadOnlyContentBase content, string alias) + public bool IsCompressed(IReadOnlyContentBase content, string alias, bool published) { - var compressedStorage = _isCompressedCache.GetOrAdd((content.ContentTypeId, alias), x => + var compressedStorage = _isCompressedCache.GetOrAdd((content.ContentTypeId, alias, published), x => { if (!_contentTypes.TryGetValue(x.contentTypeId, out var ct)) return false; @@ -40,7 +40,7 @@ namespace Umbraco.Core.PropertyEditors if (!_propertyEditors.TryGet(propertyType.PropertyEditorAlias, out var propertyEditor)) return false; - return _compressionOptions.IsCompressed(content, propertyType, propertyEditor); + return _compressionOptions.IsCompressed(content, propertyType, propertyEditor, published); }); return compressedStorage; diff --git a/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs b/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs index b3543dad1a..9acad53364 100644 --- a/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs +++ b/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs @@ -56,14 +56,14 @@ namespace Umbraco.Tests.PublishedContent var content = Mock.Of(x => x.ContentTypeId == 1); - var json = jsonSerializer.Serialize(content, cacheModel).StringData; - var msgPack = msgPackSerializer.Serialize(content, cacheModel).ByteData; + var json = jsonSerializer.Serialize(content, cacheModel,false).StringData; + var msgPack = msgPackSerializer.Serialize(content, cacheModel, false).ByteData; Console.WriteLine(json); Console.WriteLine(msgPackSerializer.ToJson(msgPack)); - var jsonContent = jsonSerializer.Deserialize(content, json, null); - var msgPackContent = msgPackSerializer.Deserialize(content, null, msgPack); + var jsonContent = jsonSerializer.Deserialize(content, json, null,false); + var msgPackContent = msgPackSerializer.Deserialize(content, null, msgPack,false); CollectionAssert.AreEqual(jsonContent.CultureData.Keys, msgPackContent.CultureData.Keys); diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs index c112cc6efa..5ec51bffb8 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs @@ -393,12 +393,13 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } else { - var deserializedContent = serializer.Deserialize(dto, dto.EditData, dto.EditDataRaw); + bool published = false; + var deserializedContent = serializer.Deserialize(dto, dto.EditData, dto.EditDataRaw, published); d = new ContentData { Name = dto.EditName, - Published = false, + Published = published, TemplateId = dto.EditTemplateId, VersionId = dto.VersionId, VersionDate = dto.EditVersionDate, @@ -420,13 +421,14 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } else { - var deserializedContent = serializer.Deserialize(dto, dto.PubData, dto.PubDataRaw); + bool published = true; + var deserializedContent = serializer.Deserialize(dto, dto.PubData, dto.PubDataRaw, published); p = new ContentData { Name = dto.PubName, UrlSegment = deserializedContent.UrlSegment, - Published = true, + Published = published, TemplateId = dto.PubTemplateId, VersionId = dto.VersionId, VersionDate = dto.PubVersionDate, @@ -456,12 +458,13 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource if (dto.EditData == null && dto.EditDataRaw == null) throw new InvalidOperationException("No data for media " + dto.Id); - var deserializedMedia = serializer.Deserialize(dto, dto.EditData, dto.EditDataRaw); + bool published = true; + var deserializedMedia = serializer.Deserialize(dto, dto.EditData, dto.EditDataRaw, published); var p = new ContentData { Name = dto.EditName, - Published = true, + Published = published, TemplateId = -1, VersionId = dto.VersionId, VersionDate = dto.EditVersionDate, diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializer.cs index d1a83d8452..6c37046e8b 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializer.cs @@ -14,12 +14,12 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource /// /// Deserialize the data into a /// - ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData); + ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData, bool published); /// /// Serializes the /// - ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model); + ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model,bool published); } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs index 21cd0bf763..0cc10f5f98 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs @@ -24,7 +24,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource DateFormatString = "o" }; private readonly JsonNameTable _propertyNameTable = new DefaultJsonNameTable(); - public ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData) + public ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData,bool published) { if (stringData == null && byteData != null) throw new NotSupportedException($"{typeof(JsonContentNestedDataSerializer)} does not support byte[] serialization"); @@ -39,7 +39,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } } - public ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model) + public ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model,bool published) { // note that numeric values (which are Int32) are serialized without their // type (eg "value":1234) and JsonConvert by default deserializes them as Int64 diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs index 6ae872ef69..f69232aad3 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs @@ -39,7 +39,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource _options = defaultOptions .WithResolver(resolver) - .WithCompression(MessagePackCompression.Lz4BlockArray); + .WithCompression(MessagePackCompression.Lz4BlockArray) + .WithSecurity(MessagePackSecurity.UntrustedData); } public string ToJson(byte[] bin) @@ -48,12 +49,12 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource return json; } - public ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData) + public ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData,bool published) { if (byteData != null) { var cacheModel = MessagePackSerializer.Deserialize(byteData, _options); - Expand(content, cacheModel); + Expand(content, cacheModel, published); return cacheModel; } else if (stringData != null) @@ -61,7 +62,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource // NOTE: We don't really support strings but it's possible if manually used (i.e. tests) var bin = Convert.FromBase64String(stringData); var cacheModel = MessagePackSerializer.Deserialize(bin, _options); - Expand(content, cacheModel); + Expand(content, cacheModel,published); return cacheModel; } else @@ -70,9 +71,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } } - public ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model) + public ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model,bool published) { - Compress(content, model); + Compress(content, model, published); var bytes = MessagePackSerializer.Serialize(model, _options); return new ContentCacheDataSerializationResult(null, bytes); } @@ -88,11 +89,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource /// read/decompressed as a string to be displayed on the front-end. This allows for potentially a significant /// memory savings but could also affect performance of first rendering pages while decompression occurs. /// - private void Compress(IReadOnlyContentBase content, ContentCacheDataModel model) + private void Compress(IReadOnlyContentBase content, ContentCacheDataModel model,bool published) { foreach(var propertyAliasToData in model.PropertyData) { - if (_propertyOptions.IsCompressed(content, propertyAliasToData.Key)) + if (_propertyOptions.IsCompressed(content, propertyAliasToData.Key,published)) { foreach(var property in propertyAliasToData.Value.Where(x => x.Value != null && x.Value is string)) { @@ -106,11 +107,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource /// Used during deserialization to map the property data as lazy or expand the value /// /// - private void Expand(IReadOnlyContentBase content, ContentCacheDataModel nestedData) + private void Expand(IReadOnlyContentBase content, ContentCacheDataModel nestedData,bool published) { foreach (var propertyAliasToData in nestedData.PropertyData) { - if (_propertyOptions.IsCompressed(content, propertyAliasToData.Key)) + if (_propertyOptions.IsCompressed(content, propertyAliasToData.Key,published)) { foreach (var property in propertyAliasToData.Value.Where(x => x.Value != null)) { diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializerFactory.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializerFactory.cs index fcc3fa2bb8..29378caf0f 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializerFactory.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializerFactory.cs @@ -13,7 +13,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource private readonly IMemberTypeService _memberTypeService; private readonly PropertyEditorCollection _propertyEditors; private readonly IPropertyCacheCompressionOptions _compressionOptions; - private readonly ConcurrentDictionary<(int, string), bool> _isCompressedCache = new ConcurrentDictionary<(int, string), bool>(); + private readonly ConcurrentDictionary<(int, string,bool), bool> _isCompressedCache = new ConcurrentDictionary<(int, string,bool), bool>(); public MsgPackContentNestedDataSerializerFactory( IContentTypeService contentTypeService, diff --git a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs index f9c25b7b35..7ca425aad9 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs @@ -1491,7 +1491,7 @@ namespace Umbraco.Web.PublishedCache.NuCache UrlSegment = content.GetUrlSegment(_urlSegmentProviders) }; - var serialized = serializer.Serialize(ReadOnlyContentBaseAdapter.Create(content), contentCacheData); + var serialized = serializer.Serialize(ReadOnlyContentBaseAdapter.Create(content), contentCacheData,published); var dto = new ContentNuDto { From 044c900c2ba52428b7acecb8d15fc0769c5b9eaa Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Fri, 9 Jul 2021 20:14:35 +1200 Subject: [PATCH 07/24] Make IPropertyCacheCompressionOptions useful out of the box. key = "Umbraco.Web.PublishedCache.NuCache.CompressUnPublishedContent" value = "true" will compress all ntext properties on unpublished content --- ...dContentPropertyCacheCompressionOptions.cs | 29 +++++++++++++++++++ src/Umbraco.Core/Umbraco.Core.csproj | 1 + .../PublishedCache/NuCache/NuCacheComposer.cs | 12 ++++++-- .../NuCache/NuCacheSerializerComponent.cs | 1 + 4 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 src/Umbraco.Core/PropertyEditors/UnPublishedContentPropertyCacheCompressionOptions.cs diff --git a/src/Umbraco.Core/PropertyEditors/UnPublishedContentPropertyCacheCompressionOptions.cs b/src/Umbraco.Core/PropertyEditors/UnPublishedContentPropertyCacheCompressionOptions.cs new file mode 100644 index 0000000000..51433e476e --- /dev/null +++ b/src/Umbraco.Core/PropertyEditors/UnPublishedContentPropertyCacheCompressionOptions.cs @@ -0,0 +1,29 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Umbraco.Core.Models; + +namespace Umbraco.Core.PropertyEditors +{ + /// + /// Compress large, non published text properties + /// + internal class UnPublishedContentPropertyCacheCompressionOptions : IPropertyCacheCompressionOptions + { + public bool IsCompressed(IReadOnlyContentBase content, PropertyType propertyType, IDataEditor dataEditor, bool published) + { + if (published) + { + return false; + } + if (propertyType.SupportsPublishing && propertyType.ValueStorageType == ValueStorageType.Ntext) + { + //Only compress non published content that supports publishing and the property is text + return true; + } + return false; + } + } +} diff --git a/src/Umbraco.Core/Umbraco.Core.csproj b/src/Umbraco.Core/Umbraco.Core.csproj index 931cef070f..f6a523441e 100755 --- a/src/Umbraco.Core/Umbraco.Core.csproj +++ b/src/Umbraco.Core/Umbraco.Core.csproj @@ -171,6 +171,7 @@ + diff --git a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs index c6b214102b..4b3d7b20fa 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs @@ -26,8 +26,16 @@ namespace Umbraco.Web.PublishedCache.NuCache { composition.RegisterUnique(); } - - composition.RegisterUnique(); + var unPublishedContentCompression = ConfigurationManager.AppSettings[NuCacheSerializerComponent.Nucache_UnPublishedContentCompression_Key]; + if ("MsgPack" == serializer && "true" == unPublishedContentCompression) + { + composition.RegisterUnique(); + } + else + { + composition.RegisterUnique(); + } + composition.RegisterUnique(factory => new ContentDataSerializer(new DictionaryOfPropertyDataSerializer())); diff --git a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComponent.cs b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComponent.cs index a1d3ed2b12..c2d24b16ac 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComponent.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComponent.cs @@ -16,6 +16,7 @@ namespace Umbraco.Web.PublishedCache.NuCache public class NuCacheSerializerComponent : IComponent { internal const string Nucache_Serializer_Key = "Umbraco.Web.PublishedCache.NuCache.Serializer"; + internal const string Nucache_UnPublishedContentCompression_Key = "Umbraco.Web.PublishedCache.NuCache.CompressUnPublishedContent"; private const string JSON_SERIALIZER_VALUE = "JSON"; private readonly Lazy _service; private readonly IKeyValueService _keyValueService; From b119201480da1f99bf463776f71d2e73901f2aef Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Fri, 9 Jul 2021 20:55:03 +1200 Subject: [PATCH 08/24] fix serializer swap message --- .../PublishedCache/NuCache/NuCacheSerializerComponent.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComponent.cs b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComponent.cs index c2d24b16ac..41afb2b781 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComponent.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComponent.cs @@ -55,7 +55,7 @@ namespace Umbraco.Web.PublishedCache.NuCache { _profilingLogger.Warn($"Database NuCache was serialized using {currentSerializer}. Currently configured NuCache serializer {serializer}. Rebuilding Nucache"); - using (_profilingLogger.TraceDuration($"Rebuilding NuCache database with {currentSerializer} serializer")) + using (_profilingLogger.TraceDuration($"Rebuilding NuCache database with {serializer} serializer")) { _service.Value.Rebuild(); _keyValueService.SetValue(Nucache_Serializer_Key, serializer); From 6a8ed8012cc282e87b8d55c23f518f9991c0daba Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Fri, 9 Jul 2021 21:22:30 +1200 Subject: [PATCH 09/24] simplify --- .../UnPublishedContentPropertyCacheCompressionOptions.cs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/Umbraco.Core/PropertyEditors/UnPublishedContentPropertyCacheCompressionOptions.cs b/src/Umbraco.Core/PropertyEditors/UnPublishedContentPropertyCacheCompressionOptions.cs index 51433e476e..7a5212a985 100644 --- a/src/Umbraco.Core/PropertyEditors/UnPublishedContentPropertyCacheCompressionOptions.cs +++ b/src/Umbraco.Core/PropertyEditors/UnPublishedContentPropertyCacheCompressionOptions.cs @@ -14,11 +14,7 @@ namespace Umbraco.Core.PropertyEditors { public bool IsCompressed(IReadOnlyContentBase content, PropertyType propertyType, IDataEditor dataEditor, bool published) { - if (published) - { - return false; - } - if (propertyType.SupportsPublishing && propertyType.ValueStorageType == ValueStorageType.Ntext) + if (!published && propertyType.SupportsPublishing && propertyType.ValueStorageType == ValueStorageType.Ntext) { //Only compress non published content that supports publishing and the property is text return true; From 5e8b4572a0a322b8cfbe27fea3038367c76415fd Mon Sep 17 00:00:00 2001 From: Shannon Date: Fri, 9 Jul 2021 09:40:31 -0600 Subject: [PATCH 10/24] Ensures that the content nu data column is updated to support null during migration. --- .../Migrations/Upgrade/V_8_15_0/AddCmsContentNuByteColumn.cs | 3 +++ .../PublishedCache/NuCache/NuCacheSerializerComponent.cs | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Umbraco.Core/Migrations/Upgrade/V_8_15_0/AddCmsContentNuByteColumn.cs b/src/Umbraco.Core/Migrations/Upgrade/V_8_15_0/AddCmsContentNuByteColumn.cs index 3eab1a812e..9eeaa67189 100644 --- a/src/Umbraco.Core/Migrations/Upgrade/V_8_15_0/AddCmsContentNuByteColumn.cs +++ b/src/Umbraco.Core/Migrations/Upgrade/V_8_15_0/AddCmsContentNuByteColumn.cs @@ -16,6 +16,9 @@ namespace Umbraco.Core.Migrations.Upgrade.V_8_15_0 var columns = SqlSyntax.GetColumnsInSchema(Context.Database).ToList(); AddColumnIfNotExists(columns, "dataRaw"); + + // allow null + AlterColumn(Constants.DatabaseSchema.Tables.NodeData, "data"); } } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComponent.cs b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComponent.cs index a1d3ed2b12..fcbb5a3715 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComponent.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComponent.cs @@ -54,7 +54,7 @@ namespace Umbraco.Web.PublishedCache.NuCache { _profilingLogger.Warn($"Database NuCache was serialized using {currentSerializer}. Currently configured NuCache serializer {serializer}. Rebuilding Nucache"); - using (_profilingLogger.TraceDuration($"Rebuilding NuCache database with {currentSerializer} serializer")) + using (_profilingLogger.TraceDuration($"Rebuilding NuCache database with {serializer} serializer")) { _service.Value.Rebuild(); _keyValueService.SetValue(Nucache_Serializer_Key, serializer); From 7dc03256edf1bbe15b5b44c5d44a033369ce0e07 Mon Sep 17 00:00:00 2001 From: Shannon Date: Fri, 9 Jul 2021 12:05:19 -0600 Subject: [PATCH 11/24] Fixes SQLCE migration for nucache table. --- .../Create/Table/CreateTableOfDtoBuilder.cs | 2 + .../V_8_15_0/AddCmsContentNuByteColumn.cs | 47 +++++++++++++++++-- .../SqlSyntax/SqlSyntaxProviderBase.cs | 15 +++--- 3 files changed, 54 insertions(+), 10 deletions(-) diff --git a/src/Umbraco.Core/Migrations/Expressions/Create/Table/CreateTableOfDtoBuilder.cs b/src/Umbraco.Core/Migrations/Expressions/Create/Table/CreateTableOfDtoBuilder.cs index 4b73e9435d..a5e2fca1f7 100644 --- a/src/Umbraco.Core/Migrations/Expressions/Create/Table/CreateTableOfDtoBuilder.cs +++ b/src/Umbraco.Core/Migrations/Expressions/Create/Table/CreateTableOfDtoBuilder.cs @@ -9,6 +9,8 @@ namespace Umbraco.Core.Migrations.Expressions.Create.Table public class CreateTableOfDtoBuilder : IExecutableBuilder { private readonly IMigrationContext _context; + + // TODO: This doesn't do anything. private readonly DatabaseType[] _supportedDatabaseTypes; public CreateTableOfDtoBuilder(IMigrationContext context, params DatabaseType[] supportedDatabaseTypes) diff --git a/src/Umbraco.Core/Migrations/Upgrade/V_8_15_0/AddCmsContentNuByteColumn.cs b/src/Umbraco.Core/Migrations/Upgrade/V_8_15_0/AddCmsContentNuByteColumn.cs index 9eeaa67189..5217fc9870 100644 --- a/src/Umbraco.Core/Migrations/Upgrade/V_8_15_0/AddCmsContentNuByteColumn.cs +++ b/src/Umbraco.Core/Migrations/Upgrade/V_8_15_0/AddCmsContentNuByteColumn.cs @@ -1,5 +1,10 @@ -using System.Linq; +using NPoco; +using System.Data; +using System.Linq; +using Umbraco.Core.Persistence; +using Umbraco.Core.Persistence.DatabaseAnnotations; using Umbraco.Core.Persistence.Dtos; +using Umbraco.Core.PropertyEditors; namespace Umbraco.Core.Migrations.Upgrade.V_8_15_0 { @@ -13,12 +18,46 @@ namespace Umbraco.Core.Migrations.Upgrade.V_8_15_0 public override void Migrate() { + // allow null for the `data` field + if (DatabaseType.IsSqlCe()) + { + // SQLCE does not support altering NTEXT, so we have to jump through some hoops to do it + // All column ordering must remain the same as what is defined in the DTO so we need to create a temp table, + // drop orig and then re-create/copy. + Create.Table(withoutKeysAndIndexes: true).Do(); + Execute.Sql($"INSERT INTO [{TempTableName}] SELECT nodeId, published, data, rv FROM [{Constants.DatabaseSchema.Tables.NodeData}]").Do(); + Delete.Table(Constants.DatabaseSchema.Tables.NodeData).Do(); + Create.Table().Do(); + Execute.Sql($"INSERT INTO [{Constants.DatabaseSchema.Tables.NodeData}] SELECT nodeId, published, data, rv, NULL FROM [{TempTableName}]").Do(); + } + else + { + AlterColumn(Constants.DatabaseSchema.Tables.NodeData, "data"); + } + var columns = SqlSyntax.GetColumnsInSchema(Context.Database).ToList(); - AddColumnIfNotExists(columns, "dataRaw"); + } - // allow null - AlterColumn(Constants.DatabaseSchema.Tables.NodeData, "data"); + private const string TempTableName = Constants.DatabaseSchema.TableNamePrefix + "cms" + "ContentNuTEMP"; + + [TableName(TempTableName)] + [ExplicitColumns] + private class ContentNuDtoTemp + { + [Column("nodeId")] + public int NodeId { get; set; } + + [Column("published")] + public bool Published { get; set; } + + [Column("data")] + [SpecialDbType(SpecialDbTypes.NTEXT)] + [NullSetting(NullSetting = NullSettings.Null)] + public string Data { get; set; } + + [Column("rv")] + public long Rv { get; set; } } } } diff --git a/src/Umbraco.Core/Persistence/SqlSyntax/SqlSyntaxProviderBase.cs b/src/Umbraco.Core/Persistence/SqlSyntax/SqlSyntaxProviderBase.cs index 4d6b2eeea1..0d2d7aeb21 100644 --- a/src/Umbraco.Core/Persistence/SqlSyntax/SqlSyntaxProviderBase.cs +++ b/src/Umbraco.Core/Persistence/SqlSyntax/SqlSyntaxProviderBase.cs @@ -352,7 +352,7 @@ namespace Umbraco.Core.Persistence.SqlSyntax sql.Append(" "); sql.Append(FormatIdentity(column)); - var isNullable = column.IsNullable; + //var isNullable = column.IsNullable; //var constraint = FormatConstraint(column)?.TrimStart("CONSTRAINT "); //var hasConstraint = !string.IsNullOrWhiteSpace(constraint); @@ -360,11 +360,14 @@ namespace Umbraco.Core.Persistence.SqlSyntax //var defaultValue = FormatDefaultValue(column); //var hasDefaultValue = !string.IsNullOrWhiteSpace(defaultValue); - if (isNullable /*&& !hasConstraint && !hasDefaultValue*/) - { - sqls = Enumerable.Empty(); - return sql.ToString(); - } + // TODO: This used to exit if nullable but that means this would never work + // to return SQL if the column was nullable?!? I don't get it. This was here + // 4 years ago, I've removed it so that this works for nullable columns. + //if (isNullable /*&& !hasConstraint && !hasDefaultValue*/) + //{ + // sqls = Enumerable.Empty(); + // return sql.ToString(); + //} var msql = new List(); sqls = msql; From 93f5dd76317b437f14f37db49f1ac39ace300c39 Mon Sep 17 00:00:00 2001 From: Shannon Date: Fri, 9 Jul 2021 13:35:43 -0600 Subject: [PATCH 12/24] Changes bulk copy timeout to be infinite. Changes bulk copy to not allocate an arbitrary array. --- .../NPocoDatabaseExtensions-Bulk.cs | 35 ++++++++++++++----- .../Persistence/PocoDataDataReader.cs | 5 +-- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/src/Umbraco.Core/Persistence/NPocoDatabaseExtensions-Bulk.cs b/src/Umbraco.Core/Persistence/NPocoDatabaseExtensions-Bulk.cs index bff682d095..77cc0d6601 100644 --- a/src/Umbraco.Core/Persistence/NPocoDatabaseExtensions-Bulk.cs +++ b/src/Umbraco.Core/Persistence/NPocoDatabaseExtensions-Bulk.cs @@ -62,26 +62,33 @@ namespace Umbraco.Core.Persistence /// The number of records that were inserted. public static int BulkInsertRecords(this IUmbracoDatabase database, IEnumerable records, bool useNativeBulkInsert = true) { - var recordsA = records.ToArray(); - if (recordsA.Length == 0) return 0; + if (!records.Any()) return 0; var pocoData = database.PocoDataFactory.ForType(typeof(T)); if (pocoData == null) throw new InvalidOperationException("Could not find PocoData for " + typeof(T)); if (database.DatabaseType.IsSqlCe()) { - if (useNativeBulkInsert) return BulkInsertRecordsSqlCe(database, pocoData, recordsA); + if (useNativeBulkInsert) + { + return BulkInsertRecordsSqlCe(database, pocoData, records); + } + // else, no other choice - foreach (var record in recordsA) + var count = 0; + foreach (var record in records) + { database.Insert(record); - return recordsA.Length; + count++; + } + return count; } if (database.DatabaseType.IsSqlServer()) { return useNativeBulkInsert && database.DatabaseType.IsSqlServer2008OrLater() - ? BulkInsertRecordsSqlServer(database, pocoData, recordsA) - : BulkInsertRecordsWithCommands(database, recordsA); + ? BulkInsertRecordsSqlServer(database, pocoData, records) + : BulkInsertRecordsWithCommands(database, records.ToArray()); } throw new NotSupportedException(); } @@ -96,7 +103,9 @@ namespace Umbraco.Core.Persistence private static int BulkInsertRecordsWithCommands(IUmbracoDatabase database, T[] records) { foreach (var command in database.GenerateBulkInsertCommands(records)) + { command.ExecuteNonQuery(); + } return records.Length; // what else? } @@ -241,6 +250,10 @@ namespace Umbraco.Core.Persistence /// The number of records that were inserted. internal static int BulkInsertRecordsSqlServer(IUmbracoDatabase database, PocoData pocoData, IEnumerable records) { + // TODO: The main reason this exists is because the NPoco InsertBulk method doesn't return the number of items. + // It is worth investigating the performance of this vs NPoco's because we use a custom BulkDataReader + // which in theory should be more efficient than NPocos way of building up an in-memory DataTable. + // create command against the original database.Connection using (var command = database.CreateCommand(database.Connection, CommandType.Text, string.Empty)) { @@ -252,7 +265,13 @@ namespace Umbraco.Core.Persistence var syntax = database.SqlContext.SqlSyntax as SqlServerSyntaxProvider; if (syntax == null) throw new NotSupportedException("SqlSyntax must be SqlServerSyntaxProvider."); - using (var copy = new SqlBulkCopy(tConnection, SqlBulkCopyOptions.Default, tTransaction) { BulkCopyTimeout = 10000, DestinationTableName = tableName }) + using (var copy = new SqlBulkCopy(tConnection, SqlBulkCopyOptions.Default, tTransaction) + { + BulkCopyTimeout = 0, // 0 = no bulk copy timeout. If a timeout occurs it will be an connection/command timeout. + DestinationTableName = tableName, + // be consistent with NPoco: https://github.com/schotime/NPoco/blob/5117a55fde57547e928246c044fd40bd00b2d7d1/src/NPoco.SqlServer/SqlBulkCopyHelper.cs#L50 + BatchSize = 4096 + }) using (var bulkReader = new PocoDataDataReader(records, pocoData, syntax)) { //we need to add column mappings here because otherwise columns will be matched by their order and if the order of them are different in the DB compared diff --git a/src/Umbraco.Core/Persistence/PocoDataDataReader.cs b/src/Umbraco.Core/Persistence/PocoDataDataReader.cs index 460a4d3d90..397d903cc2 100644 --- a/src/Umbraco.Core/Persistence/PocoDataDataReader.cs +++ b/src/Umbraco.Core/Persistence/PocoDataDataReader.cs @@ -40,9 +40,10 @@ namespace Umbraco.Core.Persistence _tableDefinition = DefinitionFactory.GetTableDefinition(pd.Type, sqlSyntaxProvider); if (_tableDefinition == null) throw new InvalidOperationException("No table definition found for type " + pd.Type); - // only real columns, exclude result columns + // only real columns, exclude result/computed columns + // Like NPoco does: https://github.com/schotime/NPoco/blob/5117a55fde57547e928246c044fd40bd00b2d7d1/src/NPoco.SqlServer/SqlBulkCopyHelper.cs#L59 _readerColumns = pd.Columns - .Where(x => x.Value.ResultColumn == false) + .Where(x => x.Value.ResultColumn == false && x.Value.ComputedColumn == false) .Select(x => x.Value) .ToArray(); From a37f6c604f399b1c6f387adcc97fa71d0dc9abea Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Sat, 10 Jul 2021 14:25:26 +0200 Subject: [PATCH 13/24] Inconsistent formatting --- .../IPropertyCacheCompression.cs | 4 ++-- .../IPropertyCacheCompressionOptions.cs | 2 +- .../NoopPropertyCacheCompressionOptions.cs | 2 +- ...edContentPropertyCacheCompressionOptions.cs | 7 +------ .../ContentSerializationTests.cs | 6 +++--- .../DataSource/IContentCacheDataSerializer.cs | 4 ++-- .../JsonContentNestedDataSerializer.cs | 4 ++-- .../MsgPackContentNestedDataSerializer.cs | 18 +++++++++++------- ...sgPackContentNestedDataSerializerFactory.cs | 2 +- .../PublishedCache/NuCache/NuCacheComposer.cs | 8 ++++---- .../NuCache/PublishedSnapshotService.cs | 8 ++++---- 11 files changed, 32 insertions(+), 33 deletions(-) diff --git a/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompression.cs b/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompression.cs index eb89173581..69d10a2276 100644 --- a/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompression.cs +++ b/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompression.cs @@ -6,7 +6,7 @@ namespace Umbraco.Core.PropertyEditors /// Determines if a property type's value should be compressed in memory /// /// - /// + /// /// public interface IPropertyCacheCompression { @@ -16,6 +16,6 @@ namespace Umbraco.Core.PropertyEditors /// The content /// The property to compress or not /// Whether this content is the published version - bool IsCompressed(IReadOnlyContentBase content, string propertyTypeAlias,bool published); + bool IsCompressed(IReadOnlyContentBase content, string propertyTypeAlias, bool published); } } diff --git a/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompressionOptions.cs b/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompressionOptions.cs index 71eb782ee2..e4603d55e3 100644 --- a/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompressionOptions.cs +++ b/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompressionOptions.cs @@ -11,6 +11,6 @@ namespace Umbraco.Core.PropertyEditors /// The property to compress or not /// The datatype of the property to compress or not /// Whether this content is the published version - bool IsCompressed(IReadOnlyContentBase content, PropertyType propertyType, IDataEditor dataEditor,bool published); + bool IsCompressed(IReadOnlyContentBase content, PropertyType propertyType, IDataEditor dataEditor, bool published); } } diff --git a/src/Umbraco.Core/PropertyEditors/NoopPropertyCacheCompressionOptions.cs b/src/Umbraco.Core/PropertyEditors/NoopPropertyCacheCompressionOptions.cs index f566172cd9..ea3b8d8a2e 100644 --- a/src/Umbraco.Core/PropertyEditors/NoopPropertyCacheCompressionOptions.cs +++ b/src/Umbraco.Core/PropertyEditors/NoopPropertyCacheCompressionOptions.cs @@ -7,6 +7,6 @@ namespace Umbraco.Core.PropertyEditors /// internal class NoopPropertyCacheCompressionOptions : IPropertyCacheCompressionOptions { - public bool IsCompressed(IReadOnlyContentBase content, PropertyType propertyType, IDataEditor dataEditor,bool published) => false; + public bool IsCompressed(IReadOnlyContentBase content, PropertyType propertyType, IDataEditor dataEditor, bool published) => false; } } diff --git a/src/Umbraco.Core/PropertyEditors/UnPublishedContentPropertyCacheCompressionOptions.cs b/src/Umbraco.Core/PropertyEditors/UnPublishedContentPropertyCacheCompressionOptions.cs index 7a5212a985..ece25479cc 100644 --- a/src/Umbraco.Core/PropertyEditors/UnPublishedContentPropertyCacheCompressionOptions.cs +++ b/src/Umbraco.Core/PropertyEditors/UnPublishedContentPropertyCacheCompressionOptions.cs @@ -1,9 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Umbraco.Core.Models; +using Umbraco.Core.Models; namespace Umbraco.Core.PropertyEditors { diff --git a/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs b/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs index 9acad53364..9a44cf35f9 100644 --- a/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs +++ b/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs @@ -56,14 +56,14 @@ namespace Umbraco.Tests.PublishedContent var content = Mock.Of(x => x.ContentTypeId == 1); - var json = jsonSerializer.Serialize(content, cacheModel,false).StringData; + var json = jsonSerializer.Serialize(content, cacheModel, false).StringData; var msgPack = msgPackSerializer.Serialize(content, cacheModel, false).ByteData; Console.WriteLine(json); Console.WriteLine(msgPackSerializer.ToJson(msgPack)); - var jsonContent = jsonSerializer.Deserialize(content, json, null,false); - var msgPackContent = msgPackSerializer.Deserialize(content, null, msgPack,false); + var jsonContent = jsonSerializer.Deserialize(content, json, null, false); + var msgPackContent = msgPackSerializer.Deserialize(content, null, msgPack, false); CollectionAssert.AreEqual(jsonContent.CultureData.Keys, msgPackContent.CultureData.Keys); diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializer.cs index 6c37046e8b..4bdf7e9665 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializer.cs @@ -17,9 +17,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData, bool published); /// - /// Serializes the + /// Serializes the /// - ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model,bool published); + ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model, bool published); } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs index 0cc10f5f98..358561cabd 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs @@ -24,7 +24,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource DateFormatString = "o" }; private readonly JsonNameTable _propertyNameTable = new DefaultJsonNameTable(); - public ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData,bool published) + public ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData, bool published) { if (stringData == null && byteData != null) throw new NotSupportedException($"{typeof(JsonContentNestedDataSerializer)} does not support byte[] serialization"); @@ -39,7 +39,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } } - public ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model,bool published) + public ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model, bool published) { // note that numeric values (which are Int32) are serialized without their // type (eg "value":1234) and JsonConvert by default deserializes them as Int64 diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs index f69232aad3..f1400382e6 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs @@ -40,7 +40,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource _options = defaultOptions .WithResolver(resolver) .WithCompression(MessagePackCompression.Lz4BlockArray) - .WithSecurity(MessagePackSecurity.UntrustedData); + .WithSecurity(MessagePackSecurity.UntrustedData); } public string ToJson(byte[] bin) @@ -49,7 +49,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource return json; } - public ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData,bool published) + public ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData, bool published) { if (byteData != null) { @@ -62,7 +62,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource // NOTE: We don't really support strings but it's possible if manually used (i.e. tests) var bin = Convert.FromBase64String(stringData); var cacheModel = MessagePackSerializer.Deserialize(bin, _options); - Expand(content, cacheModel,published); + Expand(content, cacheModel, published); return cacheModel; } else @@ -71,7 +71,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } } - public ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model,bool published) + public ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model, bool published) { Compress(content, model, published); var bytes = MessagePackSerializer.Serialize(model, _options); @@ -81,7 +81,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource /// /// Used during serialization to compress properties /// + /// /// + /// /// /// This will essentially 'double compress' property data. The MsgPack data as a whole will already be compressed /// but this will go a step further and double compress property data so that it is stored in the nucache file @@ -89,11 +91,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource /// read/decompressed as a string to be displayed on the front-end. This allows for potentially a significant /// memory savings but could also affect performance of first rendering pages while decompression occurs. /// - private void Compress(IReadOnlyContentBase content, ContentCacheDataModel model,bool published) + private void Compress(IReadOnlyContentBase content, ContentCacheDataModel model, bool published) { foreach(var propertyAliasToData in model.PropertyData) { - if (_propertyOptions.IsCompressed(content, propertyAliasToData.Key,published)) + if (_propertyOptions.IsCompressed(content, propertyAliasToData.Key, published)) { foreach(var property in propertyAliasToData.Value.Where(x => x.Value != null && x.Value is string)) { @@ -106,8 +108,10 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource /// /// Used during deserialization to map the property data as lazy or expand the value /// + /// /// - private void Expand(IReadOnlyContentBase content, ContentCacheDataModel nestedData,bool published) + /// + private void Expand(IReadOnlyContentBase content, ContentCacheDataModel nestedData, bool published) { foreach (var propertyAliasToData in nestedData.PropertyData) { diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializerFactory.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializerFactory.cs index 29378caf0f..5245df8353 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializerFactory.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializerFactory.cs @@ -13,7 +13,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource private readonly IMemberTypeService _memberTypeService; private readonly PropertyEditorCollection _propertyEditors; private readonly IPropertyCacheCompressionOptions _compressionOptions; - private readonly ConcurrentDictionary<(int, string,bool), bool> _isCompressedCache = new ConcurrentDictionary<(int, string,bool), bool>(); + private readonly ConcurrentDictionary<(int, string,bool), bool> _isCompressedCache = new ConcurrentDictionary<(int, string, bool), bool>(); public MsgPackContentNestedDataSerializerFactory( IContentTypeService contentTypeService, diff --git a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs index 4b3d7b20fa..dd3907e254 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs @@ -24,10 +24,10 @@ namespace Umbraco.Web.PublishedCache.NuCache } else { - composition.RegisterUnique(); + composition.RegisterUnique(); } - var unPublishedContentCompression = ConfigurationManager.AppSettings[NuCacheSerializerComponent.Nucache_UnPublishedContentCompression_Key]; - if ("MsgPack" == serializer && "true" == unPublishedContentCompression) + var unPublishedContentCompression = ConfigurationManager.AppSettings[NuCacheSerializerComponent.Nucache_UnPublishedContentCompression_Key]; + if (serializer == "MsgPack" && unPublishedContentCompression == "true") { composition.RegisterUnique(); } @@ -35,7 +35,7 @@ namespace Umbraco.Web.PublishedCache.NuCache { composition.RegisterUnique(); } - + composition.RegisterUnique(factory => new ContentDataSerializer(new DictionaryOfPropertyDataSerializer())); diff --git a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs index 7ca425aad9..f3373dab6c 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs @@ -89,7 +89,7 @@ namespace Umbraco.Web.PublishedCache.NuCache IPublishedModelFactory publishedModelFactory, UrlSegmentProviderCollection urlSegmentProviders, ISyncBootStateAccessor syncBootStateAccessor, - IContentCacheDataSerializerFactory contentCacheDataSerializerFactory, + IContentCacheDataSerializerFactory contentCacheDataSerializerFactory, ContentDataSerializer contentDataSerializer = null) : base(publishedSnapshotAccessor, variationContextAccessor) { @@ -262,7 +262,7 @@ namespace Umbraco.Web.PublishedCache.NuCache if (!okMedia) _logger.Warn("Loading media from local db raised warnings, will reload from database."); } - + if (!okContent) LockAndLoadContent(scope => LoadContentFromDatabaseLocked(scope, true)); @@ -1168,7 +1168,7 @@ namespace Umbraco.Web.PublishedCache.NuCache if (Volatile.Read(ref _isReady) == false) { throw new InvalidOperationException("The published snapshot service has not properly initialized."); - } + } var preview = previewToken.IsNullOrWhiteSpace() == false; return new PublishedSnapshot(this, preview); @@ -1491,7 +1491,7 @@ namespace Umbraco.Web.PublishedCache.NuCache UrlSegment = content.GetUrlSegment(_urlSegmentProviders) }; - var serialized = serializer.Serialize(ReadOnlyContentBaseAdapter.Create(content), contentCacheData,published); + var serialized = serializer.Serialize(ReadOnlyContentBaseAdapter.Create(content), contentCacheData, published); var dto = new ContentNuDto { From 2a6bdf73300b358ef5be2b1e4b3a480d2e790ef8 Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Mon, 12 Jul 2021 10:48:21 +0200 Subject: [PATCH 14/24] Fixes umbraco/Umbraco-CMS#10402 We added an overload that would conflict for people with existing templates, that was a bit too premature. Disabled for now with a note to look into it for v9 again. --- src/Umbraco.Web/UrlHelperRenderExtensions.cs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/Umbraco.Web/UrlHelperRenderExtensions.cs b/src/Umbraco.Web/UrlHelperRenderExtensions.cs index 2c547c841e..33e657e7cf 100644 --- a/src/Umbraco.Web/UrlHelperRenderExtensions.cs +++ b/src/Umbraco.Web/UrlHelperRenderExtensions.cs @@ -281,14 +281,15 @@ namespace Umbraco.Web return CreateHtmlString(url, htmlEncode); } - public static IHtmlString GetCropUrl(this UrlHelper urlHelper, ImageCropperValue imageCropperValue, string cropAlias, bool htmlEncode = true) - { - if (imageCropperValue == null || string.IsNullOrEmpty(imageCropperValue.Src)) return EmptyHtmlString; - - var url = imageCropperValue.Src.GetCropUrl(imageCropperValue, cropAlias: cropAlias, useCropDimensions: true); - - return CreateHtmlString(url, htmlEncode); - } + // TODO: enable again in v9 and make sure to document that `@Url.GetCropUrl(Model.Property, cropAlias: "Featured")` needs to be updated - see https://github.com/umbraco/Umbraco-CMS/pull/10527 for alternatives + // public static IHtmlString GetCropUrl(this UrlHelper urlHelper, ImageCropperValue imageCropperValue, string cropAlias, bool htmlEncode = true) + // { + // if (imageCropperValue == null || string.IsNullOrEmpty(imageCropperValue.Src)) return EmptyHtmlString; + // + // var url = imageCropperValue.Src.GetCropUrl(imageCropperValue, cropAlias: cropAlias, useCropDimensions: true); + // + // return CreateHtmlString(url, htmlEncode); + // } public static IHtmlString GetCropUrl(this UrlHelper urlHelper, ImageCropperValue imageCropperValue, From 8ebb37a5963c020bd49f1bda940efd6776f069fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niels=20Lyngs=C3=B8?= Date: Fri, 9 Jul 2021 11:29:01 +0200 Subject: [PATCH 15/24] add vm.model.onValueChanged callback --- .../umbMediaPicker3PropertyEditor.component.js | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/mediapicker3/umbMediaPicker3PropertyEditor.component.js b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/mediapicker3/umbMediaPicker3PropertyEditor.component.js index 96f3126288..02b9f0b928 100644 --- a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/mediapicker3/umbMediaPicker3PropertyEditor.component.js +++ b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/mediapicker3/umbMediaPicker3PropertyEditor.component.js @@ -98,6 +98,10 @@ vm.model.value.forEach(mediaEntry => updateMediaEntryData(mediaEntry)); + // set the onValueChanged callback, this will tell us if the media picker model changed on the server + // once the data is submitted. If so we need to re-initialize + vm.model.onValueChanged = onServerValueChanged; + userService.getCurrentUser().then(function (userData) { if (!vm.model.config.startNodeId) { @@ -120,6 +124,15 @@ }; + function onServerValueChanged(newVal, oldVal) { + if(newVal === null || !Array.isArray(newVal)) { + newVal = []; + vm.model.value = newVal; + } + + vm.model.value.forEach(mediaEntry => updateMediaEntryData(mediaEntry)); + } + function setDirty() { if (vm.propertyForm) { vm.propertyForm.$setDirty(); @@ -259,7 +272,7 @@ function setActiveMedia(mediaEntryOrNull) { vm.activeMediaEntry = mediaEntryOrNull; } - + function editMedia(mediaEntry, options, $event) { if($event) From 3def2ee2ad2897e1d1c6d9b1d077e4c4652d01d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niels=20Lyngs=C3=B8?= Date: Fri, 9 Jul 2021 11:29:01 +0200 Subject: [PATCH 16/24] add vm.model.onValueChanged callback (cherry picked from commit 8ebb37a5963c020bd49f1bda940efd6776f069fc) # Conflicts: # src/Umbraco.Web.UI.Client/src/views/propertyeditors/mediapicker3/umbMediaPicker3PropertyEditor.component.js --- .../umbMediaPicker3PropertyEditor.component.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/mediapicker3/umbMediaPicker3PropertyEditor.component.js b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/mediapicker3/umbMediaPicker3PropertyEditor.component.js index 675381d46e..3ad3309085 100644 --- a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/mediapicker3/umbMediaPicker3PropertyEditor.component.js +++ b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/mediapicker3/umbMediaPicker3PropertyEditor.component.js @@ -93,6 +93,10 @@ vm.model.value.forEach(mediaEntry => updateMediaEntryData(mediaEntry)); + // set the onValueChanged callback, this will tell us if the media picker model changed on the server + // once the data is submitted. If so we need to re-initialize + vm.model.onValueChanged = onServerValueChanged; + userService.getCurrentUser().then(function (userData) { if (!vm.model.config.startNodeId) { @@ -115,6 +119,15 @@ }; + function onServerValueChanged(newVal, oldVal) { + if(newVal === null || !Array.isArray(newVal)) { + newVal = []; + vm.model.value = newVal; + } + + vm.model.value.forEach(mediaEntry => updateMediaEntryData(mediaEntry)); + } + function setDirty() { if (vm.propertyForm) { vm.propertyForm.$setDirty(); From 1f4cc81af8c5ce7b286238e29773f8a2be9c4182 Mon Sep 17 00:00:00 2001 From: Matt Brailsford Date: Mon, 19 Jul 2021 11:01:32 +0100 Subject: [PATCH 17/24] Removes `inert` attribute when `editorService.closeAll()` is called Fixes #10708 by removing the `inert` attribute on `#mainWrapper` when `editorService.closeAll()` is called. This code is necessary because when close all is called, there is no editor passed, and so `removeEditor` is never called, and so the `focusLockService.removeInertAttribute();` method isn't called. I add to the if block that handles the close all code instead. --- .../directives/components/editor/umbeditors.directive.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Umbraco.Web.UI.Client/src/common/directives/components/editor/umbeditors.directive.js b/src/Umbraco.Web.UI.Client/src/common/directives/components/editor/umbeditors.directive.js index 590f9627ab..1c9c85b075 100644 --- a/src/Umbraco.Web.UI.Client/src/common/directives/components/editor/umbeditors.directive.js +++ b/src/Umbraco.Web.UI.Client/src/common/directives/components/editor/umbeditors.directive.js @@ -126,7 +126,9 @@ // close all editors if (args && !args.editor && args.editors.length === 0) { editorCount = 0; - scope.editors = []; + scope.editors = []; + // Remove the inert attribute from the #mainwrapper + focusLockService.removeInertAttribute(); } })); From 98266d462c5821327d8d4320a08224447c370e8a Mon Sep 17 00:00:00 2001 From: Matt Brailsford Date: Mon, 19 Jul 2021 11:01:32 +0100 Subject: [PATCH 18/24] Removes `inert` attribute when `editorService.closeAll()` is called Fixes #10708 by removing the `inert` attribute on `#mainWrapper` when `editorService.closeAll()` is called. This code is necessary because when close all is called, there is no editor passed, and so `removeEditor` is never called, and so the `focusLockService.removeInertAttribute();` method isn't called. I add to the if block that handles the close all code instead. (cherry picked from commit 1f4cc81af8c5ce7b286238e29773f8a2be9c4182) --- .../directives/components/editor/umbeditors.directive.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Umbraco.Web.UI.Client/src/common/directives/components/editor/umbeditors.directive.js b/src/Umbraco.Web.UI.Client/src/common/directives/components/editor/umbeditors.directive.js index 590f9627ab..1c9c85b075 100644 --- a/src/Umbraco.Web.UI.Client/src/common/directives/components/editor/umbeditors.directive.js +++ b/src/Umbraco.Web.UI.Client/src/common/directives/components/editor/umbeditors.directive.js @@ -126,7 +126,9 @@ // close all editors if (args && !args.editor && args.editors.length === 0) { editorCount = 0; - scope.editors = []; + scope.editors = []; + // Remove the inert attribute from the #mainwrapper + focusLockService.removeInertAttribute(); } })); From fe9d0db763fc34644c9feedc015b5d24f6876639 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niels=20Lyngs=C3=B8?= Date: Thu, 22 Jul 2021 15:03:09 +0200 Subject: [PATCH 19/24] 10718 Enables more configured Blocks in Block List Editor (#10726) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Niels Lyngsø Co-authored-by: Mole --- .../src/common/resources/content.resource.js | 11 ++++---- .../blockeditormodelobject.service.js | 14 ++++++---- src/Umbraco.Web/Editors/ContentController.cs | 26 +++++++++++++++--- .../ContentEditing/ContentTypesByKeys.cs | 27 +++++++++++++++++++ src/Umbraco.Web/Umbraco.Web.csproj | 1 + 5 files changed, 66 insertions(+), 13 deletions(-) create mode 100644 src/Umbraco.Web/Models/ContentEditing/ContentTypesByKeys.cs diff --git a/src/Umbraco.Web.UI.Client/src/common/resources/content.resource.js b/src/Umbraco.Web.UI.Client/src/common/resources/content.resource.js index 01fff53931..a33e85c9a2 100644 --- a/src/Umbraco.Web.UI.Client/src/common/resources/content.resource.js +++ b/src/Umbraco.Web.UI.Client/src/common/resources/content.resource.js @@ -57,7 +57,7 @@ function contentResource($q, $http, umbDataFormatter, umbRequestHelper) { * Do stuff... * }); * - * + * * @returns {Promise} resourcePromise object. * */ @@ -691,11 +691,12 @@ function contentResource($q, $http, umbDataFormatter, umbRequestHelper) { getScaffoldByKeys: function (parentId, scaffoldKeys) { return umbRequestHelper.resourcePromise( - $http.get( + $http.post( umbRequestHelper.getApiUrl( "contentApiBaseUrl", - "GetEmptyByKeys", - { contentTypeKeys: scaffoldKeys, parentId: parentId })), + "GetEmptyByKeys"), + { contentTypeKeys: scaffoldKeys, parentId: parentId } + ), 'Failed to retrieve data for empty content items ids' + scaffoldKeys.join(", ")) .then(function (result) { Object.keys(result).map(function(key) { @@ -804,7 +805,7 @@ function contentResource($q, $http, umbDataFormatter, umbRequestHelper) { else if (options.orderDirection === "desc") { options.orderDirection = "Descending"; } - + //converts the value to a js bool function toBool(v) { if (Utilities.isNumber(v)) { diff --git a/src/Umbraco.Web.UI.Client/src/common/services/blockeditormodelobject.service.js b/src/Umbraco.Web.UI.Client/src/common/services/blockeditormodelobject.service.js index 4bcbbc89d6..ee96cfbaaa 100644 --- a/src/Umbraco.Web.UI.Client/src/common/services/blockeditormodelobject.service.js +++ b/src/Umbraco.Web.UI.Client/src/common/services/blockeditormodelobject.service.js @@ -487,7 +487,7 @@ * @returns {Object | null} Scaffold model for the that content type. Or null if the scaffolding model dosnt exist in this context. */ getScaffoldFromKey: function (contentTypeKey) { - return this.scaffolds.find(o => o.contentTypeKey === contentTypeKey); + return this.scaffolds.find(o => o.contentTypeKey === contentTypeKey) || null; }, /** @@ -499,7 +499,7 @@ * @returns {Object | null} Scaffold model for the that content type. Or null if the scaffolding model dosnt exist in this context. */ getScaffoldFromAlias: function (contentTypeAlias) { - return this.scaffolds.find(o => o.contentTypeAlias === contentTypeAlias); + return this.scaffolds.find(o => o.contentTypeAlias === contentTypeAlias) || null; }, /** @@ -609,10 +609,14 @@ blockObject.settingsData = settingsData; // make basics from scaffold - blockObject.settings = Utilities.copy(settingsScaffold); - ensureUdiAndKey(blockObject.settings, settingsUdi); + if (settingsScaffold !== null) {// We might not have settingsScaffold + blockObject.settings = Utilities.copy(settingsScaffold); + ensureUdiAndKey(blockObject.settings, settingsUdi); - mapToElementModel(blockObject.settings, settingsData); + mapToElementModel(blockObject.settings, settingsData); + } else { + blockObject.settings = null; + } // add settings content-app appendSettingsContentApp(blockObject.content, this.__labels.settingsName); diff --git a/src/Umbraco.Web/Editors/ContentController.cs b/src/Umbraco.Web/Editors/ContentController.cs index 55eee7b2a2..8293d1274f 100644 --- a/src/Umbraco.Web/Editors/ContentController.cs +++ b/src/Umbraco.Web/Editors/ContentController.cs @@ -449,6 +449,13 @@ namespace Umbraco.Web.Editors return result; } + private IDictionary GetEmptyByKeysInternal(Guid[] contentTypeKeys, int parentId) + { + using var scope = _scopeProvider.CreateScope(autoComplete: true); + var contentTypes = Services.ContentTypeService.GetAll(contentTypeKeys).ToList(); + return GetEmpties(contentTypes, parentId).ToDictionary(x => x.ContentTypeKey); + } + /// /// Gets a collection of empty content items for all document types. /// @@ -457,9 +464,22 @@ namespace Umbraco.Web.Editors [OutgoingEditorModelEvent] public IDictionary GetEmptyByKeys([FromUri] Guid[] contentTypeKeys, [FromUri] int parentId) { - using var scope = _scopeProvider.CreateScope(autoComplete: true); - var contentTypes = Services.ContentTypeService.GetAll(contentTypeKeys).ToList(); - return GetEmpties(contentTypes, parentId).ToDictionary(x => x.ContentTypeKey); + return GetEmptyByKeysInternal(contentTypeKeys, parentId); + } + + /// + /// Gets a collection of empty content items for all document types. + /// + /// + /// This is a post request in order to support a large amount of GUIDs without hitting the URL length limit. + /// + /// + /// + [HttpPost] + [OutgoingEditorModelEvent] + public IDictionary GetEmptyByKeys(ContentTypesByKeys contentTypeByKeys) + { + return GetEmptyByKeysInternal(contentTypeByKeys.ContentTypeKeys, contentTypeByKeys.ParentId); } [OutgoingEditorModelEvent] diff --git a/src/Umbraco.Web/Models/ContentEditing/ContentTypesByKeys.cs b/src/Umbraco.Web/Models/ContentEditing/ContentTypesByKeys.cs new file mode 100644 index 0000000000..8bf1d452bc --- /dev/null +++ b/src/Umbraco.Web/Models/ContentEditing/ContentTypesByKeys.cs @@ -0,0 +1,27 @@ +using System; +using System.ComponentModel.DataAnnotations; +using System.Runtime.Serialization; + +namespace Umbraco.Web.Models.ContentEditing +{ + /// + /// A model for retrieving multiple content types based on their keys. + /// + [DataContract(Name = "contentTypes", Namespace = "")] + public class ContentTypesByKeys + { + /// + /// ID of the parent of the content type. + /// + [DataMember(Name = "parentId")] + [Required] + public int ParentId { get; set; } + + /// + /// The id of every content type to get. + /// + [DataMember(Name = "contentTypeKeys")] + [Required] + public Guid[] ContentTypeKeys { get; set; } + } +} diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index 4dce8a2da0..e830722cc7 100644 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -241,6 +241,7 @@ + From 071a89a59ba8ec0acf1766e0ef4c0c4f248f4413 Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Fri, 23 Jul 2021 13:32:30 +0200 Subject: [PATCH 20/24] Bump version to 8.14.2 --- src/SolutionInfo.cs | 4 ++-- src/Umbraco.Web.UI/Umbraco.Web.UI.csproj | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/SolutionInfo.cs b/src/SolutionInfo.cs index 3a43aec402..625cbe3fb7 100644 --- a/src/SolutionInfo.cs +++ b/src/SolutionInfo.cs @@ -18,5 +18,5 @@ using System.Resources; [assembly: AssemblyVersion("8.0.0")] // these are FYI and changed automatically -[assembly: AssemblyFileVersion("8.14.1")] -[assembly: AssemblyInformationalVersion("8.14.1")] +[assembly: AssemblyFileVersion("8.14.2")] +[assembly: AssemblyInformationalVersion("8.14.2")] diff --git a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj index f238984205..ce465c6dc5 100644 --- a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj +++ b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj @@ -348,9 +348,9 @@ False True - 8141 + 8142 / - http://localhost:8141 + http://localhost:8142 8131 / http://localhost:8131 From 9807365800880dcfb0b7d71fd25553a396ba8bed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niels=20Lyngs=C3=B8?= Date: Fri, 23 Jul 2021 14:26:12 +0200 Subject: [PATCH 21/24] Escape HTML in localization tokens. (#10729) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Niels Lyngsø --- .../common/services/localization.service.js | 60 +++++++++---------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/src/Umbraco.Web.UI.Client/src/common/services/localization.service.js b/src/Umbraco.Web.UI.Client/src/common/services/localization.service.js index 99162eaf53..805afae5b8 100644 --- a/src/Umbraco.Web.UI.Client/src/common/services/localization.service.js +++ b/src/Umbraco.Web.UI.Client/src/common/services/localization.service.js @@ -22,7 +22,7 @@ */ angular.module('umbraco.services') -.factory('localizationService', function ($http, $q, eventsService, $window, $filter, userService) { +.factory('localizationService', function ($http, $q, eventsService) { // TODO: This should be injected as server vars var url = "LocalizedText"; @@ -61,14 +61,14 @@ angular.module('umbraco.services') return "[" + alias + "]"; } - + var service = { - + // loads the language resource file from the server initLocalizedResources: function () { - // TODO: This promise handling is super ugly, we should just be returnning the promise from $http and returning inner values. + // TODO: This promise handling is super ugly, we should just be returnning the promise from $http and returning inner values. var deferred = $q.defer(); @@ -120,7 +120,7 @@ angular.module('umbraco.services') * @description * Helper to tokenize and compile a localization string * @param {String} value the value to tokenize - * @param {Object} scope the $scope object + * @param {Object} scope the $scope object * @returns {String} tokenized resource string */ tokenize: function (value, scope) { @@ -138,8 +138,8 @@ angular.module('umbraco.services') } return value; }, - - + + /** * @ngdoc method * @name umbraco.services.localizationService#tokenReplace @@ -148,19 +148,19 @@ angular.module('umbraco.services') * @description * Helper to replace tokens * @param {String} value the text-string to manipulate - * @param {Array} tekens An array of tokens values + * @param {Array} tekens An array of tokens values * @returns {String} Replaced test-string */ tokenReplace: function (text, tokens) { if (tokens) { for (var i = 0; i < tokens.length; i++) { - text = text.replace("%" + i + "%", tokens[i]); + text = text.replace("%" + i + "%", _.escape(tokens[i])); } } return text; }, - - + + /** * @ngdoc method * @name umbraco.services.localizationService#localize @@ -168,16 +168,16 @@ angular.module('umbraco.services') * * @description * Checks the dictionary for a localized resource string - * @param {String} value the area/key to localize in the format of 'section_key' + * @param {String} value the area/key to localize in the format of 'section_key' * alternatively if no section is set such as 'key' then we assume the key is to be looked in * the 'general' section - * + * * @param {Array} tokens if specified this array will be sent as parameter values * This replaces %0% and %1% etc in the dictionary key value with the passed in strings - * - * @param {String} fallbackValue if specified this string will be returned if no matching + * + * @param {String} fallbackValue if specified this string will be returned if no matching * entry was found in the dictionary - * + * * @returns {String} localized resource string */ localize: function (value, tokens, fallbackValue) { @@ -194,7 +194,7 @@ angular.module('umbraco.services') * @description * Checks the dictionary for multipe localized resource strings at once, preventing the need for nested promises * with localizationService.localize - * + * * ##Usage * 
          * localizationService.localizeMany(["speechBubbles_templateErrorHeader", "speechBubbles_templateErrorText"]).then(function(data){
@@ -203,11 +203,11 @@ angular.module('umbraco.services')
          *      notificationService.error(header, message);
          * });
          *
- * - * @param {Array} keys is an array of strings of the area/key to localize in the format of 'section_key' + * + * @param {Array} keys is an array of strings of the area/key to localize in the format of 'section_key' * alternatively if no section is set such as 'key' then we assume the key is to be looked in * the 'general' section - * + * * @returns {Array} An array of localized resource string in the same order */ localizeMany: function(keys) { @@ -234,18 +234,18 @@ angular.module('umbraco.services') * @description * Checks the dictionary for multipe localized resource strings at once & concats them to a single string * Which was not possible with localizationSerivce.localize() due to returning a promise - * + * * ##Usage * 
          * localizationService.concat(["speechBubbles_templateErrorHeader", "speechBubbles_templateErrorText"]).then(function(data){
          *      var combinedText = data;
          * });
          *
- * - * @param {Array} keys is an array of strings of the area/key to localize in the format of 'section_key' + * + * @param {Array} keys is an array of strings of the area/key to localize in the format of 'section_key' * alternatively if no section is set such as 'key' then we assume the key is to be looked in * the 'general' section - * + * * @returns {String} An concatenated string of localized resource string passed into the function in the same order */ concat: function(keys) { @@ -280,7 +280,7 @@ angular.module('umbraco.services') * @description * Checks the dictionary for multipe localized resource strings at once & formats a tokenized message * Which was not possible with localizationSerivce.localize() due to returning a promise - * + * * ##Usage * 
          * localizationService.format(["template_insert", "template_insertSections"], "%0% %1%").then(function(data){
@@ -288,14 +288,14 @@ angular.module('umbraco.services')
          *      var formattedResult = data;
          * });
          *
- * - * @param {Array} keys is an array of strings of the area/key to localize in the format of 'section_key' + * + * @param {Array} keys is an array of strings of the area/key to localize in the format of 'section_key' * alternatively if no section is set such as 'key' then we assume the key is to be looked in * the 'general' section - * + * * @param {String} message is the string you wish to replace containing tokens in the format of %0% and %1% * with the localized resource strings - * + * * @returns {String} An concatenated string of localized resource string passed into the function in the same order */ format: function(keys, message){ @@ -330,7 +330,7 @@ angular.module('umbraco.services') resourceFileLoadStatus = "none"; resourceLoadingPromise = []; }); - + // return the local instance when called return service; From f7529e0cf52220759ec458f8328268d5dbc3cecb Mon Sep 17 00:00:00 2001 From: Bjarne Fyrstenborg Date: Tue, 27 Jul 2021 14:52:01 +0200 Subject: [PATCH 22/24] Inject windowResizeListener in Image Crop directive (#10745) (cherry picked from commit 2eb554f1b46d50ae8445dcdedc2d759fcb9178da) --- .../components/imaging/umbimagecrop.directive.js | 12 ++++++------ .../common/services/windowresizelistener.service.js | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/Umbraco.Web.UI.Client/src/common/directives/components/imaging/umbimagecrop.directive.js b/src/Umbraco.Web.UI.Client/src/common/directives/components/imaging/umbimagecrop.directive.js index 60ba71d7a5..1f5d506bdd 100644 --- a/src/Umbraco.Web.UI.Client/src/common/directives/components/imaging/umbimagecrop.directive.js +++ b/src/Umbraco.Web.UI.Client/src/common/directives/components/imaging/umbimagecrop.directive.js @@ -6,7 +6,7 @@ **/ angular.module("umbraco.directives") .directive('umbImageCrop', - function ($timeout, $window, cropperHelper) { + function ($timeout, cropperHelper, windowResizeListener) { const MAX_SCALE = 4; @@ -26,7 +26,7 @@ angular.module("umbraco.directives") forceUpdate: '@?' }, - link: function (scope, element, attrs, windowResizeListener) { + link: function (scope, element, attrs) { var unsubscribe = []; let sliderRef = null; @@ -72,7 +72,7 @@ angular.module("umbraco.directives") }; function updateSlider() { - if(sliderRef) { + if (sliderRef) { // Update slider range min/max sliderRef.noUiSlider.updateOptions({ "range": { @@ -102,7 +102,7 @@ angular.module("umbraco.directives") // cross-browser wheel delta var delta = Math.max(-50, Math.min(50, (event.wheelDelta || -event.detail))); - if(sliderRef) { + if (sliderRef) { var currentScale =sliderRef.noUiSlider.get(); var newScale = Math.min(Math.max(currentScale + delta*.001*scope.dimensions.image.ratio, scope.dimensions.scale.min), scope.dimensions.scale.max); @@ -127,8 +127,8 @@ angular.module("umbraco.directives") 'left': (parseInt(scope.dimensions.margin.left, 10)) + 'px' } }; - updateStyles(); + updateStyles(); //elements var $viewport = element.find(".viewport"); @@ -138,11 +138,11 @@ angular.module("umbraco.directives") $overlay.bind("focus", function () { $overlay.bind("DOMMouseScroll mousewheel onmousewheel", onScroll); }); + $overlay.bind("blur", function () { $overlay.unbind("DOMMouseScroll mousewheel onmousewheel", onScroll); }); - //default constraints for drag n drop var constraints = { left: { max: 0, min: 0 }, top: { max: 0, min: 0 } }; scope.constraints = constraints; diff --git a/src/Umbraco.Web.UI.Client/src/common/services/windowresizelistener.service.js b/src/Umbraco.Web.UI.Client/src/common/services/windowresizelistener.service.js index 68691b8629..3c13228a9b 100644 --- a/src/Umbraco.Web.UI.Client/src/common/services/windowresizelistener.service.js +++ b/src/Umbraco.Web.UI.Client/src/common/services/windowresizelistener.service.js @@ -11,7 +11,7 @@ */ function windowResizeListener($rootScope) { - var WinReszier = (function () { + var WinResizer = (function () { var registered = []; var inited = false; var resize = _.debounce(function(ev) { @@ -51,7 +51,7 @@ function windowResizeListener($rootScope) { * @param {Function} cb */ register: function (cb) { - WinReszier.register(cb); + WinResizer.register(cb); }, /** @@ -59,9 +59,9 @@ function windowResizeListener($rootScope) { * @param {Function} cb */ unregister: function(cb) { - WinReszier.unregister(cb); + WinResizer.unregister(cb); } }; } -angular.module('umbraco.services').factory('windowResizeListener', windowResizeListener); \ No newline at end of file +angular.module('umbraco.services').factory('windowResizeListener', windowResizeListener); From 7c336554db55fc3f0725b29718a4b6649b5fa97f Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Thu, 29 Jul 2021 13:54:05 +0200 Subject: [PATCH 23/24] Fix: Issue when upgraded from 8.14 to 8.15 when inviting user into the backoffice. #10746 (#10767) * Remove unusable method * Fixes #10746 by removing the recursive call ending in an infinite loop Also: the culture is already known, don't take the `CurrentUICulture` (cherry picked from commit cb06442de44d93b6437fcf4fa92bb4d12dc68f5e) --- .../LocalizedTextServiceExtensions.cs | 26 +------------------ 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/src/Umbraco.Core/Services/LocalizedTextServiceExtensions.cs b/src/Umbraco.Core/Services/LocalizedTextServiceExtensions.cs index 2911441578..e062f11306 100644 --- a/src/Umbraco.Core/Services/LocalizedTextServiceExtensions.cs +++ b/src/Umbraco.Core/Services/LocalizedTextServiceExtensions.cs @@ -71,30 +71,6 @@ namespace Umbraco.Core.Services #pragma warning restore CS0618 // Type or member is obsolete } - /// - /// Localize using the current thread culture - /// - /// - /// - /// - /// - /// - public static string Localize(this ILocalizedTextService manager, string area, string alias, IDictionary tokens = null) - { - if (manager is ILocalizedTextService2 manager2) - { - return manager2.Localize(area, alias, Thread.CurrentThread.CurrentUICulture, tokens); - } - var fullKey = alias; - if (area != null) - { - fullKey = string.Concat(area, "/", alias); - } -#pragma warning disable CS0618 // Type or member is obsolete - return manager.Localize(fullKey, Thread.CurrentThread.CurrentUICulture, tokens); -#pragma warning restore CS0618 // Type or member is obsolete - } - /// /// Localize a key without any variables /// @@ -108,7 +84,7 @@ namespace Umbraco.Core.Services { if (manager is ILocalizedTextService2 manager2) { - return manager2.Localize(area, alias, Thread.CurrentThread.CurrentUICulture, tokens); + return manager2.Localize(area, alias, culture, ConvertToDictionaryVars(tokens)); } var fullKey = alias; if (area != null) From b070a36039f02b94a01d630bd1dfb5a1d52d66b9 Mon Sep 17 00:00:00 2001 From: Julia Gru <56249914+julczka@users.noreply.github.com> Date: Mon, 2 Aug 2021 15:11:42 +0200 Subject: [PATCH 24/24] reset confirmDiscardChanges on save (#10777) --- .../src/views/content/content.rights.controller.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Umbraco.Web.UI.Client/src/views/content/content.rights.controller.js b/src/Umbraco.Web.UI.Client/src/views/content/content.rights.controller.js index 78521af2fe..2fb032d301 100644 --- a/src/Umbraco.Web.UI.Client/src/views/content/content.rights.controller.js +++ b/src/Umbraco.Web.UI.Client/src/views/content/content.rights.controller.js @@ -154,9 +154,10 @@ currentForm.$dirty = false; } }); - + $scope.dialog.confirmDiscardChanges = false; vm.saveState = "success"; vm.saveSuccces = true; + }, function(error){ vm.saveState = "error"; vm.saveError = error;