From f04e27ca197ae99975ed90aedaa4e53f95a2aee7 Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Tue, 22 Jun 2021 14:16:05 +0200 Subject: [PATCH 001/123] Bump version to 8.14.1 --- src/SolutionInfo.cs | 4 ++-- src/Umbraco.Web.UI/Umbraco.Web.UI.csproj | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/SolutionInfo.cs b/src/SolutionInfo.cs index 3619dc1371..3a43aec402 100644 --- a/src/SolutionInfo.cs +++ b/src/SolutionInfo.cs @@ -18,5 +18,5 @@ using System.Resources; [assembly: AssemblyVersion("8.0.0")] // these are FYI and changed automatically -[assembly: AssemblyFileVersion("8.14.0")] -[assembly: AssemblyInformationalVersion("8.14.0")] +[assembly: AssemblyFileVersion("8.14.1")] +[assembly: AssemblyInformationalVersion("8.14.1")] diff --git a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj index f3652a1273..f238984205 100644 --- a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj +++ b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj @@ -348,9 +348,9 @@ False True - 8140 + 8141 / - http://localhost:8140 + http://localhost:8141 8131 / http://localhost:8131 From 6816ae7ca41fb825b54dd23c9ffdfb1de32722a2 Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Fri, 2 Jul 2021 11:47:39 +0200 Subject: [PATCH 002/123] Fix dependencies to make 8.15 installable via NuGet (cherry picked from commit 3e208904356f1fe1262a7edee052dbf9663a6ad3) --- build/NuSpecs/UmbracoCms.Web.nuspec | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build/NuSpecs/UmbracoCms.Web.nuspec b/build/NuSpecs/UmbracoCms.Web.nuspec index e96a217f4e..7aebfae108 100644 --- a/build/NuSpecs/UmbracoCms.Web.nuspec +++ b/build/NuSpecs/UmbracoCms.Web.nuspec @@ -43,7 +43,8 @@ - + + From 9b32c83182526b6532300fb6fbacea75a195c842 Mon Sep 17 00:00:00 2001 From: Mole Date: Tue, 6 Jul 2021 13:03:44 +0200 Subject: [PATCH 003/123] Update PropertyTypeGroupDto IdentitySeed to no longer collide with new media types --- src/Umbraco.Core/Persistence/Dtos/PropertyTypeGroupDto.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Umbraco.Core/Persistence/Dtos/PropertyTypeGroupDto.cs b/src/Umbraco.Core/Persistence/Dtos/PropertyTypeGroupDto.cs index c48a6697ef..3de376ee0a 100644 --- a/src/Umbraco.Core/Persistence/Dtos/PropertyTypeGroupDto.cs +++ b/src/Umbraco.Core/Persistence/Dtos/PropertyTypeGroupDto.cs @@ -12,7 +12,7 @@ namespace Umbraco.Core.Persistence.Dtos internal class PropertyTypeGroupDto { [Column("id")] - [PrimaryKeyColumn(IdentitySeed = 12)] + [PrimaryKeyColumn(IdentitySeed = 56)] public int Id { get; set; } [Column("contenttypeNodeId")] From dfa161adb485036236883bda0ba4b24ee7d1e7f3 Mon Sep 17 00:00:00 2001 From: Mole Date: Tue, 6 Jul 2021 13:08:20 +0200 Subject: [PATCH 004/123] Add migration --- .../Migrations/Upgrade/UmbracoPlan.cs | 1 + .../V_8_15_0/UpdateCmsPropertyGroupIdSeed.cs | 21 +++++++++++++++++++ src/Umbraco.Core/Umbraco.Core.csproj | 1 + 3 files changed, 23 insertions(+) create mode 100644 src/Umbraco.Core/Migrations/Upgrade/V_8_15_0/UpdateCmsPropertyGroupIdSeed.cs diff --git a/src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs b/src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs index d9a71a1cc2..15ae30d3b4 100644 --- a/src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs +++ b/src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs @@ -206,6 +206,7 @@ namespace Umbraco.Core.Migrations.Upgrade // to 8.15.0... To("{8DDDCD0B-D7D5-4C97-BD6A-6B38CA65752F}"); To("{4695D0C9-0729-4976-985B-048D503665D8}"); + To("{5C424554-A32D-4852-8ED1-A13508187901}"); //FINAL } diff --git a/src/Umbraco.Core/Migrations/Upgrade/V_8_15_0/UpdateCmsPropertyGroupIdSeed.cs b/src/Umbraco.Core/Migrations/Upgrade/V_8_15_0/UpdateCmsPropertyGroupIdSeed.cs new file mode 100644 index 0000000000..908b5cda60 --- /dev/null +++ b/src/Umbraco.Core/Migrations/Upgrade/V_8_15_0/UpdateCmsPropertyGroupIdSeed.cs @@ -0,0 +1,21 @@ +using NPoco; +using Umbraco.Core.Persistence; +using Umbraco.Core.Persistence.Dtos; + +namespace Umbraco.Core.Migrations.Upgrade.V_8_15_0 +{ + public class UpdateCmsPropertyGroupIdSeed : MigrationBase + { + public UpdateCmsPropertyGroupIdSeed(IMigrationContext context) : base(context) + { + } + + public override void Migrate() + { + if (DatabaseType.IsSqlCe()) + { + Database.Execute(Sql("ALTER TABLE [cmsPropertyTypeGroup] ALTER COLUMN [id] IDENTITY (56,1)")); + } + } + } +} diff --git a/src/Umbraco.Core/Umbraco.Core.csproj b/src/Umbraco.Core/Umbraco.Core.csproj index 931cef070f..d37907ae0f 100755 --- a/src/Umbraco.Core/Umbraco.Core.csproj +++ b/src/Umbraco.Core/Umbraco.Core.csproj @@ -141,6 +141,7 @@ + From 4ac29927deb0183e9e27b42db32eaad661e1536d Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Wed, 7 Jul 2021 11:22:50 +0200 Subject: [PATCH 005/123] Bump version to 8.15.0 --- src/SolutionInfo.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SolutionInfo.cs b/src/SolutionInfo.cs index 6ef4d6ce85..201213cffa 100644 --- a/src/SolutionInfo.cs +++ b/src/SolutionInfo.cs @@ -19,4 +19,4 @@ using System.Resources; // these are FYI and changed automatically [assembly: AssemblyFileVersion("8.15.0")] -[assembly: AssemblyInformationalVersion("8.15.0-rc")] +[assembly: AssemblyInformationalVersion("8.15.0")] From 9101cfdc09141fb7e00a82d096900fca3e510d73 Mon Sep 17 00:00:00 2001 From: Mole Date: Wed, 7 Jul 2021 14:23:24 +0200 Subject: [PATCH 006/123] Make GetLocalCropUrl use GetCropUrl on local crops instead of calling itself --- .../Extensions/FriendlyImageCropperTemplateExtensions.cs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Umbraco.Web.Common/Extensions/FriendlyImageCropperTemplateExtensions.cs b/src/Umbraco.Web.Common/Extensions/FriendlyImageCropperTemplateExtensions.cs index e8d580968e..2fe6353429 100644 --- a/src/Umbraco.Web.Common/Extensions/FriendlyImageCropperTemplateExtensions.cs +++ b/src/Umbraco.Web.Common/Extensions/FriendlyImageCropperTemplateExtensions.cs @@ -1,4 +1,5 @@ -using Microsoft.Extensions.DependencyInjection; +using System; +using Microsoft.Extensions.DependencyInjection; using Umbraco.Cms.Core.Media; using Umbraco.Cms.Core.Models; using Umbraco.Cms.Core.Models.PublishedContent; @@ -337,10 +338,14 @@ namespace Umbraco.Extensions ); + [Obsolete("Use GetCrop to merge local and media crops, get automatic cache buster value and have more parameters.")] public static string GetLocalCropUrl( this MediaWithCrops mediaWithCrops, string alias, string cacheBusterValue = null) - => mediaWithCrops.GetLocalCropUrl(alias, cacheBusterValue); + { + return mediaWithCrops.LocalCrops.Src + + mediaWithCrops.LocalCrops.GetCropUrl(alias, ImageUrlGenerator, cacheBusterValue: cacheBusterValue); + } } } From f56c4e529a1f59270004f184d98890cc4e1e1b5d Mon Sep 17 00:00:00 2001 From: Bjarke Berg Date: Wed, 7 Jul 2021 18:50:36 +0200 Subject: [PATCH 007/123] Version number --- src/Directory.Build.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Directory.Build.props b/src/Directory.Build.props index 431bd07fd3..b45e31a4a6 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -2,7 +2,7 @@ 9.0.0 9.0.0 - 9.0.0-rc001.pre001 + 9.0.0-rc001 9.0.0 9.0 en-US From 005b09d58b79835ca3fd75a8dc542795acd940a3 Mon Sep 17 00:00:00 2001 From: Bjarke Berg Date: Thu, 8 Jul 2021 09:39:40 +0200 Subject: [PATCH 008/123] Fixed version numbers in templates --- build/templates/UmbracoPackage/.template.config/template.json | 2 +- build/templates/UmbracoProject/.template.config/template.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build/templates/UmbracoPackage/.template.config/template.json b/build/templates/UmbracoPackage/.template.config/template.json index 0a4c8f2f2b..b2bc86399a 100644 --- a/build/templates/UmbracoPackage/.template.config/template.json +++ b/build/templates/UmbracoPackage/.template.config/template.json @@ -24,7 +24,7 @@ "version": { "type": "parameter", "datatype": "string", - "defaultValue": "9.0.0-beta003", + "defaultValue": "9.0.0-rc001", "description": "The version of Umbraco to load using NuGet", "replaces": "UMBRACO_VERSION_FROM_TEMPLATE" }, diff --git a/build/templates/UmbracoProject/.template.config/template.json b/build/templates/UmbracoProject/.template.config/template.json index 8a8c396dcb..ba75721bc4 100644 --- a/build/templates/UmbracoProject/.template.config/template.json +++ b/build/templates/UmbracoProject/.template.config/template.json @@ -57,7 +57,7 @@ "version": { "type": "parameter", "datatype": "string", - "defaultValue": "9.0.0-beta004", + "defaultValue": "9.0.0-rc001", "description": "The version of Umbraco to load using NuGet", "replaces": "UMBRACO_VERSION_FROM_TEMPLATE" }, From dd8d719f6520a1ec6f170d6848721c54c3e621c7 Mon Sep 17 00:00:00 2001 From: Bjarke Berg Date: Thu, 8 Jul 2021 10:07:11 +0200 Subject: [PATCH 009/123] Fixed potential issue with PublishedSnapshot not available. --- src/Umbraco.Infrastructure/Routing/RedirectTrackingHandler.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Umbraco.Infrastructure/Routing/RedirectTrackingHandler.cs b/src/Umbraco.Infrastructure/Routing/RedirectTrackingHandler.cs index 7b3b3f86ed..6fa7789bcf 100644 --- a/src/Umbraco.Infrastructure/Routing/RedirectTrackingHandler.cs +++ b/src/Umbraco.Infrastructure/Routing/RedirectTrackingHandler.cs @@ -87,8 +87,8 @@ namespace Umbraco.Cms.Core.Routing private void StoreOldRoute(IContent entity, OldRoutesDictionary oldRoutes) { - var contentCache = _publishedSnapshotAccessor.PublishedSnapshot.Content; - var entityContent = contentCache.GetById(entity.Id); + var contentCache = _publishedSnapshotAccessor.PublishedSnapshot?.Content; + var entityContent = contentCache?.GetById(entity.Id); if (entityContent == null) return; From 4f06c1f33a54daf4752a439778626202960db68c Mon Sep 17 00:00:00 2001 From: Bjarke Berg Date: Thu, 8 Jul 2021 11:09:12 +0200 Subject: [PATCH 010/123] Missing translation --- src/Umbraco.Web.UI.NetCore/umbraco/config/lang/en.xml | 2 ++ src/Umbraco.Web.UI.NetCore/umbraco/config/lang/en_us.xml | 1 + 2 files changed, 3 insertions(+) diff --git a/src/Umbraco.Web.UI.NetCore/umbraco/config/lang/en.xml b/src/Umbraco.Web.UI.NetCore/umbraco/config/lang/en.xml index 3fae82eed4..60909ee544 100644 --- a/src/Umbraco.Web.UI.NetCore/umbraco/config/lang/en.xml +++ b/src/Umbraco.Web.UI.NetCore/umbraco/config/lang/en.xml @@ -1278,6 +1278,8 @@ To manage your website, simply open the Umbraco backoffice and start adding cont Package version Package already installed This package cannot be installed, it requires a minimum Umbraco version of + Verified to work on Umbraco Cloud + Paste with full formatting (Not recommended) diff --git a/src/Umbraco.Web.UI.NetCore/umbraco/config/lang/en_us.xml b/src/Umbraco.Web.UI.NetCore/umbraco/config/lang/en_us.xml index 7057587e5e..5aeea645bf 100644 --- a/src/Umbraco.Web.UI.NetCore/umbraco/config/lang/en_us.xml +++ b/src/Umbraco.Web.UI.NetCore/umbraco/config/lang/en_us.xml @@ -1293,6 +1293,7 @@ To manage your website, simply open the Umbraco backoffice and start adding cont Upgrading from version Package already installed This package cannot be installed, it requires a minimum Umbraco version of + Verified to work on Umbraco Cloud Paste with full formatting (Not recommended) From 6b6525cc9891cc82d5be8701b91394bdb8ff5a9d Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Fri, 9 Jul 2021 19:33:42 +1200 Subject: [PATCH 011/123] Add whether the current content is published. Allowing for compression of just non published, non media content. --- .../IPropertyCacheCompression.cs | 10 +++++++-- .../IPropertyCacheCompressionOptions.cs | 9 +++++++- .../NoopPropertyCacheCompressionOptions.cs | 2 +- .../PropertyCacheCompression.cs | 10 ++++----- .../ContentSerializationTests.cs | 8 +++---- .../NuCache/DataSource/DatabaseDataSource.cs | 15 +++++++------ .../DataSource/IContentCacheDataSerializer.cs | 4 ++-- .../JsonContentNestedDataSerializer.cs | 4 ++-- .../MsgPackContentNestedDataSerializer.cs | 21 ++++++++++--------- ...gPackContentNestedDataSerializerFactory.cs | 2 +- .../NuCache/PublishedSnapshotService.cs | 2 +- 11 files changed, 52 insertions(+), 35 deletions(-) diff --git a/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompression.cs b/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompression.cs index 96a559630b..eb89173581 100644 --- a/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompression.cs +++ b/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompression.cs @@ -9,7 +9,13 @@ namespace Umbraco.Core.PropertyEditors /// /// public interface IPropertyCacheCompression - { - bool IsCompressed(IReadOnlyContentBase content, string propertyTypeAlias); + { + /// + /// Whether a property on the content is/should be compressed + /// + /// The content + /// The property to compress or not + /// Whether this content is the published version + bool IsCompressed(IReadOnlyContentBase content, string propertyTypeAlias,bool published); } } diff --git a/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompressionOptions.cs b/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompressionOptions.cs index 2fa0153f9e..71eb782ee2 100644 --- a/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompressionOptions.cs +++ b/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompressionOptions.cs @@ -4,6 +4,13 @@ namespace Umbraco.Core.PropertyEditors { public interface IPropertyCacheCompressionOptions { - bool IsCompressed(IReadOnlyContentBase content, PropertyType propertyType, IDataEditor dataEditor); + /// + /// Whether a property on the content is/should be compressed + /// + /// The content + /// The property to compress or not + /// The datatype of the property to compress or not + /// Whether this content is the published version + bool IsCompressed(IReadOnlyContentBase content, PropertyType propertyType, IDataEditor dataEditor,bool published); } } diff --git a/src/Umbraco.Core/PropertyEditors/NoopPropertyCacheCompressionOptions.cs b/src/Umbraco.Core/PropertyEditors/NoopPropertyCacheCompressionOptions.cs index 1f12d45769..f566172cd9 100644 --- a/src/Umbraco.Core/PropertyEditors/NoopPropertyCacheCompressionOptions.cs +++ b/src/Umbraco.Core/PropertyEditors/NoopPropertyCacheCompressionOptions.cs @@ -7,6 +7,6 @@ namespace Umbraco.Core.PropertyEditors /// internal class NoopPropertyCacheCompressionOptions : IPropertyCacheCompressionOptions { - public bool IsCompressed(IReadOnlyContentBase content, PropertyType propertyType, IDataEditor dataEditor) => false; + public bool IsCompressed(IReadOnlyContentBase content, PropertyType propertyType, IDataEditor dataEditor,bool published) => false; } } diff --git a/src/Umbraco.Core/PropertyEditors/PropertyCacheCompression.cs b/src/Umbraco.Core/PropertyEditors/PropertyCacheCompression.cs index 6be21fca7f..be0553ce65 100644 --- a/src/Umbraco.Core/PropertyEditors/PropertyCacheCompression.cs +++ b/src/Umbraco.Core/PropertyEditors/PropertyCacheCompression.cs @@ -14,13 +14,13 @@ namespace Umbraco.Core.PropertyEditors private readonly IPropertyCacheCompressionOptions _compressionOptions; private readonly IReadOnlyDictionary _contentTypes; private readonly PropertyEditorCollection _propertyEditors; - private readonly ConcurrentDictionary<(int contentTypeId, string propertyAlias), bool> _isCompressedCache; + private readonly ConcurrentDictionary<(int contentTypeId, string propertyAlias,bool published), bool> _isCompressedCache; public PropertyCacheCompression( IPropertyCacheCompressionOptions compressionOptions, IReadOnlyDictionary contentTypes, PropertyEditorCollection propertyEditors, - ConcurrentDictionary<(int, string), bool> compressedStoragePropertyEditorCache) + ConcurrentDictionary<(int, string,bool), bool> compressedStoragePropertyEditorCache) { _compressionOptions = compressionOptions; _contentTypes = contentTypes ?? throw new System.ArgumentNullException(nameof(contentTypes)); @@ -28,9 +28,9 @@ namespace Umbraco.Core.PropertyEditors _isCompressedCache = compressedStoragePropertyEditorCache; } - public bool IsCompressed(IReadOnlyContentBase content, string alias) + public bool IsCompressed(IReadOnlyContentBase content, string alias, bool published) { - var compressedStorage = _isCompressedCache.GetOrAdd((content.ContentTypeId, alias), x => + var compressedStorage = _isCompressedCache.GetOrAdd((content.ContentTypeId, alias, published), x => { if (!_contentTypes.TryGetValue(x.contentTypeId, out var ct)) return false; @@ -40,7 +40,7 @@ namespace Umbraco.Core.PropertyEditors if (!_propertyEditors.TryGet(propertyType.PropertyEditorAlias, out var propertyEditor)) return false; - return _compressionOptions.IsCompressed(content, propertyType, propertyEditor); + return _compressionOptions.IsCompressed(content, propertyType, propertyEditor, published); }); return compressedStorage; diff --git a/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs b/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs index b3543dad1a..9acad53364 100644 --- a/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs +++ b/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs @@ -56,14 +56,14 @@ namespace Umbraco.Tests.PublishedContent var content = Mock.Of(x => x.ContentTypeId == 1); - var json = jsonSerializer.Serialize(content, cacheModel).StringData; - var msgPack = msgPackSerializer.Serialize(content, cacheModel).ByteData; + var json = jsonSerializer.Serialize(content, cacheModel,false).StringData; + var msgPack = msgPackSerializer.Serialize(content, cacheModel, false).ByteData; Console.WriteLine(json); Console.WriteLine(msgPackSerializer.ToJson(msgPack)); - var jsonContent = jsonSerializer.Deserialize(content, json, null); - var msgPackContent = msgPackSerializer.Deserialize(content, null, msgPack); + var jsonContent = jsonSerializer.Deserialize(content, json, null,false); + var msgPackContent = msgPackSerializer.Deserialize(content, null, msgPack,false); CollectionAssert.AreEqual(jsonContent.CultureData.Keys, msgPackContent.CultureData.Keys); diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs index c112cc6efa..5ec51bffb8 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs @@ -393,12 +393,13 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } else { - var deserializedContent = serializer.Deserialize(dto, dto.EditData, dto.EditDataRaw); + bool published = false; + var deserializedContent = serializer.Deserialize(dto, dto.EditData, dto.EditDataRaw, published); d = new ContentData { Name = dto.EditName, - Published = false, + Published = published, TemplateId = dto.EditTemplateId, VersionId = dto.VersionId, VersionDate = dto.EditVersionDate, @@ -420,13 +421,14 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } else { - var deserializedContent = serializer.Deserialize(dto, dto.PubData, dto.PubDataRaw); + bool published = true; + var deserializedContent = serializer.Deserialize(dto, dto.PubData, dto.PubDataRaw, published); p = new ContentData { Name = dto.PubName, UrlSegment = deserializedContent.UrlSegment, - Published = true, + Published = published, TemplateId = dto.PubTemplateId, VersionId = dto.VersionId, VersionDate = dto.PubVersionDate, @@ -456,12 +458,13 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource if (dto.EditData == null && dto.EditDataRaw == null) throw new InvalidOperationException("No data for media " + dto.Id); - var deserializedMedia = serializer.Deserialize(dto, dto.EditData, dto.EditDataRaw); + bool published = true; + var deserializedMedia = serializer.Deserialize(dto, dto.EditData, dto.EditDataRaw, published); var p = new ContentData { Name = dto.EditName, - Published = true, + Published = published, TemplateId = -1, VersionId = dto.VersionId, VersionDate = dto.EditVersionDate, diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializer.cs index d1a83d8452..6c37046e8b 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializer.cs @@ -14,12 +14,12 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource /// /// Deserialize the data into a /// - ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData); + ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData, bool published); /// /// Serializes the /// - ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model); + ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model,bool published); } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs index 21cd0bf763..0cc10f5f98 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs @@ -24,7 +24,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource DateFormatString = "o" }; private readonly JsonNameTable _propertyNameTable = new DefaultJsonNameTable(); - public ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData) + public ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData,bool published) { if (stringData == null && byteData != null) throw new NotSupportedException($"{typeof(JsonContentNestedDataSerializer)} does not support byte[] serialization"); @@ -39,7 +39,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } } - public ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model) + public ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model,bool published) { // note that numeric values (which are Int32) are serialized without their // type (eg "value":1234) and JsonConvert by default deserializes them as Int64 diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs index 6ae872ef69..f69232aad3 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs @@ -39,7 +39,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource _options = defaultOptions .WithResolver(resolver) - .WithCompression(MessagePackCompression.Lz4BlockArray); + .WithCompression(MessagePackCompression.Lz4BlockArray) + .WithSecurity(MessagePackSecurity.UntrustedData); } public string ToJson(byte[] bin) @@ -48,12 +49,12 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource return json; } - public ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData) + public ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData,bool published) { if (byteData != null) { var cacheModel = MessagePackSerializer.Deserialize(byteData, _options); - Expand(content, cacheModel); + Expand(content, cacheModel, published); return cacheModel; } else if (stringData != null) @@ -61,7 +62,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource // NOTE: We don't really support strings but it's possible if manually used (i.e. tests) var bin = Convert.FromBase64String(stringData); var cacheModel = MessagePackSerializer.Deserialize(bin, _options); - Expand(content, cacheModel); + Expand(content, cacheModel,published); return cacheModel; } else @@ -70,9 +71,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } } - public ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model) + public ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model,bool published) { - Compress(content, model); + Compress(content, model, published); var bytes = MessagePackSerializer.Serialize(model, _options); return new ContentCacheDataSerializationResult(null, bytes); } @@ -88,11 +89,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource /// read/decompressed as a string to be displayed on the front-end. This allows for potentially a significant /// memory savings but could also affect performance of first rendering pages while decompression occurs. /// - private void Compress(IReadOnlyContentBase content, ContentCacheDataModel model) + private void Compress(IReadOnlyContentBase content, ContentCacheDataModel model,bool published) { foreach(var propertyAliasToData in model.PropertyData) { - if (_propertyOptions.IsCompressed(content, propertyAliasToData.Key)) + if (_propertyOptions.IsCompressed(content, propertyAliasToData.Key,published)) { foreach(var property in propertyAliasToData.Value.Where(x => x.Value != null && x.Value is string)) { @@ -106,11 +107,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource /// Used during deserialization to map the property data as lazy or expand the value /// /// - private void Expand(IReadOnlyContentBase content, ContentCacheDataModel nestedData) + private void Expand(IReadOnlyContentBase content, ContentCacheDataModel nestedData,bool published) { foreach (var propertyAliasToData in nestedData.PropertyData) { - if (_propertyOptions.IsCompressed(content, propertyAliasToData.Key)) + if (_propertyOptions.IsCompressed(content, propertyAliasToData.Key,published)) { foreach (var property in propertyAliasToData.Value.Where(x => x.Value != null)) { diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializerFactory.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializerFactory.cs index fcc3fa2bb8..29378caf0f 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializerFactory.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializerFactory.cs @@ -13,7 +13,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource private readonly IMemberTypeService _memberTypeService; private readonly PropertyEditorCollection _propertyEditors; private readonly IPropertyCacheCompressionOptions _compressionOptions; - private readonly ConcurrentDictionary<(int, string), bool> _isCompressedCache = new ConcurrentDictionary<(int, string), bool>(); + private readonly ConcurrentDictionary<(int, string,bool), bool> _isCompressedCache = new ConcurrentDictionary<(int, string,bool), bool>(); public MsgPackContentNestedDataSerializerFactory( IContentTypeService contentTypeService, diff --git a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs index f9c25b7b35..7ca425aad9 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs @@ -1491,7 +1491,7 @@ namespace Umbraco.Web.PublishedCache.NuCache UrlSegment = content.GetUrlSegment(_urlSegmentProviders) }; - var serialized = serializer.Serialize(ReadOnlyContentBaseAdapter.Create(content), contentCacheData); + var serialized = serializer.Serialize(ReadOnlyContentBaseAdapter.Create(content), contentCacheData,published); var dto = new ContentNuDto { From 044c900c2ba52428b7acecb8d15fc0769c5b9eaa Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Fri, 9 Jul 2021 20:14:35 +1200 Subject: [PATCH 012/123] Make IPropertyCacheCompressionOptions useful out of the box. key = "Umbraco.Web.PublishedCache.NuCache.CompressUnPublishedContent" value = "true" will compress all ntext properties on unpublished content --- ...dContentPropertyCacheCompressionOptions.cs | 29 +++++++++++++++++++ src/Umbraco.Core/Umbraco.Core.csproj | 1 + .../PublishedCache/NuCache/NuCacheComposer.cs | 12 ++++++-- .../NuCache/NuCacheSerializerComponent.cs | 1 + 4 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 src/Umbraco.Core/PropertyEditors/UnPublishedContentPropertyCacheCompressionOptions.cs diff --git a/src/Umbraco.Core/PropertyEditors/UnPublishedContentPropertyCacheCompressionOptions.cs b/src/Umbraco.Core/PropertyEditors/UnPublishedContentPropertyCacheCompressionOptions.cs new file mode 100644 index 0000000000..51433e476e --- /dev/null +++ b/src/Umbraco.Core/PropertyEditors/UnPublishedContentPropertyCacheCompressionOptions.cs @@ -0,0 +1,29 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Umbraco.Core.Models; + +namespace Umbraco.Core.PropertyEditors +{ + /// + /// Compress large, non published text properties + /// + internal class UnPublishedContentPropertyCacheCompressionOptions : IPropertyCacheCompressionOptions + { + public bool IsCompressed(IReadOnlyContentBase content, PropertyType propertyType, IDataEditor dataEditor, bool published) + { + if (published) + { + return false; + } + if (propertyType.SupportsPublishing && propertyType.ValueStorageType == ValueStorageType.Ntext) + { + //Only compress non published content that supports publishing and the property is text + return true; + } + return false; + } + } +} diff --git a/src/Umbraco.Core/Umbraco.Core.csproj b/src/Umbraco.Core/Umbraco.Core.csproj index 931cef070f..f6a523441e 100755 --- a/src/Umbraco.Core/Umbraco.Core.csproj +++ b/src/Umbraco.Core/Umbraco.Core.csproj @@ -171,6 +171,7 @@ + diff --git a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs index c6b214102b..4b3d7b20fa 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs @@ -26,8 +26,16 @@ namespace Umbraco.Web.PublishedCache.NuCache { composition.RegisterUnique(); } - - composition.RegisterUnique(); + var unPublishedContentCompression = ConfigurationManager.AppSettings[NuCacheSerializerComponent.Nucache_UnPublishedContentCompression_Key]; + if ("MsgPack" == serializer && "true" == unPublishedContentCompression) + { + composition.RegisterUnique(); + } + else + { + composition.RegisterUnique(); + } + composition.RegisterUnique(factory => new ContentDataSerializer(new DictionaryOfPropertyDataSerializer())); diff --git a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComponent.cs b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComponent.cs index a1d3ed2b12..c2d24b16ac 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComponent.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComponent.cs @@ -16,6 +16,7 @@ namespace Umbraco.Web.PublishedCache.NuCache public class NuCacheSerializerComponent : IComponent { internal const string Nucache_Serializer_Key = "Umbraco.Web.PublishedCache.NuCache.Serializer"; + internal const string Nucache_UnPublishedContentCompression_Key = "Umbraco.Web.PublishedCache.NuCache.CompressUnPublishedContent"; private const string JSON_SERIALIZER_VALUE = "JSON"; private readonly Lazy _service; private readonly IKeyValueService _keyValueService; From b119201480da1f99bf463776f71d2e73901f2aef Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Fri, 9 Jul 2021 20:55:03 +1200 Subject: [PATCH 013/123] fix serializer swap message --- .../PublishedCache/NuCache/NuCacheSerializerComponent.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComponent.cs b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComponent.cs index c2d24b16ac..41afb2b781 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComponent.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComponent.cs @@ -55,7 +55,7 @@ namespace Umbraco.Web.PublishedCache.NuCache { _profilingLogger.Warn($"Database NuCache was serialized using {currentSerializer}. Currently configured NuCache serializer {serializer}. Rebuilding Nucache"); - using (_profilingLogger.TraceDuration($"Rebuilding NuCache database with {currentSerializer} serializer")) + using (_profilingLogger.TraceDuration($"Rebuilding NuCache database with {serializer} serializer")) { _service.Value.Rebuild(); _keyValueService.SetValue(Nucache_Serializer_Key, serializer); From 6a8ed8012cc282e87b8d55c23f518f9991c0daba Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Fri, 9 Jul 2021 21:22:30 +1200 Subject: [PATCH 014/123] simplify --- .../UnPublishedContentPropertyCacheCompressionOptions.cs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/Umbraco.Core/PropertyEditors/UnPublishedContentPropertyCacheCompressionOptions.cs b/src/Umbraco.Core/PropertyEditors/UnPublishedContentPropertyCacheCompressionOptions.cs index 51433e476e..7a5212a985 100644 --- a/src/Umbraco.Core/PropertyEditors/UnPublishedContentPropertyCacheCompressionOptions.cs +++ b/src/Umbraco.Core/PropertyEditors/UnPublishedContentPropertyCacheCompressionOptions.cs @@ -14,11 +14,7 @@ namespace Umbraco.Core.PropertyEditors { public bool IsCompressed(IReadOnlyContentBase content, PropertyType propertyType, IDataEditor dataEditor, bool published) { - if (published) - { - return false; - } - if (propertyType.SupportsPublishing && propertyType.ValueStorageType == ValueStorageType.Ntext) + if (!published && propertyType.SupportsPublishing && propertyType.ValueStorageType == ValueStorageType.Ntext) { //Only compress non published content that supports publishing and the property is text return true; From 5e8b4572a0a322b8cfbe27fea3038367c76415fd Mon Sep 17 00:00:00 2001 From: Shannon Date: Fri, 9 Jul 2021 09:40:31 -0600 Subject: [PATCH 015/123] Ensures that the content nu data column is updated to support null during migration. --- .../Migrations/Upgrade/V_8_15_0/AddCmsContentNuByteColumn.cs | 3 +++ .../PublishedCache/NuCache/NuCacheSerializerComponent.cs | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Umbraco.Core/Migrations/Upgrade/V_8_15_0/AddCmsContentNuByteColumn.cs b/src/Umbraco.Core/Migrations/Upgrade/V_8_15_0/AddCmsContentNuByteColumn.cs index 3eab1a812e..9eeaa67189 100644 --- a/src/Umbraco.Core/Migrations/Upgrade/V_8_15_0/AddCmsContentNuByteColumn.cs +++ b/src/Umbraco.Core/Migrations/Upgrade/V_8_15_0/AddCmsContentNuByteColumn.cs @@ -16,6 +16,9 @@ namespace Umbraco.Core.Migrations.Upgrade.V_8_15_0 var columns = SqlSyntax.GetColumnsInSchema(Context.Database).ToList(); AddColumnIfNotExists(columns, "dataRaw"); + + // allow null + AlterColumn(Constants.DatabaseSchema.Tables.NodeData, "data"); } } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComponent.cs b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComponent.cs index a1d3ed2b12..fcbb5a3715 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComponent.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComponent.cs @@ -54,7 +54,7 @@ namespace Umbraco.Web.PublishedCache.NuCache { _profilingLogger.Warn($"Database NuCache was serialized using {currentSerializer}. Currently configured NuCache serializer {serializer}. Rebuilding Nucache"); - using (_profilingLogger.TraceDuration($"Rebuilding NuCache database with {currentSerializer} serializer")) + using (_profilingLogger.TraceDuration($"Rebuilding NuCache database with {serializer} serializer")) { _service.Value.Rebuild(); _keyValueService.SetValue(Nucache_Serializer_Key, serializer); From 7dc03256edf1bbe15b5b44c5d44a033369ce0e07 Mon Sep 17 00:00:00 2001 From: Shannon Date: Fri, 9 Jul 2021 12:05:19 -0600 Subject: [PATCH 016/123] Fixes SQLCE migration for nucache table. --- .../Create/Table/CreateTableOfDtoBuilder.cs | 2 + .../V_8_15_0/AddCmsContentNuByteColumn.cs | 47 +++++++++++++++++-- .../SqlSyntax/SqlSyntaxProviderBase.cs | 15 +++--- 3 files changed, 54 insertions(+), 10 deletions(-) diff --git a/src/Umbraco.Core/Migrations/Expressions/Create/Table/CreateTableOfDtoBuilder.cs b/src/Umbraco.Core/Migrations/Expressions/Create/Table/CreateTableOfDtoBuilder.cs index 4b73e9435d..a5e2fca1f7 100644 --- a/src/Umbraco.Core/Migrations/Expressions/Create/Table/CreateTableOfDtoBuilder.cs +++ b/src/Umbraco.Core/Migrations/Expressions/Create/Table/CreateTableOfDtoBuilder.cs @@ -9,6 +9,8 @@ namespace Umbraco.Core.Migrations.Expressions.Create.Table public class CreateTableOfDtoBuilder : IExecutableBuilder { private readonly IMigrationContext _context; + + // TODO: This doesn't do anything. private readonly DatabaseType[] _supportedDatabaseTypes; public CreateTableOfDtoBuilder(IMigrationContext context, params DatabaseType[] supportedDatabaseTypes) diff --git a/src/Umbraco.Core/Migrations/Upgrade/V_8_15_0/AddCmsContentNuByteColumn.cs b/src/Umbraco.Core/Migrations/Upgrade/V_8_15_0/AddCmsContentNuByteColumn.cs index 9eeaa67189..5217fc9870 100644 --- a/src/Umbraco.Core/Migrations/Upgrade/V_8_15_0/AddCmsContentNuByteColumn.cs +++ b/src/Umbraco.Core/Migrations/Upgrade/V_8_15_0/AddCmsContentNuByteColumn.cs @@ -1,5 +1,10 @@ -using System.Linq; +using NPoco; +using System.Data; +using System.Linq; +using Umbraco.Core.Persistence; +using Umbraco.Core.Persistence.DatabaseAnnotations; using Umbraco.Core.Persistence.Dtos; +using Umbraco.Core.PropertyEditors; namespace Umbraco.Core.Migrations.Upgrade.V_8_15_0 { @@ -13,12 +18,46 @@ namespace Umbraco.Core.Migrations.Upgrade.V_8_15_0 public override void Migrate() { + // allow null for the `data` field + if (DatabaseType.IsSqlCe()) + { + // SQLCE does not support altering NTEXT, so we have to jump through some hoops to do it + // All column ordering must remain the same as what is defined in the DTO so we need to create a temp table, + // drop orig and then re-create/copy. + Create.Table(withoutKeysAndIndexes: true).Do(); + Execute.Sql($"INSERT INTO [{TempTableName}] SELECT nodeId, published, data, rv FROM [{Constants.DatabaseSchema.Tables.NodeData}]").Do(); + Delete.Table(Constants.DatabaseSchema.Tables.NodeData).Do(); + Create.Table().Do(); + Execute.Sql($"INSERT INTO [{Constants.DatabaseSchema.Tables.NodeData}] SELECT nodeId, published, data, rv, NULL FROM [{TempTableName}]").Do(); + } + else + { + AlterColumn(Constants.DatabaseSchema.Tables.NodeData, "data"); + } + var columns = SqlSyntax.GetColumnsInSchema(Context.Database).ToList(); - AddColumnIfNotExists(columns, "dataRaw"); + } - // allow null - AlterColumn(Constants.DatabaseSchema.Tables.NodeData, "data"); + private const string TempTableName = Constants.DatabaseSchema.TableNamePrefix + "cms" + "ContentNuTEMP"; + + [TableName(TempTableName)] + [ExplicitColumns] + private class ContentNuDtoTemp + { + [Column("nodeId")] + public int NodeId { get; set; } + + [Column("published")] + public bool Published { get; set; } + + [Column("data")] + [SpecialDbType(SpecialDbTypes.NTEXT)] + [NullSetting(NullSetting = NullSettings.Null)] + public string Data { get; set; } + + [Column("rv")] + public long Rv { get; set; } } } } diff --git a/src/Umbraco.Core/Persistence/SqlSyntax/SqlSyntaxProviderBase.cs b/src/Umbraco.Core/Persistence/SqlSyntax/SqlSyntaxProviderBase.cs index 4d6b2eeea1..0d2d7aeb21 100644 --- a/src/Umbraco.Core/Persistence/SqlSyntax/SqlSyntaxProviderBase.cs +++ b/src/Umbraco.Core/Persistence/SqlSyntax/SqlSyntaxProviderBase.cs @@ -352,7 +352,7 @@ namespace Umbraco.Core.Persistence.SqlSyntax sql.Append(" "); sql.Append(FormatIdentity(column)); - var isNullable = column.IsNullable; + //var isNullable = column.IsNullable; //var constraint = FormatConstraint(column)?.TrimStart("CONSTRAINT "); //var hasConstraint = !string.IsNullOrWhiteSpace(constraint); @@ -360,11 +360,14 @@ namespace Umbraco.Core.Persistence.SqlSyntax //var defaultValue = FormatDefaultValue(column); //var hasDefaultValue = !string.IsNullOrWhiteSpace(defaultValue); - if (isNullable /*&& !hasConstraint && !hasDefaultValue*/) - { - sqls = Enumerable.Empty(); - return sql.ToString(); - } + // TODO: This used to exit if nullable but that means this would never work + // to return SQL if the column was nullable?!? I don't get it. This was here + // 4 years ago, I've removed it so that this works for nullable columns. + //if (isNullable /*&& !hasConstraint && !hasDefaultValue*/) + //{ + // sqls = Enumerable.Empty(); + // return sql.ToString(); + //} var msql = new List(); sqls = msql; From 93f5dd76317b437f14f37db49f1ac39ace300c39 Mon Sep 17 00:00:00 2001 From: Shannon Date: Fri, 9 Jul 2021 13:35:43 -0600 Subject: [PATCH 017/123] Changes bulk copy timeout to be infinite. Changes bulk copy to not allocate an arbitrary array. --- .../NPocoDatabaseExtensions-Bulk.cs | 35 ++++++++++++++----- .../Persistence/PocoDataDataReader.cs | 5 +-- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/src/Umbraco.Core/Persistence/NPocoDatabaseExtensions-Bulk.cs b/src/Umbraco.Core/Persistence/NPocoDatabaseExtensions-Bulk.cs index bff682d095..77cc0d6601 100644 --- a/src/Umbraco.Core/Persistence/NPocoDatabaseExtensions-Bulk.cs +++ b/src/Umbraco.Core/Persistence/NPocoDatabaseExtensions-Bulk.cs @@ -62,26 +62,33 @@ namespace Umbraco.Core.Persistence /// The number of records that were inserted. public static int BulkInsertRecords(this IUmbracoDatabase database, IEnumerable records, bool useNativeBulkInsert = true) { - var recordsA = records.ToArray(); - if (recordsA.Length == 0) return 0; + if (!records.Any()) return 0; var pocoData = database.PocoDataFactory.ForType(typeof(T)); if (pocoData == null) throw new InvalidOperationException("Could not find PocoData for " + typeof(T)); if (database.DatabaseType.IsSqlCe()) { - if (useNativeBulkInsert) return BulkInsertRecordsSqlCe(database, pocoData, recordsA); + if (useNativeBulkInsert) + { + return BulkInsertRecordsSqlCe(database, pocoData, records); + } + // else, no other choice - foreach (var record in recordsA) + var count = 0; + foreach (var record in records) + { database.Insert(record); - return recordsA.Length; + count++; + } + return count; } if (database.DatabaseType.IsSqlServer()) { return useNativeBulkInsert && database.DatabaseType.IsSqlServer2008OrLater() - ? BulkInsertRecordsSqlServer(database, pocoData, recordsA) - : BulkInsertRecordsWithCommands(database, recordsA); + ? BulkInsertRecordsSqlServer(database, pocoData, records) + : BulkInsertRecordsWithCommands(database, records.ToArray()); } throw new NotSupportedException(); } @@ -96,7 +103,9 @@ namespace Umbraco.Core.Persistence private static int BulkInsertRecordsWithCommands(IUmbracoDatabase database, T[] records) { foreach (var command in database.GenerateBulkInsertCommands(records)) + { command.ExecuteNonQuery(); + } return records.Length; // what else? } @@ -241,6 +250,10 @@ namespace Umbraco.Core.Persistence /// The number of records that were inserted. internal static int BulkInsertRecordsSqlServer(IUmbracoDatabase database, PocoData pocoData, IEnumerable records) { + // TODO: The main reason this exists is because the NPoco InsertBulk method doesn't return the number of items. + // It is worth investigating the performance of this vs NPoco's because we use a custom BulkDataReader + // which in theory should be more efficient than NPocos way of building up an in-memory DataTable. + // create command against the original database.Connection using (var command = database.CreateCommand(database.Connection, CommandType.Text, string.Empty)) { @@ -252,7 +265,13 @@ namespace Umbraco.Core.Persistence var syntax = database.SqlContext.SqlSyntax as SqlServerSyntaxProvider; if (syntax == null) throw new NotSupportedException("SqlSyntax must be SqlServerSyntaxProvider."); - using (var copy = new SqlBulkCopy(tConnection, SqlBulkCopyOptions.Default, tTransaction) { BulkCopyTimeout = 10000, DestinationTableName = tableName }) + using (var copy = new SqlBulkCopy(tConnection, SqlBulkCopyOptions.Default, tTransaction) + { + BulkCopyTimeout = 0, // 0 = no bulk copy timeout. If a timeout occurs it will be an connection/command timeout. + DestinationTableName = tableName, + // be consistent with NPoco: https://github.com/schotime/NPoco/blob/5117a55fde57547e928246c044fd40bd00b2d7d1/src/NPoco.SqlServer/SqlBulkCopyHelper.cs#L50 + BatchSize = 4096 + }) using (var bulkReader = new PocoDataDataReader(records, pocoData, syntax)) { //we need to add column mappings here because otherwise columns will be matched by their order and if the order of them are different in the DB compared diff --git a/src/Umbraco.Core/Persistence/PocoDataDataReader.cs b/src/Umbraco.Core/Persistence/PocoDataDataReader.cs index 460a4d3d90..397d903cc2 100644 --- a/src/Umbraco.Core/Persistence/PocoDataDataReader.cs +++ b/src/Umbraco.Core/Persistence/PocoDataDataReader.cs @@ -40,9 +40,10 @@ namespace Umbraco.Core.Persistence _tableDefinition = DefinitionFactory.GetTableDefinition(pd.Type, sqlSyntaxProvider); if (_tableDefinition == null) throw new InvalidOperationException("No table definition found for type " + pd.Type); - // only real columns, exclude result columns + // only real columns, exclude result/computed columns + // Like NPoco does: https://github.com/schotime/NPoco/blob/5117a55fde57547e928246c044fd40bd00b2d7d1/src/NPoco.SqlServer/SqlBulkCopyHelper.cs#L59 _readerColumns = pd.Columns - .Where(x => x.Value.ResultColumn == false) + .Where(x => x.Value.ResultColumn == false && x.Value.ComputedColumn == false) .Select(x => x.Value) .ToArray(); From a37f6c604f399b1c6f387adcc97fa71d0dc9abea Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Sat, 10 Jul 2021 14:25:26 +0200 Subject: [PATCH 018/123] Inconsistent formatting --- .../IPropertyCacheCompression.cs | 4 ++-- .../IPropertyCacheCompressionOptions.cs | 2 +- .../NoopPropertyCacheCompressionOptions.cs | 2 +- ...edContentPropertyCacheCompressionOptions.cs | 7 +------ .../ContentSerializationTests.cs | 6 +++--- .../DataSource/IContentCacheDataSerializer.cs | 4 ++-- .../JsonContentNestedDataSerializer.cs | 4 ++-- .../MsgPackContentNestedDataSerializer.cs | 18 +++++++++++------- ...sgPackContentNestedDataSerializerFactory.cs | 2 +- .../PublishedCache/NuCache/NuCacheComposer.cs | 8 ++++---- .../NuCache/PublishedSnapshotService.cs | 8 ++++---- 11 files changed, 32 insertions(+), 33 deletions(-) diff --git a/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompression.cs b/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompression.cs index eb89173581..69d10a2276 100644 --- a/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompression.cs +++ b/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompression.cs @@ -6,7 +6,7 @@ namespace Umbraco.Core.PropertyEditors /// Determines if a property type's value should be compressed in memory /// /// - /// + /// /// public interface IPropertyCacheCompression { @@ -16,6 +16,6 @@ namespace Umbraco.Core.PropertyEditors /// The content /// The property to compress or not /// Whether this content is the published version - bool IsCompressed(IReadOnlyContentBase content, string propertyTypeAlias,bool published); + bool IsCompressed(IReadOnlyContentBase content, string propertyTypeAlias, bool published); } } diff --git a/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompressionOptions.cs b/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompressionOptions.cs index 71eb782ee2..e4603d55e3 100644 --- a/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompressionOptions.cs +++ b/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompressionOptions.cs @@ -11,6 +11,6 @@ namespace Umbraco.Core.PropertyEditors /// The property to compress or not /// The datatype of the property to compress or not /// Whether this content is the published version - bool IsCompressed(IReadOnlyContentBase content, PropertyType propertyType, IDataEditor dataEditor,bool published); + bool IsCompressed(IReadOnlyContentBase content, PropertyType propertyType, IDataEditor dataEditor, bool published); } } diff --git a/src/Umbraco.Core/PropertyEditors/NoopPropertyCacheCompressionOptions.cs b/src/Umbraco.Core/PropertyEditors/NoopPropertyCacheCompressionOptions.cs index f566172cd9..ea3b8d8a2e 100644 --- a/src/Umbraco.Core/PropertyEditors/NoopPropertyCacheCompressionOptions.cs +++ b/src/Umbraco.Core/PropertyEditors/NoopPropertyCacheCompressionOptions.cs @@ -7,6 +7,6 @@ namespace Umbraco.Core.PropertyEditors /// internal class NoopPropertyCacheCompressionOptions : IPropertyCacheCompressionOptions { - public bool IsCompressed(IReadOnlyContentBase content, PropertyType propertyType, IDataEditor dataEditor,bool published) => false; + public bool IsCompressed(IReadOnlyContentBase content, PropertyType propertyType, IDataEditor dataEditor, bool published) => false; } } diff --git a/src/Umbraco.Core/PropertyEditors/UnPublishedContentPropertyCacheCompressionOptions.cs b/src/Umbraco.Core/PropertyEditors/UnPublishedContentPropertyCacheCompressionOptions.cs index 7a5212a985..ece25479cc 100644 --- a/src/Umbraco.Core/PropertyEditors/UnPublishedContentPropertyCacheCompressionOptions.cs +++ b/src/Umbraco.Core/PropertyEditors/UnPublishedContentPropertyCacheCompressionOptions.cs @@ -1,9 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Umbraco.Core.Models; +using Umbraco.Core.Models; namespace Umbraco.Core.PropertyEditors { diff --git a/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs b/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs index 9acad53364..9a44cf35f9 100644 --- a/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs +++ b/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs @@ -56,14 +56,14 @@ namespace Umbraco.Tests.PublishedContent var content = Mock.Of(x => x.ContentTypeId == 1); - var json = jsonSerializer.Serialize(content, cacheModel,false).StringData; + var json = jsonSerializer.Serialize(content, cacheModel, false).StringData; var msgPack = msgPackSerializer.Serialize(content, cacheModel, false).ByteData; Console.WriteLine(json); Console.WriteLine(msgPackSerializer.ToJson(msgPack)); - var jsonContent = jsonSerializer.Deserialize(content, json, null,false); - var msgPackContent = msgPackSerializer.Deserialize(content, null, msgPack,false); + var jsonContent = jsonSerializer.Deserialize(content, json, null, false); + var msgPackContent = msgPackSerializer.Deserialize(content, null, msgPack, false); CollectionAssert.AreEqual(jsonContent.CultureData.Keys, msgPackContent.CultureData.Keys); diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializer.cs index 6c37046e8b..4bdf7e9665 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializer.cs @@ -17,9 +17,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData, bool published); /// - /// Serializes the + /// Serializes the /// - ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model,bool published); + ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model, bool published); } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs index 0cc10f5f98..358561cabd 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs @@ -24,7 +24,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource DateFormatString = "o" }; private readonly JsonNameTable _propertyNameTable = new DefaultJsonNameTable(); - public ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData,bool published) + public ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData, bool published) { if (stringData == null && byteData != null) throw new NotSupportedException($"{typeof(JsonContentNestedDataSerializer)} does not support byte[] serialization"); @@ -39,7 +39,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } } - public ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model,bool published) + public ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model, bool published) { // note that numeric values (which are Int32) are serialized without their // type (eg "value":1234) and JsonConvert by default deserializes them as Int64 diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs index f69232aad3..f1400382e6 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs @@ -40,7 +40,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource _options = defaultOptions .WithResolver(resolver) .WithCompression(MessagePackCompression.Lz4BlockArray) - .WithSecurity(MessagePackSecurity.UntrustedData); + .WithSecurity(MessagePackSecurity.UntrustedData); } public string ToJson(byte[] bin) @@ -49,7 +49,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource return json; } - public ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData,bool published) + public ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData, bool published) { if (byteData != null) { @@ -62,7 +62,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource // NOTE: We don't really support strings but it's possible if manually used (i.e. tests) var bin = Convert.FromBase64String(stringData); var cacheModel = MessagePackSerializer.Deserialize(bin, _options); - Expand(content, cacheModel,published); + Expand(content, cacheModel, published); return cacheModel; } else @@ -71,7 +71,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } } - public ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model,bool published) + public ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model, bool published) { Compress(content, model, published); var bytes = MessagePackSerializer.Serialize(model, _options); @@ -81,7 +81,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource /// /// Used during serialization to compress properties /// + /// /// + /// /// /// This will essentially 'double compress' property data. The MsgPack data as a whole will already be compressed /// but this will go a step further and double compress property data so that it is stored in the nucache file @@ -89,11 +91,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource /// read/decompressed as a string to be displayed on the front-end. This allows for potentially a significant /// memory savings but could also affect performance of first rendering pages while decompression occurs. /// - private void Compress(IReadOnlyContentBase content, ContentCacheDataModel model,bool published) + private void Compress(IReadOnlyContentBase content, ContentCacheDataModel model, bool published) { foreach(var propertyAliasToData in model.PropertyData) { - if (_propertyOptions.IsCompressed(content, propertyAliasToData.Key,published)) + if (_propertyOptions.IsCompressed(content, propertyAliasToData.Key, published)) { foreach(var property in propertyAliasToData.Value.Where(x => x.Value != null && x.Value is string)) { @@ -106,8 +108,10 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource /// /// Used during deserialization to map the property data as lazy or expand the value /// + /// /// - private void Expand(IReadOnlyContentBase content, ContentCacheDataModel nestedData,bool published) + /// + private void Expand(IReadOnlyContentBase content, ContentCacheDataModel nestedData, bool published) { foreach (var propertyAliasToData in nestedData.PropertyData) { diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializerFactory.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializerFactory.cs index 29378caf0f..5245df8353 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializerFactory.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializerFactory.cs @@ -13,7 +13,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource private readonly IMemberTypeService _memberTypeService; private readonly PropertyEditorCollection _propertyEditors; private readonly IPropertyCacheCompressionOptions _compressionOptions; - private readonly ConcurrentDictionary<(int, string,bool), bool> _isCompressedCache = new ConcurrentDictionary<(int, string,bool), bool>(); + private readonly ConcurrentDictionary<(int, string,bool), bool> _isCompressedCache = new ConcurrentDictionary<(int, string, bool), bool>(); public MsgPackContentNestedDataSerializerFactory( IContentTypeService contentTypeService, diff --git a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs index 4b3d7b20fa..dd3907e254 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs @@ -24,10 +24,10 @@ namespace Umbraco.Web.PublishedCache.NuCache } else { - composition.RegisterUnique(); + composition.RegisterUnique(); } - var unPublishedContentCompression = ConfigurationManager.AppSettings[NuCacheSerializerComponent.Nucache_UnPublishedContentCompression_Key]; - if ("MsgPack" == serializer && "true" == unPublishedContentCompression) + var unPublishedContentCompression = ConfigurationManager.AppSettings[NuCacheSerializerComponent.Nucache_UnPublishedContentCompression_Key]; + if (serializer == "MsgPack" && unPublishedContentCompression == "true") { composition.RegisterUnique(); } @@ -35,7 +35,7 @@ namespace Umbraco.Web.PublishedCache.NuCache { composition.RegisterUnique(); } - + composition.RegisterUnique(factory => new ContentDataSerializer(new DictionaryOfPropertyDataSerializer())); diff --git a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs index 7ca425aad9..f3373dab6c 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs @@ -89,7 +89,7 @@ namespace Umbraco.Web.PublishedCache.NuCache IPublishedModelFactory publishedModelFactory, UrlSegmentProviderCollection urlSegmentProviders, ISyncBootStateAccessor syncBootStateAccessor, - IContentCacheDataSerializerFactory contentCacheDataSerializerFactory, + IContentCacheDataSerializerFactory contentCacheDataSerializerFactory, ContentDataSerializer contentDataSerializer = null) : base(publishedSnapshotAccessor, variationContextAccessor) { @@ -262,7 +262,7 @@ namespace Umbraco.Web.PublishedCache.NuCache if (!okMedia) _logger.Warn("Loading media from local db raised warnings, will reload from database."); } - + if (!okContent) LockAndLoadContent(scope => LoadContentFromDatabaseLocked(scope, true)); @@ -1168,7 +1168,7 @@ namespace Umbraco.Web.PublishedCache.NuCache if (Volatile.Read(ref _isReady) == false) { throw new InvalidOperationException("The published snapshot service has not properly initialized."); - } + } var preview = previewToken.IsNullOrWhiteSpace() == false; return new PublishedSnapshot(this, preview); @@ -1491,7 +1491,7 @@ namespace Umbraco.Web.PublishedCache.NuCache UrlSegment = content.GetUrlSegment(_urlSegmentProviders) }; - var serialized = serializer.Serialize(ReadOnlyContentBaseAdapter.Create(content), contentCacheData,published); + var serialized = serializer.Serialize(ReadOnlyContentBaseAdapter.Create(content), contentCacheData, published); var dto = new ContentNuDto { From f69ca3392a723083b60fbc3e64428ca557e4c743 Mon Sep 17 00:00:00 2001 From: Marc Goodson Date: Sun, 11 Jul 2021 12:35:08 +0100 Subject: [PATCH 019/123] Typo in DataTypeList Media Picke (legacy) should be Media Picker (legacy) --- .../Migrations/Install/DatabaseDataCreator.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Umbraco.Infrastructure/Migrations/Install/DatabaseDataCreator.cs b/src/Umbraco.Infrastructure/Migrations/Install/DatabaseDataCreator.cs index 83adbc6e8a..a7cf92e2a9 100644 --- a/src/Umbraco.Infrastructure/Migrations/Install/DatabaseDataCreator.cs +++ b/src/Umbraco.Infrastructure/Migrations/Install/DatabaseDataCreator.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Linq; using Microsoft.Extensions.Logging; using NPoco; @@ -151,7 +151,7 @@ namespace Umbraco.Cms.Infrastructure.Migrations.Install //New UDI pickers with newer Ids _database.Insert(Cms.Core.Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = 1046, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = "-1,1046", SortOrder = 2, UniqueId = new Guid("FD1E0DA5-5606-4862-B679-5D0CF3A52A59"), Text = "Content Picker", NodeObjectType = Cms.Core.Constants.ObjectTypes.DataType, CreateDate = DateTime.Now }); _database.Insert(Cms.Core.Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = 1047, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = "-1,1047", SortOrder = 2, UniqueId = new Guid("1EA2E01F-EBD8-4CE1-8D71-6B1149E63548"), Text = "Member Picker", NodeObjectType = Cms.Core.Constants.ObjectTypes.DataType, CreateDate = DateTime.Now }); - _database.Insert(Cms.Core.Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = 1048, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = "-1,1048", SortOrder = 2, UniqueId = new Guid("135D60E0-64D9-49ED-AB08-893C9BA44AE5"), Text = "Media Picke (legacy)", NodeObjectType = Cms.Core.Constants.ObjectTypes.DataType, CreateDate = DateTime.Now }); + _database.Insert(Cms.Core.Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = 1048, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = "-1,1048", SortOrder = 2, UniqueId = new Guid("135D60E0-64D9-49ED-AB08-893C9BA44AE5"), Text = "Media Picker (legacy)", NodeObjectType = Cms.Core.Constants.ObjectTypes.DataType, CreateDate = DateTime.Now }); _database.Insert(Cms.Core.Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = 1049, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = "-1,1049", SortOrder = 2, UniqueId = new Guid("9DBBCBBB-2327-434A-B355-AF1B84E5010A"), Text = "Multiple Media Picker (legacy)", NodeObjectType = Cms.Core.Constants.ObjectTypes.DataType, CreateDate = DateTime.Now }); _database.Insert(Cms.Core.Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = 1050, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = "-1,1050", SortOrder = 2, UniqueId = new Guid("B4E3535A-1753-47E2-8568-602CF8CFEE6F"), Text = "Multi URL Picker", NodeObjectType = Cms.Core.Constants.ObjectTypes.DataType, CreateDate = DateTime.Now }); From 2a6bdf73300b358ef5be2b1e4b3a480d2e790ef8 Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Mon, 12 Jul 2021 10:48:21 +0200 Subject: [PATCH 020/123] Fixes umbraco/Umbraco-CMS#10402 We added an overload that would conflict for people with existing templates, that was a bit too premature. Disabled for now with a note to look into it for v9 again. --- src/Umbraco.Web/UrlHelperRenderExtensions.cs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/Umbraco.Web/UrlHelperRenderExtensions.cs b/src/Umbraco.Web/UrlHelperRenderExtensions.cs index 2c547c841e..33e657e7cf 100644 --- a/src/Umbraco.Web/UrlHelperRenderExtensions.cs +++ b/src/Umbraco.Web/UrlHelperRenderExtensions.cs @@ -281,14 +281,15 @@ namespace Umbraco.Web return CreateHtmlString(url, htmlEncode); } - public static IHtmlString GetCropUrl(this UrlHelper urlHelper, ImageCropperValue imageCropperValue, string cropAlias, bool htmlEncode = true) - { - if (imageCropperValue == null || string.IsNullOrEmpty(imageCropperValue.Src)) return EmptyHtmlString; - - var url = imageCropperValue.Src.GetCropUrl(imageCropperValue, cropAlias: cropAlias, useCropDimensions: true); - - return CreateHtmlString(url, htmlEncode); - } + // TODO: enable again in v9 and make sure to document that `@Url.GetCropUrl(Model.Property, cropAlias: "Featured")` needs to be updated - see https://github.com/umbraco/Umbraco-CMS/pull/10527 for alternatives + // public static IHtmlString GetCropUrl(this UrlHelper urlHelper, ImageCropperValue imageCropperValue, string cropAlias, bool htmlEncode = true) + // { + // if (imageCropperValue == null || string.IsNullOrEmpty(imageCropperValue.Src)) return EmptyHtmlString; + // + // var url = imageCropperValue.Src.GetCropUrl(imageCropperValue, cropAlias: cropAlias, useCropDimensions: true); + // + // return CreateHtmlString(url, htmlEncode); + // } public static IHtmlString GetCropUrl(this UrlHelper urlHelper, ImageCropperValue imageCropperValue, From d624eaff335c67fcbadc35fab2ca767b4612182b Mon Sep 17 00:00:00 2001 From: Vitor Rodrigues Date: Tue, 13 Jul 2021 01:26:03 +0200 Subject: [PATCH 021/123] Added MaybeNullAttribute to ModelsBuilder generated files The added attribute will enable consuming projects that have Nullable Reference Types enabled to receive proper warnings when access null checks aren't performed. --- .../ModelsBuilder/Building/TextBuilder.cs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/Umbraco.Infrastructure/ModelsBuilder/Building/TextBuilder.cs b/src/Umbraco.Infrastructure/ModelsBuilder/Building/TextBuilder.cs index 9084cc902f..a5186963ab 100644 --- a/src/Umbraco.Infrastructure/ModelsBuilder/Building/TextBuilder.cs +++ b/src/Umbraco.Infrastructure/ModelsBuilder/Building/TextBuilder.cs @@ -98,6 +98,13 @@ namespace Umbraco.Cms.Infrastructure.ModelsBuilder.Building sb.AppendFormat("{0}[global::System.CodeDom.Compiler.GeneratedCodeAttribute(\"Umbraco.ModelsBuilder.Embedded\", \"{1}\")]\n", tabs, ApiVersion.Current.Version); } + // writes an attribute that specifies that an output may be null. + // (useful for consuming projects with nullable reference types enabled) + private static void WriteMaybeNullAttribute(StringBuilder sb, string tabs, bool isReturn = false) + { + sb.AppendFormat("{0}[{1}global::System.Diagnostics.CodeAnalysis.MaybeNull]\n", tabs, isReturn ? "return: " : ""); + } + private void WriteContentType(StringBuilder sb, TypeModel type) { string sep; @@ -185,9 +192,11 @@ namespace Umbraco.Cms.Infrastructure.ModelsBuilder.Building sb.AppendFormat("\t\tpublic new const PublishedItemType ModelItemType = PublishedItemType.{0};\n", itemType); WriteGeneratedCodeAttribute(sb, "\t\t"); + WriteMaybeNullAttribute(sb, "\t\t", true); sb.Append("\t\tpublic new static IPublishedContentType GetModelContentType(IPublishedSnapshotAccessor publishedSnapshotAccessor)\n"); sb.Append("\t\t\t=> PublishedModelUtility.GetModelContentType(publishedSnapshotAccessor, ModelItemType, ModelTypeAlias);\n"); WriteGeneratedCodeAttribute(sb, "\t\t"); + WriteMaybeNullAttribute(sb, "\t\t", true); sb.AppendFormat("\t\tpublic static IPublishedPropertyType GetModelPropertyType(IPublishedSnapshotAccessor publishedSnapshotAccessor, Expression> selector)\n", type.ClrName); sb.Append("\t\t\t=> PublishedModelUtility.GetModelPropertyType(GetModelContentType(publishedSnapshotAccessor), selector);\n"); @@ -305,6 +314,8 @@ namespace Umbraco.Cms.Infrastructure.ModelsBuilder.Building } WriteGeneratedCodeAttribute(sb, "\t\t"); + if (!property.ModelClrType.IsValueType) + WriteMaybeNullAttribute(sb, "\t\t"); sb.AppendFormat("\t\t[ImplementPropertyType(\"{0}\")]\n", property.Alias); if (mixinStatic) @@ -349,6 +360,8 @@ namespace Umbraco.Cms.Infrastructure.ModelsBuilder.Building sb.AppendFormat("\t\t/// Static getter for {0}\n", XmlCommentString(property.Name)); WriteGeneratedCodeAttribute(sb, "\t\t"); + if (!property.ModelClrType.IsValueType) + WriteMaybeNullAttribute(sb, "\t\t", true); sb.Append("\t\tpublic static "); WriteClrType(sb, property.ClrTypeName); sb.AppendFormat(" {0}(I{1} that, IPublishedValueFallback publishedValueFallback) => that.Value", @@ -404,6 +417,9 @@ namespace Umbraco.Cms.Infrastructure.ModelsBuilder.Building if (!string.IsNullOrWhiteSpace(property.Name)) sb.AppendFormat("\t\t/// {0}\n", XmlCommentString(property.Name)); WriteGeneratedCodeAttribute(sb, "\t\t"); + if (!property.ModelClrType.IsValueType) + WriteMaybeNullAttribute(sb, "\t\t"); + sb.Append("\t\t"); WriteClrType(sb, property.ClrTypeName); sb.AppendFormat(" {0} {{ get; }}\n", From 4f9265240bd7ead2e1c729612f5c3dee15ed41bd Mon Sep 17 00:00:00 2001 From: Vitor Rodrigues Date: Tue, 13 Jul 2021 01:47:26 +0200 Subject: [PATCH 022/123] #10648: Updated public IUmbracoBuilder extension methods to allow chaining --- .../Composing/CompositionExtensions.cs | 9 ++- .../UmbracoBuilder.DistributedCache.cs | 46 +++++++----- .../UmbracoBuilder.Uniques.cs | 70 ++++++++++++------- .../WebsiteUmbracoBuilderExtensions.cs | 18 +++-- 4 files changed, 91 insertions(+), 52 deletions(-) diff --git a/src/Umbraco.Core/Composing/CompositionExtensions.cs b/src/Umbraco.Core/Composing/CompositionExtensions.cs index 74f30b81b6..d087af77d8 100644 --- a/src/Umbraco.Core/Composing/CompositionExtensions.cs +++ b/src/Umbraco.Core/Composing/CompositionExtensions.cs @@ -11,9 +11,10 @@ namespace Umbraco.Extensions /// /// The builder. /// A function creating a published snapshot service. - public static void SetPublishedSnapshotService(this IUmbracoBuilder builder, Func factory) + public static IUmbracoBuilder SetPublishedSnapshotService(this IUmbracoBuilder builder, Func factory) { builder.Services.AddUnique(factory); + return builder; } /// @@ -21,10 +22,11 @@ namespace Umbraco.Extensions /// /// The type of the published snapshot service. /// The builder. - public static void SetPublishedSnapshotService(this IUmbracoBuilder builder) + public static IUmbracoBuilder SetPublishedSnapshotService(this IUmbracoBuilder builder) where T : class, IPublishedSnapshotService { builder.Services.AddUnique(); + return builder; } /// @@ -32,9 +34,10 @@ namespace Umbraco.Extensions /// /// The builder. /// A published snapshot service. - public static void SetPublishedSnapshotService(this IUmbracoBuilder builder, IPublishedSnapshotService service) + public static IUmbracoBuilder SetPublishedSnapshotService(this IUmbracoBuilder builder, IPublishedSnapshotService service) { builder.Services.AddUnique(service); + return builder; } } } diff --git a/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.DistributedCache.cs b/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.DistributedCache.cs index 05dba2cc0f..71ea85d80f 100644 --- a/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.DistributedCache.cs +++ b/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.DistributedCache.cs @@ -2,12 +2,8 @@ using System; using Microsoft.Extensions.DependencyInjection; using Umbraco.Cms.Core.Cache; using Umbraco.Cms.Core.DependencyInjection; -using Umbraco.Cms.Core.Events; using Umbraco.Cms.Core.Notifications; -using Umbraco.Cms.Core.PublishedCache; -using Umbraco.Cms.Core.Services.Changes; using Umbraco.Cms.Core.Sync; -using Umbraco.Cms.Infrastructure.Search; using Umbraco.Cms.Infrastructure.Sync; using Umbraco.Extensions; @@ -40,49 +36,67 @@ namespace Umbraco.Cms.Infrastructure.DependencyInjection /// /// The type of the server registrar. /// The builder. - public static void SetServerRegistrar(this IUmbracoBuilder builder) + public static IUmbracoBuilder SetServerRegistrar(this IUmbracoBuilder builder) where T : class, IServerRoleAccessor - => builder.Services.AddUnique(); + { + builder.Services.AddUnique(); + return builder; + } /// /// Sets the server registrar. /// /// The builder. /// A function creating a server registrar. - public static void SetServerRegistrar(this IUmbracoBuilder builder, Func factory) - => builder.Services.AddUnique(factory); + public static IUmbracoBuilder SetServerRegistrar(this IUmbracoBuilder builder, Func factory) + { + builder.Services.AddUnique(factory); + return builder; + } /// /// Sets the server registrar. /// /// The builder. /// A server registrar. - public static void SetServerRegistrar(this IUmbracoBuilder builder, IServerRoleAccessor registrar) - => builder.Services.AddUnique(registrar); + public static IUmbracoBuilder SetServerRegistrar(this IUmbracoBuilder builder, IServerRoleAccessor registrar) + { + builder.Services.AddUnique(registrar); + return builder; + } /// /// Sets the server messenger. /// /// The type of the server registrar. /// The builder. - public static void SetServerMessenger(this IUmbracoBuilder builder) + public static IUmbracoBuilder SetServerMessenger(this IUmbracoBuilder builder) where T : class, IServerMessenger - => builder.Services.AddUnique(); + { + builder.Services.AddUnique(); + return builder; + } /// /// Sets the server messenger. /// /// The builder. /// A function creating a server messenger. - public static void SetServerMessenger(this IUmbracoBuilder builder, Func factory) - => builder.Services.AddUnique(factory); + public static IUmbracoBuilder SetServerMessenger(this IUmbracoBuilder builder, Func factory) + { + builder.Services.AddUnique(factory); + return builder; + } /// /// Sets the server messenger. /// /// The builder. /// A server messenger. - public static void SetServerMessenger(this IUmbracoBuilder builder, IServerMessenger registrar) - => builder.Services.AddUnique(registrar); + public static IUmbracoBuilder SetServerMessenger(this IUmbracoBuilder builder, IServerMessenger registrar) + { + builder.Services.AddUnique(registrar); + return builder; + } } } diff --git a/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.Uniques.cs b/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.Uniques.cs index cbbaa6a3e0..b311b1f0da 100644 --- a/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.Uniques.cs +++ b/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.Uniques.cs @@ -1,6 +1,5 @@ using System; using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.DependencyInjection.Extensions; using Umbraco.Cms.Core.DependencyInjection; using Umbraco.Cms.Core.Dictionary; using Umbraco.Cms.Core.IO; @@ -21,10 +20,11 @@ namespace Umbraco.Cms.Infrastructure.DependencyInjection /// /// The type of the factory. /// The builder. - public static void SetCultureDictionaryFactory(this IUmbracoBuilder builder) + public static IUmbracoBuilder SetCultureDictionaryFactory(this IUmbracoBuilder builder) where T : class, ICultureDictionaryFactory { builder.Services.AddUnique(); + return builder; } /// @@ -32,9 +32,10 @@ namespace Umbraco.Cms.Infrastructure.DependencyInjection /// /// The builder. /// A function creating a culture dictionary factory. - public static void SetCultureDictionaryFactory(this IUmbracoBuilder builder, Func factory) + public static IUmbracoBuilder SetCultureDictionaryFactory(this IUmbracoBuilder builder, Func factory) { builder.Services.AddUnique(factory); + return builder; } /// @@ -42,9 +43,10 @@ namespace Umbraco.Cms.Infrastructure.DependencyInjection /// /// The builder. /// A factory. - public static void SetCultureDictionaryFactory(this IUmbracoBuilder builder, ICultureDictionaryFactory factory) + public static IUmbracoBuilder SetCultureDictionaryFactory(this IUmbracoBuilder builder, ICultureDictionaryFactory factory) { builder.Services.AddUnique(factory); + return builder; } /// @@ -52,10 +54,11 @@ namespace Umbraco.Cms.Infrastructure.DependencyInjection /// /// The type of the factory. /// The builder. - public static void SetPublishedContentModelFactory(this IUmbracoBuilder builder) + public static IUmbracoBuilder SetPublishedContentModelFactory(this IUmbracoBuilder builder) where T : class, IPublishedModelFactory { builder.Services.AddUnique(); + return builder; } /// @@ -63,9 +66,10 @@ namespace Umbraco.Cms.Infrastructure.DependencyInjection /// /// The builder. /// A function creating a published content model factory. - public static void SetPublishedContentModelFactory(this IUmbracoBuilder builder, Func factory) + public static IUmbracoBuilder SetPublishedContentModelFactory(this IUmbracoBuilder builder, Func factory) { builder.Services.AddUnique(factory); + return builder; } /// @@ -73,9 +77,10 @@ namespace Umbraco.Cms.Infrastructure.DependencyInjection /// /// The builder. /// A published content model factory. - public static void SetPublishedContentModelFactory(this IUmbracoBuilder builder, IPublishedModelFactory factory) + public static IUmbracoBuilder SetPublishedContentModelFactory(this IUmbracoBuilder builder, IPublishedModelFactory factory) { builder.Services.AddUnique(factory); + return builder; } /// @@ -83,10 +88,11 @@ namespace Umbraco.Cms.Infrastructure.DependencyInjection /// /// The type of the short string helper. /// The builder. - public static void SetShortStringHelper(this IUmbracoBuilder builder) + public static IUmbracoBuilder SetShortStringHelper(this IUmbracoBuilder builder) where T : class, IShortStringHelper { builder.Services.AddUnique(); + return builder; } /// @@ -94,9 +100,10 @@ namespace Umbraco.Cms.Infrastructure.DependencyInjection /// /// The builder. /// A function creating a short string helper. - public static void SetShortStringHelper(this IUmbracoBuilder builder, Func factory) + public static IUmbracoBuilder SetShortStringHelper(this IUmbracoBuilder builder, Func factory) { builder.Services.AddUnique(factory); + return builder; } /// @@ -104,9 +111,10 @@ namespace Umbraco.Cms.Infrastructure.DependencyInjection /// /// A builder. /// A short string helper. - public static void SetShortStringHelper(this IUmbracoBuilder builder, IShortStringHelper helper) + public static IUmbracoBuilder SetShortStringHelper(this IUmbracoBuilder builder, IShortStringHelper helper) { builder.Services.AddUnique(helper); + return builder; } /// @@ -114,19 +122,23 @@ namespace Umbraco.Cms.Infrastructure.DependencyInjection /// /// A builder. /// Factory method to create an IFileSystem implementation used in the MediaFileManager - public static void SetMediaFileSystem(this IUmbracoBuilder builder, - Func filesystemFactory) => builder.Services.AddUnique( - provider => - { - IFileSystem filesystem = filesystemFactory(provider); - // We need to use the Filesystems to create a shadow wrapper, - // because shadow wrapper requires the IsScoped delegate from the FileSystems. - // This is used by the scope provider when taking control of the filesystems. - FileSystems fileSystems = provider.GetRequiredService(); - IFileSystem shadow = fileSystems.CreateShadowWrapper(filesystem, "media"); + public static IUmbracoBuilder SetMediaFileSystem(this IUmbracoBuilder builder, + Func filesystemFactory) + { + builder.Services.AddUnique( + provider => + { + IFileSystem filesystem = filesystemFactory(provider); + // We need to use the Filesystems to create a shadow wrapper, + // because shadow wrapper requires the IsScoped delegate from the FileSystems. + // This is used by the scope provider when taking control of the filesystems. + FileSystems fileSystems = provider.GetRequiredService(); + IFileSystem shadow = fileSystems.CreateShadowWrapper(filesystem, "media"); - return provider.CreateInstance(shadow); - }); + return provider.CreateInstance(shadow); + }); + return builder; + } /// /// Register FileSystems with a method to configure the . @@ -135,7 +147,7 @@ namespace Umbraco.Cms.Infrastructure.DependencyInjection /// Method that configures the . /// Throws exception if is null. /// Throws exception if full path can't be resolved successfully. - public static void ConfigureFileSystems(this IUmbracoBuilder builder, + public static IUmbracoBuilder ConfigureFileSystems(this IUmbracoBuilder builder, Action configure) { if (configure == null) @@ -150,6 +162,7 @@ namespace Umbraco.Cms.Infrastructure.DependencyInjection configure(provider, fileSystems); return fileSystems; }); + return builder; } /// @@ -157,10 +170,11 @@ namespace Umbraco.Cms.Infrastructure.DependencyInjection /// /// The type of the log viewer. /// The builder. - public static void SetLogViewer(this IUmbracoBuilder builder) + public static IUmbracoBuilder SetLogViewer(this IUmbracoBuilder builder) where T : class, ILogViewer { builder.Services.AddUnique(); + return builder; } /// @@ -168,19 +182,21 @@ namespace Umbraco.Cms.Infrastructure.DependencyInjection /// /// The builder. /// A function creating a log viewer. - public static void SetLogViewer(this IUmbracoBuilder builder, Func factory) + public static IUmbracoBuilder SetLogViewer(this IUmbracoBuilder builder, Func factory) { builder.Services.AddUnique(factory); + return builder; } /// /// Sets the log viewer. /// /// A builder. - /// A log viewer. - public static void SetLogViewer(this IUmbracoBuilder builder, ILogViewer viewer) + /// A log viewer. + public static IUmbracoBuilder SetLogViewer(this IUmbracoBuilder builder, ILogViewer viewer) { builder.Services.AddUnique(viewer); + return builder; } } } diff --git a/src/Umbraco.Web.Website/Extensions/WebsiteUmbracoBuilderExtensions.cs b/src/Umbraco.Web.Website/Extensions/WebsiteUmbracoBuilderExtensions.cs index 72994992fa..65bff41a59 100644 --- a/src/Umbraco.Web.Website/Extensions/WebsiteUmbracoBuilderExtensions.cs +++ b/src/Umbraco.Web.Website/Extensions/WebsiteUmbracoBuilderExtensions.cs @@ -16,10 +16,11 @@ namespace Umbraco.Extensions /// /// The type of the content last chance finder. /// The builder. - public static void SetContentLastChanceFinder(this IUmbracoBuilder builder) + public static IUmbracoBuilder SetContentLastChanceFinder(this IUmbracoBuilder builder) where T : class, IContentLastChanceFinder { builder.Services.AddUnique(); + return builder; } /// @@ -27,9 +28,10 @@ namespace Umbraco.Extensions /// /// The builder. /// A function creating a last chance finder. - public static void SetContentLastChanceFinder(this IUmbracoBuilder builder, Func factory) + public static IUmbracoBuilder SetContentLastChanceFinder(this IUmbracoBuilder builder, Func factory) { builder.Services.AddUnique(factory); + return builder; } /// @@ -37,9 +39,10 @@ namespace Umbraco.Extensions /// /// The builder. /// A last chance finder. - public static void SetContentLastChanceFinder(this IUmbracoBuilder builder, IContentLastChanceFinder finder) + public static IUmbracoBuilder SetContentLastChanceFinder(this IUmbracoBuilder builder, IContentLastChanceFinder finder) { builder.Services.AddUnique(finder); + return builder; } /// @@ -47,10 +50,11 @@ namespace Umbraco.Extensions /// /// The type of the site domain helper. /// - public static void SetSiteDomainHelper(this IUmbracoBuilder builder) + public static IUmbracoBuilder SetSiteDomainHelper(this IUmbracoBuilder builder) where T : class, ISiteDomainMapper { builder.Services.AddUnique(); + return builder; } /// @@ -58,9 +62,10 @@ namespace Umbraco.Extensions /// /// The builder. /// A function creating a helper. - public static void SetSiteDomainHelper(this IUmbracoBuilder builder, Func factory) + public static IUmbracoBuilder SetSiteDomainHelper(this IUmbracoBuilder builder, Func factory) { builder.Services.AddUnique(factory); + return builder; } /// @@ -68,9 +73,10 @@ namespace Umbraco.Extensions /// /// The builder. /// A helper. - public static void SetSiteDomainHelper(this IUmbracoBuilder builder, ISiteDomainMapper helper) + public static IUmbracoBuilder SetSiteDomainHelper(this IUmbracoBuilder builder, ISiteDomainMapper helper) { builder.Services.AddUnique(helper); + return builder; } #endregion From 89e758127dd9c96b7a23ad61543f16a868886cb5 Mon Sep 17 00:00:00 2001 From: Vitor Rodrigues Date: Tue, 13 Jul 2021 10:15:03 +0200 Subject: [PATCH 023/123] Updated ModelsBuilder unit tests --- .../Umbraco.ModelsBuilder.Embedded/BuilderTests.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Umbraco.Tests.UnitTests/Umbraco.ModelsBuilder.Embedded/BuilderTests.cs b/src/Umbraco.Tests.UnitTests/Umbraco.ModelsBuilder.Embedded/BuilderTests.cs index 346ac0239d..4dea81facb 100644 --- a/src/Umbraco.Tests.UnitTests/Umbraco.ModelsBuilder.Embedded/BuilderTests.cs +++ b/src/Umbraco.Tests.UnitTests/Umbraco.ModelsBuilder.Embedded/BuilderTests.cs @@ -77,9 +77,11 @@ namespace Umbraco.Cms.Web.Common.PublishedModels [global::System.CodeDom.Compiler.GeneratedCodeAttribute(""Umbraco.ModelsBuilder.Embedded"", """ + version + @""")] public new const PublishedItemType ModelItemType = PublishedItemType.Content; [global::System.CodeDom.Compiler.GeneratedCodeAttribute(""Umbraco.ModelsBuilder.Embedded"", """ + version + @""")] + [return: global::System.Diagnostics.CodeAnalysis.MaybeNull] public new static IPublishedContentType GetModelContentType(IPublishedSnapshotAccessor publishedSnapshotAccessor) => PublishedModelUtility.GetModelContentType(publishedSnapshotAccessor, ModelItemType, ModelTypeAlias); [global::System.CodeDom.Compiler.GeneratedCodeAttribute(""Umbraco.ModelsBuilder.Embedded"", """ + version + @""")] + [return: global::System.Diagnostics.CodeAnalysis.MaybeNull] public static IPublishedPropertyType GetModelPropertyType(IPublishedSnapshotAccessor publishedSnapshotAccessor, Expression> selector) => PublishedModelUtility.GetModelPropertyType(GetModelContentType(publishedSnapshotAccessor), selector); #pragma warning restore 0109 @@ -96,6 +98,7 @@ namespace Umbraco.Cms.Web.Common.PublishedModels // properties [global::System.CodeDom.Compiler.GeneratedCodeAttribute(""Umbraco.ModelsBuilder.Embedded"", """ + version + @""")] + [global::System.Diagnostics.CodeAnalysis.MaybeNull] [ImplementPropertyType(""prop1"")] public virtual string Prop1 => this.Value(_publishedValueFallback, ""prop1""); } @@ -183,9 +186,11 @@ namespace Umbraco.Cms.Web.Common.PublishedModels [global::System.CodeDom.Compiler.GeneratedCodeAttribute(""Umbraco.ModelsBuilder.Embedded"", """ + version + @""")] public new const PublishedItemType ModelItemType = PublishedItemType.Content; [global::System.CodeDom.Compiler.GeneratedCodeAttribute(""Umbraco.ModelsBuilder.Embedded"", """ + version + @""")] + [return: global::System.Diagnostics.CodeAnalysis.MaybeNull] public new static IPublishedContentType GetModelContentType(IPublishedSnapshotAccessor publishedSnapshotAccessor) => PublishedModelUtility.GetModelContentType(publishedSnapshotAccessor, ModelItemType, ModelTypeAlias); [global::System.CodeDom.Compiler.GeneratedCodeAttribute(""Umbraco.ModelsBuilder.Embedded"", """ + version + @""")] + [return: global::System.Diagnostics.CodeAnalysis.MaybeNull] public static IPublishedPropertyType GetModelPropertyType(IPublishedSnapshotAccessor publishedSnapshotAccessor, Expression> selector) => PublishedModelUtility.GetModelPropertyType(GetModelContentType(publishedSnapshotAccessor), selector); #pragma warning restore 0109 @@ -202,6 +207,7 @@ namespace Umbraco.Cms.Web.Common.PublishedModels // properties [global::System.CodeDom.Compiler.GeneratedCodeAttribute(""Umbraco.ModelsBuilder.Embedded"", """ + version + @""")] + [global::System.Diagnostics.CodeAnalysis.MaybeNull] [ImplementPropertyType(""foo"")] public virtual global::System.Collections.Generic.IEnumerable Foo => this.Value>(_publishedValueFallback, ""foo""); } From d045003fb5ff352611b38d75fbe032be884cab6c Mon Sep 17 00:00:00 2001 From: Evan Moore Date: Tue, 13 Jul 2021 11:02:45 -0400 Subject: [PATCH 024/123] add userId to IX_ExternalLogin_LoginProvider --- .../Migrations/Upgrade/V_9_0_0/ExternalLoginTableIndexes.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Umbraco.Infrastructure/Migrations/Upgrade/V_9_0_0/ExternalLoginTableIndexes.cs b/src/Umbraco.Infrastructure/Migrations/Upgrade/V_9_0_0/ExternalLoginTableIndexes.cs index ef29207093..a582157c6c 100644 --- a/src/Umbraco.Infrastructure/Migrations/Upgrade/V_9_0_0/ExternalLoginTableIndexes.cs +++ b/src/Umbraco.Infrastructure/Migrations/Upgrade/V_9_0_0/ExternalLoginTableIndexes.cs @@ -45,8 +45,8 @@ namespace Umbraco.Cms.Infrastructure.Migrations.Upgrade.V_9_0_0 Create .Index(indexName1) .OnTable(ExternalLoginDto.TableName) - .OnColumn("loginProvider") - .Ascending() + .OnColumn("loginProvider").Ascending() + .OnColumn("userId").Ascending() .WithOptions() .Unique() .WithOptions() From d4760f546794cc82ce65507d5b22ef2da5762f3d Mon Sep 17 00:00:00 2001 From: Evan Moore Date: Tue, 13 Jul 2021 13:19:28 -0400 Subject: [PATCH 025/123] include userId in IX_umbracoExternalLogin_LoginProvider --- .../Persistence/Dtos/ExternalLoginDto.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Umbraco.Infrastructure/Persistence/Dtos/ExternalLoginDto.cs b/src/Umbraco.Infrastructure/Persistence/Dtos/ExternalLoginDto.cs index 3fc65b28a5..95ff357abe 100644 --- a/src/Umbraco.Infrastructure/Persistence/Dtos/ExternalLoginDto.cs +++ b/src/Umbraco.Infrastructure/Persistence/Dtos/ExternalLoginDto.cs @@ -25,9 +25,9 @@ namespace Umbraco.Cms.Infrastructure.Persistence.Dtos public int UserId { get; set; } [Column("loginProvider")] - [Length(4000)] // TODO: This value seems WAY too high, this is just a name + [Length(400)] [NullSetting(NullSetting = NullSettings.NotNull)] - [Index(IndexTypes.UniqueNonClustered, Name = "IX_" + TableName + "_LoginProvider")] + [Index(IndexTypes.UniqueNonClustered, ForColumns = "loginProvider,userId", Name = "IX_" + TableName + "_LoginProvider")] public string LoginProvider { get; set; } [Column("providerKey")] From 0ca4dd176ca35a9d43d0c4db292a7c2a99a90942 Mon Sep 17 00:00:00 2001 From: Evan Moore Date: Tue, 13 Jul 2021 13:55:44 -0400 Subject: [PATCH 026/123] Add some comments --- .../Persistence/Dtos/ExternalLoginDto.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Umbraco.Infrastructure/Persistence/Dtos/ExternalLoginDto.cs b/src/Umbraco.Infrastructure/Persistence/Dtos/ExternalLoginDto.cs index 95ff357abe..69bf1b837e 100644 --- a/src/Umbraco.Infrastructure/Persistence/Dtos/ExternalLoginDto.cs +++ b/src/Umbraco.Infrastructure/Persistence/Dtos/ExternalLoginDto.cs @@ -24,12 +24,18 @@ namespace Umbraco.Cms.Infrastructure.Persistence.Dtos [Index(IndexTypes.NonClustered)] public int UserId { get; set; } + /// + /// Used to store the name of the provider (i.e. Facebook, Google) + /// [Column("loginProvider")] [Length(400)] [NullSetting(NullSetting = NullSettings.NotNull)] [Index(IndexTypes.UniqueNonClustered, ForColumns = "loginProvider,userId", Name = "IX_" + TableName + "_LoginProvider")] public string LoginProvider { get; set; } + /// + /// Stores the key the provider uses to lookup the login + /// [Column("providerKey")] [Length(4000)] [NullSetting(NullSetting = NullSettings.NotNull)] From 0c125492f4200fe6b856509632951a0e6aa3a851 Mon Sep 17 00:00:00 2001 From: Evan Moore Date: Tue, 13 Jul 2021 20:58:37 -0400 Subject: [PATCH 027/123] Fix External Login Providers using wrong property for icon --- .../src/views/components/application/umb-login.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Umbraco.Web.UI.Client/src/views/components/application/umb-login.html b/src/Umbraco.Web.UI.Client/src/views/components/application/umb-login.html index 02429fa164..56d5aa8c93 100644 --- a/src/Umbraco.Web.UI.Client/src/views/components/application/umb-login.html +++ b/src/Umbraco.Web.UI.Client/src/views/components/application/umb-login.html @@ -129,7 +129,7 @@ ng-class="login.properties.ButtonStyle" id="{{login.authType}}" name="provider" value="{{login.authType}}" title="Log in using your {{login.caption}} account"> - + Sign in with {{login.caption}} From acda7706bec0a6e038a438bc8f1856d0a080019c Mon Sep 17 00:00:00 2001 From: Evan Moore Date: Tue, 13 Jul 2021 21:04:47 -0400 Subject: [PATCH 028/123] Remove changes from temp-10656 --- .../Migrations/Upgrade/V_9_0_0/ExternalLoginTableIndexes.cs | 4 ++-- .../Persistence/Dtos/ExternalLoginDto.cs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Umbraco.Infrastructure/Migrations/Upgrade/V_9_0_0/ExternalLoginTableIndexes.cs b/src/Umbraco.Infrastructure/Migrations/Upgrade/V_9_0_0/ExternalLoginTableIndexes.cs index a582157c6c..ef29207093 100644 --- a/src/Umbraco.Infrastructure/Migrations/Upgrade/V_9_0_0/ExternalLoginTableIndexes.cs +++ b/src/Umbraco.Infrastructure/Migrations/Upgrade/V_9_0_0/ExternalLoginTableIndexes.cs @@ -45,8 +45,8 @@ namespace Umbraco.Cms.Infrastructure.Migrations.Upgrade.V_9_0_0 Create .Index(indexName1) .OnTable(ExternalLoginDto.TableName) - .OnColumn("loginProvider").Ascending() - .OnColumn("userId").Ascending() + .OnColumn("loginProvider") + .Ascending() .WithOptions() .Unique() .WithOptions() diff --git a/src/Umbraco.Infrastructure/Persistence/Dtos/ExternalLoginDto.cs b/src/Umbraco.Infrastructure/Persistence/Dtos/ExternalLoginDto.cs index 95ff357abe..3fc65b28a5 100644 --- a/src/Umbraco.Infrastructure/Persistence/Dtos/ExternalLoginDto.cs +++ b/src/Umbraco.Infrastructure/Persistence/Dtos/ExternalLoginDto.cs @@ -25,9 +25,9 @@ namespace Umbraco.Cms.Infrastructure.Persistence.Dtos public int UserId { get; set; } [Column("loginProvider")] - [Length(400)] + [Length(4000)] // TODO: This value seems WAY too high, this is just a name [NullSetting(NullSetting = NullSettings.NotNull)] - [Index(IndexTypes.UniqueNonClustered, ForColumns = "loginProvider,userId", Name = "IX_" + TableName + "_LoginProvider")] + [Index(IndexTypes.UniqueNonClustered, Name = "IX_" + TableName + "_LoginProvider")] public string LoginProvider { get; set; } [Column("providerKey")] From 96837a18a18ce60854d94726be1b51e8b4f794a3 Mon Sep 17 00:00:00 2001 From: Mole Date: Wed, 14 Jul 2021 11:03:09 +0200 Subject: [PATCH 029/123] Correctly return false from TryGetMediaPath if the value is null --- .../PropertyEditors/MediaUrlGeneratorCollection.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Umbraco.Core/PropertyEditors/MediaUrlGeneratorCollection.cs b/src/Umbraco.Core/PropertyEditors/MediaUrlGeneratorCollection.cs index cd3a36c607..35913d3fe4 100644 --- a/src/Umbraco.Core/PropertyEditors/MediaUrlGeneratorCollection.cs +++ b/src/Umbraco.Core/PropertyEditors/MediaUrlGeneratorCollection.cs @@ -12,6 +12,14 @@ namespace Umbraco.Cms.Core.PropertyEditors public bool TryGetMediaPath(string propertyEditorAlias, object value, out string mediaPath) { + // We can't get a media path from a null value + // The value will be null when uploading a brand new image, since we try to get the "old path" which doesn't exist yet. + if (value is null) + { + mediaPath = null; + return false; + } + foreach(IMediaUrlGenerator generator in this) { if (generator.TryGetMediaPath(propertyEditorAlias, value, out var mp)) From ef4621e6f139009094cfa04fb1ec8571e6d4c6b4 Mon Sep 17 00:00:00 2001 From: Evan Moore Date: Wed, 14 Jul 2021 09:10:57 -0400 Subject: [PATCH 030/123] Login screen replace i tag with umb-icon directive --- .../views/components/application/umb-login.html | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Umbraco.Web.UI.Client/src/views/components/application/umb-login.html b/src/Umbraco.Web.UI.Client/src/views/components/application/umb-login.html index 56d5aa8c93..eb3a334f53 100644 --- a/src/Umbraco.Web.UI.Client/src/views/components/application/umb-login.html +++ b/src/Umbraco.Web.UI.Client/src/views/components/application/umb-login.html @@ -124,14 +124,14 @@
- +
From 8e20b7fda5f0023a21c17f790495a474e241768c Mon Sep 17 00:00:00 2001 From: Evan Moore Date: Wed, 14 Jul 2021 12:29:59 -0400 Subject: [PATCH 031/123] Fix broken icon on user overlay --- .../src/views/common/overlays/user/user.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Umbraco.Web.UI.Client/src/views/common/overlays/user/user.html b/src/Umbraco.Web.UI.Client/src/views/common/overlays/user/user.html index 330a57ab7d..d5f9a0e363 100644 --- a/src/Umbraco.Web.UI.Client/src/views/common/overlays/user/user.html +++ b/src/Umbraco.Web.UI.Client/src/views/common/overlays/user/user.html @@ -59,7 +59,7 @@ ng-class="login.properties.ButtonStyle" id="{{login.authType}}"> - + Link your {{login.caption}} account From f978ef15d02ed3b608a162b3bdff88877763573c Mon Sep 17 00:00:00 2001 From: Adam Hearn Date: Thu, 15 Jul 2021 01:28:24 +0100 Subject: [PATCH 032/123] Support member names using Down-Level Logon Name format --- .../Configuration/Models/SecuritySettings.cs | 7 +++++++ .../Security/ConfigureMemberIdentityOptions.cs | 11 +++++++++-- src/Umbraco.Web.UI.NetCore/appsettings.json | 1 + 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/Umbraco.Core/Configuration/Models/SecuritySettings.cs b/src/Umbraco.Core/Configuration/Models/SecuritySettings.cs index 48e08d596a..7d4dd45fb8 100644 --- a/src/Umbraco.Core/Configuration/Models/SecuritySettings.cs +++ b/src/Umbraco.Core/Configuration/Models/SecuritySettings.cs @@ -15,6 +15,7 @@ namespace Umbraco.Cms.Core.Configuration.Models internal const bool StaticHideDisabledUsersInBackOffice = false; internal const bool StaticAllowPasswordReset = true; internal const string StaticAuthCookieName = "UMB_UCONTEXT"; + internal const string StaticAllowedUserNameCharacters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._@+\\"; /// /// Gets or sets a value indicating whether to keep the user logged in. @@ -50,6 +51,12 @@ namespace Umbraco.Cms.Core.Configuration.Models /// public bool UsernameIsEmail { get; set; } = true; + /// + /// Gets or sets the set of allowed characters for a username + /// + [DefaultValue(StaticAllowedUserNameCharacters)] + public string AllowedUserNameCharacters { get; set; } = StaticAllowedUserNameCharacters; + /// /// Gets or sets a value for the user password settings. /// diff --git a/src/Umbraco.Web.Common/Security/ConfigureMemberIdentityOptions.cs b/src/Umbraco.Web.Common/Security/ConfigureMemberIdentityOptions.cs index cc19670f83..db82ff2b05 100644 --- a/src/Umbraco.Web.Common/Security/ConfigureMemberIdentityOptions.cs +++ b/src/Umbraco.Web.Common/Security/ConfigureMemberIdentityOptions.cs @@ -10,9 +10,13 @@ namespace Umbraco.Cms.Web.Common.Security public sealed class ConfigureMemberIdentityOptions : IConfigureOptions { private readonly MemberPasswordConfigurationSettings _memberPasswordConfiguration; + private readonly SecuritySettings _securitySettings; - public ConfigureMemberIdentityOptions(IOptions memberPasswordConfiguration) - => _memberPasswordConfiguration = memberPasswordConfiguration.Value; + public ConfigureMemberIdentityOptions(IOptions memberPasswordConfiguration, IOptions securitySettings) + { + _memberPasswordConfiguration = memberPasswordConfiguration.Value; + _securitySettings = securitySettings.Value; + } public void Configure(IdentityOptions options) { @@ -22,6 +26,9 @@ namespace Umbraco.Cms.Web.Common.Security options.User.RequireUniqueEmail = true; + // Support validation of member names using Down-Level Logon Name format + options.User.AllowedUserNameCharacters = _securitySettings.AllowedUserNameCharacters; + options.Lockout.AllowedForNewUsers = true; // TODO: Implement this options.Lockout.DefaultLockoutTimeSpan = TimeSpan.FromDays(30); diff --git a/src/Umbraco.Web.UI.NetCore/appsettings.json b/src/Umbraco.Web.UI.NetCore/appsettings.json index 8fe1993bcb..a2c0e9c938 100644 --- a/src/Umbraco.Web.UI.NetCore/appsettings.json +++ b/src/Umbraco.Web.UI.NetCore/appsettings.json @@ -46,6 +46,7 @@ "KeepUserLoggedIn": false, "UsernameIsEmail": true, "HideDisabledUsersInBackoffice": false, + "AllowedUserNameCharacters": "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._@+\\", "UserPassword": { "RequiredLength": 10, "RequireNonLetterOrDigit": false, From 33f53fcf354ca1ca491f1038e4625c5b2b4a3283 Mon Sep 17 00:00:00 2001 From: Mole Date: Thu, 15 Jul 2021 14:10:52 +0200 Subject: [PATCH 033/123] Force authentication of user when getting localized text and get culture from user identity --- .../Controllers/BackOfficeController.cs | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/src/Umbraco.Web.BackOffice/Controllers/BackOfficeController.cs b/src/Umbraco.Web.BackOffice/Controllers/BackOfficeController.cs index 161896628b..b3e48963e4 100644 --- a/src/Umbraco.Web.BackOffice/Controllers/BackOfficeController.cs +++ b/src/Umbraco.Web.BackOffice/Controllers/BackOfficeController.cs @@ -4,7 +4,6 @@ using System.Globalization; using System.IO; using System.Linq; using System.Security.Claims; -using System.Threading; using System.Threading.Tasks; using Microsoft.AspNetCore.Authentication; using Microsoft.AspNetCore.Authorization; @@ -225,17 +224,25 @@ namespace Umbraco.Cms.Web.BackOffice.Controllers /// [HttpGet] [AllowAnonymous] - public Dictionary> LocalizedText(string culture = null) + public async Task>> LocalizedText(string culture = null) { - var isAuthenticated = _backofficeSecurityAccessor.BackOfficeSecurity.IsAuthenticated(); + CultureInfo cultureInfo; + if (string.IsNullOrWhiteSpace(culture)) + { + // Force authentication to occur since this is not an authorized endpoint, we need this to get a user. + AuthenticateResult authenticationResult = await this.AuthenticateBackOfficeAsync(); + // We have to get the culture from the Identity, we can't rely on thread culture + // It's entirely likely for a user to have a different culture in the backoffice, than their system. + var user = authenticationResult.Principal?.Identity; - var cultureInfo = string.IsNullOrWhiteSpace(culture) - //if the user is logged in, get their culture, otherwise default to 'en' - ? isAuthenticated - //current culture is set at the very beginning of each request - ? Thread.CurrentThread.CurrentCulture - : CultureInfo.GetCultureInfo(_globalSettings.DefaultUILanguage) - : CultureInfo.GetCultureInfo(culture); + cultureInfo = (authenticationResult.Succeeded && user is not null) + ? user.GetCulture() + : CultureInfo.GetCultureInfo(_globalSettings.DefaultUILanguage); + } + else + { + cultureInfo = CultureInfo.GetCultureInfo(culture); + } var allValues = _textService.GetAllStoredValues(cultureInfo); var pathedValues = allValues.Select(kv => From df9c4a016030b553d3bd88f6114709a842829282 Mon Sep 17 00:00:00 2001 From: Shannon Date: Thu, 15 Jul 2021 13:26:32 -0600 Subject: [PATCH 034/123] Adds bundle options to the package manifest to more control over how bundling works for static file processing for app_plugins --- .../Configuration/Grid/GridEditorsConfig.cs | 2 +- .../ContentAppFactoryCollectionBuilder.cs | 2 +- .../Dashboards/DashboardCollectionBuilder.cs | 4 +- src/Umbraco.Core/Manifest/BundleOptions.cs | 26 ++++ .../Manifest/CompositePackageManifest.cs | 67 ++++++++++ src/Umbraco.Core/Manifest/IManifestParser.cs | 4 +- src/Umbraco.Core/Manifest/ManifestAssets.cs | 17 +++ src/Umbraco.Core/Manifest/PackageManifest.cs | 3 + .../ParameterEditorCollection.cs | 2 +- .../PropertyEditorCollection.cs | 2 +- .../Sections/SectionCollectionBuilder.cs | 2 +- .../Manifest/ManifestParser.cs | 67 ++++++---- .../WebAssets/BackOfficeWebAssets.cs | 74 +++++++++-- .../WebAssets/RuntimeMinifierExtensions.cs | 41 ------ .../Controllers/BackOfficeController.cs | 10 +- .../Extensions/RuntimeMinifierExtensions.cs | 124 ++++++++++++++++++ .../SmidgeRuntimeMinifier.cs | 16 ++- 17 files changed, 373 insertions(+), 90 deletions(-) create mode 100644 src/Umbraco.Core/Manifest/BundleOptions.cs create mode 100644 src/Umbraco.Core/Manifest/CompositePackageManifest.cs create mode 100644 src/Umbraco.Core/Manifest/ManifestAssets.cs delete mode 100644 src/Umbraco.Infrastructure/WebAssets/RuntimeMinifierExtensions.cs create mode 100644 src/Umbraco.Web.BackOffice/Extensions/RuntimeMinifierExtensions.cs diff --git a/src/Umbraco.Core/Configuration/Grid/GridEditorsConfig.cs b/src/Umbraco.Core/Configuration/Grid/GridEditorsConfig.cs index ab6a7e9396..f1a4f0643c 100644 --- a/src/Umbraco.Core/Configuration/Grid/GridEditorsConfig.cs +++ b/src/Umbraco.Core/Configuration/Grid/GridEditorsConfig.cs @@ -64,7 +64,7 @@ namespace Umbraco.Cms.Core.Configuration.Grid } // add manifest editors, skip duplicates - foreach (var gridEditor in _manifestParser.Manifest.GridEditors) + foreach (var gridEditor in _manifestParser.CombinedManifest.GridEditors) { if (editors.Contains(gridEditor) == false) editors.Add(gridEditor); } diff --git a/src/Umbraco.Core/ContentApps/ContentAppFactoryCollectionBuilder.cs b/src/Umbraco.Core/ContentApps/ContentAppFactoryCollectionBuilder.cs index ff1223d983..a80c79a3ef 100644 --- a/src/Umbraco.Core/ContentApps/ContentAppFactoryCollectionBuilder.cs +++ b/src/Umbraco.Core/ContentApps/ContentAppFactoryCollectionBuilder.cs @@ -33,7 +33,7 @@ namespace Umbraco.Cms.Core.ContentApps // its dependencies too, and that can create cycles or other oddities var manifestParser = factory.GetRequiredService(); var ioHelper = factory.GetRequiredService(); - return base.CreateItems(factory).Concat(manifestParser.Manifest.ContentApps.Select(x => new ManifestContentAppFactory(x, ioHelper))); + return base.CreateItems(factory).Concat(manifestParser.CombinedManifest.ContentApps.Select(x => new ManifestContentAppFactory(x, ioHelper))); } } } diff --git a/src/Umbraco.Core/Dashboards/DashboardCollectionBuilder.cs b/src/Umbraco.Core/Dashboards/DashboardCollectionBuilder.cs index 55e840ad8e..348e81e383 100644 --- a/src/Umbraco.Core/Dashboards/DashboardCollectionBuilder.cs +++ b/src/Umbraco.Core/Dashboards/DashboardCollectionBuilder.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using Microsoft.Extensions.DependencyInjection; @@ -19,7 +19,7 @@ namespace Umbraco.Cms.Core.Dashboards // its dependencies too, and that can create cycles or other oddities var manifestParser = factory.GetRequiredService(); - var dashboardSections = Merge(base.CreateItems(factory), manifestParser.Manifest.Dashboards); + var dashboardSections = Merge(base.CreateItems(factory), manifestParser.CombinedManifest.Dashboards); return dashboardSections; } diff --git a/src/Umbraco.Core/Manifest/BundleOptions.cs b/src/Umbraco.Core/Manifest/BundleOptions.cs new file mode 100644 index 0000000000..810efb6c45 --- /dev/null +++ b/src/Umbraco.Core/Manifest/BundleOptions.cs @@ -0,0 +1,26 @@ +namespace Umbraco.Cms.Core.Manifest +{ + public enum BundleOptions + { + /// + /// The default bundling behavior for assets in the package folder. + /// + /// + /// The assets will be bundled with the typical packages bundle. + /// + Default = 0, + + /// + /// The assets in the package will not be processed at all and will all be requested as individual assets. + /// + /// + /// This will essentially be a bundle that has composite processing turned off for both debug and production. + /// + None = 1, + + /// + /// The packages assets will be processed as it's own separate bundle. (in debug, files will not be processed) + /// + Independent = 2 + } +} diff --git a/src/Umbraco.Core/Manifest/CompositePackageManifest.cs b/src/Umbraco.Core/Manifest/CompositePackageManifest.cs new file mode 100644 index 0000000000..939d635fc3 --- /dev/null +++ b/src/Umbraco.Core/Manifest/CompositePackageManifest.cs @@ -0,0 +1,67 @@ +using System; +using System.Collections.Generic; +using Umbraco.Cms.Core.PropertyEditors; + +namespace Umbraco.Cms.Core.Manifest +{ + + /// + /// A package manifest made up of all combined manifests + /// + public class CompositePackageManifest + { + public CompositePackageManifest( + IReadOnlyList propertyEditors, + IReadOnlyList parameterEditors, + IReadOnlyList gridEditors, + IReadOnlyList contentApps, + IReadOnlyList dashboards, + IReadOnlyList sections, + IReadOnlyDictionary> scripts, + IReadOnlyDictionary> stylesheets) + { + PropertyEditors = propertyEditors ?? throw new ArgumentNullException(nameof(propertyEditors)); + ParameterEditors = parameterEditors ?? throw new ArgumentNullException(nameof(parameterEditors)); + GridEditors = gridEditors ?? throw new ArgumentNullException(nameof(gridEditors)); + ContentApps = contentApps ?? throw new ArgumentNullException(nameof(contentApps)); + Dashboards = dashboards ?? throw new ArgumentNullException(nameof(dashboards)); + Sections = sections ?? throw new ArgumentNullException(nameof(sections)); + Scripts = scripts ?? throw new ArgumentNullException(nameof(scripts)); + Stylesheets = stylesheets ?? throw new ArgumentNullException(nameof(stylesheets)); + } + + /// + /// Gets or sets the property editors listed in the manifest. + /// + public IReadOnlyList PropertyEditors { get; } + + /// + /// Gets or sets the parameter editors listed in the manifest. + /// + public IReadOnlyList ParameterEditors { get; } + + /// + /// Gets or sets the grid editors listed in the manifest. + /// + public IReadOnlyList GridEditors { get; } + + /// + /// Gets or sets the content apps listed in the manifest. + /// + public IReadOnlyList ContentApps { get; } + + /// + /// Gets or sets the dashboards listed in the manifest. + /// + public IReadOnlyList Dashboards { get; } + + /// + /// Gets or sets the sections listed in the manifest. + /// + public IReadOnlyCollection Sections { get; } + + public IReadOnlyDictionary> Scripts { get; } + + public IReadOnlyDictionary> Stylesheets { get; } + } +} diff --git a/src/Umbraco.Core/Manifest/IManifestParser.cs b/src/Umbraco.Core/Manifest/IManifestParser.cs index dc3a19714e..9e56ed17fc 100644 --- a/src/Umbraco.Core/Manifest/IManifestParser.cs +++ b/src/Umbraco.Core/Manifest/IManifestParser.cs @@ -4,13 +4,13 @@ namespace Umbraco.Cms.Core.Manifest { public interface IManifestParser { - string Path { get; set; } + //string Path { get; set; } /// /// Gets all manifests, merged into a single manifest object. /// /// - PackageManifest Manifest { get; } + CompositePackageManifest CombinedManifest { get; } /// /// Parses a manifest. diff --git a/src/Umbraco.Core/Manifest/ManifestAssets.cs b/src/Umbraco.Core/Manifest/ManifestAssets.cs new file mode 100644 index 0000000000..ad5dfaa0f0 --- /dev/null +++ b/src/Umbraco.Core/Manifest/ManifestAssets.cs @@ -0,0 +1,17 @@ +using System; +using System.Collections.Generic; + +namespace Umbraco.Cms.Core.Manifest +{ + public class ManifestAssets + { + public ManifestAssets(string packageName, IReadOnlyList assets) + { + PackageName = packageName ?? throw new ArgumentNullException(nameof(packageName)); + Assets = assets ?? throw new ArgumentNullException(nameof(assets)); + } + + public string PackageName { get; } + public IReadOnlyList Assets { get; } + } +} diff --git a/src/Umbraco.Core/Manifest/PackageManifest.cs b/src/Umbraco.Core/Manifest/PackageManifest.cs index 56c69ebb15..753ec0613a 100644 --- a/src/Umbraco.Core/Manifest/PackageManifest.cs +++ b/src/Umbraco.Core/Manifest/PackageManifest.cs @@ -6,6 +6,7 @@ using Umbraco.Extensions; namespace Umbraco.Cms.Core.Manifest { + /// /// Represents the content of a package manifest. /// @@ -47,6 +48,8 @@ namespace Umbraco.Cms.Core.Manifest /// [IgnoreDataMember] public string Source { get; set; } + [DataMember(Name = "bundleOptions")] + public BundleOptions BundleOptions { get; set; } /// /// Gets or sets the scripts listed in the manifest. diff --git a/src/Umbraco.Core/PropertyEditors/ParameterEditorCollection.cs b/src/Umbraco.Core/PropertyEditors/ParameterEditorCollection.cs index 039de8cb6a..39647bb753 100644 --- a/src/Umbraco.Core/PropertyEditors/ParameterEditorCollection.cs +++ b/src/Umbraco.Core/PropertyEditors/ParameterEditorCollection.cs @@ -9,7 +9,7 @@ namespace Umbraco.Cms.Core.PropertyEditors public ParameterEditorCollection(DataEditorCollection dataEditors, IManifestParser manifestParser) : base(() => dataEditors .Where(x => (x.Type & EditorType.MacroParameter) > 0) - .Union(manifestParser.Manifest.PropertyEditors)) + .Union(manifestParser.CombinedManifest.PropertyEditors)) { } // note: virtual so it can be mocked diff --git a/src/Umbraco.Core/PropertyEditors/PropertyEditorCollection.cs b/src/Umbraco.Core/PropertyEditors/PropertyEditorCollection.cs index 4b551d3257..1ddf150f93 100644 --- a/src/Umbraco.Core/PropertyEditors/PropertyEditorCollection.cs +++ b/src/Umbraco.Core/PropertyEditors/PropertyEditorCollection.cs @@ -9,7 +9,7 @@ namespace Umbraco.Cms.Core.PropertyEditors public PropertyEditorCollection(DataEditorCollection dataEditors, IManifestParser manifestParser) : base(() => dataEditors .Where(x => (x.Type & EditorType.PropertyValue) > 0) - .Union(manifestParser.Manifest.PropertyEditors)) + .Union(manifestParser.CombinedManifest.PropertyEditors)) { } public PropertyEditorCollection(DataEditorCollection dataEditors) diff --git a/src/Umbraco.Core/Sections/SectionCollectionBuilder.cs b/src/Umbraco.Core/Sections/SectionCollectionBuilder.cs index 0c5b2d7ba9..219d634261 100644 --- a/src/Umbraco.Core/Sections/SectionCollectionBuilder.cs +++ b/src/Umbraco.Core/Sections/SectionCollectionBuilder.cs @@ -18,7 +18,7 @@ namespace Umbraco.Cms.Core.Sections // its dependencies too, and that can create cycles or other oddities var manifestParser = factory.GetRequiredService(); - return base.CreateItems(factory).Concat(manifestParser.Manifest.Sections); + return base.CreateItems(factory).Concat(manifestParser.CombinedManifest.Sections); } } } diff --git a/src/Umbraco.Infrastructure/Manifest/ManifestParser.cs b/src/Umbraco.Infrastructure/Manifest/ManifestParser.cs index 7d98a19091..529a148093 100644 --- a/src/Umbraco.Infrastructure/Manifest/ManifestParser.cs +++ b/src/Umbraco.Infrastructure/Manifest/ManifestParser.cs @@ -62,22 +62,22 @@ namespace Umbraco.Cms.Core.Manifest /// /// Initializes a new instance of the class. /// - private ManifestParser(AppCaches appCaches, ManifestValueValidatorCollection validators, ManifestFilterCollection filters, string path, ILogger logger, IIOHelper ioHelper, IHostingEnvironment hostingEnvironment) + private ManifestParser(AppCaches appCaches, ManifestValueValidatorCollection validators, ManifestFilterCollection filters, string appPluginsPath, ILogger logger, IIOHelper ioHelper, IHostingEnvironment hostingEnvironment) { if (appCaches == null) throw new ArgumentNullException(nameof(appCaches)); _cache = appCaches.RuntimeCache; _validators = validators ?? throw new ArgumentNullException(nameof(validators)); _filters = filters ?? throw new ArgumentNullException(nameof(filters)); - if (path == null) throw new ArgumentNullException(nameof(path)); - if (string.IsNullOrWhiteSpace(path)) throw new ArgumentException("Value can't be empty or consist only of white-space characters.", nameof(path)); + if (appPluginsPath == null) throw new ArgumentNullException(nameof(appPluginsPath)); + if (string.IsNullOrWhiteSpace(appPluginsPath)) throw new ArgumentException("Value can't be empty or consist only of white-space characters.", nameof(appPluginsPath)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _ioHelper = ioHelper; _hostingEnvironment = hostingEnvironment; - Path = path; + AppPluginsPath = appPluginsPath; } - public string Path + public string AppPluginsPath { get => _path; set => _path = value.StartsWith("~/") ? _hostingEnvironment.MapPathContentRoot(value) : value; @@ -87,11 +87,12 @@ namespace Umbraco.Cms.Core.Manifest /// Gets all manifests, merged into a single manifest object. /// /// - public PackageManifest Manifest - => _cache.GetCacheItem("Umbraco.Core.Manifest.ManifestParser::Manifests", () => + public CompositePackageManifest CombinedManifest + => _cache.GetCacheItem("Umbraco.Core.Manifest.ManifestParser::Manifests", () => { - var manifests = GetManifests(); + IEnumerable manifests = GetManifests(); return MergeManifests(manifests); + }, new TimeSpan(0, 4, 0)); /// @@ -130,10 +131,10 @@ namespace Umbraco.Cms.Core.Manifest /// /// Merges all manifests into one. /// - private static PackageManifest MergeManifests(IEnumerable manifests) + private static CompositePackageManifest MergeManifests(IEnumerable manifests) { - var scripts = new HashSet(); - var stylesheets = new HashSet(); + var scripts = new Dictionary>(); + var stylesheets = new Dictionary>(); var propertyEditors = new List(); var parameterEditors = new List(); var gridEditors = new List(); @@ -141,10 +142,28 @@ namespace Umbraco.Cms.Core.Manifest var dashboards = new List(); var sections = new List(); - foreach (var manifest in manifests) + foreach (PackageManifest manifest in manifests) { - if (manifest.Scripts != null) foreach (var script in manifest.Scripts) scripts.Add(script); - if (manifest.Stylesheets != null) foreach (var stylesheet in manifest.Stylesheets) stylesheets.Add(stylesheet); + if (manifest.Scripts != null) + { + if (!scripts.TryGetValue(manifest.BundleOptions, out List scriptsPerBundleOption)) + { + scriptsPerBundleOption = new List(); + scripts[manifest.BundleOptions] = scriptsPerBundleOption; + } + scriptsPerBundleOption.Add(new ManifestAssets(manifest.PackageName, manifest.Scripts)); + } + + if (manifest.Stylesheets != null) + { + if (!stylesheets.TryGetValue(manifest.BundleOptions, out List stylesPerBundleOption)) + { + stylesPerBundleOption = new List(); + stylesheets[manifest.BundleOptions] = stylesPerBundleOption; + } + stylesPerBundleOption.Add(new ManifestAssets(manifest.PackageName, manifest.Stylesheets)); + } + if (manifest.PropertyEditors != null) propertyEditors.AddRange(manifest.PropertyEditors); if (manifest.ParameterEditors != null) parameterEditors.AddRange(manifest.ParameterEditors); if (manifest.GridEditors != null) gridEditors.AddRange(manifest.GridEditors); @@ -153,17 +172,15 @@ namespace Umbraco.Cms.Core.Manifest if (manifest.Sections != null) sections.AddRange(manifest.Sections.DistinctBy(x => x.Alias.ToLowerInvariant())); } - return new PackageManifest - { - Scripts = scripts.ToArray(), - Stylesheets = stylesheets.ToArray(), - PropertyEditors = propertyEditors.ToArray(), - ParameterEditors = parameterEditors.ToArray(), - GridEditors = gridEditors.ToArray(), - ContentApps = contentApps.ToArray(), - Dashboards = dashboards.ToArray(), - Sections = sections.ToArray() - }; + return new CompositePackageManifest( + propertyEditors, + parameterEditors, + gridEditors, + contentApps, + dashboards, + sections, + scripts.ToDictionary(x => x.Key, x => (IReadOnlyList)x.Value), + stylesheets.ToDictionary(x => x.Key, x => (IReadOnlyList)x.Value)); } // gets all manifest files (recursively) diff --git a/src/Umbraco.Infrastructure/WebAssets/BackOfficeWebAssets.cs b/src/Umbraco.Infrastructure/WebAssets/BackOfficeWebAssets.cs index 74668d3090..07de655fb8 100644 --- a/src/Umbraco.Infrastructure/WebAssets/BackOfficeWebAssets.cs +++ b/src/Umbraco.Infrastructure/WebAssets/BackOfficeWebAssets.cs @@ -73,25 +73,66 @@ namespace Umbraco.Cms.Infrastructure.WebAssets _runtimeMinifier.CreateJsBundle(UmbracoCoreJsBundleName, false, FormatPaths(GetScriptsForBackOfficeCore())); + + // get the property editor assets var propertyEditorAssets = ScanPropertyEditors() .GroupBy(x => x.AssetType) .ToDictionary(x => x.Key, x => x.Select(c => c.FilePath)); + // get the back office custom assets var customAssets = _customBackOfficeAssetsCollection.GroupBy(x => x.DependencyType).ToDictionary(x => x.Key, x => x.Select(c => c.FilePath)); - var jsAssets = (customAssets.TryGetValue(AssetType.Javascript, out var customScripts) ? customScripts : Enumerable.Empty()) - .Union(propertyEditorAssets.TryGetValue(AssetType.Javascript, out var scripts) ? scripts : Enumerable.Empty()); + // This bundle includes all scripts from property editor assets, + // custom back office assets, and any scripts found in package manifests + // that have the default bundle options. + + IEnumerable jsAssets = (customAssets.TryGetValue(AssetType.Javascript, out IEnumerable customScripts) ? customScripts : Enumerable.Empty()) + .Union(propertyEditorAssets.TryGetValue(AssetType.Javascript, out IEnumerable scripts) ? scripts : Enumerable.Empty()); + _runtimeMinifier.CreateJsBundle( - UmbracoExtensionsJsBundleName, true, + UmbracoExtensionsJsBundleName, + true, FormatPaths( GetScriptsForBackOfficeExtensions(jsAssets))); - var cssAssets = (customAssets.TryGetValue(AssetType.Css, out var customStyles) ? customStyles : Enumerable.Empty()) - .Union(propertyEditorAssets.TryGetValue(AssetType.Css, out var styles) ? styles : Enumerable.Empty()); + // Create a bundle per package manifest that is declaring an Independent bundle type + RegisterPackageBundlesForIndependentOptions(_parser.CombinedManifest.Scripts, AssetType.Javascript); + + // This bundle includes all CSS from property editor assets, + // custom back office assets, and any CSS found in package manifests + // that have the default bundle options. + + IEnumerable cssAssets = (customAssets.TryGetValue(AssetType.Css, out IEnumerable customStyles) ? customStyles : Enumerable.Empty()) + .Union(propertyEditorAssets.TryGetValue(AssetType.Css, out IEnumerable styles) ? styles : Enumerable.Empty()); + _runtimeMinifier.CreateCssBundle( - UmbracoCssBundleName, true, + UmbracoCssBundleName, + true, FormatPaths( GetStylesheetsForBackOffice(cssAssets))); + + // Create a bundle per package manifest that is declaring an Independent bundle type + RegisterPackageBundlesForIndependentOptions(_parser.CombinedManifest.Stylesheets, AssetType.Css); + } + + public static string GetIndependentPackageBundleName(ManifestAssets manifestAssets, AssetType assetType) + => $"{manifestAssets.PackageName.ToLowerInvariant()}-{(assetType == AssetType.Css ? "css" : "js")}"; + + private void RegisterPackageBundlesForIndependentOptions( + IReadOnlyDictionary> combinedPackageManifestAssets, + AssetType assetType) + { + // Create a bundle per package manifest that is declaring the matching BundleOptions + if (combinedPackageManifestAssets.TryGetValue(BundleOptions.Independent, out IReadOnlyList manifestAssetList)) + { + foreach (ManifestAssets manifestAssets in manifestAssetList) + { + _runtimeMinifier.CreateJsBundle( + GetIndependentPackageBundleName(manifestAssets, assetType), + true, + FormatPaths(manifestAssets.Assets.ToArray())); + } + } } /// @@ -100,10 +141,15 @@ namespace Umbraco.Cms.Infrastructure.WebAssets /// private string[] GetScriptsForBackOfficeExtensions(IEnumerable propertyEditorScripts) { - var scripts = new HashSet(); - foreach (string script in _parser.Manifest.Scripts) + var scripts = new HashSet(StringComparer.InvariantCultureIgnoreCase); + + // only include scripts with the default bundle options here + if (_parser.CombinedManifest.Scripts.TryGetValue(BundleOptions.Default, out IReadOnlyList manifestAssets)) { - scripts.Add(script); + foreach (string script in manifestAssets.SelectMany(x => x.Assets)) + { + scripts.Add(script); + } } foreach (string script in propertyEditorScripts) @@ -130,11 +176,15 @@ namespace Umbraco.Cms.Infrastructure.WebAssets /// private string[] GetStylesheetsForBackOffice(IEnumerable propertyEditorStyles) { - var stylesheets = new HashSet(); + var stylesheets = new HashSet(StringComparer.InvariantCultureIgnoreCase); - foreach (string script in _parser.Manifest.Stylesheets) + // only include css with the default bundle options here + if (_parser.CombinedManifest.Stylesheets.TryGetValue(BundleOptions.Default, out IReadOnlyList manifestAssets)) { - stylesheets.Add(script); + foreach (string script in manifestAssets.SelectMany(x => x.Assets)) + { + stylesheets.Add(script); + } } foreach (string stylesheet in propertyEditorStyles) diff --git a/src/Umbraco.Infrastructure/WebAssets/RuntimeMinifierExtensions.cs b/src/Umbraco.Infrastructure/WebAssets/RuntimeMinifierExtensions.cs deleted file mode 100644 index f33d48d8fd..0000000000 --- a/src/Umbraco.Infrastructure/WebAssets/RuntimeMinifierExtensions.cs +++ /dev/null @@ -1,41 +0,0 @@ -using System; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Umbraco.Cms.Core.Configuration.Models; -using Umbraco.Cms.Core.Hosting; -using Umbraco.Cms.Core.WebAssets; -using Umbraco.Cms.Infrastructure.WebAssets; - -namespace Umbraco.Extensions -{ - public static class RuntimeMinifierExtensions - { - /// - /// Returns the JavaScript to load the back office's assets - /// - /// - public static async Task GetScriptForLoadingBackOfficeAsync(this IRuntimeMinifier minifier, GlobalSettings globalSettings, IHostingEnvironment hostingEnvironment) - { - var coreScripts = await minifier.GetJsAssetPathsAsync(BackOfficeWebAssets.UmbracoCoreJsBundleName); - var extensionsScripts = await minifier.GetJsAssetPathsAsync(BackOfficeWebAssets.UmbracoExtensionsJsBundleName); - var result = BackOfficeJavaScriptInitializer.GetJavascriptInitialization(coreScripts.Union(extensionsScripts), "umbraco", globalSettings, hostingEnvironment); - result += await GetStylesheetInitializationAsync(minifier); - - return result; - } - - /// - /// Gets the back office css bundle paths and formats a JS call to lazy load them - /// - private static async Task GetStylesheetInitializationAsync(IRuntimeMinifier minifier) - { - var files = await minifier.GetCssAssetPathsAsync(BackOfficeWebAssets.UmbracoCssBundleName); - var sb = new StringBuilder(); - foreach (var file in files) - sb.AppendFormat("{0}LazyLoad.css('{1}');", Environment.NewLine, file); - return sb.ToString(); - } - - } -} diff --git a/src/Umbraco.Web.BackOffice/Controllers/BackOfficeController.cs b/src/Umbraco.Web.BackOffice/Controllers/BackOfficeController.cs index 161896628b..ca1aea69a7 100644 --- a/src/Umbraco.Web.BackOffice/Controllers/BackOfficeController.cs +++ b/src/Umbraco.Web.BackOffice/Controllers/BackOfficeController.cs @@ -17,6 +17,7 @@ using Umbraco.Cms.Core.Cache; using Umbraco.Cms.Core.Configuration.Grid; using Umbraco.Cms.Core.Configuration.Models; using Umbraco.Cms.Core.Hosting; +using Umbraco.Cms.Core.Manifest; using Umbraco.Cms.Core.Models; using Umbraco.Cms.Core.Security; using Umbraco.Cms.Core.Serialization; @@ -63,6 +64,7 @@ namespace Umbraco.Cms.Web.BackOffice.Controllers private readonly IBackOfficeExternalLoginProviders _externalLogins; private readonly IHttpContextAccessor _httpContextAccessor; private readonly IBackOfficeTwoFactorOptions _backOfficeTwoFactorOptions; + private readonly IManifestParser _manifestParser; private readonly ServerVariablesParser _serverVariables; public BackOfficeController( @@ -81,6 +83,7 @@ namespace Umbraco.Cms.Web.BackOffice.Controllers IBackOfficeExternalLoginProviders externalLogins, IHttpContextAccessor httpContextAccessor, IBackOfficeTwoFactorOptions backOfficeTwoFactorOptions, + IManifestParser manifestParser, ServerVariablesParser serverVariables) { _userManager = userManager; @@ -98,6 +101,7 @@ namespace Umbraco.Cms.Web.BackOffice.Controllers _externalLogins = externalLogins; _httpContextAccessor = httpContextAccessor; _backOfficeTwoFactorOptions = backOfficeTwoFactorOptions; + _manifestParser = manifestParser; _serverVariables = serverVariables; } @@ -213,7 +217,11 @@ namespace Umbraco.Cms.Web.BackOffice.Controllers [AllowAnonymous] public async Task Application() { - var result = await _runtimeMinifier.GetScriptForLoadingBackOfficeAsync(_globalSettings, _hostingEnvironment); + var result = await _runtimeMinifier.GetScriptForLoadingBackOfficeAsync( + _globalSettings, + _hostingEnvironment, + _manifestParser, + Url); return new JavaScriptResult(result); } diff --git a/src/Umbraco.Web.BackOffice/Extensions/RuntimeMinifierExtensions.cs b/src/Umbraco.Web.BackOffice/Extensions/RuntimeMinifierExtensions.cs new file mode 100644 index 0000000000..e436a57d83 --- /dev/null +++ b/src/Umbraco.Web.BackOffice/Extensions/RuntimeMinifierExtensions.cs @@ -0,0 +1,124 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Mvc; +using Umbraco.Cms.Core.Configuration.Models; +using Umbraco.Cms.Core.Hosting; +using Umbraco.Cms.Core.Manifest; +using Umbraco.Cms.Core.WebAssets; +using Umbraco.Cms.Infrastructure.WebAssets; + +namespace Umbraco.Extensions +{ + public static class RuntimeMinifierExtensions + { + /// + /// Returns the JavaScript to load the back office's assets + /// + /// + public static async Task GetScriptForLoadingBackOfficeAsync( + this IRuntimeMinifier minifier, + GlobalSettings globalSettings, + IHostingEnvironment hostingEnvironment, + IManifestParser manifestParser, + IUrlHelper urlHelper) + { + var files = new HashSet(StringComparer.InvariantCultureIgnoreCase); + foreach(var file in await minifier.GetJsAssetPathsAsync(BackOfficeWebAssets.UmbracoCoreJsBundleName)) + { + files.Add(file); + } + + foreach (var file in await minifier.GetJsAssetPathsAsync(BackOfficeWebAssets.UmbracoExtensionsJsBundleName)) + { + files.Add(file); + } + + // process the independent bundles + if (manifestParser.CombinedManifest.Scripts.TryGetValue(BundleOptions.Independent, out IReadOnlyList independentManifestAssetsList)) + { + foreach (ManifestAssets manifestAssets in independentManifestAssetsList) + { + var bundleName = BackOfficeWebAssets.GetIndependentPackageBundleName(manifestAssets, AssetType.Javascript); + foreach(var asset in await minifier.GetJsAssetPathsAsync(bundleName)) + { + files.Add(asset); + } + } + } + + // process the "None" bundles, meaning we'll just render the script as-is + if (manifestParser.CombinedManifest.Scripts.TryGetValue(BundleOptions.None, out IReadOnlyList noneManifestAssetsList)) + { + foreach (ManifestAssets manifestAssets in noneManifestAssetsList) + { + foreach(var asset in manifestAssets.Assets) + { + files.Add(urlHelper.Content(asset)); + } + } + } + + var result = BackOfficeJavaScriptInitializer.GetJavascriptInitialization( + files, + "umbraco", + globalSettings, + hostingEnvironment); + + result += await GetStylesheetInitializationAsync(minifier, manifestParser, urlHelper); + + return result; + } + + /// + /// Gets the back office css bundle paths and formats a JS call to lazy load them + /// + private static async Task GetStylesheetInitializationAsync( + IRuntimeMinifier minifier, + IManifestParser manifestParser, + IUrlHelper urlHelper) + { + var files = new HashSet(StringComparer.InvariantCultureIgnoreCase); + foreach(var file in await minifier.GetCssAssetPathsAsync(BackOfficeWebAssets.UmbracoCssBundleName)) + { + files.Add(file); + } + + // process the independent bundles + if (manifestParser.CombinedManifest.Stylesheets.TryGetValue(BundleOptions.Independent, out IReadOnlyList independentManifestAssetsList)) + { + foreach (ManifestAssets manifestAssets in independentManifestAssetsList) + { + var bundleName = BackOfficeWebAssets.GetIndependentPackageBundleName(manifestAssets, AssetType.Css); + foreach (var asset in await minifier.GetCssAssetPathsAsync(bundleName)) + { + files.Add(asset); + } + } + } + + // process the "None" bundles, meaning we'll just render the script as-is + if (manifestParser.CombinedManifest.Stylesheets.TryGetValue(BundleOptions.None, out IReadOnlyList noneManifestAssetsList)) + { + foreach (ManifestAssets manifestAssets in noneManifestAssetsList) + { + foreach (var asset in manifestAssets.Assets) + { + files.Add(urlHelper.Content(asset)); + } + } + } + + var sb = new StringBuilder(); + foreach (string file in files) + { + sb.AppendFormat("{0}LazyLoad.css('{1}');", Environment.NewLine, file); + } + + return sb.ToString(); + } + + } +} diff --git a/src/Umbraco.Web.Common/RuntimeMinification/SmidgeRuntimeMinifier.cs b/src/Umbraco.Web.Common/RuntimeMinification/SmidgeRuntimeMinifier.cs index 4353b2ee1a..c202c49980 100644 --- a/src/Umbraco.Web.Common/RuntimeMinification/SmidgeRuntimeMinifier.cs +++ b/src/Umbraco.Web.Common/RuntimeMinification/SmidgeRuntimeMinifier.cs @@ -60,9 +60,21 @@ namespace Umbraco.Cms.Web.Common.RuntimeMinification // replace the default JsMinifier with NuglifyJs and CssMinifier with NuglifyCss in the default pipelines // for use with our bundles only (not modifying global options) _jsOptimizedPipeline = new Lazy(() => bundles.PipelineFactory.DefaultJs().Replace(_bundles.PipelineFactory)); - _jsNonOptimizedPipeline = new Lazy(() => bundles.PipelineFactory.DefaultJs()); + _jsNonOptimizedPipeline = new Lazy(() => + { + PreProcessPipeline defaultJs = bundles.PipelineFactory.DefaultJs(); + // remove minification from this pipeline + defaultJs.Processors.RemoveAll(x => x is JsMinifier); + return defaultJs; + }); _cssOptimizedPipeline = new Lazy(() => bundles.PipelineFactory.DefaultCss().Replace(_bundles.PipelineFactory)); - _cssNonOptimizedPipeline = new Lazy(() => bundles.PipelineFactory.DefaultCss()); + _cssNonOptimizedPipeline = new Lazy(() => + { + PreProcessPipeline defaultCss = bundles.PipelineFactory.DefaultCss(); + // remove minification from this pipeline + defaultCss.Processors.RemoveAll(x => x is CssMinifier); + return defaultCss; + }); Type cacheBusterType = _runtimeMinificationSettings.CacheBuster switch { From b16bf2f58379feac0ba7b13cefd9ad18190dc315 Mon Sep 17 00:00:00 2001 From: Shannon Date: Thu, 15 Jul 2021 14:06:33 -0600 Subject: [PATCH 035/123] Adds BundlingOptions for better bundling support and moves all logic to the bundler (don't manually process urls). --- src/Umbraco.Core/WebAssets/BundlingOptions.cs | 44 ++++++++++ .../WebAssets/IRuntimeMinifier.cs | 4 +- .../WebAssets/BackOfficeWebAssets.cs | 54 ++++++++++-- .../Controllers/BackOfficeController.cs | 3 +- .../Extensions/RuntimeMinifierExtensions.cs | 28 ++---- .../SmidgeRuntimeMinifier.cs | 88 ++++++------------- 6 files changed, 128 insertions(+), 93 deletions(-) create mode 100644 src/Umbraco.Core/WebAssets/BundlingOptions.cs diff --git a/src/Umbraco.Core/WebAssets/BundlingOptions.cs b/src/Umbraco.Core/WebAssets/BundlingOptions.cs new file mode 100644 index 0000000000..e9234b06a4 --- /dev/null +++ b/src/Umbraco.Core/WebAssets/BundlingOptions.cs @@ -0,0 +1,44 @@ +using System; + +namespace Umbraco.Cms.Core.WebAssets +{ + public struct BundlingOptions : IEquatable + { + public static BundlingOptions OptimizedAndComposite => new BundlingOptions(true, true); + public static BundlingOptions OptimizedNotComposite => new BundlingOptions(true, false); + public static BundlingOptions NotOptimizedNotComposite => new BundlingOptions(false, false); + public static BundlingOptions NotOptimizedAndComposite => new BundlingOptions(false, false); + + public BundlingOptions(bool optimizeOutput = true, bool enabledCompositeFiles = true) + { + OptimizeOutput = optimizeOutput; + EnabledCompositeFiles = enabledCompositeFiles; + } + + /// + /// If true, the files in the bundle will be minified + /// + public bool OptimizeOutput { get; } + + /// + /// If true, the files in the bundle will be combined, if false the files + /// will be served as individual files. + /// + public bool EnabledCompositeFiles { get; } + + public override bool Equals(object obj) => obj is BundlingOptions options && Equals(options); + public bool Equals(BundlingOptions other) => OptimizeOutput == other.OptimizeOutput && EnabledCompositeFiles == other.EnabledCompositeFiles; + + public override int GetHashCode() + { + int hashCode = 2130304063; + hashCode = hashCode * -1521134295 + OptimizeOutput.GetHashCode(); + hashCode = hashCode * -1521134295 + EnabledCompositeFiles.GetHashCode(); + return hashCode; + } + + public static bool operator ==(BundlingOptions left, BundlingOptions right) => left.Equals(right); + + public static bool operator !=(BundlingOptions left, BundlingOptions right) => !(left == right); + } +} diff --git a/src/Umbraco.Core/WebAssets/IRuntimeMinifier.cs b/src/Umbraco.Core/WebAssets/IRuntimeMinifier.cs index f287c722ad..7ca1cd883b 100644 --- a/src/Umbraco.Core/WebAssets/IRuntimeMinifier.cs +++ b/src/Umbraco.Core/WebAssets/IRuntimeMinifier.cs @@ -25,7 +25,7 @@ namespace Umbraco.Cms.Core.WebAssets /// /// Thrown if any of the paths specified are not absolute /// - void CreateCssBundle(string bundleName, bool optimizeOutput, params string[] filePaths); + void CreateCssBundle(string bundleName, BundlingOptions bundleOptions, params string[] filePaths); /// /// Renders the html link tag for the bundle @@ -48,7 +48,7 @@ namespace Umbraco.Cms.Core.WebAssets /// /// Thrown if any of the paths specified are not absolute /// - void CreateJsBundle(string bundleName, bool optimizeOutput, params string[] filePaths); + void CreateJsBundle(string bundleName, BundlingOptions bundleOptions, params string[] filePaths); /// /// Renders the html script tag for the bundle diff --git a/src/Umbraco.Infrastructure/WebAssets/BackOfficeWebAssets.cs b/src/Umbraco.Infrastructure/WebAssets/BackOfficeWebAssets.cs index 07de655fb8..8944691404 100644 --- a/src/Umbraco.Infrastructure/WebAssets/BackOfficeWebAssets.cs +++ b/src/Umbraco.Infrastructure/WebAssets/BackOfficeWebAssets.cs @@ -21,6 +21,8 @@ namespace Umbraco.Cms.Infrastructure.WebAssets public const string UmbracoInitCssBundleName = "umbraco-backoffice-init-css"; public const string UmbracoCoreJsBundleName = "umbraco-backoffice-js"; public const string UmbracoExtensionsJsBundleName = "umbraco-backoffice-extensions-js"; + public const string UmbracoNonOptimizedPackageJsBundleName = "umbraco-backoffice-non-optimized-js"; + public const string UmbracoNonOptimizedPackageCssBundleName = "umbraco-backoffice-non-optimized-css"; public const string UmbracoTinyMceJsBundleName = "umbraco-tinymce-js"; public const string UmbracoUpgradeCssBundleName = "umbraco-authorize-upgrade-css"; @@ -51,26 +53,32 @@ namespace Umbraco.Cms.Infrastructure.WebAssets { // Create bundles - _runtimeMinifier.CreateCssBundle(UmbracoInitCssBundleName, false, + _runtimeMinifier.CreateCssBundle(UmbracoInitCssBundleName, + BundlingOptions.NotOptimizedAndComposite, FormatPaths("lib/bootstrap-social/bootstrap-social.css", "assets/css/umbraco.min.css", "lib/font-awesome/css/font-awesome.min.css")); - _runtimeMinifier.CreateCssBundle(UmbracoUpgradeCssBundleName, false, + _runtimeMinifier.CreateCssBundle(UmbracoUpgradeCssBundleName, + BundlingOptions.NotOptimizedAndComposite, FormatPaths("assets/css/umbraco.min.css", "lib/bootstrap-social/bootstrap-social.css", "lib/font-awesome/css/font-awesome.min.css")); - _runtimeMinifier.CreateCssBundle(UmbracoPreviewCssBundleName, false, + _runtimeMinifier.CreateCssBundle(UmbracoPreviewCssBundleName, + BundlingOptions.NotOptimizedAndComposite, FormatPaths("assets/css/canvasdesigner.min.css")); - _runtimeMinifier.CreateJsBundle(UmbracoPreviewJsBundleName, false, + _runtimeMinifier.CreateJsBundle(UmbracoPreviewJsBundleName, + BundlingOptions.NotOptimizedAndComposite, FormatPaths(GetScriptsForPreview())); - _runtimeMinifier.CreateJsBundle(UmbracoTinyMceJsBundleName, false, + _runtimeMinifier.CreateJsBundle(UmbracoTinyMceJsBundleName, + BundlingOptions.NotOptimizedAndComposite, FormatPaths(GetScriptsForTinyMce())); - _runtimeMinifier.CreateJsBundle(UmbracoCoreJsBundleName, false, + _runtimeMinifier.CreateJsBundle(UmbracoCoreJsBundleName, + BundlingOptions.NotOptimizedAndComposite, FormatPaths(GetScriptsForBackOfficeCore())); @@ -91,13 +99,16 @@ namespace Umbraco.Cms.Infrastructure.WebAssets _runtimeMinifier.CreateJsBundle( UmbracoExtensionsJsBundleName, - true, + BundlingOptions.OptimizedAndComposite, FormatPaths( GetScriptsForBackOfficeExtensions(jsAssets))); // Create a bundle per package manifest that is declaring an Independent bundle type RegisterPackageBundlesForIndependentOptions(_parser.CombinedManifest.Scripts, AssetType.Javascript); + // Create a single non-optimized (no file processing) bundle for all manifests declaring None as a bundle option + RegisterPackageBundlesForNoneOption(_parser.CombinedManifest.Scripts, UmbracoNonOptimizedPackageJsBundleName); + // This bundle includes all CSS from property editor assets, // custom back office assets, and any CSS found in package manifests // that have the default bundle options. @@ -107,17 +118,42 @@ namespace Umbraco.Cms.Infrastructure.WebAssets _runtimeMinifier.CreateCssBundle( UmbracoCssBundleName, - true, + BundlingOptions.OptimizedAndComposite, FormatPaths( GetStylesheetsForBackOffice(cssAssets))); // Create a bundle per package manifest that is declaring an Independent bundle type RegisterPackageBundlesForIndependentOptions(_parser.CombinedManifest.Stylesheets, AssetType.Css); + + // Create a single non-optimized (no file processing) bundle for all manifests declaring None as a bundle option + RegisterPackageBundlesForNoneOption(_parser.CombinedManifest.Stylesheets, UmbracoNonOptimizedPackageCssBundleName); } public static string GetIndependentPackageBundleName(ManifestAssets manifestAssets, AssetType assetType) => $"{manifestAssets.PackageName.ToLowerInvariant()}-{(assetType == AssetType.Css ? "css" : "js")}"; + private void RegisterPackageBundlesForNoneOption( + IReadOnlyDictionary> combinedPackageManifestAssets, + string bundleName) + { + var assets = new HashSet(StringComparer.InvariantCultureIgnoreCase); + + // Create a bundle per package manifest that is declaring the matching BundleOptions + if (combinedPackageManifestAssets.TryGetValue(BundleOptions.None, out IReadOnlyList manifestAssetList)) + { + foreach(var asset in manifestAssetList.SelectMany(x => x.Assets)) + { + assets.Add(asset); + } + } + + _runtimeMinifier.CreateJsBundle( + bundleName, + // no optimization, no composite files, just render individual files + BundlingOptions.NotOptimizedNotComposite, + FormatPaths(assets.ToArray())); + } + private void RegisterPackageBundlesForIndependentOptions( IReadOnlyDictionary> combinedPackageManifestAssets, AssetType assetType) @@ -129,7 +165,7 @@ namespace Umbraco.Cms.Infrastructure.WebAssets { _runtimeMinifier.CreateJsBundle( GetIndependentPackageBundleName(manifestAssets, assetType), - true, + BundlingOptions.OptimizedAndComposite, FormatPaths(manifestAssets.Assets.ToArray())); } } diff --git a/src/Umbraco.Web.BackOffice/Controllers/BackOfficeController.cs b/src/Umbraco.Web.BackOffice/Controllers/BackOfficeController.cs index ca1aea69a7..67dfaa29fe 100644 --- a/src/Umbraco.Web.BackOffice/Controllers/BackOfficeController.cs +++ b/src/Umbraco.Web.BackOffice/Controllers/BackOfficeController.cs @@ -220,8 +220,7 @@ namespace Umbraco.Cms.Web.BackOffice.Controllers var result = await _runtimeMinifier.GetScriptForLoadingBackOfficeAsync( _globalSettings, _hostingEnvironment, - _manifestParser, - Url); + _manifestParser); return new JavaScriptResult(result); } diff --git a/src/Umbraco.Web.BackOffice/Extensions/RuntimeMinifierExtensions.cs b/src/Umbraco.Web.BackOffice/Extensions/RuntimeMinifierExtensions.cs index e436a57d83..0be88c9d2a 100644 --- a/src/Umbraco.Web.BackOffice/Extensions/RuntimeMinifierExtensions.cs +++ b/src/Umbraco.Web.BackOffice/Extensions/RuntimeMinifierExtensions.cs @@ -22,8 +22,7 @@ namespace Umbraco.Extensions this IRuntimeMinifier minifier, GlobalSettings globalSettings, IHostingEnvironment hostingEnvironment, - IManifestParser manifestParser, - IUrlHelper urlHelper) + IManifestParser manifestParser) { var files = new HashSet(StringComparer.InvariantCultureIgnoreCase); foreach(var file in await minifier.GetJsAssetPathsAsync(BackOfficeWebAssets.UmbracoCoreJsBundleName)) @@ -50,15 +49,9 @@ namespace Umbraco.Extensions } // process the "None" bundles, meaning we'll just render the script as-is - if (manifestParser.CombinedManifest.Scripts.TryGetValue(BundleOptions.None, out IReadOnlyList noneManifestAssetsList)) + foreach (var asset in await minifier.GetJsAssetPathsAsync(BackOfficeWebAssets.UmbracoNonOptimizedPackageJsBundleName)) { - foreach (ManifestAssets manifestAssets in noneManifestAssetsList) - { - foreach(var asset in manifestAssets.Assets) - { - files.Add(urlHelper.Content(asset)); - } - } + files.Add(asset); } var result = BackOfficeJavaScriptInitializer.GetJavascriptInitialization( @@ -67,7 +60,7 @@ namespace Umbraco.Extensions globalSettings, hostingEnvironment); - result += await GetStylesheetInitializationAsync(minifier, manifestParser, urlHelper); + result += await GetStylesheetInitializationAsync(minifier, manifestParser); return result; } @@ -77,8 +70,7 @@ namespace Umbraco.Extensions /// private static async Task GetStylesheetInitializationAsync( IRuntimeMinifier minifier, - IManifestParser manifestParser, - IUrlHelper urlHelper) + IManifestParser manifestParser) { var files = new HashSet(StringComparer.InvariantCultureIgnoreCase); foreach(var file in await minifier.GetCssAssetPathsAsync(BackOfficeWebAssets.UmbracoCssBundleName)) @@ -100,15 +92,9 @@ namespace Umbraco.Extensions } // process the "None" bundles, meaning we'll just render the script as-is - if (manifestParser.CombinedManifest.Stylesheets.TryGetValue(BundleOptions.None, out IReadOnlyList noneManifestAssetsList)) + foreach (var asset in await minifier.GetCssAssetPathsAsync(BackOfficeWebAssets.UmbracoNonOptimizedPackageCssBundleName)) { - foreach (ManifestAssets manifestAssets in noneManifestAssetsList) - { - foreach (var asset in manifestAssets.Assets) - { - files.Add(urlHelper.Content(asset)); - } - } + files.Add(asset); } var sb = new StringBuilder(); diff --git a/src/Umbraco.Web.Common/RuntimeMinification/SmidgeRuntimeMinifier.cs b/src/Umbraco.Web.Common/RuntimeMinification/SmidgeRuntimeMinifier.cs index c202c49980..96188ba08c 100644 --- a/src/Umbraco.Web.Common/RuntimeMinification/SmidgeRuntimeMinifier.cs +++ b/src/Umbraco.Web.Common/RuntimeMinification/SmidgeRuntimeMinifier.cs @@ -90,7 +90,7 @@ namespace Umbraco.Cms.Web.Common.RuntimeMinification public string CacheBuster => (_cacheBuster ??= _cacheBusterResolver.GetCacheBuster(_cacheBusterType)).GetValue(); // only issue with creating bundles like this is that we don't have full control over the bundle options, though that could - public void CreateCssBundle(string bundleName, bool optimizeOutput, params string[] filePaths) + public void CreateCssBundle(string bundleName, BundlingOptions bundleOptions, params string[] filePaths) { if (filePaths.Any(f => !f.StartsWith("/") && !f.StartsWith("~/"))) { @@ -102,39 +102,17 @@ namespace Umbraco.Cms.Web.Common.RuntimeMinification throw new InvalidOperationException($"The bundle name {bundleName} already exists"); } - if (optimizeOutput) - { - var bundle = _bundles.Create(bundleName, _cssOptimizedPipeline.Value, WebFileType.Css, filePaths) - .WithEnvironmentOptions( - BundleEnvironmentOptions.Create() - .ForDebug(builder => builder - // auto-invalidate bundle if files change in debug - .EnableFileWatcher() - // use the cache buster defined in config - .SetCacheBusterType(_cacheBusterType)) - .ForProduction(builder => builder - // use the cache buster defined in config - .SetCacheBusterType(_cacheBusterType)) - .Build()); - } - else - { - var bundle = _bundles.Create(bundleName, _cssNonOptimizedPipeline.Value, WebFileType.Css, filePaths) - .WithEnvironmentOptions( - BundleEnvironmentOptions.Create() - .ForDebug(builder => builder - // use the cache buster defined in config - .SetCacheBusterType(_cacheBusterType)) - .ForProduction(builder => builder - // use the cache buster defined in config - .SetCacheBusterType(_cacheBusterType)) - .Build()); - } + PreProcessPipeline pipeline = bundleOptions.OptimizeOutput + ? _cssOptimizedPipeline.Value + : _cssNonOptimizedPipeline.Value; + + Bundle bundle = _bundles.Create(bundleName, pipeline, WebFileType.Css, filePaths); + bundle.WithEnvironmentOptions(ConfigureBundleEnvironmentOptions(bundleOptions)); } public async Task RenderCssHereAsync(string bundleName) => (await _smidge.SmidgeHelper.CssHereAsync(bundleName, _hostingEnvironment.IsDebugMode)).ToString(); - public void CreateJsBundle(string bundleName, bool optimizeOutput, params string[] filePaths) + public void CreateJsBundle(string bundleName, BundlingOptions bundleOptions, params string[] filePaths) { if (filePaths.Any(f => !f.StartsWith("/") && !f.StartsWith("~/"))) { @@ -146,40 +124,32 @@ namespace Umbraco.Cms.Web.Common.RuntimeMinification throw new InvalidOperationException($"The bundle name {bundleName} already exists"); } - if (optimizeOutput) - { - var bundle = _bundles.Create(bundleName, _jsOptimizedPipeline.Value, WebFileType.Js, filePaths) - .WithEnvironmentOptions( - BundleEnvironmentOptions.Create() - .ForDebug(builder => builder - // auto-invalidate bundle if files change in debug - .EnableFileWatcher() - // use the cache buster defined in config - .SetCacheBusterType(_cacheBusterType)) - .ForProduction(builder => builder - // use the cache buster defined in config - .SetCacheBusterType(_cacheBusterType)) - .Build()); - } - else - { - var bundle = _bundles.Create(bundleName, _jsNonOptimizedPipeline.Value, WebFileType.Js, filePaths) - .WithEnvironmentOptions( - BundleEnvironmentOptions.Create() - .ForDebug(builder => builder - // use the cache buster defined in config - .SetCacheBusterType(_cacheBusterType)) - .ForProduction(builder => builder - // use the cache buster defined in config - .SetCacheBusterType(_cacheBusterType)) - .Build()); - } + PreProcessPipeline pipeline = bundleOptions.OptimizeOutput + ? _jsOptimizedPipeline.Value + : _jsNonOptimizedPipeline.Value; + + Bundle bundle = _bundles.Create(bundleName, pipeline, WebFileType.Js, filePaths); + bundle.WithEnvironmentOptions(ConfigureBundleEnvironmentOptions(bundleOptions)); + } + + private BundleEnvironmentOptions ConfigureBundleEnvironmentOptions(BundlingOptions bundleOptions) + { + var bundleEnvironmentOptions = new BundleEnvironmentOptions(); + // auto-invalidate bundle if files change in debug + bundleEnvironmentOptions.DebugOptions.FileWatchOptions.Enabled = true; + // set cache busters + bundleEnvironmentOptions.DebugOptions.SetCacheBusterType(_cacheBusterType); + bundleEnvironmentOptions.ProductionOptions.SetCacheBusterType(_cacheBusterType); + // config if the files should be combined + bundleEnvironmentOptions.ProductionOptions.ProcessAsCompositeFile = bundleOptions.EnabledCompositeFiles; + + return bundleEnvironmentOptions; } public async Task RenderJsHereAsync(string bundleName) => (await _smidge.SmidgeHelper.JsHereAsync(bundleName, _hostingEnvironment.IsDebugMode)).ToString(); - public async Task> GetJsAssetPathsAsync(string bundleName) => await _smidge.SmidgeHelper.GenerateJsUrlsAsync(bundleName, _hostingEnvironment.IsDebugMode); + public async Task> GetCssAssetPathsAsync(string bundleName) => await _smidge.SmidgeHelper.GenerateCssUrlsAsync(bundleName, _hostingEnvironment.IsDebugMode); /// From c2d51f8d37e00e3966fa543dc3021f008d4eabc8 Mon Sep 17 00:00:00 2001 From: Shannon Date: Fri, 16 Jul 2021 10:13:23 -0600 Subject: [PATCH 036/123] re-adds AppPluginsPath --- src/Umbraco.Core/Manifest/IManifestParser.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Umbraco.Core/Manifest/IManifestParser.cs b/src/Umbraco.Core/Manifest/IManifestParser.cs index 9e56ed17fc..09d3ccbe1c 100644 --- a/src/Umbraco.Core/Manifest/IManifestParser.cs +++ b/src/Umbraco.Core/Manifest/IManifestParser.cs @@ -4,7 +4,7 @@ namespace Umbraco.Cms.Core.Manifest { public interface IManifestParser { - //string Path { get; set; } + string AppPluginsPath { get; set; } /// /// Gets all manifests, merged into a single manifest object. From 37571700433256d480106f7627450f19003ef87d Mon Sep 17 00:00:00 2001 From: Shannon Deminick Date: Sat, 17 Jul 2021 02:13:56 +1000 Subject: [PATCH 037/123] Update src/Umbraco.Core/WebAssets/BundlingOptions.cs Co-authored-by: Mole --- src/Umbraco.Core/WebAssets/BundlingOptions.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Umbraco.Core/WebAssets/BundlingOptions.cs b/src/Umbraco.Core/WebAssets/BundlingOptions.cs index e9234b06a4..6a3c0b9bd1 100644 --- a/src/Umbraco.Core/WebAssets/BundlingOptions.cs +++ b/src/Umbraco.Core/WebAssets/BundlingOptions.cs @@ -7,7 +7,7 @@ namespace Umbraco.Cms.Core.WebAssets public static BundlingOptions OptimizedAndComposite => new BundlingOptions(true, true); public static BundlingOptions OptimizedNotComposite => new BundlingOptions(true, false); public static BundlingOptions NotOptimizedNotComposite => new BundlingOptions(false, false); - public static BundlingOptions NotOptimizedAndComposite => new BundlingOptions(false, false); + public static BundlingOptions NotOptimizedAndComposite => new BundlingOptions(false, true); public BundlingOptions(bool optimizeOutput = true, bool enabledCompositeFiles = true) { From 6118091da25a2a6c26f4c0aed2b97c6072840c7b Mon Sep 17 00:00:00 2001 From: Shannon Date: Mon, 19 Jul 2021 16:34:32 -0600 Subject: [PATCH 038/123] fixes css independent registration. --- .../WebAssets/BackOfficeWebAssets.cs | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/Umbraco.Infrastructure/WebAssets/BackOfficeWebAssets.cs b/src/Umbraco.Infrastructure/WebAssets/BackOfficeWebAssets.cs index 8944691404..00cbdce532 100644 --- a/src/Umbraco.Infrastructure/WebAssets/BackOfficeWebAssets.cs +++ b/src/Umbraco.Infrastructure/WebAssets/BackOfficeWebAssets.cs @@ -163,10 +163,20 @@ namespace Umbraco.Cms.Infrastructure.WebAssets { foreach (ManifestAssets manifestAssets in manifestAssetList) { - _runtimeMinifier.CreateJsBundle( - GetIndependentPackageBundleName(manifestAssets, assetType), - BundlingOptions.OptimizedAndComposite, - FormatPaths(manifestAssets.Assets.ToArray())); + string bundleName = GetIndependentPackageBundleName(manifestAssets, assetType); + string[] filePaths = FormatPaths(manifestAssets.Assets.ToArray()); + + switch (assetType) + { + case AssetType.Javascript: + _runtimeMinifier.CreateJsBundle(bundleName, BundlingOptions.OptimizedAndComposite, filePaths); + break; + case AssetType.Css: + _runtimeMinifier.CreateCssBundle(bundleName, BundlingOptions.OptimizedAndComposite, filePaths); + break; + default: + throw new IndexOutOfRangeException(); + } } } } From 3f0a47897434263db3f362e3016c1b2a6b393801 Mon Sep 17 00:00:00 2001 From: Vitor Rodrigues Date: Tue, 20 Jul 2021 01:29:00 +0200 Subject: [PATCH 039/123] Exposed MVC configuration action in AddBackOffice - #10709 --- .../DependencyInjection/UmbracoBuilderExtensions.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Umbraco.Web.BackOffice/DependencyInjection/UmbracoBuilderExtensions.cs b/src/Umbraco.Web.BackOffice/DependencyInjection/UmbracoBuilderExtensions.cs index b24b0c32fc..aeb329efb4 100644 --- a/src/Umbraco.Web.BackOffice/DependencyInjection/UmbracoBuilderExtensions.cs +++ b/src/Umbraco.Web.BackOffice/DependencyInjection/UmbracoBuilderExtensions.cs @@ -1,3 +1,4 @@ +using System; using System.Linq; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; @@ -31,7 +32,7 @@ namespace Umbraco.Extensions /// /// Adds all required components to run the Umbraco back office /// - public static IUmbracoBuilder AddBackOffice(this IUmbracoBuilder builder) => builder + public static IUmbracoBuilder AddBackOffice(this IUmbracoBuilder builder, Action configureMvc = null) => builder .AddConfiguration() .AddUmbracoCore() .AddWebComponents() @@ -42,7 +43,7 @@ namespace Umbraco.Extensions .AddMembersIdentity() .AddBackOfficeAuthorizationPolicies() .AddUmbracoProfiler() - .AddMvcAndRazor() + .AddMvcAndRazor(configureMvc) .AddWebServer() .AddPreviewSupport() .AddHostedServices() From 8ebb37a5963c020bd49f1bda940efd6776f069fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niels=20Lyngs=C3=B8?= Date: Fri, 9 Jul 2021 11:29:01 +0200 Subject: [PATCH 040/123] add vm.model.onValueChanged callback --- .../umbMediaPicker3PropertyEditor.component.js | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/mediapicker3/umbMediaPicker3PropertyEditor.component.js b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/mediapicker3/umbMediaPicker3PropertyEditor.component.js index 96f3126288..02b9f0b928 100644 --- a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/mediapicker3/umbMediaPicker3PropertyEditor.component.js +++ b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/mediapicker3/umbMediaPicker3PropertyEditor.component.js @@ -98,6 +98,10 @@ vm.model.value.forEach(mediaEntry => updateMediaEntryData(mediaEntry)); + // set the onValueChanged callback, this will tell us if the media picker model changed on the server + // once the data is submitted. If so we need to re-initialize + vm.model.onValueChanged = onServerValueChanged; + userService.getCurrentUser().then(function (userData) { if (!vm.model.config.startNodeId) { @@ -120,6 +124,15 @@ }; + function onServerValueChanged(newVal, oldVal) { + if(newVal === null || !Array.isArray(newVal)) { + newVal = []; + vm.model.value = newVal; + } + + vm.model.value.forEach(mediaEntry => updateMediaEntryData(mediaEntry)); + } + function setDirty() { if (vm.propertyForm) { vm.propertyForm.$setDirty(); @@ -259,7 +272,7 @@ function setActiveMedia(mediaEntryOrNull) { vm.activeMediaEntry = mediaEntryOrNull; } - + function editMedia(mediaEntry, options, $event) { if($event) From 3def2ee2ad2897e1d1c6d9b1d077e4c4652d01d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niels=20Lyngs=C3=B8?= Date: Fri, 9 Jul 2021 11:29:01 +0200 Subject: [PATCH 041/123] add vm.model.onValueChanged callback (cherry picked from commit 8ebb37a5963c020bd49f1bda940efd6776f069fc) # Conflicts: # src/Umbraco.Web.UI.Client/src/views/propertyeditors/mediapicker3/umbMediaPicker3PropertyEditor.component.js --- .../umbMediaPicker3PropertyEditor.component.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/mediapicker3/umbMediaPicker3PropertyEditor.component.js b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/mediapicker3/umbMediaPicker3PropertyEditor.component.js index 675381d46e..3ad3309085 100644 --- a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/mediapicker3/umbMediaPicker3PropertyEditor.component.js +++ b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/mediapicker3/umbMediaPicker3PropertyEditor.component.js @@ -93,6 +93,10 @@ vm.model.value.forEach(mediaEntry => updateMediaEntryData(mediaEntry)); + // set the onValueChanged callback, this will tell us if the media picker model changed on the server + // once the data is submitted. If so we need to re-initialize + vm.model.onValueChanged = onServerValueChanged; + userService.getCurrentUser().then(function (userData) { if (!vm.model.config.startNodeId) { @@ -115,6 +119,15 @@ }; + function onServerValueChanged(newVal, oldVal) { + if(newVal === null || !Array.isArray(newVal)) { + newVal = []; + vm.model.value = newVal; + } + + vm.model.value.forEach(mediaEntry => updateMediaEntryData(mediaEntry)); + } + function setDirty() { if (vm.propertyForm) { vm.propertyForm.$setDirty(); From 053f1b2795bf1222115cfe48bdf90d1832bfa49d Mon Sep 17 00:00:00 2001 From: Lars-Erik Date: Tue, 20 Jul 2021 13:08:55 +0200 Subject: [PATCH 042/123] Moved the one remaining SqlCe dependant integration test into separate assembly. --- .gitignore | 5 +- .../Persistence/DatabaseBuilderTests.cs | 7 +- .../Umbraco.Tests.Integration.SqlCe.csproj | 32 +++ .../Umbraco.Tests.Integration.csproj | 192 +++++++++--------- src/umbraco-netcore-only.sln | 19 +- 5 files changed, 150 insertions(+), 105 deletions(-) rename src/{Umbraco.Tests.Integration => Umbraco.Tests.Integration.SqlCe}/Umbraco.Infrastructure/Persistence/DatabaseBuilderTests.cs (95%) create mode 100644 src/Umbraco.Tests.Integration.SqlCe/Umbraco.Tests.Integration.SqlCe.csproj diff --git a/.gitignore b/.gitignore index e38c71eef8..f5249bb70b 100644 --- a/.gitignore +++ b/.gitignore @@ -207,6 +207,7 @@ src/Umbraco.Tests/TEMP/ src/Umbraco.Tests.UnitTests/umbraco/Data/TEMP/ /src/Umbraco.Web.UI.NetCore/appsettings.Local.json -src/Umbraco.Tests.Integration/DatabaseContextTests.sdf +src/Umbraco.Tests.Integration.SqlCe/umbraco/Data/TEMP/ +src/Umbraco.Tests.Integration.SqlCe/DatabaseContextTests.sdf -src/Umbraco.Web.UI.NetCore/umbraco/config/appsettings-schema.json +src/Umbraco.Web.UI.NetCore/umbraco/config/appsettings-schema.json diff --git a/src/Umbraco.Tests.Integration/Umbraco.Infrastructure/Persistence/DatabaseBuilderTests.cs b/src/Umbraco.Tests.Integration.SqlCe/Umbraco.Infrastructure/Persistence/DatabaseBuilderTests.cs similarity index 95% rename from src/Umbraco.Tests.Integration/Umbraco.Infrastructure/Persistence/DatabaseBuilderTests.cs rename to src/Umbraco.Tests.Integration.SqlCe/Umbraco.Infrastructure/Persistence/DatabaseBuilderTests.cs index 8082fb8b16..80f72541a6 100644 --- a/src/Umbraco.Tests.Integration/Umbraco.Infrastructure/Persistence/DatabaseBuilderTests.cs +++ b/src/Umbraco.Tests.Integration.SqlCe/Umbraco.Infrastructure/Persistence/DatabaseBuilderTests.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.IO; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; @@ -24,6 +24,11 @@ namespace Umbraco.Cms.Tests.Integration.Umbraco.Infrastructure.Persistence private IUmbracoDatabaseFactory UmbracoDatabaseFactory => GetRequiredService(); private IEmbeddedDatabaseCreator EmbeddedDatabaseCreator => GetRequiredService(); + public DatabaseBuilderTests() + { + TestOptionAttributeBase.ScanAssemblies.Add(typeof(DatabaseBuilderTests).Assembly); + } + [Test] public void CreateDatabase() { diff --git a/src/Umbraco.Tests.Integration.SqlCe/Umbraco.Tests.Integration.SqlCe.csproj b/src/Umbraco.Tests.Integration.SqlCe/Umbraco.Tests.Integration.SqlCe.csproj new file mode 100644 index 0000000000..9d831257e9 --- /dev/null +++ b/src/Umbraco.Tests.Integration.SqlCe/Umbraco.Tests.Integration.SqlCe.csproj @@ -0,0 +1,32 @@ + + + + net5.0 + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + + + + diff --git a/src/Umbraco.Tests.Integration/Umbraco.Tests.Integration.csproj b/src/Umbraco.Tests.Integration/Umbraco.Tests.Integration.csproj index 687df3b46a..663c7705c8 100644 --- a/src/Umbraco.Tests.Integration/Umbraco.Tests.Integration.csproj +++ b/src/Umbraco.Tests.Integration/Umbraco.Tests.Integration.csproj @@ -1,106 +1,106 @@ - + - - Exe - net5.0 - false - Umbraco.Cms.Tests.Integration - + + net5.0 + Umbraco.Cms.Tests.Integration + Umbraco.Cms.Tests.Integration + Umbraco CMS Integration Tests + Contains helper classes for integration tests with Umbraco, including all internal integration tests. + true + - - IS_WINDOWS - + + IS_WINDOWS + - - - - - - - - - - - - - - - - - ResXFileCodeGenerator - TestFiles.Designer.cs - - - ResXFileCodeGenerator - ImportResources.Designer.cs - Designer - - - True - True - TestFiles.resx - - - True - True - ImportResources.resx - - + + + + + + + + + + + + + + + + + ResXFileCodeGenerator + TestFiles.Designer.cs + + + ResXFileCodeGenerator + ImportResources.Designer.cs + Designer + + + True + True + TestFiles.resx + + + True + True + ImportResources.resx + + - - - + + + - - - - Designer - - - - - - - - - Designer - - - - - - + + + + Designer + + + + + + + + + Designer + + + + + + - - - - - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + - - - - - - - - - - + + + + + + + + - - - Designer - - + + + Designer + + diff --git a/src/umbraco-netcore-only.sln b/src/umbraco-netcore-only.sln index 9535e362d1..893d1823d9 100644 --- a/src/umbraco-netcore-only.sln +++ b/src/umbraco-netcore-only.sln @@ -6,16 +6,16 @@ MinimumVisualStudioVersion = 10.0.40219.1 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Build", "Build", "{2849E9D4-3B4E-40A3-A309-F3CB4F0E125F}" ProjectSection(SolutionItems) = preProject ..\linting\.editorconfig = ..\linting\.editorconfig + ..\build\azure-pipelines.yml = ..\build\azure-pipelines.yml ..\build\build-bootstrap.ps1 = ..\build\build-bootstrap.ps1 ..\build\build.ps1 = ..\build\build.ps1 - ..\NuGet.Config = ..\NuGet.Config - SolutionInfo.cs = SolutionInfo.cs - ..\linting\stylecop.json = ..\linting\stylecop.json ..\linting\codeanalysis.ruleset = ..\linting\codeanalysis.ruleset ..\linting\codeanalysis.tests.ruleset = ..\linting\codeanalysis.tests.ruleset ..\Directory.Build.props = ..\Directory.Build.props ..\Directory.Build.targets = ..\Directory.Build.targets - ..\build\azure-pipelines.yml = ..\build\azure-pipelines.yml + ..\NuGet.Config = ..\NuGet.Config + SolutionInfo.cs = SolutionInfo.cs + ..\linting\stylecop.json = ..\linting\stylecop.json EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Docs", "Docs", "{FD962632-184C-4005-A5F3-E705D92FC645}" @@ -140,9 +140,11 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Umbraco.Tests.UnitTests", " EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Umbraco.Web.Common", "Umbraco.Web.Common\Umbraco.Web.Common.csproj", "{79E4293D-C92C-4649-AEC8-F1EFD95BDEB1}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Umbraco.Tests.Integration", "Umbraco.Tests.Integration\Umbraco.Tests.Integration.csproj", "{1B885D2F-1599-4557-A4EC-474CC74DEB10}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Umbraco.Tests.Integration", "Umbraco.Tests.Integration\Umbraco.Tests.Integration.csproj", "{1B885D2F-1599-4557-A4EC-474CC74DEB10}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Umbraco.Examine.Lucene", "Umbraco.Examine.Lucene\Umbraco.Examine.Lucene.csproj", "{B03560E9-2AEB-49C4-8031-84BCDAEB48C4}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Umbraco.Examine.Lucene", "Umbraco.Examine.Lucene\Umbraco.Examine.Lucene.csproj", "{B03560E9-2AEB-49C4-8031-84BCDAEB48C4}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Umbraco.Tests.Integration.SqlCe", "Umbraco.Tests.Integration.SqlCe\Umbraco.Tests.Integration.SqlCe.csproj", "{7A58F7CB-786F-43D6-A946-7BFA1B70D0AA}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -198,6 +200,10 @@ Global {B03560E9-2AEB-49C4-8031-84BCDAEB48C4}.Debug|Any CPU.Build.0 = Debug|Any CPU {B03560E9-2AEB-49C4-8031-84BCDAEB48C4}.Release|Any CPU.ActiveCfg = Release|Any CPU {B03560E9-2AEB-49C4-8031-84BCDAEB48C4}.Release|Any CPU.Build.0 = Release|Any CPU + {7A58F7CB-786F-43D6-A946-7BFA1B70D0AA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7A58F7CB-786F-43D6-A946-7BFA1B70D0AA}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7A58F7CB-786F-43D6-A946-7BFA1B70D0AA}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7A58F7CB-786F-43D6-A946-7BFA1B70D0AA}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -213,6 +219,7 @@ Global {9102ABDF-E537-4E46-B525-C9ED4833EED0} = {B5BD12C1-A454-435E-8A46-FF4A364C0382} {1B885D2F-1599-4557-A4EC-474CC74DEB10} = {B5BD12C1-A454-435E-8A46-FF4A364C0382} {B03560E9-2AEB-49C4-8031-84BCDAEB48C4} = {2849E9D4-3B4E-40A3-A309-F3CB4F0E125F} + {7A58F7CB-786F-43D6-A946-7BFA1B70D0AA} = {B5BD12C1-A454-435E-8A46-FF4A364C0382} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {7A0F2E34-D2AF-4DAB-86A0-7D7764B3D0EC} From 37066bb0e154e06710b60589ab0b99be80893c38 Mon Sep 17 00:00:00 2001 From: Lars-Erik Date: Tue, 20 Jul 2021 17:34:19 +0200 Subject: [PATCH 043/123] =?UTF-8?q?The=20rest=20of=20the=20internals.=20?= =?UTF-8?q?=F0=9F=A5=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../DataSource/BTree.ContentDataSerializer.cs | 4 ++-- src/Umbraco.PublishedCache.NuCache/DataSource/BTree.cs | 4 ++-- .../DataSource/IDictionaryOfPropertyDataSerializer.cs | 2 +- .../Umbraco.Tests.Integration.SqlCe.csproj | 1 + 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/Umbraco.PublishedCache.NuCache/DataSource/BTree.ContentDataSerializer.cs b/src/Umbraco.PublishedCache.NuCache/DataSource/BTree.ContentDataSerializer.cs index 59b73c21b0..99ce9365fc 100644 --- a/src/Umbraco.PublishedCache.NuCache/DataSource/BTree.ContentDataSerializer.cs +++ b/src/Umbraco.PublishedCache.NuCache/DataSource/BTree.ContentDataSerializer.cs @@ -1,4 +1,4 @@ -using System.IO; +using System.IO; using CSharpTest.Net.Serialization; namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource @@ -6,7 +6,7 @@ namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource /// /// Serializes/Deserializes data to BTree data source for /// - internal class ContentDataSerializer : ISerializer + public class ContentDataSerializer : ISerializer { public ContentDataSerializer(IDictionaryOfPropertyDataSerializer dictionaryOfPropertyDataSerializer = null) { diff --git a/src/Umbraco.PublishedCache.NuCache/DataSource/BTree.cs b/src/Umbraco.PublishedCache.NuCache/DataSource/BTree.cs index 1b8089d8ba..c813e428d2 100644 --- a/src/Umbraco.PublishedCache.NuCache/DataSource/BTree.cs +++ b/src/Umbraco.PublishedCache.NuCache/DataSource/BTree.cs @@ -1,11 +1,11 @@ -using System.Configuration; +using System.Configuration; using CSharpTest.Net.Collections; using CSharpTest.Net.Serialization; using Umbraco.Cms.Core.Configuration.Models; namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource { - internal class BTree + public class BTree { public static BPlusTree GetTree(string filepath, bool exists, NuCacheSettings settings, ContentDataSerializer contentDataSerializer = null) { diff --git a/src/Umbraco.PublishedCache.NuCache/DataSource/IDictionaryOfPropertyDataSerializer.cs b/src/Umbraco.PublishedCache.NuCache/DataSource/IDictionaryOfPropertyDataSerializer.cs index bffa66898d..702689a995 100644 --- a/src/Umbraco.PublishedCache.NuCache/DataSource/IDictionaryOfPropertyDataSerializer.cs +++ b/src/Umbraco.PublishedCache.NuCache/DataSource/IDictionaryOfPropertyDataSerializer.cs @@ -3,7 +3,7 @@ using System.IO; namespace Umbraco.Cms.Infrastructure.PublishedCache.DataSource { - internal interface IDictionaryOfPropertyDataSerializer + public interface IDictionaryOfPropertyDataSerializer { IDictionary ReadFrom(Stream stream); void WriteTo(IDictionary value, Stream stream); diff --git a/src/Umbraco.Tests.Integration.SqlCe/Umbraco.Tests.Integration.SqlCe.csproj b/src/Umbraco.Tests.Integration.SqlCe/Umbraco.Tests.Integration.SqlCe.csproj index 9d831257e9..a8d2d5e55a 100644 --- a/src/Umbraco.Tests.Integration.SqlCe/Umbraco.Tests.Integration.SqlCe.csproj +++ b/src/Umbraco.Tests.Integration.SqlCe/Umbraco.Tests.Integration.SqlCe.csproj @@ -2,6 +2,7 @@ net5.0 + false From 1155210209e62fff8dc731c95c40adc1c3760c57 Mon Sep 17 00:00:00 2001 From: Shannon Date: Wed, 21 Jul 2021 14:32:55 -0600 Subject: [PATCH 044/123] Fixes error reporting for external login --- .../Controllers/BackOfficeController.cs | 14 ++++++++++---- .../Security/AutoLinkSignInResult.cs | 4 ++-- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/Umbraco.Web.BackOffice/Controllers/BackOfficeController.cs b/src/Umbraco.Web.BackOffice/Controllers/BackOfficeController.cs index 161896628b..5201c91c87 100644 --- a/src/Umbraco.Web.BackOffice/Controllers/BackOfficeController.cs +++ b/src/Umbraco.Web.BackOffice/Controllers/BackOfficeController.cs @@ -33,6 +33,7 @@ using Umbraco.Cms.Web.Common.Controllers; using Umbraco.Cms.Web.Common.Filters; using Umbraco.Extensions; using Constants = Umbraco.Cms.Core.Constants; +using SignInResult = Microsoft.AspNetCore.Identity.SignInResult; namespace Umbraco.Cms.Web.BackOffice.Controllers { @@ -430,7 +431,7 @@ namespace Umbraco.Cms.Web.BackOffice.Controllers if (response == null) throw new ArgumentNullException(nameof(response)); // Sign in the user with this external login provider (which auto links, etc...) - var result = await _signInManager.ExternalLoginSignInAsync(loginInfo, isPersistent: false); + SignInResult result = await _signInManager.ExternalLoginSignInAsync(loginInfo, isPersistent: false); var errors = new List(); @@ -467,17 +468,17 @@ namespace Umbraco.Cms.Web.BackOffice.Controllers return verifyResponse; } - else if (result == Microsoft.AspNetCore.Identity.SignInResult.LockedOut) + else if (result == SignInResult.LockedOut) { errors.Add($"The local user {loginInfo.Principal.Identity.Name} for the external provider {loginInfo.ProviderDisplayName} is locked out."); } - else if (result == Microsoft.AspNetCore.Identity.SignInResult.NotAllowed) + else if (result == SignInResult.NotAllowed) { // This occurs when SignInManager.CanSignInAsync fails which is when RequireConfirmedEmail , RequireConfirmedPhoneNumber or RequireConfirmedAccount fails // however since we don't enforce those rules (yet) this shouldn't happen. errors.Add($"The user {loginInfo.Principal.Identity.Name} for the external provider {loginInfo.ProviderDisplayName} has not confirmed their details and cannot sign in."); } - else if (result == Microsoft.AspNetCore.Identity.SignInResult.Failed) + else if (result == SignInResult.Failed) { // Failed only occurs when the user does not exist errors.Add("The requested provider (" + loginInfo.LoginProvider + ") has not been linked to an account, the provider must be linked from the back office."); @@ -494,6 +495,11 @@ namespace Umbraco.Cms.Web.BackOffice.Controllers { errors.AddRange(autoLinkSignInResult.Errors); } + else if (!result.Succeeded) + { + // this shouldn't occur, the above should catch the correct error but we'll be safe just in case + errors.Add($"An unknown error with the requested provider ({loginInfo.LoginProvider}) occurred."); + } if (errors.Count > 0) { diff --git a/src/Umbraco.Web.BackOffice/Security/AutoLinkSignInResult.cs b/src/Umbraco.Web.BackOffice/Security/AutoLinkSignInResult.cs index 3da2553d04..3901e96fbd 100644 --- a/src/Umbraco.Web.BackOffice/Security/AutoLinkSignInResult.cs +++ b/src/Umbraco.Web.BackOffice/Security/AutoLinkSignInResult.cs @@ -9,12 +9,12 @@ namespace Umbraco.Cms.Web.BackOffice.Security /// public class AutoLinkSignInResult : SignInResult { - public static AutoLinkSignInResult FailedNotLinked => new AutoLinkSignInResult() + public static AutoLinkSignInResult FailedNotLinked { get; } = new AutoLinkSignInResult() { Succeeded = false }; - public static AutoLinkSignInResult FailedNoEmail => new AutoLinkSignInResult() + public static AutoLinkSignInResult FailedNoEmail { get; } = new AutoLinkSignInResult() { Succeeded = false }; From 7280c22e0adb60b482b3cc78f11d0292fc093838 Mon Sep 17 00:00:00 2001 From: Shannon Date: Wed, 21 Jul 2021 14:53:25 -0600 Subject: [PATCH 045/123] Changes BackOfficeExternalLoginProviderOptions to use the IOptions pattern with named options so they can be set after DI. --- .../BackOfficeAuthenticationBuilder.cs | 20 +++++++++------- .../BackOfficeExternalLoginProvider.cs | 12 +++++++--- .../BackOfficeExternalLoginProviderOptions.cs | 24 +++++++++++-------- .../BackOfficeExternalLoginsBuilder.cs | 8 +++---- 4 files changed, 38 insertions(+), 26 deletions(-) diff --git a/src/Umbraco.Web.BackOffice/Security/BackOfficeAuthenticationBuilder.cs b/src/Umbraco.Web.BackOffice/Security/BackOfficeAuthenticationBuilder.cs index 5ccd1c0aa1..bc35e9ad44 100644 --- a/src/Umbraco.Web.BackOffice/Security/BackOfficeAuthenticationBuilder.cs +++ b/src/Umbraco.Web.BackOffice/Security/BackOfficeAuthenticationBuilder.cs @@ -12,18 +12,16 @@ namespace Umbraco.Cms.Web.BackOffice.Security /// public class BackOfficeAuthenticationBuilder : AuthenticationBuilder { - private readonly BackOfficeExternalLoginProviderOptions _loginProviderOptions; + private readonly Action _loginProviderOptions; - public BackOfficeAuthenticationBuilder(IServiceCollection services, BackOfficeExternalLoginProviderOptions loginProviderOptions) + public BackOfficeAuthenticationBuilder( + IServiceCollection services, + Action loginProviderOptions = null) : base(services) - { - _loginProviderOptions = loginProviderOptions; - } + => _loginProviderOptions = loginProviderOptions ?? (x => { }); public string SchemeForBackOffice(string scheme) - { - return Constants.Security.BackOfficeExternalAuthenticationTypePrefix + scheme; - } + => Constants.Security.BackOfficeExternalAuthenticationTypePrefix + scheme; /// /// Overridden to track the final authenticationScheme being registered for the external login @@ -43,7 +41,11 @@ namespace Umbraco.Cms.Web.BackOffice.Security } // add our login provider to the container along with a custom options configuration - Services.AddSingleton(x => new BackOfficeExternalLoginProvider(displayName, authenticationScheme, _loginProviderOptions)); + Services.Configure(authenticationScheme, _loginProviderOptions); + Services.AddSingleton(x => new BackOfficeExternalLoginProvider( + displayName, + authenticationScheme, + x.GetRequiredService>())); Services.TryAddEnumerable(ServiceDescriptor.Singleton, EnsureBackOfficeScheme>()); return base.AddRemoteScheme(authenticationScheme, displayName, configureOptions); diff --git a/src/Umbraco.Web.BackOffice/Security/BackOfficeExternalLoginProvider.cs b/src/Umbraco.Web.BackOffice/Security/BackOfficeExternalLoginProvider.cs index ff2a64f155..42798022b7 100644 --- a/src/Umbraco.Web.BackOffice/Security/BackOfficeExternalLoginProvider.cs +++ b/src/Umbraco.Web.BackOffice/Security/BackOfficeExternalLoginProvider.cs @@ -1,4 +1,5 @@ -using System; +using System; +using Microsoft.Extensions.Options; namespace Umbraco.Cms.Web.BackOffice.Security { @@ -7,11 +8,16 @@ namespace Umbraco.Cms.Web.BackOffice.Security /// public class BackOfficeExternalLoginProvider : IEquatable { - public BackOfficeExternalLoginProvider(string name, string authenticationType, BackOfficeExternalLoginProviderOptions properties) + public BackOfficeExternalLoginProvider(string name, string authenticationType, IOptions properties) { + if (properties is null) + { + throw new ArgumentNullException(nameof(properties)); + } + Name = name ?? throw new ArgumentNullException(nameof(name)); AuthenticationType = authenticationType ?? throw new ArgumentNullException(nameof(authenticationType)); - Options = properties ?? throw new ArgumentNullException(nameof(properties)); + Options = properties.Value; } public string Name { get; } diff --git a/src/Umbraco.Web.BackOffice/Security/BackOfficeExternalLoginProviderOptions.cs b/src/Umbraco.Web.BackOffice/Security/BackOfficeExternalLoginProviderOptions.cs index fa1c1fe487..cdbcd0b8e8 100644 --- a/src/Umbraco.Web.BackOffice/Security/BackOfficeExternalLoginProviderOptions.cs +++ b/src/Umbraco.Web.BackOffice/Security/BackOfficeExternalLoginProviderOptions.cs @@ -1,14 +1,13 @@ -namespace Umbraco.Cms.Web.BackOffice.Security +namespace Umbraco.Cms.Web.BackOffice.Security { - - /// /// Options used to configure back office external login providers /// public class BackOfficeExternalLoginProviderOptions { public BackOfficeExternalLoginProviderOptions( - string buttonStyle, string icon, + string buttonStyle, + string icon, ExternalSignInAutoLinkOptions autoLinkOptions = null, bool denyLocalLogin = false, bool autoRedirectLoginToExternalProvider = false, @@ -22,18 +21,23 @@ CustomBackOfficeView = customBackOfficeView; } - public string ButtonStyle { get; } - public string Icon { get; } + public BackOfficeExternalLoginProviderOptions() + { + } + + public string ButtonStyle { get; set; } = "btn-openid"; + + public string Icon { get; set; } = "fa-user"; /// /// Options used to control how users can be auto-linked/created/updated based on the external login provider /// - public ExternalSignInAutoLinkOptions AutoLinkOptions { get; } + public ExternalSignInAutoLinkOptions AutoLinkOptions { get; set; } /// /// When set to true will disable all local user login functionality /// - public bool DenyLocalLogin { get; } + public bool DenyLocalLogin { get; set; } /// /// When specified this will automatically redirect to the OAuth login provider instead of prompting the user to click on the OAuth button first. @@ -42,7 +46,7 @@ /// This is generally used in conjunction with . If more than one OAuth provider specifies this, the last registered /// provider's redirect settings will win. /// - public bool AutoRedirectLoginToExternalProvider { get; } + public bool AutoRedirectLoginToExternalProvider { get; set; } /// /// A virtual path to a custom angular view that is used to replace the entire UI that renders the external login button that the user interacts with @@ -51,6 +55,6 @@ /// If this view is specified it is 100% up to the user to render the html responsible for rendering the link/un-link buttons along with showing any errors /// that occur. This overrides what Umbraco normally does by default. /// - public string CustomBackOfficeView { get; } + public string CustomBackOfficeView { get; set; } } } diff --git a/src/Umbraco.Web.BackOffice/Security/BackOfficeExternalLoginsBuilder.cs b/src/Umbraco.Web.BackOffice/Security/BackOfficeExternalLoginsBuilder.cs index daea904a49..cab3ea10d1 100644 --- a/src/Umbraco.Web.BackOffice/Security/BackOfficeExternalLoginsBuilder.cs +++ b/src/Umbraco.Web.BackOffice/Security/BackOfficeExternalLoginsBuilder.cs @@ -1,4 +1,4 @@ -using System; +using System; using Microsoft.Extensions.DependencyInjection; namespace Umbraco.Cms.Web.BackOffice.Security @@ -21,9 +21,9 @@ namespace Umbraco.Cms.Web.BackOffice.Security /// /// /// - public BackOfficeExternalLoginsBuilder AddBackOfficeLogin( - BackOfficeExternalLoginProviderOptions loginProviderOptions, - Action build) + public BackOfficeExternalLoginsBuilder AddBackOfficeLogin( + Action build, + Action loginProviderOptions = null) { build(new BackOfficeAuthenticationBuilder(_services, loginProviderOptions)); return this; From 615008c0ab32e501cff65f173cd25e440b15c459 Mon Sep 17 00:00:00 2001 From: Mole Date: Thu, 22 Jul 2021 11:10:02 +0200 Subject: [PATCH 046/123] Rename udi to id in EntityController.GetUrl --- .../Controllers/EntityController.cs | 8 ++++---- .../src/common/resources/entity.resource.js | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Umbraco.Web.BackOffice/Controllers/EntityController.cs b/src/Umbraco.Web.BackOffice/Controllers/EntityController.cs index fcd0c81891..3169c9f64b 100644 --- a/src/Umbraco.Web.BackOffice/Controllers/EntityController.cs +++ b/src/Umbraco.Web.BackOffice/Controllers/EntityController.cs @@ -258,16 +258,16 @@ namespace Umbraco.Cms.Web.BackOffice.Controllers /// /// Gets the URL of an entity /// - /// UDI of the entity to fetch URL for + /// UDI of the entity to fetch URL for /// The culture to fetch the URL for /// The URL or path to the item - public IActionResult GetUrl(Udi udi, string culture = "*") + public IActionResult GetUrl(Udi id, string culture = "*") { - var intId = _entityService.GetId(udi); + var intId = _entityService.GetId(id); if (!intId.Success) return NotFound(); UmbracoEntityTypes entityType; - switch (udi.EntityType) + switch (id.EntityType) { case Constants.UdiEntityType.Document: entityType = UmbracoEntityTypes.Document; diff --git a/src/Umbraco.Web.UI.Client/src/common/resources/entity.resource.js b/src/Umbraco.Web.UI.Client/src/common/resources/entity.resource.js index 9f0ebf8b73..2dc6aeb6a6 100644 --- a/src/Umbraco.Web.UI.Client/src/common/resources/entity.resource.js +++ b/src/Umbraco.Web.UI.Client/src/common/resources/entity.resource.js @@ -142,7 +142,7 @@ function entityResource($q, $http, umbRequestHelper) { umbRequestHelper.getApiUrl( "entityApiBaseUrl", "GetUrl", - [{ udi: udi }, {culture: culture }])), + [{ id: udi }, {culture: culture }])), 'Failed to retrieve url for UDI:' + udi); }, From 1f4cc81af8c5ce7b286238e29773f8a2be9c4182 Mon Sep 17 00:00:00 2001 From: Matt Brailsford Date: Mon, 19 Jul 2021 11:01:32 +0100 Subject: [PATCH 047/123] Removes `inert` attribute when `editorService.closeAll()` is called Fixes #10708 by removing the `inert` attribute on `#mainWrapper` when `editorService.closeAll()` is called. This code is necessary because when close all is called, there is no editor passed, and so `removeEditor` is never called, and so the `focusLockService.removeInertAttribute();` method isn't called. I add to the if block that handles the close all code instead. --- .../directives/components/editor/umbeditors.directive.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Umbraco.Web.UI.Client/src/common/directives/components/editor/umbeditors.directive.js b/src/Umbraco.Web.UI.Client/src/common/directives/components/editor/umbeditors.directive.js index 590f9627ab..1c9c85b075 100644 --- a/src/Umbraco.Web.UI.Client/src/common/directives/components/editor/umbeditors.directive.js +++ b/src/Umbraco.Web.UI.Client/src/common/directives/components/editor/umbeditors.directive.js @@ -126,7 +126,9 @@ // close all editors if (args && !args.editor && args.editors.length === 0) { editorCount = 0; - scope.editors = []; + scope.editors = []; + // Remove the inert attribute from the #mainwrapper + focusLockService.removeInertAttribute(); } })); From 98266d462c5821327d8d4320a08224447c370e8a Mon Sep 17 00:00:00 2001 From: Matt Brailsford Date: Mon, 19 Jul 2021 11:01:32 +0100 Subject: [PATCH 048/123] Removes `inert` attribute when `editorService.closeAll()` is called Fixes #10708 by removing the `inert` attribute on `#mainWrapper` when `editorService.closeAll()` is called. This code is necessary because when close all is called, there is no editor passed, and so `removeEditor` is never called, and so the `focusLockService.removeInertAttribute();` method isn't called. I add to the if block that handles the close all code instead. (cherry picked from commit 1f4cc81af8c5ce7b286238e29773f8a2be9c4182) --- .../directives/components/editor/umbeditors.directive.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Umbraco.Web.UI.Client/src/common/directives/components/editor/umbeditors.directive.js b/src/Umbraco.Web.UI.Client/src/common/directives/components/editor/umbeditors.directive.js index 590f9627ab..1c9c85b075 100644 --- a/src/Umbraco.Web.UI.Client/src/common/directives/components/editor/umbeditors.directive.js +++ b/src/Umbraco.Web.UI.Client/src/common/directives/components/editor/umbeditors.directive.js @@ -126,7 +126,9 @@ // close all editors if (args && !args.editor && args.editors.length === 0) { editorCount = 0; - scope.editors = []; + scope.editors = []; + // Remove the inert attribute from the #mainwrapper + focusLockService.removeInertAttribute(); } })); From fe9d0db763fc34644c9feedc015b5d24f6876639 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niels=20Lyngs=C3=B8?= Date: Thu, 22 Jul 2021 15:03:09 +0200 Subject: [PATCH 049/123] 10718 Enables more configured Blocks in Block List Editor (#10726) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Niels Lyngsø Co-authored-by: Mole --- .../src/common/resources/content.resource.js | 11 ++++---- .../blockeditormodelobject.service.js | 14 ++++++---- src/Umbraco.Web/Editors/ContentController.cs | 26 +++++++++++++++--- .../ContentEditing/ContentTypesByKeys.cs | 27 +++++++++++++++++++ src/Umbraco.Web/Umbraco.Web.csproj | 1 + 5 files changed, 66 insertions(+), 13 deletions(-) create mode 100644 src/Umbraco.Web/Models/ContentEditing/ContentTypesByKeys.cs diff --git a/src/Umbraco.Web.UI.Client/src/common/resources/content.resource.js b/src/Umbraco.Web.UI.Client/src/common/resources/content.resource.js index 01fff53931..a33e85c9a2 100644 --- a/src/Umbraco.Web.UI.Client/src/common/resources/content.resource.js +++ b/src/Umbraco.Web.UI.Client/src/common/resources/content.resource.js @@ -57,7 +57,7 @@ function contentResource($q, $http, umbDataFormatter, umbRequestHelper) { * Do stuff... * }); * - * + * * @returns {Promise} resourcePromise object. * */ @@ -691,11 +691,12 @@ function contentResource($q, $http, umbDataFormatter, umbRequestHelper) { getScaffoldByKeys: function (parentId, scaffoldKeys) { return umbRequestHelper.resourcePromise( - $http.get( + $http.post( umbRequestHelper.getApiUrl( "contentApiBaseUrl", - "GetEmptyByKeys", - { contentTypeKeys: scaffoldKeys, parentId: parentId })), + "GetEmptyByKeys"), + { contentTypeKeys: scaffoldKeys, parentId: parentId } + ), 'Failed to retrieve data for empty content items ids' + scaffoldKeys.join(", ")) .then(function (result) { Object.keys(result).map(function(key) { @@ -804,7 +805,7 @@ function contentResource($q, $http, umbDataFormatter, umbRequestHelper) { else if (options.orderDirection === "desc") { options.orderDirection = "Descending"; } - + //converts the value to a js bool function toBool(v) { if (Utilities.isNumber(v)) { diff --git a/src/Umbraco.Web.UI.Client/src/common/services/blockeditormodelobject.service.js b/src/Umbraco.Web.UI.Client/src/common/services/blockeditormodelobject.service.js index 4bcbbc89d6..ee96cfbaaa 100644 --- a/src/Umbraco.Web.UI.Client/src/common/services/blockeditormodelobject.service.js +++ b/src/Umbraco.Web.UI.Client/src/common/services/blockeditormodelobject.service.js @@ -487,7 +487,7 @@ * @returns {Object | null} Scaffold model for the that content type. Or null if the scaffolding model dosnt exist in this context. */ getScaffoldFromKey: function (contentTypeKey) { - return this.scaffolds.find(o => o.contentTypeKey === contentTypeKey); + return this.scaffolds.find(o => o.contentTypeKey === contentTypeKey) || null; }, /** @@ -499,7 +499,7 @@ * @returns {Object | null} Scaffold model for the that content type. Or null if the scaffolding model dosnt exist in this context. */ getScaffoldFromAlias: function (contentTypeAlias) { - return this.scaffolds.find(o => o.contentTypeAlias === contentTypeAlias); + return this.scaffolds.find(o => o.contentTypeAlias === contentTypeAlias) || null; }, /** @@ -609,10 +609,14 @@ blockObject.settingsData = settingsData; // make basics from scaffold - blockObject.settings = Utilities.copy(settingsScaffold); - ensureUdiAndKey(blockObject.settings, settingsUdi); + if (settingsScaffold !== null) {// We might not have settingsScaffold + blockObject.settings = Utilities.copy(settingsScaffold); + ensureUdiAndKey(blockObject.settings, settingsUdi); - mapToElementModel(blockObject.settings, settingsData); + mapToElementModel(blockObject.settings, settingsData); + } else { + blockObject.settings = null; + } // add settings content-app appendSettingsContentApp(blockObject.content, this.__labels.settingsName); diff --git a/src/Umbraco.Web/Editors/ContentController.cs b/src/Umbraco.Web/Editors/ContentController.cs index 55eee7b2a2..8293d1274f 100644 --- a/src/Umbraco.Web/Editors/ContentController.cs +++ b/src/Umbraco.Web/Editors/ContentController.cs @@ -449,6 +449,13 @@ namespace Umbraco.Web.Editors return result; } + private IDictionary GetEmptyByKeysInternal(Guid[] contentTypeKeys, int parentId) + { + using var scope = _scopeProvider.CreateScope(autoComplete: true); + var contentTypes = Services.ContentTypeService.GetAll(contentTypeKeys).ToList(); + return GetEmpties(contentTypes, parentId).ToDictionary(x => x.ContentTypeKey); + } + /// /// Gets a collection of empty content items for all document types. /// @@ -457,9 +464,22 @@ namespace Umbraco.Web.Editors [OutgoingEditorModelEvent] public IDictionary GetEmptyByKeys([FromUri] Guid[] contentTypeKeys, [FromUri] int parentId) { - using var scope = _scopeProvider.CreateScope(autoComplete: true); - var contentTypes = Services.ContentTypeService.GetAll(contentTypeKeys).ToList(); - return GetEmpties(contentTypes, parentId).ToDictionary(x => x.ContentTypeKey); + return GetEmptyByKeysInternal(contentTypeKeys, parentId); + } + + /// + /// Gets a collection of empty content items for all document types. + /// + /// + /// This is a post request in order to support a large amount of GUIDs without hitting the URL length limit. + /// + /// + /// + [HttpPost] + [OutgoingEditorModelEvent] + public IDictionary GetEmptyByKeys(ContentTypesByKeys contentTypeByKeys) + { + return GetEmptyByKeysInternal(contentTypeByKeys.ContentTypeKeys, contentTypeByKeys.ParentId); } [OutgoingEditorModelEvent] diff --git a/src/Umbraco.Web/Models/ContentEditing/ContentTypesByKeys.cs b/src/Umbraco.Web/Models/ContentEditing/ContentTypesByKeys.cs new file mode 100644 index 0000000000..8bf1d452bc --- /dev/null +++ b/src/Umbraco.Web/Models/ContentEditing/ContentTypesByKeys.cs @@ -0,0 +1,27 @@ +using System; +using System.ComponentModel.DataAnnotations; +using System.Runtime.Serialization; + +namespace Umbraco.Web.Models.ContentEditing +{ + /// + /// A model for retrieving multiple content types based on their keys. + /// + [DataContract(Name = "contentTypes", Namespace = "")] + public class ContentTypesByKeys + { + /// + /// ID of the parent of the content type. + /// + [DataMember(Name = "parentId")] + [Required] + public int ParentId { get; set; } + + /// + /// The id of every content type to get. + /// + [DataMember(Name = "contentTypeKeys")] + [Required] + public Guid[] ContentTypeKeys { get; set; } + } +} diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index 4dce8a2da0..e830722cc7 100644 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -241,6 +241,7 @@ + From c3953bb0c1fcf8a3c4a1f7f63b5077545e503bea Mon Sep 17 00:00:00 2001 From: Shannon Date: Thu, 22 Jul 2021 13:54:46 -0600 Subject: [PATCH 050/123] Ensure to resolve the named options --- .../Security/BackOfficeAuthenticationBuilder.cs | 2 +- .../Security/BackOfficeExternalLoginProvider.cs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Umbraco.Web.BackOffice/Security/BackOfficeAuthenticationBuilder.cs b/src/Umbraco.Web.BackOffice/Security/BackOfficeAuthenticationBuilder.cs index bc35e9ad44..27b763218e 100644 --- a/src/Umbraco.Web.BackOffice/Security/BackOfficeAuthenticationBuilder.cs +++ b/src/Umbraco.Web.BackOffice/Security/BackOfficeAuthenticationBuilder.cs @@ -45,7 +45,7 @@ namespace Umbraco.Cms.Web.BackOffice.Security Services.AddSingleton(x => new BackOfficeExternalLoginProvider( displayName, authenticationScheme, - x.GetRequiredService>())); + x.GetRequiredService>())); Services.TryAddEnumerable(ServiceDescriptor.Singleton, EnsureBackOfficeScheme>()); return base.AddRemoteScheme(authenticationScheme, displayName, configureOptions); diff --git a/src/Umbraco.Web.BackOffice/Security/BackOfficeExternalLoginProvider.cs b/src/Umbraco.Web.BackOffice/Security/BackOfficeExternalLoginProvider.cs index 42798022b7..07f16734a2 100644 --- a/src/Umbraco.Web.BackOffice/Security/BackOfficeExternalLoginProvider.cs +++ b/src/Umbraco.Web.BackOffice/Security/BackOfficeExternalLoginProvider.cs @@ -8,7 +8,7 @@ namespace Umbraco.Cms.Web.BackOffice.Security /// public class BackOfficeExternalLoginProvider : IEquatable { - public BackOfficeExternalLoginProvider(string name, string authenticationType, IOptions properties) + public BackOfficeExternalLoginProvider(string name, string authenticationType, IOptionsSnapshot properties) { if (properties is null) { @@ -17,7 +17,7 @@ namespace Umbraco.Cms.Web.BackOffice.Security Name = name ?? throw new ArgumentNullException(nameof(name)); AuthenticationType = authenticationType ?? throw new ArgumentNullException(nameof(authenticationType)); - Options = properties.Value; + Options = properties.Get(authenticationType); } public string Name { get; } From 560d49d6a5400f9b4e53bf1f1e3d926620152fc0 Mon Sep 17 00:00:00 2001 From: Shannon Date: Thu, 22 Jul 2021 14:33:32 -0600 Subject: [PATCH 051/123] create scope to resolve named options. --- .../Security/BackOfficeAuthenticationBuilder.cs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/Umbraco.Web.BackOffice/Security/BackOfficeAuthenticationBuilder.cs b/src/Umbraco.Web.BackOffice/Security/BackOfficeAuthenticationBuilder.cs index 27b763218e..dd360b04f6 100644 --- a/src/Umbraco.Web.BackOffice/Security/BackOfficeAuthenticationBuilder.cs +++ b/src/Umbraco.Web.BackOffice/Security/BackOfficeAuthenticationBuilder.cs @@ -42,10 +42,17 @@ namespace Umbraco.Cms.Web.BackOffice.Security // add our login provider to the container along with a custom options configuration Services.Configure(authenticationScheme, _loginProviderOptions); - Services.AddSingleton(x => new BackOfficeExternalLoginProvider( - displayName, - authenticationScheme, - x.GetRequiredService>())); + base.Services.AddSingleton(x => + { + // need to create a scope to resolve IOptionsSnapshot + using (x.CreateScope()) + { + return new BackOfficeExternalLoginProvider( + displayName, + authenticationScheme, + x.GetRequiredService>()); + } + }); Services.TryAddEnumerable(ServiceDescriptor.Singleton, EnsureBackOfficeScheme>()); return base.AddRemoteScheme(authenticationScheme, displayName, configureOptions); From 9ec86f2f86a30fa1043d46d23d21ee197a1b2cdd Mon Sep 17 00:00:00 2001 From: Shannon Date: Thu, 22 Jul 2021 14:47:17 -0600 Subject: [PATCH 052/123] create scope to resolve named options. --- .../Security/BackOfficeAuthenticationBuilder.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Umbraco.Web.BackOffice/Security/BackOfficeAuthenticationBuilder.cs b/src/Umbraco.Web.BackOffice/Security/BackOfficeAuthenticationBuilder.cs index dd360b04f6..633135b2a1 100644 --- a/src/Umbraco.Web.BackOffice/Security/BackOfficeAuthenticationBuilder.cs +++ b/src/Umbraco.Web.BackOffice/Security/BackOfficeAuthenticationBuilder.cs @@ -45,12 +45,12 @@ namespace Umbraco.Cms.Web.BackOffice.Security base.Services.AddSingleton(x => { // need to create a scope to resolve IOptionsSnapshot - using (x.CreateScope()) + using (IServiceScope serviceScope = x.CreateScope()) { return new BackOfficeExternalLoginProvider( displayName, authenticationScheme, - x.GetRequiredService>()); + serviceScope.ServiceProvider.GetRequiredService>()); } }); Services.TryAddEnumerable(ServiceDescriptor.Singleton, EnsureBackOfficeScheme>()); From ed5ab8b866ae1241a7aba336864859a443cd1e40 Mon Sep 17 00:00:00 2001 From: Shannon Date: Thu, 22 Jul 2021 16:34:30 -0600 Subject: [PATCH 053/123] Fix delete user clause --- .../Repositories/Implement/UserRepository.cs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/Umbraco.Infrastructure/Persistence/Repositories/Implement/UserRepository.cs b/src/Umbraco.Infrastructure/Persistence/Repositories/Implement/UserRepository.cs index 0dd6e2d43c..71bc5b8d33 100644 --- a/src/Umbraco.Infrastructure/Persistence/Repositories/Implement/UserRepository.cs +++ b/src/Umbraco.Infrastructure/Persistence/Repositories/Implement/UserRepository.cs @@ -432,11 +432,12 @@ ORDER BY colName"; { var list = new List { - "DELETE FROM umbracoUser2UserGroup WHERE userId = @id", - "DELETE FROM umbracoUser2NodeNotify WHERE userId = @id", - "DELETE FROM umbracoUserStartNode WHERE userId = @id", - "DELETE FROM umbracoUser WHERE id = @id", - "DELETE FROM umbracoExternalLogin WHERE id = @id" + $"DELETE FROM {Constants.DatabaseSchema.Tables.UserLogin} WHERE userId = @id", + $"DELETE FROM {Constants.DatabaseSchema.Tables.User2UserGroup} WHERE userId = @id", + $"DELETE FROM {Constants.DatabaseSchema.Tables.User2NodeNotify} WHERE userId = @id", + $"DELETE FROM {Constants.DatabaseSchema.Tables.UserStartNode} WHERE userId = @id", + $"DELETE FROM {Constants.DatabaseSchema.Tables.User} WHERE id = @id", + $"DELETE FROM {Constants.DatabaseSchema.Tables.ExternalLogin} WHERE id = @id" }; return list; } From 0f7e5a2f2f73cb112b8350bdf00d8ea6b9736e81 Mon Sep 17 00:00:00 2001 From: Shannon Date: Thu, 22 Jul 2021 16:39:06 -0600 Subject: [PATCH 054/123] fix null auto link options. --- .../BackOfficeExternalLoginProviderOptions.cs | 2 +- .../services/externallogininfo.service.js | 103 +++++++++--------- 2 files changed, 54 insertions(+), 51 deletions(-) diff --git a/src/Umbraco.Web.BackOffice/Security/BackOfficeExternalLoginProviderOptions.cs b/src/Umbraco.Web.BackOffice/Security/BackOfficeExternalLoginProviderOptions.cs index cdbcd0b8e8..977f981e52 100644 --- a/src/Umbraco.Web.BackOffice/Security/BackOfficeExternalLoginProviderOptions.cs +++ b/src/Umbraco.Web.BackOffice/Security/BackOfficeExternalLoginProviderOptions.cs @@ -32,7 +32,7 @@ namespace Umbraco.Cms.Web.BackOffice.Security /// /// Options used to control how users can be auto-linked/created/updated based on the external login provider /// - public ExternalSignInAutoLinkOptions AutoLinkOptions { get; set; } + public ExternalSignInAutoLinkOptions AutoLinkOptions { get; set; } = new ExternalSignInAutoLinkOptions(); /// /// When set to true will disable all local user login functionality diff --git a/src/Umbraco.Web.UI.Client/src/common/services/externallogininfo.service.js b/src/Umbraco.Web.UI.Client/src/common/services/externallogininfo.service.js index b44f79dd65..10092aaf38 100644 --- a/src/Umbraco.Web.UI.Client/src/common/services/externallogininfo.service.js +++ b/src/Umbraco.Web.UI.Client/src/common/services/externallogininfo.service.js @@ -5,63 +5,66 @@ **/ function externalLoginInfoService(externalLoginInfo, umbRequestHelper) { - function getLoginProvider(provider) { - if (provider) { - var found = _.find(externalLoginInfo.providers, x => x.authType == provider); - return found; - } - return null; + function getLoginProvider(provider) { + if (provider) { + var found = _.find(externalLoginInfo.providers, x => x.authType == provider); + return found; } + return null; + } - function getLoginProviderView(provider) { - if (provider && provider.properties && provider.properties.CustomBackOfficeView) { - return umbRequestHelper.convertVirtualToAbsolutePath(provider.properties.CustomBackOfficeView); - } - return null; + function getLoginProviderView(provider) { + if (provider && provider.properties && provider.properties.CustomBackOfficeView) { + return umbRequestHelper.convertVirtualToAbsolutePath(provider.properties.CustomBackOfficeView); } + return null; + } - /** - * Returns true if any provider denies local login if `provider` is null, else whether the passed - * @param {any} provider - */ - function hasDenyLocalLogin(provider) { - if (!provider) { - return _.some(externalLoginInfo.providers, x => x.properties && (x.properties.DenyLocalLogin === true)); - } - else { - return provider && provider.properties && (provider.properties.DenyLocalLogin === true); - } + /** + * Returns true if any provider denies local login if `provider` is null, else whether the passed + * @param {any} provider + */ + function hasDenyLocalLogin(provider) { + if (!provider) { + return _.some(externalLoginInfo.providers, x => x.properties && (x.properties.DenyLocalLogin === true)); } - - /** - * Returns all login providers - */ - function getLoginProviders() { - return externalLoginInfo.providers; + else { + return provider && provider.properties && (provider.properties.DenyLocalLogin === true); } + } - /** Returns all logins providers that have options that the user can interact with */ - function getLoginProvidersWithOptions() { - // only include providers that allow manual linking or ones that provide a custom view - var providers = _.filter(externalLoginInfo.providers, x => { - // transform the data and also include the custom view as a nicer property - x.customView = getLoginProviderView(x); - if (x.customView) { - return true; - } - else { - return x.properties.AutoLinkOptions.AllowManualLinking; - } - }); - return providers; - } + /** + * Returns all login providers + */ + function getLoginProviders() { + return externalLoginInfo.providers; + } - return { - hasDenyLocalLogin: hasDenyLocalLogin, - getLoginProvider: getLoginProvider, - getLoginProviders: getLoginProviders, - getLoginProvidersWithOptions: getLoginProvidersWithOptions, - getLoginProviderView: getLoginProviderView - }; + /** Returns all logins providers that have options that the user can interact with */ + function getLoginProvidersWithOptions() { + // only include providers that allow manual linking or ones that provide a custom view + var providers = _.filter(externalLoginInfo.providers, x => { + // transform the data and also include the custom view as a nicer property + x.customView = getLoginProviderView(x); + if (x.customView) { + return true; + } + else if (x.properties.AutoLinkOptions) { + return x.properties.AutoLinkOptions.AllowManualLinking; + } + else { + return false; + } + }); + return providers; + } + + return { + hasDenyLocalLogin: hasDenyLocalLogin, + getLoginProvider: getLoginProvider, + getLoginProviders: getLoginProviders, + getLoginProvidersWithOptions: getLoginProvidersWithOptions, + getLoginProviderView: getLoginProviderView + }; } angular.module('umbraco.services').factory('externalLoginInfoService', externalLoginInfoService); From 071a89a59ba8ec0acf1766e0ef4c0c4f248f4413 Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Fri, 23 Jul 2021 13:32:30 +0200 Subject: [PATCH 055/123] Bump version to 8.14.2 --- src/SolutionInfo.cs | 4 ++-- src/Umbraco.Web.UI/Umbraco.Web.UI.csproj | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/SolutionInfo.cs b/src/SolutionInfo.cs index 3a43aec402..625cbe3fb7 100644 --- a/src/SolutionInfo.cs +++ b/src/SolutionInfo.cs @@ -18,5 +18,5 @@ using System.Resources; [assembly: AssemblyVersion("8.0.0")] // these are FYI and changed automatically -[assembly: AssemblyFileVersion("8.14.1")] -[assembly: AssemblyInformationalVersion("8.14.1")] +[assembly: AssemblyFileVersion("8.14.2")] +[assembly: AssemblyInformationalVersion("8.14.2")] diff --git a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj index f238984205..ce465c6dc5 100644 --- a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj +++ b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj @@ -348,9 +348,9 @@ False True - 8141 + 8142 / - http://localhost:8141 + http://localhost:8142 8131 / http://localhost:8131 From 9807365800880dcfb0b7d71fd25553a396ba8bed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niels=20Lyngs=C3=B8?= Date: Fri, 23 Jul 2021 14:26:12 +0200 Subject: [PATCH 056/123] Escape HTML in localization tokens. (#10729) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Niels Lyngsø --- .../common/services/localization.service.js | 60 +++++++++---------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/src/Umbraco.Web.UI.Client/src/common/services/localization.service.js b/src/Umbraco.Web.UI.Client/src/common/services/localization.service.js index 99162eaf53..805afae5b8 100644 --- a/src/Umbraco.Web.UI.Client/src/common/services/localization.service.js +++ b/src/Umbraco.Web.UI.Client/src/common/services/localization.service.js @@ -22,7 +22,7 @@ */ angular.module('umbraco.services') -.factory('localizationService', function ($http, $q, eventsService, $window, $filter, userService) { +.factory('localizationService', function ($http, $q, eventsService) { // TODO: This should be injected as server vars var url = "LocalizedText"; @@ -61,14 +61,14 @@ angular.module('umbraco.services') return "[" + alias + "]"; } - + var service = { - + // loads the language resource file from the server initLocalizedResources: function () { - // TODO: This promise handling is super ugly, we should just be returnning the promise from $http and returning inner values. + // TODO: This promise handling is super ugly, we should just be returnning the promise from $http and returning inner values. var deferred = $q.defer(); @@ -120,7 +120,7 @@ angular.module('umbraco.services') * @description * Helper to tokenize and compile a localization string * @param {String} value the value to tokenize - * @param {Object} scope the $scope object + * @param {Object} scope the $scope object * @returns {String} tokenized resource string */ tokenize: function (value, scope) { @@ -138,8 +138,8 @@ angular.module('umbraco.services') } return value; }, - - + + /** * @ngdoc method * @name umbraco.services.localizationService#tokenReplace @@ -148,19 +148,19 @@ angular.module('umbraco.services') * @description * Helper to replace tokens * @param {String} value the text-string to manipulate - * @param {Array} tekens An array of tokens values + * @param {Array} tekens An array of tokens values * @returns {String} Replaced test-string */ tokenReplace: function (text, tokens) { if (tokens) { for (var i = 0; i < tokens.length; i++) { - text = text.replace("%" + i + "%", tokens[i]); + text = text.replace("%" + i + "%", _.escape(tokens[i])); } } return text; }, - - + + /** * @ngdoc method * @name umbraco.services.localizationService#localize @@ -168,16 +168,16 @@ angular.module('umbraco.services') * * @description * Checks the dictionary for a localized resource string - * @param {String} value the area/key to localize in the format of 'section_key' + * @param {String} value the area/key to localize in the format of 'section_key' * alternatively if no section is set such as 'key' then we assume the key is to be looked in * the 'general' section - * + * * @param {Array} tokens if specified this array will be sent as parameter values * This replaces %0% and %1% etc in the dictionary key value with the passed in strings - * - * @param {String} fallbackValue if specified this string will be returned if no matching + * + * @param {String} fallbackValue if specified this string will be returned if no matching * entry was found in the dictionary - * + * * @returns {String} localized resource string */ localize: function (value, tokens, fallbackValue) { @@ -194,7 +194,7 @@ angular.module('umbraco.services') * @description * Checks the dictionary for multipe localized resource strings at once, preventing the need for nested promises * with localizationService.localize - * + * * ##Usage * 
          * localizationService.localizeMany(["speechBubbles_templateErrorHeader", "speechBubbles_templateErrorText"]).then(function(data){
@@ -203,11 +203,11 @@ angular.module('umbraco.services')
          *      notificationService.error(header, message);
          * });
          *
- * - * @param {Array} keys is an array of strings of the area/key to localize in the format of 'section_key' + * + * @param {Array} keys is an array of strings of the area/key to localize in the format of 'section_key' * alternatively if no section is set such as 'key' then we assume the key is to be looked in * the 'general' section - * + * * @returns {Array} An array of localized resource string in the same order */ localizeMany: function(keys) { @@ -234,18 +234,18 @@ angular.module('umbraco.services') * @description * Checks the dictionary for multipe localized resource strings at once & concats them to a single string * Which was not possible with localizationSerivce.localize() due to returning a promise - * + * * ##Usage * 
          * localizationService.concat(["speechBubbles_templateErrorHeader", "speechBubbles_templateErrorText"]).then(function(data){
          *      var combinedText = data;
          * });
          *
- * - * @param {Array} keys is an array of strings of the area/key to localize in the format of 'section_key' + * + * @param {Array} keys is an array of strings of the area/key to localize in the format of 'section_key' * alternatively if no section is set such as 'key' then we assume the key is to be looked in * the 'general' section - * + * * @returns {String} An concatenated string of localized resource string passed into the function in the same order */ concat: function(keys) { @@ -280,7 +280,7 @@ angular.module('umbraco.services') * @description * Checks the dictionary for multipe localized resource strings at once & formats a tokenized message * Which was not possible with localizationSerivce.localize() due to returning a promise - * + * * ##Usage * 
          * localizationService.format(["template_insert", "template_insertSections"], "%0% %1%").then(function(data){
@@ -288,14 +288,14 @@ angular.module('umbraco.services')
          *      var formattedResult = data;
          * });
          *
- * - * @param {Array} keys is an array of strings of the area/key to localize in the format of 'section_key' + * + * @param {Array} keys is an array of strings of the area/key to localize in the format of 'section_key' * alternatively if no section is set such as 'key' then we assume the key is to be looked in * the 'general' section - * + * * @param {String} message is the string you wish to replace containing tokens in the format of %0% and %1% * with the localized resource strings - * + * * @returns {String} An concatenated string of localized resource string passed into the function in the same order */ format: function(keys, message){ @@ -330,7 +330,7 @@ angular.module('umbraco.services') resourceFileLoadStatus = "none"; resourceLoadingPromise = []; }); - + // return the local instance when called return service; From 5bd00c8fc9ed005c6390fc4f4d34ec6f43ea30b3 Mon Sep 17 00:00:00 2001 From: Andy Butland Date: Mon, 26 Jul 2021 09:32:08 +0200 Subject: [PATCH 057/123] Migrated Umbraco.TestData to .NET 5. --- .../Configuration/TestDataSettings.cs | 16 + .../Extensions/UmbracoBuilderExtensions.cs | 54 +++ src/Umbraco.TestData/LoadTestComponent.cs | 35 -- src/Umbraco.TestData/LoadTestComposer.cs | 24 +- src/Umbraco.TestData/LoadTestController.cs | 309 +++++++++--------- .../Properties/AssemblyInfo.cs | 36 -- src/Umbraco.TestData/SegmentTestController.cs | 62 +++- src/Umbraco.TestData/Umbraco.TestData.csproj | 88 +---- .../UmbracoTestDataController.cs | 75 +++-- src/Umbraco.TestData/readme.md | 16 +- .../Umbraco.Web.UI.NetCore.csproj | 3 +- 11 files changed, 354 insertions(+), 364 deletions(-) create mode 100644 src/Umbraco.TestData/Configuration/TestDataSettings.cs create mode 100644 src/Umbraco.TestData/Extensions/UmbracoBuilderExtensions.cs delete mode 100644 src/Umbraco.TestData/LoadTestComponent.cs delete mode 100644 src/Umbraco.TestData/Properties/AssemblyInfo.cs diff --git a/src/Umbraco.TestData/Configuration/TestDataSettings.cs b/src/Umbraco.TestData/Configuration/TestDataSettings.cs new file mode 100644 index 0000000000..78084f726a --- /dev/null +++ b/src/Umbraco.TestData/Configuration/TestDataSettings.cs @@ -0,0 +1,16 @@ +namespace Umbraco.TestData.Configuration +{ + public class TestDataSettings + { + /// + /// Gets or sets a value indicating whether the test data generation is enabled. + /// + public bool Enabled { get; set; } = false; + + /// + /// Gets or sets a value indicating whether persisted local database cache files for content and media are disabled. + /// + /// The URL path. + public bool IgnoreLocalDb { get; set; } = false; + } +} diff --git a/src/Umbraco.TestData/Extensions/UmbracoBuilderExtensions.cs b/src/Umbraco.TestData/Extensions/UmbracoBuilderExtensions.cs new file mode 100644 index 0000000000..a11e7b12ef --- /dev/null +++ b/src/Umbraco.TestData/Extensions/UmbracoBuilderExtensions.cs @@ -0,0 +1,54 @@ +using System.Linq; +using Microsoft.AspNetCore.Builder; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Umbraco.Cms.Core.DependencyInjection; +using Umbraco.Cms.Infrastructure.PublishedCache; +using Umbraco.Cms.Web.Common.ApplicationBuilder; +using Umbraco.TestData.Configuration; + +namespace Umbraco.TestData.Extensions +{ + public static class UmbracoBuilderExtensions + { + public static IUmbracoBuilder AddUmbracoTestData(this IUmbracoBuilder builder) + { + if (builder.Services.Any(x => x.ServiceType == typeof(LoadTestController))) + { + // We assume forms are composed if any implementations of LoadTestController exists + return builder; + } + + IConfigurationSection testDataSection = builder.Config.GetSection("Umbraco:CMS:TestData"); + TestDataSettings config = testDataSection.Get(); + if (config == null || config.Enabled == false) + { + return builder; + } + + builder.Services.Configure(testDataSection); + + if (config.IgnoreLocalDb) + { + builder.Services.AddSingleton(factory => new PublishedSnapshotServiceOptions + { + IgnoreLocalDb = true + }); + } + + builder.Services.Configure(options => + options.AddFilter(new UmbracoPipelineFilter(nameof(LoadTestController)) + { + Endpoints = app => app.UseEndpoints(endpoints => + endpoints.MapControllerRoute( + "LoadTest", + "/LoadTest/{action}", + new { controller = "LoadTest", Action = "Index" })) + })); + + builder.Services.AddScoped(typeof(LoadTestController)); + + return builder; + } + } +} diff --git a/src/Umbraco.TestData/LoadTestComponent.cs b/src/Umbraco.TestData/LoadTestComponent.cs deleted file mode 100644 index cfd923cd07..0000000000 --- a/src/Umbraco.TestData/LoadTestComponent.cs +++ /dev/null @@ -1,35 +0,0 @@ -using System.Web.Mvc; -using System.Web.Routing; -using System.Configuration; -using Umbraco.Cms.Core.Composing; - -// see https://github.com/Shazwazza/UmbracoScripts/tree/master/src/LoadTesting - -namespace Umbraco.TestData -{ - public class LoadTestComponent : IComponent - { - public void Initialize() - { - if (ConfigurationManager.AppSettings["Umbraco.TestData.Enabled"] != "true") - return; - - - - RouteTable.Routes.MapRoute( - name: "LoadTest", - url: "LoadTest/{action}", - defaults: new - { - controller = "LoadTest", - action = "Index" - }, - namespaces: new[] { "Umbraco.TestData" } - ); - } - - public void Terminate() - { - } - } -} diff --git a/src/Umbraco.TestData/LoadTestComposer.cs b/src/Umbraco.TestData/LoadTestComposer.cs index e5b16e5ab1..8d66c4965d 100644 --- a/src/Umbraco.TestData/LoadTestComposer.cs +++ b/src/Umbraco.TestData/LoadTestComposer.cs @@ -1,31 +1,13 @@ -using System.Configuration; -using Microsoft.Extensions.DependencyInjection; using Umbraco.Cms.Core.Composing; using Umbraco.Cms.Core.DependencyInjection; -using Umbraco.Cms.Infrastructure.PublishedCache; +using Umbraco.TestData.Extensions; // see https://github.com/Shazwazza/UmbracoScripts/tree/master/src/LoadTesting namespace Umbraco.TestData { - public class LoadTestComposer : ComponentComposer, IUserComposer + public class LoadTestComposer : IUserComposer { - public override void Compose(IUmbracoBuilder builder) - { - base.Compose(builder); - - if (ConfigurationManager.AppSettings["Umbraco.TestData.Enabled"] != "true") - return; - - builder.Services.AddScoped(typeof(LoadTestController), typeof(LoadTestController)); - - if (ConfigurationManager.AppSettings["Umbraco.TestData.IgnoreLocalDb"] == "true") - { - builder.Services.AddSingleton(factory => new PublishedSnapshotServiceOptions - { - IgnoreLocalDb = true - }); - } - } + public void Compose(IUmbracoBuilder builder) => builder.AddUmbracoTestData(); } } diff --git a/src/Umbraco.TestData/LoadTestController.cs b/src/Umbraco.TestData/LoadTestController.cs index e1494fbdab..3033d2febb 100644 --- a/src/Umbraco.TestData/LoadTestController.cs +++ b/src/Umbraco.TestData/LoadTestController.cs @@ -1,21 +1,15 @@ -using System; -using System.Configuration; +using System; using System.Diagnostics; +using System.IO; using System.Linq; +using System.Text; using System.Threading; -using System.Web; -using System.Web.Hosting; -using System.Web.Mvc; -using System.Web.Routing; -using Microsoft.Extensions.DependencyInjection; -using Umbraco.Cms.Core.Composing; -using Umbraco.Cms.Core.DependencyInjection; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Hosting; +using Umbraco.Cms.Core; using Umbraco.Cms.Core.Models; using Umbraco.Cms.Core.Services; using Umbraco.Cms.Core.Strings; -using System.IO; -using Umbraco.Cms.Core; -using Umbraco.Cms.Core.Hosting; // see https://github.com/Shazwazza/UmbracoScripts/tree/master/src/LoadTesting @@ -23,27 +17,17 @@ namespace Umbraco.TestData { public class LoadTestController : Controller { - public LoadTestController( - ServiceContext serviceContext, - IShortStringHelper shortStringHelper, - IHostingEnvironment hostingEnvironment) - { - _serviceContext = serviceContext; - _shortStringHelper = shortStringHelper; - _hostingEnvironment = hostingEnvironment; - } + private static readonly Random s_random = new Random(); + private static readonly object s_locko = new object(); - private static readonly Random _random = new Random(); - private static readonly object _locko = new object(); + private static volatile int s_containerId = -1; - private static volatile int _containerId = -1; + private const string ContainerAlias = "LoadTestContainer"; + private const string ContentAlias = "LoadTestContent"; + private const int TextboxDefinitionId = -88; + private const int MaxCreate = 1000; - private const string _containerAlias = "LoadTestContainer"; - private const string _contentAlias = "LoadTestContent"; - private const int _textboxDefinitionId = -88; - private const int _maxCreate = 1000; - - private static readonly string HeadHtml = @" + private static readonly string s_headHtml = @" LoadTest