From e368710364a99f7a13a9bc9391a755888f2ea669 Mon Sep 17 00:00:00 2001 From: Nikolaj Geisle <70372949+Zeegaan@users.noreply.github.com> Date: Fri, 6 Dec 2024 13:20:57 +0100 Subject: [PATCH 1/6] V15: Add custom serializer for hybrid cache (#17727) * Add custom serializer * Add migration to rebuild cache * Rename migration namespace to 15.1 * Also clear media cache * Remove failed cache items * Refactor to only use keys for document cache repository --------- Co-authored-by: nikolajlauridsen --- .../Migrations/Upgrade/UmbracoPlan.cs | 1 + .../Upgrade/V_15_1_0/RebuildCacheMigration.cs | 23 ++++++++ .../ContentCacheNode.cs | 2 +- .../ContentData.cs | 2 +- .../UmbracoBuilderExtensions.cs | 3 +- .../Persistence/DatabaseCacheRepository.cs | 43 -------------- .../Persistence/IDatabaseCacheRepository.cs | 4 -- .../Serialization/HybridCacheSerializer.cs | 40 +++++++++++++ .../Services/DocumentCacheService.cs | 58 ++++++++++--------- .../Services/MediaCacheService.cs | 32 +++++----- .../DocumentHybridCacheMockTests.cs | 17 +++--- 11 files changed, 123 insertions(+), 102 deletions(-) create mode 100644 src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_1_0/RebuildCacheMigration.cs create mode 100644 src/Umbraco.PublishedCache.HybridCache/Serialization/HybridCacheSerializer.cs diff --git a/src/Umbraco.Infrastructure/Migrations/Upgrade/UmbracoPlan.cs b/src/Umbraco.Infrastructure/Migrations/Upgrade/UmbracoPlan.cs index 745e8fb5af..7d92bb9385 100644 --- a/src/Umbraco.Infrastructure/Migrations/Upgrade/UmbracoPlan.cs +++ b/src/Umbraco.Infrastructure/Migrations/Upgrade/UmbracoPlan.cs @@ -104,5 +104,6 @@ public class UmbracoPlan : MigrationPlan To("{9D3CE7D4-4884-41D4-98E8-302EB6CB0CF6}"); To("{37875E80-5CDD-42FF-A21A-7D4E3E23E0ED}"); To("{42E44F9E-7262-4269-922D-7310CB48E724}"); + To("{7B51B4DE-5574-4484-993E-05D12D9ED703}"); } } diff --git a/src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_1_0/RebuildCacheMigration.cs b/src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_1_0/RebuildCacheMigration.cs new file mode 100644 index 0000000000..5ef83f73ca --- /dev/null +++ b/src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_1_0/RebuildCacheMigration.cs @@ -0,0 +1,23 @@ +using Umbraco.Cms.Core.PublishedCache; + +namespace Umbraco.Cms.Infrastructure.Migrations.Upgrade.V_15_1_0; + +[Obsolete("Will be removed in V18")] +public class RebuildCacheMigration : MigrationBase +{ + private readonly IDocumentCacheService _documentCacheService; + private readonly IMediaCacheService _mediaCacheService; + + public RebuildCacheMigration(IMigrationContext context, IDocumentCacheService documentCacheService, IMediaCacheService mediaCacheService) : base(context) + { + _documentCacheService = documentCacheService; + _mediaCacheService = mediaCacheService; + } + + protected override void Migrate() + { + _documentCacheService.ClearMemoryCacheAsync(CancellationToken.None).GetAwaiter().GetResult(); + _mediaCacheService.ClearMemoryCacheAsync(CancellationToken.None).GetAwaiter().GetResult(); + } + +} diff --git a/src/Umbraco.PublishedCache.HybridCache/ContentCacheNode.cs b/src/Umbraco.PublishedCache.HybridCache/ContentCacheNode.cs index e72d4f234b..bdd4364844 100644 --- a/src/Umbraco.PublishedCache.HybridCache/ContentCacheNode.cs +++ b/src/Umbraco.PublishedCache.HybridCache/ContentCacheNode.cs @@ -4,7 +4,7 @@ namespace Umbraco.Cms.Infrastructure.HybridCache; // This is for cache performance reasons, see https://learn.microsoft.com/en-us/aspnet/core/performance/caching/hybrid?view=aspnetcore-9.0#reuse-objects [ImmutableObject(true)] -internal sealed class ContentCacheNode +public sealed class ContentCacheNode { public int Id { get; set; } diff --git a/src/Umbraco.PublishedCache.HybridCache/ContentData.cs b/src/Umbraco.PublishedCache.HybridCache/ContentData.cs index c314241479..e3304e7139 100644 --- a/src/Umbraco.PublishedCache.HybridCache/ContentData.cs +++ b/src/Umbraco.PublishedCache.HybridCache/ContentData.cs @@ -7,7 +7,7 @@ namespace Umbraco.Cms.Infrastructure.HybridCache; /// // This is for cache performance reasons, see https://learn.microsoft.com/en-us/aspnet/core/performance/caching/hybrid?view=aspnetcore-9.0#reuse-objects [ImmutableObject(true)] -internal sealed class ContentData +public sealed class ContentData { public ContentData(string? name, string? urlSegment, int versionId, DateTime versionDate, int writerId, int? templateId, bool published, Dictionary? properties, IReadOnlyDictionary? cultureInfos) { diff --git a/src/Umbraco.PublishedCache.HybridCache/DependencyInjection/UmbracoBuilderExtensions.cs b/src/Umbraco.PublishedCache.HybridCache/DependencyInjection/UmbracoBuilderExtensions.cs index 031ab0d0c5..ca625dacdf 100644 --- a/src/Umbraco.PublishedCache.HybridCache/DependencyInjection/UmbracoBuilderExtensions.cs +++ b/src/Umbraco.PublishedCache.HybridCache/DependencyInjection/UmbracoBuilderExtensions.cs @@ -1,7 +1,6 @@  using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; -using Umbraco.Cms.Core.Cache; using Umbraco.Cms.Core.Configuration.Models; using Umbraco.Cms.Core.DependencyInjection; using Umbraco.Cms.Core.Notifications; @@ -35,7 +34,7 @@ public static class UmbracoBuilderExtensions // We'll be a bit friendlier and default this to a higher value, you quickly hit the 1MB limit with a few languages and especially blocks. // This can be overwritten later if needed. options.MaximumPayloadBytes = 1024 * 1024 * 100; // 100MB - }); + }).AddSerializer(); #pragma warning restore EXTEXP0018 builder.Services.AddSingleton(); builder.Services.AddSingleton(); diff --git a/src/Umbraco.PublishedCache.HybridCache/Persistence/DatabaseCacheRepository.cs b/src/Umbraco.PublishedCache.HybridCache/Persistence/DatabaseCacheRepository.cs index df357c90c9..8947d37027 100644 --- a/src/Umbraco.PublishedCache.HybridCache/Persistence/DatabaseCacheRepository.cs +++ b/src/Umbraco.PublishedCache.HybridCache/Persistence/DatabaseCacheRepository.cs @@ -189,30 +189,6 @@ AND cmsContentNu.nodeId IS NULL return count == 0; } - public async Task GetContentSourceAsync(int id, bool preview = false) - { - Sql? sql = SqlContentSourcesSelect() - .Append(SqlObjectTypeNotTrashed(SqlContext, Constants.ObjectTypes.Document)) - .Append(SqlWhereNodeId(SqlContext, id)) - .Append(SqlOrderByLevelIdSortOrder(SqlContext)); - - ContentSourceDto? dto = await Database.FirstOrDefaultAsync(sql); - - if (dto == null) - { - return null; - } - - if (preview is false && dto.PubDataRaw is null && dto.PubData is null) - { - return null; - } - - IContentCacheDataSerializer serializer = - _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Document); - return CreateContentNodeKit(dto, serializer, preview); - } - public async Task GetContentSourceAsync(Guid key, bool preview = false) { Sql? sql = SqlContentSourcesSelect() @@ -292,25 +268,6 @@ AND cmsContentNu.nodeId IS NULL public IEnumerable GetDocumentKeysByContentTypeKeys(IEnumerable keys, bool published = false) => GetContentSourceByDocumentTypeKey(keys, Constants.ObjectTypes.Document).Where(x => x.Published == published).Select(x => x.Key); - public async Task GetMediaSourceAsync(int id) - { - Sql? sql = SqlMediaSourcesSelect() - .Append(SqlObjectTypeNotTrashed(SqlContext, Constants.ObjectTypes.Media)) - .Append(SqlWhereNodeId(SqlContext, id)) - .Append(SqlOrderByLevelIdSortOrder(SqlContext)); - - ContentSourceDto? dto = await Database.FirstOrDefaultAsync(sql); - - if (dto is null) - { - return null; - } - - IContentCacheDataSerializer serializer = - _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Media); - return CreateMediaNodeKit(dto, serializer); - } - public async Task GetMediaSourceAsync(Guid key) { Sql? sql = SqlMediaSourcesSelect() diff --git a/src/Umbraco.PublishedCache.HybridCache/Persistence/IDatabaseCacheRepository.cs b/src/Umbraco.PublishedCache.HybridCache/Persistence/IDatabaseCacheRepository.cs index 93a589d926..129e78bc2f 100644 --- a/src/Umbraco.PublishedCache.HybridCache/Persistence/IDatabaseCacheRepository.cs +++ b/src/Umbraco.PublishedCache.HybridCache/Persistence/IDatabaseCacheRepository.cs @@ -7,12 +7,8 @@ internal interface IDatabaseCacheRepository { Task DeleteContentItemAsync(int id); - Task GetContentSourceAsync(int id, bool preview = false); - Task GetContentSourceAsync(Guid key, bool preview = false); - Task GetMediaSourceAsync(int id); - Task GetMediaSourceAsync(Guid key); diff --git a/src/Umbraco.PublishedCache.HybridCache/Serialization/HybridCacheSerializer.cs b/src/Umbraco.PublishedCache.HybridCache/Serialization/HybridCacheSerializer.cs new file mode 100644 index 0000000000..dbe88358df --- /dev/null +++ b/src/Umbraco.PublishedCache.HybridCache/Serialization/HybridCacheSerializer.cs @@ -0,0 +1,40 @@ +using System.Buffers; +using MessagePack; +using MessagePack.Resolvers; +using Microsoft.Extensions.Caching.Hybrid; +using Microsoft.Extensions.Logging; + +namespace Umbraco.Cms.Infrastructure.HybridCache.Serialization; + +internal class HybridCacheSerializer : IHybridCacheSerializer +{ + private readonly ILogger _logger; + private readonly MessagePackSerializerOptions _options; + + public HybridCacheSerializer(ILogger logger) + { + _logger = logger; + MessagePackSerializerOptions defaultOptions = ContractlessStandardResolver.Options; + IFormatterResolver resolver = CompositeResolver.Create(defaultOptions.Resolver); + + _options = defaultOptions + .WithResolver(resolver) + .WithCompression(MessagePackCompression.Lz4BlockArray) + .WithSecurity(MessagePackSecurity.UntrustedData); + } + + public ContentCacheNode Deserialize(ReadOnlySequence source) + { + try + { + return MessagePackSerializer.Deserialize(source, _options); + } + catch (MessagePackSerializationException ex) + { + _logger.LogError(ex, "Error deserializing ContentCacheNode"); + return null!; + } + } + + public void Serialize(ContentCacheNode value, IBufferWriter target) => target.Write(MessagePackSerializer.Serialize(value, _options)); +} diff --git a/src/Umbraco.PublishedCache.HybridCache/Services/DocumentCacheService.cs b/src/Umbraco.PublishedCache.HybridCache/Services/DocumentCacheService.cs index e3bb827393..d1e80d9509 100644 --- a/src/Umbraco.PublishedCache.HybridCache/Services/DocumentCacheService.cs +++ b/src/Umbraco.PublishedCache.HybridCache/Services/DocumentCacheService.cs @@ -74,23 +74,7 @@ internal sealed class DocumentCacheService : IDocumentCacheService { bool calculatedPreview = preview ?? GetPreview(); - ContentCacheNode? contentCacheNode = await _hybridCache.GetOrCreateAsync( - GetCacheKey(key, calculatedPreview), // Unique key to the cache entry - async cancel => - { - using ICoreScope scope = _scopeProvider.CreateCoreScope(); - ContentCacheNode? contentCacheNode = await _databaseCacheRepository.GetContentSourceAsync(key, calculatedPreview); - scope.Complete(); - return contentCacheNode; - }, - GetEntryOptions(key)); - - return contentCacheNode is null ? null : _publishedContentFactory.ToIPublishedContent(contentCacheNode, calculatedPreview).CreateModel(_publishedModelFactory); - } - - private bool GetPreview() - { - return _previewService.IsInPreview(); + return await GetNodeAsync(key, calculatedPreview); } public async Task GetByIdAsync(int id, bool? preview = null) @@ -104,17 +88,37 @@ internal sealed class DocumentCacheService : IDocumentCacheService bool calculatedPreview = preview ?? GetPreview(); Guid key = keyAttempt.Result; - ContentCacheNode? contentCacheNode = await _hybridCache.GetOrCreateAsync( - GetCacheKey(keyAttempt.Result, calculatedPreview), // Unique key to the cache entry - async cancel => - { - using ICoreScope scope = _scopeProvider.CreateCoreScope(); - ContentCacheNode? contentCacheNode = await _databaseCacheRepository.GetContentSourceAsync(id, calculatedPreview); - scope.Complete(); - return contentCacheNode; - }, GetEntryOptions(key)); + return await GetNodeAsync(key, calculatedPreview); + } - return contentCacheNode is null ? null : _publishedContentFactory.ToIPublishedContent(contentCacheNode, calculatedPreview).CreateModel(_publishedModelFactory);; + private async Task GetNodeAsync(Guid key, bool preview) + { + var cacheKey = GetCacheKey(key, preview); + + ContentCacheNode? contentCacheNode = await _hybridCache.GetOrCreateAsync( + cacheKey, + async cancel => + { + using ICoreScope scope = _scopeProvider.CreateCoreScope(); + ContentCacheNode? contentCacheNode = await _databaseCacheRepository.GetContentSourceAsync(key, preview); + scope.Complete(); + return contentCacheNode; + }, + GetEntryOptions(key)); + + // We don't want to cache removed items, this may cause issues if the L2 serializer changes. + if (contentCacheNode is null) + { + await _hybridCache.RemoveAsync(cacheKey); + return null; + } + + return _publishedContentFactory.ToIPublishedContent(contentCacheNode, preview).CreateModel(_publishedModelFactory); + } + + private bool GetPreview() + { + return _previewService.IsInPreview(); } public IEnumerable GetByContentType(IPublishedContentType contentType) diff --git a/src/Umbraco.PublishedCache.HybridCache/Services/MediaCacheService.cs b/src/Umbraco.PublishedCache.HybridCache/Services/MediaCacheService.cs index af5396262e..7913193512 100644 --- a/src/Umbraco.PublishedCache.HybridCache/Services/MediaCacheService.cs +++ b/src/Umbraco.PublishedCache.HybridCache/Services/MediaCacheService.cs @@ -76,17 +76,7 @@ internal class MediaCacheService : IMediaCacheService return null; } - ContentCacheNode? contentCacheNode = await _hybridCache.GetOrCreateAsync( - $"{key}", // Unique key to the cache entry - async cancel => - { - using ICoreScope scope = _scopeProvider.CreateCoreScope(); - ContentCacheNode? mediaCacheNode = await _databaseCacheRepository.GetMediaSourceAsync(idAttempt.Result); - scope.Complete(); - return mediaCacheNode; - }, GetEntryOptions(key)); - - return contentCacheNode is null ? null : _publishedContentFactory.ToIPublishedMedia(contentCacheNode).CreateModel(_publishedModelFactory); + return await GetNodeAsync(key); } public async Task GetByIdAsync(int id) @@ -96,19 +86,33 @@ internal class MediaCacheService : IMediaCacheService { return null; } + Guid key = keyAttempt.Result; + return await GetNodeAsync(key); + } + + private async Task GetNodeAsync(Guid key) + { + var cacheKey = $"{key}"; ContentCacheNode? contentCacheNode = await _hybridCache.GetOrCreateAsync( - $"{keyAttempt.Result}", // Unique key to the cache entry + cacheKey, // Unique key to the cache entry async cancel => { using ICoreScope scope = _scopeProvider.CreateCoreScope(); - ContentCacheNode? mediaCacheNode = await _databaseCacheRepository.GetMediaSourceAsync(id); + ContentCacheNode? mediaCacheNode = await _databaseCacheRepository.GetMediaSourceAsync(key); scope.Complete(); return mediaCacheNode; }, GetEntryOptions(key)); - return contentCacheNode is null ? null : _publishedContentFactory.ToIPublishedMedia(contentCacheNode).CreateModel(_publishedModelFactory); + // We don't want to cache removed items, this may cause issues if the L2 serializer changes. + if (contentCacheNode is null) + { + await _hybridCache.RemoveAsync(cacheKey); + return null; + } + + return _publishedContentFactory.ToIPublishedMedia(contentCacheNode).CreateModel(_publishedModelFactory); } public async Task HasContentByIdAsync(int id) diff --git a/tests/Umbraco.Tests.Integration/Umbraco.PublishedCache.HybridCache/DocumentHybridCacheMockTests.cs b/tests/Umbraco.Tests.Integration/Umbraco.PublishedCache.HybridCache/DocumentHybridCacheMockTests.cs index 065e2db3cd..32988c4472 100644 --- a/tests/Umbraco.Tests.Integration/Umbraco.PublishedCache.HybridCache/DocumentHybridCacheMockTests.cs +++ b/tests/Umbraco.Tests.Integration/Umbraco.PublishedCache.HybridCache/DocumentHybridCacheMockTests.cs @@ -76,12 +76,6 @@ public class DocumentHybridCacheMockTests : UmbracoIntegrationTestWithContent IsDraft = false, }; - _mockedNucacheRepository.Setup(r => r.GetContentSourceAsync(It.IsAny(), true)) - .ReturnsAsync(draftTestCacheNode); - - _mockedNucacheRepository.Setup(r => r.GetContentSourceAsync(It.IsAny(), false)) - .ReturnsAsync(publishedTestCacheNode); - _mockedNucacheRepository.Setup(r => r.GetContentSourceAsync(It.IsAny(), true)) .ReturnsAsync(draftTestCacheNode); @@ -153,7 +147,7 @@ public class DocumentHybridCacheMockTests : UmbracoIntegrationTestWithContent var textPage2 = await _mockedCache.GetByIdAsync(Textpage.Id, true); AssertTextPage(textPage); AssertTextPage(textPage2); - _mockedNucacheRepository.Verify(x => x.GetContentSourceAsync(It.IsAny(), It.IsAny()), Times.Exactly(1)); + _mockedNucacheRepository.Verify(x => x.GetContentSourceAsync(It.IsAny(), It.IsAny()), Times.Exactly(1)); } [Test] @@ -171,10 +165,12 @@ public class DocumentHybridCacheMockTests : UmbracoIntegrationTestWithContent _cacheSettings.ContentTypeKeys = [ Textpage.ContentType.Key ]; await _mockDocumentCacheService.SeedAsync(CancellationToken.None); + _mockedNucacheRepository.Verify(x => x.GetContentSourceAsync(It.IsAny(), It.IsAny()), Times.Exactly(1)); + var textPage = await _mockedCache.GetByIdAsync(Textpage.Id); AssertTextPage(textPage); - _mockedNucacheRepository.Verify(x => x.GetContentSourceAsync(It.IsAny(), It.IsAny()), Times.Exactly(0)); + _mockedNucacheRepository.Verify(x => x.GetContentSourceAsync(It.IsAny(), It.IsAny()), Times.Exactly(1)); } [Test] @@ -192,10 +188,11 @@ public class DocumentHybridCacheMockTests : UmbracoIntegrationTestWithContent _cacheSettings.ContentTypeKeys = [ Textpage.ContentType.Key ]; await _mockDocumentCacheService.SeedAsync(CancellationToken.None); + _mockedNucacheRepository.Verify(x => x.GetContentSourceAsync(It.IsAny(), It.IsAny()), Times.Exactly(1)); var textPage = await _mockedCache.GetByIdAsync(Textpage.Key); AssertTextPage(textPage); - _mockedNucacheRepository.Verify(x => x.GetContentSourceAsync(It.IsAny(), It.IsAny()), Times.Exactly(0)); + _mockedNucacheRepository.Verify(x => x.GetContentSourceAsync(It.IsAny(), It.IsAny()), Times.Exactly(1)); } [Test] @@ -209,7 +206,7 @@ public class DocumentHybridCacheMockTests : UmbracoIntegrationTestWithContent var textPage = await _mockedCache.GetByIdAsync(Textpage.Id, true); AssertTextPage(textPage); - _mockedNucacheRepository.Verify(x => x.GetContentSourceAsync(It.IsAny(), It.IsAny()), Times.Exactly(1)); + _mockedNucacheRepository.Verify(x => x.GetContentSourceAsync(It.IsAny(), It.IsAny()), Times.Exactly(1)); } [Test] From ca57e7f516a02d743cedea7590c70ef19c5649dd Mon Sep 17 00:00:00 2001 From: nikolajlauridsen Date: Mon, 9 Dec 2024 09:15:01 +0100 Subject: [PATCH 2/6] Set version --- version.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.json b/version.json index 6e1550ea5c..e3f7978da7 100644 --- a/version.json +++ b/version.json @@ -1,6 +1,6 @@ { "$schema": "https://raw.githubusercontent.com/dotnet/Nerdbank.GitVersioning/main/src/NerdBank.GitVersioning/version.schema.json", - "version": "15.1.0", + "version": "15.1.0-rc2", "assemblyVersion": { "precision": "build" }, From e757b90ee18ebbf6911bbe37171676e44d5815e3 Mon Sep 17 00:00:00 2001 From: nikolajlauridsen Date: Mon, 9 Dec 2024 10:51:08 +0100 Subject: [PATCH 3/6] Bump to final --- version.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.json b/version.json index e3f7978da7..6e1550ea5c 100644 --- a/version.json +++ b/version.json @@ -1,6 +1,6 @@ { "$schema": "https://raw.githubusercontent.com/dotnet/Nerdbank.GitVersioning/main/src/NerdBank.GitVersioning/version.schema.json", - "version": "15.1.0-rc2", + "version": "15.1.0", "assemblyVersion": { "precision": "build" }, From 03eb878d30464c682a28d92ad4f19cdf95515a01 Mon Sep 17 00:00:00 2001 From: Mole Date: Tue, 10 Dec 2024 10:25:50 +0100 Subject: [PATCH 4/6] V15: Fix docker (#17722) * Bump dotnet version * Ensure LF line endings for shell scripts --- .gitattributes | 1 + templates/UmbracoProject/Dockerfile | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitattributes b/.gitattributes index 8ac58cf5fe..6e707b7796 100644 --- a/.gitattributes +++ b/.gitattributes @@ -46,6 +46,7 @@ *.xml text=auto *.resx text=auto *.yml text eol=lf core.whitespace whitespace=tab-in-indent,trailing-space,tabwidth=2 +*.sh eol=lf *.csproj text=auto merge=union *.vbproj text=auto merge=union diff --git a/templates/UmbracoProject/Dockerfile b/templates/UmbracoProject/Dockerfile index e3eda648dd..0e1b379b20 100644 --- a/templates/UmbracoProject/Dockerfile +++ b/templates/UmbracoProject/Dockerfile @@ -1,10 +1,10 @@ -FROM mcr.microsoft.com/dotnet/aspnet:8.0 AS base +FROM mcr.microsoft.com/dotnet/aspnet:9.0 AS base USER $APP_UID WORKDIR /app EXPOSE 8080 EXPOSE 8081 -FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build +FROM mcr.microsoft.com/dotnet/sdk:9.0 AS build ARG BUILD_CONFIGURATION=Release WORKDIR /src COPY ["UmbracoProject/UmbracoProject.csproj", "UmbracoProject/"] From 888c96e77746a608629361d4f9d9f5e90ca3ee85 Mon Sep 17 00:00:00 2001 From: Jacob Overgaard <752371+iOvergaard@users.noreply.github.com> Date: Tue, 10 Dec 2024 15:14:58 +0100 Subject: [PATCH 5/6] build: change the fetchDepth to version the backoffice package --- build/azure-pipelines.yml | 51 +++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/build/azure-pipelines.yml b/build/azure-pipelines.yml index 1a60bb492a..810027c643 100644 --- a/build/azure-pipelines.yml +++ b/build/azure-pipelines.yml @@ -117,8 +117,7 @@ stages: - checkout: self submodules: false lfs: false, - fetchDepth: 1 - fetchFilter: tree:0 + fetchDepth: 500 - template: templates/backoffice-install.yml - script: npm run build:for:npm displayName: Run build:for:npm @@ -302,27 +301,27 @@ stages: LinuxPart1Of3: vmImage: "ubuntu-latest" # Filter tests that are part of the Umbraco.Infrastructure namespace but not part of the Umbraco.Infrastructure.Service namespace - testFilter: '(FullyQualifiedName~Umbraco.Infrastructure) & (FullyQualifiedName!~Umbraco.Infrastructure.Service)' + testFilter: "(FullyQualifiedName~Umbraco.Infrastructure) & (FullyQualifiedName!~Umbraco.Infrastructure.Service)" LinuxPart2Of3: vmImage: "ubuntu-latest" # Filter tests that are part of the Umbraco.Infrastructure.Service namespace - testFilter: '(FullyQualifiedName~Umbraco.Infrastructure.Service)' + testFilter: "(FullyQualifiedName~Umbraco.Infrastructure.Service)" LinuxPart3Of3: vmImage: "ubuntu-latest" # Filter tests that are not part of the Umbraco.Infrastructure namespace. So this will run all tests that are not part of the Umbraco.Infrastructure namespace - testFilter: '(FullyQualifiedName!~Umbraco.Infrastructure)' + testFilter: "(FullyQualifiedName!~Umbraco.Infrastructure)" macOSPart1Of3: vmImage: "macOS-latest" # Filter tests that are part of the Umbraco.Infrastructure namespace but not part of the Umbraco.Infrastructure.Service namespace - testFilter: '(FullyQualifiedName~Umbraco.Infrastructure) & (FullyQualifiedName!~Umbraco.Infrastructure.Service)' + testFilter: "(FullyQualifiedName~Umbraco.Infrastructure) & (FullyQualifiedName!~Umbraco.Infrastructure.Service)" macOSPart2Of3: vmImage: "macOS-latest" # Filter tests that are part of the Umbraco.Infrastructure.Service namespace - testFilter: '(FullyQualifiedName~Umbraco.Infrastructure.Service)' + testFilter: "(FullyQualifiedName~Umbraco.Infrastructure.Service)" macOSPart3Of3: vmImage: "macOS-latest" # Filter tests that are not part of the Umbraco.Infrastructure namespace. - testFilter: '(FullyQualifiedName!~Umbraco.Infrastructure)' + testFilter: "(FullyQualifiedName!~Umbraco.Infrastructure)" pool: vmImage: $(vmImage) variables: @@ -374,40 +373,40 @@ stages: Tests__Database__DatabaseType: LocalDb Tests__Database__SQLServerMasterConnectionString: N/A # Filter tests that are part of the Umbraco.Infrastructure namespace but not part of the Umbraco.Infrastructure.Service namespace - testFilter: '(FullyQualifiedName~Umbraco.Infrastructure) & (FullyQualifiedName!~Umbraco.Infrastructure.Service)' + testFilter: "(FullyQualifiedName~Umbraco.Infrastructure) & (FullyQualifiedName!~Umbraco.Infrastructure.Service)" WindowsPart2Of3: vmImage: "windows-latest" Tests__Database__DatabaseType: LocalDb Tests__Database__SQLServerMasterConnectionString: N/A # Filter tests that are part of the Umbraco.Infrastructure.Service namespace - testFilter: '(FullyQualifiedName~Umbraco.Infrastructure.Service)' + testFilter: "(FullyQualifiedName~Umbraco.Infrastructure.Service)" WindowsPart3Of3: vmImage: "windows-latest" Tests__Database__DatabaseType: LocalDb Tests__Database__SQLServerMasterConnectionString: N/A # Filter tests that are not part of the Umbraco.Infrastructure namespace. So this will run all tests that are not part of the Umbraco.Infrastructure namespace - testFilter: '(FullyQualifiedName!~Umbraco.Infrastructure)' + testFilter: "(FullyQualifiedName!~Umbraco.Infrastructure)" LinuxPart1Of3: vmImage: "ubuntu-latest" SA_PASSWORD: UmbracoIntegration123! Tests__Database__DatabaseType: SqlServer Tests__Database__SQLServerMasterConnectionString: "Server=(local);User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True" # Filter tests that are part of the Umbraco.Infrastructure namespace but not part of the Umbraco.Infrastructure.Service namespace - testFilter: '(FullyQualifiedName~Umbraco.Infrastructure) & (FullyQualifiedName!~Umbraco.Infrastructure.Service)' + testFilter: "(FullyQualifiedName~Umbraco.Infrastructure) & (FullyQualifiedName!~Umbraco.Infrastructure.Service)" LinuxPart2Of3: vmImage: "ubuntu-latest" SA_PASSWORD: UmbracoIntegration123! Tests__Database__DatabaseType: SqlServer Tests__Database__SQLServerMasterConnectionString: "Server=(local);User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True" # Filter tests that are part of the Umbraco.Infrastructure.Service namespace - testFilter: '(FullyQualifiedName~Umbraco.Infrastructure.Service)' + testFilter: "(FullyQualifiedName~Umbraco.Infrastructure.Service)" LinuxPart3Of3: vmImage: "ubuntu-latest" SA_PASSWORD: UmbracoIntegration123! Tests__Database__DatabaseType: SqlServer Tests__Database__SQLServerMasterConnectionString: "Server=(local);User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True" # Filter tests that are not part of the Umbraco.Infrastructure namespace. So this will run all tests that are not part of the Umbraco.Infrastructure namespace - testFilter: '(FullyQualifiedName!~Umbraco.Infrastructure)' + testFilter: "(FullyQualifiedName!~Umbraco.Infrastructure)" pool: vmImage: $(vmImage) steps: @@ -493,22 +492,22 @@ stages: matrix: LinuxPart1Of3: vmImage: "ubuntu-latest" - testCommand: "npx playwright test DefaultConfig --grep \"@smoke\"--shard=1/3" + testCommand: 'npx playwright test DefaultConfig --grep "@smoke"--shard=1/3' LinuxPart2Of3: vmImage: "ubuntu-latest" - testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --shard=2/3" + testCommand: 'npx playwright test DefaultConfig --grep "@smoke" --shard=2/3' LinuxPart3Of3: vmImage: "ubuntu-latest" - testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --shard=3/3" + testCommand: 'npx playwright test DefaultConfig --grep "@smoke" --shard=3/3' WindowsPart1Of3: vmImage: "windows-latest" - testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --shard=1/3" + testCommand: 'npx playwright test DefaultConfig --grep "@smoke" --shard=1/3' WindowsPart2Of3: vmImage: "windows-latest" - testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --shard=2/3" + testCommand: 'npx playwright test DefaultConfig --grep "@smoke" --shard=2/3' WindowsPart3Of3: vmImage: "windows-latest" - testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --shard=3/3" + testCommand: 'npx playwright test DefaultConfig --grep "@smoke" --shard=3/3' pool: vmImage: $(vmImage) steps: @@ -636,29 +635,29 @@ stages: matrix: ${{ if eq(parameters.sqlServerLinuxAcceptanceTests, True) }}: LinuxPart1Of3: - testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --grep-invert \"Users\" --shard=1/3" + testCommand: 'npx playwright test DefaultConfig --grep "@smoke" --grep-invert "Users" --shard=1/3' vmImage: "ubuntu-latest" SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD) CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True" LinuxPart2Of3: - testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --grep-invert \"Users\" --shard=2/3" + testCommand: 'npx playwright test DefaultConfig --grep "@smoke" --grep-invert "Users" --shard=2/3' vmImage: "ubuntu-latest" SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD) CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True" LinuxPart3Of3: - testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --grep-invert \"Users\" --shard=3/3" + testCommand: 'npx playwright test DefaultConfig --grep "@smoke" --grep-invert "Users" --shard=3/3' vmImage: "ubuntu-latest" SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD) CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True" WindowsPart1Of3: vmImage: "windows-latest" - testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --grep-invert \"Users\" --shard=1/3" + testCommand: 'npx playwright test DefaultConfig --grep "@smoke" --grep-invert "Users" --shard=1/3' WindowsPart2Of3: vmImage: "windows-latest" - testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --grep-invert \"Users\" --shard=2/3" + testCommand: 'npx playwright test DefaultConfig --grep "@smoke" --grep-invert "Users" --shard=2/3' WindowsPart3Of3: vmImage: "windows-latest" - testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --grep-invert \"Users\" --shard=3/3" + testCommand: 'npx playwright test DefaultConfig --grep "@smoke" --grep-invert "Users" --shard=3/3' pool: vmImage: $(vmImage) steps: From 2cfd52a9411d59122a7315fdffcb356d871726f8 Mon Sep 17 00:00:00 2001 From: Jacob Overgaard <752371+iOvergaard@users.noreply.github.com> Date: Tue, 10 Dec 2024 15:31:17 +0100 Subject: [PATCH 6/6] build: overwrite existing files to quiet the extractor --- build/azure-pipelines.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build/azure-pipelines.yml b/build/azure-pipelines.yml index 810027c643..e6d92201b6 100644 --- a/build/azure-pipelines.yml +++ b/build/azure-pipelines.yml @@ -935,6 +935,7 @@ stages: inputs: archiveFilePatterns: $(Build.SourcesDirectory)/csharp-docs.zip destinationFolder: $(Build.ArtifactStagingDirectory)/csharp-docs + overwriteExistingFiles: true - task: AzureFileCopy@4 displayName: "Copy C# Docs to blob storage" inputs: @@ -958,6 +959,7 @@ stages: inputs: archiveFilePatterns: $(Build.SourcesDirectory)/ui-docs.zip destinationFolder: $(Build.ArtifactStagingDirectory)/ui-docs + overwriteExistingFiles: true - task: AzureFileCopy@4 displayName: "Copy Storybook to blob storage" inputs: @@ -981,6 +983,7 @@ stages: inputs: archiveFilePatterns: $(Build.SourcesDirectory)/ui-api-docs.zip destinationFolder: $(Build.ArtifactStagingDirectory)/ui-api-docs + overwriteExistingFiles: true - task: AzureFileCopy@4 displayName: "Copy UI API Docs to blob storage" inputs: