From b8ee161e08d4d00d235b525b43e7e92fd62e35c7 Mon Sep 17 00:00:00 2001
From: Andreas Zerbst <73799582+andr317c@users.noreply.github.com>
Date: Wed, 15 Jan 2025 10:21:36 +0100
Subject: [PATCH] V15 QA updated our E2E pipeline to output test artifacts in a
 clearer way (#17826)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Updated pipeline to publish acceptance test artifacts for each os. and to run tests from npm command

* Adjustment to the targetPath

* Publish all in results folder

* Moved folder

* Reverted changes

* Added all results in test folder

* Updated naming

* Used semicolon

* Additional semicolons

---------

Co-authored-by: Niels Lyngsø <nsl@umbraco.dk>
---
 build/azure-pipelines.yml            | 34 +++++++++++++---------------
 build/nightly-E2E-test-pipelines.yml | 32 +++++++++++++-------------
 2 files changed, 32 insertions(+), 34 deletions(-)

diff --git a/build/azure-pipelines.yml b/build/azure-pipelines.yml
index 865d97081d..318ae7907a 100644
--- a/build/azure-pipelines.yml
+++ b/build/azure-pipelines.yml
@@ -492,22 +492,22 @@ stages:
           matrix:
             LinuxPart1Of3:
               vmImage: "ubuntu-latest"
-              testCommand: 'npx playwright test DefaultConfig --grep "@smoke" --shard=1/3'
+              testCommand: "npm run smokeTest -- --shard=1/3"
             LinuxPart2Of3:
               vmImage: "ubuntu-latest"
-              testCommand: 'npx playwright test DefaultConfig --grep "@smoke" --shard=2/3'
+              testCommand: "npm run smokeTest -- --shard=2/3"
             LinuxPart3Of3:
               vmImage: "ubuntu-latest"
-              testCommand: 'npx playwright test DefaultConfig --grep "@smoke" --shard=3/3'
+              testCommand: "npm run smokeTest -- --shard=3/3"
             WindowsPart1Of3:
               vmImage: "windows-latest"
-              testCommand: 'npx playwright test DefaultConfig --grep "@smoke" --shard=1/3'
+              testCommand: "npm run smokeTest -- --shard=1/3"
             WindowsPart2Of3:
               vmImage: "windows-latest"
-              testCommand: 'npx playwright test DefaultConfig --grep "@smoke" --shard=2/3'
+              testCommand: "npm run smokeTest -- --shard=2/3"
             WindowsPart3Of3:
               vmImage: "windows-latest"
-              testCommand: 'npx playwright test DefaultConfig --grep "@smoke" --shard=3/3'
+              testCommand: "npm run smokeTest -- --shard=3/3"
         pool:
           vmImage: $(vmImage)
         steps:
@@ -612,7 +612,7 @@ stages:
           # Copy artifacts
           - pwsh: |
               if (Test-Path tests/Umbraco.Tests.AcceptanceTest/results/*) {
-                Copy-Item tests/Umbraco.Tests.AcceptanceTest/results $(Build.ArtifactStagingDirectory) -Recurse
+                Copy-Item tests/Umbraco.Tests.AcceptanceTest/results/* $(Build.ArtifactStagingDirectory) -Recurse
               }
             displayName: Copy Playwright results
             condition: succeededOrFailed()
@@ -623,8 +623,7 @@ stages:
             condition: succeededOrFailed()
             inputs:
               targetPath: $(Build.ArtifactStagingDirectory)
-              artifact: "Acceptance Tests - $(Agent.JobName) - Attempt #$(System.JobAttempt)"
-
+              artifact: "Acceptance Test Results - $(Agent.JobName) - Attempt #$(System.JobAttempt)"
       - job:
         displayName: E2E Tests (SQL Server)
         variables:
@@ -635,29 +634,29 @@ stages:
           matrix:
             ${{ if eq(parameters.sqlServerLinuxAcceptanceTests, True) }}:
               LinuxPart1Of3:
-                testCommand: 'npx playwright test DefaultConfig --grep "@smoke" --grep-invert "Users" --shard=1/3'
+                testCommand: "npm run smokeTestSqlite -- --shard=1/3"
                 vmImage: "ubuntu-latest"
                 SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
                 CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True"
               LinuxPart2Of3:
-                testCommand: 'npx playwright test DefaultConfig --grep "@smoke" --grep-invert "Users" --shard=2/3'
+                testCommand: "npm run smokeTestSqlite -- --shard=2/3"
                 vmImage: "ubuntu-latest"
                 SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
                 CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True"
               LinuxPart3Of3:
-                testCommand: 'npx playwright test DefaultConfig --grep "@smoke" --grep-invert "Users" --shard=3/3'
+                testCommand: "npm run smokeTestSqlite -- --shard=3/3"
                 vmImage: "ubuntu-latest"
                 SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
                 CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True"
             WindowsPart1Of3:
               vmImage: "windows-latest"
-              testCommand: 'npx playwright test DefaultConfig --grep "@smoke" --grep-invert "Users" --shard=1/3'
+              testCommand: "npm run smokeTestSqlite -- --shard=1/3"
             WindowsPart2Of3:
               vmImage: "windows-latest"
-              testCommand: 'npx playwright test DefaultConfig --grep "@smoke" --grep-invert "Users" --shard=2/3'
+              testCommand: "npm run smokeTestSqlite -- --shard=2/3"
             WindowsPart3Of3:
               vmImage: "windows-latest"
-              testCommand: 'npx playwright test DefaultConfig --grep "@smoke" --grep-invert "Users" --shard=3/3'
+              testCommand: "npm run smokeTestSqlite -- --shard=3/3"
         pool:
           vmImage: $(vmImage)
         steps:
@@ -779,7 +778,7 @@ stages:
           # Copy artifacts
           - pwsh: |
               if (Test-Path tests/Umbraco.Tests.AcceptanceTest/results/*) {
-                Copy-Item tests/Umbraco.Tests.AcceptanceTest/results $(Build.ArtifactStagingDirectory) -Recurse
+                Copy-Item tests/Umbraco.Tests.AcceptanceTest/results/* $(Build.ArtifactStagingDirectory) -Recurse
               }
             displayName: Copy Playwright results
             condition: succeededOrFailed()
@@ -790,8 +789,7 @@ stages:
             condition: succeededOrFailed()
             inputs:
               targetPath: $(Build.ArtifactStagingDirectory)
-              artifact: "Acceptance Tests - $(Agent.JobName) - Attempt #$(System.JobAttempt)"
-
+              artifact: "Acceptance Test Results - $(Agent.JobName) - Attempt #$(System.JobAttempt)"
   ###############################################
   ## Release
   ###############################################
diff --git a/build/nightly-E2E-test-pipelines.yml b/build/nightly-E2E-test-pipelines.yml
index 42cc5973c5..9435aadfd8 100644
--- a/build/nightly-E2E-test-pipelines.yml
+++ b/build/nightly-E2E-test-pipelines.yml
@@ -111,22 +111,22 @@ stages:
           matrix:
             LinuxPart1Of3:
               vmImage: "ubuntu-latest"
-              testCommand: "npx playwright test DefaultConfig --shard=1/3"
+              testCommand: "npm run test -- --shard=1/3"
             LinuxPart2Of3:
               vmImage: "ubuntu-latest"
-              testCommand: "npx playwright test DefaultConfig --shard=2/3"
+              testCommand: "npm run test -- --shard=2/3"
             LinuxPart3Of3:
               vmImage: "ubuntu-latest"
-              testCommand: "npx playwright test DefaultConfig --shard=3/3"
+              testCommand: "npm run test -- --shard=3/3"
             WindowsPart1Of3:
               vmImage: "windows-latest"
-              testCommand: "npx playwright test DefaultConfig --shard=1/3"
+              testCommand: "npm run test -- --shard=1/3"
             WindowsPart2Of3:
               vmImage: "windows-latest"
-              testCommand: "npx playwright test DefaultConfig --shard=2/3"
+              testCommand: "npm run test -- --shard=2/3"
             WindowsPart3Of3:
               vmImage: "windows-latest"
-              testCommand: "npx playwright test DefaultConfig --shard=3/3"
+              testCommand: "npm run test -- --shard=3/3"
         pool:
           vmImage: $(vmImage)
         steps:
@@ -235,7 +235,7 @@ stages:
           # Copy artifacts
           - pwsh: |
               if (Test-Path tests/Umbraco.Tests.AcceptanceTest/results/*) {
-                Copy-Item tests/Umbraco.Tests.AcceptanceTest/results $(Build.ArtifactStagingDirectory) -Recurse
+                Copy-Item tests/Umbraco.Tests.AcceptanceTest/results/* $(Build.ArtifactStagingDirectory) -Recurse
               }
             displayName: Copy Playwright results
             condition: succeededOrFailed()
@@ -246,7 +246,7 @@ stages:
             condition: succeededOrFailed()
             inputs:
               targetPath: $(Build.ArtifactStagingDirectory)
-              artifact: 'Acceptance Tests - $(Agent.JobName) - Attempt #$(System.JobAttempt)'
+              artifact: "Acceptance Test Results - $(Agent.JobName) - Attempt #$(System.JobAttempt)"
 
       - job:
         displayName: E2E Tests (SQL Server)
@@ -258,29 +258,29 @@ stages:
         strategy:
           matrix:
             LinuxPart1Of3:
-              testCommand: "npx playwright test DefaultConfig --grep-invert \"Users\" --shard=1/3"
+              testCommand: "npm run testSqlite -- --shard=1/3"
               vmImage: "ubuntu-latest"
               SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
               CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True"
             LinuxPart2Of3:
-              testCommand: "npx playwright test DefaultConfig --grep-invert \"Users\" --shard=2/3"
+              testCommand: "npm run testSqlite -- --shard=2/3"
               vmImage: "ubuntu-latest"
               SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
               CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True"
             LinuxPart3Of3:
-              testCommand: "npx playwright test DefaultConfig  --grep-invert \"Users\" --shard=3/3"
+              testCommand: "npm run testSqlite -- --shard=3/3"
               vmImage: "ubuntu-latest"
               SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
               CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True"
             WindowsPart1Of3:
               vmImage: "windows-latest"
-              testCommand: "npx playwright test DefaultConfig --grep-invert \"Users\" --shard=1/3"
+              testCommand: "npm run testSqlite -- --shard=1/3"
             WindowsPart2Of3:
               vmImage: "windows-latest"
-              testCommand: "npx playwright test DefaultConfig --grep-invert \"Users\" --shard=2/3"
+              testCommand: "npm run testSqlite -- --shard=2/3"
             WindowsPart3Of3:
               vmImage: "windows-latest"
-              testCommand: "npx playwright test DefaultConfig --grep-invert \"Users\" --shard=3/3"
+              testCommand: "npm run testSqlite -- --shard=3/3"
         pool:
           vmImage: $(vmImage)
         steps:
@@ -406,7 +406,7 @@ stages:
           # Copy artifacts
           - pwsh: |
               if (Test-Path tests/Umbraco.Tests.AcceptanceTest/results/*) {
-                Copy-Item tests/Umbraco.Tests.AcceptanceTest/results $(Build.ArtifactStagingDirectory) -Recurse
+                Copy-Item tests/Umbraco.Tests.AcceptanceTest/results/* $(Build.ArtifactStagingDirectory) -Recurse
               }
             displayName: Copy Playwright results
             condition: succeededOrFailed()
@@ -417,4 +417,4 @@ stages:
             condition: succeededOrFailed()
             inputs:
               targetPath: $(Build.ArtifactStagingDirectory)
-              artifact: 'Acceptance Tests - $(Agent.JobName) - Attempt #$(System.JobAttempt)'
+              artifact: "Acceptance Test Results - $(Agent.JobName) - Attempt #$(System.JobAttempt)"