V15 QA add parallelization of e2e tests (#17691)

* Tries parallelizing the E2E tests

* Updated variable

* Updated SqlServer

* Updated playwright config

* Bumped to use 3 shards

* Implemented shards on nightly pipeline

* Fixed indentation

* Added a smokeTest

* Updated test to run

* Added test command

* Update Templates.spec.ts

build/azure-pipelines.yml

@@ -439,10 +439,24 @@ stages:
           CONNECTIONSTRINGS__UMBRACODBDSN_PROVIDERNAME: Microsoft.Data.Sqlite
         strategy:
           matrix:
-            Linux:
+            LinuxPart1Of3:
               vmImage: "ubuntu-latest"
-            Windows:
+              testCommand: "npx playwright test DefaultConfig --grep \"@smoke\"--shard=1/3"
+            LinuxPart2Of3:
+              vmImage: "ubuntu-latest"
+              testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --shard=2/3"
+            LinuxPart3Of3:
+              vmImage: "ubuntu-latest"
+              testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --shard=3/3"
+            WindowsPart1Of3:
               vmImage: "windows-latest"
+              testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --shard=1/3"
+            WindowsPart2Of3:
+              vmImage: "windows-latest"
+              testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --shard=2/3"
+            WindowsPart3Of3:
+              vmImage: "windows-latest"
+              testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --shard=3/3"
         pool:
           vmImage: $(vmImage)
         steps:
@@ -526,7 +540,7 @@ stages:
             workingDirectory: tests/Umbraco.Tests.AcceptanceTest
 
           # Test
-          - pwsh: npm run smokeTestSqlite --ignore-certificate-errors
+          - pwsh: $(testCommand)
             displayName: Run Playwright tests
             continueOnError: true
             workingDirectory: tests/Umbraco.Tests.AcceptanceTest
@@ -568,13 +582,31 @@ stages:
           CONNECTIONSTRINGS__UMBRACODBDSN_PROVIDERNAME: Microsoft.Data.SqlClient
         strategy:
           matrix:
-            ${{ if eq(parameters.sqlServerLinuxAcceptanceTests, True) }} :
+            ${{ if eq(parameters.sqlServerLinuxAcceptanceTests, True) }}:
-              Linux:
+              LinuxPart1Of3:
+                testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --grep-invert \"Users\" --shard=1/3"
                 vmImage: "ubuntu-latest"
                 SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
                 CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True"
-            Windows:
+              LinuxPart2Of3:
+                testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --grep-invert \"Users\" --shard=2/3"
+                vmImage: "ubuntu-latest"
+                SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
+                CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True"
+              LinuxPart3Of3:
+                testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --grep-invert \"Users\" --shard=3/3"
+                vmImage: "ubuntu-latest"
+                SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
+                CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True"
+            WindowsPart1Of3:
               vmImage: "windows-latest"
+              testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --grep-invert \"Users\" --shard=1/3"
+            WindowsPart2Of3:
+              vmImage: "windows-latest"
+              testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --grep-invert \"Users\" --shard=2/3"
+            WindowsPart3Of3:
+              vmImage: "windows-latest"
+              testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --grep-invert \"Users\" --shard=3/3"
         pool:
           vmImage: $(vmImage)
         steps:
@@ -666,7 +698,7 @@ stages:
             workingDirectory: tests/Umbraco.Tests.AcceptanceTest
 
           # Test
-          - pwsh: npm run smokeTest --ignore-certificate-errors
+          - pwsh: $(testCommand)
             displayName: Run Playwright tests
             continueOnError: true
             workingDirectory: tests/Umbraco.Tests.AcceptanceTest

build/nightly-E2E-test-pipelines.yml

@@ -23,12 +23,6 @@ variables:
   npm_config_cache: $(Pipeline.Workspace)/.npm_client
   NODE_OPTIONS: --max_old_space_size=16384
 
-parameters:
-  - name: runSmokeTests
-    displayName: Run the smoke tests
-    type: boolean
-    default: false
-
 stages:
   ###############################################
   ## Build
@@ -115,10 +109,24 @@ stages:
           CONNECTIONSTRINGS__UMBRACODBDSN_PROVIDERNAME: Microsoft.Data.Sqlite
         strategy:
           matrix:
-            Linux:
+            LinuxPart1Of3:
-              vmImage: 'ubuntu-latest'
+              vmImage: "ubuntu-latest"
-            Windows:
+              testCommand: "npx playwright test DefaultConfig --shard=1/3"
-              vmImage: 'windows-latest'
+            LinuxPart2Of3:
+              vmImage: "ubuntu-latest"
+              testCommand: "npx playwright test DefaultConfig --shard=2/3"
+            LinuxPart3Of3:
+              vmImage: "ubuntu-latest"
+              testCommand: "npx playwright test DefaultConfig --shard=3/3"
+            WindowsPart1Of3:
+              vmImage: "windows-latest"
+              testCommand: "npx playwright test DefaultConfig --shard=1/3"
+            WindowsPart2Of3:
+              vmImage: "windows-latest"
+              testCommand: "npx playwright test DefaultConfig --shard=2/3"
+            WindowsPart3Of3:
+              vmImage: "windows-latest"
+              testCommand: "npx playwright test DefaultConfig --shard=3/3"
         pool:
           vmImage: $(vmImage)
         steps:
@@ -206,10 +214,7 @@ stages:
             workingDirectory: tests/Umbraco.Tests.AcceptanceTest
 
           # Test
-          - ${{ if eq(parameters.runSmokeTests, true) }}:
+          - pwsh: $(testCommand)
-              pwsh: npm run smokeTestSqlite --ignore-certificate-errors
-            ${{ else }}:
-              pwsh: npm run testSqlite --ignore-certificate-errors
             displayName: Run Playwright tests
             continueOnError: true
             workingDirectory: tests/Umbraco.Tests.AcceptanceTest
@@ -252,12 +257,30 @@ stages:
           CONNECTIONSTRINGS__UMBRACODBDSN_PROVIDERNAME: Microsoft.Data.SqlClient
         strategy:
           matrix:
-            Linux:
+            LinuxPart1Of3:
-              vmImage: 'ubuntu-latest'
+              testCommand: "npx playwright test DefaultConfig --grep-invert \"Users\" --shard=1/3"
+              vmImage: "ubuntu-latest"
               SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
-              CONNECTIONSTRINGS__UMBRACODBDSN: 'Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True'
+              CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True"
-            Windows:
+            LinuxPart2Of3:
-              vmImage: 'windows-latest'
+              testCommand: "npx playwright test DefaultConfig --grep-invert \"Users\" --shard=2/3"
+              vmImage: "ubuntu-latest"
+              SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
+              CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True"
+            LinuxPart3Of3:
+              testCommand: "npx playwright test DefaultConfig  --grep-invert \"Users\" --shard=3/3"
+              vmImage: "ubuntu-latest"
+              SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
+              CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True"
+            WindowsPart1Of3:
+              vmImage: "windows-latest"
+              testCommand: "npx playwright test DefaultConfig --grep-invert \"Users\" --shard=1/3"
+            WindowsPart2Of3:
+              vmImage: "windows-latest"
+              testCommand: "npx playwright test DefaultConfig --grep-invert \"Users\" --shard=2/3"
+            WindowsPart3Of3:
+              vmImage: "windows-latest"
+              testCommand: "npx playwright test DefaultConfig --grep-invert \"Users\" --shard=3/3"
         pool:
           vmImage: $(vmImage)
         steps:
@@ -353,10 +376,7 @@ stages:
             workingDirectory: tests/Umbraco.Tests.AcceptanceTest
 
           # Test
-          - ${{ if eq(parameters.runSmokeTests, true) }}:
+          - pwsh: $(testCommand)
-              pwsh: npm run smokeTest --ignore-certificate-errors
-            ${{ else }}:
-              pwsh: npm run test --ignore-certificate-errors
             displayName: Run Playwright tests
             continueOnError: true
             workingDirectory: tests/Umbraco.Tests.AcceptanceTest

tests/Umbraco.Tests.AcceptanceTest/playwright.config.ts

@@ -47,6 +47,7 @@ export default defineConfig({
       use: {
         ...devices['Desktop Chrome'],
         // Use prepared auth state.
+        ignoreHTTPSErrors: true,
         storageState: STORAGE_STATE,
       },
     },