V15 QA add parallelization of e2e tests (#17691)

* Tries parallelizing the E2E tests * Updated variable * Updated SqlServer * Updated playwright config * Bumped to use 3 shards * Implemented shards on nightly pipeline * Fixed indentation * Added a smokeTest * Updated test to run * Added test command * Update Templates.spec.ts
2024-12-02 14:48:15 +01:00
parent 01a19b8a9f
commit 303758d1b2
3 changed files with 83 additions and 30 deletions
--- a/build/azure-pipelines.yml
+++ b/build/azure-pipelines.yml
@@ -439,10 +439,24 @@ stages:
          CONNECTIONSTRINGS__UMBRACODBDSN_PROVIDERNAME: Microsoft.Data.Sqlite
        strategy:
          matrix:
-            Linux:
+            LinuxPart1Of3:
              vmImage: "ubuntu-latest"
-            Windows:
+              testCommand: "npx playwright test DefaultConfig --grep \"@smoke\"--shard=1/3"
+            LinuxPart2Of3:
+              vmImage: "ubuntu-latest"
+              testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --shard=2/3"
+            LinuxPart3Of3:
+              vmImage: "ubuntu-latest"
+              testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --shard=3/3"
+            WindowsPart1Of3:
              vmImage: "windows-latest"
+              testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --shard=1/3"
+            WindowsPart2Of3:
+              vmImage: "windows-latest"
+              testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --shard=2/3"
+            WindowsPart3Of3:
+              vmImage: "windows-latest"
+              testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --shard=3/3"
        pool:
          vmImage: $(vmImage)
        steps:
@@ -526,7 +540,7 @@ stages:
            workingDirectory: tests/Umbraco.Tests.AcceptanceTest

          # Test
-          - pwsh: npm run smokeTestSqlite --ignore-certificate-errors
+          - pwsh: $(testCommand)
            displayName: Run Playwright tests
            continueOnError: true
            workingDirectory: tests/Umbraco.Tests.AcceptanceTest
@@ -569,12 +583,30 @@ stages:
        strategy:
          matrix:
            ${{ if eq(parameters.sqlServerLinuxAcceptanceTests, True) }}:
-              Linux:
+              LinuxPart1Of3:
+                testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --grep-invert \"Users\" --shard=1/3"
                vmImage: "ubuntu-latest"
                SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
                CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True"
-            Windows:
+              LinuxPart2Of3:
+                testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --grep-invert \"Users\" --shard=2/3"
+                vmImage: "ubuntu-latest"
+                SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
+                CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True"
+              LinuxPart3Of3:
+                testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --grep-invert \"Users\" --shard=3/3"
+                vmImage: "ubuntu-latest"
+                SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
+                CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True"
+            WindowsPart1Of3:
              vmImage: "windows-latest"
+              testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --grep-invert \"Users\" --shard=1/3"
+            WindowsPart2Of3:
+              vmImage: "windows-latest"
+              testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --grep-invert \"Users\" --shard=2/3"
+            WindowsPart3Of3:
+              vmImage: "windows-latest"
+              testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --grep-invert \"Users\" --shard=3/3"
        pool:
          vmImage: $(vmImage)
        steps:
@@ -666,7 +698,7 @@ stages:
            workingDirectory: tests/Umbraco.Tests.AcceptanceTest

          # Test
-          - pwsh: npm run smokeTest --ignore-certificate-errors
+          - pwsh: $(testCommand)
            displayName: Run Playwright tests
            continueOnError: true
            workingDirectory: tests/Umbraco.Tests.AcceptanceTest
--- a/build/nightly-E2E-test-pipelines.yml
+++ b/build/nightly-E2E-test-pipelines.yml
@@ -23,12 +23,6 @@ variables:
  npm_config_cache: $(Pipeline.Workspace)/.npm_client
  NODE_OPTIONS: --max_old_space_size=16384

-parameters:
-  - name: runSmokeTests
-    displayName: Run the smoke tests
-    type: boolean
-    default: false
-
 stages:
  ###############################################
  ## Build
@@ -115,10 +109,24 @@ stages:
          CONNECTIONSTRINGS__UMBRACODBDSN_PROVIDERNAME: Microsoft.Data.Sqlite
        strategy:
          matrix:
-            Linux:
-              vmImage: 'ubuntu-latest'
-            Windows:
-              vmImage: 'windows-latest'
+            LinuxPart1Of3:
+              vmImage: "ubuntu-latest"
+              testCommand: "npx playwright test DefaultConfig --shard=1/3"
+            LinuxPart2Of3:
+              vmImage: "ubuntu-latest"
+              testCommand: "npx playwright test DefaultConfig --shard=2/3"
+            LinuxPart3Of3:
+              vmImage: "ubuntu-latest"
+              testCommand: "npx playwright test DefaultConfig --shard=3/3"
+            WindowsPart1Of3:
+              vmImage: "windows-latest"
+              testCommand: "npx playwright test DefaultConfig --shard=1/3"
+            WindowsPart2Of3:
+              vmImage: "windows-latest"
+              testCommand: "npx playwright test DefaultConfig --shard=2/3"
+            WindowsPart3Of3:
+              vmImage: "windows-latest"
+              testCommand: "npx playwright test DefaultConfig --shard=3/3"
        pool:
          vmImage: $(vmImage)
        steps:
@@ -206,10 +214,7 @@ stages:
            workingDirectory: tests/Umbraco.Tests.AcceptanceTest

          # Test
-          - ${{ if eq(parameters.runSmokeTests, true) }}:
-              pwsh: npm run smokeTestSqlite --ignore-certificate-errors
-            ${{ else }}:
-              pwsh: npm run testSqlite --ignore-certificate-errors
+          - pwsh: $(testCommand)
            displayName: Run Playwright tests
            continueOnError: true
            workingDirectory: tests/Umbraco.Tests.AcceptanceTest
@@ -252,12 +257,30 @@ stages:
          CONNECTIONSTRINGS__UMBRACODBDSN_PROVIDERNAME: Microsoft.Data.SqlClient
        strategy:
          matrix:
-            Linux:
-              vmImage: 'ubuntu-latest'
+            LinuxPart1Of3:
+              testCommand: "npx playwright test DefaultConfig --grep-invert \"Users\" --shard=1/3"
+              vmImage: "ubuntu-latest"
              SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
-              CONNECTIONSTRINGS__UMBRACODBDSN: 'Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True'
-            Windows:
-              vmImage: 'windows-latest'
+              CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True"
+            LinuxPart2Of3:
+              testCommand: "npx playwright test DefaultConfig --grep-invert \"Users\" --shard=2/3"
+              vmImage: "ubuntu-latest"
+              SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
+              CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True"
+            LinuxPart3Of3:
+              testCommand: "npx playwright test DefaultConfig  --grep-invert \"Users\" --shard=3/3"
+              vmImage: "ubuntu-latest"
+              SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
+              CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True"
+            WindowsPart1Of3:
+              vmImage: "windows-latest"
+              testCommand: "npx playwright test DefaultConfig --grep-invert \"Users\" --shard=1/3"
+            WindowsPart2Of3:
+              vmImage: "windows-latest"
+              testCommand: "npx playwright test DefaultConfig --grep-invert \"Users\" --shard=2/3"
+            WindowsPart3Of3:
+              vmImage: "windows-latest"
+              testCommand: "npx playwright test DefaultConfig --grep-invert \"Users\" --shard=3/3"
        pool:
          vmImage: $(vmImage)
        steps:
@@ -353,10 +376,7 @@ stages:
            workingDirectory: tests/Umbraco.Tests.AcceptanceTest

          # Test
-          - ${{ if eq(parameters.runSmokeTests, true) }}:
-              pwsh: npm run smokeTest --ignore-certificate-errors
-            ${{ else }}:
-              pwsh: npm run test --ignore-certificate-errors
+          - pwsh: $(testCommand)
            displayName: Run Playwright tests
            continueOnError: true
            workingDirectory: tests/Umbraco.Tests.AcceptanceTest
--- a/tests/Umbraco.Tests.AcceptanceTest/playwright.config.ts
+++ b/tests/Umbraco.Tests.AcceptanceTest/playwright.config.ts
@@ -47,6 +47,7 @@ export default defineConfig({
      use: {
        ...devices['Desktop Chrome'],
        // Use prepared auth state.
+        ignoreHTTPSErrors: true,
        storageState: STORAGE_STATE,
      },
    },