From 41737a35d9eabca04e21a991bec178c09ecc9245 Mon Sep 17 00:00:00 2001 From: Andreas Zerbst <73799582+andr317c@users.noreply.github.com> Date: Thu, 13 Oct 2022 14:49:48 +0200 Subject: [PATCH] V10/bugfix/create simple package test (#13162) * Fixed assert to hopefully find the package each time so it isnt flaky anymore * Updated so it retries 5 times instead of 2 * Dont submit html-report * Dont have output defined in npm run * Copy over playwright trace.zip files before publishing * Updated assert so it looks after the package in the table * updated so we get the first fail as the trace file * Bumped version for testhelpers * Updated so the test checks if the package actually exists. Added a wait that checks if the created packages button is visible * Updated package lock * Fixed so it now calls the correct testhelper Co-authored-by: Zeegaan --- build/azure-pipelines.yml | 17 +++++++++++++++-- .../package-lock.json | 14 +++++++------- tests/Umbraco.Tests.AcceptanceTest/package.json | 2 +- .../playwright.config.ts | 6 +++--- .../DefaultConfig/Packages/packages.spec.ts | 10 +++++++++- 5 files changed, 35 insertions(+), 14 deletions(-) diff --git a/build/azure-pipelines.yml b/build/azure-pipelines.yml index 8dc3a611ab..e4d751b51b 100644 --- a/build/azure-pipelines.yml +++ b/build/azure-pipelines.yml @@ -444,13 +444,12 @@ stages: - pwsh: npx playwright install --with-deps displayName: Install Playwright workingDirectory: tests/Umbraco.Tests.AcceptanceTest - - pwsh: npm run test --ignore-certificate-errors --output $(Build.ArtifactStagingDirectory)\test-results + - pwsh: npm run test --ignore-certificate-errors displayName: Run Playwright (Desktop) continueOnError: true workingDirectory: tests/Umbraco.Tests.AcceptanceTest env: CI: true - PLAYWRIGHT_HTML_REPORT: $(Build.ArtifactStagingDirectory)\playwright-report - pwsh: | docker logs $(dockerImageName) > $(Build.ArtifactStagingDirectory)/playwright.log 2>&1 docker stop $(dockerImageName) @@ -459,6 +458,19 @@ stages: - pwsh: Stop-Process $env:AcceptanceTestProcessId condition: eq(variables['Agent.OS'], 'Windows_NT') displayName: Stop app (Windows only) + - task: PowerShell@2 + displayName: Check if artifacts folder exists + inputs: + targetType: inline + script: | + $MyVariable = Test-Path -Path $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest/results + Write-Host "##vso[task.setvariable variable=resultFolderExists;]$MyVariable" + - task: CopyFiles@2 + displayName: Prepare artifacts + condition: eq(variables.resultFolderExists, 'True') + inputs: + sourceFolder: $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest/results/ + targetFolder: $(Build.ArtifactStagingDirectory)/playwright - task: PublishPipelineArtifact@1 condition: always() displayName: Publish test artifacts @@ -466,6 +478,7 @@ stages: targetPath: $(Build.ArtifactStagingDirectory) artifact: 'E2E artifacts - $(Agent.OS) - Attempt #$(System.JobAttempt)' + ############################################### ## Release ############################################### diff --git a/tests/Umbraco.Tests.AcceptanceTest/package-lock.json b/tests/Umbraco.Tests.AcceptanceTest/package-lock.json index 70229528bb..d9d26331d6 100644 --- a/tests/Umbraco.Tests.AcceptanceTest/package-lock.json +++ b/tests/Umbraco.Tests.AcceptanceTest/package-lock.json @@ -8,7 +8,7 @@ "hasInstallScript": true, "dependencies": { "@umbraco/json-models-builders": "^1.0.0", - "@umbraco/playwright-testhelpers": "^1.0.2", + "@umbraco/playwright-testhelpers": "^1.0.3", "camelize": "^1.0.0", "dotenv": "^16.0.2", "faker": "^4.1.0", @@ -101,9 +101,9 @@ } }, "node_modules/@umbraco/playwright-testhelpers": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@umbraco/playwright-testhelpers/-/playwright-testhelpers-1.0.2.tgz", - "integrity": "sha512-j1y6YRq2Rg5AXyYk/304P2rTrDCLU7Sz67/MMfkPBHSvadjdof7EW8649Aio29xGAg1YAR4y+Zeyw6XnM35ZkA==", + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@umbraco/playwright-testhelpers/-/playwright-testhelpers-1.0.3.tgz", + "integrity": "sha512-PmUnIaoKitxAC4JWSiPEOPg74Ypt6DNLjUQEATV0n9yVbw5aFQhql+KrdN4F30gFNr1c6Gw6I5iDXzNmq5/zfg==", "dependencies": { "@umbraco/json-models-builders": "^1.0.0", "camelize": "^1.0.0", @@ -906,9 +906,9 @@ } }, "@umbraco/playwright-testhelpers": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@umbraco/playwright-testhelpers/-/playwright-testhelpers-1.0.2.tgz", - "integrity": "sha512-j1y6YRq2Rg5AXyYk/304P2rTrDCLU7Sz67/MMfkPBHSvadjdof7EW8649Aio29xGAg1YAR4y+Zeyw6XnM35ZkA==", + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@umbraco/playwright-testhelpers/-/playwright-testhelpers-1.0.3.tgz", + "integrity": "sha512-PmUnIaoKitxAC4JWSiPEOPg74Ypt6DNLjUQEATV0n9yVbw5aFQhql+KrdN4F30gFNr1c6Gw6I5iDXzNmq5/zfg==", "requires": { "@umbraco/json-models-builders": "^1.0.0", "camelize": "^1.0.0", diff --git a/tests/Umbraco.Tests.AcceptanceTest/package.json b/tests/Umbraco.Tests.AcceptanceTest/package.json index 8996b7dbaa..7464e119e0 100644 --- a/tests/Umbraco.Tests.AcceptanceTest/package.json +++ b/tests/Umbraco.Tests.AcceptanceTest/package.json @@ -19,7 +19,7 @@ }, "dependencies": { "@umbraco/json-models-builders": "^1.0.0", - "@umbraco/playwright-testhelpers": "^1.0.2", + "@umbraco/playwright-testhelpers": "^1.0.3", "camelize": "^1.0.0", "faker": "^4.1.0", "form-data": "^4.0.0", diff --git a/tests/Umbraco.Tests.AcceptanceTest/playwright.config.ts b/tests/Umbraco.Tests.AcceptanceTest/playwright.config.ts index 9f9a1a226b..f5c9bf5fde 100644 --- a/tests/Umbraco.Tests.AcceptanceTest/playwright.config.ts +++ b/tests/Umbraco.Tests.AcceptanceTest/playwright.config.ts @@ -21,11 +21,11 @@ const config: PlaywrightTestConfig = { /* Fail the build on CI if you accidentally left test.only in the source code. */ forbidOnly: !!process.env.CI, /* Retry on CI only */ - retries: process.env.CI ? 2 : 5, + retries: process.env.CI ? 5 : 2, // We don't want to run parallel, as tests might differ in state workers: 1, /* Reporter to use. See https://playwright.dev/docs/test-reporters */ - reporter: process.env.CI ? 'html' : 'line', + reporter: process.env.CI ? 'line' : 'html', outputDir : "./results", /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ @@ -36,7 +36,7 @@ const config: PlaywrightTestConfig = { // baseURL: 'http://localhost:44332', /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ - trace: 'on-first-retry', + trace: 'retain-on-failure', ignoreHTTPSErrors: true, }, diff --git a/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Packages/packages.spec.ts b/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Packages/packages.spec.ts index b0bb06e2a8..ba91e6ba82 100644 --- a/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Packages/packages.spec.ts +++ b/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Packages/packages.spec.ts @@ -89,9 +89,17 @@ test.describe('Packages', () => { // Navigate pack to packages and Assert the file is created // Waits until the button download is visible await page.locator('[label-key="general_download"]').isVisible(); + // Checks if the packages was created + const doesExist = await umbracoApi.packages.doesNameExist(packageName); + await expect(doesExist).toBe(true); await umbracoUi.goToSection(ConstantHelper.sections.packages); + + // Needs to wait until the page has loaded and the button is clickable + await page.locator('[data-element="sub-view-umbCreatedPackages"]').isVisible(); await page.locator('[data-element="sub-view-umbCreatedPackages"]').click(); - await expect(await page.locator("body", {hasText: packageName})).toBeVisible(); + // Asserts that the package can be found in the table + await expect(await page.locator('.table-hover')).toHaveCount(1); + await expect(await page.locator('.table-hover').first()).toContainText(packageName); // Cleanup await umbracoApi.packages.ensureNameNotExists(packageName);