V16 QA Implement new release tag for E2E tests (#19697)
* Removed this as these tests are covered in other files * Added release tags * Make all tests with @smoke and @release tags run in the pipeline * Updated npx command * Updated npx command * Updated npx command * Fixed failed tests related to document type folder * Cleaned up * Used grep in yaml file instead of package.json file * Updated yml file * Updated testCommand * Fixed command * Added releaseTest command * Added another job to run regression test in the release build * Fixed comments * Updated name of test job * Make all release tests run in the pipeline * Updated warning message * Reverted npm command
This commit is contained in:
Nhu Dinh
@@ -513,9 +513,9 @@ stages:
|
||||
UMBRACO__CMS__WEBROUTING__UMBRACOAPPLICATIONURL: https://localhost:44331/
|
||||
ASPNETCORE_URLS: https://localhost:44331
|
||||
jobs:
|
||||
# E2E Tests
|
||||
# E2E Smoke Tests
|
||||
- job:
|
||||
displayName: E2E Tests (SQLite)
|
||||
displayName: E2E Smoke Tests (SQLite)
|
||||
# currently disabled due to DB locks randomly occuring.
|
||||
condition: eq(${{parameters.sqliteAcceptanceTests}}, True)
|
||||
variables:
|
||||
@@ -678,7 +678,7 @@ stages:
|
||||
testRunTitle: "$(Agent.JobName)"
|
||||
|
||||
- job:
|
||||
displayName: E2E Tests (SQL Server)
|
||||
displayName: E2E Smoke Tests (SQL Server)
|
||||
variables:
|
||||
# Connection string
|
||||
CONNECTIONSTRINGS__UMBRACODBDSN: Data Source=(localdb)\MSSQLLocalDB;AttachDbFilename=|DataDirectory|\Umbraco.mdf;Integrated Security=True
|
||||
@@ -862,6 +862,176 @@ stages:
|
||||
searchFolder: "tests/Umbraco.Tests.AcceptanceTest/results"
|
||||
testRunTitle: "$(Agent.JobName)"
|
||||
|
||||
- job:
|
||||
displayName: E2E Release Tests (SQL Server)
|
||||
variables:
|
||||
# Connection string
|
||||
CONNECTIONSTRINGS__UMBRACODBDSN: Data Source=(localdb)\MSSQLLocalDB;AttachDbFilename=|DataDirectory|\Umbraco.mdf;Integrated Security=True
|
||||
CONNECTIONSTRINGS__UMBRACODBDSN_PROVIDERNAME: Microsoft.Data.SqlClient
|
||||
condition: eq(dependencies.Build.outputs['A.build.NBGV_PublicRelease'], 'True')
|
||||
strategy:
|
||||
matrix:
|
||||
WindowsPart1Of3:
|
||||
vmImage: "windows-latest"
|
||||
testCommand: "npm run releaseTest -- --shard=1/3"
|
||||
WindowsPart2Of3:
|
||||
vmImage: "windows-latest"
|
||||
testCommand: "npm run releaseTest -- --shard=2/3"
|
||||
WindowsPart3Of3:
|
||||
vmImage: "windows-latest"
|
||||
testCommand: "npm run releaseTest -- --shard=3/3"
|
||||
pool:
|
||||
vmImage: $(vmImage)
|
||||
steps:
|
||||
# Setup test environment
|
||||
- task: DownloadPipelineArtifact@2
|
||||
displayName: Download NuGet artifacts
|
||||
inputs:
|
||||
artifact: nupkg
|
||||
path: $(Agent.BuildDirectory)/app/nupkg
|
||||
|
||||
- task: NodeTool@0
|
||||
displayName: Use Node.js $(nodeVersion)
|
||||
inputs:
|
||||
versionSpec: $(nodeVersion)
|
||||
|
||||
- task: UseDotNet@2
|
||||
displayName: Use .NET SDK from global.json
|
||||
inputs:
|
||||
useGlobalJson: true
|
||||
|
||||
- pwsh: |
|
||||
"UMBRACO_USER_LOGIN=$(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSEREMAIL)
|
||||
UMBRACO_USER_PASSWORD=$(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
|
||||
URL=$(ASPNETCORE_URLS)
|
||||
STORAGE_STAGE_PATH=$(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest/playwright/.auth/user.json
|
||||
CONSOLE_ERRORS_PATH=$(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest/console-errors.json" | Out-File .env
|
||||
displayName: Generate .env
|
||||
workingDirectory: $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest
|
||||
|
||||
# Cache and restore NPM packages
|
||||
- task: Cache@2
|
||||
displayName: Cache NPM packages
|
||||
inputs:
|
||||
key: 'npm_e2e | "$(Agent.OS)" | $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest/package-lock.json'
|
||||
restoreKeys: |
|
||||
npm_e2e | "$(Agent.OS)"
|
||||
npm_e2e
|
||||
path: $(npm_config_cache)
|
||||
|
||||
- script: npm ci --no-fund --no-audit --prefer-offline
|
||||
workingDirectory: $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest
|
||||
displayName: Restore NPM packages
|
||||
|
||||
# Build application
|
||||
- pwsh: |
|
||||
$cmsVersion = "$(Build.BuildNumber)" -replace "\+",".g"
|
||||
dotnet new nugetconfig
|
||||
dotnet nuget add source ./nupkg --name Local
|
||||
dotnet new install Umbraco.Templates::$cmsVersion
|
||||
dotnet new umbraco --name UmbracoProject --version $cmsVersion --exclude-gitignore --no-restore --no-update-check
|
||||
dotnet restore UmbracoProject
|
||||
cp $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest.UmbracoProject/*.cs UmbracoProject
|
||||
dotnet build UmbracoProject --configuration $(buildConfiguration) --no-restore
|
||||
dotnet dev-certs https
|
||||
displayName: Build application
|
||||
workingDirectory: $(Agent.BuildDirectory)/app
|
||||
|
||||
# Start SQL Server
|
||||
- powershell: docker run --name mssql -d -p 1433:1433 -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=$(SA_PASSWORD)" mcr.microsoft.com/mssql/server:2022-latest
|
||||
displayName: Start SQL Server Docker image (Linux)
|
||||
condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
|
||||
|
||||
- pwsh: SqlLocalDB start MSSQLLocalDB
|
||||
displayName: Start SQL Server LocalDB (Windows)
|
||||
condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
|
||||
|
||||
# Run application
|
||||
- bash: |
|
||||
nohup dotnet run --project UmbracoProject --configuration $(buildConfiguration) --no-build --no-launch-profile > $(Build.ArtifactStagingDirectory)/playwright.log 2>&1 &
|
||||
echo "##vso[task.setvariable variable=AcceptanceTestProcessId]$!"
|
||||
displayName: Run application (Linux)
|
||||
condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
|
||||
workingDirectory: $(Agent.BuildDirectory)/app
|
||||
|
||||
- pwsh: |
|
||||
$process = Start-Process dotnet "run --project UmbracoProject --configuration $(buildConfiguration) --no-build --no-launch-profile 2>&1" -PassThru -NoNewWindow -RedirectStandardOutput $(Build.ArtifactStagingDirectory)/playwright.log
|
||||
Write-Host "##vso[task.setvariable variable=AcceptanceTestProcessId]$($process.Id)"
|
||||
displayName: Run application (Windows)
|
||||
condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
|
||||
workingDirectory: $(Agent.BuildDirectory)/app
|
||||
|
||||
# Wait for application to start responding to requests
|
||||
- pwsh: npx wait-on -v --interval 1000 --timeout 120000 $(ASPNETCORE_URLS)
|
||||
displayName: Wait for application
|
||||
workingDirectory: tests/Umbraco.Tests.AcceptanceTest
|
||||
|
||||
# Install Playwright and dependencies
|
||||
- pwsh: npx playwright install chromium
|
||||
displayName: Install Playwright only with Chromium browser
|
||||
workingDirectory: tests/Umbraco.Tests.AcceptanceTest
|
||||
|
||||
# Test
|
||||
- pwsh: $(testCommand)
|
||||
displayName: Run Playwright tests
|
||||
workingDirectory: tests/Umbraco.Tests.AcceptanceTest
|
||||
env:
|
||||
CI: true
|
||||
CommitId: $(Build.SourceVersion)
|
||||
AgentOs: $(Agent.OS)
|
||||
|
||||
# Stop application
|
||||
- bash: kill -15 $(AcceptanceTestProcessId)
|
||||
displayName: Stop application (Linux)
|
||||
condition: and(ne(variables.AcceptanceTestProcessId, ''), eq(variables['Agent.OS'], 'Linux'))
|
||||
|
||||
- pwsh: Stop-Process -Id $(AcceptanceTestProcessId)
|
||||
displayName: Stop application (Windows)
|
||||
condition: and(ne(variables.AcceptanceTestProcessId, ''), eq(variables['Agent.OS'], 'Windows_NT'))
|
||||
|
||||
# Stop SQL Server
|
||||
- pwsh: docker stop mssql
|
||||
displayName: Stop SQL Server Docker image (Linux)
|
||||
condition: eq(variables['Agent.OS'], 'Linux')
|
||||
|
||||
- pwsh: SqlLocalDB stop MSSQLLocalDB
|
||||
displayName: Stop SQL Server LocalDB (Windows)
|
||||
condition: eq(variables['Agent.OS'], 'Windows_NT')
|
||||
|
||||
# Copy artifacts
|
||||
- pwsh: |
|
||||
if (Test-Path tests/Umbraco.Tests.AcceptanceTest/results/*) {
|
||||
Copy-Item tests/Umbraco.Tests.AcceptanceTest/results/* $(Build.ArtifactStagingDirectory) -Recurse
|
||||
}
|
||||
displayName: Copy Playwright results
|
||||
condition: succeededOrFailed()
|
||||
|
||||
# Copy console error log
|
||||
- pwsh: |
|
||||
if (Test-Path tests/Umbraco.Tests.AcceptanceTest/console-errors.json) {
|
||||
Copy-Item tests/Umbraco.Tests.AcceptanceTest/console-errors.json $(Build.ArtifactStagingDirectory)
|
||||
}
|
||||
displayName: Copy console error log
|
||||
condition: succeededOrFailed()
|
||||
|
||||
# Publish test artifacts
|
||||
- task: PublishPipelineArtifact@1
|
||||
displayName: Publish test artifacts
|
||||
condition: succeededOrFailed()
|
||||
inputs:
|
||||
targetPath: $(Build.ArtifactStagingDirectory)
|
||||
artifact: "Acceptance Test Results - $(Agent.JobName) - Attempt #$(System.JobAttempt)"
|
||||
|
||||
# Publish test results
|
||||
- task: PublishTestResults@2
|
||||
displayName: "Publish test results"
|
||||
condition: succeededOrFailed()
|
||||
inputs:
|
||||
testResultsFormat: 'JUnit'
|
||||
testResultsFiles: '*.xml'
|
||||
searchFolder: "tests/Umbraco.Tests.AcceptanceTest/results"
|
||||
testRunTitle: "$(Agent.JobName)"
|
||||
|
||||
###############################################
|
||||
## Release
|
||||
###############################################
|
||||
@@ -1067,4 +1237,4 @@ stages:
|
||||
storage: umbracoapidocs
|
||||
ContainerName: "$web"
|
||||
BlobPrefix: v$(umbracoMajorVersion)/ui-api
|
||||
CleanTargetBeforeCopy: true
|
||||
CleanTargetBeforeCopy: true
|
||||
Reference in New Issue
Block a user