V15 QA added a fix for flaky integration tests run on SQL Server Linux (#18965)
This commit is contained in:
Andreas Zerbst
@@ -432,6 +432,33 @@ stages:
|
||||
displayName: Start SQL Server Docker image (Linux)
|
||||
condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
|
||||
|
||||
- powershell: |
|
||||
$maxAttempts = 12
|
||||
$attempt = 0
|
||||
$status = ""
|
||||
|
||||
while (($status -ne 'running') -and ($attempt -lt $maxAttempts)) {
|
||||
Start-Sleep -Seconds 5
|
||||
# We use the docker inspect command to check the status of the container. If the container is not running, we wait 5 seconds and try again. And if reaches 12 attempts, we fail the build.
|
||||
$status = docker inspect -f '{{.State.Status}}' mssql
|
||||
|
||||
if ($status -ne 'running') {
|
||||
Write-Host "Waiting for SQL Server to be ready... Attempt $($attempt + 1)"
|
||||
$attempt++
|
||||
}
|
||||
}
|
||||
|
||||
if ($status -eq 'running') {
|
||||
Write-Host "SQL Server container is running"
|
||||
docker ps -a
|
||||
} else {
|
||||
Write-Host "SQL Server did not become ready in time. Last known status: $status"
|
||||
docker logs mssql
|
||||
exit 1
|
||||
}
|
||||
displayName: Wait for SQL Server to be ready (Linux)
|
||||
condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
|
||||
|
||||
- pwsh: SqlLocalDB start MSSQLLocalDB
|
||||
displayName: Start SQL Server LocalDB (Windows)
|
||||
condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
|
||||
|
||||
@@ -93,6 +93,189 @@ stages:
|
||||
targetPath: $(Build.ArtifactStagingDirectory)/npm
|
||||
artifactName: npm
|
||||
|
||||
- stage: Integration
|
||||
displayName: Integration Tests
|
||||
dependsOn: Build
|
||||
jobs:
|
||||
# Integration Tests (SQLite)
|
||||
- job:
|
||||
timeoutInMinutes: 180
|
||||
displayName: Integration Tests (SQLite)
|
||||
strategy:
|
||||
matrix:
|
||||
# Windows:
|
||||
# vmImage: 'windows-latest'
|
||||
# We split the tests into 3 parts for each OS to reduce the time it takes to run them on the pipeline
|
||||
LinuxPart1Of3:
|
||||
vmImage: "ubuntu-latest"
|
||||
# Filter tests that are part of the Umbraco.Infrastructure namespace but not part of the Umbraco.Infrastructure.Service namespace
|
||||
testFilter: "(FullyQualifiedName~Umbraco.Infrastructure) & (FullyQualifiedName!~Umbraco.Infrastructure.Service)"
|
||||
LinuxPart2Of3:
|
||||
vmImage: "ubuntu-latest"
|
||||
# Filter tests that are part of the Umbraco.Infrastructure.Service namespace
|
||||
testFilter: "(FullyQualifiedName~Umbraco.Infrastructure.Service)"
|
||||
LinuxPart3Of3:
|
||||
vmImage: "ubuntu-latest"
|
||||
# Filter tests that are not part of the Umbraco.Infrastructure namespace. So this will run all tests that are not part of the Umbraco.Infrastructure namespace
|
||||
testFilter: "(FullyQualifiedName!~Umbraco.Infrastructure)"
|
||||
macOSPart1Of3:
|
||||
vmImage: "macOS-latest"
|
||||
# Filter tests that are part of the Umbraco.Infrastructure namespace but not part of the Umbraco.Infrastructure.Service namespace
|
||||
testFilter: "(FullyQualifiedName~Umbraco.Infrastructure) & (FullyQualifiedName!~Umbraco.Infrastructure.Service)"
|
||||
macOSPart2Of3:
|
||||
vmImage: "macOS-latest"
|
||||
# Filter tests that are part of the Umbraco.Infrastructure.Service namespace
|
||||
testFilter: "(FullyQualifiedName~Umbraco.Infrastructure.Service)"
|
||||
macOSPart3Of3:
|
||||
vmImage: "macOS-latest"
|
||||
# Filter tests that are not part of the Umbraco.Infrastructure namespace.
|
||||
testFilter: "(FullyQualifiedName!~Umbraco.Infrastructure)"
|
||||
pool:
|
||||
vmImage: $(vmImage)
|
||||
variables:
|
||||
Tests__Database__DatabaseType: "Sqlite"
|
||||
steps:
|
||||
- checkout: self
|
||||
submodules: false
|
||||
lfs: false,
|
||||
fetchDepth: 1
|
||||
fetchFilter: tree:0
|
||||
# Setup test environment
|
||||
- task: DownloadPipelineArtifact@2
|
||||
displayName: Download build artifacts
|
||||
inputs:
|
||||
artifact: build_output
|
||||
path: $(Build.SourcesDirectory)
|
||||
|
||||
- task: UseDotNet@2
|
||||
displayName: Use .NET SDK from global.json
|
||||
inputs:
|
||||
useGlobalJson: true
|
||||
|
||||
# Test
|
||||
- task: DotNetCoreCLI@2
|
||||
displayName: Run dotnet test
|
||||
inputs:
|
||||
command: test
|
||||
projects: "tests/Umbraco.Tests.Integration/Umbraco.Tests.Integration.csproj"
|
||||
testRunTitle: Integration Tests SQLite - $(Agent.OS)
|
||||
arguments: '--filter "$(testFilter)" --configuration $(buildConfiguration) --no-build'
|
||||
|
||||
# Integration Tests (SQL Server)
|
||||
- job:
|
||||
timeoutInMinutes: 180
|
||||
displayName: Integration Tests (SQL Server)
|
||||
strategy:
|
||||
matrix:
|
||||
# We split the tests into 3 parts for each OS to reduce the time it takes to run them on the pipeline
|
||||
WindowsPart1Of3:
|
||||
vmImage: "windows-latest"
|
||||
Tests__Database__DatabaseType: LocalDb
|
||||
Tests__Database__SQLServerMasterConnectionString: N/A
|
||||
# Filter tests that are part of the Umbraco.Infrastructure namespace but not part of the Umbraco.Infrastructure.Service namespace
|
||||
testFilter: "(FullyQualifiedName~Umbraco.Infrastructure) & (FullyQualifiedName!~Umbraco.Infrastructure.Service)"
|
||||
WindowsPart2Of3:
|
||||
vmImage: "windows-latest"
|
||||
Tests__Database__DatabaseType: LocalDb
|
||||
Tests__Database__SQLServerMasterConnectionString: N/A
|
||||
# Filter tests that are part of the Umbraco.Infrastructure.Service namespace
|
||||
testFilter: "(FullyQualifiedName~Umbraco.Infrastructure.Service)"
|
||||
WindowsPart3Of3:
|
||||
vmImage: "windows-latest"
|
||||
Tests__Database__DatabaseType: LocalDb
|
||||
Tests__Database__SQLServerMasterConnectionString: N/A
|
||||
# Filter tests that are not part of the Umbraco.Infrastructure namespace. So this will run all tests that are not part of the Umbraco.Infrastructure namespace
|
||||
testFilter: "(FullyQualifiedName!~Umbraco.Infrastructure)"
|
||||
LinuxPart1Of3:
|
||||
vmImage: "ubuntu-latest"
|
||||
SA_PASSWORD: UmbracoIntegration123!
|
||||
Tests__Database__DatabaseType: SqlServer
|
||||
Tests__Database__SQLServerMasterConnectionString: "Server=(local);User Id=sa;Password=$(SA_PASSWORD);Encrypt=True;TrustServerCertificate=True"
|
||||
# Filter tests that are part of the Umbraco.Infrastructure namespace but not part of the Umbraco.Infrastructure.Service namespace
|
||||
testFilter: "(FullyQualifiedName~Umbraco.Infrastructure) & (FullyQualifiedName!~Umbraco.Infrastructure.Service)"
|
||||
LinuxPart2Of3:
|
||||
vmImage: "ubuntu-latest"
|
||||
SA_PASSWORD: UmbracoIntegration123!
|
||||
Tests__Database__DatabaseType: SqlServer
|
||||
Tests__Database__SQLServerMasterConnectionString: "Server=(local);User Id=sa;Password=$(SA_PASSWORD);Encrypt=True;TrustServerCertificate=True"
|
||||
# Filter tests that are part of the Umbraco.Infrastructure.Service namespace
|
||||
testFilter: "(FullyQualifiedName~Umbraco.Infrastructure.Service)"
|
||||
LinuxPart3Of3:
|
||||
vmImage: "ubuntu-latest"
|
||||
SA_PASSWORD: UmbracoIntegration123!
|
||||
Tests__Database__DatabaseType: SqlServer
|
||||
Tests__Database__SQLServerMasterConnectionString: "Server=(local);User Id=sa;Password=$(SA_PASSWORD);Encrypt=True;TrustServerCertificate=True"
|
||||
# Filter tests that are not part of the Umbraco.Infrastructure namespace. So this will run all tests that are not part of the Umbraco.Infrastructure namespace
|
||||
testFilter: "(FullyQualifiedName!~Umbraco.Infrastructure)"
|
||||
pool:
|
||||
vmImage: $(vmImage)
|
||||
steps:
|
||||
# Setup test environment
|
||||
- task: DownloadPipelineArtifact@2
|
||||
displayName: Download build artifacts
|
||||
inputs:
|
||||
artifact: build_output
|
||||
path: $(Build.SourcesDirectory)
|
||||
|
||||
- task: UseDotNet@2
|
||||
displayName: Use .NET SDK from global.json
|
||||
inputs:
|
||||
useGlobalJson: true
|
||||
|
||||
# Start SQL Server
|
||||
- powershell: docker run --name mssql -d -p 1433:1433 -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=$(SA_PASSWORD)" mcr.microsoft.com/mssql/server:2022-latest
|
||||
displayName: Start SQL Server Docker image (Linux)
|
||||
condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
|
||||
|
||||
- powershell: |
|
||||
$maxAttempts = 12
|
||||
$attempt = 0
|
||||
$status = ""
|
||||
|
||||
while (($status -ne 'running') -and ($attempt -lt $maxAttempts)) {
|
||||
Start-Sleep -Seconds 5
|
||||
# We use the docker inspect command to check the status of the container. If the container is not running, we wait 5 seconds and try again. And if reaches 12 attempts, we fail the build.
|
||||
$status = docker inspect -f '{{.State.Status}}' mssql
|
||||
|
||||
if ($status -ne 'running') {
|
||||
Write-Host "Waiting for SQL Server to be ready... Attempt $($attempt + 1)"
|
||||
$attempt++
|
||||
}
|
||||
}
|
||||
|
||||
if ($status -eq 'running') {
|
||||
Write-Host "SQL Server container is running"
|
||||
docker ps -a
|
||||
} else {
|
||||
Write-Host "SQL Server did not become ready in time. Last known status: $status"
|
||||
docker logs mssql
|
||||
exit 1
|
||||
}
|
||||
displayName: Wait for SQL Server to be ready (Linux)
|
||||
condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
|
||||
|
||||
- pwsh: SqlLocalDB start MSSQLLocalDB
|
||||
displayName: Start SQL Server LocalDB (Windows)
|
||||
condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
|
||||
|
||||
# Test
|
||||
- task: DotNetCoreCLI@2
|
||||
displayName: Run dotnet test
|
||||
inputs:
|
||||
command: test
|
||||
projects: "tests/Umbraco.Tests.Integration/Umbraco.Tests.Integration.csproj"
|
||||
testRunTitle: Integration Tests SQL Server - $(Agent.OS)
|
||||
arguments: '--filter "$(testFilter)" --configuration $(buildConfiguration) --no-build'
|
||||
|
||||
# Stop SQL Server
|
||||
- pwsh: docker stop mssql
|
||||
displayName: Stop SQL Server Docker image (Linux)
|
||||
condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
|
||||
|
||||
- pwsh: SqlLocalDB stop MSSQLLocalDB
|
||||
displayName: Stop SQL Server LocalDB (Windows)
|
||||
condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
|
||||
|
||||
- stage: E2E
|
||||
displayName: E2E Tests
|
||||
dependsOn: Build
|
||||
@@ -290,17 +473,17 @@ stages:
|
||||
testCommand: "npm run testSqlite -- --shard=1/3"
|
||||
vmImage: "ubuntu-latest"
|
||||
SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
|
||||
CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True"
|
||||
CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);Encrypt=True;TrustServerCertificate=True"
|
||||
LinuxPart2Of3:
|
||||
testCommand: "npm run testSqlite -- --shard=2/3"
|
||||
vmImage: "ubuntu-latest"
|
||||
SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
|
||||
CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True"
|
||||
CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);Encrypt=True;TrustServerCertificate=True"
|
||||
LinuxPart3Of3:
|
||||
testCommand: "npm run testSqlite -- --shard=3/3"
|
||||
vmImage: "ubuntu-latest"
|
||||
SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
|
||||
CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True"
|
||||
CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);Encrypt=True;TrustServerCertificate=True"
|
||||
WindowsPart1Of3:
|
||||
vmImage: "windows-latest"
|
||||
testCommand: "npm run testSqlite -- --shard=1/3"
|
||||
@@ -371,6 +554,33 @@ stages:
|
||||
displayName: Start SQL Server Docker image (Linux)
|
||||
condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
|
||||
|
||||
- powershell: |
|
||||
$maxAttempts = 12
|
||||
$attempt = 0
|
||||
$status = ""
|
||||
|
||||
while (($status -ne 'running') -and ($attempt -lt $maxAttempts)) {
|
||||
Start-Sleep -Seconds 5
|
||||
# We use the docker inspect command to check the status of the container. If the container is not running, we wait 5 seconds and try again. And if reaches 12 attempts, we fail the build.
|
||||
$status = docker inspect -f '{{.State.Status}}' mssql
|
||||
|
||||
if ($status -ne 'running') {
|
||||
Write-Host "Waiting for SQL Server to be ready... Attempt $($attempt + 1)"
|
||||
$attempt++
|
||||
}
|
||||
}
|
||||
|
||||
if ($status -eq 'running') {
|
||||
Write-Host "SQL Server container is running"
|
||||
docker ps -a
|
||||
} else {
|
||||
Write-Host "SQL Server did not become ready in time. Last known status: $status"
|
||||
docker logs mssql
|
||||
exit 1
|
||||
}
|
||||
displayName: Wait for SQL Server to be ready (Linux)
|
||||
condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
|
||||
|
||||
- pwsh: SqlLocalDB start MSSQLLocalDB
|
||||
displayName: Start SQL Server LocalDB (Windows)
|
||||
condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
|
||||
@@ -447,7 +657,7 @@ stages:
|
||||
inputs:
|
||||
targetPath: $(Build.ArtifactStagingDirectory)
|
||||
artifact: "Acceptance Test Results - $(Agent.JobName) - Attempt #$(System.JobAttempt)"
|
||||
|
||||
|
||||
# Publish test results
|
||||
- task: PublishTestResults@2
|
||||
displayName: "Publish test results"
|
||||
|
||||
Reference in New Issue
Block a user