diff --git a/build/azure-pipelines.yml b/build/azure-pipelines.yml index a33ee744f7..92621c30d3 100644 --- a/build/azure-pipelines.yml +++ b/build/azure-pipelines.yml @@ -432,6 +432,33 @@ stages: displayName: Start SQL Server Docker image (Linux) condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux')) + - powershell: | + $maxAttempts = 12 + $attempt = 0 + $status = "" + + while (($status -ne 'running') -and ($attempt -lt $maxAttempts)) { + Start-Sleep -Seconds 5 + # We use the docker inspect command to check the status of the container. If the container is not running, we wait 5 seconds and try again. And if reaches 12 attempts, we fail the build. + $status = docker inspect -f '{{.State.Status}}' mssql + + if ($status -ne 'running') { + Write-Host "Waiting for SQL Server to be ready... Attempt $($attempt + 1)" + $attempt++ + } + } + + if ($status -eq 'running') { + Write-Host "SQL Server container is running" + docker ps -a + } else { + Write-Host "SQL Server did not become ready in time. Last known status: $status" + docker logs mssql + exit 1 + } + displayName: Wait for SQL Server to be ready (Linux) + condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux')) + - pwsh: SqlLocalDB start MSSQLLocalDB displayName: Start SQL Server LocalDB (Windows) condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) diff --git a/build/nightly-E2E-test-pipelines.yml b/build/nightly-E2E-test-pipelines.yml index 54af8efc06..0740eed585 100644 --- a/build/nightly-E2E-test-pipelines.yml +++ b/build/nightly-E2E-test-pipelines.yml @@ -93,6 +93,189 @@ stages: targetPath: $(Build.ArtifactStagingDirectory)/npm artifactName: npm + - stage: Integration + displayName: Integration Tests + dependsOn: Build + jobs: + # Integration Tests (SQLite) + - job: + timeoutInMinutes: 180 + displayName: Integration Tests (SQLite) + strategy: + matrix: + # Windows: + # vmImage: 'windows-latest' + # We split the tests into 3 parts for each OS to reduce the time it takes to run them on the pipeline + LinuxPart1Of3: + vmImage: "ubuntu-latest" + # Filter tests that are part of the Umbraco.Infrastructure namespace but not part of the Umbraco.Infrastructure.Service namespace + testFilter: "(FullyQualifiedName~Umbraco.Infrastructure) & (FullyQualifiedName!~Umbraco.Infrastructure.Service)" + LinuxPart2Of3: + vmImage: "ubuntu-latest" + # Filter tests that are part of the Umbraco.Infrastructure.Service namespace + testFilter: "(FullyQualifiedName~Umbraco.Infrastructure.Service)" + LinuxPart3Of3: + vmImage: "ubuntu-latest" + # Filter tests that are not part of the Umbraco.Infrastructure namespace. So this will run all tests that are not part of the Umbraco.Infrastructure namespace + testFilter: "(FullyQualifiedName!~Umbraco.Infrastructure)" + macOSPart1Of3: + vmImage: "macOS-latest" + # Filter tests that are part of the Umbraco.Infrastructure namespace but not part of the Umbraco.Infrastructure.Service namespace + testFilter: "(FullyQualifiedName~Umbraco.Infrastructure) & (FullyQualifiedName!~Umbraco.Infrastructure.Service)" + macOSPart2Of3: + vmImage: "macOS-latest" + # Filter tests that are part of the Umbraco.Infrastructure.Service namespace + testFilter: "(FullyQualifiedName~Umbraco.Infrastructure.Service)" + macOSPart3Of3: + vmImage: "macOS-latest" + # Filter tests that are not part of the Umbraco.Infrastructure namespace. + testFilter: "(FullyQualifiedName!~Umbraco.Infrastructure)" + pool: + vmImage: $(vmImage) + variables: + Tests__Database__DatabaseType: "Sqlite" + steps: + - checkout: self + submodules: false + lfs: false, + fetchDepth: 1 + fetchFilter: tree:0 + # Setup test environment + - task: DownloadPipelineArtifact@2 + displayName: Download build artifacts + inputs: + artifact: build_output + path: $(Build.SourcesDirectory) + + - task: UseDotNet@2 + displayName: Use .NET SDK from global.json + inputs: + useGlobalJson: true + + # Test + - task: DotNetCoreCLI@2 + displayName: Run dotnet test + inputs: + command: test + projects: "tests/Umbraco.Tests.Integration/Umbraco.Tests.Integration.csproj" + testRunTitle: Integration Tests SQLite - $(Agent.OS) + arguments: '--filter "$(testFilter)" --configuration $(buildConfiguration) --no-build' + + # Integration Tests (SQL Server) + - job: + timeoutInMinutes: 180 + displayName: Integration Tests (SQL Server) + strategy: + matrix: + # We split the tests into 3 parts for each OS to reduce the time it takes to run them on the pipeline + WindowsPart1Of3: + vmImage: "windows-latest" + Tests__Database__DatabaseType: LocalDb + Tests__Database__SQLServerMasterConnectionString: N/A + # Filter tests that are part of the Umbraco.Infrastructure namespace but not part of the Umbraco.Infrastructure.Service namespace + testFilter: "(FullyQualifiedName~Umbraco.Infrastructure) & (FullyQualifiedName!~Umbraco.Infrastructure.Service)" + WindowsPart2Of3: + vmImage: "windows-latest" + Tests__Database__DatabaseType: LocalDb + Tests__Database__SQLServerMasterConnectionString: N/A + # Filter tests that are part of the Umbraco.Infrastructure.Service namespace + testFilter: "(FullyQualifiedName~Umbraco.Infrastructure.Service)" + WindowsPart3Of3: + vmImage: "windows-latest" + Tests__Database__DatabaseType: LocalDb + Tests__Database__SQLServerMasterConnectionString: N/A + # Filter tests that are not part of the Umbraco.Infrastructure namespace. So this will run all tests that are not part of the Umbraco.Infrastructure namespace + testFilter: "(FullyQualifiedName!~Umbraco.Infrastructure)" + LinuxPart1Of3: + vmImage: "ubuntu-latest" + SA_PASSWORD: UmbracoIntegration123! + Tests__Database__DatabaseType: SqlServer + Tests__Database__SQLServerMasterConnectionString: "Server=(local);User Id=sa;Password=$(SA_PASSWORD);Encrypt=True;TrustServerCertificate=True" + # Filter tests that are part of the Umbraco.Infrastructure namespace but not part of the Umbraco.Infrastructure.Service namespace + testFilter: "(FullyQualifiedName~Umbraco.Infrastructure) & (FullyQualifiedName!~Umbraco.Infrastructure.Service)" + LinuxPart2Of3: + vmImage: "ubuntu-latest" + SA_PASSWORD: UmbracoIntegration123! + Tests__Database__DatabaseType: SqlServer + Tests__Database__SQLServerMasterConnectionString: "Server=(local);User Id=sa;Password=$(SA_PASSWORD);Encrypt=True;TrustServerCertificate=True" + # Filter tests that are part of the Umbraco.Infrastructure.Service namespace + testFilter: "(FullyQualifiedName~Umbraco.Infrastructure.Service)" + LinuxPart3Of3: + vmImage: "ubuntu-latest" + SA_PASSWORD: UmbracoIntegration123! + Tests__Database__DatabaseType: SqlServer + Tests__Database__SQLServerMasterConnectionString: "Server=(local);User Id=sa;Password=$(SA_PASSWORD);Encrypt=True;TrustServerCertificate=True" + # Filter tests that are not part of the Umbraco.Infrastructure namespace. So this will run all tests that are not part of the Umbraco.Infrastructure namespace + testFilter: "(FullyQualifiedName!~Umbraco.Infrastructure)" + pool: + vmImage: $(vmImage) + steps: + # Setup test environment + - task: DownloadPipelineArtifact@2 + displayName: Download build artifacts + inputs: + artifact: build_output + path: $(Build.SourcesDirectory) + + - task: UseDotNet@2 + displayName: Use .NET SDK from global.json + inputs: + useGlobalJson: true + + # Start SQL Server + - powershell: docker run --name mssql -d -p 1433:1433 -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=$(SA_PASSWORD)" mcr.microsoft.com/mssql/server:2022-latest + displayName: Start SQL Server Docker image (Linux) + condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux')) + + - powershell: | + $maxAttempts = 12 + $attempt = 0 + $status = "" + + while (($status -ne 'running') -and ($attempt -lt $maxAttempts)) { + Start-Sleep -Seconds 5 + # We use the docker inspect command to check the status of the container. If the container is not running, we wait 5 seconds and try again. And if reaches 12 attempts, we fail the build. + $status = docker inspect -f '{{.State.Status}}' mssql + + if ($status -ne 'running') { + Write-Host "Waiting for SQL Server to be ready... Attempt $($attempt + 1)" + $attempt++ + } + } + + if ($status -eq 'running') { + Write-Host "SQL Server container is running" + docker ps -a + } else { + Write-Host "SQL Server did not become ready in time. Last known status: $status" + docker logs mssql + exit 1 + } + displayName: Wait for SQL Server to be ready (Linux) + condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux')) + + - pwsh: SqlLocalDB start MSSQLLocalDB + displayName: Start SQL Server LocalDB (Windows) + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) + + # Test + - task: DotNetCoreCLI@2 + displayName: Run dotnet test + inputs: + command: test + projects: "tests/Umbraco.Tests.Integration/Umbraco.Tests.Integration.csproj" + testRunTitle: Integration Tests SQL Server - $(Agent.OS) + arguments: '--filter "$(testFilter)" --configuration $(buildConfiguration) --no-build' + + # Stop SQL Server + - pwsh: docker stop mssql + displayName: Stop SQL Server Docker image (Linux) + condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux')) + + - pwsh: SqlLocalDB stop MSSQLLocalDB + displayName: Stop SQL Server LocalDB (Windows) + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) + - stage: E2E displayName: E2E Tests dependsOn: Build @@ -290,17 +473,17 @@ stages: testCommand: "npm run testSqlite -- --shard=1/3" vmImage: "ubuntu-latest" SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD) - CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True" + CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);Encrypt=True;TrustServerCertificate=True" LinuxPart2Of3: testCommand: "npm run testSqlite -- --shard=2/3" vmImage: "ubuntu-latest" SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD) - CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True" + CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);Encrypt=True;TrustServerCertificate=True" LinuxPart3Of3: testCommand: "npm run testSqlite -- --shard=3/3" vmImage: "ubuntu-latest" SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD) - CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True" + CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);Encrypt=True;TrustServerCertificate=True" WindowsPart1Of3: vmImage: "windows-latest" testCommand: "npm run testSqlite -- --shard=1/3" @@ -371,6 +554,33 @@ stages: displayName: Start SQL Server Docker image (Linux) condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux')) + - powershell: | + $maxAttempts = 12 + $attempt = 0 + $status = "" + + while (($status -ne 'running') -and ($attempt -lt $maxAttempts)) { + Start-Sleep -Seconds 5 + # We use the docker inspect command to check the status of the container. If the container is not running, we wait 5 seconds and try again. And if reaches 12 attempts, we fail the build. + $status = docker inspect -f '{{.State.Status}}' mssql + + if ($status -ne 'running') { + Write-Host "Waiting for SQL Server to be ready... Attempt $($attempt + 1)" + $attempt++ + } + } + + if ($status -eq 'running') { + Write-Host "SQL Server container is running" + docker ps -a + } else { + Write-Host "SQL Server did not become ready in time. Last known status: $status" + docker logs mssql + exit 1 + } + displayName: Wait for SQL Server to be ready (Linux) + condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux')) + - pwsh: SqlLocalDB start MSSQLLocalDB displayName: Start SQL Server LocalDB (Windows) condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) @@ -447,7 +657,7 @@ stages: inputs: targetPath: $(Build.ArtifactStagingDirectory) artifact: "Acceptance Test Results - $(Agent.JobName) - Attempt #$(System.JobAttempt)" - + # Publish test results - task: PublishTestResults@2 displayName: "Publish test results"