diff --git a/build/azure-pipelines.yml b/build/azure-pipelines.yml index 977ce443d0..dc5ef96bde 100644 --- a/build/azure-pipelines.yml +++ b/build/azure-pipelines.yml @@ -517,10 +517,7 @@ stages: workingDirectory: tests/Umbraco.Tests.AcceptanceTest # Test - - ${{ if eq(parameters.isNightly, true) }}: - pwsh: npm run test --ignore-certificate-errors - ${{ else }}: - pwsh: npm run smokeTest --ignore-certificate-errors + - pwsh: npm run smokeTest --ignore-certificate-errors displayName: Run Playwright tests continueOnError: true workingDirectory: tests/Umbraco.Tests.AcceptanceTest @@ -660,10 +657,7 @@ stages: workingDirectory: tests/Umbraco.Tests.AcceptanceTest # Test - - ${{ if eq(parameters.isNightly, true) }}: - pwsh: npm run test --ignore-certificate-errors - ${{ else }}: - pwsh: npm run smokeTest --ignore-certificate-errors + - pwsh: npm run smokeTest --ignore-certificate-errors displayName: Run Playwright tests continueOnError: true workingDirectory: tests/Umbraco.Tests.AcceptanceTest diff --git a/build/nightly-E2E-test-pipelines.yml b/build/nightly-E2E-test-pipelines.yml new file mode 100644 index 0000000000..b50b2405d6 --- /dev/null +++ b/build/nightly-E2E-test-pipelines.yml @@ -0,0 +1,395 @@ +name: Nightly_E2E_Test_$(TeamProject)_$(Build.DefinitionName)_$(SourceBranchName)_$(Date:yyyyMMdd)$(Rev:.r) + +pr: none +trigger: none + +schedules: +- cron: '0 0 * * *' + displayName: Daily midnight build + branches: + include: + - v14/dev + ## Uncomment after merged to v15/dev + ## - v15/dev + +variables: + nodeVersion: 20 + solution: umbraco.sln + buildConfiguration: Release + UMBRACO__CMS__GLOBAL__ID: 00000000-0000-0000-0000-000000000042 + DOTNET_NOLOGO: true + DOTNET_GENERATE_ASPNET_CERTIFICATE: false + DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true + DOTNET_CLI_TELEMETRY_OPTOUT: true + npm_config_cache: $(Pipeline.Workspace)/.npm_client + NODE_OPTIONS: --max_old_space_size=16384 + +parameters: + - name: runSqlServerE2ETests + displayName: Run the SQL Server E2E Tests + type: boolean + default: false + +stages: + ############################################### + ## Build + ############################################### + - stage: Build + jobs: + - job: A + displayName: Build Umbraco CMS + pool: + vmImage: 'ubuntu-latest' + steps: + - checkout: self + fetchDepth: 10 + submodules: true + - task: UseDotNet@2 + displayName: Use .NET SDK from global.json + inputs: + useGlobalJson: true + - template: templates/backoffice-install.yml + - script: npm run build:for:cms + displayName: Run build (Bellissima) + workingDirectory: src/Umbraco.Web.UI.Client + - script: npm ci --no-fund --no-audit --prefer-offline + displayName: Run npm ci (Login) + workingDirectory: src/Umbraco.Web.UI.Login + - script: npm run build + displayName: Run npm build (Login) + workingDirectory: src/Umbraco.Web.UI.Login + - task: DotNetCoreCLI@2 + displayName: Run dotnet restore + inputs: + command: restore + projects: $(solution) + - task: DotNetCoreCLI@2 + name: build + displayName: Run dotnet build and generate NuGet packages + inputs: + command: build + projects: $(solution) + arguments: '--configuration $(buildConfiguration) --no-restore --property:ContinuousIntegrationBuild=true --property:GeneratePackageOnBuild=true --property:PackageOutputPath=$(Build.ArtifactStagingDirectory)/nupkg' + - task: PublishPipelineArtifact@1 + displayName: Publish nupkg + inputs: + targetPath: $(Build.ArtifactStagingDirectory)/nupkg + artifactName: nupkg + - task: PublishPipelineArtifact@1 + displayName: Publish build artifacts + inputs: + targetPath: $(Build.SourcesDirectory) + artifactName: build_output + + - stage: E2E + displayName: E2E Tests + dependsOn: Build + variables: + npm_config_cache: $(Pipeline.Workspace)/.npm_e2e + # Enable console logging in Release mode + SERILOG__WRITETO__0__NAME: Async + SERILOG__WRITETO__0__ARGS__CONFIGURE__0__NAME: Console + # Set unattended install settings + UMBRACO__CMS__UNATTENDED__INSTALLUNATTENDED: true + UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERNAME: Playwright Test + UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD: UmbracoAcceptance123! + UMBRACO__CMS__UNATTENDED__UNATTENDEDUSEREMAIL: playwright@umbraco.com + # Custom Umbraco settings + UMBRACO__CMS__CONTENT__CONTENTVERSIONCLEANUPPOLICY__ENABLECLEANUP: false + UMBRACO__CMS__GLOBAL__DISABLEELECTIONFORSINGLESERVER: true + UMBRACO__CMS__GLOBAL__INSTALLMISSINGDATABASE: true + UMBRACO__CMS__GLOBAL__ID: 00000000-0000-0000-0000-000000000042 + UMBRACO__CMS__GLOBAL__VERSIONCHECKPERIOD: 0 + UMBRACO__CMS__GLOBAL__USEHTTPS: true + UMBRACO__CMS__HEALTHCHECKS__NOTIFICATION__ENABLED: false + UMBRACO__CMS__KEEPALIVE__DISABLEKEEPALIVETASK: true + UMBRACO__CMS__WEBROUTING__UMBRACOAPPLICATIONURL: https://localhost:44331/ + ASPNETCORE_URLS: https://localhost:44331 + jobs: + # E2E Tests + - job: + displayName: E2E Tests (SQLite) + timeoutInMinutes: 120 + variables: + # Connection string + CONNECTIONSTRINGS__UMBRACODBDSN: Data Source=Umbraco;Mode=Memory;Cache=Shared;Foreign Keys=True;Pooling=True + CONNECTIONSTRINGS__UMBRACODBDSN_PROVIDERNAME: Microsoft.Data.Sqlite + strategy: + matrix: + Linux: + vmImage: 'ubuntu-latest' + Windows: + vmImage: 'windows-latest' + pool: + vmImage: $(vmImage) + steps: + # Setup test environment + - task: DownloadPipelineArtifact@2 + displayName: Download NuGet artifacts + inputs: + artifact: nupkg + path: $(Agent.BuildDirectory)/app/nupkg + + - task: NodeTool@0 + displayName: Use Node.js $(nodeVersion) + retryCountOnTaskFailure: 3 + inputs: + versionSpec: $(nodeVersion) + + - task: UseDotNet@2 + displayName: Use .NET SDK from global.json + inputs: + useGlobalJson: true + + - pwsh: | + "UMBRACO_USER_LOGIN=$(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSEREMAIL) + UMBRACO_USER_PASSWORD=$(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD) + URL=$(ASPNETCORE_URLS) + STORAGE_STAGE_PATH=$(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest/playwright/.auth/user.json" | Out-File .env + displayName: Generate .env + workingDirectory: $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest + + # Cache and restore NPM packages + - task: Cache@2 + displayName: Cache NPM packages + inputs: + key: 'npm_e2e | "$(Agent.OS)" | $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest/package-lock.json' + restoreKeys: | + npm_e2e | "$(Agent.OS)" + npm_e2e + path: $(npm_config_cache) + + - script: npm ci --no-fund --no-audit --prefer-offline + workingDirectory: $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest + displayName: Restore NPM packages + + # Build application + - pwsh: | + $cmsVersion = "$(Build.BuildNumber)" -replace "\+",".g" + dotnet new nugetconfig + dotnet nuget add source ./nupkg --name Local + dotnet new install Umbraco.Templates::$cmsVersion + dotnet new umbraco --name UmbracoProject --version $cmsVersion --exclude-gitignore --no-restore --no-update-check + dotnet restore UmbracoProject + cp $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest.UmbracoProject/*.cs UmbracoProject + dotnet build UmbracoProject --configuration $(buildConfiguration) --no-restore + dotnet dev-certs https + displayName: Build application + workingDirectory: $(Agent.BuildDirectory)/app + + # Run application + - bash: | + nohup dotnet run --project UmbracoProject --configuration $(buildConfiguration) --no-build --no-launch-profile > $(Build.ArtifactStagingDirectory)/playwright.log 2>&1 & + echo "##vso[task.setvariable variable=AcceptanceTestProcessId]$!" + displayName: Run application (Linux) + condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux')) + workingDirectory: $(Agent.BuildDirectory)/app + + - pwsh: | + $process = Start-Process dotnet "run --project UmbracoProject --configuration $(buildConfiguration) --no-build --no-launch-profile 2>&1" -PassThru -NoNewWindow -RedirectStandardOutput $(Build.ArtifactStagingDirectory)/playwright.log + Write-Host "##vso[task.setvariable variable=AcceptanceTestProcessId]$($process.Id)" + displayName: Run application (Windows) + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) + workingDirectory: $(Agent.BuildDirectory)/app + + # Ensures we have the package wait-on installed + - pwsh: npm install wait-on + displayName: Install wait-on package + + # Wait for application to start responding to requests + - pwsh: npx wait-on -v --interval 1000 --timeout 120000 $(ASPNETCORE_URLS) + displayName: Wait for application + workingDirectory: tests/Umbraco.Tests.AcceptanceTest + + # Install Playwright and dependencies + - pwsh: npx playwright install --with-deps + displayName: Install Playwright + workingDirectory: tests/Umbraco.Tests.AcceptanceTest + + # Test + - pwsh: npm run test --ignore-certificate-errors + displayName: Run Playwright tests + continueOnError: true + workingDirectory: tests/Umbraco.Tests.AcceptanceTest + env: + CI: true + CommitId: $(Build.SourceVersion) + AgentOs: $(Agent.OS) + + # Stop application + - bash: kill -15 $(AcceptanceTestProcessId) + displayName: Stop application (Linux) + condition: and(succeeded(), ne(variables.AcceptanceTestProcessId, ''), eq(variables['Agent.OS'], 'Linux')) + + - pwsh: Stop-Process -Id $(AcceptanceTestProcessId) + displayName: Stop application (Windows) + condition: and(succeeded(), ne(variables.AcceptanceTestProcessId, ''), eq(variables['Agent.OS'], 'Windows_NT')) + + # Copy artifacts + - pwsh: | + if (Test-Path tests/Umbraco.Tests.AcceptanceTest/results/*) { + Copy-Item tests/Umbraco.Tests.AcceptanceTest/results $(Build.ArtifactStagingDirectory) -Recurse + } + displayName: Copy Playwright results + condition: succeededOrFailed() + + # Publish + - task: PublishPipelineArtifact@1 + displayName: Publish test artifacts + condition: succeededOrFailed() + inputs: + targetPath: $(Build.ArtifactStagingDirectory) + artifact: 'Acceptance Tests - $(Agent.JobName) - Attempt #$(System.JobAttempt)' + + - job: + displayName: E2E Tests (SQL Server) + condition: and(succeeded(), ${{ eq(parameters.runSqlServerE2ETests, true) }}) + timeoutInMinutes: 120 + variables: + # Connection string + CONNECTIONSTRINGS__UMBRACODBDSN: Data Source=(localdb)\MSSQLLocalDB;AttachDbFilename=|DataDirectory|\Umbraco.mdf;Integrated Security=True + CONNECTIONSTRINGS__UMBRACODBDSN_PROVIDERNAME: Microsoft.Data.SqlClient + strategy: + matrix: + Linux: + vmImage: 'ubuntu-latest' + SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD) + CONNECTIONSTRINGS__UMBRACODBDSN: 'Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True' + Windows: + vmImage: 'windows-latest' + pool: + vmImage: $(vmImage) + steps: + # Setup test environment + - task: DownloadPipelineArtifact@2 + displayName: Download NuGet artifacts + inputs: + artifact: nupkg + path: $(Agent.BuildDirectory)/app/nupkg + + - task: NodeTool@0 + displayName: Use Node.js $(nodeVersion) + inputs: + versionSpec: $(nodeVersion) + + - task: UseDotNet@2 + displayName: Use .NET SDK from global.json + inputs: + useGlobalJson: true + + - pwsh: | + "UMBRACO_USER_LOGIN=$(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSEREMAIL) + UMBRACO_USER_PASSWORD=$(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD) + URL=$(ASPNETCORE_URLS)" | Out-File .env + displayName: Generate .env + workingDirectory: $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest + + # Cache and restore NPM packages + - task: Cache@2 + displayName: Cache NPM packages + inputs: + key: 'npm_e2e | "$(Agent.OS)" | $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest/package-lock.json' + restoreKeys: | + npm_e2e | "$(Agent.OS)" + npm_e2e + path: $(npm_config_cache) + + - script: npm ci --no-fund --no-audit --prefer-offline + workingDirectory: $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest + displayName: Restore NPM packages + + # Build application + - pwsh: | + $cmsVersion = "$(Build.BuildNumber)" -replace "\+",".g" + dotnet new nugetconfig + dotnet nuget add source ./nupkg --name Local + dotnet new install Umbraco.Templates::$cmsVersion + dotnet new umbraco --name UmbracoProject --version $cmsVersion --exclude-gitignore --no-restore --no-update-check + dotnet restore UmbracoProject + cp $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest.UmbracoProject/*.cs UmbracoProject + dotnet build UmbracoProject --configuration $(buildConfiguration) --no-restore + dotnet dev-certs https + displayName: Build application + workingDirectory: $(Agent.BuildDirectory)/app + + # Start SQL Server + - powershell: docker run --name mssql -d -p 1433:1433 -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=$(SA_PASSWORD)" mcr.microsoft.com/mssql/server:2022-latest + displayName: Start SQL Server Docker image (Linux) + condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux')) + + - pwsh: SqlLocalDB start MSSQLLocalDB + displayName: Start SQL Server LocalDB (Windows) + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) + + # Run application + - bash: | + nohup dotnet run --project UmbracoProject --configuration $(buildConfiguration) --no-build --no-launch-profile > $(Build.ArtifactStagingDirectory)/playwright.log 2>&1 & + echo "##vso[task.setvariable variable=AcceptanceTestProcessId]$!" + displayName: Run application (Linux) + condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux')) + workingDirectory: $(Agent.BuildDirectory)/app + + - pwsh: | + $process = Start-Process dotnet "run --project UmbracoProject --configuration $(buildConfiguration) --no-build --no-launch-profile 2>&1" -PassThru -NoNewWindow -RedirectStandardOutput $(Build.ArtifactStagingDirectory)/playwright.log + Write-Host "##vso[task.setvariable variable=AcceptanceTestProcessId]$($process.Id)" + displayName: Run application (Windows) + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) + workingDirectory: $(Agent.BuildDirectory)/app + + # Ensures we have the package wait-on installed + - pwsh: npm install wait-on + displayName: Install wait-on package + + # Wait for application to start responding to requests + - pwsh: npx wait-on -v --interval 1000 --timeout 120000 $(ASPNETCORE_URLS) + displayName: Wait for application + workingDirectory: tests/Umbraco.Tests.AcceptanceTest + + # Install Playwright and dependencies + - pwsh: npx playwright install --with-deps + displayName: Install Playwright + workingDirectory: tests/Umbraco.Tests.AcceptanceTest + + # Test + - pwsh: npm run test --ignore-certificate-errors + displayName: Run Playwright tests + continueOnError: true + workingDirectory: tests/Umbraco.Tests.AcceptanceTest + env: + CI: true + CommitId: $(Build.SourceVersion) + AgentOs: $(Agent.OS) + + # Stop application + - bash: kill -15 $(AcceptanceTestProcessId) + displayName: Stop application (Linux) + condition: and(succeeded(), ne(variables.AcceptanceTestProcessId, ''), eq(variables['Agent.OS'], 'Linux')) + + - pwsh: Stop-Process -Id $(AcceptanceTestProcessId) + displayName: Stop application (Windows) + condition: and(succeeded(), ne(variables.AcceptanceTestProcessId, ''), eq(variables['Agent.OS'], 'Windows_NT')) + + # Stop SQL Server + - pwsh: docker stop mssql + displayName: Stop SQL Server Docker image (Linux) + condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux')) + + - pwsh: SqlLocalDB stop MSSQLLocalDB + displayName: Stop SQL Server LocalDB (Windows) + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) + + # Copy artifacts + - pwsh: | + if (Test-Path tests/Umbraco.Tests.AcceptanceTest/results/*) { + Copy-Item tests/Umbraco.Tests.AcceptanceTest/results $(Build.ArtifactStagingDirectory) -Recurse + } + displayName: Copy Playwright results + condition: succeededOrFailed() + + # Publish + - task: PublishPipelineArtifact@1 + displayName: Publish test artifacts + condition: succeededOrFailed() + inputs: + targetPath: $(Build.ArtifactStagingDirectory) + artifact: 'Acceptance Tests - $(Agent.JobName) - Attempt #$(System.JobAttempt)' \ No newline at end of file diff --git a/umbraco.sln b/umbraco.sln index 5e26e18d8c..87d95422f8 100644 --- a/umbraco.sln +++ b/umbraco.sln @@ -141,6 +141,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "build", "build", "{20CE9C97 build\azure-pipelines.yml = build\azure-pipelines.yml build\nightly-build-trigger.yml = build\nightly-build-trigger.yml build\templates\backoffice-install.yml = build\templates\backoffice-install.yml + build\nightly-E2E-test-pipelines.yml = build\nightly-E2E-test-pipelines.yml EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "csharp-docs", "csharp-docs", "{F2BF84D9-0A14-40AF-A0F3-B9BBBBC16A44}"