Publish E2E application logs (#13128)

* Publish E2E application logs * Stop app after running E2E tests * Update GenerateAppsettingsSchema task * Enable console logging in Release mode * Cleanup variables and enable logging in Linux docker * Remove --yes from npx commands * Set custom Umbraco settings (disable keep-alive, health checks, etc.) * Update Docker image tags/versions * Change PowerShell@2 task to pwsh and write results to artifact staging directory * Ignore Playwright .env file
2022-10-10 09:09:59 +02:00
parent b5262d74eb
commit 049ea6dca5
4 changed files with 82 additions and 95 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -90,10 +90,7 @@ preserve.belle
 /src/Umbraco.Web.UI/appsettings.Local.json

 # Tests
-cypress.env.json
-/tests/Umbraco.Tests.AcceptanceTest/cypress/screenshots/
-/tests/Umbraco.Tests.AcceptanceTest/cypress/support/chainable.ts
-/tests/Umbraco.Tests.AcceptanceTest/cypress/videos/
+/tests/Umbraco.Tests.AcceptanceTest/.env
 /tests/Umbraco.Tests.Integration.SqlCe/DatabaseContextTests.sdf
 /tests/Umbraco.Tests.Integration.SqlCe/[Uu]mbraco/[Dd]ata/TEMP/
 /tests/Umbraco.Tests.Integration/appsettings.Tests.Local.json
--- a/build/azure-pipelines.yml
+++ b/build/azure-pipelines.yml
@@ -349,24 +349,10 @@ stages:
      - job:
        displayName: E2E Tests
        variables:
-          - name: Umbraco__CMS__Unattended__InstallUnattended  # Windows only
-            value: true
-          - name: Umbraco__CMS__Unattended__UnattendedUserName  # Windows only
-            value: Playwright Test
-          - name: Umbraco__CMS__Unattended__UnattendedUserEmail  # Windows only
-            value: playwright@umbraco.com
-          - name: Umbraco__CMS__Unattended__UnattendedUserPassword  # Windows only
-            value: UmbracoAcceptance123!
-          - name: Umbraco__CMS__Global__InstallMissingDatabase  # Windows only
-            value: true
-          - name: UmbracoDatabaseServer # Windows only
-            value: (LocalDB)\MSSQLLocalDB
-          - name: UmbracoDatabaseName  # Windows only
-            value: Playwright
-          - name: ConnectionStrings__umbracoDbDSN  # Windows only
-            value: Server=$(UmbracoDatabaseServer);Database=$(UmbracoDatabaseName);Integrated Security=true;
-          - name: PLAYWRIGHT_BASE_URL
-            value: https://localhost:8443
+          Umbraco__CMS__Unattended__UnattendedUserName: Playwright Test
+          Umbraco__CMS__Unattended__UnattendedUserPassword: UmbracoAcceptance123!
+          Umbraco__CMS__Unattended__UnattendedUserEmail: playwright@umbraco.com
+          ASPNETCORE_URLS: https://localhost:8443
        strategy:
          matrix:
            Linux:
@@ -375,6 +361,20 @@ stages:
              dockerImageName: umbraco-linux
            Windows:
              vmImage: 'windows-latest'
+              # Enable console logging in Release mode
+              Serilog__WriteTo__0__Name: Async
+              Serilog__WriteTo__0__Args__configure__0__Name: Console
+              # Set unattended install settings
+              Umbraco__CMS__Unattended__InstallUnattended: true
+              Umbraco__CMS__Global__InstallMissingDatabase: true
+              UmbracoDatabaseServer: (LocalDB)\MSSQLLocalDB
+              UmbracoDatabaseName: Playwright
+              ConnectionStrings__umbracoDbDSN: Server=$(UmbracoDatabaseServer);Database=$(UmbracoDatabaseName);Integrated Security=true;
+              # Custom Umbraco settings
+              Umbraco__CMS__Global__VersionCheckPeriod: 0
+              Umbraco__CMS__Global__UseHttps: true
+              Umbraco__CMS__HealthChecks__Notification__Enabled: false
+              Umbraco__CMS__KeepAlive__DisableKeepAliveTask: true
        pool:
          vmImage: $(vmImage)
        steps:
@@ -395,22 +395,19 @@ stages:
                "npm_e2e" | "$(Agent.OS)"
                "npm_e2e"
              path: $(npm_config_cache)
-          - task: PowerShell@2
+          - pwsh: |
+              New-Item -Path "." -Name ".env" -ItemType "file" -Value "UMBRACO_USER_LOGIN=$(Umbraco__CMS__Unattended__UnattendedUserEmail)
+              UMBRACO_USER_PASSWORD=$(Umbraco__CMS__Unattended__UnattendedUserPassword)
+              URL=$(ASPNETCORE_URLS)"
            displayName: Generate .env
-            inputs:
-              targetType: inline
-              workingDirectory: $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest/
-              script: |
-                New-Item -Path "." -Name ".env" -ItemType "file" -Value "UMBRACO_USER_LOGIN=$(Umbraco__CMS__Unattended__UnattendedUserEmail)
-                UMBRACO_USER_PASSWORD=$(Umbraco__CMS__Unattended__UnattendedUserPassword)
-                URL=$(PLAYWRIGHT_BASE_URL)"
+            workingDirectory: $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest/
          - script: npm ci --no-fund --no-audit --prefer-offline
            workingDirectory: $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest/
            displayName: Run npm ci
-          - powershell: sqllocaldb start mssqllocaldb
+          - pwsh: sqllocaldb start mssqllocaldb
            displayName: Start localdb (Windows only)
            condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
-          - powershell:  Invoke-Sqlcmd -Query "CREATE DATABASE $env:UmbracoDatabaseName" -ServerInstance $env:UmbracoDatabaseServer
+          - pwsh: Invoke-Sqlcmd -Query "CREATE DATABASE $env:UmbracoDatabaseName" -ServerInstance $env:UmbracoDatabaseServer
            displayName: Create database (Windows only)
            condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
          - task: UseDotNet@2
@@ -419,72 +416,52 @@ stages:
              version: $(dotnetVersion)
              performMultiLevelLookup: true
              includePreviewVersions: $(dotnetIncludePreviewVersions)
-          # Linux containers smooth
-          - task: PowerShell@2
+          - pwsh: |
+              $sha = 'g$(Build.SourceVersion)'.substring(0, 8)
+              docker build -t $(dockerImageName):$sha -f $(dockerfile) .
+              mkdir -p $(Build.ArtifactStagingDirectory)/docker-images
+              docker save -o $(Build.ArtifactStagingDirectory)/docker-images/$(dockerImageName).$sha.tar $(dockerImageName):$sha
+              dotnet dev-certs https -ep ${HOME}/.aspnet/https/aspnetapp.pfx -p $(Umbraco__CMS__Unattended__UnattendedUserPassword)
+              docker run --name $(dockerImageName) -dp 8080:5000 -dp 8443:5001 -e UMBRACO__CMS__GLOBAL__ID=$(UMBRACO__CMS__GLOBAL__ID) -e ASPNETCORE_Kestrel__Certificates__Default__Password="$(Umbraco__CMS__Unattended__UnattendedUserPassword)" -e ASPNETCORE_Kestrel__Certificates__Default__Path=/https/aspnetapp.pfx -v ${HOME}/.aspnet/https:/https/ $(dockerImageName):$sha
+              docker ps
            condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
-            displayName: Build & run container (Linux only)
-            inputs:
-              workingDirectory: tests/Umbraco.Tests.AcceptanceTest/misc
-              targetType: inline
-              script: |
-                $sha = 'g$(Build.SourceVersion)'.substring(0, 8)
-                docker build -t $(dockerImageName):$sha -f $(dockerfile) .
-                mkdir -p $(Build.ArtifactStagingDirectory)/docker-images
-                docker save -o $(Build.ArtifactStagingDirectory)/docker-images/$(dockerImageName).$sha.tar $(dockerImageName):$sha
-                dotnet dev-certs https -ep ${HOME}/.aspnet/https/aspnetapp.pfx -p UmbracoAcceptance123!
-                docker run --name $(dockerImageName) -dp 8080:5000 -dp 8443:5001 -e UMBRACO__CMS__GLOBAL__ID=$(UMBRACO__CMS__GLOBAL__ID) -e ASPNETCORE_Kestrel__Certificates__Default__Password="UmbracoAcceptance123!" -e ASPNETCORE_Kestrel__Certificates__Default__Path=/https/aspnetapp.pfx -v ${HOME}/.aspnet/https:/https/ $(dockerImageName):$sha
-                docker ps
-          # Urls matching docker setup.
-          - task: PowerShell@2
+            displayName: Build and run container (Linux only)
+            workingDirectory: tests/Umbraco.Tests.AcceptanceTest/misc
+          - pwsh: |
+              dotnet new --install ./nupkg/Umbraco.Templates.*.nupkg
+              dotnet new umbraco --name Playwright --no-restore --output .
+              dotnet restore --configfile ./nuget.config
+              dotnet build --configuration $(buildConfiguration) --no-restore
+              dotnet dev-certs https
+              $process = Start-Process -FilePath "dotnet" -ArgumentList "run --configuration $(buildConfiguration) --no-build --no-launch-profile 2>&1" -PassThru -RedirectStandardOutput $(Build.ArtifactStagingDirectory)/playwright.log
+              Write-Host "##vso[task.setvariable variable=AcceptanceTestProcessId]$($process.Id)"
            condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
-            displayName: Build & run app (Windows only)
-            inputs:
-              workingDirectory: tests/Umbraco.Tests.AcceptanceTest/misc
-              targetType: inline
-              script: |
-                dotnet new --install ./nupkg/Umbraco.Templates.*.nupkg
-                dotnet new umbraco --name Playwright --no-restore --output .
-                dotnet restore --configfile ./nuget.config
-                dotnet build --configuration $(buildConfiguration) --no-restore
-                dotnet dev-certs https
-                Start-Process -FilePath "dotnet" -ArgumentList "run --configuration $(buildConfiguration) --no-build --no-launch-profile --urls $(PLAYWRIGHT_BASE_URL)"
-          - task: PowerShell@2
+            displayName: Build and run app (Windows only)
+            workingDirectory: tests/Umbraco.Tests.AcceptanceTest/misc
+          - pwsh: npx wait-on -v --interval 1000 --timeout 120000 $(ASPNETCORE_URLS)
            displayName: Wait for app
-            inputs:
-              targetType: inline
-              workingDirectory: tests/Umbraco.Tests.AcceptanceTest
-              script: |
-                npm i -g wait-on
-                wait-on -v --interval 1000 --timeout 120000 $(PLAYWRIGHT_BASE_URL)
-          - task: PowerShell@2
+            workingDirectory: tests/Umbraco.Tests.AcceptanceTest
+          - pwsh: npx playwright install --with-deps
            displayName: Install Playwright
-            inputs:
-              targetType: inline
-              workingDirectory: tests/Umbraco.Tests.AcceptanceTest
-              script: npx playwright install
-          - task: PowerShell@2
+            workingDirectory: tests/Umbraco.Tests.AcceptanceTest
+          - pwsh: npm run test --ignore-certificate-errors --output $(Build.ArtifactStagingDirectory)\test-results
            displayName: Run Playwright (Desktop)
            continueOnError: true
-            inputs:
-              targetType: inline
-              workingDirectory: tests/Umbraco.Tests.AcceptanceTest
-              script: 'npm run test --ignore-certificate-errors'
-          - task: PowerShell@2
-            displayName: Check if artifacts folder exists
-            inputs:
-              targetType: inline
-              script: |
-                $MyVariable = Test-Path -Path $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest/results
-                Write-Host "##vso[task.setvariable variable=resultFolderExists;]$MyVariable"
-          - task: CopyFiles@2
-            displayName: Prepare artifacts
-            condition: eq(variables.resultFolderExists, 'True')
-            inputs:
-              sourceFolder: $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest/results/
-              targetFolder: $(Build.ArtifactStagingDirectory)/playwright
+            workingDirectory: tests/Umbraco.Tests.AcceptanceTest
+            env:
+              CI: true
+              PLAYWRIGHT_HTML_REPORT: $(Build.ArtifactStagingDirectory)\playwright-report
+          - pwsh: |
+              docker logs $(dockerImageName) > $(Build.ArtifactStagingDirectory)/playwright.log 2>&1
+              docker stop $(dockerImageName)
+            condition: eq(variables['Agent.OS'], 'Linux')
+            displayName: Stop app (Linux only)
+          - pwsh: Stop-Process $env:AcceptanceTestProcessId
+            condition: eq(variables['Agent.OS'], 'Windows_NT')
+            displayName: Stop app (Windows only)
          - task: PublishPipelineArtifact@1
-            displayName: "Publish test artifacts"
-            condition: eq(variables.resultFolderExists, 'True')
+            condition: always()
+            displayName: Publish test artifacts
            inputs:
              targetPath: $(Build.ArtifactStagingDirectory)
              artifact: 'E2E artifacts - $(Agent.OS) - Attempt #$(System.JobAttempt)'
--- a/src/Umbraco.Cms/Umbraco.Cms.csproj
+++ b/src/Umbraco.Cms/Umbraco.Cms.csproj
@@ -26,6 +26,6 @@
  </PropertyGroup>
  <Target Name="GenerateAppsettingsSchema" BeforeTargets="Build" Condition="!Exists('$(JsonSchemaPath)')">
    <Message Text="Generating appsettings-schema.json because it doesn't exist" Importance="high" />
-    <Exec WorkingDirectory="$(JsonSchemaProjectPath)" Command="dotnet run -c Release -o &quot;$(JsonSchemaPath)&quot;" />
+    <Exec WorkingDirectory="$(JsonSchemaProjectPath)" Command="dotnet run --configuration Release -- --outputFile &quot;$(JsonSchemaPath)&quot;" />
  </Target>
 </Project>
--- a/tests/Umbraco.Tests.AcceptanceTest/misc/umbraco-linux.docker
+++ b/tests/Umbraco.Tests.AcceptanceTest/misc/umbraco-linux.docker
@@ -2,7 +2,7 @@
 ## Build
 ############################################

-FROM mcr.microsoft.com/dotnet/sdk:6.0.300 AS build
+FROM mcr.microsoft.com/dotnet/sdk:6.0.401 AS build

 COPY nuget.config .

@@ -20,12 +20,16 @@ RUN dotnet publish --configuration Release --no-build --output /dist
 ## Run
 ############################################

-FROM mcr.microsoft.com/dotnet/aspnet:6.0.5 AS run
+FROM mcr.microsoft.com/dotnet/aspnet:6.0.9 AS run

 WORKDIR /app
 COPY --from=build dist .

-ENV ASPNETCORE_URLS="http://0.0.0.0:5000;https://0.0.0.0:5001"
+# Enable console logging in Release mode
+ENV Serilog__WriteTo__0__Name=Async
+ENV Serilog__WriteTo__0__Args__configure__0__Name=Console
+
+# Set unattended install settings
 ENV ConnectionStrings__umbracoDbDSN_ProviderName="Microsoft.Data.Sqlite"
 ENV ConnectionStrings__umbracoDbDSN="Data Source=|DataDirectory|/Umbraco.sqlite.db;Cache=Shared;Foreign Keys=True;Pooling=True"
 ENV Umbraco__CMS__Unattended__InstallUnattended="true"
@@ -33,4 +37,13 @@ ENV Umbraco__CMS__Unattended__UnattendedUserName="Playwright Test"
 ENV Umbraco__CMS__Unattended__UnattendedUserEmail="playwright@umbraco.com"
 ENV Umbraco__CMS__Unattended__UnattendedUserPassword="UmbracoAcceptance123!"

+# Custom Umbraco settings
+ENV Umbraco__CMS__Global__VersionCheckPeriod="0"
+ENV Umbraco__CMS__Global__UseHttps="true"
+ENV Umbraco__CMS__HealthChecks__Notification__Enabled="false"
+ENV Umbraco__CMS__KeepAlive__DisableKeepAliveTask="true"
+
+# Set application URL
+ENV ASPNETCORE_URLS="http://0.0.0.0:5000;https://0.0.0.0:5001"
+
 CMD dotnet Playwright.dll