Umbraco-CMS/build/azure-pipelines.yml

name: $(TeamProject)_$(Build.DefinitionName)_$(SourceBranchName)_$(Date:yyyyMMdd)$(Rev:.r)

parameters:
  - name: sqlServerIntegrationTests
    displayName: Run SQL Server Integration Tests
    type: boolean
    default: false
  - name: myGetDeploy
    displayName: Deploy to MyGet
    type: boolean
    default: false
  - name: nuGetDeploy
    displayName: Deploy to NuGet
    type: boolean
    default: false
  - name: buildApiDocs
    displayName: Build API docs
    type: boolean
    default: false
  - name: uploadApiDocs
    displayName: Upload API docs
    type: boolean
    default: false

variables:
  nodeVersion: 16.17.0
  dotnetVersion: 7.x
  dotnetIncludePreviewVersions: true
  solution: umbraco.sln
  buildConfiguration: Release
  UMBRACO__CMS__GLOBAL__ID: 00000000-0000-0000-0000-000000000042
  DOTNET_NOLOGO: true
  DOTNET_GENERATE_ASPNET_CERTIFICATE: false
  DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
  DOTNET_CLI_TELEMETRY_OPTOUT: true

stages:
  ###############################################
  ## Build
  ###############################################
  - stage: Build
    variables:
      npm_config_cache: $(Pipeline.Workspace)/.npm_client
    jobs:
      - job: A
        displayName: Build Umbraco CMS
        pool:
          vmImage: 'ubuntu-latest'
        steps:
          - task: NodeTool@0
            displayName: Use Node.js $(nodeVersion)
            inputs:
              versionSpec: $(nodeVersion)
          - task: Cache@2
            displayName: Cache node_modules
            inputs:
              key: '"npm_client" | "$(Agent.OS)" | $(Build.SourcesDirectory)/src/Umbraco.Web.UI.Client/package-lock.json'
              restoreKeys: |
                "npm_client" | "$(Agent.OS)"
                "npm_client"
              path: $(npm_config_cache)
          - script: npm ci --no-fund --no-audit --prefer-offline
            workingDirectory:  src/Umbraco.Web.UI.Client
            displayName: Run npm ci
          - task: gulp@0
            displayName: Run gulp build
            inputs:
              gulpFile: src/Umbraco.Web.UI.Client/gulpfile.js
              targets: coreBuild
              workingDirectory: src/Umbraco.Web.UI.Client
          - task: UseDotNet@2
            displayName: Use .NET $(dotnetVersion)
            inputs:
              version: $(dotnetVersion)
              includePreviewVersions: $(dotnetIncludePreviewVersions)
          - task: DotNetCoreCLI@2
            displayName: Run dotnet restore
            inputs:
              command: restore
              projects: $(solution)
          - task: DotNetCoreCLI@2
            displayName: Run dotnet build
            inputs:
              command: build
              projects: $(solution)
              arguments: '--configuration $(buildConfiguration) --no-restore -p:ContinuousIntegrationBuild=true'
          - script: |
              version="$(Build.BuildNumber)"
              echo "varsion: $version"

              major="$(echo $version | cut -d '.' -f 1)"
              echo "major version: $major"

              echo "##vso[task.setvariable variable=majorVersion;isOutput=true]$major"
            displayName: Set major version
            name: determineMajorVersion
          - task: PowerShell@2
            displayName: Prepare nupkg
            inputs:
              targetType: inline
              script: |
                $umbracoVersion = "$(Build.BuildNumber)" -replace "\+",".g"
                $templatePaths = Get-ChildItem 'templates/**/.template.config/template.json'

                foreach ($templatePath in $templatePaths) {
                  $a = Get-Content $templatePath -Raw | ConvertFrom-Json
                  if ($a.symbols -and $a.symbols.UmbracoVersion) {
                    $a.symbols.UmbracoVersion.defaultValue = $umbracoVersion
                    $a | ConvertTo-Json -Depth 32 | Set-Content $templatePath
                  }
                }

                dotnet pack $(solution) --configuration $(buildConfiguration) --no-build --output $(Build.ArtifactStagingDirectory)/nupkg
          - script: |
              sha="$(Build.SourceVersion)"
              sha=${sha:0:7}
              buildnumber="$(Build.BuildNumber)_$(Build.BuildId)_$sha"
              echo "##vso[build.updatebuildnumber]$buildnumber"
            displayName: Update build number
          - task: PublishPipelineArtifact@1
            displayName: Publish nupkg
            inputs:
              targetPath: $(Build.ArtifactStagingDirectory)/nupkg
              artifactName: nupkg
          - task: PublishPipelineArtifact@1
            displayName: Publish build artifacts
            inputs:
              targetPath: $(Build.SourcesDirectory)
              artifactName: build_output

  - stage: Build_Docs
    condition: and(succeeded(), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), ${{parameters.buildApiDocs}}))
    displayName: Prepare API Documentation
    dependsOn: Build
    variables:
      umbracoMajorVersion: $[ stageDependencies.Build.A.outputs['determineMajorVersion.majorVersion'] ]
    jobs:
      # C# API Reference
      - job:
        displayName: Build C# API Reference
        pool:
          vmImage: 'windows-latest'
        steps:
          - task: PowerShell@2
            displayName: Install DocFX
            inputs:
              targetType: inline
              script: |
                choco install docfx --version=2.59.2 -y
                if ($lastexitcode -ne 0){
                    throw ("Error installing DocFX")
                }
          - task: PowerShell@2
            displayName: Generate metadata
            inputs:
              targetType: inline
              script: |
                docfx metadata "$(Build.SourcesDirectory)/build/csharp-docs/docfx.json"
                if ($lastexitcode -ne 0){
                    throw ("Error generating metadata.")
                }
          - task: PowerShell@2
            displayName: Generate documentation
            inputs:
              targetType: inline
              script: |
                docfx build "$(Build.SourcesDirectory)/build/csharp-docs/docfx.json"
                if ($lastexitcode -ne 0){
                      throw ("Error generating documentation.")
                }
          - task: ArchiveFiles@2
            displayName: Archive C# Docs
            inputs:
              rootFolderOrFile: $(Build.SourcesDirectory)/build/csharp-docs/_site
              includeRootFolder: false
              archiveFile: $(Build.ArtifactStagingDirectory)/csharp-docs.zip
          - task: PublishPipelineArtifact@1
            displayName: Publish C# Docs
            inputs:
              targetPath: $(Build.ArtifactStagingDirectory)/csharp-docs.zip
              artifact: csharp-docs

      # js API Reference
      - job:
        displayName: Build js API Reference
        pool:
          vmImage: 'ubuntu-latest'
        steps:
          - task: NodeTool@0
            displayName: Use Node.js 10.15.0
            inputs:
              versionSpec: 10.15.0 # Won't work with higher versions
          - script: |
              npm ci --no-fund --no-audit --prefer-offline
              npx gulp docs

              major="$(umbracoMajorVersion)"
              echo "major version: $major"

              baseUrl="https://apidocs.umbraco.com/v$major/ui/"
              echo "baseUrl: $baseUrl"

              sed -i "s|baseUrl = .*|baseUrl = '$baseUrl',|" api/index.html
            displayName: Generate js Docs
            workingDirectory:  $(Build.SourcesDirectory)/src/Umbraco.Web.UI.Docs
          - task: ArchiveFiles@2
            displayName: Archive js Docs
            inputs:
              rootFolderOrFile: $(Build.SourcesDirectory)/src/Umbraco.Web.UI.Docs/api
              includeRootFolder: false
              archiveFile: $(Build.ArtifactStagingDirectory)/ui-docs.zip
          - task: PublishPipelineArtifact@1
            displayName: Publish js Docs
            inputs:
              targetPath: $(Build.ArtifactStagingDirectory)/ui-docs.zip
              artifact: ui-docs

  ###############################################
  ## Test
  ###############################################
  - stage: Unit
    displayName: Unit Tests
    dependsOn: Build
    jobs:
      # Unit Tests
      - job:
        displayName: Unit Tests
        strategy:
          matrix:
            Windows:
              vmImage: 'windows-latest'
            Linux:
              vmImage: 'ubuntu-latest'
            macOS:
              vmImage: 'macOS-latest'
        pool:
          vmImage: $(vmImage)
        steps:
          - task: DownloadPipelineArtifact@2
            displayName: Download build artifacts
            inputs:
              artifact: build_output
              path: $(Build.SourcesDirectory)
          - task: UseDotNet@2
            displayName: Use .NET $(dotnetVersion)
            inputs:
              version: $(dotnetVersion)
              includePreviewVersions: $(dotnetIncludePreviewVersions)
          - task: DotNetCoreCLI@2
            displayName: Run dotnet test
            inputs:
              command: test
              projects: '**/*.Tests.UnitTests.csproj'
              arguments: '--configuration $(buildConfiguration) --no-build'
              testRunTitle: Unit Tests - $(Agent.OS)

  - stage: Integration
    displayName: Integration Tests
    dependsOn: Build
    jobs:
      # Integration Tests (SQLite)
      - job:
        displayName: Integration Tests (SQLite)
        strategy:
          matrix:
            Windows:
              vmImage: 'windows-latest'
            Linux:
              vmImage: 'ubuntu-latest'
            macOS:
              vmImage: 'macOS-latest'
        pool:
          vmImage: $(vmImage)
        steps:
          - task: DownloadPipelineArtifact@2
            displayName: Download build artifacts
            inputs:
              artifact: build_output
              path: $(Build.SourcesDirectory)
          - task: UseDotNet@2
            displayName: Use .NET $(dotnetVersion)
            inputs:
              version: $(dotnetVersion)
              includePreviewVersions: $(dotnetIncludePreviewVersions)
          - task: DotNetCoreCLI@2
            displayName: Run dotnet test
            inputs:
              command: test
              projects: '**/*.Tests.Integration.csproj'
              arguments: '--configuration $(buildConfiguration) --no-build'
              testRunTitle: Integration Tests SQLite - $(Agent.OS)
            env:
              Tests__Database__DatabaseType: 'Sqlite'
              Umbraco__CMS__Global__MainDomLock: 'FileSystemMainDomLock'

      # Integration Tests (SQL Server)
      - job:
        timeoutInMinutes: 120
        condition: or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), ${{parameters.sqlServerIntegrationTests}})
        displayName: Integration Tests (SQL Server)
        strategy:
          matrix:
            Windows:
              vmImage: 'windows-latest'
              testDb: LocalDb
              connectionString: N/A
            Linux:
              vmImage: 'ubuntu-latest'
              testDb: SqlServer
              connectionString: 'Server=localhost,1433;User Id=sa;Password=$(SA_PASSWORD);'
        pool:
          vmImage: $(vmImage)
        variables:
          SA_PASSWORD: UmbracoIntegration123!
        steps:
          - task: DownloadPipelineArtifact@2
            displayName: Download build artifacts
            inputs:
              artifact: build_output
              path: $(Build.SourcesDirectory)
          - task: UseDotNet@2
            displayName: Use .NET $(dotnetVersion)
            inputs:
              version: $(dotnetVersion)
              includePreviewVersions: $(dotnetIncludePreviewVersions)
          - powershell: sqllocaldb start mssqllocaldb
            displayName: Start localdb (Windows only)
            condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
          - powershell: docker run --name mssql -d -p 1433:1433 -e ACCEPT_EULA=Y -e SA_PASSWORD=$(SA_PASSWORD) -e MSSQL_PID=Developer mcr.microsoft.com/mssql/server:2019-latest
            displayName: Start SQL Server (Linux only)
            condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
          - task: DotNetCoreCLI@2
            displayName: Run dotnet test
            inputs:
              command: test
              projects: '**/*.Tests.Integration.csproj'
              arguments: '--configuration $(buildConfiguration) --no-build'
              testRunTitle: Integration Tests SQL Server - $(Agent.OS)
            env:
              Tests__Database__DatabaseType: $(testDb)
              Tests__Database__SQLServerMasterConnectionString: $(connectionString)
              Umbraco__CMS__Global__MainDomLock: 'SqlMainDomLock'

  - stage: E2E
    variables:
      npm_config_cache: $(Pipeline.Workspace)/.npm_e2e
    displayName: E2E Tests
    dependsOn: Build
    jobs:
      # E2E Tests
      - job:
        displayName: E2E Tests
        variables:
          Umbraco__CMS__Unattended__UnattendedUserName: Playwright Test
          Umbraco__CMS__Unattended__UnattendedUserPassword: UmbracoAcceptance123!
          Umbraco__CMS__Unattended__UnattendedUserEmail: playwright@umbraco.com
          ASPNETCORE_URLS: https://localhost:8443
        strategy:
          matrix:
            Linux:
              vmImage: 'ubuntu-latest'
              dockerfile: umbraco-linux.docker
              dockerImageName: umbraco-linux
            Windows:
              vmImage: 'windows-latest'
              DOTNET_GENERATE_ASPNET_CERTIFICATE: true # Automatically generate HTTPS development certificate on Windows
              # Enable console logging in Release mode
              Serilog__WriteTo__0__Name: Async
              Serilog__WriteTo__0__Args__configure__0__Name: Console
              # Set unattended install settings
              Umbraco__CMS__Unattended__InstallUnattended: true
              Umbraco__CMS__Global__InstallMissingDatabase: true
              UmbracoDatabaseServer: (LocalDB)\MSSQLLocalDB
              UmbracoDatabaseName: Playwright
              ConnectionStrings__umbracoDbDSN: Server=$(UmbracoDatabaseServer);Database=$(UmbracoDatabaseName);Integrated Security=true;
              # Custom Umbraco settings
              Umbraco__CMS__Global__VersionCheckPeriod: 0
              Umbraco__CMS__Global__UseHttps: true
              Umbraco__CMS__HealthChecks__Notification__Enabled: false
              Umbraco__CMS__KeepAlive__DisableKeepAliveTask: true
        pool:
          vmImage: $(vmImage)
        steps:
          - task: DownloadPipelineArtifact@2
            displayName: Download nupkg
            inputs:
              artifact: nupkg
              path: $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest/misc/nupkg
          - task: NodeTool@0
            displayName: Use Node.js $(nodeVersion)
            inputs:
              versionSpec: $(nodeVersion)
          - task: Cache@2
            displayName: Cache node_modules
            inputs:
              key: '"npm_e2e" | "$(Agent.OS)" | $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest/package-lock.json'
              restoreKeys: |
                "npm_e2e" | "$(Agent.OS)"
                "npm_e2e"
              path: $(npm_config_cache)
          - pwsh: |
              New-Item -Path "." -Name ".env" -ItemType "file" -Value "UMBRACO_USER_LOGIN=$(Umbraco__CMS__Unattended__UnattendedUserEmail)
              UMBRACO_USER_PASSWORD=$(Umbraco__CMS__Unattended__UnattendedUserPassword)
              URL=$(ASPNETCORE_URLS)"
            displayName: Generate .env
            workingDirectory: $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest/
          - script: npm ci --no-fund --no-audit --prefer-offline
            workingDirectory: $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest/
            displayName: Run npm ci
          - pwsh: sqllocaldb start mssqllocaldb
            displayName: Start localdb (Windows only)
            condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
          - pwsh: Invoke-Sqlcmd -Query "CREATE DATABASE $env:UmbracoDatabaseName" -ServerInstance $env:UmbracoDatabaseServer
            displayName: Create database (Windows only)
            condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
          - task: UseDotNet@2
            displayName: Use .NET $(dotnetVersion)
            inputs:
              version: $(dotnetVersion)
              includePreviewVersions: $(dotnetIncludePreviewVersions)
          - pwsh: |
              $sha = 'g$(Build.SourceVersion)'.substring(0, 8)
              docker build -t $(dockerImageName):$sha -f $(dockerfile) .
              mkdir -p $(Build.ArtifactStagingDirectory)/docker-images
              docker save -o $(Build.ArtifactStagingDirectory)/docker-images/$(dockerImageName).$sha.tar $(dockerImageName):$sha
              dotnet dev-certs https -ep ${HOME}/.aspnet/https/aspnetapp.pfx -p $(Umbraco__CMS__Unattended__UnattendedUserPassword)
              docker run --name $(dockerImageName) -dp 8080:5000 -dp 8443:5001 -e UMBRACO__CMS__GLOBAL__ID=$(UMBRACO__CMS__GLOBAL__ID) -e ASPNETCORE_Kestrel__Certificates__Default__Password="$(Umbraco__CMS__Unattended__UnattendedUserPassword)" -e ASPNETCORE_Kestrel__Certificates__Default__Path=/https/aspnetapp.pfx -v ${HOME}/.aspnet/https:/https/ $(dockerImageName):$sha
              docker ps
            condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
            displayName: Build and run container (Linux only)
            workingDirectory: tests/Umbraco.Tests.AcceptanceTest/misc
          - pwsh: |
              dotnet new --install ./nupkg/Umbraco.Templates.*.nupkg
              dotnet new umbraco --name Playwright --no-restore --output .
              dotnet restore --configfile ./nuget.config
              dotnet build --configuration $(buildConfiguration) --no-restore
              dotnet dev-certs https
              $process = Start-Process -FilePath "dotnet" -ArgumentList "run --configuration $(buildConfiguration) --no-build --no-launch-profile 2>&1" -PassThru -RedirectStandardOutput $(Build.ArtifactStagingDirectory)/playwright.log
              Write-Host "##vso[task.setvariable variable=AcceptanceTestProcessId]$($process.Id)"
            condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
            displayName: Build and run app (Windows only)
            workingDirectory: tests/Umbraco.Tests.AcceptanceTest/misc
          - pwsh: npx wait-on -v --interval 1000 --timeout 120000 $(ASPNETCORE_URLS)
            displayName: Wait for app
            workingDirectory: tests/Umbraco.Tests.AcceptanceTest
          - pwsh: npx playwright install --with-deps
            displayName: Install Playwright
            workingDirectory: tests/Umbraco.Tests.AcceptanceTest
          - pwsh: npm run test --ignore-certificate-errors
            displayName: Run Playwright (Desktop)
            continueOnError: true
            workingDirectory: tests/Umbraco.Tests.AcceptanceTest
            env:
              CI: true
          - pwsh: |
              docker logs $(dockerImageName) > $(Build.ArtifactStagingDirectory)/playwright.log 2>&1
              docker stop $(dockerImageName)
            condition: eq(variables['Agent.OS'], 'Linux')
            displayName: Stop app (Linux only)
          - pwsh: Stop-Process $env:AcceptanceTestProcessId
            condition: eq(variables['Agent.OS'], 'Windows_NT')
            displayName: Stop app (Windows only)
          - task: PowerShell@2
            displayName: Check if artifacts folder exists
            inputs:
              targetType: inline
              script: |
                $MyVariable = Test-Path -Path $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest/results
                Write-Host "##vso[task.setvariable variable=resultFolderExists;]$MyVariable"
          - task: CopyFiles@2
            displayName: Prepare artifacts
            condition: eq(variables.resultFolderExists, 'True')
            inputs:
              sourceFolder: $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest/results/
              targetFolder: $(Build.ArtifactStagingDirectory)/playwright
          - task: PublishPipelineArtifact@1
            condition: always()
            displayName: Publish test artifacts
            inputs:
              targetPath: $(Build.ArtifactStagingDirectory)
              artifact: 'E2E artifacts - $(Agent.OS) - Attempt #$(System.JobAttempt)'


  ###############################################
  ## Release
  ###############################################
  - stage: Deploy_MyGet
    displayName: MyGet pre-release
    dependsOn:
      - Unit
      - Integration
      # - E2E # TODO: Enable when stable.
    condition: and(succeeded(), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), ${{parameters.myGetDeploy}}))
    jobs:
      - job:
        displayName: Push to pre-release feed
        steps:
          - checkout: none
          - task: DownloadPipelineArtifact@2
            displayName: Download nupkg
            inputs:
              artifact: nupkg
              path: $(Build.ArtifactStagingDirectory)/nupkg
          - task: NuGetCommand@2
            displayName: NuGet push
            inputs:
              command: restore
              projects: $(solution)
          # TODO: Use NuGetCommand instead of DotNetCoreCLI
          #                - task: NuGetCommand@2
          #                  displayName: Restore NuGet Packages
          #                  inputs:
          #                      restoreSolution: $(solution)
          #                      feedsToUse: config
  - stage: Deploy_NuGet
    displayName: NuGet release
    dependsOn:
      - Deploy_MyGet
      - Build_Docs
    condition: and(succeeded(), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), ${{parameters.nuGetDeploy}}))
    jobs:
      - job:
        displayName: Push to NuGet
        steps:
          - checkout: none
          - task: DownloadPipelineArtifact@2
            displayName: Download nupkg
            inputs:
              artifact: nupkg
              path: $(Build.ArtifactStagingDirectory)/nupkg
          - task: NuGetCommand@2
            displayName: NuGet push
            inputs:
              command: restore
              projects: $(solution)

  - stage: Upload_API_Docs
    pool:
      vmImage: 'windows-latest' # Apparently AzureFileCopy is windows only :(
    variables:
      umbracoMajorVersion:  $[ stageDependencies.Build.A.outputs['determineMajorVersion.majorVersion'] ]
    displayName: Upload API Documention
    dependsOn:
      - Build
      - Deploy_NuGet
    condition: and(succeeded(), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), ${{parameters.uploadApiDocs}}))
    jobs:
      - job:
        displayName: Upload C# Docs
        steps:
          - checkout: none
          - task: DownloadPipelineArtifact@2
            displayName: Download artifact
            inputs:
              artifact: csharp-docs
              path: $(Build.SourcesDirectory)
          - task: ExtractFiles@1
            inputs:
              archiveFilePatterns: $(Build.SourcesDirectory)/csharp-docs.zip
              destinationFolder: $(Build.ArtifactStagingDirectory)/csharp-docs
          - task: AzureFileCopy@4
            displayName: 'Copy C# Docs to blob storage'
            inputs:
              SourcePath: '$(Build.ArtifactStagingDirectory)/csharp-docs/*'
              azureSubscription: umbraco-storage
              Destination: AzureBlob
              storage: umbracoapidocs
              ContainerName: '$web'
              BlobPrefix: v$(umbracoMajorVersion)/csharp
      - job:
        displayName: Upload js Docs
        steps:
          - checkout: none
          - task: DownloadPipelineArtifact@2
            displayName: Download artifact
            inputs:
              artifact: ui-docs
              path: $(Build.SourcesDirectory)
          - task: ExtractFiles@1
            inputs:
              archiveFilePatterns: $(Build.SourcesDirectory)/ui-docs.zip
              destinationFolder: $(Build.ArtifactStagingDirectory)/ui-docs
          - task: AzureFileCopy@4
            displayName: 'Copy UI Docs to blob storage'
            inputs:
              SourcePath: '$(Build.ArtifactStagingDirectory)/ui-docs/*'
              azureSubscription: umbraco-storage
              Destination: AzureBlob
              storage: umbracoapidocs
              ContainerName: '$web'
              BlobPrefix: v$(umbracoMajorVersion)/ui