Umbraco-CMS/build/azure-pipelines.yml

name: $(TeamProject)_$(Build.DefinitionName)_$(SourceBranchName)_$(Date:yyyyMMdd)$(Rev:.r)

parameters:
  - name: sqlServerIntegrationTests
    displayName: Run SQL Server Integration Tests
    type: boolean
    default: false
  - name: myGetDeploy
    displayName: Deploy to MyGet
    type: boolean
    default: false
  - name: nuGetDeploy
    displayName: Deploy to NuGet
    type: boolean
    default: false
  - name: buildApiDocs
    displayName: Build API docs
    type: boolean
    default: false
  - name: uploadApiDocs
    displayName: Upload API docs
    type: boolean
    default: false

variables:
  nodeVersion: 14.18.1
  dotnetVersion: 6.x
  solution: umbraco.sln
  buildConfiguration: Release
  UMBRACO__CMS__GLOBAL__ID: 00000000-0000-0000-0000-000000000042
  DOTNET_NOLOGO: true
  DOTNET_GENERATE_ASPNET_CERTIFICATE: false
  DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
  DOTNET_CLI_TELEMETRY_OPTOUT: true

stages:
  ###############################################
  ## Build
  ###############################################
  - stage: Build
    variables:
      npm_config_cache: $(Pipeline.Workspace)/.npm_client
    jobs:
      - job: A
        displayName: Build Umbraco CMS
        pool:
          vmImage: 'ubuntu-latest'
        steps:
          - task: NodeTool@0
            displayName: Use Node.js $(nodeVersion)
            inputs:
              versionSpec: $(nodeVersion)
          - task: Cache@2
            displayName: Cache node_modules
            inputs:
              key: '"npm_client" | "$(Agent.OS)" | $(Build.SourcesDirectory)/src/Umbraco.Web.UI.Client/package-lock.json'
              restoreKeys: |
                "npm_client" | "$(Agent.OS)"
                "npm_client"
              path: $(npm_config_cache)
          - script: npm ci --no-fund --no-audit --prefer-offline
            workingDirectory:  src/Umbraco.Web.UI.Client
            displayName: Run npm ci
          - task: gulp@0
            displayName: Run gulp build
            inputs:
              gulpFile: src/Umbraco.Web.UI.Client/gulpfile.js
              targets: coreBuild
              workingDirectory: src/Umbraco.Web.UI.Client
          - task: UseDotNet@2
            displayName: Use .NET $(dotnetVersion)
            inputs:
              version: $(dotnetVersion)
              performMultiLevelLookup: true
          - task: DotNetCoreCLI@2
            displayName: Run dotnet restore
            inputs:
              command: restore
              projects: $(solution)
          - task: DotNetCoreCLI@2
            displayName: Run dotnet build
            inputs:
              command: build
              projects: $(solution)
              arguments: '--configuration $(buildConfiguration) --no-restore -p:ContinuousIntegrationBuild=true'
          - script: |
              version="$(Build.BuildNumber)"
              echo "varsion: $version"

              major="$(echo $version | cut -d '.' -f 1)"
              echo "major version: $major"

              echo "##vso[task.setvariable variable=majorVersion;isOutput=true]$major"
            displayName: Set major version
            name: determineMajorVersion
          - task: PowerShell@2
            displayName: Prepare nupkg
            inputs:
              targetType: inline
              script: |
                $umbracoVersion = "$(Build.BuildNumber)" -replace "\+",".g"
                $templatePaths = Get-ChildItem 'templates/**/.template.config/template.json'

                foreach ($templatePath in $templatePaths) {
                  $a = Get-Content $templatePath -Raw | ConvertFrom-Json
                  if ($a.symbols -and $a.symbols.UmbracoVersion) {
                    $a.symbols.UmbracoVersion.defaultValue = $umbracoVersion
                    $a | ConvertTo-Json -Depth 32 | Set-Content $templatePath
                  }
                }

                dotnet pack $(solution) --configuration $(buildConfiguration) --no-build -p:BuildProjectReferences=false --output $(Build.ArtifactStagingDirectory)/nupkg
          - script: |
              sha="$(Build.SourceVersion)"
              sha=${sha:0:7}
              buildnumber="$(Build.BuildNumber)_$(Build.BuildId)_$sha"
              echo "##vso[build.updatebuildnumber]$buildnumber"
            displayName: Update build number
          - task: PublishPipelineArtifact@1
            displayName: Publish nupkg
            inputs:
              targetPath: $(Build.ArtifactStagingDirectory)/nupkg
              artifactName: nupkg
          - task: PublishPipelineArtifact@1
            displayName: Publish build artifacts
            inputs:
              targetPath: $(Build.SourcesDirectory)
              artifactName: build_output

  - stage: Build_Docs
    condition: and(succeeded(), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), ${{parameters.buildApiDocs}}))
    displayName: Prepare API Documentation
    dependsOn: Build
    variables:
      umbracoMajorVersion: $[ stageDependencies.Build.A.outputs['determineMajorVersion.majorVersion'] ]
    jobs:
      # C# API Reference
      - job:
        displayName: Build C# API Reference
        pool:
          vmImage: 'windows-latest'
        steps:
          - task: PowerShell@2
            displayName: Install DocFX
            inputs:
                targetType: inline
                script: |
                  choco install docfx --version=2.59.2 -y
                  if ($lastexitcode -ne 0){
                      throw ("Error installing DocFX")
                  }
          - task: PowerShell@2
            displayName: Generate metadata
            inputs:
                targetType: inline
                script: |
                  docfx metadata "$(Build.SourcesDirectory)/build/csharp-docs/docfx.json"
                  if ($lastexitcode -ne 0){
                      throw ("Error generating metadata.")
                  }
          - task: PowerShell@2
            displayName: Generate documentation
            inputs:
                targetType: inline
                script: |
                  docfx build "$(Build.SourcesDirectory)/build/csharp-docs/docfx.json"
                  if ($lastexitcode -ne 0){
                        throw ("Error generating documentation.")
                  }
          - task: ArchiveFiles@2
            displayName: Archive C# Docs
            inputs:
                rootFolderOrFile: $(Build.SourcesDirectory)/build/csharp-docs/_site
                includeRootFolder: false
                archiveFile: $(Build.ArtifactStagingDirectory)/csharp-docs.zip
          - task: PublishPipelineArtifact@1
            displayName: Publish C# Docs
            inputs:
                targetPath: $(Build.ArtifactStagingDirectory)/csharp-docs.zip
                artifact: csharp-docs

      # js API Reference
      - job:
        displayName: Build js API Reference
        pool:
          vmImage: 'ubuntu-latest'
        steps:
          - task: NodeTool@0
            displayName: Use Node.js 10.15.0
            inputs:
              versionSpec: 10.15.0 # Won't work with 14.18.1
          - script: |
              npm ci --no-fund --no-audit --prefer-offline
              npx gulp docs

              major="$(umbracoMajorVersion)"
              echo "major version: $major"

              baseUrl="https://apidocs.umbraco.com/v$major/ui/"
              echo "baseUrl: $baseUrl"

              sed -i "s|baseUrl = .*|baseUrl = '$baseUrl',|" api/index.html
            displayName: Generate js Docs
            workingDirectory:  $(Build.SourcesDirectory)/src/Umbraco.Web.UI.Docs
          - task: ArchiveFiles@2
            displayName: Archive js Docs
            inputs:
                rootFolderOrFile: $(Build.SourcesDirectory)/src/Umbraco.Web.UI.Docs/api
                includeRootFolder: false
                archiveFile: $(Build.ArtifactStagingDirectory)/ui-docs.zip
          - task: PublishPipelineArtifact@1
            displayName: Publish js Docs
            inputs:
                targetPath: $(Build.ArtifactStagingDirectory)/ui-docs.zip
                artifact: ui-docs

  ###############################################
  ## Test
  ###############################################
  - stage: Unit
    displayName: Unit Tests
    dependsOn: Build
    jobs:
      # Unit Tests
      - job:
        displayName: Unit Tests
        strategy:
          matrix:
            Windows:
              vmImage: 'windows-latest'
            Linux:
              vmImage: 'ubuntu-latest'
            macOS:
              vmImage: 'macOS-latest'
        pool:
          vmImage: $(vmImage)
        steps:
          - task: DownloadPipelineArtifact@2
            displayName: Download build artifacts
            inputs:
              artifact: build_output
              path: $(Build.SourcesDirectory)
          - task: UseDotNet@2
            displayName: Use .NET $(dotnetVersion)
            inputs:
              version: $(dotnetVersion)
              performMultiLevelLookup: true
          - task: DotNetCoreCLI@2
            displayName: Run dotnet test
            inputs:
              command: test
              projects: '**/*.Tests.UnitTests.csproj'
              arguments: '--configuration $(buildConfiguration) --no-build'
              testRunTitle: Unit Tests - $(Agent.OS)

  - stage: Integration
    displayName: Integration Tests
    dependsOn: Build
    jobs:
      # Integration Tests (SQLite)
      - job:
        displayName: Integration Tests (SQLite)
        strategy:
          matrix:
            Windows:
              vmImage: 'windows-latest'
            Linux:
              vmImage: 'ubuntu-latest'
            macOS:
              vmImage: 'macOS-latest'
        pool:
          vmImage: $(vmImage)
        steps:
          - task: DownloadPipelineArtifact@2
            displayName: Download build artifacts
            inputs:
              artifact: build_output
              path: $(Build.SourcesDirectory)
          - task: UseDotNet@2
            displayName: Use .NET $(dotnetVersion)
            inputs:
              version: $(dotnetVersion)
              performMultiLevelLookup: true
          - task: DotNetCoreCLI@2
            displayName: Run dotnet test
            inputs:
              command: test
              projects: '**/*.Tests.Integration.csproj'
              arguments: '--configuration $(buildConfiguration) --no-build'
              testRunTitle: Integration Tests SQLite - $(Agent.OS)
            env:
              Tests__Database__DatabaseType: 'Sqlite'
              Umbraco__CMS__Global__MainDomLock: 'FileSystemMainDomLock'

      # Integration Tests (SQL Server)
      - job:
        timeoutInMinutes: 120
        condition: or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), ${{parameters.sqlServerIntegrationTests}})
        displayName: Integration Tests (SQL Server)
        strategy:
          matrix:
            Windows:
              vmImage: 'windows-latest'
              testDb: LocalDb
              connectionString: N/A
            Linux:
              vmImage: 'ubuntu-latest'
              testDb: SqlServer
              connectionString: 'Server=localhost,1433;User Id=sa;Password=$(SA_PASSWORD);'
        pool:
          vmImage: $(vmImage)
        variables:
          SA_PASSWORD: UmbracoIntegration123!
        steps:
          - task: DownloadPipelineArtifact@2
            displayName: Download build artifacts
            inputs:
              artifact: build_output
              path: $(Build.SourcesDirectory)
          - powershell: sqllocaldb start mssqllocaldb
            displayName: Start localdb (Windows only)
            condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
          - powershell: docker run --name mssql -d -p 1433:1433 -e ACCEPT_EULA=Y -e SA_PASSWORD=$(SA_PASSWORD) -e MSSQL_PID=Developer mcr.microsoft.com/mssql/server:2019-latest 
            displayName: Start SQL Server (Linux only)
            condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
          - task: DotNetCoreCLI@2
            displayName: Run dotnet test
            inputs:
              command: test
              projects: '**/*.Tests.Integration.csproj'
              arguments: '--configuration $(buildConfiguration) --no-build'
              testRunTitle: Integration Tests SQL Server - $(Agent.OS)
            env:
              Tests__Database__DatabaseType: $(testDb)
              Tests__Database__SQLServerMasterConnectionString: $(connectionString)
              Umbraco__CMS__Global__MainDomLock: 'SqlMainDomLock'

  - stage: E2E
    variables:
      npm_config_cache: $(Pipeline.Workspace)/.npm_e2e
      CYPRESS_CACHE_FOLDER: $(Pipeline.Workspace)/cypress_binaries
    displayName: E2E Tests
    dependsOn: Build
    jobs:
      # E2E Tests
      - job:
        displayName: E2E Tests
        variables:
          - name: Umbraco__CMS__Unattended__InstallUnattended  # Windows only
            value: true
          - name: Umbraco__CMS__Unattended__UnattendedUserName  # Windows only
            value: Cypress Test
          - name: Umbraco__CMS__Unattended__UnattendedUserEmail  # Windows only
            value: cypress@umbraco.com
          - name: Umbraco__CMS__Unattended__UnattendedUserPassword  # Windows only
            value: UmbracoAcceptance123!
          - name: Umbraco__CMS__Global__InstallMissingDatabase  # Windows only
            value: true
          - name: UmbracoDatabaseServer # Windows only
            value: (LocalDB)\MSSQLLocalDB
          - name: UmbracoDatabaseName  # Windows only
            value: Cypress
          - name: ConnectionStrings__umbracoDbDSN  # Windows only
            value: Server=$(UmbracoDatabaseServer);Database=$(UmbracoDatabaseName);Integrated Security=true;
          - name: CYPRESS_BASE_URL
            value: http://localhost:8080
        strategy:
          matrix:
            Linux:
              vmImage: 'ubuntu-latest'
              dockerfile: umbraco-linux.docker
              dockerImageName: umbraco-linux
            Windows:
              vmImage: 'windows-latest'
        pool:
          vmImage: $(vmImage)
        steps:
          - task: DownloadPipelineArtifact@2
            displayName: Download nupkg
            inputs:
              artifact: nupkg
              path: $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest/misc/nupkg
          - task: NodeTool@0
            displayName: Use Node.js $(nodeVersion)
            inputs:
              versionSpec: $(nodeVersion)
          - task: Cache@2
            displayName: Cache node_modules
            inputs:
              key: '"npm_e2e" | "$(Agent.OS)" | $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest/package-lock.json'
              restoreKeys: |
                "npm_e2e" | "$(Agent.OS)"
                "npm_e2e"
              path: $(npm_config_cache)
          - task: Cache@2
            displayName: Cache cypress binaries
            inputs:
              key: '"cypress_binaries" | "$(Agent.OS)" | $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest/package-lock.json'
              path: $(CYPRESS_CACHE_FOLDER)
          - task: PowerShell@2
            displayName: Generate Cypress.env.json
            inputs:
              targetType: inline
              script: >
                @{ username = "$(Umbraco__CMS__Unattended__UnattendedUserEmail)"; password = "$(Umbraco__CMS__Unattended__UnattendedUserPassword)" } | ConvertTo-Json | Set-Content -Path "tests/Umbraco.Tests.AcceptanceTest/cypress.env.json"#
          - script: npm ci --no-fund --no-audit --prefer-offline
            workingDirectory:  $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest/
            displayName: Run npm ci
          - powershell: sqllocaldb start mssqllocaldb
            displayName: Start localdb (Windows only)
            condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
          - powershell:  Invoke-Sqlcmd -Query "CREATE DATABASE $env:UmbracoDatabaseName" -ServerInstance $env:UmbracoDatabaseServer
            displayName: Create database (Windows only)
            condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
          - task: UseDotNet@2
            displayName: Use .NET $(dotnetVersion)
            inputs:
              version: $(dotnetVersion)
              performMultiLevelLookup: true
          # Linux containers smooth
          - task: PowerShell@2
            condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
            displayName: Build & run container (Linux only)
            inputs:
              workingDirectory: tests/Umbraco.Tests.AcceptanceTest/misc
              targetType: inline
              script: |
                $sha = 'g$(Build.SourceVersion)'.substring(0, 8)
                docker build -t $(dockerImageName):$sha -f $(dockerfile) .
                mkdir -p $(Build.ArtifactStagingDirectory)/docker-images
                docker save -o $(Build.ArtifactStagingDirectory)/docker-images/$(dockerImageName).$sha.tar $(dockerImageName):$sha
                docker run --name $(dockerImageName) -dp 8080:5000 -e UMBRACO__CMS__GLOBAL__ID=$(UMBRACO__CMS__GLOBAL__ID) $(dockerImageName):$sha
                docker ps
          # Windows containers take forever.
          # --no-launch-profile stops ASPNETCORE_ENVIRONMENT=Development which breaks the users.ts tests (smtp config = invite user button)
          # Urls matching docker setup.
          - task: PowerShell@2
            condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
            displayName: Build & run app (Windows only)
            inputs:
              workingDirectory: tests/Umbraco.Tests.AcceptanceTest/misc
              targetType: inline
              script: |
                dotnet new --install ./nupkg/Umbraco.Templates.*.nupkg
                dotnet new umbraco --name Cypress -o . --no-restore
                dotnet restore --configfile ./nuget.config
                dotnet build --configuration $(buildConfiguration) --no-restore
                Start-Process -FilePath "dotnet" -ArgumentList "run --configuration $(buildConfiguration) --no-build --no-launch-profile --urls $(CYPRESS_BASE_URL)"
          - task: PowerShell@2
            displayName: Wait for app
            inputs:
              targetType: inline
              workingDirectory: tests/Umbraco.Tests.AcceptanceTest
              script: |
                npm i -g wait-on
                wait-on -v --interval 1000 --timeout 120000 $(CYPRESS_BASE_URL)
          - task: PowerShell@2
            displayName: Run Cypress (Desktop)
            continueOnError: true
            inputs:
              targetType: inline
              workingDirectory: tests/Umbraco.Tests.AcceptanceTest
              script: 'npm run test -- --reporter junit --reporter-options "mochaFile=results/test-output-D-[hash].xml,toConsole=true" --config="viewportHeight=1600,viewportWidth=2560,screenshotsFolder=cypress/artifacts/desktop/screenshots,videosFolder=cypress/artifacts/desktop/videos,videoUploadOnPasses=false"'
          - task: PublishTestResults@2
            displayName: Publish test results
            condition: always()
            inputs:
              testResultsFormat: 'JUnit'
              testResultsFiles: 'tests/Umbraco.Tests.AcceptanceTest/results/test-output-D-*.xml'
              mergeTestResults: true
              testRunTitle: "e2e - $(Agent.OS)"
          - task: CopyFiles@2
            displayName: Prepare artifacts
            condition: always()
            inputs:
              sourceFolder: $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest/cypress/artifacts
              targetFolder: $(Build.ArtifactStagingDirectory)/cypresss
          - task: PublishPipelineArtifact@1
            displayName: "Publish test artifacts"
            condition: always()
            inputs:
              targetPath: $(Build.ArtifactStagingDirectory)
              artifact: 'E2E artifacts - $(Agent.OS) - Attempt #$(System.JobAttempt)'

  ###############################################
  ## Release
  ###############################################
  - stage: Deploy_MyGet
    displayName: MyGet pre-release
    dependsOn:
      - Unit
      - Integration
      # - E2E # TODO: Enable when stable.
    condition: and(succeeded(), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), ${{parameters.myGetDeploy}}))
    jobs:
      - job:
        displayName: Push to pre-release feed
        steps:
          - checkout: none
          - task: DownloadPipelineArtifact@2
            displayName: Download nupkg
            inputs:
              artifact: nupkg
              path: $(Build.ArtifactStagingDirectory)/nupkg
          - task: NuGetCommand@2
            displayName: NuGet push
            inputs:
              command: 'push'
              packagesToPush: $(Build.ArtifactStagingDirectory)/**/*.nupkg
              nuGetFeedType: 'external'
              publishFeedCredentials: 'MyGet - Pre-releases'

  - stage: Deploy_NuGet
    displayName: NuGet release
    dependsOn:
      - Deploy_MyGet
      - Build_Docs
    condition: and(succeeded(), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), ${{parameters.nuGetDeploy}}))
    jobs:
      - job:
        displayName: Push to NuGet
        steps:
          - checkout: none
          - task: DownloadPipelineArtifact@2
            displayName: Download nupkg
            inputs:
              artifact: nupkg
              path: $(Build.ArtifactStagingDirectory)/nupkg
          - task: NuGetCommand@2
            displayName: NuGet push
            inputs:
              command: 'push'
              packagesToPush: $(Build.ArtifactStagingDirectory)/**/*.nupkg
              nuGetFeedType: 'external'
              publishFeedCredentials: 'NuGet - Umbraco.*'

  - stage: Upload_API_Docs
    pool:
      vmImage: 'windows-latest' # Apparently AzureFileCopy is windows only :(
    variables:
      umbracoMajorVersion:  $[ stageDependencies.Build.A.outputs['determineMajorVersion.majorVersion'] ]
    displayName: Upload API Documention
    dependsOn:
      - Build
      - Deploy_NuGet
    condition: and(succeeded(), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), ${{parameters.uploadApiDocs}}))
    jobs:
      - job:
        displayName: Upload C# Docs
        steps:
          - checkout: none
          - task: DownloadPipelineArtifact@2
            displayName: Download artifact
            inputs:
              artifact: csharp-docs
              path: $(Build.SourcesDirectory)
          - task: ExtractFiles@1
            inputs:
              archiveFilePatterns: $(Build.SourcesDirectory)/csharp-docs.zip
              destinationFolder: $(Build.ArtifactStagingDirectory)/csharp-docs
          - task: AzureFileCopy@4
            displayName: 'Copy C# Docs to blob storage'
            inputs:
              SourcePath: '$(Build.ArtifactStagingDirectory)/csharp-docs/*'
              azureSubscription: umbraco-storage
              Destination: AzureBlob
              storage: umbracoapidocs
              ContainerName: '$web'
              BlobPrefix: v$(umbracoMajorVersion)/csharp
      - job:
        displayName: Upload js Docs
        steps:
          - checkout: none
          - task: DownloadPipelineArtifact@2
            displayName: Download artifact
            inputs:
              artifact: ui-docs
              path: $(Build.SourcesDirectory)
          - task: ExtractFiles@1
            inputs:
              archiveFilePatterns: $(Build.SourcesDirectory)/ui-docs.zip
              destinationFolder: $(Build.ArtifactStagingDirectory)/ui-docs
          - task: AzureFileCopy@4
            displayName: 'Copy UI Docs to blob storage'
            inputs:
              SourcePath: '$(Build.ArtifactStagingDirectory)/ui-docs/*'
              azureSubscription: umbraco-storage
              Destination: AzureBlob
              storage: umbracoapidocs
              ContainerName: '$web'
              BlobPrefix: v$(umbracoMajorVersion)/ui