From 25e17bbb5cc8566702767a8d0fbf7377a334e0b4 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Tue, 20 Jul 2021 16:27:35 -0700 Subject: [PATCH 1/5] Setup publish stage in release pipeline This splits the release into three stages: build, sign, and publish. The last stage requires a manual approval on its deployment job's associated environment. When approved, it creates a draft GitHub release using our ReleaseTools module and securely uploads the artifacts directly from the build agent, thus taking developer machines out of the release process. --- .vsts-ci/azure-pipelines-release.yml | 59 ++++++++++++++++++-------- .vsts-ci/templates/publish-general.yml | 12 ++++++ .vsts-ci/templates/release-general.yml | 46 ++++++++++++-------- 3 files changed, 82 insertions(+), 35 deletions(-) create mode 100644 .vsts-ci/templates/publish-general.yml diff --git a/.vsts-ci/azure-pipelines-release.yml b/.vsts-ci/azure-pipelines-release.yml index 4b861a570..aecf27d07 100644 --- a/.vsts-ci/azure-pipelines-release.yml +++ b/.vsts-ci/azure-pipelines-release.yml @@ -1,3 +1,5 @@ +name: Release-$(Build.SourceBranchName)-$(Date:yyyyMMdd)$(Rev:.rr) + variables: # Don't download unneeded packages - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE @@ -18,24 +20,45 @@ resources: repositories: - repository: ComplianceRepo type: github - endpoint: ComplianceGHRepo + endpoint: GitHub name: PowerShell/compliance + - repository: vscode-powershell + type: git + name: vscode-powershell + +stages: +- stage: Build + displayName: Build the release + jobs: + - job: Build + pool: + vmImage: vs2017-win2016 + steps: + - template: templates/ci-general.yml -jobs: -- job: 'ReleaseBuild' - displayName: 'Build release' - pool: - vmImage: 'vs2017-win2016' - steps: - - template: templates/ci-general.yml +- stage: Sign + displayName: Sign the release + jobs: + - job: Sign + pool: + name: 1ES + demands: ImageOverride -equals MMS2019 + variables: + - group: ESRP + steps: + - template: templates/release-general.yml -- job: 'SignBuild' - displayName: Signing Build - dependsOn: 'ReleaseBuild' - pool: - name: '1ES' - demands: ImageOverride -equals MMS2019 - variables: - - group: ESRP - steps: - - template: templates/release-general.yml +- stage: Publish + displayName: Publish the draft release + jobs: + - deployment: Publish + environment: PowerShellEditorServices + pool: + name: 1ES + variables: + - group: Publish + strategy: + runOnce: + deploy: + steps: + - template: templates/publish-general.yml diff --git a/.vsts-ci/templates/publish-general.yml b/.vsts-ci/templates/publish-general.yml new file mode 100644 index 000000000..4f99a53f0 --- /dev/null +++ b/.vsts-ci/templates/publish-general.yml @@ -0,0 +1,12 @@ +steps: +- checkout: self +- checkout: vscode-powershell +- pwsh: | + Set-PSRepository -Name PSGallery -InstallationPolicy Trusted | Out-Null + Install-Module -Name PowerShellForGitHub -Scope CurrentUser -Force -Confirm:$false + Import-Module '$(Build.SourcesDirectory)/vscode-powershell/tools/ReleaseTools.psm1' + Set-GitHubConfiguration -SuppressTelemetryReminder + $password = ConvertTo-SecureString -String '$(GitHubToken)' -AsPlainText -Force + Set-GitHubAuthentication -Credential (New-Object System.Management.Automation.PSCredential ("token", $password)) + New-DraftRelease -RepositoryName PowerShellEditorServices -Assets '$(Pipeline.Workspace)/PowerShellEditorServices/PowerShellEditorServices.zip' + displayName: Drafting a GitHub Release diff --git a/.vsts-ci/templates/release-general.yml b/.vsts-ci/templates/release-general.yml index 9a9ec2029..cb7113769 100644 --- a/.vsts-ci/templates/release-general.yml +++ b/.vsts-ci/templates/release-general.yml @@ -1,27 +1,29 @@ steps: +# TODO: Replace build artifacts with pipeline artifacts - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' + displayName: Download build artifacts inputs: downloadType: specific - task: ExtractFiles@1 - displayName: 'Extract Build Zip' + displayName: Unzip build artifacts inputs: - archiveFilePatterns: '$(Build.ArtifactStagingDirectory)/PowerShellEditorServices-CI/PowerShellEditorServices*.zip' - destinationFolder: '$(Build.ArtifactStagingDirectory)/PowerShellEditorServices' + archiveFilePatterns: $(Build.ArtifactStagingDirectory)/PowerShellEditorServices-CI/PowerShellEditorServices*.zip + destinationFolder: $(Build.ArtifactStagingDirectory)/Unsigned - checkout: ComplianceRepo - displayName: 'Checkout the ComplianceRepo' +# NOTE: The signing templates explicitly copy everything along as they run, so +# the last output path has every signed (and intentionally unsigned) file. - template: EsrpSign.yml@ComplianceRepo parameters: - buildOutputPath: '$(Build.ArtifactStagingDirectory)/PowerShellEditorServices' - signOutputPath: '$(Build.ArtifactStagingDirectory)/FirstPartySigned' - alwaysCopy: true # So publishing works - certificateId: 'CP-230012' # Authenticode certificate - useMinimatch: true # This enables the use of globbing + buildOutputPath: $(Build.ArtifactStagingDirectory)/Unsigned + signOutputPath: $(Build.ArtifactStagingDirectory)/FirstPartySigned + alwaysCopy: true + certificateId: CP-230012 # Authenticode certificate shouldSign: true # We always want to sign + useMinimatch: true # This enables the use of globbing pattern: | # PowerShellEditorServices Script PowerShellEditorServices/*.{ps1,psd1,psm1,ps1xml} @@ -35,12 +37,12 @@ steps: - template: EsrpSign.yml@ComplianceRepo parameters: - buildOutputPath: '$(Build.ArtifactStagingDirectory)/FirstPartySigned' - signOutputPath: '$(Build.ArtifactStagingDirectory)/ThirdPartySigned' - alwaysCopy: true # So publishing works - certificateId: 'CP-231522' # Third-party certificate - useMinimatch: true # This enables the use of globbing + buildOutputPath: $(Build.ArtifactStagingDirectory)/FirstPartySigned + signOutputPath: $(Build.ArtifactStagingDirectory)/ThirdPartySigned + alwaysCopy: true + certificateId: CP-231522 # Third-party certificate shouldSign: true # We always want to sign + useMinimatch: true # This enables the use of globbing pattern: | **/MediatR.dll **/Nerdbank.Streams.dll @@ -49,9 +51,19 @@ steps: **/Serilog*.dll **/UnixConsoleEcho.dll -- publish: $(Build.ArtifactStagingDirectory)/ThirdPartySigned +- task: ArchiveFiles@2 + displayName: Zip finished assets + inputs: + rootFolderOrFile: $(Build.ArtifactStagingDirectory)/ThirdPartySigned + includeRootFolder: false + archiveType: zip + archiveFile: $(Build.ArtifactStagingDirectory)/PowerShellEditorServices.zip + replaceExistingArchive: true + verbose: true + +- publish: $(Build.ArtifactStagingDirectory)/PowerShellEditorServices.zip artifact: PowerShellEditorServices - displayName: 'Publish signed (and unsigned) artifacts' + displayName: Publish signed pipeline artifacts - checkout: self From 4792107c06c9fe545b0dc5fd9a0cc598d998afab Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Wed, 21 Jul 2021 14:08:00 -0700 Subject: [PATCH 2/5] Move `azurePipelinesBuild.ps1` to `tools` folder So we have fewer folders. --- .github/workflows/codeql-analysis.yml | 2 +- .vsts-ci/templates/ci-general.yml | 2 +- {scripts => tools}/azurePipelinesBuild.ps1 | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename {scripts => tools}/azurePipelinesBuild.ps1 (100%) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 31d0cd118..cf4227d7a 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -37,7 +37,7 @@ jobs: - name: Build shell: pwsh - run: scripts/azurePipelinesBuild.ps1 + run: tools/azurePipelinesBuild.ps1 - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v1 diff --git a/.vsts-ci/templates/ci-general.yml b/.vsts-ci/templates/ci-general.yml index 34b81b7c1..3348ca793 100644 --- a/.vsts-ci/templates/ci-general.yml +++ b/.vsts-ci/templates/ci-general.yml @@ -24,7 +24,7 @@ steps: condition: ne(variables['Build.Reason'], 'PullRequest') - task: PowerShell@2 inputs: - filePath: scripts/azurePipelinesBuild.ps1 + filePath: tools/azurePipelinesBuild.ps1 pwsh: ${{ parameters.pwsh }} - task: PublishTestResults@2 inputs: diff --git a/scripts/azurePipelinesBuild.ps1 b/tools/azurePipelinesBuild.ps1 similarity index 100% rename from scripts/azurePipelinesBuild.ps1 rename to tools/azurePipelinesBuild.ps1 From 4da48af97c48a26b0c737d1008a37ee2f0f32a2d Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Wed, 21 Jul 2021 14:44:01 -0700 Subject: [PATCH 3/5] Use pipeline artifacts instead of build artifacts Also move the creation of said artifacts into Azure DevOps templates themselves, instead of in the build system. Finally, download only the signed pipeline artifacts in the deployment, which by default downloads all artifacts. --- .vsts-ci/templates/ci-general.yml | 13 +++++---- .vsts-ci/templates/publish-general.yml | 3 ++ .vsts-ci/templates/release-general.yml | 39 ++++++++++---------------- PowerShellEditorServices.build.ps1 | 39 +------------------------- 4 files changed, 27 insertions(+), 67 deletions(-) diff --git a/.vsts-ci/templates/ci-general.yml b/.vsts-ci/templates/ci-general.yml index 3348ca793..28d9a5d19 100644 --- a/.vsts-ci/templates/ci-general.yml +++ b/.vsts-ci/templates/ci-general.yml @@ -22,22 +22,25 @@ steps: - pwsh: Write-Host "##vso[build.updatebuildnumber]$env:BUILD_SOURCEBRANCHNAME-$env:BUILD_SOURCEVERSION-$((get-date).ToString("yyyyMMddhhmmss"))" displayName: Set Build Name for Non-PR condition: ne(variables['Build.Reason'], 'PullRequest') + - task: PowerShell@2 inputs: filePath: tools/azurePipelinesBuild.ps1 pwsh: ${{ parameters.pwsh }} + + - publish: module + artifact: PowerShellEditorServices-Build-$(System.JobId) + displayName: Publish unsigned pipeline artifacts + condition: succeededOrFailed() + - task: PublishTestResults@2 inputs: testRunner: VSTest testResultsFiles: '**/*.trx' condition: succeededOrFailed() + - task: PublishTestResults@2 inputs: testRunner: NUnit testResultsFiles: '**/TestResults.xml' condition: succeededOrFailed() - - task: PublishBuildArtifacts@1 - inputs: - ArtifactName: PowerShellEditorServices-CI - PathtoPublish: '$(Build.ArtifactStagingDirectory)' - condition: succeededOrFailed() diff --git a/.vsts-ci/templates/publish-general.yml b/.vsts-ci/templates/publish-general.yml index 4f99a53f0..56177e389 100644 --- a/.vsts-ci/templates/publish-general.yml +++ b/.vsts-ci/templates/publish-general.yml @@ -1,6 +1,9 @@ steps: - checkout: self - checkout: vscode-powershell +- download: current + artifact: PowerShellEditorServices + displayName: Download signed pipeline artifacts - pwsh: | Set-PSRepository -Name PSGallery -InstallationPolicy Trusted | Out-Null Install-Module -Name PowerShellForGitHub -Scope CurrentUser -Force -Confirm:$false diff --git a/.vsts-ci/templates/release-general.yml b/.vsts-ci/templates/release-general.yml index cb7113769..a56229448 100644 --- a/.vsts-ci/templates/release-general.yml +++ b/.vsts-ci/templates/release-general.yml @@ -1,16 +1,7 @@ steps: -# TODO: Replace build artifacts with pipeline artifacts -- task: DownloadBuildArtifacts@0 - displayName: Download build artifacts - inputs: - downloadType: specific - -- task: ExtractFiles@1 - displayName: Unzip build artifacts - inputs: - archiveFilePatterns: $(Build.ArtifactStagingDirectory)/PowerShellEditorServices-CI/PowerShellEditorServices*.zip - destinationFolder: $(Build.ArtifactStagingDirectory)/Unsigned +- download: current + displayName: Download unsigned pipeline artifacts - checkout: ComplianceRepo @@ -18,8 +9,8 @@ steps: # the last output path has every signed (and intentionally unsigned) file. - template: EsrpSign.yml@ComplianceRepo parameters: - buildOutputPath: $(Build.ArtifactStagingDirectory)/Unsigned - signOutputPath: $(Build.ArtifactStagingDirectory)/FirstPartySigned + buildOutputPath: $(Pipeline.Workspace)/PowerShellEditorServices-Build-* + signOutputPath: $(Pipeline.Workspace)/FirstPartySigned alwaysCopy: true certificateId: CP-230012 # Authenticode certificate shouldSign: true # We always want to sign @@ -37,8 +28,8 @@ steps: - template: EsrpSign.yml@ComplianceRepo parameters: - buildOutputPath: $(Build.ArtifactStagingDirectory)/FirstPartySigned - signOutputPath: $(Build.ArtifactStagingDirectory)/ThirdPartySigned + buildOutputPath: $(Pipeline.Workspace)/FirstPartySigned + signOutputPath: $(Pipeline.Workspace)/ThirdPartySigned alwaysCopy: true certificateId: CP-231522 # Third-party certificate shouldSign: true # We always want to sign @@ -54,14 +45,14 @@ steps: - task: ArchiveFiles@2 displayName: Zip finished assets inputs: - rootFolderOrFile: $(Build.ArtifactStagingDirectory)/ThirdPartySigned + rootFolderOrFile: $(Pipeline.Workspace)/ThirdPartySigned includeRootFolder: false archiveType: zip - archiveFile: $(Build.ArtifactStagingDirectory)/PowerShellEditorServices.zip + archiveFile: PowerShellEditorServices.zip replaceExistingArchive: true verbose: true -- publish: $(Build.ArtifactStagingDirectory)/PowerShellEditorServices.zip +- publish: PowerShellEditorServices.zip artifact: PowerShellEditorServices displayName: Publish signed pipeline artifacts @@ -70,18 +61,18 @@ steps: - template: assembly-module-compliance.yml@ComplianceRepo parameters: # binskim - AnalyzeTarget: '$(Build.ArtifactStagingDirectory)/*.dll' + AnalyzeTarget: $(Pipeline.Workspace)/*.dll AnalyzeSymPath: 'SRV*' # component-governance - sourceScanPath: '$(Build.SourcesDirectory)/PowerShellEditorServices' + sourceScanPath: $(Build.SourcesDirectory)/PowerShellEditorServices # credscan suppressionsFile: '' # TermCheck AKA PoliCheck - targetArgument: '$(Build.SourcesDirectory)/PowerShellEditorServices' - optionsUEPATH: '$(Build.SourcesDirectory)/PowerShellEditorServices/tools/terms/UserExclusions.xml' + targetArgument: $(Build.SourcesDirectory)/PowerShellEditorServices + optionsUEPATH: $(Build.SourcesDirectory)/PowerShellEditorServices/tools/terms/UserExclusions.xml optionsRulesDBPath: '' - optionsFTPath: '$(Build.SourcesDirectory)/PowerShellEditorServices/tools/terms/FileTypeSet.xml' + optionsFTPath: $(Build.SourcesDirectory)/PowerShellEditorServices/tools/terms/FileTypeSet.xml # tsa-upload - codeBaseName: 'PowerShell_PowerShellEditorServices_20210201' + codeBaseName: PowerShell_PowerShellEditorServices_20210201 # selections APIScan: false diff --git a/PowerShellEditorServices.build.ps1 b/PowerShellEditorServices.build.ps1 index fbd126708..60274440e 100644 --- a/PowerShellEditorServices.build.ps1 +++ b/PowerShellEditorServices.build.ps1 @@ -146,31 +146,6 @@ task Clean BinClean,{ } } -task GetProductVersion -Before PackageModule, UploadArtifacts { - [xml]$props = Get-Content .\PowerShellEditorServices.Common.props - - $script:BuildNumber = 9999 - $script:VersionSuffix = $props.Project.PropertyGroup.VersionSuffix - - if ($env:TF_BUILD) { - # SYSTEM_PHASENAME is the Job name. - # Job names can only include `_` but that's not a valid character for versions. - $jobname = $env:SYSTEM_PHASENAME -replace '_', '' - $script:BuildNumber = "$jobname-$env:BUILD_BUILDID" - } - - if ($script:VersionSuffix -ne $null) { - $script:VersionSuffix = "$script:VersionSuffix-$script:BuildNumber" - } - else { - $script:VersionSuffix = "$script:BuildNumber" - } - - $script:FullVersion = "$($props.Project.PropertyGroup.VersionPrefix)-$script:VersionSuffix" - - Write-Host "`n### Product Version: $script:FullVersion`n" -ForegroundColor Green -} - task CreateBuildInfo -Before Build { $buildVersion = "" $buildOrigin = "Development" @@ -435,17 +410,5 @@ task BuildCmdletHelp { New-ExternalHelp -Path $PSScriptRoot\module\PowerShellEditorServices.VSCode\docs -OutputPath $PSScriptRoot\module\PowerShellEditorServices.VSCode\en-US -Force } -task PackageModule { - [System.IO.Compression.ZipFile]::CreateFromDirectory( - "$PSScriptRoot/module/", - "$PSScriptRoot/PowerShellEditorServices-$($script:FullVersion).zip", - [System.IO.Compression.CompressionLevel]::Optimal, - $false) -} - -task UploadArtifacts -If ($null -ne $env:TF_BUILD) { - Copy-Item -Path .\PowerShellEditorServices-$($script:FullVersion).zip -Destination $env:BUILD_ARTIFACTSTAGINGDIRECTORY -Force -} - # The default task is to run the entire CI build -task . GetProductVersion, Clean, Build, Test, BuildCmdletHelp, PackageModule, UploadArtifacts +task . Clean, Build, Test, BuildCmdletHelp From 148f4235d78c0c2f30edfca21f66255794d0b52c Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Thu, 22 Jul 2021 10:19:45 -0700 Subject: [PATCH 4/5] Stop using PowerShell daily for CI It leads to bugs I cannot solve. --- .vsts-ci/templates/ci-general.yml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/.vsts-ci/templates/ci-general.yml b/.vsts-ci/templates/ci-general.yml index 28d9a5d19..d62ea210c 100644 --- a/.vsts-ci/templates/ci-general.yml +++ b/.vsts-ci/templates/ci-general.yml @@ -2,20 +2,6 @@ parameters: pwsh: true steps: - - powershell: | - Write-Host "Installing PowerShell Daily..." - - # Use `AGENT_TEMPDIRECTORY` to make sure the downloaded PowerShell is cleaned up. - $powerShellPath = Join-Path -Path $env:AGENT_TEMPDIRECTORY -ChildPath 'powershell' - Invoke-WebRequest -Uri https://aka.ms/install-powershell.ps1 -OutFile ./install-powershell.ps1 - - ./install-powershell.ps1 -Destination $powerShellPath -Daily - - # Using `prependpath` to update the PATH just for this build. - Write-Host "##vso[task.prependpath]$powerShellPath" - displayName: Install PowerShell Daily - continueOnError: true - - pwsh: '$PSVersionTable' displayName: Display PowerShell version information From 85eae6264941e03a796991e13d953169e0dca6fc Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Thu, 22 Jul 2021 10:21:20 -0700 Subject: [PATCH 5/5] Clean up `ci-general.yml` template - Use proper `parameters` syntax - Use cleaner indentation - Add decent display names - Remove setting build name for non-PR builds - Remove unused NUnit test results --- .vsts-ci/templates/ci-general.yml | 46 ++++++++++++++----------------- 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/.vsts-ci/templates/ci-general.yml b/.vsts-ci/templates/ci-general.yml index d62ea210c..f3e237e88 100644 --- a/.vsts-ci/templates/ci-general.yml +++ b/.vsts-ci/templates/ci-general.yml @@ -1,32 +1,26 @@ parameters: - pwsh: true +- name: pwsh + type: boolean + default: true steps: - - pwsh: '$PSVersionTable' - displayName: Display PowerShell version information +- pwsh: '$PSVersionTable' + displayName: PowerShell version - - pwsh: Write-Host "##vso[build.updatebuildnumber]$env:BUILD_SOURCEBRANCHNAME-$env:BUILD_SOURCEVERSION-$((get-date).ToString("yyyyMMddhhmmss"))" - displayName: Set Build Name for Non-PR - condition: ne(variables['Build.Reason'], 'PullRequest') +- task: PowerShell@2 + displayName: Build and test + inputs: + filePath: tools/azurePipelinesBuild.ps1 + pwsh: ${{ parameters.pwsh }} - - task: PowerShell@2 - inputs: - filePath: tools/azurePipelinesBuild.ps1 - pwsh: ${{ parameters.pwsh }} +- publish: module + artifact: PowerShellEditorServices-Build-$(System.JobId) + displayName: Publish unsigned pipeline artifacts + condition: succeededOrFailed() - - publish: module - artifact: PowerShellEditorServices-Build-$(System.JobId) - displayName: Publish unsigned pipeline artifacts - condition: succeededOrFailed() - - - task: PublishTestResults@2 - inputs: - testRunner: VSTest - testResultsFiles: '**/*.trx' - condition: succeededOrFailed() - - - task: PublishTestResults@2 - inputs: - testRunner: NUnit - testResultsFiles: '**/TestResults.xml' - condition: succeededOrFailed() +- task: PublishTestResults@2 + displayName: Publish test results + inputs: + testRunner: VSTest + testResultsFiles: '**/*.trx' + condition: succeededOrFailed()