diff --git a/eng/common/TestResources/New-TestResources.ps1 b/eng/common/TestResources/New-TestResources.ps1 index 9e281d04eb66..90e496d2e83d 100644 --- a/eng/common/TestResources/New-TestResources.ps1 +++ b/eng/common/TestResources/New-TestResources.ps1 @@ -154,6 +154,9 @@ if ($ProvisionerApplicationId) { $subscriptionArgs = if ($SubscriptionId) { @{SubscriptionId = $SubscriptionId} } + else { + @{} + } $provisionerAccount = Retry { Connect-AzAccount -Force:$Force -Tenant $TenantId -Credential $provisionerCredential -ServicePrincipal -Environment $Environment @subscriptionArgs @@ -161,7 +164,11 @@ if ($ProvisionerApplicationId) { $exitActions += { Write-Verbose "Logging out of service principal '$($provisionerAccount.Context.Account)'" - $null = Disconnect-AzAccount -AzureContext $provisionerAccount.Context + + # Only attempt to disconnect if the -WhatIf flag was not set. Otherwise, this call is not necessary and will fail. + if ($PSCmdlet.ShouldProcess($ProvisionerApplicationId)) { + $null = Disconnect-AzAccount -AzureContext $provisionerAccount.Context + } } } @@ -176,7 +183,6 @@ if ($TestApplicationId -and !$TestApplicationOid) { } } - # If the ServiceDirectory is an absolute path use the last directory name # (e.g. D:\foo\bar\ -> bar) $serviceName = if (Split-Path -IsAbsolute $ServiceDirectory) { @@ -229,6 +235,13 @@ if ($resourceGroup.ProvisioningState -eq 'Succeeded') { # New-AzResourceGroup would've written an error and stopped the pipeline by default anyway. Write-Verbose "Successfully created resource group '$($resourceGroup.ResourceGroupName)'" } +elseif (($resourceGroup -eq $null) -and (-not $PSCmdlet.ShouldProcess($resourceGroupName))) { + # If the -WhatIf flag was passed, there will be no resource group created. Fake it. + $resourceGroup = [PSCustomObject]@{ + ResourceGroupName = $resourceGroupName + Location = $Location + } +} # Populate the template parameters and merge any additional specified. $templateParameters = @{ @@ -299,6 +312,9 @@ foreach ($templateFile in $templateFiles) { "$($serviceDirectoryPrefix)RESOURCE_GROUP" = $resourceGroup.ResourceGroupName; "$($serviceDirectoryPrefix)LOCATION" = $resourceGroup.Location; "$($serviceDirectoryPrefix)ENVIRONMENT" = $context.Environment.Name; + "$($serviceDirectoryPrefix)AZURE_AUTHORITY_HOST" = $context.Environment.ActiveDirectoryAuthority; + "$($serviceDirectoryPrefix)RESOURCE_MANAGER_URL" = $context.Environment.ResourceManagerUrl; + "$($serviceDirectoryPrefix)SERVICE_MANAGEMENT_URL" = $context.Environment.ServiceManagementUrl; } foreach ($key in $deployment.Outputs.Keys) { @@ -331,7 +347,7 @@ foreach ($templateFile in $templateFiles) { } else { - + if (!$CI) { # Write an extra new line to isolate the environment variables for easy reading. Log "Persist the following environment variables based on your detected shell ($shell):`n" @@ -340,7 +356,7 @@ foreach ($templateFile in $templateFiles) { foreach ($key in $deploymentOutputs.Keys) { $value = $deploymentOutputs[$key] - + if ($CI) { # Treat all ARM template output variables as secrets since "SecureString" variables do not set values. # In order to mask secrets but set environment variables for any given ARM template, we set variables twice as shown below. diff --git a/eng/common/scripts/ChangeLog-Operations.ps1 b/eng/common/scripts/ChangeLog-Operations.ps1 new file mode 100644 index 000000000000..8397e1d0a05a --- /dev/null +++ b/eng/common/scripts/ChangeLog-Operations.ps1 @@ -0,0 +1,122 @@ +# Common Changelog Operations + +$RELEASE_TITLE_REGEX = "(?^\#+.*(?\b\d+\.\d+\.\d+([^0-9\s][^\s:]+)?)(\s(?\(Unreleased\)|\(\d{4}-\d{2}-\d{2}\)))?)" + +# Returns a Collection of changeLogEntry object containing changelog info for all version present in the gived CHANGELOG +function Get-ChangeLogEntries { + param ( + [Parameter(Mandatory = $true)] + [String]$ChangeLogLocation + ) + + $changeLogEntries = @{} + if (!(Test-Path $ChangeLogLocation)) { + Write-Error "ChangeLog[${ChangeLogLocation}] does not exist" + return $null + } + + try { + $contents = Get-Content $ChangeLogLocation + # walk the document, finding where the version specifiers are and creating lists + $changeLogEntry = $null + foreach ($line in $contents) { + if ($line -match $RELEASE_TITLE_REGEX) { + $changeLogEntry = [pscustomobject]@{ + ReleaseVersion = $matches["version"] + ReleaseStatus = $matches["releaseStatus"] + ReleaseTitle = $line + ReleaseContent = @() # Release content without the version title + } + $changeLogEntries[$changeLogEntry.ReleaseVersion] = $changeLogEntry + } + else { + if ($changeLogEntry) { + $changeLogEntry.ReleaseContent += $line + } + } + } + } + catch { + Write-Host "Error parsing $ChangeLogLocation." + Write-Host $_.Exception.Message + } + return $changeLogEntries +} + +# Returns single changeLogEntry object containing the ChangeLog for a particular version +function Get-ChangeLogEntry { + param ( + [Parameter(Mandatory = $true)] + [String]$ChangeLogLocation, + [Parameter(Mandatory = $true)] + [String]$VersionString + ) + $changeLogEntries = Get-ChangeLogEntries -ChangeLogLocation $ChangeLogLocation + + if ($changeLogEntries -and $changeLogEntries.ContainsKey($VersionString)) { + return $changeLogEntries[$VersionString] + } + return $null +} + +#Returns the changelog for a particular version as string +function Get-ChangeLogEntryAsString { + param ( + [Parameter(Mandatory = $true)] + [String]$ChangeLogLocation, + [Parameter(Mandatory = $true)] + [String]$VersionString + ) + + $changeLogEntry = Get-ChangeLogEntry -ChangeLogLocation $ChangeLogLocation -VersionString $VersionString + return ChangeLogEntryAsString $changeLogEntry +} + +function ChangeLogEntryAsString($changeLogEntry) { + if (!$changeLogEntry) { + return "[Missing change log entry]" + } + [string]$releaseTitle = $changeLogEntry.ReleaseTitle + [string]$releaseContent = $changeLogEntry.ReleaseContent -Join [Environment]::NewLine + return $releaseTitle, $releaseContent -Join [Environment]::NewLine +} + +function Confirm-ChangeLogEntry { + param ( + [Parameter(Mandatory = $true)] + [String]$ChangeLogLocation, + [Parameter(Mandatory = $true)] + [String]$VersionString, + [boolean]$ForRelease = $false + ) + + $changeLogEntry = Get-ChangeLogEntry -ChangeLogLocation $ChangeLogLocation -VersionString $VersionString + + if (!$changeLogEntry) { + Write-Error "ChangeLog[${ChangeLogLocation}] does not have an entry for version ${VersionString}." + return $false + } + + Write-Host "Found the following change log entry for version '${VersionString}' in [${ChangeLogLocation}]." + Write-Host "-----" + Write-Host (ChangeLogEntryAsString $changeLogEntry) + Write-Host "-----" + + if ([System.String]::IsNullOrEmpty($changeLogEntry.ReleaseStatus)) { + Write-Error "Entry does not have a correct release status. Please ensure the status is set to a date '(yyyy-MM-dd)' or '(Unreleased)' if not yet released." + return $false + } + + if ($ForRelease -eq $True) { + if ($changeLogEntry.ReleaseStatus -eq "(Unreleased)") { + Write-Error "Entry has no release date set. Please ensure to set a release date with format 'yyyy-MM-dd'." + return $false + } + + if ([System.String]::IsNullOrWhiteSpace($changeLogEntry.ReleaseContent)) { + Write-Error "Entry has no content. Please ensure to provide some content of what changed in this version." + return $false + } + } + return $true +} \ No newline at end of file diff --git a/eng/common/scripts/Package-Properties.ps1 b/eng/common/scripts/Package-Properties.ps1 new file mode 100644 index 000000000000..cba89157a879 --- /dev/null +++ b/eng/common/scripts/Package-Properties.ps1 @@ -0,0 +1,182 @@ +# Helper functions for retireving useful information from azure-sdk-for-* repo +# Example Use : Import-Module .\eng\common\scripts\modules +class PackageProps +{ + [string]$pkgName + [string]$pkgVersion + [string]$pkgDirectoryPath + [string]$pkgServiceName + [string]$pkgReadMePath + [string]$pkgChangeLogPath + [string]$pkgGroup + + PackageProps([string]$pkgName,[string]$pkgVersion,[string]$pkgDirectoryPath,[string]$pkgServiceName) + { + $this.Initialize($pkgName, $pkgVersion, $pkgDirectoryPath, $pkgServiceName) + } + + PackageProps([string]$pkgName,[string]$pkgVersion,[string]$pkgDirectoryPath,[string]$pkgServiceName,[string]$pkgGroup="") + { + $this.Initialize($pkgName, $pkgVersion, $pkgDirectoryPath, $pkgServiceName, $pkgGroup) + } + + hidden [void]Initialize( + [string]$pkgName, + [string]$pkgVersion, + [string]$pkgDirectoryPath, + [string]$pkgServiceName + ) + { + $this.pkgName = $pkgName + $this.pkgVersion = $pkgVersion + $this.pkgDirectoryPath = $pkgDirectoryPath + $this.pkgServiceName = $pkgServiceName + + if (Test-Path (Join-Path $pkgDirectoryPath "README.md")) + { + $this.pkgReadMePath = Join-Path $pkgDirectoryPath "README.md" + } + else + { + $this.pkgReadMePath = $null + } + + if (Test-Path (Join-Path $pkgDirectoryPath "CHANGELOG.md")) + { + $this.pkgChangeLogPath = Join-Path $pkgDirectoryPath "CHANGELOG.md" + } + else + { + $this.pkgChangeLogPath = $null + } + } + + hidden [void]Initialize( + [string]$pkgName, + [string]$pkgVersion, + [string]$pkgDirectoryPath, + [string]$pkgServiceName, + [string]$pkgGroup + ) + { + $this.Initialize($pkgName, $pkgVersion, $pkgDirectoryPath, $pkgServiceName) + $this.pkgGroup = $pkgGroup + } +} + +# Takes package name and service Name +# Returns important properties of the package as related to the language repo +# Returns a PS Object with properties @ { pkgName, pkgVersion, pkgDirectoryPath, pkgReadMePath, pkgChangeLogPath } +# Note: python is required for parsing python package properties. +function Get-PkgProperties +{ + Param + ( + [Parameter(Mandatory=$true)] + [string]$PackageName, + [Parameter(Mandatory=$true)] + [string]$ServiceName + ) + + $pkgDirectoryName = $null + $pkgDirectoryPath = $null + $serviceDirectoryPath = Join-Path $RepoRoot "sdk" $ServiceName + if (!(Test-Path $serviceDirectoryPath)) + { + Write-Error "Service Directory $ServiceName does not exist" + exit 1 + } + + $directoriesPresent = Get-ChildItem $serviceDirectoryPath -Directory + + foreach ($directory in $directoriesPresent) + { + $pkgDirectoryPath = Join-Path $serviceDirectoryPath $directory.Name + if ($ExtractPkgProps) + { + $pkgProps = &$ExtractPkgProps -pkgPath $pkgDirectoryPath -serviceName $ServiceName -pkgName $PackageName + } + else + { + Write-Error "The function '${ExtractPkgProps}' was not found." + } + + if ($pkgProps -ne $null) + { + return $pkgProps + } + } + Write-Error "Failed to retrive Properties for $PackageName" +} + +# Takes ServiceName and Repo Root Directory +# Returns important properties for each package in the specified service, or entire repo if the serviceName is not specified +# Returns an Table of service key to array values of PS Object with properties @ { pkgName, pkgVersion, pkgDirectoryPath, pkgReadMePath, pkgChangeLogPath } +function Get-AllPkgProperties ([string]$ServiceName=$null) +{ + $pkgPropsResult = @() + + if ([string]::IsNullOrEmpty($ServiceName)) + { + $searchDir = Join-Path $RepoRoot "sdk" + foreach ($dir in (Get-ChildItem $searchDir -Directory)) + { + $serviceDir = Join-Path $searchDir $dir.Name + + if (Test-Path (Join-Path $serviceDir "ci.yml")) + { + $activePkgList = Get-PkgListFromYml -ciYmlPath (Join-Path $serviceDir "ci.yml") + if ($activePkgList -ne $null) + { + $pkgPropsResult = Operate-OnPackages -activePkgList $activePkgList -serviceName $dir.Name -pkgPropsResult $pkgPropsResult + } + } + } + } + else + { + $serviceDir = Join-Path $RepoRoot "sdk" $ServiceName + if (Test-Path (Join-Path $serviceDir "ci.yml")) + { + $activePkgList = Get-PkgListFromYml -ciYmlPath (Join-Path $serviceDir "ci.yml") + if ($activePkgList -ne $null) + { + $pkgPropsResult = Operate-OnPackages -activePkgList $activePkgList -serviceName $ServiceName -pkgPropsResult $pkgPropsResult + } + } + } + + return $pkgPropsResult +} + +function Operate-OnPackages ($activePkgList, $serviceName, [Array]$pkgPropsResult) +{ + foreach ($pkg in $activePkgList) + { + $pkgProps = Get-PkgProperties -PackageName $pkg["name"] -ServiceName $serviceName + $pkgPropsResult += $pkgProps + } + return $pkgPropsResult +} + +function Get-PkgListFromYml ($ciYmlPath) +{ + $ProgressPreference = "SilentlyContinue" + Register-PSRepository -Default -ErrorAction:SilentlyContinue + Install-Module -Name powershell-yaml -RequiredVersion 0.4.1 -Force -Scope CurrentUser + $ciYmlContent = Get-Content $ciYmlPath -Raw + $ciYmlObj = ConvertFrom-Yaml $ciYmlContent -Ordered + if ($ciYmlObj.Contains("stages")) + { + $artifactsInCI = $ciYmlObj["stages"][0]["parameters"]["Artifacts"] + } + elseif ($ciYmlObj.Contains("extends")) + { + $artifactsInCI = $ciYmlObj["extends"]["parameters"]["Artifacts"] + } + if ($artifactsInCI -eq $null) + { + Write-Error "Failed to retrive package names in ci $ciYmlPath" + } + return $artifactsInCI +} \ No newline at end of file diff --git a/eng/common/scripts/artifact-metadata-parsing.ps1 b/eng/common/scripts/artifact-metadata-parsing.ps1 index 40b99e632de7..af7c9401e644 100644 --- a/eng/common/scripts/artifact-metadata-parsing.ps1 +++ b/eng/common/scripts/artifact-metadata-parsing.ps1 @@ -406,7 +406,7 @@ function IsPythonPackageVersionPublished($pkgId, $pkgVersion) { # Retrieves the list of all tags that exist on the target repository function GetExistingTags($apiUrl) { try { - return (Invoke-WebRequest -Method "GET" -Uri "$apiUrl/git/refs/tags" -MaximumRetryCount 3 -RetryIntervalSec 10) | % { $_.ref.Replace("refs/tags/", "") } + return (Invoke-RestMethod -Method "GET" -Uri "$apiUrl/git/refs/tags" -MaximumRetryCount 3 -RetryIntervalSec 10) | % { $_.ref.Replace("refs/tags/", "") } } catch { Write-Host $_ diff --git a/eng/common/scripts/common.ps1 b/eng/common/scripts/common.ps1 new file mode 100644 index 000000000000..f1677b21425f --- /dev/null +++ b/eng/common/scripts/common.ps1 @@ -0,0 +1,27 @@ +$global:RepoRoot = Resolve-Path "${PSScriptRoot}..\..\..\.." +$global:EngDir = Join-Path $global:RepoRoot "eng" +$global:EngCommonDir = Join-Path $global:EngDir "common" +$global:EngCommonScriptsDir = Join-Path $global:EngCommonDir "scripts" +$global:EngScriptsDir = Join-Path $global:EngDir "scripts" + +# Import required scripts +. (Join-Path $global:EngCommonScriptsDir SemVer.ps1) +. (Join-Path $global:EngCommonScriptsDir Changelog-Operations.ps1) +. (Join-Path $global:EngCommonScriptsDir Package-Properties.ps1) + +# Setting expected from common languages settings +$global:Language = "Unknown" +$global:PackageRepository = "Unknown" +$global:packagePattern = "Unknown" +$global:MetadataUri = "Unknown" + +# Import common language settings +$EngScriptsLanguageSettings = Join-path $global:EngScriptsDir "Language-Settings.ps1" +if (Test-Path $EngScriptsLanguageSettings) { + . $EngScriptsLanguageSettings +} + +# Transformed Functions +$GetPackageInfoFromRepoFn = "Get-${Language}-PackageInfoFromRepo" +$GetPackageInfoFromPackageFileFn = "Get-${Language}-PackageInfoFromPackageFile" +$PublishGithubIODocsFn = "Publish-${Language}-GithubIODocs" \ No newline at end of file