Sync eng/common directory with azure-sdk-tools repository

eng/common/TestResources/New-TestResources.ps1

@@ -154,14 +154,21 @@ if ($ProvisionerApplicationId) {
     $subscriptionArgs = if ($SubscriptionId) {
         @{SubscriptionId = $SubscriptionId}
     }
+    else {
+        @{}
+    }
 
     $provisionerAccount = Retry {
         Connect-AzAccount -Force:$Force -Tenant $TenantId -Credential $provisionerCredential -ServicePrincipal -Environment $Environment @subscriptionArgs
     }
 
     $exitActions += {
         Write-Verbose "Logging out of service principal '$($provisionerAccount.Context.Account)'"
-        $null = Disconnect-AzAccount -AzureContext $provisionerAccount.Context
+
+        # Only attempt to disconnect if the -WhatIf flag was not set.  Otherwise, this call is not necessary and will fail.
+        if ($PSCmdlet.ShouldProcess($ProvisionerApplicationId)) {
+            $null = Disconnect-AzAccount -AzureContext $provisionerAccount.Context
+        }
     }
 }
 
@@ -176,7 +183,6 @@ if ($TestApplicationId -and !$TestApplicationOid) {
     }
 }
 
-
 # If the ServiceDirectory is an absolute path use the last directory name
 # (e.g. D:\foo\bar\ -> bar)
 $serviceName = if (Split-Path -IsAbsolute  $ServiceDirectory) {
@@ -229,6 +235,13 @@ if ($resourceGroup.ProvisioningState -eq 'Succeeded') {
     # New-AzResourceGroup would've written an error and stopped the pipeline by default anyway.
     Write-Verbose "Successfully created resource group '$($resourceGroup.ResourceGroupName)'"
 }
+elseif (($resourceGroup -eq $null) -and (-not $PSCmdlet.ShouldProcess($resourceGroupName))) {
+    # If the -WhatIf flag was passed, there will be no resource group created.  Fake it.
+    $resourceGroup = [PSCustomObject]@{
+        ResourceGroupName = $resourceGroupName
+        Location = $Location
+    }
+}
 
 # Populate the template parameters and merge any additional specified.
 $templateParameters = @{
@@ -299,6 +312,9 @@ foreach ($templateFile in $templateFiles) {
         "$($serviceDirectoryPrefix)RESOURCE_GROUP" = $resourceGroup.ResourceGroupName;
         "$($serviceDirectoryPrefix)LOCATION" = $resourceGroup.Location;
         "$($serviceDirectoryPrefix)ENVIRONMENT" = $context.Environment.Name;
+        "$($serviceDirectoryPrefix)AZURE_AUTHORITY_HOST" = $context.Environment.ActiveDirectoryAuthority;
+        "$($serviceDirectoryPrefix)RESOURCE_MANAGER_URL" = $context.Environment.ResourceManagerUrl;
+        "$($serviceDirectoryPrefix)SERVICE_MANAGEMENT_URL" = $context.Environment.ServiceManagementUrl;
     }
 
     foreach ($key in $deployment.Outputs.Keys) {
@@ -331,7 +347,7 @@ foreach ($templateFile in $templateFiles) {
     }
     else
     {
-        
+
         if (!$CI) {
             # Write an extra new line to isolate the environment variables for easy reading.
             Log "Persist the following environment variables based on your detected shell ($shell):`n"
@@ -340,7 +356,7 @@ foreach ($templateFile in $templateFiles) {
         foreach ($key in $deploymentOutputs.Keys)
         {
             $value = $deploymentOutputs[$key]
-            
+
             if ($CI) {
                 # Treat all ARM template output variables as secrets since "SecureString" variables do not set values.
                 # In order to mask secrets but set environment variables for any given ARM template, we set variables twice as shown below.

eng/common/scripts/ChangeLog-Operations.ps1

@@ -0,0 +1,122 @@
+# Common Changelog Operations
+
+$RELEASE_TITLE_REGEX = "(?<releaseNoteTitle>^\#+.*(?<version>\b\d+\.\d+\.\d+([^0-9\s][^\s:]+)?)(\s(?<releaseStatus>\(Unreleased\)|\(\d{4}-\d{2}-\d{2}\)))?)"
+
+# Returns a Collection of changeLogEntry object containing changelog info for all version present in the gived CHANGELOG
+function Get-ChangeLogEntries {
+  param (
+    [Parameter(Mandatory = $true)]
+    [String]$ChangeLogLocation
+  )
+
+  $changeLogEntries = @{}
+  if (!(Test-Path $ChangeLogLocation)) {
+    Write-Error "ChangeLog[${ChangeLogLocation}] does not exist"
+    return $null
+  }
+
+  try {
+    $contents = Get-Content $ChangeLogLocation
+    # walk the document, finding where the version specifiers are and creating lists
+    $changeLogEntry = $null
+    foreach ($line in $contents) {
+      if ($line -match $RELEASE_TITLE_REGEX) {
+        $changeLogEntry = [pscustomobject]@{ 
+          ReleaseVersion = $matches["version"]
+          ReleaseStatus  = $matches["releaseStatus"]
+          ReleaseTitle   = $line
+          ReleaseContent = @() # Release content without the version title
+        }
+        $changeLogEntries[$changeLogEntry.ReleaseVersion] = $changeLogEntry
+      }
+      else {
+        if ($changeLogEntry) {
+          $changeLogEntry.ReleaseContent += $line
+        }
+      }
+    }
+  }
+  catch {
+    Write-Host "Error parsing $ChangeLogLocation."
+    Write-Host $_.Exception.Message
+  }
+  return $changeLogEntries
+}
+
+# Returns single changeLogEntry object containing the ChangeLog for a particular version
+function Get-ChangeLogEntry {
+  param (
+    [Parameter(Mandatory = $true)]
+    [String]$ChangeLogLocation,
+    [Parameter(Mandatory = $true)]
+    [String]$VersionString
+  )
+  $changeLogEntries = Get-ChangeLogEntries -ChangeLogLocation $ChangeLogLocation
+
+  if ($changeLogEntries -and $changeLogEntries.ContainsKey($VersionString)) {
+    return $changeLogEntries[$VersionString]
+  }
+  return $null
+}
+
+#Returns the changelog for a particular version as string
+function Get-ChangeLogEntryAsString {
+  param (
+    [Parameter(Mandatory = $true)]
+    [String]$ChangeLogLocation,
+    [Parameter(Mandatory = $true)]
+    [String]$VersionString
+  )
+
+  $changeLogEntry = Get-ChangeLogEntry -ChangeLogLocation $ChangeLogLocation -VersionString $VersionString
+  return ChangeLogEntryAsString $changeLogEntry
+}
+
+function ChangeLogEntryAsString($changeLogEntry) {
+  if (!$changeLogEntry) {
+    return "[Missing change log entry]"
+  }
+  [string]$releaseTitle = $changeLogEntry.ReleaseTitle
+  [string]$releaseContent = $changeLogEntry.ReleaseContent -Join [Environment]::NewLine
+  return $releaseTitle, $releaseContent -Join [Environment]::NewLine
+}
+
+function Confirm-ChangeLogEntry {
+  param (
+    [Parameter(Mandatory = $true)]
+    [String]$ChangeLogLocation,
+    [Parameter(Mandatory = $true)]
+    [String]$VersionString,
+    [boolean]$ForRelease = $false
+  )
+
+  $changeLogEntry = Get-ChangeLogEntry -ChangeLogLocation $ChangeLogLocation -VersionString $VersionString
+
+  if (!$changeLogEntry) {
+    Write-Error "ChangeLog[${ChangeLogLocation}] does not have an entry for version ${VersionString}."
+    return $false
+  }
+
+  Write-Host "Found the following change log entry for version '${VersionString}' in [${ChangeLogLocation}]."
+  Write-Host "-----"
+  Write-Host (ChangeLogEntryAsString $changeLogEntry)
+  Write-Host "-----"
+
+  if ([System.String]::IsNullOrEmpty($changeLogEntry.ReleaseStatus)) {
+    Write-Error "Entry does not have a correct release status. Please ensure the status is set to a date '(yyyy-MM-dd)' or '(Unreleased)' if not yet released."
+    return $false
+  }
+
+  if ($ForRelease -eq $True) {
+    if ($changeLogEntry.ReleaseStatus -eq "(Unreleased)") {
+      Write-Error "Entry has no release date set. Please ensure to set a release date with format 'yyyy-MM-dd'."
+      return $false
+    }
+
+    if ([System.String]::IsNullOrWhiteSpace($changeLogEntry.ReleaseContent)) {
+      Write-Error "Entry has no content. Please ensure to provide some content of what changed in this version."
+      return $false
+    }
+  }
+  return $true
+}

eng/common/scripts/Package-Properties.ps1

@@ -0,0 +1,182 @@
+# Helper functions for retireving useful information from azure-sdk-for-* repo
+# Example Use : Import-Module .\eng\common\scripts\modules
+class PackageProps
+{
+    [string]$pkgName
+    [string]$pkgVersion
+    [string]$pkgDirectoryPath
+    [string]$pkgServiceName
+    [string]$pkgReadMePath
+    [string]$pkgChangeLogPath
+    [string]$pkgGroup
+
+    PackageProps([string]$pkgName,[string]$pkgVersion,[string]$pkgDirectoryPath,[string]$pkgServiceName)
+    {
+        $this.Initialize($pkgName, $pkgVersion, $pkgDirectoryPath, $pkgServiceName)
+    }
+
+    PackageProps([string]$pkgName,[string]$pkgVersion,[string]$pkgDirectoryPath,[string]$pkgServiceName,[string]$pkgGroup="")
+    {
+        $this.Initialize($pkgName, $pkgVersion, $pkgDirectoryPath, $pkgServiceName, $pkgGroup)
+    }
+
+    hidden [void]Initialize(
+        [string]$pkgName,
+        [string]$pkgVersion,
+        [string]$pkgDirectoryPath,
+        [string]$pkgServiceName
+    )
+    {
+        $this.pkgName = $pkgName
+        $this.pkgVersion = $pkgVersion
+        $this.pkgDirectoryPath = $pkgDirectoryPath
+        $this.pkgServiceName = $pkgServiceName
+
+        if (Test-Path (Join-Path $pkgDirectoryPath "README.md"))
+        {
+            $this.pkgReadMePath = Join-Path $pkgDirectoryPath "README.md"
+        } 
+        else
+        {
+            $this.pkgReadMePath = $null
+        }
+
+        if (Test-Path (Join-Path $pkgDirectoryPath "CHANGELOG.md"))
+        {
+            $this.pkgChangeLogPath = Join-Path $pkgDirectoryPath "CHANGELOG.md"
+        } 
+        else
+        {
+            $this.pkgChangeLogPath = $null
+        }
+    }
+
+    hidden [void]Initialize(
+        [string]$pkgName,
+        [string]$pkgVersion,
+        [string]$pkgDirectoryPath,
+        [string]$pkgServiceName,
+        [string]$pkgGroup
+    )
+    {
+        $this.Initialize($pkgName, $pkgVersion, $pkgDirectoryPath, $pkgServiceName)
+        $this.pkgGroup = $pkgGroup
+    }
+}
+
+# Takes package name and service Name
+# Returns important properties of the package as related to the language repo
+# Returns a PS Object with properties @ { pkgName, pkgVersion, pkgDirectoryPath, pkgReadMePath, pkgChangeLogPath }
+# Note: python is required for parsing python package properties.
+function Get-PkgProperties
+{
+    Param
+    (
+        [Parameter(Mandatory=$true)]
+        [string]$PackageName,
+        [Parameter(Mandatory=$true)]
+        [string]$ServiceName
+    )
+
+    $pkgDirectoryName = $null
+    $pkgDirectoryPath = $null
+    $serviceDirectoryPath = Join-Path $RepoRoot "sdk" $ServiceName
+    if (!(Test-Path $serviceDirectoryPath))
+    {
+        Write-Error "Service Directory $ServiceName does not exist"
+        exit 1
+    }
+
+    $directoriesPresent = Get-ChildItem $serviceDirectoryPath -Directory
+
+    foreach ($directory in $directoriesPresent)
+    {
+        $pkgDirectoryPath = Join-Path $serviceDirectoryPath $directory.Name
+        if ($ExtractPkgProps)
+        {
+            $pkgProps = &$ExtractPkgProps -pkgPath $pkgDirectoryPath -serviceName $ServiceName -pkgName $PackageName
+        }
+        else
+        {
+            Write-Error "The function '${ExtractPkgProps}' was not found."
+        }
+
+        if ($pkgProps -ne $null)
+        {
+            return $pkgProps
+        }
+    }
+    Write-Error "Failed to retrive Properties for $PackageName"
+}
+
+# Takes ServiceName and Repo Root Directory
+# Returns important properties for each package in the specified service, or entire repo if the serviceName is not specified
+# Returns an Table of service key to array values of PS Object with properties @ { pkgName, pkgVersion, pkgDirectoryPath, pkgReadMePath, pkgChangeLogPath }
+function Get-AllPkgProperties ([string]$ServiceName=$null)
+{
+    $pkgPropsResult = @()
+
+    if ([string]::IsNullOrEmpty($ServiceName))
+    {
+        $searchDir = Join-Path $RepoRoot "sdk"
+        foreach ($dir in (Get-ChildItem $searchDir -Directory))
+        {
+            $serviceDir = Join-Path $searchDir $dir.Name
+
+            if (Test-Path (Join-Path $serviceDir "ci.yml"))
+            {
+                $activePkgList = Get-PkgListFromYml -ciYmlPath (Join-Path $serviceDir "ci.yml")
+                if ($activePkgList -ne $null)
+                {
+                    $pkgPropsResult = Operate-OnPackages -activePkgList $activePkgList -serviceName $dir.Name -pkgPropsResult $pkgPropsResult
+                }
+            }
+        }
+    } 
+    else
+    {
+        $serviceDir = Join-Path $RepoRoot "sdk" $ServiceName
+        if (Test-Path (Join-Path $serviceDir "ci.yml"))
+        {
+            $activePkgList = Get-PkgListFromYml -ciYmlPath (Join-Path $serviceDir "ci.yml")
+            if ($activePkgList -ne $null)
+            {
+                $pkgPropsResult = Operate-OnPackages -activePkgList $activePkgList -serviceName $ServiceName -pkgPropsResult $pkgPropsResult
+            }
+        }
+    }
+
+    return $pkgPropsResult
+}
+
+function Operate-OnPackages ($activePkgList, $serviceName, [Array]$pkgPropsResult)
+{
+    foreach ($pkg in $activePkgList)
+    {
+        $pkgProps = Get-PkgProperties -PackageName $pkg["name"] -ServiceName $serviceName
+        $pkgPropsResult += $pkgProps
+    }
+    return $pkgPropsResult
+}
+
+function Get-PkgListFromYml ($ciYmlPath)
+{
+    $ProgressPreference = "SilentlyContinue"
+    Register-PSRepository -Default -ErrorAction:SilentlyContinue
+    Install-Module -Name powershell-yaml -RequiredVersion 0.4.1 -Force -Scope CurrentUser
+    $ciYmlContent = Get-Content $ciYmlPath -Raw
+    $ciYmlObj = ConvertFrom-Yaml $ciYmlContent -Ordered
+    if ($ciYmlObj.Contains("stages"))
+    {
+      $artifactsInCI = $ciYmlObj["stages"][0]["parameters"]["Artifacts"]
+    }
+    elseif ($ciYmlObj.Contains("extends")) 
+    {
+      $artifactsInCI = $ciYmlObj["extends"]["parameters"]["Artifacts"]
+    }
+    if ($artifactsInCI -eq $null)
+    {
+        Write-Error "Failed to retrive package names in ci $ciYmlPath"
+    }
+    return $artifactsInCI
+}

eng/common/scripts/artifact-metadata-parsing.ps1

@@ -406,7 +406,7 @@ function IsPythonPackageVersionPublished($pkgId, $pkgVersion) {
 # Retrieves the list of all tags that exist on the target repository
 function GetExistingTags($apiUrl) {
   try {
-    return (Invoke-WebRequest -Method "GET" -Uri "$apiUrl/git/refs/tags"  -MaximumRetryCount 3 -RetryIntervalSec 10) | % { $_.ref.Replace("refs/tags/", "") }
+    return (Invoke-RestMethod -Method "GET" -Uri "$apiUrl/git/refs/tags"  -MaximumRetryCount 3 -RetryIntervalSec 10) | % { $_.ref.Replace("refs/tags/", "") }
   }
   catch {
     Write-Host $_