Merge branch 'master' of https://github.com/Azure/azure-sdk-for-python into regenerate_keys

* 'master' of https://github.com/Azure/azure-sdk-for-python: (24 commits)
  rename FormField.type -> FormField.value_type (Azure#12708)
  20200703 codereport (Azure#12361)
  Use the common Cosmos emulator template. (Azure#12678)
  Add azurestackhci ci.yml for pipeline generation. (Azure#12701)
  [ServiceBus] Resend received message (Azure#12457)
  AZURE_CLIENT_ID sets user-assigned managed identity for DefaultAzureCredential (Azure#12689)
  Add better autolockrenew on-failure handling capabilities. (Azure#12307)
  Update CODEOWNERS (Azure#12418)
  close client session in async tests (Azure#12656)
  Sync eng/common directory with azure-sdk-tools repository (Azure#12502)
  [Event Hubs] Use EventHubManagementClient to create test resources (Azure#12501)
  Increment package version after release of azure_identity (Azure#12675)
  [formrecognizer] include error code for errors that occur during polling (Azure#12621)
  Cross-language consistency changes (Azure#12669)
  Enable APIStubGen step (Azure#12615)
  Adding the ability to create a container with analytical storage turned on. (Azure#12408)
  [ServiceBus] Adjust user-agent following the guideline (Azure#12596)
  Release for Cognitive Services - Anomaly Detector (Azure#12582)
  [formrecognizer] Add type to FormField (Azure#12561)
  Add example summary for azure-identity readme.md (Azure#12509)
  ...

Author: iscai-msft

.github/CODEOWNERS

@@ -9,36 +9,83 @@
 /sdk/ @mayurid @lmazuel
 
 # Core
+# PRLabel: %Azure.Core
 /sdk/core/                                                           @lmazuel @xiangyan99 @johanste @rakshith91
 /sdk/core/azure-core/                                                @xiangyan99 @lmazuel
 
+####
 # Service team
+####
+# PRLabel: %Azure.Identity
 /sdk/identity/                                                       @chlowell @schaabs
+
+# PRLabel: %Event Hubs
 /sdk/eventhub/                                                       @annatisch @yunhaoling @YijunXieMS
-/sdk/storage/                                                        @amishra-dev @zezha-msft @annatisch @rakshith91 @xiafu-msft
-/sdk/applicationinsights/                                            @alexeldeib
+
+# PRLabel: %Storage
+/sdk/storage/                                                        @amishra-dev @zezha-msft @annatisch @rakshith91 @xiafu-msft    
+
+/sdk/applicationinsights/                                            @alexeldeib      
+
+# PRLabel: %Batch
 /sdk/batch/                                                          @bgklein @xingwu1
-/sdk/cognitiveservices/azure-cognitiveservices-vision-customvision/  @areddish
+/sdk/cognitiveservices/azure-cognitiveservices-vision-customvision/  @areddish          
+
+# PRLabel: %KeyVault
 /sdk/keyvault/                                                       @schaabs @chlowell @iscai-msft
+
+# PRLabel: %Monitor - LogAnalytics
 /sdk/loganalytics/                                                   @alexeldeib
+
+# PRLabel: %Consumption
 /sdk/consumption/                                                    @sandeepnl
+
+# PRLabel: %Container Instances
 /sdk/containerinstance/                                              @samkreter @xizhamsft
+
+# PRLabel: %Container Registry
 /sdk/containerregistry/                                              @djyou
 /sdk/containerservice/                                               @samkreter @zqingqing1 @GaneshaThirumurthi
+
+# PRLabel: %Cosmos
 /sdk/cosmos/                                                         @southpolesteve @zfoster
+
+# PRLabel: %Data Factory
 /sdk/datafactory/                                                    @hvermis
-/sdk/datalake/                                                       @ro-joowan
+/sdk/datalake/                                                       @ro-joowan                                                     
 /sdk/datadatamigration/                                              @vchske
+
+# PRLabel: %Event Grid
 /sdk/eventgrid/                                                      @kalyanaj
+
+# PRLabel: %HDInsight
 /sdk/hdinsight/                                                      @idear1203
+
+# PRLabel: %Machine Learning Compute
 /sdk/machinelearningcompute/                                         @shutchings
+
+# PRLabel: %Recovery Services
 /sdk/recoveryservices/                                               @DheerendraRathor
+
+# PRLabel: %Search
 /sdk/search/                                                         @xiangyan99 @rakshith91
+
+# PRLabel: %Service Fabric
 /sdk/servicefabric/                                                  @QingChenmsft @samedder
+
+# PRLabel: %SQL
 /sql/sql/                                                            @jaredmoo
+
+# PRLabel: %Service Bus
 /sdk/servicebus/                                                     @annatisch @yunhaoling @YijunXieMS @KieranBrantnerMagee
+
+# PRLabel: %Synapse
 /sdk/synapse/                                                        @aim-for-better @idear1203
+
+# PRLabel: %Cognitive - Text Analytics
 /sdk/textanalytics/                                                  @kristapratico @iscai-msft
+
+# PRLabel: %Cognitive - Form Recognizer
 /sdk/formrecognizer/                                                 @kristapratico @iscai-msft @rakshith91
 
 # Smoke Tests

eng/common/TestResources/deploy-test-resources.yml

@@ -13,7 +13,7 @@ parameters:
 #   "TestApplicationId": "<test app id>",
 #   "TestApplicationSecret": "<test app secret>",
 #   "ProvisionerApplicationId": "<provisoner app id>",
-#   "ProvisoinerApplicationSecret": "<provisoner app secert>",
+#   "ProvisionerApplicationSecret": "<provisoner app secert>",
 #   "Environment": "AzureCloud | AzureGov | AzureChina | <other environment>"
 # }
 

eng/common/TestResources/remove-test-resources.yml

@@ -32,5 +32,5 @@ steps:
         -Force `
         -Verbose
     displayName: Remove test resources
-    condition: and(ne(variables['AZURE_RESOURCEGROUP_NAME'], ''), succeededOrFailed())
+    condition: ne(variables['AZURE_RESOURCEGROUP_NAME'], '')
     continueOnError: true

eng/pipelines/templates/steps/cosmos-emulator.yml renamed to eng/common/pipelines/templates/steps/cosmos-emulator.yml

@@ -12,6 +12,8 @@ parameters:
   WorkingDirectory: $(System.DefaultWorkingDirectory)
   PRTitle: not-specified
   ScriptDirectory: eng/common/scripts
+  GHReviewersVariable: '' 
+  GHTeamReviewersVariable: ''
 
 steps:
 
@@ -63,3 +65,18 @@ steps:
       -PRBranch "${{ parameters.PRBranchName }}"
       -AuthToken "$(azuresdk-github-pat)"
       -PRTitle "${{ parameters.PRTitle }}"
+
+- task: PowerShell@2
+  displayName: Tag a Reviewer on PR
+  condition: and(succeeded(), eq(variables['HasChanges'], 'true'))
+  inputs:
+    pwsh: true
+    workingDirectory: ${{ parameters.WorkingDirectory }}
+    filePath: ${{ parameters.ScriptDirectory }}/add-pullrequest-reviewers.ps1
+    arguments: >
+      -RepoOwner "${{ parameters.RepoOwner }}"
+      -RepoName "${{ parameters.RepoName }}"
+      -AuthToken "$(azuresdk-github-pat)"
+      -GitHubUsers "$(${{ parameters.GHReviewersVariable }})"
+      -GitHubTeams "$(${{ parameters.GHTeamReviewersVariable }})"
+      -PRNumber "$(Submitted.PullRequest.Number)"

eng/common/pipelines/templates/steps/create-pull-request.yml

@@ -12,6 +12,8 @@ parameters:
   ArtifactName: ''
   Language: ''
   DocRepoDestinationPath: '' #usually docs-ref-services/
+  GHReviewersVariable: '' 
+  GHTeamReviewersVariable: '' # externally set, as eng-common does not have the identity-resolver. Run as pre-step
 
 steps:
 - pwsh: |
@@ -56,3 +58,5 @@ steps:
     BaseBranchName: smoke-test
     WorkingDirectory: ${{ parameters.WorkingDirectory }}/repo
     ScriptDirectory: ${{ parameters.WorkingDirectory }}/${{ parameters.ScriptDirectory }}
+    GHReviewersVariable: ${{ parameters.GHReviewersVariable }}
+    GHTeamReviewersVariable: ${{ parameters.GHTeamReviewersVariable }} 

eng/common/pipelines/templates/steps/docs-metadata-release.yml

@@ -0,0 +1,46 @@
+parameters: 
+  TargetVariable: ''
+  ServiceDirectory: ''
+
+steps:
+  - pwsh: |
+      git clone https://github.com/Azure/azure-sdk-tools.git $(Build.SourcesDirectory)/tools_repo
+      cd $(Build.SourcesDirectory)/tools_repo
+      git checkout 564ad63ae72d18422533fa1da9d396e7703c1cb5
+    displayName: Setup Identity Resolver
+
+  - pwsh: |
+      $result = dotnet run -v q -- `
+        --aad-app-id-var APP_ID `
+        --aad-app-secret-var APP_SECRET `
+        --aad-tenant-var AAD_TENANT `
+        --kusto-url-var KUSTO_URL `
+        --kusto-database-var KUSTO_DB `
+        --kusto-table-var KUSTO_TABLE `
+        --identity "$(Build.QueuedBy)"
+      $resolvedIdentity = $result[-1] | ConvertFrom-Json
+
+      Write-Host $resolvedIdentity
+
+      Write-Output "##vso[task.setvariable variable=${{ parameters.TargetVariable }}]$($resolvedIdentity.GithubUserName)"
+    displayName: 'Resolving Queuing User'
+    workingDirectory: $(Build.SourcesDirectory)/tools_repo/tools/notification-configuration/identity-resolver
+    env:
+      APP_ID: $(notification-aad-app-id)
+      APP_SECRET: $(notification-aad-secret)
+      AAD_TENANT: $(notification-aad-tenant)
+      KUSTO_URL: $(notification-kusto-url)
+      KUSTO_DB: $(notification-kusto-db)
+      KUSTO_TABLE: $(notification-kusto-table)
+
+  - pwsh: |
+      Remove-Item -Force -Recurse $(Build.SourcesDirectory)/tools_repo
+    displayName: Clean Up Cloned Tools Repo
+
+  - pwsh: |
+      $originalValue = "$(${{ parameters.TargetVariable }})"
+      $result = $(Build.SourcesDirectory)/eng/common/scripts/get-codeowners.ps1 -TargetDirectory /sdk/${{ parameters.ServiceDirectory }}/ -RootDirectory $(Build.SourcesDirectory)
+      if ($result) {
+        Write-Output "##vso[task.setvariable variable=${{ parameters.TargetVariable }}]$originalValue,$result"
+      }
+    displayName: Add CodeOwners if Present

eng/common/pipelines/templates/steps/get-pr-owners.yml

@@ -58,6 +58,9 @@ $resp | Write-Verbose
 
 if ($resp.Count -gt 0) {
     Write-Host -f green "Pull request already exists $($resp[0].html_url)"
+
+    # setting variable to reference the pull request by number
+    Write-Host "##vso[task.setvariable variable=Submitted.PullRequest.Number]$($resp[0].number)"
 }
 else {
   $data = @{
@@ -80,4 +83,7 @@ else {
 
   $resp | Write-Verbose
   Write-Host -f green "Pull request created https://github.com/$RepoOwner/$RepoName/pull/$($resp.number)"
+
+  # setting variable to reference the pull request by number
+  Write-Host "##vso[task.setvariable variable=Submitted.PullRequest.Number]$($resp.number)"
 }

eng/common/scripts/Submit-PullRequest.ps1

@@ -125,7 +125,18 @@ function CheckLink ([System.Uri]$linkUri)
   }
   else {
     try {
-      $response = Invoke-WebRequest -Uri $linkUri
+      $headRequestSucceeded = $true
+      try {
+        # Attempt HEAD request first
+        $response = Invoke-WebRequest -Uri $linkUri -Method HEAD
+      }
+      catch {
+        $headRequestSucceeded = $false
+      }
+      if (!$headRequestSucceeded) {
+        # Attempt a GET request if the HEAD request failed.
+        $response = Invoke-WebRequest -Uri $linkUri -Method GET
+      }
       $statusCode = $response.StatusCode
       if ($statusCode -ne 200) {
         Write-Host "[$statusCode] while requesting $linkUri"

eng/common/scripts/Verify-Links.ps1

@@ -0,0 +1,79 @@
+param(
+    [Parameter(Mandatory = $true)]
+    $RepoOwner,
+
+    [Parameter(Mandatory = $true)]
+    $RepoName,
+
+    [Parameter(Mandatory = $false)]
+    $GitHubUsers = "",
+
+    [Parameter(Mandatory = $false)]
+    $GitHubTeams = "",
+
+    [Parameter(Mandatory = $true)]
+    $PRNumber,
+  
+    [Parameter(Mandatory = $true)]
+    $AuthToken
+)
+
+# at least one of these needs to be populated
+if (-not $GitHubUsers -and -not $GitHubTeams) {
+  Write-Host "No user provided for addition, exiting."
+  exit 0
+}
+
+$userAdditions = @($GitHubUsers.Split(",") | % { $_.Trim() } | ? { return $_ })
+$teamAdditions = @($GitHubTeams.Split(",") | % { $_.Trim() } | ? { return $_ })
+
+$headers = @{
+  Authorization = "bearer $AuthToken"
+}
+$uri = "https://api.github.com/repos/$RepoOwner/$RepoName/pulls/$PRNumber/requested_reviewers"
+
+try {
+  $resp = Invoke-RestMethod -Headers $headers $uri -MaximumRetryCount 3
+}
+catch {
+  Write-Error "Invoke-RestMethod [$uri] failed with exception:`n$_"
+  exit 1
+}
+
+# the response object takes this form: https://developer.github.com/v3/pulls/review_requests/#response-1
+# before we can push a new reviewer, we need to pull the simple Ids out of the complex objects that came back in the response
+$userReviewers = @($resp.users | % { return $_.login })
+$teamReviewers = @($resp.teams | % { return $_.slug })
+
+if (!$usersReviewers) { $modifiedUserReviewers = @() } else { $modifiedUserReviewers = $usersReviewers.Clone() }
+$modifiedUserReviewers += ($modifiedUserReviewers | ? { !$usersReviews.Contains($_) })
+
+if ($teamReviewers) { $modifiedTeamReviewers = @() } else { $modifiedTeamReviewers = $teamReviewers.Clone() }
+$modifiedTeamReviewers += ($modifiedUserReviewers | ? { !$teamReviewers.Contains($_) })
+
+$detectedUserDiffs = Compare-Object -ReferenceObject $userReviewers -DifferenceObject $modifiedUserReviewers
+$detectedTeamDiffs = Compare-Object -ReferenceObject $teamReviewers -DifferenceObject $modifiedTeamReviewers
+
+# Compare-Object returns values when there is a difference between the comparied objects.
+# we only want to run the update if there IS a difference.
+if ($detectedUserDiffs -or $detectedTeamDiffs) {
+  $postResp = @{}
+
+  if ($modifiedUserReviewers) { $postResp["reviewers"] = $modifiedUserReviewers }
+  if ($modifiedTeamReviewers) { $postResp["team_reviewers"] = $modifiedTeamReviewers }
+
+  $postResp = $postResp | ConvertTo-Json
+
+  try {
+    $resp = Invoke-RestMethod -Method Post -Headers $headers -Body $postResp -Uri $uri -MaximumRetryCount 3
+    $resp | Write-Verbose
+  }
+  catch {
+    Write-Error "Unable to update PR reviewers. `n$_"
+  }
+}
+else {
+  $results = $GitHubUsers + $GitHubTeams
+  Write-Host "Reviewers $results already added. Exiting."
+  exit(0)
+}

eng/common/scripts/add-pullrequest-reviewers.ps1

@@ -347,6 +347,38 @@ function ParseCArtifact($pkg, $workingDirectory) {
   }
 }
 
+function ParseCppArtifact($pkg, $workingDirectory) {
+  $packageInfo = Get-Content -Raw -Path $pkg | ConvertFrom-JSON
+  $packageArtifactLocation = (Get-ItemProperty $pkg).Directory.FullName
+  $releaseNotes = ""
+  $readmeContent = ""
+
+  $pkgVersion = $packageInfo.version
+  $pkgName = $packageInfo.name
+
+  $changeLogLoc = @(Get-ChildItem -Path $packageArtifactLocation -Recurse -Include "CHANGELOG.md")[0]
+  if ($changeLogLoc)
+  {
+    $releaseNotes = Get-ChangeLogEntryAsString -ChangeLogLocation $changeLogLoc -VersionString $pkgVersion
+  }
+
+  $readmeContentLoc = @(Get-ChildItem -Path $packageArtifactLocation -Recurse -Include "README.md")[0]
+  if ($readmeContentLoc) {
+    $readmeContent = Get-Content -Raw $readmeContentLoc
+  }
+
+  return New-Object PSObject -Property @{
+    PackageId      = $pkgName
+    PackageVersion = $pkgVersion
+    # Artifact info is always considered deployable for now becasue it is not
+    # deployed anywhere. Dealing with duplicate tags happens downstream in
+    # CheckArtifactShaAgainstTagsList
+    Deployable     = $true
+    ReleaseNotes   = $releaseNotes
+  }
+}
+
+
 # Returns the pypi publish status of a package id and version.
 function IsPythonPackageVersionPublished($pkgId, $pkgVersion) {
   try {
@@ -374,9 +406,10 @@ function IsPythonPackageVersionPublished($pkgId, $pkgVersion) {
 # Retrieves the list of all tags that exist on the target repository
 function GetExistingTags($apiUrl) {
   try {
-    return (Invoke-WebRequest-WithHandling -Method "GET" -url "$apiUrl/git/refs/tags"  ) | % { $_.ref.Replace("refs/tags/", "") }
+    return (Invoke-WebRequest -Method "GET" -Uri "$apiUrl/git/refs/tags"  -MaximumRetryCount 3 -RetryIntervalSec 10) | % { $_.ref.Replace("refs/tags/", "") }
   }
   catch {
+    Write-Host $_
     $statusCode = $_.Exception.Response.StatusCode.value__
     $statusDescription = $_.Exception.Response.StatusDescription
 
@@ -386,7 +419,7 @@ function GetExistingTags($apiUrl) {
 
     # Return an empty list if there are no tags in the repo
     if ($statusCode -eq 404) {
-      return @()
+      return ,@()
     }
 
     exit(1)
@@ -424,6 +457,10 @@ function VerifyPackages($pkgRepository, $artifactLocation, $workingDirectory, $a
       $ParsePkgInfoFn = "ParseCArtifact"
       $packagePattern = "*.json"
     }
+    "CPP" {
+      $ParsePkgInfoFn = "ParseCppArtifact"
+      $packagePattern = "*.json"
+    }
     default {
       Write-Host "Unrecognized Language: $language"
       exit(1)

eng/common/scripts/artifact-metadata-parsing.ps1

@@ -350,4 +350,10 @@ if ($Language -eq "c")
     # used to publish multiple docs packages in a single invocation.
     $pkgInfo = Get-Content $DocLocation/package-info.json | ConvertFrom-Json
     Upload-Blobs -DocDir $DocLocation -PkgName 'docs' -DocVersion $pkgInfo.version
+}
+
+if ($Language -eq "cpp")
+{
+    $packageInfo = (Get-Content (Join-Path $DocLocation 'package-info.json') | ConvertFrom-Json)
+    Upload-Blobs -DocDir $DocLocation -PkgName $packageInfo.name -DocVersion $packageInfo.version
 }