Merge branch 'master' of https://github.com/Azure/azure-sdk-for-python into add_back_pii

* 'master' of https://github.com/Azure/azure-sdk-for-python: (40 commits)
  [text analytics] generate multiapi client, made v3.1-preview.1 default version (Azure#12720)
  Prevent AttributeError during get_certificate_version (Azure#12747)
  re-apply updates after resetting master (Azure#12771)
  Sync eng/common directory with azure-sdk-tools repository (Azure#12745)
  Enable bandit for servicebus (Azure#12763)
  Skip pypy3 for service bus CI pipeline (Azure#12732)
  Update CODEOWNERS (Azure#12765)
  updated tox file (Azure#11087)
  Updating to match Microsoft recommended template (Azure#11045)
  [text analytics] update version in master (Azure#12749)
  Explicitly stringify message prints in samples per manual doc pass recommendation. (Azure#12709)
  Run only analyse dependency step for aggregate report (Azure#12728)
  VSCodeCredential supports Python 2.7 on Windows (Azure#12731)
  Enable bandit (Azure#12722)
  Move InteractiveCredential to a new module (Azure#12706)
  Add foundations for eng/common restructure (Azure#12725)
  Sync eng/common directory with azure-sdk-tools repository (Azure#12730)
  fix flaky test (Azure#12721)
  python link verification (Azure#12543)
  [ServiceBus] Remove legacy control client (Azure#12644)
  ...

Author: iscai-msft

.github/CODEOWNERS

@@ -9,36 +9,83 @@
 /sdk/ @mayurid @lmazuel
 
 # Core
+# PRLabel: %Azure.Core
 /sdk/core/                                                           @lmazuel @xiangyan99 @johanste @rakshith91
 /sdk/core/azure-core/                                                @xiangyan99 @lmazuel
 
+####
 # Service team
+####
+# PRLabel: %Azure.Identity
 /sdk/identity/                                                       @chlowell @schaabs
+
+# PRLabel: %Event Hubs
 /sdk/eventhub/                                                       @annatisch @yunhaoling @YijunXieMS
-/sdk/storage/                                                        @amishra-dev @zezha-msft @annatisch @rakshith91 @xiafu-msft
-/sdk/applicationinsights/                                            @alexeldeib
+
+# PRLabel: %Storage
+/sdk/storage/                                                        @amishra-dev @zezha-msft @annatisch @rakshith91 @xiafu-msft @kasobol-msft
+
+/sdk/applicationinsights/                                            @alexeldeib      
+
+# PRLabel: %Batch
 /sdk/batch/                                                          @bgklein @xingwu1
-/sdk/cognitiveservices/azure-cognitiveservices-vision-customvision/  @areddish
+/sdk/cognitiveservices/azure-cognitiveservices-vision-customvision/  @areddish          
+
+# PRLabel: %KeyVault
 /sdk/keyvault/                                                       @schaabs @chlowell @iscai-msft
+
+# PRLabel: %Monitor - LogAnalytics
 /sdk/loganalytics/                                                   @alexeldeib
+
+# PRLabel: %Consumption
 /sdk/consumption/                                                    @sandeepnl
+
+# PRLabel: %Container Instances
 /sdk/containerinstance/                                              @samkreter @xizhamsft
+
+# PRLabel: %Container Registry
 /sdk/containerregistry/                                              @djyou
 /sdk/containerservice/                                               @samkreter @zqingqing1 @GaneshaThirumurthi
+
+# PRLabel: %Cosmos
 /sdk/cosmos/                                                         @southpolesteve @zfoster
+
+# PRLabel: %Data Factory
 /sdk/datafactory/                                                    @hvermis
-/sdk/datalake/                                                       @ro-joowan
+/sdk/datalake/                                                       @ro-joowan                                                     
 /sdk/datadatamigration/                                              @vchske
+
+# PRLabel: %Event Grid
 /sdk/eventgrid/                                                      @kalyanaj
+
+# PRLabel: %HDInsight
 /sdk/hdinsight/                                                      @idear1203
+
+# PRLabel: %Machine Learning Compute
 /sdk/machinelearningcompute/                                         @shutchings
+
+# PRLabel: %Recovery Services
 /sdk/recoveryservices/                                               @DheerendraRathor
+
+# PRLabel: %Search
 /sdk/search/                                                         @xiangyan99 @rakshith91
+
+# PRLabel: %Service Fabric
 /sdk/servicefabric/                                                  @QingChenmsft @samedder
+
+# PRLabel: %SQL
 /sql/sql/                                                            @jaredmoo
+
+# PRLabel: %Service Bus
 /sdk/servicebus/                                                     @annatisch @yunhaoling @YijunXieMS @KieranBrantnerMagee
+
+# PRLabel: %Synapse
 /sdk/synapse/                                                        @aim-for-better @idear1203
+
+# PRLabel: %Cognitive - Text Analytics
 /sdk/textanalytics/                                                  @kristapratico @iscai-msft
+
+# PRLabel: %Cognitive - Form Recognizer
 /sdk/formrecognizer/                                                 @kristapratico @iscai-msft @rakshith91
 
 # Smoke Tests

NOTICE.txt

@@ -1,7 +1,17 @@
-This file list any third-party libraries or other resources that may be
-distributed under licenses different than the Azure SDK for Python software.
+NOTICES AND INFORMATION
+Do Not Translate or Localize
 
-In the event that we accidentally failed to list a required notice, please
-bring it to our attention by opening an issue.
+This software incorporates material from third parties. Microsoft makes certain
+open source code available at https://3rdpartysource.microsoft.com, or you may
+send a check or money order for US $5.00, including the product name, the open
+source component name, and version number, to:
 
-The attached notices are provided for information only.
+Source Code Compliance Team
+Microsoft Corporation
+One Microsoft Way
+Redmond, WA 98052
+USA
+
+Notwithstanding any other terms, you may reverse engineer this software to the
+extent required to debug changes to any libraries licensed under the GNU Lesser
+General Public License.

README.md

@@ -47,7 +47,7 @@ Management libraries can be identified by namespaces that start with `azure-mgmt
 ## Need help?
 
 * For detailed documentation visit our [Azure SDK for Python documentation](https://aka.ms/python-docs)
-* File an issue via [Github Issues](../../issues)
+* File an issue via [Github Issues](https://github.com/Azure/azure-sdk-for-python/issues)
 * Check [previous questions](https://stackoverflow.com/questions/tagged/azure+python) or ask new ones on StackOverflow using `azure` and `python` tags.
 
 ### Community

doc/dev/mgmt/swagger/multi_api/readme.md

@@ -8,7 +8,7 @@ This is the AutoRest configuration file for Network.
 
 ## Getting Started
 
-To build the SDK for Network, simply [Install AutoRest](https://aka.ms/autorest/install) and in this folder, run:
+To build the SDK for Network, simply [Install AutoRest](https://github.com/Azure/autorest/blob/master/docs/installing-autorest.md) and in this folder, run:
 
 > `autorest`
 
@@ -66,4 +66,4 @@ swagger-to-sdk:
 
 ## Python
 
-See configuration in [readme.python.md](./readme.python.md)
+See configuration in [readme.python.md](https://github.com/Azure/azure-sdk-for-python/blob/master/doc/dev/mgmt/swagger/multi_api/readme.python.md)

doc/dev/mgmt/swagger/single_api/readme.md

@@ -8,7 +8,7 @@ This is the AutoRest configuration file for Cdn.
 
 ---
 ## Getting Started
-To build the SDK for Cdn, simply [Install AutoRest](https://aka.ms/autorest/install) and in this folder, run:
+To build the SDK for Cdn, simply [Install AutoRest](https://github.com/Azure/autorest/blob/master/docs/installing-autorest.md) and in this folder, run:
 
 > `autorest`
 

doc/dev/mgmt/swagger_conf.md

@@ -18,38 +18,38 @@ Writing the readme is the responsibility of the Python SDK team. There is curren
 - Readme that handles only one API version, and generates packages that handle one API version only
 - Readme that handles several API versions, and generates packages with multiples API and profile supports
 
-These templates can be found in the [single_api](./single_api) and the [multi_api](./multi_api) folders.
+These templates can be found in the [single_api](https://github.com/Azure/azure-sdk-for-python/blob/master/doc/dev/mgmt/swagger/single_api) and the [multi_api](https://github.com/Azure/azure-sdk-for-python/blob/master/doc/dev/mgmt/swagger/multi_api) folders.
 
 
 ### Single API readmes
 
 This one is the most simple:
-- Copy the [readme.python.md](./single_api/readme.python.md) and replace `servicetoreplace` by your service name
-- Be sure the main [readme.md](./single_api/readme.md) contains a "swagger-to-sdk" section with Python
+- Copy the [readme.python.md](https://github.com/Azure/azure-sdk-for-python/blob/master/doc/dev/mgmt/swagger/single_api/readme.python.md) and replace `servicetoreplace` by your service name
+- Be sure the main [readme.md](https://github.com/Azure/azure-sdk-for-python/blob/master/doc/dev/mgmt/swagger/single_api/readme.md) contains a "swagger-to-sdk" section with Python
 
 ### Multi API readmes
 
-When doing multi-api packages, it means you have shipping several "tags" of the main readme as one package. Autorest is calling this process a "batch" call, and this is the purpose of the "batch" section in [readme.python.md](./multi_api/readme.python.md).
+When doing multi-api packages, it means you have shipping several "tags" of the main readme as one package. Autorest is calling this process a "batch" call, and this is the purpose of the "batch" section in [readme.python.md](https://github.com/Azure/azure-sdk-for-python/blob/master/doc/dev/mgmt/swagger/multi_api/readme.python.md).
 
 In order to be sure the correct tags exist, you can use the following script:
 ```shell
 python ./scripts/multi_api_readme_help.py /azure-rest-api-specs/specification/service/resource-manager/
 ```
 
 This script will analyze the Swaggers available, and suggests on stdout:
-- A list of tags for the main [readme.md](./multi_api/readme.md)
-- A batch declaration for the [readme.python.md](./multi_api/readme.python.md)
+- A list of tags for the main [readme.md](https://github.com/Azure/azure-sdk-for-python/blob/master/doc/dev/mgmt/swagger/multi_api/readme.md)
+- A batch declaration for the [readme.python.md](https://github.com/Azure/azure-sdk-for-python/blob/master/doc/dev/mgmt/swagger/multi_api/readme.python.md)
 
 This script is not perfect that it *does* require manual review of the output and not a direct copy/paste.
 
 It's important for Python that tags represents only *ONE* unique API version. It's why it's pretty common that Python uses a set of tags that other languages don't use.
 
 Once you know the list of tags you need to generate:
 
-- Copy the [readme.python.md](./multi_api/readme.python.md) and replace `servicetoreplace` by your service name
-- Update the batch list of [readme.python.md](./multi_api/readme.python.md)
-- Be sure you have one tag section for each batch entry in [readme.python.md](./multi_api/readme.python.md)
-- Be sure the main [readme.md](./multi_api/readme.md) contains a "swagger-to-sdk" section with Python with an `afterscripts` section like the one in the template.
+- Copy the [readme.python.md](https://github.com/Azure/azure-sdk-for-python/blob/master/doc/dev/mgmt/swagger/multi_api/readme.python.md) and replace `servicetoreplace` by your service name
+- Update the batch list of [readme.python.md](https://github.com/Azure/azure-sdk-for-python/blob/master/doc/dev/mgmt/swagger/multi_api/readme.python.md)
+- Be sure you have one tag section for each batch entry in [readme.python.md](https://github.com/Azure/azure-sdk-for-python/blob/master/doc/dev/mgmt/swagger/multi_api/readme.python.md)
+- Be sure the main [readme.md](https://github.com/Azure/azure-sdk-for-python/blob/master/doc/dev/mgmt/swagger/multi_api/readme.md) contains a "swagger-to-sdk" section with Python with an `afterscripts` section like the one in the template.
 
 The `afterscripts` will execute a Jinja template to create a client to link together all the batch generated autorest ([example](https://github.com/Azure/azure-sdk-for-python/blob/4a7c67189591b052fe2b5769847ff68f7845386d/sdk/storage/azure-mgmt-storage/azure/mgmt/storage/_storage_management_client.py))
 

eng/common/TestResources/New-TestResources.ps1

@@ -154,14 +154,21 @@ if ($ProvisionerApplicationId) {
     $subscriptionArgs = if ($SubscriptionId) {
         @{SubscriptionId = $SubscriptionId}
     }
+    else {
+        @{}
+    }
 
     $provisionerAccount = Retry {
         Connect-AzAccount -Force:$Force -Tenant $TenantId -Credential $provisionerCredential -ServicePrincipal -Environment $Environment @subscriptionArgs
     }
 
     $exitActions += {
         Write-Verbose "Logging out of service principal '$($provisionerAccount.Context.Account)'"
-        $null = Disconnect-AzAccount -AzureContext $provisionerAccount.Context
+
+        # Only attempt to disconnect if the -WhatIf flag was not set.  Otherwise, this call is not necessary and will fail.
+        if ($PSCmdlet.ShouldProcess($ProvisionerApplicationId)) {
+            $null = Disconnect-AzAccount -AzureContext $provisionerAccount.Context
+        }
     }
 }
 
@@ -176,7 +183,6 @@ if ($TestApplicationId -and !$TestApplicationOid) {
     }
 }
 
-
 # If the ServiceDirectory is an absolute path use the last directory name
 # (e.g. D:\foo\bar\ -> bar)
 $serviceName = if (Split-Path -IsAbsolute  $ServiceDirectory) {
@@ -229,6 +235,13 @@ if ($resourceGroup.ProvisioningState -eq 'Succeeded') {
     # New-AzResourceGroup would've written an error and stopped the pipeline by default anyway.
     Write-Verbose "Successfully created resource group '$($resourceGroup.ResourceGroupName)'"
 }
+elseif (($resourceGroup -eq $null) -and (-not $PSCmdlet.ShouldProcess($resourceGroupName))) {
+    # If the -WhatIf flag was passed, there will be no resource group created.  Fake it.
+    $resourceGroup = [PSCustomObject]@{
+        ResourceGroupName = $resourceGroupName
+        Location = $Location
+    }
+}
 
 # Populate the template parameters and merge any additional specified.
 $templateParameters = @{
@@ -299,6 +312,9 @@ foreach ($templateFile in $templateFiles) {
         "$($serviceDirectoryPrefix)RESOURCE_GROUP" = $resourceGroup.ResourceGroupName;
         "$($serviceDirectoryPrefix)LOCATION" = $resourceGroup.Location;
         "$($serviceDirectoryPrefix)ENVIRONMENT" = $context.Environment.Name;
+        "$($serviceDirectoryPrefix)AZURE_AUTHORITY_HOST" = $context.Environment.ActiveDirectoryAuthority;
+        "$($serviceDirectoryPrefix)RESOURCE_MANAGER_URL" = $context.Environment.ResourceManagerUrl;
+        "$($serviceDirectoryPrefix)SERVICE_MANAGEMENT_URL" = $context.Environment.ServiceManagementUrl;
     }
 
     foreach ($key in $deployment.Outputs.Keys) {
@@ -331,7 +347,7 @@ foreach ($templateFile in $templateFiles) {
     }
     else
     {
-        
+
         if (!$CI) {
             # Write an extra new line to isolate the environment variables for easy reading.
             Log "Persist the following environment variables based on your detected shell ($shell):`n"
@@ -340,7 +356,7 @@ foreach ($templateFile in $templateFiles) {
         foreach ($key in $deploymentOutputs.Keys)
         {
             $value = $deploymentOutputs[$key]
-            
+
             if ($CI) {
                 # Treat all ARM template output variables as secrets since "SecureString" variables do not set values.
                 # In order to mask secrets but set environment variables for any given ARM template, we set variables twice as shown below.

eng/common/TestResources/deploy-test-resources.yml

@@ -13,7 +13,7 @@ parameters:
 #   "TestApplicationId": "<test app id>",
 #   "TestApplicationSecret": "<test app secret>",
 #   "ProvisionerApplicationId": "<provisoner app id>",
-#   "ProvisoinerApplicationSecret": "<provisoner app secert>",
+#   "ProvisionerApplicationSecret": "<provisoner app secert>",
 #   "Environment": "AzureCloud | AzureGov | AzureChina | <other environment>"
 # }
 

eng/common/TestResources/remove-test-resources.yml

@@ -32,5 +32,5 @@ steps:
         -Force `
         -Verbose
     displayName: Remove test resources
-    condition: and(ne(variables['AZURE_RESOURCEGROUP_NAME'], ''), succeededOrFailed())
+    condition: ne(variables['AZURE_RESOURCEGROUP_NAME'], '')
     continueOnError: true

eng/pipelines/templates/steps/cosmos-emulator.yml renamed to eng/common/pipelines/templates/steps/cosmos-emulator.yml

@@ -12,6 +12,8 @@ parameters:
   WorkingDirectory: $(System.DefaultWorkingDirectory)
   PRTitle: not-specified
   ScriptDirectory: eng/common/scripts
+  GHReviewersVariable: '' 
+  GHTeamReviewersVariable: ''
 
 steps:
 
@@ -63,3 +65,18 @@ steps:
       -PRBranch "${{ parameters.PRBranchName }}"
       -AuthToken "$(azuresdk-github-pat)"
       -PRTitle "${{ parameters.PRTitle }}"
+
+- task: PowerShell@2
+  displayName: Tag a Reviewer on PR
+  condition: and(succeeded(), eq(variables['HasChanges'], 'true'))
+  inputs:
+    pwsh: true
+    workingDirectory: ${{ parameters.WorkingDirectory }}
+    filePath: ${{ parameters.ScriptDirectory }}/add-pullrequest-reviewers.ps1
+    arguments: >
+      -RepoOwner "${{ parameters.RepoOwner }}"
+      -RepoName "${{ parameters.RepoName }}"
+      -AuthToken "$(azuresdk-github-pat)"
+      -GitHubUsers "$(${{ parameters.GHReviewersVariable }})"
+      -GitHubTeams "$(${{ parameters.GHTeamReviewersVariable }})"
+      -PRNumber "$(Submitted.PullRequest.Number)"

eng/common/pipelines/templates/steps/create-pull-request.yml

@@ -12,6 +12,8 @@ parameters:
   ArtifactName: ''
   Language: ''
   DocRepoDestinationPath: '' #usually docs-ref-services/
+  GHReviewersVariable: '' 
+  GHTeamReviewersVariable: '' # externally set, as eng-common does not have the identity-resolver. Run as pre-step
 
 steps:
 - pwsh: |
@@ -56,3 +58,5 @@ steps:
     BaseBranchName: smoke-test
     WorkingDirectory: ${{ parameters.WorkingDirectory }}/repo
     ScriptDirectory: ${{ parameters.WorkingDirectory }}/${{ parameters.ScriptDirectory }}
+    GHReviewersVariable: ${{ parameters.GHReviewersVariable }}
+    GHTeamReviewersVariable: ${{ parameters.GHTeamReviewersVariable }} 

eng/common/pipelines/templates/steps/docs-metadata-release.yml

@@ -0,0 +1,46 @@
+parameters: 
+  TargetVariable: ''
+  ServiceDirectory: ''
+
+steps:
+  - pwsh: |
+      git clone https://github.com/Azure/azure-sdk-tools.git $(Build.SourcesDirectory)/tools_repo
+      cd $(Build.SourcesDirectory)/tools_repo
+      git checkout 564ad63ae72d18422533fa1da9d396e7703c1cb5
+    displayName: Setup Identity Resolver
+
+  - pwsh: |
+      $result = dotnet run -v q -- `
+        --aad-app-id-var APP_ID `
+        --aad-app-secret-var APP_SECRET `
+        --aad-tenant-var AAD_TENANT `
+        --kusto-url-var KUSTO_URL `
+        --kusto-database-var KUSTO_DB `
+        --kusto-table-var KUSTO_TABLE `
+        --identity "$(Build.QueuedBy)"
+      $resolvedIdentity = $result[-1] | ConvertFrom-Json
+
+      Write-Host $resolvedIdentity
+
+      Write-Output "##vso[task.setvariable variable=${{ parameters.TargetVariable }}]$($resolvedIdentity.GithubUserName)"
+    displayName: 'Resolving Queuing User'
+    workingDirectory: $(Build.SourcesDirectory)/tools_repo/tools/notification-configuration/identity-resolver
+    env:
+      APP_ID: $(notification-aad-app-id)
+      APP_SECRET: $(notification-aad-secret)
+      AAD_TENANT: $(notification-aad-tenant)
+      KUSTO_URL: $(notification-kusto-url)
+      KUSTO_DB: $(notification-kusto-db)
+      KUSTO_TABLE: $(notification-kusto-table)
+
+  - pwsh: |
+      Remove-Item -Force -Recurse $(Build.SourcesDirectory)/tools_repo
+    displayName: Clean Up Cloned Tools Repo
+
+  - pwsh: |
+      $originalValue = "$(${{ parameters.TargetVariable }})"
+      $result = $(Build.SourcesDirectory)/eng/common/scripts/get-codeowners.ps1 -TargetDirectory /sdk/${{ parameters.ServiceDirectory }}/ -RootDirectory $(Build.SourcesDirectory)
+      if ($result) {
+        Write-Output "##vso[task.setvariable variable=${{ parameters.TargetVariable }}]$originalValue,$result"
+      }
+    displayName: Add CodeOwners if Present