Merge branch 'master' of https://github.com/Azure/azure-sdk-for-python into make_canary_test

* 'master' of https://github.com/Azure/azure-sdk-for-python: (28 commits)
  Kaihuis maps (Azure#11574)
  [formrecognizer] support Copy API (Azure#11372)
  Artifact Powered Docs.MS Release (Azure#11395)
  Sync eng/common directory with azure-sdk-tools repository (Azure#11562)
  Updated changelog (Azure#11554)
  update pinned versions in autorest_req.txt (Azure#11557)
  Add user authentication API to UsernamePasswordCredential (Azure#11528)
  Reduce Build Matrix (Azure#11539)
  Prevent Key Vault test failure due to operation timing (Azure#11552)
  update tests (Azure#11534)
  Datashare 2019 11 01 (Azure#11540)
  Search Renames / Regen (Azure#11342)
  update unit test according to the latest uamqp update (Azure#11533)
  Sync eng/common directory with azure-sdk-tools repository (Azure#11472)
  rename input parameters (Azure#11518)
  [cosmos] readme review feedback (Azure#11527)
  Update CODEOWNERS (Azure#11516)
  [Core] Support multipart changesets (Azure#10972)
  fix AttributeException (Azure#11463)
  release-for-hanaonazure-mgmt (Azure#11441)
  ...

Author: iscai-msft

.github/CODEOWNERS

@@ -10,7 +10,7 @@
 
 # Core
 /sdk/core/                                                           @lmazuel @xiangyan99 @johanste
-/sdk/core/azure-core/                                                @xiangyan99 @bryevdv @lmazuel
+/sdk/core/azure-core/                                                @xiangyan99 @lmazuel
 
 # Service team
 /sdk/identity/                                                       @chlowell @schaabs

common/smoketest/smoke-test.yml

@@ -9,58 +9,58 @@ jobs:
           PythonVersion: '2.7'
           InstallAsyncRequirements: false
           OSVmImage: ubuntu-18.04
-          CloudType: AzureCloud
+          SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
           ArmTemplateParameters: $(azureCloudArmParameters)
         Python_37_Linux (AzureCloud):
           PythonVersion: '3.7'
           OSVmImage: ubuntu-18.04
-          CloudType: AzureCloud
+          SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
           ArmTemplateParameters: $(azureCloudArmParameters)
         Python_38_Linux (AzureCloud):
           PythonVersion: '3.8'
           OSVmImage: ubuntu-18.04
-          CloudType: AzureCloud
+          SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
           ArmTemplateParameters: $(azureCloudArmParameters)
         Python_37_Windows (AzureCloud):
           PythonVersion: '3.7'
           OSVmImage: windows-2019
-          CloudType: AzureCloud
+          SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
           ArmTemplateParameters: $(azureCloudArmParameters)
         Python_38_Windows (AzureCloud):
           PythonVersion: '3.8'
           OSVmImage: windows-2019
-          CloudType: AzureCloud
+          SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
           ArmTemplateParameters: $(azureCloudArmParameters)
         Python_37_Mac (AzureCloud):
           PythonVersion: '3.7'
           OSVmImage: macOS-10.15
-          CloudType: AzureCloud
+          SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
           ArmTemplateParameters: $(azureCloudArmParameters)
         Python_38_Mac (AzureCloud):
           PythonVersion: '3.8'
           OSVmImage: macOS-10.15
-          CloudType: AzureCloud
+          SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
           ArmTemplateParameters: $(azureCloudArmParameters)
         Python_38_Linux (AzureUSGovernment):
           PythonVersion: '3.8'
           OSVmImage: ubuntu-18.04
-          CloudType: AzureUSGovernment
+          SubscriptionConfiguration: $(sub-config-gov-test-resources)
           ArmTemplateParameters: $(azureUSGovernmentArmParameters)
         Python_37_Windows (AzureUSGovernment):
           PythonVersion: '3.7'
           OSVmImage: windows-2019
-          CloudType: AzureUSGovernment
+          SubscriptionConfiguration: $(sub-config-gov-test-resources)
           ArmTemplateParameters: $(azureUSGovernmentArmParameters)
         Python_38_Linux (AzureChinaCloud):
           PythonVersion: '3.8'
           OSVmImage: ubuntu-18.04
-          CloudType: AzureChinaCloud
+          SubscriptionConfiguration: $(sub-config-cn-test-resources)
           Location: 'chinanorth'
           ArmTemplateParameters: $(azureChinaCloudArmParameters)
         Python_37_Windows (AzureChinaCloud):
           PythonVersion: '3.7'
           OSVmImage: windows-2019
-          CloudType: AzureChinaCloud
+          SubscriptionConfiguration: $(sub-config-cn-test-resources)
           Location: 'chinanorth'
           ArmTemplateParameters: $(azureChinaCloudArmParameters)
 
@@ -103,7 +103,7 @@ jobs:
       - template: /eng/common/TestResources/deploy-test-resources.yml
         parameters:
           ServiceDirectory: '$(Build.SourcesDirectory)/common/smoketest/'
-          CloudType: $(CloudType)
+          SubscriptionConfiguration: $(SubscriptionConfiguration)
           Location: $(Location)
           ArmTemplateParameters: $(ArmTemplateParameters)
 
@@ -113,3 +113,5 @@ jobs:
       - template: /eng/common/TestResources/remove-test-resources.yml
         parameters:
           ServiceDirectory: '$(Build.SourcesDirectory)/common/smoketest/'
+          SubscriptionConfiguration: $(SubscriptionConfiguration)
+

eng/autorest_req.txt

@@ -1,8 +1,8 @@
-pytest==5.4.1; python_version >= '3.5'
+pytest==5.4.2; python_version >= '3.5'
 pytest==4.6.9; python_version == '2.7'
 pytest-cov==2.8.1
-pytest-asyncio==0.10.0; python_version >= '3.5'
+pytest-asyncio==0.12.0; python_version >= '3.5'
 isodate==0.6.0
-msrest==0.6.13
+msrest==0.6.14
 aiohttp==3.6.2
-wheel
+wheel==0.34.2

eng/common/TestResources/remove-test-resources.yml

@@ -32,4 +32,5 @@ steps:
         -Force `
         -Verbose
     displayName: Remove test resources
+    condition: and(ne(variables['AZURE_RESOURCEGROUP_NAME'], ''), succeededOrFailed())
     continueOnError: true

eng/common/pipelines/templates/steps/docs-metadata-release.yml

@@ -55,7 +55,4 @@ steps:
     PRTitle: "Docs.MS Readme Update."
     BaseBranchName: smoke-test
     WorkingDirectory: ${{parameters.WorkingDirectory}}/repo
-    ScriptDirectory: ${{parameters.WorkingDirectory}}/${{parameters.ScriptDirectory}}
-
-
-
+    ScriptDirectory: ${{parameters.WorkingDirectory}}/${{parameters.ScriptDirectory}}

eng/common/scripts/artifact-metadata-parsing.ps1

@@ -358,7 +358,7 @@ function GetExistingTags($apiUrl) {
 }
 
 # Walk across all build artifacts, check them against the appropriate repository, return a list of tags/releases
-function VerifyPackages($pkgRepository, $artifactLocation, $workingDirectory, $apiUrl, $releaseSha, $exitOnError = $True) {
+function VerifyPackages($pkgRepository, $artifactLocation, $workingDirectory, $apiUrl, $releaseSha,  $continueOnError = $false) {
   $pkgList = [array]@()
   $ParsePkgInfoFn = ""
   $packagePattern = ""
@@ -404,7 +404,7 @@ function VerifyPackages($pkgRepository, $artifactLocation, $workingDirectory, $a
         continue
       }
 
-      if ($parsedPackage.Deployable -ne $True -and $exitOnError) {
+      if ($parsedPackage.Deployable -ne $True -and !$continueOnError) {
         Write-Host "Package $($parsedPackage.PackageId) is marked with version $($parsedPackage.PackageVersion), the version $($parsedPackage.PackageVersion) has already been deployed to the target repository."
         Write-Host "Maybe a pkg version wasn't updated properly?"
         exit(1)
@@ -430,8 +430,8 @@ function VerifyPackages($pkgRepository, $artifactLocation, $workingDirectory, $a
 
   $intersect = $results | % { $_.Tag } | ? { $existingTags -contains $_ }
 
-  if ($intersect.Length -gt 0 -and $exitOnError) {
-    CheckArtifactShaAgainstTagsList -priorExistingTagList $intersect -releaseSha $releaseSha -apiUrl $apiUrl -exitOnError $exitOnError
+  if ($intersect.Length -gt 0 -and !$continueOnError) {
+    CheckArtifactShaAgainstTagsList -priorExistingTagList $intersect -releaseSha $releaseSha -apiUrl $apiUrl -continueOnError $continueOnError
 
     # all the tags are clean. remove them from the list of releases we will publish.
     $results = $results | ? { -not ($intersect -contains $_.Tag ) }
@@ -443,7 +443,7 @@ function VerifyPackages($pkgRepository, $artifactLocation, $workingDirectory, $a
 # given a set of tags that we want to release, we need to ensure that if they already DO exist.
 # if they DO exist, quietly exit if the commit sha of the artifact matches that of the tag
 # if the commit sha does not match, exit with error and report both problem shas
-function CheckArtifactShaAgainstTagsList($priorExistingTagList, $releaseSha, $apiUrl, $exitOnError) {
+function CheckArtifactShaAgainstTagsList($priorExistingTagList, $releaseSha, $apiUrl, $continueOnError) {
   $headers = @{
     "Content-Type"  = "application/json"
     "Authorization" = "token $($env:GH_TOKEN)"
@@ -465,7 +465,7 @@ function CheckArtifactShaAgainstTagsList($priorExistingTagList, $releaseSha, $ap
     }
   }
 
-  if ($unmatchedTags.Length -gt 0 -and $exitOnError) {
+  if ($unmatchedTags.Length -gt 0 -and !$continueOnError) {
     Write-Host "Tags already existing with different SHA versions. Exiting."
     exit(1)
   }

eng/common/scripts/create-tags-and-git-release.ps1

@@ -15,7 +15,7 @@ param (
   $repoOwner = "", # the owning organization of the repository. EG "Azure"
   $repoName = "", # the name of the repository. EG "azure-sdk-for-java"
   $repoId = "$repoOwner/$repoName", # full repo id. EG azure/azure-sdk-for-net  DevOps: $(Build.Repository.Id),
-  [switch]$forceCreate = $false
+  [switch]$continueOnError = $false
 )
 
 Write-Host "> $PSCommandPath $args"
@@ -26,7 +26,7 @@ $apiUrl = "https://api.github.com/repos/$repoId"
 Write-Host "Using API URL $apiUrl"
 
 # VERIFY PACKAGES
-$pkgList = VerifyPackages -pkgRepository $packageRepository -artifactLocation $artifactLocation -workingDirectory $workingDirectory -apiUrl $apiUrl -releaseSha $releaseSha
+$pkgList = VerifyPackages -pkgRepository $packageRepository -artifactLocation $artifactLocation -workingDirectory $workingDirectory -apiUrl $apiUrl -releaseSha $releaseSha -continueOnError $continueOnError
 
 if ($pkgList) {
   Write-Host "Given the visible artifacts, github releases will be created for the following:"

eng/common/scripts/update-docs-metadata.ps1

@@ -86,7 +86,7 @@ $pkgs = VerifyPackages -pkgRepository $Repository `
   -workingDirectory $WorkDirectory `
   -apiUrl $apiUrl `
   -releaseSha $ReleaseSHA `
-  -exitOnError $False
+  -continueOnError $True
 
 if ($pkgs) {
   Write-Host "Given the visible artifacts, readmes will be copied for the following packages"

eng/pipelines/templates/jobs/archetype-sdk-client.yml

@@ -20,18 +20,6 @@ parameters:
       PythonVersion: '3.5'
       CoverageArg: ''
       RunForPR: false
-    Linux_Python36:
-      OSName: 'Linux'
-      OSVmImage: 'ubuntu-18.04'
-      PythonVersion: '3.6'
-      CoverageArg: ''
-      RunForPR: false
-    Linux_Python37:
-      OSName: 'Linux'
-      OSVmImage: 'ubuntu-18.04'
-      PythonVersion: '3.7'
-      CoverageArg: ''
-      RunForPR: false
     Linux_Python38:
       OSName: 'Linux'
       OSVmImage: 'ubuntu-18.04'
@@ -90,6 +78,7 @@ jobs:
       parameters: 
         ServiceDirectory: ${{ parameters.ServiceDirectory }}
         BuildTargetingString: ${{ parameters.BuildTargetingString }}
+        TestMarkArgument: ${{ parameters.TestMarkArgument }}
 
   - job: 'Test'
     condition: and(succeededOrFailed(), ne(variables['Skip.Test'], 'true'))
@@ -146,44 +135,6 @@ jobs:
               ServiceDirectory: ${{ parameters.ServiceDirectory }}
               BuildTargetingString: ${{ parameters.BuildTargetingString }}
 
-  - job: 'RunMyPy'
-    condition: and(succeededOrFailed(), ne(variables['Skip.MyPy'], 'true'))
-    displayName: 'Run MyPy'
-    variables:
-    - template: ../variables/globals.yml
-
-    dependsOn:
-      - 'Build'
-
-    pool:
-      vmImage: 'ubuntu-18.04'
-
-    steps:
-    - template: ../steps/run_mypy.yml
-      parameters: 
-        ServiceDirectory: ${{ parameters.ServiceDirectory }}
-        BuildTargetingString: ${{ parameters.BuildTargetingString }}
-        TestMarkArgument: ${{ parameters.TestMarkArgument }}
-
-  - job: 'RunPyLint'
-    condition: and(succeededOrFailed(), ne(variables['Skip.Pylint'], 'true'))
-    displayName: 'Run Pylint'
-    variables:
-    - template: ../variables/globals.yml
-
-    dependsOn:
-      - 'Build'
-
-    pool:
-      vmImage: 'ubuntu-18.04'
-
-    steps:
-    - template: ../steps/run_pylint.yml
-      parameters: 
-        ServiceDirectory: ${{ parameters.ServiceDirectory }}
-        BuildTargetingString: ${{ parameters.BuildTargetingString }}
-        TestMarkArgument: ${{ parameters.TestMarkArgument }}
-
   - job: 'RunRegression'
     condition: and(succeededOrFailed(), or(eq(variables['Run.Regression'], 'true'), and(eq(variables['Build.Reason'], 'Schedule'), eq(variables['System.TeamProject'],'internal'))))
     displayName: 'Run Regression'

eng/pipelines/templates/stages/archetype-python-release.yml

@@ -4,6 +4,8 @@ parameters:
   DependsOn: Build
   DocArtifact: 'documentation'
   DevFeedName: public/azure-sdk-for-python
+  TargetDocRepoOwner: ''
+  TargetDocRepoName: ''
 
 stages:
   - ${{if and(eq(variables['Build.Reason'], 'Manual'), eq(variables['System.TeamProject'], 'internal'))}}:
@@ -26,15 +28,15 @@ stages:
                 deploy:
                   steps:
                     - checkout: self
+                    - template: /eng/pipelines/templates/steps/stage-filtered-artifacts.yml
+                      parameters:
+                        SourceFolder: ${{parameters.ArtifactName}} 
+                        TargetFolder: ${{artifact.safeName}}
+                        PackageName: ${{artifact.name}}
                     - pwsh: |
-                        Get-ChildItem $(Pipeline.Workspace)/${{parameters.ArtifactName}}
-                        New-Item -Type Directory -Name ${{artifact.safeName}} -Path $(Pipeline.Workspace)
-                        $underscorePrefix = "${{artifact.name}}"
-                        $dashPrefix = "${{artifact.name}}".Replace("_", "-")
-                        Copy-Item $(Pipeline.Workspace)/${{parameters.ArtifactName}}/$dashPrefix-[0-9]*.[0-9]*.[0-9]* $(Pipeline.Workspace)/${{artifact.safeName}}
-                        Copy-Item $(Pipeline.Workspace)/${{parameters.ArtifactName}}/$underscorePrefix-[0-9]*.[0-9]*.[0-9]* $(Pipeline.Workspace)/${{artifact.safeName}}
-                        Get-ChildItem $(Pipeline.Workspace)/${{artifact.safeName}}
-                      displayName: Stage artifacts
+                        Get-ChildItem -Recurse $(Pipeline.Workspace)/${{artifact.safeName}}
+                      workingDirectory: $(Pipeline.Workspace)
+                      displayName: Output Visible Artifacts
                     - template: /eng/common/pipelines/templates/steps/create-tags-and-git-release.yml
                       parameters:
                         ArtifactLocation: $(Pipeline.Workspace)/${{artifact.safeName}}
@@ -62,14 +64,11 @@ stages:
                         artifact: ${{parameters.ArtifactName}}
                         timeoutInMinutes: 5
 
-                      - pwsh: |
-                          Get-ChildItem $(Pipeline.Workspace)/${{parameters.ArtifactName}}
-                          New-Item -Type Directory -Name ${{artifact.safeName}} -Path $(Pipeline.Workspace)
-                          $underscorePrefix = "${{artifact.name}}"
-                          $dashPrefix = "${{artifact.name}}".Replace("_", "-")
-                          Copy-Item $(Pipeline.Workspace)/${{parameters.ArtifactName}}/$dashPrefix-[0-9]*.[0-9]*.[0-9]* $(Pipeline.Workspace)/${{artifact.safeName}}
-                          Copy-Item $(Pipeline.Workspace)/${{parameters.ArtifactName}}/$underscorePrefix-[0-9]*.[0-9]*.[0-9]* $(Pipeline.Workspace)/${{artifact.safeName}}
-                          Get-ChildItem $(Pipeline.Workspace)/${{artifact.safeName}}
+                      - template: /eng/pipelines/templates/steps/stage-filtered-artifacts.yml
+                        parameters:
+                          SourceFolder: ${{parameters.ArtifactName}} 
+                          TargetFolder: ${{artifact.safeName}}
+                          PackageName: ${{artifact.name}}
 
                       - task: UsePythonVersion@0
 
@@ -105,7 +104,7 @@ stages:
                         displayName: 'Publish package to feed: ${{parameters.DevFeedName}}'
 
           - ${{if ne(artifact.options.skipPublishDocs, 'true')}}:
-            - deployment: PublishDocs
+            - deployment: PublishGitHubIODocs
               displayName: Publish Docs to GitHubIO Blob Storage
               condition: and(succeeded(), ne(variables['Skip.PublishDocs'], 'true'))
               environment: githubio
@@ -119,12 +118,12 @@ stages:
                   deploy:
                     steps:
                       - checkout: self
-                      - pwsh: |
-                          Get-ChildItem $(Pipeline.Workspace)/${{parameters.DocArtifact}}
-                          New-Item -Type Directory -Name ${{artifact.safeName}} -Path $(Pipeline.Workspace)
-                          $dashPrefix = "${{artifact.name}}".Replace("_", "-")
-                          Copy-Item $(Pipeline.Workspace)/${{parameters.DocArtifact}}/$dashPrefix.zip $(Pipeline.Workspace)/${{artifact.safeName}}
-                        displayName: Stage artifacts
+                      - template: /eng/pipelines/templates/steps/stage-filtered-artifacts.yml
+                        parameters:
+                          SourceFolder: ${{parameters.DocArtifact}}
+                          TargetFolder: ${{artifact.safeName}}
+                          PackageName: ${{artifact.name}}
+                          AdditionalRegex: '.zip'
                       - pwsh: |
                           Get-ChildItem -Recurse $(Pipeline.Workspace)/${{artifact.safeName}}
                         workingDirectory: $(Pipeline.Workspace)
@@ -138,6 +137,46 @@ stages:
                           # we override the regular script path because we have cloned the build tools repo as a separate artifact.
                           ScriptPath: 'eng/common/scripts/copy-docs-to-blobstorage.ps1'
 
+          - ${{if ne(artifact.options.skipPublishDocs, 'true')}}:
+            - deployment: PublishDocs
+              displayName: "Docs.MS Release"
+              condition: and(succeeded(), ne(variables['Skip.PublishDocs'], 'true'))
+              environment: githubio
+              dependsOn: PublishPackage
+
+              pool:
+                vmImage: ubuntu-18.04
+
+              strategy:
+                runOnce:
+                  deploy:
+                    steps:
+                      - checkout: self
+                      - template: /eng/pipelines/templates/steps/stage-filtered-artifacts.yml
+                        parameters:
+                          SourceFolder: ${{parameters.ArtifactName}} 
+                          TargetFolder: ${{artifact.safeName}}
+                          PackageName: ${{artifact.name}}
+                      - pwsh: |
+                          Get-ChildItem -Recurse $(Pipeline.Workspace)/${{artifact.safeName}}
+                        workingDirectory: $(Pipeline.Workspace)
+                        displayName: Output Visible Artifacts
+                      - template: /eng/common/pipelines/templates/steps/docs-metadata-release.yml
+                        parameters:
+                          ArtifactLocation: $(Pipeline.Workspace)/${{artifact.safeName}}
+                          PackageRepository: PyPI
+                          ReleaseSha: $(Build.SourceVersion)
+                          RepoId: Azure/azure-sdk-for-python
+                          WorkingDirectory: $(System.DefaultWorkingDirectory)
+                          TargetDocRepo: 'MicrosoftDocs/azure-docs-sdk-python'
+                          TargetDocRepoOwner: ${{parameters.TargetDocRepoOwner}}
+                          TargetDocRepoName: ${{parameters.TargetDocRepoName}}
+                          PRBranchName: 'smoke-test-rdme'
+                          ArtifactName: ${{parameters.ArtifactName}}
+                          Language: 'python'
+                          ServiceDirectory: ${{ parameters.ServiceDirectory }}
+                          DocRepoDestinationPath: 'docs-ref-services/' 
+
           - ${{if ne(artifact.options.skipUpdatePackageVersion, 'true')}}:
             - deployment: UpdatePackageVersion
               displayName: "Update Package Version"