Merge branch 'master' of https://github.com/Azure/azure-sdk-for-python into python-track2-test

Author: msyyc

.github/CODEOWNERS

@@ -128,7 +128,7 @@
 /common/smoketest/                                                   @lmazuel @chlowell @annatisch @rakshith91 @shurd @southpolesteve
 
 # Management Plane
-/**/*mgmt*/     @00Kai0 @msyyc @jsntcy @changlong-liu
+/**/*mgmt*/                                                          @RAY-316 @msyyc 
 
 
 ###########

SUPPORT.md

@@ -0,0 +1,29 @@
+# Support
+
+## How to file issues and get help  
+
+Customers with an [Azure support plan](https://azure.microsoft.com/support/options/) can open an [Azure support ticket](https://azure.microsoft.com/support/create-ticket/).
+**We recommend this option if your problem requires immediate attention.** 
+
+### Github issues
+We use [GitHub Issues](https://github.com/Azure/azure-sdk-for-python/issues/new/choose) to track bugs, questions, and feature requests. 
+GitHub issues are free, but **response time is not guaranteed.** See [GitHub issues support process](https://devblogs.microsoft.com/azure-sdk/github-issue-support-process/) for more details.
+
+### Community resources
+- Search for similar issues in [our GitHub repository](https://github.com/Azure/azure-sdk-for-python/issues)
+- Ask a question on [StackOverflow](https://stackoverflow.com/questions/tagged/azure-sdk-python) and tag it with azure-sdk-python
+- Share or upvote feature requests on [Feedback Page](https://feedback.azure.com/forums/34192--general-feedback).
+- Take a look at the [Azure SDK blog](https://devblogs.microsoft.com/azure-sdk/).
+- Chat with other community members on [gitter](https://gitter.im/Azure/azure-sdk-for-python?source=orgpage)
+- Ask a question on [Twitter](https://twitter.com/AzureSDK)
+- Ask a question at [Microsoft Q&A](https://docs.microsoft.com/answers/products/azure?WT.mc_id=Portal-Microsoft_Azure_Support&product=all)
+- Ask a question at [Microsoft Tech Community](https://techcommunity.microsoft.com/t5/azure/ct-p/Azure)
+
+### Security bugs
+Security issues and bugs should be reported privately, via email, to the Microsoft Security Response Center([email protected]). 
+You should receive a response within 24 hours. 
+Further information, including the MSRC PGP key, can be found in the [Security TechCenter](https://www.microsoft.com/msrc/faqs-report-an-issue?rtc=1)
+
+## Microsoft Support Policy
+
+Please refer to [Azure SDK Support and Lifecycle information](https://azure.github.io/azure-sdk/policies_support.html)

doc/dev/conda-builds.md

@@ -4,6 +4,8 @@
 
 Follow the instructions [here](https://docs.conda.io/projects/conda-build/en/latest/install-conda-build.html) to install `conda` and `conda-build`.
 
+**The Azure SDK Conda artifacts support `python3.8` and `python3.9` only.**
+
 ## CI Build Process
 
 There will be a `CondaArtifact` defined in the `ci.yml` of each service directory. (`sdk/<service>`)
@@ -15,9 +17,18 @@ A Conda Artifact defines:
 - Any other necessary details.
 
 ## How to Build an Azure SDK Conda Package Locally
+#### If using powershell, you will need to prep your environment before proceeding to the next step
+
+```
+powershell -ExecutionPolicy ByPass -NoExit -Command "& '<path-to-conda-folder>\shell\condabin\conda-hook.ps1' ; conda activate '<path-to-conda-folder>' "
+```
+
+Afterwards, invoke `conda init powershell` and re-create the pshell session.
 
+By default, your powershell environment will now load `conda`. If you want pure pip, you will need to use explicit invocations of your `python` locations to create virtual envs.
 ### Set up your conda environment
 
+
 You will notice that all the azure-sdk conda distributions have the **same** version number and requirement set. This is due to the fact that the azure-sdk team pushes our conda packages out in waves. To support this, all versions are set via a common environment variable `AZURESDK_CONDA_VERSION`.
 
 We keep this environment variable set properly across all our builds by using a common `conda_env.yml` when creating our build environment. This environment definition ensures that: 

eng/common/pipelines/templates/steps/docindex.yml renamed to eng/common/pipelines/templates/jobs/docindex.yml

@@ -1,24 +1,65 @@
 # intended to be used as part of a release process
 parameters:
-  ArtifactLocation: 'not-specified'
-  PackageRepository: 'not-specified'
-  ReleaseSha: 'not-specified'
-  RepoId: $(Build.Repository.Name)
-  WorkingDirectory: ''
-  ScriptDirectory: eng/common/scripts
-  TargetDocRepoName: ''
-  TargetDocRepoOwner: ''
-  PRBranchName: 'master-rdme'
-  PRLabels: 'auto-merge'
-  ArtifactName: ''
-  Language: ''
-  DocRepoDestinationPath: '' #usually docs-ref-services/
-  CIConfigs: '[]'
-  GHReviewersVariable: '' 
-  GHTeamReviewersVariable: '' # externally set, as eng-common does not have the identity-resolver. Run as pre-step
-  OnboardingBranch: ''
-  CloseAfterOpenForTesting: false
-  SkipPackageJson: false
+  - name: ArtifactLocation
+    type: string
+    default: 'not-specified'
+  - name: PackageRepository
+    type: string
+    default: 'not-specified'
+  - name: ReleaseSha
+    type: string
+    default: 'not-specified'
+  - name: RepoId
+    type: string
+    default: $(Build.Repository.Name)
+  - name: WorkingDirectory
+    type: string
+    default: ''
+  - name: ScriptDirectory
+    type: string
+    default: eng/common/scripts
+  - name: TargetDocRepoName
+    type: string
+    default: ''
+  - name: TargetDocRepoOwner
+    type: string
+    default: ''
+  - name: PRBranchName
+    type: string
+    default: 'master-rdme'
+  - name: PRLabels
+    type: string
+    default: 'auto-merge'
+  - name: ArtifactName
+    type: string
+    default: ''
+  - name: Language
+    type: string
+    default: ''
+  - name: DocRepoDestinationPath
+    type: string
+    default: '' #usually docs-ref-services/
+  - name: CIConfigs
+    type: string
+    default: '[]'
+  - name: GHReviewersVariable
+    type: string
+    default: ''
+  - name: GHTeamReviewersVariable
+    type: string
+    default: ''  # externally set, as eng-common does not have the identity-resolver. Run as pre-step
+  - name: OnboardingBranch
+    type: string
+    default: ''
+  - name: CloseAfterOpenForTesting
+    type: boolean
+    default: false
+  - name: SkipPackageJson
+    type: object
+    default: false
+  - name: SparseCheckoutPaths
+    type: object
+    default: null
 
 steps:
 - pwsh: |
@@ -30,14 +71,25 @@ steps:
       Write-Host "This script is not executing on Windows, skipping registry modification."
     }
   displayName: Enable Long Paths if Necessary
-  
-- pwsh: |
-    git clone https://github.com/${{ parameters.TargetDocRepoOwner }}/${{ parameters.TargetDocRepoName }} ${{ parameters.WorkingDirectory }}/repo
-  displayName: Clone Documentation Repository
-  ignoreLASTEXITCODE: false
+
+- ${{ if not(parameters.SparseCheckoutPaths) }}:
+  - pwsh: |
+      git clone https://github.com/${{ parameters.TargetDocRepoOwner }}/${{ parameters.TargetDocRepoName }} ${{ parameters.WorkingDirectory }}/repo
+    displayName: Clone Documentation Repository
+    ignoreLASTEXITCODE: false
+
+- ${{ if parameters.SparseCheckoutPaths }}:
+  - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
+    parameters:
+      SkipDefaultCheckout: true
+      Repositories:
+        - Name: ${{ parameters.TargetDocRepoOwner }}/${{ parameters.TargetDocRepoName }}
+          WorkingDirectory: ${{ parameters.WorkingDirectory }}/repo
+      Paths: ${{ parameters.SparseCheckoutPaths }}
+
 - template: /eng/common/pipelines/templates/steps/set-default-branch.yml
   parameters:
-    WorkingDirectory: ${{ parameters.WorkingDirectory }}/repo 
+    WorkingDirectory: ${{ parameters.WorkingDirectory }}/repo
 - task: PowerShell@2
   displayName: 'Apply Documentation Updates From Artifact'
   inputs:

eng/common/pipelines/templates/steps/docs-metadata-release.yml

@@ -16,22 +16,59 @@ steps:
   - ${{ if not(parameters.SkipDefaultCheckout) }}:
     - checkout: none
 
-  - ${{ each repo in parameters.Repositories }}:
-    - pwsh: |
-        $dir = "${{ coalesce(repo.WorkingDirectory, format('{0}/{1}', '$(System.DefaultWorkingDirectory)', repo.Name)) }}"
-        New-Item $dir -ItemType Directory -Force
-      displayName: Create ${{ repo.Name }} directories
-
-    - pwsh: |
-        git clone --no-checkout --filter=tree:0 git://github.com/${{ repo.Name }} .
-        git sparse-checkout init
-        $paths = ('${{ convertToJson(parameters.Paths) }}' | ConvertFrom-Json) -Join ' '
-        Invoke-Expression -Command "git sparse-checkout set eng $paths"
-        Write-Host "Set sparse checkout paths to:"
-        Get-Content .git/info/sparse-checkout
-      displayName: Init sparse checkout ${{ repo.Name }}
-      workingDirectory: ${{ coalesce(repo.WorkingDirectory, format('{0}/{1}', '$(System.DefaultWorkingDirectory)', repo.Name)) }}
-
-    - pwsh: git checkout ${{ repo.Commitish }}
-      displayName: Sparse checkout at ${{ repo.Commitish }}
-      workingDirectory: ${{ coalesce(repo.WorkingDirectory, format('{0}/{1}', '$(System.DefaultWorkingDirectory)', repo.Name)) }}
+  - task: PowerShell@2
+    displayName: 'Sparse checkout repositories'
+    inputs:
+      targetType: inline
+      # Define this inline, because of the chicken/egg problem with loading a script when nothing
+      # has been checked out yet.
+      script: |
+        function SparseCheckout([Array]$paths, [Hashtable]$repository)
+        {
+            $paths = $paths -Join ' '
+
+            $dir = $repository.WorkingDirectory
+            if (!$dir) {
+              $dir = "./$($repository.Name)"
+            }
+            New-Item $dir -ItemType Directory -Force
+            Push-Location $dir
+
+            if (Test-Path .git/info/sparse-checkout) {
+              $hasInitialized = $true
+              Write-Host "Repository $($repository.Name) has already been initialized. Skipping this step."
+            } else {
+              Write-Host "Repository $($repository.Name) is being initialized."
+              git clone --no-checkout --filter=tree:0 git://github.com/$($repository.Name) .
+              git sparse-checkout init
+              git sparse-checkout set eng
+            }
+
+            $gitsparsecmd = "git sparse-checkout add $paths"
+            Write-Host $gitsparsecmd
+            Invoke-Expression -Command $gitsparsecmd
+
+            Write-Host "Set sparse checkout paths to:"
+            Get-Content .git/info/sparse-checkout
+
+            # sparse-checkout commands after initial checkout will auto-checkout again
+            if (!$hasInitialized) {
+              Write-Host "git checkout $($repository.Commitish)"
+              git checkout $($repository.Commitish)  # this will use the default branch if repo.Commitish is empty
+            } else {
+              Write-Host "Skipping checkout as repo has already been initialized"
+            }
+
+            Pop-Location
+        }
+
+        # Paths may be sourced as a yaml object literal OR a dynamically generated variable json string.
+        # If the latter, convertToJson will wrap the 'string' in quotes, so remove them.
+        $paths = '${{ convertToJson(parameters.Paths) }}'.Trim('"') | ConvertFrom-Json
+        # Replace windows backslash paths, as Azure Pipelines default directories are sometimes formatted like 'D:\a\1\s'
+        $repositories = '${{ convertToJson(parameters.Repositories) }}' -replace '\\', '/' | ConvertFrom-Json -AsHashtable
+        foreach ($repo in $Repositories) {
+          SparseCheckout $paths $repo
+        }
+      pwsh: true
+      workingDirectory: $(System.DefaultWorkingDirectory)

eng/common/pipelines/templates/steps/sparse-checkout.yml

@@ -13,17 +13,35 @@ function Get-ChangeLogEntries {
     [String]$ChangeLogLocation
   )
 
-  $changeLogEntries = [Ordered]@{}
   if (!(Test-Path $ChangeLogLocation)) {
     LogError "ChangeLog[${ChangeLogLocation}] does not exist"
     return $null
   }
+  LogDebug "Extracting entries from [${ChangeLogLocation}]."
+  return Get-ChangeLogEntriesFromContent (Get-Content -Path $ChangeLogLocation)
+}
 
+function Get-ChangeLogEntriesFromContent {
+  param (
+    [Parameter(Mandatory = $true)]
+    $changeLogContent
+  )
+
+  if ($changeLogContent -is [string])
+  {
+    $changeLogContent = $changeLogContent.Split("`n")
+  }
+  elseif($changeLogContent -isnot [array])
+  {
+    LogError "Invalid ChangelogContent passed"
+    return $null
+  }
+
+  $changeLogEntries = [Ordered]@{}
   try {
-    $contents = Get-Content $ChangeLogLocation
     # walk the document, finding where the version specifiers are and creating lists
     $changeLogEntry = $null
-    foreach ($line in $contents) {
+    foreach ($line in $changeLogContent) {
       if ($line -match $RELEASE_TITLE_REGEX) {
         $changeLogEntry = [pscustomobject]@{ 
           ReleaseVersion = $matches["version"]
@@ -41,7 +59,7 @@ function Get-ChangeLogEntries {
     }
   }
   catch {
-    Write-Host "Error parsing $ChangeLogLocation."
+    Write-Host "Error parsing Changelog."
     Write-Host $_.Exception.Message
   }
   return $changeLogEntries

eng/common/scripts/ChangeLog-Operations.ps1

@@ -1,2 +1,2 @@
 variables:
-  AZURESDK_CONDA_VERSION: '2021.05.01'
+  AZURESDK_CONDA_VERSION: '2021.05.01b1'

eng/conda_env.yml

@@ -0,0 +1,12 @@
+# install from root of repo
+aiohttp>=3.0; python_version >= '3.5'
+tools/azure-devtools
+tools/azure-sdk-tools
+mock;
+aiodns>=2.0; python_version >= '3.5'
+parameterized>=0.7.3; python_version >= '3.0'
+trio; python_version >= '3.5'
+typing_extensions>=3.7.2
+futures==3.3.0; python_version <= '2.7'
+cryptography
+adal

eng/conda_test_requirements.txt

@@ -0,0 +1,4 @@
+trigger: none
+
+jobs:
+  - template: /eng/common/pipelines/templates/jobs/docindex.yml