Merge branch 'master' of https://github.com/Azure/azure-sdk-for-python into correct_healthcare

* 'master' of https://github.com/Azure/azure-sdk-for-python: (30 commits)
  [EventGrid] Improve docs (Azure#16986)
  Fixed delete_blob method signature (Azure#16983)
  Fixed Content-MD5 serialization and typing (Azure#16987)
  [eventgrid] Black formatting and tests suite (Azure#16989)
  Appconfig perf test (Azure#16809)
  adds support for milliseconds on datetime objects (Azure#16847)
  [ServiceBus] Improve AMQP Error handling (Azure#16427)
  T2 resource 2021 02 26 (Azure#16961)
  T2 servicefabricmanagedclusters 2021 02 26 (Azure#16958)
  Update Prepare-Release script (Azure#16976)
  the previous filter was erroneously filtering out real files. (Azure#16978)
  Update azure-core minimum version (Azure#16968)
  update documention in depends section of eng_sys_checks.md (Azure#16967)
  version (Azure#16959)
  T2 web 2021 02 24 (Azure#16901)
  Change test pipeline to run in eastus2 (Azure#16950)
  Adding azure-devtools to mr-auth dev_requirements (Azure#16953)
  Add perf tests for eventgrid (Azure#16949)
  temp fix for canary tests (Azure#16945)
  Update get_package_properties to also return path to setup.py (Azure#16946)
  ...

Author: iscai-msft

ci_template.yml

@@ -5,6 +5,7 @@ trigger:
   branches:
     include:
     - master
+    - main
     - hotfix/*
     - release/*
     - restapi*
@@ -16,6 +17,7 @@ pr:
   branches:
     include:
     - master
+    - main
     - feature/*
     - hotfix/*
     - release/*

doc/eng_sys_checks.md

@@ -48,7 +48,7 @@ Analyze job in both nightly CI and pull request validation pipeline runs a set o
 Change log verification is added to ensure package has valid change log for current version. Guidelines to properly maintain the change log is documented [here](https://github.com/Azure/azure-sdk-for-python/blob/master/doc/)
 
 ## PR Validation Checks
-Each pull request runs various tests using `pytest` in addition to all the tests mentioned above in analyze check. Pull request validation performs 3 different types of test: `whl, sdist and depends`. Following section explains the purpose of each of these tests and how to execute them locally. All pull requests are validated on multiple python versions across different platforms and below is the test matrix for pull request.
+Each pull request runs various tests using `pytest` in addition to all the tests mentioned above in analyze check. Pull request validation performs 3 different types of test: `whl, sdist and depends`. The following section explains the purpose of each of these tests and how to execute them locally. All pull requests are validated on multiple python versions across different platforms. Find the test matrix below.
 
 
 |`Python  Version`|`Platform`  |
@@ -73,8 +73,8 @@ This test installs sdist of the package being tested and runs all tests cases in
 2. Run following command
    ``tox -e sdist -c ../../../eng/tox/tox.ini``
 
-####depends
-This test is to ensure all modules in the package being tested can be successfully imported. This is to ensure all package requirement is properly set in setup.py as well as to ensure modules are imported using valid namespace. This test install the package and it's required packages and executes `from <package-root-namespace> import *`. For e.g. `from azure.core import *`.
+#### depends
+The `depends` check ensures all modules in a target package can be successfully imported. Actually installing and importing will verify that all package requirements are properly set in setup.py and that the `__all__` set for the package is properly defined. This test installs the package and its required packages, then executes `from <package-root-namespace> import *`. For example from `azure-core`, the following would be invoked:  `from azure.core import *`.
 
 Following is the command to run this test environment locally.
 

eng/.docsettings.yml

@@ -124,6 +124,7 @@ known_content_issues:
   - ['sdk/core/azure-mgmt-nspkg/README.rst', 'nspkg and common']
   - ['sdk/core/azure-nspkg/README.rst', 'nspkg and common']
   - ['sdk/keyvault/azure-keyvault-nspkg/README.md', 'nspkg and common']
+  - ['sdk/mixedreality/azure-mixedreality-nspkg/README.md', 'nspkg and common']
   - ['sdk/search/azure-search-nspkg/README.md', 'nspkg and common']
   - ['sdk/storage/azure-storage-blob/samples/README.md', 'nspkg and common']
   - ['sdk/storage/azure-storage-file-datalake/samples/README.md', 'nspkg and common']

eng/common/scripts/Helpers/DevOps-WorkItem-Helpers.ps1

@@ -127,7 +127,6 @@ function FindPackageWorkItem($lang, $packageName, $version, $outputCommand = $tr
   $fields += "ID"
   $fields += "State"
   $fields += "System.AssignedTo"
-  $fields += "Microsoft.VSTS.Common.StateChangeDate"
   $fields += "Parent"
   $fields += "Language"
   $fields += "Package"
@@ -265,19 +264,6 @@ function CreateWorkItem($title, $type, $iteration, $area, $fields, $assignedTo,
   return $workItem
 }
 
-function ResetWorkItemState($workItem, $resetState = $null, $outputCommand = $true)
-{
-  if (!$resetState -or $resetState -eq "New") {
-    $resetState = "Next Release Unknown"
-  }
-  if ($workItem.fields["System.State"] -ne $resetState)
-  {
-    Write-Verbose "Resetting state for [$($workItem.id)] from '$($workItem.fields['System.State'])' to '$resetState'"
-    return UpdateWorkItem $workItem.id -state $resetState -outputCommand $outputCommand
-  }
-  return $workItem
-}
-
 function UpdateWorkItem($id, $fields, $title, $state, $assignedTo, $outputCommand = $true)
 {
   $parameters = $ReleaseDevOpsCommonParameters
@@ -305,6 +291,30 @@ function UpdatePackageWorkItemReleaseState($id, $state, $releaseType, $outputCom
   return UpdateWorkItem -id $id -state $state -fields $fields -outputCommand $outputCommand
 }
 
+function FindOrCreateClonePackageWorkItem($lang, $pkg, $verMajorMinor, $outputCommand = $false)
+{
+  $workItem = FindPackageWorkItem -lang $lang -packageName $pkg.Package -version $verMajorMinor -includeClosed $true -outputCommand $outputCommand
+
+  if (!$workItem) {
+    $latestVersionItem = FindLatestPackageWorkItem -lang $lang -packageName $pkg.Package -outputCommand $outputCommand
+    $assignedTo = "me"
+    if ($latestVersionItem) {
+      Write-Verbose "Copying data from latest matching [$($latestVersionItem.id)] with version $($latestVersionItem.fields["Custom.PackageVersionMajorMinor"])"
+      if ($latestVersionItem.fields["System.AssignedTo"]) {
+        $assignedTo = $latestVersionItem.fields["System.AssignedTo"]["uniqueName"]
+      }
+      $pkg.DisplayName = $latestVersionItem.fields["Custom.PackageDisplayName"]
+      $pkg.ServiceName = $latestVersionItem.fields["Custom.ServiceName"]
+      if (!$pkg.RepoPath -and $pkg.RepoPath -ne "NA" -and $pkg.fields["Custom.PackageRepoPath"]) {
+        $pkg.RepoPath = $pkg.fields["Custom.PackageRepoPath"]
+      }
+    }
+    $workItem = CreateOrUpdatePackageWorkItem $lang $pkg $verMajorMinor -existingItem $null -assignedTo $assignedTo -outputCommand $outputCommand
+  }
+
+  return $workItem
+}
+
 function CreateOrUpdatePackageWorkItem($lang, $pkg, $verMajorMinor, $existingItem, $assignedTo = $null, $outputCommand = $true)
 {
   if (!$lang -or !$pkg -or !$verMajorMinor) {
@@ -347,14 +357,18 @@ function CreateOrUpdatePackageWorkItem($lang, $pkg, $verMajorMinor, $existingIte
       Write-Host "At least field $changedField ($($existingItem.fields[$changedField])) changed so updating."
     }
 
-    $beforeState = $existingItem.fields["System.State"]
-
     if ($changedField) {
+      $beforeState = $existingItem.fields["System.State"]
+
       # Need to set to New to be able to update
       $existingItem = UpdateWorkItem -id $existingItem.id -fields $fields -title $title -state "New" -assignedTo $assignedTo -outputCommand $outputCommand
       Write-Host "[$($existingItem.id)]$lang - $pkgName($verMajorMinor) - Updated"
+
+      if ($beforeState -ne $existingItem.fields['System.State']) {
+        Write-Verbose "Resetting state for [$($existingItem.id)] from '$($existingItem.fields['System.State'])' to '$beforeState'"
+        $existingItem = UpdateWorkItem $existingItem.id -state $beforeState -outputCommand $outputCommand
+      }
     }
-    $existingItem = ResetWorkItemState $existingItem $beforeState -outputCommand $outputCommand
 
     $newparentItem = FindOrCreatePackageGroupParent $serviceName $pkgDisplayName -outputCommand $false
     UpdateWorkItemParent $existingItem $newParentItem -outputCommand $outputCommand
@@ -363,7 +377,6 @@ function CreateOrUpdatePackageWorkItem($lang, $pkg, $verMajorMinor, $existingIte
 
   $parentItem = FindOrCreatePackageGroupParent $serviceName $pkgDisplayName -outputCommand $false
   $workItem = CreateWorkItem $title "Package" "Release" "Release" $fields $assignedTo $parentItem.id -outputCommand $outputCommand
-  $workItem = ResetWorkItemState $workItem -outputCommand $outputCommand
   Write-Host "[$($workItem.id)]$lang - $pkgName($verMajorMinor) - Created"
   return $workItem
 }
@@ -752,25 +765,6 @@ function UpdatePackageVersions($pkgWorkItem, $plannedVersions, $shippedVersions)
   "value": "$shippedPackages"
 }
 "@
-
-    # If we shipped a version after we set "In Release" state then reset the state to "Next Release Unknown"
-    if ($pkgWorkItem.fields["System.State"] -eq "In Release")
-    {
-      $lastShippedDate = [DateTime]$newShippedVersions[0].Date
-      $markedInReleaseDate = ([DateTime]$pkgWorkItem.fields["Microsoft.VSTS.Common.StateChangeDate"])
-
-      # We just shipped so lets set the state to "Next Release Unknown"
-      if ($markedInReleaseDate -le $lastShippedDate)
-      {
-        $fieldUpdates += @'
-{
-  "op": "replace",
-  "path": "/fields/State",
-  "value": "Next Release Unknown"
-}
-'@
-      }
-    }
   }
 
   # Full merged version set

eng/common/scripts/Prepare-Release.ps1

@@ -2,60 +2,67 @@
 
 [CmdletBinding()]
 param(
-    [Parameter(Mandatory=$true)]
-    [string]$PackageName,
-    [string]$ServiceDirectory,
-    [string]$ReleaseDate, # Pass Date in the form MM/dd/yyyy"
-    [string]$BuildType # For Java
+  [Parameter(Mandatory = $true)]
+  [string]$PackageName,
+  [string]$ServiceDirectory,
+  [string]$ReleaseDate # Pass Date in the form MM/dd/yyyy"
 )
+Set-StrictMode -Version 3
 
 . ${PSScriptRoot}\common.ps1
 
 function Get-ReleaseDay($baseDate)
 {
-    # Find first friday
-    while ($baseDate.DayOfWeek -ne 5)
-    {
-        $baseDate = $baseDate.AddDays(1)
-    }
-    
-    # Go to Tuesday
-    $baseDate = $baseDate.AddDays(4)
-
-    return $baseDate;
+  # Find first friday
+  while ($baseDate.DayOfWeek -ne 5)
+  {
+    $baseDate = $baseDate.AddDays(1)
+  }
+
+  # Go to Tuesday
+  $baseDate = $baseDate.AddDays(4)
+
+  return $baseDate;
 }
 
 $ErrorPreference = 'Stop'
 
 $packageProperties = Get-PkgProperties -PackageName $PackageName -ServiceDirectory $serviceDirectory
 
+if (!$packageProperties)
+{
+  Write-Error "Could not find a package with name [ $packageName ], please verify the package name matches the exact name."
+  exit 1
+}
+
+Write-Host "Package Name [ $($packageProperties.Name) ]"
 Write-Host "Source directory [ $serviceDirectory ]"
 
 if (!$ReleaseDate)
 {
-    $currentDate = Get-Date
-    $thisMonthReleaseDate = Get-ReleaseDay((Get-Date -Day 1));
-    $nextMonthReleaseDate = Get-ReleaseDay((Get-Date -Day 1).AddMonths(1));
-
-    if ($thisMonthReleaseDate -ge $currentDate)
-    {
-        # On track for this month release
-        $ParsedReleaseDate = $thisMonthReleaseDate
-    }
-    elseif ($currentDate.Day -lt 15)
-    {
-        # Catching up to this month release
-        $ParsedReleaseDate = $currentDate
-    }
-    else 
-    {
-        # Next month release
-        $ParsedReleaseDate = $nextMonthReleaseDate
-    }
+  $currentDate = Get-Date
+  $thisMonthReleaseDate = Get-ReleaseDay((Get-Date -Day 1));
+  $nextMonthReleaseDate = Get-ReleaseDay((Get-Date -Day 1).AddMonths(1));
+
+  if ($thisMonthReleaseDate -ge $currentDate)
+  {
+    # On track for this month release
+    $ParsedReleaseDate = $thisMonthReleaseDate
+  }
+  elseif ($currentDate.Day -lt 15)
+  {
+    # Catching up to this month release
+    $ParsedReleaseDate = $currentDate
+  }
+  else
+  {
+    # Next month release
+    $ParsedReleaseDate = $nextMonthReleaseDate
+  }
 }
 else
 {
-    $ParsedReleaseDate = [datetime]$ReleaseDate
+  $ParsedReleaseDate = [datetime]$ReleaseDate
 }
 
 $releaseDateString = $ParsedReleaseDate.ToString("MM/dd/yyyy")
@@ -70,40 +77,42 @@ $newVersion = Read-Host -Prompt "Input the new version, or press Enter to use us
 
 if (!$newVersion)
 {
-    $newVersion = $currentProjectVersion;
+  $newVersion = $currentProjectVersion;
 }
 
 $newVersionParsed = [AzureEngSemanticVersion]::ParseVersionString($newVersion)
 if ($null -eq $newVersionParsed)
 {
-    Write-Error "Invalid version $newVersion. Version must follow standard SemVer rules, see https://aka.ms/azsdk/engsys/packageversioning"
-    exit 1
+  Write-Error "Invalid version $newVersion. Version must follow standard SemVer rules, see https://aka.ms/azsdk/engsys/packageversioning"
+  exit 1
 }
 
 if (Test-Path "Function:SetPackageVersion")
 {
-    SetPackageVersion -PackageName $packageProperties.Name -Version $newVersion -ServiceDirectory $serviceDirectory -ReleaseDate $releaseDateString `
-    -BuildType $BuildType -GroupId $packageProperties.Group
+  SetPackageVersion -PackageName $packageProperties.Name -Version $newVersion -ServiceDirectory $serviceDirectory -ReleaseDate $releaseDateString `
+    -PackageProperties $packageProperties
 }
 else
 {
-    LogError "The function 'SetPackageVersion' was not found.`
+  LogError "The function 'SetPackageVersion' was not found.`
     Make sure it is present in eng/scripts/Language-Settings.ps1.`
     See https://github.com/Azure/azure-sdk-tools/blob/master/doc/common/common_engsys.md#code-structure"
-    exit 1
+  exit 1
 }
 
 &$EngCommonScriptsDir/Update-DevOps-Release-WorkItem.ps1 `
--language $LanguageDisplayName `
--packageName $packageProperties.Name `
--version $newVersion `
--plannedDate $releaseDateString `
--packageRepoPath $packageProperties.serviceDirectory
+  -language $LanguageDisplayName `
+  -packageName $packageProperties.Name `
+  -version $newVersion `
+  -plannedDate $releaseDateString `
+  -packageRepoPath $packageProperties.serviceDirectory `
+  -packageType $packageProperties.SDKType `
+  -packageNewLibrary $packageProperties.IsNewSDK
 
 git diff -s --exit-code $packageProperties.DirectoryPath
 if ($LASTEXITCODE -ne 0)
 {
-    git status
-    Write-Host "Some changes were made to the repo source" -ForegroundColor Green
-    Write-Host "Submit a pull request with the necessary changes to the repo" -ForegroundColor Green
-}
+  git status
+  Write-Host "Some changes were made to the repo source" -ForegroundColor Green
+  Write-Host "Submit a pull request with the necessary changes to the repo" -ForegroundColor Green
+}

eng/common/scripts/common.ps1

@@ -25,12 +25,12 @@ if (Test-Path $EngScriptsLanguageSettings) {
   . $EngScriptsLanguageSettings
 }
 
-if (-not $LanguageShort)
+if (!(Get-Variable -Name "LangaugeShort" -ValueOnly -ErrorAction "Ignore"))
 {
   $LangaugeShort = $Language
 }
 
-if (-not $LanguageDisplayName)
+if (!(Get-Variable -Name "LanguageDisplayName" -ValueOnly -ErrorAction "Ignore"))
 {
   $LanguageDisplayName = $Language
 }

eng/common/scripts/logging.ps1

@@ -1,7 +1,4 @@
-if (-not $isDevOpsRun)
-{
-  $isDevOpsRun = ($null -ne $env:SYSTEM_TEAMPROJECTID)
-}
+$isDevOpsRun = ($null -ne $env:SYSTEM_TEAMPROJECTID)
 
 function LogWarning
 {
@@ -37,4 +34,4 @@ function LogDebug
     {
         Write-Debug "$args"
     }
-}
+}

eng/pipelines/aggregate-reports.yml

@@ -54,6 +54,7 @@ jobs:
       # Used for generating baseline file.   
       # GdnBreakOutputBaselineFile: python
       # GdnBreakOutputBaseline: baseline
+    continueOnError: true
   - task: securedevelopmentteam.vss-secure-development-tools.build-task-publishsecurityanalysislogs.PublishSecurityAnalysisLogs@3
     displayName: 'Publish Security Analysis Logs'
     continueOnError: true

eng/pipelines/templates/stages/archetype-python-release.yml

@@ -286,14 +286,15 @@ stages:
               $packageDirectory = "${{artifact.name}}".Replace("_", "-")
               echo "##vso[task.setvariable variable=Package.Name]$packageDirectory"
           - pwsh: |
-              $fileCount = (Get-ChildItem $(Pipeline.Workspace)/${{parameters.ArtifactName}}/$packageDirectory  | Measure-Object).Count
+              $fileCount = (Get-ChildItem $(Pipeline.Workspace)/${{parameters.ArtifactName}}/$(Package.Name) | ? {$_.Name -match "-[0-9]*.[0-9]*.[0-9]*a[0-9]*" } | Measure-Object).Count
+
               if ($fileCount -eq 0) {
                 Write-Host "No alpha packages for ${{artifact.safeName}} to publish."
                 exit 0
               }
 
-              twine upload --repository $(DevFeedName) --config-file $(PYPIRC_PATH) $(Pipeline.Workspace)/${{parameters.ArtifactName}}/$(Package.Name)/*a*.whl
+              twine upload --repository $(DevFeedName) --config-file $(PYPIRC_PATH) $(Pipeline.Workspace)/${{parameters.ArtifactName}}/$(Package.Name)/*-*a*.whl
               echo "Uploaded whl to devops feed $(DevFeedName)"
-              twine upload --repository $(DevFeedName) --config-file $(PYPIRC_PATH) $(Pipeline.Workspace)/${{parameters.ArtifactName}}/$(Package.Name)/*a*.zip
+              twine upload --repository $(DevFeedName) --config-file $(PYPIRC_PATH) $(Pipeline.Workspace)/${{parameters.ArtifactName}}/$(Package.Name)/*-*a*.zip
               echo "Uploaded sdist to devops feed $(DevFeedName)"
             displayName: 'Publish ${{artifact.name}} alpha package'