Merge branch 'main' of https://github.com/Azure/azure-sdk-for-python into update_package_json

* 'main' of https://github.com/Azure/azure-sdk-for-python: (136 commits)
  Re-record (Azure#32584)
  Update CODEOWNERS (Azure#32583)
  capture request data with aiohttp multipart/form-data (Azure#32543)
  [Core] Create generic azure-core package: `corehttp` (Azure#32191)
  overwrite .amlignore file (Azure#32546)
  Fix Sphinx error in core (Azure#32417)
  skip pyright for now (Azure#32569)
  Fix identity-broker sdist (Azure#32568)
  Updates to interactions with `pyproject.toml` (Azure#32480)
  fix: fixing local variable's name(last_modified_exising -> last_modified_existing) typo via Cspell (Azure#32556)
  Update feature store related entities to use the GA version (Azure#32541)
  Removing unused RST files (Azure#32547)
  Update python version for virtualenv issues (Azure#32528)
  [AutoRelease] t2-keyvault-2023-10-13-36681(can only be merged by SDK owner) (Azure#32478)
  [Monitor Distro] Update broken link (Azure#32505)
  Bump python stress images and addons (Azure#32554)
  Bump (Azure#32552)
  Increment package version after release of azure-cosmos (Azure#32550)
  repair aggregate report pipeline (Azure#32509)
  update readme and samples (Azure#32548)
  ...

Author: iscai-msft

.github/CODEOWNERS

@@ -29,6 +29,9 @@
 /sdk/ @lmazuel
 
 # Core
+# PRLabel: %Core.Http
+/sdk/core/corehttp/                                                  @pvaneck @lmazuel @annatisch @johanste @xiangyan99 @iscai-msft @kashifkhan
+
 # PRLabel: %Azure.Core
 /sdk/core/                                                           @lmazuel @annatisch @johanste
 /sdk/core/azure-core/                                                @lmazuel @annatisch @johanste @xiangyan99 @tjprescott @iscai-msft @kashifkhan
@@ -104,7 +107,7 @@
 /sdk/keyvault/                                                       @schaabs @mccoyp
 
 # PRLabel: %Load Testing
-/sdk/loadtestservice/azure-developer-loadtesting/                    @msyyc @iscai-msft
+/sdk/loadtesting/azure-developer-loadtesting/                        @msyyc @iscai-msft
 
 # PRLabel: %Monitor
 /sdk/loganalytics/azure-loganalytics/                                @azmonapplicationinsights @pvaneck
@@ -142,7 +145,7 @@
 # PRLabel: %Data Factory
 /sdk/datafactory/                                                    @hvermis
 /sdk/datalake/                                                       @ro-joowan
-/sdk/datadatamigration/                                              @vchske
+/sdk/datamigration/                                                  @vchske
 
 # PRLabel: %Device Provisioning
 /sdk/iothub/azure-iot-deviceprovisioning                             @c-ryan-k @digimaun
@@ -160,7 +163,7 @@
 /sdk/modelsrepository/                                               @cartertinney @digimaun
 
 # PRLabel: %Machine Learning Compute
-/sdk/machinelearningcompute/                                         @shutchings
+/sdk/machinelearning/                                                @shutchings
 
 # PRLabel: %Machine Learning
 /sdk/ml/                                                             @paulshealy1 @singankit @diondrapeck @luigiw @kdestin @MilesHolland @needuv @ninghu @YusakuNo1
@@ -281,7 +284,7 @@
 /sdk/servicefabric/                                                  @QingChenmsft @samedder
 
 # PRLabel: %SQL
-/sql/sql/                                                            @jaredmoo
+/sdk/sql/                                                            @jaredmoo
 
 # PRLabel: %Service Bus
 /sdk/servicebus/                                                     @annatisch @kashifkhan @swathipil @l0lawrence

.github/workflows/event-processor.yml

@@ -55,7 +55,7 @@ jobs:
         run: >
           dotnet tool install
           Azure.Sdk.Tools.GitHubEventProcessor
-          --version 1.0.0-dev.20230713.2
+          --version 1.0.0-dev.20231010.2
           --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json
           --global
         shell: bash

.github/workflows/scheduled-event-processor.yml

@@ -34,7 +34,7 @@ jobs:
         run: >
           dotnet tool install
           Azure.Sdk.Tools.GitHubEventProcessor
-          --version 1.0.0-dev.20230713.2
+          --version 1.0.0-dev.20231010.2
           --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json
           --global
         shell: bash

.vscode/cspell.json

@@ -55,6 +55,7 @@
     "sdk/core/azure-core-tracing-opencensus/**",
     "sdk/core/azure-core-tracing-opentelemetry/**",
     "sdk/core/azure-servicemanagement-legacy/**",
+    "sdk/core/corehttp/**",
     "sdk/digitaltwins/azure-digitaltwins-core/**",
     "sdk/eventhub/azure-eventhub-checkpointstoretable/**",
     "sdk/eventhub/azure-eventhub-checkpointstoreblob-aio/**",
@@ -97,6 +98,9 @@
     "sdk/translation/azure-ai-translation-text/tests/test_break_sentence.py",
     "sdk/translation/azure-ai-translation-text/tests/test_translation.py",
     "sdk/translation/azure-ai-translation-text/tests/test_transliteration.py",
+    "sdk/translation/azure-ai-translation-text/tests/test_break_sentence_async.py",
+    "sdk/translation/azure-ai-translation-text/tests/test_translation_async.py",
+    "sdk/translation/azure-ai-translation-text/tests/test_transliteration_async.py",
     "sdk/translation/test-resources.json",
     "eng/**/*.json",
     "eng/*.txt",
@@ -120,6 +124,11 @@
     "conda/conda-releaselogs/azure-mgmt.md"
   ],
   "words": [
+    "pyprojects",
+    "certifi",
+    "cffi",
+    "tomlw",
+    "xdist",
     "pyprojecttoml",
     "tomli",
     "sdkrel",
@@ -201,6 +210,7 @@
     "ekus",
     "encryptor",
     "engsys",
+    "entra",
     "envname",
     "evals",
     "fileno",
@@ -493,7 +503,7 @@
       "words": [
         "wammsa"
       ]
-    },    
+    },
     {
       "filename": "sdk/tables/azure-data-tables/tests/**/*.py",
       "words": [
@@ -828,7 +838,8 @@
     {
       "filename": "sdk/keyvault/**",
       "words": [
-        "eddsa"
+        "eddsa",
+        "Thawte"
       ]
     },
     {

doc/eng_sys_checks.md

@@ -118,6 +118,8 @@ Starting with [this pr](https://github.com/Azure/azure-sdk-for-python/pull/28345
 
 We default to **enabling** most of our checks like `pylint`, `mypy`, etc. Due to that, most `pyproject.toml` settings will likely be **disabling** checks.
 
+There is also an additional setting to turn on strict sphinx validation, for docstring validation.
+
 Here's an example:
 
 ```toml
@@ -130,6 +132,7 @@ verifytypes = false
 pyright = false
 pylint = false
 black = false
+strict_sphinx = false
 ```
 
 If a package does not yet have a `pyproject.toml`, creating one with just the section `[tool.azure-sdk-build]` will do no harm to the release of the package in question.
@@ -195,9 +198,20 @@ Analyze job in both nightly CI and pull request validation pipeline runs a set o
 
 1. Go to root of the package.
 2. Execute following command: `tox run -e pylint -c ../../../eng/tox/tox.ini --root .`
-  
+
 Note that the `pylint` environment is configured to run against the **earliest supported python version**. This means that users **must** have `python 3.7` installed on their machine to run this check locally.
 
+### Sphinx and docstring checker
+
+[`Sphinx`](https://www.sphinx-doc.org/en/master/) is the preferred documentation builder for Python libraries. The documentation is always built and attached to each PR builds. Sphinx can be configured to
+fail if docstring are invalid, helping to ensure the resulting documentation will be of high quality. Following are the steps to run `sphinx` locally for a specific package with strict docstring checking:
+
+1. Go to root of the package.
+2. Make sure the `pyproject.toml` file contains `strict_sphinx = true`
+3. Execute following command: `tox run -e sphinx -c ../../../eng/tox/tox.ini --root .`
+
+Note: While as of now the default value is `False`, it will become `True` by mid 2024.
+
 ### Bandit
 
 `Bandit` is static security analysis tool. This check is triggered for all Azure SDK package as part of analyze job. Following are the steps to `Bandit` tool locally for a specific package.

doc/sphinx/python_mgmt_migration_guide.rst

@@ -102,7 +102,7 @@ To the show the code snippets for the change:
     from azure.identity import ClientSecretCredential
 
     credential = ClientSecretCredential(
-        client_secret=client_secret,
+        tenant_id=tenant_id,
         client_id=client_id,
         client_secret=client_secret
     )

eng/.docsettings.yml

@@ -103,6 +103,7 @@ known_content_issues:
   - ['sdk/appconfiguration/azure-appconfiguration/swagger/README.md', '#4554']
   - ['sdk/attestation/azure-security-attestation/swagger/README.md', '#4554']
   - ['sdk/core/azure-core/tests/testserver_tests/coretestserver/README.rst', '#4554']
+  - ['sdk/core/corehttp/tests/testserver_tests/coretestserver/README.rst', '#4554']
   - ['sdk/media/azure-media-analytics-edge/README.md', '#4554']
   - ['sdk/remoterendering/azure-mixedreality-remoterendering/README.md', '#4554']
   - ['sdk/modelsrepository/azure-iot-modelsrepository/README.md', '#4554']
@@ -115,6 +116,8 @@ known_content_issues:
   - ['sdk/core/azure-common/README.rst', 'common']
   - ['sdk/core/azure-core/README.md', 'common']
   - ['sdk/core/azure-core/samples/README.md', 'common']
+  - ['sdk/core/corehttp/README.md', 'common']
+  - ['sdk/core/corehttp/samples/README.md', 'common']
   - ['sdk/search/azure-search-documents/README.md', 'common']
   - ['sdk/storage/azure-storage-blob/samples/README.md', 'common']
   - ['sdk/storage/azure-storage-file-datalake/samples/README.md', 'common']
@@ -153,6 +156,7 @@ known_content_issues:
   - ['sdk/appconfiguration/azure-appconfiguration/swagger/README.md', '#4554']
   - ['sdk/core/azure-core/tests/testserver_tests/coretestserver/README.md', '#4554']
   - ['sdk/core/azure-core-experimental/README.md', '#4554']
+  - ['sdk/identity/azure-identity-broker/README.md', '#4554']
 package_indexing_exclusion_list:
   - 'azure-sdk-tools'
   - 'azure-template'

eng/ci_tools.txt

@@ -1,19 +1,18 @@
 # requirements leveraged by ci tools
 setuptools==67.6.0; python_version >= '3.5'
-virtualenv==20.23.0
+virtualenv==20.24.3
 wheel==0.37.0
 packaging==23.1
-wheel==0.37.0
 tox==4.5.0
-pyproject-api<1.6
 pathlib2==2.3.5
 doc-warden==0.7.2
 beautifulsoup4==4.9.1
 pkginfo==1.9.6
-pip==20.3.3
+pip==23.2.1
 wrapt==1.12.1; python_version <= '3.10'
 wrapt==1.15.0; python_version >= '3.11'
 typing-extensions<=4.6.3
+pyproject-api<1.6
 
 # requirements leveraged for testing
 pytest==7.3.1
@@ -33,7 +32,7 @@ pyyaml==6.0.1
 urllib3==1.26.16
 PyGithub==1.59.1
 ConfigArgParse==1.2.3
-six==1.14.0
+six==1.16.0
 
 # local dev packages
 ./tools/azure-devtools

eng/common/TestResources/New-TestResources.ps1

@@ -771,7 +771,6 @@ try {
                     -TemplateParameterObject $templateFileParameters `
                     -Force:$Force
         }
-
         if ($deployment.ProvisioningState -ne 'Succeeded') {
             Write-Host "Deployment '$($deployment.DeploymentName)' has state '$($deployment.ProvisioningState)' with CorrelationId '$($deployment.CorrelationId)'. Exiting..."
             Write-Host @'
@@ -803,6 +802,9 @@ try {
             Write-Verbose "Removing compiled bicep file $($templateFile.jsonFilePath)"
             Remove-Item $templateFile.jsonFilePath
         }
+
+        Write-Host "Deleting ARM deployment as it may contain secrets. Deployed resources will not be affected."
+        $null = $deployment | Remove-AzResourceGroupDeployment
     }
 
 } finally {

eng/common/pipelines/templates/steps/eng-common-workflow-enforcer.yml

@@ -11,7 +11,7 @@ steps:
 
         if ((!"$(System.PullRequest.SourceBranch)".StartsWith("sync-eng/common")) -and "$(System.PullRequest.TargetBranch)" -match "^(refs/heads/)?$(DefaultBranch)$")
         {
-          $filesInCommonDir  = & "eng/common/scripts/get-changedfiles.ps1" -DiffPath 'eng/common/*'
+          $filesInCommonDir  = & "eng/common/scripts/get-changedfiles.ps1" -DiffPath 'eng/common/*' -DiffFilterType ""
           if (($LASTEXITCODE -eq 0) -and ($filesInCommonDir.Count -gt 0))
           {
             Write-Host "##vso[task.LogIssue type=error;]Changes to files under 'eng/common' directory should not be made in this Repo`n${filesInCommonDir}"
@@ -21,7 +21,7 @@ steps:
         }
         if ((!"$(System.PullRequest.SourceBranch)".StartsWith("sync-.github/workflows")) -and "$(System.PullRequest.TargetBranch)" -match "^(refs/heads/)?$(DefaultBranch)$")
         {
-          $filesInCommonDir  = & "eng/common/scripts/get-changedfiles.ps1" -DiffPath '.github/workflows/*'
+          $filesInCommonDir  = & "eng/common/scripts/get-changedfiles.ps1" -DiffPath '.github/workflows/*' -DiffFilterType ""
           if (($LASTEXITCODE -eq 0) -and ($filesInCommonDir.Count -gt 0))
           {
             Write-Host "##vso[task.LogIssue type=error;]Changes to files under '.github/workflows' directory should not be made in this Repo`n${filesInCommonDir}"
@@ -30,4 +30,4 @@ steps:
           }
         }
       displayName: Prevent changes to eng/common and .github/workflows outside of azure-sdk-tools repo
-      condition: and(succeeded(), ne(variables['Skip.EngCommonWorkflowEnforcer'], 'true'), not(endsWith(variables['Build.Repository.Name'], '-pr')))
+      condition: and(succeeded(), ne(variables['Skip.EngCommonWorkflowEnforcer'], 'true'), not(endsWith(variables['Build.Repository.Name'], '-pr')))

eng/common/pipelines/templates/steps/sparse-checkout.yml

@@ -17,7 +17,10 @@ steps:
       - checkout: none
 
   - task: PowerShell@2
-    displayName: 'Sparse checkout repositories'
+    ${{ if eq(length(parameters.Repositories), 1) }}:
+      displayName: 'Sparse checkout ${{ parameters.Repositories[0].Name }}'
+    ${{ else }}:
+      displayName: 'Sparse checkout repositories'
     inputs:
       targetType: inline
       # Define this inline, because of the chicken/egg problem with loading a script when nothing

eng/common/scripts/Helpers/DevOps-WorkItem-Helpers.ps1

@@ -182,56 +182,6 @@ function FindParentWorkItem($serviceName, $packageDisplayName, $outputCommand =
   return $null
 }
 
-$releasePlanWorkItems = @{}
-function FindReleasePlanWorkItem($serviceName, $packageDisplayName, $outputCommand = $false, $ignoreReleasePlannerTests = $true)
-{
-  $key = BuildHashKey $serviceName $packageDisplayName
-  if ($key -and $releasePlanWorkItems.ContainsKey($key)) {
-    return $releasePlanWorkItems[$key]
-  }
-
-  if ($serviceName) {
-    $condition = "[ServiceName] = '${serviceName}'"
-    if ($packageDisplayName) {
-      $condition += " AND [PackageDisplayName] = '${packageDisplayName}'"
-    }
-    else {
-      $condition += " AND [PackageDisplayName] = ''"
-    }
-  }
-  else {
-    $condition = "[ServiceName] <> ''"
-  }
-  $condition += " AND [System.State] <> 'Finished'"
-  if($ignoreReleasePlannerTests){
-    $condition += " AND [Tags] NOT CONTAINS 'Release Planner App Test'"
-  }
-
-  $query = "SELECT [ID], [ServiceName], [PackageDisplayName], [Parent] FROM WorkItems WHERE [Work Item Type] = 'Release Plan' AND ${condition}"
-
-  $fields = @("System.Id", "Custom.ServiceName", "Custom.PackageDisplayName", "System.Parent", "System.Tags")
-
-  $workItems = Invoke-Query $fields $query $outputCommand
-
-  foreach ($wi in $workItems)
-  {
-    $localKey = BuildHashKey $wi.fields["Custom.ServiceName"] $wi.fields["Custom.PackageDisplayName"]
-    if (!$localKey) { continue }
-    if ($releasePlanWorkItems.ContainsKey($localKey) -and $releasePlanWorkItems[$localKey].id -ne $wi.id) {
-      Write-Warning "Already found parent [$($releasePlanWorkItems[$localKey].id)] with key [$localKey], using that one instead of [$($wi.id)]."
-    }
-    else {
-      Write-Verbose "[$($wi.id)]$localKey - Cached"
-      $releasePlanWorkItems[$localKey] = $wi
-    }
-  }
-
-  if ($key -and $releasePlanWorkItems.ContainsKey($key)) {
-    return $releasePlanWorkItems[$key]
-  }
-  return $null
-}
-
 $packageWorkItems = @{}
 $packageWorkItemWithoutKeyFields = @{}
 
@@ -545,40 +495,17 @@ function CreateOrUpdatePackageWorkItem($lang, $pkg, $verMajorMinor, $existingIte
       }
     }
 
-    $newparentItem = FindOrCreateReleasePlanParent $serviceName $pkgDisplayName -outputCommand $false
+    $newparentItem = FindOrCreatePackageGroupParent $serviceName $pkgDisplayName -outputCommand $false
     UpdateWorkItemParent $existingItem $newParentItem -outputCommand $outputCommand
     return $existingItem
   }
 
-  $parentItem = FindOrCreateReleasePlanParent $serviceName $pkgDisplayName -outputCommand $false
+  $parentItem = FindOrCreatePackageGroupParent $serviceName $pkgDisplayName -outputCommand $false
   $workItem = CreateWorkItem $title "Package" "Release" "Release" $fields $assignedTo $parentItem.id -outputCommand $outputCommand
   Write-Host "[$($workItem.id)]$lang - $pkgName($verMajorMinor) - Created"
   return $workItem
 }
 
-function FindOrCreateReleasePlanParent($serviceName, $packageDisplayName, $outputCommand = $true, $ignoreReleasePlannerTests = $true)
-{
-  $existingItem = FindReleasePlanWorkItem $serviceName $packageDisplayName -outputCommand $outputCommand -ignoreReleasePlannerTests $ignoreReleasePlannerTests
-  if ($existingItem) {
-    Write-Host "Found existing release plan work item [$($existingItem.id)]"
-    $newparentItem = FindOrCreatePackageGroupParent $serviceName $packageDisplayName -outputCommand $outputCommand -ignoreReleasePlannerTests $ignoreReleasePlannerTests
-    UpdateWorkItemParent $existingItem $newParentItem
-    return $existingItem
-  }
-
-  $fields = @()
-  $fields += "`"PackageDisplayName=${packageDisplayName}`""
-  $fields += "`"ServiceName=${serviceName}`""
-  $productParentItem = FindOrCreatePackageGroupParent $serviceName $packageDisplayName -outputCommand $outputCommand -ignoreReleasePlannerTests $ignoreReleasePlannerTests
-  $title = "Release Plan - $($packageDisplayName)"
-  $workItem = CreateWorkItem $title "Release Plan" "Release" "Release" $fields $null $productParentItem.id
-
-  $localKey = BuildHashKey $serviceName $packageDisplayName
-  Write-Host "[$($workItem.id)]$localKey - Created release plan work item"
-  $releasePlanWorkItems[$localKey] = $workItem
-  return $workItem
-}
-
 function FindOrCreatePackageGroupParent($serviceName, $packageDisplayName, $outputCommand = $true, $ignoreReleasePlannerTests = $true)
 {
   $existingItem = FindParentWorkItem $serviceName $packageDisplayName -outputCommand $outputCommand -ignoreReleasePlannerTests $ignoreReleasePlannerTests

eng/common/scripts/TypeSpec-Project-Generate.ps1

@@ -53,13 +53,6 @@ function NpmInstallForProject([string]$workingDirectory) {
             Copy-Item -Path $emitterPackageLock -Destination "package-lock.json" -Force
         }
 
-        $useAlphaNpmRegistry = (Get-Content $replacementPackageJson -Raw).Contains("-alpha.")
-
-        if($useAlphaNpmRegistry) {
-            Write-Host "Package.json contains '-alpha.' in the version, Creating .npmrc using public/azure-sdk-for-js-test-autorest feed."
-            "registry=https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-js-test-autorest@local/npm/registry/ `n`nalways-auth=true" | Out-File '.npmrc'
-        }
-
         if ($usingLockFile) {
             Invoke-LoggedCommand "npm ci"
         }