Merge branch 'main' of https://github.com/Azure/azure-sdk-for-python into header_tuples

* 'main' of https://github.com/Azure/azure-sdk-for-python: (23 commits)
  make core tests offline (#19986)
  [translation] naming alignment (#20406)
  Address API view issues (#20407)
  skip custom translation model samples (#20414)
  [purview account] first release for azure-purview-account (#20335)
  improve docs and samples for glossaries and custom models (#18587)
  Sync eng/common directory with azure-sdk-tools for PR 1918 (#20342)
  Update .docsettings.yml (#20402)
  improve begin_translation docstring (#20401)
  [translation] fix poller.details (#20392)
  Chain exceptions from LibsecretPersistence (#20380)
  [AutoRelease] t2-datamigration-2021-08-19-04035 (#20349)
  [AutoRelease] t2-policyinsights-2021-08-23-78735 (#20381)
  [AutoRelease] t2-servicebus-2021-08-19-00756 (#20350)
  [AutoRelease] t2-costmanagement-2021-07-27-84467 (#19949)
  keyvault and storage have a conflict between mindependency and what local azure-identity requires. resolve it (#20391)
  Add Cloud Configuration section to Identity readme (#20373)
  Consistency related changes (#20385)
  Document Pod Identity's special use of client_id (#20377)
  [rest] use azure json encoder for json input bodies (#20361)
  ...

Author: iscai-msft

eng/.docsettings.yml

@@ -10,6 +10,7 @@ omitted_paths:
   - sdk/identity/azure-identity/tests/*
   - sdk/**/tests/perfstress_tests/*
   - sdk/nspkg/*
+  - sdk/**/swagger/*
 
 language: python
 root_check_enabled: True
@@ -91,6 +92,7 @@ known_content_issues:
   - ['sdk/purview/azure-purview-catalog/swagger/README.md',  '#4554']
   - ['sdk/purview/azure-purview-scanning/swagger/README.md',  '#4554']
   - ['sdk/agrifood/azure-agrifood-farming/swagger/README.md',  '#4554']
+  - ['sdk/purview/azure-purview-account/swagger/README.md', '#4554']
   - ['sdk/containerregistry/azure-containerregistry/swagger/README.md', '#4554']
   - ['sdk/appconfiguration/azure-appconfiguration/swagger/README.md', '#4554']
   - ['sdk/attestation/azure-security-attestation/swagger/README.md', '#4554']

eng/common/scripts/stress-testing/deploy-stress-tests.ps1

@@ -20,9 +20,8 @@ $ErrorActionPreference = 'Stop'
 . $PSScriptRoot/find-all-stress-packages.ps1
 $FailedCommands = New-Object Collections.Generic.List[hashtable]
 
-if (!(Get-Module powershell-yaml)) {
-    Install-Module -Name powershell-yaml -RequiredVersion 0.4.1 -Force -Scope CurrentUser
-}
+. (Join-Path $PSScriptRoot "../Helpers" PSModule-Helpers.ps1)
+Install-ModuleIfNotInstalled "powershell-yaml" "0.4.1" | Import-Module
 
 # Powershell does not (at time of writing) treat exit codes from external binaries
 # as cause for stopping execution, so do this via a wrapper function.
@@ -51,7 +50,10 @@ function Login([string]$subscription, [string]$clusterGroup, [boolean]$pushImage
         RunOrExitOnFailure az login --allow-no-subscriptions
     }
 
-    $clusterName = (az aks list -g $clusterGroup -o json| ConvertFrom-Json).name
+    # Discover cluster name, only one cluster per group is expected
+    Write-Host "Listing AKS cluster in $subscription/$clusterGroup"
+    $cluster = RunOrExitOnFailure az aks list -g $clusterGroup --subscription $subscription -o json
+    $clusterName = ($cluster | ConvertFrom-Json).name
 
     RunOrExitOnFailure az aks get-credentials `
         -n "$clusterName" `
@@ -60,8 +62,9 @@ function Login([string]$subscription, [string]$clusterGroup, [boolean]$pushImage
         --overwrite-existing
 
     if ($pushImages) {
-        $registry = (az acr list -g $clusterGroup -o json | ConvertFrom-Json).name
-        RunOrExitOnFailure az acr login -n $registry
+        $registry = RunOrExitOnFailure az acr list -g $clusterGroup --subscription $subscription -o json
+        $registryName = ($registry | ConvertFrom-Json).name
+        RunOrExitOnFailure az acr login -n $registryName
     }
 }
 
@@ -110,11 +113,8 @@ function DeployStressPackage(
     [string]$repository,
     [boolean]$pushImages
 ) {
-    $registry = (az acr list -g $clusterGroup -o json | ConvertFrom-Json).name
-    if (!$registry) {
-        Write-Host "Could not find container registry in resource group $clusterGroup"
-        exit 1
-    }
+    $registry = RunOrExitOnFailure az acr list -g $clusterGroup --subscription $subscription -o json
+    $registryName = ($registry | ConvertFrom-Json).name
 
     Run helm dependency update $pkg.Directory
     if ($LASTEXITCODE) { return }
@@ -133,7 +133,7 @@ function DeployStressPackage(
             if (!$imageName) {
                 $imageName = $dockerFile.Directory.Name
             }
-            $imageTag = "${registry}.azurecr.io/$($repository.ToLower())/$($imageName):$deployId"
+            $imageTag = "${registryName}.azurecr.io/$($repository.ToLower())/$($imageName):$deployId"
             Write-Host "Building and pushing stress test docker image '$imageTag'"
             Run docker build -t $imageTag -f $dockerFile.FullName $dockerFile.DirectoryName
             if ($LASTEXITCODE) { return }
@@ -154,7 +154,7 @@ function DeployStressPackage(
     Run helm upgrade $pkg.ReleaseName $pkg.Directory `
         -n $pkg.Namespace `
         --install `
-        --set repository=$registry.azurecr.io/$repository `
+        --set repository=$registryName.azurecr.io/$repository `
         --set tag=$deployId `
         --set stress-test-addons.env=$environment
     if ($LASTEXITCODE) {
@@ -176,4 +176,7 @@ function DeployStressPackage(
     Run kubectl label secret -n $pkg.Namespace --overwrite $helmReleaseConfig deployId=$deployId
 }
 
-DeployStressTests @PSBoundParameters
+# Don't call functions when the script is being dot sourced
+if ($MyInvocation.InvocationName -ne ".") {
+    DeployStressTests @PSBoundParameters
+}

eng/pipelines/templates/jobs/smoke.tests.yml

@@ -159,15 +159,15 @@ jobs:
           timeoutInMinutes: 5
 
         - pwsh: |
-            $packages = Get-ChildItem "$(Pipeline.Workspace)/${{ parameters.ArtifactName }}/*.zip"
+            $packages = Get-ChildItem "$(Pipeline.Workspace)/${{ parameters.ArtifactName }}/${{ parameters.Artifact.name }}/*.zip"
             Write-Host "Artifacts found:"
             $artifacts = $packages | ForEach-Object {
               if ($_.Name -match "([a-zA-Z\-]+)\-(.*).zip") {
                 Write-Host "$($matches[1]): $($matches[2])"
                 return @{ "name" = $matches[1]; "version" = $matches[2] }
               }
             }
-            $dependencies = Get-Content $env:REQUIREMENTS | ForEach-Object {
+            $dependencies = Get-Content $(requirements) | ForEach-Object {
               $line = $_
               if ($line -match "([a-zA-Z\-]+)(\W+)(.*)") {
                   $override = ($artifacts | Where-Object { $_.Name -eq $matches[1] }).Version
@@ -179,36 +179,44 @@ jobs:
               return $line
             }
 
-            $dependencies | Out-File $env:REQUIREMENTS
+            $dependencies | Out-File $(requirements)
 
           displayName: Override requirements with pipeline build artifact versions
-          env:
-            REQUIREMENTS: $(requirements)
 
-        - pwsh: pip install -r "$env:REQUIREMENTS" --no-deps --upgrade
+        # Retry for pip install due to delay in package availability after publish
+        # The package is expected to be available for download/installation within 10 minutes 
+        - pwsh: |
+            $ErrorActionPreference = "Continue"
+            while ($retries++ -lt 15) {
+              Write-Host "pip install -r $(requirements) --no-deps --upgrade --no-cache-dir"
+              pip install -r "$(requirements)" --no-deps --upgrade --no-cache-dir
+              if ($LASTEXITCODE) {
+                if ($retries -ge 15) {
+                  exit $LASTEXITCODE
+                }
+                Write-Host "Installation failed, retrying in 1 minute..."
+                sleep 60
+              } else {
+                break
+              }
+            }
           displayName: Install requirements without dependencies
-          env:
-            REQUIREMENTS: $(requirements)
 
       - ${{ if eq(parameters.Daily, true) }}:
         - pwsh: |
-            pip install -r "$env:REQUIREMENTS" --pre --no-deps --upgrade `
+            pip install -r "$(requirements)" --pre --no-deps --upgrade `
               --index-url https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple
 
           displayName: Install requirements from dev feed without dependencies
-          env:
-            REQUIREMENTS: $(requirements)
 
       - pwsh: pip install -r $(Build.SourcesDirectory)/common/smoketest/requirements_async.txt
         displayName: "Install requirements_async.txt"
         condition: and(succeeded(), ne(variables['SkipAsyncInstall'], true))
 
       - pwsh: |
-          python $(Build.SourcesDirectory)/common/smoketest/dependencies.py -r "$env:REQUIREMENTS" `
+          python $(Build.SourcesDirectory)/common/smoketest/dependencies.py -r "$(requirements)" `
             | Out-File $(Build.SourcesDirectory)/common/smoketest/requirements_dependencies.txt
         displayName: Create dependency list from installed packages
-        env:
-          REQUIREMENTS: $(requirements)
 
       - script: pip install -r $(Build.SourcesDirectory)/common/smoketest/requirements_dependencies.txt
         displayName: Install package dependencies from PyPI

eng/tox/allowed_pylint_failures.py

@@ -54,6 +54,7 @@
     "azure-purview-nspkg",
     "azure-purview-scanning",
     "azure-purview-catalog",
+    "azure-purview-account",
     "azure-messaging-nspkg",
     "azure-agrifood-farming",
     "azure-eventhub",

scripts/devops_tasks/test_run_samples.py

@@ -123,6 +123,8 @@
         "sample_list_translations_with_filters_async.py",
         "sample_list_document_statuses_with_filters.py",
         "sample_list_translations_with_filters.py",
+        "sample_translation_with_custom_model.py",
+        "sample_translation_with_custom_model_async.py",
     ]
 }
 

sdk/core/azure-core/CHANGELOG.md

@@ -4,6 +4,8 @@
 
 ### Features Added
 
+- We now use `azure.core.serialization.AzureJSONEncoder` to serialize `json` input to `azure.core.rest.HttpRequest`.
+
 ### Breaking Changes in the Provisional `azure.core.rest` package
 
 - The `text` property on `azure.core.rest.HttpResponse` and `azure.core.rest.AsyncHttpResponse` has changed to a method, which also takes

sdk/core/azure-core/azure/core/rest/_helpers.py

@@ -50,6 +50,7 @@
     from urlparse import urlparse  # type: ignore
 except ImportError:
     from urllib.parse import urlparse
+from azure.core.serialization import AzureJSONEncoder
 
 ################################### TYPES SECTION #########################
 
@@ -182,7 +183,7 @@ def set_content_body(content):
 
 def set_json_body(json):
     # type: (Any) -> Tuple[Dict[str, str], Any]
-    body = dumps(json)
+    body = dumps(json, cls=AzureJSONEncoder)
     return {
         "Content-Type": "application/json",
         "Content-Length": str(len(body))

sdk/core/azure-core/tests/testserver_tests/async_tests/conftest.py renamed to sdk/core/azure-core/tests/async_tests/conftest.py

@@ -66,4 +66,4 @@ async def __aenter__(self):
         return self
 
     async def __aexit__(self, *exc_details) -> None:
-        await self._client.__aexit__(*exc_details)
+        await self._client.__aexit__(*exc_details)

sdk/core/azure-core/tests/testserver_tests/async_tests/rest_client_async.py renamed to sdk/core/azure-core/tests/async_tests/rest_client_async.py

@@ -3,14 +3,6 @@
 # Licensed under the MIT License. See LICENSE.txt in the project root for
 # license information.
 # -------------------------------------------------------------------------
-from six.moves.http_client import HTTPConnection
-import time
-
-try:
-    from unittest import mock
-except ImportError:
-    import mock
-
 from azure.core.pipeline.transport import HttpRequest, AsyncHttpResponse, AsyncHttpTransport, AioHttpTransport
 from azure.core.pipeline.policies import HeadersPolicy
 from azure.core.pipeline import AsyncPipeline
@@ -42,9 +34,9 @@ def body(self):
 
 
 @pytest.mark.asyncio
-async def test_basic_options_aiohttp():
+async def test_basic_options_aiohttp(port):
 
-    request = HttpRequest("OPTIONS", "https://httpbin.org")
+    request = HttpRequest("OPTIONS", "http://localhost:{}/basic/string".format(port))
     async with AsyncPipeline(AioHttpTransport(), policies=[]) as pipeline:
         response = await pipeline.run(request)
 

sdk/core/azure-core/tests/async_tests/test_basic_transport_async.py

@@ -41,7 +41,7 @@ def emit(self, record):
 
     policy = HttpLoggingPolicy(logger=logger)
 
-    universal_request = HttpRequest('GET', 'http://127.0.0.1/')
+    universal_request = HttpRequest('GET', 'http://localhost/')
     http_response = HttpResponse(universal_request, None)
     http_response.status_code = 202
     request = PipelineRequest(universal_request, PipelineContext(None))
@@ -54,7 +54,7 @@ def emit(self, record):
 
     assert all(m.levelname == 'INFO' for m in mock_handler.messages)
     assert len(mock_handler.messages) == 6
-    assert mock_handler.messages[0].message == "Request URL: 'http://127.0.0.1/'"
+    assert mock_handler.messages[0].message == "Request URL: 'http://localhost/'"
     assert mock_handler.messages[1].message == "Request method: 'GET'"
     assert mock_handler.messages[2].message == 'Request headers:'
     assert mock_handler.messages[3].message == 'No body was attached to the request'
@@ -75,13 +75,13 @@ def emit(self, record):
 
     assert all(m.levelname == 'INFO' for m in mock_handler.messages)
     assert len(mock_handler.messages) == 12
-    assert mock_handler.messages[0].message == "Request URL: 'http://127.0.0.1/'"
+    assert mock_handler.messages[0].message == "Request URL: 'http://localhost/'"
     assert mock_handler.messages[1].message == "Request method: 'GET'"
     assert mock_handler.messages[2].message == 'Request headers:'
     assert mock_handler.messages[3].message == 'No body was attached to the request'
     assert mock_handler.messages[4].message == 'Response status: 202'
     assert mock_handler.messages[5].message == 'Response headers:'
-    assert mock_handler.messages[6].message == "Request URL: 'http://127.0.0.1/'"
+    assert mock_handler.messages[6].message == "Request URL: 'http://localhost/'"
     assert mock_handler.messages[7].message == "Request method: 'GET'"
     assert mock_handler.messages[8].message == 'Request headers:'
     assert mock_handler.messages[9].message == 'No body was attached to the request'
@@ -102,15 +102,15 @@ def emit(self, record):
         "Content-Type": "Caramel",
         "HateToo": "Chocolat",
     }
-    universal_request.url = "http://127.0.0.1/?country=france&city=aix"
+    universal_request.url = "http://localhost/?country=france&city=aix"
 
     policy.on_request(request)
     response = PipelineResponse(request, http_response, request.context)
     policy.on_response(request, response)
 
     assert all(m.levelname == 'INFO' for m in mock_handler.messages)
     assert len(mock_handler.messages) == 10
-    assert mock_handler.messages[0].message == "Request URL: 'http://127.0.0.1/?country=france&city=REDACTED'"
+    assert mock_handler.messages[0].message == "Request URL: 'http://localhost/?country=france&city=REDACTED'"
     assert mock_handler.messages[1].message == "Request method: 'GET'"
     assert mock_handler.messages[2].message == "Request headers:"
     # Dict not ordered in Python, exact logging order doesn't matter
@@ -156,7 +156,7 @@ def emit(self, record):
     policy = HttpLoggingPolicy()
     kwargs={'logger': logger}
 
-    universal_request = HttpRequest('GET', 'http://127.0.0.1/')
+    universal_request = HttpRequest('GET', 'http://localhost/')
     http_response = HttpResponse(universal_request, None)
     http_response.status_code = 202
     request = PipelineRequest(universal_request, PipelineContext(None, **kwargs))
@@ -169,7 +169,7 @@ def emit(self, record):
 
     assert all(m.levelname == 'INFO' for m in mock_handler.messages)
     assert len(mock_handler.messages) == 6
-    assert mock_handler.messages[0].message == "Request URL: 'http://127.0.0.1/'"
+    assert mock_handler.messages[0].message == "Request URL: 'http://localhost/'"
     assert mock_handler.messages[1].message == "Request method: 'GET'"
     assert mock_handler.messages[2].message == 'Request headers:'
     assert mock_handler.messages[3].message == 'No body was attached to the request'
@@ -192,13 +192,13 @@ def emit(self, record):
 
     assert all(m.levelname == 'INFO' for m in mock_handler.messages)
     assert len(mock_handler.messages) == 12
-    assert mock_handler.messages[0].message == "Request URL: 'http://127.0.0.1/'"
+    assert mock_handler.messages[0].message == "Request URL: 'http://localhost/'"
     assert mock_handler.messages[1].message == "Request method: 'GET'"
     assert mock_handler.messages[2].message == 'Request headers:'
     assert mock_handler.messages[3].message == 'No body was attached to the request'
     assert mock_handler.messages[4].message == 'Response status: 202'
     assert mock_handler.messages[5].message == 'Response headers:'
-    assert mock_handler.messages[6].message == "Request URL: 'http://127.0.0.1/'"
+    assert mock_handler.messages[6].message == "Request URL: 'http://localhost/'"
     assert mock_handler.messages[7].message == "Request method: 'GET'"
     assert mock_handler.messages[8].message == 'Request headers:'
     assert mock_handler.messages[9].message == 'No body was attached to the request'
@@ -226,7 +226,7 @@ def emit(self, record):
 
     policy = HttpLoggingPolicy(logger=logger)
 
-    universal_request = HttpRequest('GET', 'http://127.0.0.1/')
+    universal_request = HttpRequest('GET', 'http://localhost/')
     universal_request.body = "testbody"
     http_response = HttpResponse(universal_request, None)
     http_response.status_code = 202
@@ -238,7 +238,7 @@ def emit(self, record):
 
     assert all(m.levelname == 'INFO' for m in mock_handler.messages)
     assert len(mock_handler.messages) == 6
-    assert mock_handler.messages[0].message == "Request URL: 'http://127.0.0.1/'"
+    assert mock_handler.messages[0].message == "Request URL: 'http://localhost/'"
     assert mock_handler.messages[1].message == "Request method: 'GET'"
     assert mock_handler.messages[2].message == 'Request headers:'
     assert mock_handler.messages[3].message == 'A body is sent with the request'
@@ -267,7 +267,7 @@ def emit(self, record):
 
     policy = HttpLoggingPolicy(logger=logger)
 
-    universal_request = HttpRequest('GET', 'http://127.0.0.1/')
+    universal_request = HttpRequest('GET', 'http://localhost/')
     mock = Mock()
     mock.__class__ = types.AsyncGeneratorType
     universal_request.body = mock
@@ -281,7 +281,7 @@ def emit(self, record):
 
     assert all(m.levelname == 'INFO' for m in mock_handler.messages)
     assert len(mock_handler.messages) == 6
-    assert mock_handler.messages[0].message == "Request URL: 'http://127.0.0.1/'"
+    assert mock_handler.messages[0].message == "Request URL: 'http://localhost/'"
     assert mock_handler.messages[1].message == "Request method: 'GET'"
     assert mock_handler.messages[2].message == 'Request headers:'
     assert mock_handler.messages[3].message == 'File upload'