Merge pull request #5 from Azure/master

get updates from azure repo

Author: LibbaLawrence

.github/CODEOWNERS

@@ -10,7 +10,7 @@
 
 # Core
 /sdk/core/                                                           @lmazuel @xiangyan99 @johanste
-/sdk/core/azure-core/                                                @xiangyan99 @bryevdv @lmazuel
+/sdk/core/azure-core/                                                @xiangyan99 @lmazuel
 
 # Service team
 /sdk/identity/                                                       @chlowell @schaabs
@@ -40,6 +40,9 @@
 /sdk/textanalytics/                                                  @kristapratico @iscai-msft
 /sdk/formrecognizer/                                                 @kristapratico @iscai-msft
 
+# Smoke Tests
+/common/smoketest/                                                   @lmazuel @chlowell @annatisch @rakshith91 @shurd @southpolesteve
+
 # Management Plane
 /**/*mgmt*/     @zikalino
 

common/smoketest/README.md

@@ -74,7 +74,7 @@ pip install -r requiriments.txt
 pip install -r requiriments_async.txt
 ```
 
-If a python version below 3.5 is being used, it is still possible to run the samples. When it gets to the async tests a message `'Async not suported'` will be displayed.
+If a python version below 3.5 is being used, it is still possible to run the samples. When it gets to the async tests a message `'Async not supported'` will be displayed.
 
 ## Key concepts
 

common/smoketest/dependencies.py

@@ -0,0 +1,71 @@
+import argparse
+import pkg_resources
+
+try:
+    # pip < 20
+    from pip._internal.req import parse_requirements
+    from pip._internal.download import PipSession
+except:
+    # pip >= 20
+    from pip._internal.req import parse_requirements
+    from pip._internal.network.session import PipSession
+
+def combine_requirements(requirements):
+    name = requirements[0].project_name
+    specs = []
+    for req in requirements:
+        if len(req.specs) == 0:
+            continue
+
+        specs.extend([s[0] + s[1] for s in req.specs])
+
+    return name + ",".join(specs)
+
+def get_dependencies(packages):
+    requirements = []
+    for package in packages:
+        package_info = pkg_resources.working_set.by_key[package]
+
+        applicable_requirements = [r for r in package_info.requires() if r.marker is None or r.marker.evaluate()]
+        requirements.extend(applicable_requirements)
+
+    return requirements
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="List dependencies for a given requirements.txt file"
+    )
+
+    parser.add_argument(
+        "-r",
+        "--requirements",
+        dest="requirements_file",
+        help="File containing list of packages for which to find dependencies",
+        required=True
+    )
+
+    args = parser.parse_args()
+    # Get package names from requirements.txt
+    requirements = parse_requirements(args.requirements_file, session=PipSession())
+    package_names = [item.req.name for item in requirements]
+
+    dependencies = get_dependencies(package_names)
+
+    # It may be the case that packages have multiple sets of dependency
+    # requirements, for example:
+    # Package A requires Foo>=1.0.0,<2.0.0
+    # Package B requires Foo>=1.0.0,<1.2.3
+    # This combines all required versions into one string for pip to resolve
+    # Output: Foo>=1.0.0,<2.0.0,>=1.0.0,<1.2.3
+    # Pip parses this value using the Requirement object (https://setuptools.readthedocs.io/en/latest/pkg_resources.html#requirement-objects)
+    # According to https://packaging.python.org/glossary/#term-requirement-specifier
+    grouped_dependencies = {}
+    for dep in dependencies:
+        if dep.key in grouped_dependencies:
+            grouped_dependencies[dep.key].append(dep)
+        else:
+            grouped_dependencies[dep.key] = [dep]
+
+    final_dependencies = [combine_requirements(r) for r in grouped_dependencies.values()]
+
+    print("\n".join(final_dependencies))

common/smoketest/smoke-test.yml

@@ -9,58 +9,58 @@ jobs:
           PythonVersion: '2.7'
           InstallAsyncRequirements: false
           OSVmImage: ubuntu-18.04
-          CloudType: AzureCloud
+          SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
           ArmTemplateParameters: $(azureCloudArmParameters)
         Python_37_Linux (AzureCloud):
           PythonVersion: '3.7'
           OSVmImage: ubuntu-18.04
-          CloudType: AzureCloud
+          SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
           ArmTemplateParameters: $(azureCloudArmParameters)
         Python_38_Linux (AzureCloud):
           PythonVersion: '3.8'
           OSVmImage: ubuntu-18.04
-          CloudType: AzureCloud
+          SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
           ArmTemplateParameters: $(azureCloudArmParameters)
         Python_37_Windows (AzureCloud):
           PythonVersion: '3.7'
           OSVmImage: windows-2019
-          CloudType: AzureCloud
+          SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
           ArmTemplateParameters: $(azureCloudArmParameters)
         Python_38_Windows (AzureCloud):
           PythonVersion: '3.8'
           OSVmImage: windows-2019
-          CloudType: AzureCloud
+          SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
           ArmTemplateParameters: $(azureCloudArmParameters)
         Python_37_Mac (AzureCloud):
           PythonVersion: '3.7'
           OSVmImage: macOS-10.15
-          CloudType: AzureCloud
+          SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
           ArmTemplateParameters: $(azureCloudArmParameters)
         Python_38_Mac (AzureCloud):
           PythonVersion: '3.8'
           OSVmImage: macOS-10.15
-          CloudType: AzureCloud
+          SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
           ArmTemplateParameters: $(azureCloudArmParameters)
         Python_38_Linux (AzureUSGovernment):
           PythonVersion: '3.8'
           OSVmImage: ubuntu-18.04
-          CloudType: AzureUSGovernment
+          SubscriptionConfiguration: $(sub-config-gov-test-resources)
           ArmTemplateParameters: $(azureUSGovernmentArmParameters)
         Python_37_Windows (AzureUSGovernment):
           PythonVersion: '3.7'
           OSVmImage: windows-2019
-          CloudType: AzureUSGovernment
+          SubscriptionConfiguration: $(sub-config-gov-test-resources)
           ArmTemplateParameters: $(azureUSGovernmentArmParameters)
         Python_38_Linux (AzureChinaCloud):
           PythonVersion: '3.8'
           OSVmImage: ubuntu-18.04
-          CloudType: AzureChinaCloud
+          SubscriptionConfiguration: $(sub-config-cn-test-resources)
           Location: 'chinanorth'
           ArmTemplateParameters: $(azureChinaCloudArmParameters)
         Python_37_Windows (AzureChinaCloud):
           PythonVersion: '3.7'
           OSVmImage: windows-2019
-          CloudType: AzureChinaCloud
+          SubscriptionConfiguration: $(sub-config-cn-test-resources)
           Location: 'chinanorth'
           ArmTemplateParameters: $(azureChinaCloudArmParameters)
 
@@ -84,29 +84,34 @@ jobs:
           pip --version
         displayName: pip --version
 
-      - script: pip install packaging pkginfo
-        displayName: Install requirements for dev tools
-
-      - script: pip install -r ./common/smoketest/requirements.txt
-        displayName: "Install requirements.txt"
+      - script: pip install -r ./common/smoketest/requirements.txt --pre --no-deps --index-url https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple
+        displayName: Install requirements from dev feed without dependencies
 
       - script: pip install -r ./common/smoketest/requirements_async.txt
         displayName: "Install requirements_async.txt"
         condition: and(succeeded(), eq(variables['InstallAsyncRequirements'], 'true'))
 
-      - script: python ./eng/tox/install_dev_build_dependency.py -r ./common/smoketest/requirements.txt
-        displayName: "Install dev dependencies from feed"
+      - script: python ./common/smoketest/dependencies.py -r ./common/smoketest/requirements.txt | tee ./common/smoketest/requirements_dependencies.txt
+        displayName: Create dependency list from installed dev packages
+
+      - script: pip install -r ./common/smoketest/requirements_dependencies.txt
+        displayName: Install dev package dependencies from PyPI
+
+      - script: pip freeze
+        displayName: Show installed packages (pip freeze)
 
       - template: /eng/common/TestResources/deploy-test-resources.yml
         parameters:
           ServiceDirectory: '$(Build.SourcesDirectory)/common/smoketest/'
-          CloudType: $(CloudType)
+          SubscriptionConfiguration: $(SubscriptionConfiguration)
           Location: $(Location)
           ArmTemplateParameters: $(ArmTemplateParameters)
 
       - script: python ./common/smoketest/program.py
-        displayName: "Run Smoke Test"
+        displayName: Run Smoke Test
 
       - template: /eng/common/TestResources/remove-test-resources.yml
         parameters:
           ServiceDirectory: '$(Build.SourcesDirectory)/common/smoketest/'
+          SubscriptionConfiguration: $(SubscriptionConfiguration)
+

doc/dev/mgmt/generating-integration-test.md

@@ -127,7 +127,7 @@ now you can run live integration test:
 
 ## Fixing Test
 
-It's obvious that when running test for the first time someting is not going to work.
+It's obvious that when running test for the first time something is not going to work.
 
 The best approach is to:
 - fix the test manually

doc/dev/mgmt/generation.md

@@ -8,7 +8,7 @@ IMPORTANT NOTE: All the commands prefixed by `python` in this page assumes you h
 
 ### Autorest versioning
 
-A few notes on [Autorest for Python versionning](https://github.com/Azure/autorest.python/blob/master/ChangeLog.md):
+A few notes on [Autorest for Python versioning](https://github.com/Azure/autorest.python/blob/master/ChangeLog.md):
 - Autorest for Python v2.x is deprecated, and should not be used anymore for any generation under any circumstances.
 - Autorest for Python v3.x is the most currently used one. Should not be used, but still ok if service team are still in v3.x and they want to avoid breaking changes for a given version (rare).
 - Autorest for Python v4.x is the current recommendation. This generator can generates async code, but this should be disabled with --no-async. No package should be shipped with async based on v4

doc/dev/mgmt/multiapi.md

@@ -14,7 +14,7 @@ Because there is different flavors of Azure that are not necessarly provided wit
 
 ### Why a multi-api package?
 
-Indeed, a simple solution would be to write down explictly what version of SDK supports what API version. Example: 1.0 supports 2015-06-01, 2.0 supports 2017-07-01, etc. The story for customers then would be to pin the specific SDK version for the specific API version they need. However, this was considered unacceptable in an end-to-end scenario:
+Indeed, a simple solution would be to write down explicitly what version of SDK supports what API version. Example: 1.0 supports 2015-06-01, 2.0 supports 2017-07-01, etc. The story for customers then would be to pin the specific SDK version for the specific API version they need. However, this was considered unacceptable in an end-to-end scenario:
 - It means you cannot install in the same Python environment packages that would target different cloud (Python doesn't allow installation of different versions of the same package together). Azure CLI or Ansible supports for different clouds would then be extremely complicated.
 - This forces customers to use old SDK, that might have been fixed on different axis than API version (security fixes, new SDK features like async, etc.)
 - Customers rarely needs only one package, but a set of them (storage, compute, network, etc.) and having to keep track of the correct list of packages is challenging.
@@ -44,7 +44,7 @@ Network interfaces operations are defines in a [network interface file](https://
 
 **Python multi-api packaging is based on the assumptions that it's true.** If it's not, it's usually ok but requires a little more subtle packaging (see final section here)
 
-Being that a given Swagger defines only *one* fixed API version, doing multi-api version in one package implies shipping several Swagger files into one package. This is achived by the `batch` directive of Autorest. More details on how to write Readme for Swagger in the specific page for it [swagger_conf.md](./swagger_conf.md).
+Being that a given Swagger defines only *one* fixed API version, doing multi-api version in one package implies shipping several Swagger files into one package. This is archived by the `batch` directive of Autorest. More details on how to write Readme for Swagger in the specific page for it [swagger_conf.md](./swagger_conf.md).
 
 Python SDK team is responsible to design the correct set of tags to set for the `batch` node. Each line of the batch directive should contains only *one* api version to match the folder name used. this might require adding new tags in the readme.md that are specific to only one API version. These tags are usually suffixed by "-only" ([example with compute](https://github.com/Azure/azure-rest-api-specs/tree/master/specification/compute/resource-manager#tag-package-2019-03-01-only))
 

doc/dev/mgmt/swagger_conf.md

@@ -14,7 +14,7 @@ In practical terms, we want to control the version of Autorest used, the output
 
 ## Writing the readme
 
-Writing the readme is the responsability of the Python SDK team. There is currently two types of templates for Python readmes:
+Writing the readme is the responsibility of the Python SDK team. There is currently two types of templates for Python readmes:
 - Readme that handles only one API version, and generates packages that handle one API version only
 - Readme that handles several API versions, and generates packages with multiples API and profile supports
 

doc/sphinx/conf.py

@@ -67,8 +67,8 @@
     'trio': ('https://trio.readthedocs.io/en/stable/', None),
     'msal': ('https://msal-python.readthedocs.io/en/latest/', None),
     # Azure packages
-    'azure-core': ('https://azuresdkdocs.blob.core.windows.net/$web/python/azure-core/1.1.1/', None),
-    'azure-identity': ('https://azuresdkdocs.blob.core.windows.net/$web/python/azure-identity/1.1.0/', None),
+    'azure-core': ('https://azuresdkdocs.blob.core.windows.net/$web/python/azure-core/latest/', None),
+    'azure-identity': ('https://azuresdkdocs.blob.core.windows.net/$web/python/azure-identity/latest/', None),
 }
 
 autodoc_member_order = 'groupwise'

eng/.docsettings.yml

@@ -78,6 +78,7 @@ known_content_issues:
   - ['sdk/graphrbac/azure-graphrbac/README.md', '#4554']
   - ['sdk/loganalytics/azure-loganalytics/README.md', '#4554']
   - ['sdk/servicebus/azure-servicebus/README.md', '#4554']
+  - ['sdk/servicebus/azure-servicebus/swagger/README.md', '#4554']
   - ['sdk/servicefabric/azure-servicefabric/README.md', '#4554']
   - ['sdk/storage/azure-storage-nspkg/README.md', '#4554']
   - ['sdk/storage/azure-storage-blob/swagger/README.md', '#4554']

eng/autorest_req.txt

@@ -1,8 +1,8 @@
-pytest==5.4.1; python_version >= '3.5'
+pytest==5.4.2; python_version >= '3.5'
 pytest==4.6.9; python_version == '2.7'
 pytest-cov==2.8.1
-pytest-asyncio==0.10.0; python_version >= '3.5'
+pytest-asyncio==0.12.0; python_version >= '3.5'
 isodate==0.6.0
-msrest==0.6.13
+msrest==0.6.14
 aiohttp==3.6.2
-wheel
+wheel==0.34.2

eng/ci_tools.txt

@@ -3,7 +3,7 @@ setuptools==44.1.0; python_version == '2.7'
 setuptools==45.1.0; python_version >= '3.5'
 wheel==0.34.2 
 Jinja2==2.11.1
-packaging==20.3
+packaging==20.4
 tox==3.14.6
 tox-monorepo==0.1.2
 twine==1.15.0

eng/common/TestResources/New-TestResources.ps1

@@ -156,7 +156,7 @@ if ($ProvisionerApplicationId) {
     }
 
     $provisionerAccount = Retry {
-        Connect-AzAccount -Tenant $TenantId -Credential $provisionerCredential -ServicePrincipal -Environment $Environment @subscriptionArgs
+        Connect-AzAccount -Force:$Force -Tenant $TenantId -Credential $provisionerCredential -ServicePrincipal -Environment $Environment @subscriptionArgs
     }
 
     $exitActions += {