iscai-msft
diff --git a/‎.github/CODEOWNERS
+2-1 b/‎.github/CODEOWNERS
+2-1
diff --git a/‎doc/dev/README.md
+1-1 b/‎doc/dev/README.md
+1-1
diff --git a/‎doc/dev/get_unreleased_package_guide.md
+1-1 b/‎doc/dev/get_unreleased_package_guide.md
+1-1
diff --git a/‎doc/dev/package_version/package_version_rule.md
+44 b/‎doc/dev/package_version/package_version_rule.md
+44
diff --git a/‎doc/dev/package_version/version summary.xlsx
11.1 KB b/‎doc/dev/package_version/version summary.xlsx
11.1 KB
diff --git a/‎doc/dev/package_version/version_summary.png
14.2 KB b/‎doc/dev/package_version/version_summary.png
14.2 KB
diff --git a/‎doc/dev/packaging.md
+62-26 b/‎doc/dev/packaging.md
+62-26
diff --git a/‎doc/dev/perfstress_tests.md
+1-1 b/‎doc/dev/perfstress_tests.md
+1-1
diff --git a/‎doc/dev/test_proxy_migration_guide.md
+50-8 b/‎doc/dev/test_proxy_migration_guide.md
+50-8
diff --git a/‎eng/common/TestResources/New-TestResources.ps1
+24-6 b/‎eng/common/TestResources/New-TestResources.ps1
+24-6
@@ -41,6 +41,7 @@
 /sdk/communication/                                                  @acsdevx-msft
 /sdk/communication/azure-communication-phonenumbers/                 @RoyHerrod @danielav7 @whisper6284 @AlonsoMondal
 /sdk/communication/azure-communication-sms/                          @RoyHerrod @arifibrahim4
+/sdk/communication/azure-communication-identity/                     @Azure/acs-identity-sdk
 
 # PRLabel: %KeyVault
 /sdk/keyvault/                                                       @schaabs @chlowell @mccoyp @YalinLi0312
@@ -65,7 +66,7 @@
 /sdk/containerservice/                                               @samkreter @zqingqing1 @GaneshaThirumurthi
 
 # PRLabel: %Cosmos
-/sdk/cosmos/                                                         @southpolesteve @zfoster
+/sdk/cosmos/                                                         @kushagraThapar @simorenoh @simplynaveen20 @xinlian12 @moderakh 
 
 # PRLabel: %Data Factory
 /sdk/datafactory/                                                    @hvermis
 
@@ -7,7 +7,7 @@ Overview of the documents:
 - [Release](https://github.com/Azure/azure-sdk-for-python/blob/main/doc/dev/release.md) : How to release a package when ready
 - [Packaging](https://github.com/Azure/azure-sdk-for-python/blob/main/doc/dev/packaging.md) : How to organize packaging information for packages under `azure`
 - [Testing](https://github.com/Azure/azure-sdk-for-python/blob/main/doc/dev/tests.md): How to write unit and functional tests for a library
-- [Docstrings and Type hints](https://github.com/Azure/azure-sdk-for-python/blob/main/doc/dev/docstring_typehint.md): How to document an SDK for compatability with internal tools (API View) and our documentation at [MS Docs][ms_docs] and the [azure.github.io][azure_github_io] site.
+- [Docstrings and Type hints](https://github.com/Azure/azure-sdk-for-python/blob/main/doc/dev/docstring_typehint.md): How to document an SDK for compatibility with internal tools (API View) and our documentation at [MS Docs][ms_docs] and the [azure.github.io][azure_github_io] site.
 
 The [mgmt](https://github.com/Azure/azure-sdk-for-python/blob/main/doc/dev/mgmt) folder contains information specific to management packages (i.e. packages prefixed by `azure-mgmt`)
 
 
@@ -16,4 +16,4 @@ The following figure shows the wheel and zip of the package.Click to download th
 (1) If there is no link in the figure above, it may be folded. You can also find it in the check.
 ![img.png](unreleased_package_guide_example3.png)
 
-(2) [Private repo](https://github.com/Azure/azure-rest-api-specs-pr) can only be triggered when the target branch is `main`
+(2) The private Azure/azure-rest-api-specs-pr repo can only be triggered when the target branch is `main`
@@ -0,0 +1,44 @@
+This file claims the rules how Python decide next version number for package.
+
+The package version contains two part:
+1. the package is preview or stable?
+2. version number
+
+# How to judge preview or stable?
+Python SDK is generated with [swagger](https://github.com/Azure/azure-rest-api-specs), so if swagger content is preview, 
+the package is preview; if swagger content is stable, the package is stable.
+
+(1) For single api package(for example: [datadog](https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/datadog)),
+ as long as the current tag is preview, the package version should be preview
+
+(2) For multi api package(for example: [network](https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/network/azure-mgmt-network)), 
+there will be `DEFAULT_API_VERSION`(for example: [`DEFAULT_API_VERSION` of network](https://github.com/Azure/azure-sdk-for-python/blob/59709af16b7cd29a51d562137bc5bbfdf53f9327/sdk/network/azure-mgmt-network/azure/mgmt/network/_network_management_client.py#L60)). 
+As long as it is preview, then the package version is preview.
+
+(note1: If the name of tag contains 'preview' or the tag contains files of 'preview' folder, then the tag is preview tag. 
+For exampe: [preview tag](https://github.com/Azure/azure-rest-api-specs/tree/main/specification/compute/resource-manager#tag-package-2021-06-01-preview))
+
+(note2: If the api-version contains 'preview', then it is preview api-version. for example: [preview api-version](https://github.com/Azure/azure-rest-api-specs/blob/69eacf00a36d565d3220d5dd6f4a5293664f1ae9/specification/network/resource-manager/Microsoft.Network/preview/2015-05-01-preview/network.json#L6))
+
+(note3: The difference about single api and multi api, please see the detailed file)
+
+(note4: preview package version contains `b`, for example: `1.0.0b1`)
+
+# How to decide next version number
+1\. If current version is preview version, the new tag is preview tag, then next version is `x.x.xbx+1`
+
+2\. If current version is stable version, the new tag is stable tag, then :
+ * if there is breaking change, next version is `x+1.x.x`
+ * if there is new feature but no breaking change, next version is `x.x+1.x`
+ * if there is only bugfix, next version is `x.x.x+1`
+
+3\. If current version is stable version, the new tag is preview tag, calculate version number according to `2` 
+and then append `b1` in the result
+
+
+According to the up rules, we could summarize all the possibilities in the following table:
+
+![img.png](version_summary.png)
+
+(`-` means that this item doesn't influence result)
+ 
@@ -1,26 +1,25 @@
 # Azure packaging
 
-This article describes how to declare setup.py and all packaging information for packages inside the `azure` namespace
+This article describes the recommendations for defining namespace packaging to release a package inside the `azure` namespace. Being inside the `azure` namespace means that a service `myservice` can be imported using:
+```python
+import azure.myservice
+```
 
 Namespace packaging is complicated in Python, here's a few reading if you still doubt it:
 - https://packaging.python.org/guides/packaging-namespace-packages/
 - https://www.python.org/dev/peps/pep-0420/
 - https://github.com/pypa/sample-namespace-packages
 
-This article describes the recommendation on how to define namespace packaging to release a package inside the `azure` namespace. Being inside the `azure` namespace meaning you have a service `myservice` that you want to import using:
-```python
-import azure.myservice
-```
-
 Note:
-- This article is not about setup.py or setup.cfg or the right way to *write* the packaging, it's about what instructions you should use to achieve this. If you are fluent in setuptools, and prefer to write the suggestions in setup.cfg and not in setup.py, this is not a concern.
+While this article provides an example using setup.py, this can also be achieved with setup.cfg or other methods, as long as the constraints on the final wheels/sdist are met.
+
+*This page has been updated to be Python 3 only packages as we do not recommend supporting Python 2 after January 1st 2022.* If you still want to support Python 2 for some reasons, there is a section at the bottom with some details (or you have the Github history, to read the page as it was on November 1st 2021).
 
 # What are the constraints?
 
 We want to build sdist and wheels in order to follow the following constraints:
 - Solution should work with *recent* versions of pip and setuptools (not the very latest only, but not archaeology either)
-- Wheels must work with Python 2.7 and 3.6+
-- easy-install scenario is a plus, but cannot be considered critical anymore
+- Wheels must work with Python 3.7+
 - mixed dev installation and PyPI installation should be explicitly addressed
 
 # What do I do in my files to achieve that
@@ -55,13 +54,9 @@ The "packages" section MUST EXCLUDE the `azure` package. Example:
     ]),
 ```
 
-The "extras_requires" section MUST include a conditional dependency on "azure-nspkg" for Python 2. There is also a conditional dependency on "typing" for Python 3.5 because of the type-hinting for Python 3.5 and above. Example:
-
+Since the package is Python 3 only, you must notify it in the setup.py as well:
 ```python
-    extras_require={
-        ":python_version<'3.0'": ['azure-nspkg'],
-        ":python_version<'3.5'": ['typing'],
-    }
+    python_requires=">=3.7",
 ```
 
 Example of a full setup.py
@@ -113,16 +108,15 @@ setup(
     classifiers=[
         'Development Status :: 4 - Beta',
         'Programming Language :: Python',
-        'Programming Language :: Python :: 2',
-        'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: 3 :: Only',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.5',
-        'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3.9',
+        'Programming Language :: Python :: 3.10',
         'License :: OSI Approved :: MIT License',
     ],
+    python_requires=">=3.7",
     zip_safe=False,
     packages=find_packages(exclude=[
         'tests',
@@ -134,17 +128,59 @@ setup(
         'msrestazure>=0.4.32,<2.0.0',
         'azure-common~=1.1',
     ],
-    extras_require={
-        ":python_version<'3.0'": ['azure-nspkg'],
-        ":python_version<'3.5'": ['typing'],
-    }
 )
 ```
 
-This syntax works with setuptools >= 17.1 and pip >= 6.0, which is considered enough to support in 2019.
+This syntax works with setuptools >= 24.2.0 (July 2016) and pip >= 9.0 (Nov 2016), which is considered enough to support in 2021.
+
+Since the package is Python 3 only, do NOT make this wheel universal. This usually means you should NOT have `universal=1` in the `setup.cfg`. It may mean you can completely remove the file if `universal` was the only configuration option inside.
 
 # How can I check if my packages are built correctly?
 
-- wheels must NOT contain a `azure/__init__.py` file (you can open it with a zip util to check)
-- wheels installs `azure-nskpg` ONLY on Python 2.
+- wheel file must NOT contain a `azure/__init__.py` file (you can open it with a zip util to check)
+- wheel file name suffix is `py3-none-any`, and NOT `py2.py3-none-any`.
 - sdist must contain a `azure/__init__.py` file that declares `azure` as a namespace package using the `pkgutil` syntax
+
+# I already have a package that supports Python 2, can I get short version on how to udpate to Python 3 only?
+
+- Remove "universal" from setup.cfg, or completly remove the file if it was the only option
+- In setup.py:
+  - Remove `extra_requires`
+  - Add `python_requires=">=3.7",`
+  - Remove the Python 2 and 3.5/3.6 classifiers
+  - Add classifier `Programming Language :: Python :: 3 :: Only`
+  - Remove the "azure" check if applicable (see next note)
+
+# Note on checking old Azure packages
+
+You may see code in `setup.py` looking like this:
+```python
+# azure v0.x is not compatible with this package
+# azure v0.x used to have a __version__ attribute (newer versions don't)
+try:
+    import azure
+
+    try:
+        VER = azure.__version__  # type: ignore
+        raise Exception(
+            "This package is incompatible with azure=={}. ".format(VER) + 'Uninstall it with "pip uninstall azure".'
+        )
+    except AttributeError:
+        pass
+except ImportError:
+    pass
+```
+
+This was to prevent some difficult update scenario 6 years ago, and can be safely removed from your setup.py
+
+# Note on Python 2
+
+The "extras_requires" section MUST include a conditional dependency on "azure-nspkg" for Python 2. Example:
+
+```python
+    extras_require={
+        ":python_version<'3.0'": ['azure-nspkg'],
+    }
+```
+
+An additional verification is that wheels installs `azure-nskpg` ONLY on Python 2.
@@ -82,7 +82,7 @@ The framework has a series of common command line options built in:
 - `-w --warm-up=5` Number of seconds to spend warming up the connection before measuring begins. Default is 5.
 - `--sync` Whether to run the tests in sync or async. Default is False (async).
 - `--no-cleanup` Whether to keep newly created resources after test run. Default is False (resources will be deleted).
-- `-x --test-proxy` Whether to run the tests against the test proxy server. Specfiy the URL for the proxy endpoint (e.g. "https://localhost:5001").
+- `-x --test-proxies` Whether to run the tests against the test proxy server. Specify the URL(s) for the proxy endpoint(s) (e.g. "https://localhost:5001").
 - `--profile` Whether to run the perftest with cProfile. If enabled (default is False), the output file of the **last completed single iteration** will be written to the current working directory in the format `"cProfile-<TestClassName>-<TestID>-<sync/async>.pstats"`.
 
 
 
@@ -28,25 +28,25 @@ class TestExample(AzureTestCase):
 ### New test structure
 
 To use the proxy, test classes should inherit from AzureRecordedTestCase and recorded test methods should use a
-RecordedByProxy decorator:
+`recorded_by_proxy` decorator:
 
 ```py
-from devtools_testutils import AzureRecordedTestCase, RecordedByProxy
+from devtools_testutils import AzureRecordedTestCase, recorded_by_proxy
 
 class TestExample(AzureRecordedTestCase):
 
-    @RecordedByProxy
+    @recorded_by_proxy
     def test_example(self):
         ...
 
     @ExamplePreparer()
-    @RecordedByProxy
+    @recorded_by_proxy
     def test_example_with_preparer(self):
         ...
 ```
 
-For async tests, import the RecordedByProxyAsync decorator from `devtools_testutils.aio` and use it in the same
-way as RecordedByProxy.
+For async tests, import the `recorded_by_proxy_async` decorator from `devtools_testutils.aio` and use it in the same
+way as `recorded_by_proxy`.
 
 > **Note:** since AzureRecordedTestCase doesn't inherit from `unittest.TestCase`, test class names need to start
 > with "Test" in order to be properly collected by pytest by default. For more information, please refer to
@@ -135,6 +135,48 @@ made to `https://fakeendpoint-secondary.table.core.windows.net`, and URIs will a
 
 For more details about sanitizers and their options, please refer to [devtools_testutils/sanitizers.py][py_sanitizers].
 
+### Enabling Testproxy on the CI
+
+To enable test proxy on the CI, you need to set the parameter `TestProxy: true` on ci.yml and on tests.yml are present in the service level folder.
+
+![image](https://user-images.githubusercontent.com/45376673/142270668-5be58bca-87e5-45f5-b593-44f8b1f757bc.png)
+### Record test variables
+
+To run recorded tests successfully when there's an element of non-secret randomness to them, the test proxy provides a
+[`variables` API](https://github.com/Azure/azure-sdk-tools/tree/main/tools/test-proxy/Azure.Sdk.Tools.TestProxy#storing-variables).
+This makes it possible for a test to record the values of variables that were used during recording and use the same
+values in playback mode without a sanitizer.
+
+For example, imagine that a test uses a randomized `table_name` variable when creating resources. The same random value
+for `table_name` can be used in playback mode by using this `variables` API.
+
+There are two requirements for a test to use recorded variables. First, the test method should accept `**kwargs` and/or
+a `variables` parameter. Second, the test method should `return` a dictionary with any test variables that it wants to
+record. This dictionary will be stored in the recording when the test is run live, and will be passed to the test as a
+`variables` keyword argument when the test is run in playback.
+
+Below is a code example of how a test method could use recorded variables:
+
+```python
+from devtools_testutils import AzureRecordedTestCase, recorded_by_proxy
+
+class TestExample(AzureRecordedTestCase):
+
+    @recorded_by_proxy
+    def test_example(self, variables):
+        # in live mode, variables is an empty dictionary
+        # in playback mode, the value of variables is {"table_name": "random-value"}
+        if self.is_live:
+            table_name = "random-value"
+            variables = {"table_name": table_name}
+        
+        # use variables["table_name"] when using the table name throughout the test
+        ...
+
+        # return the variables at the end of the test
+        return variables
+```
+
 ## Implementation details
 
 ### What does the test proxy do?
@@ -147,7 +189,7 @@ For example, if an operation would typically make a GET request to
 `https://localhost:5001/Tables` instead. The original endpoint should be stored in an `x-recording-upstream-base-uri` --
 the proxy will send the original request and record the result.
 
-The RecordedByProxy and RecordedByProxyAsync decorators patch test requests to do this for you.
+The `recorded_by_proxy` and `recorded_by_proxy_async` decorators patch test requests to do this for you.
 
 ### How does the test proxy know when and what to record or play back?
 
@@ -179,7 +221,7 @@ Running tests in playback follows the same pattern, except that requests will be
 `/playback/stop` instead. A header, `x-recording-mode`, should be set to `record` for all requests when recording and
 `playback` when playing recordings back. More details can be found [here][detailed_docs].
 
-The RecordedByProxy and RecordedByProxyAsync decorators send the appropriate requests at the start and end of each test
+The `recorded_by_proxy` and `recorded_by_proxy_async` decorators send the appropriate requests at the start and end of each test
 case.
 
 [detailed_docs]: https://github.com/Azure/azure-sdk-tools/tree/main/tools/test-proxy/Azure.Sdk.Tools.TestProxy/README.md
 
@@ -84,11 +84,13 @@ if (!$PSBoundParameters.ContainsKey('ErrorAction')) {
     $ErrorActionPreference = 'Stop'
 }
 
-function Log($Message) {
+function Log($Message)
+{
     Write-Host ('{0} - {1}' -f [DateTime]::Now.ToLongTimeString(), $Message)
 }
 
-function Retry([scriptblock] $Action, [int] $Attempts = 5) {
+function Retry([scriptblock] $Action, [int] $Attempts = 5)
+{
     $attempt = 0
     $sleep = 5
 
@@ -109,7 +111,20 @@ function Retry([scriptblock] $Action, [int] $Attempts = 5) {
     }
 }
 
-function MergeHashes([hashtable] $source, [psvariable] $dest) {
+function LoadCloudConfig([string] $env)
+{
+    $configPath = "$PSScriptRoot/clouds/$env.json"
+    if (!(Test-Path $configPath)) {
+        Write-Warning "Could not find cloud configuration for environment '$env'"
+        return @{}
+    }
+
+    $config = Get-Content $configPath | ConvertFrom-Json -AsHashtable
+    return $config
+}
+
+function MergeHashes([hashtable] $source, [psvariable] $dest)
+{
     foreach ($key in $source.Keys) {
         if ($dest.Value.ContainsKey($key) -and $dest.Value[$key] -ne $source[$key]) {
             Write-Warning ("Overwriting '$($dest.Name).$($key)' with value '$($dest.Value[$key])' " +
@@ -119,7 +134,8 @@ function MergeHashes([hashtable] $source, [psvariable] $dest) {
     }
 }
 
-function BuildBicepFile([System.IO.FileSystemInfo] $file) {
+function BuildBicepFile([System.IO.FileSystemInfo] $file)
+{
     if (!(Get-Command bicep -ErrorAction Ignore)) {
         Write-Error "A bicep file was found at '$($file.FullName)' but the Azure Bicep CLI is not installed. See https://aka.ms/install-bicep-pwsh"
         throw
@@ -334,7 +350,7 @@ try {
     $serviceName = if (Split-Path $ServiceDirectory) {
         Split-Path -Leaf $ServiceDirectory
     } else {
-        $ServiceDirectory
+        $ServiceDirectory.Trim('/')
     }
 
     $ResourceGroupName = if ($ResourceGroupName) {
@@ -492,6 +508,8 @@ try {
         $templateParameters.Add('testApplicationSecret', $TestApplicationSecret)
     }
 
+    $defaultCloudParameters = LoadCloudConfig $Environment
+    MergeHashes $defaultCloudParameters $(Get-Variable templateParameters)
     MergeHashes $ArmTemplateParameters $(Get-Variable templateParameters)
     MergeHashes $AdditionalParameters $(Get-Variable templateParameters)
 
@@ -600,7 +618,7 @@ try {
             $outputFile = "$($templateFile.originalFilePath).env"
 
             $environmentText = $deploymentOutputs | ConvertTo-Json;
-            $bytes = ([System.Text.Encoding]::UTF8).GetBytes($environmentText)
+            $bytes = [System.Text.Encoding]::UTF8.GetBytes($environmentText)
             $protectedBytes = [Security.Cryptography.ProtectedData]::Protect($bytes, $null, [Security.Cryptography.DataProtectionScope]::CurrentUser)
 
             Set-Content $outputFile -Value $protectedBytes -AsByteStream -Force