Merge remote-tracking branch 'origin/master' into lro_continuation_token

Author: lmazuel

eng/autorest_req.txt

@@ -4,4 +4,5 @@ pytest-cov==2.8.1
 pytest-asyncio==0.10.0; python_version >= '3.5'
 isodate==0.6.0
 msrest==0.6.13
-aiohttp==3.6.2
+aiohttp==3.6.2
+wheel

eng/common/TestResources/New-TestResources.ps1

@@ -62,7 +62,10 @@ param (
     [switch] $CI = ($null -ne $env:SYSTEM_TEAMPROJECTID),
 
     [Parameter()]
-    [switch] $Force
+    [switch] $Force,
+
+    [Parameter()]
+    [switch] $OutFile
 )
 
 # By default stop for any error.
@@ -108,7 +111,8 @@ trap {
 }
 
 # Enumerate test resources to deploy. Fail if none found.
-$root = [System.IO.Path]::Combine("$PSScriptRoot/../../../sdk", $ServiceDirectory) | Resolve-Path
+$repositoryRoot = "$PSScriptRoot/../../.." | Resolve-Path
+$root = [System.IO.Path]::Combine($repositoryRoot, "sdk", $ServiceDirectory) | Resolve-Path
 $templateFileName = 'test-resources.json'
 $templateFiles = @()
 
@@ -172,19 +176,20 @@ if ($TestApplicationId -and !$TestApplicationOid) {
     }
 }
 
+
+# If the ServiceDirectory is an absolute path use the last directory name
+# (e.g. D:\foo\bar\ -> bar)
+$serviceName = if (Split-Path -IsAbsolute  $ServiceDirectory) {
+    Split-Path -Leaf $ServiceDirectory
+} else {
+    $ServiceDirectory
+}
+
 # Format the resource group name based on resource group naming recommendations and limitations.
 $resourceGroupName = if ($CI) {
     $BaseName = 't' + (New-Guid).ToString('n').Substring(0, 16)
     Write-Verbose "Generated base name '$BaseName' for CI build"
 
-    # If the ServiceDirectory is an absolute path use the last directory name
-    # (e.g. D:\foo\bar\ -> bar)
-    $serviceName = if (Split-Path -IsAbsolute  $ServiceDirectory) {
-        Split-Path -Leaf $ServiceDirectory
-    } else {
-        $ServiceDirectory
-    }
-
     "rg-{0}-$BaseName" -f ($serviceName -replace '[\\\/:]', '-').Substring(0, [Math]::Min($serviceName.Length, 90 - $BaseName.Length - 4)).Trim('-')
 } else {
     "rg-$BaseName"
@@ -281,12 +286,21 @@ foreach ($templateFile in $templateFiles) {
         Write-Verbose "Successfully deployed template '$templateFile' to resource group '$($resourceGroup.ResourceGroupName)'"
     }
 
-    if ($deployment.Outputs.Count -and !$CI) {
-        # Write an extra new line to isolate the environment variables for easy reading.
-        Log "Persist the following environment variables based on your detected shell ($shell):`n"
+    $serviceDirectoryPrefix = $serviceName.ToUpperInvariant() + "_"
+
+    $context = Get-AzContext;
+
+    # Add default values
+    $deploymentOutputs = @{
+        "$($serviceDirectoryPrefix)CLIENT_ID" = $TestApplicationId;
+        "$($serviceDirectoryPrefix)CLIENT_SECRET" = $TestApplicationSecret;
+        "$($serviceDirectoryPrefix)TENANT_ID" = $context.Tenant.Id;
+        "$($serviceDirectoryPrefix)SUBSCRIPTION_ID" =  $context.Subscription.Id;
+        "$($serviceDirectoryPrefix)RESOURCE_GROUP" = $resourceGroup.ResourceGroupName;
+        "$($serviceDirectoryPrefix)LOCATION" = $resourceGroup.Location;
+        "$($serviceDirectoryPrefix)ENVIRONMENT" = $context.Environment.Name;
     }
 
-    $deploymentOutputs = @{}
     foreach ($key in $deployment.Outputs.Keys) {
         $variable = $deployment.Outputs[$key]
 
@@ -295,23 +309,54 @@ foreach ($templateFile in $templateFiles) {
 
         if ($variable.Type -eq 'String' -or $variable.Type -eq 'SecureString') {
             $deploymentOutputs[$key] = $variable.Value
+        }
+    }
 
+    if ($OutFile)
+    {
+        if (!$IsWindows)
+        {
+            Write-Host "File option is supported only on Windows"
+        }
+
+        $outputFile = "$templateFile.env"
+
+        $environmentText = $deploymentOutputs | ConvertTo-Json;
+        $bytes = ([System.Text.Encoding]::UTF8).GetBytes($environmentText)
+        $protectedBytes = [Security.Cryptography.ProtectedData]::Protect($bytes, $null, [Security.Cryptography.DataProtectionScope]::CurrentUser)
+
+        Set-Content $outputFile -Value $protectedBytes -AsByteStream -Force
+
+        Write-Host "Test environment settings`n $environmentText`nstored into encrypted $outputFile"
+    }
+    else
+    {
+        
+        if (!$CI) {
+            # Write an extra new line to isolate the environment variables for easy reading.
+            Log "Persist the following environment variables based on your detected shell ($shell):`n"
+        }
+
+        foreach ($key in $deploymentOutputs.Keys)
+        {
+            $value = $deploymentOutputs[$key]
+            
             if ($CI) {
                 # Treat all ARM template output variables as secrets since "SecureString" variables do not set values.
                 # In order to mask secrets but set environment variables for any given ARM template, we set variables twice as shown below.
                 Write-Host "Setting variable '$key': ***"
-                Write-Host "##vso[task.setvariable variable=_$key;issecret=true;]$($variable.Value)"
-                Write-Host "##vso[task.setvariable variable=$key;]$($variable.Value)"
+                Write-Host "##vso[task.setvariable variable=_$key;issecret=true;]$($value)"
+                Write-Host "##vso[task.setvariable variable=$key;]$($value)"
             } else {
-                Write-Host ($shellExportFormat -f $key, $variable.Value)
+                Write-Host ($shellExportFormat -f $key, $value)
             }
         }
-    }
 
-    if ($key) {
-        # Isolate the environment variables for easy reading.
-        Write-Host "`n"
-        $key = $null
+        if ($key) {
+            # Isolate the environment variables for easy reading.
+            Write-Host "`n"
+            $key = $null
+        }
     }
 
     $postDeploymentScript = $templateFile | Split-Path | Join-Path -ChildPath 'test-resources-post.ps1'
@@ -442,6 +487,10 @@ Deployment (CI/CD) build (only Azure Pipelines is currently supported).
 .PARAMETER Force
 Force creation of resources instead of being prompted.
 
+.PARAMETER OutFile
+Save test environment settings into a test-resources.json.env file next to test-resources.json. File is protected via DPAPI. Supported only on windows.
+The environment file would be scoped to the current repository directory.
+
 .EXAMPLE
 Connect-AzAccount -Subscription "REPLACE_WITH_SUBSCRIPTION_ID"
 $testAadApp = New-AzADServicePrincipal -Role Owner -DisplayName 'azure-sdk-live-test-app'

eng/common/TestResources/Remove-TestResources.ps1

@@ -38,6 +38,9 @@ param (
     [Parameter(ParameterSetName = 'Default+Provisioner', Mandatory = $true)]
     [Parameter(ParameterSetName = 'ResourceGroup+Provisioner', Mandatory = $true)]
     [string] $ProvisionerApplicationSecret,
+    
+    [Parameter()]
+    [string] $ServiceDirectory,
 
     [Parameter()]
     [ValidateSet('AzureCloud', 'AzureUSGovernment', 'AzureChinaCloud')]
@@ -116,6 +119,15 @@ if (!$ResourceGroupName) {
     $ResourceGroupName = "rg-$BaseName"
 }
 
+if (![string]::IsNullOrWhiteSpace($ServiceDirectory)) {
+    $root = [System.IO.Path]::Combine("$PSScriptRoot/../../../sdk", $ServiceDirectory) | Resolve-Path
+    $preRemovalScript = Join-Path -Path $root -ChildPath 'remove-test-resources-pre.ps1'
+    if (Test-Path $preRemovalScript) {
+        Log "Invoking pre resource removal script '$preRemovalScript'"
+        &$preRemovalScript -ResourceGroupName $ResourceGroupName @PSBoundParameters
+    }
+}
+
 Log "Deleting resource group '$ResourceGroupName'"
 if (Retry { Remove-AzResourceGroup -Name "$ResourceGroupName" -Force:$Force }) {
     Write-Verbose "Successfully deleted resource group '$ResourceGroupName'"
@@ -157,6 +169,10 @@ A service principal ID to provision test resources when a provisioner is specifi
 .PARAMETER ProvisionerApplicationSecret
 A service principal secret (password) to provision test resources when a provisioner is specified.
 
+.PARAMETER ServiceDirectory
+A directory under 'sdk' in the repository root - optionally with subdirectories
+specified - in which to discover pre removal script named 'remove-test-resources-pre.json'.
+
 .PARAMETER Environment
 Name of the cloud environment. The default is the Azure Public Cloud
 ('PublicCloud')

eng/common/TestResources/Remove-TestResources.ps1.md

@@ -169,6 +169,22 @@ Accept pipeline input: False
 Accept wildcard characters: False
 ```
 
+### -ServiceDirectory
+A directory under 'sdk' in the repository root - optionally with subdirectories
+specified - specified - in which to discover pre removal script named 'remove-test-resources-pre.json'.
+
+```yaml
+Type: String
+Parameter Sets: (All)
+Aliases:
+
+Required: False
+Position: Named
+Default value: None
+Accept pipeline input: False
+Accept wildcard characters: False
+```
+
 ### -Environment
 Name of the cloud environment.
 The default is the Azure Public Cloud

eng/common/TestResources/remove-test-resources.yml

@@ -7,6 +7,9 @@
 # Assumes steps in deploy-test-resources.yml was run previously. Requires
 # environment variable: AZURE_RESOURCEGROUP_NAME and Az PowerShell module
 
+parameters:
+  ServiceDirectory: ''
+
 steps:
   - pwsh: >
       eng/common/TestResources/Remove-TestResources.ps1
@@ -15,6 +18,7 @@ steps:
       -SubscriptionId '$(azure-subscription-id)'
       -ProvisionerApplicationId '$(aad-azure-sdk-test-client-id)'
       -ProvisionerApplicationSecret '$(aad-azure-sdk-test-client-secret)'
+      -ServiceDirectory '${{ parameters.ServiceDirectory }}'
       -Environment 'AzureCloud'
       -Force
       -Verbose
@@ -29,6 +33,7 @@ steps:
       -SubscriptionId '$(azure-subscription-id-gov)'
       -ProvisionerApplicationId '$(aad-azure-sdk-test-client-id-gov)'
       -ProvisionerApplicationSecret '$(aad-azure-sdk-test-client-secret-gov)'
+      -ServiceDirectory '${{ parameters.ServiceDirectory }}'
       -Environment 'AzureUSGovernment'
       -Force
       -Verbose
@@ -43,6 +48,7 @@ steps:
       -SubscriptionId '$(azure-subscription-id-cn)'
       -ProvisionerApplicationId '$(aad-azure-sdk-test-client-id-cn)'
       -ProvisionerApplicationSecret '$(aad-azure-sdk-test-client-secret-cn)'
+      -ServiceDirectory '${{ parameters.ServiceDirectory }}'
       -Environment 'AzureChinaCloud'
       -Force
       -Verbose

eng/common/pipelines/templates/steps/bypass-local-dns.yml

@@ -0,0 +1,6 @@
+
+steps:
+    # https://github.com/actions/virtual-environments/issues/798
+    - script: sudo ln -sf /run/systemd/resolve/resolv.conf /etc/resolv.conf
+      displayName: Bypass local DNS server to workaround issue resolving cognitiveservices names
+      condition: and(succeededOrFailed(), eq(variables['OSVmImage'], 'ubuntu-18.04'))

eng/common/scripts/copy-readmes-to-docs.ps1

@@ -0,0 +1,85 @@
+# Example Usage: ./copy-readmes-to-docs.ps1 -CodeRepo C:/repo/sdk-for-python -DocRepo C:/repo/azure-docs-sdk-python
+# run the link updating script before this to get links fixed to the release
+# Highly recommended that you use Powershell Core.
+# git reset --hard origin/smoke-test on the doc repo prior to running this
+
+param (
+  [String]$CodeRepo,
+  [String]$DocRepo,
+  [String]$TargetServices
+)
+
+$PACKAGE_README_REGEX = ".*[\/\\]sdk[\\\/][^\/\\]+[\\\/][^\/\\]+[\/\\]README\.md" 
+
+Write-Host "repo is $CodeRepo"
+
+$date = Get-Date -Format "MM/dd/yyyy"
+
+if ($CodeRepo -Match "net")
+{
+  $lang = ".NET"
+  $TARGET_FOLDER = Join-Path -Path $DocRepo  -ChildPath "api/overview/azure"
+  $metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/releases/latest/dotnet-packages.csv"
+}
+if ($CodeRepo -Match "python"){
+  $lang = "Python"
+  $TARGET_FOLDER = Join-Path -Path $DocRepo  -ChildPath "docs-ref-services"
+  $metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/releases/latest/python-packages.csv"
+}
+if ($CodeRepo -Match "java"){
+  $lang = "Java"
+  $TARGET_FOLDER = Join-Path -Path $DocRepo  -ChildPath "docs-ref-services"
+  $metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/releases/latest/java-packages.csv"
+}
+if ($CodeRepo -Match "js"){
+  $lang = "JavaScript"
+  $TARGET_FOLDER = Join-Path -Path $DocRepo  -ChildPath "docs-ref-services"
+  $metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/releases/latest/js-packages.csv"
+}
+
+
+
+$metadataResponse = Invoke-WebRequest -Uri $metadataUri | ConvertFrom-Csv
+
+if ([string]::IsNullOrWhiteSpace($TargetServices))
+{
+  $selectedServices = $metadataResponse | ForEach-Object -Process {$_.RepoPath} | Get-Unique
+}
+else {
+  $selectedServices = $TargetServices -Split "," | % { return $_.Trim() }
+}
+
+
+foreach($service in $selectedServices){
+  $readmePath = Join-Path -Path $CodeRepo  -ChildPath "sdk/$service"
+  Write-Host "Examining: $readmePath" 
+
+  $libraries = $metadataResponse | Where-Object { $_.RepoPath -eq $service }
+  
+  foreach($library in $libraries){
+
+    $package = $library.Package
+    $version = If ([string]::IsNullOrWhiteSpace($library.VersionGA)) {$library.VersionPreview} Else {$library.VersionGA}
+
+    $file = Join-Path -Path $readmePath -ChildPath "/$package/README.md" | Get-Item
+    Write-Host "`tOutputting $($file.FullName)"
+
+    $fileContent = Get-Content $file
+
+    $fileContent = $fileContent -Join "`n"
+
+    $fileMatch = (Select-String -InputObject $fileContent -Pattern 'Azure .+? (client|plugin|shared) library for (JavaScript|Java|Python|\.NET)').Matches[0]
+
+    $header = "---`r`ntitle: $fileMatch`r`nkeywords: Azure, $lang, SDK, API, $service, $package`r`nauthor: maggiepint`r`nms.author: magpint`r`nms.date: $date`r`nms.topic: article`r`nms.prod: azure`r`nms.technology: azure`r`nms.devlang: $lang`r`nms.service: $service`r`n---`r`n"
+
+    $fileContent = $fileContent -replace $fileMatch, "$fileMatch - Version $version `r`n"
+
+    $fileContent = "$header $fileContent"
+
+    $readmeName = "$($file.Directory.Name.Replace('azure-','').Replace('Azure.', '').ToLower())-readme.md"
+
+    $readmeOutputLocation = Join-Path $TARGET_FOLDER -ChildPath $readmeName
+
+    Set-Content -Path $readmeOutputLocation -Value $fileContent
+  }
+}

eng/pipelines/autorest_checks.yml

@@ -17,8 +17,9 @@ variables:
   NodeVersion: '10.x'
   PythonVersion: '3.6'
   auto_rest_clone_url: 'https://github.com/Azure/autorest.python.git'
-  repo_branch: 'azure-core'
-  source_path: 'sdk/core/azure-core'
+  repo_branch: 'autorestv3'
+  source_path_azure_core: 'sdk/core/azure-core'
+  source_path_azure_mgmt_core: 'sdk/core/azure-mgmt-core'
 
 jobs:
   - job: 'Run_AutoRest'
@@ -40,13 +41,24 @@ jobs:
 
       - script: |
           pip install -r eng/autorest_req.txt
-          pip install $(Build.SourcesDirectory)/$(source_path)
-        displayName: 'Prepare Environment'
-
-      - script: |
-          git clone --recursive $(auto_rest_clone_url)
+          git clone $(auto_rest_clone_url)
           cd autorest.python
           git checkout $(repo_branch)
           npm install
-          pytest test/vanilla
-        displayName: 'Run Test'
+        displayName: 'Prepare Environment'
+
+      - script: |
+          cd $(Build.SourcesDirectory)/autorest.python/test/vanilla
+          pip install $(Build.SourcesDirectory)/$(source_path_azure_core)
+          pip install -r requirements.txt
+          pip freeze
+          pytest $(Build.SourcesDirectory)/autorest.python/test/vanilla
+        displayName: 'Install azure-core and Test Vanilla'
+
+      - script: |
+          cd $(Build.SourcesDirectory)/autorest.python/test/azure          
+          pip install $(Build.SourcesDirectory)/$(source_path_azure_mgmt_core)          
+          pip install -r requirements.txt
+          pip freeze
+          pytest $(Build.SourcesDirectory)/autorest.python/test/azure
+        displayName: 'Install azure-mgmt-core and Test Azure'