merging with upstream/master

Author: toswedlu

doc/dev/mgmt/tests.md

@@ -176,7 +176,7 @@ providing several features useful for the SDK tests, for example:
 *   Patches for overriding functions and methods that don't work well with tests
     (such as long-running operations)
 
-Code in the [`azure-sdk-tools/devtools_testutils`](tools/azure-sdk-tools/devtools_testutils) directory
+Code in the [`azure-sdk-tools/devtools_testutils`](https://github.com/Azure/azure-sdk-for-python/tree/master/tools/azure-sdk-tools/devtools_testutils) directory
 provides concrete implementations of the features provided in `scenario_tests`
 that are oriented around use in SDK testing
 and that you can use directly in your unit tests.

doc/sphinx/package_service_mapping.json

@@ -901,6 +901,11 @@
     "service_name": "Storage",
     "manually_generated": true
   },
+  "azure-storage-blob-changefeed": {
+    "category": "Client",
+    "service_name": "Storage",
+    "manually_generated": true
+  },
   "azure-storage-file-share": {
     "category": "Client",
     "service_name": "Storage",

eng/.docsettings.yml

@@ -114,6 +114,7 @@ known_content_issues:
   - ['sdk/search/azure-search-nspkg/README.md', 'nspkg and common']
   - ['sdk/storage/azure-storage-blob/samples/README.md', 'nspkg and common']
   - ['sdk/storage/azure-storage-file-datalake/samples/README.md', 'nspkg and common']
+  - ['sdk/storage/azure-storage-blob-changefeed/samples/README.md', 'nspkg and common']
   - ['sdk/storage/azure-storage-file-share/samples/README.md', 'nspkg and common']
   - ['sdk/storage/azure-storage-queue/samples/README.md', 'nspkg and common']
   - ['sdk/textanalytics/azure-ai-nspkg/README.md', 'nspkg and common']

eng/ci_tools.txt

@@ -1,36 +1,35 @@
 # requirements leveraged by ci tools
 setuptools==44.1.0; python_version == '2.7'
-setuptools==45.1.0; python_version >= '3.5'
+setuptools==46.4.0; python_version >= '3.5'
 virtualenv==20.0.23
 wheel==0.34.2 
-Jinja2==2.11.1
+Jinja2==2.11.2
 packaging==20.4
-tox==3.14.6
+tox==3.15.0
 tox-monorepo==0.1.2
-twine==1.15.0
+twine==1.15.0; python_version == '2.7' or python_version == '3.5'
+twine==3.1.1; python_version >= '3.6'
 pathlib2==2.3.5
 readme-renderer[md]==25.0
-doc-warden==0.5.4
+doc-warden==0.7.1
+# we pin coverage to 4.5.4 because there is an bug with `pytest-cov`. the generated coverage files cannot be `coverage combine`ed
 coverage==4.5.4
-codecov==2.0.22 
-beautifulsoup4==4.8.2
+codecov==2.1.0
+beautifulsoup4==4.9.1
 pkginfo==1.5.0.1
 
 # locking packages defined as deps from azure-sdk-tools or azure-devtools
 pytoml==0.1.21
 pyOpenSSL==19.1.0
 json-delta==2.0
-ConfigArgParse==1.1
+ConfigArgParse==1.2.3
 six==1.14.0
 vcrpy==3.0.0
 pyyaml==5.3.1
-pytest==5.4.1; python_version >= '3.5'
+pytest==5.4.2; python_version >= '3.5'
 pytest==4.6.9; python_version == '2.7'
 pytest-cov==2.8.1
 
 # local dev packages
 ./tools/azure-devtools
 ./tools/azure-sdk-tools
-
-
-

eng/common/Update-Change-Log.ps1

@@ -27,10 +27,10 @@ function Get-ChangelogPath($Path)
 { 
    # Check if CHANGELOG.md is present in path 
    $ChangeLogPath = Join-Path -Path $Path -ChildPath "CHANGELOG.md"
-   if ((Test-Path -Path $ChangeLogPath) -eq $False){
+   if ((Test-Path -Path $ChangeLogPath) -eq $False) {
       # Check if change log exists with name HISTORY.md
       $ChangeLogPath = Join-Path -Path $Path -ChildPath "HISTORY.md"
-      if ((Test-Path -Path $ChangeLogPath) -eq $False){
+      if ((Test-Path -Path $ChangeLogPath) -eq $False) {
          Write-Host "Change log is not found in path[$Path]"
          exit(1)
       }
@@ -45,7 +45,7 @@ function Get-VersionTitle($Version, $Unreleased)
 {
    # Generate version title
    $newVersionTitle = "## $Version $UNRELEASED_TAG"
-   if ($Unreleased -eq $False){
+   if ($Unreleased -eq $False) {
       $releaseDate = Get-Date -Format "(yyyy-MM-dd)"
       $newVersionTitle = "## $Version $releaseDate"
    }
@@ -67,10 +67,10 @@ function Get-NewChangeLog( [System.Collections.ArrayList]$ChangelogLines, $Versi
    # Version increment tool passes replaceversion as False and Unreleased as True
    $is_version_increment = $ReplaceVersion -eq $False -and $Unreleased -eq $True
 
-   for(; $Index -lt $ChangelogLines.Count; $Index++){
-      if (Version-Matches($ChangelogLines[$Index])){
+   for (; $Index -lt $ChangelogLines.Count; $Index++) {
+      if (Version-Matches($ChangelogLines[$Index])) {
          # Find current title in change log
-         if( -not $CurrentTitle){
+         if( -not $CurrentTitle) {
             $CurrentTitle = $ChangelogLines[$Index]
             $CurrentIndex = $Index
             Write-Host "Current Version title: $CurrentTitle"
@@ -80,7 +80,7 @@ function Get-NewChangeLog( [System.Collections.ArrayList]$ChangelogLines, $Versi
          # update change log script is triggered for all packages with current version for Java ( or any language where version is maintained in common file)
          # and this can cause an issue if someone changes changelog manually to prepare for release without updating actual version in central version file
          # Do not add new line or replace existing title when version is already present and script is triggered to add new line
-         if ($is_version_increment -and $ChangelogLines[$Index].Contains($Version)){
+         if ($is_version_increment -and $ChangelogLines[$Index].Contains($Version)) {
             Write-Host "Version is already present in change log."
             exit(0)
          }
@@ -90,26 +90,24 @@ function Get-NewChangeLog( [System.Collections.ArrayList]$ChangelogLines, $Versi
    # Generate version title
    $newVersionTitle = Get-VersionTitle -Version $Version -Unreleased $Unreleased
 
-   if( $newVersionTitle -eq $CurrentTitle){
+   if( $newVersionTitle -eq $CurrentTitle) {
       Write-Host "No change is required in change log. Version is already present."
       exit(0)
    }
 
-   
-
-   if (($ReplaceVersion -eq $True) -and ($Unreleased -eq $False) -and (-not $CurrentTitle.Contains($UNRELEASED_TAG))){
+   if (($ReplaceVersion -eq $True) -and ($Unreleased -eq $False) -and $CurrentTitle.Contains($version) -and (-not $CurrentTitle.Contains($UNRELEASED_TAG))) {
       Write-Host "Version is already present in change log with a release date."
       exit(0)
    }
 
    # if current version title already has new version then we should replace title to update it
-   if ($CurrentTitle.Contains($Version) -and $ReplaceVersion -eq $False){
+   if ($CurrentTitle.Contains($Version) -and $ReplaceVersion -eq $False) {
       Write-Host "Version is already present in title. Updating version title"
       $ReplaceVersion = $True
    }
 
    # if version is already found and not replacing then nothing to do
-   if ($ReplaceVersion -eq $False){
+   if ($ReplaceVersion -eq $False) {
       Write-Host "Adding version title $newVersionTitle"
       $ChangelogLines.insert($CurrentIndex, "")
       $ChangelogLines.insert($CurrentIndex, "")
@@ -121,24 +119,28 @@ function Get-NewChangeLog( [System.Collections.ArrayList]$ChangelogLines, $Versi
       $ChangelogLines[$CurrentIndex] = $newVersionTitle
    }
 
-   return $ChangelogLines      
+   return $ChangelogLines
 }
 
 
 # Make sure path is valid
-if ((Test-Path -Path $ChangeLogPath) -eq $False){
+if ((Test-Path -Path $ChangeLogPath) -eq $False) {
    Write-Host "Change log path is invalid. [$ChangeLogPath]"
    exit(1)
 }
 
 # probe change log path if path is directory 
-if (Test-Path -Path $ChangeLogPath -PathType Container)
-{   
+if (Test-Path -Path $ChangeLogPath -PathType Container) {
    $ChangeLogPath = Get-ChangelogPath -Path $ChangeLogPath
 }
 
 # Read current change logs and add/update version
 $ChangelogLines = [System.Collections.ArrayList](Get-Content -Path $ChangeLogPath)
+
+if ($null -eq $ChangelogLines) { 
+   $ChangelogLines = @() 
+}
+
 $NewContents = Get-NewChangeLog -ChangelogLines $ChangelogLines -Version $Version -Unreleased $Unreleased -ReplaceVersion $ReplaceVersion
 
 Write-Host "Writing change log to file [$ChangeLogPath]"

eng/common/pipelines/templates/steps/daily-dev-build-variable.yml

@@ -0,0 +1,12 @@
+# This script fragment is used across our repos to set a variable "SetDevVersion" which
+# is used when this pipeline is going to be generating and publishing daily dev builds.
+
+steps:
+- pwsh: |
+    $setDailyDevBuild = "false"
+    if (('$(Build.Reason)' -eq 'Schedule') -and ('$(System.TeamProject)' -eq 'internal')) {
+      $setDailyDevBuild = "true"
+    }
+    echo "##vso[task.setvariable variable=SetDevVersion]$setDailyDevBuild"
+  displayName: "Setup Versioning Properties"
+  condition: eq(variables['SetDevVersion'], '')

eng/common/pipelines/templates/steps/verify-links.yml

@@ -1,5 +1,7 @@
 parameters:
   Directory: 'not-specified'
+  IgnoreLinksFile: "$(Build.SourcesDirectory)/eng/ignore-links.txt"
+  
 
 steps:
   - task: PowerShell@2
@@ -9,4 +11,4 @@ steps:
       workingDirectory: $(Build.SourcesDirectory)/${{ parameters.Directory }}
       filePath: eng/common/scripts/Verify-Links.ps1 
       arguments: >
-        -urls $(dir -r -i *.md) -rootUrl "file://$(Build.SourcesDirectory)/${{ parameters.Directory }}"
+        -urls $(dir -r -i *.md) -rootUrl "file://$(Build.SourcesDirectory)/${{ parameters.Directory }}" -recursive:$false -ignoreLinksFile ${{ parameters.IgnoreLinksFile }}

eng/common/scripts/Verify-ChangeLog.ps1

@@ -1,31 +1,48 @@
 # Wrapper Script for ChangeLog Verification
 param (
-    [String]$ChangeLogLocation,
-    [String]$VersionString,
-    [string]$PackageName,
-    [string]$ServiceName,
-    [string]$RepoRoot,
-    [ValidateSet("net","java","js","python")]
-    [string]$Language,
-    [string]$RepoName,
-    [boolean]$ForRelease=$False
+  [String]$ChangeLogLocation,
+  [String]$VersionString,
+  [string]$PackageName,
+  [string]$ServiceName,
+  [string]$RepoRoot,
+  [ValidateSet("net", "java", "js", "python")]
+  [string]$Language,
+  [string]$RepoName,
+  [boolean]$ForRelease = $False
 )
 
+$ProgressPreference = "SilentlyContinue"
 . (Join-Path $PSScriptRoot SemVer.ps1)
 Import-Module (Join-Path $PSScriptRoot modules ChangeLog-Operations.psm1)
 
-if ((Test-Path $ChangeLogLocation) -and -not([System.String]::IsNullOrEmpty($VersionString)))
+$validChangeLog = $false
+if ($ChangeLogLocation -and $VersionString) 
 {
-    Confirm-ChangeLogEntry -ChangeLogLocation $ChangeLogLocation -VersionString $VersionString -ForRelease $ForRelease
+  $validChangeLog = Confirm-ChangeLogEntry -ChangeLogLocation $ChangeLogLocation -VersionString $VersionString -ForRelease $ForRelease
 }
-else 
+else
 {
-    Import-Module (Join-Path $PSScriptRoot modules Package-Properties.psm1)
-    if ([System.String]::IsNullOrEmpty($Language))
+  Import-Module (Join-Path $PSScriptRoot modules Package-Properties.psm1)
+  if ([System.String]::IsNullOrEmpty($Language))
+  {
+    if ($RepoName -match "azure-sdk-for-(?<lang>[^-]+)")
     {
-        $Language = $RepoName.Substring($RepoName.LastIndexOf('-') + 1)
+      $Language = $matches["lang"]
     }
+    else
+    {
+      Write-Error "Failed to set Language automatically. Please pass the appropriate Language as a parameter."
+      exit 1
+    }
+  }
+
+  $PackageProp = Get-PkgProperties -PackageName $PackageName -ServiceName $ServiceName -Language $Language -RepoRoot $RepoRoot
+  $validChangeLog = Confirm-ChangeLogEntry -ChangeLogLocation $PackageProp.pkgChangeLogPath -VersionString $PackageProp.pkgVersion -ForRelease $ForRelease
+}
+
+if (!$validChangeLog)
+{
+  exit 1
+}
 
-    $PackageProp = Get-PkgProperties -PackageName $PackageName -ServiceName $ServiceName -Language $Language -RepoRoot $RepoRoot
-    Confirm-ChangeLogEntry -ChangeLogLocation $PackageProp.pkgChangeLogPath -VersionString $PackageProp.pkgVersion -ForRelease $ForRelease
-}
+exit 0

eng/common/scripts/modules/ChangeLog-Operations.psm1

@@ -11,8 +11,8 @@ function Get-ChangeLogEntries {
 
   $changeLogEntries = @{}
   if (!(Test-Path $ChangeLogLocation)) {
-    Write-Host "ChangeLog '{0}' was not found" -f $ChangeLogLocation
-    exit 1
+    Write-Error "ChangeLog[${ChangeLogLocation}] does not exist"
+    return $null
   }
 
   try {
@@ -51,14 +51,12 @@ function Get-ChangeLogEntry {
     [Parameter(Mandatory = $true)]
     [String]$VersionString
   )
-
   $changeLogEntries = Get-ChangeLogEntries -ChangeLogLocation $ChangeLogLocation
 
-  if ($changeLogEntries.ContainsKey($VersionString)) {
+  if ($changeLogEntries -and $changeLogEntries.ContainsKey($VersionString)) {
     return $changeLogEntries[$VersionString]
   }
-  Write-Error "Release Notes for the Specified version ${VersionString} was not found"
-  exit 1
+  return $null
 }
 
 #Returns the changelog for a particular version as string
@@ -70,9 +68,16 @@ function Get-ChangeLogEntryAsString {
     [String]$VersionString
   )
 
-  $changeLogEntries = Get-ChangeLogEntry -ChangeLogLocation $ChangeLogLocation -VersionString $VersionString
-  [string]$releaseTitle = $changeLogEntries.ReleaseTitle
-  [string]$releaseContent = $changeLogEntries.ReleaseContent -Join [Environment]::NewLine
+  $changeLogEntry = Get-ChangeLogEntry -ChangeLogLocation $ChangeLogLocation -VersionString $VersionString
+  return ChangeLogEntryAsString $changeLogEntry
+}
+
+function ChangeLogEntryAsString($changeLogEntry) {
+  if (!$changeLogEntry) {
+    return "[Missing change log entry]"
+  }
+  [string]$releaseTitle = $changeLogEntry.ReleaseTitle
+  [string]$releaseContent = $changeLogEntry.ReleaseContent -Join [Environment]::NewLine
   return $releaseTitle, $releaseContent -Join [Environment]::NewLine
 }
 
@@ -87,27 +92,33 @@ function Confirm-ChangeLogEntry {
 
   $changeLogEntry = Get-ChangeLogEntry -ChangeLogLocation $ChangeLogLocation -VersionString $VersionString
 
+  if (!$changeLogEntry) {
+    Write-Error "ChangeLog[${ChangeLogLocation}] does not have an entry for version ${VersionString}."
+    return $false
+  }
+
+  Write-Host "Found the following change log entry for version '${VersionString}' in [${ChangeLogLocation}]."
+  Write-Host "-----"
+  Write-Host (ChangeLogEntryAsString $changeLogEntry)
+  Write-Host "-----"
+
   if ([System.String]::IsNullOrEmpty($changeLogEntry.ReleaseStatus)) {
-    Write-Host ("##[error]Changelog '{0}' has wrong release note title" -f $ChangeLogLocation)
-    Write-Host "##[info]Ensure the release date is included i.e. (yyyy-MM-dd) or (Unreleased) if not yet released"
-    exit 1
+    Write-Error "Entry does not have a correct release status. Please ensure the status is set to a date '(yyyy-MM-dd)' or '(Unreleased)' if not yet released."
+    return $false
   }
 
   if ($ForRelease -eq $True) {
     if ($changeLogEntry.ReleaseStatus -eq "(Unreleased)") {
-      Write-Host ("##[error]No release date set. Please set a release date with format 'yyyy-MM-dd' in the heading for version '{0}' in the changelog '{1}'." -f $VersionString, $ChangelogLocation)
-      exit 1
+      Write-Error "Entry has no release date set. Please ensure to set a release date with format 'yyyy-MM-dd'."
+      return $false
     }
 
     if ([System.String]::IsNullOrWhiteSpace($changeLogEntry.ReleaseContent)) {
-      Write-Host ("##[error]Empty Release Notes for '{0}' in '{1}'" -f $VersionString, $ChangeLogLocation)
-      Write-Host "##[info]Please ensure there is a release notes entry before releasing the package."
-      exit 1
+      Write-Error "Entry has no content. Please ensure to provide some content of what changed in this version."
+      return $false
     }
   }
-
-  Write-Host $changeLogEntry.ReleaseTitle
-  Write-Host $changeLogEntry.ReleaseContent
+  return $true
 }
 
 Export-ModuleMember -Function 'Get-ChangeLogEntries'