Merge branch 'main' of https://github.com/Azure/azure-sdk-for-python into header_tuples

* 'main' of https://github.com/Azure/azure-sdk-for-python: (21 commits)
  [Key Vault] Add 7.3-preview support for certificates (#20477)
  Use dummy values and correct code rendering in README (#20322)
  Use CredScan-suppressed dummy password in sample (#20468)
  Fix type annotation in azure.storage.blob (#20084)
  Adding static checks (#20457)
  release_iseus_status_auto_reply (#20441)
  Clarify LogsQueryClient query parameter description (#20467)
  Sync eng/common directory with azure-sdk-tools for PR 1953 (#20466)
  Skip eng common workflow enforcer for private repos (#20462)
  remove iter_text and iter_lines (#20460)
  [ServiceBus] Improve test stability and cpu usage (#20352)
  t2-Netapp-update-to-2021-06-01 (#20453)
  [AutoRelease] t2-keyvault-2021-08-26-55443 (#20423)
  Sync eng/common directory with azure-sdk-tools for PR 1943 (#20450)
  New dummy password in README (#20328)
  ensure test principal creation succeeds properly (#20446)
  Update question-answering readme links (#20439)
  Stip empty changelog sections before release (#20437)
  [AutoRelease] t2-healthcareapis-2021-08-26-27542 (#20422)
  [AutoRelease] t2-iothub-2021-08-25-25696 (#20409)
  ...

Author: iscai-msft

eng/common/TestResources/New-TestResources.ps1

@@ -289,7 +289,8 @@ try {
                 $AzureTestPrincipal
             } else {
                 Log "TestApplicationId was not specified; creating a new service principal in subscription '$SubscriptionId'"
-                $global:AzureTestPrincipal = New-AzADServicePrincipal -Role Owner -Scope "/subscriptions/$SubscriptionId"
+                $suffix = (New-Guid).ToString('n').Substring(0, 4)
+                $global:AzureTestPrincipal = New-AzADServicePrincipal -Role Owner -Scope "/subscriptions/$SubscriptionId" -DisplayName "test-resources-$($baseName)$suffix.microsoft.com"
                 $global:AzureTestSubscription = $SubscriptionId
 
                 Log "Created service principal '$($AzureTestPrincipal.ApplicationId)'"
@@ -477,7 +478,13 @@ try {
             &$preDeploymentScript -ResourceGroupName $ResourceGroupName @PSBoundParameters
         }
 
-        Log "Deploying template '$($templateFile.originalFilePath)' to resource group '$($resourceGroup.ResourceGroupName)'"
+        $msg = if ($templateFile.jsonFilePath -ne $templateFile.originalFilePath) {
+            "Deployment template $($templateFile.jsonFilePath) from $($templateFile.originalFilePath) to resource group $($resourceGroup.ResourceGroupName)"
+        } else {
+            "Deployment template $($templateFile.jsonFilePath) to resource group $($resourceGroup.ResourceGroupName)"
+        }
+        Log $msg
+        
         $deployment = Retry {
             $lastDebugPreference = $DebugPreference
             try {
@@ -538,7 +545,7 @@ try {
                 Write-Host 'File option is supported only on Windows'
             }
 
-            $outputFile = "$($templateFile.jsonFilePath).env"
+            $outputFile = "$($templateFile.originalFilePath).env"
 
             $environmentText = $deploymentOutputs | ConvertTo-Json;
             $bytes = ([System.Text.Encoding]::UTF8).GetBytes($environmentText)

eng/common/pipelines/templates/steps/eng-common-workflow-enforcer.yml

@@ -21,4 +21,4 @@ steps:
           }
         }
       displayName: Prevent changes to eng/common outside of azure-sdk-tools repo
-      condition: and(succeeded(), ne(variables['Skip.EngCommonWorkflowEnforcer'], 'true'))
+      condition: and(succeeded(), ne(variables['Skip.EngCommonWorkflowEnforcer'], 'true'), not(endsWith(variables['Build.Repository.Name'], '-pr')))

eng/common/scripts/Prepare-Release.ps1

@@ -184,6 +184,13 @@ else
   exit 1
 }
 
+$changelogIsValid = Confirm-ChangeLogEntry -ChangeLogLocation $packageProperties.ChangeLogPath -VersionString $newVersion -ForRelease $true
+
+if (!$changelogIsValid)
+{
+  Write-Host "The changelog [$($packageProperties.ChangeLogPath)] is not valid for release. Please make sure it is valid before queuing release build." -ForegroundColor Red
+}
+
 git diff -s --exit-code $packageProperties.DirectoryPath
 if ($LASTEXITCODE -ne 0)
 {

eng/common/scripts/Update-ChangeLog.ps1

@@ -106,7 +106,47 @@ if ($LatestsSorted[0] -ne $Version) {
 
 if ($ReplaceLatestEntryTitle)
 {
-    $newChangeLogEntry = New-ChangeLogEntry -Version $Version -Status $ReleaseStatus -Content $ChangeLogEntries[$LatestVersion].ReleaseContent
+    # Remove empty sections from content
+    $sanitizedContent = @()
+    $sectionContent = @()
+    $sectionContentCount = 0
+    $latesVersionContent = $ChangeLogEntries[$LatestVersion].ReleaseContent
+
+    foreach ($line in $latesVersionContent)
+    {
+        if ($line.StartsWith("### ") -or $sectionContentCount -gt 0)
+        {
+            if ($line.StartsWith("#") -and $sectionContentCount -gt 1)
+            {
+                $sanitizedContent += $sectionContent
+                $sectionContent = @()
+                $sectionContentCount = 0
+            }
+    
+            if ($line.StartsWith("#") -and $sectionContentCount -eq 1)
+            {
+                $sectionContent = @()
+                $sectionContentCount = 0
+            }
+
+            $sectionContent += $line
+            if (-not [System.String]::IsNullOrWhiteSpace($line))
+            {
+                $sectionContentCount++
+            }
+        }
+        elseif ($sectionContent.Count -eq 0)
+        {
+            $sanitizedContent += $line
+        }
+    }
+    
+    if ($sectionContentCount -gt 1)
+    {
+        $sanitizedContent += $sectionContent
+    }
+
+    $newChangeLogEntry = New-ChangeLogEntry -Version $Version -Status $ReleaseStatus -Content $sanitizedContent
     LogDebug "Resetting latest entry title to [$($newChangeLogEntry.ReleaseTitle)]"
     $ChangeLogEntries.Remove($LatestVersion)
     if ($newChangeLogEntry) {

eng/tox/mypy_hard_failure_packages.py

@@ -13,6 +13,7 @@
   "azure-servicebus",
   "azure-ai-textanalytics",
   "azure-ai-formrecognizer",
+  "azure-ai-translation-document",
   "azure-ai-metricsadvisor",
   "azure-eventgrid",
   "azure-appconfiguration",

scripts/release_issue_status/main.py

@@ -5,12 +5,15 @@
 from datetime import date, datetime
 import subprocess as sp
 from azure.storage.blob import BlobClient
+import reply_generator as rg
+from update_issue_body import update_issue_body, find_readme_link
+import traceback
 
 _NULL = ' '
 _FILE_OUT = 'release_issue_status.csv'
+_FILE_OUT_PYTHON = 'release_python_status.md'
 _PYTHON_SDK_ADMINISTRATORS = {'msyyc', 'RAY-316', 'BigCat20196'}
 
-
 def my_print(cmd):
     print('==' + cmd + ' ==\n')
 
@@ -20,6 +23,20 @@ def print_check(cmd):
     sp.check_call(cmd, shell=True)
 
 
+def output_python_md(issue_status_python):
+    with open(_FILE_OUT_PYTHON, 'w') as file_out:
+        file_out.write('| issue | author | package | assignee | bot advice | created date of issue | delay from created date |\n')
+        file_out.write('| ------ | ------ | ------ | ------ | ------ | ------ | :-----: |\n')
+        file_out.writelines([item.output_python() for item in sorted(issue_status_python, key=_key_select)])
+
+
+def output_csv(issue_status):
+    with open(_FILE_OUT, 'w') as file_out:
+        file_out.write('language,issue,author,package,created date,delay from created date,latest update time,'
+                       'delay from latest update,status,bot advice\n')
+        file_out.writelines([item.output() for item in sorted(issue_status, key=_key_select)])
+  
+
 class IssueStatus:
     link = _NULL
     author = _NULL
@@ -36,6 +53,7 @@ class IssueStatus:
     whether_author_comment = True
     issue_object = _NULL
     labels = _NULL
+    assignee = _NULL
 
     def output(self):
         return '{},{},{},{},{},{},{},{},{},{}\n'.format(self.language, self.link, self.author,
@@ -46,6 +64,13 @@ def output(self):
                                                         self.delay_from_latest_update,
                                                         self.status, self.bot_advice)
 
+    
+    def output_python(self):
+        return '| [#{}]({}) | {} | {} | {} | {} | {} | {} |\n'.format(self.link.split('/')[-1], self.link, self.author, 
+                                                                      self.package, self.assignee, self.bot_advice, 
+                                                                      str(date.fromtimestamp(self.create_date)), 
+                                                                      self.delay_from_create_date)
+
 
 def _extract(str_list, key_word):
     for item in str_list:
@@ -86,8 +111,7 @@ def _extract_author_latest_comment(comments):
 def _whether_author_comment(comments):
     q = set(comment.user.login for comment in comments)
     diff = q.difference(_PYTHON_SDK_ADMINISTRATORS)
-
-    return  len(diff) > 0
+    return len(diff) > 0
 
 def _latest_comment_time(comments, delay_from_create_date):
     q = [(comment.updated_at.timestamp(), comment.user.login)
@@ -96,13 +120,50 @@ def _latest_comment_time(comments, delay_from_create_date):
 
     return delay_from_create_date if not q else int((time.time() - q[-1][0]) / 3600 / 24)
 
+
+def auto_reply(item, sdk_repo, rest_repo, duplicated_issue):
+    print("==========new issue number: {}".format(item.issue_object.number))
+    if 'auto-link' not in item.labels:
+        try:
+            package_name, readme_link = update_issue_body(sdk_repo, rest_repo, item.issue_object.number)
+            print("pkname, readme", package_name, readme_link)
+            item.package = package_name
+            key = ('Python', item.package)
+            duplicated_issue[key] = duplicated_issue.get(key, 0) + 1
+        except Exception as e:
+            item.bot_advice = 'failed to modify the body of the new issue. Please modify manually'
+            item.labels.append('attention')
+            print(e)
+            raise
+        item.labels.append('auto-link')
+        item.issue_object.set_labels(*item.labels)
+    else:
+        try:
+            readme_link = find_readme_link(sdk_repo, item.issue_object.number)
+        except Exception as e:
+            print('Issue: {}  updates body failed'.format(item.issue_object.number))
+            item.bot_advice = 'failed to find Readme link, Please check !!'
+            item.labels.append('attention')
+            raise
+    try:
+        reply = rg.begin_reply_generate(item=item, rest_repo=rest_repo, readme_link=readme_link)
+    except Exception as e:
+        item.bot_advice = 'auto reply failed, Please intervene manually !!'
+        print('Error from auto reply ========================')
+        print('Issue:{}'.format(item.issue_object.number))
+        print(traceback.format_exc())
+        print('==============================================')
+
+
 def main():
     # get latest issue status
     g = Github(os.getenv('TOKEN'))  # please fill user_token
-    repo = g.get_repo('Azure/sdk-release-request')
-    label1 = repo.get_label('ManagementPlane')
-    open_issues = repo.get_issues(state='open', labels=[label1])
+    sdk_repo = g.get_repo('Azure/sdk-release-request')
+    rest_repo = g.get_repo('Azure/azure-rest-api-specs')
+    label1 = sdk_repo.get_label('ManagementPlane')
+    open_issues = sdk_repo.get_issues(state='open', labels=[label1])
     issue_status = []
+    issue_status_python = []
     duplicated_issue = dict()
     start_time = time.time()
     for item in open_issues:
@@ -124,7 +185,9 @@ def main():
         issue.issue_object = item
         issue.labels = [label.name for label in item.labels]
         issue.days_from_latest_commit = _latest_comment_time(item.get_comments(), issue.delay_from_create_date)
-
+        if item.assignee:
+            issue.assignee = item.assignee.login
+        
         issue_status.append(issue)
         key = (issue.language, issue.package)
         duplicated_issue[key] = duplicated_issue.get(key, 0) + 1
@@ -135,18 +198,23 @@ def main():
     # rule2: if latest comment is from author, need response asap
     # rule3: if comment num is 0, it is new issue, better to deal with it asap
     # rule4: if delay from latest update is over 7 days, better to deal with it soon.
-    # rule5: if delay from created date is over 30 days and owner never reply, close it.
-    # rule6: if delay from created date is over 15 days and owner never reply, remind owner to handle it.
+    # rule5: if delay from created date is over 30 days, better to close.
+    # rule6: if delay from created date is over 30 days and owner never reply, close it.
+    # rule7: if delay from created date is over 15 days and owner never reply, remind owner to handle it.
     for item in issue_status:
         if item.status == 'release':
             item.bot_advice = 'better to release asap.'
-        elif item.author == item.author_latest_comment:
-            item.bot_advice = 'new comment for author.'
-        elif item.comment_num == 0:
+        elif item.comment_num == 0 and 'Python' in item.labels:
             item.bot_advice = 'new issue and better to confirm quickly.'
+            try:
+                auto_reply(item, sdk_repo, rest_repo, duplicated_issue)
+            except Exception as e:
+                continue
+        elif not item.author_latest_comment in _PYTHON_SDK_ADMINISTRATORS:
+            item.bot_advice = 'new comment for author.'
         elif item.delay_from_latest_update >= 7:
             item.bot_advice = 'delay for a long time and better to handle now.'
-  
+
         if item.days_from_latest_commit >= 30 and item.language == 'Python' and '30days attention' not in item.labels:
             item.labels.append('30days attention')
             item.issue_object.set_labels(*item.labels)
@@ -157,17 +225,17 @@ def main():
                                              ' please deal with it ASAP. We will close the issue if there is still no response after 15 days!')
             item.labels.append('15days attention')
             item.issue_object.set_labels(*item.labels)
-            
-            
+
         # judge whether there is duplicated issue for same package
         if item.package != _NULL and duplicated_issue.get((item.language, item.package)) > 1:
             item.bot_advice = f'Warning:There is duplicated issue for {item.package}. ' + item.bot_advice
+            
+        if item.language == 'Python':
+            issue_status_python.append(item)
 
     # output result
-    with open(_FILE_OUT, 'w') as file_out:
-        file_out.write('language,issue,author,package,created date,delay from created date,latest update time,'
-                       'delay from latest update,status,bot advice\n')
-        file_out.writelines([item.output() for item in sorted(issue_status, key=_key_select)])
+    output_python_md(issue_status_python)
+    output_csv(issue_status)
 
     # commit to github
     print_check('git add .')
@@ -179,7 +247,7 @@ def main():
                                              blob_name=_FILE_OUT)
     with open(_FILE_OUT, 'rb') as data:
         blob.upload_blob(data, overwrite=True)
-
+        
 
 if __name__ == '__main__':
     main()