Enable py2docfx docs gen tool, remove the dockerimage docs validation (#38085)

* Enable py2docfx docs gen tool, remove the dockerimage docs validation

* Remove the --no-venv-required flag from the command line which was a leftover copy/paste from the original PR

* Change the version to a pre-release for testing as the released version doesn't appear to have these changes

* Update py2docfx version since previous version had problems on Linux

* Change the python command to add -u so the sdtout and stderr streams are unbuffered and we see the full output

* Updates for feedback and to see what's going on with the output

* Remove the Get-Content line which should have been removed in the last commit

* Print the command output on actual different lines

* Finish the sentence in the comment

Author: JimSuplizio

eng/pipelines/docindex.yml

@@ -14,20 +14,16 @@ jobs:
       DocRepoName: azure-docs-sdk-python
       DocValidationImageId: azuresdkimages.azurecr.io/pyrefautocr:latest
     steps:
-      # Docs CI uses Python 3.9.13
+      # py2docfx requires Python >= 3.11
       - task: UsePythonVersion@0
-        displayName: 'Use Python 3.9.13'
+        displayName: 'Use Python 3.11'
         inputs:
-          versionSpec: '3.9.13'
+          versionSpec: '3.11'
 
       # Docs CI upgrades pip, wheel, and setuptools
       - pwsh: python -m pip install --upgrade pip wheel setuptools
         displayName: Update python tools for package verification
 
-      # Pull and build the docker image.
-      - template: /eng/common/pipelines/templates/steps/docker-pull-image.yml
-        parameters:
-          ImageId: "$(DocValidationImageId)"
       # Sync docs repo onboarding files/folders
       - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
         parameters:
@@ -41,6 +37,8 @@ jobs:
             - Name: $(DocRepoOwner)/$(DocRepoName)
               WorkingDirectory: $(DocRepoLocation)
 
+      - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml
+
       - task: Powershell@2
         inputs:
           pwsh: true
@@ -53,7 +51,7 @@ jobs:
         inputs:
           pwsh: true
           filePath: eng/common/scripts/Update-DocsMsPackages.ps1
-          arguments: -DocRepoLocation $(DocRepoLocation) -ImageId '$(DocValidationImageId)'
+          arguments: -DocRepoLocation $(DocRepoLocation)
         displayName: Update Docs Onboarding for main branch
         condition: and(succeeded(), or(eq(variables['Build.Reason'], 'Schedule'), eq(variables['Force.MainUpdate'], 'true')))
 
@@ -92,7 +90,7 @@ jobs:
         parameters:
           BaseRepoBranch: $(DefaultBranch)
           BaseRepoOwner: $(DocRepoOwner)
-          CommitMsg: "Update docs CI configuration"
+          CommitMsg: "Update docs CI configuration Build: $(System.CollectionUri)$(System.TeamProject)/_build/results?buildId=$(Build.BuildId)"
           TargetRepoName: $(DocRepoName)
           TargetRepoOwner: $(DocRepoOwner)
           WorkingDirectory: $(DocRepoLocation)

eng/pipelines/templates/stages/archetype-python-release.yml

@@ -9,7 +9,6 @@ parameters:
   TargetDocRepoOwner: ''
   TargetDocRepoName: ''
   PackageSourceOverride: "https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/"
-  DocValidationImageId: "azuresdkimages.azurecr.io/pyrefautocr:latest"
 
 stages:
   - ${{if and(in(variables['Build.Reason'], 'Manual', ''), eq(variables['System.TeamProject'], 'internal'))}}:
@@ -232,6 +231,15 @@ stages:
                                 - sdk/**/*.md
                                 - .github/CODEOWNERS
                           - download: current
+
+                          # py2docfx requires Python >= 3.11
+                          - task: UsePythonVersion@0
+                            displayName: 'Use Python 3.11'
+                            inputs:
+                              versionSpec: '3.11'
+
+                          - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml
+
                           - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml
                             parameters:
                               PackageInfoLocations:
@@ -244,7 +252,6 @@ stages:
                                 - docs-ref-services/
                                 - metadata/
                               PackageSourceOverride: ${{parameters.PackageSourceOverride}}
-                              DocValidationImageId: ${{parameters.DocValidationImageId}}
 
           - deployment: UpdatePackageVersion
             displayName: "Update Package Version"
@@ -351,6 +358,14 @@ stages:
               Get-ChildItem -Recurse $(Pipeline.Workspace)/${{parameters.ArtifactName}}/
             displayName: Show visible artifacts
 
+          # py2docfx requires Python >= 3.11
+          - task: UsePythonVersion@0
+            displayName: 'Use Python 3.11'
+            inputs:
+              versionSpec: '3.11'
+
+          - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml
+
           - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml
             parameters:
               PackageInfoLocations:
@@ -366,6 +381,5 @@ stages:
                 - docs-ref-services/
                 - metadata/
               PackageSourceOverride: ${{parameters.PackageSourceOverride}}
-              DocValidationImageId: ${{parameters.DocValidationImageId}}
 
           - template: /eng/common/pipelines/templates/steps/docsms-ensure-validation.yml

eng/pipelines/templates/steps/install-rex-validation-tool.yml

@@ -0,0 +1,8 @@
+steps:
+  - pwsh: |
+      Write-Host "python -m pip install -r eng/scripts/docs/py2docfx_requirements.txt"
+      python -m pip install -r eng/scripts/docs/py2docfx_requirements.txt
+      # After install py2docfx,
+      Write-Host "Testing the install. Running python -m py2docfx -h to display the help"
+      python -m py2docfx -h
+    displayName: Install py2docfx for package validation

eng/scripts/Language-Settings.ps1

@@ -440,36 +440,6 @@ function Import-Dev-Cert-python
     "The variables SSL_CERT_DIR, SSL_CERT_FILE, and REQUESTS_CA_BUNDLE are now dynamically set in proxy_startup.py"
 }
 
-# Defined in common.ps1 as:
-# $ValidateDocsMsPackagesFn = "Validate-${Language}-DocMsPackages"
-function Validate-Python-DocMsPackages ($PackageInfo, $PackageInfos, $PackageSourceOverride, $DocValidationImageId)
-{
-  # While eng/common/scripts/Update-DocsMsMetadata.ps1 is still passing a single packageInfo, process as a batch
-  if (!$PackageInfos) {
-    $PackageInfos =  @($PackageInfo)
-  }
-
-  $allSucceeded = $true
-  foreach ($item in $PackageInfos) {
-    # If the Version is IGNORE that means it's a source install and those aren't run through ValidatePackage
-    if ($item.Version -eq 'IGNORE') {
-      continue
-    }
-
-    $result = ValidatePackage `
-      -packageName $item.Name `
-      -packageVersion "==$($item.Version)" `
-      -PackageSourceOverride $PackageSourceOverride `
-      -DocValidationImageId $DocValidationImageId
-
-    if (!$result) {
-      $allSucceeded = $false
-    }
-  }
-
-  return $allSucceeded
-}
-
 function Get-python-EmitterName() {
   return "@azure-tools/typespec-python"
 }