Remove unnecessary variable defaults (Azure#37604)

* apply black and prepare for parsing the ci.yml so that we can understand if testproxy is enabled or disabled
* The language specific cert trust in Language-Settings has been updated to no longer globally set environment variables SSL_CERT_DIR, SSL_CERT_FILE, and REQUEST_CA_BUNDLE. These are dynamically set in proxy_startup.py

Author: scbedd

.vscode/cspell.json

@@ -470,7 +470,8 @@
     "dicom",
     "WINDOWSVMIMAGE",
     "LINUXVMIMAGE",
-    "MACVMIMAGE"
+    "MACVMIMAGE",
+    "myuseragent"
   ],
   "overrides": [
     {

eng/pipelines/templates/variables/globals.yml

@@ -11,3 +11,4 @@ variables:
   REPOROOT: $(Build.SourcesDirectory)
   WINDOWS_OUTPUTROOT: $(REPOROOT)\out
   WindowsContainerImage: 'onebranch.azurecr.io/windows/ltsc2019/vse2022:latest'
+  GDN_SUPPRESS_FORKED_BUILD_WARNING: true

eng/scripts/Language-Settings.ps1

@@ -391,11 +391,8 @@ function Get-python-DocsMsMetadataForPackage($PackageInfo) {
 
 function Import-Dev-Cert-python
 {
-  Write-Host "Python Trust Methodology"
-
-  $pathToScript = Resolve-Path (Join-Path -Path $PSScriptRoot -ChildPath "../../scripts/devops_tasks/trust_proxy_cert.py")
-  python -m pip install requests
-  python $pathToScript
+  Write-Host "Python no longer requires an out of proc trust methodology." `
+    "The variables SSL_CERT_DIR, SSL_CERT_FILE, and REQUESTS_CA_BUNDLE are now dynamically set in proxy_startup.py"
 }
 
 # Defined in common.ps1 as:

scripts/devops_tasks/tox_harness.py

@@ -354,9 +354,9 @@ def prep_and_run_tox(targeted_packages: List[str], parsed_args: Namespace) -> No
                     "setup_execute_tests.py -> tox_harness.py::prep_and_run_tox",
             )
 
-    return_code = execute_tox_serial(tox_command_tuples)
+    return_result = execute_tox_serial(tox_command_tuples)
 
     if not parsed_args.disablecov:
         collect_tox_coverage_files(targeted_packages)
 
-    sys.exit(return_code)
+    sys.exit(return_result) #type: ignore

tools/azure-sdk-tools/ci_tools/build.py

@@ -9,6 +9,7 @@
 from ci_tools.versioning.version_shared import set_version_py, set_dev_classifier
 from ci_tools.versioning.version_set_dev import get_dev_version, format_build_id
 
+
 def build() -> None:
     parser = argparse.ArgumentParser(
         description="""This is the primary entrypoint for the "build" action. This command is used to build any package within the azure-sdk-for-python repository.""",
@@ -102,7 +103,9 @@ def build() -> None:
     else:
         target_dir = repo_root
 
-    logging.debug(f"Searching for packages starting from {target_dir} with glob string {args.glob_string} and package filter {args.package_filter_string}")
+    logging.debug(
+        f"Searching for packages starting from {target_dir} with glob string {args.glob_string} and package filter {args.package_filter_string}"
+    )
 
     targeted_packages = discover_targeted_packages(
         args.glob_string,

tools/azure-sdk-tools/ci_tools/conda/conda_functions.py

@@ -239,10 +239,12 @@ def create_setup_files(
     with open(cfg_location, "w") as f:
         f.write(SETUP_CFG)
 
+
 def tolerant_match(pkg_name, input_string):
-    pattern = pkg_name.replace('-', '[-_]')
+    pattern = pkg_name.replace("-", "[-_]")
     return fnmatch.fnmatch(input_string, f"*{pattern}*")
 
+
 def create_combined_sdist(
     conda_build: CondaConfiguration, config_assembly_folder: str, config_assembled_folder: str
 ) -> str:
@@ -276,7 +278,8 @@ def create_combined_sdist(
                     [
                         os.path.join(config_assembled_folder, a)
                         for a in os.listdir(config_assembled_folder)
-                        if os.path.isfile(os.path.join(config_assembled_folder, a)) and tolerant_match(conda_build.name, a)
+                        if os.path.isfile(os.path.join(config_assembled_folder, a))
+                        and tolerant_match(conda_build.name, a)
                     ]
                 )
             )
@@ -526,15 +529,22 @@ def prep_and_create_environment(environment_dir: str) -> None:
 
     subprocess.run(["conda", "env", "create", "--prefix", environment_dir], cwd=environment_dir, check=True)
     subprocess.run(
-        ["conda", "install", "--yes", "--quiet", "--prefix", environment_dir, "conda-build", "conda-verify", "typing-extensions", "conda-index"],
-        cwd=environment_dir,
-        check=True
-    )
-    subprocess.run(
-        ["conda", "run", "--prefix", environment_dir, "conda", "list"],
+        [
+            "conda",
+            "install",
+            "--yes",
+            "--quiet",
+            "--prefix",
+            environment_dir,
+            "conda-build",
+            "conda-verify",
+            "typing-extensions",
+            "conda-index",
+        ],
         cwd=environment_dir,
-        check=True
+        check=True,
     )
+    subprocess.run(["conda", "run", "--prefix", environment_dir, "conda", "list"], cwd=environment_dir, check=True)
 
 
 def copy_channel_files(coalescing_channel_dir: str, additional_channel_dir: str) -> None:
@@ -573,9 +583,17 @@ def build_conda_packages(
     if additional_channel_folders:
         for channel in additional_channel_folders:
             copy_channel_files(conda_output_dir, channel)
-            subprocess.run(["conda", "run", "--prefix", conda_env_dir, "python", "-m", "conda_index", conda_output_dir], cwd=repo_root, check=True)
+            subprocess.run(
+                ["conda", "run", "--prefix", conda_env_dir, "python", "-m", "conda_index", conda_output_dir],
+                cwd=repo_root,
+                check=True,
+            )
     else:
-        subprocess.run(["conda", "run", "--prefix", conda_env_dir, "python", "-m", "conda_index", conda_output_dir], cwd=repo_root, check=True)
+        subprocess.run(
+            ["conda", "run", "--prefix", conda_env_dir, "python", "-m", "conda_index", conda_output_dir],
+            cwd=repo_root,
+            check=True,
+        )
 
     for conda_build in conda_configurations:
         conda_build_folder = os.path.join(conda_sdist_dir, conda_build.name).replace("\\", "/")
@@ -595,7 +613,18 @@ def invoke_conda_build(
     channels: List[str] = [],
 ) -> None:
 
-    command = ["conda", "run", "--prefix", conda_env_dir, "conda-build", ".", "--output-folder", conda_output_dir, "-c", conda_output_dir]
+    command = [
+        "conda",
+        "run",
+        "--prefix",
+        conda_env_dir,
+        "conda-build",
+        ".",
+        "--output-folder",
+        conda_output_dir,
+        "-c",
+        conda_output_dir,
+    ]
     for channel in channels:
         command.extend(["-c", channel])
 
@@ -641,8 +670,10 @@ def entrypoint():
 
     args = parser.parse_args()
 
-    if (not args.config and not args.config_file):
-        raise argparse.ArgumentError("config arg", "One of either -c (--config) or -f (--file) argument must be provided.")
+    if not args.config and not args.config_file:
+        raise argparse.ArgumentError(
+            "config arg", "One of either -c (--config) or -f (--file) argument must be provided."
+        )
 
     if args.config_file:
         with open(args.config_file, "r") as f:

tools/azure-sdk-tools/ci_tools/dependency_analysis.py

@@ -11,6 +11,7 @@
 import sys
 import textwrap
 from typing import List, Set, Dict, Tuple, Any
+
 try:
     from collections import Sized
 except:

tools/azure-sdk-tools/ci_tools/functions.py

@@ -42,10 +42,7 @@
 ]
 
 TEST_COMPATIBILITY_MAP = {}
-TEST_PYTHON_DISTRO_INCOMPATIBILITY_MAP = {
-    "azure-storage-blob": "pypy",
-    "azure-eventhub": "pypy"
-}
+TEST_PYTHON_DISTRO_INCOMPATIBILITY_MAP = {"azure-storage-blob": "pypy", "azure-eventhub": "pypy"}
 
 omit_regression = (
     lambda x: "nspkg" not in x
@@ -196,11 +193,13 @@ def discover_targeted_packages(
 
     # glob the starting package set
     collected_packages = glob_packages(glob_string, target_root_dir)
-    logging.info(f"Results for glob_string \"{glob_string}\" and root directory \"{target_root_dir}\" are: {collected_packages}")
+    logging.info(
+        f'Results for glob_string "{glob_string}" and root directory "{target_root_dir}" are: {collected_packages}'
+    )
 
     # apply the additional contains filter
     collected_packages = [pkg for pkg in collected_packages if additional_contains_filter in pkg]
-    logging.info(f"Results after additional contains filter: \"{additional_contains_filter}\" {collected_packages}")
+    logging.info(f'Results after additional contains filter: "{additional_contains_filter}" {collected_packages}')
 
     # filter for compatibility, this means excluding a package that doesn't support py36 when we are running a py36 executable
     if compatibility_filter:
@@ -406,9 +405,7 @@ def find_sdist(dist_dir: str, pkg_name: str, pkg_version: str) -> str:
     packages = [os.path.relpath(w, dist_dir) for w in packages]
 
     if not packages:
-        logging.error(
-            f"No sdist is found in directory {dist_dir} with package name format {pkg_format}."
-        )
+        logging.error(f"No sdist is found in directory {dist_dir} with package name format {pkg_format}.")
         return
     return packages[0]
 

tools/azure-sdk-tools/ci_tools/git_tools.py

@@ -1,5 +1,6 @@
 """Pure git tools for managing local folder Git.
 """
+
 import logging
 
 from git import Repo, GitCommandError

tools/azure-sdk-tools/ci_tools/github_tools.py

@@ -1,5 +1,6 @@
 """Github tools.
 """
+
 from contextlib import contextmanager
 import logging
 import os

tools/azure-sdk-tools/ci_tools/parsing/__init__.py

@@ -3,23 +3,23 @@
     parse_require,
     get_name_from_specifier,
     ParsedSetup,
-    parse_freeze_output,
     read_setup_py_content,
     get_build_config,
     get_config_setting,
     update_build_config,
-    compare_string_to_glob_array
+    compare_string_to_glob_array,
+    get_ci_config,
 )
 
 __all__ = [
     "parse_setup",
     "parse_require",
     "get_name_from_specifier",
     "ParsedSetup",
-    "parse_freeze_output",
     "read_setup_py_content",
     "get_build_config",
     "get_config_setting",
     "update_build_config",
-    "compare_string_to_glob_array"
+    "compare_string_to_glob_array",
+    "get_ci_config",
 ]

tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py

@@ -3,6 +3,7 @@
 import textwrap
 import re
 import fnmatch
+import logging
 
 try:
     # py 311 adds this library natively
@@ -11,7 +12,7 @@
     # otherwise fall back to pypi package tomli
     import tomli as toml
 
-from typing import Dict, List, Tuple, Any
+from typing import Dict, List, Tuple, Any, Optional
 
 # Assumes the presence of setuptools
 from pkg_resources import (
@@ -102,7 +103,7 @@ def from_path(cls, parse_directory_or_file: str):
             ext_modules,
         )
 
-    def get_build_config(self) -> Dict[str, Any]:
+    def get_build_config(self) -> Optional[Dict[str, Any]]:
         return get_build_config(self.folder)
 
     def get_config_setting(self, setting: str, default: Any = True) -> Any:
@@ -162,7 +163,7 @@ def get_config_setting(package_path: str, setting: str, default: Any = True) ->
     return default
 
 
-def get_build_config(package_path: str) -> Dict[str, Any]:
+def get_build_config(package_path: str) -> Optional[Dict[str, Any]]:
     """
     Attempts to retrieve all values within [tools.azure-sdk-build] section of a pyproject.toml.
 
@@ -185,6 +186,29 @@ def get_build_config(package_path: str) -> Dict[str, Any]:
             return {}
 
 
+def get_ci_config(package_path: str) -> Optional[Dict[str, Any]]:
+    """
+    Attempts to retrieve the parsed toml content of a CI.yml associated with this package.
+    """
+    if os.path.isfile(package_path):
+        package_path = os.path.dirname(package_path)
+
+    # this checks exactly one directory up
+    # for sdk/core/azure-core
+    # sdk/core/ci.yml is checked only
+    ci_file = os.path.join(os.path.dirname(package_path), "ci.yml")
+
+    if os.path.exists(ci_file):
+        import yaml
+        try:
+            with open(ci_file, "r") as f:
+                return yaml.safe_load(f)
+        except Exception as e:
+            logging.error(f"Failed to load ci.yml at {ci_file} due to exception {e}")
+
+    return None
+
+
 def read_setup_py_content(setup_filename: str) -> str:
     """
     Get setup.py content, returns a string.
@@ -193,10 +217,9 @@ def read_setup_py_content(setup_filename: str) -> str:
         content = setup_file.read()
         return content
 
-
 def parse_setup(
     setup_filename: str,
-) -> Tuple[str, str, str, List[str], bool, str, str, Dict[str, Any], bool, List[str], str, List[Extension]]:
+) -> Tuple[str, str, str, List[str], bool, str, str, Dict[str, Any], bool, List[str], List[str], str, List[Extension]]:
     """
     Used to evaluate a setup.py (or a directory containing a setup.py) and return a tuple containing:
     (
@@ -234,7 +257,7 @@ def setup(*args, **kwargs):
             not isinstance(node, ast.Expr)
             or not isinstance(node.value, ast.Call)
             or not hasattr(node.value.func, "id")
-            or node.value.func.id != "setup"
+            or node.value.func.id != "setup"  # type: ignore
         ):
             continue
         parsed.body[index:index] = parsed_mock_setup.body
@@ -279,22 +302,23 @@ def setup(*args, **kwargs):
     ext_package = kwargs.get("ext_package", None)
     ext_modules = kwargs.get("ext_modules", [])
 
+    # fmt: off
     return (
-        name,
-        version,
-        python_requires,
-        requires,
-        is_new_sdk,
-        setup_filename,
-        name_space,
-        package_data,
-        include_package_data,
-        classifiers,
-        keywords,
-        ext_package,
-        ext_modules,
+        name,                   # str
+        version,                # str
+        python_requires,        # str
+        requires,               # List[str]
+        is_new_sdk,             # bool
+        setup_filename,         # str
+        name_space,             # str,
+        package_data,           # Dict[str, Any],
+        include_package_data,   # bool,
+        classifiers,            # List[str],
+        keywords,               # List[str] ADJUSTED
+        ext_package,            # str
+        ext_modules,            # List[Extension]
     )
-
+    # fmt: on
 
 def get_install_requires(setup_path: str) -> List[str]:
     """
@@ -312,16 +336,6 @@ def parse_require(req: str) -> Requirement:
     return Requirement.parse(req)
 
 
-def parse_freeze_output(file_location: str) -> Dict[str, str]:
-    """
-    Takes a python requirements file and returns a dictionary representing the contents.
-    """
-    with open(file_location, "r") as f:
-        reqs = f.read()
-
-    return dict((req.name, req) for req in parse_requirements(reqs))
-
-
 def get_name_from_specifier(version: str) -> str:
     """
     Given a specifier string of format of <package-name><comparison><versionNumber>, returns the package name.