Fail "new primer" runs on fatal errors, like the "old primer" (#6746)

Author: jacobtylerwalls

.github/workflows/primer_run_main.yaml

@@ -96,7 +96,11 @@ jobs:
         run: |
           . venv/bin/activate
           pip install -e .
-          python tests/primer/primer_tool.py run --type=main
+          python tests/primer/primer_tool.py run --type=main 2>warnings.txt
+          WARNINGS=$(head -c 65000 < warnings.txt)
+          if [[ $WARNINGS ]]
+          then echo "::warning ::$WARNINGS"
+          fi
       - name: Upload output
         uses: actions/upload-artifact@v3
         with:

.github/workflows/primer_run_pr.yaml

@@ -167,7 +167,11 @@ jobs:
         run: |
           . venv/bin/activate
           pip install -e .
-          python tests/primer/primer_tool.py run --type=pr
+          python tests/primer/primer_tool.py run --type=pr 2>warnings.txt
+          WARNINGS=$(head -c 65000 < warnings.txt)
+          if [[ $WARNINGS ]]
+          then echo "::warning ::$WARNINGS"
+          fi
       - name: Upload output of PR
         uses: actions/upload-artifact@v3
         with:

tests/primer/primer_tool.py

@@ -7,7 +7,9 @@
 import argparse
 import json
 import sys
+import warnings
 from io import StringIO
+from itertools import chain
 from pathlib import Path
 from typing import Dict, List, Union
 
@@ -134,6 +136,22 @@ def _handle_run_command(self) -> None:
         ) as f:
             json.dump(packages, f)
 
+        # Fail loudly (and fail CI pipelines) if any fatal errors are found,
+        # unless they are astroid-errors, in which case just warn.
+        # This is to avoid introducing a dependency on bleeding-edge astroid
+        # for pylint CI pipelines generally, even though we want to use astroid main
+        # for the purpose of diffing emitted messages and generating PR comments.
+        messages = list(chain.from_iterable(packages.values()))
+        astroid_errors = [msg for msg in messages if msg["symbol"] == "astroid-error"]
+        other_fatal_msgs = [
+            msg
+            for msg in messages
+            if msg["type"] == "fatal" and msg["symbol"] != "astroid-error"
+        ]
+        if astroid_errors:
+            warnings.warn(f"Fatal errors traced to astroid:  {astroid_errors}")
+        assert not other_fatal_msgs, other_fatal_msgs
+
     def _handle_compare_command(self) -> None:
         with open(self.config.base_file, encoding="utf-8") as f:
             main_dict: PackageMessages = json.load(f)
@@ -246,7 +264,7 @@ def _lint_package(self, data: PackageToLint) -> list[dict[str, str | int]]:
             arguments += [f"--rcfile={data.pylintrc_relpath}"]
         output = StringIO()
         reporter = JSONReporter(output)
-        Run(arguments, reporter=reporter, do_exit=False)
+        Run(arguments, reporter=reporter, exit=False)
         return json.loads(output.getvalue())
 
     @staticmethod

tests/profile/test_profile_against_externals.py

@@ -46,7 +46,7 @@ def test_run(tmp_path, name, git_repo):
     filepaths = _get_py_files(scanpath=str(checkoutdir))
     print(f"Have {len(filepaths)} files")
 
-    runner = Run(filepaths, reporter=Reporter(), do_exit=False)
+    runner = Run(filepaths, reporter=Reporter(), exit=False)
 
     print(
         f"Had {len(filepaths)} files with {len(runner.linter.reporter.messages)} messages"