CLN: Clean up of CI scripts (pandas-dev#28482)

Author: datapythonista

ci/azure/posix.yml

@@ -60,15 +60,21 @@ jobs:
         echo "Creating Environment"
         ci/setup_env.sh
       displayName: 'Setup environment and build pandas'
+
     - script: |
         source activate pandas-dev
         ci/run_tests.sh
       displayName: 'Test'
+
     - script: source activate pandas-dev && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd
+      displayName: 'Build versions'
+
     - task: PublishTestResults@2
       inputs:
         testResultsFiles: 'test-data-*.xml'
         testRunTitle: ${{ format('{0}-$(CONDA_PY)', parameters.name) }}
+      displayName: 'Publish test results'
+
     - powershell: |
         $junitXml = "test-data-single.xml"
         $(Get-Content $junitXml | Out-String) -match 'failures="(.*?)"'
@@ -94,6 +100,7 @@ jobs:
           Write-Error "$($matches[1]) tests failed"
         }
       displayName: 'Check for test failures'
+
     - script: |
         source activate pandas-dev
         python ci/print_skipped.py

ci/print_skipped.py

@@ -1,52 +1,40 @@
 #!/usr/bin/env python
-
-import math
 import os
-import sys
 import xml.etree.ElementTree as et
 
 
-def parse_results(filename):
+def main(filename):
+    if not os.path.isfile(filename):
+        return
+
     tree = et.parse(filename)
     root = tree.getroot()
-    skipped = []
-
     current_class = ""
-    i = 1
-    assert i - 1 == len(skipped)
     for el in root.findall("testcase"):
         cn = el.attrib["classname"]
         for sk in el.findall("skipped"):
             old_class = current_class
             current_class = cn
-            name = "{classname}.{name}".format(
-                classname=current_class, name=el.attrib["name"]
-            )
-            msg = sk.attrib["message"]
-            out = ""
             if old_class != current_class:
-                ndigits = int(math.log(i, 10) + 1)
-
-                # 4 for : + space + # + space
-                out += "-" * (len(name + msg) + 4 + ndigits) + "\n"
-            out += "#{i} {name}: {msg}".format(i=i, name=name, msg=msg)
-            skipped.append(out)
-            i += 1
-            assert i - 1 == len(skipped)
-    assert i - 1 == len(skipped)
-    # assert len(skipped) == int(root.attrib['skip'])
-    return "\n".join(skipped)
-
-
-def main():
-    test_files = ["test-data-single.xml", "test-data-multiple.xml", "test-data.xml"]
-
-    print("SKIPPED TESTS:")
-    for fn in test_files:
-        if os.path.isfile(fn):
-            print(parse_results(fn))
-    return 0
+                yield None
+            yield {
+                "class_name": current_class,
+                "test_name": el.attrib["name"],
+                "message": sk.attrib["message"],
+            }
 
 
 if __name__ == "__main__":
-    sys.exit(main())
+    print("SKIPPED TESTS:")
+    i = 1
+    for file_type in ("-single", "-multiple", ""):
+        for test_data in main("test-data{}.xml".format(file_type)):
+            if test_data is None:
+                print("-" * 80)
+            else:
+                print(
+                    "#{i} {class_name}.{test_name}: {message}".format(
+                        **dict(test_data, i=i)
+                    )
+                )
+                i += 1

ci/run_tests.sh

@@ -1,13 +1,6 @@
-#!/bin/bash
+#!/bin/bash -e
 
-set -e
-
-if [ "$DOC" ]; then
-    echo "We are not running pytest as this is a doc-build"
-    exit 0
-fi
-
-# Workaround for pytest-xdist flaky collection order
+# Workaround for pytest-xdist (it collects different tests in the workers if PYTHONHASHSEED is not set)
 # https://github.com/pytest-dev/pytest/issues/920
 # https://github.com/pytest-dev/pytest/issues/1075
 export PYTHONHASHSEED=$(python -c 'import random; print(random.randint(1, 4294967295))')
@@ -16,7 +9,7 @@ if [ -n "$LOCALE_OVERRIDE" ]; then
     export LC_ALL="$LOCALE_OVERRIDE"
     export LANG="$LOCALE_OVERRIDE"
     PANDAS_LOCALE=`python -c 'import pandas; pandas.get_option("display.encoding")'`
-    if [[ "$LOCALE_OVERIDE" != "$PANDAS_LOCALE" ]]; then
+    if [[ "$LOCALE_OVERRIDE" != "$PANDAS_LOCALE" ]]; then
         echo "pandas could not detect the locale. System locale: $LOCALE_OVERRIDE, pandas detected: $PANDAS_LOCALE"
         # TODO Not really aborting the tests until https://github.com/pandas-dev/pandas/issues/23923 is fixed
         # exit 1