Use pytest-randomly for reproducible random test parameters

.github/workflows/ci.yml

@@ -135,7 +135,8 @@ jobs:
       - name: Install dependencies
         run: pip install -r dev_tools/requirements/isolated-base.env.txt
       - name: Test each module in isolation
-        run: pytest -n auto -m slow dev_tools/packaging/isolated_packages_test.py
+        # TODO(#5870) - remove --randomly-dont-reorganize after fixing order-related test failures
+        run: pytest --randomly-dont-reorganize -n auto -m slow dev_tools/packaging/isolated_packages_test.py
   pytest:
     name: Pytest Ubuntu
     strategy:
@@ -159,7 +160,7 @@ jobs:
       - name: Run Quil dependencies
         run: docker-compose -f cirq-rigetti/docker-compose.test.yaml up -d
       - name: Pytest check
-        run: check/pytest -n auto --ignore=cirq-core/cirq/contrib --actually-quiet --rigetti-integration
+        run: check/pytest -n auto --ignore=cirq-core/cirq/contrib --rigetti-integration
       - name: Stop Quil dependencies
         run: docker-compose -f cirq-rigetti/docker-compose.test.yaml down
   build_docs:
@@ -240,7 +241,7 @@ jobs:
       - name: Pytest Windows
         run: |
           source dev_tools/pypath
-          check/pytest -n auto --ignore=cirq-core/cirq/contrib --actually-quiet
+          check/pytest -n auto --ignore=cirq-core/cirq/contrib
         shell: bash
   macos:
     name: Pytest MacOS

check/pytest

@@ -4,9 +4,7 @@
 # Runs pytest on the repository.
 #
 # Usage:
-#     check/pytest [--actually-quiet] [--flags for pytest] [file-paths-relative-to-repo-root]
-#
-# The --actually-quiet argument filters out any progress output from pytest.
+#     check/pytest [--flags for pytest] [file-paths-relative-to-repo-root]
 #
 # You may specify pytest flags and specific files to test. The file paths
 # must be relative to the repository root. If no files are specified, everything
@@ -18,17 +16,15 @@ cd "$(dirname "${BASH_SOURCE[0]}")"
 cd "$(git rev-parse --show-toplevel)"
 
 PYTEST_ARGS=()
-ACTUALLY_QUIET=""
+# TODO(#5870) - remove this after fixing order-related test failures
+PYTEST_ARGS+=("--randomly-dont-reorganize")
+
 PARALLEL=""
 for arg in "$@"; do
-    if [[ "${arg}" == "--actually-quiet" ]]; then
-        ACTUALLY_QUIET=1
-    else
-        if [[ "${arg}" == "-n" ]]; then
-            PARALLEL=1
-        fi
-        PYTEST_ARGS+=("${arg}")
+    if [[ "${arg}" == "-n" ]]; then
+	PARALLEL=1
     fi
+    PYTEST_ARGS+=("${arg}")
 done
 
 if [ -z "${PARALLEL}" ]; then
@@ -41,18 +37,8 @@ if [ "$PYTHON_VERSION" -lt "37" ]; then
   PYTEST_ARGS+=("--ignore=cirq-rigetti")
 fi
 
-: ${CIRQ_TESTING_RANDOM_SEED=$(git log -1 --pretty="%ct")}
-export CIRQ_TESTING_RANDOM_SEED
-
-if [ -z "${ACTUALLY_QUIET}" ]; then
-    pytest "${PYTEST_ARGS[@]}"
-    RESULT=$?
-else
-    # Filter out lines like "...F....x...      [ 42%]", with coloring.
-    pytest -q --color=yes "${PYTEST_ARGS[@]}" |
-        perl -nle'print if not m{^(.\[0m)?[\.FEsx]+(.\[36m)?\s+\[\s*\d+%\](.\[0m)?$}'
-    RESULT=${PIPESTATUS[0]}
-fi
+pytest "${PYTEST_ARGS[@]}"
+RESULT=$?
 
 if [ "$RESULT" = 5 ]; then
   echo "[exit 5] No tests collected, but ignored."

check/pytest-and-incremental-coverage

@@ -71,8 +71,7 @@ fi
 source dev_tools/pypath
 
 # Run tests while producing coverage files.
-check/pytest --actually-quiet \
-             --cov \
+check/pytest --cov \
              --cov-config=dev_tools/conf/.coveragerc \
              "${PYTEST_ARGS[@]}"
 pytest_result=$?

check/pytest-changed-files

@@ -77,7 +77,4 @@ fi
 
 source dev_tools/pypath
 
-: ${CIRQ_TESTING_RANDOM_SEED=$(git log -1 --pretty="%ct")}
-export CIRQ_TESTING_RANDOM_SEED
-
 pytest ${rest} "${changed[@]}"

cirq-core/cirq/contrib/routing/initialization_test.py

@@ -39,8 +39,9 @@ def test_initialization_reproducible_with_seed(seed):
     eq.add_equality_group(*mappings)
 
 
-@pytest.mark.parametrize('graph_seed,state', [(random.randint(0, 2**32), np.random.get_state())])
-def test_initialization_with_no_seed(graph_seed, state):
+def test_initialization_with_no_seed():
+    graph_seed = random.randint(0, 2**32)
+    state = np.random.get_state()
     mappings = []
     for _ in range(3):
         np.random.set_state(state)

cirq-core/cirq/ops/boolean_hamiltonian_test.py

@@ -127,9 +127,7 @@ def test_gray_code_sorting(n_bits, expected_hs):
             x //= 2
         hs_template.append(tuple(sorted(h)))
 
-    for seed in range(10):
-        random.seed(seed)
-
+    for _ in range(10):
         hs = hs_template.copy()
         random.shuffle(hs)
 

cirq-core/cirq/sim/clifford/clifford_simulator_test.py

@@ -335,8 +335,6 @@ def test_clifford_circuit(split):
     (q0, q1) = (cirq.LineQubit(0), cirq.LineQubit(1))
     circuit = cirq.Circuit()
 
-    np.random.seed(0)
-
     for _ in range(100):
         x = np.random.randint(7)
 

cirq-core/cirq/transformers/analytical_decompositions/controlled_gate_decomposition_test.py

@@ -88,7 +88,6 @@ def test_decompose_specific_matrices():
 
 
 def test_decompose_random_unitary():
-    np.random.seed(0)
     for controls_count in range(5):
         for _ in range(10):
             _test_decompose(_random_unitary(), controls_count)
@@ -97,7 +96,6 @@ def test_decompose_random_unitary():
 
 
 def test_decompose_random_special_unitary():
-    np.random.seed(0)
     for controls_count in range(5):
         for _ in range(10):
             _test_decompose(_random_special_unitary(), controls_count)

conftest.py

@@ -12,18 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
-import random
-
 import pytest
 
-# allow CI execution of isolated_packages_test.py without numpy
-try:
-    import numpy
-except ImportError:
-    # coverage: ignore
-    numpy = None
-
 
 def pytest_configure(config):
     # Ignore deprecation warnings in python code generated from our protobuf definitions.
@@ -55,10 +45,3 @@ def closefigures():
 
     yield
     plt.close('all')
-
-
-# skip seeding for unset or empty CIRQ_TESTING_RANDOM_SEED
-if numpy is not None and os.environ.get('CIRQ_TESTING_RANDOM_SEED'):
-    rngseed = int(os.environ['CIRQ_TESTING_RANDOM_SEED'])
-    random.seed(rngseed)
-    numpy.random.seed(rngseed)

dev_tools/bash_scripts_test.py

@@ -349,8 +349,7 @@ def test_pytest_and_incremental_coverage_branch_selection(tmpdir_factory):
     assert result.returncode == 0
     assert result.stdout == (
         'INTERCEPTED check/pytest '
-        '--actually-quiet --cov '
-        '--cov-config=dev_tools/conf/.coveragerc\n'
+        '--cov --cov-config=dev_tools/conf/.coveragerc\n'
         'The annotate command will be removed in a future version.\n'
         'Get in touch if you still use it: [email protected]\n'
         'No data to report.\n'
@@ -376,8 +375,7 @@ def test_pytest_and_incremental_coverage_branch_selection(tmpdir_factory):
     assert result.returncode == 0
     assert result.stdout == (
         'INTERCEPTED check/pytest '
-        '--actually-quiet --cov '
-        '--cov-config=dev_tools/conf/.coveragerc\n'
+        '--cov --cov-config=dev_tools/conf/.coveragerc\n'
         'The annotate command will be removed in a future version.\n'
         'Get in touch if you still use it: [email protected]\n'
         'No data to report.\n'
@@ -395,8 +393,7 @@ def test_pytest_and_incremental_coverage_branch_selection(tmpdir_factory):
     assert result.returncode == 0
     assert result.stdout == (
         'INTERCEPTED check/pytest '
-        '--actually-quiet --cov '
-        '--cov-config=dev_tools/conf/.coveragerc\n'
+        '--cov --cov-config=dev_tools/conf/.coveragerc\n'
         'The annotate command will be removed in a future version.\n'
         'Get in touch if you still use it: [email protected]\n'
         'No data to report.\n'
@@ -414,8 +411,7 @@ def test_pytest_and_incremental_coverage_branch_selection(tmpdir_factory):
     assert result.returncode == 0
     assert result.stdout == (
         'INTERCEPTED check/pytest '
-        '--actually-quiet --cov '
-        '--cov-config=dev_tools/conf/.coveragerc\n'
+        '--cov --cov-config=dev_tools/conf/.coveragerc\n'
         'The annotate command will be removed in a future version.\n'
         'Get in touch if you still use it: [email protected]\n'
         'No data to report.\n'
@@ -433,8 +429,7 @@ def test_pytest_and_incremental_coverage_branch_selection(tmpdir_factory):
     assert result.returncode == 0
     assert result.stdout == (
         'INTERCEPTED check/pytest '
-        '--actually-quiet --cov '
-        '--cov-config=dev_tools/conf/.coveragerc\n'
+        '--cov --cov-config=dev_tools/conf/.coveragerc\n'
         'The annotate command will be removed in a future version.\n'
         'Get in touch if you still use it: [email protected]\n'
         'No data to report.\n'
@@ -464,8 +459,7 @@ def test_pytest_and_incremental_coverage_branch_selection(tmpdir_factory):
     assert result.returncode == 0
     assert result.stdout == (
         'INTERCEPTED check/pytest '
-        '--actually-quiet --cov '
-        '--cov-config=dev_tools/conf/.coveragerc\n'
+        '--cov --cov-config=dev_tools/conf/.coveragerc\n'
         'The annotate command will be removed in a future version.\n'
         'Get in touch if you still use it: [email protected]\n'
         'No data to report.\n'
@@ -483,8 +477,7 @@ def test_pytest_and_incremental_coverage_branch_selection(tmpdir_factory):
     assert result.returncode == 0
     assert result.stdout == (
         'INTERCEPTED check/pytest '
-        '--actually-quiet --cov '
-        '--cov-config=dev_tools/conf/.coveragerc\n'
+        '--cov --cov-config=dev_tools/conf/.coveragerc\n'
         'The annotate command will be removed in a future version.\n'
         'Get in touch if you still use it: [email protected]\n'
         'No data to report.\n'
@@ -509,8 +502,7 @@ def test_pytest_and_incremental_coverage_branch_selection(tmpdir_factory):
     assert result.returncode == 0
     assert result.stdout.startswith(
         'INTERCEPTED check/pytest '
-        '--actually-quiet --cov '
-        '--cov-config=dev_tools/conf/.coveragerc\n'
+        '--cov --cov-config=dev_tools/conf/.coveragerc\n'
         'The annotate command will be removed in a future version.\n'
         'Get in touch if you still use it: [email protected]\n'
         'No data to report.\n'

dev_tools/packaging/isolated_packages_test.py

@@ -48,8 +48,9 @@ def test_isolated_packages(cloned_env, module):
     )
     assert result.returncode == 0, f"Failed to install {module.name}:\n{result.stderr}"
 
+    # TODO(#5870) - remove --randomly-dont-reorganize after fixing order-related test failures
     result = shell_tools.run(
-        f"{env}/bin/pytest ./{module.root} --ignore ./cirq-core/cirq/contrib".split(),
+        f"{env}/bin/pytest --randomly-dont-reorganize ./{module.root} --ignore ./cirq-core/cirq/contrib".split(),
         capture_output=True,
         check=False,
     )

dev_tools/requirements/deps/pytest.txt

@@ -3,6 +3,7 @@
 pytest
 pytest-asyncio
 pytest-cov
+pytest-randomly
 # Notebook >=6.4.8 + coverage > 6.2 hangs CI: https://github.com/quantumlib/Cirq/issues/4897
 coverage<=6.2
 

examples/direct_fidelity_estimation_test.py

@@ -111,7 +111,6 @@ def noisy_moment(self, moment, system_qubits):
                     ),
                 ]
 
-        np.random.seed(0)
         noise = NoiseOnLastQubitOnly()
         noisy_simulator = cirq.DensityMatrixSimulator(noise=noise)