condition can be evaluated in the real time

use the skipping.evaluate_condition from pytest in order to keep the same behavior with pytest.mark.skipif. It evaluates the condition in real time so that we can decide whether the case should be rerun in runtime

Change-Id: I2575d368c79480223c84498513a4ef605db0c576

Author: likanglin

README.rst

@@ -108,6 +108,20 @@ You can also specify an optional ``condition`` in the re-run marker:
       import random
       assert random.choice([True, False])
 
+You can use ``@pytest.mark.flaky(condition)`` similarly as ``@pytest.mark.skipif(condition)``, see `pytest-mark-skipif <https://docs.pytest.org/en/6.2.x/reference.html#pytest-mark-skipif>`_
+
+.. code-block:: python
+
+    @pytest.mark.flaky(reruns=2,condition="sys.platform.startswith('win32')")
+    def test_example():
+        import random
+        assert random.choice([True, False])
+    # totally same as the above
+    @pytest.mark.flaky(reruns=2,condition=sys.platform.startswith("win32"))
+    def test_example():
+      import random
+      assert random.choice([True, False])
+
 Note that the test will re-run for any ``condition`` that is truthy.
 
 Output

pytest_rerunfailures.py

@@ -1,9 +1,14 @@
+import os
+import platform
 import re
+import sys
 import time
+import traceback
 import warnings
 
 import pkg_resources
 import pytest
+from _pytest.outcomes import fail
 from _pytest.runner import runtestprotocol
 
 HAS_RESULTLOG = False
@@ -184,11 +189,61 @@ def get_reruns_condition(item):
 
     condition = True
     if rerun_marker is not None and "condition" in rerun_marker.kwargs:
-        condition = rerun_marker.kwargs["condition"]
+        condition = evaluate_condition(
+            item, rerun_marker, rerun_marker.kwargs["condition"]
+        )
 
     return condition
 
 
+def evaluate_condition(item, mark, condition: object) -> bool:
+    """
+    copy from python3.8 _pytest.skipping.py
+    """
+    result = False
+    # String condition.
+    if isinstance(condition, str):
+        globals_ = {
+            "os": os,
+            "sys": sys,
+            "platform": platform,
+            "config": item.config,
+        }
+        if hasattr(item, "obj"):
+            globals_.update(item.obj.__globals__)  # type: ignore[attr-defined]
+        try:
+            filename = f"<{mark.name} condition>"
+            condition_code = compile(condition, filename, "eval")
+            result = eval(condition_code, globals_)
+        except SyntaxError as exc:
+            msglines = [
+                "Error evaluating %r condition" % mark.name,
+                "    " + condition,
+                "    " + " " * (exc.offset or 0) + "^",
+                "SyntaxError: invalid syntax",
+            ]
+            fail("\n".join(msglines), pytrace=False)
+        except Exception as exc:
+            msglines = [
+                "Error evaluating %r condition" % mark.name,
+                "    " + condition,
+                *traceback.format_exception_only(type(exc), exc),
+            ]
+            fail("\n".join(msglines), pytrace=False)
+
+    # Boolean condition.
+    else:
+        try:
+            result = bool(condition)
+        except Exception as exc:
+            msglines = [
+                "Error evaluating %r condition as a boolean" % mark.name,
+                *traceback.format_exception_only(type(exc), exc),
+            ]
+            fail("\n".join(msglines), pytrace=False)
+    return result
+
+
 def _remove_cached_results_from_failed_fixtures(item):
     """
     Note: remove all cached_result attribute from every fixture

test_pytest_rerunfailures.py

@@ -529,8 +529,6 @@ def test_only_rerun_flag(testdir, only_rerun_texts, should_rerun):
         (False, 0),
         (1, 2),
         (0, 0),
-        ("'non-empty'", 2),
-        ("''", 0),
         (["list"], 2),
         ([], 0),
         ({"dict": 1}, 2),
@@ -550,3 +548,41 @@ def test_fail_two():
 
     result = testdir.runpytest()
     assert_outcomes(result, passed=0, failed=1, rerun=expected_reruns)
+
+
+@pytest.mark.parametrize(
+    "condition, expected_reruns",
+    [
+        ('sys.platform.startswith("non-exists") == False', 2),
+        ('os.getpid() != -1', 2)
+    ]
+)
+# before evaluating the condition expression, sys&os&platform package has been imported
+def test_reruns_with_string_condition(testdir, condition, expected_reruns):
+    testdir.makepyfile(
+        f"""
+           import pytest
+            
+           @pytest.mark.flaky(reruns=2, condition='{condition}')
+           def test_fail_two():
+               assert False"""
+    )
+    result = testdir.runpytest()
+    assert_outcomes(result, passed=0, failed=1, rerun=2)
+
+
+def test_reruns_with_string_condition_with_global_var(testdir):
+    testdir.makepyfile(
+        f"""
+              import pytest
+
+              rerunBool = False
+              
+              @pytest.mark.flaky(reruns=2, condition='rerunBool')
+              def test_fail_two():
+                  global rerunBool
+                  rerunBool = True
+                  assert False"""
+    )
+    result = testdir.runpytest()
+    assert_outcomes(result, passed=0, failed=1, rerun=2)