Change RemovedInPytest4Warnings to errors by default

To keep existing tests which emit RemovedInPytest4Warnings running, decided
to go with a command line option because:

* Is harder to integrate an ini option with tests which already use an ini file
* It also marks tests which need to be removed/updated in 4.1, when
  RemovedInPytest4Warning and related functionality are removed.

Fix pytest-dev#3737

Author: nicoddemus

Diff for: changelog/3737.removal.rst

@@ -0,0 +1,23 @@
+**RemovedInPytest4Warnings are now errors by default.**
+
+Following our plan to remove deprecated features with as little disruption as
+possible, all warnings of type ``RemovedInPytest4Warnings`` now generate errors
+instead of warning messages.
+
+**The affected features will be effectively removed in pytest 4.1**, so please consult the
+`Deprecations and Removals <https://docs.pytest.org/en/latest/deprecations.html>`__
+section in the docs for directions on how to update existing code.
+
+In the pytest ``4.0.X`` series, it is possible to change the errors back into warnings as a stop
+gap measure by adding this to your ``pytest.ini`` file:
+
+.. code-block:: ini
+
+    [pytest]
+    filterwarnings =
+        ignore::pytest.RemovedInPytest4Warning
+
+But this will stop working when pytest ``4.1`` is released.
+
+**If you have concerns** about the removal of a specific feature, please add a
+comment to `#4348 <https://github.com/pytest-dev/pytest/issues/4348>`__.

Diff for: doc/en/example/assertion/failure_demo.py

@@ -1,6 +1,7 @@
 import six
 
 import _pytest._code
+import pytest
 from pytest import raises
 
 
@@ -16,15 +17,11 @@ def otherfunc_multi(a, b):
     assert a == b
 
 
+@pytest.mark.parametrize("param1, param2", [(3, 6)])
 def test_generative(param1, param2):
     assert param1 * 2 < param2
 
 
-def pytest_generate_tests(metafunc):
-    if "param1" in metafunc.fixturenames:
-        metafunc.addcall(funcargs=dict(param1=3, param2=6))
-
-
 class TestFailing(object):
     def test_simple(self):
         def f():

Diff for: src/_pytest/warnings.py

@@ -9,6 +9,8 @@
 import pytest
 from _pytest import compat
 
+SHOW_PYTEST_WARNINGS_ARG = "-Walways::pytest.RemovedInPytest4Warning"
+
 
 def _setoption(wmod, arg):
     """
@@ -77,6 +79,8 @@ def catch_warnings_for_item(config, ihook, when, item):
             warnings.filterwarnings("always", category=DeprecationWarning)
             warnings.filterwarnings("always", category=PendingDeprecationWarning)
 
+        warnings.filterwarnings("error", category=pytest.RemovedInPytest4Warning)
+
         # filters should have this precedence: mark, cmdline options, ini
         # filters should be applied in the inverse order of precedence
         for arg in inifilters:

Diff for: testing/acceptance_test.py

@@ -14,6 +14,7 @@
 import pytest
 from _pytest.main import EXIT_NOTESTSCOLLECTED
 from _pytest.main import EXIT_USAGEERROR
+from _pytest.warnings import SHOW_PYTEST_WARNINGS_ARG
 
 
 def prepend_pythonpath(*dirs):
@@ -307,7 +308,7 @@ def pytest_runtest_setup(item):
         """
         )
         p = testdir.makepyfile("""def test_func(x): pass""")
-        res = testdir.runpytest(p)
+        res = testdir.runpytest(p, SHOW_PYTEST_WARNINGS_ARG)
         assert res.ret == 0
         res.stdout.fnmatch_lines(["*1 skipped*"])
 
@@ -321,7 +322,9 @@ def test_func(i):
                 pass
         """
         )
-        res = testdir.runpytest(p.basename + "::" + "test_func[1]")
+        res = testdir.runpytest(
+            p.basename + "::" + "test_func[1]", SHOW_PYTEST_WARNINGS_ARG
+        )
         assert res.ret == 0
         res.stdout.fnmatch_lines(["*1 passed*"])
 

Diff for: testing/deprecated_test.py

@@ -5,11 +5,11 @@
 import os
 
 import pytest
+from _pytest.warnings import SHOW_PYTEST_WARNINGS_ARG
 
 pytestmark = pytest.mark.pytester_example_path("deprecated")
 
 
-@pytest.mark.filterwarnings("default")
 def test_yield_tests_deprecation(testdir):
     testdir.makepyfile(
         """
@@ -23,7 +23,7 @@ def test_gen2():
                 yield func1, 1, 1
     """
     )
-    result = testdir.runpytest()
+    result = testdir.runpytest(SHOW_PYTEST_WARNINGS_ARG)
     result.stdout.fnmatch_lines(
         [
             "*test_yield_tests_deprecation.py:3:*yield tests are deprecated*",
@@ -41,7 +41,7 @@ def test_foo(request):
             print(request.node.Module)
     """
     )
-    result = testdir.runpytest()
+    result = testdir.runpytest(SHOW_PYTEST_WARNINGS_ARG)
     result.stdout.fnmatch_lines(
         [
             "*test_compat_properties_deprecation.py:2:*usage of Function.Module is deprecated, "
@@ -63,7 +63,7 @@ def test_foo(fix):
             assert fix == 1
     """
     )
-    result = testdir.runpytest()
+    result = testdir.runpytest(SHOW_PYTEST_WARNINGS_ARG)
     result.stdout.fnmatch_lines(
         [
             "*test_cached_setup_deprecation.py:4:*cached_setup is deprecated*",
@@ -93,7 +93,7 @@ def test_foo(self):
                 pass
     """
     )
-    result = testdir.runpytest()
+    result = testdir.runpytest(SHOW_PYTEST_WARNINGS_ARG)
     result.stdout.fnmatch_lines(
         [
             '*test_custom_class_deprecation.py:1:*"Class" objects in collectors of type "MyModule*',
@@ -102,7 +102,6 @@ def test_foo(self):
     )
 
 
-@pytest.mark.filterwarnings("default")
 def test_funcarg_prefix_deprecation(testdir):
     testdir.makepyfile(
         """
@@ -113,7 +112,7 @@ def test_funcarg_prefix(value):
             assert value == 10
     """
     )
-    result = testdir.runpytest("-ra")
+    result = testdir.runpytest("-ra", SHOW_PYTEST_WARNINGS_ARG)
     result.stdout.fnmatch_lines(
         [
             (
@@ -198,7 +197,6 @@ def test():
     )
 
 
-@pytest.mark.filterwarnings("always:Metafunc.addcall is deprecated")
 def test_metafunc_addcall_deprecated(testdir):
     testdir.makepyfile(
         """
@@ -209,7 +207,7 @@ def test_func(i):
             pass
     """
     )
-    res = testdir.runpytest("-s")
+    res = testdir.runpytest("-s", SHOW_PYTEST_WARNINGS_ARG)
     assert res.ret == 0
     res.stdout.fnmatch_lines(
         ["*Metafunc.addcall is deprecated*", "*2 passed, 2 warnings*"]
@@ -263,7 +261,7 @@ def test_func():
             pass
     """
     )
-    res = testdir.runpytest()
+    res = testdir.runpytest(SHOW_PYTEST_WARNINGS_ARG)
     assert res.ret == 0
     msg = str(PYTEST_PLUGINS_FROM_NON_TOP_LEVEL_CONFTEST).splitlines()[0]
     res.stdout.fnmatch_lines(
@@ -292,6 +290,7 @@ def test_pytest_plugins_in_non_top_level_conftest_deprecated_pyargs(
     testdir.syspathinsert(testdir.tmpdir.join("src"))
 
     args = ("--pyargs", "pkg") if use_pyargs else ()
+    args += (SHOW_PYTEST_WARNINGS_ARG,)
     res = testdir.runpytest(*args)
     assert res.ret == 0
     msg = str(PYTEST_PLUGINS_FROM_NON_TOP_LEVEL_CONFTEST).splitlines()[0]

Diff for: testing/python/collect.py

@@ -7,6 +7,7 @@
 import pytest
 from _pytest.main import EXIT_NOTESTSCOLLECTED
 from _pytest.nodes import Collector
+from _pytest.warnings import SHOW_PYTEST_WARNINGS_ARG
 
 
 class TestModule(object):
@@ -370,7 +371,7 @@ def assert_order_of_execution():
                 yield assert_order_of_execution
         """
         )
-        reprec = testdir.inline_run(o)
+        reprec = testdir.inline_run(o, SHOW_PYTEST_WARNINGS_ARG)
         passed, skipped, failed = reprec.countoutcomes()
         assert passed == 7
         assert not skipped and not failed
@@ -404,7 +405,7 @@ def assert_order_of_execution():
                 yield assert_order_of_execution
         """
         )
-        reprec = testdir.inline_run(o)
+        reprec = testdir.inline_run(o, SHOW_PYTEST_WARNINGS_ARG)
         passed, skipped, failed = reprec.countoutcomes()
         assert passed == 4
         assert not skipped and not failed
@@ -448,7 +449,7 @@ def test_setuplist():
                 assert setuplist[1] != setuplist[2], setuplist
         """
         )
-        reprec = testdir.inline_run(o, "-v")
+        reprec = testdir.inline_run(o, "-v", SHOW_PYTEST_WARNINGS_ARG)
         passed, skipped, failed = reprec.countoutcomes()
         assert passed == 4
         assert not skipped and not failed
@@ -1380,7 +1381,7 @@ def test_hello():
             pass
     """
     )
-    result = testdir.runpytest()
+    result = testdir.runpytest(SHOW_PYTEST_WARNINGS_ARG)
     result.stdout.fnmatch_lines(["*1 passed*"])
 
 
@@ -1407,7 +1408,7 @@ def test_hello(self):
                 pass
     """
     )
-    result = testdir.runpytest("--collect-only")
+    result = testdir.runpytest("--collect-only", SHOW_PYTEST_WARNINGS_ARG)
     result.stdout.fnmatch_lines(
         ["*MyClass*", "*MyInstance*", "*MyFunction*test_hello*"]
     )

Diff for: testing/python/fixture.py

@@ -8,6 +8,7 @@
 from _pytest.fixtures import FixtureRequest
 from _pytest.pathlib import Path
 from _pytest.pytester import get_public_names
+from _pytest.warnings import SHOW_PYTEST_WARNINGS_ARG
 
 
 def test_getfuncargnames():
@@ -975,7 +976,8 @@ def test_func1(something):
             class TestClass(object):
                 def test_func1a(self, something):
                     assert something == "hello"
-        """
+        """,
+            SHOW_PYTEST_WARNINGS_ARG,
         )
         reprec.assertoutcome(passed=2)
 
@@ -997,7 +999,8 @@ def test_func1a(self, something):
                     assert something == "hello"
                 def test_func2b(self, something):
                     assert something == "hello"
-        """
+        """,
+            SHOW_PYTEST_WARNINGS_ARG,
         )
         reprec.assertoutcome(passed=4)
 
@@ -1057,7 +1060,7 @@ def test_two_different_setups(arg1, arg2):
                 assert arg1 != arg2
         """
         )
-        result = testdir.runpytest("-v")
+        result = testdir.runpytest("-v", SHOW_PYTEST_WARNINGS_ARG)
         result.stdout.fnmatch_lines(["*1 passed*"])
 
     def test_request_cached_setup_getfixturevalue(self, testdir):
@@ -1076,7 +1079,7 @@ def test_two_funcarg(arg1):
                 assert arg1 == 11
         """
         )
-        result = testdir.runpytest("-v")
+        result = testdir.runpytest("-v", SHOW_PYTEST_WARNINGS_ARG)
         result.stdout.fnmatch_lines(["*1 passed*"])
 
     def test_request_cached_setup_functional(self, testdir):
@@ -1107,7 +1110,7 @@ def test_check_test0_has_teardown_correct():
                 assert test_0.values == [2]
         """
         )
-        result = testdir.runpytest("-v")
+        result = testdir.runpytest("-v", SHOW_PYTEST_WARNINGS_ARG)
         result.stdout.fnmatch_lines(["*3 passed*"])
 
     def test_issue117_sessionscopeteardown(self, testdir):
@@ -1126,7 +1129,7 @@ def test_func(app):
                 pass
         """
         )
-        result = testdir.runpytest()
+        result = testdir.runpytest(SHOW_PYTEST_WARNINGS_ARG)
         assert result.ret != 0
         result.stdout.fnmatch_lines(["*3/x*", "*ZeroDivisionError*"])
 
@@ -1868,7 +1871,7 @@ def f(hello):
                 yield f, -3
         """
         )
-        reprec = testdir.inline_run()
+        reprec = testdir.inline_run(SHOW_PYTEST_WARNINGS_ARG)
         reprec.assertoutcome(passed=2)
 
     def test_funcarg_and_setup(self, testdir):
@@ -2348,7 +2351,7 @@ def test_1(arg):
             """
             % method
         )
-        result = testdir.runpytest()
+        result = testdir.runpytest(SHOW_PYTEST_WARNINGS_ARG)
         assert result.ret != 0
         result.stdout.fnmatch_lines(
             ["*ScopeMismatch*You tried*function*session*request*"]