Turn warnings into errors in pytest's own test suite

Fix pytest-dev#2588

Author: nicoddemus

changelog/2588.trivial

@@ -0,0 +1 @@
+Turn warnings into errors in pytest's own test suite in order to catch regressions due to deprecations more promptly.

testing/python/collect.py

@@ -12,8 +12,11 @@
 )
 
 
+ignore_parametrized_marks = pytest.mark.filterwarnings('ignore:Applying marks directly to parameters')
+
+
 class TestModule(object):
-    def test_failing_import(self, testdir):
+    def test_failing_import(self, testdir):          
         modcol = testdir.getmodulecol("import alksdjalskdjalkjals")
         pytest.raises(Collector.CollectError, modcol.collect)
 
@@ -567,7 +570,8 @@ def test_overridden_via_param(value):
         rec = testdir.inline_run()
         rec.assertoutcome(passed=1)
 
-    def test_parametrize_with_mark(selfself, testdir):
+    @ignore_parametrized_marks
+    def test_parametrize_with_mark(self, testdir):
         items = testdir.getitems("""
             import pytest
             @pytest.mark.foo
@@ -640,6 +644,7 @@ def test2(self, x, y):
         assert colitems[2].name == 'test2[a-c]'
         assert colitems[3].name == 'test2[b-c]'
 
+    @ignore_parametrized_marks
     def test_parametrize_skipif(self, testdir):
         testdir.makepyfile("""
             import pytest
@@ -653,6 +658,7 @@ def test_skip_if(x):
         result = testdir.runpytest()
         result.stdout.fnmatch_lines('* 2 passed, 1 skipped in *')
 
+    @ignore_parametrized_marks
     def test_parametrize_skip(self, testdir):
         testdir.makepyfile("""
             import pytest
@@ -666,6 +672,7 @@ def test_skip(x):
         result = testdir.runpytest()
         result.stdout.fnmatch_lines('* 2 passed, 1 skipped in *')
 
+    @ignore_parametrized_marks
     def test_parametrize_skipif_no_skip(self, testdir):
         testdir.makepyfile("""
             import pytest
@@ -679,6 +686,7 @@ def test_skipif_no_skip(x):
         result = testdir.runpytest()
         result.stdout.fnmatch_lines('* 1 failed, 2 passed in *')
 
+    @ignore_parametrized_marks
     def test_parametrize_xfail(self, testdir):
         testdir.makepyfile("""
             import pytest
@@ -692,6 +700,7 @@ def test_xfail(x):
         result = testdir.runpytest()
         result.stdout.fnmatch_lines('* 2 passed, 1 xfailed in *')
 
+    @ignore_parametrized_marks
     def test_parametrize_passed(self, testdir):
         testdir.makepyfile("""
             import pytest
@@ -705,6 +714,7 @@ def test_xfail(x):
         result = testdir.runpytest()
         result.stdout.fnmatch_lines('* 2 passed, 1 xpassed in *')
 
+    @ignore_parametrized_marks
     def test_parametrize_xfail_passed(self, testdir):
         testdir.makepyfile("""
             import pytest

testing/python/metafunc.py

@@ -1289,8 +1289,9 @@ def pytest_generate_tests(metafunc):
         assert output.count('preparing foo-3') == 1
 
 
+@pytest.mark.filterwarnings('ignore:Applying marks directly to parameters')
+@pytest.mark.issue308
 class TestMarkersWithParametrization(object):
-    pytestmark = pytest.mark.issue308
 
     def test_simple_mark(self, testdir):
         s = """

testing/python/setup_only.py

@@ -187,7 +187,7 @@ def dynamically_requested_fixture():
             pass
         @pytest.fixture()
         def dependent_fixture(request):
-            request.getfuncargvalue('dynamically_requested_fixture')
+            request.getfixturevalue('dynamically_requested_fixture')
         def test_dyn(dependent_fixture):
             pass
     ''')

testing/test_mark.py

@@ -787,6 +787,7 @@ def assert_test_is_not_selected(keyword):
                   marks=[pytest.mark.xfail, pytest.mark.skip], id=None)),
 
 ])
+@pytest.mark.filterwarnings('ignore')
 def test_parameterset_extractfrom(argval, expected):
     extracted = ParameterSet.extract_from(argval)
     assert extracted == expected

testing/test_recwarn.py

@@ -2,7 +2,6 @@
 import warnings
 import re
 import py
-import sys
 
 import pytest
 from _pytest.recwarn import WarningsRecorder
@@ -125,6 +124,7 @@ def f():
     @pytest.mark.parametrize('warning_type', [PendingDeprecationWarning, DeprecationWarning])
     @pytest.mark.parametrize('mode', ['context_manager', 'call'])
     @pytest.mark.parametrize('call_f_first', [True, False])
+    @pytest.mark.filterwarnings('ignore')
     def test_deprecated_call_modes(self, warning_type, mode, call_f_first):
         """Ensure deprecated_call() captures a deprecation warning as expected inside its
         block/function.
@@ -170,32 +170,6 @@ def f():
                 with pytest.deprecated_call():
                     f()
 
-    def test_deprecated_function_already_called(self, testdir):
-        """deprecated_call should be able to catch a call to a deprecated
-        function even if that function has already been called in the same
-        module. See #1190.
-        """
-        testdir.makepyfile("""
-            import warnings
-            import pytest
-
-            def deprecated_function():
-                warnings.warn("deprecated", DeprecationWarning)
-
-            def test_one():
-                deprecated_function()
-
-            def test_two():
-                pytest.deprecated_call(deprecated_function)
-        """)
-        result = testdir.runpytest()
-        # for some reason in py26 catch_warnings manages to catch the deprecation warning
-        # from deprecated_function(), even with default filters active (which ignore deprecation
-        # warnings)
-        py26 = sys.version_info[:2] == (2, 6)
-        expected = '*=== 2 passed in *===' if not py26 else '*=== 2 passed, 1 warnings in *==='
-        result.stdout.fnmatch_lines(expected)
-
 
 class TestWarns(object):
     def test_strings(self):

testing/test_unittest.py

@@ -9,9 +9,9 @@ def test_simple_unittest(testdir):
         import unittest
         class MyTestCase(unittest.TestCase):
             def testpassing(self):
-                self.assertEquals('foo', 'foo')
+                self.assertEqual('foo', 'foo')
             def test_failing(self):
-                self.assertEquals('foo', 'bar')
+                self.assertEqual('foo', 'bar')
     """)
     reprec = testdir.inline_run(testpath)
     assert reprec.matchreport("testpassing").passed
@@ -23,10 +23,10 @@ def test_runTest_method(testdir):
         import unittest
         class MyTestCaseWithRunTest(unittest.TestCase):
             def runTest(self):
-                self.assertEquals('foo', 'foo')
+                self.assertEqual('foo', 'foo')
         class MyTestCaseWithoutRunTest(unittest.TestCase):
             def runTest(self):
-                self.assertEquals('foo', 'foo')
+                self.assertEqual('foo', 'foo')
             def test_something(self):
                 pass
         """)
@@ -59,7 +59,7 @@ def setUp(self):
             def setup_method(self, method):
                 self.foo2 = 1
             def test_both(self):
-                self.assertEquals(1, self.foo)
+                self.assertEqual(1, self.foo)
                 assert self.foo2 == 1
             def teardown_method(self, method):
                 assert 0, "42"
@@ -136,7 +136,7 @@ def tearDown(self):
                 self.l.append(None)
         class Second(unittest.TestCase):
             def test_check(self):
-                self.assertEquals(MyTestCase.l, [None])
+                self.assertEqual(MyTestCase.l, [None])
     """)
     reprec = testdir.inline_run(testpath)
     passed, skipped, failed = reprec.countoutcomes()
@@ -598,7 +598,7 @@ def test_unittest_not_shown_in_traceback(testdir):
         class t(unittest.TestCase):
             def test_hello(self):
                 x = 3
-                self.assertEquals(x, 4)
+                self.assertEqual(x, 4)
     """)
     res = testdir.runpytest()
     assert "failUnlessEqual" not in res.stdout.str()

testing/test_warnings.py

@@ -33,6 +33,7 @@ def test_func():
     })
 
 
+@pytest.mark.filterwarnings('always')
 def test_normal_flow(testdir, pyfile_with_warnings):
     """
     Check that the warnings section is displayed, containing test node ids followed by
@@ -54,6 +55,7 @@ def test_normal_flow(testdir, pyfile_with_warnings):
     assert result.stdout.str().count('test_normal_flow.py::test_func') == 1
 
 
+@pytest.mark.filterwarnings('always')
 def test_setup_teardown_warnings(testdir, pyfile_with_warnings):
     testdir.makepyfile('''
         import warnings
@@ -115,6 +117,7 @@ def test_ignore(testdir, pyfile_with_warnings, method):
 
 @pytest.mark.skipif(sys.version_info < (3, 0),
                     reason='warnings message is unicode is ok in python3')
+@pytest.mark.filterwarnings('always')
 def test_unicode(testdir, pyfile_with_warnings):
     testdir.makepyfile('''
         # -*- coding: utf8 -*-
@@ -152,6 +155,7 @@ def fix():
             warnings.warn(u"测试")
             yield
 
+        @pytest.mark.filterwarnings('always')
         def test_func(fix):
             pass
     ''')

tox.ini

@@ -193,6 +193,7 @@ python_classes = Test Acceptance
 python_functions = test
 norecursedirs = .tox ja .hg cx_freeze_source
 filterwarnings =
+    error
     # produced by path.local
     ignore:bad escape.*:DeprecationWarning:re
     # produced by path.readlines