Introduce functional tests for configuration file

Author: Pierre-Sassoulas

tests/config/issue_4580/basic_example.toml renamed to tests/config/functional/issue_4580/basic_example.toml

@@ -0,0 +1,2 @@
+************* Module {abspath}
+{relpath}:1:0: E0014: Incorrect setting encountered in top level configuration-section 'load-plugins' : '[]' (bad-configuration-option-value)

tests/config/issue_4580/correct_basic_name_group.toml renamed to tests/config/functional/issue_4580/correct_basic_name_group.toml

@@ -0,0 +1,3 @@
+************* Module {abspath}
+{relpath}:1:0: E0014: Incorrect setting encountered in top level configuration-section 'load-plugins' : '[]' (bad-configuration-option-value)
+{relpath}:1:0: E0014: Incorrect setting encountered in top level configuration-section 'disable' : 'logging-not-lazy,logging-format-interpolation' (bad-configuration-option-value)

tests/config/issue_4580/correct_import_preferred_module.toml renamed to tests/config/functional/issue_4580/correct_import_preferred_module.toml

@@ -0,0 +1,2 @@
+************* Module {abspath}
+{relpath}:1:0: E0014: Incorrect setting encountered in top level configuration-section 'disable' : 'logging-not-lazy,logging-format-interpolation' (bad-configuration-option-value)

tests/config/issue_4580/empty_list.toml renamed to tests/config/functional/issue_4580/empty_list.toml

@@ -0,0 +1,2 @@
+************* Module {abspath}
+{relpath}:1:0: E0013: Plugin 'pylint_websockets' is impossible to load, is it installed ? ('No module named 'pylint_websockets'') (bad-plugin-value)

tests/config/functional/issue_4580/empty_list.toml.expected

@@ -0,0 +1,51 @@
+from pathlib import Path
+from typing import Tuple
+from unittest.mock import patch
+
+import pytest
+
+from pylint import run_pylint
+
+HERE = Path(__file__).parent
+USER_SPECIFIC = str(HERE.parent.parent)
+FUNCTIONAL_DIR = HERE / "functional"
+# We use string then recast to path so we can use -k in pytest otherwise
+# we get 'pyproject_toml_path0' as a test name
+TOML_PATHS = [str(p) for p in (FUNCTIONAL_DIR).rglob("*.toml")]
+FILE_TO_LINT = str(HERE / "file_to_lint.py")
+
+
+def get_relpath(path: str) -> str:
+    """Get the relative path we want without user specific information"""
+    return path.replace(USER_SPECIFIC, "")[1:]
+
+
+def get_expected(pyproject_toml_path: str) -> Tuple[int, str]:
+    expected_output = ""
+    expected_code = 0
+    expected_result_path = Path(pyproject_toml_path + ".expected")
+    if expected_result_path.exists():
+        expected_code = 2
+        with open(expected_result_path, encoding="utf8") as f:
+            expected_output = f.read()
+        expected_output = expected_output.format(
+            abspath=pyproject_toml_path, relpath=get_relpath(pyproject_toml_path)
+        )
+    return expected_code, expected_output
+
+
+@pytest.mark.parametrize("pyproject_toml_path", TOML_PATHS)
+def test_config_pyproject_toml(pyproject_toml_path, capsys):
+    """Functional tests for configurations."""
+    msg = f"Wrong result with configuration {pyproject_toml_path}"
+    expected_code, expected_output = get_expected(pyproject_toml_path)
+    with patch(
+        "sys.argv", ["pylint", "--rcfile", pyproject_toml_path, FILE_TO_LINT, "-r", "n"]
+    ):
+        with pytest.raises(SystemExit) as exc_info:
+            run_pylint()
+        assert exc_info.value.code == expected_code, msg
+        out, err = capsys.readouterr()
+        # rstrip() applied so we can have a final newline in the expected test file
+        assert expected_output.rstrip() == out.rstrip(), msg
+        assert not err, msg