diff --git a/CONFIGURATION.md b/CONFIGURATION.md index a1d4773e..91553698 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -60,6 +60,8 @@ This server can be configured using `workspace/didChangeConfiguration` method. E | `pylsp.plugins.pylint.enabled` | `boolean` | Enable or disable the plugin. | `false` | | `pylsp.plugins.pylint.args` | `array` of non-unique `string` items | Arguments to pass to pylint. | `[]` | | `pylsp.plugins.pylint.executable` | `string` | Executable to run pylint with. Enabling this will run pylint on unsaved files via stdin. Can slow down workflow. Only works with python3. | `null` | +| `pylsp.plugins.rope_autoimport.enabled` | `boolean` | Enable or disable autoimport. | `false` | +| `pylsp.plugins.rope_autoimport.memory` | `boolean` | Make the autoimport database memory only. Drastically increases startup time. | `false` | | `pylsp.plugins.rope_completion.enabled` | `boolean` | Enable or disable the plugin. | `false` | | `pylsp.plugins.rope_completion.eager` | `boolean` | Resolve documentation and detail eagerly. | `false` | | `pylsp.plugins.yapf.enabled` | `boolean` | Enable or disable the plugin. | `true` | diff --git a/README.md b/README.md index 3832ad1c..8b80fedd 100644 --- a/README.md +++ b/README.md @@ -140,6 +140,7 @@ pip install 'python-lsp-server[websockets]' ## LSP Server Features * Auto Completion +* [Autoimport](docs/autoimport.md) * Code Linting * Signature Help * Go to definition diff --git a/docs/autoimport.md b/docs/autoimport.md new file mode 100644 index 00000000..832de652 --- /dev/null +++ b/docs/autoimport.md @@ -0,0 +1,25 @@ +# Autoimport for pylsp + +Requirements: + +1. install `python-lsp-server[rope]` +2. set `pylsp.plugins.rope_autoimport.enabled` to `true` + +## Startup + +Autoimport will generate an autoimport sqllite3 database in .ropefolder/autoimport.db on startup. +This will take a few seconds but should be much quicker on future runs. + +## Usage + +Autoimport will provide suggestions to import names from everything in `sys.path`. You can change this by changing where pylsp is running or by setting rope's 'python_path' option. +It will suggest modules, submodules, keywords, functions, and classes. + +Since autoimport inserts everything towards the end of the import group, its recommended you use the isort [plugin](https://github.com/paradoxxxzero/pyls-isort). + +## Credits + +- Most of the code was written by me, @bageljrkhanofemus +- [lyz-code](https://github.com/lyz-code/autoimport) for inspiration and some ideas +- [rope](https://github.com/python-rope/rope), especially @lieryan +- [pyright](https://github.com/Microsoft/pyright) for details on language server implementation diff --git a/pylsp/config/config.py b/pylsp/config/config.py index 4ddb4988..218afdf6 100644 --- a/pylsp/config/config.py +++ b/pylsp/config/config.py @@ -81,7 +81,6 @@ def __init__(self, root_uri, init_opts, process_id, capabilities): if plugin is not None: log.info("Loaded pylsp plugin %s from %s", name, plugin) - # pylint: disable=no-member for plugin_conf in self._pm.hook.pylsp_settings(config=self): self._plugin_settings = _utils.merge_dicts(self._plugin_settings, plugin_conf) diff --git a/pylsp/config/schema.json b/pylsp/config/schema.json index 9e744ac3..aa3d3178 100644 --- a/pylsp/config/schema.json +++ b/pylsp/config/schema.json @@ -354,6 +354,16 @@ "default": null, "description": "Executable to run pylint with. Enabling this will run pylint on unsaved files via stdin. Can slow down workflow. Only works with python3." }, + "pylsp.plugins.rope_autoimport.enabled": { + "type": "boolean", + "default": false, + "description": "Enable or disable autoimport." + }, + "pylsp.plugins.rope_autoimport.memory": { + "type": "boolean", + "default": false, + "description": "Make the autoimport database memory only. Drastically increases startup time." + }, "pylsp.plugins.rope_completion.enabled": { "type": "boolean", "default": false, diff --git a/pylsp/plugins/rope_autoimport.py b/pylsp/plugins/rope_autoimport.py new file mode 100644 index 00000000..c598f42e --- /dev/null +++ b/pylsp/plugins/rope_autoimport.py @@ -0,0 +1,219 @@ +# Copyright 2022- Python Language Server Contributors. + +import logging +from typing import Any, Dict, Generator, List, Set + +import parso +from jedi import Script +from parso.python import tree +from parso.tree import NodeOrLeaf +from rope.base.resources import Resource +from rope.contrib.autoimport.defs import SearchResult +from rope.contrib.autoimport.sqlite import AutoImport + +from pylsp import hookimpl +from pylsp.config.config import Config +from pylsp.workspace import Document, Workspace + +log = logging.getLogger(__name__) + +_score_pow = 5 +_score_max = 10**_score_pow +MAX_RESULTS = 1000 + + +@hookimpl +def pylsp_settings() -> Dict[str, Dict[str, Dict[str, Any]]]: + # Default rope_completion to disabled + return {"plugins": {"rope_autoimport": {"enabled": True, "memory": False}}} + + +def _should_insert(expr: tree.BaseNode, word_node: tree.Leaf) -> bool: + """ + Check if we should insert the word_node on the given expr. + + Works for both correct and incorrect code. This is because the + user is often working on the code as they write it. + """ + if len(expr.children) == 0: + return True + first_child = expr.children[0] + if isinstance(first_child, tree.EndMarker): + if "#" in first_child.prefix: + return False # Check for single line comment + if first_child == word_node: + return True # If the word is the first word then its fine + if len(expr.children) > 1: + if any(node.type == "operator" and "." in node.value or + node.type == "trailer" for node in expr.children): + return False # Check if we're on a method of a function + if isinstance(first_child, (tree.PythonErrorNode, tree.PythonNode)): + # The tree will often include error nodes like this to indicate errors + # we want to ignore errors since the code is being written + return _should_insert(first_child, word_node) + return _handle_first_child(first_child, expr, word_node) + + +def _handle_first_child(first_child: NodeOrLeaf, expr: tree.BaseNode, + word_node: tree.Leaf) -> bool: + """Check if we suggest imports given the following first child.""" + if isinstance(first_child, tree.Import): + return False + if isinstance(first_child, (tree.PythonLeaf, tree.PythonErrorLeaf)): + # Check if the first item is a from or import statement even when incomplete + if first_child.value in ("import", "from"): + return False + if isinstance(first_child, tree.Keyword): + if first_child.value == "def": + return _should_import_function(word_node, expr) + if first_child.value == "class": + return _should_import_class(word_node, expr) + return True + + +def _should_import_class(word_node: tree.Leaf, expr: tree.BaseNode) -> bool: + prev_node = None + for node in expr.children: + if isinstance(node, tree.Name): + if isinstance(prev_node, tree.Operator): + if node == word_node and prev_node.value == "(": + return True + prev_node = node + + return False + + +def _should_import_function(word_node: tree.Leaf, expr: tree.BaseNode) -> bool: + prev_node = None + for node in expr.children: + if _handle_argument(node, word_node): + return True + if isinstance(prev_node, tree.Operator): + if prev_node.value == "->": + if node == word_node: + return True + prev_node = node + return False + + +def _handle_argument(node: NodeOrLeaf, word_node: tree.Leaf): + if isinstance(node, tree.PythonNode): + if node.type == "tfpdef": + if node.children[2] == word_node: + return True + if node.type == "parameters": + for parameter in node.children: + if _handle_argument(parameter, word_node): + return True + return False + + +def _process_statements( + suggestions: List[SearchResult], + doc_uri: str, + word: str, + autoimport: AutoImport, + document: Document, +) -> Generator[Dict[str, Any], None, None]: + for suggestion in suggestions: + insert_line = autoimport.find_insertion_line(document.source) - 1 + start = {"line": insert_line, "character": 0} + edit_range = {"start": start, "end": start} + edit = { + "range": edit_range, + "newText": suggestion.import_statement + "\n" + } + score = _get_score(suggestion.source, suggestion.import_statement, + suggestion.name, word) + if score > _score_max: + continue + # TODO make this markdown + yield { + "label": suggestion.name, + "kind": suggestion.itemkind, + "sortText": _sort_import(score), + "data": { + "doc_uri": doc_uri + }, + "detail": _document(suggestion.import_statement), + "additionalTextEdits": [edit], + } + + +def get_names(script: Script) -> Set[str]: + """Get all names to ignore from the current file.""" + raw_names = script.get_names(definitions=True) + log.debug(raw_names) + return set(name.name for name in raw_names) + + +@hookimpl +def pylsp_completions(config: Config, workspace: Workspace, document: Document, + position): + """Get autoimport suggestions.""" + line = document.lines[position["line"]] + expr = parso.parse(line) + word_node = expr.get_leaf_for_position((1, position["character"])) + if not _should_insert(expr, word_node): + return [] + word = word_node.value + log.debug(f"autoimport: searching for word: {word}") + rope_config = config.settings(document_path=document.path).get("rope", {}) + ignored_names: Set[str] = get_names( + document.jedi_script(use_document_path=True)) + autoimport = workspace._rope_autoimport(rope_config) + suggestions = list( + autoimport.search_full(word, ignored_names=ignored_names)) + results = list( + sorted( + _process_statements(suggestions, document.uri, word, autoimport, + document), + key=lambda statement: statement["sortText"], + )) + if len(results) > MAX_RESULTS: + results = results[:MAX_RESULTS] + return results + + +def _document(import_statement: str) -> str: + return """# Auto-Import\n""" + import_statement + + +def _get_score(source: int, full_statement: str, suggested_name: str, + desired_name) -> int: + import_length = len("import") + full_statement_score = len(full_statement) - import_length + suggested_name_score = ((len(suggested_name) - len(desired_name)))**2 + source_score = 20 * source + return suggested_name_score + full_statement_score + source_score + + +def _sort_import(score: int) -> str: + score = max(min(score, (_score_max) - 1), 0) + # Since we are using ints, we need to pad them. + # We also want to prioritize autoimport behind everything since its the last priority. + # The minimum is to prevent score from overflowing the pad + return "[z" + str(score).rjust(_score_pow, "0") + + +@hookimpl +def pylsp_initialize(config: Config, workspace: Workspace): + """Initialize AutoImport. Generates the cache for local and global items.""" + memory: bool = config.plugin_settings("rope_autoimport").get( + "memory", False) + rope_config = config.settings().get("rope", {}) + autoimport = workspace._rope_autoimport(rope_config, memory) + autoimport.generate_modules_cache() + autoimport.generate_cache() + + +@hookimpl +def pylsp_document_did_save(config: Config, workspace: Workspace, + document: Document): + """Update the names associated with this document.""" + rope_config = config.settings().get("rope", {}) + rope_doucment: Resource = document._rope_resource(rope_config) + autoimport = workspace._rope_autoimport(rope_config) + autoimport.generate_cache(resources=[rope_doucment]) + # Might as well using saving the document as an indicator to regenerate the module cache + autoimport.generate_modules_cache() diff --git a/pylsp/python_lsp.py b/pylsp/python_lsp.py index 94e7a8cf..e663a8a0 100644 --- a/pylsp/python_lsp.py +++ b/pylsp/python_lsp.py @@ -34,7 +34,6 @@ class _StreamHandlerWrapper(socketserver.StreamRequestHandler): def setup(self): super().setup() - # pylint: disable=no-member self.delegate = self.DELEGATE_CLASS(self.rfile, self.wfile) def handle(self): @@ -48,7 +47,6 @@ def handle(self): if isinstance(e, WindowsError) and e.winerror == 10054: pass - # pylint: disable=no-member self.SHUTDOWN_CALL() @@ -212,6 +210,8 @@ def __getitem__(self, item): raise KeyError() def m_shutdown(self, **_kwargs): + for workspace in self.workspaces.values(): + workspace.close() self._shutdown = True def m_exit(self, **_kwargs): @@ -351,6 +351,9 @@ def definitions(self, doc_uri, position): def document_symbols(self, doc_uri): return flatten(self._hook('pylsp_document_symbols', doc_uri)) + def document_did_save(self, doc_uri): + return self._hook("pylsp_document_did_save", doc_uri) + def execute_command(self, command, arguments): return self._hook('pylsp_execute_command', command=command, arguments=arguments) @@ -417,6 +420,7 @@ def m_text_document__did_change(self, contentChanges=None, textDocument=None, ** def m_text_document__did_save(self, textDocument=None, **_kwargs): self.lint(textDocument['uri'], is_saved=True) + self.document_did_save(textDocument['uri']) def m_text_document__code_action(self, textDocument=None, range=None, context=None, **_kwargs): return self.code_actions(textDocument['uri'], range, context) diff --git a/pylsp/workspace.py b/pylsp/workspace.py index bf312f62..eae1a767 100644 --- a/pylsp/workspace.py +++ b/pylsp/workspace.py @@ -7,6 +7,7 @@ import re import functools from threading import RLock +from typing import Optional import jedi @@ -48,6 +49,15 @@ def __init__(self, root_uri, endpoint, config=None): # Whilst incubating, keep rope private self.__rope = None self.__rope_config = None + self.__rope_autoimport = None + + def _rope_autoimport(self, rope_config: Optional, memory: bool = False): + # pylint: disable=import-outside-toplevel + from rope.contrib.autoimport.sqlite import AutoImport + if self.__rope_autoimport is None: + project = self._rope_project_builder(rope_config) + self.__rope_autoimport = AutoImport(project, memory=memory) + return self.__rope_autoimport def _rope_project_builder(self, rope_config): # pylint: disable=import-outside-toplevel @@ -56,8 +66,12 @@ def _rope_project_builder(self, rope_config): # TODO: we could keep track of dirty files and validate only those if self.__rope is None or self.__rope_config != rope_config: rope_folder = rope_config.get('ropeFolder') - self.__rope = Project(self._root_path, ropefolder=rope_folder) - self.__rope.prefs.set('extension_modules', rope_config.get('extensionModules', [])) + if rope_folder: + self.__rope = Project(self._root_path, ropefolder=rope_folder) + else: + self.__rope = Project(self._root_path) + self.__rope.prefs.set('extension_modules', + rope_config.get('extensionModules', [])) self.__rope.prefs.set('ignore_syntax_errors', True) self.__rope.prefs.set('ignore_bad_imports', True) self.__rope.validate() @@ -128,6 +142,10 @@ def _create_document(self, doc_uri, source=None, version=None): rope_project_builder=self._rope_project_builder, ) + def close(self): + if self.__rope_autoimport is not None: + self.__rope_autoimport.close() + class Document: diff --git a/pyproject.toml b/pyproject.toml index 076f17fe..d07cfc4b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ all = [ "pydocstyle>=2.0.0", "pyflakes>=2.4.0,<2.5.0", "pylint>=2.5.0", - "rope>=0.10.5", + "rope>1.2.0", "yapf", "whatthepatch" ] @@ -44,7 +44,7 @@ pycodestyle = ["pycodestyle>=2.8.0,<2.9.0"] pydocstyle = ["pydocstyle>=2.0.0"] pyflakes = ["pyflakes>=2.4.0,<2.5.0"] pylint = ["pylint>=2.5.0"] -rope = ["rope>0.10.5"] +rope = ["rope>1.2.0"] yapf = ["yapf", "whatthepatch>=1.0.2,<2.0.0"] websockets = ["websockets>=10.3"] test = [ @@ -79,6 +79,7 @@ pyflakes = "pylsp.plugins.pyflakes_lint" pylint = "pylsp.plugins.pylint_lint" rope_completion = "pylsp.plugins.rope_completion" rope_rename = "pylsp.plugins.rope_rename" +rope_autoimport = "pylsp.plugins.rope_autoimport" yapf = "pylsp.plugins.yapf_format" [project.scripts] @@ -99,3 +100,7 @@ write_to_template = "__version__ = \"{version}\"\n" # VERSION_INFO is populated [tool.pytest.ini_options] testpaths = ["test"] addopts = "--cov-report html --cov-report term --junitxml=pytest.xml --cov pylsp --cov test" + +[tool.coverage.run] +concurrency = ["multiprocessing", "thread"] + diff --git a/test/fixtures.py b/test/fixtures.py index e57bda6b..ad1f8ce3 100644 --- a/test/fixtures.py +++ b/test/fixtures.py @@ -67,7 +67,8 @@ def workspace(tmpdir): """Return a workspace.""" ws = Workspace(uris.from_fs_path(str(tmpdir)), Mock()) ws._config = Config(ws.root_uri, {}, 0, {}) - return ws + yield ws + ws.close() @pytest.fixture diff --git a/test/plugins/test_autoimport.py b/test/plugins/test_autoimport.py new file mode 100644 index 00000000..b017b92f --- /dev/null +++ b/test/plugins/test_autoimport.py @@ -0,0 +1,238 @@ +# Copyright 2022- Python Language Server Contributors. + +from typing import Dict, List +from unittest.mock import Mock + +import jedi +import parso +import pytest + +from pylsp import lsp, uris +from pylsp.config.config import Config +from pylsp.plugins.rope_autoimport import _get_score, _should_insert, get_names +from pylsp.plugins.rope_autoimport import \ + pylsp_completions as pylsp_autoimport_completions +from pylsp.plugins.rope_autoimport import pylsp_initialize +from pylsp.workspace import Workspace + +DOC_URI = uris.from_fs_path(__file__) + + +@pytest.fixture(scope="session") +def autoimport_workspace(tmp_path_factory) -> Workspace: + "Special autoimport workspace. Persists across sessions to make in-memory sqlite3 database fast." + workspace = Workspace(uris.from_fs_path(str(tmp_path_factory.mktemp("pylsp"))), Mock()) + workspace._config = Config(workspace.root_uri, {}, 0, {}) + workspace._config.update({"rope_autoimport": {"memory": True, "enabled": True}}) + pylsp_initialize(workspace._config, workspace) + yield workspace + workspace.close() + + +# pylint: disable=redefined-outer-name +@pytest.fixture +def completions(config: Config, autoimport_workspace: Workspace, request): + document, position = request.param + com_position = {"line": 0, "character": position} + autoimport_workspace.put_document(DOC_URI, source=document) + doc = autoimport_workspace.get_document(DOC_URI) + yield pylsp_autoimport_completions(config, autoimport_workspace, doc, com_position) + autoimport_workspace.rm_document(DOC_URI) + + +def should_insert(phrase: str, position: int): + expr = parso.parse(phrase) + word_node = expr.get_leaf_for_position((1, position)) + return _should_insert(expr, word_node) + + +def check_dict(query: Dict, results: List[Dict]) -> bool: + for result in results: + if all(result[key] == query[key] for key in query.keys()): + return True + return False + + +@pytest.mark.parametrize("completions", [("""pathli """, 6)], indirect=True) +def test_autoimport_completion(completions): + assert completions + assert check_dict( + { + "label": "pathlib", + "kind": lsp.CompletionItemKind.Module + }, completions) + + +@pytest.mark.parametrize("completions", [("""import """, 7)], indirect=True) +def test_autoimport_import(completions): + assert len(completions) == 0 + + +@pytest.mark.parametrize("completions", [("""pathlib""", 2)], indirect=True) +def test_autoimport_pathlib(completions): + assert completions[0]["label"] == "pathlib" + + start = {"line": 0, "character": 0} + edit_range = {"start": start, "end": start} + assert completions[0]["additionalTextEdits"] == [{ + "range": + edit_range, + "newText": + "import pathlib\n" + }] + + +@pytest.mark.parametrize("completions", [("""import test\n""", 10)], + indirect=True) +def test_autoimport_import_with_name(completions): + assert len(completions) == 0 + + +@pytest.mark.parametrize("completions", [("""def func(s""", 10)], + indirect=True) +def test_autoimport_function(completions): + + assert len(completions) == 0 + + +@pytest.mark.parametrize("completions", [("""class Test""", 10)], + indirect=True) +def test_autoimport_class(completions): + assert len(completions) == 0 + + +@pytest.mark.parametrize("completions", [("""class Test(NamedTupl):""", 20)], + indirect=True) +def test_autoimport_class_complete(completions): + assert len(completions) > 0 + + +@pytest.mark.parametrize("completions", [("""class Test(NamedTupl""", 20)], + indirect=True) +def test_autoimport_class_incomplete(completions): + assert len(completions) > 0 + + +@pytest.mark.parametrize("completions", [("""def func(s:Lis""", 12)], + indirect=True) +def test_autoimport_function_typing(completions): + assert len(completions) > 0 + assert check_dict({"label": "List"}, completions) + + +@pytest.mark.parametrize("completions", [("""def func(s : Lis ):""", 16)], + indirect=True) +def test_autoimport_function_typing_complete(completions): + assert len(completions) > 0 + assert check_dict({"label": "List"}, completions) + + +@pytest.mark.parametrize("completions", + [("""def func(s : Lis ) -> Generat:""", 29)], + indirect=True) +def test_autoimport_function_typing_return(completions): + assert len(completions) > 0 + assert check_dict({"label": "Generator"}, completions) + + +def test_autoimport_defined_name(config, workspace): + document = """List = "hi"\nLis""" + com_position = {"line": 1, "character": 3} + workspace.put_document(DOC_URI, source=document) + doc = workspace.get_document(DOC_URI) + completions = pylsp_autoimport_completions(config, workspace, doc, + com_position) + workspace.rm_document(DOC_URI) + assert not check_dict({"label": "List"}, completions) + + +# This test has several large issues. +# 1. autoimport relies on its sources being written to disk. This makes testing harder +# 2. the hook doesn't handle removed files +# 3. The testing framework cannot access the actual autoimport object so it cannot clear the cache +# def test_autoimport_update_module(config: Config, workspace: Workspace): +# document2 = "SomethingYouShouldntWrite = 1" +# document = """SomethingYouShouldntWrit""" +# com_position = { +# "line": 0, +# "character": 3, +# } +# doc2_path = workspace.root_path + "/test_file_no_one_should_write_to.py" +# if os.path.exists(doc2_path): +# os.remove(doc2_path) +# DOC2_URI = uris.from_fs_path(doc2_path) +# workspace.put_document(DOC_URI, source=document) +# doc = workspace.get_document(DOC_URI) +# completions = pylsp_autoimport_completions(config, workspace, doc, com_position) +# assert len(completions) == 0 +# with open(doc2_path, "w") as f: +# f.write(document2) +# workspace.put_document(DOC2_URI, source=document2) +# doc2 = workspace.get_document(DOC2_URI) +# pylsp_document_did_save(config, workspace, doc2) +# assert check_dict({"label": "SomethingYouShouldntWrite"}, completions) +# workspace.put_document(DOC2_URI, source="\n") +# doc2 = workspace.get_document(DOC2_URI) +# os.remove(doc2_path) +# pylsp_document_did_save(config, workspace, doc2) +# completions = pylsp_autoimport_completions(config, workspace, doc, com_position) +# assert len(completions) == 0 +# workspace.rm_document(DOC_URI) + + +class TestShouldInsert: + + def test_dot(self): + + assert not should_insert("""str.""", 4) + + def test_dot_partial(self): + + assert not should_insert("""str.metho\n""", 9) + + def test_comment(self): + assert not should_insert("""#""", 1) + + def test_comment_indent(self): + + assert not should_insert(""" # """, 5) + + def test_from(self): + assert not should_insert("""from """, 5) + assert should_insert("""from """, 4) + + +def test_sort_sources(): + result1 = _get_score(1, "import pathlib", "pathlib", "pathli") + result2 = _get_score(2, "import pathlib", "pathlib", "pathli") + assert result1 < result2 + + +def test_sort_statements(): + result1 = _get_score(2, "from importlib_metadata import pathlib", + "pathlib", "pathli") + result2 = _get_score(2, "import pathlib", "pathlib", "pathli") + assert result1 > result2 + + +def test_sort_both(): + result1 = _get_score(3, "from importlib_metadata import pathlib", + "pathlib", "pathli") + result2 = _get_score(2, "import pathlib", "pathlib", "pathli") + assert result1 > result2 + + +def test_get_names(): + source = """ + from a import s as e + import blah, bleh + hello = "str" + a, b = 1, 2 + def someone(): + soemthing + class sfa: + sfiosifo + """ + results = get_names(jedi.Script(code=source)) + assert results == set( + ["blah", "bleh", "e", "hello", "someone", "sfa", "a", "b"])