Merge branch 'main' into 436-create-a-json-file-that-integrates-the-dictionaries-from-and

Author: GideonKoenig

package-parser/.gitignore

@@ -30,3 +30,6 @@ out/
 
 # VSCode Settings
 .vscode/
+
+# IntelliJ/Pycharm settings
+.idea/

package-parser/package-parser.iml

@@ -11,4 +11,4 @@
     <orderEntry type="jdk" jdkName="Poetry (api-editor)" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
-</module>
+</module>

package-parser/package_parser/commands/generate_annotations/__init__.py

@@ -1 +1,2 @@
 from ._combine import write_json
+from ._generate_annotations import generate_annotations

package-parser/package_parser/commands/generate_annotations/_generate_annotations.py

@@ -0,0 +1,190 @@
+import json
+from io import TextIOWrapper
+from pathlib import Path
+from typing import Any
+
+from package_parser.commands.find_usages import (
+    ClassUsage,
+    FunctionUsage,
+    UsageStore,
+    ValueUsage,
+)
+from package_parser.commands.get_api import API
+from package_parser.utils import parent_qname
+
+
+def generate_annotations(
+    api_file: TextIOWrapper, usages_file: TextIOWrapper, out_dir: Path
+):
+    with api_file:
+        api_json = json.load(api_file)
+        api = API.from_json(api_json)
+
+    with usages_file:
+        usages_json = json.load(usages_file)
+        usages = UsageStore.from_json(usages_json)
+
+    # out_dir.mkdir(parents=True, exist_ok=True)
+    # base_file_name = api_file.name.replace("__api.json", "")
+
+    __preprocess_usages(usages, api)
+    constant_parameters = __find_constant_parameters(usages, api)
+    return constant_parameters
+
+
+def __preprocess_usages(usages: UsageStore, api: API) -> None:
+    __remove_internal_usages(usages, api)
+    __add_unused_api_elements(usages, api)
+    __add_implicit_usages_of_default_value(usages, api)
+
+
+def __remove_internal_usages(usages: UsageStore, api: API) -> None:
+    """
+    Removes usages of internal parts of the API. It might incorrectly remove some calls to methods that are inherited
+    from internal classes into a public class but these are just fit/predict/etc., i.e. something we want to keep
+    unchanged anyway.
+
+    :param usages: Usage store
+    :param api: Description of the API
+    """
+
+    # Internal classes
+    for class_qname in list(usages.class_usages.keys()):
+        if not api.is_public_class(class_qname):
+            print(f"Removing usages of internal class {class_qname}")
+            usages.remove_class(class_qname)
+
+    # Internal functions
+    for function_qname in list(usages.function_usages.keys()):
+        if not api.is_public_function(function_qname):
+            print(f"Removing usages of internal function {function_qname}")
+            usages.remove_function(function_qname)
+
+    # Internal parameters
+    parameter_qnames = set(api.parameters().keys())
+
+    for parameter_qname in list(usages.parameter_usages.keys()):
+        function_qname = parent_qname(parameter_qname)
+        if parameter_qname not in parameter_qnames or not api.is_public_function(
+            function_qname
+        ):
+            print(f"Removing usages of internal parameter {parameter_qname}")
+            usages.remove_parameter(parameter_qname)
+
+
+def __add_unused_api_elements(usages: UsageStore, api: API) -> None:
+    """
+    Adds unused API elements to the UsageStore. When a class, function or parameter is not used, it is not content of
+    the UsageStore, so we need to add it.
+
+    :param usages: Usage store
+    :param api: Description of the API
+    """
+
+    # Public classes
+    for class_qname in api.classes:
+        if api.is_public_class(class_qname):
+            usages.init_class(class_qname)
+
+    # Public functions
+    for function in api.functions.values():
+        if api.is_public_function(function.qname):
+            usages.init_function(function.qname)
+
+            # "Public" parameters
+            for parameter in function.parameters:
+                parameter_qname = f"{function.qname}.{parameter.name}"
+                usages.init_parameter(parameter_qname)
+                usages.init_value(parameter_qname)
+
+
+def __add_implicit_usages_of_default_value(usages: UsageStore, api: API) -> None:
+    """
+    Adds the implicit usages of a parameters default value. When a function is called and a parameter is used with its
+    default value, that usage of a value is not part of the UsageStore, so  we need to add it.
+
+    :param usages: Usage store
+    :param api: Description of the API
+    """
+
+    for parameter_qname, parameter_usage_list in list(usages.parameter_usages.items()):
+        default_value = api.get_default_value(parameter_qname)
+        if default_value is None:
+            continue
+
+        function_qname = parent_qname(parameter_qname)
+        function_usage_list = usages.function_usages[function_qname]
+
+        locations_of_implicit_usages_of_default_value = set(
+            [it.location for it in function_usage_list]
+        ) - set([it.location for it in parameter_usage_list])
+
+        for location in locations_of_implicit_usages_of_default_value:
+            usages.add_value_usage(parameter_qname, default_value, location)
+
+
+def __find_constant_parameters(
+    usages: UsageStore, api: API
+) -> dict[str, dict[str, str]]:
+    """
+    Returns all parameters that are only ever assigned a single value.
+
+    :param usages: Usage store
+    """
+
+    result = {}
+
+    for parameter_qname in list(usages.parameter_usages.keys()):
+
+        if len(usages.value_usages[parameter_qname].values()) == 0:
+            continue
+
+        if len(usages.value_usages[parameter_qname].keys()) == 1:
+            target_name = __qname_to_target_name(api, parameter_qname)
+            default_type, default_value = __get_default_type_from_value(
+                str(usages.most_common_value(parameter_qname))
+            )
+            print(target_name)
+            result[target_name] = {
+                "target": target_name,
+                "defaultType": default_type,
+                "defaultValue": default_value,
+            }
+
+    print(json.dumps(result))
+    return result
+
+
+def __qname_to_target_name(api: API, qname: str) -> str:
+    target_elements = qname.split(".")
+
+    package_name = api.package
+    module_name = class_name = function_name = parameter_name = ""
+
+    if ".".join(target_elements) in api.parameters().keys():
+        parameter_name = "/" + target_elements.pop()
+    if ".".join(target_elements) in api.functions.keys():
+        function_name = f"/{target_elements.pop()}"
+    if ".".join(target_elements) in api.classes.keys():
+        class_name = f"/{target_elements.pop()}"
+    if ".".join(target_elements) in api.modules.keys():
+        module_name = "/" + ".".join(target_elements)
+
+    return package_name + module_name + class_name + function_name + parameter_name
+
+
+def __get_default_type_from_value(default_value: str) -> tuple[str, str]:
+    default_value = str(default_value)[1:-1]
+
+    if default_value == "null":
+        default_type = "none"
+    elif default_value == "True" or default_value == "False":
+        default_type = "boolean"
+    elif default_value.isnumeric():
+        default_type = "number"
+        default_value = default_value
+    else:
+        default_type = "string"
+        default_value = default_value
+
+    return default_type, default_value

package-parser/package_parser/commands/generate_annotations/_generate_unused_annotations.py

@@ -0,0 +1,44 @@
+import json
+import re
+from typing import Dict, List, Tuple
+
+
+def generate_unused_annotations(in_file_path: str):
+    """
+    Returns a Dict of unused functions or classes
+
+    :param in_file_path: JSON file that contains a list of unused functions or classes
+    """
+
+    with open(in_file_path, "r", encoding="UTF-8") as in_file:
+        data = json.load(in_file)
+
+    unuseds: Dict[str, Dict[str, str]] = {}
+    for name in data:
+        formatted_name = format_name(name)
+        unuseds[formatted_name] = {"target": formatted_name}
+
+    return unuseds
+
+
+def format_name(name: str):
+    if name is None:
+        return None
+
+    parts = re.split("\\.", name)
+    newname = "sklearn/" + parts[0]
+
+    if len(parts) == 1:
+        return newname
+
+    slash = False
+    for part in parts[1:-1]:
+        if not slash and re.match("^_{0,2}[A-Z]", part):
+            slash = True
+        if slash:
+            newname += "/" + part
+        else:
+            newname += "." + part
+
+    newname += "/" + parts[-1]
+    return newname

package-parser/tests/commands/generate_annotations/test_determine_constant_parameters.py

@@ -0,0 +1,59 @@
+import json
+import os
+
+import pytest
+from package_parser.commands.find_usages._model import UsageStore
+from package_parser.commands.generate_annotations._generate_annotations import (
+    generate_annotations,
+)
+
+# Expected output:
+# @Unused annotations should be created for the following declarations:
+#
+# test.Unused_Class
+# test.unused_global_function
+# test.Commonly_Used_Class.unused_method
+#
+# @Constant annotations should be created for the following parameters:
+#
+# test.commonly_used_global_function.useless_required_parameter (with value "'blup'")
+# test.commonly_used_global_function.unused_optional_parameter (with value "'bla'", i.e. the default value)
+# test.commonly_used_global_function.useless_optional_parameter (with value "'bla'")
+
+
+def test_determination_of_constant_parameters():
+
+    expected = {
+        "test/test/commonly_used_global_function/useless_required_parameter": {
+            "target": "test/test/commonly_used_global_function/useless_required_parameter",
+            "defaultType": "string",
+            "defaultValue": "blup",
+        },
+        "test/test/commonly_used_global_function/unused_optional_parameter": {
+            "target": "test/test/commonly_used_global_function/unused_optional_parameter",
+            "defaultType": "string",
+            "defaultValue": "bla",
+        },
+        "test/test/commonly_used_global_function/useless_optional_parameter": {
+            "target": "test/test/commonly_used_global_function/useless_optional_parameter",
+            "defaultType": "string",
+            "defaultValue": "bla",
+        },
+    }
+
+    api_json_path = os.path.join(
+        os.getcwd(), "tests", "data", "constant", "api_data.json"
+    )
+    usages_json_path = os.path.join(
+        os.getcwd(), "tests", "data", "constant", "usage_data.json"
+    )
+
+    api_file = open(api_json_path)
+    usages_file = open(usages_json_path)
+
+    constant_parameters = generate_annotations(api_file, usages_file, "/.")
+
+    api_file.close()
+    usages_file.close()
+
+    assert constant_parameters == expected

package-parser/tests/commands/generate_annotations/test_generate_unused_annotations.py

@@ -0,0 +1,67 @@
+import pytest
+from package_parser.commands.generate_annotations._generate_unused_annotations import (
+    format_name,
+    generate_unused_annotations,
+)
+
+EXPECTED_VALUE = {
+    "sklearn/sklearn.base/_BaseEstimator/__setstate__": {
+        "target": "sklearn/sklearn.base/_BaseEstimator/__setstate__"
+    },
+    "sklearn/sklearn.base/is_regressor": {
+        "target": "sklearn/sklearn.base/is_regressor"
+    },
+    "sklearn/sklearn.cluster._agglomerative/linkage_tree": {
+        "target": "sklearn/sklearn.cluster._agglomerative/linkage_tree"
+    },
+    "sklearn/sklearn.cluster._kmeans/MiniBatchKMeans/init_size_": {
+        "target": "sklearn/sklearn.cluster._kmeans/MiniBatchKMeans/init_size_"
+    },
+}
+
+
+def test_format_underscores():
+    assert (
+        format_name("sklearn.cluster._kmeans._MiniBatchKMeans.random_state_")
+        == "sklearn/sklearn.cluster._kmeans/_MiniBatchKMeans/random_state_"
+    )
+
+
+def test_format_uppercase():
+    assert (
+        format_name("sklearn.cluster._kmeans.MiniBatchKMeans.random_state_")
+        == "sklearn/sklearn.cluster._kmeans/MiniBatchKMeans/random_state_"
+    )
+
+
+def test_format_normal():
+    assert (
+        format_name("sklearn.cluster._mean_shift.get_bin_seeds")
+        == "sklearn/sklearn.cluster._mean_shift/get_bin_seeds"
+    )
+
+
+def test_format_one_part():
+    assert format_name("test") == "sklearn/test"
+
+
+def test_format_none():
+    assert format_name(None) is None
+
+
+def test_format_empty():
+    assert format_name("") == "sklearn/"
+
+
+def test_generate():
+    assert (
+        generate_unused_annotations(
+            "tests/commands/generate_annotations/unused_functions_list.json"
+        )
+        == EXPECTED_VALUE
+    )
+
+
+def test_generate_bad_path():
+    with pytest.raises(FileNotFoundError):
+        generate_unused_annotations("aaaaaaaaaaaAAAAAAAAAAAA")

package-parser/tests/commands/generate_annotations/unused_functions_list.json

@@ -0,0 +1,6 @@
+[
+    "sklearn.base._BaseEstimator.__setstate__",
+    "sklearn.base.is_regressor",
+    "sklearn.cluster._agglomerative.linkage_tree",
+    "sklearn.cluster._kmeans.MiniBatchKMeans.init_size_"
+]

package-parser/tests/data/constant/usage_data.json

@@ -25,8 +25,8 @@
             },
             {
                 "file": "test.py",
-                "line": 1,
-                "column": 1
+                "line": 2,
+                "column": 2
             }
         ]
     },