Merge pull request #79 from FalkorDB/staging

Staging

Author: AviAvni

api/analyzers/analyzer.py

@@ -54,7 +54,7 @@ def resolve(self, files: dict[Path, File], lsp: SyncLanguageServer, file_path: P
             return []
 
     @abstractmethod
-    def add_dependencies(self, path: Path, files: dict[Path, File]):
+    def add_dependencies(self, path: Path, files: list[Path]):
         """
         Add dependencies to the files.
 

api/analyzers/java/analyzer.py

@@ -19,7 +19,7 @@ class JavaAnalyzer(AbstractAnalyzer):
     def __init__(self) -> None:
         super().__init__(Language(tsjava.language()))
 
-    def add_dependencies(self, path: Path, files: dict[Path, File]):
+    def add_dependencies(self, path: Path, files: list[Path]):
         # if not Path("java-decompiler-engine-243.23654.153.jar").is_file():
         #     subprocess.run(["wget", "https://www.jetbrains.com/intellij-repository/releases/com/jetbrains/intellij/java/java-decompiler-engine/243.23654.153/java-decompiler-engine-243.23654.153.jar"])
         subprocess.run(["rm", "-rf", f"{path}/temp_deps"])
@@ -35,6 +35,7 @@ def add_dependencies(self, path: Path, files: dict[Path, File]):
             # subprocess.run(["java", "-jar", "java-decompiler-engine-243.23654.153.jar", "-hdc=0 -iib=1 -rsy=1 -rbr=1 -dgs=1 -din=1 -den=1 -asc=1 -bsm=1", jar_path, f"{path}/temp_deps/{artifactId}-{version}"])
             subprocess.run(["cp", jar_path, f"{artifactId}-{version}.jar"], cwd=f"{path}/temp_deps/{artifactId}-{version}")
             subprocess.run(["unzip", f"{artifactId}-{version}.jar"], cwd=f"{path}/temp_deps/{artifactId}-{version}")
+        files.extend(Path(f"{path}/temp_deps").rglob("*.java"))
 
     def get_entity_label(self, node: Node) -> str:
         if node.type == 'class_declaration':

api/analyzers/python/analyzer.py

@@ -1,6 +1,9 @@
+import os
 import subprocess
 from multilspy import SyncLanguageServer
 from pathlib import Path
+
+import toml
 from ...entities import *
 from typing import Optional
 from ..analyzer import AbstractAnalyzer
@@ -15,15 +18,23 @@ class PythonAnalyzer(AbstractAnalyzer):
     def __init__(self) -> None:
         super().__init__(Language(tspython.language()))
 
-    def add_dependencies(self, path: Path, files: dict[Path, File]):
+    def add_dependencies(self, path: Path, files: list[Path]):
         if Path(f"{path}/venv").is_dir():
             return
-        subprocess.run(["python3", "-m", "venv", f"{path}/venv"])
-        if Path(f"{path}/requirements.txt").is_file():
-            subprocess.run([f"{path}/venv/bin/pip", "install", "-r", "requirements.txt"])
+        subprocess.run(["python3", "-m", "venv", "venv"], cwd=str(path))
         if Path(f"{path}/pyproject.toml").is_file():
-            subprocess.run([f"{path}/venv/bin/pip", "install", "poetry"])
-            subprocess.run([f"{path}/venv/bin/poetry", "install"])
+            subprocess.run(["pip", "install", "poetry"], cwd=str(path), env={"VIRTUAL_ENV": f"{path}/venv", "PATH": f"{path}/venv/bin:{os.environ['PATH']}"})
+            subprocess.run(["poetry", "install"], cwd=str(path), env={"VIRTUAL_ENV": f"{path}/venv", "PATH": f"{path}/venv/bin:{os.environ['PATH']}"})
+            with open(f"{path}/pyproject.toml", 'r') as file:
+                pyproject_data = toml.load(file)
+                for requirement in pyproject_data.get("tool").get("poetry").get("dependencies"):
+                    files.extend(Path(f"{path}/venv/lib").rglob(f"**/site-packages/{requirement}/*.py"))
+        elif Path(f"{path}/requirements.txt").is_file():
+            subprocess.run(["pip", "install", "-r", "requirements.txt"], cwd=str(path), env={"VIRTUAL_ENV": f"{path}/venv", "PATH": f"{path}/venv/bin:{os.environ['PATH']}"})
+            with open(f"{path}/requirements.txt", 'r') as file:
+                requirements = [line.strip().split("==") for line in file if line.strip()]
+                for requirement in requirements:
+                    files.extend(Path(f"{path}/venv/lib/").rglob(f"**/site-packages/{requirement}/*.py"))
 
     def get_entity_label(self, node: Node) -> str:
         if node.type == 'class_definition':

api/analyzers/source_analyzer.py

@@ -38,7 +38,7 @@ def supported_types(self) -> list[str]:
         """
         """
         return list(analyzers.keys())
-    
+
     def create_entity_hierarchy(self, entity: Entity, file: File, analyzer: AbstractAnalyzer, graph: Graph):
         types = analyzer.get_entity_types()
         stack = list(entity.node.children)
@@ -72,7 +72,7 @@ def create_hierarchy(self, file: File, analyzer: AbstractAnalyzer, graph: Graph)
             else:
                 stack.extend(node.children)
 
-    def first_pass(self, path: Path, ignore: list[str], graph: Graph) -> None:
+    def first_pass(self, path: Path, files: list[Path], ignore: list[str], graph: Graph) -> None:
         """
         Perform the first pass analysis on source files in the given directory tree.
 
@@ -81,12 +81,10 @@ def first_pass(self, path: Path, ignore: list[str], graph: Graph) -> None:
             executor (concurrent.futures.Executor): The executor to run tasks concurrently.
         """
 
-        if any(path.rglob('*.java')):
-            analyzers[".java"].add_dependencies(path, self.files)
-        if any(path.rglob('*.py')):
-            analyzers[".py"].add_dependencies(path, self.files)
-
-        files = list(path.rglob('*.*'))
+        supoorted_types = self.supported_types()
+        for ext in set([file.suffix for file in files if file.suffix in supoorted_types]):
+            analyzers[ext].add_dependencies(path, files)
+        
         files_len = len(files)
         for i, file_path in enumerate(files):
             # Skip none supported files
@@ -115,7 +113,7 @@ def first_pass(self, path: Path, ignore: list[str], graph: Graph) -> None:
             graph.add_file(file)
             self.create_hierarchy(file, analyzer, graph)
 
-    def second_pass(self, graph: Graph, path: Path) -> None:
+    def second_pass(self, graph: Graph, files: list[Path], path: Path) -> None:
         """
         Recursively analyze the contents of a directory.
 
@@ -140,7 +138,8 @@ def second_pass(self, graph: Graph, path: Path) -> None:
             lsps[".py"] = NullLanguageServer()
         with lsps[".java"].start_server(), lsps[".py"].start_server():
             files_len = len(self.files)
-            for i, (file_path, file) in enumerate(self.files.items()):
+            for i, file_path in enumerate(files):
+                file = self.files[file_path]
                 logging.info(f'Processing file ({i + 1}/{files_len}): {file_path}')
                 for _, entity in file.entities.items():
                     entity.resolved_symbol(lambda key, symbol: analyzers[file_path.suffix].resolve_symbol(self.files, lsps[file_path.suffix], file_path, path, key, symbol))
@@ -159,22 +158,17 @@ def second_pass(self, graph: Graph, path: Path) -> None:
                             elif key == "parameters":
                                 graph.connect_entities("PARAMETERS", entity.id, symbol.id)
 
-    def analyze_file(self, file_path: Path, path: Path, graph: Graph) -> None:
-        ext = file_path.suffix
-        logging.info(f"analyze_file: path: {file_path}")
-        logging.info(f"analyze_file: ext: {ext}")
-        if ext not in analyzers:
-            return
-
-        self.first_pass(file_path, [], graph)
-        self.second_pass(graph, path)
+    def analyze_files(self, files: list[Path], path: Path, graph: Graph) -> None:
+        self.first_pass(path, files, [], graph)
+        self.second_pass(graph, files, path)
 
     def analyze_sources(self, path: Path, ignore: list[str], graph: Graph) -> None:
+        files = list(path.rglob("*.java")) + list(path.rglob("*.py"))
         # First pass analysis of the source code
-        self.first_pass(path, ignore, graph)
+        self.first_pass(path, files, ignore, graph)
 
         # Second pass analysis of the source code
-        self.second_pass(graph, path)
+        self.second_pass(graph, files, path)
 
     def analyze_local_folder(self, path: str, g: Graph, ignore: Optional[list[str]] = []) -> None:
         """
@@ -203,14 +197,14 @@ def analyze_local_repository(self, path: str, ignore: Optional[list[str]] = None
             path (str): Path to a local git repository
             ignore (List(str)): List of paths to skip
         """
-        from git import Repo
+        from pygit2.repository import Repository
 
         self.analyze_local_folder(path, ignore)
 
         # Save processed commit hash to the DB
-        repo = Repo(path)
+        repo = Repository(path)
         head = repo.commit("HEAD")
-        self.graph.set_graph_commit(head.hexsha)
+        self.graph.set_graph_commit(head.short_id)
 
         return self.graph
 

api/auto_complete.py

@@ -1,6 +1,6 @@
 from .graph import Graph
 
 def prefix_search(repo: str, prefix: str) -> str:
+    """ Returns a list of all entities in the repository that start with the given prefix. """
     g = Graph(repo)
     return g.prefix_search(prefix)
-

api/git_utils/git_graph.py

@@ -46,9 +46,9 @@ def add_commit(self, commit: Commit) -> None:
         """
             Add a new commit to the graph
         """
-        date    = commit.committed_date
+        date    = commit.commit_time
         author  = commit.author.name
-        hexsha  = commit.hexsha
+        hexsha  = commit.short_id
         message = commit.message
         logging.info(f"Adding commit {hexsha}: {message}")