enable second pass, introduce git graph

Author: swilly22

code_graph/analyzers/source_analyzer.py

@@ -73,8 +73,7 @@ def process_file(path: Path) -> None:
         # Wait for all tasks to complete
         #concurrent.futures.wait(tasks)
 
-    def second_pass(self, base: str, root: str,
-                    executor: concurrent.futures.Executor) -> None:
+    def second_pass(self, ignore: List[str], executor: concurrent.futures.Executor) -> None:
         """
         Recursively analyze the contents of a directory.
 
@@ -85,7 +84,16 @@ def second_pass(self, base: str, root: str,
         """
 
         tasks = []
-        for dirpath, dirnames, filenames in os.walk(root):
+        for dirpath, dirnames, filenames in os.walk("."):
+
+            # skip current directory if it is within the ignore list
+            if dirpath in ignore:
+                # in-place clear dirnames to prevent os.walk from recursing into
+                # any of the nested directories
+                logger.info(f'ignoring directory: {dirpath}')
+                dirnames[:] = []
+                continue
+
             logger.info(f'Processing directory: {dirpath}')
 
             # Process each file in the current directory
@@ -101,9 +109,8 @@ def second_pass(self, base: str, root: str,
 
                 def process_file(path: Path) -> None:
                     with open(path, 'rb') as f:
-                        relative_path = str(path).replace(base, '')
                         ext = path.suffix
-                        analyzers[ext].second_pass(Path(relative_path), f, self.graph)
+                        analyzers[ext].second_pass(path, f, self.graph)
 
                 task = executor.submit(process_file, file_path)
                 tasks.append(task)
@@ -117,7 +124,7 @@ def analyze_sources(self, ignore: List[str]) -> None:
             self.first_pass(ignore, executor)
 
             # Second pass analysis of the source code
-            #self.second_pass(ignore, executor)
+            self.second_pass(ignore, executor)
 
     def analyze_github_repository(self, url: str) -> None:
         """

code_graph/git_utils/git_graph.py

@@ -0,0 +1,70 @@
+class GitGraph():
+    """
+    Represents a git commit graph
+    nodes are commits where one commit leads to its parents and children
+    edges contains queries and parameters for transitioning the code-graph
+    from the current commit to parent / child
+    """
+
+    def __init__(self, name: str, host: str = 'localhost', port: int = 6379,
+                 username: Optional[str] = None, password: Optional[str] = None):
+
+        self.db = FalkorDB(host=host, port=port, username=username,
+                           password=password)
+        self.g = self.db.select_graph(name)
+
+        # create indicies
+        # index commit hash
+        try:
+            self.g.create_node_range_index("Commit", "hash")
+        except Exception:
+            pass
+
+
+    def add_commit(self, commit_hash: str, author: str, message: str, date, int) -> None:
+        """
+            Add a new commit to the graph
+        """
+        q = "MERGE (c:Commit {hash: $hash, author: $author, message: $message, date: $date})"
+        params = {'hash': commit_hash, 'author': author, 'message': message, 'date': $date}
+        self.g.query(q, params)
+
+
+    def connect_commits(child: str, parent: str) -> None:
+        """
+            connect commits via both PARENT and CHILD edges
+        """
+
+        q = """MATCH (child :Commit {hash: $child_hash}), (parent :Commit {hash: $parent_hash})
+               MERGE (child)-[:PARENT]->(parent)
+               MERGE (parent)-[:CHILD]->(child)"""
+
+        params = {'child_hash': child, 'parent_hash': parent}
+
+        self.g.query(q, parent)
+
+    def set_parent_transition(child: str, parent: str, queries: [tuple[str: dict]]) -> None:
+        """
+            Sets the queries and parameters needed to transition the code-graph
+            from the child commit to the parent commit
+        """
+
+        q = """MATCH (child: Commit {hash: $child})-[e:PARENT]->(parent {hash: $parent})
+               SET e.queries = $queries, e.params = $params"""
+
+        params = {'child': child, 'parent': parent, 'queries': queries}
+
+        self.g.query(q, params)
+
+    def set_child_transition(child: str, parent: str, queries: [tuple[str: dict]]) -> None:
+        """
+            Sets the queries and parameters needed to transition the code-graph
+            from the parent commit to the child commit
+        """
+
+        q = """MATCH (parent {hash: $parent})-[e:CHILD]->(child: Commit {hash: $child})
+               SET e.queries = $queries, e.params = $params"""
+
+        params = {'child': child, 'parent': parent, 'queries': queries}
+
+        self.g.query(q, params)

code_graph/git_utils/git_utils.py

@@ -104,8 +104,7 @@ def build_commit_graph(path: str):
     repo_name = os.path.split(os.path.normpath(path))[-1]
     g = Graph(repo_name)
 
-    # TODO: need to wait for replication to sync with its master
-    setup_replication()
+    #setup_replication()
 
     # start monitoring graph effects
     # these capture the changes a graph goes through when moving from one
@@ -160,12 +159,15 @@ def build_commit_graph(path: str):
                      'ext' : os.path.splitext(path)[1]} for path in deleted]
 
             # remove deleted files from the graph
-            g.delete_files(deleted_files)
+            q, params = g.delete_files(deleted_files, True)
+            if(q is not None):
+                # log transition action
+
             input("Press Enter to continue...")
 
     # clean up
-    stop_monitor_effects()
-    teardown_replica()
+    #stop_monitor_effects()
+    #teardown_replica()
 
 if __name__ == "__main__":
     build_commit_graph("/Users/roilipman/Dev/FalkorDB")

code_graph/graph.py

@@ -240,7 +240,7 @@ def add_file(self, file: File) -> None:
         res = self.g.query(q, params)
         file.id = res.result_set[0][0]
 
-    def delete_files(self, files: List[dict]) -> None:
+    def delete_files(self, files: List[dict], log: bool = False) -> tuple[str, dict]:
         """
         Deletes file(s) from the graph in addition to any other entity
         defined in the file
@@ -260,7 +260,12 @@ def delete_files(self, files: List[dict]) -> None:
         """
 
         params = {'files': files}
-        self.g.query(q, params)
+        res = self.g.query(q, params)
+
+        if log and (res.relationships_deleted > 0 or res.nodes_deleted > 0):
+            return (q, params)
+
+        return None
 
     def get_file(self, path: str, name: str, ext: str) -> Optional[File]:
         """