neo4j-labs
diff --git a/‎backend/example.env
+1-1 b/‎backend/example.env
+1-1
diff --git a/‎backend/score.py
+4-13 b/‎backend/score.py
+4-13
diff --git a/‎backend/src/graphDB_dataAccess.py
+12-2 b/‎backend/src/graphDB_dataAccess.py
+12-2
diff --git a/‎backend/src/graph_query.py
+3-1 b/‎backend/src/graph_query.py
+3-1
diff --git a/‎backend/src/llm.py
+2-2 b/‎backend/src/llm.py
+2-2
diff --git a/‎backend/src/make_relationships.py
+1-30 b/‎backend/src/make_relationships.py
+1-30
diff --git a/‎backend/test_integrationqa.py
+63-56 b/‎backend/test_integrationqa.py
+63-56
@@ -23,7 +23,7 @@ GCS_FILE_CACHE = "" #save the file into GCS or local, SHould be True or False
 NEO4J_USER_AGENT = ""
 ENABLE_USER_AGENT = ""
 LLM_MODEL_CONFIG_model_version=""
-ENTITY_EMBEDDING="TRUE"   # TRUE or FALSE based on whether to create embeddings for entities suitable for entity vector mode
+ENTITY_EMBEDDING="" True or False
 DUPLICATE_SCORE_VALUE =0.97
 DUPLICATE_TEXT_DISTANCE =3
 DEFAULT_DIFFBOT_CHAT_MODEL="openai_gpt_4o"  #whichever model specified here , need to add config for that model in below format)
 
@@ -18,7 +18,7 @@
 from src.communities import create_communities
 from src.neighbours import get_neighbour_nodes
 import json
-from typing import List, Mapping, Union
+from typing import List
 from starlette.middleware.sessions import SessionMiddleware
 from google.oauth2.credentials import Credentials
 import os
@@ -30,8 +30,7 @@
 from Secweb.XFrameOptions import XFrame
 from fastapi.middleware.gzip import GZipMiddleware
 from src.ragas_eval import *
-from starlette.types import ASGIApp, Message, Receive, Scope, Send
-import gzip
+from starlette.types import ASGIApp, Receive, Scope, Send
 from langchain_neo4j import Neo4jGraph
 from src.entities.source_node import sourceNode
 
@@ -598,8 +597,6 @@ async def generate():
                 # get the current status of document node
 
                 else:
-                    graph = create_graph_database_connection(uri, userName, decoded_password, database)
-                    graphDb_data_Access = graphDBdataAccess(graph)
                     result = graphDb_data_Access.get_current_status_document_node(file_name)
                     print(f'Result of document status in SSE : {result}')
                     if len(result) > 0:
@@ -904,10 +901,9 @@ async def fetch_chunktext(
        gc.collect()
 
 
-@app.post("/backend_connection_configuation")
-async def backend_connection_configuation():
+@app.post("/backend_connection_configuration")
+async def backend_connection_configuration():
     try:
-        start = time.time()
         uri = os.getenv('NEO4J_URI')
         username= os.getenv('NEO4J_USERNAME')
         database= os.getenv('NEO4J_DATABASE')
@@ -928,11 +924,6 @@ async def backend_connection_configuation():
                 result["database"] = database
                 result["password"] = encoded_password
                 result['gcs_file_cache'] = gcs_file_cache
-                end = time.time()
-                elapsed_time = end - start
-                result['api_name'] = 'backend_connection_configuration'
-                result['elapsed_api_time'] = f'{elapsed_time:.2f}'
-                logger.log_struct(result, "INFO")
                 return create_api_response('Success',message=f"Backend connection successful",data=result)
         else:
             graph_connection = False
 
@@ -46,14 +46,24 @@ def create_source_node(self, obj_source_node:sourceNode):
                             d.relationshipCount = $r_count, d.model= $model, d.gcsBucket=$gcs_bucket, 
                             d.gcsBucketFolder= $gcs_bucket_folder, d.language= $language,d.gcsProjectId= $gcs_project_id,
                             d.is_cancelled=False, d.total_chunks=0, d.processed_chunk=0,
-                            d.access_token=$access_token""",
+                            d.access_token=$access_token,
+                            d.chunkNodeCount=$chunkNodeCount,d.chunkRelCount=$chunkRelCount,
+                            d.entityNodeCount=$entityNodeCount,d.entityEntityRelCount=$entityEntityRelCount,
+                            d.communityNodeCount=$communityNodeCount,d.communityRelCount=$communityRelCount""",
                             {"fn":obj_source_node.file_name, "fs":obj_source_node.file_size, "ft":obj_source_node.file_type, "st":job_status, 
                             "url":obj_source_node.url,
                             "awsacc_key_id":obj_source_node.awsAccessKeyId, "f_source":obj_source_node.file_source, "c_at":obj_source_node.created_at,
                             "u_at":obj_source_node.created_at, "pt":0, "e_message":'', "n_count":0, "r_count":0, "model":obj_source_node.model,
                             "gcs_bucket": obj_source_node.gcsBucket, "gcs_bucket_folder": obj_source_node.gcsBucketFolder, 
                             "language":obj_source_node.language, "gcs_project_id":obj_source_node.gcsProjectId,
-                            "access_token":obj_source_node.access_token})
+                            "access_token":obj_source_node.access_token,
+                            "chunkNodeCount":obj_source_node.chunkNodeCount,
+                            "chunkRelCount":obj_source_node.chunkRelCount,
+                            "entityNodeCount":obj_source_node.entityNodeCount,
+                            "entityEntityRelCount":obj_source_node.entityEntityRelCount,
+                            "communityNodeCount":obj_source_node.communityNodeCount,
+                            "communityRelCount":obj_source_node.communityRelCount
+                            })
         except Exception as e:
             error_message = str(e)
             logging.info(f"error_message = {error_message}")
 
@@ -223,6 +223,7 @@ def get_graph_results(uri, username, password,database,document_names):
 
 def get_chunktext_results(uri, username, password, database, document_name, page_no):
    """Retrieves chunk text, position, and page number from graph data with pagination."""
+   driver = None
    try:
        logging.info("Starting chunk text query process")
        offset = 10
@@ -251,4 +252,5 @@ def get_chunktext_results(uri, username, password, database, document_name, page
        logging.error(f"An error occurred in get_chunktext_results. Error: {str(e)}")
        raise Exception("An error occurred in get_chunktext_results. Please check the logs for more details.") from e
    finally:
-       driver.close()
+       if driver:
+           driver.close()
@@ -191,7 +191,7 @@ async def get_graph_document_list(
     return graph_document_list
 
 
-async def get_graph_from_llm(model, chunkId_chunkDoc_list, allowedNodes, allowedRelationship, additional_instructions=None):
+async def get_graph_from_llm(model, chunkId_chunkDoc_list, allowedNodes, allowedRelationship):
     try:
         llm, model_name = get_llm(model)
         combined_chunk_document_list = get_combined_chunks(chunkId_chunkDoc_list)
@@ -206,7 +206,7 @@ async def get_graph_from_llm(model, chunkId_chunkDoc_list, allowedNodes, allowed
             allowedRelationship = allowedRelationship.split(',')
 
         graph_document_list = await get_graph_document_list(
-            llm, combined_chunk_document_list, allowedNodes, allowedRelationship, additional_instructions
+            llm, combined_chunk_document_list, allowedNodes, allowedRelationship
         )
         return graph_document_list
     except Exception as e:
 
@@ -41,7 +41,7 @@ def merge_relationship_between_chunk_and_entites(graph: Neo4jGraph, graph_docume
 
 
 def create_chunk_embeddings(graph, chunkId_chunkDoc_list, file_name):
-    
+    #create embedding
     isEmbedding = os.getenv('IS_EMBEDDING')
 
     embeddings, dimension = EMBEDDING_FUNCTION , EMBEDDING_DIMENSION
@@ -54,35 +54,6 @@ def create_chunk_embeddings(graph, chunkId_chunkDoc_list, file_name):
                 "chunkId": row['chunk_id'],
                 "embeddings": embeddings_arr
             })
-            # graph.query("""MATCH (d:Document {fileName : $fileName})
-            #                MERGE (c:Chunk {id:$chunkId}) SET c.embedding = $embeddings 
-            #                MERGE (c)-[:PART_OF]->(d)
-            #             """,
-            #             {
-            #                 "fileName" : file_name,
-            #                 "chunkId": row['chunk_id'],
-            #                 "embeddings" : embeddings_arr
-            #             }
-            #             )
-            # logging.info('create vector index on chunk embedding')
-            result = graph.query("SHOW INDEXES YIELD * WHERE labelsOrTypes = ['__Chunk__'] and name = 'vector'")
-            vector_index = graph.query("SHOW INDEXES YIELD * WHERE labelsOrTypes = ['Chunk'] and type = 'VECTOR' AND name = 'vector' return options")
-            if result:
-                logging.info(f"vector index dropped for 'Chunk'")
-                graph.query("DROP INDEX vector IF EXISTS;")
-
-            if len(vector_index) == 0:
-                logging.info(f'vector index is not exist, will create in next query')
-                graph.query("""CREATE VECTOR INDEX `vector` if not exists for (c:Chunk) on (c.embedding)
-                                OPTIONS {indexConfig: {
-                                `vector.dimensions`: $dimensions,
-                                `vector.similarity_function`: 'cosine'
-                                }}
-                            """,
-                            {
-                                "dimensions" : dimension
-                            }
-                            )
 
     query_to_create_embedding = """
         UNWIND $data AS row
 
@@ -98,41 +98,44 @@ def test_graph_from_wikipedia(model_name):
        file_name = "Apollo_program"
        create_source_node_graph_url_wikipedia(graph, model_name, wiki_query, source_type)
 
-    wiki_result = extract_graph_from_file_Wikipedia(URI, USERNAME, PASSWORD, DATABASE, model_name, file_name, 1, 'en', '', '')
-    logging.info("Wikipedia test done")
-    print(wiki_result)
-
-    try:
-        assert weburl_result['status'] == 'Completed'
-        assert weburl_result['nodeCount'] > 0
-        assert weburl_result['relationshipCount'] > 0
-        print("Success")
-    except AssertionError as e:
-        print("Fail: ", e)
-    return weburl_result
-
+       wiki_result = asyncio.run(extract_graph_from_file_Wikipedia(URI, USERNAME, PASSWORD, DATABASE, model_name, wiki_query, 'en',file_name, '', '',None))
+       logging.info("Wikipedia test done")
+       print(wiki_result)
+    #    try:
+    #        assert wiki_result['status'] == 'Completed'
+    #        assert wiki_result['nodeCount'] > 0
+    #        assert wiki_result['relationshipCount'] > 0
+    #        print("Success")
+    #    except AssertionError as e:
+    #        print("Fail: ", e)
+  
+       return wiki_result
+    except Exception as ex:
+        print('Hello error herte')
+        print(ex)
 
 def test_graph_website(model_name):
     """Test graph creation from a Website page."""
      #graph, model, source_url, source_type
-    source_url = 'https://www.amazon.com/'
+    source_url = 'https://www.cloudskillsboost.google/'
     source_type = 'web-url'
+    file_name = 'Google Cloud Skills Boost'
+    # file_name = []
     create_source_node_graph_web_url(graph, model_name, source_url, source_type)
 
-    weburl_result = extract_graph_from_web_page(URI, USERNAME, PASSWORD, DATABASE, model_name, source_url, '', '')
+    weburl_result = asyncio.run(extract_graph_from_web_page(URI, USERNAME, PASSWORD, DATABASE, model_name, source_url,file_name, '', '',None))
     logging.info("WebUrl test done")
     print(weburl_result)
 
-    try:
-        assert weburl_result['status'] == 'Completed'
-        assert weburl_result['nodeCount'] > 0
-        assert weburl_result['relationshipCount'] > 0
-        print("Success")
-    except AssertionError as e:
-        print("Fail: ", e)
+    # try:
+    #     assert weburl_result['status'] == 'Completed'
+    #     assert weburl_result['nodeCount'] > 0
+    #     assert weburl_result['relationshipCount'] > 0
+    #     print("Success")
+    # except AssertionError as e:
+    #     print("Fail: ", e)
     return weburl_result
 
-
 def test_graph_from_youtube_video(model_name):
    """Test graph creation from a YouTube video."""
    source_url = 'https://www.youtube.com/watch?v=T-qy-zPWgqA'
@@ -290,39 +293,43 @@ def test_populate_graph_schema_from_text(model):
 #             print(f"Result {i} differs from result {i+1}")
 
 def run_tests():
-    final_list = []
-    error_list = []
-    models = ['openai-gpt-3.5', 'openai-gpt-4o']
-
-    for model_name in models:
-        try:
-              final_list.append(test_graph_from_file_local(model_name))
-              final_list.append(test_graph_from_wikipedia(model_name))
-              final_list.append(test_populate_graph_schema_from_text(model_name))
-              final_list.append(test_graph_website(model_name))
-              final_list.append(test_graph_from_youtube_video(model_name))
-              final_list.append(test_chatbot_qna(model_name))
-              final_list.append(test_chatbot_qna(model_name, mode='vector'))
-              final_list.append(test_chatbot_qna(model_name, mode='graph+vector+fulltext'))
-        except Exception as e:
-            error_list.append((model_name, str(e)))
-    # #Compare and log diffrences in graph results
-    # # compare_graph_results(final_list)  # Pass the final_list to comapre_graph_results
-    # test_populate_graph_schema_from_text('openai-gpt-4o')
-    dis_elementid, dis_status = disconected_nodes()
-    lst_element_id = [dis_elementid]
-    delt = delete_disconected_nodes(lst_element_id)
-    dup = get_duplicate_nodes()
-    # schma = test_populate_graph_schema_from_text(model)
-    # Save final results to CSV
-    df = pd.DataFrame(final_list)
-    print(df)
-    df['execution_date'] = dt.today().strftime('%Y-%m-%d')
-    df['disconnected_nodes']=dis_status
-    df['get_duplicate_nodes']=dup
-    df['delete_disconected_nodes']=delt
-    # df['test_populate_graph_schema_from_text'] = schma
-    df.to_csv(f"Integration_TestResult_{dt.now().strftime('%Y%m%d_%H%M%S')}.csv", index=False)
+   final_list = []
+   error_list = []
+   
+   models = ['openai_gpt_4','openai_gpt_4o','openai_gpt_4o_mini','gemini_1.5_pro','gemini_1.5_flash']
+
+   for model_name in models:
+       try:
+                final_list.append(test_graph_from_file_local(model_name))
+                final_list.append(test_graph_from_wikipedia(model_name))
+                final_list.append(test_graph_website(model_name))
+                final_list.append(test_populate_graph_schema_from_text(model_name))
+                final_list.append(test_graph_from_youtube_video(model_name))
+                final_list.append(test_chatbot_qna(model_name))
+                final_list.append(test_chatbot_qna(model_name, mode='vector'))
+                final_list.append(test_chatbot_qna(model_name, mode='graph+vector'))
+                final_list.append(test_chatbot_qna(model_name, mode='fulltext'))
+                final_list.append(test_chatbot_qna(model_name, mode='graph+vector+fulltext'))
+                final_list.append(test_chatbot_qna(model_name, mode='entity search+vector'))
+                
+       except Exception as e:
+           error_list.append((model_name, str(e)))
+
+#    test_populate_graph_schema_from_text('openai-gpt-4o')
+#delete diconnected nodes
+   dis_elementid, dis_status = disconected_nodes()
+   lst_element_id = [dis_elementid]
+   delt = delete_disconected_nodes(lst_element_id)
+   dup = get_duplicate_nodes()
+   print(final_list)
+   schma = test_populate_graph_schema_from_text(model_name)
+   # Save final results to CSV
+   df = pd.DataFrame(final_list)
+   print(df)
+   df['execution_date'] = dt.today().strftime('%Y-%m-%d')
+#diconnected nodes   
+   df['disconnected_nodes']=dis_status
+   df['get_duplicate_nodes']=dup
 
    df['delete_disconected_nodes']=delt
    df['test_populate_graph_schema_from_text'] = schma