Global search fulltext (#767)

* added global search+vector+fulltext mode

* added community details in chunk entities

* added node ids

* updated vector graph query

* added entities and modified chat response

* added params

* api response changes

* added chunk entity query

* modifies query

* payload changes

* added nodetails properties

* payload new changes

* communities check

* communities selecetion check

* Communities bug solutions (#770)

* added local chat history

* added write access check

* added write access param

* labels cahnge for nodes

* added fulltext creation

* disabled the write and delete actions for read only user mode

* modified query

* test updates

* test uupdated

* enable communities

* removed the selected prop

* Read Only User Support (#766)

* added local chat history

* added write access check

* added write access param

* added fulltext creation

* disabled the write and delete actions for read only user mode

* modified query

---------

Co-authored-by: vasanthasaikalluri <[email protected]>

* storing the gds status and write access on refresh

* enable communities label change

---------

Co-authored-by: vasanthasaikalluri <[email protected]>
Co-authored-by: kartikpersistent <[email protected]>
Co-authored-by: abhishekkumar-27 <[email protected]>

* readonly fixed on refresh

* clear chat history

* slectedFiles check for Chatbot

* clear history

---------

Co-authored-by: vasanthasaikalluri <[email protected]>
Co-authored-by: kartikpersistent <[email protected]>
Co-authored-by: abhishekkumar-27 <[email protected]>

Author: 4 people

backend/score.py

@@ -272,8 +272,8 @@ async def post_processing(uri=Form(), userName=Form(), password=Form(), database
             json_obj = {'api_name': 'post_processing/create_entity_embedding', 'db_url': uri, 'logging_time': formatted_time(datetime.now(timezone.utc))}
             logger.log_struct(json_obj)
             logging.info(f'Entity Embeddings created')
-
-        if "create_communities" in tasks:
+            
+        if "enable_communities" in tasks:
             model = "openai-gpt-4o"
             await asyncio.to_thread(create_communities, uri, userName, password, database,model)
             josn_obj = {'api_name': 'post_processing/create_communities', 'db_url': uri, 'logging_time': formatted_time(datetime.now(timezone.utc))}
@@ -321,10 +321,9 @@ async def chat_bot(uri=Form(),model=Form(None),userName=Form(), password=Form(),
         gc.collect()
 
 @app.post("/chunk_entities")
-async def chunk_entities(uri=Form(),userName=Form(), password=Form(), database=Form(), chunk_ids=Form(None),is_entity=Form()):
+async def chunk_entities(uri=Form(),userName=Form(), password=Form(), database=Form(), nodedetails=Form(None),entities=Form(),mode=Form()):
     try:
-        logging.info(f"URI: {uri}, Username: {userName}, chunk_ids: {chunk_ids}")
-        result = await asyncio.to_thread(get_entities_from_chunkids,uri=uri, username=userName, password=password, database=database,chunk_ids=chunk_ids,is_entity=json.loads(is_entity.lower()))
+        result = await asyncio.to_thread(get_entities_from_chunkids,uri=uri, username=userName, password=password, database=database,nodedetails=nodedetails,entities=entities,mode=mode)
         json_obj = {'api_name':'chunk_entities','db_url':uri, 'logging_time': formatted_time(datetime.now(timezone.utc))}
         logger.log_struct(json_obj)
         return create_api_response('Success',data=result)

backend/src/chunkid_entities.py

@@ -81,16 +81,16 @@ def process_chunk_data(chunk_data):
     except Exception as e:
         logging.error(f"chunkid_entities module: An error occurred while extracting the Chunk text from records: {e}")
 
-def process_chunkids(driver, chunk_ids):
+def process_chunkids(driver, chunk_ids, entities):
     """
     Processes chunk IDs to retrieve chunk data.
     """
     try:
         logging.info(f"Starting graph query process for chunk ids: {chunk_ids}")
-        chunk_ids_list = chunk_ids.split(",")
-
-        records, summary, keys = driver.execute_query(CHUNK_QUERY, chunksIds=chunk_ids_list)
+        records, summary, keys = driver.execute_query(CHUNK_QUERY, chunksIds=chunk_ids,entityIds=entities["entityids"], relationshipIds=entities["relationshipids"])
         result = process_records(records)
+        result["nodes"].extend(records[0]["nodes"])
+        result["nodes"] = remove_duplicate_nodes(result["nodes"])
         logging.info(f"Nodes and relationships are processed")
 
         result["chunk_data"] = process_chunk_data(records)
@@ -118,79 +118,95 @@ def remove_duplicate_nodes(nodes,property="element_id"):
 
     return unique_nodes
 
-def process_entityids(driver, chunk_ids):
+def process_entityids(driver, entity_ids):
     """
     Processes entity IDs to retrieve local community data.
     """
     try:
-        logging.info(f"Starting graph query process for entity ids: {chunk_ids}")
-        entity_ids_list = chunk_ids.split(",")
+        logging.info(f"Starting graph query process for entity ids: {entity_ids}")
         query_body = LOCAL_COMMUNITY_SEARCH_QUERY.format(
             topChunks=LOCAL_COMMUNITY_TOP_CHUNKS,
             topCommunities=LOCAL_COMMUNITY_TOP_COMMUNITIES,
             topOutsideRels=LOCAL_COMMUNITY_TOP_OUTSIDE_RELS
         )
         query = LOCAL_COMMUNITY_DETAILS_QUERY_PREFIX + query_body + LOCAL_COMMUNITY_DETAILS_QUERY_SUFFIX
 
-        records, summary, keys = driver.execute_query(query, entityIds=entity_ids_list)
+        records, summary, keys = driver.execute_query(query, entityIds=entity_ids)
 
         result = process_records(records)
         if records:
             result["nodes"].extend(records[0]["nodes"])
             result["nodes"] = remove_duplicate_nodes(result["nodes"])
+
             logging.info(f"Nodes and relationships are processed")
+
             result["chunk_data"] = records[0]["chunks"]
             result["community_data"] = records[0]["communities"]
         else:
             result["chunk_data"] = list()
             result["community_data"] = list()
-        logging.info(f"Query process completed successfully for chunk ids: {chunk_ids}")
+        logging.info(f"Query process completed successfully for chunk ids: {entity_ids}")
         return result
     except Exception as e:
-        logging.error(f"chunkid_entities module: Error processing entity ids: {chunk_ids}. Error: {e}")
+        logging.error(f"chunkid_entities module: Error processing entity ids: {entity_ids}. Error: {e}")
         raise  
 
-def get_entities_from_chunkids(uri, username, password, database ,chunk_ids,is_entity=False):
-    """
-    Retrieve and process nodes and relationships from a graph database given a list of chunk IDs.
+def process_communityids(driver, community_ids):
+    """Processes community IDs to retrieve community data."""
+    try:
+        logging.info(f"Starting graph query process for community ids: {community_ids}")
+        query = GLOBAL_COMMUNITY_DETAILS_QUERY
+        records, summary, keys = driver.execute_query(query, communityids=community_ids)
+
+        result = {"nodes": [], "relationships": [], "chunk_data": []}
+        result["community_data"] = records[0]["communities"] if records else []
 
-    Parameters:
-    uri (str): The URI of the graph database.
-    username (str): The username for the database authentication.
-    password (str): The password for the database authentication.
-    chunk_ids (str): A comma-separated string of chunk IDs.
+        logging.info(f"Query process completed successfully for community ids: {community_ids}")
+        return result
+    except Exception as e:
+        logging.error(f"chunkid_entities module: Error processing community ids: {community_ids}. Error: {e}")
+        raise 
 
-    Returns:
-    dict: A dictionary with 'nodes' and 'relationships' keys containing processed data, or an error message.
-    """    
+def get_entities_from_chunkids(uri, username, password, database ,nodedetails,entities,mode):   
     try:
 
         driver = get_graphDB_driver(uri, username, password,database)
-        if not is_entity:
-            if chunk_ids:
-                logging.info(f"chunkid_entities module: Starting for chunk ids : {chunk_ids}")
-                result = process_chunkids(driver,chunk_ids)
+        default_response = {"nodes": list(),"relationships": list(),"chunk_data": list(),"community_data": list(),}
+
+        nodedetails = json.loads(nodedetails)
+        entities = json.loads(entities)
+
+        if mode == CHAT_GLOBAL_VECTOR_FULLTEXT_MODE:
+
+            if "communitydetails" in nodedetails and nodedetails["communitydetails"]:
+                community_ids = [item["id"] for item in nodedetails["communitydetails"]]
+                logging.info(f"chunkid_entities module: Starting for community ids: {community_ids}")
+                return process_communityids(driver, community_ids)
+            else:
+                logging.info("chunkid_entities module: No community ids are passed")
+                return default_response
+            
+        elif mode == CHAT_ENTITY_VECTOR_MODE:
+
+            if "entitydetails" in nodedetails and nodedetails["entitydetails"]:
+                entity_ids = [item["id"] for item in nodedetails["entitydetails"]]
+                logging.info(f"chunkid_entities module: Starting for entity ids: {entity_ids}")
+                return process_entityids(driver, entity_ids)
             else:
-                logging.info(f"chunkid_entities module: No chunk ids are passed")
-                result = {
-                    "nodes": [],
-                    "relationships": [],
-                    "chunk_data":[]
-                }
-            return result
-        if chunk_ids:
-            result = process_entityids(driver,chunk_ids)
-            logging.info(f"chunkid_entities module: Starting for entity ids : {chunk_ids}")
+                logging.info("chunkid_entities module: No entity ids are passed")
+                return default_response  
+            
         else:
-            logging.info(f"chunkid_entities module: No entity ids are passed")
-            result = {
-                "nodes": [],
-                "relationships": [],
-                "chunk_data":[],
-                "community_data":[]
-            }
-        return result
+
+            if "chunkdetails" in nodedetails and nodedetails["chunkdetails"]:
+                chunk_ids = [item["id"] for item in nodedetails["chunkdetails"]]
+                logging.info(f"chunkid_entities module: Starting for chunk ids: {chunk_ids}")
+                return process_chunkids(driver, chunk_ids, entities)
+            else:
+                logging.info("chunkid_entities module: No chunk ids are passed")
+                return default_response
 
     except Exception as e:
         logging.error(f"chunkid_entities module: An error occurred in get_entities_from_chunkids. Error: {str(e)}")
-        raise Exception(f"chunkid_entities module: An error occurred in get_entities_from_chunkids. Please check the logs for more details.") from e
+        raise Exception(f"chunkid_entities module: An error occurred in get_entities_from_chunkids. Please check the logs for more details.") from e 
+