User flow changes for recreating supported vector index (#682)

* removed the if check

* Add one more check for create vector index when chunks are exist without embeddings

* removed local files

* condition changes

* chunks exists check

* chunk exists without embeddings check

* vector Index issue fixed

* vector index with different dimension

* Update graphDB_dataAccess.py

---------

Co-authored-by: Pravesh Kumar <[email protected]>

Author: kartikpersistent

backend/score.py

@@ -611,11 +611,11 @@ async def merge_duplicate_nodes(uri=Form(), userName=Form(), password=Form(), da
         gc.collect()
 
 @app.post("/drop_create_vector_index")
-async def merge_duplicate_nodes(uri=Form(), userName=Form(), password=Form(), database=Form(), is_vector_index_recreate=Form()):
+async def merge_duplicate_nodes(uri=Form(), userName=Form(), password=Form(), database=Form(), isVectorIndexExist=Form()):
     try:
         graph = create_graph_database_connection(uri, userName, password, database)
         graphDb_data_Access = graphDBdataAccess(graph)
-        result = graphDb_data_Access.drop_create_vector_index(is_vector_index_recreate)
+        result = graphDb_data_Access.drop_create_vector_index(isVectorIndexExist)
         return create_api_response('Success',message=result)
     except Exception as e:
         job_status = "Failed"

backend/src/graphDB_dataAccess.py

@@ -158,23 +158,34 @@ def connection_check_and_get_vector_dimensions(self):
                                     WHERE type = 'VECTOR' AND name = 'vector'
                                     RETURN options.indexConfig['vector.dimensions'] AS vector_dimensions
                                 """)
+        
+        result_chunks = self.graph.query("""match (c:Chunk) return size(c.embedding) as embeddingSize, count(*) as chunks, 
+                                                    count(c.embedding) as hasEmbedding
+                                """)
+        
         embedding_model = os.getenv('EMBEDDING_MODEL')
         embeddings, application_dimension = load_embedding_model(embedding_model)
         logging.info(f'embedding model:{embeddings} and dimesion:{application_dimension}')
+        # print(chunks_exists)
 
         if self.graph:
             if len(db_vector_dimension) > 0:
                 return {'db_vector_dimension': db_vector_dimension[0]['vector_dimensions'], 'application_dimension':application_dimension, 'message':"Connection Successful"}
             else:
-                logging.info("Vector index does not exist in database")
-                return {'db_vector_dimension': 0, 'application_dimension':application_dimension, 'message':"Connection Successful"}
+                if len(db_vector_dimension) == 0 and len(result_chunks) == 0:
+                    logging.info("Chunks and vector index does not exists in database")
+                    return {'db_vector_dimension': 0, 'application_dimension':application_dimension, 'message':"Connection Successful","chunks_exists":False}
+                elif len(db_vector_dimension) == 0 and result_chunks[0]['hasEmbedding']==0 and result_chunks[0]['chunks'] > 0:
+                    return {'db_vector_dimension': 0, 'application_dimension':application_dimension, 'message':"Connection Successful","chunks_exists":True}
+                else:
+                    return {'message':"Connection Successful"}
 
     def execute_query(self, query, param=None):
         return self.graph.query(query, param)
 
     def get_current_status_document_node(self, file_name):
         query = """
-                MATCH(d:Document {fileName : $file_name}) RETURN d.status AS Status , d.processingTime AS processingTime, 
+                MATCH(d:Document {fileName : $file_name}) RETURN d.stats AS Status , d.processingTime AS processingTime, 
                 d.nodeCount AS nodeCount, d.model as model, d.relationshipCount as relationshipCount,
                 d.total_pages AS total_pages, d.total_chunks AS total_chunks , d.fileSize as fileSize, 
                 d.is_cancelled as is_cancelled, d.processed_chunk as processed_chunk, d.fileSource as fileSource
@@ -322,15 +333,16 @@ def merge_duplicate_nodes(self,duplicate_nodes_list):
         param = {"rows":nodes_list}
         return self.execute_query(query,param)
 
-    def drop_create_vector_index(self, is_vector_index_recreate):
+    def drop_create_vector_index(self, isVectorIndexExist):
         """
         drop and create the vector index when vector index dimesion are different.
         """
         embedding_model = os.getenv('EMBEDDING_MODEL')
         embeddings, dimension = load_embedding_model(embedding_model)
-        if is_vector_index_recreate == 'true':
-            self.graph.query("""drop index vector""")
 
+        if isVectorIndexExist == 'true':
+            self.graph.query("""drop index vector""")
+        # self.graph.query("""drop index vector""")
         self.graph.query("""CREATE VECTOR INDEX `vector` if not exists for (c:Chunk) on (c.embedding)
                             OPTIONS {indexConfig: {
                             `vector.dimensions`: $dimensions,
@@ -341,4 +353,4 @@ def drop_create_vector_index(self, is_vector_index_recreate):
                             "dimensions" : dimension
                         }
                         )
-        return "Drop and Re-Create vector index succesfully"
+        return "Drop and Re-Create vector index succesfully"

frontend/src/components/Content.tsx

@@ -48,9 +48,10 @@ const Content: React.FC<ContentProps> = ({
   const isTablet = useMediaQuery(`(min-width:${breakpoints.xs}) and (max-width: ${breakpoints.lg})`);
   const [init, setInit] = useState<boolean>(false);
   const [openConnection, setOpenConnection] = useState<connectionState>({
-    isvectorIndexMatch: true,
     openPopUp: false,
-    novectorindexInDB: true,
+    chunksExists: false,
+    vectorIndexMisMatch: false,
+    chunksExistsWithDifferentDimension: false,
   });
   const [openGraphView, setOpenGraphView] = useState<boolean>(false);
   const [inspectedName, setInspectedName] = useState<string>('');
@@ -594,14 +595,24 @@ const Content: React.FC<ContentProps> = ({
               userDbVectorIndex: response.data.data.db_vector_dimension,
             })
           );
-          if (response.data.data.application_dimension === response.data.data.db_vector_dimension) {
+          if (
+            (response.data.data.application_dimension === response.data.data.db_vector_dimension ||
+              response.data.data.db_vector_dimension == 0) &&
+            !response.data.data.chunks_exists
+          ) {
             setConnectionStatus(true);
             setOpenConnection((prev) => ({ ...prev, openPopUp: false }));
           } else {
             setOpenConnection({
-              isvectorIndexMatch: false,
               openPopUp: true,
-              novectorindexInDB: response.data.data.db_vector_dimension === 0,
+              chunksExists: response.data.data.chunks_exists as boolean,
+              vectorIndexMisMatch:
+                response.data.data.db_vector_dimension > 0 &&
+                response.data.data.db_vector_dimension != response.data.data.application_dimension,
+              chunksExistsWithDifferentDimension:
+                response.data.data.db_vector_dimension > 0 &&
+                response.data.data.db_vector_dimension != response.data.data.application_dimension &&
+                (response.data.data.chunks_exists ?? true),
             });
             setConnectionStatus(false);
           }
@@ -716,8 +727,9 @@ const Content: React.FC<ContentProps> = ({
               open={openConnection.openPopUp}
               setOpenConnection={setOpenConnection}
               setConnectionStatus={setConnectionStatus}
-              isVectorIndexMatch={openConnection.isvectorIndexMatch}
-              noVectorIndexFound={openConnection.novectorindexInDB}
+              isVectorIndexMatch={openConnection.vectorIndexMisMatch}
+              chunksExistsWithoutEmbedding={openConnection.chunksExists}
+              chunksExistsWithDifferentEmbedding={openConnection.chunksExistsWithDifferentDimension}
             />
           </Suspense>
 

frontend/src/components/Popups/ConnectionModal/ConnectionModal.tsx

@@ -13,7 +13,8 @@ export default function ConnectionModal({
   setOpenConnection,
   setConnectionStatus,
   isVectorIndexMatch,
-  noVectorIndexFound,
+  chunksExistsWithoutEmbedding,
+  chunksExistsWithDifferentEmbedding,
 }: ConnectionModalProps) {
   let prefilledconnection = localStorage.getItem('neo4j.connection');
   let initialuri;
@@ -98,20 +99,21 @@ export default function ConnectionModal({
     [userCredentials, userDbVectorIndex]
   );
   useEffect(() => {
-    if (!isVectorIndexMatch) {
+    if (isVectorIndexMatch || chunksExistsWithoutEmbedding) {
       setMessage({
         type: 'danger',
         content: (
           <VectorIndexMisMatchAlert
             vectorIndexLoading={vectorIndexLoading}
-            recreateVectorIndex={() => recreateVectorIndex(!noVectorIndexFound)}
-            isVectorIndexAlreadyExists={!noVectorIndexFound}
+            recreateVectorIndex={() => recreateVectorIndex(chunksExistsWithDifferentEmbedding)}
+            isVectorIndexAlreadyExists={chunksExistsWithDifferentEmbedding || isVectorIndexMatch}
             userVectorIndexDimension={JSON.parse(localStorage.getItem('neo4j.connection') ?? 'null').userDbVectorIndex}
+            chunksExists={chunksExistsWithoutEmbedding}
           />
         ),
       });
     }
-  }, [isVectorIndexMatch, vectorIndexLoading, noVectorIndexFound]);
+  }, [isVectorIndexMatch, vectorIndexLoading, chunksExistsWithDifferentEmbedding, chunksExistsWithoutEmbedding]);
 
   const parseAndSetURI = (uri: string, urlparams = false) => {
     const uriParts: string[] = uri.split('://');
@@ -189,46 +191,80 @@ export default function ConnectionModal({
     const connectionURI = `${protocol}://${URI}${URI.split(':')[1] ? '' : `:${port}`}`;
     setUserCredentials({ uri: connectionURI, userName: username, password: password, database: database, port: port });
     setIsLoading(true);
-    const response = await connectAPI(connectionURI, username, password, database);
-    if (response?.data?.status === 'Success') {
-      setUserDbVectorIndex(response.data.data.db_vector_dimension);
-      if (response.data.data.db_vector_dimension === response.data.data.application_dimension) {
-        setConnectionStatus(true);
-        setOpenConnection((prev) => ({ ...prev, openPopUp: false }));
-        setMessage({
-          type: 'success',
-          content: response.data.data.message,
-        });
+    try {
+      const response = await connectAPI(connectionURI, username, password, database);
+      setIsLoading(false);
+      if (response?.data?.status !== 'Success') {
+        throw new Error(response.data.error);
       } else {
-        setMessage({
-          type: 'danger',
-          content: (
-            <VectorIndexMisMatchAlert
-              vectorIndexLoading={vectorIndexLoading}
-              recreateVectorIndex={() => recreateVectorIndex(response.data.data.db_vector_dimension != 0)}
-              isVectorIndexAlreadyExists={response.data.data.db_vector_dimension != 0}
-              userVectorIndexDimension={response.data.data.db_vector_dimension}
-            />
-          ),
-        });
+        setUserDbVectorIndex(response.data.data.db_vector_dimension);
+        if (
+          (response.data.data.application_dimension === response.data.data.db_vector_dimension ||
+            response.data.data.db_vector_dimension == 0) &&
+          !response.data.data.chunks_exists
+        ) {
+          setConnectionStatus(true);
+          setOpenConnection((prev) => ({ ...prev, openPopUp: false }));
+          setMessage({
+            type: 'success',
+            content: response.data.data.message,
+          });
+        } else if ((response.data.data.chunks_exists ?? true) && response.data.data.db_vector_dimension == 0) {
+          setMessage({
+            type: 'danger',
+            content: (
+              <VectorIndexMisMatchAlert
+                vectorIndexLoading={vectorIndexLoading}
+                recreateVectorIndex={() =>
+                  recreateVectorIndex(
+                    !(
+                      response.data.data.db_vector_dimension > 0 &&
+                      response.data.data.db_vector_dimension != response.data.data.application_dimension
+                    )
+                  )
+                }
+                isVectorIndexAlreadyExists={response.data.data.db_vector_dimension != 0}
+                chunksExists={true}
+              />
+            ),
+          });
+        } else {
+          setMessage({
+            type: 'danger',
+            content: (
+              <VectorIndexMisMatchAlert
+                vectorIndexLoading={vectorIndexLoading}
+                recreateVectorIndex={() => recreateVectorIndex(true)}
+                isVectorIndexAlreadyExists={
+                  response.data.data.db_vector_dimension != 0 &&
+                  response.data.data.db_vector_dimension != response.data.data.application_dimension
+                }
+                chunksExists={true}
+                userVectorIndexDimension={response.data.data.db_vector_dimension}
+              />
+            ),
+          });
+        }
+        localStorage.setItem(
+          'neo4j.connection',
+          JSON.stringify({
+            uri: connectionURI,
+            user: username,
+            password: password,
+            database: database,
+            userDbVectorIndex,
+          })
+        );
+      }
+    } catch (error) {
+      setIsLoading(false);
+      if (error instanceof Error) {
+        setMessage({ type: 'danger', content: error.message });
+        setOpenConnection((prev) => ({ ...prev, openPopUp: true }));
+        setPassword('');
+        setConnectionStatus(false);
       }
-      localStorage.setItem(
-        'neo4j.connection',
-        JSON.stringify({
-          uri: connectionURI,
-          user: username,
-          password: password,
-          database: database,
-          userDbVectorIndex,
-        })
-      );
-    } else {
-      setMessage({ type: 'danger', content: response.data.error });
-      setOpenConnection((prev) => ({ ...prev, openPopUp: true }));
-      setPassword('');
-      setConnectionStatus(false);
     }
-    setIsLoading(false);
     setTimeout(() => {
       setPassword('');
     }, 3000);

frontend/src/components/Popups/ConnectionModal/VectorIndexMisMatchAlert.tsx

@@ -8,11 +8,13 @@ export default function VectorIndexMisMatchAlert({
   recreateVectorIndex,
   isVectorIndexAlreadyExists,
   userVectorIndexDimension,
+  chunksExists,
 }: {
   vectorIndexLoading: boolean;
   recreateVectorIndex: () => Promise<void>;
   isVectorIndexAlreadyExists: boolean;
-  userVectorIndexDimension: number;
+  userVectorIndexDimension?: number;
+  chunksExists: boolean;
 }) {
   const { userCredentials } = useCredentials();
   return (
@@ -25,8 +27,9 @@ The existing Neo4j vector index dimension (${userVectorIndexDimension}) is incom
 To proceed, please choose one of the following options: 
 1.**Recreate Vector Index:** Click "Re-Create Vector Index" to generate a compatible vector index. 
 2.**Use a Different Instance:** Connect to a Neo4j instance with a compatible vector index configuration  `
-            : `**Vector index not found**.
-              To leverage AI-powered search, please create a vector index.This will enable efficient similarity search within your Neo4j database`}
+            : chunksExists
+            ? `A vector index is essential for performing efficient similarity searches within your data. Without it, some chunks of data will be invisible to queries based on meaning and context. Creating a vector index unlocks the full potential of your data by allowing you to find related information quickly and accurately.`
+            : ''}
         </Markdown>
       </Box>
       <Box className='n-size-full n-flex n-flex-col n-items-center n-justify-center'>

frontend/src/components/Popups/GraphEnhancementDialog/Deduplication/index.tsx

@@ -80,12 +80,12 @@ export default function DeduplicationTab() {
   const onRemove = (nodeid: string, similarNodeId: string) => {
     setDuplicateNodes((prev) => {
       return prev.map((d) =>
-        d.e.elementId === nodeid
+        (d.e.elementId === nodeid
           ? {
               ...d,
               similar: d.similar.filter((n) => n.elementId != similarNodeId),
             }
-          : d
+          : d)
       );
     });
   };

frontend/src/services/vectorIndexCreation.ts

@@ -8,7 +8,7 @@ export const createVectorIndex = async (userCredentials: UserCredentials, isVect
   formData.append('database', userCredentials?.database ?? '');
   formData.append('userName', userCredentials?.userName ?? '');
   formData.append('password', userCredentials?.password ?? '');
-  formData.append('is_vector_index_recreate', JSON.stringify(isVectorIndexExists));
+  formData.append('isVectorIndexExist', JSON.stringify(isVectorIndexExists));
   try {
     const response = await axios.post<commonserverresponse>(`${url()}/drop_create_vector_index`, formData);
     return response;

frontend/src/types.ts

@@ -613,9 +613,10 @@ export interface ExtendedRelationship extends Relationship {
   labels: string[];
 }
 export interface connectionState {
-  isvectorIndexMatch: boolean;
   openPopUp: boolean;
-  novectorindexInDB: boolean;
+  chunksExists: boolean;
+  vectorIndexMisMatch: boolean;
+  chunksExistsWithDifferentDimension: boolean;
 }
 export interface Message {
   type: 'success' | 'info' | 'warning' | 'danger' | 'unknown';
@@ -627,7 +628,8 @@ export interface ConnectionModalProps {
   setOpenConnection: Dispatch<SetStateAction<connectionState>>;
   setConnectionStatus: Dispatch<SetStateAction<boolean>>;
   isVectorIndexMatch: boolean;
-  noVectorIndexFound: boolean;
+  chunksExistsWithoutEmbedding: boolean;
+  chunksExistsWithDifferentEmbedding: boolean;
 }
 export interface ReusableDropdownProps extends DropdownProps {
   options: string[] | OptionType[];