Additional metrics using ground truth (#855)

* Updating ragas metrics

* added the service for additional metrics

* additional metrics api

* Adding Rouge to requirement

* changes done for additional metrics for gemini model

* Additional metrics changes related to gemini model

* Adding Rouge_Score Version

* Api Integration

* payload changes

* payload fix

* Fixing Eval Error

* Adding fact_score metric

* code refactoring

* table integration

* data binding

* Integrated  additional metrics on multimodes

* removed fact score

* Removing Fact Score

* fix: Multimode fix

* custommiddleware for gzip

* removed unused state

* message changes

* uncommented gzipmiddleware

* code refactoring

* removed settings modal code

* Table UI Fixes

* removed state

* UX improvements for chunks popup

* added the status check

* ndl version changes

* tip and dropdown changes

* icon fixes

* contextmenu fix

* Box CSS fix

* icon fixes

* icon changes

* IsRoot fix

* added the tooltip for metrics

* Menu fix inside modal

* hover color fix

* menu changes

* format and lint fixes

---------

Co-authored-by: a-s-poorna <[email protected]>
Co-authored-by: kaustubh-darekar <[email protected]>

Author: 3 people

backend/requirements.txt

@@ -179,5 +179,5 @@ PyMuPDF==1.24.5
 pypandoc==1.13
 graphdatascience==1.10
 Secweb==1.11.0
-ragas==0.1.14
-
+ragas==0.2.2
+rouge_score==0.1.2

backend/score.py

@@ -85,7 +85,6 @@ async def __call__(self, scope: Scope, receive: Receive, send: Send):
 app.add_middleware(ContentSecurityPolicy, Option={'default-src': ["'self'"], 'base-uri': ["'self'"], 'block-all-mixed-content': []}, script_nonce=False, style_nonce=False, report_only=False)
 app.add_middleware(XContentTypeOptions)
 app.add_middleware(XFrame, Option={'X-Frame-Options': 'DENY'})
-#app.add_middleware(GZipMiddleware, minimum_size=1000, compresslevel=5)
 app.add_middleware(CustomGZipMiddleware, minimum_size=1000, compresslevel=5,paths=["/sources_list","/url/scan","/extract","/chat_bot","/chunk_entities","/get_neighbours","/graph_query","/schema","/populate_graph_schema","/get_unconnected_nodes_list","/get_duplicate_nodes","/fetch_chunktext"])
 app.add_middleware(
     CORSMiddleware,
@@ -847,6 +846,40 @@ async def calculate_metric(question: str = Form(),
        )
    finally:
        gc.collect()
+       
+
+@app.post('/additional_metrics')
+async def calculate_additional_metrics(question: str = Form(),
+                                        context: str = Form(),
+                                        answer: str = Form(),
+                                        reference: str = Form(),
+                                        model: str = Form(),
+                                        mode: str = Form(),
+):
+   try:
+       context_list = [str(item).strip() for item in json.loads(context)] if context else []
+       answer_list = [str(item).strip() for item in json.loads(answer)] if answer else []
+       mode_list = [str(item).strip() for item in json.loads(mode)] if mode else []
+       result = await get_additional_metrics(question, context_list,answer_list, reference, model)
+       if result is None or "error" in result:
+           return create_api_response(
+               'Failed',
+               message='Failed to calculate evaluation metrics.',
+               error=result.get("error", "Ragas evaluation returned null")
+           )
+       data = {mode: {metric: result[i][metric] for metric in result[i]} for i, mode in enumerate(mode_list)}
+       return create_api_response('Success', data=data)
+   except Exception as e:
+       logging.exception(f"Error while calculating evaluation metrics: {e}")
+       return create_api_response(
+           'Failed',
+           message="Error while calculating evaluation metrics",
+           error=str(e)
+       )
+   finally:
+       gc.collect()
+
+
 
 @app.post("/fetch_chunktext")
 async def fetch_chunktext(

backend/src/QA_integration.py

@@ -662,7 +662,7 @@ def QA_RAG(graph,model, question, document_names, session_id, mode, write_access
         if document_names and not chat_mode_settings["document_filter"]:
             result =  {
                 "session_id": "",  
-                "message": "This chat mode does support document selection",
+                "message": "Please deselect all documents in the table before using this chat mode",
                 "info": {
                     "sources": [],
                     "model": "",

backend/src/ragas_eval.py

@@ -7,6 +7,16 @@
 from ragas import evaluate
 from ragas.metrics import answer_relevancy, faithfulness
 from src.shared.common_fn import load_embedding_model 
+from ragas.dataset_schema import SingleTurnSample
+from ragas.metrics import BleuScore, RougeScore, SemanticSimilarity, ContextEntityRecall
+from ragas.metrics._factual_correctness import FactualCorrectness
+from ragas.llms import LangchainLLMWrapper
+from langchain_openai import ChatOpenAI
+from langchain.embeddings import OpenAIEmbeddings
+from ragas.embeddings import LangchainEmbeddingsWrapper
+import nltk
+
+nltk.download('punkt')
 load_dotenv()
 
 EMBEDDING_MODEL = os.getenv("RAGAS_EMBEDDING_MODEL")
@@ -52,3 +62,41 @@ def get_ragas_metrics(question: str, context: list, answer: list, model: str):
     except Exception as e:
        logging.exception(f"Error during metrics evaluation: {e}")
        return {"error": str(e)}
+
+
+async def get_additional_metrics(question: str, contexts: list, answers: list, reference: str, model_name: str):
+   """Calculates multiple metrics for given question, answers, contexts, and reference."""
+   try:
+       if ("diffbot" in model_name) or ("ollama" in model_name):
+           raise ValueError(f"Unsupported model for evaluation: {model_name}")
+       llm, model_name = get_llm(model=model_name)
+       ragas_llm = LangchainLLMWrapper(llm)
+       embeddings = EMBEDDING_FUNCTION
+       embedding_model = LangchainEmbeddingsWrapper(embeddings=embeddings)
+       rouge_scorer = RougeScore()
+       semantic_scorer = SemanticSimilarity()
+       entity_recall_scorer = ContextEntityRecall()
+       entity_recall_scorer.llm = ragas_llm
+       semantic_scorer.embeddings = embedding_model
+       metrics = []
+       for response, context in zip(answers, contexts):
+           sample = SingleTurnSample(response=response, reference=reference)
+           rouge_score = await rouge_scorer.single_turn_ascore(sample)
+           rouge_score = round(rouge_score,4)
+           semantic_score = await semantic_scorer.single_turn_ascore(sample)
+           semantic_score = round(semantic_score, 4)
+           if "gemini" in model_name:
+               entity_recall_score = "Not Available"
+           else:
+               entity_sample = SingleTurnSample(reference=reference, retrieved_contexts=[context])
+               entity_recall_score = await entity_recall_scorer.single_turn_ascore(entity_sample)
+               entity_recall_score = round(entity_recall_score, 4)
+           metrics.append({
+               "rouge_score": rouge_score,
+               "semantic_score": semantic_score,
+               "context_entity_recall_score": entity_recall_score
+           })
+       return metrics
+   except Exception as e:
+       logging.exception("Error in get_additional_metrics")
+       return {"error": str(e)}

frontend/package.json

@@ -15,11 +15,12 @@
     "@mui/material": "^5.15.10",
     "@mui/styled-engine": "^5.15.9",
     "@neo4j-devtools/word-color": "^0.0.8",
-    "@neo4j-ndl/base": "^2.12.7",
-    "@neo4j-ndl/react": "^2.16.9",
-    "@neo4j-nvl/base": "^0.3.3",
-    "@neo4j-nvl/react": "^0.3.3",
+    "@neo4j-ndl/base": "^3.0.10",
+    "@neo4j-ndl/react": "^3.0.17",
+    "@neo4j-nvl/base": "^0.3.6",
+    "@neo4j-nvl/react": "^0.3.6",
     "@react-oauth/google": "^0.12.1",
+    "@tanstack/react-table": "^8.20.5",
     "@types/uuid": "^9.0.7",
     "axios": "^1.6.5",
     "clsx": "^2.1.1",

frontend/src/App.css

@@ -25,7 +25,7 @@
 }
 
 .contentWithExpansion {
-  width: calc(-840px + 100dvw);
+  width: calc(-807px + 100dvw);
   height: calc(100dvh - 58px);
   padding: 3px;
   display: flex;
@@ -386,4 +386,13 @@
 .custom-menu {
   min-width: 250px;
   max-width: 305px;
+}
+.ndl-modal-root{
+  z-index: 39 !important;
+}
+.tbody-dark .ndl-data-grid-tr:hover {
+  --cell-background: rgb(60 63 68) !important;
+}
+.tbody-light .ndl-data-grid-tr:hover {
+  --cell-background: rgb(226 227 229) !important;
 }

frontend/src/HOC/CustomModal.tsx

@@ -16,7 +16,7 @@ const CustomModal: React.FC<CustomModalProps> = ({
   return (
     <Dialog
       size='small'
-      open={open}
+      isOpen={open}
       modalProps={{
         id: 'default-menu',
       }}
@@ -25,16 +25,17 @@ const CustomModal: React.FC<CustomModalProps> = ({
       <Dialog.Content className='n-flex n-flex-col n-gap-token-4 mt-6'>
         {status !== 'unknown' && (
           <Banner
-            closeable
+            isCloseable
             description={statusMessage}
             onClose={() => setStatus('unknown')}
             type={status}
             name='Custom Banner'
+            usage='inline'
           />
         )}
         <div className='n-flex n-flex-row n-flex-wrap'>{children}</div>
         <Dialog.Actions className='mt-4'>
-          <Button onClick={submitHandler} size='medium' disabled={isDisabled}>
+          <Button onClick={submitHandler} size='medium' isDisabled={isDisabled}>
             {submitLabel}
           </Button>
         </Dialog.Actions>

frontend/src/HOC/withVisibility.tsx

@@ -0,0 +1,14 @@
+interface VisibilityProps {
+  isVisible: boolean;
+}
+export function withVisibility<P>(WrappedComponent: React.ComponentType<P>) {
+  const VisibityControlled = (props: P & VisibilityProps) => {
+    if (props.isVisible === false) {
+      return null;
+    }
+
+    return <WrappedComponent {...props} />;
+  };
+
+  return VisibityControlled;
+}