azure-ai-search-rag-eval-quotient-ai.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Azure AI Search, Azure OpenAI, and QuotientAI RAG Pipeline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This pipeline combines Azure AI Search for information retrieval, Azure OpenAI for generation, and QuotientAI for evaluation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install quotientai"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "from azure.core.credentials import AzureKeyCredential\n",
    "from azure.search.documents import SearchClient\n",
    "import os\n",
    "import numpy as np\n",
    "from typing import List, Dict, Any\n",
    "from azure.search.documents import SearchClient\n",
    "from azure.core.credentials import AzureKeyCredential\n",
    "from azure.identity import DefaultAzureCredential\n",
    "from azure.search.documents.indexes import SearchIndexClient\n",
    "from azure.search.documents.models import (\n",
    "    VectorizableTextQuery,\n",
    "    VectorizedQuery\n",
    ")\n",
    "from quotientai import QuotientAI\n",
    "from openai import AzureOpenAI\n",
    "from dotenv import load_dotenv\n",
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from typing import List, Dict, Any\n",
    "from dataclasses import dataclass\n",
    "from datetime import datetime\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "# Azure Search\n",
    "from azure.search.documents import SearchClient\n",
    "from azure.core.credentials import AzureKeyCredential\n",
    "from azure.search.documents.models import VectorizedQuery\n",
    "\n",
    "# OpenAI\n",
    "from openai import AzureOpenAI\n",
    "\n",
    "# Load environment variables\n",
    "load_dotenv()\n",
    "\n",
    "AZURE_SEARCH_ENDPOINT = os.getenv(\"AZURE_SEARCH_SERVICE_ENDPOINT\")\n",
    "AZURE_SEARCH_KEY = os.getenv(\"AZURE_SEARCH_ADMIN_KEY\")\n",
    "AZURE_OPENAI_ENDPOINT = os.getenv(\"AZURE_OPENAI_ENDPOINT\")\n",
    "AZURE_OPENAI_EMBEDDING_DEPLOYED_MODEL_NAME=os.getenv(\"AZURE_OPENAI_EMBEDDING_DEPLOYED_MODEL_NAME\")\n",
    "AZURE_OPENAI_CHAT_COMPLETION_DEPLOYED_MODEL_NAME = os.getenv(\"AZURE_OPENAI_CHAT_COMPLETION_DEPLOYED_MODEL_NAME\")\n",
    "AZURE_OPENAI_KEY = os.getenv(\"AZURE_OPENAI_API_KEY\")\n",
    "QUOTIENT_API_KEY = os.getenv(\"QUOTIENT_API_KEY\")\n",
    "os.environ[\"QUOTIENT_API_KEY\"] = QUOTIENT_API_KEY\n",
    "\n",
    "# Initialize Azure clients\n",
    "search_client = SearchClient(endpoint=AZURE_SEARCH_ENDPOINT, index_name=\"llamaindex-azure-aisearch-rag-literal-ai\", credential=AzureKeyCredential(AZURE_SEARCH_KEY))\n",
    "openai_client = AzureOpenAI(\n",
    "    api_key=AZURE_OPENAI_KEY,\n",
    "    api_version=\"2024-10-21\",\n",
    "    azure_endpoint=AZURE_OPENAI_ENDPOINT\n",
    ")\n",
    "# Initialize QuotientAI client\n",
    "quotient_client = QuotientAI()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. QuotientAI minimal resource classes and client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "@dataclass\n",
    "class ModelProvider:\n",
    "    id: str\n",
    "    name: str\n",
    "\n",
    "@dataclass\n",
    "class Model:\n",
    "    id: str\n",
    "    name: str\n",
    "    provider: ModelProvider\n",
    "    created_at: str\n",
    "\n",
    "class ModelsResource:\n",
    "    def __init__(self, client):\n",
    "        self._client = client\n",
    "    def list(self) -> List[Model]:\n",
    "        # Placeholder\n",
    "        return []\n",
    "\n",
    "@dataclass\n",
    "class DatasetRowMetadata:\n",
    "    annotation: str = \"ungraded\"\n",
    "    annotation_note: str = \"\"\n",
    "\n",
    "@dataclass\n",
    "class DatasetRow:\n",
    "    id: str\n",
    "    input: str\n",
    "    context: str\n",
    "    expected: str\n",
    "    metadata: DatasetRowMetadata\n",
    "    created_by: str\n",
    "    created_at: datetime\n",
    "    updated_at: datetime\n",
    "\n",
    "@dataclass\n",
    "class Dataset:\n",
    "    id: str\n",
    "    name: str\n",
    "    created_by: str\n",
    "    created_at: datetime\n",
    "    updated_at: datetime\n",
    "    description: str = \"\"\n",
    "    rows: List[DatasetRow] = None\n",
    "\n",
    "class DatasetsResource:\n",
    "    def __init__(self, client):\n",
    "        self._client = client\n",
    "\n",
    "    def create(self, name: str, description: str = None, rows: List[dict] = None) -> Dataset:\n",
    "        # Mock in-memory creation for demonstration\n",
    "        now = datetime.utcnow()\n",
    "        ds_id = \"dataset-\" + str(int(now.timestamp()))\n",
    "        row_objs = []\n",
    "        for i, r in enumerate(rows or []):\n",
    "            row_objs.append(\n",
    "                DatasetRow(\n",
    "                    id=f\"{ds_id}-{i}\",\n",
    "                    input=r.get(\"input\", \"\"),\n",
    "                    context=r.get(\"context\", \"\"),\n",
    "                    expected=r.get(\"expected\", \"\"),\n",
    "                    metadata=DatasetRowMetadata(\n",
    "                        annotation=r[\"metadata\"].get(\"annotation\", \"ungraded\"),\n",
    "                        annotation_note=r[\"metadata\"].get(\"annotation_note\", \"\"),\n",
    "                    ),\n",
    "                    created_at=now,\n",
    "                    updated_at=now,\n",
    "                    created_by=\"PlaceholderUser\",\n",
    "                )\n",
    "            )\n",
    "        return Dataset(\n",
    "            id=ds_id,\n",
    "            name=name,\n",
    "            description=description or \"\",\n",
    "            created_at=now,\n",
    "            updated_at=now,\n",
    "            created_by=\"PlaceholderUser\",\n",
    "            rows=row_objs,\n",
    "        )\n",
    "\n",
    "@dataclass\n",
    "class Prompt:\n",
    "    id: str\n",
    "    name: str\n",
    "    version: int\n",
    "    system_prompt: str\n",
    "    user_prompt: str\n",
    "    created_at: datetime\n",
    "    updated_at: datetime\n",
    "\n",
    "class PromptsResource:\n",
    "    def __init__(self, client):\n",
    "        self._client = client\n",
    "\n",
    "    def create(self, name: str, system_prompt: str = \"\", user_prompt: str = \"\") -> Prompt:\n",
    "        now = datetime.utcnow()\n",
    "        prompt_id = \"prompt-\" + str(int(now.timestamp()))\n",
    "        return Prompt(\n",
    "            id=prompt_id,\n",
    "            name=name,\n",
    "            version=1,\n",
    "            system_prompt=system_prompt,\n",
    "            user_prompt=user_prompt,\n",
    "            created_at=now,\n",
    "            updated_at=now,\n",
    "        )\n",
    "\n",
    "class QuotientAI:\n",
    "    def __init__(self):\n",
    "        self.models = ModelsResource(self)\n",
    "        self.datasets = DatasetsResource(self)\n",
    "        self.prompts = PromptsResource(self)\n",
    "\n",
    "    # Mock internal request methods\n",
    "    def _get(self, path: str):\n",
    "        pass\n",
    "    def _post(self, path: str, data: dict):\n",
    "        return data\n",
    "    def _patch(self, path: str, data: dict):\n",
    "        return data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Helper: Generate Embeddings with Azure OpenAI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_embeddings(text: str):\n",
    "    \"\"\"\n",
    "    Generate embeddings using Azure OpenAI.\n",
    "    \"\"\"\n",
    "    embeddings_response = openai_client.embeddings.create(\n",
    "        model=AZURE_OPENAI_EMBEDDING_DEPLOYED_MODEL_NAME, \n",
    "        input=text\n",
    "    )\n",
    "    return embeddings_response.data[0].embedding\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def call_chat_model(system_prompt: str, user_query: str) -> str:\n",
    "    \"\"\"\n",
    "    Call the Azure OpenAI Chat Completion (GPT-4o or similar).\n",
    "    You can also pass in context here if you prefer.\n",
    "    \"\"\"\n",
    "    completion = openai_client.chat_completions.create(\n",
    "        model=AZURE_OPENAI_CHAT_COMPLETION_DEPLOYED_MODEL_NAME,\n",
    "        messages=[\n",
    "            {\"role\": \"system\", \"content\": system_prompt},\n",
    "            {\"role\": \"user\", \"content\": user_query}\n",
    "        ],\n",
    "    )\n",
    "    return completion.choices[0].message.content"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Vectorized Retrieval from Azure AI Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def retrieve_documents(question: str, top_k: int = 3) -> List[Dict]:\n",
    "    \"\"\"\n",
    "    Retrieve top_k documents from Azure Search using vector embeddings.\n",
    "    \"\"\"\n",
    "    query_embedding = generate_embeddings(question)\n",
    "    vector_query = VectorizedQuery(\n",
    "        vector=query_embedding,\n",
    "        k_nearest_neighbors=top_k,\n",
    "        fields=\"embedding\"  # your index vector field name\n",
    "    )\n",
    "    results = search_client.search(\n",
    "        search_text=None,\n",
    "        vector_queries=[vector_query],\n",
    "        top=top_k\n",
    "    )\n",
    "    return list(results)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "5. Example: Evaluate retrieval for multiple top_k values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "==== Querying with top_k=3 ====\n",
      "[Result #1]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.7765124\n",
      "------------------------------------------------------------\n",
      "[Result #2]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.7631142\n",
      "------------------------------------------------------------\n",
      "[Result #3]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.7506997\n",
      "------------------------------------------------------------\n",
      "\n",
      "==== Querying with top_k=10 ====\n",
      "[Result #1]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.7765124\n",
      "------------------------------------------------------------\n",
      "[Result #2]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.7631142\n",
      "------------------------------------------------------------\n",
      "[Result #3]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.7506997\n",
      "------------------------------------------------------------\n",
      "[Result #4]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.7367365\n",
      "------------------------------------------------------------\n",
      "[Result #5]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.7309735\n",
      "------------------------------------------------------------\n",
      "[Result #6]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.73024464\n",
      "------------------------------------------------------------\n",
      "[Result #7]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.72153836\n",
      "------------------------------------------------------------\n",
      "[Result #8]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.71948296\n",
      "------------------------------------------------------------\n",
      "[Result #9]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.71884054\n",
      "------------------------------------------------------------\n",
      "[Result #10]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.7145346\n",
      "------------------------------------------------------------\n",
      "\n",
      "==== Querying with top_k=50 ====\n",
      "[Result #1]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.7765124\n",
      "------------------------------------------------------------\n",
      "[Result #2]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.7631142\n",
      "------------------------------------------------------------\n",
      "[Result #3]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.7506997\n",
      "------------------------------------------------------------\n",
      "[Result #4]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.7367365\n",
      "------------------------------------------------------------\n",
      "[Result #5]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.7309735\n",
      "------------------------------------------------------------\n",
      "[Result #6]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.73024464\n",
      "------------------------------------------------------------\n",
      "[Result #7]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.72153836\n",
      "------------------------------------------------------------\n",
      "[Result #8]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.71948296\n",
      "------------------------------------------------------------\n",
      "[Result #9]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.71884054\n",
      "------------------------------------------------------------\n",
      "[Result #10]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.7145346\n",
      "------------------------------------------------------------\n",
      "[Result #11]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.7115476\n",
      "------------------------------------------------------------\n",
      "[Result #12]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.7074393\n",
      "------------------------------------------------------------\n",
      "[Result #13]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.7068772\n",
      "------------------------------------------------------------\n",
      "[Result #14]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.706722\n",
      "------------------------------------------------------------\n",
      "[Result #15]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.70505863\n",
      "------------------------------------------------------------\n",
      "[Result #16]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.70396394\n",
      "------------------------------------------------------------\n",
      "[Result #17]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.70382386\n",
      "------------------------------------------------------------\n",
      "[Result #18]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.70356816\n",
      "------------------------------------------------------------\n",
      "[Result #19]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.7030631\n",
      "------------------------------------------------------------\n",
      "[Result #20]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.70244133\n",
      "------------------------------------------------------------\n",
      "[Result #21]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.70240504\n",
      "------------------------------------------------------------\n",
      "[Result #22]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.7019276\n",
      "------------------------------------------------------------\n",
      "[Result #23]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.7013003\n",
      "------------------------------------------------------------\n",
      "[Result #24]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.7007803\n",
      "------------------------------------------------------------\n",
      "[Result #25]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.69987965\n",
      "------------------------------------------------------------\n",
      "[Result #26]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.69928235\n",
      "------------------------------------------------------------\n",
      "[Result #27]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.69849664\n",
      "------------------------------------------------------------\n",
      "[Result #28]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.6984239\n",
      "------------------------------------------------------------\n",
      "[Result #29]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.6981494\n",
      "------------------------------------------------------------\n",
      "[Result #30]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.6980157\n",
      "------------------------------------------------------------\n",
      "[Result #31]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.69672555\n",
      "------------------------------------------------------------\n",
      "[Result #32]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.69662696\n",
      "------------------------------------------------------------\n",
      "[Result #33]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.69653386\n",
      "------------------------------------------------------------\n",
      "[Result #34]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.69628453\n",
      "------------------------------------------------------------\n",
      "[Result #35]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.6959157\n",
      "------------------------------------------------------------\n",
      "[Result #36]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.69485486\n",
      "------------------------------------------------------------\n",
      "[Result #37]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.6923996\n",
      "------------------------------------------------------------\n",
      "[Result #38]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.6915822\n",
      "------------------------------------------------------------\n",
      "[Result #39]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.69142413\n",
      "------------------------------------------------------------\n",
      "[Result #40]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.6913037\n",
      "------------------------------------------------------------\n",
      "[Result #41]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.6910848\n",
      "------------------------------------------------------------\n",
      "[Result #42]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.69086313\n",
      "------------------------------------------------------------\n",
      "[Result #43]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.6906489\n",
      "------------------------------------------------------------\n",
      "[Result #44]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.68987817\n",
      "------------------------------------------------------------\n",
      "[Result #45]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.6886098\n",
      "------------------------------------------------------------\n",
      "[Result #46]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.6885524\n",
      "------------------------------------------------------------\n",
      "[Result #47]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.6881464\n",
      "------------------------------------------------------------\n",
      "[Result #48]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.68802994\n",
      "------------------------------------------------------------\n",
      "[Result #49]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.68761516\n",
      "------------------------------------------------------------\n",
      "[Result #50]\n",
      "Title: No title\n",
      "Content snippet: ...\n",
      "Score: 0.68753934\n",
      "------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "test_query = \"What is the annual deductible for individual coverage under the Northwind Health Plus plan?\"\n",
    "\n",
    "for k in [3, 10, 50]:\n",
    "    print(f\"\\n==== Querying with top_k={k} ====\")\n",
    "    docs = query_azure_search(test_query, top_k=k)\n",
    "    for i, doc in enumerate(docs):\n",
    "        print(f\"[Result #{i+1}]\")\n",
    "        print(f\"Title: {doc.get('title', 'No title')}\")\n",
    "        print(f\"Content snippet: {doc.get('content','')[:120]}...\")\n",
    "        print(f\"Score: {doc.get('@search.score','No score')}\")\n",
    "        print(\"-\" * 60)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  6. Create an Evaluation Dataset in Quotient"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Created dataset in QuotientAI:\n",
      "Dataset ID: ea38ea5c-2ed5-427c-8568-73a30d3d158f\n",
      "Name: Northwind Health Evaluation\n",
      "Number of rows: 10\n"
     ]
    }
   ],
   "source": [
    "# Provided evaluation dataset\n",
    "eval_dataset = [\n",
    "    {\n",
    "        \"id\": \"Q001\",\n",
    "        \"input\": \"What is the annual deductible for individual coverage under the Northwind Health Plus plan?\",\n",
    "        \"context\": \"The Northwind Health Plus plan has a calendar year deductible of $1,500 for individuals for in-network services.\",\n",
    "        \"expected\": \"$1,500\",\n",
    "        \"metadata\": {\n",
    "            \"annotation\": \"good\",\n",
    "            \"annotation_note\": \"Verified deductible from Northwind Health Plus benefits summary.\",\n",
    "        },\n",
    "    },\n",
    "    {\n",
    "        \"id\": \"Q002\",\n",
    "        \"input\": \"Does the Northwind Standard plan cover out-of-network emergency services?\",\n",
    "        \"context\": \"Northwind Standard does not offer coverage for emergency services, mental health and substance abuse coverage, or out-of-network services.\",\n",
    "        \"expected\": \"No\",\n",
    "        \"metadata\": {\n",
    "            \"annotation\": \"good\",\n",
    "            \"annotation_note\": \"Aligned with coverage exclusions listed for the Northwind Standard plan.\",\n",
    "        },\n",
    "    },\n",
    "    {\n",
    "        \"id\": \"Q003\",\n",
    "        \"input\": \"What fitness activities are covered under the PerksPlus program?\",\n",
    "        \"context\": \"PerksPlus covers gym memberships, personal training sessions, yoga and Pilates classes, fitness equipment purchases, sports team fees, health retreats and spas, outdoor activities like rock climbing, and virtual fitness programs.\",\n",
    "        \"expected\": \"Gym memberships, personal training, yoga, Pilates, fitness equipment purchases, sports fees, health retreats, outdoor activities, and virtual programs.\",\n",
    "        \"metadata\": {\n",
    "            \"annotation\": \"good\",\n",
    "            \"annotation_note\": \"Cross-referenced with PerksPlus health and wellness program details.\",\n",
    "        },\n",
    "    },\n",
    "    {\n",
    "        \"id\": \"Q004\",\n",
    "        \"input\": \"What is the copayment for a specialist visit under the Northwind Standard plan?\",\n",
    "        \"context\": \"The copayment for specialist visits under the Northwind Standard plan is $50.\",\n",
    "        \"expected\": \"$50\",\n",
    "        \"metadata\": {\n",
    "            \"annotation\": \"good\",\n",
    "            \"annotation_note\": \"Matched with the cost-sharing section for the Northwind Standard plan.\",\n",
    "        },\n",
    "    },\n",
    "    {\n",
    "        \"id\": \"Q005\",\n",
    "        \"input\": \"Does the Northwind Health Plus plan offer coverage for mental health services?\",\n",
    "        \"context\": \"Northwind Health Plus provides coverage for mental health and substance abuse services.\",\n",
    "        \"expected\": \"Yes\",\n",
    "        \"metadata\": {\n",
    "            \"annotation\": \"good\",\n",
    "            \"annotation_note\": \"Validated against the coverage summary for Northwind Health Plus.\",\n",
    "        },\n",
    "    },\n",
    "    {\n",
    "        \"id\": \"Q006\",\n",
    "        \"input\": \"Are acupuncture services covered under the Northwind Health Plus plan?\",\n",
    "        \"context\": \"Northwind Health Plus covers up to 12 medically necessary acupuncture visits per year, with prior approval required.\",\n",
    "        \"expected\": \"Yes, up to 12 visits per year with prior approval.\",\n",
    "        \"metadata\": {\n",
    "            \"annotation\": \"good\",\n",
    "            \"annotation_note\": \"Aligned with the acupuncture coverage section.\",\n",
    "        },\n",
    "    },\n",
    "    {\n",
    "        \"id\": \"Q007\",\n",
    "        \"input\": \"What is the reimbursement limit for fitness-related expenses under PerksPlus?\",\n",
    "        \"context\": \"Employees can expense up to $1,000 annually for fitness-related programs under PerksPlus.\",\n",
    "        \"expected\": \"$1,000\",\n",
    "        \"metadata\": {\n",
    "            \"annotation\": \"good\",\n",
    "            \"annotation_note\": \"Confirmed from the PerksPlus overview.\",\n",
    "        },\n",
    "    },\n",
    "    {\n",
    "        \"id\": \"Q008\",\n",
    "        \"input\": \"What types of dental services are covered under Northwind Health Plus?\",\n",
    "        \"context\": \"Northwind Health Plus covers dental exams, cleanings, and fillings as part of its vision and dental coverage.\",\n",
    "        \"expected\": \"Dental exams, cleanings, and fillings.\",\n",
    "        \"metadata\": {\n",
    "            \"annotation\": \"good\",\n",
    "            \"annotation_note\": \"Verified against the dental services coverage for Northwind Health Plus.\",\n",
    "        },\n",
    "    },\n",
    "    {\n",
    "        \"id\": \"Q009\",\n",
    "        \"input\": \"What is the out-of-pocket maximum for family coverage under Northwind Standard?\",\n",
    "        \"context\": \"The out-of-pocket maximum for family coverage under Northwind Standard is $12,700.\",\n",
    "        \"expected\": \"$12,700\",\n",
    "        \"metadata\": {\n",
    "            \"annotation\": \"good\",\n",
    "            \"annotation_note\": \"Cross-checked with Northwind Standard plan details.\",\n",
    "        },\n",
    "    },\n",
    "    {\n",
    "        \"id\": \"Q010\",\n",
    "        \"input\": \"Does the Northwind Health Plus plan protect against balance billing?\",\n",
    "        \"context\": \"Northwind Health Plus protects members from balance billing for in-network services.\",\n",
    "        \"expected\": \"Yes, for in-network services.\",\n",
    "        \"metadata\": {\n",
    "            \"annotation\": \"good\",\n",
    "            \"annotation_note\": \"Aligned with balance billing protection details.\",\n",
    "        },\n",
    "    },\n",
    "]\n",
    "\n",
    "dataset_rows = []\n",
    "for row in eval_dataset:\n",
    "    dataset_rows.append({\n",
    "        \"input\": row[\"input\"],\n",
    "        \"context\": row[\"context\"],\n",
    "        \"expected\": row[\"expected\"],\n",
    "        \"metadata\": {\n",
    "            \"annotation\": row[\"metadata\"][\"annotation\"],\n",
    "            \"annotation_note\": row[\"metadata\"][\"annotation_note\"],\n",
    "        }\n",
    "    })\n",
    "\n",
    "rag_dataset = quotient_client.datasets.create(\n",
    "    name=\"Northwind Health Evaluation\",\n",
    "    description=\"Evaluation dataset for RAG with Northwind Health data.\",\n",
    "    rows=dataset_rows\n",
    ")\n",
    "\n",
    "print(\"\\nCreated dataset in QuotientAI:\")\n",
    "print(f\"Dataset ID: {rag_dataset.id}\")\n",
    "print(f\"Name: {rag_dataset.name}\")\n",
    "print(f\"Number of rows: {len(rag_dataset.rows)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 7. Create a Prompt in Quotient"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Created Prompt in QuotientAI:\n",
      "Prompt ID: 9757aa01-2508-439f-9a84-d33d8700ced0\n",
      "Prompt Name: my-northwind-prompt\n",
      "System Prompt: You are a helpful assistant with knowledge of Contoso HR. Answer accurately based on context.\n",
      "User Prompt: User query: {input}\n",
      "Context: {context}\n",
      "Please provide a concise, accurate answer.\n"
     ]
    }
   ],
   "source": [
    "prompt = quotient_client.prompts.create(\n",
    "    name=\"my-northwind-prompt\",\n",
    "    system_prompt=\"You are a helpful assistant with knowledge of Contoso HR. Answer accurately based on context.\",\n",
    "    user_prompt=\"User query: {input}\\nContext: {context}\\nPlease provide a concise, accurate answer.\"\n",
    ")\n",
    "\n",
    "print(\"\\nCreated Prompt in QuotientAI:\")\n",
    "print(f\"Prompt ID: {prompt.id}\")\n",
    "print(f\"Prompt Name: {prompt.name}\")\n",
    "print(f\"System Prompt: {prompt.system_prompt}\")\n",
    "print(f\"User Prompt: {prompt.user_prompt}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}