azure-ai-search-rag-eval-promptflow.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install promptflow-evals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "import azure.identity\n",
    "import dotenv\n",
    "import openai\n",
    "from azure.search.documents import SearchClient\n",
    "from azure.search.documents.models import VectorizedQuery\n",
    "\n",
    "dotenv.load_dotenv()\n",
    "\n",
    "azure_credential = azure.identity.AzureDeveloperCliCredential(tenant_id=os.getenv(\"AZURE_TENANT_ID\"))\n",
    "\n",
    "# Initialize Azure OpenAI client\n",
    "AZURE_OPENAI_EMBEDDING_DEPLOYED_MODEL_NAME = os.getenv(\"AZURE_OPENAI_EMBEDDING_DEPLOYED_MODEL_NAME\")\n",
    "\n",
    "token_provider = azure.identity.get_bearer_token_provider(azure_credential, \"https://cognitiveservices.azure.com/.default\")\n",
    "openai_client = openai.AzureOpenAI(\n",
    "    api_version=\"2024-07-01-preview\",\n",
    "    azure_endpoint=os.getenv(\"AZURE_OPENAI_ENDPOINT\"),\n",
    "    azure_ad_token_provider=token_provider)\n",
    "\n",
    "def get_embedding(text):\n",
    "    get_embeddings_response = openai_client.embeddings.create(model=AZURE_OPENAI_EMBEDDING_DEPLOYED_MODEL_NAME, input=text)\n",
    "    return get_embeddings_response.data[0].embedding\n",
    "\n",
    "# Initialize Azure search client\n",
    "AZURE_SEARCH_SERVICE_ENDPOINT = os.getenv(\"AZURE_SEARCH_SERVICE_ENDPOINT\")\n",
    "\n",
    "AZURE_SEARCH_FULL_INDEX = \"dbpedia-1m-all\"\n",
    "search_client = SearchClient(AZURE_SEARCH_SERVICE_ENDPOINT, AZURE_SEARCH_FULL_INDEX, credential=azure_credential)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "A product manager investigates, selects, and drives the development of products for an organization, performing the activities of product management. They consider factors such as the intended demographic, the products offered by the competition, and how well the product fits with the company's business model. Generally, a product manager manages one or more tangible products [4a5576f9-1604-4ca2-b1d2-464c30f3498c].\n"
     ]
    }
   ],
   "source": [
    "user_question = \"What does a product manager do?\"\n",
    "user_question_vector = get_embedding(user_question)\n",
    "\n",
    "# Perform the search query\n",
    "r = search_client.search(\n",
    "    user_question,\n",
    "    top=5, \n",
    "    vector_queries=[\n",
    "        VectorizedQuery(vector=user_question_vector, k_nearest_neighbors=50, fields=\"embedding\")\n",
    "    ],\n",
    "    query_type=\"semantic\",\n",
    "    semantic_configuration_name=\"default\"\n",
    ")\n",
    "\n",
    "# Format the sources to include id, title, and text\n",
    "sources = \"\\n\\n\".join([f\"[{doc['id']}]: {doc['title']} - {doc['text']}\\n\" for doc in r])\n",
    "\n",
    "# Define the system message\n",
    "SYSTEM_MESSAGE = \"\"\"\n",
    "Assistant helps users find relevant information from the DBpedia dataset. DBpedia is a structured version of Wikipedia, containing factual information.\n",
    "Be concise in your answers, and answer ONLY with the facts listed in the sources below.\n",
    "If there isn't enough information from the sources, respond by stating you don't know. Avoid generating answers not based on the sources below.\n",
    "Each source has a name followed by the actual information. Always include the source name for each fact you use.\n",
    "Use square brackets to reference the source, for example [doc1].\n",
    "\"\"\"\n",
    "\n",
    "# Define the user message\n",
    "USER_MESSAGE = user_question + \"\\nSources: \" + sources\n",
    "\n",
    "# Generate the response using OpenAI\n",
    "response = openai_client.chat.completions.create(\n",
    "    model=os.getenv(\"AZURE_OPENAI_CHAT_COMPLETION_DEPLOYED_MODEL_NAME\"),\n",
    "    temperature=0.7,\n",
    "    messages=[\n",
    "        {\"role\": \"system\", \"content\": SYSTEM_MESSAGE},\n",
    "        {\"role\": \"user\", \"content\": USER_MESSAGE},\n",
    "    ],\n",
    ")\n",
    "\n",
    "# Print the answer\n",
    "answer = response.choices[0].message.content\n",
    "print(answer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'gpt_relevance': 5.0}\n",
      "{'gpt_groundedness': 5.0}\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "from promptflow.core import AzureOpenAIModelConfiguration\n",
    "from promptflow.evals.evaluators import GroundednessEvaluator, RelevanceEvaluator\n",
    "\n",
    "model_config = AzureOpenAIModelConfiguration(\n",
    "    azure_endpoint=os.getenv(\"AZURE_OEPNAI_ENDPOINT\"),\n",
    "    azure_deployment=os.environ.get(\"AZURE_OPENAI_CHAT_COMPLETION_DEPLOYED_MODEL_NAME\"),\n",
    ")\n",
    "\n",
    "relevance_eval = RelevanceEvaluator(model_config)\n",
    "groundedness_eval = GroundednessEvaluator(model_config)\n",
    "\n",
    "relevance_score = relevance_eval(\n",
    "    question=user_question,\n",
    "    answer=answer,\n",
    "    context=sources,\n",
    ")\n",
    "print(relevance_score)\n",
    "\n",
    "groundedness_score = groundedness_eval(\n",
    "    answer=answer,\n",
    "    context=sources,\n",
    ")\n",
    "print(groundedness_score)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}