azure-ai-search-rag-eval-w&b.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install azure-identity\n",
    "!pip install azure-search-documents==11.4.0\n",
    "!pip install llama-index\n",
    "!pip install llama-index-embeddings-azure-openai\n",
    "!pip install llama-index-llms-azure-openai\n",
    "!pip install llama-index-vector-stores-azureaisearch\n",
    "!pip install nest-asyncio\n",
    "!pip install python-dotenv\n",
    "!pip install wandb\n",
    "!pip install weave\n",
    "!pip install llama-index-callbacks-wandb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initial Setup\n",
    "Load environment variables and initialize the necessary clients and models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'dotenv'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[2], line 2\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mos\u001b[39;00m\n\u001b[1;32m----> 2\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mdotenv\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m load_dotenv\n\u001b[0;32m      3\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mazure\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcore\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcredentials\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m AzureKeyCredential\n\u001b[0;32m      4\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mazure\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01msearch\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdocuments\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m SearchClient\n",
      "\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'dotenv'"
     ]
    }
   ],
   "source": [
    "import os\n",
    "from dotenv import load_dotenv\n",
    "from azure.core.credentials import AzureKeyCredential\n",
    "from azure.search.documents import SearchClient\n",
    "from azure.search.documents.indexes import SearchIndexClient\n",
    "from llama_index.core import StorageContext, VectorStoreIndex\n",
    "from llama_index.embeddings.azure_openai import AzureOpenAIEmbedding\n",
    "from llama_index.llms.azure_openai import AzureOpenAI\n",
    "from llama_index.vector_stores.azureaisearch import AzureAISearchVectorStore, IndexManagement\n",
    "from llama_index.core.callbacks import LlamaDebugHandler\n",
    "\n",
    "# Load environment variables\n",
    "load_dotenv()\n",
    "\n",
    "# Environment Variables\n",
    "AZURE_OPENAI_ENDPOINT = os.getenv(\"AZURE_OPENAI_ENDPOINT\")\n",
    "AZURE_OPENAI_API_KEY = os.getenv(\"AZURE_OPENAI_API_KEY\")\n",
    "AZURE_OPENAI_CHAT_COMPLETION_DEPLOYED_MODEL_NAME = os.getenv(\"AZURE_OPENAI_CHAT_COMPLETION_DEPLOYED_MODEL_NAME\") # I'm using GPT-3.5-turbo\n",
    "AZURE_OPENAI_EMBEDDING_DEPLOYED_MODEL_NAME = os.getenv(\"AZURE_OPENAI_EMBEDDING_DEPLOYED_MODEL_NAME\") # I'm using text-embedding-ada-002\n",
    "SEARCH_SERVICE_ENDPOINT = os.getenv(\"AZURE_SEARCH_SERVICE_ENDPOINT\")\n",
    "SEARCH_SERVICE_API_KEY = os.getenv(\"AZURE_SEARCH_ADMIN_KEY\")\n",
    "INDEX_NAME = \"contoso-hr-docs\"\n",
    "WANDB_PROJECT=\"llamaindex-contos-hr-docs\"\n",
    "# WANDB_API_KEY = os.getenv(\"WANDB_API_KEY\")\n",
    "# os.environ['WANDB_API_KEY'] = WANDB_API_KEY\n",
    "\n",
    "# Initialize Azure OpenAI and embedding models\n",
    "llm = AzureOpenAI(\n",
    "    model=AZURE_OPENAI_CHAT_COMPLETION_DEPLOYED_MODEL_NAME,\n",
    "    deployment_name=AZURE_OPENAI_CHAT_COMPLETION_DEPLOYED_MODEL_NAME,\n",
    "    api_key=AZURE_OPENAI_API_KEY,\n",
    "    azure_endpoint=AZURE_OPENAI_ENDPOINT,\n",
    "    api_version=\"2024-02-01\"\n",
    ")\n",
    "\n",
    "embed_model = AzureOpenAIEmbedding(\n",
    "    model=AZURE_OPENAI_EMBEDDING_DEPLOYED_MODEL_NAME,\n",
    "    deployment_name=AZURE_OPENAI_EMBEDDING_DEPLOYED_MODEL_NAME,\n",
    "    api_key=AZURE_OPENAI_API_KEY,\n",
    "    azure_endpoint=AZURE_OPENAI_ENDPOINT,\n",
    "    api_version=\"2024-02-01\"\n",
    ")\n",
    "\n",
    "# Initialize search clients\n",
    "credential = AzureKeyCredential(SEARCH_SERVICE_API_KEY)\n",
    "index_client = SearchIndexClient(endpoint=SEARCH_SERVICE_ENDPOINT, credential=credential)\n",
    "search_client = SearchClient(endpoint=SEARCH_SERVICE_ENDPOINT, index_name=INDEX_NAME, credential=credential)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Weights & Biases Callback Manager Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "`np.float_` was removed in the NumPy 2.0 release. Use `np.float64` instead.",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[2], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mwandb\u001b[39;00m\n\u001b[0;32m      2\u001b[0m wandb\u001b[38;5;241m.\u001b[39mlogin()\n",
      "File \u001b[1;32mc:\\Dev\\azure-ai-search-python-playground\\.venv\\Lib\\site-packages\\wandb\\__init__.py:27\u001b[0m\n\u001b[0;32m     24\u001b[0m \u001b[38;5;66;03m# This needs to be early as other modules call it.\u001b[39;00m\n\u001b[0;32m     25\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mwandb\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01merrors\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mterm\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m termsetup, termlog, termerror, termwarn\n\u001b[1;32m---> 27\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mwandb\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m sdk \u001b[38;5;28;01mas\u001b[39;00m wandb_sdk\n\u001b[0;32m     29\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mwandb\u001b[39;00m\n\u001b[0;32m     31\u001b[0m wandb\u001b[38;5;241m.\u001b[39mwandb_lib \u001b[38;5;241m=\u001b[39m wandb_sdk\u001b[38;5;241m.\u001b[39mlib  \u001b[38;5;66;03m# type: ignore\u001b[39;00m\n",
      "File \u001b[1;32mc:\\Dev\\azure-ai-search-python-playground\\.venv\\Lib\\site-packages\\wandb\\sdk\\__init__.py:25\u001b[0m\n\u001b[0;32m      3\u001b[0m __all__ \u001b[38;5;241m=\u001b[39m (\n\u001b[0;32m      4\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mConfig\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m      5\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mSettings\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m     21\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhelper\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m     22\u001b[0m )\n\u001b[0;32m     24\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m wandb_helper \u001b[38;5;28;01mas\u001b[39;00m helper\n\u001b[1;32m---> 25\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01martifacts\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01martifact\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Artifact\n\u001b[0;32m     26\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mwandb_alerts\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m AlertLevel\n\u001b[0;32m     27\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mwandb_config\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Config\n",
      "File \u001b[1;32mc:\\Dev\\azure-ai-search-python-playground\\.venv\\Lib\\site-packages\\wandb\\sdk\\artifacts\\artifact.py:45\u001b[0m\n\u001b[0;32m     42\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mrequests\u001b[39;00m\n\u001b[0;32m     44\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mwandb\u001b[39;00m\n\u001b[1;32m---> 45\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mwandb\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m data_types, env, util\n\u001b[0;32m     46\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mwandb\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mapis\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mnormalize\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m normalize_exceptions\n\u001b[0;32m     47\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mwandb\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mapis\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mpublic\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m ArtifactCollection, ArtifactFiles, RetryingClient, Run\n",
      "File \u001b[1;32mc:\\Dev\\azure-ai-search-python-playground\\.venv\\Lib\\site-packages\\wandb\\data_types.py:32\u001b[0m\n\u001b[0;32m     29\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mwandb\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m util\n\u001b[0;32m     30\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mwandb\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01msdk\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mlib\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m filesystem\n\u001b[1;32m---> 32\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01msdk\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdata_types\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m _dtypes\n\u001b[0;32m     33\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01msdk\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdata_types\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_private\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m MEDIA_TMP\n\u001b[0;32m     34\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01msdk\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdata_types\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mbase_types\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mmedia\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[0;32m     35\u001b[0m     BatchableMedia,\n\u001b[0;32m     36\u001b[0m     Media,\n\u001b[0;32m     37\u001b[0m     _numpy_arrays_to_lists,\n\u001b[0;32m     38\u001b[0m )\n",
      "File \u001b[1;32mc:\\Dev\\azure-ai-search-python-playground\\.venv\\Lib\\site-packages\\wandb\\sdk\\data_types\\_dtypes.py:393\u001b[0m\n\u001b[0;32m    391\u001b[0m NumberType\u001b[38;5;241m.\u001b[39mtypes\u001b[38;5;241m.\u001b[39mappend(np\u001b[38;5;241m.\u001b[39mfloat32)\n\u001b[0;32m    392\u001b[0m NumberType\u001b[38;5;241m.\u001b[39mtypes\u001b[38;5;241m.\u001b[39mappend(np\u001b[38;5;241m.\u001b[39mfloat64)\n\u001b[1;32m--> 393\u001b[0m NumberType\u001b[38;5;241m.\u001b[39mtypes\u001b[38;5;241m.\u001b[39mappend(\u001b[43mnp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfloat_\u001b[49m)\n\u001b[0;32m    394\u001b[0m NumberType\u001b[38;5;241m.\u001b[39mtypes\u001b[38;5;241m.\u001b[39mappend(np\u001b[38;5;241m.\u001b[39mcomplex64)\n\u001b[0;32m    395\u001b[0m NumberType\u001b[38;5;241m.\u001b[39mtypes\u001b[38;5;241m.\u001b[39mappend(np\u001b[38;5;241m.\u001b[39mcomplex128)\n",
      "File \u001b[1;32mc:\\Dev\\azure-ai-search-python-playground\\.venv\\Lib\\site-packages\\numpy\\__init__.py:411\u001b[0m, in \u001b[0;36m__getattr__\u001b[1;34m(attr)\u001b[0m\n\u001b[0;32m    408\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mAttributeError\u001b[39;00m(__former_attrs__[attr])\n\u001b[0;32m    410\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m attr \u001b[38;5;129;01min\u001b[39;00m __expired_attributes__:\n\u001b[1;32m--> 411\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mAttributeError\u001b[39;00m(\n\u001b[0;32m    412\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m`np.\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mattr\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m` was removed in the NumPy 2.0 release. \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    413\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m__expired_attributes__[attr]\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    414\u001b[0m     )\n\u001b[0;32m    416\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m attr \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mchararray\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[0;32m    417\u001b[0m     warnings\u001b[38;5;241m.\u001b[39mwarn(\n\u001b[0;32m    418\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m`np.chararray` is deprecated and will be removed from \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    419\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mthe main namespace in the future. Use an array with a string \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    420\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mor bytes dtype instead.\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;167;01mDeprecationWarning\u001b[39;00m, stacklevel\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m2\u001b[39m)\n",
      "\u001b[1;31mAttributeError\u001b[0m: `np.float_` was removed in the NumPy 2.0 release. Use `np.float64` instead."
     ]
    }
   ],
   "source": [
    "import wandb\n",
    "wandb.login()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Instrument LlamaIndex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "`np.float_` was removed in the NumPy 2.0 release. Use `np.float64` instead.",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[3], line 2\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mllama_index\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcore\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m set_global_handler\n\u001b[1;32m----> 2\u001b[0m \u001b[43mset_global_handler\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mwandb\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mrun_args\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m{\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mproject\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mWANDB_PROJECT\u001b[49m\u001b[43m}\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32mc:\\Dev\\azure-ai-search-python-playground\\.venv\\Lib\\site-packages\\llama_index\\core\\callbacks\\global_handlers.py:10\u001b[0m, in \u001b[0;36mset_global_handler\u001b[1;34m(eval_mode, **eval_params)\u001b[0m\n\u001b[0;32m      7\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Set global eval handlers.\"\"\"\u001b[39;00m\n\u001b[0;32m      8\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mllama_index\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcore\u001b[39;00m\n\u001b[1;32m---> 10\u001b[0m handler \u001b[38;5;241m=\u001b[39m \u001b[43mcreate_global_handler\u001b[49m\u001b[43m(\u001b[49m\u001b[43meval_mode\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43meval_params\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     11\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m handler:\n\u001b[0;32m     12\u001b[0m     llama_index\u001b[38;5;241m.\u001b[39mcore\u001b[38;5;241m.\u001b[39mglobal_handler \u001b[38;5;241m=\u001b[39m handler\n",
      "File \u001b[1;32mc:\\Dev\\azure-ai-search-python-playground\\.venv\\Lib\\site-packages\\llama_index\\core\\callbacks\\global_handlers.py:31\u001b[0m, in \u001b[0;36mcreate_global_handler\u001b[1;34m(eval_mode, **eval_params)\u001b[0m\n\u001b[0;32m     25\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m:\n\u001b[0;32m     26\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m(\n\u001b[0;32m     27\u001b[0m             \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mWandbCallbackHandler is not installed. \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m     28\u001b[0m             \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mPlease install it using `pip install llama-index-callbacks-wandb`\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m     29\u001b[0m         )\n\u001b[1;32m---> 31\u001b[0m     handler: BaseCallbackHandler \u001b[38;5;241m=\u001b[39m \u001b[43mWandbCallbackHandler\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43meval_params\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     32\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m eval_mode \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mopeninference\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[0;32m     33\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n",
      "File \u001b[1;32mc:\\Dev\\azure-ai-search-python-playground\\.venv\\Lib\\site-packages\\llama_index\\callbacks\\wandb\\base.py:121\u001b[0m, in \u001b[0;36mWandbCallbackHandler.__init__\u001b[1;34m(self, run_args, tokenizer, event_starts_to_ignore, event_ends_to_ignore)\u001b[0m\n\u001b[0;32m    113\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\n\u001b[0;32m    114\u001b[0m     \u001b[38;5;28mself\u001b[39m,\n\u001b[0;32m    115\u001b[0m     run_args: Optional[WandbRunArgs] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    118\u001b[0m     event_ends_to_ignore: Optional[List[CBEventType]] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[0;32m    119\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m    120\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m--> 121\u001b[0m         \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mwandb\u001b[39;00m\n\u001b[0;32m    122\u001b[0m         \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mwandb\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01msdk\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdata_types\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m trace_tree\n\u001b[0;32m    124\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_wandb \u001b[38;5;241m=\u001b[39m wandb\n",
      "File \u001b[1;32mc:\\Dev\\azure-ai-search-python-playground\\.venv\\Lib\\site-packages\\wandb\\__init__.py:27\u001b[0m\n\u001b[0;32m     24\u001b[0m \u001b[38;5;66;03m# This needs to be early as other modules call it.\u001b[39;00m\n\u001b[0;32m     25\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mwandb\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01merrors\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mterm\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m termsetup, termlog, termerror, termwarn\n\u001b[1;32m---> 27\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mwandb\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m sdk \u001b[38;5;28;01mas\u001b[39;00m wandb_sdk\n\u001b[0;32m     29\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mwandb\u001b[39;00m\n\u001b[0;32m     31\u001b[0m wandb\u001b[38;5;241m.\u001b[39mwandb_lib \u001b[38;5;241m=\u001b[39m wandb_sdk\u001b[38;5;241m.\u001b[39mlib  \u001b[38;5;66;03m# type: ignore\u001b[39;00m\n",
      "File \u001b[1;32mc:\\Dev\\azure-ai-search-python-playground\\.venv\\Lib\\site-packages\\wandb\\sdk\\__init__.py:25\u001b[0m\n\u001b[0;32m      3\u001b[0m __all__ \u001b[38;5;241m=\u001b[39m (\n\u001b[0;32m      4\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mConfig\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m      5\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mSettings\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m     21\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhelper\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m     22\u001b[0m )\n\u001b[0;32m     24\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m wandb_helper \u001b[38;5;28;01mas\u001b[39;00m helper\n\u001b[1;32m---> 25\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01martifacts\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01martifact\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Artifact\n\u001b[0;32m     26\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mwandb_alerts\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m AlertLevel\n\u001b[0;32m     27\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mwandb_config\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Config\n",
      "File \u001b[1;32mc:\\Dev\\azure-ai-search-python-playground\\.venv\\Lib\\site-packages\\wandb\\sdk\\artifacts\\artifact.py:45\u001b[0m\n\u001b[0;32m     42\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mrequests\u001b[39;00m\n\u001b[0;32m     44\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mwandb\u001b[39;00m\n\u001b[1;32m---> 45\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mwandb\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m data_types, env, util\n\u001b[0;32m     46\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mwandb\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mapis\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mnormalize\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m normalize_exceptions\n\u001b[0;32m     47\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mwandb\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mapis\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mpublic\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m ArtifactCollection, ArtifactFiles, RetryingClient, Run\n",
      "File \u001b[1;32mc:\\Dev\\azure-ai-search-python-playground\\.venv\\Lib\\site-packages\\wandb\\data_types.py:32\u001b[0m\n\u001b[0;32m     29\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mwandb\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m util\n\u001b[0;32m     30\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mwandb\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01msdk\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mlib\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m filesystem\n\u001b[1;32m---> 32\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01msdk\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdata_types\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m _dtypes\n\u001b[0;32m     33\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01msdk\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdata_types\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_private\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m MEDIA_TMP\n\u001b[0;32m     34\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01msdk\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdata_types\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mbase_types\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mmedia\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[0;32m     35\u001b[0m     BatchableMedia,\n\u001b[0;32m     36\u001b[0m     Media,\n\u001b[0;32m     37\u001b[0m     _numpy_arrays_to_lists,\n\u001b[0;32m     38\u001b[0m )\n",
      "File \u001b[1;32mc:\\Dev\\azure-ai-search-python-playground\\.venv\\Lib\\site-packages\\wandb\\sdk\\data_types\\_dtypes.py:393\u001b[0m\n\u001b[0;32m    391\u001b[0m NumberType\u001b[38;5;241m.\u001b[39mtypes\u001b[38;5;241m.\u001b[39mappend(np\u001b[38;5;241m.\u001b[39mfloat32)\n\u001b[0;32m    392\u001b[0m NumberType\u001b[38;5;241m.\u001b[39mtypes\u001b[38;5;241m.\u001b[39mappend(np\u001b[38;5;241m.\u001b[39mfloat64)\n\u001b[1;32m--> 393\u001b[0m NumberType\u001b[38;5;241m.\u001b[39mtypes\u001b[38;5;241m.\u001b[39mappend(\u001b[43mnp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfloat_\u001b[49m)\n\u001b[0;32m    394\u001b[0m NumberType\u001b[38;5;241m.\u001b[39mtypes\u001b[38;5;241m.\u001b[39mappend(np\u001b[38;5;241m.\u001b[39mcomplex64)\n\u001b[0;32m    395\u001b[0m NumberType\u001b[38;5;241m.\u001b[39mtypes\u001b[38;5;241m.\u001b[39mappend(np\u001b[38;5;241m.\u001b[39mcomplex128)\n",
      "File \u001b[1;32mc:\\Dev\\azure-ai-search-python-playground\\.venv\\Lib\\site-packages\\numpy\\__init__.py:411\u001b[0m, in \u001b[0;36m__getattr__\u001b[1;34m(attr)\u001b[0m\n\u001b[0;32m    408\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mAttributeError\u001b[39;00m(__former_attrs__[attr])\n\u001b[0;32m    410\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m attr \u001b[38;5;129;01min\u001b[39;00m __expired_attributes__:\n\u001b[1;32m--> 411\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mAttributeError\u001b[39;00m(\n\u001b[0;32m    412\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m`np.\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mattr\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m` was removed in the NumPy 2.0 release. \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    413\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m__expired_attributes__[attr]\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    414\u001b[0m     )\n\u001b[0;32m    416\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m attr \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mchararray\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[0;32m    417\u001b[0m     warnings\u001b[38;5;241m.\u001b[39mwarn(\n\u001b[0;32m    418\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m`np.chararray` is deprecated and will be removed from \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    419\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mthe main namespace in the future. Use an array with a string \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    420\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mor bytes dtype instead.\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;167;01mDeprecationWarning\u001b[39;00m, stacklevel\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m2\u001b[39m)\n",
      "\u001b[1;31mAttributeError\u001b[0m: `np.float_` was removed in the NumPy 2.0 release. Use `np.float64` instead."
     ]
    }
   ],
   "source": [
    "from llama_index.core import set_global_handler\n",
    "set_global_handler(\"wandb\", run_args={\"project\": WANDB_PROJECT})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: weave in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (0.50.13)\n",
      "Requirement already satisfied: typing-extensions>=4.0.0 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from weave) (4.12.2)\n",
      "Requirement already satisfied: pyarrow<17.0.0,>=14.0.1 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from weave) (16.1.0)\n",
      "Requirement already satisfied: openai>=1.0.0 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from weave) (1.37.1)\n",
      "Requirement already satisfied: tiktoken>=0.4.0 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from weave) (0.7.0)\n",
      "Requirement already satisfied: pydantic>=2.0.0 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from weave) (2.8.2)\n",
      "Requirement already satisfied: rich>=13.7.0 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from weave) (13.7.1)\n",
      "Requirement already satisfied: aiohttp>=3.8.3 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from weave) (3.10.0)\n",
      "Requirement already satisfied: aiofiles>=22.1.0 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from weave) (24.1.0)\n",
      "Requirement already satisfied: aioprocessing>=2.0.1 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from weave) (2.0.1)\n",
      "Requirement already satisfied: Werkzeug>=3.0.3 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from weave) (3.0.3)\n",
      "Requirement already satisfied: janus>=1.0.0 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from weave) (1.0.0)\n",
      "Requirement already satisfied: python-json-logger>=2.0.4 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from weave) (2.0.7)\n",
      "Requirement already satisfied: numpy>=1.21 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from weave) (2.0.1)\n",
      "Requirement already satisfied: wandb>=0.16.4 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from weave) (0.16.6)\n",
      "Requirement already satisfied: graphql-core>3 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from weave) (3.2.3)\n",
      "Requirement already satisfied: gql>=3.4.1 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from gql[requests]>=3.4.1->weave) (3.5.0)\n",
      "Collecting numpy>=1.21 (from weave)\n",
      "  Using cached numpy-1.26.4-cp311-cp311-win_amd64.whl.metadata (61 kB)\n",
      "Requirement already satisfied: analytics-python>=1.2.9 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from weave) (1.2.9)\n",
      "Requirement already satisfied: packaging>=21.0 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from weave) (24.1)\n",
      "Requirement already satisfied: tenacity!=8.4.0,>=8.3.0 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from weave) (8.5.0)\n",
      "Requirement already satisfied: emoji>=2.12.1 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from weave) (2.12.1)\n",
      "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from aiohttp>=3.8.3->weave) (2.3.2)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from aiohttp>=3.8.3->weave) (1.3.1)\n",
      "Requirement already satisfied: attrs>=17.3.0 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from aiohttp>=3.8.3->weave) (23.2.0)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from aiohttp>=3.8.3->weave) (1.4.1)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from aiohttp>=3.8.3->weave) (6.0.5)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from aiohttp>=3.8.3->weave) (1.9.4)\n",
      "Requirement already satisfied: requests<3.0,>=2.7 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from analytics-python>=1.2.9->weave) (2.32.3)\n",
      "Requirement already satisfied: six>=1.5 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from analytics-python>=1.2.9->weave) (1.16.0)\n",
      "Requirement already satisfied: python-dateutil>2.1 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from analytics-python>=1.2.9->weave) (2.9.0.post0)\n",
      "Requirement already satisfied: backoff<3.0,>=1.11.1 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from gql>=3.4.1->gql[requests]>=3.4.1->weave) (2.2.1)\n",
      "Requirement already satisfied: anyio<5,>=3.0 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from gql>=3.4.1->gql[requests]>=3.4.1->weave) (4.4.0)\n",
      "Requirement already satisfied: requests-toolbelt<2,>=1.0.0 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from gql[requests]>=3.4.1->weave) (1.0.0)\n",
      "Requirement already satisfied: distro<2,>=1.7.0 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from openai>=1.0.0->weave) (1.9.0)\n",
      "Requirement already satisfied: httpx<1,>=0.23.0 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from openai>=1.0.0->weave) (0.27.0)\n",
      "Requirement already satisfied: sniffio in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from openai>=1.0.0->weave) (1.3.1)\n",
      "Requirement already satisfied: tqdm>4 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from openai>=1.0.0->weave) (4.66.4)\n",
      "Requirement already satisfied: annotated-types>=0.4.0 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from pydantic>=2.0.0->weave) (0.7.0)\n",
      "Requirement already satisfied: pydantic-core==2.20.1 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from pydantic>=2.0.0->weave) (2.20.1)\n",
      "Requirement already satisfied: markdown-it-py>=2.2.0 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from rich>=13.7.0->weave) (3.0.0)\n",
      "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from rich>=13.7.0->weave) (2.18.0)\n",
      "Requirement already satisfied: regex>=2022.1.18 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from tiktoken>=0.4.0->weave) (2024.7.24)\n",
      "Requirement already satisfied: Click!=8.0.0,>=7.1 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from wandb>=0.16.4->weave) (8.1.7)\n",
      "Requirement already satisfied: GitPython!=3.1.29,>=1.0.0 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from wandb>=0.16.4->weave) (3.1.43)\n",
      "Requirement already satisfied: psutil>=5.0.0 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from wandb>=0.16.4->weave) (6.0.0)\n",
      "Requirement already satisfied: sentry-sdk>=1.0.0 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from wandb>=0.16.4->weave) (2.12.0)\n",
      "Requirement already satisfied: docker-pycreds>=0.4.0 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from wandb>=0.16.4->weave) (0.4.0)\n",
      "Requirement already satisfied: PyYAML in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from wandb>=0.16.4->weave) (6.0.1)\n",
      "Requirement already satisfied: setproctitle in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from wandb>=0.16.4->weave) (1.3.3)\n",
      "Requirement already satisfied: setuptools in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from wandb>=0.16.4->weave) (65.5.0)\n",
      "Requirement already satisfied: appdirs>=1.4.3 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from wandb>=0.16.4->weave) (1.4.4)\n",
      "Requirement already satisfied: protobuf!=4.21.0,<5,>=3.19.0 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from wandb>=0.16.4->weave) (4.25.4)\n",
      "Requirement already satisfied: MarkupSafe>=2.1.1 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from Werkzeug>=3.0.3->weave) (2.1.5)\n",
      "Requirement already satisfied: idna>=2.8 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from anyio<5,>=3.0->gql>=3.4.1->gql[requests]>=3.4.1->weave) (3.7)\n",
      "Requirement already satisfied: colorama in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from Click!=8.0.0,>=7.1->wandb>=0.16.4->weave) (0.4.6)\n",
      "Requirement already satisfied: gitdb<5,>=4.0.1 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from GitPython!=3.1.29,>=1.0.0->wandb>=0.16.4->weave) (4.0.11)\n",
      "Requirement already satisfied: certifi in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from httpx<1,>=0.23.0->openai>=1.0.0->weave) (2024.7.4)\n",
      "Requirement already satisfied: httpcore==1.* in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from httpx<1,>=0.23.0->openai>=1.0.0->weave) (1.0.5)\n",
      "Requirement already satisfied: h11<0.15,>=0.13 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from httpcore==1.*->httpx<1,>=0.23.0->openai>=1.0.0->weave) (0.14.0)\n",
      "Requirement already satisfied: mdurl~=0.1 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from markdown-it-py>=2.2.0->rich>=13.7.0->weave) (0.1.2)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from requests<3.0,>=2.7->analytics-python>=1.2.9->weave) (3.3.2)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from requests<3.0,>=2.7->analytics-python>=1.2.9->weave) (2.2.2)\n",
      "Requirement already satisfied: smmap<6,>=3.0.1 in c:\\dev\\azure-ai-search-python-playground\\.venv\\lib\\site-packages (from gitdb<5,>=4.0.1->GitPython!=3.1.29,>=1.0.0->wandb>=0.16.4->weave) (5.0.1)\n",
      "Using cached numpy-1.26.4-cp311-cp311-win_amd64.whl (15.8 MB)\n",
      "Installing collected packages: numpy\n",
      "  Attempting uninstall: numpy\n",
      "    Found existing installation: numpy 2.0.1\n",
      "    Uninstalling numpy-2.0.1:\n",
      "      Successfully uninstalled numpy-2.0.1\n",
      "Successfully installed numpy-1.26.4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "[notice] A new release of pip is available: 24.0 -> 24.2\n",
      "[notice] To update, run: python.exe -m pip install --upgrade pip\n",
      "ERROR: Ignored the following versions that require a different python version: 1.21.2 Requires-Python >=3.7,<3.11; 1.21.3 Requires-Python >=3.7,<3.11; 1.21.4 Requires-Python >=3.7,<3.11; 1.21.5 Requires-Python >=3.7,<3.11; 1.21.6 Requires-Python >=3.7,<3.11\n",
      "ERROR: Could not find a version that satisfies the requirement numpy==1.21.6 (from versions: 1.3.0, 1.4.1, 1.5.0, 1.5.1, 1.6.0, 1.6.1, 1.6.2, 1.7.0, 1.7.1, 1.7.2, 1.8.0, 1.8.1, 1.8.2, 1.9.0, 1.9.1, 1.9.2, 1.9.3, 1.10.0.post2, 1.10.1, 1.10.2, 1.10.4, 1.11.0, 1.11.1, 1.11.2, 1.11.3, 1.12.0, 1.12.1, 1.13.0, 1.13.1, 1.13.3, 1.14.0, 1.14.1, 1.14.2, 1.14.3, 1.14.4, 1.14.5, 1.14.6, 1.15.0, 1.15.1, 1.15.2, 1.15.3, 1.15.4, 1.16.0, 1.16.1, 1.16.2, 1.16.3, 1.16.4, 1.16.5, 1.16.6, 1.17.0, 1.17.1, 1.17.2, 1.17.3, 1.17.4, 1.17.5, 1.18.0, 1.18.1, 1.18.2, 1.18.3, 1.18.4, 1.18.5, 1.19.0, 1.19.1, 1.19.2, 1.19.3, 1.19.4, 1.19.5, 1.20.0, 1.20.1, 1.20.2, 1.20.3, 1.21.0, 1.21.1, 1.22.0, 1.22.1, 1.22.2, 1.22.3, 1.22.4, 1.23.0rc1, 1.23.0rc2, 1.23.0rc3, 1.23.0, 1.23.1, 1.23.2, 1.23.3, 1.23.4, 1.23.5, 1.24.0rc1, 1.24.0rc2, 1.24.0, 1.24.1, 1.24.2, 1.24.3, 1.24.4, 1.25.0rc1, 1.25.0, 1.25.1, 1.25.2, 1.26.0b1, 1.26.0rc1, 1.26.0, 1.26.1, 1.26.2, 1.26.3, 1.26.4, 2.0.0b1, 2.0.0rc1, 2.0.0rc2, 2.0.0, 2.0.1)\n",
      "ERROR: No matching distribution found for numpy==1.21.6\n",
      "\n",
      "[notice] A new release of pip is available: 24.0 -> 24.2\n",
      "[notice] To update, run: python.exe -m pip install --upgrade pip\n"
     ]
    }
   ],
   "source": [
    "! pip install --upgrade weave\n",
    "! pip install numpy==1.21.6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "`np.float_` was removed in the NumPy 2.0 release. Use `np.float64` instead.",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[12], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mweave\u001b[39;00m\n\u001b[0;32m      2\u001b[0m weave\u001b[38;5;241m.\u001b[39minit(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mWANDB_PROJECT\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n",
      "File \u001b[1;32mc:\\Dev\\azure-ai-search-python-playground\\.venv\\Lib\\site-packages\\weave\\__init__.py:14\u001b[0m\n\u001b[0;32m     11\u001b[0m _loading_builtins_token \u001b[38;5;241m=\u001b[39m _context_state\u001b[38;5;241m.\u001b[39mset_loading_built_ins()\n\u001b[0;32m     13\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m weave_types \u001b[38;5;28;01mas\u001b[39;00m types\n\u001b[1;32m---> 14\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m storage\n\u001b[0;32m     16\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mapi\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;241m*\u001b[39m\n\u001b[0;32m     19\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01merrors\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;241m*\u001b[39m\n",
      "File \u001b[1;32mc:\\Dev\\azure-ai-search-python-playground\\.venv\\Lib\\site-packages\\weave\\storage.py:11\u001b[0m\n\u001b[0;32m      8\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mtyping\u001b[39;00m\n\u001b[0;32m     10\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mweave\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m client_context\n\u001b[1;32m---> 11\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mweave\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mlegacy\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[0;32m     12\u001b[0m     artifact_base,\n\u001b[0;32m     13\u001b[0m     artifact_fs,\n\u001b[0;32m     14\u001b[0m     artifact_local,\n\u001b[0;32m     15\u001b[0m     artifact_mem,\n\u001b[0;32m     16\u001b[0m     artifact_wandb,\n\u001b[0;32m     17\u001b[0m     box,\n\u001b[0;32m     18\u001b[0m     graph,\n\u001b[0;32m     19\u001b[0m     mappers_python,\n\u001b[0;32m     20\u001b[0m     timestamp,\n\u001b[0;32m     21\u001b[0m )\n\u001b[0;32m     23\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m errors, ref_base\n\u001b[0;32m     24\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m weave_types \u001b[38;5;28;01mas\u001b[39;00m types\n",
      "File \u001b[1;32mc:\\Dev\\azure-ai-search-python-playground\\.venv\\Lib\\site-packages\\weave\\legacy\\artifact_base.py:11\u001b[0m\n\u001b[0;32m      8\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mtyping\u001b[39;00m\n\u001b[0;32m     10\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mweave\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m weave_types \u001b[38;5;28;01mas\u001b[39;00m types\n\u001b[1;32m---> 11\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mweave\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m ref_base\n\u001b[0;32m     14\u001b[0m \u001b[38;5;28;01mclass\u001b[39;00m \u001b[38;5;21;01mArtifact\u001b[39;00m:\n\u001b[0;32m     15\u001b[0m     \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mset\u001b[39m(\u001b[38;5;28mself\u001b[39m, key: \u001b[38;5;28mstr\u001b[39m, type_: types\u001b[38;5;241m.\u001b[39mType, obj: typing\u001b[38;5;241m.\u001b[39mAny) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mArtifactRef\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n",
      "File \u001b[1;32mc:\\Dev\\azure-ai-search-python-playground\\.venv\\Lib\\site-packages\\weave\\ref_base.py:10\u001b[0m\n\u001b[0;32m      8\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mweave\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m client_context\n\u001b[0;32m      9\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mweave\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mlegacy\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m box, context_state, object_context\n\u001b[1;32m---> 10\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mweave\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mlegacy\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mlanguage_features\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mtagging\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m tag_store\n\u001b[0;32m     12\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m errors\n\u001b[0;32m     13\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m weave_types \u001b[38;5;28;01mas\u001b[39;00m types\n",
      "File \u001b[1;32mc:\\Dev\\azure-ai-search-python-playground\\.venv\\Lib\\site-packages\\weave\\legacy\\language_features\\tagging\\tag_store.py:25\u001b[0m\n\u001b[0;32m     22\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mcollections\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m defaultdict\n\u001b[0;32m     23\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mcontextlib\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m contextmanager\n\u001b[1;32m---> 25\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mweave\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m engine_trace, errors\n\u001b[0;32m     26\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mweave\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m weave_types \u001b[38;5;28;01mas\u001b[39;00m types\n\u001b[0;32m     27\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mweave\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mlegacy\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m box\n",
      "File \u001b[1;32mc:\\Dev\\azure-ai-search-python-playground\\.venv\\Lib\\site-packages\\weave\\engine_trace.py:21\u001b[0m\n\u001b[0;32m     18\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mtime\u001b[39;00m\n\u001b[0;32m     19\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mtyping\u001b[39;00m\n\u001b[1;32m---> 21\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m environment, logs, stream_data_interfaces\n\u001b[0;32m     24\u001b[0m \u001b[38;5;66;03m# Thanks co-pilot!\u001b[39;00m\n\u001b[0;32m     25\u001b[0m \u001b[38;5;28;01mclass\u001b[39;00m \u001b[38;5;21;01mDummySpan\u001b[39;00m:\n",
      "File \u001b[1;32mc:\\Dev\\azure-ai-search-python-playground\\.venv\\Lib\\site-packages\\weave\\stream_data_interfaces.py:11\u001b[0m\n\u001b[0;32m      8\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mtyping\u001b[39;00m\n\u001b[0;32m      9\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01muuid\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m uuid4\n\u001b[1;32m---> 11\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mwandb\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01msdk\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdata_types\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mtrace_tree\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Span \u001b[38;5;28;01mas\u001b[39;00m WBSpan\n\u001b[0;32m     14\u001b[0m \u001b[38;5;66;03m# TraceSpanDict is the lowest common denominator for a span\u001b[39;00m\n\u001b[0;32m     15\u001b[0m \u001b[38;5;66;03m# in the Weave system. It conforms to the OpenTelemetry spec\u001b[39;00m\n\u001b[0;32m     16\u001b[0m \u001b[38;5;28;01mclass\u001b[39;00m \u001b[38;5;21;01mTraceSpanDict\u001b[39;00m(typing\u001b[38;5;241m.\u001b[39mTypedDict):\n\u001b[0;32m     17\u001b[0m     \u001b[38;5;66;03m# **OpenTelemetry spec**\u001b[39;00m\n\u001b[0;32m     18\u001b[0m \n\u001b[0;32m     19\u001b[0m     \u001b[38;5;66;03m# The ID of the span - typically a UUID\u001b[39;00m\n",
      "File \u001b[1;32mc:\\Dev\\azure-ai-search-python-playground\\.venv\\Lib\\site-packages\\wandb\\sdk\\data_types\\trace_tree.py:16\u001b[0m\n\u001b[0;32m     13\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01menum\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Enum\n\u001b[0;32m     14\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtyping\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m TYPE_CHECKING, Any, Dict, List, Optional, Union\n\u001b[1;32m---> 16\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mwandb\u001b[39;00m\n\u001b[0;32m     17\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mwandb\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdata_types\u001b[39;00m\n\u001b[0;32m     18\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mwandb\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01msdk\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdata_types\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m _dtypes\n",
      "File \u001b[1;32mc:\\Dev\\azure-ai-search-python-playground\\.venv\\Lib\\site-packages\\wandb\\__init__.py:27\u001b[0m\n\u001b[0;32m     24\u001b[0m \u001b[38;5;66;03m# This needs to be early as other modules call it.\u001b[39;00m\n\u001b[0;32m     25\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mwandb\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01merrors\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mterm\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m termsetup, termlog, termerror, termwarn\n\u001b[1;32m---> 27\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mwandb\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m sdk \u001b[38;5;28;01mas\u001b[39;00m wandb_sdk\n\u001b[0;32m     29\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mwandb\u001b[39;00m\n\u001b[0;32m     31\u001b[0m wandb\u001b[38;5;241m.\u001b[39mwandb_lib \u001b[38;5;241m=\u001b[39m wandb_sdk\u001b[38;5;241m.\u001b[39mlib  \u001b[38;5;66;03m# type: ignore\u001b[39;00m\n",
      "File \u001b[1;32mc:\\Dev\\azure-ai-search-python-playground\\.venv\\Lib\\site-packages\\wandb\\sdk\\__init__.py:25\u001b[0m\n\u001b[0;32m      3\u001b[0m __all__ \u001b[38;5;241m=\u001b[39m (\n\u001b[0;32m      4\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mConfig\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m      5\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mSettings\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m     21\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhelper\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m     22\u001b[0m )\n\u001b[0;32m     24\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m wandb_helper \u001b[38;5;28;01mas\u001b[39;00m helper\n\u001b[1;32m---> 25\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01martifacts\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01martifact\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Artifact\n\u001b[0;32m     26\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mwandb_alerts\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m AlertLevel\n\u001b[0;32m     27\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mwandb_config\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Config\n",
      "File \u001b[1;32mc:\\Dev\\azure-ai-search-python-playground\\.venv\\Lib\\site-packages\\wandb\\sdk\\artifacts\\artifact.py:45\u001b[0m\n\u001b[0;32m     42\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mrequests\u001b[39;00m\n\u001b[0;32m     44\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mwandb\u001b[39;00m\n\u001b[1;32m---> 45\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mwandb\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m data_types, env, util\n\u001b[0;32m     46\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mwandb\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mapis\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mnormalize\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m normalize_exceptions\n\u001b[0;32m     47\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mwandb\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mapis\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mpublic\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m ArtifactCollection, ArtifactFiles, RetryingClient, Run\n",
      "File \u001b[1;32mc:\\Dev\\azure-ai-search-python-playground\\.venv\\Lib\\site-packages\\wandb\\data_types.py:32\u001b[0m\n\u001b[0;32m     29\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mwandb\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m util\n\u001b[0;32m     30\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mwandb\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01msdk\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mlib\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m filesystem\n\u001b[1;32m---> 32\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01msdk\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdata_types\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m _dtypes\n\u001b[0;32m     33\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01msdk\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdata_types\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_private\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m MEDIA_TMP\n\u001b[0;32m     34\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01msdk\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdata_types\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mbase_types\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mmedia\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[0;32m     35\u001b[0m     BatchableMedia,\n\u001b[0;32m     36\u001b[0m     Media,\n\u001b[0;32m     37\u001b[0m     _numpy_arrays_to_lists,\n\u001b[0;32m     38\u001b[0m )\n",
      "File \u001b[1;32mc:\\Dev\\azure-ai-search-python-playground\\.venv\\Lib\\site-packages\\wandb\\sdk\\data_types\\_dtypes.py:393\u001b[0m\n\u001b[0;32m    391\u001b[0m NumberType\u001b[38;5;241m.\u001b[39mtypes\u001b[38;5;241m.\u001b[39mappend(np\u001b[38;5;241m.\u001b[39mfloat32)\n\u001b[0;32m    392\u001b[0m NumberType\u001b[38;5;241m.\u001b[39mtypes\u001b[38;5;241m.\u001b[39mappend(np\u001b[38;5;241m.\u001b[39mfloat64)\n\u001b[1;32m--> 393\u001b[0m NumberType\u001b[38;5;241m.\u001b[39mtypes\u001b[38;5;241m.\u001b[39mappend(\u001b[43mnp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfloat_\u001b[49m)\n\u001b[0;32m    394\u001b[0m NumberType\u001b[38;5;241m.\u001b[39mtypes\u001b[38;5;241m.\u001b[39mappend(np\u001b[38;5;241m.\u001b[39mcomplex64)\n\u001b[0;32m    395\u001b[0m NumberType\u001b[38;5;241m.\u001b[39mtypes\u001b[38;5;241m.\u001b[39mappend(np\u001b[38;5;241m.\u001b[39mcomplex128)\n",
      "File \u001b[1;32mc:\\Dev\\azure-ai-search-python-playground\\.venv\\Lib\\site-packages\\numpy\\__init__.py:411\u001b[0m, in \u001b[0;36m__getattr__\u001b[1;34m(attr)\u001b[0m\n\u001b[0;32m    408\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mAttributeError\u001b[39;00m(__former_attrs__[attr])\n\u001b[0;32m    410\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m attr \u001b[38;5;129;01min\u001b[39;00m __expired_attributes__:\n\u001b[1;32m--> 411\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mAttributeError\u001b[39;00m(\n\u001b[0;32m    412\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m`np.\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mattr\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m` was removed in the NumPy 2.0 release. \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    413\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m__expired_attributes__[attr]\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    414\u001b[0m     )\n\u001b[0;32m    416\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m attr \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mchararray\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[0;32m    417\u001b[0m     warnings\u001b[38;5;241m.\u001b[39mwarn(\n\u001b[0;32m    418\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m`np.chararray` is deprecated and will be removed from \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    419\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mthe main namespace in the future. Use an array with a string \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    420\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mor bytes dtype instead.\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;167;01mDeprecationWarning\u001b[39;00m, stacklevel\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m2\u001b[39m)\n",
      "\u001b[1;31mAttributeError\u001b[0m: `np.float_` was removed in the NumPy 2.0 release. Use `np.float64` instead."
     ]
    }
   ],
   "source": [
    "import weave\n",
    "weave.init(f\"{WANDB_PROJECT}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "unexpected character after line continuation character (903547445.py, line 1)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;36m  Cell \u001b[1;32mIn[43], line 1\u001b[1;36m\u001b[0m\n\u001b[1;33m    dir %USERPROFILE%\\.netrc\u001b[0m\n\u001b[1;37m                      ^\u001b[0m\n\u001b[1;31mSyntaxError\u001b[0m\u001b[1;31m:\u001b[0m unexpected character after line continuation character\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Vector Store Initialization\n",
    "Set up the vector store using Azure AI Search."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.core.settings import Settings\n",
    "\n",
    "Settings.llm = llm\n",
    "Settings.embed_model = embed_model\n",
    "\n",
    "# Initialize the vector store\n",
    "vector_store = AzureAISearchVectorStore(\n",
    "    search_or_index_client=index_client,\n",
    "    index_name=INDEX_NAME,\n",
    "    index_management=IndexManagement.VALIDATE_INDEX,\n",
    "    id_field_key=\"id\",\n",
    "    chunk_field_key=\"text\",\n",
    "    embedding_field_key=\"embedding\",\n",
    "    embedding_dimensionality=1536,\n",
    "    metadata_string_field_key=\"metadata\",\n",
    "    doc_id_field_key=\"doc_id\",\n",
    "    language_analyzer=\"en.lucene\",\n",
    "    vector_algorithm_type=\"exhaustiveKnn\",\n",
    ")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Use Existing Index\n",
    "I'm going to use my existing \"contoso-hr-docs\" index that I created. For how to create an index and load documents from scratch, see [here](https://github.com/farzad528/azure-ai-search-python-playground/blob/addb1a29e70ee9dbf1bb9a39bbe367aa15e4cf5f/azure-ai-search-rag-eval-trulens.ipynb#L145)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: Logged trace tree to W&B.\n"
     ]
    }
   ],
   "source": [
    "storage_context = StorageContext.from_defaults(vector_store=vector_store)\n",
    "index = VectorStoreIndex.from_documents(\n",
    "    [],\n",
    "    storage_context=storage_context,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: Logged trace tree to W&B.\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "**`Final Response:`** It is important to review the plan's evidence of coverage to determine if scuba diving is covered under the health plan. Additionally, discussing this with the healthcare provider and reviewing the list of excluded services and prescriptions is recommended to ensure coverage for scuba diving. If scuba diving is not covered under the plan, exploring alternative coverage options or discussing payment options with the healthcare provider may be necessary."
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "page_label: 90\n",
      "file_path: c:\\Dev\\azure-ai-search-python-playground\\data\\pdf\\Northwind_Health_Plus_Benefits_Details.pdf\n",
      "\n",
      "benefits for mental health and \n",
      "substance abuse services as it does for medical and surgical benefits. This includes covering \n",
      "services that are medically necessary, suc h as inpatient and outpatient services, medication \n",
      "management, and psychological and psychosocial therapies.  \n",
      "It is important to note that the plan may not provide coverage or impose any limits or \n",
      "exclusions that are not in compliance with applicable laws a nd regulations. Additionally, the \n",
      "plan may not discriminate against individuals based on their medical condition or health \n",
      "status. Individuals who feel they have been discriminated against should contact the \n",
      "Department of Labor, who can investigate the iss ue. \n",
      "Finally, it is important to note that the plan may not provide coverage or impose any limits \n",
      "or exclusions that are not in compliance with applicable laws and regulations. Additionally, \n",
      "the plan may not discriminate against individuals based on their m edical condition or \n",
      "health status. Individuals who feel they have been discriminated against should contact the \n",
      "Department of Labor, who can investigate the issue.\n",
      "page_label: 75\n",
      "file_path: c:\\Dev\\azure-ai-search-python-playground\\data\\pdf\\Northwind_Health_Plus_Benefits_Details.pdf\n",
      "\n",
      "Health Plus does not cover preventive care services provided \n",
      "by a non -network  provider.  \n",
      "Tips for Avoiding Exclusions  \n",
      "When considering a medical service or treatment, it is important to review the plan's \n",
      "evidence of coverage to ensure that the service or treatment is covered under the plan. You \n",
      "should also discuss the service or treatment with your doctor to ensure that i t is medically \n",
      "necessary. Additionally, you should review the list of excluded services and prescriptions to \n",
      "ensure that you are not seeking treatment for an excluded service or prescription.  \n",
      "If you are considering a medical service or treatment that is n ot covered under the plan, you \n",
      "should discuss payment options with your doctor or healthcare provider. Additionally, you \n",
      "may need to consider other payment sources, such as private insurance, flexible spending \n",
      "accounts, or state or federal programs.\n",
      "page_label: 95\n",
      "file_path: c:\\Dev\\azure-ai-search-python-playground\\data\\pdf\\Northwind_Standard_Benefits_Details.pdf\n",
      "\n",
      "When you enroll in the Northwind Standard plan, you may be eligible to continue coverage \n",
      "under other health plans. This could include coverage from your spouse’s or a parent’s \n",
      "employer, or from a government -sponsored program such as Medica re or Medicaid.  \n",
      "If you and your spouse have coverage under different plans, you must determine which plan \n",
      "is the primary plan and which is secondary. This is important, as the primary plan will pay \n",
      "first; the secondary plan will pay what the primary plan does not.  \n",
      "If you have other coverage, you must notify Northwind Health of any changes in your \n",
      "coverage status or any changes in any of your other coverage. If you fail to do so, you may be \n",
      "responsible for any charges that Northwind Health would have paid i f you had notified \n",
      "them of the other coverage.  \n",
      "In certain circumstances, you may be able to keep your other coverage and still be eligible \n",
      "for coverage under the Northwind Standard plan. However, if the other coverage is primary, \n",
      "you will be responsible fo r any charges that would have been paid by the Northwind \n",
      "Standard plan.  \n",
      "It is also important to note that if you have coverage through a government -sponsored \n",
      "program such as Medicare or Medicaid, you may be subject to certain restrictions. For \n",
      "example, you  may be required to obtain certain services through the government -\n",
      "sponsored plan.  \n",
      "Tips for Employees  \n",
      "To ensure you get the most out of your Northwind Standard coverage, here are some tips:  \n",
      "• Make sure you understand the terms and conditions of your other coverage and any \n",
      "restrictions associated with it.  \n",
      "• Know which plan is primary and which is secondary.  \n",
      "• Notify Northwind Health of any changes in your coverage status or any changes in any of \n",
      "your other coverage.  \n",
      "• Understand any restrictions associated w ith any government -sponsored programs you \n",
      "may be enrolled in.  \n",
      "• Your Northwind Standard plan does not cover certain services, such as emergency care, \n",
      "mental health and substance abuse coverage, or out -of-network services. Be sure to explore \n",
      "alternative co verage options if you need coverage for these services.  \n",
      "• Take advantage of preventive care services and prescription drug coverage available \n",
      "through your Northwind Standard plan.  \n",
      "•\n"
     ]
    }
   ],
   "source": [
    "# Query execution\n",
    "from llama_index.core.schema import MetadataMode\n",
    "from llama_index.core.response.notebook_utils import display_response\n",
    "query = \"Does my health plan cover scuba diving?\"\n",
    "query_engine = index.as_query_engine(llm, similarity_top_k=3)\n",
    "response = query_engine.query(query)\n",
    "\n",
    "# Print the response\n",
    "display_response(response)\n",
    "print(\"\\n\")\n",
    "\n",
    "# Print what the LLM sees\n",
    "for node in response.source_nodes:\n",
    "    print(node.get_content(metadata_mode=MetadataMode.LLM))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Above you can now visualize and see tracers in action. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evalaute RAG with Weights & Biases"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using `DatasetGenerator` lets run some evaluations using a synthetic test dataset!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Let's use GPT 3.5 as our LLM of choice.\n",
    "service_context = ServiceContext.from_defaults(llm=llm, callback_manager=callback_manager)\n",
    "\n",
    "\n",
    "# Let's just use a meaningful subset of the shuffled documents.\n",
    "random_documents = copy.deepcopy(documents)\n",
    "random.shuffle(random_documents)\n",
    "random_documents = random_documents[:10]\n",
    "\n",
    "\n",
    "# Let's reduce the number of questions per chunk.\n",
    "data_generator = DatasetGenerator.from_documents(\n",
    "    random_documents, service_context=service_context, num_questions_per_chunk=2\n",
    ")\n",
    "# Let's reduce the number of questions per chunk from 10 to 2.\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}