azure-ai-search-private-equity-due-dilligence-agent.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install llama-index llama-parse pydantic nest_asyncio\n",
    "!pip install llama-index-vector-stores-azureaisearch\n",
    "!pip install azure-search-documents \n",
    "!pip install llama-index-embeddings-azure-openai\n",
    "!pip install llama-index-llms-azure-openai\n",
    "!pip install llama-index-readers-docling llama-index-node-parser-docling\n",
    "!pip install llama-index-readers-file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "import time\n",
    "import logging\n",
    "from copy import deepcopy\n",
    "from dotenv import load_dotenv\n",
    "import nest_asyncio\n",
    "from typing import Optional\n",
    "\n",
    "from azure.core.credentials import AzureKeyCredential\n",
    "from azure.search.documents import SearchClient\n",
    "from azure.search.documents.indexes import SearchIndexClient\n",
    "from llama_index.core.node_parser import TokenTextSplitter\n",
    "from llama_index.core import StorageContext, VectorStoreIndex\n",
    "from llama_index.core import SimpleDirectoryReader, StorageContext, VectorStoreIndex\n",
    "from llama_index.core.schema import TextNode, ImageNode, NodeWithScore, MetadataMode\n",
    "from llama_index.core.settings import Settings\n",
    "from llama_index.core.query_engine import CustomQueryEngine, SimpleMultiModalQueryEngine\n",
    "from llama_index.core.retrievers import BaseRetriever\n",
    "from llama_index.core.prompts import PromptTemplate\n",
    "from llama_index.core.base.response.schema import Response\n",
    "from llama_index.embeddings.azure_openai import AzureOpenAIEmbedding\n",
    "from llama_index.llms.azure_openai import AzureOpenAI\n",
    "from llama_index.vector_stores.azureaisearch import AzureAISearchVectorStore, IndexManagement, MetadataIndexFieldType\n",
    "from llama_parse import LlamaParse\n",
    "from llama_index.core.node_parser import MarkdownNodeParser\n",
    "import os\n",
    "import json\n",
    "import time\n",
    "import logging\n",
    "from copy import deepcopy\n",
    "from dotenv import load_dotenv\n",
    "import nest_asyncio\n",
    "from typing import Optional, List\n",
    "\n",
    "from azure.core.credentials import AzureKeyCredential\n",
    "from azure.search.documents import SearchClient\n",
    "from azure.search.documents.indexes import SearchIndexClient\n",
    "\n",
    "from llama_index.core import StorageContext, VectorStoreIndex\n",
    "from llama_index.core.schema import TextNode, ImageNode, NodeWithScore, MetadataMode\n",
    "from llama_index.core.settings import Settings\n",
    "from llama_index.core.workflow import Workflow, Event, StartEvent, StopEvent, step, Context\n",
    "from llama_index.core.prompts import ChatPromptTemplate\n",
    "from llama_index.embeddings.azure_openai import AzureOpenAIEmbedding\n",
    "from llama_index.llms.azure_openai import AzureOpenAI\n",
    "from llama_index.vector_stores.azureaisearch import AzureAISearchVectorStore, IndexManagement, MetadataIndexFieldType\n",
    "from llama_parse import LlamaParse\n",
    "from pydantic import BaseModel, Field"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load environment variables\n",
    "load_dotenv()\n",
    "\n",
    "# Azure OpenAI and Azure Search configuration\n",
    "AZURE_OPENAI_ENDPOINT = os.getenv(\"AZURE_OPENAI_ENDPOINT\")\n",
    "AZURE_OPENAI_API_KEY = os.getenv(\"AZURE_OPENAI_API_KEY\")\n",
    "AZURE_OPENAI_CHAT_COMPLETION_DEPLOYED_MODEL_NAME = os.getenv(\"AZURE_OPENAI_CHAT_COMPLETION_DEPLOYED_MODEL_NAME\")\n",
    "AZURE_OPENAI_EMBEDDING_DEPLOYED_MODEL_NAME = os.getenv(\"AZURE_OPENAI_EMBEDDING_DEPLOYED_MODEL_NAME\")\n",
    "SEARCH_SERVICE_ENDPOINT = os.getenv(\"AZURE_SEARCH_SERVICE_ENDPOINT\")\n",
    "SEARCH_SERVICE_API_KEY = os.getenv(\"AZURE_SEARCH_ADMIN_KEY\")\n",
    "INDEX_NAME = \"private-equity-due-diligence\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize Azure OpenAI models\n",
    "llm = AzureOpenAI(\n",
    "    model=AZURE_OPENAI_CHAT_COMPLETION_DEPLOYED_MODEL_NAME,\n",
    "    deployment_name=AZURE_OPENAI_CHAT_COMPLETION_DEPLOYED_MODEL_NAME,\n",
    "    api_key=AZURE_OPENAI_API_KEY,\n",
    "    azure_endpoint=AZURE_OPENAI_ENDPOINT,\n",
    "    api_version=\"2024-10-21\"\n",
    ")\n",
    "\n",
    "embed_model = AzureOpenAIEmbedding(\n",
    "    model=AZURE_OPENAI_EMBEDDING_DEPLOYED_MODEL_NAME,\n",
    "    deployment_name=AZURE_OPENAI_EMBEDDING_DEPLOYED_MODEL_NAME,\n",
    "    api_key=AZURE_OPENAI_API_KEY,\n",
    "    azure_endpoint=AZURE_OPENAI_ENDPOINT,\n",
    "    api_version=\"2024-10-21\"\n",
    ")\n",
    "\n",
    "Settings.llm = llm\n",
    "Settings.embed_model = embed_model\n",
    "\n",
    "# Initialize Azure Search clients\n",
    "credential = AzureKeyCredential(SEARCH_SERVICE_API_KEY)\n",
    "index_client = SearchIndexClient(endpoint=SEARCH_SERVICE_ENDPOINT, credential=credential)\n",
    "search_client = SearchClient(endpoint=SEARCH_SERVICE_ENDPOINT, index_name=INDEX_NAME, credential=credential)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create Azure AI Search Vector Store\n",
    "metadata_fields = {\n",
    "    # Define any custom metadata fields for filtering if needed\n",
    "}\n",
    "\n",
    "vector_store = AzureAISearchVectorStore(\n",
    "    search_or_index_client=index_client,\n",
    "    filterable_metadata_field_keys=metadata_fields,\n",
    "    index_name=INDEX_NAME,\n",
    "    index_management=IndexManagement.CREATE_IF_NOT_EXISTS,\n",
    "    id_field_key=\"id\",\n",
    "    chunk_field_key=\"chunk\",\n",
    "    embedding_field_key=\"embedding\",\n",
    "    embedding_dimensionality=1536,  # Update based on embedding model\n",
    "    metadata_string_field_key=\"metadata\",\n",
    "    doc_id_field_key=\"doc_id\",\n",
    "    language_analyzer=\"en.lucene\",\n",
    "    vector_algorithm_type=\"exhaustiveKnn\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Started parsing the file under job_id 5c5f746d-943d-4821-8ac3-f9a4f373ad80\n",
      "Started parsing the file under job_id 92b60da5-95a5-4c6e-83e4-364ebad254f7\n",
      "Started parsing the file under job_id 449af97a-dd76-4d90-a16e-d2d9f6237de5\n",
      "Started parsing the file under job_id 658c2de2-1a71-41b2-bce2-1dfe902b3442\n",
      "Started parsing the file under job_id f449284b-971d-474d-a01b-ea287aa7d4c7\n",
      "Started parsing the file under job_id b68b3cd9-5b41-4221-abf7-c33f43b27a6d\n",
      "Number of documents parsed: 464\n"
     ]
    }
   ],
   "source": [
    "# Use LlamaParse for ingestion\n",
    "parser = LlamaParse(\n",
    "    result_type=\"markdown\",\n",
    "    auto_mode=True,\n",
    "    auto_mode_trigger_on_image_in_page=True,\n",
    "    auto_mode_trigger_on_table_in_page=True\n",
    ")\n",
    "\n",
    "# Parse documents from directory\n",
    "file_extractor = {\".pdf\": parser}\n",
    "documents = SimpleDirectoryReader(\n",
    "    input_dir=\"data/portfolio-companies\", \n",
    "    file_extractor=file_extractor, \n",
    ").load_data()\n",
    "\n",
    "print(f\"Number of documents parsed: {len(documents)}\")\n",
    "\n",
    "# Apply token splitting transformation\n",
    "text_splitter = TokenTextSplitter(chunk_size=512, chunk_overlap=50)\n",
    "\n",
    "# Ingest split documents into Azure AI Search\n",
    "storage_context = StorageContext.from_defaults(vector_store=vector_store)\n",
    "index = VectorStoreIndex.from_documents(\n",
    "    documents=documents,\n",
    "    storage_context=storage_context,\n",
    "    embed_model=embed_model,\n",
    "    transformations=[text_splitter],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Q: Summarize the key risks identified in AMD Q3 report.\n",
      "A: The key risks identified in the AMD Q3 report include:\n",
      "\n",
      "1. **Economic and Strategic Risks**:\n",
      "   - Competition from dominant players like Intel and Nvidia may hinder AMD's ability to compete effectively.\n",
      "   - The semiconductor industry is cyclical and can experience severe downturns.\n",
      "   - Product demand is influenced by market conditions in various industries.\n",
      "   - Timely introduction of products with valuable features is crucial for success.\n",
      "   - Loss of significant customers could adversely impact the business.\n",
      "   - Economic uncertainty may negatively affect operations and results.\n",
      "   - Quarterly and seasonal sales patterns can influence operating results.\n",
      "   - Inadequate protection of intellectual property could lead to competitive disadvantages.\n",
      "   - Currency exchange rate fluctuations may have adverse effects.\n",
      "\n",
      "2. **Operational and Technology Risks**:\n",
      "   - Dependence on third-party manufacturers poses risks if they cannot deliver products timely or with competitive technology.\n",
      "   - Availability of essential manufacturing resources may be uncertain.\n",
      "   - Manufacturing yields may not meet expectations.\n",
      "\n",
      "3. **Cybersecurity Risks**:\n",
      "   - Vulnerabilities related to security may lead to litigation and damage to reputation.\n",
      "   - IT outages, data breaches, and cyberattacks could disrupt operations and compromise sensitive information.\n",
      "   - The reliance on both owned and third-party IT systems increases exposure to cybersecurity threats.\n",
      "   - Increased remote working arrangements present additional risks to IT systems.\n",
      "   - Cyber threats can come from various sources, including sophisticated attackers and social engineering tactics.\n",
      "\n",
      "Sources:\n",
      "# PART II. OTHER INFORMATION\n",
      "\n",
      "# ITEM 1. LEGAL PROCEEDINGS\n",
      "\n",
      "For a discussion of our legal proceedings, refer to Note 13—Commitments and Contingencies of the Notes to Condensed Consolidated Financial Statements (Part I, Item 1 of this Form 10-Q).\n",
      "\n",
      "# ITEM 1A. RISK FACTORS\n",
      "\n",
      "The risks and uncertainties described below are not the only ones we face. If any of the following risks actually occurs, our business, financial condition or results of operations could be materially adversely affected. In addition, you should consider the interrelationship and compounding effects of two or more risks occurring simultaneously.\n",
      "\n",
      "# Risk Factors Summary\n",
      "\n",
      "The following is a summary of the principal risks that could adversely affect our business, financial condition and results of operations.\n",
      "\n",
      "# Economic and Strategic Risks\n",
      "\n",
      "- Intel Corporation’s dominance of the microprocessor market and its aggressive business practices may limit our ability to compete effectively on a level playing field.\n",
      "- Nvidia’s dominance in the graphics processing unit market and its aggressive business practices may limit our ability to compete effectively on a level playing field.\n",
      "- The markets in which our products are sold are highly competitive and rapidly evolving.\n",
      "- The semiconductor industry is highly cyclical and has experienced severe downturns.\n",
      "- The demand for our products depends in part on the market conditions in the industries into which they are sold.\n",
      "- The success of our business depends on our ability to introduce products on a timely basis with features and performance levels that provide value to our customers while supporting significant industry transitions.\n",
      "- The loss of a significant customer may have a material adverse effect on us.\n",
      "- Economic and market uncertainty may adversely impact our business and operating results.\n",
      "- Our operating results are subject to quarterly and seasonal sales patterns.\n",
      "- If we cannot adequately protect our technology or other intellectual property through patents, copyrights, trade secrets, trademarks and other measures, we may lose a competitive advantage and incur significant expenses.\n",
      "- Unfavorable currency exchange rate fluctuations could adversely affect us.\n",
      "\n",
      "# Operational and Technology Risks\n",
      "\n",
      "- We rely on third parties to manufacture our products, and if they are unable to do so on a timely basis in sufficient quantities and using competitive technologies, our business could be materially adversely affected.\n",
      "- Essential equipment, materials, substrates or manufacturing processes may not be available to us.\n",
      "- We may fail to achieve expected manufacturing yields for our products.\n",
      "- {'file_path': 'c:\\\\Dev\\\\azure-ai-search-python-playground\\\\data\\\\portfolio-companies\\\\AMD Q3 2024 10-Q.pdf', 'file_name': 'AMD Q3 2024 10-Q.pdf', 'file_type': 'application/pdf', 'file_size': 2702839, 'creation_date': '2024-12-24', 'last_modified_date': '2024-12-24'}\n",
      "could adversely affect our revenue. We have and may in the future be subject to claims and litigation related to security vulnerabilities. Actual or perceived security vulnerabilities of our products may subject us to adverse publicity, damage to our brand and reputation, and could materially harm our business or results of operations.\n",
      "\n",
      "IT outages, data loss, data breaches and cyberattacks could disrupt operations and compromise our intellectual property or other sensitive information, be costly to remediate or cause significant damage to our business, reputation, financial condition and results of operations.\n",
      "\n",
      "Our business relies on technology hardware, software, cloud services, infrastructure, networks and systems (collectively, IT Systems). We own and manage some IT Systems but also rely on critical third-party IT Systems, products and services. In the ordinary course of business, we and various third-party providers and business partners process and maintain sensitive data, including personal information about workers, customers and others, as well as intellectual property and proprietary or confidential information relating to our business and that of our customers and business partners (collectively, Confidential Data). Maintaining the availability, integrity and security of our IT Systems and Confidential Data is critical to our business and reputation. While we and others have implemented various controls and defenses, AMD and companies like AMD and our vendors and customers have been and are increasingly subject to cybersecurity attacks, risks and threats. Threat actors range in sophistication from individual hackers and insiders to ransom gangs and state-sponsored attackers. Cyber threats may be generic, or they may be custom-crafted against our IT Systems or supply chain. The increased prevalence of remote working arrangements at AMD and our providers present additional operational risks and attack vectors to our IT Systems. Our IT Systems and Confidential Data are vulnerable to a range of cybersecurity risks and threats, including malicious code that is added to widely available open-source software, compromised commercial software or security vulnerabilities in our products or systems, or those of a third party, that are being used by attackers prior to mitigations being put in place, such as zero-day attacks. Cyberattacks have and may come into our IT Systems through the compromise of users’ access credentials. Users’ access credentials can be compromised by phishing, vishing, smishing, multi-factor authentication (MFA) prompt bombing, hacking, or other social engineering, cybersecurity, or theft activities. {'file_path': 'c:\\\\Dev\\\\azure-ai-search-python-playground\\\\data\\\\portfolio-companies\\\\AMD Q3 2024 10-Q.pdf', 'file_name': 'AMD Q3 2024 10-Q.pdf', 'file_type': 'application/pdf', 'file_size': 2702839, 'creation_date': '2024-12-24', 'last_modified_date': '2024-12-24'}\n"
     ]
    }
   ],
   "source": [
    "# Query the index to validate ingestion\n",
    "QUERY = \"Summarize the key risks identified in AMD Q3 report.\"\n",
    "result = index.as_query_engine(llm=llm).query(QUERY)\n",
    "\n",
    "# Display results\n",
    "print(f\"Q: {QUERY}\\nA: {result.response.strip()}\\n\\nSources:\")\n",
    "for n in result.source_nodes:\n",
    "    print(n.text, n.metadata)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}