From e4d98ac89fdd55e33c145d155c7168bb10dc7baa Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Fri, 28 Mar 2025 09:47:48 -0700 Subject: [PATCH 01/11] Initial changes for text-embedding-3 --- app/backend/app.py | 4 +-- docs/deploy_existing.md | 4 +-- docs/deploy_features.md | 61 +++++++++++++++++++++++++++++-------- docs/gpt4v.md | 4 +-- infra/main.bicep | 10 +++--- tests/conftest.py | 2 +- tests/test_prepdocs.py | 2 +- tests/test_searchmanager.py | 2 +- tests/test_upload.py | 2 +- 9 files changed, 64 insertions(+), 27 deletions(-) diff --git a/app/backend/app.py b/app/backend/app.py index b5e4084f76..65615185ca 100644 --- a/app/backend/app.py +++ b/app/backend/app.py @@ -421,8 +421,8 @@ async def setup_clients(): # Shared by all OpenAI deployments OPENAI_HOST = os.getenv("OPENAI_HOST", "azure") OPENAI_CHATGPT_MODEL = os.environ["AZURE_OPENAI_CHATGPT_MODEL"] - OPENAI_EMB_MODEL = os.getenv("AZURE_OPENAI_EMB_MODEL_NAME", "text-embedding-ada-002") - OPENAI_EMB_DIMENSIONS = int(os.getenv("AZURE_OPENAI_EMB_DIMENSIONS") or 1536) + OPENAI_EMB_MODEL = os.environ["AZURE_OPENAI_EMB_MODEL_NAME"] + OPENAI_EMB_DIMENSIONS = int(os.environ["AZURE_OPENAI_EMB_DIMENSIONS"]) # Used with Azure OpenAI deployments AZURE_OPENAI_SERVICE = os.getenv("AZURE_OPENAI_SERVICE") AZURE_OPENAI_GPT4V_DEPLOYMENT = os.environ.get("AZURE_OPENAI_GPT4V_DEPLOYMENT") diff --git a/docs/deploy_existing.md b/docs/deploy_existing.md index 4275dcf2f6..d0f64ecc1e 100644 --- a/docs/deploy_existing.md +++ b/docs/deploy_existing.md @@ -31,8 +31,8 @@ You should set these values before running `azd up`. Once you've set them, retur 1. Run `azd env set AZURE_OPENAI_CHATGPT_DEPLOYMENT_VERSION {Version string for existing chat deployment}`. Only needed if your chat deployment model version is not the default '0125'. You definitely need to change this if you changed the model. 1. Run `azd env set AZURE_OPENAI_CHATGPT_DEPLOYMENT_SKU {Name of SKU for existing chat deployment}`. Only needed if your chat deployment SKU is not the default 'Standard', like if it is 'GlobalStandard' instead. 1. Run `azd env set AZURE_OPENAI_EMB_DEPLOYMENT {Name of existing embedding deployment}`. Only needed if your embeddings deployment is not the default 'embedding'. -1. Run `azd env set AZURE_OPENAI_EMB_MODEL_NAME {Model name of existing embedding deployment}`. Only needed if your embeddings model is not the default 'text-embedding-ada-002'. -1. Run `azd env set AZURE_OPENAI_EMB_DIMENSIONS {Dimensions for existing embedding deployment}`. Only needed if your embeddings model is not the default 'text-embedding-ada-002'. +1. Run `azd env set AZURE_OPENAI_EMB_MODEL_NAME {Model name of existing embedding deployment}`. Only needed if your embeddings model is not the default 'text-embedding-3-large'. +1. Run `azd env set AZURE_OPENAI_EMB_DIMENSIONS {Dimensions for existing embedding deployment}`. Only needed if your embeddings model is not the default 'text-embedding-3-large'. 1. Run `azd env set AZURE_OPENAI_EMB_DEPLOYMENT_VERSION {Version string for existing embedding deployment}`. If your embeddings deployment is one of the 'text-embedding-3' models, set this to the number 1. 1. This project does *not* use keys when authenticating to Azure OpenAI. However, if your Azure OpenAI service must have key access enabled for some reason (like for use by other projects), then run `azd env set AZURE_OPENAI_DISABLE_KEYS false`. The default value is `true` so you should only run the command if you need key access. diff --git a/docs/deploy_features.md b/docs/deploy_features.md index c1390b05ec..5c2b2fd7a5 100644 --- a/docs/deploy_features.md +++ b/docs/deploy_features.md @@ -5,8 +5,8 @@ This document covers optional features that can be enabled in the deployed Azure You should typically enable these features before running `azd up`. Once you've set them, return to the [deployment steps](../README.md#deploying). * [Using different chat completion models](#using-different-chat-completion-models) -* [Using text-embedding-3 models](#using-text-embedding-3-models) -* [Enabling GPT-4 Turbo with Vision](#enabling-gpt-4-turbo-with-vision) +* [Using different embedding models](#using-different-embedding-models) +* [Enabling GPT vision feature](#enabling-gpt-vision-feature) * [Enabling media description with Azure Content Understanding](#enabling-media-description-with-azure-content-understanding) * [Enabling client-side chat history](#enabling-client-side-chat-history) * [Enabling persistent chat history with Azure Cosmos DB](#enabling-persistent-chat-history-with-azure-cosmos-db) @@ -121,12 +121,16 @@ This process does *not* delete your previous model deployment. If you want to de > [!NOTE] > To revert back to a previous model, run the same commands with the previous model name and version. -## Using text-embedding-3 models +## Using different embedding models -By default, the deployed Azure web app uses the `text-embedding-ada-002` embedding model. If you want to use one of the text-embedding-3 models, you can do so by following these steps: +By default, the deployed Azure web app uses the `text-embedding-3-large` embedding model. If you want to use a different embeddig model, you can do so by following these steps: 1. Run one of the following commands to set the desired model: + ```shell + azd env set AZURE_OPENAI_EMB_MODEL_NAME text-embedding-ada-002 + ``` + ```shell azd env set AZURE_OPENAI_EMB_MODEL_NAME text-embedding-3-small ``` @@ -137,17 +141,53 @@ By default, the deployed Azure web app uses the `text-embedding-ada-002` embeddi 2. Specify the desired dimensions of the model: (from 256-3072, model dependent) + Default dimensions for text-embedding-ada-002 + + ```shell + azd env set AZURE_OPENAI_EMB_DIMENSIONS 1536 + ``` + + Default dimensions for text-embedding-3-small + ```shell - azd env set AZURE_OPENAI_EMB_DIMENSIONS 256 + azd env set AZURE_OPENAI_EMB_DIMENSIONS 1536 ``` -3. Set the model version to "1" (the only version as of March 2024): + Default dimensions for text-embedding-3-large + + ```shell + azd env set AZURE_OPENAI_EMB_DIMENSIONS 3072 + ``` + +3. Set the model version, depending on the model you are using: + + For text-embedding-ada-002: + + ```shell + azd env set AZURE_OPENAI_EMB_DEPLOYMENT_VERSION 2 + ``` + + For text-embedding-3-small and text-embedding-3-large: ```shell azd env set AZURE_OPENAI_EMB_DEPLOYMENT_VERSION 1 ``` -4. When prompted during `azd up`, make sure to select a region for the OpenAI resource group location that supports the text-embedding-3 models. There are [limited regions available](https://learn.microsoft.com/azure/ai-services/openai/concepts/models#embeddings-models). +4. To set the embedding model deployment SKU name, run this command with [the desired SKU name](https://learn.microsoft.com/azure/ai-services/openai/how-to/deployment-types#deployment-types). + + For GlobalStandard: + + ```bash + azd env set AZURE_OPENAI_EMB_DEPLOYMENT_SKU GlobalStandard + ``` + + For Standard: + + ```bash + azd env set AZURE_OPENAI_EMB_DEPLOYMENT_SKU Standard + ``` + +5. When prompted during `azd up`, make sure to select a region for the OpenAI resource group location that supports the desired embedding model and deployment SKU. There are [limited regions available](https://learn.microsoft.com/azure/ai-services/openai/concepts/models?tabs=global-standard%2Cstandard-chat-completions#models-by-deployment-type). If you have already deployed: @@ -155,14 +195,11 @@ If you have already deployed: * You'll need to create a new index, and re-index all of the data using the new model. You can either delete the current index in the Azure Portal, or create an index with a different name by running `azd env set AZURE_SEARCH_INDEX new-index-name`. When you next run `azd up`, the new index will be created and the data will be re-indexed. * If your OpenAI resource is not in one of the supported regions, you should delete `openAiResourceGroupLocation` from `.azure/YOUR-ENV-NAME/config.json`. When running `azd up`, you will be prompted to select a new region. -> ![NOTE] -> The text-embedding-3 models are not currently supported by the integrated vectorization feature. - -## Enabling GPT-4 Turbo with Vision +## Enabling GPT vision feature ⚠️ This feature is not currently compatible with [integrated vectorization](#enabling-integrated-vectorization). -This section covers the integration of GPT-4 Vision with Azure AI Search. Learn how to enhance your search capabilities with the power of image and text indexing, enabling advanced search functionalities over diverse document types. For a detailed guide on setup and usage, visit our [Enabling GPT-4 Turbo with Vision](gpt4v.md) page. +This section covers the integration of GPT vision models with Azure AI Search. Learn how to enhance your search capabilities with the power of image and text indexing, enabling advanced search functionalities over diverse document types. For a detailed guide on setup and usage, visit our page on [Using GPT vision model with RAG approach](gpt4v.md). ## Enabling media description with Azure Content Understanding diff --git a/docs/gpt4v.md b/docs/gpt4v.md index 8825bc2742..7bb4890a17 100644 --- a/docs/gpt4v.md +++ b/docs/gpt4v.md @@ -21,10 +21,10 @@ For more details on how this feature works, read [this blog post](https://techco * Create a [AI Vision account in Azure Portal first](https://ms.portal.azure.com/#create/Microsoft.CognitiveServicesComputerVision), so that you can agree to the Responsible AI terms for that resource. You can delete that account after agreeing. * The ability to deploy a gpt-4o model in the [supported regions](https://learn.microsoft.com/azure/ai-services/openai/concepts/models#standard-deployment-model-availability). If you're not sure, try to create a gpt-4o deployment from your Azure OpenAI deployments page. -* Ensure that you can deploy the Azure OpenAI resource group in [a region where all required components are available](https://learn.microsoft.com/azure/cognitive-services/openai/concepts/models#model-summary-table-and-region-availability): +* Ensure that you can deploy the Azure OpenAI resource group in [a region and deployment SKU where all required components are available](https://learn.microsoft.com/azure/cognitive-services/openai/concepts/models#model-summary-table-and-region-availability): * Azure OpenAI models * gpt-4o-mini - * text-embedding-ada-002 + * text-embedding-3-large * gpt-4o (for vision/evaluation features) * [Azure AI Vision](https://learn.microsoft.com/azure/ai-services/computer-vision/) diff --git a/infra/main.bicep b/infra/main.bicep index fa7a87c1b5..e40bab159b 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -146,12 +146,12 @@ param embeddingDeploymentSkuName string = '' param embeddingDeploymentCapacity int = 0 param embeddingDimensions int = 0 var embedding = { - modelName: !empty(embeddingModelName) ? embeddingModelName : 'text-embedding-ada-002' - deploymentName: !empty(embeddingDeploymentName) ? embeddingDeploymentName : 'embedding' - deploymentVersion: !empty(embeddingDeploymentVersion) ? embeddingDeploymentVersion : '2' - deploymentSkuName: !empty(embeddingDeploymentSkuName) ? embeddingDeploymentSkuName : 'Standard' + modelName: !empty(embeddingModelName) ? embeddingModelName : 'text-embedding-3-large' + deploymentName: !empty(embeddingDeploymentName) ? embeddingDeploymentName : 'text-embedding-3-large' + deploymentVersion: !empty(embeddingDeploymentVersion) ? embeddingDeploymentVersion : '1' + deploymentSkuName: !empty(embeddingDeploymentSkuName) ? embeddingDeploymentSkuName : 'GlobalStandard' deploymentCapacity: embeddingDeploymentCapacity != 0 ? embeddingDeploymentCapacity : 30 - dimensions: embeddingDimensions != 0 ? embeddingDimensions : 1536 + dimensions: embeddingDimensions != 0 ? embeddingDimensions : 3072 } param gpt4vModelName string = '' diff --git a/tests/conftest.py b/tests/conftest.py index fd2fa139a0..f0885aa769 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -95,7 +95,7 @@ async def mock_acreate(*args, **kwargs): object="embedding", ) ], - model="text-embedding-ada-002", + model="text-embedding-3-large", usage=Usage(prompt_tokens=8, total_tokens=8), ) diff --git a/tests/test_prepdocs.py b/tests/test_prepdocs.py index 79489c93ff..2d598dc4c0 100644 --- a/tests/test_prepdocs.py +++ b/tests/test_prepdocs.py @@ -51,7 +51,7 @@ async def mock_create_client(*args, **kwargs): object="embedding", ) ], - model="text-embedding-ada-002", + model="text-embedding-3-large", usage=Usage(prompt_tokens=8, total_tokens=8), ) ) diff --git a/tests/test_searchmanager.py b/tests/test_searchmanager.py index cc8403bb51..1c5a98be85 100644 --- a/tests/test_searchmanager.py +++ b/tests/test_searchmanager.py @@ -258,7 +258,7 @@ async def mock_create_client(*args, **kwargs): object="embedding", ) ], - model="text-embedding-ada-002", + model="text-embedding-3-large", usage=Usage(prompt_tokens=8, total_tokens=8), ) ) diff --git a/tests/test_upload.py b/tests/test_upload.py index 0a9b1ef34a..9a758c1f0a 100644 --- a/tests/test_upload.py +++ b/tests/test_upload.py @@ -75,7 +75,7 @@ async def mock_create_client(self, *args, **kwargs): object="embedding", ) ], - model="text-embedding-ada-002", + model="text-embedding-3-large", usage=Usage(prompt_tokens=8, total_tokens=8), ) ) From 2fddbd318ff8a2c4b931c3536a8bf9dc0ef80171 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Fri, 28 Mar 2025 16:23:00 -0700 Subject: [PATCH 02/11] Change text-embedding-3 --- app/backend/approaches/approach.py | 12 ++++--- .../approaches/chatreadretrievereadvision.py | 10 +++--- .../approaches/retrievethenreadvision.py | 10 +++--- .../integratedvectorizerstrategy.py | 9 ++++-- app/backend/prepdocslib/searchmanager.py | 32 +++++++++++++++---- infra/main.bicep | 22 ++++++++++--- 6 files changed, 68 insertions(+), 27 deletions(-) diff --git a/app/backend/approaches/approach.py b/app/backend/approaches/approach.py index 3edf85989a..a713dc6621 100644 --- a/app/backend/approaches/approach.py +++ b/app/backend/approaches/approach.py @@ -44,10 +44,9 @@ class Document: reranker_score: Optional[float] = None def serialize_for_results(self) -> dict[str, Any]: - return { + result_dict = { "id": self.id, "content": self.content, - "embedding": Document.trim_embedding(self.embedding), "imageEmbedding": Document.trim_embedding(self.image_embedding), "category": self.category, "sourcepage": self.sourcepage, @@ -69,6 +68,8 @@ def serialize_for_results(self) -> dict[str, Any]: "score": self.score, "reranker_score": self.reranker_score, } + result_dict[self.embedding_field] = Document.trim_embedding(self.embedding) + return result_dict @classmethod def trim_embedding(cls, embedding: Optional[List[float]]) -> Optional[str]: @@ -102,6 +103,7 @@ def __init__( embedding_deployment: Optional[str], # Not needed for non-Azure OpenAI or for retrieval_mode="text" embedding_model: str, embedding_dimensions: int, + embedding_field: str, openai_host: str, vision_endpoint: str, vision_token_provider: Callable[[], Awaitable[str]], @@ -115,6 +117,7 @@ def __init__( self.embedding_deployment = embedding_deployment self.embedding_model = embedding_model self.embedding_dimensions = embedding_dimensions + self.embedding_field = embedding_field self.openai_host = openai_host self.vision_endpoint = vision_endpoint self.vision_token_provider = vision_token_provider @@ -178,7 +181,7 @@ async def search( Document( id=document.get("id"), content=document.get("content"), - embedding=document.get("embedding"), + embedding=document.get(self.embedding_field), image_embedding=document.get("imageEmbedding"), category=document.get("category"), sourcepage=document.get("sourcepage"), @@ -254,7 +257,8 @@ class ExtraArgs(TypedDict, total=False): **dimensions_args, ) query_vector = embedding.data[0].embedding - return VectorizedQuery(vector=query_vector, k_nearest_neighbors=50, fields="embedding") + # TODO: use optimizations from rag time journey 3 + return VectorizedQuery(vector=query_vector, k_nearest_neighbors=50, fields=self.embedding) async def compute_image_embedding(self, q: str): endpoint = urljoin(self.vision_endpoint, "computervision/retrieval:vectorizeText") diff --git a/app/backend/approaches/chatreadretrievereadvision.py b/app/backend/approaches/chatreadretrievereadvision.py index 559f15bd1a..012e9f2791 100644 --- a/app/backend/approaches/chatreadretrievereadvision.py +++ b/app/backend/approaches/chatreadretrievereadvision.py @@ -38,6 +38,7 @@ def __init__( embedding_deployment: Optional[str], # Not needed for non-Azure OpenAI or for retrieval_mode="text" embedding_model: str, embedding_dimensions: int, + embedding_field: str, sourcepage_field: str, content_field: str, query_language: str, @@ -57,6 +58,7 @@ def __init__( self.embedding_deployment = embedding_deployment self.embedding_model = embedding_model self.embedding_dimensions = embedding_dimensions + self.embedding_field = embedding_field self.sourcepage_field = sourcepage_field self.content_field = content_field self.query_language = query_language @@ -86,7 +88,7 @@ async def run_until_final_call( minimum_reranker_score = overrides.get("minimum_reranker_score", 0.0) filter = self.build_filter(overrides, auth_claims) - vector_fields = overrides.get("vector_fields", ["embedding"]) + vector_fields = overrides.get("vector_fields", [self.embedding_field]) send_text_to_gptvision = overrides.get("gpt4v_input") in ["textAndImages", "texts", None] send_images_to_gptvision = overrides.get("gpt4v_input") in ["textAndImages", "images", None] @@ -121,9 +123,9 @@ async def run_until_final_call( if use_vector_search: for field in vector_fields: vector = ( - await self.compute_text_embedding(query_text) - if field == "embedding" - else await self.compute_image_embedding(query_text) + await self.compute_image_embedding(query_text) + if field.startswith("image") + else await self.compute_text_embedding(query_text) ) vectors.append(vector) diff --git a/app/backend/approaches/retrievethenreadvision.py b/app/backend/approaches/retrievethenreadvision.py index d532f16c72..1fd5279856 100644 --- a/app/backend/approaches/retrievethenreadvision.py +++ b/app/backend/approaches/retrievethenreadvision.py @@ -32,6 +32,7 @@ def __init__( embedding_deployment: Optional[str], # Not needed for non-Azure OpenAI or for retrieval_mode="text" embedding_model: str, embedding_dimensions: int, + embedding_field: str, sourcepage_field: str, content_field: str, query_language: str, @@ -47,6 +48,7 @@ def __init__( self.embedding_model = embedding_model self.embedding_deployment = embedding_deployment self.embedding_dimensions = embedding_dimensions + self.embedding_field = embedding_field self.sourcepage_field = sourcepage_field self.content_field = content_field self.gpt4v_deployment = gpt4v_deployment @@ -81,7 +83,7 @@ async def run( minimum_reranker_score = overrides.get("minimum_reranker_score", 0.0) filter = self.build_filter(overrides, auth_claims) - vector_fields = overrides.get("vector_fields", ["embedding"]) + vector_fields = overrides.get("vector_fields", [self.embedding_field]) send_text_to_gptvision = overrides.get("gpt4v_input") in ["textAndImages", "texts", None] send_images_to_gptvision = overrides.get("gpt4v_input") in ["textAndImages", "images", None] @@ -90,9 +92,9 @@ async def run( if use_vector_search: for field in vector_fields: vector = ( - await self.compute_text_embedding(q) - if field == "embedding" - else await self.compute_image_embedding(q) + await self.compute_image_embedding(q) + if field.startswith("image") + else await self.compute_text_embedding(q) ) vectors.append(vector) diff --git a/app/backend/prepdocslib/integratedvectorizerstrategy.py b/app/backend/prepdocslib/integratedvectorizerstrategy.py index 66e8e4a346..7496e2c050 100644 --- a/app/backend/prepdocslib/integratedvectorizerstrategy.py +++ b/app/backend/prepdocslib/integratedvectorizerstrategy.py @@ -60,7 +60,10 @@ def __init__( self.category = category self.search_info = search_info - async def create_embedding_skill(self, index_name: str): + async def create_embedding_skill(self, index_name: str, embedding_field: str) -> SearchIndexerSkillset: + """ + Create a skillset for the indexer to chunk documents and generate embeddings + """ skillset_name = f"{index_name}-skillset" split_skill = SplitSkill( @@ -87,7 +90,7 @@ async def create_embedding_skill(self, index_name: str): inputs=[ InputFieldMappingEntry(name="text", source="/document/pages/*"), ], - outputs=[OutputFieldMappingEntry(name="embedding", target_name="vector")], + outputs=[OutputFieldMappingEntry(name=embedding_field, target_name="vector")], ) index_projection = SearchIndexerIndexProjection( @@ -98,7 +101,7 @@ async def create_embedding_skill(self, index_name: str): source_context="/document/pages/*", mappings=[ InputFieldMappingEntry(name="content", source="/document/pages/*"), - InputFieldMappingEntry(name="embedding", source="/document/pages/*/vector"), + InputFieldMappingEntry(name=embedding_field, source="/document/pages/*/vector"), InputFieldMappingEntry(name="sourcepage", source="/document/metadata_storage_name"), ], ), diff --git a/app/backend/prepdocslib/searchmanager.py b/app/backend/prepdocslib/searchmanager.py index f75af03514..e5cf3ed55d 100644 --- a/app/backend/prepdocslib/searchmanager.py +++ b/app/backend/prepdocslib/searchmanager.py @@ -55,6 +55,7 @@ def __init__( use_acls: bool = False, use_int_vectorization: bool = False, embeddings: Optional[OpenAIEmbeddings] = None, + embedding_field: str = "embedding3", # can we make this not have a default? search_images: bool = False, ): self.search_info = search_info @@ -63,7 +64,9 @@ def __init__( self.use_int_vectorization = use_int_vectorization self.embeddings = embeddings # Integrated vectorization uses the ada-002 model with 1536 dimensions - self.embedding_dimensions = self.embeddings.open_ai_dimensions if self.embeddings else 1536 + # TODO: Update integrated vectorization too! + self.embedding_dimensions = self.embeddings.open_ai_dimensions if self.embeddings else None + self.embedding_field = embedding_field self.search_images = search_images async def create_index(self, vectorizers: Optional[List[VectorSearchVectorizer]] = None): @@ -93,7 +96,7 @@ async def create_index(self, vectorizers: Optional[List[VectorSearchVectorizer]] analyzer_name=self.search_analyzer_name, ), SearchField( - name="embedding", + name=self.embedding_field, type=SearchFieldDataType.Collection(SearchFieldDataType.Single), hidden=False, searchable=True, @@ -204,9 +207,7 @@ async def create_index(self, vectorizers: Optional[List[VectorSearchVectorizer]] VectorSearchProfile( name="embedding_config", algorithm_configuration_name="hnsw_config", - vectorizer_name=( - f"{self.search_info.index_name}-vectorizer" if self.use_int_vectorization else None - ), + vectorizer_name=(f"{self.search_info.index_name}-vectorizer"), ), ], vectorizers=vectorizers, @@ -228,7 +229,24 @@ async def create_index(self, vectorizers: Optional[List[VectorSearchVectorizer]] ), ) await search_index_client.create_or_update_index(existing_index) - + # check if embedding field exists + if not any(field.name == self.embedding_field for field in existing_index.fields): + logger.info("Adding embedding field to index %s", self.search_info.index_name) + existing_index.fields.append( + SearchField( + name=self.embedding_field, + type=SearchFieldDataType.Collection(SearchFieldDataType.Single), + hidden=False, + searchable=True, + filterable=False, + sortable=False, + facetable=False, + # TODO: use optimizations here + vector_search_dimensions=self.embedding_dimensions, + vector_search_profile_name="embedding_config", + ), + ) + await search_index_client.create_or_update_index(existing_index) if existing_index.vector_search is not None and ( existing_index.vector_search.vectorizers is None or len(existing_index.vector_search.vectorizers) == 0 @@ -289,7 +307,7 @@ async def update_content( texts=[section.split_page.text for section in batch] ) for i, document in enumerate(documents): - document["embedding"] = embeddings[i] + document[self.embedding_field] = embeddings[i] if image_embeddings: for i, (document, section) in enumerate(zip(documents, batch)): document["imageEmbedding"] = image_embeddings[section.split_page.page_num] diff --git a/infra/main.bicep b/infra/main.bicep index e40bab159b..cbafd13c49 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -77,10 +77,13 @@ param chatHistoryVersion string = 'cosmosdb-v2' // https://learn.microsoft.com/azure/ai-services/openai/concepts/models?tabs=python-secure%2Cstandard%2Cstandard-chat-completions#standard-deployment-model-availability @description('Location for the OpenAI resource group') @allowed([ + 'australiaeast' + 'brazilsouth' 'canadaeast' 'eastus' 'eastus2' 'francecentral' + 'germanywestcentral' 'switzerlandnorth' 'uksouth' 'japaneast' @@ -135,7 +138,7 @@ var chatGpt = { modelName: !empty(chatGptModelName) ? chatGptModelName : 'gpt-4o-mini' deploymentName: !empty(chatGptDeploymentName) ? chatGptDeploymentName : 'gpt-4o-mini' deploymentVersion: !empty(chatGptDeploymentVersion) ? chatGptDeploymentVersion : '2024-07-18' - deploymentSkuName: !empty(chatGptDeploymentSkuName) ? chatGptDeploymentSkuName : 'Standard' + deploymentSkuName: !empty(chatGptDeploymentSkuName) ? chatGptDeploymentSkuName : 'Standard' // TODO, but it will break existing deployments deploymentCapacity: chatGptDeploymentCapacity != 0 ? chatGptDeploymentCapacity : 30 } @@ -148,8 +151,8 @@ param embeddingDimensions int = 0 var embedding = { modelName: !empty(embeddingModelName) ? embeddingModelName : 'text-embedding-3-large' deploymentName: !empty(embeddingDeploymentName) ? embeddingDeploymentName : 'text-embedding-3-large' - deploymentVersion: !empty(embeddingDeploymentVersion) ? embeddingDeploymentVersion : '1' - deploymentSkuName: !empty(embeddingDeploymentSkuName) ? embeddingDeploymentSkuName : 'GlobalStandard' + deploymentVersion: !empty(embeddingDeploymentVersion) ? embeddingDeploymentVersion : (embeddingModelName == 'text-embedding-ada-002' ? '2' : '1') + deploymentSkuName: !empty(embeddingDeploymentSkuName) ? embeddingDeploymentSkuName : (embeddingModelName == 'text-embedding-ada-002' ? 'Standard' : 'GlobalStandard') deploymentCapacity: embeddingDeploymentCapacity != 0 ? embeddingDeploymentCapacity : 30 dimensions: embeddingDimensions != 0 ? embeddingDimensions : 3072 } @@ -163,7 +166,7 @@ var gpt4v = { modelName: !empty(gpt4vModelName) ? gpt4vModelName : 'gpt-4o' deploymentName: !empty(gpt4vDeploymentName) ? gpt4vDeploymentName : 'gpt-4o' deploymentVersion: !empty(gpt4vModelVersion) ? gpt4vModelVersion : '2024-08-06' - deploymentSkuName: !empty(gpt4vDeploymentSkuName) ? gpt4vDeploymentSkuName : 'Standard' + deploymentSkuName: !empty(gpt4vDeploymentSkuName) ? gpt4vDeploymentSkuName : 'Standard' // TODO, but it will break existing deployments deploymentCapacity: gpt4vDeploymentCapacity != 0 ? gpt4vDeploymentCapacity : 10 } @@ -176,7 +179,7 @@ var eval = { modelName: !empty(evalModelName) ? evalModelName : 'gpt-4o' deploymentName: !empty(evalDeploymentName) ? evalDeploymentName : 'gpt-4o' deploymentVersion: !empty(evalModelVersion) ? evalModelVersion : '2024-08-06' - deploymentSkuName: !empty(evalDeploymentSkuName) ? evalDeploymentSkuName : 'Standard' + deploymentSkuName: !empty(evalDeploymentSkuName) ? evalDeploymentSkuName : 'Standard' // TODO, but it will break existing deployments deploymentCapacity: evalDeploymentCapacity != 0 ? evalDeploymentCapacity : 30 } @@ -1235,6 +1238,7 @@ output AZURE_RESOURCE_GROUP string = resourceGroup.name // Shared by all OpenAI deployments output OPENAI_HOST string = openAiHost output AZURE_OPENAI_EMB_MODEL_NAME string = embedding.modelName +output AZURE_OPENAI_EMB_DIMENSIONS int = embedding.dimensions output AZURE_OPENAI_CHATGPT_MODEL string = chatGpt.modelName output AZURE_OPENAI_GPT4V_MODEL string = gpt4v.modelName @@ -1243,9 +1247,17 @@ output AZURE_OPENAI_SERVICE string = isAzureOpenAiHost && deployAzureOpenAi ? op output AZURE_OPENAI_API_VERSION string = isAzureOpenAiHost ? azureOpenAiApiVersion : '' output AZURE_OPENAI_RESOURCE_GROUP string = isAzureOpenAiHost ? openAiResourceGroup.name : '' output AZURE_OPENAI_CHATGPT_DEPLOYMENT string = isAzureOpenAiHost ? chatGpt.deploymentName : '' +output AZURE_OPENAI_CHATGPT_DEPLOYMENT_VERSION string = isAzureOpenAiHost ? chatGpt.deploymentVersion : '' +output AZURE_OPENAI_CHATGPT_DEPLOYMENT_SKU string = isAzureOpenAiHost ? chatGpt.deploymentSkuName : '' output AZURE_OPENAI_EMB_DEPLOYMENT string = isAzureOpenAiHost ? embedding.deploymentName : '' +output AZURE_OPENAI_EMB_DEPLOYMENT_VERSION string = isAzureOpenAiHost ? embedding.deploymentVersion : '' +output AZURE_OPENAI_EMB_DEPLOYMENT_SKU string = isAzureOpenAiHost ? embedding.deploymentSkuName : '' output AZURE_OPENAI_GPT4V_DEPLOYMENT string = isAzureOpenAiHost && useGPT4V ? gpt4v.deploymentName : '' +output AZURE_OPENAI_GPT4V_DEPLOYMENT_VERSION string = isAzureOpenAiHost && useGPT4V ? gpt4v.deploymentVersion : '' +output AZURE_OPENAI_GPT4V_DEPLOYMENT_SKU string = isAzureOpenAiHost && useGPT4V ? gpt4v.deploymentSkuName : '' output AZURE_OPENAI_EVAL_DEPLOYMENT string = isAzureOpenAiHost && useEval ? eval.deploymentName : '' +output AZURE_OPENAI_EVAL_DEPLOYMENT_VERSION string = isAzureOpenAiHost && useEval ? eval.deploymentVersion : '' +output AZURE_OPENAI_EVAL_DEPLOYMENT_SKU string = isAzureOpenAiHost && useEval ? eval.deploymentSkuName : '' output AZURE_OPENAI_EVAL_MODEL string = isAzureOpenAiHost && useEval ? eval.modelName : '' output AZURE_SPEECH_SERVICE_ID string = useSpeechOutputAzure ? speech.outputs.resourceId : '' From 3e6d743dfd59f6d60838d0c33a5148b13b615077 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Fri, 28 Mar 2025 22:54:07 -0700 Subject: [PATCH 03/11] Bicep fixes --- infra/main.bicep | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/infra/main.bicep b/infra/main.bicep index cbafd13c49..a19882ad2b 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -74,7 +74,7 @@ param chatHistoryDatabaseName string = 'chat-database' param chatHistoryContainerName string = 'chat-history-v2' param chatHistoryVersion string = 'cosmosdb-v2' -// https://learn.microsoft.com/azure/ai-services/openai/concepts/models?tabs=python-secure%2Cstandard%2Cstandard-chat-completions#standard-deployment-model-availability +// https://learn.microsoft.com/azure/ai-services/openai/concepts/models?tabs=global-standard%2Cstandard-chat-completions#models-by-deployment-type @description('Location for the OpenAI resource group') @allowed([ 'australiaeast' @@ -84,12 +84,22 @@ param chatHistoryVersion string = 'cosmosdb-v2' 'eastus2' 'francecentral' 'germanywestcentral' - 'switzerlandnorth' - 'uksouth' 'japaneast' + 'koreacentral' 'northcentralus' - 'australiaeast' + 'norwayeast' + 'polandcentral' + 'southafricanorth' + 'southcentralus' + 'southindia' + 'spaincentral' 'swedencentral' + 'switzerlandnorth' + 'uaenorth' + 'uksouth' + 'westeurope' + 'westus' + 'westus3' ]) @metadata({ azd: { @@ -138,7 +148,7 @@ var chatGpt = { modelName: !empty(chatGptModelName) ? chatGptModelName : 'gpt-4o-mini' deploymentName: !empty(chatGptDeploymentName) ? chatGptDeploymentName : 'gpt-4o-mini' deploymentVersion: !empty(chatGptDeploymentVersion) ? chatGptDeploymentVersion : '2024-07-18' - deploymentSkuName: !empty(chatGptDeploymentSkuName) ? chatGptDeploymentSkuName : 'Standard' // TODO, but it will break existing deployments + deploymentSkuName: !empty(chatGptDeploymentSkuName) ? chatGptDeploymentSkuName : 'GlobalStandard' // Not backward-compatible deploymentCapacity: chatGptDeploymentCapacity != 0 ? chatGptDeploymentCapacity : 30 } @@ -166,7 +176,7 @@ var gpt4v = { modelName: !empty(gpt4vModelName) ? gpt4vModelName : 'gpt-4o' deploymentName: !empty(gpt4vDeploymentName) ? gpt4vDeploymentName : 'gpt-4o' deploymentVersion: !empty(gpt4vModelVersion) ? gpt4vModelVersion : '2024-08-06' - deploymentSkuName: !empty(gpt4vDeploymentSkuName) ? gpt4vDeploymentSkuName : 'Standard' // TODO, but it will break existing deployments + deploymentSkuName: !empty(gpt4vDeploymentSkuName) ? gpt4vDeploymentSkuName : 'GlobalStandard' // Not-backward compatible deploymentCapacity: gpt4vDeploymentCapacity != 0 ? gpt4vDeploymentCapacity : 10 } @@ -179,7 +189,7 @@ var eval = { modelName: !empty(evalModelName) ? evalModelName : 'gpt-4o' deploymentName: !empty(evalDeploymentName) ? evalDeploymentName : 'gpt-4o' deploymentVersion: !empty(evalModelVersion) ? evalModelVersion : '2024-08-06' - deploymentSkuName: !empty(evalDeploymentSkuName) ? evalDeploymentSkuName : 'Standard' // TODO, but it will break existing deployments + deploymentSkuName: !empty(evalDeploymentSkuName) ? evalDeploymentSkuName : 'GlobalStandard' // Not backward-compatible deploymentCapacity: evalDeploymentCapacity != 0 ? evalDeploymentCapacity : 30 } From 1b3d1005cfe7e0b4f65a9371c08339b2cb285359 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Tue, 1 Apr 2025 10:41:29 -0700 Subject: [PATCH 04/11] More embedding related changes --- app/backend/app.py | 6 + app/backend/approaches/approach.py | 5 +- .../approaches/chatreadretrieveread.py | 2 + app/backend/approaches/retrievethenread.py | 2 + app/backend/prepdocslib/searchmanager.py | 164 ++++++++++-------- app/backend/requirements.txt | 2 +- infra/main.bicep | 3 + infra/main.parameters.json | 3 + tests/e2e.py | 2 + 9 files changed, 109 insertions(+), 80 deletions(-) diff --git a/app/backend/app.py b/app/backend/app.py index 65615185ca..e781dee240 100644 --- a/app/backend/app.py +++ b/app/backend/app.py @@ -456,6 +456,8 @@ async def setup_clients(): AZURE_SEARCH_QUERY_SPELLER = os.getenv("AZURE_SEARCH_QUERY_SPELLER") or "lexicon" AZURE_SEARCH_SEMANTIC_RANKER = os.getenv("AZURE_SEARCH_SEMANTIC_RANKER", "free").lower() AZURE_SEARCH_QUERY_REWRITING = os.getenv("AZURE_SEARCH_QUERY_REWRITING", "false").lower() + # This defaults to the previous field name "embedding", for backwards compatibility + AZURE_SEARCH_FIELD_NAME_EMBEDDING = os.getenv("AZURE_SEARCH_FIELD_NAME_EMBEDDING", "embedding") AZURE_SPEECH_SERVICE_ID = os.getenv("AZURE_SPEECH_SERVICE_ID") AZURE_SPEECH_SERVICE_LOCATION = os.getenv("AZURE_SPEECH_SERVICE_LOCATION") @@ -662,6 +664,7 @@ async def setup_clients(): embedding_model=OPENAI_EMB_MODEL, embedding_deployment=AZURE_OPENAI_EMB_DEPLOYMENT, embedding_dimensions=OPENAI_EMB_DIMENSIONS, + embedding_field=AZURE_SEARCH_FIELD_NAME_EMBEDDING, sourcepage_field=KB_FIELDS_SOURCEPAGE, content_field=KB_FIELDS_CONTENT, query_language=AZURE_SEARCH_QUERY_LANGUAGE, @@ -679,6 +682,7 @@ async def setup_clients(): embedding_model=OPENAI_EMB_MODEL, embedding_deployment=AZURE_OPENAI_EMB_DEPLOYMENT, embedding_dimensions=OPENAI_EMB_DIMENSIONS, + embedding_field=AZURE_SEARCH_FIELD_NAME_EMBEDDING, sourcepage_field=KB_FIELDS_SOURCEPAGE, content_field=KB_FIELDS_CONTENT, query_language=AZURE_SEARCH_QUERY_LANGUAGE, @@ -704,6 +708,7 @@ async def setup_clients(): embedding_model=OPENAI_EMB_MODEL, embedding_deployment=AZURE_OPENAI_EMB_DEPLOYMENT, embedding_dimensions=OPENAI_EMB_DIMENSIONS, + embedding_field=AZURE_SEARCH_FIELD_NAME_EMBEDDING, sourcepage_field=KB_FIELDS_SOURCEPAGE, content_field=KB_FIELDS_CONTENT, query_language=AZURE_SEARCH_QUERY_LANGUAGE, @@ -725,6 +730,7 @@ async def setup_clients(): embedding_model=OPENAI_EMB_MODEL, embedding_deployment=AZURE_OPENAI_EMB_DEPLOYMENT, embedding_dimensions=OPENAI_EMB_DIMENSIONS, + embedding_field=AZURE_SEARCH_FIELD_NAME_EMBEDDING, sourcepage_field=KB_FIELDS_SOURCEPAGE, content_field=KB_FIELDS_CONTENT, query_language=AZURE_SEARCH_QUERY_LANGUAGE, diff --git a/app/backend/approaches/approach.py b/app/backend/approaches/approach.py index a713dc6621..f03ed087b6 100644 --- a/app/backend/approaches/approach.py +++ b/app/backend/approaches/approach.py @@ -47,6 +47,8 @@ def serialize_for_results(self) -> dict[str, Any]: result_dict = { "id": self.id, "content": self.content, + # Should we rename to its actual field name in the index? + "embedding": Document.trim_embedding(self.embedding), "imageEmbedding": Document.trim_embedding(self.image_embedding), "category": self.category, "sourcepage": self.sourcepage, @@ -68,7 +70,6 @@ def serialize_for_results(self) -> dict[str, Any]: "score": self.score, "reranker_score": self.reranker_score, } - result_dict[self.embedding_field] = Document.trim_embedding(self.embedding) return result_dict @classmethod @@ -258,7 +259,7 @@ class ExtraArgs(TypedDict, total=False): ) query_vector = embedding.data[0].embedding # TODO: use optimizations from rag time journey 3 - return VectorizedQuery(vector=query_vector, k_nearest_neighbors=50, fields=self.embedding) + return VectorizedQuery(vector=query_vector, k_nearest_neighbors=50, fields=self.embedding_field) async def compute_image_embedding(self, q: str): endpoint = urljoin(self.vision_endpoint, "computervision/retrieval:vectorizeText") diff --git a/app/backend/approaches/chatreadretrieveread.py b/app/backend/approaches/chatreadretrieveread.py index c839b03d30..87265b2232 100644 --- a/app/backend/approaches/chatreadretrieveread.py +++ b/app/backend/approaches/chatreadretrieveread.py @@ -34,6 +34,7 @@ def __init__( embedding_deployment: Optional[str], # Not needed for non-Azure OpenAI or for retrieval_mode="text" embedding_model: str, embedding_dimensions: int, + embedding_field: str, sourcepage_field: str, content_field: str, query_language: str, @@ -48,6 +49,7 @@ def __init__( self.embedding_deployment = embedding_deployment self.embedding_model = embedding_model self.embedding_dimensions = embedding_dimensions + self.embedding_field = embedding_field self.sourcepage_field = sourcepage_field self.content_field = content_field self.query_language = query_language diff --git a/app/backend/approaches/retrievethenread.py b/app/backend/approaches/retrievethenread.py index fc87132fa7..4e10c94654 100644 --- a/app/backend/approaches/retrievethenread.py +++ b/app/backend/approaches/retrievethenread.py @@ -28,6 +28,7 @@ def __init__( embedding_model: str, embedding_deployment: Optional[str], # Not needed for non-Azure OpenAI or for retrieval_mode="text" embedding_dimensions: int, + embedding_field: str, sourcepage_field: str, content_field: str, query_language: str, @@ -43,6 +44,7 @@ def __init__( self.embedding_dimensions = embedding_dimensions self.chatgpt_deployment = chatgpt_deployment self.embedding_deployment = embedding_deployment + self.embedding_field = embedding_field self.sourcepage_field = sourcepage_field self.content_field = content_field self.query_language = query_language diff --git a/app/backend/prepdocslib/searchmanager.py b/app/backend/prepdocslib/searchmanager.py index e5cf3ed55d..3a18dc98ce 100644 --- a/app/backend/prepdocslib/searchmanager.py +++ b/app/backend/prepdocslib/searchmanager.py @@ -6,8 +6,10 @@ from azure.search.documents.indexes.models import ( AzureOpenAIVectorizer, AzureOpenAIVectorizerParameters, + BinaryQuantizationCompression, HnswAlgorithmConfiguration, HnswParameters, + RescoringOptions, SearchableField, SearchField, SearchFieldDataType, @@ -18,8 +20,8 @@ SemanticSearch, SimpleField, VectorSearch, + VectorSearchCompressionRescoreStorageMethod, VectorSearchProfile, - VectorSearchVectorizer, ) from .blobmanager import BlobManager @@ -69,11 +71,44 @@ def __init__( self.embedding_field = embedding_field self.search_images = search_images - async def create_index(self, vectorizers: Optional[List[VectorSearchVectorizer]] = None): + async def create_index(self): logger.info("Checking whether search index %s exists...", self.search_info.index_name) async with self.search_info.create_search_index_client() as search_index_client: + vectorizer = None + embedding_field = None + if self.embeddings and isinstance(self.embeddings, AzureOpenAIEmbeddingService): + vectorizer = AzureOpenAIVectorizer( + vectorizer_name=f"{self.search_info.index_name}-vectorizer", + parameters=AzureOpenAIVectorizerParameters( + resource_url=self.embeddings.open_ai_endpoint, + deployment_name=self.embeddings.open_ai_deployment, + model_name=self.embeddings.open_ai_model_name, + ), + ) + if self.embeddings: + if self.embedding_dimensions is None: + raise ValueError( + "Embedding dimensions must be set in order to add an embedding field to the search index" + ) + if self.embedding_field is None: + raise ValueError( + "Embedding field must be set in order to add an embedding field to the search index" + ) + embedding_field = SearchField( + name=self.embedding_field, + type=SearchFieldDataType.Collection(SearchFieldDataType.Single), + hidden=True, + searchable=True, + filterable=False, + sortable=False, + facetable=False, + vector_search_dimensions=self.embedding_dimensions, + vector_search_profile_name="embedding_config", + stored=False, + ) + if self.search_info.index_name not in [name async for name in search_index_client.list_index_names()]: logger.info("Creating new search index %s", self.search_info.index_name) fields = [ @@ -95,17 +130,6 @@ async def create_index(self, vectorizers: Optional[List[VectorSearchVectorizer]] type="Edm.String", analyzer_name=self.search_analyzer_name, ), - SearchField( - name=self.embedding_field, - type=SearchFieldDataType.Collection(SearchFieldDataType.Single), - hidden=False, - searchable=True, - filterable=False, - sortable=False, - facetable=False, - vector_search_dimensions=self.embedding_dimensions, - vector_search_profile_name="embedding_config", - ), SimpleField(name="category", type="Edm.String", filterable=True, facetable=True), SimpleField( name="sourcepage", @@ -160,27 +184,50 @@ async def create_index(self, vectorizers: Optional[List[VectorSearchVectorizer]] ), ) - vectorizers = [] - if self.embeddings and isinstance(self.embeddings, AzureOpenAIEmbeddingService): - logger.info( - "Including vectorizer for search index %s, using Azure OpenAI service %s", - self.search_info.index_name, - self.embeddings.open_ai_service, - ) - vectorizers.append( - AzureOpenAIVectorizer( - vectorizer_name=f"{self.search_info.index_name}-vectorizer", - parameters=AzureOpenAIVectorizerParameters( - resource_url=self.embeddings.open_ai_endpoint, - deployment_name=self.embeddings.open_ai_deployment, - model_name=self.embeddings.open_ai_model_name, - ), + vector_search = None + if self.embeddings: + logger.info("Including embedding field in new index %s", self.search_info.index_name) + fields.append(embedding_field) + + vectorizers = [] + if vectorizer is not None: + logger.info("Including vectorizer in new index %s", self.search_info.index_name) + vectorizers.append(vectorizer) + else: + logger.info( + "New index %s will not have vectorizer, since no Azure OpenAI service is set", + self.search_info.index_name, ) - ) - else: - logger.info( - "Not including vectorizer for search index %s, no Azure OpenAI service found", - self.search_info.index_name, + + vector_search = VectorSearch( + profiles=[ + VectorSearchProfile( + name="embedding_config", + algorithm_configuration_name="hnsw_config", + compression_name="binary-quantization", + **({"vectorizer_name": vectorizer.vectorizer_name if vectorizer else None}), + ), + ], + algorithms=[ + HnswAlgorithmConfiguration( + name="hnsw_config", + parameters=HnswParameters(metric="cosine"), + ) + ], + vectorizers=vectorizers, + compressions=[ + BinaryQuantizationCompression( + compression_name="binary-quantization", + rescoring_options=RescoringOptions( + enable_rescoring=True, + default_oversampling=10, + rescore_storage_method=VectorSearchCompressionRescoreStorageMethod.PRESERVE_ORIGINALS, + ), + # Explicitly set deprecated parameters to None + rerank_with_original_vectors=None, + default_oversampling=None, + ) + ], ) index = SearchIndex( @@ -196,22 +243,7 @@ async def create_index(self, vectorizers: Optional[List[VectorSearchVectorizer]] ) ] ), - vector_search=VectorSearch( - algorithms=[ - HnswAlgorithmConfiguration( - name="hnsw_config", - parameters=HnswParameters(metric="cosine"), - ) - ], - profiles=[ - VectorSearchProfile( - name="embedding_config", - algorithm_configuration_name="hnsw_config", - vectorizer_name=(f"{self.search_info.index_name}-vectorizer"), - ), - ], - vectorizers=vectorizers, - ), + vector_search=vector_search, ) await search_index_client.create_index(index) @@ -229,23 +261,10 @@ async def create_index(self, vectorizers: Optional[List[VectorSearchVectorizer]] ), ) await search_index_client.create_or_update_index(existing_index) - # check if embedding field exists - if not any(field.name == self.embedding_field for field in existing_index.fields): + # check if embedding field exists - TODO: will this really work if we havent redfined vector search? + if self.embeddings and not any(field.name == self.embedding_field for field in existing_index.fields): logger.info("Adding embedding field to index %s", self.search_info.index_name) - existing_index.fields.append( - SearchField( - name=self.embedding_field, - type=SearchFieldDataType.Collection(SearchFieldDataType.Single), - hidden=False, - searchable=True, - filterable=False, - sortable=False, - facetable=False, - # TODO: use optimizations here - vector_search_dimensions=self.embedding_dimensions, - vector_search_profile_name="embedding_config", - ), - ) + existing_index.fields.append(embedding_field) await search_index_client.create_or_update_index(existing_index) if existing_index.vector_search is not None and ( existing_index.vector_search.vectorizers is None @@ -253,21 +272,12 @@ async def create_index(self, vectorizers: Optional[List[VectorSearchVectorizer]] ): if self.embeddings is not None and isinstance(self.embeddings, AzureOpenAIEmbeddingService): logger.info("Adding vectorizer to search index %s", self.search_info.index_name) - existing_index.vector_search.vectorizers = [ - AzureOpenAIVectorizer( - vectorizer_name=f"{self.search_info.index_name}-vectorizer", - parameters=AzureOpenAIVectorizerParameters( - resource_url=self.embeddings.open_ai_endpoint, - deployment_name=self.embeddings.open_ai_deployment, - model_name=self.embeddings.open_ai_model_name, - ), - ) - ] + existing_index.vector_search.vectorizers = [vectorizer] await search_index_client.create_or_update_index(existing_index) else: logger.info( - "Can't add vectorizer to search index %s since no Azure OpenAI embeddings service is defined", - self.search_info, + "Search index %s will not have vectorizer, since no Azure OpenAI service is set", + self.search_info.index_name, ) async def update_content( diff --git a/app/backend/requirements.txt b/app/backend/requirements.txt index 82f0574846..cf3a5b3624 100644 --- a/app/backend/requirements.txt +++ b/app/backend/requirements.txt @@ -57,7 +57,7 @@ azure-monitor-opentelemetry==1.6.1 # via -r requirements.in azure-monitor-opentelemetry-exporter==1.0.0b32 # via azure-monitor-opentelemetry -azure-search-documents==11.6.0b9 +azure-search-documents==11.6.0b11 # via -r requirements.in azure-storage-blob==12.22.0 # via diff --git a/infra/main.bicep b/infra/main.bicep index a19882ad2b..ca552b034f 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -27,6 +27,7 @@ param searchIndexName string // Set in main.parameters.json param searchQueryLanguage string // Set in main.parameters.json param searchQuerySpeller string // Set in main.parameters.json param searchServiceSemanticRankerLevel string // Set in main.parameters.json +param searchFieldNameEmbedding string // Set in main.parameters.json var actualSearchServiceSemanticRankerLevel = (searchServiceSkuName == 'free') ? 'disabled' : searchServiceSemanticRankerLevel @@ -390,6 +391,7 @@ var appEnvVariables = { AZURE_VISION_ENDPOINT: useGPT4V ? computerVision.outputs.endpoint : '' AZURE_SEARCH_QUERY_LANGUAGE: searchQueryLanguage AZURE_SEARCH_QUERY_SPELLER: searchQuerySpeller + AZURE_SEARCH_FIELD_NAME_EMBEDDING: searchFieldNameEmbedding APPLICATIONINSIGHTS_CONNECTION_STRING: useApplicationInsights ? monitoring.outputs.applicationInsightsConnectionString : '' @@ -1284,6 +1286,7 @@ output AZURE_SEARCH_SERVICE string = searchService.outputs.name output AZURE_SEARCH_SERVICE_RESOURCE_GROUP string = searchServiceResourceGroup.name output AZURE_SEARCH_SEMANTIC_RANKER string = actualSearchServiceSemanticRankerLevel output AZURE_SEARCH_SERVICE_ASSIGNED_USERID string = searchService.outputs.principalId +output AZURE_SEARCH_FIELD_NAME_EMBEDDING string = searchFieldNameEmbedding output AZURE_COSMOSDB_ACCOUNT string = (useAuthentication && useChatHistoryCosmos) ? cosmosDb.outputs.name : '' output AZURE_CHAT_HISTORY_DATABASE string = chatHistoryDatabaseName diff --git a/infra/main.parameters.json b/infra/main.parameters.json index 6f825b99fd..a6527bcc5d 100644 --- a/infra/main.parameters.json +++ b/infra/main.parameters.json @@ -83,6 +83,9 @@ "searchServiceQueryRewriting": { "value": "${AZURE_SEARCH_QUERY_REWRITING=false}" }, + "searchFieldNameEmbedding": { + "value": "${AZURE_SEARCH_FIELD_NAME_EMBEDDING=embedding3}" + }, "storageAccountName": { "value": "${AZURE_STORAGE_ACCOUNT}" }, diff --git a/tests/e2e.py b/tests/e2e.py index 509890e5b1..7edfdc2066 100644 --- a/tests/e2e.py +++ b/tests/e2e.py @@ -57,6 +57,8 @@ def run_server(port: int): "AZURE_SPEECH_SERVICE_LOCATION": "eastus", "AZURE_OPENAI_SERVICE": "test-openai-service", "AZURE_OPENAI_CHATGPT_MODEL": "gpt-4o-mini", + "AZURE_OPENAI_EMB_MODEL_NAME": "text-embedding-3-large", + "AZURE_OPENAI_EMB_DIMENSIONS": "3072", }, clear=True, ): From d2b2e9fa87bc6bcb0da777ac7cdd21d5aee96e0f Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Tue, 1 Apr 2025 10:54:49 -0700 Subject: [PATCH 05/11] Add dimension truncation --- app/backend/prepdocslib/searchmanager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/backend/prepdocslib/searchmanager.py b/app/backend/prepdocslib/searchmanager.py index 3a18dc98ce..0185250538 100644 --- a/app/backend/prepdocslib/searchmanager.py +++ b/app/backend/prepdocslib/searchmanager.py @@ -218,6 +218,7 @@ async def create_index(self): compressions=[ BinaryQuantizationCompression( compression_name="binary-quantization", + truncation_dimension=1024, rescoring_options=RescoringOptions( enable_rescoring=True, default_oversampling=10, From 6f07ce8360b3ca2d7649ee577db822ee3f252096 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Tue, 1 Apr 2025 11:15:14 -0700 Subject: [PATCH 06/11] Mypy fix --- app/backend/prepdocslib/searchmanager.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/backend/prepdocslib/searchmanager.py b/app/backend/prepdocslib/searchmanager.py index 0185250538..1b1b09fbb0 100644 --- a/app/backend/prepdocslib/searchmanager.py +++ b/app/backend/prepdocslib/searchmanager.py @@ -185,14 +185,14 @@ async def create_index(self): ) vector_search = None - if self.embeddings: + if self.embeddings and embedding_field: logger.info("Including embedding field in new index %s", self.search_info.index_name) fields.append(embedding_field) - vectorizers = [] + vectorizers = None if vectorizer is not None: logger.info("Including vectorizer in new index %s", self.search_info.index_name) - vectorizers.append(vectorizer) + vectorizers = [vectorizer] else: logger.info( "New index %s will not have vectorizer, since no Azure OpenAI service is set", From 121521a1da63bcaa077958189656af82d12953d7 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Tue, 1 Apr 2025 14:31:06 -0700 Subject: [PATCH 07/11] Fix mypy issues --- app/backend/prepdocs.py | 4 + app/backend/prepdocslib/filestrategy.py | 25 +- .../integratedvectorizerstrategy.py | 14 +- app/backend/prepdocslib/searchmanager.py | 252 +++++++++++------- 4 files changed, 188 insertions(+), 107 deletions(-) diff --git a/app/backend/prepdocs.py b/app/backend/prepdocs.py index 57cfe52e6f..35e50ce6e8 100644 --- a/app/backend/prepdocs.py +++ b/app/backend/prepdocs.py @@ -398,6 +398,8 @@ async def main(strategy: Strategy, setup_index: bool = True): blob_manager=blob_manager, document_action=document_action, embeddings=openai_embeddings_service, + search_field_name_embedding=os.environ["AZURE_SEARCH_FIELD_NAME_EMBEDDING"], + search_field_name_image_embedding=os.environ["AZURE_SEARCH_FIELD_NAME_IMAGE_EMBEDDING"], subscription_id=os.environ["AZURE_SUBSCRIPTION_ID"], search_service_user_assigned_id=args.searchserviceassignedid, search_analyzer_name=os.getenv("AZURE_SEARCH_ANALYZER_NAME"), @@ -430,6 +432,8 @@ async def main(strategy: Strategy, setup_index: bool = True): embeddings=openai_embeddings_service, image_embeddings=image_embeddings_service, search_analyzer_name=os.getenv("AZURE_SEARCH_ANALYZER_NAME"), + search_field_name_embedding=os.getenv("AZURE_SEARCH_FIELD_NAME_EMBEDDING"), + search_field_name_image_embedding=os.getenv("AZURE_SEARCH_FIELD_NAME_IMAGE_EMBEDDING"), use_acls=use_acls, category=args.category, use_content_understanding=use_content_understanding, diff --git a/app/backend/prepdocslib/filestrategy.py b/app/backend/prepdocslib/filestrategy.py index 3748f67a09..8dd28427ae 100644 --- a/app/backend/prepdocslib/filestrategy.py +++ b/app/backend/prepdocslib/filestrategy.py @@ -51,6 +51,8 @@ def __init__( embeddings: Optional[OpenAIEmbeddings] = None, image_embeddings: Optional[ImageEmbeddings] = None, search_analyzer_name: Optional[str] = None, + search_field_name_embedding: Optional[str] = None, + search_field_name_image_embedding: Optional[str] = None, use_acls: bool = False, category: Optional[str] = None, use_content_understanding: bool = False, @@ -63,22 +65,29 @@ def __init__( self.embeddings = embeddings self.image_embeddings = image_embeddings self.search_analyzer_name = search_analyzer_name + self.search_field_name_embedding = search_field_name_embedding + self.search_field_name_image_embedding = search_field_name_image_embedding self.search_info = search_info self.use_acls = use_acls self.category = category self.use_content_understanding = use_content_understanding self.content_understanding_endpoint = content_understanding_endpoint - async def setup(self): - search_manager = SearchManager( + def setup_search_manager(self): + self.search_manager = SearchManager( self.search_info, self.search_analyzer_name, self.use_acls, False, self.embeddings, + field_name_embedding=self.search_field_name_embedding, + field_name_image_embedding=self.search_field_name_image_embedding, search_images=self.image_embeddings is not None, ) - await search_manager.create_index() + + async def setup(self): + self.setup_search_manager() + await self.search_manager.create_index() if self.use_content_understanding: if self.content_understanding_endpoint is None: @@ -91,9 +100,7 @@ async def setup(self): await cu_manager.create_analyzer() async def run(self): - search_manager = SearchManager( - self.search_info, self.search_analyzer_name, self.use_acls, False, self.embeddings - ) + self.setup_search_manager() if self.document_action == DocumentAction.Add: files = self.list_file_strategy.list() async for file in files: @@ -104,7 +111,7 @@ async def run(self): blob_image_embeddings: Optional[List[List[float]]] = None if self.image_embeddings and blob_sas_uris: blob_image_embeddings = await self.image_embeddings.create_embeddings(blob_sas_uris) - await search_manager.update_content(sections, blob_image_embeddings, url=file.url) + await self.search_manager.update_content(sections, blob_image_embeddings, url=file.url) finally: if file: file.close() @@ -112,10 +119,10 @@ async def run(self): paths = self.list_file_strategy.list_paths() async for path in paths: await self.blob_manager.remove_blob(path) - await search_manager.remove_content(path) + await self.search_manager.remove_content(path) elif self.document_action == DocumentAction.RemoveAll: await self.blob_manager.remove_blob() - await search_manager.remove_content() + await self.search_manager.remove_content() class UploadUserFileStrategy: diff --git a/app/backend/prepdocslib/integratedvectorizerstrategy.py b/app/backend/prepdocslib/integratedvectorizerstrategy.py index 7496e2c050..ce9981260c 100644 --- a/app/backend/prepdocslib/integratedvectorizerstrategy.py +++ b/app/backend/prepdocslib/integratedvectorizerstrategy.py @@ -41,6 +41,8 @@ def __init__( blob_manager: BlobManager, search_info: SearchInfo, embeddings: AzureOpenAIEmbeddingService, + search_field_name_embedding: str, + search_field_name_image_embedding: str, subscription_id: str, search_service_user_assigned_id: str, document_action: DocumentAction = DocumentAction.Add, @@ -53,6 +55,8 @@ def __init__( self.blob_manager = blob_manager self.document_action = document_action self.embeddings = embeddings + self.search_field_name_embedding = search_field_name_embedding + self.search_field_name_image_embedding = search_field_name_image_embedding self.subscription_id = subscription_id self.search_user_assigned_identity = search_service_user_assigned_id self.search_analyzer_name = search_analyzer_name @@ -60,7 +64,7 @@ def __init__( self.category = category self.search_info = search_info - async def create_embedding_skill(self, index_name: str, embedding_field: str) -> SearchIndexerSkillset: + async def create_embedding_skill(self, index_name: str) -> SearchIndexerSkillset: """ Create a skillset for the indexer to chunk documents and generate embeddings """ @@ -90,7 +94,7 @@ async def create_embedding_skill(self, index_name: str, embedding_field: str) -> inputs=[ InputFieldMappingEntry(name="text", source="/document/pages/*"), ], - outputs=[OutputFieldMappingEntry(name=embedding_field, target_name="vector")], + outputs=[OutputFieldMappingEntry(name=self.search_field_name_embedding, target_name="vector")], ) index_projection = SearchIndexerIndexProjection( @@ -101,8 +105,10 @@ async def create_embedding_skill(self, index_name: str, embedding_field: str) -> source_context="/document/pages/*", mappings=[ InputFieldMappingEntry(name="content", source="/document/pages/*"), - InputFieldMappingEntry(name=embedding_field, source="/document/pages/*/vector"), InputFieldMappingEntry(name="sourcepage", source="/document/metadata_storage_name"), + InputFieldMappingEntry( + name=self.search_field_name_embedding, source="/document/pages/*/vector" + ), ], ), ], @@ -128,6 +134,8 @@ async def setup(self): use_acls=self.use_acls, use_int_vectorization=True, embeddings=self.embeddings, + field_name_embedding=self.search_field_name_embedding, + field_name_image_embedding=self.search_field_name_image_embedding, search_images=False, ) diff --git a/app/backend/prepdocslib/searchmanager.py b/app/backend/prepdocslib/searchmanager.py index 1b1b09fbb0..cc7008f42b 100644 --- a/app/backend/prepdocslib/searchmanager.py +++ b/app/backend/prepdocslib/searchmanager.py @@ -20,8 +20,11 @@ SemanticSearch, SimpleField, VectorSearch, + VectorSearchAlgorithmConfiguration, + VectorSearchCompression, VectorSearchCompressionRescoreStorageMethod, VectorSearchProfile, + VectorSearchVectorizer, ) from .blobmanager import BlobManager @@ -57,7 +60,8 @@ def __init__( use_acls: bool = False, use_int_vectorization: bool = False, embeddings: Optional[OpenAIEmbeddings] = None, - embedding_field: str = "embedding3", # can we make this not have a default? + field_name_embedding: Optional[str] = None, + field_name_image_embedding: Optional[str] = None, search_images: bool = False, ): self.search_info = search_info @@ -65,10 +69,9 @@ def __init__( self.use_acls = use_acls self.use_int_vectorization = use_int_vectorization self.embeddings = embeddings - # Integrated vectorization uses the ada-002 model with 1536 dimensions - # TODO: Update integrated vectorization too! self.embedding_dimensions = self.embeddings.open_ai_dimensions if self.embeddings else None - self.embedding_field = embedding_field + self.field_name_embedding = field_name_embedding + self.field_name_image_embedding = field_name_image_embedding self.search_images = search_images async def create_index(self): @@ -76,28 +79,60 @@ async def create_index(self): async with self.search_info.create_search_index_client() as search_index_client: - vectorizer = None embedding_field = None - if self.embeddings and isinstance(self.embeddings, AzureOpenAIEmbeddingService): - vectorizer = AzureOpenAIVectorizer( - vectorizer_name=f"{self.search_info.index_name}-vectorizer", - parameters=AzureOpenAIVectorizerParameters( - resource_url=self.embeddings.open_ai_endpoint, - deployment_name=self.embeddings.open_ai_deployment, - model_name=self.embeddings.open_ai_model_name, - ), - ) + image_embedding_field = None + text_vector_search_profile = None + text_vector_algorithm = None + text_vector_compression = None + image_vector_search_profile = None + image_vector_algorithm = None + if self.embeddings: if self.embedding_dimensions is None: raise ValueError( "Embedding dimensions must be set in order to add an embedding field to the search index" ) - if self.embedding_field is None: + if self.field_name_embedding is None: raise ValueError( "Embedding field must be set in order to add an embedding field to the search index" ) + + text_vectorizer = None + if isinstance(self.embeddings, AzureOpenAIEmbeddingService): + text_vectorizer = AzureOpenAIVectorizer( + vectorizer_name=f"{self.embeddings.open_ai_model_name}-vectorizer", + parameters=AzureOpenAIVectorizerParameters( + resource_url=self.embeddings.open_ai_endpoint, + deployment_name=self.embeddings.open_ai_deployment, + model_name=self.embeddings.open_ai_model_name, + ), + ) + + text_vector_algorithm = HnswAlgorithmConfiguration( + name="hnsw_config", + parameters=HnswParameters(metric="cosine"), + ) + text_vector_compression = BinaryQuantizationCompression( + compression_name=f"{self.field_name_embedding}-compression", + truncation_dimension=1024, # should this be a parameter? maybe not yet? + rescoring_options=RescoringOptions( + enable_rescoring=True, + default_oversampling=10, + rescore_storage_method=VectorSearchCompressionRescoreStorageMethod.PRESERVE_ORIGINALS, + ), + # Explicitly set deprecated parameters to None + rerank_with_original_vectors=None, + default_oversampling=None, + ) + text_vector_search_profile = VectorSearchProfile( + name=f"{self.field_name_embedding}-profile", + algorithm_configuration_name=text_vector_algorithm.name, + compression_name=text_vector_compression.compression_name, + **({"vectorizer_name": text_vectorizer.vectorizer_name if text_vectorizer else None}), + ) + embedding_field = SearchField( - name=self.embedding_field, + name=self.field_name_embedding, type=SearchFieldDataType.Collection(SearchFieldDataType.Single), hidden=True, searchable=True, @@ -105,10 +140,35 @@ async def create_index(self): sortable=False, facetable=False, vector_search_dimensions=self.embedding_dimensions, - vector_search_profile_name="embedding_config", + vector_search_profile_name=f"{self.field_name_embedding}-profile", stored=False, ) + if self.search_images: + if self.field_name_image_embedding is None: + raise ValueError( + "Image embedding field must be set in order to add an image embedding field to the search index" + ) + image_vector_algorithm = HnswAlgorithmConfiguration( + name="image_hnsw_config", + parameters=HnswParameters(metric="cosine"), + ) + image_vector_search_profile = VectorSearchProfile( + name=f"{self.field_name_image_embedding}-profile", + algorithm_configuration_name=image_vector_algorithm.name, + ) + image_embedding_field = SearchField( + name=self.field_name_image_embedding, + type=SearchFieldDataType.Collection(SearchFieldDataType.Single), + hidden=False, + searchable=True, + filterable=False, + sortable=False, + facetable=False, + vector_search_dimensions=1024, + vector_search_profile_name=image_vector_search_profile.name, + ) + if self.search_info.index_name not in [name async for name in search_index_client.list_index_names()]: logger.info("Creating new search index %s", self.search_info.index_name) fields = [ @@ -165,71 +225,37 @@ async def create_index(self): filterable=True, ) ) + if self.use_int_vectorization: - logger.info("Including parent_id field in new index %s", self.search_info.index_name) + logger.info("Including parent_id field for integrated vectorization support in new index") fields.append(SearchableField(name="parent_id", type="Edm.String", filterable=True)) - if self.search_images: - logger.info("Including imageEmbedding field in new index %s", self.search_info.index_name) - fields.append( - SearchField( - name="imageEmbedding", - type=SearchFieldDataType.Collection(SearchFieldDataType.Single), - hidden=False, - searchable=True, - filterable=False, - sortable=False, - facetable=False, - vector_search_dimensions=1024, - vector_search_profile_name="embedding_config", - ), - ) - vector_search = None - if self.embeddings and embedding_field: - logger.info("Including embedding field in new index %s", self.search_info.index_name) + vectorizers: List[VectorSearchVectorizer] = [] + vector_search_profiles = [] + vector_algorithms: list[VectorSearchAlgorithmConfiguration] = [] + vector_compressions: list[VectorSearchCompression] = [] + if embedding_field: + logger.info("Including %s field for text vectors in new index", embedding_field.name) fields.append(embedding_field) + if text_vectorizer is not None: + vectorizers.append(text_vectorizer) + if ( + text_vector_search_profile is None + or text_vector_algorithm is None + or text_vector_compression is None + ): + raise ValueError("Text vector search profile, algorithm and compression must be set") + vector_search_profiles.append(text_vector_search_profile) + vector_algorithms.append(text_vector_algorithm) + vector_compressions.append(text_vector_compression) - vectorizers = None - if vectorizer is not None: - logger.info("Including vectorizer in new index %s", self.search_info.index_name) - vectorizers = [vectorizer] - else: - logger.info( - "New index %s will not have vectorizer, since no Azure OpenAI service is set", - self.search_info.index_name, - ) - - vector_search = VectorSearch( - profiles=[ - VectorSearchProfile( - name="embedding_config", - algorithm_configuration_name="hnsw_config", - compression_name="binary-quantization", - **({"vectorizer_name": vectorizer.vectorizer_name if vectorizer else None}), - ), - ], - algorithms=[ - HnswAlgorithmConfiguration( - name="hnsw_config", - parameters=HnswParameters(metric="cosine"), - ) - ], - vectorizers=vectorizers, - compressions=[ - BinaryQuantizationCompression( - compression_name="binary-quantization", - truncation_dimension=1024, - rescoring_options=RescoringOptions( - enable_rescoring=True, - default_oversampling=10, - rescore_storage_method=VectorSearchCompressionRescoreStorageMethod.PRESERVE_ORIGINALS, - ), - # Explicitly set deprecated parameters to None - rerank_with_original_vectors=None, - default_oversampling=None, - ) - ], - ) + if image_embedding_field: + logger.info("Including %s field for image vectors in new index", image_embedding_field.name) + fields.append(image_embedding_field) + if image_vector_search_profile is None or image_vector_algorithm is None: + raise ValueError("Image search profile and algorithm must be set") + vector_search_profiles.append(image_vector_search_profile) + vector_algorithms.append(image_vector_algorithm) index = SearchIndex( name=self.search_info.index_name, @@ -244,7 +270,12 @@ async def create_index(self): ) ] ), - vector_search=vector_search, + vector_search=VectorSearch( + profiles=vector_search_profiles, + algorithms=vector_algorithms, + compressions=vector_compressions, + vectorizers=vectorizers, + ), ) await search_index_client.create_index(index) @@ -262,24 +293,51 @@ async def create_index(self): ), ) await search_index_client.create_or_update_index(existing_index) - # check if embedding field exists - TODO: will this really work if we havent redfined vector search? - if self.embeddings and not any(field.name == self.embedding_field for field in existing_index.fields): - logger.info("Adding embedding field to index %s", self.search_info.index_name) + + if embedding_field and not any( + field.name == self.field_name_embedding for field in existing_index.fields + ): + logger.info("Adding %s field for text embeddings", self.field_name_embedding) existing_index.fields.append(embedding_field) + if existing_index.vector_search is None: + raise ValueError("Vector search is not enabled for the existing index") + if text_vectorizer is not None: + if existing_index.vector_search.vectorizers is None: + existing_index.vector_search.vectorizers = [] + existing_index.vector_search.vectorizers.append(text_vectorizer) + if ( + text_vector_search_profile is None + or text_vector_algorithm is None + or text_vector_compression is None + ): + raise ValueError("Text vector search profile, algorithm and compression must be set") + if existing_index.vector_search.profiles is None: + existing_index.vector_search.profiles = [] + existing_index.vector_search.profiles.append(text_vector_search_profile) + if existing_index.vector_search.algorithms is None: + existing_index.vector_search.algorithms = [] + existing_index.vector_search.algorithms.append(text_vector_algorithm) + if existing_index.vector_search.compressions is None: + existing_index.vector_search.compressions = [] + existing_index.vector_search.compressions.append(text_vector_compression) await search_index_client.create_or_update_index(existing_index) - if existing_index.vector_search is not None and ( - existing_index.vector_search.vectorizers is None - or len(existing_index.vector_search.vectorizers) == 0 + + if image_embedding_field and not any( + field.name == self.field_name_image_embedding for field in existing_index.fields ): - if self.embeddings is not None and isinstance(self.embeddings, AzureOpenAIEmbeddingService): - logger.info("Adding vectorizer to search index %s", self.search_info.index_name) - existing_index.vector_search.vectorizers = [vectorizer] - await search_index_client.create_or_update_index(existing_index) - else: - logger.info( - "Search index %s will not have vectorizer, since no Azure OpenAI service is set", - self.search_info.index_name, - ) + logger.info("Adding %s field for image embeddings", image_embedding_field.name) + existing_index.fields.append(image_embedding_field) + if image_vector_search_profile is None or image_vector_algorithm is None: + raise ValueError("Image vector search profile and algorithm must be set") + if existing_index.vector_search is None: + raise ValueError("Image vector search is not enabled for the existing index") + if existing_index.vector_search.profiles is None: + existing_index.vector_search.profiles = [] + existing_index.vector_search.profiles.append(image_vector_search_profile) + if existing_index.vector_search.algorithms is None: + existing_index.vector_search.algorithms = [] + existing_index.vector_search.algorithms.append(image_vector_algorithm) + await search_index_client.create_or_update_index(existing_index) async def update_content( self, sections: List[Section], image_embeddings: Optional[List[List[float]]] = None, url: Optional[str] = None @@ -314,14 +372,18 @@ async def update_content( for document in documents: document["storageUrl"] = url if self.embeddings: + if self.field_name_embedding is None: + raise ValueError("Embedding field name must be set") embeddings = await self.embeddings.create_embeddings( texts=[section.split_page.text for section in batch] ) for i, document in enumerate(documents): - document[self.embedding_field] = embeddings[i] + document[self.field_name_embedding] = embeddings[i] if image_embeddings: + if self.field_name_image_embedding is None: + raise ValueError("Image embedding field name must be set") for i, (document, section) in enumerate(zip(documents, batch)): - document["imageEmbedding"] = image_embeddings[section.split_page.page_num] + document[self.field_name_image_embedding] = image_embeddings[section.split_page.page_num] await search_client.upload_documents(documents) From bfb74e647dfa71ee8ca2635463cabccdb98011a8 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Tue, 1 Apr 2025 16:21:50 -0700 Subject: [PATCH 08/11] Fix tests, add parameter --- app/backend/app.py | 7 ++++++- app/backend/prepdocs.py | 5 +++-- app/backend/prepdocslib/filestrategy.py | 15 +++++++++++++- infra/main.bicep | 3 +++ infra/main.parameters.json | 3 +++ tests/conftest.py | 8 ++++++++ tests/test_app_config.py | 2 ++ tests/test_chatapproach.py | 3 +++ tests/test_chatvisionapproach.py | 3 ++- tests/test_searchmanager.py | 26 +++++++++++++++++++------ 10 files changed, 64 insertions(+), 11 deletions(-) diff --git a/app/backend/app.py b/app/backend/app.py index e781dee240..4798d9de69 100644 --- a/app/backend/app.py +++ b/app/backend/app.py @@ -458,6 +458,7 @@ async def setup_clients(): AZURE_SEARCH_QUERY_REWRITING = os.getenv("AZURE_SEARCH_QUERY_REWRITING", "false").lower() # This defaults to the previous field name "embedding", for backwards compatibility AZURE_SEARCH_FIELD_NAME_EMBEDDING = os.getenv("AZURE_SEARCH_FIELD_NAME_EMBEDDING", "embedding") + AZURE_SEARCH_FIELD_NAME_IMAGE_EMBEDDING = os.getenv("AZURE_SEARCH_FIELD_NAME_IMAGE_EMBEDDING", "imageEmbedding") AZURE_SPEECH_SERVICE_ID = os.getenv("AZURE_SPEECH_SERVICE_ID") AZURE_SPEECH_SERVICE_LOCATION = os.getenv("AZURE_SPEECH_SERVICE_LOCATION") @@ -574,7 +575,11 @@ async def setup_clients(): disable_vectors=os.getenv("USE_VECTORS", "").lower() == "false", ) ingester = UploadUserFileStrategy( - search_info=search_info, embeddings=text_embeddings_service, file_processors=file_processors + search_info=search_info, + embeddings=text_embeddings_service, + file_processors=file_processors, + search_field_name_embedding=AZURE_SEARCH_FIELD_NAME_EMBEDDING, + search_field_name_image_embedding=AZURE_SEARCH_FIELD_NAME_IMAGE_EMBEDDING, ) current_app.config[CONFIG_INGESTER] = ingester diff --git a/app/backend/prepdocs.py b/app/backend/prepdocs.py index 35e50ce6e8..7f7d0ab5bc 100644 --- a/app/backend/prepdocs.py +++ b/app/backend/prepdocs.py @@ -432,8 +432,9 @@ async def main(strategy: Strategy, setup_index: bool = True): embeddings=openai_embeddings_service, image_embeddings=image_embeddings_service, search_analyzer_name=os.getenv("AZURE_SEARCH_ANALYZER_NAME"), - search_field_name_embedding=os.getenv("AZURE_SEARCH_FIELD_NAME_EMBEDDING"), - search_field_name_image_embedding=os.getenv("AZURE_SEARCH_FIELD_NAME_IMAGE_EMBEDDING"), + # Default to the previous field names for backward compatibility + search_field_name_embedding=os.getenv("AZURE_SEARCH_FIELD_NAME_EMBEDDING", "embedding"), + search_field_name_image_embedding=os.getenv("AZURE_SEARCH_FIELD_NAME_IMAGE_EMBEDDING", "imageEmbedding"), use_acls=use_acls, category=args.category, use_content_understanding=use_content_understanding, diff --git a/app/backend/prepdocslib/filestrategy.py b/app/backend/prepdocslib/filestrategy.py index 8dd28427ae..4c88de94d2 100644 --- a/app/backend/prepdocslib/filestrategy.py +++ b/app/backend/prepdocslib/filestrategy.py @@ -136,12 +136,25 @@ def __init__( file_processors: dict[str, FileProcessor], embeddings: Optional[OpenAIEmbeddings] = None, image_embeddings: Optional[ImageEmbeddings] = None, + search_field_name_embedding: Optional[str] = None, + search_field_name_image_embedding: Optional[str] = None, ): self.file_processors = file_processors self.embeddings = embeddings self.image_embeddings = image_embeddings self.search_info = search_info - self.search_manager = SearchManager(self.search_info, None, True, False, self.embeddings) + self.search_manager = SearchManager( + search_info=self.search_info, + search_analyzer_name=None, + use_acls=True, + use_int_vectorization=False, + embeddings=self.embeddings, + field_name_embedding=search_field_name_embedding, + field_name_image_embedding=search_field_name_image_embedding, + search_images=False, + ) + self.search_field_name_embedding = search_field_name_embedding + self.search_field_name_image_embedding = search_field_name_image_embedding async def add_file(self, file: File): if self.image_embeddings: diff --git a/infra/main.bicep b/infra/main.bicep index 26160e2c65..550a79dc49 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -28,6 +28,7 @@ param searchQueryLanguage string // Set in main.parameters.json param searchQuerySpeller string // Set in main.parameters.json param searchServiceSemanticRankerLevel string // Set in main.parameters.json param searchFieldNameEmbedding string // Set in main.parameters.json +param searchFieldNameImageEmbedding string // Set in main.parameters.json var actualSearchServiceSemanticRankerLevel = (searchServiceSkuName == 'free') ? 'disabled' : searchServiceSemanticRankerLevel @@ -392,6 +393,7 @@ var appEnvVariables = { AZURE_SEARCH_QUERY_LANGUAGE: searchQueryLanguage AZURE_SEARCH_QUERY_SPELLER: searchQuerySpeller AZURE_SEARCH_FIELD_NAME_EMBEDDING: searchFieldNameEmbedding + AZURE_SEARCH_FIELD_NAME_IMAGE_EMBEDDING: searchFieldNameImageEmbedding APPLICATIONINSIGHTS_CONNECTION_STRING: useApplicationInsights ? monitoring.outputs.applicationInsightsConnectionString : '' @@ -1288,6 +1290,7 @@ output AZURE_SEARCH_SERVICE_RESOURCE_GROUP string = searchServiceResourceGroup.n output AZURE_SEARCH_SEMANTIC_RANKER string = actualSearchServiceSemanticRankerLevel output AZURE_SEARCH_SERVICE_ASSIGNED_USERID string = searchService.outputs.principalId output AZURE_SEARCH_FIELD_NAME_EMBEDDING string = searchFieldNameEmbedding +output AZURE_SEARCH_FIELD_NAME_IMAGE_EMBEDDING string = searchFieldNameEmbedding output AZURE_COSMOSDB_ACCOUNT string = (useAuthentication && useChatHistoryCosmos) ? cosmosDb.outputs.name : '' output AZURE_CHAT_HISTORY_DATABASE string = chatHistoryDatabaseName diff --git a/infra/main.parameters.json b/infra/main.parameters.json index a6527bcc5d..4f73f95222 100644 --- a/infra/main.parameters.json +++ b/infra/main.parameters.json @@ -86,6 +86,9 @@ "searchFieldNameEmbedding": { "value": "${AZURE_SEARCH_FIELD_NAME_EMBEDDING=embedding3}" }, + "searchFieldNameImageEmbedding": { + "value": "${AZURE_SEARCH_FIELD_NAME_IMAGE_EMBEDDING=imageEmbedding}" + }, "storageAccountName": { "value": "${AZURE_STORAGE_ACCOUNT}" }, diff --git a/tests/conftest.py b/tests/conftest.py index f0885aa769..7cb44a654a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -246,12 +246,16 @@ def mock_blob_container_client(monkeypatch): "OPENAI_HOST": "openai", "OPENAI_API_KEY": "secretkey", "OPENAI_ORGANIZATION": "organization", + "AZURE_OPENAI_EMB_MODEL_NAME": "text-embedding-3-large", + "AZURE_OPENAI_EMB_DIMENSIONS": "3072", }, { "OPENAI_HOST": "azure", "AZURE_OPENAI_SERVICE": "test-openai-service", "AZURE_OPENAI_CHATGPT_DEPLOYMENT": "test-chatgpt", "AZURE_OPENAI_EMB_DEPLOYMENT": "test-ada", + "AZURE_OPENAI_EMB_MODEL_NAME": "text-embedding-3-large", + "AZURE_OPENAI_EMB_DIMENSIONS": "3072", "USE_GPT4V": "true", "AZURE_OPENAI_GPT4V_MODEL": "gpt-4", "VISION_ENDPOINT": "https://testvision.cognitiveservices.azure.com/", @@ -264,6 +268,8 @@ def mock_blob_container_client(monkeypatch): "AZURE_OPENAI_SERVICE": "test-openai-service", "AZURE_OPENAI_CHATGPT_DEPLOYMENT": "test-chatgpt", "AZURE_OPENAI_EMB_DEPLOYMENT": "test-ada", + "AZURE_OPENAI_EMB_MODEL_NAME": "text-embedding-3-large", + "AZURE_OPENAI_EMB_DIMENSIONS": "3072", "AZURE_USE_AUTHENTICATION": "true", "AZURE_USER_STORAGE_ACCOUNT": "test-user-storage-account", "AZURE_USER_STORAGE_CONTAINER": "test-user-storage-container", @@ -280,6 +286,8 @@ def mock_blob_container_client(monkeypatch): "AZURE_OPENAI_SERVICE": "test-openai-service", "AZURE_OPENAI_CHATGPT_DEPLOYMENT": "test-chatgpt", "AZURE_OPENAI_EMB_DEPLOYMENT": "test-ada", + "AZURE_OPENAI_EMB_MODEL_NAME": "text-embedding-3-large", + "AZURE_OPENAI_EMB_DIMENSIONS": "3072", "AZURE_USE_AUTHENTICATION": "true", "AZURE_ENABLE_GLOBAL_DOCUMENT_ACCESS": "true", "AZURE_ENABLE_UNAUTHENTICATED_ACCESS": "true", diff --git a/tests/test_app_config.py b/tests/test_app_config.py index f5fa64c5ae..8f820bd3a5 100644 --- a/tests/test_app_config.py +++ b/tests/test_app_config.py @@ -16,6 +16,8 @@ def minimal_env(monkeypatch): monkeypatch.setenv("AZURE_SEARCH_SERVICE", "test-search-service") monkeypatch.setenv("AZURE_OPENAI_SERVICE", "test-openai-service") monkeypatch.setenv("AZURE_OPENAI_CHATGPT_MODEL", "gpt-4o-mini") + monkeypatch.setenv("AZURE_OPENAI_EMB_MODEL_NAME", "text-embedding-3-large") + monkeypatch.setenv("AZURE_OPENAI_EMB_DIMENSIONS", 3072) yield diff --git a/tests/test_chatapproach.py b/tests/test_chatapproach.py index a12c3e4147..9900ae88bc 100644 --- a/tests/test_chatapproach.py +++ b/tests/test_chatapproach.py @@ -30,6 +30,7 @@ def chat_approach(): embedding_deployment="embeddings", embedding_model=MOCK_EMBEDDING_MODEL_NAME, embedding_dimensions=MOCK_EMBEDDING_DIMENSIONS, + embedding_field="embedding3", sourcepage_field="", content_field="", query_language="en-us", @@ -176,6 +177,7 @@ async def test_search_results_filtering_by_scores( embedding_deployment="embeddings", embedding_model=MOCK_EMBEDDING_MODEL_NAME, embedding_dimensions=MOCK_EMBEDDING_DIMENSIONS, + embedding_field="embedding3", sourcepage_field="", content_field="", query_language="en-us", @@ -214,6 +216,7 @@ async def test_search_results_query_rewriting(monkeypatch): embedding_deployment="embeddings", embedding_model=MOCK_EMBEDDING_MODEL_NAME, embedding_dimensions=MOCK_EMBEDDING_DIMENSIONS, + embedding_field="embedding3", sourcepage_field="", content_field="", query_language="en-us", diff --git a/tests/test_chatvisionapproach.py b/tests/test_chatvisionapproach.py index d2c450efca..7039cae395 100644 --- a/tests/test_chatvisionapproach.py +++ b/tests/test_chatvisionapproach.py @@ -60,6 +60,7 @@ def chat_approach(openai_client, mock_confidential_client_success): embedding_deployment="embeddings", embedding_model=MOCK_EMBEDDING_MODEL_NAME, embedding_dimensions=MOCK_EMBEDDING_DIMENSIONS, + embedding_field="embedding3", sourcepage_field="", content_field="", query_language="en-us", @@ -149,4 +150,4 @@ async def test_compute_text_embedding(chat_approach, openai_client, mock_openai_ assert isinstance(result, VectorizedQuery) assert result.vector == [0.0023064255, -0.009327292, -0.0028842222] assert result.k_nearest_neighbors == 50 - assert result.fields == "embedding" + assert result.fields == "embedding3" diff --git a/tests/test_searchmanager.py b/tests/test_searchmanager.py index 1c5a98be85..a59e5d3623 100644 --- a/tests/test_searchmanager.py +++ b/tests/test_searchmanager.py @@ -50,11 +50,16 @@ async def mock_list_index_names(self): monkeypatch.setattr(SearchIndexClient, "create_index", mock_create_index) monkeypatch.setattr(SearchIndexClient, "list_index_names", mock_list_index_names) - manager = SearchManager(search_info) + manager = SearchManager( + search_info, + use_int_vectorization=False, + field_name_embedding="embedding", + field_name_image_embedding="imageEmbedding", + ) await manager.create_index() assert len(indexes) == 1, "It should have created one index" assert indexes[0].name == "test" - assert len(indexes[0].fields) == 7 + assert len(indexes[0].fields) == 6 @pytest.mark.asyncio @@ -71,11 +76,16 @@ async def mock_list_index_names(self): monkeypatch.setattr(SearchIndexClient, "create_index", mock_create_index) monkeypatch.setattr(SearchIndexClient, "list_index_names", mock_list_index_names) - manager = SearchManager(search_info, use_int_vectorization=True) + manager = SearchManager( + search_info, + use_int_vectorization=True, + field_name_embedding="embedding", + field_name_image_embedding="image_embedding", + ) await manager.create_index() assert len(indexes) == 1, "It should have created one index" assert indexes[0].name == "test" - assert len(indexes[0].fields) == 8 + assert len(indexes[0].fields) == 7 @pytest.mark.asyncio @@ -165,11 +175,13 @@ async def mock_list_index_names(self): manager = SearchManager( search_info, use_acls=True, + field_name_embedding="embedding", + field_name_image_embedding="image_embedding", ) await manager.create_index() assert len(indexes) == 1, "It should have created one index" assert indexes[0].name == "test" - assert len(indexes[0].fields) == 9 + assert len(indexes[0].fields) == 8 @pytest.mark.asyncio @@ -283,6 +295,8 @@ async def mock_upload_documents(self, documents): manager = SearchManager( search_info, embeddings=embeddings, + field_name_embedding="embedding3", + field_name_image_embedding="image_embedding", ) test_io = io.BytesIO(b"test content") @@ -303,7 +317,7 @@ async def mock_upload_documents(self, documents): ) assert len(documents_uploaded) == 1, "It should have uploaded one document" - assert documents_uploaded[0]["embedding"] == [ + assert documents_uploaded[0]["embedding3"] == [ 0.0023064255, -0.009327292, -0.0028842222, From e9b822cd8faea8d446bc56ac471dea6067d11293 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Wed, 2 Apr 2025 15:22:37 -0700 Subject: [PATCH 09/11] Upgrade int vect for new embedding model --- .../integratedvectorizerstrategy.py | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/app/backend/prepdocslib/integratedvectorizerstrategy.py b/app/backend/prepdocslib/integratedvectorizerstrategy.py index ce9981260c..c80a90033f 100644 --- a/app/backend/prepdocslib/integratedvectorizerstrategy.py +++ b/app/backend/prepdocslib/integratedvectorizerstrategy.py @@ -6,7 +6,6 @@ ) from azure.search.documents.indexes.models import ( AzureOpenAIEmbeddingSkill, - FieldMapping, IndexProjectionMode, InputFieldMappingEntry, OutputFieldMappingEntry, @@ -63,15 +62,18 @@ def __init__( self.use_acls = use_acls self.category = category self.search_info = search_info + prefix = f"{self.search_info.index_name}-{self.search_field_name_embedding}" + self.skillset_name = f"{prefix}-skillset" + self.indexer_name = f"{prefix}-indexer" + self.data_source_name = f"{prefix}-blob" async def create_embedding_skill(self, index_name: str) -> SearchIndexerSkillset: """ Create a skillset for the indexer to chunk documents and generate embeddings """ - skillset_name = f"{index_name}-skillset" split_skill = SplitSkill( - name=f"{index_name}-split-skill", + name="split-skill", description="Split skill to chunk documents", text_split_mode="pages", context="/document", @@ -84,7 +86,7 @@ async def create_embedding_skill(self, index_name: str) -> SearchIndexerSkillset ) embedding_skill = AzureOpenAIEmbeddingSkill( - name=f"{index_name}-embedding-skill", + name="embedding-skill", description="Skill to generate embeddings via Azure OpenAI", context="/document/pages/*", resource_url=f"https://{self.embeddings.open_ai_service}.openai.azure.com", @@ -94,7 +96,7 @@ async def create_embedding_skill(self, index_name: str) -> SearchIndexerSkillset inputs=[ InputFieldMappingEntry(name="text", source="/document/pages/*"), ], - outputs=[OutputFieldMappingEntry(name=self.search_field_name_embedding, target_name="vector")], + outputs=[OutputFieldMappingEntry(name="embedding", target_name="vector")], ) index_projection = SearchIndexerIndexProjection( @@ -106,6 +108,8 @@ async def create_embedding_skill(self, index_name: str) -> SearchIndexerSkillset mappings=[ InputFieldMappingEntry(name="content", source="/document/pages/*"), InputFieldMappingEntry(name="sourcepage", source="/document/metadata_storage_name"), + InputFieldMappingEntry(name="sourcefile", source="/document/metadata_storage_name"), + InputFieldMappingEntry(name="storageUrl", source="/document/metadata_storage_path"), InputFieldMappingEntry( name=self.search_field_name_embedding, source="/document/pages/*/vector" ), @@ -118,7 +122,7 @@ async def create_embedding_skill(self, index_name: str) -> SearchIndexerSkillset ) skillset = SearchIndexerSkillset( - name=skillset_name, + name=self.skillset_name, description="Skillset to chunk documents and generate embeddings", skills=[split_skill, embedding_skill], index_projection=index_projection, @@ -144,7 +148,7 @@ async def setup(self): ds_client = self.search_info.create_search_indexer_client() ds_container = SearchIndexerDataContainer(name=self.blob_manager.container) data_source_connection = SearchIndexerDataSourceConnection( - name=f"{self.search_info.index_name}-blob", + name=self.data_source_name, type=SearchIndexerDataSourceType.AZURE_BLOB, connection_string=self.blob_manager.get_managedidentity_connectionstring(), container=ds_container, @@ -174,23 +178,19 @@ async def run(self): await self.blob_manager.remove_blob() # Create an indexer - indexer_name = f"{self.search_info.index_name}-indexer" - indexer = SearchIndexer( - name=indexer_name, + name=self.indexer_name, description="Indexer to index documents and generate embeddings", - skillset_name=f"{self.search_info.index_name}-skillset", + skillset_name=self.skillset_name, target_index_name=self.search_info.index_name, - data_source_name=f"{self.search_info.index_name}-blob", - # Map the metadata_storage_name field to the title field in the index to display the PDF title in the search results - field_mappings=[FieldMapping(source_field_name="metadata_storage_name", target_field_name="title")], + data_source_name=self.data_source_name, ) indexer_client = self.search_info.create_search_indexer_client() indexer_result = await indexer_client.create_or_update_indexer(indexer) # Run the indexer - await indexer_client.run_indexer(indexer_name) + await indexer_client.run_indexer(self.indexer_name) await indexer_client.close() logger.info( From 78cd4c1bd000f194170d15486624a6f752aa9f47 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Wed, 2 Apr 2025 15:59:32 -0700 Subject: [PATCH 10/11] Add missing env vars in other files --- .azdo/pipelines/azure-dev.yml | 2 ++ .github/workflows/azure-dev.yml | 2 ++ azure.yaml | 2 ++ 3 files changed, 6 insertions(+) diff --git a/.azdo/pipelines/azure-dev.yml b/.azdo/pipelines/azure-dev.yml index b500d40e03..fb869dd1d1 100644 --- a/.azdo/pipelines/azure-dev.yml +++ b/.azdo/pipelines/azure-dev.yml @@ -60,6 +60,8 @@ steps: AZURE_SEARCH_QUERY_SPELLER: $(AZURE_SEARCH_QUERY_SPELLER) AZURE_SEARCH_SEMANTIC_RANKER: $(AZURE_SEARCH_SEMANTIC_RANKER) AZURE_SEARCH_QUERY_REWRITING: $(AZURE_SEARCH_QUERY_REWRITING) + AZURE_SEARCH_FIELD_NAME_EMBEDDING: $(AZURE_SEARCH_FIELD_NAME_EMBEDDING) + AZURE_SEARCH_FIELD_NAME_IMAGE_EMBEDDING: $(AZURE_SEARCH_FIELD_NAME_IMAGE_EMBEDDING) AZURE_STORAGE_ACCOUNT: $(AZURE_STORAGE_ACCOUNT) AZURE_STORAGE_RESOURCE_GROUP: $(AZURE_STORAGE_RESOURCE_GROUP) AZURE_STORAGE_SKU: $(AZURE_STORAGE_SKU) diff --git a/.github/workflows/azure-dev.yml b/.github/workflows/azure-dev.yml index d20cc20f90..1f2f5ddccd 100644 --- a/.github/workflows/azure-dev.yml +++ b/.github/workflows/azure-dev.yml @@ -50,6 +50,8 @@ jobs: AZURE_SEARCH_QUERY_SPELLER: ${{ vars.AZURE_SEARCH_QUERY_SPELLER }} AZURE_SEARCH_SEMANTIC_RANKER: ${{ vars.AZURE_SEARCH_SEMANTIC_RANKER }} AZURE_SEARCH_QUERY_REWRITING: ${{ vars.AZURE_SEARCH_QUERY_REWRITING }} + AZURE_SEARCH_FIELD_NAME_EMBEDDING: ${{ vars.AZURE_SEARCH_FIELD_NAME_EMBEDDING }} + AZURE_SEARCH_FIELD_NAME_IMAGE_EMBEDDING: ${{ vars.AZURE_SEARCH_FIELD_NAME_IMAGE_EMBEDDING }} AZURE_STORAGE_ACCOUNT: ${{ vars.AZURE_STORAGE_ACCOUNT }} AZURE_STORAGE_RESOURCE_GROUP: ${{ vars.AZURE_STORAGE_RESOURCE_GROUP }} AZURE_STORAGE_SKU: ${{ vars.AZURE_STORAGE_SKU }} diff --git a/azure.yaml b/azure.yaml index 0793545f3a..f0306b08de 100644 --- a/azure.yaml +++ b/azure.yaml @@ -57,6 +57,8 @@ pipeline: - AZURE_SEARCH_QUERY_SPELLER - AZURE_SEARCH_SEMANTIC_RANKER - AZURE_SEARCH_QUERY_REWRITING + - AZURE_SEARCH_FIELD_NAME_EMBEDDING + - AZURE_SEARCH_FIELD_NAME_IMAGE_EMBEDDING - AZURE_STORAGE_ACCOUNT - AZURE_STORAGE_RESOURCE_GROUP - AZURE_STORAGE_SKU From 14713d292f8a9a7339c1e40be45196abb343704d Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Wed, 2 Apr 2025 16:32:17 -0700 Subject: [PATCH 11/11] Remove en-us from markdown --- docs/deploy_features.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/deploy_features.md b/docs/deploy_features.md index c7107fc176..8084388b23 100644 --- a/docs/deploy_features.md +++ b/docs/deploy_features.md @@ -194,7 +194,7 @@ By default, the deployed Azure web app uses the `text-embedding-3-large` embeddi azd env set AZURE_OPENAI_EMB_DEPLOYMENT_SKU Standard ``` -5. When prompted during `azd up`, make sure to select a region for the OpenAI resource group location that supports the desired embedding model and deployment SKU. There are [limited regions available](https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models?tabs=global-standard%2Cstandard-chat-completions#models-by-deployment-type). +5. When prompted during `azd up`, make sure to select a region for the OpenAI resource group location that supports the desired embedding model and deployment SKU. There are [limited regions available](https://learn.microsoft.com/azure/ai-services/openai/concepts/models?tabs=global-standard%2Cstandard-chat-completions#models-by-deployment-type). If you have already deployed: