Jhakulin/image input for assistants

sdk/ai/azure-ai-projects/apiview-properties.json

@@ -60,7 +60,13 @@
         "azure.ai.projects.models.MessageDeltaTextUrlCitationDetails": "Azure.AI.Projects.Agents.MessageDeltaTextUrlCitationDetails",
         "azure.ai.projects.models.MessageImageFileContent": "Azure.AI.Projects.Agents.MessageImageFileContent",
         "azure.ai.projects.models.MessageImageFileDetails": "Azure.AI.Projects.Agents.MessageImageFileDetails",
+        "azure.ai.projects.models.MessageImageFileParam": "Azure.AI.Projects.Agents.MessageImageFileParam",
+        "azure.ai.projects.models.MessageImageUrlParam": "Azure.AI.Projects.Agents.MessageImageUrlParam",
         "azure.ai.projects.models.MessageIncompleteDetails": "Azure.AI.Projects.Agents.MessageIncompleteDetails",
+        "azure.ai.projects.models.MessageInputContentBlock": "Azure.AI.Projects.Agents.MessageInputContentBlock",
+        "azure.ai.projects.models.MessageInputImageFileBlock": "Azure.AI.Projects.Agents.MessageInputImageFileBlock",
+        "azure.ai.projects.models.MessageInputImageUrlBlock": "Azure.AI.Projects.Agents.MessageInputImageUrlBlock",
+        "azure.ai.projects.models.MessageInputTextBlock": "Azure.AI.Projects.Agents.MessageInputTextBlock",
         "azure.ai.projects.models.MessageTextAnnotation": "Azure.AI.Projects.Agents.MessageTextAnnotation",
         "azure.ai.projects.models.MessageTextContent": "Azure.AI.Projects.Agents.MessageTextContent",
         "azure.ai.projects.models.MessageTextDetails": "Azure.AI.Projects.Agents.MessageTextDetails",
@@ -155,7 +161,6 @@
         "azure.ai.projects.models.UpdateCodeInterpreterToolResourceOptions": "Azure.AI.Projects.Agents.UpdateCodeInterpreterToolResourceOptions",
         "azure.ai.projects.models.UpdateFileSearchToolResourceOptions": "Azure.AI.Projects.Agents.UpdateFileSearchToolResourceOptions",
         "azure.ai.projects.models.UpdateToolResourcesOptions": "Azure.AI.Projects.Agents.UpdateToolResourcesOptions",
-        "azure.ai.projects.models.UploadFileRequest": "Azure.AI.Projects.Agents.uploadFile.Request.anonymous",
         "azure.ai.projects.models.VectorStore": "Azure.AI.Projects.Agents.VectorStore",
         "azure.ai.projects.models.VectorStoreChunkingStrategyRequest": "Azure.AI.Projects.Agents.VectorStoreChunkingStrategyRequest",
         "azure.ai.projects.models.VectorStoreAutoChunkingStrategyRequest": "Azure.AI.Projects.Agents.VectorStoreAutoChunkingStrategyRequest",
@@ -181,6 +186,8 @@
         "azure.ai.projects.models.ResponseFormat": "Azure.AI.Projects.Agents.ResponseFormat",
         "azure.ai.projects.models.ListSortOrder": "Azure.AI.Projects.Agents.ListSortOrder",
         "azure.ai.projects.models.MessageRole": "Azure.AI.Projects.Agents.MessageRole",
+        "azure.ai.projects.models.MessageBlockType": "Azure.AI.Projects.Agents.MessageBlockType",
+        "azure.ai.projects.models.ImageDetailLevel": "Azure.AI.Projects.Agents.ImageDetailLevel",
         "azure.ai.projects.models.MessageStatus": "Azure.AI.Projects.Agents.MessageStatus",
         "azure.ai.projects.models.MessageIncompleteDetailsReason": "Azure.AI.Projects.Agents.MessageIncompleteDetailsReason",
         "azure.ai.projects.models.RunStatus": "Azure.AI.Projects.Agents.RunStatus",

sdk/ai/azure-ai-projects/azure/ai/projects/_types.py

@@ -6,7 +6,7 @@
 # Changes may cause incorrect behavior and will be lost if the code is regenerated.
 # --------------------------------------------------------------------------
 
-from typing import TYPE_CHECKING, Union
+from typing import List, TYPE_CHECKING, Union
 
 if TYPE_CHECKING:
     from . import models as _models
@@ -17,5 +17,6 @@
     "_models.AgentsApiResponseFormat",
     "_models.ResponseFormatJsonSchemaType",
 ]
+MessageInputContent = Union[str, List["_models.MessageInputContentBlock"]]
 MessageAttachmentToolDefinition = Union["_models.CodeInterpreterToolDefinition", "_models.FileSearchToolDefinition"]
 AgentsApiToolChoiceOption = Union[str, str, "_models.AgentsApiToolChoiceOptionMode", "_models.AgentsNamedToolChoice"]

sdk/ai/azure-ai-projects/azure/ai/projects/aio/operations/_operations.py

@@ -1347,7 +1347,7 @@ async def create_message(
         thread_id: str,
         *,
         role: Union[str, _models.MessageRole],
-        content: str,
+        content: "_types.MessageInputContent",
         content_type: str = "application/json",
         attachments: Optional[List[_models.MessageAttachment]] = None,
         metadata: Optional[Dict[str, str]] = None,
@@ -1359,17 +1359,18 @@ async def create_message(
         :type thread_id: str
         :keyword role: The role of the entity that is creating the message. Allowed values include:
 
-
-         * ``user``\\ : Indicates the message is sent by an actual user and should be used in most
-           cases to represent user-generated messages.
-         * ``assistant``\\ : Indicates the message is generated by the agent. Use this value to insert
-           messages from the agent into the
-           conversation. Known values are: "user" and "assistant". Required.
+         * `user`: Indicates the message is sent by an actual user and should be used in most
+         cases to represent user-generated messages.
+         * `assistant`: Indicates the message is generated by the agent. Use this value to insert
+         messages from the agent into the
+         conversation. Known values are: "user" and "assistant". Required.
         :paramtype role: str or ~azure.ai.projects.models.MessageRole
-        :keyword content: The textual content of the initial message. Currently, robust input including
-         images and annotated text may only be provided via
-         a separate call to the create message API. Required.
-        :paramtype content: str
+        :keyword content: The content of the initial message. This may be:
+
+         * A basic string, if you only need text, or
+         * An array of typed content blocks (text, image_file, image_url, etc.). Is either a str type
+         or a [MessageInputContentBlock] type. Required.
+        :paramtype content: str or list[~azure.ai.projects.models.MessageInputContentBlock]
         :keyword content_type: Body Parameter content-type. Content type parameter for JSON body.
          Default value is "application/json".
         :paramtype content_type: str
@@ -1429,7 +1430,7 @@ async def create_message(
         body: Union[JSON, IO[bytes]] = _Unset,
         *,
         role: Union[str, _models.MessageRole] = _Unset,
-        content: str = _Unset,
+        content: "_types.MessageInputContent" = _Unset,
         attachments: Optional[List[_models.MessageAttachment]] = None,
         metadata: Optional[Dict[str, str]] = None,
         **kwargs: Any
@@ -1442,16 +1443,18 @@ async def create_message(
         :type body: JSON or IO[bytes]
         :keyword role: The role of the entity that is creating the message. Allowed values include:
 
-         * ``user``\\ : Indicates the message is sent by an actual user and should be used in most
-           cases to represent user-generated messages.
-         * ``assistant``\\ : Indicates the message is generated by the agent. Use this value to insert
-           messages from the agent into the
-           conversation. Known values are: "user" and "assistant". Required.
+         * `user`: Indicates the message is sent by an actual user and should be used in most
+         cases to represent user-generated messages.
+         * `assistant`: Indicates the message is generated by the agent. Use this value to insert
+         messages from the agent into the
+         conversation. Known values are: "user" and "assistant". Required.
         :paramtype role: str or ~azure.ai.projects.models.MessageRole
-        :keyword content: The textual content of the initial message. Currently, robust input including
-         images and annotated text may only be provided via
-         a separate call to the create message API. Required.
-        :paramtype content: str
+        :keyword content: The content of the initial message. This may be:
+
+         * A basic string, if you only need text, or
+         * An array of typed content blocks (text, image_file, image_url, etc.). Is either a str type
+         or a [MessageInputContentBlock] type. Required.
+        :paramtype content: str or list[~azure.ai.projects.models.MessageInputContentBlock]
         :keyword attachments: A list of files attached to the message, and the tools they should be
          added to. Default value is None.
         :paramtype attachments: list[~azure.ai.projects.models.MessageAttachment]
@@ -3540,12 +3543,16 @@ async def upload_file(
             params=_params,
         )
         path_format_arguments = {
-            "endpoint": self._serialize.url("self._config.endpoint", self._config.endpoint, "str"),
-            "subscriptionId": self._serialize.url("self._config.subscription_id", self._config.subscription_id, "str"),
+            "endpoint": self._serialize.url("self._config.endpoint", self._config.endpoint, "str", skip_quote=True),
+            "subscriptionId": self._serialize.url(
+                "self._config.subscription_id", self._config.subscription_id, "str", skip_quote=True
+            ),
             "resourceGroupName": self._serialize.url(
-                "self._config.resource_group_name", self._config.resource_group_name, "str"
+                "self._config.resource_group_name", self._config.resource_group_name, "str", skip_quote=True
+            ),
+            "projectName": self._serialize.url(
+                "self._config.project_name", self._config.project_name, "str", skip_quote=True
             ),
-            "projectName": self._serialize.url("self._config.project_name", self._config.project_name, "str"),
         }
         _request.url = self._client.format_url(_request.url, **path_format_arguments)
 

sdk/ai/azure-ai-projects/azure/ai/projects/models/__init__.py

@@ -70,7 +70,13 @@
     MessageDeltaTextUrlCitationDetails,
     MessageImageFileContent,
     MessageImageFileDetails,
+    MessageImageFileParam,
+    MessageImageUrlParam,
     MessageIncompleteDetails,
+    MessageInputContentBlock,
+    MessageInputImageFileBlock,
+    MessageInputImageUrlBlock,
+    MessageInputTextBlock,
     MessageTextAnnotation,
     MessageTextContent,
     MessageTextDetails,
@@ -168,7 +174,6 @@
     UpdateCodeInterpreterToolResourceOptions,
     UpdateFileSearchToolResourceOptions,
     UpdateToolResourcesOptions,
-    UploadFileRequest,
     VectorStore,
     VectorStoreAutoChunkingStrategyRequest,
     VectorStoreAutoChunkingStrategyResponse,
@@ -202,8 +207,10 @@
     FilePurpose,
     FileState,
     Frequency,
+    ImageDetailLevel,
     IncompleteDetailsReason,
     ListSortOrder,
+    MessageBlockType,
     MessageIncompleteDetailsReason,
     MessageRole,
     MessageStatus,
@@ -291,7 +298,13 @@
     "MessageDeltaTextUrlCitationDetails",
     "MessageImageFileContent",
     "MessageImageFileDetails",
+    "MessageImageFileParam",
+    "MessageImageUrlParam",
     "MessageIncompleteDetails",
+    "MessageInputContentBlock",
+    "MessageInputImageFileBlock",
+    "MessageInputImageUrlBlock",
+    "MessageInputTextBlock",
     "MessageTextAnnotation",
     "MessageTextContent",
     "MessageTextDetails",
@@ -389,7 +402,6 @@
     "UpdateCodeInterpreterToolResourceOptions",
     "UpdateFileSearchToolResourceOptions",
     "UpdateToolResourcesOptions",
-    "UploadFileRequest",
     "VectorStore",
     "VectorStoreAutoChunkingStrategyRequest",
     "VectorStoreAutoChunkingStrategyResponse",
@@ -420,8 +432,10 @@
     "FilePurpose",
     "FileState",
     "Frequency",
+    "ImageDetailLevel",
     "IncompleteDetailsReason",
     "ListSortOrder",
+    "MessageBlockType",
     "MessageIncompleteDetailsReason",
     "MessageRole",
     "MessageStatus",

sdk/ai/azure-ai-projects/azure/ai/projects/models/_enums.py

@@ -251,6 +251,17 @@ class Frequency(str, Enum, metaclass=CaseInsensitiveEnumMeta):
     MINUTE = "Minute"
 
 
+class ImageDetailLevel(str, Enum, metaclass=CaseInsensitiveEnumMeta):
+    """Specifies an image's detail level. Can be 'auto', 'low', 'high', or an unknown future value."""
+
+    AUTO = "auto"
+    """Automatically select an appropriate detail level."""
+    LOW = "low"
+    """Use a lower detail level to reduce bandwidth or cost."""
+    HIGH = "high"
+    """Use a higher detail level—potentially more resource-intensive."""
+
+
 class IncompleteDetailsReason(str, Enum, metaclass=CaseInsensitiveEnumMeta):
     """The reason why the run is incomplete. This will point to which specific token limit was reached
     over the course of the run.
@@ -271,6 +282,19 @@ class ListSortOrder(str, Enum, metaclass=CaseInsensitiveEnumMeta):
     """Specifies a descending sort order."""
 
 
+class MessageBlockType(str, Enum, metaclass=CaseInsensitiveEnumMeta):
+    """Specifies the kind of content block within a message. Could be text, an image file, an external
+    image URL, or an unknown future type.
+    """
+
+    TEXT = "text"
+    """Indicates a block containing text content."""
+    IMAGE_FILE = "image_file"
+    """Indicates a block referencing an internally uploaded image file."""
+    IMAGE_URL = "image_url"
+    """Indicates a block referencing an external image URL."""
+
+
 class MessageIncompleteDetailsReason(str, Enum, metaclass=CaseInsensitiveEnumMeta):
     """A set of reasons describing why a message is marked as incomplete."""