From 1c396c95b040fb3d1a2523b09eaad4ff62d96846 Mon Sep 17 00:00:00 2001 From: Vincent Date: Mon, 16 Dec 2024 13:12:53 +0100 Subject: [PATCH 1/9] fix(cli/migrate): change grit binaries prefix (#1951) --- src/openai/cli/_tools/migrate.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/openai/cli/_tools/migrate.py b/src/openai/cli/_tools/migrate.py index 7a0b0f90f6..841b777528 100644 --- a/src/openai/cli/_tools/migrate.py +++ b/src/openai/cli/_tools/migrate.py @@ -92,8 +92,8 @@ def install() -> Path: install_dir = dir_name / ".install" target_dir = install_dir / "bin" - target_path = target_dir / "marzano" - temp_file = target_dir / "marzano.tmp" + target_path = target_dir / "grit" + temp_file = target_dir / "grit.tmp" if target_path.exists(): _debug(f"{target_path} already exists") @@ -110,7 +110,7 @@ def install() -> Path: arch = _get_arch() _debug(f"Using architecture {arch}") - file_name = f"marzano-{arch}-{platform}" + file_name = f"grit-{arch}-{platform}" download_url = f"https://github.com/getgrit/gritql/releases/latest/download/{file_name}.tar.gz" sys.stdout.write(f"Downloading Grit CLI from {download_url}\n") From 9a4c69c383bc6719b6521a485f2c7e62a9c036a9 Mon Sep 17 00:00:00 2001 From: Robert Craigie Date: Mon, 16 Dec 2024 12:22:00 +0000 Subject: [PATCH 2/9] fix(assistants): correctly send `include` query param --- src/openai/resources/beta/threads/runs/runs.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/openai/resources/beta/threads/runs/runs.py b/src/openai/resources/beta/threads/runs/runs.py index 620cc270e5..0418d570ba 100644 --- a/src/openai/resources/beta/threads/runs/runs.py +++ b/src/openai/resources/beta/threads/runs/runs.py @@ -1106,7 +1106,6 @@ def stream( body=maybe_transform( { "assistant_id": assistant_id, - "include": include, "additional_instructions": additional_instructions, "additional_messages": additional_messages, "instructions": instructions, @@ -1126,7 +1125,11 @@ def stream( run_create_params.RunCreateParams, ), options=make_request_options( - extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + query=maybe_transform({"include": include}, run_create_params.RunCreateParams), ), cast_to=Run, stream=True, @@ -1899,7 +1902,6 @@ async def create( body=await async_maybe_transform( { "assistant_id": assistant_id, - "include": include, "additional_instructions": additional_instructions, "additional_messages": additional_messages, "instructions": instructions, @@ -2472,7 +2474,6 @@ def stream( body=maybe_transform( { "assistant_id": assistant_id, - "include": include, "additional_instructions": additional_instructions, "additional_messages": additional_messages, "instructions": instructions, @@ -2492,7 +2493,11 @@ def stream( run_create_params.RunCreateParams, ), options=make_request_options( - extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + query=maybe_transform({"include": include}, run_create_params.RunCreateParams), ), cast_to=Run, stream=True, From ad802551d8aaf4e6eff711118676ec4e64392638 Mon Sep 17 00:00:00 2001 From: "stainless-app[bot]" <142633134+stainless-app[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2024 16:04:05 +0000 Subject: [PATCH 3/9] docs(readme): example snippet for client context manager (#1953) --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index 780ee261fe..cbcfdb4447 100644 --- a/README.md +++ b/README.md @@ -652,6 +652,16 @@ client.with_options(http_client=DefaultHttpxClient(...)) By default the library closes underlying HTTP connections whenever the client is [garbage collected](https://docs.python.org/3/reference/datamodel.html#object.__del__). You can manually close the client using the `.close()` method if desired, or with a context manager that closes when exiting. +```py +from openai import OpenAI + +with OpenAI() as client: + # make requests here + ... + +# HTTP client is now closed +``` + ## Microsoft Azure OpenAI To use this library with [Azure OpenAI](https://learn.microsoft.com/azure/ai-services/openai/overview), use the `AzureOpenAI` From 628dead660c00435bf46e09081c7b90b7bbe4a8a Mon Sep 17 00:00:00 2001 From: "stainless-app[bot]" <142633134+stainless-app[bot]@users.noreply.github.com> Date: Tue, 17 Dec 2024 17:26:44 +0000 Subject: [PATCH 4/9] chore(internal): fix some typos (#1955) --- tests/test_client.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_client.py b/tests/test_client.py index 7caa8cb319..7751e7d463 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -349,11 +349,11 @@ def test_default_query_option(self) -> None: FinalRequestOptions( method="get", url="/foo", - params={"foo": "baz", "query_param": "overriden"}, + params={"foo": "baz", "query_param": "overridden"}, ) ) url = httpx.URL(request.url) - assert dict(url.params) == {"foo": "baz", "query_param": "overriden"} + assert dict(url.params) == {"foo": "baz", "query_param": "overridden"} def test_request_extra_json(self) -> None: request = self.client._build_request( @@ -1201,11 +1201,11 @@ def test_default_query_option(self) -> None: FinalRequestOptions( method="get", url="/foo", - params={"foo": "baz", "query_param": "overriden"}, + params={"foo": "baz", "query_param": "overridden"}, ) ) url = httpx.URL(request.url) - assert dict(url.params) == {"foo": "baz", "query_param": "overriden"} + assert dict(url.params) == {"foo": "baz", "query_param": "overridden"} def test_request_extra_json(self) -> None: request = self.client._build_request( From ec22ffb129c524525caa33b088405d27c271e631 Mon Sep 17 00:00:00 2001 From: "stainless-app[bot]" <142633134+stainless-app[bot]@users.noreply.github.com> Date: Tue, 17 Dec 2024 17:59:13 +0000 Subject: [PATCH 5/9] feat(api): new o1 and GPT-4o models + preference fine-tuning (#1956) learn more here: https://platform.openai.com/docs/changelog --- .stats.yml | 4 +- api.md | 16 + src/openai/resources/beta/beta.py | 32 ++ .../resources/beta/realtime/__init__.py | 33 ++ .../resources/beta/realtime/realtime.py | 102 ++++++ .../resources/beta/realtime/sessions.py | 337 ++++++++++++++++++ src/openai/resources/chat/completions.py | 240 ++++++++----- src/openai/resources/fine_tuning/jobs/jobs.py | 22 +- src/openai/types/beta/realtime/__init__.py | 6 + .../beta/realtime/session_create_params.py | 149 ++++++++ .../beta/realtime/session_create_response.py | 150 ++++++++ src/openai/types/chat/__init__.py | 4 + ...chat_completion_developer_message_param.py | 25 ++ .../chat/chat_completion_message_param.py | 2 + .../chat/chat_completion_reasoning_effort.py | 7 + .../types/chat/completion_create_params.py | 34 +- src/openai/types/chat_model.py | 7 +- .../types/fine_tuning/fine_tuning_job.py | 106 +++++- .../fine_tuning/fine_tuning_job_event.py | 13 + .../types/fine_tuning/job_create_params.py | 94 ++++- tests/api_resources/beta/realtime/__init__.py | 1 + .../beta/realtime/test_sessions.py | 146 ++++++++ tests/api_resources/chat/test_completions.py | 44 +-- tests/api_resources/fine_tuning/test_jobs.py | 36 ++ tests/test_client.py | 16 +- 25 files changed, 1475 insertions(+), 151 deletions(-) create mode 100644 src/openai/resources/beta/realtime/__init__.py create mode 100644 src/openai/resources/beta/realtime/realtime.py create mode 100644 src/openai/resources/beta/realtime/sessions.py create mode 100644 src/openai/types/beta/realtime/__init__.py create mode 100644 src/openai/types/beta/realtime/session_create_params.py create mode 100644 src/openai/types/beta/realtime/session_create_response.py create mode 100644 src/openai/types/chat/chat_completion_developer_message_param.py create mode 100644 src/openai/types/chat/chat_completion_reasoning_effort.py create mode 100644 tests/api_resources/beta/realtime/__init__.py create mode 100644 tests/api_resources/beta/realtime/test_sessions.py diff --git a/.stats.yml b/.stats.yml index 3cc042fe0a..e3a0040a5a 100644 --- a/.stats.yml +++ b/.stats.yml @@ -1,2 +1,2 @@ -configured_endpoints: 68 -openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/openai-2e0e0678be19d1118fd796af291822075e40538dba326611e177e9f3dc245a53.yml +configured_endpoints: 69 +openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/openai-779ea2754025daf5e18eb8ceb203ec321692636bc3a999338556a479178efa6c.yml diff --git a/api.md b/api.md index 7def07bb79..91b2a9c2fd 100644 --- a/api.md +++ b/api.md @@ -47,6 +47,7 @@ from openai.types.chat import ( ChatCompletionContentPartInputAudio, ChatCompletionContentPartRefusal, ChatCompletionContentPartText, + ChatCompletionDeveloperMessageParam, ChatCompletionFunctionCallOption, ChatCompletionFunctionMessageParam, ChatCompletionMessage, @@ -55,6 +56,7 @@ from openai.types.chat import ( ChatCompletionModality, ChatCompletionNamedToolChoice, ChatCompletionPredictionContent, + ChatCompletionReasoningEffort, ChatCompletionRole, ChatCompletionStreamOptions, ChatCompletionSystemMessageParam, @@ -235,6 +237,20 @@ Methods: # Beta +## Realtime + +### Sessions + +Types: + +```python +from openai.types.beta.realtime import Session, SessionCreateResponse +``` + +Methods: + +- client.beta.realtime.sessions.create(\*\*params) -> SessionCreateResponse + ## VectorStores Types: diff --git a/src/openai/resources/beta/beta.py b/src/openai/resources/beta/beta.py index 5079c989a5..1ffa6c8e79 100644 --- a/src/openai/resources/beta/beta.py +++ b/src/openai/resources/beta/beta.py @@ -21,6 +21,14 @@ ThreadsWithStreamingResponse, AsyncThreadsWithStreamingResponse, ) +from .realtime.realtime import ( + Realtime, + AsyncRealtime, + RealtimeWithRawResponse, + AsyncRealtimeWithRawResponse, + RealtimeWithStreamingResponse, + AsyncRealtimeWithStreamingResponse, +) from .vector_stores.vector_stores import ( VectorStores, AsyncVectorStores, @@ -38,6 +46,10 @@ class Beta(SyncAPIResource): def chat(self) -> Chat: return Chat(self._client) + @cached_property + def realtime(self) -> Realtime: + return Realtime(self._client) + @cached_property def vector_stores(self) -> VectorStores: return VectorStores(self._client) @@ -75,6 +87,10 @@ class AsyncBeta(AsyncAPIResource): def chat(self) -> AsyncChat: return AsyncChat(self._client) + @cached_property + def realtime(self) -> AsyncRealtime: + return AsyncRealtime(self._client) + @cached_property def vector_stores(self) -> AsyncVectorStores: return AsyncVectorStores(self._client) @@ -111,6 +127,10 @@ class BetaWithRawResponse: def __init__(self, beta: Beta) -> None: self._beta = beta + @cached_property + def realtime(self) -> RealtimeWithRawResponse: + return RealtimeWithRawResponse(self._beta.realtime) + @cached_property def vector_stores(self) -> VectorStoresWithRawResponse: return VectorStoresWithRawResponse(self._beta.vector_stores) @@ -128,6 +148,10 @@ class AsyncBetaWithRawResponse: def __init__(self, beta: AsyncBeta) -> None: self._beta = beta + @cached_property + def realtime(self) -> AsyncRealtimeWithRawResponse: + return AsyncRealtimeWithRawResponse(self._beta.realtime) + @cached_property def vector_stores(self) -> AsyncVectorStoresWithRawResponse: return AsyncVectorStoresWithRawResponse(self._beta.vector_stores) @@ -145,6 +169,10 @@ class BetaWithStreamingResponse: def __init__(self, beta: Beta) -> None: self._beta = beta + @cached_property + def realtime(self) -> RealtimeWithStreamingResponse: + return RealtimeWithStreamingResponse(self._beta.realtime) + @cached_property def vector_stores(self) -> VectorStoresWithStreamingResponse: return VectorStoresWithStreamingResponse(self._beta.vector_stores) @@ -162,6 +190,10 @@ class AsyncBetaWithStreamingResponse: def __init__(self, beta: AsyncBeta) -> None: self._beta = beta + @cached_property + def realtime(self) -> AsyncRealtimeWithStreamingResponse: + return AsyncRealtimeWithStreamingResponse(self._beta.realtime) + @cached_property def vector_stores(self) -> AsyncVectorStoresWithStreamingResponse: return AsyncVectorStoresWithStreamingResponse(self._beta.vector_stores) diff --git a/src/openai/resources/beta/realtime/__init__.py b/src/openai/resources/beta/realtime/__init__.py new file mode 100644 index 0000000000..474434e6e1 --- /dev/null +++ b/src/openai/resources/beta/realtime/__init__.py @@ -0,0 +1,33 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from .realtime import ( + Realtime, + AsyncRealtime, + RealtimeWithRawResponse, + AsyncRealtimeWithRawResponse, + RealtimeWithStreamingResponse, + AsyncRealtimeWithStreamingResponse, +) +from .sessions import ( + Sessions, + AsyncSessions, + SessionsWithRawResponse, + AsyncSessionsWithRawResponse, + SessionsWithStreamingResponse, + AsyncSessionsWithStreamingResponse, +) + +__all__ = [ + "Sessions", + "AsyncSessions", + "SessionsWithRawResponse", + "AsyncSessionsWithRawResponse", + "SessionsWithStreamingResponse", + "AsyncSessionsWithStreamingResponse", + "Realtime", + "AsyncRealtime", + "RealtimeWithRawResponse", + "AsyncRealtimeWithRawResponse", + "RealtimeWithStreamingResponse", + "AsyncRealtimeWithStreamingResponse", +] diff --git a/src/openai/resources/beta/realtime/realtime.py b/src/openai/resources/beta/realtime/realtime.py new file mode 100644 index 0000000000..e57e0be503 --- /dev/null +++ b/src/openai/resources/beta/realtime/realtime.py @@ -0,0 +1,102 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from .sessions import ( + Sessions, + AsyncSessions, + SessionsWithRawResponse, + AsyncSessionsWithRawResponse, + SessionsWithStreamingResponse, + AsyncSessionsWithStreamingResponse, +) +from ...._compat import cached_property +from ...._resource import SyncAPIResource, AsyncAPIResource + +__all__ = ["Realtime", "AsyncRealtime"] + + +class Realtime(SyncAPIResource): + @cached_property + def sessions(self) -> Sessions: + return Sessions(self._client) + + @cached_property + def with_raw_response(self) -> RealtimeWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return the + the raw response object instead of the parsed content. + + For more information, see https://www.github.com/openai/openai-python#accessing-raw-response-data-eg-headers + """ + return RealtimeWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> RealtimeWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://www.github.com/openai/openai-python#with_streaming_response + """ + return RealtimeWithStreamingResponse(self) + + +class AsyncRealtime(AsyncAPIResource): + @cached_property + def sessions(self) -> AsyncSessions: + return AsyncSessions(self._client) + + @cached_property + def with_raw_response(self) -> AsyncRealtimeWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return the + the raw response object instead of the parsed content. + + For more information, see https://www.github.com/openai/openai-python#accessing-raw-response-data-eg-headers + """ + return AsyncRealtimeWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> AsyncRealtimeWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://www.github.com/openai/openai-python#with_streaming_response + """ + return AsyncRealtimeWithStreamingResponse(self) + + +class RealtimeWithRawResponse: + def __init__(self, realtime: Realtime) -> None: + self._realtime = realtime + + @cached_property + def sessions(self) -> SessionsWithRawResponse: + return SessionsWithRawResponse(self._realtime.sessions) + + +class AsyncRealtimeWithRawResponse: + def __init__(self, realtime: AsyncRealtime) -> None: + self._realtime = realtime + + @cached_property + def sessions(self) -> AsyncSessionsWithRawResponse: + return AsyncSessionsWithRawResponse(self._realtime.sessions) + + +class RealtimeWithStreamingResponse: + def __init__(self, realtime: Realtime) -> None: + self._realtime = realtime + + @cached_property + def sessions(self) -> SessionsWithStreamingResponse: + return SessionsWithStreamingResponse(self._realtime.sessions) + + +class AsyncRealtimeWithStreamingResponse: + def __init__(self, realtime: AsyncRealtime) -> None: + self._realtime = realtime + + @cached_property + def sessions(self) -> AsyncSessionsWithStreamingResponse: + return AsyncSessionsWithStreamingResponse(self._realtime.sessions) diff --git a/src/openai/resources/beta/realtime/sessions.py b/src/openai/resources/beta/realtime/sessions.py new file mode 100644 index 0000000000..1d1ee701e5 --- /dev/null +++ b/src/openai/resources/beta/realtime/sessions.py @@ -0,0 +1,337 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import List, Union, Iterable +from typing_extensions import Literal + +import httpx + +from .... import _legacy_response +from ...._types import NOT_GIVEN, Body, Query, Headers, NotGiven +from ...._utils import ( + maybe_transform, + async_maybe_transform, +) +from ...._compat import cached_property +from ...._resource import SyncAPIResource, AsyncAPIResource +from ...._response import to_streamed_response_wrapper, async_to_streamed_response_wrapper +from ...._base_client import make_request_options +from ....types.beta.realtime import session_create_params +from ....types.beta.realtime.session_create_response import SessionCreateResponse + +__all__ = ["Sessions", "AsyncSessions"] + + +class Sessions(SyncAPIResource): + @cached_property + def with_raw_response(self) -> SessionsWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return the + the raw response object instead of the parsed content. + + For more information, see https://www.github.com/openai/openai-python#accessing-raw-response-data-eg-headers + """ + return SessionsWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> SessionsWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://www.github.com/openai/openai-python#with_streaming_response + """ + return SessionsWithStreamingResponse(self) + + def create( + self, + *, + model: Literal[ + "gpt-4o-realtime-preview", + "gpt-4o-realtime-preview-2024-10-01", + "gpt-4o-realtime-preview-2024-12-17", + "gpt-4o-mini-realtime-preview", + "gpt-4o-mini-realtime-preview-2024-12-17", + ], + input_audio_format: Literal["pcm16", "g711_ulaw", "g711_alaw"] | NotGiven = NOT_GIVEN, + input_audio_transcription: session_create_params.InputAudioTranscription | NotGiven = NOT_GIVEN, + instructions: str | NotGiven = NOT_GIVEN, + max_response_output_tokens: Union[int, Literal["inf"]] | NotGiven = NOT_GIVEN, + modalities: List[Literal["text", "audio"]] | NotGiven = NOT_GIVEN, + output_audio_format: Literal["pcm16", "g711_ulaw", "g711_alaw"] | NotGiven = NOT_GIVEN, + temperature: float | NotGiven = NOT_GIVEN, + tool_choice: str | NotGiven = NOT_GIVEN, + tools: Iterable[session_create_params.Tool] | NotGiven = NOT_GIVEN, + turn_detection: session_create_params.TurnDetection | NotGiven = NOT_GIVEN, + voice: Literal["alloy", "ash", "ballad", "coral", "echo", "sage", "shimmer", "verse"] | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> SessionCreateResponse: + """ + Create an ephemeral API token for use in client-side applications with the + Realtime API. Can be configured with the same session parameters as the + `session.update` client event. + + It responds with a session object, plus a `client_secret` key which contains a + usable ephemeral API token that can be used to authenticate browser clients for + the Realtime API. + + Args: + model: The Realtime model used for this session. + + input_audio_format: The format of input audio. Options are `pcm16`, `g711_ulaw`, or `g711_alaw`. + + input_audio_transcription: Configuration for input audio transcription, defaults to off and can be set to + `null` to turn off once on. Input audio transcription is not native to the + model, since the model consumes audio directly. Transcription runs + asynchronously through Whisper and should be treated as rough guidance rather + than the representation understood by the model. + + instructions: The default system instructions (i.e. system message) prepended to model calls. + This field allows the client to guide the model on desired responses. The model + can be instructed on response content and format, (e.g. "be extremely succinct", + "act friendly", "here are examples of good responses") and on audio behavior + (e.g. "talk quickly", "inject emotion into your voice", "laugh frequently"). The + instructions are not guaranteed to be followed by the model, but they provide + guidance to the model on the desired behavior. + + Note that the server sets default instructions which will be used if this field + is not set and are visible in the `session.created` event at the start of the + session. + + max_response_output_tokens: Maximum number of output tokens for a single assistant response, inclusive of + tool calls. Provide an integer between 1 and 4096 to limit output tokens, or + `inf` for the maximum available tokens for a given model. Defaults to `inf`. + + modalities: The set of modalities the model can respond with. To disable audio, set this to + ["text"]. + + output_audio_format: The format of output audio. Options are `pcm16`, `g711_ulaw`, or `g711_alaw`. + + temperature: Sampling temperature for the model, limited to [0.6, 1.2]. Defaults to 0.8. + + tool_choice: How the model chooses tools. Options are `auto`, `none`, `required`, or specify + a function. + + tools: Tools (functions) available to the model. + + turn_detection: Configuration for turn detection. Can be set to `null` to turn off. Server VAD + means that the model will detect the start and end of speech based on audio + volume and respond at the end of user speech. + + voice: The voice the model uses to respond. Voice cannot be changed during the session + once the model has responded with audio at least once. Current voice options are + `alloy`, `ash`, `ballad`, `coral`, `echo` `sage`, `shimmer` and `verse`. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + extra_headers = {"OpenAI-Beta": "assistants=v2", **(extra_headers or {})} + return self._post( + "/realtime/sessions", + body=maybe_transform( + { + "model": model, + "input_audio_format": input_audio_format, + "input_audio_transcription": input_audio_transcription, + "instructions": instructions, + "max_response_output_tokens": max_response_output_tokens, + "modalities": modalities, + "output_audio_format": output_audio_format, + "temperature": temperature, + "tool_choice": tool_choice, + "tools": tools, + "turn_detection": turn_detection, + "voice": voice, + }, + session_create_params.SessionCreateParams, + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=SessionCreateResponse, + ) + + +class AsyncSessions(AsyncAPIResource): + @cached_property + def with_raw_response(self) -> AsyncSessionsWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return the + the raw response object instead of the parsed content. + + For more information, see https://www.github.com/openai/openai-python#accessing-raw-response-data-eg-headers + """ + return AsyncSessionsWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> AsyncSessionsWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://www.github.com/openai/openai-python#with_streaming_response + """ + return AsyncSessionsWithStreamingResponse(self) + + async def create( + self, + *, + model: Literal[ + "gpt-4o-realtime-preview", + "gpt-4o-realtime-preview-2024-10-01", + "gpt-4o-realtime-preview-2024-12-17", + "gpt-4o-mini-realtime-preview", + "gpt-4o-mini-realtime-preview-2024-12-17", + ], + input_audio_format: Literal["pcm16", "g711_ulaw", "g711_alaw"] | NotGiven = NOT_GIVEN, + input_audio_transcription: session_create_params.InputAudioTranscription | NotGiven = NOT_GIVEN, + instructions: str | NotGiven = NOT_GIVEN, + max_response_output_tokens: Union[int, Literal["inf"]] | NotGiven = NOT_GIVEN, + modalities: List[Literal["text", "audio"]] | NotGiven = NOT_GIVEN, + output_audio_format: Literal["pcm16", "g711_ulaw", "g711_alaw"] | NotGiven = NOT_GIVEN, + temperature: float | NotGiven = NOT_GIVEN, + tool_choice: str | NotGiven = NOT_GIVEN, + tools: Iterable[session_create_params.Tool] | NotGiven = NOT_GIVEN, + turn_detection: session_create_params.TurnDetection | NotGiven = NOT_GIVEN, + voice: Literal["alloy", "ash", "ballad", "coral", "echo", "sage", "shimmer", "verse"] | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> SessionCreateResponse: + """ + Create an ephemeral API token for use in client-side applications with the + Realtime API. Can be configured with the same session parameters as the + `session.update` client event. + + It responds with a session object, plus a `client_secret` key which contains a + usable ephemeral API token that can be used to authenticate browser clients for + the Realtime API. + + Args: + model: The Realtime model used for this session. + + input_audio_format: The format of input audio. Options are `pcm16`, `g711_ulaw`, or `g711_alaw`. + + input_audio_transcription: Configuration for input audio transcription, defaults to off and can be set to + `null` to turn off once on. Input audio transcription is not native to the + model, since the model consumes audio directly. Transcription runs + asynchronously through Whisper and should be treated as rough guidance rather + than the representation understood by the model. + + instructions: The default system instructions (i.e. system message) prepended to model calls. + This field allows the client to guide the model on desired responses. The model + can be instructed on response content and format, (e.g. "be extremely succinct", + "act friendly", "here are examples of good responses") and on audio behavior + (e.g. "talk quickly", "inject emotion into your voice", "laugh frequently"). The + instructions are not guaranteed to be followed by the model, but they provide + guidance to the model on the desired behavior. + + Note that the server sets default instructions which will be used if this field + is not set and are visible in the `session.created` event at the start of the + session. + + max_response_output_tokens: Maximum number of output tokens for a single assistant response, inclusive of + tool calls. Provide an integer between 1 and 4096 to limit output tokens, or + `inf` for the maximum available tokens for a given model. Defaults to `inf`. + + modalities: The set of modalities the model can respond with. To disable audio, set this to + ["text"]. + + output_audio_format: The format of output audio. Options are `pcm16`, `g711_ulaw`, or `g711_alaw`. + + temperature: Sampling temperature for the model, limited to [0.6, 1.2]. Defaults to 0.8. + + tool_choice: How the model chooses tools. Options are `auto`, `none`, `required`, or specify + a function. + + tools: Tools (functions) available to the model. + + turn_detection: Configuration for turn detection. Can be set to `null` to turn off. Server VAD + means that the model will detect the start and end of speech based on audio + volume and respond at the end of user speech. + + voice: The voice the model uses to respond. Voice cannot be changed during the session + once the model has responded with audio at least once. Current voice options are + `alloy`, `ash`, `ballad`, `coral`, `echo` `sage`, `shimmer` and `verse`. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + extra_headers = {"OpenAI-Beta": "assistants=v2", **(extra_headers or {})} + return await self._post( + "/realtime/sessions", + body=await async_maybe_transform( + { + "model": model, + "input_audio_format": input_audio_format, + "input_audio_transcription": input_audio_transcription, + "instructions": instructions, + "max_response_output_tokens": max_response_output_tokens, + "modalities": modalities, + "output_audio_format": output_audio_format, + "temperature": temperature, + "tool_choice": tool_choice, + "tools": tools, + "turn_detection": turn_detection, + "voice": voice, + }, + session_create_params.SessionCreateParams, + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=SessionCreateResponse, + ) + + +class SessionsWithRawResponse: + def __init__(self, sessions: Sessions) -> None: + self._sessions = sessions + + self.create = _legacy_response.to_raw_response_wrapper( + sessions.create, + ) + + +class AsyncSessionsWithRawResponse: + def __init__(self, sessions: AsyncSessions) -> None: + self._sessions = sessions + + self.create = _legacy_response.async_to_raw_response_wrapper( + sessions.create, + ) + + +class SessionsWithStreamingResponse: + def __init__(self, sessions: Sessions) -> None: + self._sessions = sessions + + self.create = to_streamed_response_wrapper( + sessions.create, + ) + + +class AsyncSessionsWithStreamingResponse: + def __init__(self, sessions: AsyncSessions) -> None: + self._sessions = sessions + + self.create = async_to_streamed_response_wrapper( + sessions.create, + ) diff --git a/src/openai/resources/chat/completions.py b/src/openai/resources/chat/completions.py index 60ab5138ba..728c744327 100644 --- a/src/openai/resources/chat/completions.py +++ b/src/openai/resources/chat/completions.py @@ -22,6 +22,7 @@ from ..._streaming import Stream, AsyncStream from ...types.chat import ( ChatCompletionAudioParam, + ChatCompletionReasoningEffort, completion_create_params, ) from ..._base_client import make_request_options @@ -32,6 +33,7 @@ from ...types.chat.chat_completion_tool_param import ChatCompletionToolParam from ...types.chat.chat_completion_audio_param import ChatCompletionAudioParam from ...types.chat.chat_completion_message_param import ChatCompletionMessageParam +from ...types.chat.chat_completion_reasoning_effort import ChatCompletionReasoningEffort from ...types.chat.chat_completion_stream_options_param import ChatCompletionStreamOptionsParam from ...types.chat.chat_completion_prediction_content_param import ChatCompletionPredictionContentParam from ...types.chat.chat_completion_tool_choice_option_param import ChatCompletionToolChoiceOptionParam @@ -79,6 +81,7 @@ def create( parallel_tool_calls: bool | NotGiven = NOT_GIVEN, prediction: Optional[ChatCompletionPredictionContentParam] | NotGiven = NOT_GIVEN, presence_penalty: Optional[float] | NotGiven = NOT_GIVEN, + reasoning_effort: ChatCompletionReasoningEffort | NotGiven = NOT_GIVEN, response_format: completion_create_params.ResponseFormat | NotGiven = NOT_GIVEN, seed: Optional[int] | NotGiven = NOT_GIVEN, service_tier: Optional[Literal["auto", "default"]] | NotGiven = NOT_GIVEN, @@ -106,6 +109,12 @@ def create( [vision](https://platform.openai.com/docs/guides/vision), and [audio](https://platform.openai.com/docs/guides/audio) guides. + Parameter support can differ depending on the model used to generate the + response, particularly for newer reasoning models. Parameters that are only + supported for reasoning models are noted below. For the current state of + unsupported parameters in reasoning models, + [refer to the reasoning guide](https://platform.openai.com/docs/guides/reasoning). + Args: messages: A list of messages comprising the conversation so far. Depending on the [model](https://platform.openai.com/docs/models) you use, different message @@ -126,16 +135,18 @@ def create( existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim. - [See more information about frequency and presence penalties.](https://platform.openai.com/docs/guides/text-generation) - function_call: Deprecated in favor of `tool_choice`. - Controls which (if any) function is called by the model. `none` means the model - will not call a function and instead generates a message. `auto` means the model - can pick between generating a message or calling a function. Specifying a - particular function via `{"name": "my_function"}` forces the model to call that + Controls which (if any) function is called by the model. + + `none` means the model will not call a function and instead generates a message. + + `auto` means the model can pick between generating a message or calling a function. + Specifying a particular function via `{"name": "my_function"}` forces the model + to call that function. + `none` is the default when no functions are present. `auto` is the default if functions are present. @@ -197,13 +208,14 @@ def create( whether they appear in the text so far, increasing the model's likelihood to talk about new topics. - [See more information about frequency and presence penalties.](https://platform.openai.com/docs/guides/text-generation) + reasoning_effort: **o1 models only** + + Constrains effort on reasoning for + [reasoning models](https://platform.openai.com/docs/guides/reasoning). Currently + supported values are `low`, `medium`, and `high`. Reducing reasoning effort can + result in faster responses and fewer tokens used on reasoning in a response. - response_format: An object specifying the format that the model must output. Compatible with - [GPT-4o](https://platform.openai.com/docs/models#gpt-4o), - [GPT-4o mini](https://platform.openai.com/docs/models#gpt-4o-mini), - [GPT-4 Turbo](https://platform.openai.com/docs/models#gpt-4-turbo-and-gpt-4) and - all GPT-3.5 Turbo models newer than `gpt-3.5-turbo-1106`. + response_format: An object specifying the format that the model must output. Setting to `{ "type": "json_schema", "json_schema": {...} }` enables Structured Outputs which ensures the model will match your supplied JSON schema. Learn more @@ -259,9 +271,8 @@ def create( temperature: What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more - focused and deterministic. - - We generally recommend altering this or `top_p` but not both. + focused and deterministic. We generally recommend altering this or `top_p` but + not both. tool_choice: Controls which (if any) tool is called by the model. `none` means the model will not call any tool and instead generates a message. `auto` means the model can @@ -322,6 +333,7 @@ def create( parallel_tool_calls: bool | NotGiven = NOT_GIVEN, prediction: Optional[ChatCompletionPredictionContentParam] | NotGiven = NOT_GIVEN, presence_penalty: Optional[float] | NotGiven = NOT_GIVEN, + reasoning_effort: ChatCompletionReasoningEffort | NotGiven = NOT_GIVEN, response_format: completion_create_params.ResponseFormat | NotGiven = NOT_GIVEN, seed: Optional[int] | NotGiven = NOT_GIVEN, service_tier: Optional[Literal["auto", "default"]] | NotGiven = NOT_GIVEN, @@ -348,6 +360,12 @@ def create( [vision](https://platform.openai.com/docs/guides/vision), and [audio](https://platform.openai.com/docs/guides/audio) guides. + Parameter support can differ depending on the model used to generate the + response, particularly for newer reasoning models. Parameters that are only + supported for reasoning models are noted below. For the current state of + unsupported parameters in reasoning models, + [refer to the reasoning guide](https://platform.openai.com/docs/guides/reasoning). + Args: messages: A list of messages comprising the conversation so far. Depending on the [model](https://platform.openai.com/docs/models) you use, different message @@ -375,16 +393,18 @@ def create( existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim. - [See more information about frequency and presence penalties.](https://platform.openai.com/docs/guides/text-generation) - function_call: Deprecated in favor of `tool_choice`. - Controls which (if any) function is called by the model. `none` means the model - will not call a function and instead generates a message. `auto` means the model - can pick between generating a message or calling a function. Specifying a - particular function via `{"name": "my_function"}` forces the model to call that + Controls which (if any) function is called by the model. + + `none` means the model will not call a function and instead generates a message. + + `auto` means the model can pick between generating a message or calling a function. + Specifying a particular function via `{"name": "my_function"}` forces the model + to call that function. + `none` is the default when no functions are present. `auto` is the default if functions are present. @@ -446,13 +466,14 @@ def create( whether they appear in the text so far, increasing the model's likelihood to talk about new topics. - [See more information about frequency and presence penalties.](https://platform.openai.com/docs/guides/text-generation) + reasoning_effort: **o1 models only** + + Constrains effort on reasoning for + [reasoning models](https://platform.openai.com/docs/guides/reasoning). Currently + supported values are `low`, `medium`, and `high`. Reducing reasoning effort can + result in faster responses and fewer tokens used on reasoning in a response. - response_format: An object specifying the format that the model must output. Compatible with - [GPT-4o](https://platform.openai.com/docs/models#gpt-4o), - [GPT-4o mini](https://platform.openai.com/docs/models#gpt-4o-mini), - [GPT-4 Turbo](https://platform.openai.com/docs/models#gpt-4-turbo-and-gpt-4) and - all GPT-3.5 Turbo models newer than `gpt-3.5-turbo-1106`. + response_format: An object specifying the format that the model must output. Setting to `{ "type": "json_schema", "json_schema": {...} }` enables Structured Outputs which ensures the model will match your supplied JSON schema. Learn more @@ -501,9 +522,8 @@ def create( temperature: What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more - focused and deterministic. - - We generally recommend altering this or `top_p` but not both. + focused and deterministic. We generally recommend altering this or `top_p` but + not both. tool_choice: Controls which (if any) tool is called by the model. `none` means the model will not call any tool and instead generates a message. `auto` means the model can @@ -564,6 +584,7 @@ def create( parallel_tool_calls: bool | NotGiven = NOT_GIVEN, prediction: Optional[ChatCompletionPredictionContentParam] | NotGiven = NOT_GIVEN, presence_penalty: Optional[float] | NotGiven = NOT_GIVEN, + reasoning_effort: ChatCompletionReasoningEffort | NotGiven = NOT_GIVEN, response_format: completion_create_params.ResponseFormat | NotGiven = NOT_GIVEN, seed: Optional[int] | NotGiven = NOT_GIVEN, service_tier: Optional[Literal["auto", "default"]] | NotGiven = NOT_GIVEN, @@ -590,6 +611,12 @@ def create( [vision](https://platform.openai.com/docs/guides/vision), and [audio](https://platform.openai.com/docs/guides/audio) guides. + Parameter support can differ depending on the model used to generate the + response, particularly for newer reasoning models. Parameters that are only + supported for reasoning models are noted below. For the current state of + unsupported parameters in reasoning models, + [refer to the reasoning guide](https://platform.openai.com/docs/guides/reasoning). + Args: messages: A list of messages comprising the conversation so far. Depending on the [model](https://platform.openai.com/docs/models) you use, different message @@ -617,16 +644,18 @@ def create( existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim. - [See more information about frequency and presence penalties.](https://platform.openai.com/docs/guides/text-generation) - function_call: Deprecated in favor of `tool_choice`. - Controls which (if any) function is called by the model. `none` means the model - will not call a function and instead generates a message. `auto` means the model - can pick between generating a message or calling a function. Specifying a - particular function via `{"name": "my_function"}` forces the model to call that + Controls which (if any) function is called by the model. + + `none` means the model will not call a function and instead generates a message. + + `auto` means the model can pick between generating a message or calling a function. + Specifying a particular function via `{"name": "my_function"}` forces the model + to call that function. + `none` is the default when no functions are present. `auto` is the default if functions are present. @@ -688,13 +717,14 @@ def create( whether they appear in the text so far, increasing the model's likelihood to talk about new topics. - [See more information about frequency and presence penalties.](https://platform.openai.com/docs/guides/text-generation) + reasoning_effort: **o1 models only** - response_format: An object specifying the format that the model must output. Compatible with - [GPT-4o](https://platform.openai.com/docs/models#gpt-4o), - [GPT-4o mini](https://platform.openai.com/docs/models#gpt-4o-mini), - [GPT-4 Turbo](https://platform.openai.com/docs/models#gpt-4-turbo-and-gpt-4) and - all GPT-3.5 Turbo models newer than `gpt-3.5-turbo-1106`. + Constrains effort on reasoning for + [reasoning models](https://platform.openai.com/docs/guides/reasoning). Currently + supported values are `low`, `medium`, and `high`. Reducing reasoning effort can + result in faster responses and fewer tokens used on reasoning in a response. + + response_format: An object specifying the format that the model must output. Setting to `{ "type": "json_schema", "json_schema": {...} }` enables Structured Outputs which ensures the model will match your supplied JSON schema. Learn more @@ -743,9 +773,8 @@ def create( temperature: What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more - focused and deterministic. - - We generally recommend altering this or `top_p` but not both. + focused and deterministic. We generally recommend altering this or `top_p` but + not both. tool_choice: Controls which (if any) tool is called by the model. `none` means the model will not call any tool and instead generates a message. `auto` means the model can @@ -805,6 +834,7 @@ def create( parallel_tool_calls: bool | NotGiven = NOT_GIVEN, prediction: Optional[ChatCompletionPredictionContentParam] | NotGiven = NOT_GIVEN, presence_penalty: Optional[float] | NotGiven = NOT_GIVEN, + reasoning_effort: ChatCompletionReasoningEffort | NotGiven = NOT_GIVEN, response_format: completion_create_params.ResponseFormat | NotGiven = NOT_GIVEN, seed: Optional[int] | NotGiven = NOT_GIVEN, service_tier: Optional[Literal["auto", "default"]] | NotGiven = NOT_GIVEN, @@ -846,6 +876,7 @@ def create( "parallel_tool_calls": parallel_tool_calls, "prediction": prediction, "presence_penalty": presence_penalty, + "reasoning_effort": reasoning_effort, "response_format": response_format, "seed": seed, "service_tier": service_tier, @@ -911,6 +942,7 @@ async def create( parallel_tool_calls: bool | NotGiven = NOT_GIVEN, prediction: Optional[ChatCompletionPredictionContentParam] | NotGiven = NOT_GIVEN, presence_penalty: Optional[float] | NotGiven = NOT_GIVEN, + reasoning_effort: ChatCompletionReasoningEffort | NotGiven = NOT_GIVEN, response_format: completion_create_params.ResponseFormat | NotGiven = NOT_GIVEN, seed: Optional[int] | NotGiven = NOT_GIVEN, service_tier: Optional[Literal["auto", "default"]] | NotGiven = NOT_GIVEN, @@ -938,6 +970,12 @@ async def create( [vision](https://platform.openai.com/docs/guides/vision), and [audio](https://platform.openai.com/docs/guides/audio) guides. + Parameter support can differ depending on the model used to generate the + response, particularly for newer reasoning models. Parameters that are only + supported for reasoning models are noted below. For the current state of + unsupported parameters in reasoning models, + [refer to the reasoning guide](https://platform.openai.com/docs/guides/reasoning). + Args: messages: A list of messages comprising the conversation so far. Depending on the [model](https://platform.openai.com/docs/models) you use, different message @@ -958,16 +996,18 @@ async def create( existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim. - [See more information about frequency and presence penalties.](https://platform.openai.com/docs/guides/text-generation) - function_call: Deprecated in favor of `tool_choice`. - Controls which (if any) function is called by the model. `none` means the model - will not call a function and instead generates a message. `auto` means the model - can pick between generating a message or calling a function. Specifying a - particular function via `{"name": "my_function"}` forces the model to call that + Controls which (if any) function is called by the model. + + `none` means the model will not call a function and instead generates a message. + + `auto` means the model can pick between generating a message or calling a function. + Specifying a particular function via `{"name": "my_function"}` forces the model + to call that function. + `none` is the default when no functions are present. `auto` is the default if functions are present. @@ -1029,13 +1069,14 @@ async def create( whether they appear in the text so far, increasing the model's likelihood to talk about new topics. - [See more information about frequency and presence penalties.](https://platform.openai.com/docs/guides/text-generation) + reasoning_effort: **o1 models only** + + Constrains effort on reasoning for + [reasoning models](https://platform.openai.com/docs/guides/reasoning). Currently + supported values are `low`, `medium`, and `high`. Reducing reasoning effort can + result in faster responses and fewer tokens used on reasoning in a response. - response_format: An object specifying the format that the model must output. Compatible with - [GPT-4o](https://platform.openai.com/docs/models#gpt-4o), - [GPT-4o mini](https://platform.openai.com/docs/models#gpt-4o-mini), - [GPT-4 Turbo](https://platform.openai.com/docs/models#gpt-4-turbo-and-gpt-4) and - all GPT-3.5 Turbo models newer than `gpt-3.5-turbo-1106`. + response_format: An object specifying the format that the model must output. Setting to `{ "type": "json_schema", "json_schema": {...} }` enables Structured Outputs which ensures the model will match your supplied JSON schema. Learn more @@ -1091,9 +1132,8 @@ async def create( temperature: What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more - focused and deterministic. - - We generally recommend altering this or `top_p` but not both. + focused and deterministic. We generally recommend altering this or `top_p` but + not both. tool_choice: Controls which (if any) tool is called by the model. `none` means the model will not call any tool and instead generates a message. `auto` means the model can @@ -1154,6 +1194,7 @@ async def create( parallel_tool_calls: bool | NotGiven = NOT_GIVEN, prediction: Optional[ChatCompletionPredictionContentParam] | NotGiven = NOT_GIVEN, presence_penalty: Optional[float] | NotGiven = NOT_GIVEN, + reasoning_effort: ChatCompletionReasoningEffort | NotGiven = NOT_GIVEN, response_format: completion_create_params.ResponseFormat | NotGiven = NOT_GIVEN, seed: Optional[int] | NotGiven = NOT_GIVEN, service_tier: Optional[Literal["auto", "default"]] | NotGiven = NOT_GIVEN, @@ -1180,6 +1221,12 @@ async def create( [vision](https://platform.openai.com/docs/guides/vision), and [audio](https://platform.openai.com/docs/guides/audio) guides. + Parameter support can differ depending on the model used to generate the + response, particularly for newer reasoning models. Parameters that are only + supported for reasoning models are noted below. For the current state of + unsupported parameters in reasoning models, + [refer to the reasoning guide](https://platform.openai.com/docs/guides/reasoning). + Args: messages: A list of messages comprising the conversation so far. Depending on the [model](https://platform.openai.com/docs/models) you use, different message @@ -1207,16 +1254,18 @@ async def create( existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim. - [See more information about frequency and presence penalties.](https://platform.openai.com/docs/guides/text-generation) - function_call: Deprecated in favor of `tool_choice`. - Controls which (if any) function is called by the model. `none` means the model - will not call a function and instead generates a message. `auto` means the model - can pick between generating a message or calling a function. Specifying a - particular function via `{"name": "my_function"}` forces the model to call that + Controls which (if any) function is called by the model. + + `none` means the model will not call a function and instead generates a message. + + `auto` means the model can pick between generating a message or calling a function. + Specifying a particular function via `{"name": "my_function"}` forces the model + to call that function. + `none` is the default when no functions are present. `auto` is the default if functions are present. @@ -1278,13 +1327,14 @@ async def create( whether they appear in the text so far, increasing the model's likelihood to talk about new topics. - [See more information about frequency and presence penalties.](https://platform.openai.com/docs/guides/text-generation) + reasoning_effort: **o1 models only** + + Constrains effort on reasoning for + [reasoning models](https://platform.openai.com/docs/guides/reasoning). Currently + supported values are `low`, `medium`, and `high`. Reducing reasoning effort can + result in faster responses and fewer tokens used on reasoning in a response. - response_format: An object specifying the format that the model must output. Compatible with - [GPT-4o](https://platform.openai.com/docs/models#gpt-4o), - [GPT-4o mini](https://platform.openai.com/docs/models#gpt-4o-mini), - [GPT-4 Turbo](https://platform.openai.com/docs/models#gpt-4-turbo-and-gpt-4) and - all GPT-3.5 Turbo models newer than `gpt-3.5-turbo-1106`. + response_format: An object specifying the format that the model must output. Setting to `{ "type": "json_schema", "json_schema": {...} }` enables Structured Outputs which ensures the model will match your supplied JSON schema. Learn more @@ -1333,9 +1383,8 @@ async def create( temperature: What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more - focused and deterministic. - - We generally recommend altering this or `top_p` but not both. + focused and deterministic. We generally recommend altering this or `top_p` but + not both. tool_choice: Controls which (if any) tool is called by the model. `none` means the model will not call any tool and instead generates a message. `auto` means the model can @@ -1396,6 +1445,7 @@ async def create( parallel_tool_calls: bool | NotGiven = NOT_GIVEN, prediction: Optional[ChatCompletionPredictionContentParam] | NotGiven = NOT_GIVEN, presence_penalty: Optional[float] | NotGiven = NOT_GIVEN, + reasoning_effort: ChatCompletionReasoningEffort | NotGiven = NOT_GIVEN, response_format: completion_create_params.ResponseFormat | NotGiven = NOT_GIVEN, seed: Optional[int] | NotGiven = NOT_GIVEN, service_tier: Optional[Literal["auto", "default"]] | NotGiven = NOT_GIVEN, @@ -1422,6 +1472,12 @@ async def create( [vision](https://platform.openai.com/docs/guides/vision), and [audio](https://platform.openai.com/docs/guides/audio) guides. + Parameter support can differ depending on the model used to generate the + response, particularly for newer reasoning models. Parameters that are only + supported for reasoning models are noted below. For the current state of + unsupported parameters in reasoning models, + [refer to the reasoning guide](https://platform.openai.com/docs/guides/reasoning). + Args: messages: A list of messages comprising the conversation so far. Depending on the [model](https://platform.openai.com/docs/models) you use, different message @@ -1449,16 +1505,18 @@ async def create( existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim. - [See more information about frequency and presence penalties.](https://platform.openai.com/docs/guides/text-generation) - function_call: Deprecated in favor of `tool_choice`. - Controls which (if any) function is called by the model. `none` means the model - will not call a function and instead generates a message. `auto` means the model - can pick between generating a message or calling a function. Specifying a - particular function via `{"name": "my_function"}` forces the model to call that + Controls which (if any) function is called by the model. + + `none` means the model will not call a function and instead generates a message. + + `auto` means the model can pick between generating a message or calling a function. + Specifying a particular function via `{"name": "my_function"}` forces the model + to call that function. + `none` is the default when no functions are present. `auto` is the default if functions are present. @@ -1520,13 +1578,14 @@ async def create( whether they appear in the text so far, increasing the model's likelihood to talk about new topics. - [See more information about frequency and presence penalties.](https://platform.openai.com/docs/guides/text-generation) + reasoning_effort: **o1 models only** - response_format: An object specifying the format that the model must output. Compatible with - [GPT-4o](https://platform.openai.com/docs/models#gpt-4o), - [GPT-4o mini](https://platform.openai.com/docs/models#gpt-4o-mini), - [GPT-4 Turbo](https://platform.openai.com/docs/models#gpt-4-turbo-and-gpt-4) and - all GPT-3.5 Turbo models newer than `gpt-3.5-turbo-1106`. + Constrains effort on reasoning for + [reasoning models](https://platform.openai.com/docs/guides/reasoning). Currently + supported values are `low`, `medium`, and `high`. Reducing reasoning effort can + result in faster responses and fewer tokens used on reasoning in a response. + + response_format: An object specifying the format that the model must output. Setting to `{ "type": "json_schema", "json_schema": {...} }` enables Structured Outputs which ensures the model will match your supplied JSON schema. Learn more @@ -1575,9 +1634,8 @@ async def create( temperature: What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more - focused and deterministic. - - We generally recommend altering this or `top_p` but not both. + focused and deterministic. We generally recommend altering this or `top_p` but + not both. tool_choice: Controls which (if any) tool is called by the model. `none` means the model will not call any tool and instead generates a message. `auto` means the model can @@ -1637,6 +1695,7 @@ async def create( parallel_tool_calls: bool | NotGiven = NOT_GIVEN, prediction: Optional[ChatCompletionPredictionContentParam] | NotGiven = NOT_GIVEN, presence_penalty: Optional[float] | NotGiven = NOT_GIVEN, + reasoning_effort: ChatCompletionReasoningEffort | NotGiven = NOT_GIVEN, response_format: completion_create_params.ResponseFormat | NotGiven = NOT_GIVEN, seed: Optional[int] | NotGiven = NOT_GIVEN, service_tier: Optional[Literal["auto", "default"]] | NotGiven = NOT_GIVEN, @@ -1678,6 +1737,7 @@ async def create( "parallel_tool_calls": parallel_tool_calls, "prediction": prediction, "presence_penalty": presence_penalty, + "reasoning_effort": reasoning_effort, "response_format": response_format, "seed": seed, "service_tier": service_tier, diff --git a/src/openai/resources/fine_tuning/jobs/jobs.py b/src/openai/resources/fine_tuning/jobs/jobs.py index 0ed5495b0e..78eefc253c 100644 --- a/src/openai/resources/fine_tuning/jobs/jobs.py +++ b/src/openai/resources/fine_tuning/jobs/jobs.py @@ -67,6 +67,7 @@ def create( training_file: str, hyperparameters: job_create_params.Hyperparameters | NotGiven = NOT_GIVEN, integrations: Optional[Iterable[job_create_params.Integration]] | NotGiven = NOT_GIVEN, + method: job_create_params.Method | NotGiven = NOT_GIVEN, seed: Optional[int] | NotGiven = NOT_GIVEN, suffix: Optional[str] | NotGiven = NOT_GIVEN, validation_file: Optional[str] | NotGiven = NOT_GIVEN, @@ -99,17 +100,22 @@ def create( your file with the purpose `fine-tune`. The contents of the file should differ depending on if the model uses the - [chat](https://platform.openai.com/docs/api-reference/fine-tuning/chat-input) or + [chat](https://platform.openai.com/docs/api-reference/fine-tuning/chat-input), [completions](https://platform.openai.com/docs/api-reference/fine-tuning/completions-input) + format, or if the fine-tuning method uses the + [preference](https://platform.openai.com/docs/api-reference/fine-tuning/preference-input) format. See the [fine-tuning guide](https://platform.openai.com/docs/guides/fine-tuning) for more details. - hyperparameters: The hyperparameters used for the fine-tuning job. + hyperparameters: The hyperparameters used for the fine-tuning job. This value is now deprecated + in favor of `method`, and should be passed in under the `method` parameter. integrations: A list of integrations to enable for your fine-tuning job. + method: The method used for fine-tuning. + seed: The seed controls the reproducibility of the job. Passing in the same seed and job parameters should produce the same results, but may differ in rare cases. If a seed is not specified, one will be generated for you. @@ -149,6 +155,7 @@ def create( "training_file": training_file, "hyperparameters": hyperparameters, "integrations": integrations, + "method": method, "seed": seed, "suffix": suffix, "validation_file": validation_file, @@ -358,6 +365,7 @@ async def create( training_file: str, hyperparameters: job_create_params.Hyperparameters | NotGiven = NOT_GIVEN, integrations: Optional[Iterable[job_create_params.Integration]] | NotGiven = NOT_GIVEN, + method: job_create_params.Method | NotGiven = NOT_GIVEN, seed: Optional[int] | NotGiven = NOT_GIVEN, suffix: Optional[str] | NotGiven = NOT_GIVEN, validation_file: Optional[str] | NotGiven = NOT_GIVEN, @@ -390,17 +398,22 @@ async def create( your file with the purpose `fine-tune`. The contents of the file should differ depending on if the model uses the - [chat](https://platform.openai.com/docs/api-reference/fine-tuning/chat-input) or + [chat](https://platform.openai.com/docs/api-reference/fine-tuning/chat-input), [completions](https://platform.openai.com/docs/api-reference/fine-tuning/completions-input) + format, or if the fine-tuning method uses the + [preference](https://platform.openai.com/docs/api-reference/fine-tuning/preference-input) format. See the [fine-tuning guide](https://platform.openai.com/docs/guides/fine-tuning) for more details. - hyperparameters: The hyperparameters used for the fine-tuning job. + hyperparameters: The hyperparameters used for the fine-tuning job. This value is now deprecated + in favor of `method`, and should be passed in under the `method` parameter. integrations: A list of integrations to enable for your fine-tuning job. + method: The method used for fine-tuning. + seed: The seed controls the reproducibility of the job. Passing in the same seed and job parameters should produce the same results, but may differ in rare cases. If a seed is not specified, one will be generated for you. @@ -440,6 +453,7 @@ async def create( "training_file": training_file, "hyperparameters": hyperparameters, "integrations": integrations, + "method": method, "seed": seed, "suffix": suffix, "validation_file": validation_file, diff --git a/src/openai/types/beta/realtime/__init__.py b/src/openai/types/beta/realtime/__init__.py new file mode 100644 index 0000000000..1c5246db7a --- /dev/null +++ b/src/openai/types/beta/realtime/__init__.py @@ -0,0 +1,6 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from .session_create_params import SessionCreateParams as SessionCreateParams +from .session_create_response import SessionCreateResponse as SessionCreateResponse diff --git a/src/openai/types/beta/realtime/session_create_params.py b/src/openai/types/beta/realtime/session_create_params.py new file mode 100644 index 0000000000..f56f2c5c22 --- /dev/null +++ b/src/openai/types/beta/realtime/session_create_params.py @@ -0,0 +1,149 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import List, Union, Iterable +from typing_extensions import Literal, Required, TypedDict + +__all__ = ["SessionCreateParams", "InputAudioTranscription", "Tool", "TurnDetection"] + + +class SessionCreateParams(TypedDict, total=False): + model: Required[ + Literal[ + "gpt-4o-realtime-preview", + "gpt-4o-realtime-preview-2024-10-01", + "gpt-4o-realtime-preview-2024-12-17", + "gpt-4o-mini-realtime-preview", + "gpt-4o-mini-realtime-preview-2024-12-17", + ] + ] + """The Realtime model used for this session.""" + + input_audio_format: Literal["pcm16", "g711_ulaw", "g711_alaw"] + """The format of input audio. Options are `pcm16`, `g711_ulaw`, or `g711_alaw`.""" + + input_audio_transcription: InputAudioTranscription + """ + Configuration for input audio transcription, defaults to off and can be set to + `null` to turn off once on. Input audio transcription is not native to the + model, since the model consumes audio directly. Transcription runs + asynchronously through Whisper and should be treated as rough guidance rather + than the representation understood by the model. + """ + + instructions: str + """The default system instructions (i.e. + + system message) prepended to model calls. This field allows the client to guide + the model on desired responses. The model can be instructed on response content + and format, (e.g. "be extremely succinct", "act friendly", "here are examples of + good responses") and on audio behavior (e.g. "talk quickly", "inject emotion + into your voice", "laugh frequently"). The instructions are not guaranteed to be + followed by the model, but they provide guidance to the model on the desired + behavior. + + Note that the server sets default instructions which will be used if this field + is not set and are visible in the `session.created` event at the start of the + session. + """ + + max_response_output_tokens: Union[int, Literal["inf"]] + """ + Maximum number of output tokens for a single assistant response, inclusive of + tool calls. Provide an integer between 1 and 4096 to limit output tokens, or + `inf` for the maximum available tokens for a given model. Defaults to `inf`. + """ + + modalities: List[Literal["text", "audio"]] + """The set of modalities the model can respond with. + + To disable audio, set this to ["text"]. + """ + + output_audio_format: Literal["pcm16", "g711_ulaw", "g711_alaw"] + """The format of output audio. Options are `pcm16`, `g711_ulaw`, or `g711_alaw`.""" + + temperature: float + """Sampling temperature for the model, limited to [0.6, 1.2]. Defaults to 0.8.""" + + tool_choice: str + """How the model chooses tools. + + Options are `auto`, `none`, `required`, or specify a function. + """ + + tools: Iterable[Tool] + """Tools (functions) available to the model.""" + + turn_detection: TurnDetection + """Configuration for turn detection. + + Can be set to `null` to turn off. Server VAD means that the model will detect + the start and end of speech based on audio volume and respond at the end of user + speech. + """ + + voice: Literal["alloy", "ash", "ballad", "coral", "echo", "sage", "shimmer", "verse"] + """The voice the model uses to respond. + + Voice cannot be changed during the session once the model has responded with + audio at least once. Current voice options are `alloy`, `ash`, `ballad`, + `coral`, `echo` `sage`, `shimmer` and `verse`. + """ + + +class InputAudioTranscription(TypedDict, total=False): + model: str + """ + The model to use for transcription, `whisper-1` is the only currently supported + model. + """ + + +class Tool(TypedDict, total=False): + description: str + """ + The description of the function, including guidance on when and how to call it, + and guidance about what to tell the user when calling (if anything). + """ + + name: str + """The name of the function.""" + + parameters: object + """Parameters of the function in JSON Schema.""" + + type: Literal["function"] + """The type of the tool, i.e. `function`.""" + + +class TurnDetection(TypedDict, total=False): + create_response: bool + """Whether or not to automatically generate a response when VAD is enabled. + + `true` by default. + """ + + prefix_padding_ms: int + """Amount of audio to include before the VAD detected speech (in milliseconds). + + Defaults to 300ms. + """ + + silence_duration_ms: int + """Duration of silence to detect speech stop (in milliseconds). + + Defaults to 500ms. With shorter values the model will respond more quickly, but + may jump in on short pauses from the user. + """ + + threshold: float + """Activation threshold for VAD (0.0 to 1.0), this defaults to 0.5. + + A higher threshold will require louder audio to activate the model, and thus + might perform better in noisy environments. + """ + + type: str + """Type of turn detection, only `server_vad` is currently supported.""" diff --git a/src/openai/types/beta/realtime/session_create_response.py b/src/openai/types/beta/realtime/session_create_response.py new file mode 100644 index 0000000000..31f591b261 --- /dev/null +++ b/src/openai/types/beta/realtime/session_create_response.py @@ -0,0 +1,150 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import List, Union, Optional +from typing_extensions import Literal + +from ...._models import BaseModel + +__all__ = ["SessionCreateResponse", "ClientSecret", "InputAudioTranscription", "Tool", "TurnDetection"] + + +class ClientSecret(BaseModel): + expires_at: Optional[int] = None + """Timestamp for when the token expires. + + Currently, all tokens expire after one minute. + """ + + value: Optional[str] = None + """ + Ephemeral key usable in client environments to authenticate connections to the + Realtime API. Use this in client-side environments rather than a standard API + token, which should only be used server-side. + """ + + +class InputAudioTranscription(BaseModel): + model: Optional[str] = None + """ + The model to use for transcription, `whisper-1` is the only currently supported + model. + """ + + +class Tool(BaseModel): + description: Optional[str] = None + """ + The description of the function, including guidance on when and how to call it, + and guidance about what to tell the user when calling (if anything). + """ + + name: Optional[str] = None + """The name of the function.""" + + parameters: Optional[object] = None + """Parameters of the function in JSON Schema.""" + + type: Optional[Literal["function"]] = None + """The type of the tool, i.e. `function`.""" + + +class TurnDetection(BaseModel): + prefix_padding_ms: Optional[int] = None + """Amount of audio to include before the VAD detected speech (in milliseconds). + + Defaults to 300ms. + """ + + silence_duration_ms: Optional[int] = None + """Duration of silence to detect speech stop (in milliseconds). + + Defaults to 500ms. With shorter values the model will respond more quickly, but + may jump in on short pauses from the user. + """ + + threshold: Optional[float] = None + """Activation threshold for VAD (0.0 to 1.0), this defaults to 0.5. + + A higher threshold will require louder audio to activate the model, and thus + might perform better in noisy environments. + """ + + type: Optional[str] = None + """Type of turn detection, only `server_vad` is currently supported.""" + + +class SessionCreateResponse(BaseModel): + client_secret: Optional[ClientSecret] = None + """Ephemeral key returned by the API.""" + + input_audio_format: Optional[str] = None + """The format of input audio. Options are `pcm16`, `g711_ulaw`, or `g711_alaw`.""" + + input_audio_transcription: Optional[InputAudioTranscription] = None + """ + Configuration for input audio transcription, defaults to off and can be set to + `null` to turn off once on. Input audio transcription is not native to the + model, since the model consumes audio directly. Transcription runs + asynchronously through Whisper and should be treated as rough guidance rather + than the representation understood by the model. + """ + + instructions: Optional[str] = None + """The default system instructions (i.e. + + system message) prepended to model calls. This field allows the client to guide + the model on desired responses. The model can be instructed on response content + and format, (e.g. "be extremely succinct", "act friendly", "here are examples of + good responses") and on audio behavior (e.g. "talk quickly", "inject emotion + into your voice", "laugh frequently"). The instructions are not guaranteed to be + followed by the model, but they provide guidance to the model on the desired + behavior. + + Note that the server sets default instructions which will be used if this field + is not set and are visible in the `session.created` event at the start of the + session. + """ + + max_response_output_tokens: Union[int, Literal["inf"], None] = None + """ + Maximum number of output tokens for a single assistant response, inclusive of + tool calls. Provide an integer between 1 and 4096 to limit output tokens, or + `inf` for the maximum available tokens for a given model. Defaults to `inf`. + """ + + modalities: Optional[List[Literal["text", "audio"]]] = None + """The set of modalities the model can respond with. + + To disable audio, set this to ["text"]. + """ + + output_audio_format: Optional[str] = None + """The format of output audio. Options are `pcm16`, `g711_ulaw`, or `g711_alaw`.""" + + temperature: Optional[float] = None + """Sampling temperature for the model, limited to [0.6, 1.2]. Defaults to 0.8.""" + + tool_choice: Optional[str] = None + """How the model chooses tools. + + Options are `auto`, `none`, `required`, or specify a function. + """ + + tools: Optional[List[Tool]] = None + """Tools (functions) available to the model.""" + + turn_detection: Optional[TurnDetection] = None + """Configuration for turn detection. + + Can be set to `null` to turn off. Server VAD means that the model will detect + the start and end of speech based on audio volume and respond at the end of user + speech. + """ + + voice: Optional[Literal["alloy", "ash", "ballad", "coral", "echo", "sage", "shimmer", "verse"]] = None + """The voice the model uses to respond. + + Voice cannot be changed during the session once the model has responded with + audio at least once. Current voice options are `alloy`, `ash`, `ballad`, + `coral`, `echo` `sage`, `shimmer` and `verse`. + """ diff --git a/src/openai/types/chat/__init__.py b/src/openai/types/chat/__init__.py index d0a5403e79..c623a982af 100644 --- a/src/openai/types/chat/__init__.py +++ b/src/openai/types/chat/__init__.py @@ -22,6 +22,7 @@ from .chat_completion_audio_param import ChatCompletionAudioParam as ChatCompletionAudioParam from .chat_completion_message_param import ChatCompletionMessageParam as ChatCompletionMessageParam from .chat_completion_token_logprob import ChatCompletionTokenLogprob as ChatCompletionTokenLogprob +from .chat_completion_reasoning_effort import ChatCompletionReasoningEffort as ChatCompletionReasoningEffort from .chat_completion_message_tool_call import ChatCompletionMessageToolCall as ChatCompletionMessageToolCall from .chat_completion_content_part_param import ChatCompletionContentPartParam as ChatCompletionContentPartParam from .chat_completion_tool_message_param import ChatCompletionToolMessageParam as ChatCompletionToolMessageParam @@ -37,6 +38,9 @@ from .chat_completion_content_part_text_param import ( ChatCompletionContentPartTextParam as ChatCompletionContentPartTextParam, ) +from .chat_completion_developer_message_param import ( + ChatCompletionDeveloperMessageParam as ChatCompletionDeveloperMessageParam, +) from .chat_completion_message_tool_call_param import ( ChatCompletionMessageToolCallParam as ChatCompletionMessageToolCallParam, ) diff --git a/src/openai/types/chat/chat_completion_developer_message_param.py b/src/openai/types/chat/chat_completion_developer_message_param.py new file mode 100644 index 0000000000..01e4fdb654 --- /dev/null +++ b/src/openai/types/chat/chat_completion_developer_message_param.py @@ -0,0 +1,25 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import Union, Iterable +from typing_extensions import Literal, Required, TypedDict + +from .chat_completion_content_part_text_param import ChatCompletionContentPartTextParam + +__all__ = ["ChatCompletionDeveloperMessageParam"] + + +class ChatCompletionDeveloperMessageParam(TypedDict, total=False): + content: Required[Union[str, Iterable[ChatCompletionContentPartTextParam]]] + """The contents of the developer message.""" + + role: Required[Literal["developer"]] + """The role of the messages author, in this case `developer`.""" + + name: str + """An optional name for the participant. + + Provides the model information to differentiate between participants of the same + role. + """ diff --git a/src/openai/types/chat/chat_completion_message_param.py b/src/openai/types/chat/chat_completion_message_param.py index ec65d94cae..942da24304 100644 --- a/src/openai/types/chat/chat_completion_message_param.py +++ b/src/openai/types/chat/chat_completion_message_param.py @@ -10,10 +10,12 @@ from .chat_completion_system_message_param import ChatCompletionSystemMessageParam from .chat_completion_function_message_param import ChatCompletionFunctionMessageParam from .chat_completion_assistant_message_param import ChatCompletionAssistantMessageParam +from .chat_completion_developer_message_param import ChatCompletionDeveloperMessageParam __all__ = ["ChatCompletionMessageParam"] ChatCompletionMessageParam: TypeAlias = Union[ + ChatCompletionDeveloperMessageParam, ChatCompletionSystemMessageParam, ChatCompletionUserMessageParam, ChatCompletionAssistantMessageParam, diff --git a/src/openai/types/chat/chat_completion_reasoning_effort.py b/src/openai/types/chat/chat_completion_reasoning_effort.py new file mode 100644 index 0000000000..9e7946974a --- /dev/null +++ b/src/openai/types/chat/chat_completion_reasoning_effort.py @@ -0,0 +1,7 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing_extensions import Literal, TypeAlias + +__all__ = ["ChatCompletionReasoningEffort"] + +ChatCompletionReasoningEffort: TypeAlias = Literal["low", "medium", "high"] diff --git a/src/openai/types/chat/completion_create_params.py b/src/openai/types/chat/completion_create_params.py index e838858314..f168ddea6e 100644 --- a/src/openai/types/chat/completion_create_params.py +++ b/src/openai/types/chat/completion_create_params.py @@ -10,6 +10,7 @@ from .chat_completion_tool_param import ChatCompletionToolParam from .chat_completion_audio_param import ChatCompletionAudioParam from .chat_completion_message_param import ChatCompletionMessageParam +from .chat_completion_reasoning_effort import ChatCompletionReasoningEffort from ..shared_params.function_parameters import FunctionParameters from ..shared_params.response_format_text import ResponseFormatText from .chat_completion_stream_options_param import ChatCompletionStreamOptionsParam @@ -60,19 +61,21 @@ class CompletionCreateParamsBase(TypedDict, total=False): Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim. - - [See more information about frequency and presence penalties.](https://platform.openai.com/docs/guides/text-generation) """ function_call: FunctionCall """Deprecated in favor of `tool_choice`. - Controls which (if any) function is called by the model. `none` means the model - will not call a function and instead generates a message. `auto` means the model - can pick between generating a message or calling a function. Specifying a - particular function via `{"name": "my_function"}` forces the model to call that + Controls which (if any) function is called by the model. + + `none` means the model will not call a function and instead generates a message. + + `auto` means the model can pick between generating a message or calling a function. + Specifying a particular function via `{"name": "my_function"}` forces the model + to call that function. + `none` is the default when no functions are present. `auto` is the default if functions are present. """ @@ -164,18 +167,20 @@ class CompletionCreateParamsBase(TypedDict, total=False): Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics. + """ + + reasoning_effort: ChatCompletionReasoningEffort + """**o1 models only** - [See more information about frequency and presence penalties.](https://platform.openai.com/docs/guides/text-generation) + Constrains effort on reasoning for + [reasoning models](https://platform.openai.com/docs/guides/reasoning). Currently + supported values are `low`, `medium`, and `high`. Reducing reasoning effort can + result in faster responses and fewer tokens used on reasoning in a response. """ response_format: ResponseFormat """An object specifying the format that the model must output. - Compatible with [GPT-4o](https://platform.openai.com/docs/models#gpt-4o), - [GPT-4o mini](https://platform.openai.com/docs/models#gpt-4o-mini), - [GPT-4 Turbo](https://platform.openai.com/docs/models#gpt-4-turbo-and-gpt-4) and - all GPT-3.5 Turbo models newer than `gpt-3.5-turbo-1106`. - Setting to `{ "type": "json_schema", "json_schema": {...} }` enables Structured Outputs which ensures the model will match your supplied JSON schema. Learn more in the @@ -237,9 +242,8 @@ class CompletionCreateParamsBase(TypedDict, total=False): """What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while lower values like - 0.2 will make it more focused and deterministic. - - We generally recommend altering this or `top_p` but not both. + 0.2 will make it more focused and deterministic. We generally recommend altering + this or `top_p` but not both. """ tool_choice: ChatCompletionToolChoiceOptionParam diff --git a/src/openai/types/chat_model.py b/src/openai/types/chat_model.py index 3567a3ba65..e1ac464320 100644 --- a/src/openai/types/chat_model.py +++ b/src/openai/types/chat_model.py @@ -5,6 +5,8 @@ __all__ = ["ChatModel"] ChatModel: TypeAlias = Literal[ + "o1", + "o1-2024-12-17", "o1-preview", "o1-preview-2024-09-12", "o1-mini", @@ -13,10 +15,11 @@ "gpt-4o-2024-11-20", "gpt-4o-2024-08-06", "gpt-4o-2024-05-13", - "gpt-4o-realtime-preview", - "gpt-4o-realtime-preview-2024-10-01", "gpt-4o-audio-preview", "gpt-4o-audio-preview-2024-10-01", + "gpt-4o-audio-preview-2024-12-17", + "gpt-4o-mini-audio-preview", + "gpt-4o-mini-audio-preview-2024-12-17", "chatgpt-4o-latest", "gpt-4o-mini", "gpt-4o-mini-2024-07-18", diff --git a/src/openai/types/fine_tuning/fine_tuning_job.py b/src/openai/types/fine_tuning/fine_tuning_job.py index 7ac8792787..f5a11c2107 100644 --- a/src/openai/types/fine_tuning/fine_tuning_job.py +++ b/src/openai/types/fine_tuning/fine_tuning_job.py @@ -6,7 +6,16 @@ from ..._models import BaseModel from .fine_tuning_job_wandb_integration_object import FineTuningJobWandbIntegrationObject -__all__ = ["FineTuningJob", "Error", "Hyperparameters"] +__all__ = [ + "FineTuningJob", + "Error", + "Hyperparameters", + "Method", + "MethodDpo", + "MethodDpoHyperparameters", + "MethodSupervised", + "MethodSupervisedHyperparameters", +] class Error(BaseModel): @@ -24,15 +33,96 @@ class Error(BaseModel): class Hyperparameters(BaseModel): - n_epochs: Union[Literal["auto"], int] + batch_size: Union[Literal["auto"], int, None] = None + """Number of examples in each batch. + + A larger batch size means that model parameters are updated less frequently, but + with lower variance. + """ + + learning_rate_multiplier: Union[Literal["auto"], float, None] = None + """Scaling factor for the learning rate. + + A smaller learning rate may be useful to avoid overfitting. + """ + + n_epochs: Union[Literal["auto"], int, None] = None + """The number of epochs to train the model for. + + An epoch refers to one full cycle through the training dataset. + """ + + +class MethodDpoHyperparameters(BaseModel): + batch_size: Union[Literal["auto"], int, None] = None + """Number of examples in each batch. + + A larger batch size means that model parameters are updated less frequently, but + with lower variance. + """ + + beta: Union[Literal["auto"], float, None] = None + """The beta value for the DPO method. + + A higher beta value will increase the weight of the penalty between the policy + and reference model. + """ + + learning_rate_multiplier: Union[Literal["auto"], float, None] = None + """Scaling factor for the learning rate. + + A smaller learning rate may be useful to avoid overfitting. + """ + + n_epochs: Union[Literal["auto"], int, None] = None """The number of epochs to train the model for. - An epoch refers to one full cycle through the training dataset. "auto" decides - the optimal number of epochs based on the size of the dataset. If setting the - number manually, we support any number between 1 and 50 epochs. + An epoch refers to one full cycle through the training dataset. """ +class MethodDpo(BaseModel): + hyperparameters: Optional[MethodDpoHyperparameters] = None + """The hyperparameters used for the fine-tuning job.""" + + +class MethodSupervisedHyperparameters(BaseModel): + batch_size: Union[Literal["auto"], int, None] = None + """Number of examples in each batch. + + A larger batch size means that model parameters are updated less frequently, but + with lower variance. + """ + + learning_rate_multiplier: Union[Literal["auto"], float, None] = None + """Scaling factor for the learning rate. + + A smaller learning rate may be useful to avoid overfitting. + """ + + n_epochs: Union[Literal["auto"], int, None] = None + """The number of epochs to train the model for. + + An epoch refers to one full cycle through the training dataset. + """ + + +class MethodSupervised(BaseModel): + hyperparameters: Optional[MethodSupervisedHyperparameters] = None + """The hyperparameters used for the fine-tuning job.""" + + +class Method(BaseModel): + dpo: Optional[MethodDpo] = None + """Configuration for the DPO fine-tuning method.""" + + supervised: Optional[MethodSupervised] = None + """Configuration for the supervised fine-tuning method.""" + + type: Optional[Literal["supervised", "dpo"]] = None + """The type of method. Is either `supervised` or `dpo`.""" + + class FineTuningJob(BaseModel): id: str """The object identifier, which can be referenced in the API endpoints.""" @@ -61,8 +151,7 @@ class FineTuningJob(BaseModel): hyperparameters: Hyperparameters """The hyperparameters used for the fine-tuning job. - See the [fine-tuning guide](https://platform.openai.com/docs/guides/fine-tuning) - for more details. + This value will only be returned when running `supervised` jobs. """ model: str @@ -118,3 +207,6 @@ class FineTuningJob(BaseModel): integrations: Optional[List[FineTuningJobWandbIntegrationObject]] = None """A list of integrations to enable for this fine-tuning job.""" + + method: Optional[Method] = None + """The method used for fine-tuning.""" diff --git a/src/openai/types/fine_tuning/fine_tuning_job_event.py b/src/openai/types/fine_tuning/fine_tuning_job_event.py index 2d204bb980..1d728bd765 100644 --- a/src/openai/types/fine_tuning/fine_tuning_job_event.py +++ b/src/openai/types/fine_tuning/fine_tuning_job_event.py @@ -1,5 +1,7 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. +import builtins +from typing import Optional from typing_extensions import Literal from ..._models import BaseModel @@ -9,11 +11,22 @@ class FineTuningJobEvent(BaseModel): id: str + """The object identifier.""" created_at: int + """The Unix timestamp (in seconds) for when the fine-tuning job was created.""" level: Literal["info", "warn", "error"] + """The log level of the event.""" message: str + """The message of the event.""" object: Literal["fine_tuning.job.event"] + """The object type, which is always "fine_tuning.job.event".""" + + data: Optional[builtins.object] = None + """The data associated with the event.""" + + type: Optional[Literal["message", "metrics"]] = None + """The type of event.""" diff --git a/src/openai/types/fine_tuning/job_create_params.py b/src/openai/types/fine_tuning/job_create_params.py index 8814229b2e..09c3f8571c 100644 --- a/src/openai/types/fine_tuning/job_create_params.py +++ b/src/openai/types/fine_tuning/job_create_params.py @@ -5,7 +5,17 @@ from typing import List, Union, Iterable, Optional from typing_extensions import Literal, Required, TypedDict -__all__ = ["JobCreateParams", "Hyperparameters", "Integration", "IntegrationWandb"] +__all__ = [ + "JobCreateParams", + "Hyperparameters", + "Integration", + "IntegrationWandb", + "Method", + "MethodDpo", + "MethodDpoHyperparameters", + "MethodSupervised", + "MethodSupervisedHyperparameters", +] class JobCreateParams(TypedDict, total=False): @@ -26,8 +36,10 @@ class JobCreateParams(TypedDict, total=False): your file with the purpose `fine-tune`. The contents of the file should differ depending on if the model uses the - [chat](https://platform.openai.com/docs/api-reference/fine-tuning/chat-input) or + [chat](https://platform.openai.com/docs/api-reference/fine-tuning/chat-input), [completions](https://platform.openai.com/docs/api-reference/fine-tuning/completions-input) + format, or if the fine-tuning method uses the + [preference](https://platform.openai.com/docs/api-reference/fine-tuning/preference-input) format. See the [fine-tuning guide](https://platform.openai.com/docs/guides/fine-tuning) @@ -35,11 +47,17 @@ class JobCreateParams(TypedDict, total=False): """ hyperparameters: Hyperparameters - """The hyperparameters used for the fine-tuning job.""" + """ + The hyperparameters used for the fine-tuning job. This value is now deprecated + in favor of `method`, and should be passed in under the `method` parameter. + """ integrations: Optional[Iterable[Integration]] """A list of integrations to enable for your fine-tuning job.""" + method: Method + """The method used for fine-tuning.""" + seed: Optional[int] """The seed controls the reproducibility of the job. @@ -134,3 +152,73 @@ class Integration(TypedDict, total=False): can set an explicit display name for your run, add tags to your run, and set a default entity (team, username, etc) to be associated with your run. """ + + +class MethodDpoHyperparameters(TypedDict, total=False): + batch_size: Union[Literal["auto"], int] + """Number of examples in each batch. + + A larger batch size means that model parameters are updated less frequently, but + with lower variance. + """ + + beta: Union[Literal["auto"], float] + """The beta value for the DPO method. + + A higher beta value will increase the weight of the penalty between the policy + and reference model. + """ + + learning_rate_multiplier: Union[Literal["auto"], float] + """Scaling factor for the learning rate. + + A smaller learning rate may be useful to avoid overfitting. + """ + + n_epochs: Union[Literal["auto"], int] + """The number of epochs to train the model for. + + An epoch refers to one full cycle through the training dataset. + """ + + +class MethodDpo(TypedDict, total=False): + hyperparameters: MethodDpoHyperparameters + """The hyperparameters used for the fine-tuning job.""" + + +class MethodSupervisedHyperparameters(TypedDict, total=False): + batch_size: Union[Literal["auto"], int] + """Number of examples in each batch. + + A larger batch size means that model parameters are updated less frequently, but + with lower variance. + """ + + learning_rate_multiplier: Union[Literal["auto"], float] + """Scaling factor for the learning rate. + + A smaller learning rate may be useful to avoid overfitting. + """ + + n_epochs: Union[Literal["auto"], int] + """The number of epochs to train the model for. + + An epoch refers to one full cycle through the training dataset. + """ + + +class MethodSupervised(TypedDict, total=False): + hyperparameters: MethodSupervisedHyperparameters + """The hyperparameters used for the fine-tuning job.""" + + +class Method(TypedDict, total=False): + dpo: MethodDpo + """Configuration for the DPO fine-tuning method.""" + + supervised: MethodSupervised + """Configuration for the supervised fine-tuning method.""" + + type: Literal["supervised", "dpo"] + """The type of method. Is either `supervised` or `dpo`.""" diff --git a/tests/api_resources/beta/realtime/__init__.py b/tests/api_resources/beta/realtime/__init__.py new file mode 100644 index 0000000000..fd8019a9a1 --- /dev/null +++ b/tests/api_resources/beta/realtime/__init__.py @@ -0,0 +1 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. diff --git a/tests/api_resources/beta/realtime/test_sessions.py b/tests/api_resources/beta/realtime/test_sessions.py new file mode 100644 index 0000000000..65bfa27572 --- /dev/null +++ b/tests/api_resources/beta/realtime/test_sessions.py @@ -0,0 +1,146 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +import os +from typing import Any, cast + +import pytest + +from openai import OpenAI, AsyncOpenAI +from tests.utils import assert_matches_type +from openai.types.beta.realtime import SessionCreateResponse + +base_url = os.environ.get("TEST_API_BASE_URL", "http://127.0.0.1:4010") + + +class TestSessions: + parametrize = pytest.mark.parametrize("client", [False, True], indirect=True, ids=["loose", "strict"]) + + @parametrize + def test_method_create(self, client: OpenAI) -> None: + session = client.beta.realtime.sessions.create( + model="gpt-4o-realtime-preview", + ) + assert_matches_type(SessionCreateResponse, session, path=["response"]) + + @parametrize + def test_method_create_with_all_params(self, client: OpenAI) -> None: + session = client.beta.realtime.sessions.create( + model="gpt-4o-realtime-preview", + input_audio_format="pcm16", + input_audio_transcription={"model": "model"}, + instructions="instructions", + max_response_output_tokens=0, + modalities=["text"], + output_audio_format="pcm16", + temperature=0, + tool_choice="tool_choice", + tools=[ + { + "description": "description", + "name": "name", + "parameters": {}, + "type": "function", + } + ], + turn_detection={ + "create_response": True, + "prefix_padding_ms": 0, + "silence_duration_ms": 0, + "threshold": 0, + "type": "type", + }, + voice="alloy", + ) + assert_matches_type(SessionCreateResponse, session, path=["response"]) + + @parametrize + def test_raw_response_create(self, client: OpenAI) -> None: + response = client.beta.realtime.sessions.with_raw_response.create( + model="gpt-4o-realtime-preview", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + session = response.parse() + assert_matches_type(SessionCreateResponse, session, path=["response"]) + + @parametrize + def test_streaming_response_create(self, client: OpenAI) -> None: + with client.beta.realtime.sessions.with_streaming_response.create( + model="gpt-4o-realtime-preview", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + session = response.parse() + assert_matches_type(SessionCreateResponse, session, path=["response"]) + + assert cast(Any, response.is_closed) is True + + +class TestAsyncSessions: + parametrize = pytest.mark.parametrize("async_client", [False, True], indirect=True, ids=["loose", "strict"]) + + @parametrize + async def test_method_create(self, async_client: AsyncOpenAI) -> None: + session = await async_client.beta.realtime.sessions.create( + model="gpt-4o-realtime-preview", + ) + assert_matches_type(SessionCreateResponse, session, path=["response"]) + + @parametrize + async def test_method_create_with_all_params(self, async_client: AsyncOpenAI) -> None: + session = await async_client.beta.realtime.sessions.create( + model="gpt-4o-realtime-preview", + input_audio_format="pcm16", + input_audio_transcription={"model": "model"}, + instructions="instructions", + max_response_output_tokens=0, + modalities=["text"], + output_audio_format="pcm16", + temperature=0, + tool_choice="tool_choice", + tools=[ + { + "description": "description", + "name": "name", + "parameters": {}, + "type": "function", + } + ], + turn_detection={ + "create_response": True, + "prefix_padding_ms": 0, + "silence_duration_ms": 0, + "threshold": 0, + "type": "type", + }, + voice="alloy", + ) + assert_matches_type(SessionCreateResponse, session, path=["response"]) + + @parametrize + async def test_raw_response_create(self, async_client: AsyncOpenAI) -> None: + response = await async_client.beta.realtime.sessions.with_raw_response.create( + model="gpt-4o-realtime-preview", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + session = response.parse() + assert_matches_type(SessionCreateResponse, session, path=["response"]) + + @parametrize + async def test_streaming_response_create(self, async_client: AsyncOpenAI) -> None: + async with async_client.beta.realtime.sessions.with_streaming_response.create( + model="gpt-4o-realtime-preview", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + session = await response.parse() + assert_matches_type(SessionCreateResponse, session, path=["response"]) + + assert cast(Any, response.is_closed) is True diff --git a/tests/api_resources/chat/test_completions.py b/tests/api_resources/chat/test_completions.py index 1b52650b1d..393a790549 100644 --- a/tests/api_resources/chat/test_completions.py +++ b/tests/api_resources/chat/test_completions.py @@ -26,7 +26,7 @@ def test_method_create_overload_1(self, client: OpenAI) -> None: messages=[ { "content": "string", - "role": "system", + "role": "developer", } ], model="gpt-4o", @@ -39,8 +39,8 @@ def test_method_create_with_all_params_overload_1(self, client: OpenAI) -> None: messages=[ { "content": "string", - "role": "system", - "name": "string", + "role": "developer", + "name": "name", } ], model="gpt-4o", @@ -70,6 +70,7 @@ def test_method_create_with_all_params_overload_1(self, client: OpenAI) -> None: "type": "content", }, presence_penalty=-2, + reasoning_effort="low", response_format={"type": "text"}, seed=-9007199254740991, service_tier="auto", @@ -102,7 +103,7 @@ def test_raw_response_create_overload_1(self, client: OpenAI) -> None: messages=[ { "content": "string", - "role": "system", + "role": "developer", } ], model="gpt-4o", @@ -119,7 +120,7 @@ def test_streaming_response_create_overload_1(self, client: OpenAI) -> None: messages=[ { "content": "string", - "role": "system", + "role": "developer", } ], model="gpt-4o", @@ -138,7 +139,7 @@ def test_method_create_overload_2(self, client: OpenAI) -> None: messages=[ { "content": "string", - "role": "system", + "role": "developer", } ], model="gpt-4o", @@ -152,8 +153,8 @@ def test_method_create_with_all_params_overload_2(self, client: OpenAI) -> None: messages=[ { "content": "string", - "role": "system", - "name": "string", + "role": "developer", + "name": "name", } ], model="gpt-4o", @@ -184,6 +185,7 @@ def test_method_create_with_all_params_overload_2(self, client: OpenAI) -> None: "type": "content", }, presence_penalty=-2, + reasoning_effort="low", response_format={"type": "text"}, seed=-9007199254740991, service_tier="auto", @@ -215,7 +217,7 @@ def test_raw_response_create_overload_2(self, client: OpenAI) -> None: messages=[ { "content": "string", - "role": "system", + "role": "developer", } ], model="gpt-4o", @@ -232,7 +234,7 @@ def test_streaming_response_create_overload_2(self, client: OpenAI) -> None: messages=[ { "content": "string", - "role": "system", + "role": "developer", } ], model="gpt-4o", @@ -273,7 +275,7 @@ async def test_method_create_overload_1(self, async_client: AsyncOpenAI) -> None messages=[ { "content": "string", - "role": "system", + "role": "developer", } ], model="gpt-4o", @@ -286,8 +288,8 @@ async def test_method_create_with_all_params_overload_1(self, async_client: Asyn messages=[ { "content": "string", - "role": "system", - "name": "string", + "role": "developer", + "name": "name", } ], model="gpt-4o", @@ -317,6 +319,7 @@ async def test_method_create_with_all_params_overload_1(self, async_client: Asyn "type": "content", }, presence_penalty=-2, + reasoning_effort="low", response_format={"type": "text"}, seed=-9007199254740991, service_tier="auto", @@ -349,7 +352,7 @@ async def test_raw_response_create_overload_1(self, async_client: AsyncOpenAI) - messages=[ { "content": "string", - "role": "system", + "role": "developer", } ], model="gpt-4o", @@ -366,7 +369,7 @@ async def test_streaming_response_create_overload_1(self, async_client: AsyncOpe messages=[ { "content": "string", - "role": "system", + "role": "developer", } ], model="gpt-4o", @@ -385,7 +388,7 @@ async def test_method_create_overload_2(self, async_client: AsyncOpenAI) -> None messages=[ { "content": "string", - "role": "system", + "role": "developer", } ], model="gpt-4o", @@ -399,8 +402,8 @@ async def test_method_create_with_all_params_overload_2(self, async_client: Asyn messages=[ { "content": "string", - "role": "system", - "name": "string", + "role": "developer", + "name": "name", } ], model="gpt-4o", @@ -431,6 +434,7 @@ async def test_method_create_with_all_params_overload_2(self, async_client: Asyn "type": "content", }, presence_penalty=-2, + reasoning_effort="low", response_format={"type": "text"}, seed=-9007199254740991, service_tier="auto", @@ -462,7 +466,7 @@ async def test_raw_response_create_overload_2(self, async_client: AsyncOpenAI) - messages=[ { "content": "string", - "role": "system", + "role": "developer", } ], model="gpt-4o", @@ -479,7 +483,7 @@ async def test_streaming_response_create_overload_2(self, async_client: AsyncOpe messages=[ { "content": "string", - "role": "system", + "role": "developer", } ], model="gpt-4o", diff --git a/tests/api_resources/fine_tuning/test_jobs.py b/tests/api_resources/fine_tuning/test_jobs.py index aa2bf39528..1e421c30c0 100644 --- a/tests/api_resources/fine_tuning/test_jobs.py +++ b/tests/api_resources/fine_tuning/test_jobs.py @@ -50,6 +50,24 @@ def test_method_create_with_all_params(self, client: OpenAI) -> None: }, } ], + method={ + "dpo": { + "hyperparameters": { + "batch_size": "auto", + "beta": "auto", + "learning_rate_multiplier": "auto", + "n_epochs": "auto", + } + }, + "supervised": { + "hyperparameters": { + "batch_size": "auto", + "learning_rate_multiplier": "auto", + "n_epochs": "auto", + } + }, + "type": "supervised", + }, seed=42, suffix="x", validation_file="file-abc123", @@ -271,6 +289,24 @@ async def test_method_create_with_all_params(self, async_client: AsyncOpenAI) -> }, } ], + method={ + "dpo": { + "hyperparameters": { + "batch_size": "auto", + "beta": "auto", + "learning_rate_multiplier": "auto", + "n_epochs": "auto", + } + }, + "supervised": { + "hyperparameters": { + "batch_size": "auto", + "learning_rate_multiplier": "auto", + "n_epochs": "auto", + } + }, + "type": "supervised", + }, seed=42, suffix="x", validation_file="file-abc123", diff --git a/tests/test_client.py b/tests/test_client.py index 7751e7d463..e0d23403b1 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -795,7 +795,7 @@ def retry_handler(_request: httpx.Request) -> httpx.Response: messages=[ { "content": "string", - "role": "system", + "role": "developer", } ], model="gpt-4o", @@ -827,7 +827,7 @@ def retry_handler(_request: httpx.Request) -> httpx.Response: messages=[ { "content": "string", - "role": "system", + "role": "developer", } ], model="gpt-4o", @@ -859,7 +859,7 @@ def retry_handler(_request: httpx.Request) -> httpx.Response: messages=[ { "content": "string", - "role": "system", + "role": "developer", } ], model="gpt-4o", @@ -891,7 +891,7 @@ def retry_handler(_request: httpx.Request) -> httpx.Response: messages=[ { "content": "string", - "role": "system", + "role": "developer", } ], model="gpt-4o", @@ -1663,7 +1663,7 @@ def retry_handler(_request: httpx.Request) -> httpx.Response: messages=[ { "content": "string", - "role": "system", + "role": "developer", } ], model="gpt-4o", @@ -1696,7 +1696,7 @@ def retry_handler(_request: httpx.Request) -> httpx.Response: messages=[ { "content": "string", - "role": "system", + "role": "developer", } ], model="gpt-4o", @@ -1729,7 +1729,7 @@ def retry_handler(_request: httpx.Request) -> httpx.Response: messages=[ { "content": "string", - "role": "system", + "role": "developer", } ], model="gpt-4o", @@ -1762,7 +1762,7 @@ def retry_handler(_request: httpx.Request) -> httpx.Response: messages=[ { "content": "string", - "role": "system", + "role": "developer", } ], model="gpt-4o", From 97d73cf89935ca6098bb889a92f0ec2cdff16989 Mon Sep 17 00:00:00 2001 From: "stainless-app[bot]" <142633134+stainless-app[bot]@users.noreply.github.com> Date: Tue, 17 Dec 2024 18:09:06 +0000 Subject: [PATCH 6/9] feat: add Realtime API support (#1958) More information on the Realtime API can be found here: https://platform.openai.com/docs/guides/realtime --- .stats.yml | 2 +- api.md | 51 ++ pyproject.toml | 7 +- requirements-dev.lock | 2 + requirements.lock | 2 + src/openai/_client.py | 26 + src/openai/lib/azure.py | 14 + .../resources/beta/realtime/realtime.py | 852 ++++++++++++++++++ src/openai/types/__init__.py | 1 + src/openai/types/beta/realtime/__init__.py | 74 ++ .../realtime/conversation_created_event.py | 27 + .../types/beta/realtime/conversation_item.py | 61 ++ .../realtime/conversation_item_content.py | 28 + .../conversation_item_content_param.py | 27 + .../conversation_item_create_event.py | 28 + .../conversation_item_create_event_param.py | 28 + .../conversation_item_created_event.py | 25 + .../conversation_item_delete_event.py | 19 + .../conversation_item_delete_event_param.py | 18 + .../conversation_item_deleted_event.py | 18 + ...put_audio_transcription_completed_event.py | 26 + ..._input_audio_transcription_failed_event.py | 39 + .../beta/realtime/conversation_item_param.py | 62 ++ .../conversation_item_truncate_event.py | 32 + .../conversation_item_truncate_event_param.py | 31 + .../conversation_item_truncated_event.py | 24 + src/openai/types/beta/realtime/error_event.py | 36 + .../input_audio_buffer_append_event.py | 23 + .../input_audio_buffer_append_event_param.py | 22 + .../input_audio_buffer_clear_event.py | 16 + .../input_audio_buffer_clear_event_param.py | 15 + .../input_audio_buffer_cleared_event.py | 15 + .../input_audio_buffer_commit_event.py | 16 + .../input_audio_buffer_commit_event_param.py | 15 + .../input_audio_buffer_committed_event.py | 21 + ...input_audio_buffer_speech_started_event.py | 26 + ...input_audio_buffer_speech_stopped_event.py | 25 + .../realtime/rate_limits_updated_event.py | 33 + .../beta/realtime/realtime_client_event.py | 32 + .../realtime/realtime_client_event_param.py | 30 + .../beta/realtime/realtime_connect_params.py | 11 + .../types/beta/realtime/realtime_response.py | 42 + .../beta/realtime/realtime_response_status.py | 39 + .../beta/realtime/realtime_response_usage.py | 52 ++ .../beta/realtime/realtime_server_event.py | 72 ++ .../realtime/response_audio_delta_event.py | 30 + .../realtime/response_audio_done_event.py | 27 + .../response_audio_transcript_delta_event.py | 30 + .../response_audio_transcript_done_event.py | 30 + .../beta/realtime/response_cancel_event.py | 22 + .../realtime/response_cancel_event_param.py | 21 + .../response_content_part_added_event.py | 45 + .../response_content_part_done_event.py | 45 + .../beta/realtime/response_create_event.py | 115 +++ .../realtime/response_create_event_param.py | 116 +++ .../beta/realtime/response_created_event.py | 19 + .../beta/realtime/response_done_event.py | 19 + ...nse_function_call_arguments_delta_event.py | 30 + ...onse_function_call_arguments_done_event.py | 30 + .../response_output_item_added_event.py | 25 + .../response_output_item_done_event.py | 25 + .../realtime/response_text_delta_event.py | 30 + .../beta/realtime/response_text_done_event.py | 30 + src/openai/types/beta/realtime/session.py | 148 +++ .../beta/realtime/session_created_event.py | 19 + .../beta/realtime/session_update_event.py | 158 ++++ .../realtime/session_update_event_param.py | 166 ++++ .../beta/realtime/session_updated_event.py | 19 + .../types/websocket_connection_options.py | 36 + tests/api_resources/beta/test_realtime.py | 17 + 70 files changed, 3313 insertions(+), 4 deletions(-) create mode 100644 src/openai/types/beta/realtime/conversation_created_event.py create mode 100644 src/openai/types/beta/realtime/conversation_item.py create mode 100644 src/openai/types/beta/realtime/conversation_item_content.py create mode 100644 src/openai/types/beta/realtime/conversation_item_content_param.py create mode 100644 src/openai/types/beta/realtime/conversation_item_create_event.py create mode 100644 src/openai/types/beta/realtime/conversation_item_create_event_param.py create mode 100644 src/openai/types/beta/realtime/conversation_item_created_event.py create mode 100644 src/openai/types/beta/realtime/conversation_item_delete_event.py create mode 100644 src/openai/types/beta/realtime/conversation_item_delete_event_param.py create mode 100644 src/openai/types/beta/realtime/conversation_item_deleted_event.py create mode 100644 src/openai/types/beta/realtime/conversation_item_input_audio_transcription_completed_event.py create mode 100644 src/openai/types/beta/realtime/conversation_item_input_audio_transcription_failed_event.py create mode 100644 src/openai/types/beta/realtime/conversation_item_param.py create mode 100644 src/openai/types/beta/realtime/conversation_item_truncate_event.py create mode 100644 src/openai/types/beta/realtime/conversation_item_truncate_event_param.py create mode 100644 src/openai/types/beta/realtime/conversation_item_truncated_event.py create mode 100644 src/openai/types/beta/realtime/error_event.py create mode 100644 src/openai/types/beta/realtime/input_audio_buffer_append_event.py create mode 100644 src/openai/types/beta/realtime/input_audio_buffer_append_event_param.py create mode 100644 src/openai/types/beta/realtime/input_audio_buffer_clear_event.py create mode 100644 src/openai/types/beta/realtime/input_audio_buffer_clear_event_param.py create mode 100644 src/openai/types/beta/realtime/input_audio_buffer_cleared_event.py create mode 100644 src/openai/types/beta/realtime/input_audio_buffer_commit_event.py create mode 100644 src/openai/types/beta/realtime/input_audio_buffer_commit_event_param.py create mode 100644 src/openai/types/beta/realtime/input_audio_buffer_committed_event.py create mode 100644 src/openai/types/beta/realtime/input_audio_buffer_speech_started_event.py create mode 100644 src/openai/types/beta/realtime/input_audio_buffer_speech_stopped_event.py create mode 100644 src/openai/types/beta/realtime/rate_limits_updated_event.py create mode 100644 src/openai/types/beta/realtime/realtime_client_event.py create mode 100644 src/openai/types/beta/realtime/realtime_client_event_param.py create mode 100644 src/openai/types/beta/realtime/realtime_connect_params.py create mode 100644 src/openai/types/beta/realtime/realtime_response.py create mode 100644 src/openai/types/beta/realtime/realtime_response_status.py create mode 100644 src/openai/types/beta/realtime/realtime_response_usage.py create mode 100644 src/openai/types/beta/realtime/realtime_server_event.py create mode 100644 src/openai/types/beta/realtime/response_audio_delta_event.py create mode 100644 src/openai/types/beta/realtime/response_audio_done_event.py create mode 100644 src/openai/types/beta/realtime/response_audio_transcript_delta_event.py create mode 100644 src/openai/types/beta/realtime/response_audio_transcript_done_event.py create mode 100644 src/openai/types/beta/realtime/response_cancel_event.py create mode 100644 src/openai/types/beta/realtime/response_cancel_event_param.py create mode 100644 src/openai/types/beta/realtime/response_content_part_added_event.py create mode 100644 src/openai/types/beta/realtime/response_content_part_done_event.py create mode 100644 src/openai/types/beta/realtime/response_create_event.py create mode 100644 src/openai/types/beta/realtime/response_create_event_param.py create mode 100644 src/openai/types/beta/realtime/response_created_event.py create mode 100644 src/openai/types/beta/realtime/response_done_event.py create mode 100644 src/openai/types/beta/realtime/response_function_call_arguments_delta_event.py create mode 100644 src/openai/types/beta/realtime/response_function_call_arguments_done_event.py create mode 100644 src/openai/types/beta/realtime/response_output_item_added_event.py create mode 100644 src/openai/types/beta/realtime/response_output_item_done_event.py create mode 100644 src/openai/types/beta/realtime/response_text_delta_event.py create mode 100644 src/openai/types/beta/realtime/response_text_done_event.py create mode 100644 src/openai/types/beta/realtime/session.py create mode 100644 src/openai/types/beta/realtime/session_created_event.py create mode 100644 src/openai/types/beta/realtime/session_update_event.py create mode 100644 src/openai/types/beta/realtime/session_update_event_param.py create mode 100644 src/openai/types/beta/realtime/session_updated_event.py create mode 100644 src/openai/types/websocket_connection_options.py create mode 100644 tests/api_resources/beta/test_realtime.py diff --git a/.stats.yml b/.stats.yml index e3a0040a5a..12219ccaa1 100644 --- a/.stats.yml +++ b/.stats.yml @@ -1,2 +1,2 @@ configured_endpoints: 69 -openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/openai-779ea2754025daf5e18eb8ceb203ec321692636bc3a999338556a479178efa6c.yml +openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/openai-0d64ca9e45f51b4279f87b205eeb3a3576df98407698ce053f2e2302c1c08df1.yml diff --git a/api.md b/api.md index 91b2a9c2fd..ace93e0559 100644 --- a/api.md +++ b/api.md @@ -239,6 +239,57 @@ Methods: ## Realtime +Types: + +```python +from openai.types.beta.realtime import ( + ConversationCreatedEvent, + ConversationItem, + ConversationItemContent, + ConversationItemCreateEvent, + ConversationItemCreatedEvent, + ConversationItemDeleteEvent, + ConversationItemDeletedEvent, + ConversationItemInputAudioTranscriptionCompletedEvent, + ConversationItemInputAudioTranscriptionFailedEvent, + ConversationItemTruncateEvent, + ConversationItemTruncatedEvent, + ErrorEvent, + InputAudioBufferAppendEvent, + InputAudioBufferClearEvent, + InputAudioBufferClearedEvent, + InputAudioBufferCommitEvent, + InputAudioBufferCommittedEvent, + InputAudioBufferSpeechStartedEvent, + InputAudioBufferSpeechStoppedEvent, + RateLimitsUpdatedEvent, + RealtimeClientEvent, + RealtimeResponse, + RealtimeResponseStatus, + RealtimeResponseUsage, + RealtimeServerEvent, + ResponseAudioDeltaEvent, + ResponseAudioDoneEvent, + ResponseAudioTranscriptDeltaEvent, + ResponseAudioTranscriptDoneEvent, + ResponseCancelEvent, + ResponseContentPartAddedEvent, + ResponseContentPartDoneEvent, + ResponseCreateEvent, + ResponseCreatedEvent, + ResponseDoneEvent, + ResponseFunctionCallArgumentsDeltaEvent, + ResponseFunctionCallArgumentsDoneEvent, + ResponseOutputItemAddedEvent, + ResponseOutputItemDoneEvent, + ResponseTextDeltaEvent, + ResponseTextDoneEvent, + SessionCreatedEvent, + SessionUpdateEvent, + SessionUpdatedEvent, +) +``` + ### Sessions Types: diff --git a/pyproject.toml b/pyproject.toml index e03d4e798f..f83aff6fee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,9 +35,6 @@ classifiers = [ "License :: OSI Approved :: Apache Software License" ] -[project.optional-dependencies] -datalib = ["numpy >= 1", "pandas >= 1.2.3", "pandas-stubs >= 1.1.0.11"] - [project.urls] Homepage = "https://github.com/openai/openai-python" Repository = "https://github.com/openai/openai-python" @@ -45,6 +42,10 @@ Repository = "https://github.com/openai/openai-python" [project.scripts] openai = "openai.cli:main" +[project.optional-dependencies] +realtime = ["websockets >= 13, < 15"] +datalib = ["numpy >= 1", "pandas >= 1.2.3", "pandas-stubs >= 1.1.0.11"] + [tool.rye] managed = true # version pins are in requirements-dev.lock diff --git a/requirements-dev.lock b/requirements-dev.lock index 2cf6ab5ea9..94cf6aca07 100644 --- a/requirements-dev.lock +++ b/requirements-dev.lock @@ -185,5 +185,7 @@ urllib3==2.2.1 # via requests virtualenv==20.24.5 # via nox +websockets==14.1 + # via openai zipp==3.17.0 # via importlib-metadata diff --git a/requirements.lock b/requirements.lock index 826f0bc927..c10449ac20 100644 --- a/requirements.lock +++ b/requirements.lock @@ -64,3 +64,5 @@ typing-extensions==4.12.2 # via pydantic-core tzdata==2024.1 # via pandas +websockets==14.1 + # via openai diff --git a/src/openai/_client.py b/src/openai/_client.py index 5419e88f06..c784694f20 100644 --- a/src/openai/_client.py +++ b/src/openai/_client.py @@ -63,6 +63,14 @@ class OpenAI(SyncAPIClient): organization: str | None project: str | None + websocket_base_url: str | httpx.URL | None + """Base URL for WebSocket connections. + + If not specified, the default base URL will be used, with 'wss://' replacing the + 'http://' or 'https://' scheme. For example: 'http://example.com' becomes + 'wss://example.com' + """ + def __init__( self, *, @@ -70,6 +78,7 @@ def __init__( organization: str | None = None, project: str | None = None, base_url: str | httpx.URL | None = None, + websocket_base_url: str | httpx.URL | None = None, timeout: Union[float, Timeout, None, NotGiven] = NOT_GIVEN, max_retries: int = DEFAULT_MAX_RETRIES, default_headers: Mapping[str, str] | None = None, @@ -111,6 +120,8 @@ def __init__( project = os.environ.get("OPENAI_PROJECT_ID") self.project = project + self.websocket_base_url = websocket_base_url + if base_url is None: base_url = os.environ.get("OPENAI_BASE_URL") if base_url is None: @@ -172,6 +183,7 @@ def copy( api_key: str | None = None, organization: str | None = None, project: str | None = None, + websocket_base_url: str | httpx.URL | None = None, base_url: str | httpx.URL | None = None, timeout: float | Timeout | None | NotGiven = NOT_GIVEN, http_client: httpx.Client | None = None, @@ -208,6 +220,7 @@ def copy( api_key=api_key or self.api_key, organization=organization or self.organization, project=project or self.project, + websocket_base_url=websocket_base_url or self.websocket_base_url, base_url=base_url or self.base_url, timeout=self.timeout if isinstance(timeout, NotGiven) else timeout, http_client=http_client, @@ -277,6 +290,14 @@ class AsyncOpenAI(AsyncAPIClient): organization: str | None project: str | None + websocket_base_url: str | httpx.URL | None + """Base URL for WebSocket connections. + + If not specified, the default base URL will be used, with 'wss://' replacing the + 'http://' or 'https://' scheme. For example: 'http://example.com' becomes + 'wss://example.com' + """ + def __init__( self, *, @@ -284,6 +305,7 @@ def __init__( organization: str | None = None, project: str | None = None, base_url: str | httpx.URL | None = None, + websocket_base_url: str | httpx.URL | None = None, timeout: Union[float, Timeout, None, NotGiven] = NOT_GIVEN, max_retries: int = DEFAULT_MAX_RETRIES, default_headers: Mapping[str, str] | None = None, @@ -325,6 +347,8 @@ def __init__( project = os.environ.get("OPENAI_PROJECT_ID") self.project = project + self.websocket_base_url = websocket_base_url + if base_url is None: base_url = os.environ.get("OPENAI_BASE_URL") if base_url is None: @@ -386,6 +410,7 @@ def copy( api_key: str | None = None, organization: str | None = None, project: str | None = None, + websocket_base_url: str | httpx.URL | None = None, base_url: str | httpx.URL | None = None, timeout: float | Timeout | None | NotGiven = NOT_GIVEN, http_client: httpx.AsyncClient | None = None, @@ -422,6 +447,7 @@ def copy( api_key=api_key or self.api_key, organization=organization or self.organization, project=project or self.project, + websocket_base_url=websocket_base_url or self.websocket_base_url, base_url=base_url or self.base_url, timeout=self.timeout if isinstance(timeout, NotGiven) else timeout, http_client=http_client, diff --git a/src/openai/lib/azure.py b/src/openai/lib/azure.py index 54122dbecb..13d9f31838 100644 --- a/src/openai/lib/azure.py +++ b/src/openai/lib/azure.py @@ -76,6 +76,7 @@ def __init__( azure_ad_token: str | None = None, azure_ad_token_provider: AzureADTokenProvider | None = None, organization: str | None = None, + websocket_base_url: str | httpx.URL | None = None, timeout: float | Timeout | None | NotGiven = NOT_GIVEN, max_retries: int = DEFAULT_MAX_RETRIES, default_headers: Mapping[str, str] | None = None, @@ -94,6 +95,7 @@ def __init__( azure_ad_token: str | None = None, azure_ad_token_provider: AzureADTokenProvider | None = None, organization: str | None = None, + websocket_base_url: str | httpx.URL | None = None, timeout: float | Timeout | None | NotGiven = NOT_GIVEN, max_retries: int = DEFAULT_MAX_RETRIES, default_headers: Mapping[str, str] | None = None, @@ -112,6 +114,7 @@ def __init__( azure_ad_token: str | None = None, azure_ad_token_provider: AzureADTokenProvider | None = None, organization: str | None = None, + websocket_base_url: str | httpx.URL | None = None, timeout: float | Timeout | None | NotGiven = NOT_GIVEN, max_retries: int = DEFAULT_MAX_RETRIES, default_headers: Mapping[str, str] | None = None, @@ -131,6 +134,7 @@ def __init__( azure_ad_token_provider: AzureADTokenProvider | None = None, organization: str | None = None, project: str | None = None, + websocket_base_url: str | httpx.URL | None = None, base_url: str | None = None, timeout: float | Timeout | None | NotGiven = NOT_GIVEN, max_retries: int = DEFAULT_MAX_RETRIES, @@ -214,6 +218,7 @@ def __init__( default_headers=default_headers, default_query=default_query, http_client=http_client, + websocket_base_url=websocket_base_url, _strict_response_validation=_strict_response_validation, ) self._api_version = api_version @@ -227,6 +232,7 @@ def copy( api_key: str | None = None, organization: str | None = None, project: str | None = None, + websocket_base_url: str | httpx.URL | None = None, api_version: str | None = None, azure_ad_token: str | None = None, azure_ad_token_provider: AzureADTokenProvider | None = None, @@ -247,6 +253,7 @@ def copy( api_key=api_key, organization=organization, project=project, + websocket_base_url=websocket_base_url, base_url=base_url, timeout=timeout, http_client=http_client, @@ -314,6 +321,7 @@ def __init__( azure_ad_token_provider: AsyncAzureADTokenProvider | None = None, organization: str | None = None, project: str | None = None, + websocket_base_url: str | httpx.URL | None = None, timeout: float | Timeout | None | NotGiven = NOT_GIVEN, max_retries: int = DEFAULT_MAX_RETRIES, default_headers: Mapping[str, str] | None = None, @@ -333,6 +341,7 @@ def __init__( azure_ad_token_provider: AsyncAzureADTokenProvider | None = None, organization: str | None = None, project: str | None = None, + websocket_base_url: str | httpx.URL | None = None, timeout: float | Timeout | None | NotGiven = NOT_GIVEN, max_retries: int = DEFAULT_MAX_RETRIES, default_headers: Mapping[str, str] | None = None, @@ -352,6 +361,7 @@ def __init__( azure_ad_token_provider: AsyncAzureADTokenProvider | None = None, organization: str | None = None, project: str | None = None, + websocket_base_url: str | httpx.URL | None = None, timeout: float | Timeout | None | NotGiven = NOT_GIVEN, max_retries: int = DEFAULT_MAX_RETRIES, default_headers: Mapping[str, str] | None = None, @@ -372,6 +382,7 @@ def __init__( organization: str | None = None, project: str | None = None, base_url: str | None = None, + websocket_base_url: str | httpx.URL | None = None, timeout: float | Timeout | None | NotGiven = NOT_GIVEN, max_retries: int = DEFAULT_MAX_RETRIES, default_headers: Mapping[str, str] | None = None, @@ -454,6 +465,7 @@ def __init__( default_headers=default_headers, default_query=default_query, http_client=http_client, + websocket_base_url=websocket_base_url, _strict_response_validation=_strict_response_validation, ) self._api_version = api_version @@ -467,6 +479,7 @@ def copy( api_key: str | None = None, organization: str | None = None, project: str | None = None, + websocket_base_url: str | httpx.URL | None = None, api_version: str | None = None, azure_ad_token: str | None = None, azure_ad_token_provider: AsyncAzureADTokenProvider | None = None, @@ -487,6 +500,7 @@ def copy( api_key=api_key, organization=organization, project=project, + websocket_base_url=websocket_base_url, base_url=base_url, timeout=timeout, http_client=http_client, diff --git a/src/openai/resources/beta/realtime/realtime.py b/src/openai/resources/beta/realtime/realtime.py index e57e0be503..c79fd46217 100644 --- a/src/openai/resources/beta/realtime/realtime.py +++ b/src/openai/resources/beta/realtime/realtime.py @@ -2,6 +2,15 @@ from __future__ import annotations +import json +import logging +from types import TracebackType +from typing import TYPE_CHECKING, Any, Iterator, cast +from typing_extensions import AsyncIterator + +import httpx +from pydantic import BaseModel + from .sessions import ( Sessions, AsyncSessions, @@ -10,11 +19,34 @@ SessionsWithStreamingResponse, AsyncSessionsWithStreamingResponse, ) +from ...._types import NOT_GIVEN, Query, Headers, NotGiven +from ...._utils import ( + maybe_transform, + strip_not_given, + async_maybe_transform, +) from ...._compat import cached_property +from ...._models import construct_type_unchecked from ...._resource import SyncAPIResource, AsyncAPIResource +from ...._exceptions import OpenAIError +from ...._base_client import _merge_mappings +from ....types.beta.realtime import session_update_event_param, response_create_event_param +from ....types.websocket_connection_options import WebsocketConnectionOptions +from ....types.beta.realtime.realtime_client_event import RealtimeClientEvent +from ....types.beta.realtime.realtime_server_event import RealtimeServerEvent +from ....types.beta.realtime.conversation_item_param import ConversationItemParam +from ....types.beta.realtime.realtime_client_event_param import RealtimeClientEventParam + +if TYPE_CHECKING: + from websockets.sync.client import ClientConnection as WebsocketConnection + from websockets.asyncio.client import ClientConnection as AsyncWebsocketConnection + + from ...._client import OpenAI, AsyncOpenAI __all__ = ["Realtime", "AsyncRealtime"] +log: logging.Logger = logging.getLogger(__name__) + class Realtime(SyncAPIResource): @cached_property @@ -40,6 +72,33 @@ def with_streaming_response(self) -> RealtimeWithStreamingResponse: """ return RealtimeWithStreamingResponse(self) + def connect( + self, + *, + model: str, + extra_query: Query = {}, + extra_headers: Headers = {}, + websocket_connection_options: WebsocketConnectionOptions = {}, + ) -> RealtimeConnectionManager: + """ + The Realtime API enables you to build low-latency, multi-modal conversational experiences. It currently supports text and audio as both input and output, as well as function calling. + + Some notable benefits of the API include: + + - Native speech-to-speech: Skipping an intermediate text format means low latency and nuanced output. + - Natural, steerable voices: The models have natural inflection and can laugh, whisper, and adhere to tone direction. + - Simultaneous multimodal output: Text is useful for moderation; faster-than-realtime audio ensures stable playback. + + The Realtime API is a stateful, event-based API that communicates over a WebSocket. + """ + return RealtimeConnectionManager( + client=self._client, + extra_query=extra_query, + extra_headers=extra_headers, + websocket_connection_options=websocket_connection_options, + model=model, + ) + class AsyncRealtime(AsyncAPIResource): @cached_property @@ -65,6 +124,33 @@ def with_streaming_response(self) -> AsyncRealtimeWithStreamingResponse: """ return AsyncRealtimeWithStreamingResponse(self) + def connect( + self, + *, + model: str, + extra_query: Query = {}, + extra_headers: Headers = {}, + websocket_connection_options: WebsocketConnectionOptions = {}, + ) -> AsyncRealtimeConnectionManager: + """ + The Realtime API enables you to build low-latency, multi-modal conversational experiences. It currently supports text and audio as both input and output, as well as function calling. + + Some notable benefits of the API include: + + - Native speech-to-speech: Skipping an intermediate text format means low latency and nuanced output. + - Natural, steerable voices: The models have natural inflection and can laugh, whisper, and adhere to tone direction. + - Simultaneous multimodal output: Text is useful for moderation; faster-than-realtime audio ensures stable playback. + + The Realtime API is a stateful, event-based API that communicates over a WebSocket. + """ + return AsyncRealtimeConnectionManager( + client=self._client, + extra_query=extra_query, + extra_headers=extra_headers, + websocket_connection_options=websocket_connection_options, + model=model, + ) + class RealtimeWithRawResponse: def __init__(self, realtime: Realtime) -> None: @@ -100,3 +186,769 @@ def __init__(self, realtime: AsyncRealtime) -> None: @cached_property def sessions(self) -> AsyncSessionsWithStreamingResponse: return AsyncSessionsWithStreamingResponse(self._realtime.sessions) + + +class AsyncRealtimeConnection: + """Represents a live websocket connection to the Realtime API""" + + session: AsyncRealtimeSessionResource + response: AsyncRealtimeResponseResource + conversation: AsyncRealtimeConversationResource + input_audio_buffer: AsyncRealtimeInputAudioBufferResource + + _connection: AsyncWebsocketConnection + + def __init__(self, connection: AsyncWebsocketConnection) -> None: + self._connection = connection + + self.session = AsyncRealtimeSessionResource(self) + self.response = AsyncRealtimeResponseResource(self) + self.conversation = AsyncRealtimeConversationResource(self) + self.input_audio_buffer = AsyncRealtimeInputAudioBufferResource(self) + + async def __aiter__(self) -> AsyncIterator[RealtimeServerEvent]: + """ + An infinite-iterator that will continue to yield events until + the connection is closed. + """ + from websockets.exceptions import ConnectionClosedOK + + try: + while True: + yield await self.recv() + except ConnectionClosedOK: + return + + async def recv(self) -> RealtimeServerEvent: + """ + Receive the next message from the connection and parses it into a `RealtimeServerEvent` object. + + Canceling this method is safe. There's no risk of losing data. + """ + return self.parse_event(await self.recv_bytes()) + + async def recv_bytes(self) -> bytes: + """Receive the next message from the connection as raw bytes. + + Canceling this method is safe. There's no risk of losing data. + + If you want to parse the message into a `RealtimeServerEvent` object like `.recv()` does, + then you can call `.parse_event(data)`. + """ + message = await self._connection.recv(decode=False) + log.debug(f"Received websocket message: %s", message) + if not isinstance(message, bytes): + # passing `decode=False` should always result in us getting `bytes` back + raise TypeError(f"Expected `.recv(decode=False)` to return `bytes` but got {type(message)}") + + return message + + async def send(self, event: RealtimeClientEvent | RealtimeClientEventParam) -> None: + data = ( + event.to_json(use_api_names=True, exclude_defaults=True, exclude_unset=True) + if isinstance(event, BaseModel) + else json.dumps(await async_maybe_transform(event, RealtimeClientEventParam)) + ) + await self._connection.send(data) + + async def close(self, *, code: int = 1000, reason: str = "") -> None: + await self._connection.close(code=code, reason=reason) + + def parse_event(self, data: str | bytes) -> RealtimeServerEvent: + """ + Converts a raw `str` or `bytes` message into a `RealtimeServerEvent` object. + + This is helpful if you're using `.recv_bytes()`. + """ + return cast( + RealtimeServerEvent, construct_type_unchecked(value=json.loads(data), type_=cast(Any, RealtimeServerEvent)) + ) + + +class AsyncRealtimeConnectionManager: + """ + Context manager over a `AsyncRealtimeConnection` that is returned by `beta.realtime.connect()` + + This context manager ensures that the connection will be closed when it exits. + + --- + + Note that if your application doesn't work well with the context manager approach then you + can call the `.enter()` method directly to initiate a connection. + + **Warning**: You must remember to close the connection with `.close()`. + + ```py + connection = await client.beta.realtime.connect(...).enter() + # ... + await connection.close() + ``` + """ + + def __init__( + self, + *, + client: AsyncOpenAI, + model: str, + extra_query: Query, + extra_headers: Headers, + websocket_connection_options: WebsocketConnectionOptions, + ) -> None: + self.__client = client + self.__model = model + self.__connection: AsyncRealtimeConnection | None = None + self.__extra_query = extra_query + self.__extra_headers = extra_headers + self.__websocket_connection_options = websocket_connection_options + + async def __aenter__(self) -> AsyncRealtimeConnection: + """ + 👋 If your application doesn't work well with the context manager approach then you + can call this method directly to initiate a connection. + + **Warning**: You must remember to close the connection with `.close()`. + + ```py + connection = await client.beta.realtime.connect(...).enter() + # ... + await connection.close() + ``` + """ + try: + from websockets.asyncio.client import connect + except ImportError as exc: + raise OpenAIError("You need to install `openai[realtime]` to use this method") from exc + + url = self._prepare_url().copy_with( + params={ + **self.__client.base_url.params, + "model": self.__model, + **self.__extra_query, + }, + ) + log.debug("Connecting to %s", url) + if self.__websocket_connection_options: + log.debug("Connection options: %s", self.__websocket_connection_options) + + self.__connection = AsyncRealtimeConnection( + await connect( + str(url), + user_agent_header=self.__client.user_agent, + additional_headers=_merge_mappings( + { + **self.__client.auth_headers, + "OpenAI-Beta": "realtime=v1", + }, + self.__extra_headers, + ), + **self.__websocket_connection_options, + ) + ) + + return self.__connection + + enter = __aenter__ + + def _prepare_url(self) -> httpx.URL: + if self.__client.websocket_base_url is not None: + base_url = httpx.URL(self.__client.websocket_base_url) + else: + base_url = self.__client._base_url.copy_with(scheme="wss") + + merge_raw_path = base_url.raw_path.rstrip(b"/") + b"/realtime" + return base_url.copy_with(raw_path=merge_raw_path) + + async def __aexit__( + self, exc_type: type[BaseException] | None, exc: BaseException | None, exc_tb: TracebackType | None + ) -> None: + if self.__connection is not None: + await self.__connection.close() + + +class RealtimeConnection: + """Represents a live websocket connection to the Realtime API""" + + session: RealtimeSessionResource + response: RealtimeResponseResource + conversation: RealtimeConversationResource + input_audio_buffer: RealtimeInputAudioBufferResource + + _connection: WebsocketConnection + + def __init__(self, connection: WebsocketConnection) -> None: + self._connection = connection + + self.session = RealtimeSessionResource(self) + self.response = RealtimeResponseResource(self) + self.conversation = RealtimeConversationResource(self) + self.input_audio_buffer = RealtimeInputAudioBufferResource(self) + + def __iter__(self) -> Iterator[RealtimeServerEvent]: + """ + An infinite-iterator that will continue to yield events until + the connection is closed. + """ + from websockets.exceptions import ConnectionClosedOK + + try: + while True: + yield self.recv() + except ConnectionClosedOK: + return + + def recv(self) -> RealtimeServerEvent: + """ + Receive the next message from the connection and parses it into a `RealtimeServerEvent` object. + + Canceling this method is safe. There's no risk of losing data. + """ + return self.parse_event(self.recv_bytes()) + + def recv_bytes(self) -> bytes: + """Receive the next message from the connection as raw bytes. + + Canceling this method is safe. There's no risk of losing data. + + If you want to parse the message into a `RealtimeServerEvent` object like `.recv()` does, + then you can call `.parse_event(data)`. + """ + message = self._connection.recv(decode=False) + log.debug(f"Received websocket message: %s", message) + if not isinstance(message, bytes): + # passing `decode=False` should always result in us getting `bytes` back + raise TypeError(f"Expected `.recv(decode=False)` to return `bytes` but got {type(message)}") + + return message + + def send(self, event: RealtimeClientEvent | RealtimeClientEventParam) -> None: + data = ( + event.to_json(use_api_names=True, exclude_defaults=True, exclude_unset=True) + if isinstance(event, BaseModel) + else json.dumps(maybe_transform(event, RealtimeClientEventParam)) + ) + self._connection.send(data) + + def close(self, *, code: int = 1000, reason: str = "") -> None: + self._connection.close(code=code, reason=reason) + + def parse_event(self, data: str | bytes) -> RealtimeServerEvent: + """ + Converts a raw `str` or `bytes` message into a `RealtimeServerEvent` object. + + This is helpful if you're using `.recv_bytes()`. + """ + return cast( + RealtimeServerEvent, construct_type_unchecked(value=json.loads(data), type_=cast(Any, RealtimeServerEvent)) + ) + + +class RealtimeConnectionManager: + """ + Context manager over a `RealtimeConnection` that is returned by `beta.realtime.connect()` + + This context manager ensures that the connection will be closed when it exits. + + --- + + Note that if your application doesn't work well with the context manager approach then you + can call the `.enter()` method directly to initiate a connection. + + **Warning**: You must remember to close the connection with `.close()`. + + ```py + connection = client.beta.realtime.connect(...).enter() + # ... + connection.close() + ``` + """ + + def __init__( + self, + *, + client: OpenAI, + model: str, + extra_query: Query, + extra_headers: Headers, + websocket_connection_options: WebsocketConnectionOptions, + ) -> None: + self.__client = client + self.__model = model + self.__connection: RealtimeConnection | None = None + self.__extra_query = extra_query + self.__extra_headers = extra_headers + self.__websocket_connection_options = websocket_connection_options + + def __enter__(self) -> RealtimeConnection: + """ + 👋 If your application doesn't work well with the context manager approach then you + can call this method directly to initiate a connection. + + **Warning**: You must remember to close the connection with `.close()`. + + ```py + connection = client.beta.realtime.connect(...).enter() + # ... + connection.close() + ``` + """ + try: + from websockets.sync.client import connect + except ImportError as exc: + raise OpenAIError("You need to install `openai[realtime]` to use this method") from exc + + url = self._prepare_url().copy_with( + params={ + **self.__client.base_url.params, + "model": self.__model, + **self.__extra_query, + }, + ) + log.debug("Connecting to %s", url) + if self.__websocket_connection_options: + log.debug("Connection options: %s", self.__websocket_connection_options) + + self.__connection = RealtimeConnection( + connect( + str(url), + user_agent_header=self.__client.user_agent, + additional_headers=_merge_mappings( + { + **self.__client.auth_headers, + "OpenAI-Beta": "realtime=v1", + }, + self.__extra_headers, + ), + **self.__websocket_connection_options, + ) + ) + + return self.__connection + + enter = __enter__ + + def _prepare_url(self) -> httpx.URL: + if self.__client.websocket_base_url is not None: + base_url = httpx.URL(self.__client.websocket_base_url) + else: + base_url = self.__client._base_url.copy_with(scheme="wss") + + merge_raw_path = base_url.raw_path.rstrip(b"/") + b"/realtime" + return base_url.copy_with(raw_path=merge_raw_path) + + def __exit__( + self, exc_type: type[BaseException] | None, exc: BaseException | None, exc_tb: TracebackType | None + ) -> None: + if self.__connection is not None: + self.__connection.close() + + +class BaseRealtimeConnectionResource: + def __init__(self, connection: RealtimeConnection) -> None: + self._connection = connection + + +class RealtimeSessionResource(BaseRealtimeConnectionResource): + def update(self, *, session: session_update_event_param.Session, event_id: str | NotGiven = NOT_GIVEN) -> None: + """Send this event to update the session’s default configuration. + + The client may + send this event at any time to update the session configuration, and any + field may be updated at any time, except for "voice". The server will respond + with a `session.updated` event that shows the full effective configuration. + Only fields that are present are updated, thus the correct way to clear a + field like "instructions" is to pass an empty string. + """ + self._connection.send( + cast( + RealtimeClientEventParam, + strip_not_given({"type": "session.update", "session": session, "event_id": event_id}), + ) + ) + + +class RealtimeResponseResource(BaseRealtimeConnectionResource): + def cancel(self, *, event_id: str | NotGiven = NOT_GIVEN, response_id: str | NotGiven = NOT_GIVEN) -> None: + """Send this event to cancel an in-progress response. + + The server will respond + with a `response.cancelled` event or an error if there is no response to + cancel. + """ + self._connection.send( + cast( + RealtimeClientEventParam, + strip_not_given({"type": "response.cancel", "event_id": event_id, "response_id": response_id}), + ) + ) + + def create( + self, + *, + event_id: str | NotGiven = NOT_GIVEN, + response: response_create_event_param.Response | NotGiven = NOT_GIVEN, + ) -> None: + """ + This event instructs the server to create a Response, which means triggering + model inference. When in Server VAD mode, the server will create Responses + automatically. + + A Response will include at least one Item, and may have two, in which case + the second will be a function call. These Items will be appended to the + conversation history. + + The server will respond with a `response.created` event, events for Items + and content created, and finally a `response.done` event to indicate the + Response is complete. + + The `response.create` event includes inference configuration like + `instructions`, and `temperature`. These fields will override the Session's + configuration for this Response only. + """ + self._connection.send( + cast( + RealtimeClientEventParam, + strip_not_given({"type": "response.create", "event_id": event_id, "response": response}), + ) + ) + + +class RealtimeConversationResource(BaseRealtimeConnectionResource): + @cached_property + def item(self) -> RealtimeConversationItemResource: + return RealtimeConversationItemResource(self._connection) + + +class RealtimeConversationItemResource(BaseRealtimeConnectionResource): + def delete(self, *, item_id: str, event_id: str | NotGiven = NOT_GIVEN) -> None: + """Send this event when you want to remove any item from the conversation + history. + + The server will respond with a `conversation.item.deleted` event, + unless the item does not exist in the conversation history, in which case the + server will respond with an error. + """ + self._connection.send( + cast( + RealtimeClientEventParam, + strip_not_given({"type": "conversation.item.delete", "item_id": item_id, "event_id": event_id}), + ) + ) + + def create( + self, + *, + item: ConversationItemParam, + event_id: str | NotGiven = NOT_GIVEN, + previous_item_id: str | NotGiven = NOT_GIVEN, + ) -> None: + """ + Add a new Item to the Conversation's context, including messages, function + calls, and function call responses. This event can be used both to populate a + "history" of the conversation and to add new items mid-stream, but has the + current limitation that it cannot populate assistant audio messages. + + If successful, the server will respond with a `conversation.item.created` + event, otherwise an `error` event will be sent. + """ + self._connection.send( + cast( + RealtimeClientEventParam, + strip_not_given( + { + "type": "conversation.item.create", + "item": item, + "event_id": event_id, + "previous_item_id": previous_item_id, + } + ), + ) + ) + + def truncate( + self, *, audio_end_ms: int, content_index: int, item_id: str, event_id: str | NotGiven = NOT_GIVEN + ) -> None: + """Send this event to truncate a previous assistant message’s audio. + + The server + will produce audio faster than realtime, so this event is useful when the user + interrupts to truncate audio that has already been sent to the client but not + yet played. This will synchronize the server's understanding of the audio with + the client's playback. + + Truncating audio will delete the server-side text transcript to ensure there + is not text in the context that hasn't been heard by the user. + + If successful, the server will respond with a `conversation.item.truncated` + event. + """ + self._connection.send( + cast( + RealtimeClientEventParam, + strip_not_given( + { + "type": "conversation.item.truncate", + "audio_end_ms": audio_end_ms, + "content_index": content_index, + "item_id": item_id, + "event_id": event_id, + } + ), + ) + ) + + +class RealtimeInputAudioBufferResource(BaseRealtimeConnectionResource): + def clear(self, *, event_id: str | NotGiven = NOT_GIVEN) -> None: + """Send this event to clear the audio bytes in the buffer. + + The server will + respond with an `input_audio_buffer.cleared` event. + """ + self._connection.send( + cast(RealtimeClientEventParam, strip_not_given({"type": "input_audio_buffer.clear", "event_id": event_id})) + ) + + def commit(self, *, event_id: str | NotGiven = NOT_GIVEN) -> None: + """ + Send this event to commit the user input audio buffer, which will create a + new user message item in the conversation. This event will produce an error + if the input audio buffer is empty. When in Server VAD mode, the client does + not need to send this event, the server will commit the audio buffer + automatically. + + Committing the input audio buffer will trigger input audio transcription + (if enabled in session configuration), but it will not create a response + from the model. The server will respond with an `input_audio_buffer.committed` + event. + """ + self._connection.send( + cast(RealtimeClientEventParam, strip_not_given({"type": "input_audio_buffer.commit", "event_id": event_id})) + ) + + def append(self, *, audio: str, event_id: str | NotGiven = NOT_GIVEN) -> None: + """Send this event to append audio bytes to the input audio buffer. + + The audio + buffer is temporary storage you can write to and later commit. In Server VAD + mode, the audio buffer is used to detect speech and the server will decide + when to commit. When Server VAD is disabled, you must commit the audio buffer + manually. + + The client may choose how much audio to place in each event up to a maximum + of 15 MiB, for example streaming smaller chunks from the client may allow the + VAD to be more responsive. Unlike made other client events, the server will + not send a confirmation response to this event. + """ + self._connection.send( + cast( + RealtimeClientEventParam, + strip_not_given({"type": "input_audio_buffer.append", "audio": audio, "event_id": event_id}), + ) + ) + + +class BaseAsyncRealtimeConnectionResource: + def __init__(self, connection: AsyncRealtimeConnection) -> None: + self._connection = connection + + +class AsyncRealtimeSessionResource(BaseAsyncRealtimeConnectionResource): + async def update( + self, *, session: session_update_event_param.Session, event_id: str | NotGiven = NOT_GIVEN + ) -> None: + """Send this event to update the session’s default configuration. + + The client may + send this event at any time to update the session configuration, and any + field may be updated at any time, except for "voice". The server will respond + with a `session.updated` event that shows the full effective configuration. + Only fields that are present are updated, thus the correct way to clear a + field like "instructions" is to pass an empty string. + """ + await self._connection.send( + cast( + RealtimeClientEventParam, + strip_not_given({"type": "session.update", "session": session, "event_id": event_id}), + ) + ) + + +class AsyncRealtimeResponseResource(BaseAsyncRealtimeConnectionResource): + async def cancel(self, *, event_id: str | NotGiven = NOT_GIVEN, response_id: str | NotGiven = NOT_GIVEN) -> None: + """Send this event to cancel an in-progress response. + + The server will respond + with a `response.cancelled` event or an error if there is no response to + cancel. + """ + await self._connection.send( + cast( + RealtimeClientEventParam, + strip_not_given({"type": "response.cancel", "event_id": event_id, "response_id": response_id}), + ) + ) + + async def create( + self, + *, + event_id: str | NotGiven = NOT_GIVEN, + response: response_create_event_param.Response | NotGiven = NOT_GIVEN, + ) -> None: + """ + This event instructs the server to create a Response, which means triggering + model inference. When in Server VAD mode, the server will create Responses + automatically. + + A Response will include at least one Item, and may have two, in which case + the second will be a function call. These Items will be appended to the + conversation history. + + The server will respond with a `response.created` event, events for Items + and content created, and finally a `response.done` event to indicate the + Response is complete. + + The `response.create` event includes inference configuration like + `instructions`, and `temperature`. These fields will override the Session's + configuration for this Response only. + """ + await self._connection.send( + cast( + RealtimeClientEventParam, + strip_not_given({"type": "response.create", "event_id": event_id, "response": response}), + ) + ) + + +class AsyncRealtimeConversationResource(BaseAsyncRealtimeConnectionResource): + @cached_property + def item(self) -> AsyncRealtimeConversationItemResource: + return AsyncRealtimeConversationItemResource(self._connection) + + +class AsyncRealtimeConversationItemResource(BaseAsyncRealtimeConnectionResource): + async def delete(self, *, item_id: str, event_id: str | NotGiven = NOT_GIVEN) -> None: + """Send this event when you want to remove any item from the conversation + history. + + The server will respond with a `conversation.item.deleted` event, + unless the item does not exist in the conversation history, in which case the + server will respond with an error. + """ + await self._connection.send( + cast( + RealtimeClientEventParam, + strip_not_given({"type": "conversation.item.delete", "item_id": item_id, "event_id": event_id}), + ) + ) + + async def create( + self, + *, + item: ConversationItemParam, + event_id: str | NotGiven = NOT_GIVEN, + previous_item_id: str | NotGiven = NOT_GIVEN, + ) -> None: + """ + Add a new Item to the Conversation's context, including messages, function + calls, and function call responses. This event can be used both to populate a + "history" of the conversation and to add new items mid-stream, but has the + current limitation that it cannot populate assistant audio messages. + + If successful, the server will respond with a `conversation.item.created` + event, otherwise an `error` event will be sent. + """ + await self._connection.send( + cast( + RealtimeClientEventParam, + strip_not_given( + { + "type": "conversation.item.create", + "item": item, + "event_id": event_id, + "previous_item_id": previous_item_id, + } + ), + ) + ) + + async def truncate( + self, *, audio_end_ms: int, content_index: int, item_id: str, event_id: str | NotGiven = NOT_GIVEN + ) -> None: + """Send this event to truncate a previous assistant message’s audio. + + The server + will produce audio faster than realtime, so this event is useful when the user + interrupts to truncate audio that has already been sent to the client but not + yet played. This will synchronize the server's understanding of the audio with + the client's playback. + + Truncating audio will delete the server-side text transcript to ensure there + is not text in the context that hasn't been heard by the user. + + If successful, the server will respond with a `conversation.item.truncated` + event. + """ + await self._connection.send( + cast( + RealtimeClientEventParam, + strip_not_given( + { + "type": "conversation.item.truncate", + "audio_end_ms": audio_end_ms, + "content_index": content_index, + "item_id": item_id, + "event_id": event_id, + } + ), + ) + ) + + +class AsyncRealtimeInputAudioBufferResource(BaseAsyncRealtimeConnectionResource): + async def clear(self, *, event_id: str | NotGiven = NOT_GIVEN) -> None: + """Send this event to clear the audio bytes in the buffer. + + The server will + respond with an `input_audio_buffer.cleared` event. + """ + await self._connection.send( + cast(RealtimeClientEventParam, strip_not_given({"type": "input_audio_buffer.clear", "event_id": event_id})) + ) + + async def commit(self, *, event_id: str | NotGiven = NOT_GIVEN) -> None: + """ + Send this event to commit the user input audio buffer, which will create a + new user message item in the conversation. This event will produce an error + if the input audio buffer is empty. When in Server VAD mode, the client does + not need to send this event, the server will commit the audio buffer + automatically. + + Committing the input audio buffer will trigger input audio transcription + (if enabled in session configuration), but it will not create a response + from the model. The server will respond with an `input_audio_buffer.committed` + event. + """ + await self._connection.send( + cast(RealtimeClientEventParam, strip_not_given({"type": "input_audio_buffer.commit", "event_id": event_id})) + ) + + async def append(self, *, audio: str, event_id: str | NotGiven = NOT_GIVEN) -> None: + """Send this event to append audio bytes to the input audio buffer. + + The audio + buffer is temporary storage you can write to and later commit. In Server VAD + mode, the audio buffer is used to detect speech and the server will decide + when to commit. When Server VAD is disabled, you must commit the audio buffer + manually. + + The client may choose how much audio to place in each event up to a maximum + of 15 MiB, for example streaming smaller chunks from the client may allow the + VAD to be more responsive. Unlike made other client events, the server will + not send a confirmation response to this event. + """ + await self._connection.send( + cast( + RealtimeClientEventParam, + strip_not_given({"type": "input_audio_buffer.append", "audio": audio, "event_id": event_id}), + ) + ) diff --git a/src/openai/types/__init__.py b/src/openai/types/__init__.py index 7677be01b2..72950f2491 100644 --- a/src/openai/types/__init__.py +++ b/src/openai/types/__init__.py @@ -47,6 +47,7 @@ from .create_embedding_response import CreateEmbeddingResponse as CreateEmbeddingResponse from .moderation_create_response import ModerationCreateResponse as ModerationCreateResponse from .moderation_text_input_param import ModerationTextInputParam as ModerationTextInputParam +from .websocket_connection_options import WebsocketConnectionOptions as WebsocketConnectionOptions from .image_create_variation_params import ImageCreateVariationParams as ImageCreateVariationParams from .moderation_image_url_input_param import ModerationImageURLInputParam as ModerationImageURLInputParam from .moderation_multi_modal_input_param import ModerationMultiModalInputParam as ModerationMultiModalInputParam diff --git a/src/openai/types/beta/realtime/__init__.py b/src/openai/types/beta/realtime/__init__.py index 1c5246db7a..372d4ec19d 100644 --- a/src/openai/types/beta/realtime/__init__.py +++ b/src/openai/types/beta/realtime/__init__.py @@ -2,5 +2,79 @@ from __future__ import annotations +from .session import Session as Session +from .error_event import ErrorEvent as ErrorEvent +from .conversation_item import ConversationItem as ConversationItem +from .realtime_response import RealtimeResponse as RealtimeResponse +from .response_done_event import ResponseDoneEvent as ResponseDoneEvent +from .session_update_event import SessionUpdateEvent as SessionUpdateEvent +from .realtime_client_event import RealtimeClientEvent as RealtimeClientEvent +from .realtime_server_event import RealtimeServerEvent as RealtimeServerEvent +from .response_cancel_event import ResponseCancelEvent as ResponseCancelEvent +from .response_create_event import ResponseCreateEvent as ResponseCreateEvent from .session_create_params import SessionCreateParams as SessionCreateParams +from .session_created_event import SessionCreatedEvent as SessionCreatedEvent +from .session_updated_event import SessionUpdatedEvent as SessionUpdatedEvent +from .response_created_event import ResponseCreatedEvent as ResponseCreatedEvent +from .conversation_item_param import ConversationItemParam as ConversationItemParam +from .realtime_connect_params import RealtimeConnectParams as RealtimeConnectParams +from .realtime_response_usage import RealtimeResponseUsage as RealtimeResponseUsage from .session_create_response import SessionCreateResponse as SessionCreateResponse +from .realtime_response_status import RealtimeResponseStatus as RealtimeResponseStatus +from .response_text_done_event import ResponseTextDoneEvent as ResponseTextDoneEvent +from .conversation_item_content import ConversationItemContent as ConversationItemContent +from .rate_limits_updated_event import RateLimitsUpdatedEvent as RateLimitsUpdatedEvent +from .response_audio_done_event import ResponseAudioDoneEvent as ResponseAudioDoneEvent +from .response_text_delta_event import ResponseTextDeltaEvent as ResponseTextDeltaEvent +from .conversation_created_event import ConversationCreatedEvent as ConversationCreatedEvent +from .response_audio_delta_event import ResponseAudioDeltaEvent as ResponseAudioDeltaEvent +from .session_update_event_param import SessionUpdateEventParam as SessionUpdateEventParam +from .realtime_client_event_param import RealtimeClientEventParam as RealtimeClientEventParam +from .response_cancel_event_param import ResponseCancelEventParam as ResponseCancelEventParam +from .response_create_event_param import ResponseCreateEventParam as ResponseCreateEventParam +from .conversation_item_create_event import ConversationItemCreateEvent as ConversationItemCreateEvent +from .conversation_item_delete_event import ConversationItemDeleteEvent as ConversationItemDeleteEvent +from .input_audio_buffer_clear_event import InputAudioBufferClearEvent as InputAudioBufferClearEvent +from .conversation_item_content_param import ConversationItemContentParam as ConversationItemContentParam +from .conversation_item_created_event import ConversationItemCreatedEvent as ConversationItemCreatedEvent +from .conversation_item_deleted_event import ConversationItemDeletedEvent as ConversationItemDeletedEvent +from .input_audio_buffer_append_event import InputAudioBufferAppendEvent as InputAudioBufferAppendEvent +from .input_audio_buffer_commit_event import InputAudioBufferCommitEvent as InputAudioBufferCommitEvent +from .response_output_item_done_event import ResponseOutputItemDoneEvent as ResponseOutputItemDoneEvent +from .conversation_item_truncate_event import ConversationItemTruncateEvent as ConversationItemTruncateEvent +from .input_audio_buffer_cleared_event import InputAudioBufferClearedEvent as InputAudioBufferClearedEvent +from .response_content_part_done_event import ResponseContentPartDoneEvent as ResponseContentPartDoneEvent +from .response_output_item_added_event import ResponseOutputItemAddedEvent as ResponseOutputItemAddedEvent +from .conversation_item_truncated_event import ConversationItemTruncatedEvent as ConversationItemTruncatedEvent +from .response_content_part_added_event import ResponseContentPartAddedEvent as ResponseContentPartAddedEvent +from .input_audio_buffer_committed_event import InputAudioBufferCommittedEvent as InputAudioBufferCommittedEvent +from .conversation_item_create_event_param import ConversationItemCreateEventParam as ConversationItemCreateEventParam +from .conversation_item_delete_event_param import ConversationItemDeleteEventParam as ConversationItemDeleteEventParam +from .input_audio_buffer_clear_event_param import InputAudioBufferClearEventParam as InputAudioBufferClearEventParam +from .response_audio_transcript_done_event import ResponseAudioTranscriptDoneEvent as ResponseAudioTranscriptDoneEvent +from .input_audio_buffer_append_event_param import InputAudioBufferAppendEventParam as InputAudioBufferAppendEventParam +from .input_audio_buffer_commit_event_param import InputAudioBufferCommitEventParam as InputAudioBufferCommitEventParam +from .response_audio_transcript_delta_event import ( + ResponseAudioTranscriptDeltaEvent as ResponseAudioTranscriptDeltaEvent, +) +from .conversation_item_truncate_event_param import ( + ConversationItemTruncateEventParam as ConversationItemTruncateEventParam, +) +from .input_audio_buffer_speech_started_event import ( + InputAudioBufferSpeechStartedEvent as InputAudioBufferSpeechStartedEvent, +) +from .input_audio_buffer_speech_stopped_event import ( + InputAudioBufferSpeechStoppedEvent as InputAudioBufferSpeechStoppedEvent, +) +from .response_function_call_arguments_done_event import ( + ResponseFunctionCallArgumentsDoneEvent as ResponseFunctionCallArgumentsDoneEvent, +) +from .response_function_call_arguments_delta_event import ( + ResponseFunctionCallArgumentsDeltaEvent as ResponseFunctionCallArgumentsDeltaEvent, +) +from .conversation_item_input_audio_transcription_failed_event import ( + ConversationItemInputAudioTranscriptionFailedEvent as ConversationItemInputAudioTranscriptionFailedEvent, +) +from .conversation_item_input_audio_transcription_completed_event import ( + ConversationItemInputAudioTranscriptionCompletedEvent as ConversationItemInputAudioTranscriptionCompletedEvent, +) diff --git a/src/openai/types/beta/realtime/conversation_created_event.py b/src/openai/types/beta/realtime/conversation_created_event.py new file mode 100644 index 0000000000..4ba0540867 --- /dev/null +++ b/src/openai/types/beta/realtime/conversation_created_event.py @@ -0,0 +1,27 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Optional +from typing_extensions import Literal + +from ...._models import BaseModel + +__all__ = ["ConversationCreatedEvent", "Conversation"] + + +class Conversation(BaseModel): + id: Optional[str] = None + """The unique ID of the conversation.""" + + object: Optional[Literal["realtime.conversation"]] = None + """The object type, must be `realtime.conversation`.""" + + +class ConversationCreatedEvent(BaseModel): + conversation: Conversation + """The conversation resource.""" + + event_id: str + """The unique ID of the server event.""" + + type: Literal["conversation.created"] + """The event type, must be `conversation.created`.""" diff --git a/src/openai/types/beta/realtime/conversation_item.py b/src/openai/types/beta/realtime/conversation_item.py new file mode 100644 index 0000000000..4edf6c4d5f --- /dev/null +++ b/src/openai/types/beta/realtime/conversation_item.py @@ -0,0 +1,61 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import List, Optional +from typing_extensions import Literal + +from ...._models import BaseModel +from .conversation_item_content import ConversationItemContent + +__all__ = ["ConversationItem"] + + +class ConversationItem(BaseModel): + id: Optional[str] = None + """ + The unique ID of the item, this can be generated by the client to help manage + server-side context, but is not required because the server will generate one if + not provided. + """ + + arguments: Optional[str] = None + """The arguments of the function call (for `function_call` items).""" + + call_id: Optional[str] = None + """ + The ID of the function call (for `function_call` and `function_call_output` + items). If passed on a `function_call_output` item, the server will check that a + `function_call` item with the same ID exists in the conversation history. + """ + + content: Optional[List[ConversationItemContent]] = None + """The content of the message, applicable for `message` items. + + - Message items of role `system` support only `input_text` content + - Message items of role `user` support `input_text` and `input_audio` content + - Message items of role `assistant` support `text` content. + """ + + name: Optional[str] = None + """The name of the function being called (for `function_call` items).""" + + object: Optional[Literal["realtime.item"]] = None + """Identifier for the API object being returned - always `realtime.item`.""" + + output: Optional[str] = None + """The output of the function call (for `function_call_output` items).""" + + role: Optional[Literal["user", "assistant", "system"]] = None + """ + The role of the message sender (`user`, `assistant`, `system`), only applicable + for `message` items. + """ + + status: Optional[Literal["completed", "incomplete"]] = None + """The status of the item (`completed`, `incomplete`). + + These have no effect on the conversation, but are accepted for consistency with + the `conversation.item.created` event. + """ + + type: Optional[Literal["message", "function_call", "function_call_output"]] = None + """The type of the item (`message`, `function_call`, `function_call_output`).""" diff --git a/src/openai/types/beta/realtime/conversation_item_content.py b/src/openai/types/beta/realtime/conversation_item_content.py new file mode 100644 index 0000000000..b854aa0e0f --- /dev/null +++ b/src/openai/types/beta/realtime/conversation_item_content.py @@ -0,0 +1,28 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Optional +from typing_extensions import Literal + +from ...._models import BaseModel + +__all__ = ["ConversationItemContent"] + + +class ConversationItemContent(BaseModel): + id: Optional[str] = None + """ + ID of a previous conversation item (like a model response), used for + `item_reference` content types. + """ + + audio: Optional[str] = None + """Base64-encoded audio bytes, used for `input_audio` content type.""" + + text: Optional[str] = None + """The text content, used for `input_text` and `text` content types.""" + + transcript: Optional[str] = None + """The transcript of the audio, used for `input_audio` content type.""" + + type: Optional[Literal["input_text", "input_audio", "item_reference", "text"]] = None + """The content type (`input_text`, `input_audio`, `item_reference`, `text`).""" diff --git a/src/openai/types/beta/realtime/conversation_item_content_param.py b/src/openai/types/beta/realtime/conversation_item_content_param.py new file mode 100644 index 0000000000..b354d78971 --- /dev/null +++ b/src/openai/types/beta/realtime/conversation_item_content_param.py @@ -0,0 +1,27 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing_extensions import Literal, TypedDict + +__all__ = ["ConversationItemContentParam"] + + +class ConversationItemContentParam(TypedDict, total=False): + id: str + """ + ID of a previous conversation item (like a model response), used for + `item_reference` content types. + """ + + audio: str + """Base64-encoded audio bytes, used for `input_audio` content type.""" + + text: str + """The text content, used for `input_text` and `text` content types.""" + + transcript: str + """The transcript of the audio, used for `input_audio` content type.""" + + type: Literal["input_text", "input_audio", "item_reference", "text"] + """The content type (`input_text`, `input_audio`, `item_reference`, `text`).""" diff --git a/src/openai/types/beta/realtime/conversation_item_create_event.py b/src/openai/types/beta/realtime/conversation_item_create_event.py new file mode 100644 index 0000000000..50d309675b --- /dev/null +++ b/src/openai/types/beta/realtime/conversation_item_create_event.py @@ -0,0 +1,28 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Optional +from typing_extensions import Literal + +from ...._models import BaseModel +from .conversation_item import ConversationItem + +__all__ = ["ConversationItemCreateEvent"] + + +class ConversationItemCreateEvent(BaseModel): + item: ConversationItem + """The item to add to the conversation.""" + + type: Literal["conversation.item.create"] + """The event type, must be `conversation.item.create`.""" + + event_id: Optional[str] = None + """Optional client-generated ID used to identify this event.""" + + previous_item_id: Optional[str] = None + """The ID of the preceding item after which the new item will be inserted. + + If not set, the new item will be appended to the end of the conversation. If + set, it allows an item to be inserted mid-conversation. If the ID cannot be + found, an error will be returned and the item will not be added. + """ diff --git a/src/openai/types/beta/realtime/conversation_item_create_event_param.py b/src/openai/types/beta/realtime/conversation_item_create_event_param.py new file mode 100644 index 0000000000..b8c8bbc251 --- /dev/null +++ b/src/openai/types/beta/realtime/conversation_item_create_event_param.py @@ -0,0 +1,28 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing_extensions import Literal, Required, TypedDict + +from .conversation_item_param import ConversationItemParam + +__all__ = ["ConversationItemCreateEventParam"] + + +class ConversationItemCreateEventParam(TypedDict, total=False): + item: Required[ConversationItemParam] + """The item to add to the conversation.""" + + type: Required[Literal["conversation.item.create"]] + """The event type, must be `conversation.item.create`.""" + + event_id: str + """Optional client-generated ID used to identify this event.""" + + previous_item_id: str + """The ID of the preceding item after which the new item will be inserted. + + If not set, the new item will be appended to the end of the conversation. If + set, it allows an item to be inserted mid-conversation. If the ID cannot be + found, an error will be returned and the item will not be added. + """ diff --git a/src/openai/types/beta/realtime/conversation_item_created_event.py b/src/openai/types/beta/realtime/conversation_item_created_event.py new file mode 100644 index 0000000000..2f20388246 --- /dev/null +++ b/src/openai/types/beta/realtime/conversation_item_created_event.py @@ -0,0 +1,25 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing_extensions import Literal + +from ...._models import BaseModel +from .conversation_item import ConversationItem + +__all__ = ["ConversationItemCreatedEvent"] + + +class ConversationItemCreatedEvent(BaseModel): + event_id: str + """The unique ID of the server event.""" + + item: ConversationItem + """The item to add to the conversation.""" + + previous_item_id: str + """ + The ID of the preceding item in the Conversation context, allows the client to + understand the order of the conversation. + """ + + type: Literal["conversation.item.created"] + """The event type, must be `conversation.item.created`.""" diff --git a/src/openai/types/beta/realtime/conversation_item_delete_event.py b/src/openai/types/beta/realtime/conversation_item_delete_event.py new file mode 100644 index 0000000000..02ca8250ce --- /dev/null +++ b/src/openai/types/beta/realtime/conversation_item_delete_event.py @@ -0,0 +1,19 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Optional +from typing_extensions import Literal + +from ...._models import BaseModel + +__all__ = ["ConversationItemDeleteEvent"] + + +class ConversationItemDeleteEvent(BaseModel): + item_id: str + """The ID of the item to delete.""" + + type: Literal["conversation.item.delete"] + """The event type, must be `conversation.item.delete`.""" + + event_id: Optional[str] = None + """Optional client-generated ID used to identify this event.""" diff --git a/src/openai/types/beta/realtime/conversation_item_delete_event_param.py b/src/openai/types/beta/realtime/conversation_item_delete_event_param.py new file mode 100644 index 0000000000..c3f88d6627 --- /dev/null +++ b/src/openai/types/beta/realtime/conversation_item_delete_event_param.py @@ -0,0 +1,18 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing_extensions import Literal, Required, TypedDict + +__all__ = ["ConversationItemDeleteEventParam"] + + +class ConversationItemDeleteEventParam(TypedDict, total=False): + item_id: Required[str] + """The ID of the item to delete.""" + + type: Required[Literal["conversation.item.delete"]] + """The event type, must be `conversation.item.delete`.""" + + event_id: str + """Optional client-generated ID used to identify this event.""" diff --git a/src/openai/types/beta/realtime/conversation_item_deleted_event.py b/src/openai/types/beta/realtime/conversation_item_deleted_event.py new file mode 100644 index 0000000000..a35a97817a --- /dev/null +++ b/src/openai/types/beta/realtime/conversation_item_deleted_event.py @@ -0,0 +1,18 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing_extensions import Literal + +from ...._models import BaseModel + +__all__ = ["ConversationItemDeletedEvent"] + + +class ConversationItemDeletedEvent(BaseModel): + event_id: str + """The unique ID of the server event.""" + + item_id: str + """The ID of the item that was deleted.""" + + type: Literal["conversation.item.deleted"] + """The event type, must be `conversation.item.deleted`.""" diff --git a/src/openai/types/beta/realtime/conversation_item_input_audio_transcription_completed_event.py b/src/openai/types/beta/realtime/conversation_item_input_audio_transcription_completed_event.py new file mode 100644 index 0000000000..ded79cc0f7 --- /dev/null +++ b/src/openai/types/beta/realtime/conversation_item_input_audio_transcription_completed_event.py @@ -0,0 +1,26 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing_extensions import Literal + +from ...._models import BaseModel + +__all__ = ["ConversationItemInputAudioTranscriptionCompletedEvent"] + + +class ConversationItemInputAudioTranscriptionCompletedEvent(BaseModel): + content_index: int + """The index of the content part containing the audio.""" + + event_id: str + """The unique ID of the server event.""" + + item_id: str + """The ID of the user message item containing the audio.""" + + transcript: str + """The transcribed text.""" + + type: Literal["conversation.item.input_audio_transcription.completed"] + """ + The event type, must be `conversation.item.input_audio_transcription.completed`. + """ diff --git a/src/openai/types/beta/realtime/conversation_item_input_audio_transcription_failed_event.py b/src/openai/types/beta/realtime/conversation_item_input_audio_transcription_failed_event.py new file mode 100644 index 0000000000..cecac93e64 --- /dev/null +++ b/src/openai/types/beta/realtime/conversation_item_input_audio_transcription_failed_event.py @@ -0,0 +1,39 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Optional +from typing_extensions import Literal + +from ...._models import BaseModel + +__all__ = ["ConversationItemInputAudioTranscriptionFailedEvent", "Error"] + + +class Error(BaseModel): + code: Optional[str] = None + """Error code, if any.""" + + message: Optional[str] = None + """A human-readable error message.""" + + param: Optional[str] = None + """Parameter related to the error, if any.""" + + type: Optional[str] = None + """The type of error.""" + + +class ConversationItemInputAudioTranscriptionFailedEvent(BaseModel): + content_index: int + """The index of the content part containing the audio.""" + + error: Error + """Details of the transcription error.""" + + event_id: str + """The unique ID of the server event.""" + + item_id: str + """The ID of the user message item.""" + + type: Literal["conversation.item.input_audio_transcription.failed"] + """The event type, must be `conversation.item.input_audio_transcription.failed`.""" diff --git a/src/openai/types/beta/realtime/conversation_item_param.py b/src/openai/types/beta/realtime/conversation_item_param.py new file mode 100644 index 0000000000..ac0f8431e5 --- /dev/null +++ b/src/openai/types/beta/realtime/conversation_item_param.py @@ -0,0 +1,62 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import Iterable +from typing_extensions import Literal, TypedDict + +from .conversation_item_content_param import ConversationItemContentParam + +__all__ = ["ConversationItemParam"] + + +class ConversationItemParam(TypedDict, total=False): + id: str + """ + The unique ID of the item, this can be generated by the client to help manage + server-side context, but is not required because the server will generate one if + not provided. + """ + + arguments: str + """The arguments of the function call (for `function_call` items).""" + + call_id: str + """ + The ID of the function call (for `function_call` and `function_call_output` + items). If passed on a `function_call_output` item, the server will check that a + `function_call` item with the same ID exists in the conversation history. + """ + + content: Iterable[ConversationItemContentParam] + """The content of the message, applicable for `message` items. + + - Message items of role `system` support only `input_text` content + - Message items of role `user` support `input_text` and `input_audio` content + - Message items of role `assistant` support `text` content. + """ + + name: str + """The name of the function being called (for `function_call` items).""" + + object: Literal["realtime.item"] + """Identifier for the API object being returned - always `realtime.item`.""" + + output: str + """The output of the function call (for `function_call_output` items).""" + + role: Literal["user", "assistant", "system"] + """ + The role of the message sender (`user`, `assistant`, `system`), only applicable + for `message` items. + """ + + status: Literal["completed", "incomplete"] + """The status of the item (`completed`, `incomplete`). + + These have no effect on the conversation, but are accepted for consistency with + the `conversation.item.created` event. + """ + + type: Literal["message", "function_call", "function_call_output"] + """The type of the item (`message`, `function_call`, `function_call_output`).""" diff --git a/src/openai/types/beta/realtime/conversation_item_truncate_event.py b/src/openai/types/beta/realtime/conversation_item_truncate_event.py new file mode 100644 index 0000000000..cb336bba2c --- /dev/null +++ b/src/openai/types/beta/realtime/conversation_item_truncate_event.py @@ -0,0 +1,32 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Optional +from typing_extensions import Literal + +from ...._models import BaseModel + +__all__ = ["ConversationItemTruncateEvent"] + + +class ConversationItemTruncateEvent(BaseModel): + audio_end_ms: int + """Inclusive duration up to which audio is truncated, in milliseconds. + + If the audio_end_ms is greater than the actual audio duration, the server will + respond with an error. + """ + + content_index: int + """The index of the content part to truncate. Set this to 0.""" + + item_id: str + """The ID of the assistant message item to truncate. + + Only assistant message items can be truncated. + """ + + type: Literal["conversation.item.truncate"] + """The event type, must be `conversation.item.truncate`.""" + + event_id: Optional[str] = None + """Optional client-generated ID used to identify this event.""" diff --git a/src/openai/types/beta/realtime/conversation_item_truncate_event_param.py b/src/openai/types/beta/realtime/conversation_item_truncate_event_param.py new file mode 100644 index 0000000000..d3ad1e1e25 --- /dev/null +++ b/src/openai/types/beta/realtime/conversation_item_truncate_event_param.py @@ -0,0 +1,31 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing_extensions import Literal, Required, TypedDict + +__all__ = ["ConversationItemTruncateEventParam"] + + +class ConversationItemTruncateEventParam(TypedDict, total=False): + audio_end_ms: Required[int] + """Inclusive duration up to which audio is truncated, in milliseconds. + + If the audio_end_ms is greater than the actual audio duration, the server will + respond with an error. + """ + + content_index: Required[int] + """The index of the content part to truncate. Set this to 0.""" + + item_id: Required[str] + """The ID of the assistant message item to truncate. + + Only assistant message items can be truncated. + """ + + type: Required[Literal["conversation.item.truncate"]] + """The event type, must be `conversation.item.truncate`.""" + + event_id: str + """Optional client-generated ID used to identify this event.""" diff --git a/src/openai/types/beta/realtime/conversation_item_truncated_event.py b/src/openai/types/beta/realtime/conversation_item_truncated_event.py new file mode 100644 index 0000000000..36368fa28f --- /dev/null +++ b/src/openai/types/beta/realtime/conversation_item_truncated_event.py @@ -0,0 +1,24 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing_extensions import Literal + +from ...._models import BaseModel + +__all__ = ["ConversationItemTruncatedEvent"] + + +class ConversationItemTruncatedEvent(BaseModel): + audio_end_ms: int + """The duration up to which the audio was truncated, in milliseconds.""" + + content_index: int + """The index of the content part that was truncated.""" + + event_id: str + """The unique ID of the server event.""" + + item_id: str + """The ID of the assistant message item that was truncated.""" + + type: Literal["conversation.item.truncated"] + """The event type, must be `conversation.item.truncated`.""" diff --git a/src/openai/types/beta/realtime/error_event.py b/src/openai/types/beta/realtime/error_event.py new file mode 100644 index 0000000000..e020fc3848 --- /dev/null +++ b/src/openai/types/beta/realtime/error_event.py @@ -0,0 +1,36 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Optional +from typing_extensions import Literal + +from ...._models import BaseModel + +__all__ = ["ErrorEvent", "Error"] + + +class Error(BaseModel): + message: str + """A human-readable error message.""" + + type: str + """The type of error (e.g., "invalid_request_error", "server_error").""" + + code: Optional[str] = None + """Error code, if any.""" + + event_id: Optional[str] = None + """The event_id of the client event that caused the error, if applicable.""" + + param: Optional[str] = None + """Parameter related to the error, if any.""" + + +class ErrorEvent(BaseModel): + error: Error + """Details of the error.""" + + event_id: str + """The unique ID of the server event.""" + + type: Literal["error"] + """The event type, must be `error`.""" diff --git a/src/openai/types/beta/realtime/input_audio_buffer_append_event.py b/src/openai/types/beta/realtime/input_audio_buffer_append_event.py new file mode 100644 index 0000000000..a253a6488c --- /dev/null +++ b/src/openai/types/beta/realtime/input_audio_buffer_append_event.py @@ -0,0 +1,23 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Optional +from typing_extensions import Literal + +from ...._models import BaseModel + +__all__ = ["InputAudioBufferAppendEvent"] + + +class InputAudioBufferAppendEvent(BaseModel): + audio: str + """Base64-encoded audio bytes. + + This must be in the format specified by the `input_audio_format` field in the + session configuration. + """ + + type: Literal["input_audio_buffer.append"] + """The event type, must be `input_audio_buffer.append`.""" + + event_id: Optional[str] = None + """Optional client-generated ID used to identify this event.""" diff --git a/src/openai/types/beta/realtime/input_audio_buffer_append_event_param.py b/src/openai/types/beta/realtime/input_audio_buffer_append_event_param.py new file mode 100644 index 0000000000..3ad0bc737d --- /dev/null +++ b/src/openai/types/beta/realtime/input_audio_buffer_append_event_param.py @@ -0,0 +1,22 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing_extensions import Literal, Required, TypedDict + +__all__ = ["InputAudioBufferAppendEventParam"] + + +class InputAudioBufferAppendEventParam(TypedDict, total=False): + audio: Required[str] + """Base64-encoded audio bytes. + + This must be in the format specified by the `input_audio_format` field in the + session configuration. + """ + + type: Required[Literal["input_audio_buffer.append"]] + """The event type, must be `input_audio_buffer.append`.""" + + event_id: str + """Optional client-generated ID used to identify this event.""" diff --git a/src/openai/types/beta/realtime/input_audio_buffer_clear_event.py b/src/openai/types/beta/realtime/input_audio_buffer_clear_event.py new file mode 100644 index 0000000000..b0624d34df --- /dev/null +++ b/src/openai/types/beta/realtime/input_audio_buffer_clear_event.py @@ -0,0 +1,16 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Optional +from typing_extensions import Literal + +from ...._models import BaseModel + +__all__ = ["InputAudioBufferClearEvent"] + + +class InputAudioBufferClearEvent(BaseModel): + type: Literal["input_audio_buffer.clear"] + """The event type, must be `input_audio_buffer.clear`.""" + + event_id: Optional[str] = None + """Optional client-generated ID used to identify this event.""" diff --git a/src/openai/types/beta/realtime/input_audio_buffer_clear_event_param.py b/src/openai/types/beta/realtime/input_audio_buffer_clear_event_param.py new file mode 100644 index 0000000000..2bd6bc5a02 --- /dev/null +++ b/src/openai/types/beta/realtime/input_audio_buffer_clear_event_param.py @@ -0,0 +1,15 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing_extensions import Literal, Required, TypedDict + +__all__ = ["InputAudioBufferClearEventParam"] + + +class InputAudioBufferClearEventParam(TypedDict, total=False): + type: Required[Literal["input_audio_buffer.clear"]] + """The event type, must be `input_audio_buffer.clear`.""" + + event_id: str + """Optional client-generated ID used to identify this event.""" diff --git a/src/openai/types/beta/realtime/input_audio_buffer_cleared_event.py b/src/openai/types/beta/realtime/input_audio_buffer_cleared_event.py new file mode 100644 index 0000000000..632e1b94bc --- /dev/null +++ b/src/openai/types/beta/realtime/input_audio_buffer_cleared_event.py @@ -0,0 +1,15 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing_extensions import Literal + +from ...._models import BaseModel + +__all__ = ["InputAudioBufferClearedEvent"] + + +class InputAudioBufferClearedEvent(BaseModel): + event_id: str + """The unique ID of the server event.""" + + type: Literal["input_audio_buffer.cleared"] + """The event type, must be `input_audio_buffer.cleared`.""" diff --git a/src/openai/types/beta/realtime/input_audio_buffer_commit_event.py b/src/openai/types/beta/realtime/input_audio_buffer_commit_event.py new file mode 100644 index 0000000000..7b6f5e46b7 --- /dev/null +++ b/src/openai/types/beta/realtime/input_audio_buffer_commit_event.py @@ -0,0 +1,16 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Optional +from typing_extensions import Literal + +from ...._models import BaseModel + +__all__ = ["InputAudioBufferCommitEvent"] + + +class InputAudioBufferCommitEvent(BaseModel): + type: Literal["input_audio_buffer.commit"] + """The event type, must be `input_audio_buffer.commit`.""" + + event_id: Optional[str] = None + """Optional client-generated ID used to identify this event.""" diff --git a/src/openai/types/beta/realtime/input_audio_buffer_commit_event_param.py b/src/openai/types/beta/realtime/input_audio_buffer_commit_event_param.py new file mode 100644 index 0000000000..c9c927ab98 --- /dev/null +++ b/src/openai/types/beta/realtime/input_audio_buffer_commit_event_param.py @@ -0,0 +1,15 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing_extensions import Literal, Required, TypedDict + +__all__ = ["InputAudioBufferCommitEventParam"] + + +class InputAudioBufferCommitEventParam(TypedDict, total=False): + type: Required[Literal["input_audio_buffer.commit"]] + """The event type, must be `input_audio_buffer.commit`.""" + + event_id: str + """Optional client-generated ID used to identify this event.""" diff --git a/src/openai/types/beta/realtime/input_audio_buffer_committed_event.py b/src/openai/types/beta/realtime/input_audio_buffer_committed_event.py new file mode 100644 index 0000000000..3071eff357 --- /dev/null +++ b/src/openai/types/beta/realtime/input_audio_buffer_committed_event.py @@ -0,0 +1,21 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing_extensions import Literal + +from ...._models import BaseModel + +__all__ = ["InputAudioBufferCommittedEvent"] + + +class InputAudioBufferCommittedEvent(BaseModel): + event_id: str + """The unique ID of the server event.""" + + item_id: str + """The ID of the user message item that will be created.""" + + previous_item_id: str + """The ID of the preceding item after which the new item will be inserted.""" + + type: Literal["input_audio_buffer.committed"] + """The event type, must be `input_audio_buffer.committed`.""" diff --git a/src/openai/types/beta/realtime/input_audio_buffer_speech_started_event.py b/src/openai/types/beta/realtime/input_audio_buffer_speech_started_event.py new file mode 100644 index 0000000000..4f3ab082c4 --- /dev/null +++ b/src/openai/types/beta/realtime/input_audio_buffer_speech_started_event.py @@ -0,0 +1,26 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing_extensions import Literal + +from ...._models import BaseModel + +__all__ = ["InputAudioBufferSpeechStartedEvent"] + + +class InputAudioBufferSpeechStartedEvent(BaseModel): + audio_start_ms: int + """ + Milliseconds from the start of all audio written to the buffer during the + session when speech was first detected. This will correspond to the beginning of + audio sent to the model, and thus includes the `prefix_padding_ms` configured in + the Session. + """ + + event_id: str + """The unique ID of the server event.""" + + item_id: str + """The ID of the user message item that will be created when speech stops.""" + + type: Literal["input_audio_buffer.speech_started"] + """The event type, must be `input_audio_buffer.speech_started`.""" diff --git a/src/openai/types/beta/realtime/input_audio_buffer_speech_stopped_event.py b/src/openai/types/beta/realtime/input_audio_buffer_speech_stopped_event.py new file mode 100644 index 0000000000..40568170f2 --- /dev/null +++ b/src/openai/types/beta/realtime/input_audio_buffer_speech_stopped_event.py @@ -0,0 +1,25 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing_extensions import Literal + +from ...._models import BaseModel + +__all__ = ["InputAudioBufferSpeechStoppedEvent"] + + +class InputAudioBufferSpeechStoppedEvent(BaseModel): + audio_end_ms: int + """Milliseconds since the session started when speech stopped. + + This will correspond to the end of audio sent to the model, and thus includes + the `min_silence_duration_ms` configured in the Session. + """ + + event_id: str + """The unique ID of the server event.""" + + item_id: str + """The ID of the user message item that will be created.""" + + type: Literal["input_audio_buffer.speech_stopped"] + """The event type, must be `input_audio_buffer.speech_stopped`.""" diff --git a/src/openai/types/beta/realtime/rate_limits_updated_event.py b/src/openai/types/beta/realtime/rate_limits_updated_event.py new file mode 100644 index 0000000000..7e12283c46 --- /dev/null +++ b/src/openai/types/beta/realtime/rate_limits_updated_event.py @@ -0,0 +1,33 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import List, Optional +from typing_extensions import Literal + +from ...._models import BaseModel + +__all__ = ["RateLimitsUpdatedEvent", "RateLimit"] + + +class RateLimit(BaseModel): + limit: Optional[int] = None + """The maximum allowed value for the rate limit.""" + + name: Optional[Literal["requests", "tokens"]] = None + """The name of the rate limit (`requests`, `tokens`).""" + + remaining: Optional[int] = None + """The remaining value before the limit is reached.""" + + reset_seconds: Optional[float] = None + """Seconds until the rate limit resets.""" + + +class RateLimitsUpdatedEvent(BaseModel): + event_id: str + """The unique ID of the server event.""" + + rate_limits: List[RateLimit] + """List of rate limit information.""" + + type: Literal["rate_limits.updated"] + """The event type, must be `rate_limits.updated`.""" diff --git a/src/openai/types/beta/realtime/realtime_client_event.py b/src/openai/types/beta/realtime/realtime_client_event.py new file mode 100644 index 0000000000..0769184cd0 --- /dev/null +++ b/src/openai/types/beta/realtime/realtime_client_event.py @@ -0,0 +1,32 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Union +from typing_extensions import Annotated, TypeAlias + +from ...._utils import PropertyInfo +from .session_update_event import SessionUpdateEvent +from .response_cancel_event import ResponseCancelEvent +from .response_create_event import ResponseCreateEvent +from .conversation_item_create_event import ConversationItemCreateEvent +from .conversation_item_delete_event import ConversationItemDeleteEvent +from .input_audio_buffer_clear_event import InputAudioBufferClearEvent +from .input_audio_buffer_append_event import InputAudioBufferAppendEvent +from .input_audio_buffer_commit_event import InputAudioBufferCommitEvent +from .conversation_item_truncate_event import ConversationItemTruncateEvent + +__all__ = ["RealtimeClientEvent"] + +RealtimeClientEvent: TypeAlias = Annotated[ + Union[ + SessionUpdateEvent, + InputAudioBufferAppendEvent, + InputAudioBufferCommitEvent, + InputAudioBufferClearEvent, + ConversationItemCreateEvent, + ConversationItemTruncateEvent, + ConversationItemDeleteEvent, + ResponseCreateEvent, + ResponseCancelEvent, + ], + PropertyInfo(discriminator="type"), +] diff --git a/src/openai/types/beta/realtime/realtime_client_event_param.py b/src/openai/types/beta/realtime/realtime_client_event_param.py new file mode 100644 index 0000000000..4020892c33 --- /dev/null +++ b/src/openai/types/beta/realtime/realtime_client_event_param.py @@ -0,0 +1,30 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import Union +from typing_extensions import TypeAlias + +from .session_update_event_param import SessionUpdateEventParam +from .response_cancel_event_param import ResponseCancelEventParam +from .response_create_event_param import ResponseCreateEventParam +from .conversation_item_create_event_param import ConversationItemCreateEventParam +from .conversation_item_delete_event_param import ConversationItemDeleteEventParam +from .input_audio_buffer_clear_event_param import InputAudioBufferClearEventParam +from .input_audio_buffer_append_event_param import InputAudioBufferAppendEventParam +from .input_audio_buffer_commit_event_param import InputAudioBufferCommitEventParam +from .conversation_item_truncate_event_param import ConversationItemTruncateEventParam + +__all__ = ["RealtimeClientEventParam"] + +RealtimeClientEventParam: TypeAlias = Union[ + SessionUpdateEventParam, + InputAudioBufferAppendEventParam, + InputAudioBufferCommitEventParam, + InputAudioBufferClearEventParam, + ConversationItemCreateEventParam, + ConversationItemTruncateEventParam, + ConversationItemDeleteEventParam, + ResponseCreateEventParam, + ResponseCancelEventParam, +] diff --git a/src/openai/types/beta/realtime/realtime_connect_params.py b/src/openai/types/beta/realtime/realtime_connect_params.py new file mode 100644 index 0000000000..76474f3de4 --- /dev/null +++ b/src/openai/types/beta/realtime/realtime_connect_params.py @@ -0,0 +1,11 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing_extensions import Required, TypedDict + +__all__ = ["RealtimeConnectParams"] + + +class RealtimeConnectParams(TypedDict, total=False): + model: Required[str] diff --git a/src/openai/types/beta/realtime/realtime_response.py b/src/openai/types/beta/realtime/realtime_response.py new file mode 100644 index 0000000000..3e1b1406c0 --- /dev/null +++ b/src/openai/types/beta/realtime/realtime_response.py @@ -0,0 +1,42 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import List, Optional +from typing_extensions import Literal + +from ...._models import BaseModel +from .conversation_item import ConversationItem +from .realtime_response_usage import RealtimeResponseUsage +from .realtime_response_status import RealtimeResponseStatus + +__all__ = ["RealtimeResponse"] + + +class RealtimeResponse(BaseModel): + id: Optional[str] = None + """The unique ID of the response.""" + + metadata: Optional[object] = None + """Developer-provided string key-value pairs associated with this response.""" + + object: Optional[Literal["realtime.response"]] = None + """The object type, must be `realtime.response`.""" + + output: Optional[List[ConversationItem]] = None + """The list of output items generated by the response.""" + + status: Optional[Literal["completed", "cancelled", "failed", "incomplete"]] = None + """ + The final status of the response (`completed`, `cancelled`, `failed`, or + `incomplete`). + """ + + status_details: Optional[RealtimeResponseStatus] = None + """Additional details about the status.""" + + usage: Optional[RealtimeResponseUsage] = None + """Usage statistics for the Response, this will correspond to billing. + + A Realtime API session will maintain a conversation context and append new Items + to the Conversation, thus output from previous turns (text and audio tokens) + will become the input for later turns. + """ diff --git a/src/openai/types/beta/realtime/realtime_response_status.py b/src/openai/types/beta/realtime/realtime_response_status.py new file mode 100644 index 0000000000..7189cd58a1 --- /dev/null +++ b/src/openai/types/beta/realtime/realtime_response_status.py @@ -0,0 +1,39 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Optional +from typing_extensions import Literal + +from ...._models import BaseModel + +__all__ = ["RealtimeResponseStatus", "Error"] + + +class Error(BaseModel): + code: Optional[str] = None + """Error code, if any.""" + + type: Optional[str] = None + """The type of error.""" + + +class RealtimeResponseStatus(BaseModel): + error: Optional[Error] = None + """ + A description of the error that caused the response to fail, populated when the + `status` is `failed`. + """ + + reason: Optional[Literal["turn_detected", "client_cancelled", "max_output_tokens", "content_filter"]] = None + """The reason the Response did not complete. + + For a `cancelled` Response, one of `turn_detected` (the server VAD detected a + new start of speech) or `client_cancelled` (the client sent a cancel event). For + an `incomplete` Response, one of `max_output_tokens` or `content_filter` (the + server-side safety filter activated and cut off the response). + """ + + type: Optional[Literal["completed", "cancelled", "incomplete", "failed"]] = None + """ + The type of error that caused the response to fail, corresponding with the + `status` field (`completed`, `cancelled`, `incomplete`, `failed`). + """ diff --git a/src/openai/types/beta/realtime/realtime_response_usage.py b/src/openai/types/beta/realtime/realtime_response_usage.py new file mode 100644 index 0000000000..7ca822e25e --- /dev/null +++ b/src/openai/types/beta/realtime/realtime_response_usage.py @@ -0,0 +1,52 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Optional + +from ...._models import BaseModel + +__all__ = ["RealtimeResponseUsage", "InputTokenDetails", "OutputTokenDetails"] + + +class InputTokenDetails(BaseModel): + audio_tokens: Optional[int] = None + """The number of audio tokens used in the Response.""" + + cached_tokens: Optional[int] = None + """The number of cached tokens used in the Response.""" + + text_tokens: Optional[int] = None + """The number of text tokens used in the Response.""" + + +class OutputTokenDetails(BaseModel): + audio_tokens: Optional[int] = None + """The number of audio tokens used in the Response.""" + + text_tokens: Optional[int] = None + """The number of text tokens used in the Response.""" + + +class RealtimeResponseUsage(BaseModel): + input_token_details: Optional[InputTokenDetails] = None + """Details about the input tokens used in the Response.""" + + input_tokens: Optional[int] = None + """ + The number of input tokens used in the Response, including text and audio + tokens. + """ + + output_token_details: Optional[OutputTokenDetails] = None + """Details about the output tokens used in the Response.""" + + output_tokens: Optional[int] = None + """ + The number of output tokens sent in the Response, including text and audio + tokens. + """ + + total_tokens: Optional[int] = None + """ + The total number of tokens in the Response including input and output text and + audio tokens. + """ diff --git a/src/openai/types/beta/realtime/realtime_server_event.py b/src/openai/types/beta/realtime/realtime_server_event.py new file mode 100644 index 0000000000..5f8ed55b13 --- /dev/null +++ b/src/openai/types/beta/realtime/realtime_server_event.py @@ -0,0 +1,72 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Union +from typing_extensions import Annotated, TypeAlias + +from ...._utils import PropertyInfo +from .error_event import ErrorEvent +from .response_done_event import ResponseDoneEvent +from .session_created_event import SessionCreatedEvent +from .session_updated_event import SessionUpdatedEvent +from .response_created_event import ResponseCreatedEvent +from .response_text_done_event import ResponseTextDoneEvent +from .rate_limits_updated_event import RateLimitsUpdatedEvent +from .response_audio_done_event import ResponseAudioDoneEvent +from .response_text_delta_event import ResponseTextDeltaEvent +from .conversation_created_event import ConversationCreatedEvent +from .response_audio_delta_event import ResponseAudioDeltaEvent +from .conversation_item_created_event import ConversationItemCreatedEvent +from .conversation_item_deleted_event import ConversationItemDeletedEvent +from .response_output_item_done_event import ResponseOutputItemDoneEvent +from .input_audio_buffer_cleared_event import InputAudioBufferClearedEvent +from .response_content_part_done_event import ResponseContentPartDoneEvent +from .response_output_item_added_event import ResponseOutputItemAddedEvent +from .conversation_item_truncated_event import ConversationItemTruncatedEvent +from .response_content_part_added_event import ResponseContentPartAddedEvent +from .input_audio_buffer_committed_event import InputAudioBufferCommittedEvent +from .response_audio_transcript_done_event import ResponseAudioTranscriptDoneEvent +from .response_audio_transcript_delta_event import ResponseAudioTranscriptDeltaEvent +from .input_audio_buffer_speech_started_event import InputAudioBufferSpeechStartedEvent +from .input_audio_buffer_speech_stopped_event import InputAudioBufferSpeechStoppedEvent +from .response_function_call_arguments_done_event import ResponseFunctionCallArgumentsDoneEvent +from .response_function_call_arguments_delta_event import ResponseFunctionCallArgumentsDeltaEvent +from .conversation_item_input_audio_transcription_failed_event import ConversationItemInputAudioTranscriptionFailedEvent +from .conversation_item_input_audio_transcription_completed_event import ( + ConversationItemInputAudioTranscriptionCompletedEvent, +) + +__all__ = ["RealtimeServerEvent"] + +RealtimeServerEvent: TypeAlias = Annotated[ + Union[ + ErrorEvent, + SessionCreatedEvent, + SessionUpdatedEvent, + ConversationCreatedEvent, + InputAudioBufferCommittedEvent, + InputAudioBufferClearedEvent, + InputAudioBufferSpeechStartedEvent, + InputAudioBufferSpeechStoppedEvent, + ConversationItemCreatedEvent, + ConversationItemInputAudioTranscriptionCompletedEvent, + ConversationItemInputAudioTranscriptionFailedEvent, + ConversationItemTruncatedEvent, + ConversationItemDeletedEvent, + ResponseCreatedEvent, + ResponseDoneEvent, + ResponseOutputItemAddedEvent, + ResponseOutputItemDoneEvent, + ResponseContentPartAddedEvent, + ResponseContentPartDoneEvent, + ResponseTextDeltaEvent, + ResponseTextDoneEvent, + ResponseAudioTranscriptDeltaEvent, + ResponseAudioTranscriptDoneEvent, + ResponseAudioDeltaEvent, + ResponseAudioDoneEvent, + ResponseFunctionCallArgumentsDeltaEvent, + ResponseFunctionCallArgumentsDoneEvent, + RateLimitsUpdatedEvent, + ], + PropertyInfo(discriminator="type"), +] diff --git a/src/openai/types/beta/realtime/response_audio_delta_event.py b/src/openai/types/beta/realtime/response_audio_delta_event.py new file mode 100644 index 0000000000..8e0128d942 --- /dev/null +++ b/src/openai/types/beta/realtime/response_audio_delta_event.py @@ -0,0 +1,30 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing_extensions import Literal + +from ...._models import BaseModel + +__all__ = ["ResponseAudioDeltaEvent"] + + +class ResponseAudioDeltaEvent(BaseModel): + content_index: int + """The index of the content part in the item's content array.""" + + delta: str + """Base64-encoded audio data delta.""" + + event_id: str + """The unique ID of the server event.""" + + item_id: str + """The ID of the item.""" + + output_index: int + """The index of the output item in the response.""" + + response_id: str + """The ID of the response.""" + + type: Literal["response.audio.delta"] + """The event type, must be `response.audio.delta`.""" diff --git a/src/openai/types/beta/realtime/response_audio_done_event.py b/src/openai/types/beta/realtime/response_audio_done_event.py new file mode 100644 index 0000000000..68e78bc778 --- /dev/null +++ b/src/openai/types/beta/realtime/response_audio_done_event.py @@ -0,0 +1,27 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing_extensions import Literal + +from ...._models import BaseModel + +__all__ = ["ResponseAudioDoneEvent"] + + +class ResponseAudioDoneEvent(BaseModel): + content_index: int + """The index of the content part in the item's content array.""" + + event_id: str + """The unique ID of the server event.""" + + item_id: str + """The ID of the item.""" + + output_index: int + """The index of the output item in the response.""" + + response_id: str + """The ID of the response.""" + + type: Literal["response.audio.done"] + """The event type, must be `response.audio.done`.""" diff --git a/src/openai/types/beta/realtime/response_audio_transcript_delta_event.py b/src/openai/types/beta/realtime/response_audio_transcript_delta_event.py new file mode 100644 index 0000000000..3609948d10 --- /dev/null +++ b/src/openai/types/beta/realtime/response_audio_transcript_delta_event.py @@ -0,0 +1,30 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing_extensions import Literal + +from ...._models import BaseModel + +__all__ = ["ResponseAudioTranscriptDeltaEvent"] + + +class ResponseAudioTranscriptDeltaEvent(BaseModel): + content_index: int + """The index of the content part in the item's content array.""" + + delta: str + """The transcript delta.""" + + event_id: str + """The unique ID of the server event.""" + + item_id: str + """The ID of the item.""" + + output_index: int + """The index of the output item in the response.""" + + response_id: str + """The ID of the response.""" + + type: Literal["response.audio_transcript.delta"] + """The event type, must be `response.audio_transcript.delta`.""" diff --git a/src/openai/types/beta/realtime/response_audio_transcript_done_event.py b/src/openai/types/beta/realtime/response_audio_transcript_done_event.py new file mode 100644 index 0000000000..4e4436a95f --- /dev/null +++ b/src/openai/types/beta/realtime/response_audio_transcript_done_event.py @@ -0,0 +1,30 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing_extensions import Literal + +from ...._models import BaseModel + +__all__ = ["ResponseAudioTranscriptDoneEvent"] + + +class ResponseAudioTranscriptDoneEvent(BaseModel): + content_index: int + """The index of the content part in the item's content array.""" + + event_id: str + """The unique ID of the server event.""" + + item_id: str + """The ID of the item.""" + + output_index: int + """The index of the output item in the response.""" + + response_id: str + """The ID of the response.""" + + transcript: str + """The final transcript of the audio.""" + + type: Literal["response.audio_transcript.done"] + """The event type, must be `response.audio_transcript.done`.""" diff --git a/src/openai/types/beta/realtime/response_cancel_event.py b/src/openai/types/beta/realtime/response_cancel_event.py new file mode 100644 index 0000000000..c5ff991e9a --- /dev/null +++ b/src/openai/types/beta/realtime/response_cancel_event.py @@ -0,0 +1,22 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Optional +from typing_extensions import Literal + +from ...._models import BaseModel + +__all__ = ["ResponseCancelEvent"] + + +class ResponseCancelEvent(BaseModel): + type: Literal["response.cancel"] + """The event type, must be `response.cancel`.""" + + event_id: Optional[str] = None + """Optional client-generated ID used to identify this event.""" + + response_id: Optional[str] = None + """ + A specific response ID to cancel - if not provided, will cancel an in-progress + response in the default conversation. + """ diff --git a/src/openai/types/beta/realtime/response_cancel_event_param.py b/src/openai/types/beta/realtime/response_cancel_event_param.py new file mode 100644 index 0000000000..f33740730a --- /dev/null +++ b/src/openai/types/beta/realtime/response_cancel_event_param.py @@ -0,0 +1,21 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing_extensions import Literal, Required, TypedDict + +__all__ = ["ResponseCancelEventParam"] + + +class ResponseCancelEventParam(TypedDict, total=False): + type: Required[Literal["response.cancel"]] + """The event type, must be `response.cancel`.""" + + event_id: str + """Optional client-generated ID used to identify this event.""" + + response_id: str + """ + A specific response ID to cancel - if not provided, will cancel an in-progress + response in the default conversation. + """ diff --git a/src/openai/types/beta/realtime/response_content_part_added_event.py b/src/openai/types/beta/realtime/response_content_part_added_event.py new file mode 100644 index 0000000000..45c8f20f97 --- /dev/null +++ b/src/openai/types/beta/realtime/response_content_part_added_event.py @@ -0,0 +1,45 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Optional +from typing_extensions import Literal + +from ...._models import BaseModel + +__all__ = ["ResponseContentPartAddedEvent", "Part"] + + +class Part(BaseModel): + audio: Optional[str] = None + """Base64-encoded audio data (if type is "audio").""" + + text: Optional[str] = None + """The text content (if type is "text").""" + + transcript: Optional[str] = None + """The transcript of the audio (if type is "audio").""" + + type: Optional[Literal["text", "audio"]] = None + """The content type ("text", "audio").""" + + +class ResponseContentPartAddedEvent(BaseModel): + content_index: int + """The index of the content part in the item's content array.""" + + event_id: str + """The unique ID of the server event.""" + + item_id: str + """The ID of the item to which the content part was added.""" + + output_index: int + """The index of the output item in the response.""" + + part: Part + """The content part that was added.""" + + response_id: str + """The ID of the response.""" + + type: Literal["response.content_part.added"] + """The event type, must be `response.content_part.added`.""" diff --git a/src/openai/types/beta/realtime/response_content_part_done_event.py b/src/openai/types/beta/realtime/response_content_part_done_event.py new file mode 100644 index 0000000000..3d16116106 --- /dev/null +++ b/src/openai/types/beta/realtime/response_content_part_done_event.py @@ -0,0 +1,45 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Optional +from typing_extensions import Literal + +from ...._models import BaseModel + +__all__ = ["ResponseContentPartDoneEvent", "Part"] + + +class Part(BaseModel): + audio: Optional[str] = None + """Base64-encoded audio data (if type is "audio").""" + + text: Optional[str] = None + """The text content (if type is "text").""" + + transcript: Optional[str] = None + """The transcript of the audio (if type is "audio").""" + + type: Optional[Literal["text", "audio"]] = None + """The content type ("text", "audio").""" + + +class ResponseContentPartDoneEvent(BaseModel): + content_index: int + """The index of the content part in the item's content array.""" + + event_id: str + """The unique ID of the server event.""" + + item_id: str + """The ID of the item.""" + + output_index: int + """The index of the output item in the response.""" + + part: Part + """The content part that is done.""" + + response_id: str + """The ID of the response.""" + + type: Literal["response.content_part.done"] + """The event type, must be `response.content_part.done`.""" diff --git a/src/openai/types/beta/realtime/response_create_event.py b/src/openai/types/beta/realtime/response_create_event.py new file mode 100644 index 0000000000..00ba1e5dad --- /dev/null +++ b/src/openai/types/beta/realtime/response_create_event.py @@ -0,0 +1,115 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import List, Union, Optional +from typing_extensions import Literal + +from ...._models import BaseModel +from .conversation_item import ConversationItem + +__all__ = ["ResponseCreateEvent", "Response", "ResponseTool"] + + +class ResponseTool(BaseModel): + description: Optional[str] = None + """ + The description of the function, including guidance on when and how to call it, + and guidance about what to tell the user when calling (if anything). + """ + + name: Optional[str] = None + """The name of the function.""" + + parameters: Optional[object] = None + """Parameters of the function in JSON Schema.""" + + type: Optional[Literal["function"]] = None + """The type of the tool, i.e. `function`.""" + + +class Response(BaseModel): + conversation: Union[str, Literal["auto", "none"], None] = None + """Controls which conversation the response is added to. + + Currently supports `auto` and `none`, with `auto` as the default value. The + `auto` value means that the contents of the response will be added to the + default conversation. Set this to `none` to create an out-of-band response which + will not add items to default conversation. + """ + + input: Optional[List[ConversationItem]] = None + """Input items to include in the prompt for the model. + + Creates a new context for this response, without including the default + conversation. Can include references to items from the default conversation. + """ + + instructions: Optional[str] = None + """The default system instructions (i.e. + + system message) prepended to model calls. This field allows the client to guide + the model on desired responses. The model can be instructed on response content + and format, (e.g. "be extremely succinct", "act friendly", "here are examples of + good responses") and on audio behavior (e.g. "talk quickly", "inject emotion + into your voice", "laugh frequently"). The instructions are not guaranteed to be + followed by the model, but they provide guidance to the model on the desired + behavior. + + Note that the server sets default instructions which will be used if this field + is not set and are visible in the `session.created` event at the start of the + session. + """ + + max_response_output_tokens: Union[int, Literal["inf"], None] = None + """ + Maximum number of output tokens for a single assistant response, inclusive of + tool calls. Provide an integer between 1 and 4096 to limit output tokens, or + `inf` for the maximum available tokens for a given model. Defaults to `inf`. + """ + + metadata: Optional[object] = None + """Set of 16 key-value pairs that can be attached to an object. + + This can be useful for storing additional information about the object in a + structured format. Keys can be a maximum of 64 characters long and values can be + a maximum of 512 characters long. + """ + + modalities: Optional[List[Literal["text", "audio"]]] = None + """The set of modalities the model can respond with. + + To disable audio, set this to ["text"]. + """ + + output_audio_format: Optional[Literal["pcm16", "g711_ulaw", "g711_alaw"]] = None + """The format of output audio. Options are `pcm16`, `g711_ulaw`, or `g711_alaw`.""" + + temperature: Optional[float] = None + """Sampling temperature for the model, limited to [0.6, 1.2]. Defaults to 0.8.""" + + tool_choice: Optional[str] = None + """How the model chooses tools. + + Options are `auto`, `none`, `required`, or specify a function. + """ + + tools: Optional[List[ResponseTool]] = None + """Tools (functions) available to the model.""" + + voice: Optional[Literal["alloy", "ash", "ballad", "coral", "echo", "sage", "shimmer", "verse"]] = None + """The voice the model uses to respond. + + Voice cannot be changed during the session once the model has responded with + audio at least once. Current voice options are `alloy`, `ash`, `ballad`, + `coral`, `echo` `sage`, `shimmer` and `verse`. + """ + + +class ResponseCreateEvent(BaseModel): + type: Literal["response.create"] + """The event type, must be `response.create`.""" + + event_id: Optional[str] = None + """Optional client-generated ID used to identify this event.""" + + response: Optional[Response] = None + """Create a new Realtime response with these parameters""" diff --git a/src/openai/types/beta/realtime/response_create_event_param.py b/src/openai/types/beta/realtime/response_create_event_param.py new file mode 100644 index 0000000000..7c92b32df1 --- /dev/null +++ b/src/openai/types/beta/realtime/response_create_event_param.py @@ -0,0 +1,116 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import List, Union, Iterable, Optional +from typing_extensions import Literal, Required, TypedDict + +from .conversation_item_param import ConversationItemParam + +__all__ = ["ResponseCreateEventParam", "Response", "ResponseTool"] + + +class ResponseTool(TypedDict, total=False): + description: str + """ + The description of the function, including guidance on when and how to call it, + and guidance about what to tell the user when calling (if anything). + """ + + name: str + """The name of the function.""" + + parameters: object + """Parameters of the function in JSON Schema.""" + + type: Literal["function"] + """The type of the tool, i.e. `function`.""" + + +class Response(TypedDict, total=False): + conversation: Union[str, Literal["auto", "none"]] + """Controls which conversation the response is added to. + + Currently supports `auto` and `none`, with `auto` as the default value. The + `auto` value means that the contents of the response will be added to the + default conversation. Set this to `none` to create an out-of-band response which + will not add items to default conversation. + """ + + input: Iterable[ConversationItemParam] + """Input items to include in the prompt for the model. + + Creates a new context for this response, without including the default + conversation. Can include references to items from the default conversation. + """ + + instructions: str + """The default system instructions (i.e. + + system message) prepended to model calls. This field allows the client to guide + the model on desired responses. The model can be instructed on response content + and format, (e.g. "be extremely succinct", "act friendly", "here are examples of + good responses") and on audio behavior (e.g. "talk quickly", "inject emotion + into your voice", "laugh frequently"). The instructions are not guaranteed to be + followed by the model, but they provide guidance to the model on the desired + behavior. + + Note that the server sets default instructions which will be used if this field + is not set and are visible in the `session.created` event at the start of the + session. + """ + + max_response_output_tokens: Union[int, Literal["inf"]] + """ + Maximum number of output tokens for a single assistant response, inclusive of + tool calls. Provide an integer between 1 and 4096 to limit output tokens, or + `inf` for the maximum available tokens for a given model. Defaults to `inf`. + """ + + metadata: Optional[object] + """Set of 16 key-value pairs that can be attached to an object. + + This can be useful for storing additional information about the object in a + structured format. Keys can be a maximum of 64 characters long and values can be + a maximum of 512 characters long. + """ + + modalities: List[Literal["text", "audio"]] + """The set of modalities the model can respond with. + + To disable audio, set this to ["text"]. + """ + + output_audio_format: Literal["pcm16", "g711_ulaw", "g711_alaw"] + """The format of output audio. Options are `pcm16`, `g711_ulaw`, or `g711_alaw`.""" + + temperature: float + """Sampling temperature for the model, limited to [0.6, 1.2]. Defaults to 0.8.""" + + tool_choice: str + """How the model chooses tools. + + Options are `auto`, `none`, `required`, or specify a function. + """ + + tools: Iterable[ResponseTool] + """Tools (functions) available to the model.""" + + voice: Literal["alloy", "ash", "ballad", "coral", "echo", "sage", "shimmer", "verse"] + """The voice the model uses to respond. + + Voice cannot be changed during the session once the model has responded with + audio at least once. Current voice options are `alloy`, `ash`, `ballad`, + `coral`, `echo` `sage`, `shimmer` and `verse`. + """ + + +class ResponseCreateEventParam(TypedDict, total=False): + type: Required[Literal["response.create"]] + """The event type, must be `response.create`.""" + + event_id: str + """Optional client-generated ID used to identify this event.""" + + response: Response + """Create a new Realtime response with these parameters""" diff --git a/src/openai/types/beta/realtime/response_created_event.py b/src/openai/types/beta/realtime/response_created_event.py new file mode 100644 index 0000000000..a4990cf095 --- /dev/null +++ b/src/openai/types/beta/realtime/response_created_event.py @@ -0,0 +1,19 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing_extensions import Literal + +from ...._models import BaseModel +from .realtime_response import RealtimeResponse + +__all__ = ["ResponseCreatedEvent"] + + +class ResponseCreatedEvent(BaseModel): + event_id: str + """The unique ID of the server event.""" + + response: RealtimeResponse + """The response resource.""" + + type: Literal["response.created"] + """The event type, must be `response.created`.""" diff --git a/src/openai/types/beta/realtime/response_done_event.py b/src/openai/types/beta/realtime/response_done_event.py new file mode 100644 index 0000000000..9e655184b6 --- /dev/null +++ b/src/openai/types/beta/realtime/response_done_event.py @@ -0,0 +1,19 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing_extensions import Literal + +from ...._models import BaseModel +from .realtime_response import RealtimeResponse + +__all__ = ["ResponseDoneEvent"] + + +class ResponseDoneEvent(BaseModel): + event_id: str + """The unique ID of the server event.""" + + response: RealtimeResponse + """The response resource.""" + + type: Literal["response.done"] + """The event type, must be `response.done`.""" diff --git a/src/openai/types/beta/realtime/response_function_call_arguments_delta_event.py b/src/openai/types/beta/realtime/response_function_call_arguments_delta_event.py new file mode 100644 index 0000000000..cdbb64e658 --- /dev/null +++ b/src/openai/types/beta/realtime/response_function_call_arguments_delta_event.py @@ -0,0 +1,30 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing_extensions import Literal + +from ...._models import BaseModel + +__all__ = ["ResponseFunctionCallArgumentsDeltaEvent"] + + +class ResponseFunctionCallArgumentsDeltaEvent(BaseModel): + call_id: str + """The ID of the function call.""" + + delta: str + """The arguments delta as a JSON string.""" + + event_id: str + """The unique ID of the server event.""" + + item_id: str + """The ID of the function call item.""" + + output_index: int + """The index of the output item in the response.""" + + response_id: str + """The ID of the response.""" + + type: Literal["response.function_call_arguments.delta"] + """The event type, must be `response.function_call_arguments.delta`.""" diff --git a/src/openai/types/beta/realtime/response_function_call_arguments_done_event.py b/src/openai/types/beta/realtime/response_function_call_arguments_done_event.py new file mode 100644 index 0000000000..0a5db53323 --- /dev/null +++ b/src/openai/types/beta/realtime/response_function_call_arguments_done_event.py @@ -0,0 +1,30 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing_extensions import Literal + +from ...._models import BaseModel + +__all__ = ["ResponseFunctionCallArgumentsDoneEvent"] + + +class ResponseFunctionCallArgumentsDoneEvent(BaseModel): + arguments: str + """The final arguments as a JSON string.""" + + call_id: str + """The ID of the function call.""" + + event_id: str + """The unique ID of the server event.""" + + item_id: str + """The ID of the function call item.""" + + output_index: int + """The index of the output item in the response.""" + + response_id: str + """The ID of the response.""" + + type: Literal["response.function_call_arguments.done"] + """The event type, must be `response.function_call_arguments.done`.""" diff --git a/src/openai/types/beta/realtime/response_output_item_added_event.py b/src/openai/types/beta/realtime/response_output_item_added_event.py new file mode 100644 index 0000000000..c89bfdc3be --- /dev/null +++ b/src/openai/types/beta/realtime/response_output_item_added_event.py @@ -0,0 +1,25 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing_extensions import Literal + +from ...._models import BaseModel +from .conversation_item import ConversationItem + +__all__ = ["ResponseOutputItemAddedEvent"] + + +class ResponseOutputItemAddedEvent(BaseModel): + event_id: str + """The unique ID of the server event.""" + + item: ConversationItem + """The item to add to the conversation.""" + + output_index: int + """The index of the output item in the Response.""" + + response_id: str + """The ID of the Response to which the item belongs.""" + + type: Literal["response.output_item.added"] + """The event type, must be `response.output_item.added`.""" diff --git a/src/openai/types/beta/realtime/response_output_item_done_event.py b/src/openai/types/beta/realtime/response_output_item_done_event.py new file mode 100644 index 0000000000..b5910e22aa --- /dev/null +++ b/src/openai/types/beta/realtime/response_output_item_done_event.py @@ -0,0 +1,25 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing_extensions import Literal + +from ...._models import BaseModel +from .conversation_item import ConversationItem + +__all__ = ["ResponseOutputItemDoneEvent"] + + +class ResponseOutputItemDoneEvent(BaseModel): + event_id: str + """The unique ID of the server event.""" + + item: ConversationItem + """The item to add to the conversation.""" + + output_index: int + """The index of the output item in the Response.""" + + response_id: str + """The ID of the Response to which the item belongs.""" + + type: Literal["response.output_item.done"] + """The event type, must be `response.output_item.done`.""" diff --git a/src/openai/types/beta/realtime/response_text_delta_event.py b/src/openai/types/beta/realtime/response_text_delta_event.py new file mode 100644 index 0000000000..c463b3c3d0 --- /dev/null +++ b/src/openai/types/beta/realtime/response_text_delta_event.py @@ -0,0 +1,30 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing_extensions import Literal + +from ...._models import BaseModel + +__all__ = ["ResponseTextDeltaEvent"] + + +class ResponseTextDeltaEvent(BaseModel): + content_index: int + """The index of the content part in the item's content array.""" + + delta: str + """The text delta.""" + + event_id: str + """The unique ID of the server event.""" + + item_id: str + """The ID of the item.""" + + output_index: int + """The index of the output item in the response.""" + + response_id: str + """The ID of the response.""" + + type: Literal["response.text.delta"] + """The event type, must be `response.text.delta`.""" diff --git a/src/openai/types/beta/realtime/response_text_done_event.py b/src/openai/types/beta/realtime/response_text_done_event.py new file mode 100644 index 0000000000..020ff41d58 --- /dev/null +++ b/src/openai/types/beta/realtime/response_text_done_event.py @@ -0,0 +1,30 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing_extensions import Literal + +from ...._models import BaseModel + +__all__ = ["ResponseTextDoneEvent"] + + +class ResponseTextDoneEvent(BaseModel): + content_index: int + """The index of the content part in the item's content array.""" + + event_id: str + """The unique ID of the server event.""" + + item_id: str + """The ID of the item.""" + + output_index: int + """The index of the output item in the response.""" + + response_id: str + """The ID of the response.""" + + text: str + """The final text content.""" + + type: Literal["response.text.done"] + """The event type, must be `response.text.done`.""" diff --git a/src/openai/types/beta/realtime/session.py b/src/openai/types/beta/realtime/session.py new file mode 100644 index 0000000000..09cdbb02bc --- /dev/null +++ b/src/openai/types/beta/realtime/session.py @@ -0,0 +1,148 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import List, Union, Optional +from typing_extensions import Literal + +from ...._models import BaseModel + +__all__ = ["Session", "InputAudioTranscription", "Tool", "TurnDetection"] + + +class InputAudioTranscription(BaseModel): + model: Optional[str] = None + """ + The model to use for transcription, `whisper-1` is the only currently supported + model. + """ + + +class Tool(BaseModel): + description: Optional[str] = None + """ + The description of the function, including guidance on when and how to call it, + and guidance about what to tell the user when calling (if anything). + """ + + name: Optional[str] = None + """The name of the function.""" + + parameters: Optional[object] = None + """Parameters of the function in JSON Schema.""" + + type: Optional[Literal["function"]] = None + """The type of the tool, i.e. `function`.""" + + +class TurnDetection(BaseModel): + prefix_padding_ms: Optional[int] = None + """Amount of audio to include before the VAD detected speech (in milliseconds). + + Defaults to 300ms. + """ + + silence_duration_ms: Optional[int] = None + """Duration of silence to detect speech stop (in milliseconds). + + Defaults to 500ms. With shorter values the model will respond more quickly, but + may jump in on short pauses from the user. + """ + + threshold: Optional[float] = None + """Activation threshold for VAD (0.0 to 1.0), this defaults to 0.5. + + A higher threshold will require louder audio to activate the model, and thus + might perform better in noisy environments. + """ + + type: Optional[Literal["server_vad"]] = None + """Type of turn detection, only `server_vad` is currently supported.""" + + +class Session(BaseModel): + id: Optional[str] = None + """Unique identifier for the session object.""" + + input_audio_format: Optional[Literal["pcm16", "g711_ulaw", "g711_alaw"]] = None + """The format of input audio. Options are `pcm16`, `g711_ulaw`, or `g711_alaw`.""" + + input_audio_transcription: Optional[InputAudioTranscription] = None + """ + Configuration for input audio transcription, defaults to off and can be set to + `null` to turn off once on. Input audio transcription is not native to the + model, since the model consumes audio directly. Transcription runs + asynchronously through Whisper and should be treated as rough guidance rather + than the representation understood by the model. + """ + + instructions: Optional[str] = None + """The default system instructions (i.e. + + system message) prepended to model calls. This field allows the client to guide + the model on desired responses. The model can be instructed on response content + and format, (e.g. "be extremely succinct", "act friendly", "here are examples of + good responses") and on audio behavior (e.g. "talk quickly", "inject emotion + into your voice", "laugh frequently"). The instructions are not guaranteed to be + followed by the model, but they provide guidance to the model on the desired + behavior. + + Note that the server sets default instructions which will be used if this field + is not set and are visible in the `session.created` event at the start of the + session. + """ + + max_response_output_tokens: Union[int, Literal["inf"], None] = None + """ + Maximum number of output tokens for a single assistant response, inclusive of + tool calls. Provide an integer between 1 and 4096 to limit output tokens, or + `inf` for the maximum available tokens for a given model. Defaults to `inf`. + """ + + modalities: Optional[List[Literal["text", "audio"]]] = None + """The set of modalities the model can respond with. + + To disable audio, set this to ["text"]. + """ + + model: Union[ + str, + Literal[ + "gpt-4o-realtime-preview", + "gpt-4o-realtime-preview-2024-10-01", + "gpt-4o-realtime-preview-2024-12-17", + "gpt-4o-mini-realtime-preview", + "gpt-4o-mini-realtime-preview-2024-12-17", + ], + None, + ] = None + """The Realtime model used for this session.""" + + output_audio_format: Optional[Literal["pcm16", "g711_ulaw", "g711_alaw"]] = None + """The format of output audio. Options are `pcm16`, `g711_ulaw`, or `g711_alaw`.""" + + temperature: Optional[float] = None + """Sampling temperature for the model, limited to [0.6, 1.2]. Defaults to 0.8.""" + + tool_choice: Optional[str] = None + """How the model chooses tools. + + Options are `auto`, `none`, `required`, or specify a function. + """ + + tools: Optional[List[Tool]] = None + """Tools (functions) available to the model.""" + + turn_detection: Optional[TurnDetection] = None + """Configuration for turn detection. + + Can be set to `null` to turn off. Server VAD means that the model will detect + the start and end of speech based on audio volume and respond at the end of user + speech. + """ + + voice: Optional[Literal["alloy", "ash", "ballad", "coral", "echo", "sage", "shimmer", "verse"]] = None + """The voice the model uses to respond. + + Voice cannot be changed during the session once the model has responded with + audio at least once. Current voice options are `alloy`, `ash`, `ballad`, + `coral`, `echo` `sage`, `shimmer` and `verse`. + """ diff --git a/src/openai/types/beta/realtime/session_created_event.py b/src/openai/types/beta/realtime/session_created_event.py new file mode 100644 index 0000000000..baf6af388b --- /dev/null +++ b/src/openai/types/beta/realtime/session_created_event.py @@ -0,0 +1,19 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing_extensions import Literal + +from .session import Session +from ...._models import BaseModel + +__all__ = ["SessionCreatedEvent"] + + +class SessionCreatedEvent(BaseModel): + event_id: str + """The unique ID of the server event.""" + + session: Session + """Realtime session object configuration.""" + + type: Literal["session.created"] + """The event type, must be `session.created`.""" diff --git a/src/openai/types/beta/realtime/session_update_event.py b/src/openai/types/beta/realtime/session_update_event.py new file mode 100644 index 0000000000..c04220aa25 --- /dev/null +++ b/src/openai/types/beta/realtime/session_update_event.py @@ -0,0 +1,158 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import List, Union, Optional +from typing_extensions import Literal + +from ...._models import BaseModel + +__all__ = ["SessionUpdateEvent", "Session", "SessionInputAudioTranscription", "SessionTool", "SessionTurnDetection"] + + +class SessionInputAudioTranscription(BaseModel): + model: Optional[str] = None + """ + The model to use for transcription, `whisper-1` is the only currently supported + model. + """ + + +class SessionTool(BaseModel): + description: Optional[str] = None + """ + The description of the function, including guidance on when and how to call it, + and guidance about what to tell the user when calling (if anything). + """ + + name: Optional[str] = None + """The name of the function.""" + + parameters: Optional[object] = None + """Parameters of the function in JSON Schema.""" + + type: Optional[Literal["function"]] = None + """The type of the tool, i.e. `function`.""" + + +class SessionTurnDetection(BaseModel): + create_response: Optional[bool] = None + """Whether or not to automatically generate a response when VAD is enabled. + + `true` by default. + """ + + prefix_padding_ms: Optional[int] = None + """Amount of audio to include before the VAD detected speech (in milliseconds). + + Defaults to 300ms. + """ + + silence_duration_ms: Optional[int] = None + """Duration of silence to detect speech stop (in milliseconds). + + Defaults to 500ms. With shorter values the model will respond more quickly, but + may jump in on short pauses from the user. + """ + + threshold: Optional[float] = None + """Activation threshold for VAD (0.0 to 1.0), this defaults to 0.5. + + A higher threshold will require louder audio to activate the model, and thus + might perform better in noisy environments. + """ + + type: Optional[str] = None + """Type of turn detection, only `server_vad` is currently supported.""" + + +class Session(BaseModel): + model: Literal[ + "gpt-4o-realtime-preview", + "gpt-4o-realtime-preview-2024-10-01", + "gpt-4o-realtime-preview-2024-12-17", + "gpt-4o-mini-realtime-preview", + "gpt-4o-mini-realtime-preview-2024-12-17", + ] + """The Realtime model used for this session.""" + + input_audio_format: Optional[Literal["pcm16", "g711_ulaw", "g711_alaw"]] = None + """The format of input audio. Options are `pcm16`, `g711_ulaw`, or `g711_alaw`.""" + + input_audio_transcription: Optional[SessionInputAudioTranscription] = None + """ + Configuration for input audio transcription, defaults to off and can be set to + `null` to turn off once on. Input audio transcription is not native to the + model, since the model consumes audio directly. Transcription runs + asynchronously through Whisper and should be treated as rough guidance rather + than the representation understood by the model. + """ + + instructions: Optional[str] = None + """The default system instructions (i.e. + + system message) prepended to model calls. This field allows the client to guide + the model on desired responses. The model can be instructed on response content + and format, (e.g. "be extremely succinct", "act friendly", "here are examples of + good responses") and on audio behavior (e.g. "talk quickly", "inject emotion + into your voice", "laugh frequently"). The instructions are not guaranteed to be + followed by the model, but they provide guidance to the model on the desired + behavior. + + Note that the server sets default instructions which will be used if this field + is not set and are visible in the `session.created` event at the start of the + session. + """ + + max_response_output_tokens: Union[int, Literal["inf"], None] = None + """ + Maximum number of output tokens for a single assistant response, inclusive of + tool calls. Provide an integer between 1 and 4096 to limit output tokens, or + `inf` for the maximum available tokens for a given model. Defaults to `inf`. + """ + + modalities: Optional[List[Literal["text", "audio"]]] = None + """The set of modalities the model can respond with. + + To disable audio, set this to ["text"]. + """ + + output_audio_format: Optional[Literal["pcm16", "g711_ulaw", "g711_alaw"]] = None + """The format of output audio. Options are `pcm16`, `g711_ulaw`, or `g711_alaw`.""" + + temperature: Optional[float] = None + """Sampling temperature for the model, limited to [0.6, 1.2]. Defaults to 0.8.""" + + tool_choice: Optional[str] = None + """How the model chooses tools. + + Options are `auto`, `none`, `required`, or specify a function. + """ + + tools: Optional[List[SessionTool]] = None + """Tools (functions) available to the model.""" + + turn_detection: Optional[SessionTurnDetection] = None + """Configuration for turn detection. + + Can be set to `null` to turn off. Server VAD means that the model will detect + the start and end of speech based on audio volume and respond at the end of user + speech. + """ + + voice: Optional[Literal["alloy", "ash", "ballad", "coral", "echo", "sage", "shimmer", "verse"]] = None + """The voice the model uses to respond. + + Voice cannot be changed during the session once the model has responded with + audio at least once. Current voice options are `alloy`, `ash`, `ballad`, + `coral`, `echo` `sage`, `shimmer` and `verse`. + """ + + +class SessionUpdateEvent(BaseModel): + session: Session + """Realtime session object configuration.""" + + type: Literal["session.update"] + """The event type, must be `session.update`.""" + + event_id: Optional[str] = None + """Optional client-generated ID used to identify this event.""" diff --git a/src/openai/types/beta/realtime/session_update_event_param.py b/src/openai/types/beta/realtime/session_update_event_param.py new file mode 100644 index 0000000000..aa06069b04 --- /dev/null +++ b/src/openai/types/beta/realtime/session_update_event_param.py @@ -0,0 +1,166 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import List, Union, Iterable +from typing_extensions import Literal, Required, TypedDict + +__all__ = [ + "SessionUpdateEventParam", + "Session", + "SessionInputAudioTranscription", + "SessionTool", + "SessionTurnDetection", +] + + +class SessionInputAudioTranscription(TypedDict, total=False): + model: str + """ + The model to use for transcription, `whisper-1` is the only currently supported + model. + """ + + +class SessionTool(TypedDict, total=False): + description: str + """ + The description of the function, including guidance on when and how to call it, + and guidance about what to tell the user when calling (if anything). + """ + + name: str + """The name of the function.""" + + parameters: object + """Parameters of the function in JSON Schema.""" + + type: Literal["function"] + """The type of the tool, i.e. `function`.""" + + +class SessionTurnDetection(TypedDict, total=False): + create_response: bool + """Whether or not to automatically generate a response when VAD is enabled. + + `true` by default. + """ + + prefix_padding_ms: int + """Amount of audio to include before the VAD detected speech (in milliseconds). + + Defaults to 300ms. + """ + + silence_duration_ms: int + """Duration of silence to detect speech stop (in milliseconds). + + Defaults to 500ms. With shorter values the model will respond more quickly, but + may jump in on short pauses from the user. + """ + + threshold: float + """Activation threshold for VAD (0.0 to 1.0), this defaults to 0.5. + + A higher threshold will require louder audio to activate the model, and thus + might perform better in noisy environments. + """ + + type: str + """Type of turn detection, only `server_vad` is currently supported.""" + + +class Session(TypedDict, total=False): + model: Required[ + Literal[ + "gpt-4o-realtime-preview", + "gpt-4o-realtime-preview-2024-10-01", + "gpt-4o-realtime-preview-2024-12-17", + "gpt-4o-mini-realtime-preview", + "gpt-4o-mini-realtime-preview-2024-12-17", + ] + ] + """The Realtime model used for this session.""" + + input_audio_format: Literal["pcm16", "g711_ulaw", "g711_alaw"] + """The format of input audio. Options are `pcm16`, `g711_ulaw`, or `g711_alaw`.""" + + input_audio_transcription: SessionInputAudioTranscription + """ + Configuration for input audio transcription, defaults to off and can be set to + `null` to turn off once on. Input audio transcription is not native to the + model, since the model consumes audio directly. Transcription runs + asynchronously through Whisper and should be treated as rough guidance rather + than the representation understood by the model. + """ + + instructions: str + """The default system instructions (i.e. + + system message) prepended to model calls. This field allows the client to guide + the model on desired responses. The model can be instructed on response content + and format, (e.g. "be extremely succinct", "act friendly", "here are examples of + good responses") and on audio behavior (e.g. "talk quickly", "inject emotion + into your voice", "laugh frequently"). The instructions are not guaranteed to be + followed by the model, but they provide guidance to the model on the desired + behavior. + + Note that the server sets default instructions which will be used if this field + is not set and are visible in the `session.created` event at the start of the + session. + """ + + max_response_output_tokens: Union[int, Literal["inf"]] + """ + Maximum number of output tokens for a single assistant response, inclusive of + tool calls. Provide an integer between 1 and 4096 to limit output tokens, or + `inf` for the maximum available tokens for a given model. Defaults to `inf`. + """ + + modalities: List[Literal["text", "audio"]] + """The set of modalities the model can respond with. + + To disable audio, set this to ["text"]. + """ + + output_audio_format: Literal["pcm16", "g711_ulaw", "g711_alaw"] + """The format of output audio. Options are `pcm16`, `g711_ulaw`, or `g711_alaw`.""" + + temperature: float + """Sampling temperature for the model, limited to [0.6, 1.2]. Defaults to 0.8.""" + + tool_choice: str + """How the model chooses tools. + + Options are `auto`, `none`, `required`, or specify a function. + """ + + tools: Iterable[SessionTool] + """Tools (functions) available to the model.""" + + turn_detection: SessionTurnDetection + """Configuration for turn detection. + + Can be set to `null` to turn off. Server VAD means that the model will detect + the start and end of speech based on audio volume and respond at the end of user + speech. + """ + + voice: Literal["alloy", "ash", "ballad", "coral", "echo", "sage", "shimmer", "verse"] + """The voice the model uses to respond. + + Voice cannot be changed during the session once the model has responded with + audio at least once. Current voice options are `alloy`, `ash`, `ballad`, + `coral`, `echo` `sage`, `shimmer` and `verse`. + """ + + +class SessionUpdateEventParam(TypedDict, total=False): + session: Required[Session] + """Realtime session object configuration.""" + + type: Required[Literal["session.update"]] + """The event type, must be `session.update`.""" + + event_id: str + """Optional client-generated ID used to identify this event.""" diff --git a/src/openai/types/beta/realtime/session_updated_event.py b/src/openai/types/beta/realtime/session_updated_event.py new file mode 100644 index 0000000000..b9b6488eb3 --- /dev/null +++ b/src/openai/types/beta/realtime/session_updated_event.py @@ -0,0 +1,19 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing_extensions import Literal + +from .session import Session +from ...._models import BaseModel + +__all__ = ["SessionUpdatedEvent"] + + +class SessionUpdatedEvent(BaseModel): + event_id: str + """The unique ID of the server event.""" + + session: Session + """Realtime session object configuration.""" + + type: Literal["session.updated"] + """The event type, must be `session.updated`.""" diff --git a/src/openai/types/websocket_connection_options.py b/src/openai/types/websocket_connection_options.py new file mode 100644 index 0000000000..40fd24ab03 --- /dev/null +++ b/src/openai/types/websocket_connection_options.py @@ -0,0 +1,36 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import TYPE_CHECKING +from typing_extensions import Sequence, TypedDict + +if TYPE_CHECKING: + from websockets import Subprotocol + from websockets.extensions import ClientExtensionFactory + + +class WebsocketConnectionOptions(TypedDict, total=False): + """Websocket connection options copied from `websockets`. + + For example: https://websockets.readthedocs.io/en/stable/reference/asyncio/client.html#websockets.asyncio.client.connect + """ + + extensions: Sequence[ClientExtensionFactory] | None + """List of supported extensions, in order in which they should be negotiated and run.""" + + subprotocols: Sequence[Subprotocol] | None + """List of supported subprotocols, in order of decreasing preference.""" + + compression: str | None + """The “permessage-deflate” extension is enabled by default. Set compression to None to disable it. See the [compression guide](https://websockets.readthedocs.io/en/stable/topics/compression.html) for details.""" + + # limits + max_size: int | None + """Maximum size of incoming messages in bytes. None disables the limit.""" + + max_queue: int | None | tuple[int | None, int | None] + """High-water mark of the buffer where frames are received. It defaults to 16 frames. The low-water mark defaults to max_queue // 4. You may pass a (high, low) tuple to set the high-water and low-water marks. If you want to disable flow control entirely, you may set it to None, although that’s a bad idea.""" + + write_limit: int | tuple[int, int | None] + """High-water mark of write buffer in bytes. It is passed to set_write_buffer_limits(). It defaults to 32 KiB. You may pass a (high, low) tuple to set the high-water and low-water marks.""" diff --git a/tests/api_resources/beta/test_realtime.py b/tests/api_resources/beta/test_realtime.py new file mode 100644 index 0000000000..537017ffd3 --- /dev/null +++ b/tests/api_resources/beta/test_realtime.py @@ -0,0 +1,17 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +import os + +import pytest + +base_url = os.environ.get("TEST_API_BASE_URL", "http://127.0.0.1:4010") + + +class TestRealtime: + parametrize = pytest.mark.parametrize("client", [False, True], indirect=True, ids=["loose", "strict"]) + + +class TestAsyncRealtime: + parametrize = pytest.mark.parametrize("async_client", [False, True], indirect=True, ids=["loose", "strict"]) From 8829c3202dbe790ca3646476c802ec55ed47d864 Mon Sep 17 00:00:00 2001 From: Robert Craigie Date: Tue, 17 Dec 2024 18:06:18 +0000 Subject: [PATCH 7/9] fix: add reasoning_effort to all methods --- src/openai/resources/beta/chat/completions.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/openai/resources/beta/chat/completions.py b/src/openai/resources/beta/chat/completions.py index 38c09ce8dd..48cb13f7a6 100644 --- a/src/openai/resources/beta/chat/completions.py +++ b/src/openai/resources/beta/chat/completions.py @@ -15,7 +15,10 @@ from ...._resource import SyncAPIResource, AsyncAPIResource from ...._response import to_streamed_response_wrapper, async_to_streamed_response_wrapper from ...._streaming import Stream -from ....types.chat import completion_create_params +from ....types.chat import ( + ChatCompletionReasoningEffort, + completion_create_params, +) from ...._base_client import make_request_options from ....lib._parsing import ( ResponseFormatT, @@ -79,6 +82,7 @@ def parse( parallel_tool_calls: bool | NotGiven = NOT_GIVEN, prediction: Optional[ChatCompletionPredictionContentParam] | NotGiven = NOT_GIVEN, presence_penalty: Optional[float] | NotGiven = NOT_GIVEN, + reasoning_effort: ChatCompletionReasoningEffort | NotGiven = NOT_GIVEN, seed: Optional[int] | NotGiven = NOT_GIVEN, service_tier: Optional[Literal["auto", "default"]] | NotGiven = NOT_GIVEN, stop: Union[Optional[str], List[str]] | NotGiven = NOT_GIVEN, @@ -173,6 +177,7 @@ def parser(raw_completion: ChatCompletion) -> ParsedChatCompletion[ResponseForma "parallel_tool_calls": parallel_tool_calls, "prediction": prediction, "presence_penalty": presence_penalty, + "reasoning_effort": reasoning_effort, "response_format": _type_to_response_format(response_format), "seed": seed, "service_tier": service_tier, @@ -222,6 +227,7 @@ def stream( parallel_tool_calls: bool | NotGiven = NOT_GIVEN, prediction: Optional[ChatCompletionPredictionContentParam] | NotGiven = NOT_GIVEN, presence_penalty: Optional[float] | NotGiven = NOT_GIVEN, + reasoning_effort: ChatCompletionReasoningEffort | NotGiven = NOT_GIVEN, seed: Optional[int] | NotGiven = NOT_GIVEN, service_tier: Optional[Literal["auto", "default"]] | NotGiven = NOT_GIVEN, stop: Union[Optional[str], List[str]] | NotGiven = NOT_GIVEN, @@ -287,6 +293,7 @@ def stream( parallel_tool_calls=parallel_tool_calls, prediction=prediction, presence_penalty=presence_penalty, + reasoning_effort=reasoning_effort, seed=seed, service_tier=service_tier, store=store, @@ -350,6 +357,7 @@ async def parse( parallel_tool_calls: bool | NotGiven = NOT_GIVEN, prediction: Optional[ChatCompletionPredictionContentParam] | NotGiven = NOT_GIVEN, presence_penalty: Optional[float] | NotGiven = NOT_GIVEN, + reasoning_effort: ChatCompletionReasoningEffort | NotGiven = NOT_GIVEN, seed: Optional[int] | NotGiven = NOT_GIVEN, service_tier: Optional[Literal["auto", "default"]] | NotGiven = NOT_GIVEN, stop: Union[Optional[str], List[str]] | NotGiven = NOT_GIVEN, @@ -444,6 +452,7 @@ def parser(raw_completion: ChatCompletion) -> ParsedChatCompletion[ResponseForma "parallel_tool_calls": parallel_tool_calls, "prediction": prediction, "presence_penalty": presence_penalty, + "reasoning_effort": reasoning_effort, "response_format": _type_to_response_format(response_format), "seed": seed, "service_tier": service_tier, @@ -493,6 +502,7 @@ def stream( parallel_tool_calls: bool | NotGiven = NOT_GIVEN, prediction: Optional[ChatCompletionPredictionContentParam] | NotGiven = NOT_GIVEN, presence_penalty: Optional[float] | NotGiven = NOT_GIVEN, + reasoning_effort: ChatCompletionReasoningEffort | NotGiven = NOT_GIVEN, seed: Optional[int] | NotGiven = NOT_GIVEN, service_tier: Optional[Literal["auto", "default"]] | NotGiven = NOT_GIVEN, stop: Union[Optional[str], List[str]] | NotGiven = NOT_GIVEN, @@ -559,6 +569,7 @@ def stream( parallel_tool_calls=parallel_tool_calls, prediction=prediction, presence_penalty=presence_penalty, + reasoning_effort=reasoning_effort, seed=seed, service_tier=service_tier, stop=stop, From 1cb00f8fed78052aacbb9e0fac997b6ba0d44d2a Mon Sep 17 00:00:00 2001 From: Robert Craigie Date: Mon, 16 Dec 2024 19:16:25 +0000 Subject: [PATCH 8/9] docs: add examples + guidance on Realtime API support --- README.md | 61 ++++++ examples/realtime/audio_util.py | 142 +++++++++++++ examples/realtime/push_to_talk_app.py | 281 ++++++++++++++++++++++++++ mypy.ini | 5 +- pyproject.toml | 5 + 5 files changed, 493 insertions(+), 1 deletion(-) create mode 100644 examples/realtime/audio_util.py create mode 100755 examples/realtime/push_to_talk_app.py diff --git a/README.md b/README.md index cbcfdb4447..4c3ba87c97 100644 --- a/README.md +++ b/README.md @@ -258,6 +258,67 @@ We recommend that you always instantiate a client (e.g., with `client = OpenAI() - It's harder to mock for testing purposes - It's not possible to control cleanup of network connections +## Realtime API beta + +The Realtime API enables you to build low-latency, multi-modal conversational experiences. It currently supports text and audio as both input and output, as well as [function calling](https://platform.openai.com/docs/guides/function-calling) through a WebSocket connection. + +Under the hood the SDK uses the [`websockets`](https://websockets.readthedocs.io/en/stable/) library to manage connections. + +The Realtime API works through a combination of client-sent events and server-sent events. Clients can send events to do things like update session configuration or send text and audio inputs. Server events confirm when audio responses have completed, or when a text response from the model has been received. A full event reference can be found [here](platform.openai.com/docs/api-reference/realtime-client-events) and a guide can be found [here](https://platform.openai.com/docs/guides/realtime). + +Basic text based example: + +```py +import asyncio +from openai import AsyncOpenAI + +async def main(): + client = AsyncOpenAI() + + async with client.beta.realtime.connect(model="gpt-4o-realtime-preview-2024-10-01") as connection: + await connection.session.update(session={'modalities': ['text']}) + + await connection.conversation.item.create( + item={ + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "Say hello!"}], + } + ) + await connection.response.create() + + async for event in connection: + if event.type == 'response.text.delta': + print(event.delta, flush=True, end="") + + elif event.type == 'response.text.done': + print() + + elif event.type == "response.done": + break + +asyncio.run(main()) +``` + +However the real magic of the Realtime API is handling audio inputs / outputs, see this example [TUI script](https://github.com/stainless-sdks/openai-python/blob/robert/realtime-docs-preview/examples/realtime/push_to_talk_app.py) for a fully fledged example. + +### Realtime error handling + +Whenever an error occurs, the Realtime API will send an [`error` event](https://platform.openai.com/docs/guides/realtime/realtime-api-beta#handling-errors) and the connection will stay open and remain usable. This means you need to handle it yourself, as *no errors are raised directly* by the SDK when an `error` event comes in. + +```py +client = AsyncOpenAI() + +async with client.beta.realtime.connect(model="gpt-4o-realtime-preview-2024-10-01") as connection: + ... + async for event in connection: + if event.type == 'error': + print(event.error.type) + print(event.error.code) + print(event.error.event_id) + print(event.error.message) +``` + ## Using types Nested request parameters are [TypedDicts](https://docs.python.org/3/library/typing.html#typing.TypedDict). Responses are [Pydantic models](https://docs.pydantic.dev) which also provide helper methods for things like: diff --git a/examples/realtime/audio_util.py b/examples/realtime/audio_util.py new file mode 100644 index 0000000000..b073cc45be --- /dev/null +++ b/examples/realtime/audio_util.py @@ -0,0 +1,142 @@ +from __future__ import annotations + +import io +import base64 +import asyncio +import threading +from typing import Callable, Awaitable + +import numpy as np +import pyaudio +import sounddevice as sd +from pydub import AudioSegment + +from openai.resources.beta.realtime.realtime import AsyncRealtimeConnection + +CHUNK_LENGTH_S = 0.05 # 100ms +SAMPLE_RATE = 24000 +FORMAT = pyaudio.paInt16 +CHANNELS = 1 + +# pyright: reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false + + +def audio_to_pcm16_base64(audio_bytes: bytes) -> bytes: + # load the audio file from the byte stream + audio = AudioSegment.from_file(io.BytesIO(audio_bytes)) + print(f"Loaded audio: {audio.frame_rate=} {audio.channels=} {audio.sample_width=} {audio.frame_width=}") + # resample to 24kHz mono pcm16 + pcm_audio = audio.set_frame_rate(SAMPLE_RATE).set_channels(CHANNELS).set_sample_width(2).raw_data + return pcm_audio + + +class AudioPlayerAsync: + def __init__(self): + self.queue = [] + self.lock = threading.Lock() + self.stream = sd.OutputStream( + callback=self.callback, + samplerate=SAMPLE_RATE, + channels=CHANNELS, + dtype=np.int16, + blocksize=int(CHUNK_LENGTH_S * SAMPLE_RATE), + ) + self.playing = False + self._frame_count = 0 + + def callback(self, outdata, frames, time, status): # noqa + with self.lock: + data = np.empty(0, dtype=np.int16) + + # get next item from queue if there is still space in the buffer + while len(data) < frames and len(self.queue) > 0: + item = self.queue.pop(0) + frames_needed = frames - len(data) + data = np.concatenate((data, item[:frames_needed])) + if len(item) > frames_needed: + self.queue.insert(0, item[frames_needed:]) + + self._frame_count += len(data) + + # fill the rest of the frames with zeros if there is no more data + if len(data) < frames: + data = np.concatenate((data, np.zeros(frames - len(data), dtype=np.int16))) + + outdata[:] = data.reshape(-1, 1) + + def reset_frame_count(self): + self._frame_count = 0 + + def get_frame_count(self): + return self._frame_count + + def add_data(self, data: bytes): + with self.lock: + # bytes is pcm16 single channel audio data, convert to numpy array + np_data = np.frombuffer(data, dtype=np.int16) + self.queue.append(np_data) + if not self.playing: + self.start() + + def start(self): + self.playing = True + self.stream.start() + + def stop(self): + self.playing = False + self.stream.stop() + with self.lock: + self.queue = [] + + def terminate(self): + self.stream.close() + + +async def send_audio_worker_sounddevice( + connection: AsyncRealtimeConnection, + should_send: Callable[[], bool] | None = None, + start_send: Callable[[], Awaitable[None]] | None = None, +): + sent_audio = False + + device_info = sd.query_devices() + print(device_info) + + read_size = int(SAMPLE_RATE * 0.02) + + stream = sd.InputStream( + channels=CHANNELS, + samplerate=SAMPLE_RATE, + dtype="int16", + ) + stream.start() + + try: + while True: + if stream.read_available < read_size: + await asyncio.sleep(0) + continue + + data, _ = stream.read(read_size) + + if should_send() if should_send else True: + if not sent_audio and start_send: + await start_send() + await connection.send( + {"type": "input_audio_buffer.append", "audio": base64.b64encode(data).decode("utf-8")} + ) + sent_audio = True + + elif sent_audio: + print("Done, triggering inference") + await connection.send({"type": "input_audio_buffer.commit"}) + await connection.send({"type": "response.create", "response": {}}) + sent_audio = False + + await asyncio.sleep(0) + + except KeyboardInterrupt: + pass + finally: + stream.stop() + stream.close() diff --git a/examples/realtime/push_to_talk_app.py b/examples/realtime/push_to_talk_app.py new file mode 100755 index 0000000000..d46945a8ed --- /dev/null +++ b/examples/realtime/push_to_talk_app.py @@ -0,0 +1,281 @@ +#!/usr/bin/env uv run +#################################################################### +# Sample TUI app with a push to talk interface to the Realtime API # +# If you have `uv` installed and the `OPENAI_API_KEY` # +# environment variable set, you can run this example with just # +# # +# `./examples/realtime/push_to_talk_app.py` # +#################################################################### +# +# /// script +# requires-python = ">=3.9" +# dependencies = [ +# "textual", +# "numpy", +# "pyaudio", +# "pydub", +# "sounddevice", +# "openai[realtime]", +# ] +# +# [tool.uv.sources] +# openai = { path = "../../", editable = true } +# /// +from __future__ import annotations + +import base64 +import asyncio +from typing import Any, cast +from typing_extensions import override + +from textual import events +from audio_util import CHANNELS, SAMPLE_RATE, AudioPlayerAsync +from textual.app import App, ComposeResult +from textual.widgets import Button, Static, RichLog +from textual.reactive import reactive +from textual.containers import Container + +from openai import AsyncOpenAI +from openai.types.beta.realtime.session import Session +from openai.resources.beta.realtime.realtime import AsyncRealtimeConnection + + +class SessionDisplay(Static): + """A widget that shows the current session ID.""" + + session_id = reactive("") + + @override + def render(self) -> str: + return f"Session ID: {self.session_id}" if self.session_id else "Connecting..." + + +class AudioStatusIndicator(Static): + """A widget that shows the current audio recording status.""" + + is_recording = reactive(False) + + @override + def render(self) -> str: + status = ( + "🔴 Recording... (Press K to stop)" if self.is_recording else "⚪ Press K to start recording (Q to quit)" + ) + return status + + +class RealtimeApp(App[None]): + CSS = """ + Screen { + background: #1a1b26; /* Dark blue-grey background */ + } + + Container { + border: double rgb(91, 164, 91); + } + + Horizontal { + width: 100%; + } + + #input-container { + height: 5; /* Explicit height for input container */ + margin: 1 1; + padding: 1 2; + } + + Input { + width: 80%; + height: 3; /* Explicit height for input */ + } + + Button { + width: 20%; + height: 3; /* Explicit height for button */ + } + + #bottom-pane { + width: 100%; + height: 82%; /* Reduced to make room for session display */ + border: round rgb(205, 133, 63); + content-align: center middle; + } + + #status-indicator { + height: 3; + content-align: center middle; + background: #2a2b36; + border: solid rgb(91, 164, 91); + margin: 1 1; + } + + #session-display { + height: 3; + content-align: center middle; + background: #2a2b36; + border: solid rgb(91, 164, 91); + margin: 1 1; + } + + Static { + color: white; + } + """ + + client: AsyncOpenAI + should_send_audio: asyncio.Event + audio_player: AudioPlayerAsync + last_audio_item_id: str | None + connection: AsyncRealtimeConnection | None + session: Session | None + connected: asyncio.Event + + def __init__(self) -> None: + super().__init__() + self.connection = None + self.session = None + self.client = AsyncOpenAI() + self.audio_player = AudioPlayerAsync() + self.last_audio_item_id = None + self.should_send_audio = asyncio.Event() + self.connected = asyncio.Event() + + @override + def compose(self) -> ComposeResult: + """Create child widgets for the app.""" + with Container(): + yield SessionDisplay(id="session-display") + yield AudioStatusIndicator(id="status-indicator") + yield RichLog(id="bottom-pane", wrap=True, highlight=True, markup=True) + + async def on_mount(self) -> None: + self.run_worker(self.handle_realtime_connection()) + self.run_worker(self.send_mic_audio()) + + async def handle_realtime_connection(self) -> None: + async with self.client.beta.realtime.connect(model="gpt-4o-realtime-preview-2024-10-01") as conn: + self.connection = conn + self.connected.set() + + # note: this is the default and can be omitted + # if you want to manually handle VAD yourself, then set `'turn_detection': None` + await conn.session.update(session={"turn_detection": {"type": "server_vad"}}) + + acc_items: dict[str, Any] = {} + + async for event in conn: + if event.type == "session.created": + self.session = event.session + session_display = self.query_one(SessionDisplay) + assert event.session.id is not None + session_display.session_id = event.session.id + continue + + if event.type == "session.updated": + self.session = event.session + continue + + if event.type == "response.audio.delta": + if event.item_id != self.last_audio_item_id: + self.audio_player.reset_frame_count() + self.last_audio_item_id = event.item_id + + bytes_data = base64.b64decode(event.delta) + self.audio_player.add_data(bytes_data) + continue + + if event.type == "response.audio_transcript.delta": + try: + text = acc_items[event.item_id] + except KeyError: + acc_items[event.item_id] = event.delta + else: + acc_items[event.item_id] = text + event.delta + + # Clear and update the entire content because RichLog otherwise treats each delta as a new line + bottom_pane = self.query_one("#bottom-pane", RichLog) + bottom_pane.clear() + bottom_pane.write(acc_items[event.item_id]) + continue + + async def _get_connection(self) -> AsyncRealtimeConnection: + await self.connected.wait() + assert self.connection is not None + return self.connection + + async def send_mic_audio(self) -> None: + import sounddevice as sd # type: ignore + + sent_audio = False + + device_info = sd.query_devices() + print(device_info) + + read_size = int(SAMPLE_RATE * 0.02) + + stream = sd.InputStream( + channels=CHANNELS, + samplerate=SAMPLE_RATE, + dtype="int16", + ) + stream.start() + + status_indicator = self.query_one(AudioStatusIndicator) + + try: + while True: + if stream.read_available < read_size: + await asyncio.sleep(0) + continue + + await self.should_send_audio.wait() + status_indicator.is_recording = True + + data, _ = stream.read(read_size) + + connection = await self._get_connection() + if not sent_audio: + asyncio.create_task(connection.send({"type": "response.cancel"})) + sent_audio = True + + await connection.input_audio_buffer.append(audio=base64.b64encode(cast(Any, data)).decode("utf-8")) + + await asyncio.sleep(0) + except KeyboardInterrupt: + pass + finally: + stream.stop() + stream.close() + + async def on_key(self, event: events.Key) -> None: + """Handle key press events.""" + if event.key == "enter": + self.query_one(Button).press() + return + + if event.key == "q": + self.exit() + return + + if event.key == "k": + status_indicator = self.query_one(AudioStatusIndicator) + if status_indicator.is_recording: + self.should_send_audio.clear() + status_indicator.is_recording = False + + if self.session and self.session.turn_detection is None: + # The default in the API is that the model will automatically detect when the user has + # stopped talking and then start responding itself. + # + # However if we're in manual `turn_detection` mode then we need to + # manually tell the model to commit the audio buffer and start responding. + conn = await self._get_connection() + await conn.input_audio_buffer.commit() + await conn.response.create() + else: + self.should_send_audio.set() + status_indicator.is_recording = True + + +if __name__ == "__main__": + app = RealtimeApp() + app.run() diff --git a/mypy.ini b/mypy.ini index 50e5add04b..1ea1fe909d 100644 --- a/mypy.ini +++ b/mypy.ini @@ -8,7 +8,10 @@ show_error_codes = True # # We also exclude our `tests` as mypy doesn't always infer # types correctly and Pyright will still catch any type errors. -exclude = ^(src/openai/_files\.py|src/openai/_utils/_logs\.py|_dev/.*\.py|tests/.*)$ + +# realtime examples use inline `uv` script dependencies +# which means it can't be type checked +exclude = ^(src/openai/_files\.py|_dev/.*\.py|tests/.*|src/openai/_utils/_logs\.py|examples/realtime/audio_util\.py|examples/realtime/push_to_talk_app\.py)$ strict_equality = True implicit_reexport = True diff --git a/pyproject.toml b/pyproject.toml index f83aff6fee..8e78257e67 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -157,6 +157,11 @@ exclude = [ "_dev", ".venv", ".nox", + + # uses inline `uv` script dependencies + # which means it can't be type checked + "examples/realtime/audio_util.py", + "examples/realtime/push_to_talk_app.py" ] reportImplicitOverride = true From 854f13bffefc2511f3bb9c85baf221be57e15575 Mon Sep 17 00:00:00 2001 From: "stainless-app[bot]" <142633134+stainless-app[bot]@users.noreply.github.com> Date: Tue, 17 Dec 2024 18:19:24 +0000 Subject: [PATCH 9/9] release: 1.58.0 --- .release-please-manifest.json | 2 +- CHANGELOG.md | 27 +++++++++++++++++++++++++++ pyproject.toml | 2 +- src/openai/_version.py | 2 +- 4 files changed, 30 insertions(+), 3 deletions(-) diff --git a/.release-please-manifest.json b/.release-please-manifest.json index f9ae229e1a..452fa092bd 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "1.57.4" + ".": "1.58.0" } \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 02b7d0271d..5699c0cec2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,32 @@ # Changelog +## 1.58.0 (2024-12-17) + +Full Changelog: [v1.57.4...v1.58.0](https://github.com/openai/openai-python/compare/v1.57.4...v1.58.0) + +### Features + +* add Realtime API support ([#1958](https://github.com/openai/openai-python/issues/1958)) ([97d73cf](https://github.com/openai/openai-python/commit/97d73cf89935ca6098bb889a92f0ec2cdff16989)) +* **api:** new o1 and GPT-4o models + preference fine-tuning ([#1956](https://github.com/openai/openai-python/issues/1956)) ([ec22ffb](https://github.com/openai/openai-python/commit/ec22ffb129c524525caa33b088405d27c271e631)) + + +### Bug Fixes + +* add reasoning_effort to all methods ([8829c32](https://github.com/openai/openai-python/commit/8829c3202dbe790ca3646476c802ec55ed47d864)) +* **assistants:** correctly send `include` query param ([9a4c69c](https://github.com/openai/openai-python/commit/9a4c69c383bc6719b6521a485f2c7e62a9c036a9)) +* **cli/migrate:** change grit binaries prefix ([#1951](https://github.com/openai/openai-python/issues/1951)) ([1c396c9](https://github.com/openai/openai-python/commit/1c396c95b040fb3d1a2523b09eaad4ff62d96846)) + + +### Chores + +* **internal:** fix some typos ([#1955](https://github.com/openai/openai-python/issues/1955)) ([628dead](https://github.com/openai/openai-python/commit/628dead660c00435bf46e09081c7b90b7bbe4a8a)) + + +### Documentation + +* add examples + guidance on Realtime API support ([1cb00f8](https://github.com/openai/openai-python/commit/1cb00f8fed78052aacbb9e0fac997b6ba0d44d2a)) +* **readme:** example snippet for client context manager ([#1953](https://github.com/openai/openai-python/issues/1953)) ([ad80255](https://github.com/openai/openai-python/commit/ad802551d8aaf4e6eff711118676ec4e64392638)) + ## 1.57.4 (2024-12-13) Full Changelog: [v1.57.3...v1.57.4](https://github.com/openai/openai-python/compare/v1.57.3...v1.57.4) diff --git a/pyproject.toml b/pyproject.toml index 8e78257e67..1f2c9fe6c4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "openai" -version = "1.57.4" +version = "1.58.0" description = "The official Python library for the openai API" dynamic = ["readme"] license = "Apache-2.0" diff --git a/src/openai/_version.py b/src/openai/_version.py index 5b82015017..7f2f4cafb8 100644 --- a/src/openai/_version.py +++ b/src/openai/_version.py @@ -1,4 +1,4 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. __title__ = "openai" -__version__ = "1.57.4" # x-release-please-version +__version__ = "1.58.0" # x-release-please-version