From d05f44978e03454b22d3812a60131fe67b5b5674 Mon Sep 17 00:00:00 2001 From: Krista Pratico Date: Mon, 4 Nov 2024 14:19:54 -0800 Subject: [PATCH 1/9] redact API key from debug logs --- src/openai/_base_client.py | 3 ++- src/openai/_utils/__init__.py | 1 + src/openai/_utils/_logs.py | 13 +++++++++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/openai/_base_client.py b/src/openai/_base_client.py index e1d4849ae2..696934cf9e 100644 --- a/src/openai/_base_client.py +++ b/src/openai/_base_client.py @@ -62,7 +62,7 @@ HttpxRequestFiles, ModelBuilderProtocol, ) -from ._utils import is_dict, is_list, asyncify, is_given, lru_cache, is_mapping +from ._utils import APIKeyFilter, is_dict, is_list, asyncify, is_given, lru_cache, is_mapping from ._compat import model_copy, model_dump from ._models import GenericModel, FinalRequestOptions, validate_type, construct_type from ._response import ( @@ -90,6 +90,7 @@ from ._legacy_response import LegacyAPIResponse log: logging.Logger = logging.getLogger(__name__) +log.addFilter(APIKeyFilter()) # TODO: make base page type vars covariant SyncPageT = TypeVar("SyncPageT", bound="BaseSyncPage[Any]") diff --git a/src/openai/_utils/__init__.py b/src/openai/_utils/__init__.py index a7cff3c091..dd3bee5e68 100644 --- a/src/openai/_utils/__init__.py +++ b/src/openai/_utils/__init__.py @@ -1,3 +1,4 @@ +from ._logs import APIKeyFilter as APIKeyFilter from ._sync import asyncify as asyncify from ._proxy import LazyProxy as LazyProxy from ._utils import ( diff --git a/src/openai/_utils/_logs.py b/src/openai/_utils/_logs.py index e5113fd8c0..5240373fae 100644 --- a/src/openai/_utils/_logs.py +++ b/src/openai/_utils/_logs.py @@ -1,5 +1,6 @@ import os import logging +from typing_extensions import override logger: logging.Logger = logging.getLogger("openai") httpx_logger: logging.Logger = logging.getLogger("httpx") @@ -23,3 +24,15 @@ def setup_logging() -> None: _basic_config() logger.setLevel(logging.INFO) httpx_logger.setLevel(logging.INFO) + + +class APIKeyFilter(logging.Filter): + @override + def filter(self, record: logging.LogRecord) -> bool: + if hasattr(record, "args") and isinstance(record.args, dict): + if record.args.get("headers") and isinstance(record.args["headers"], dict): + if "api-key" in record.args["headers"]: + record.args["headers"]["api-key"] = "" + if "Authorization" in record.args["headers"]: + record.args["headers"]["Authorization"] = "" + return True From 7f60f7ecd348d4546293ecaa533756bd51e5f9a7 Mon Sep 17 00:00:00 2001 From: Krista Pratico Date: Mon, 4 Nov 2024 15:17:39 -0800 Subject: [PATCH 2/9] add tests --- tests/test_utils/test_logging.py | 74 ++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 tests/test_utils/test_logging.py diff --git a/tests/test_utils/test_logging.py b/tests/test_utils/test_logging.py new file mode 100644 index 0000000000..472acbe954 --- /dev/null +++ b/tests/test_utils/test_logging.py @@ -0,0 +1,74 @@ +import logging +from typing import Any, Dict, Tuple, cast + +import pytest + +from openai._utils import APIKeyFilter + + +@pytest.fixture +def logger_with_filter() -> logging.Logger: + logger = logging.getLogger("test_logger") + logger.setLevel(logging.DEBUG) + logger.addFilter(APIKeyFilter()) + return logger + + +def test_keys_redacted(logger_with_filter: logging.Logger, caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.DEBUG): + logger_with_filter.debug( + "Request options: %s", + { + "method": "post", + "url": "chat/completions", + "headers": {"api-key": "12345", "Authorization": "Bearer token"} + }, + ) + + log_record = cast(Dict[str, Any], caplog.records[0].args) + assert log_record["method"] == "post" + assert log_record["url"] == "chat/completions" + assert log_record["headers"]["api-key"] == "" + assert log_record["headers"]["Authorization"] == "" + + +def test_no_headers(logger_with_filter: logging.Logger, caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.DEBUG): + logger_with_filter.debug( + "Request options: %s", + { + "method": "post", + "url": "chat/completions" + }, + ) + + log_record = cast(Dict[str, Any], caplog.records[0].args) + assert log_record["method"] == "post" + assert log_record["url"] == "chat/completions" + assert "api-key" not in log_record + assert "Authorization" not in log_record + + +def test_headers_without_sensitive_info(logger_with_filter: logging.Logger, caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.DEBUG): + logger_with_filter.debug( + "Request options: %s", + { + "method": "post", + "url": "chat/completions", + "headers": {"custom": "value"} + } + ) + + log_record = cast(Dict[str, Any], caplog.records[0].args) + assert log_record["method"] == "post" + assert log_record["url"] == "chat/completions" + assert log_record["headers"] == {"custom": "value"} + + +def test_standard_debug_msg(logger_with_filter: logging.Logger, caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.DEBUG): + logger_with_filter.debug( + "Sending HTTP Request: %s %s", "POST", "chat/completions" + ) + assert caplog.messages[0] == "Sending HTTP Request: POST chat/completions" From 670f3f4447cefdbacb07b3b21eb02292b7d4083d Mon Sep 17 00:00:00 2001 From: Krista Pratico Date: Mon, 4 Nov 2024 15:34:17 -0800 Subject: [PATCH 3/9] fix --- src/openai/_utils/_logs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/openai/_utils/_logs.py b/src/openai/_utils/_logs.py index 5240373fae..950006b4bd 100644 --- a/src/openai/_utils/_logs.py +++ b/src/openai/_utils/_logs.py @@ -29,8 +29,8 @@ def setup_logging() -> None: class APIKeyFilter(logging.Filter): @override def filter(self, record: logging.LogRecord) -> bool: - if hasattr(record, "args") and isinstance(record.args, dict): - if record.args.get("headers") and isinstance(record.args["headers"], dict): + if isinstance(record.args, dict) and "headers" in record.args: + if isinstance(record.args["headers"], dict): if "api-key" in record.args["headers"]: record.args["headers"]["api-key"] = "" if "Authorization" in record.args["headers"]: From 69f18c2166a10da131544a64ebd07f00ca6a1849 Mon Sep 17 00:00:00 2001 From: Krista Pratico Date: Tue, 5 Nov 2024 09:14:38 -0800 Subject: [PATCH 4/9] remove unused import --- tests/test_utils/test_logging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_utils/test_logging.py b/tests/test_utils/test_logging.py index 472acbe954..79226eb246 100644 --- a/tests/test_utils/test_logging.py +++ b/tests/test_utils/test_logging.py @@ -1,5 +1,5 @@ import logging -from typing import Any, Dict, Tuple, cast +from typing import Any, Dict, cast import pytest From 4289f8c1d707d44bc755e2ded7ff90ae899f8e3c Mon Sep 17 00:00:00 2001 From: Krista Pratico Date: Tue, 5 Nov 2024 14:29:51 -0800 Subject: [PATCH 5/9] make case insensitive --- src/openai/_utils/_logs.py | 9 +++++---- tests/test_utils/test_logging.py | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/src/openai/_utils/_logs.py b/src/openai/_utils/_logs.py index 950006b4bd..f0c8e28f2b 100644 --- a/src/openai/_utils/_logs.py +++ b/src/openai/_utils/_logs.py @@ -1,5 +1,6 @@ import os import logging +from typing import Any, Dict from typing_extensions import override logger: logging.Logger = logging.getLogger("openai") @@ -31,8 +32,8 @@ class APIKeyFilter(logging.Filter): def filter(self, record: logging.LogRecord) -> bool: if isinstance(record.args, dict) and "headers" in record.args: if isinstance(record.args["headers"], dict): - if "api-key" in record.args["headers"]: - record.args["headers"]["api-key"] = "" - if "Authorization" in record.args["headers"]: - record.args["headers"]["Authorization"] = "" + logged_headers: Dict[str, Any] = record.args["headers"] + for header in logged_headers: + if header.lower() in ["api-key", "authorization"]: + logged_headers[header] = "" return True diff --git a/tests/test_utils/test_logging.py b/tests/test_utils/test_logging.py index 79226eb246..3f5b84efb5 100644 --- a/tests/test_utils/test_logging.py +++ b/tests/test_utils/test_logging.py @@ -32,6 +32,24 @@ def test_keys_redacted(logger_with_filter: logging.Logger, caplog: pytest.LogCap assert log_record["headers"]["Authorization"] == "" +def test_keys_redacted_case_insensitive(logger_with_filter: logging.Logger, caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.DEBUG): + logger_with_filter.debug( + "Request options: %s", + { + "method": "post", + "url": "chat/completions", + "headers": {"Api-key": "12345", "authorization": "Bearer token"} + }, + ) + + log_record = cast(Dict[str, Any], caplog.records[0].args) + assert log_record["method"] == "post" + assert log_record["url"] == "chat/completions" + assert log_record["headers"]["Api-key"] == "" + assert log_record["headers"]["authorization"] == "" + + def test_no_headers(logger_with_filter: logging.Logger, caplog: pytest.LogCaptureFixture) -> None: with caplog.at_level(logging.DEBUG): logger_with_filter.debug( From f3d9c66683708f6bc440d16400701aaf3bfc42da Mon Sep 17 00:00:00 2001 From: Krista Pratico Date: Wed, 6 Nov 2024 11:15:05 -0800 Subject: [PATCH 6/9] review feedback --- src/openai/_base_client.py | 4 +- src/openai/_utils/__init__.py | 2 +- src/openai/_utils/_logs.py | 19 +++--- tests/lib/test_azure.py | 100 +++++++++++++++++++++++++++++++ tests/test_utils/test_logging.py | 34 +++++++---- 5 files changed, 135 insertions(+), 24 deletions(-) diff --git a/src/openai/_base_client.py b/src/openai/_base_client.py index 696934cf9e..187518787a 100644 --- a/src/openai/_base_client.py +++ b/src/openai/_base_client.py @@ -62,7 +62,7 @@ HttpxRequestFiles, ModelBuilderProtocol, ) -from ._utils import APIKeyFilter, is_dict, is_list, asyncify, is_given, lru_cache, is_mapping +from ._utils import SensitiveHeadersFilter, is_dict, is_list, asyncify, is_given, lru_cache, is_mapping from ._compat import model_copy, model_dump from ._models import GenericModel, FinalRequestOptions, validate_type, construct_type from ._response import ( @@ -90,7 +90,7 @@ from ._legacy_response import LegacyAPIResponse log: logging.Logger = logging.getLogger(__name__) -log.addFilter(APIKeyFilter()) +log.addFilter(SensitiveHeadersFilter()) # TODO: make base page type vars covariant SyncPageT = TypeVar("SyncPageT", bound="BaseSyncPage[Any]") diff --git a/src/openai/_utils/__init__.py b/src/openai/_utils/__init__.py index dd3bee5e68..5abb34cde4 100644 --- a/src/openai/_utils/__init__.py +++ b/src/openai/_utils/__init__.py @@ -1,4 +1,4 @@ -from ._logs import APIKeyFilter as APIKeyFilter +from ._logs import SensitiveHeadersFilter as SensitiveHeadersFilter from ._sync import asyncify as asyncify from ._proxy import LazyProxy as LazyProxy from ._utils import ( diff --git a/src/openai/_utils/_logs.py b/src/openai/_utils/_logs.py index f0c8e28f2b..376946933c 100644 --- a/src/openai/_utils/_logs.py +++ b/src/openai/_utils/_logs.py @@ -1,12 +1,16 @@ import os import logging -from typing import Any, Dict from typing_extensions import override +from ._utils import is_dict + logger: logging.Logger = logging.getLogger("openai") httpx_logger: logging.Logger = logging.getLogger("httpx") +SENSITIVE_HEADERS = {"api-key", "authorization"} + + def _basic_config() -> None: # e.g. [2023-10-05 14:12:26 - openai._base_client:818 - DEBUG] HTTP Request: POST http://127.0.0.1:4010/foo/bar "200 OK" logging.basicConfig( @@ -27,13 +31,12 @@ def setup_logging() -> None: httpx_logger.setLevel(logging.INFO) -class APIKeyFilter(logging.Filter): +class SensitiveHeadersFilter(logging.Filter): @override def filter(self, record: logging.LogRecord) -> bool: - if isinstance(record.args, dict) and "headers" in record.args: - if isinstance(record.args["headers"], dict): - logged_headers: Dict[str, Any] = record.args["headers"] - for header in logged_headers: - if header.lower() in ["api-key", "authorization"]: - logged_headers[header] = "" + if is_dict(record.args) and "headers" in record.args and is_dict(record.args["headers"]): + headers = record.args["headers"] = {**record.args["headers"]} + for header in headers: + if str(header).lower() in SENSITIVE_HEADERS: + headers[header] = "" return True diff --git a/tests/lib/test_azure.py b/tests/lib/test_azure.py index a9d3478350..68411ed1a4 100644 --- a/tests/lib/test_azure.py +++ b/tests/lib/test_azure.py @@ -1,3 +1,4 @@ +import logging from typing import Union, cast from typing_extensions import Literal, Protocol @@ -5,6 +6,7 @@ import pytest from respx import MockRouter +from openai._utils import SensitiveHeadersFilter, is_dict from openai._models import FinalRequestOptions from openai.lib.azure import AzureOpenAI, AsyncAzureOpenAI @@ -28,6 +30,14 @@ class MockRequestCall(Protocol): request: httpx.Request +@pytest.fixture +def _logger_with_filter() -> logging.Logger: + logger = logging.getLogger("openai") + logger.setLevel(logging.DEBUG) + logger.addFilter(SensitiveHeadersFilter()) + return logger + + @pytest.mark.parametrize("client", [sync_client, async_client]) def test_implicit_deployment_path(client: Client) -> None: req = client._build_request( @@ -148,3 +158,93 @@ def token_provider() -> str: assert calls[0].request.headers.get("Authorization") == "Bearer first" assert calls[1].request.headers.get("Authorization") == "Bearer second" + + +@pytest.mark.respx() +def test_azure_api_key_redacted(respx_mock: MockRouter, _logger_with_filter: logging.Logger, caplog: pytest.LogCaptureFixture) -> None: + respx_mock.post( + "https://example-resource.azure.openai.com/openai/deployments/gpt-4/chat/completions?api-version=2024-06-01" + ).mock( + return_value=httpx.Response(200, json={"model": "gpt-4"}) + ) + + client = AzureOpenAI( + api_version="2024-06-01", + api_key="example_api_key", + azure_endpoint="https://example-resource.azure.openai.com", + ) + + with caplog.at_level(logging.DEBUG): + client.chat.completions.create(messages=[], model="gpt-4") + + for record in caplog.records: + if is_dict(record.args) and "headers" in record.args and is_dict(record.args["headers"]): + assert record.args["headers"]["api-key"] == "" + + +@pytest.mark.respx() +def test_azure_bearer_token_redacted(respx_mock: MockRouter, _logger_with_filter: logging.Logger, caplog: pytest.LogCaptureFixture) -> None: + respx_mock.post( + "https://example-resource.azure.openai.com/openai/deployments/gpt-4/chat/completions?api-version=2024-06-01" + ).mock( + return_value=httpx.Response(200, json={"model": "gpt-4"}) + ) + + client = AzureOpenAI( + api_version="2024-06-01", + azure_ad_token="example_token", + azure_endpoint="https://example-resource.azure.openai.com", + ) + + with caplog.at_level(logging.DEBUG): + client.chat.completions.create(messages=[], model="gpt-4") + + for record in caplog.records: + if is_dict(record.args) and "headers" in record.args and is_dict(record.args["headers"]): + assert record.args["headers"]["Authorization"] == "" + + +@pytest.mark.asyncio +@pytest.mark.respx() +async def test_azure_api_key_redacted_async(respx_mock: MockRouter, _logger_with_filter: logging.Logger, caplog: pytest.LogCaptureFixture) -> None: + respx_mock.post( + "https://example-resource.azure.openai.com/openai/deployments/gpt-4/chat/completions?api-version=2024-06-01" + ).mock( + return_value=httpx.Response(200, json={"model": "gpt-4"}) + ) + + client = AsyncAzureOpenAI( + api_version="2024-06-01", + api_key="example_api_key", + azure_endpoint="https://example-resource.azure.openai.com", + ) + + with caplog.at_level(logging.DEBUG): + await client.chat.completions.create(messages=[], model="gpt-4") + + for record in caplog.records: + if is_dict(record.args) and "headers" in record.args and is_dict(record.args["headers"]): + assert record.args["headers"]["api-key"] == "" + + +@pytest.mark.asyncio +@pytest.mark.respx() +async def test_azure_bearer_token_redacted_async(respx_mock: MockRouter, _logger_with_filter: logging.Logger, caplog: pytest.LogCaptureFixture) -> None: + respx_mock.post( + "https://example-resource.azure.openai.com/openai/deployments/gpt-4/chat/completions?api-version=2024-06-01" + ).mock( + return_value=httpx.Response(200, json={"model": "gpt-4"}) + ) + + client = AsyncAzureOpenAI( + api_version="2024-06-01", + azure_ad_token="example_token", + azure_endpoint="https://example-resource.azure.openai.com", + ) + + with caplog.at_level(logging.DEBUG): + await client.chat.completions.create(messages=[], model="gpt-4") + + for record in caplog.records: + if is_dict(record.args) and "headers" in record.args and is_dict(record.args["headers"]): + assert record.args["headers"]["Authorization"] == "" diff --git a/tests/test_utils/test_logging.py b/tests/test_utils/test_logging.py index 3f5b84efb5..cc018012e2 100644 --- a/tests/test_utils/test_logging.py +++ b/tests/test_utils/test_logging.py @@ -3,14 +3,14 @@ import pytest -from openai._utils import APIKeyFilter +from openai._utils import SensitiveHeadersFilter @pytest.fixture def logger_with_filter() -> logging.Logger: logger = logging.getLogger("test_logger") logger.setLevel(logging.DEBUG) - logger.addFilter(APIKeyFilter()) + logger.addFilter(SensitiveHeadersFilter()) return logger @@ -21,7 +21,7 @@ def test_keys_redacted(logger_with_filter: logging.Logger, caplog: pytest.LogCap { "method": "post", "url": "chat/completions", - "headers": {"api-key": "12345", "Authorization": "Bearer token"} + "headers": {"api-key": "12345", "Authorization": "Bearer token"}, }, ) @@ -30,6 +30,10 @@ def test_keys_redacted(logger_with_filter: logging.Logger, caplog: pytest.LogCap assert log_record["url"] == "chat/completions" assert log_record["headers"]["api-key"] == "" assert log_record["headers"]["Authorization"] == "" + assert ( + caplog.messages[0] + == "Request options: {'method': 'post', 'url': 'chat/completions', 'headers': {'api-key': '', 'Authorization': ''}}" + ) def test_keys_redacted_case_insensitive(logger_with_filter: logging.Logger, caplog: pytest.LogCaptureFixture) -> None: @@ -39,7 +43,7 @@ def test_keys_redacted_case_insensitive(logger_with_filter: logging.Logger, capl { "method": "post", "url": "chat/completions", - "headers": {"Api-key": "12345", "authorization": "Bearer token"} + "headers": {"Api-key": "12345", "authorization": "Bearer token"}, }, ) @@ -48,16 +52,17 @@ def test_keys_redacted_case_insensitive(logger_with_filter: logging.Logger, capl assert log_record["url"] == "chat/completions" assert log_record["headers"]["Api-key"] == "" assert log_record["headers"]["authorization"] == "" + assert ( + caplog.messages[0] + == "Request options: {'method': 'post', 'url': 'chat/completions', 'headers': {'Api-key': '', 'authorization': ''}}" + ) def test_no_headers(logger_with_filter: logging.Logger, caplog: pytest.LogCaptureFixture) -> None: with caplog.at_level(logging.DEBUG): logger_with_filter.debug( "Request options: %s", - { - "method": "post", - "url": "chat/completions" - }, + {"method": "post", "url": "chat/completions"}, ) log_record = cast(Dict[str, Any], caplog.records[0].args) @@ -65,6 +70,7 @@ def test_no_headers(logger_with_filter: logging.Logger, caplog: pytest.LogCaptur assert log_record["url"] == "chat/completions" assert "api-key" not in log_record assert "Authorization" not in log_record + assert caplog.messages[0] == "Request options: {'method': 'post', 'url': 'chat/completions'}" def test_headers_without_sensitive_info(logger_with_filter: logging.Logger, caplog: pytest.LogCaptureFixture) -> None: @@ -74,19 +80,21 @@ def test_headers_without_sensitive_info(logger_with_filter: logging.Logger, capl { "method": "post", "url": "chat/completions", - "headers": {"custom": "value"} - } + "headers": {"custom": "value"}, + }, ) log_record = cast(Dict[str, Any], caplog.records[0].args) assert log_record["method"] == "post" assert log_record["url"] == "chat/completions" assert log_record["headers"] == {"custom": "value"} + assert ( + caplog.messages[0] + == "Request options: {'method': 'post', 'url': 'chat/completions', 'headers': {'custom': 'value'}}" + ) def test_standard_debug_msg(logger_with_filter: logging.Logger, caplog: pytest.LogCaptureFixture) -> None: with caplog.at_level(logging.DEBUG): - logger_with_filter.debug( - "Sending HTTP Request: %s %s", "POST", "chat/completions" - ) + logger_with_filter.debug("Sending HTTP Request: %s %s", "POST", "chat/completions") assert caplog.messages[0] == "Sending HTTP Request: POST chat/completions" From a2984aff8b5d3ffcdbc0dcbd688ccbd934031426 Mon Sep 17 00:00:00 2001 From: Krista Pratico Date: Wed, 6 Nov 2024 12:07:01 -0800 Subject: [PATCH 7/9] fix lint --- tests/lib/test_azure.py | 341 ++++++++++++++++++++-------------------- 1 file changed, 171 insertions(+), 170 deletions(-) diff --git a/tests/lib/test_azure.py b/tests/lib/test_azure.py index 68411ed1a4..ac068b4203 100644 --- a/tests/lib/test_azure.py +++ b/tests/lib/test_azure.py @@ -30,221 +30,222 @@ class MockRequestCall(Protocol): request: httpx.Request -@pytest.fixture -def _logger_with_filter() -> logging.Logger: - logger = logging.getLogger("openai") - logger.setLevel(logging.DEBUG) - logger.addFilter(SensitiveHeadersFilter()) - return logger - - -@pytest.mark.parametrize("client", [sync_client, async_client]) -def test_implicit_deployment_path(client: Client) -> None: - req = client._build_request( - FinalRequestOptions.construct( - method="post", - url="/chat/completions", - json_data={"model": "my-deployment-model"}, +class TestAzure: + + @pytest.fixture(autouse=True) + def _logger_with_filter(self) -> logging.Logger: + logger = logging.getLogger("openai") + logger.setLevel(logging.DEBUG) + logger.addFilter(SensitiveHeadersFilter()) + return logger + + @pytest.mark.parametrize("client", [sync_client, async_client]) + def test_implicit_deployment_path(self, client: Client) -> None: + req = client._build_request( + FinalRequestOptions.construct( + method="post", + url="/chat/completions", + json_data={"model": "my-deployment-model"}, + ) + ) + assert ( + req.url + == "https://example-resource.azure.openai.com/openai/deployments/my-deployment-model/chat/completions?api-version=2023-07-01" ) - ) - assert ( - req.url - == "https://example-resource.azure.openai.com/openai/deployments/my-deployment-model/chat/completions?api-version=2023-07-01" - ) -@pytest.mark.parametrize( - "client,method", - [ - (sync_client, "copy"), - (sync_client, "with_options"), - (async_client, "copy"), - (async_client, "with_options"), - ], -) -def test_client_copying(client: Client, method: Literal["copy", "with_options"]) -> None: - if method == "copy": - copied = client.copy() - else: - copied = client.with_options() + @pytest.mark.parametrize( + "client,method", + [ + (sync_client, "copy"), + (sync_client, "with_options"), + (async_client, "copy"), + (async_client, "with_options"), + ], + ) + def test_client_copying(self, client: Client, method: Literal["copy", "with_options"]) -> None: + if method == "copy": + copied = client.copy() + else: + copied = client.with_options() - assert copied._custom_query == {"api-version": "2023-07-01"} + assert copied._custom_query == {"api-version": "2023-07-01"} -@pytest.mark.parametrize( - "client", - [sync_client, async_client], -) -def test_client_copying_override_options(client: Client) -> None: - copied = client.copy( - api_version="2022-05-01", - ) - assert copied._custom_query == {"api-version": "2022-05-01"} - - -@pytest.mark.respx() -def test_client_token_provider_refresh_sync(respx_mock: MockRouter) -> None: - respx_mock.post( - "https://example-resource.azure.openai.com/openai/deployments/gpt-4/chat/completions?api-version=2024-02-01" - ).mock( - side_effect=[ - httpx.Response(500, json={"error": "server error"}), - httpx.Response(200, json={"foo": "bar"}), - ] + @pytest.mark.parametrize( + "client", + [sync_client, async_client], ) + def test_client_copying_override_options(self, client: Client) -> None: + copied = client.copy( + api_version="2022-05-01", + ) + assert copied._custom_query == {"api-version": "2022-05-01"} + + + @pytest.mark.respx() + def test_client_token_provider_refresh_sync(self, respx_mock: MockRouter) -> None: + respx_mock.post( + "https://example-resource.azure.openai.com/openai/deployments/gpt-4/chat/completions?api-version=2024-02-01" + ).mock( + side_effect=[ + httpx.Response(500, json={"error": "server error"}), + httpx.Response(200, json={"foo": "bar"}), + ] + ) - counter = 0 + counter = 0 - def token_provider() -> str: - nonlocal counter + def token_provider() -> str: + nonlocal counter - counter += 1 + counter += 1 - if counter == 1: - return "first" + if counter == 1: + return "first" - return "second" + return "second" - client = AzureOpenAI( - api_version="2024-02-01", - azure_ad_token_provider=token_provider, - azure_endpoint="https://example-resource.azure.openai.com", - ) - client.chat.completions.create(messages=[], model="gpt-4") + client = AzureOpenAI( + api_version="2024-02-01", + azure_ad_token_provider=token_provider, + azure_endpoint="https://example-resource.azure.openai.com", + ) + client.chat.completions.create(messages=[], model="gpt-4") - calls = cast("list[MockRequestCall]", respx_mock.calls) + calls = cast("list[MockRequestCall]", respx_mock.calls) - assert len(calls) == 2 + assert len(calls) == 2 - assert calls[0].request.headers.get("Authorization") == "Bearer first" - assert calls[1].request.headers.get("Authorization") == "Bearer second" + assert calls[0].request.headers.get("Authorization") == "Bearer first" + assert calls[1].request.headers.get("Authorization") == "Bearer second" -@pytest.mark.asyncio -@pytest.mark.respx() -async def test_client_token_provider_refresh_async(respx_mock: MockRouter) -> None: - respx_mock.post( - "https://example-resource.azure.openai.com/openai/deployments/gpt-4/chat/completions?api-version=2024-02-01" - ).mock( - side_effect=[ - httpx.Response(500, json={"error": "server error"}), - httpx.Response(200, json={"foo": "bar"}), - ] - ) + @pytest.mark.asyncio + @pytest.mark.respx() + async def test_client_token_provider_refresh_async(self, respx_mock: MockRouter) -> None: + respx_mock.post( + "https://example-resource.azure.openai.com/openai/deployments/gpt-4/chat/completions?api-version=2024-02-01" + ).mock( + side_effect=[ + httpx.Response(500, json={"error": "server error"}), + httpx.Response(200, json={"foo": "bar"}), + ] + ) - counter = 0 + counter = 0 - def token_provider() -> str: - nonlocal counter + def token_provider() -> str: + nonlocal counter - counter += 1 + counter += 1 - if counter == 1: - return "first" + if counter == 1: + return "first" - return "second" + return "second" - client = AsyncAzureOpenAI( - api_version="2024-02-01", - azure_ad_token_provider=token_provider, - azure_endpoint="https://example-resource.azure.openai.com", - ) + client = AsyncAzureOpenAI( + api_version="2024-02-01", + azure_ad_token_provider=token_provider, + azure_endpoint="https://example-resource.azure.openai.com", + ) - await client.chat.completions.create(messages=[], model="gpt-4") + await client.chat.completions.create(messages=[], model="gpt-4") - calls = cast("list[MockRequestCall]", respx_mock.calls) + calls = cast("list[MockRequestCall]", respx_mock.calls) - assert len(calls) == 2 + assert len(calls) == 2 - assert calls[0].request.headers.get("Authorization") == "Bearer first" - assert calls[1].request.headers.get("Authorization") == "Bearer second" + assert calls[0].request.headers.get("Authorization") == "Bearer first" + assert calls[1].request.headers.get("Authorization") == "Bearer second" -@pytest.mark.respx() -def test_azure_api_key_redacted(respx_mock: MockRouter, _logger_with_filter: logging.Logger, caplog: pytest.LogCaptureFixture) -> None: - respx_mock.post( - "https://example-resource.azure.openai.com/openai/deployments/gpt-4/chat/completions?api-version=2024-06-01" - ).mock( - return_value=httpx.Response(200, json={"model": "gpt-4"}) - ) + @pytest.mark.respx() + def test_azure_api_key_redacted(self, respx_mock: MockRouter, _logger_with_filter: logging.Logger, caplog: pytest.LogCaptureFixture) -> None: + respx_mock.post( + "https://example-resource.azure.openai.com/openai/deployments/gpt-4/chat/completions?api-version=2024-06-01" + ).mock( + return_value=httpx.Response(200, json={"model": "gpt-4"}) + ) - client = AzureOpenAI( - api_version="2024-06-01", - api_key="example_api_key", - azure_endpoint="https://example-resource.azure.openai.com", - ) + client = AzureOpenAI( + api_version="2024-06-01", + api_key="example_api_key", + azure_endpoint="https://example-resource.azure.openai.com", + ) - with caplog.at_level(logging.DEBUG): - client.chat.completions.create(messages=[], model="gpt-4") + with caplog.at_level(logging.DEBUG): + client.chat.completions.create(messages=[], model="gpt-4") - for record in caplog.records: - if is_dict(record.args) and "headers" in record.args and is_dict(record.args["headers"]): - assert record.args["headers"]["api-key"] == "" + for record in caplog.records: + if is_dict(record.args) and "headers" in record.args and is_dict(record.args["headers"]): + assert record.args["headers"]["api-key"] == "" -@pytest.mark.respx() -def test_azure_bearer_token_redacted(respx_mock: MockRouter, _logger_with_filter: logging.Logger, caplog: pytest.LogCaptureFixture) -> None: - respx_mock.post( - "https://example-resource.azure.openai.com/openai/deployments/gpt-4/chat/completions?api-version=2024-06-01" - ).mock( - return_value=httpx.Response(200, json={"model": "gpt-4"}) - ) + @pytest.mark.respx() + def test_azure_bearer_token_redacted(self, respx_mock: MockRouter, _logger_with_filter: logging.Logger, caplog: pytest.LogCaptureFixture) -> None: + respx_mock.post( + "https://example-resource.azure.openai.com/openai/deployments/gpt-4/chat/completions?api-version=2024-06-01" + ).mock( + return_value=httpx.Response(200, json={"model": "gpt-4"}) + ) - client = AzureOpenAI( - api_version="2024-06-01", - azure_ad_token="example_token", - azure_endpoint="https://example-resource.azure.openai.com", - ) + client = AzureOpenAI( + api_version="2024-06-01", + azure_ad_token="example_token", + azure_endpoint="https://example-resource.azure.openai.com", + ) - with caplog.at_level(logging.DEBUG): - client.chat.completions.create(messages=[], model="gpt-4") + with caplog.at_level(logging.DEBUG): + client.chat.completions.create(messages=[], model="gpt-4") - for record in caplog.records: - if is_dict(record.args) and "headers" in record.args and is_dict(record.args["headers"]): - assert record.args["headers"]["Authorization"] == "" + for record in caplog.records: + if is_dict(record.args) and "headers" in record.args and is_dict(record.args["headers"]): + assert record.args["headers"]["Authorization"] == "" -@pytest.mark.asyncio -@pytest.mark.respx() -async def test_azure_api_key_redacted_async(respx_mock: MockRouter, _logger_with_filter: logging.Logger, caplog: pytest.LogCaptureFixture) -> None: - respx_mock.post( - "https://example-resource.azure.openai.com/openai/deployments/gpt-4/chat/completions?api-version=2024-06-01" - ).mock( - return_value=httpx.Response(200, json={"model": "gpt-4"}) - ) + @pytest.mark.asyncio + @pytest.mark.respx() + async def test_azure_api_key_redacted_async(self, respx_mock: MockRouter, _logger_with_filter: logging.Logger, caplog: pytest.LogCaptureFixture) -> None: + respx_mock.post( + "https://example-resource.azure.openai.com/openai/deployments/gpt-4/chat/completions?api-version=2024-06-01" + ).mock( + return_value=httpx.Response(200, json={"model": "gpt-4"}) + ) - client = AsyncAzureOpenAI( - api_version="2024-06-01", - api_key="example_api_key", - azure_endpoint="https://example-resource.azure.openai.com", - ) + client = AsyncAzureOpenAI( + api_version="2024-06-01", + api_key="example_api_key", + azure_endpoint="https://example-resource.azure.openai.com", + ) - with caplog.at_level(logging.DEBUG): - await client.chat.completions.create(messages=[], model="gpt-4") + with caplog.at_level(logging.DEBUG): + await client.chat.completions.create(messages=[], model="gpt-4") - for record in caplog.records: - if is_dict(record.args) and "headers" in record.args and is_dict(record.args["headers"]): - assert record.args["headers"]["api-key"] == "" + for record in caplog.records: + if is_dict(record.args) and "headers" in record.args and is_dict(record.args["headers"]): + assert record.args["headers"]["api-key"] == "" -@pytest.mark.asyncio -@pytest.mark.respx() -async def test_azure_bearer_token_redacted_async(respx_mock: MockRouter, _logger_with_filter: logging.Logger, caplog: pytest.LogCaptureFixture) -> None: - respx_mock.post( - "https://example-resource.azure.openai.com/openai/deployments/gpt-4/chat/completions?api-version=2024-06-01" - ).mock( - return_value=httpx.Response(200, json={"model": "gpt-4"}) - ) + @pytest.mark.asyncio + @pytest.mark.respx() + async def test_azure_bearer_token_redacted_async(self, respx_mock: MockRouter, _logger_with_filter: logging.Logger, caplog: pytest.LogCaptureFixture) -> None: + respx_mock.post( + "https://example-resource.azure.openai.com/openai/deployments/gpt-4/chat/completions?api-version=2024-06-01" + ).mock( + return_value=httpx.Response(200, json={"model": "gpt-4"}) + ) - client = AsyncAzureOpenAI( - api_version="2024-06-01", - azure_ad_token="example_token", - azure_endpoint="https://example-resource.azure.openai.com", - ) + client = AsyncAzureOpenAI( + api_version="2024-06-01", + azure_ad_token="example_token", + azure_endpoint="https://example-resource.azure.openai.com", + ) - with caplog.at_level(logging.DEBUG): - await client.chat.completions.create(messages=[], model="gpt-4") + with caplog.at_level(logging.DEBUG): + await client.chat.completions.create(messages=[], model="gpt-4") - for record in caplog.records: - if is_dict(record.args) and "headers" in record.args and is_dict(record.args["headers"]): - assert record.args["headers"]["Authorization"] == "" + for record in caplog.records: + if is_dict(record.args) and "headers" in record.args and is_dict(record.args["headers"]): + assert record.args["headers"]["Authorization"] == "" From a137655823f3853eb5ba85e52e29f1f26418f26e Mon Sep 17 00:00:00 2001 From: Krista Pratico Date: Wed, 6 Nov 2024 12:29:07 -0800 Subject: [PATCH 8/9] mypy ignore _logs.py --- mypy.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy.ini b/mypy.ini index a4517a002d..97e5de4a60 100644 --- a/mypy.ini +++ b/mypy.ini @@ -2,10 +2,10 @@ pretty = True show_error_codes = True -# Exclude _files.py because mypy isn't smart enough to apply +# Exclude _files.py and _logs.py because mypy isn't smart enough to apply # the correct type narrowing and as this is an internal module # it's fine to just use Pyright. -exclude = ^(src/openai/_files\.py|_dev/.*\.py)$ +exclude = ^(src/openai/_files\.py|src/openai/_utils/_logs\.py|_dev/.*\.py)$ strict_equality = True implicit_reexport = True From 6d5a26622271d8c18edc3acb5520dc4ff17f4361 Mon Sep 17 00:00:00 2001 From: Krista Pratico Date: Wed, 6 Nov 2024 12:52:48 -0800 Subject: [PATCH 9/9] apply test feedback + lint --- tests/lib/test_azure.py | 214 ++++++++++++++++++++-------------------- 1 file changed, 107 insertions(+), 107 deletions(-) diff --git a/tests/lib/test_azure.py b/tests/lib/test_azure.py index ac068b4203..626d7df311 100644 --- a/tests/lib/test_azure.py +++ b/tests/lib/test_azure.py @@ -30,139 +30,139 @@ class MockRequestCall(Protocol): request: httpx.Request -class TestAzure: - - @pytest.fixture(autouse=True) - def _logger_with_filter(self) -> logging.Logger: - logger = logging.getLogger("openai") - logger.setLevel(logging.DEBUG) - logger.addFilter(SensitiveHeadersFilter()) - return logger - - @pytest.mark.parametrize("client", [sync_client, async_client]) - def test_implicit_deployment_path(self, client: Client) -> None: - req = client._build_request( - FinalRequestOptions.construct( - method="post", - url="/chat/completions", - json_data={"model": "my-deployment-model"}, - ) - ) - assert ( - req.url - == "https://example-resource.azure.openai.com/openai/deployments/my-deployment-model/chat/completions?api-version=2023-07-01" +@pytest.mark.parametrize("client", [sync_client, async_client]) +def test_implicit_deployment_path(client: Client) -> None: + req = client._build_request( + FinalRequestOptions.construct( + method="post", + url="/chat/completions", + json_data={"model": "my-deployment-model"}, ) + ) + assert ( + req.url + == "https://example-resource.azure.openai.com/openai/deployments/my-deployment-model/chat/completions?api-version=2023-07-01" + ) - @pytest.mark.parametrize( - "client,method", - [ - (sync_client, "copy"), - (sync_client, "with_options"), - (async_client, "copy"), - (async_client, "with_options"), - ], - ) - def test_client_copying(self, client: Client, method: Literal["copy", "with_options"]) -> None: - if method == "copy": - copied = client.copy() - else: - copied = client.with_options() +@pytest.mark.parametrize( + "client,method", + [ + (sync_client, "copy"), + (sync_client, "with_options"), + (async_client, "copy"), + (async_client, "with_options"), + ], +) +def test_client_copying(client: Client, method: Literal["copy", "with_options"]) -> None: + if method == "copy": + copied = client.copy() + else: + copied = client.with_options() - assert copied._custom_query == {"api-version": "2023-07-01"} + assert copied._custom_query == {"api-version": "2023-07-01"} - @pytest.mark.parametrize( - "client", - [sync_client, async_client], +@pytest.mark.parametrize( + "client", + [sync_client, async_client], +) +def test_client_copying_override_options(client: Client) -> None: + copied = client.copy( + api_version="2022-05-01", + ) + assert copied._custom_query == {"api-version": "2022-05-01"} + + +@pytest.mark.respx() +def test_client_token_provider_refresh_sync(respx_mock: MockRouter) -> None: + respx_mock.post( + "https://example-resource.azure.openai.com/openai/deployments/gpt-4/chat/completions?api-version=2024-02-01" + ).mock( + side_effect=[ + httpx.Response(500, json={"error": "server error"}), + httpx.Response(200, json={"foo": "bar"}), + ] ) - def test_client_copying_override_options(self, client: Client) -> None: - copied = client.copy( - api_version="2022-05-01", - ) - assert copied._custom_query == {"api-version": "2022-05-01"} + counter = 0 - @pytest.mark.respx() - def test_client_token_provider_refresh_sync(self, respx_mock: MockRouter) -> None: - respx_mock.post( - "https://example-resource.azure.openai.com/openai/deployments/gpt-4/chat/completions?api-version=2024-02-01" - ).mock( - side_effect=[ - httpx.Response(500, json={"error": "server error"}), - httpx.Response(200, json={"foo": "bar"}), - ] - ) + def token_provider() -> str: + nonlocal counter - counter = 0 + counter += 1 - def token_provider() -> str: - nonlocal counter + if counter == 1: + return "first" - counter += 1 + return "second" - if counter == 1: - return "first" + client = AzureOpenAI( + api_version="2024-02-01", + azure_ad_token_provider=token_provider, + azure_endpoint="https://example-resource.azure.openai.com", + ) + client.chat.completions.create(messages=[], model="gpt-4") - return "second" + calls = cast("list[MockRequestCall]", respx_mock.calls) - client = AzureOpenAI( - api_version="2024-02-01", - azure_ad_token_provider=token_provider, - azure_endpoint="https://example-resource.azure.openai.com", - ) - client.chat.completions.create(messages=[], model="gpt-4") + assert len(calls) == 2 - calls = cast("list[MockRequestCall]", respx_mock.calls) + assert calls[0].request.headers.get("Authorization") == "Bearer first" + assert calls[1].request.headers.get("Authorization") == "Bearer second" - assert len(calls) == 2 - assert calls[0].request.headers.get("Authorization") == "Bearer first" - assert calls[1].request.headers.get("Authorization") == "Bearer second" +@pytest.mark.asyncio +@pytest.mark.respx() +async def test_client_token_provider_refresh_async(respx_mock: MockRouter) -> None: + respx_mock.post( + "https://example-resource.azure.openai.com/openai/deployments/gpt-4/chat/completions?api-version=2024-02-01" + ).mock( + side_effect=[ + httpx.Response(500, json={"error": "server error"}), + httpx.Response(200, json={"foo": "bar"}), + ] + ) + counter = 0 - @pytest.mark.asyncio - @pytest.mark.respx() - async def test_client_token_provider_refresh_async(self, respx_mock: MockRouter) -> None: - respx_mock.post( - "https://example-resource.azure.openai.com/openai/deployments/gpt-4/chat/completions?api-version=2024-02-01" - ).mock( - side_effect=[ - httpx.Response(500, json={"error": "server error"}), - httpx.Response(200, json={"foo": "bar"}), - ] - ) + def token_provider() -> str: + nonlocal counter - counter = 0 + counter += 1 - def token_provider() -> str: - nonlocal counter + if counter == 1: + return "first" - counter += 1 + return "second" - if counter == 1: - return "first" + client = AsyncAzureOpenAI( + api_version="2024-02-01", + azure_ad_token_provider=token_provider, + azure_endpoint="https://example-resource.azure.openai.com", + ) - return "second" + await client.chat.completions.create(messages=[], model="gpt-4") - client = AsyncAzureOpenAI( - api_version="2024-02-01", - azure_ad_token_provider=token_provider, - azure_endpoint="https://example-resource.azure.openai.com", - ) + calls = cast("list[MockRequestCall]", respx_mock.calls) - await client.chat.completions.create(messages=[], model="gpt-4") + assert len(calls) == 2 - calls = cast("list[MockRequestCall]", respx_mock.calls) + assert calls[0].request.headers.get("Authorization") == "Bearer first" + assert calls[1].request.headers.get("Authorization") == "Bearer second" - assert len(calls) == 2 - assert calls[0].request.headers.get("Authorization") == "Bearer first" - assert calls[1].request.headers.get("Authorization") == "Bearer second" +class TestAzureLogging: + @pytest.fixture(autouse=True) + def logger_with_filter(self) -> logging.Logger: + logger = logging.getLogger("openai") + logger.setLevel(logging.DEBUG) + logger.addFilter(SensitiveHeadersFilter()) + return logger @pytest.mark.respx() - def test_azure_api_key_redacted(self, respx_mock: MockRouter, _logger_with_filter: logging.Logger, caplog: pytest.LogCaptureFixture) -> None: + def test_azure_api_key_redacted(self, respx_mock: MockRouter, caplog: pytest.LogCaptureFixture) -> None: respx_mock.post( "https://example-resource.azure.openai.com/openai/deployments/gpt-4/chat/completions?api-version=2024-06-01" ).mock( @@ -179,12 +179,12 @@ def test_azure_api_key_redacted(self, respx_mock: MockRouter, _logger_with_filte client.chat.completions.create(messages=[], model="gpt-4") for record in caplog.records: - if is_dict(record.args) and "headers" in record.args and is_dict(record.args["headers"]): + if is_dict(record.args) and record.args.get("headers") and is_dict(record.args["headers"]): assert record.args["headers"]["api-key"] == "" @pytest.mark.respx() - def test_azure_bearer_token_redacted(self, respx_mock: MockRouter, _logger_with_filter: logging.Logger, caplog: pytest.LogCaptureFixture) -> None: + def test_azure_bearer_token_redacted(self, respx_mock: MockRouter, caplog: pytest.LogCaptureFixture) -> None: respx_mock.post( "https://example-resource.azure.openai.com/openai/deployments/gpt-4/chat/completions?api-version=2024-06-01" ).mock( @@ -201,13 +201,13 @@ def test_azure_bearer_token_redacted(self, respx_mock: MockRouter, _logger_with_ client.chat.completions.create(messages=[], model="gpt-4") for record in caplog.records: - if is_dict(record.args) and "headers" in record.args and is_dict(record.args["headers"]): + if is_dict(record.args) and record.args.get("headers") and is_dict(record.args["headers"]): assert record.args["headers"]["Authorization"] == "" @pytest.mark.asyncio @pytest.mark.respx() - async def test_azure_api_key_redacted_async(self, respx_mock: MockRouter, _logger_with_filter: logging.Logger, caplog: pytest.LogCaptureFixture) -> None: + async def test_azure_api_key_redacted_async(self, respx_mock: MockRouter, caplog: pytest.LogCaptureFixture) -> None: respx_mock.post( "https://example-resource.azure.openai.com/openai/deployments/gpt-4/chat/completions?api-version=2024-06-01" ).mock( @@ -224,13 +224,13 @@ async def test_azure_api_key_redacted_async(self, respx_mock: MockRouter, _logge await client.chat.completions.create(messages=[], model="gpt-4") for record in caplog.records: - if is_dict(record.args) and "headers" in record.args and is_dict(record.args["headers"]): + if is_dict(record.args) and record.args.get("headers") and is_dict(record.args["headers"]): assert record.args["headers"]["api-key"] == "" @pytest.mark.asyncio @pytest.mark.respx() - async def test_azure_bearer_token_redacted_async(self, respx_mock: MockRouter, _logger_with_filter: logging.Logger, caplog: pytest.LogCaptureFixture) -> None: + async def test_azure_bearer_token_redacted_async(self, respx_mock: MockRouter, caplog: pytest.LogCaptureFixture) -> None: respx_mock.post( "https://example-resource.azure.openai.com/openai/deployments/gpt-4/chat/completions?api-version=2024-06-01" ).mock( @@ -247,5 +247,5 @@ async def test_azure_bearer_token_redacted_async(self, respx_mock: MockRouter, _ await client.chat.completions.create(messages=[], model="gpt-4") for record in caplog.records: - if is_dict(record.args) and "headers" in record.args and is_dict(record.args["headers"]): + if is_dict(record.args) and record.args.get("headers") and is_dict(record.args["headers"]): assert record.args["headers"]["Authorization"] == ""