Skip to content

Commit f2a5cfa

Browse files
alizenhomalizenhom
committed
address test comments
1 parent ae9bc2a commit f2a5cfa

File tree

4 files changed

+58
-94
lines changed

4 files changed

+58
-94
lines changed

‎instrumentation/opentelemetry-instrumentation-openai/tests/cassettes/test_chat_completion.yaml

+20-25
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
interactions:
22
- request:
3-
body: '{"messages": [{"role": "user", "content": "Say this is a test three times"}],
4-
"model": "gpt-4", "stream": false}'
3+
body: '{"messages": [{"role": "user", "content": "Say this is a test"}], "model":
4+
"gpt-4", "stream": false}'
55
headers:
66
accept:
77
- application/json
@@ -10,7 +10,7 @@ interactions:
1010
connection:
1111
- keep-alive
1212
content-length:
13-
- '112'
13+
- '100'
1414
content-type:
1515
- application/json
1616
host:
@@ -35,44 +35,39 @@ interactions:
3535
uri: https://api.openai.com/v1/chat/completions
3636
response:
3737
body:
38-
string: !!binary |
39-
H4sIAAAAAAAAAwAAAP//hJBBS8NAEIXv+RXDntuSpLXV3BQPIihSxItI2CbTZO1md81MoFr632W3
40-
MakH8RLI+/Y93rxDBCBUKTIQRS25aJyeXt/cN+r26bH72q9f4v0Dzuu79a78KPQllWLiHXbzjgX/
41-
uGaFbZxGVtaccNGiZPSpySpdpcskThcBNLZE7W2V4+liGi+Tee+orSqQRAavEQDAIXx9N1PiXmQQ
42-
T36UBolkhSIbHgGI1mqvCEmkiKVhMRlhYQ2jCXWfa0WgCCQwEs/gn//zlBa3HUnf3nRa9/pxqKVt
43-
5Vq7oZ4P+lYZRXXeoiRrfAVi60SgxwjgLZzf/bpIuNY2jnO2OzQ+MFmc4sQ49Bm86CFblnrU06u/
44-
THmJLJWmswnFqZ8y1RgQDyXDlYI+ibHJt8pU2LpWhUXDFsfoGwAA//8DAOtohWdIAgAA
38+
string: "{\n \"id\": \"chatcmpl-AC6ajKKHHpvf6x2Qm35t6m3QE8qli\",\n \"object\":
39+
\"chat.completion\",\n \"created\": 1727448637,\n \"model\": \"gpt-4-0613\",\n
40+
\ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\":
41+
\"assistant\",\n \"content\": \"This is a test.\",\n \"refusal\":
42+
null\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n
43+
\ }\n ],\n \"usage\": {\n \"prompt_tokens\": 12,\n \"completion_tokens\":
44+
5,\n \"total_tokens\": 17,\n \"completion_tokens_details\": {\n \"reasoning_tokens\":
45+
0\n }\n },\n \"system_fingerprint\": null\n}\n"
4546
headers:
4647
CF-Cache-Status:
4748
- DYNAMIC
4849
CF-RAY:
49-
- 8c8a6a3c3e5f11ac-MRS
50+
- 8c9c4e9b7fb674d8-PMO
5051
Connection:
5152
- keep-alive
52-
Content-Encoding:
53-
- gzip
5453
Content-Type:
5554
- application/json
5655
Date:
57-
- Wed, 25 Sep 2024 10:43:45 GMT
56+
- Fri, 27 Sep 2024 14:50:37 GMT
5857
Server:
5958
- cloudflare
60-
Set-Cookie:
61-
- __cf_bm=oXnQwjslhtdFXjG89Oureoj0ycU2U3.JD0YOmXVf7Oo-1727261025-1.0.1.1-8zkkMufvVyON_EWorQBeCtOhIav5dyIQ7s5UoEMu2gTW.uaDA3owAxnO_LwCkccXJhC56ryfDhKmS49nV855yA;
62-
path=/; expires=Wed, 25-Sep-24 11:13:45 GMT; domain=.api.openai.com; HttpOnly;
63-
Secure; SameSite=None
64-
- _cfuvid=YSEnhEKHk_5duU.2X6td9ALnCbun1O0M.YvR4OF5DgU-1727261025973-0.0.1.1-604800000;
65-
path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None
59+
Set-Cookie: test_set_cookie
6660
Transfer-Encoding:
6761
- chunked
6862
X-Content-Type-Options:
6963
- nosniff
7064
access-control-expose-headers:
7165
- X-Request-ID
72-
openai-organization:
73-
- scale3-1
66+
content-length:
67+
- '551'
68+
openai-organization: test_organization
7469
openai-processing-ms:
75-
- '1074'
70+
- '434'
7671
openai-version:
7772
- '2020-10-01'
7873
strict-transport-security:
@@ -84,13 +79,13 @@ interactions:
8479
x-ratelimit-remaining-requests:
8580
- '9999'
8681
x-ratelimit-remaining-tokens:
87-
- '999975'
82+
- '999977'
8883
x-ratelimit-reset-requests:
8984
- 6ms
9085
x-ratelimit-reset-tokens:
9186
- 1ms
9287
x-request-id:
93-
- req_c4bc5f6decb9ecc1d59dd8e9435531bf
88+
- req_5f2690abaf909a9f047488694d44495e
9489
status:
9590
code: 200
9691
message: OK

‎instrumentation/opentelemetry-instrumentation-openai/tests/cassettes/test_chat_completion_streaming.yaml

+18-62
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
interactions:
22
- request:
3-
body: '{"messages": [{"role": "user", "content": "Say this is a test three times"}],
4-
"model": "gpt-4", "stream": true, "stream_options": {"include_usage": true}}'
3+
body: '{"messages": [{"role": "user", "content": "Say this is a test"}], "model":
4+
"gpt-4", "stream": true, "stream_options": {"include_usage": true}}'
55
headers:
66
accept:
77
- application/json
@@ -10,7 +10,7 @@ interactions:
1010
connection:
1111
- keep-alive
1212
content-length:
13-
- '154'
13+
- '142'
1414
content-type:
1515
- application/json
1616
host:
@@ -35,69 +35,31 @@ interactions:
3535
uri: https://api.openai.com/v1/chat/completions
3636
response:
3737
body:
38-
string: 'data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null}
38+
string: 'data: {"id":"chatcmpl-AC6akONKCxc8HS63qZ08HyjeTSq6p","object":"chat.completion.chunk","created":1727448638,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null}
3939
4040
41-
data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"This"},"logprobs":null,"finish_reason":null}],"usage":null}
41+
data: {"id":"chatcmpl-AC6akONKCxc8HS63qZ08HyjeTSq6p","object":"chat.completion.chunk","created":1727448638,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"This"},"logprobs":null,"finish_reason":null}],"usage":null}
4242
4343
44-
data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"
44+
data: {"id":"chatcmpl-AC6akONKCxc8HS63qZ08HyjeTSq6p","object":"chat.completion.chunk","created":1727448638,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"
4545
is"},"logprobs":null,"finish_reason":null}],"usage":null}
4646
4747
48-
data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"
48+
data: {"id":"chatcmpl-AC6akONKCxc8HS63qZ08HyjeTSq6p","object":"chat.completion.chunk","created":1727448638,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"
4949
a"},"logprobs":null,"finish_reason":null}],"usage":null}
5050
5151
52-
data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"
52+
data: {"id":"chatcmpl-AC6akONKCxc8HS63qZ08HyjeTSq6p","object":"chat.completion.chunk","created":1727448638,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"
5353
test"},"logprobs":null,"finish_reason":null}],"usage":null}
5454
5555
56-
data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null}
56+
data: {"id":"chatcmpl-AC6akONKCxc8HS63qZ08HyjeTSq6p","object":"chat.completion.chunk","created":1727448638,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null}
5757
5858
59-
data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"
60-
This"},"logprobs":null,"finish_reason":null}],"usage":null}
59+
data: {"id":"chatcmpl-AC6akONKCxc8HS63qZ08HyjeTSq6p","object":"chat.completion.chunk","created":1727448638,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null}
6160
6261
63-
data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"
64-
is"},"logprobs":null,"finish_reason":null}],"usage":null}
65-
66-
67-
data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"
68-
a"},"logprobs":null,"finish_reason":null}],"usage":null}
69-
70-
71-
data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"
72-
test"},"logprobs":null,"finish_reason":null}],"usage":null}
73-
74-
75-
data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null}
76-
77-
78-
data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"
79-
This"},"logprobs":null,"finish_reason":null}],"usage":null}
80-
81-
82-
data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"
83-
is"},"logprobs":null,"finish_reason":null}],"usage":null}
84-
85-
86-
data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"
87-
a"},"logprobs":null,"finish_reason":null}],"usage":null}
88-
89-
90-
data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"
91-
test"},"logprobs":null,"finish_reason":null}],"usage":null}
92-
93-
94-
data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null}
95-
96-
97-
data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null}
98-
99-
100-
data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[],"usage":{"prompt_tokens":14,"completion_tokens":15,"total_tokens":29,"completion_tokens_details":{"reasoning_tokens":0}}}
62+
data: {"id":"chatcmpl-AC6akONKCxc8HS63qZ08HyjeTSq6p","object":"chat.completion.chunk","created":1727448638,"model":"gpt-4-0613","system_fingerprint":null,"choices":[],"usage":{"prompt_tokens":12,"completion_tokens":5,"total_tokens":17,"completion_tokens_details":{"reasoning_tokens":0}}}
10163
10264
10365
data: [DONE]
@@ -108,31 +70,25 @@ interactions:
10870
CF-Cache-Status:
10971
- DYNAMIC
11072
CF-RAY:
111-
- 8c8a8f963be411a4-MRS
73+
- 8c9c4ea489d57948-PMO
11274
Connection:
11375
- keep-alive
11476
Content-Type:
11577
- text/event-stream; charset=utf-8
11678
Date:
117-
- Wed, 25 Sep 2024 11:09:15 GMT
79+
- Fri, 27 Sep 2024 14:50:38 GMT
11880
Server:
11981
- cloudflare
120-
Set-Cookie:
121-
- __cf_bm=hvo156nahnyIUFXHe9iuYr0tn0dKzveWlQN7suEYz9Q-1727262555-1.0.1.1-L1wMbo_r0VTMdA..XHJ_8JDmEIjnuzOW_umwSN1y.LlARvkoK3fluYzgsPa5W1Wd_.Hx0yB__0kriyR1pszyDw;
122-
path=/; expires=Wed, 25-Sep-24 11:39:15 GMT; domain=.api.openai.com; HttpOnly;
123-
Secure; SameSite=None
124-
- _cfuvid=imSGB7bQZURuXPnMTuFDyO6GqwZI2ELFF_y9AyZcOwk-1727262555002-0.0.1.1-604800000;
125-
path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None
82+
Set-Cookie: test_set_cookie
12683
Transfer-Encoding:
12784
- chunked
12885
X-Content-Type-Options:
12986
- nosniff
13087
access-control-expose-headers:
13188
- X-Request-ID
132-
openai-organization:
133-
- scale3-1
89+
openai-organization: test_organization
13490
openai-processing-ms:
135-
- '195'
91+
- '161'
13692
openai-version:
13793
- '2020-10-01'
13894
strict-transport-security:
@@ -144,13 +100,13 @@ interactions:
144100
x-ratelimit-remaining-requests:
145101
- '9999'
146102
x-ratelimit-remaining-tokens:
147-
- '999975'
103+
- '999977'
148104
x-ratelimit-reset-requests:
149105
- 6ms
150106
x-ratelimit-reset-tokens:
151107
- 1ms
152108
x-request-id:
153-
- req_49e07a9a2909db3c1d58e009add4f3ad
109+
- req_3fa9ac9f3693c712e4c377e26d203e58
154110
status:
155111
code: 200
156112
message: OK

‎instrumentation/opentelemetry-instrumentation-openai/tests/conftest.py

+14-1
Original file line numberDiff line numberDiff line change
@@ -44,9 +44,22 @@ def openai_client():
4444

4545
@pytest.fixture(scope="module")
4646
def vcr_config():
47-
return {"filter_headers": ["authorization", "api-key"]}
47+
return {
48+
"filter_headers": ["authorization", "api-key"],
49+
"decode_compressed_response": True,
50+
"before_record_response": scrub_response_headers,
51+
}
4852

4953

5054
@pytest.fixture(scope="session", autouse=True)
5155
def instrument():
5256
OpenAIInstrumentor().instrument()
57+
58+
59+
def scrub_response_headers(response):
60+
"""
61+
This scrubs sensitive response headers. Note they are case-sensitive!
62+
"""
63+
response["headers"]["openai-organization"] = "test_organization"
64+
response["headers"]["Set-Cookie"] = "test_set_cookie"
65+
return response

‎instrumentation/opentelemetry-instrumentation-openai/tests/test_chat_completions.py

+6-6
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,7 @@
1010
@pytest.mark.vcr()
1111
def test_chat_completion(exporter, openai_client):
1212
llm_model_value = "gpt-4"
13-
messages_value = [
14-
{"role": "user", "content": "Say this is a test three times"}
15-
]
13+
messages_value = [{"role": "user", "content": "Say this is a test"}]
1614

1715
kwargs = {
1816
"model": llm_model_value,
@@ -22,6 +20,8 @@ def test_chat_completion(exporter, openai_client):
2220

2321
response = openai_client.chat.completions.create(**kwargs)
2422
spans = exporter.get_finished_spans()
23+
# ignore by default added spans etc: http spans.
24+
# we are only interested in the last span which is the chat completion span
2525
chat_completion_span = spans[-1]
2626
# assert that the span name is correct
2727
assert chat_completion_span.name == f"chat {llm_model_value}"
@@ -81,9 +81,7 @@ def test_chat_completion(exporter, openai_client):
8181
@pytest.mark.vcr()
8282
def test_chat_completion_streaming(exporter, openai_client):
8383
llm_model_value = "gpt-4"
84-
messages_value = [
85-
{"role": "user", "content": "Say this is a test three times"}
86-
]
84+
messages_value = [{"role": "user", "content": "Say this is a test"}]
8785

8886
kwargs = {
8987
"model": llm_model_value,
@@ -108,6 +106,8 @@ def test_chat_completion_streaming(exporter, openai_client):
108106
response_stream_id = chunk.id
109107

110108
spans = exporter.get_finished_spans()
109+
# ignore by default added spans etc: http spans.
110+
# we are only interested in the last span which is the chat completion span
111111
streaming_span = spans[-1]
112112

113113
assert streaming_span.name == f"chat {llm_model_value}"

0 commit comments

Comments
 (0)