Skip to content

Commit 748c925

Browse files
aabmassemdneto
aabmass
and
emdneto
authored
VertexAI emit user, system, and assistant events (#3203)
* VertexAI emit user events * Emit system and assistant events * Fix for python 3.8 * Record events regardless of span recording * fix tests * Apply suggestions from code review Co-authored-by: Emídio Neto <[email protected]> --------- Co-authored-by: Emídio Neto <[email protected]>
1 parent 7af1918 commit 748c925

File tree

8 files changed

+534
-11
lines changed

8 files changed

+534
-11
lines changed

‎instrumentation-genai/opentelemetry-instrumentation-vertexai/CHANGELOG.md

+2
Original file line numberDiff line numberDiff line change
@@ -13,3 +13,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
1313
([#3123](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3123))
1414
- Add server attributes to Vertex AI spans
1515
([#3208](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3208))
16+
- VertexAI emit user, system, and assistant events
17+
([#3203](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3203))

‎instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/events.py

+91
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
# Copyright The OpenTelemetry Authors
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"""
16+
Factories for event types described in
17+
https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai/gen-ai-events.md#system-event.
18+
19+
Hopefully this code can be autogenerated by Weaver once Gen AI semantic conventions are
20+
schematized in YAML and the Weaver tool supports it.
21+
"""
22+
23+
from opentelemetry._events import Event
24+
from opentelemetry.semconv._incubating.attributes import gen_ai_attributes
25+
from opentelemetry.util.types import AnyValue
26+
27+
28+
def user_event(
29+
*,
30+
role: str = "user",
31+
content: AnyValue = None,
32+
) -> Event:
33+
"""Creates a User event
34+
https://github.com/open-telemetry/semantic-conventions/blob/v1.28.0/docs/gen-ai/gen-ai-events.md#user-event
35+
"""
36+
body: dict[str, AnyValue] = {
37+
"role": role,
38+
}
39+
if content is not None:
40+
body["content"] = content
41+
return Event(
42+
name="gen_ai.user.message",
43+
attributes={
44+
gen_ai_attributes.GEN_AI_SYSTEM: gen_ai_attributes.GenAiSystemValues.VERTEX_AI.value,
45+
},
46+
body=body,
47+
)
48+
49+
50+
def assistant_event(
51+
*,
52+
role: str = "assistant",
53+
content: AnyValue = None,
54+
) -> Event:
55+
"""Creates an Assistant event
56+
https://github.com/open-telemetry/semantic-conventions/blob/v1.28.0/docs/gen-ai/gen-ai-events.md#assistant-event
57+
"""
58+
body: dict[str, AnyValue] = {
59+
"role": role,
60+
}
61+
if content is not None:
62+
body["content"] = content
63+
return Event(
64+
name="gen_ai.assistant.message",
65+
attributes={
66+
gen_ai_attributes.GEN_AI_SYSTEM: gen_ai_attributes.GenAiSystemValues.VERTEX_AI.value,
67+
},
68+
body=body,
69+
)
70+
71+
72+
def system_event(
73+
*,
74+
role: str = "system",
75+
content: AnyValue = None,
76+
) -> Event:
77+
"""Creates a System event
78+
https://github.com/open-telemetry/semantic-conventions/blob/v1.28.0/docs/gen-ai/gen-ai-events.md#system-event
79+
"""
80+
body: dict[str, AnyValue] = {
81+
"role": role,
82+
}
83+
if content is not None:
84+
body["content"] = content
85+
return Event(
86+
name="gen_ai.system.message",
87+
attributes={
88+
gen_ai_attributes.GEN_AI_SYSTEM: gen_ai_attributes.GenAiSystemValues.VERTEX_AI.value,
89+
},
90+
body=body,
91+
)

‎instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/patch.py

+8-10
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
get_genai_request_attributes,
2828
get_server_attributes,
2929
get_span_name,
30+
request_to_events,
3031
)
3132
from opentelemetry.trace import SpanKind, Tracer
3233

@@ -113,12 +114,10 @@ def traced_method(
113114
kind=SpanKind.CLIENT,
114115
attributes=span_attributes,
115116
) as _span:
116-
# TODO: emit request events
117-
# if span.is_recording():
118-
# for message in kwargs.get("messages", []):
119-
# event_logger.emit(
120-
# message_to_event(message, capture_content)
121-
# )
117+
for event in request_to_events(
118+
params=params, capture_content=capture_content
119+
):
120+
event_logger.emit(event)
122121

123122
# TODO: set error.type attribute
124123
# https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai/gen-ai-spans.md
@@ -130,10 +129,9 @@ def traced_method(
130129
# )
131130

132131
# TODO: add response attributes and events
133-
# if span.is_recording():
134-
# _set_response_attributes(
135-
# span, result, event_logger, capture_content
136-
# )
132+
# _set_response_attributes(
133+
# span, result, event_logger, capture_content
134+
# )
137135
return result
138136

139137
return traced_method

‎instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/utils.py

+52-1
Original file line numberDiff line numberDiff line change
@@ -19,16 +19,24 @@
1919
from os import environ
2020
from typing import (
2121
TYPE_CHECKING,
22+
Iterable,
2223
Mapping,
2324
Sequence,
25+
cast,
2426
)
2527
from urllib.parse import urlparse
2628

29+
from opentelemetry._events import Event
30+
from opentelemetry.instrumentation.vertexai.events import (
31+
assistant_event,
32+
system_event,
33+
user_event,
34+
)
2735
from opentelemetry.semconv._incubating.attributes import (
2836
gen_ai_attributes as GenAIAttributes,
2937
)
3038
from opentelemetry.semconv.attributes import server_attributes
31-
from opentelemetry.util.types import AttributeValue
39+
from opentelemetry.util.types import AnyValue, AttributeValue
3240

3341
if TYPE_CHECKING:
3442
from google.cloud.aiplatform_v1.types import content, tool
@@ -157,3 +165,46 @@ def get_span_name(span_attributes: Mapping[str, AttributeValue]) -> str:
157165
if not model:
158166
return f"{name}"
159167
return f"{name} {model}"
168+
169+
170+
def request_to_events(
171+
*, params: GenerateContentParams, capture_content: bool
172+
) -> Iterable[Event]:
173+
# System message
174+
if params.system_instruction:
175+
request_content = _parts_to_any_value(
176+
capture_content=capture_content,
177+
parts=params.system_instruction.parts,
178+
)
179+
yield system_event(
180+
role=params.system_instruction.role, content=request_content
181+
)
182+
183+
for content in params.contents or []:
184+
# Assistant message
185+
if content.role == "model":
186+
request_content = _parts_to_any_value(
187+
capture_content=capture_content, parts=content.parts
188+
)
189+
190+
yield assistant_event(role=content.role, content=request_content)
191+
# Assume user event but role should be "user"
192+
else:
193+
request_content = _parts_to_any_value(
194+
capture_content=capture_content, parts=content.parts
195+
)
196+
yield user_event(role=content.role, content=request_content)
197+
198+
199+
def _parts_to_any_value(
200+
*,
201+
capture_content: bool,
202+
parts: Sequence[content.Part] | Sequence[content_v1beta1.Part],
203+
) -> list[dict[str, AnyValue]] | None:
204+
if not capture_content:
205+
return None
206+
207+
return [
208+
cast("dict[str, AnyValue]", type(part).to_dict(part)) # type: ignore[reportUnknownMemberType]
209+
for part in parts
210+
]

‎instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_generate_content_all_input_events.yaml

+94
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
interactions:
2+
- request:
3+
body: |-
4+
{
5+
"contents": [
6+
{
7+
"role": "user",
8+
"parts": [
9+
{
10+
"text": "My name is OpenTelemetry"
11+
}
12+
]
13+
},
14+
{
15+
"role": "model",
16+
"parts": [
17+
{
18+
"text": "Hello OpenTelemetry!"
19+
}
20+
]
21+
},
22+
{
23+
"role": "user",
24+
"parts": [
25+
{
26+
"text": "Address me by name and say this is a test"
27+
}
28+
]
29+
}
30+
],
31+
"systemInstruction": {
32+
"role": "user",
33+
"parts": [
34+
{
35+
"text": "You are a clever language model"
36+
}
37+
]
38+
}
39+
}
40+
headers:
41+
Accept:
42+
- '*/*'
43+
Accept-Encoding:
44+
- gzip, deflate
45+
Connection:
46+
- keep-alive
47+
Content-Length:
48+
- '548'
49+
Content-Type:
50+
- application/json
51+
User-Agent:
52+
- python-requests/2.32.3
53+
method: POST
54+
uri: https://us-central1-aiplatform.googleapis.com/v1/projects/fake-project/locations/us-central1/publishers/google/models/gemini-1.5-flash-002:generateContent?%24alt=json%3Benum-encoding%3Dint
55+
response:
56+
body:
57+
string: |-
58+
{
59+
"candidates": [
60+
{
61+
"content": {
62+
"role": "model",
63+
"parts": [
64+
{
65+
"text": "OpenTelemetry, this is a test.\n"
66+
}
67+
]
68+
},
69+
"finishReason": 1,
70+
"avgLogprobs": -1.1655389850299496e-06
71+
}
72+
],
73+
"usageMetadata": {
74+
"promptTokenCount": 25,
75+
"candidatesTokenCount": 9,
76+
"totalTokenCount": 34
77+
},
78+
"modelVersion": "gemini-1.5-flash-002"
79+
}
80+
headers:
81+
Content-Type:
82+
- application/json; charset=UTF-8
83+
Transfer-Encoding:
84+
- chunked
85+
Vary:
86+
- Origin
87+
- X-Origin
88+
- Referer
89+
content-length:
90+
- '422'
91+
status:
92+
code: 200
93+
message: OK
94+
version: 1

‎instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_generate_content_invalid_role.yaml

+56
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
interactions:
2+
- request:
3+
body: |-
4+
{
5+
"contents": [
6+
{
7+
"role": "invalid_role",
8+
"parts": [
9+
{
10+
"text": "Say this is a test"
11+
}
12+
]
13+
}
14+
]
15+
}
16+
headers:
17+
Accept:
18+
- '*/*'
19+
Accept-Encoding:
20+
- gzip, deflate
21+
Connection:
22+
- keep-alive
23+
Content-Length:
24+
- '149'
25+
Content-Type:
26+
- application/json
27+
User-Agent:
28+
- python-requests/2.32.3
29+
method: POST
30+
uri: https://us-central1-aiplatform.googleapis.com/v1/projects/fake-project/locations/us-central1/publishers/google/models/gemini-1.5-flash-002:generateContent?%24alt=json%3Benum-encoding%3Dint
31+
response:
32+
body:
33+
string: |-
34+
{
35+
"error": {
36+
"code": 400,
37+
"message": "Please use a valid role: user, model.",
38+
"status": "INVALID_ARGUMENT",
39+
"details": []
40+
}
41+
}
42+
headers:
43+
Content-Type:
44+
- application/json; charset=UTF-8
45+
Transfer-Encoding:
46+
- chunked
47+
Vary:
48+
- Origin
49+
- X-Origin
50+
- Referer
51+
content-length:
52+
- '416'
53+
status:
54+
code: 400
55+
message: Bad Request
56+
version: 1

0 commit comments

Comments
 (0)