Skip to content

Commit c92ba14

Browse files
authored
Add metric instrumentation for urllib (#1553)
1 parent 092d8c8 commit c92ba14

File tree

6 files changed

+314
-4
lines changed

6 files changed

+314
-4
lines changed

.github/component_owners.yml

+3
Original file line numberDiff line numberDiff line change
@@ -41,5 +41,8 @@ components:
4141
instrumentation/opentelemetry-instrumentation-tornado:
4242
- shalevr
4343

44+
instrumentation/opentelemetry-instrumentation-urllib:
45+
- shalevr
46+
4447
instrumentation/opentelemetry-instrumentation-urllib3:
4548
- shalevr

CHANGELOG.md

+2
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
99

1010
### Added
1111

12+
- Add metric instrumentation for urllib
13+
([#1553](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1553))
1214
- `opentelemetry/sdk/extension/aws` Implement [`aws.ecs.*`](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/resource/semantic_conventions/cloud_provider/aws/ecs.md) and [`aws.logs.*`](https://opentelemetry.io/docs/reference/specification/resource/semantic_conventions/cloud_provider/aws/logs/) resource attributes in the `AwsEcsResourceDetector` detector when the ECS Metadata v4 is available
1315
([#1212](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1212))
1416

instrumentation/README.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,6 @@
4040
| [opentelemetry-instrumentation-system-metrics](./opentelemetry-instrumentation-system-metrics) | psutil >= 5 | No
4141
| [opentelemetry-instrumentation-tornado](./opentelemetry-instrumentation-tornado) | tornado >= 5.1.1 | Yes
4242
| [opentelemetry-instrumentation-tortoiseorm](./opentelemetry-instrumentation-tortoiseorm) | tortoise-orm >= 0.17.0 | No
43-
| [opentelemetry-instrumentation-urllib](./opentelemetry-instrumentation-urllib) | urllib | No
43+
| [opentelemetry-instrumentation-urllib](./opentelemetry-instrumentation-urllib) | urllib | Yes
4444
| [opentelemetry-instrumentation-urllib3](./opentelemetry-instrumentation-urllib3) | urllib3 >= 1.0.0, < 2.0.0 | Yes
4545
| [opentelemetry-instrumentation-wsgi](./opentelemetry-instrumentation-wsgi) | wsgi | Yes

instrumentation/opentelemetry-instrumentation-urllib/src/opentelemetry/instrumentation/urllib/__init__.py

+60-3
Original file line numberDiff line numberDiff line change
@@ -63,10 +63,9 @@ def response_hook(span, request_obj, response)
6363
import functools
6464
import types
6565
import typing
66-
67-
# from urllib import response
6866
from http import client
69-
from typing import Collection
67+
from timeit import default_timer
68+
from typing import Collection, Dict
7069
from urllib.request import ( # pylint: disable=no-name-in-module,import-error
7170
OpenerDirector,
7271
Request,
@@ -83,7 +82,9 @@ def response_hook(span, request_obj, response)
8382
_SUPPRESS_INSTRUMENTATION_KEY,
8483
http_status_to_status_code,
8584
)
85+
from opentelemetry.metrics import Histogram, get_meter
8686
from opentelemetry.propagate import inject
87+
from opentelemetry.semconv.metrics import MetricInstruments
8788
from opentelemetry.semconv.trace import SpanAttributes
8889
from opentelemetry.trace import Span, SpanKind, get_tracer
8990
from opentelemetry.trace.status import Status
@@ -114,8 +115,15 @@ def _instrument(self, **kwargs):
114115
"""
115116
tracer_provider = kwargs.get("tracer_provider")
116117
tracer = get_tracer(__name__, __version__, tracer_provider)
118+
119+
meter_provider = kwargs.get("meter_provider")
120+
meter = get_meter(__name__, __version__, meter_provider)
121+
122+
histograms = _create_client_histograms(meter)
123+
117124
_instrument(
118125
tracer,
126+
histograms,
119127
request_hook=kwargs.get("request_hook"),
120128
response_hook=kwargs.get("response_hook"),
121129
)
@@ -132,6 +140,7 @@ def uninstrument_opener(
132140

133141
def _instrument(
134142
tracer,
143+
histograms: Dict[str, Histogram],
135144
request_hook: _RequestHookT = None,
136145
response_hook: _ResponseHookT = None,
137146
):
@@ -192,11 +201,13 @@ def _instrumented_open_call(
192201
context.set_value(_SUPPRESS_HTTP_INSTRUMENTATION_KEY, True)
193202
)
194203
try:
204+
start_time = default_timer()
195205
result = call_wrapped() # *** PROCEED
196206
except Exception as exc: # pylint: disable=W0703
197207
exception = exc
198208
result = getattr(exc, "file", None)
199209
finally:
210+
elapsed_time = round((default_timer() - start_time) * 1000)
200211
context.detach(token)
201212

202213
if result is not None:
@@ -214,6 +225,10 @@ def _instrumented_open_call(
214225
SpanAttributes.HTTP_FLAVOR
215226
] = f"{ver_[:1]}.{ver_[:-1]}"
216227

228+
_record_histograms(
229+
histograms, labels, request, result, elapsed_time
230+
)
231+
217232
if callable(response_hook):
218233
response_hook(span, request, result)
219234

@@ -248,3 +263,45 @@ def _uninstrument_from(instr_root, restore_as_bound_func=False):
248263
if restore_as_bound_func:
249264
original = types.MethodType(original, instr_root)
250265
setattr(instr_root, instr_func_name, original)
266+
267+
268+
def _create_client_histograms(meter) -> Dict[str, Histogram]:
269+
histograms = {
270+
MetricInstruments.HTTP_CLIENT_DURATION: meter.create_histogram(
271+
name=MetricInstruments.HTTP_CLIENT_DURATION,
272+
unit="ms",
273+
description="measures the duration outbound HTTP requests",
274+
),
275+
MetricInstruments.HTTP_CLIENT_REQUEST_SIZE: meter.create_histogram(
276+
name=MetricInstruments.HTTP_CLIENT_REQUEST_SIZE,
277+
unit="By",
278+
description="measures the size of HTTP request messages (compressed)",
279+
),
280+
MetricInstruments.HTTP_CLIENT_RESPONSE_SIZE: meter.create_histogram(
281+
name=MetricInstruments.HTTP_CLIENT_RESPONSE_SIZE,
282+
unit="By",
283+
description="measures the size of HTTP response messages (compressed)",
284+
),
285+
}
286+
287+
return histograms
288+
289+
290+
def _record_histograms(
291+
histograms, metric_attributes, request, response, elapsed_time
292+
):
293+
histograms[MetricInstruments.HTTP_CLIENT_DURATION].record(
294+
elapsed_time, attributes=metric_attributes
295+
)
296+
297+
data = getattr(request, "data", None)
298+
request_size = 0 if data is None else len(data)
299+
histograms[MetricInstruments.HTTP_CLIENT_REQUEST_SIZE].record(
300+
request_size, attributes=metric_attributes
301+
)
302+
303+
if response is not None:
304+
response_size = int(response.headers.get("Content-Length", 0))
305+
histograms[MetricInstruments.HTTP_CLIENT_RESPONSE_SIZE].record(
306+
response_size, attributes=metric_attributes
307+
)

instrumentation/opentelemetry-instrumentation-urllib/src/opentelemetry/instrumentation/urllib/package.py

+2
Original file line numberDiff line numberDiff line change
@@ -14,3 +14,5 @@
1414

1515

1616
_instruments = tuple()
17+
18+
_supports_metrics = True
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,246 @@
1+
# Copyright The OpenTelemetry Authors
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
16+
from timeit import default_timer
17+
from typing import Optional, Union
18+
from urllib import request
19+
from urllib.parse import urlencode
20+
21+
import httpretty
22+
23+
from opentelemetry.instrumentation.urllib import ( # pylint: disable=no-name-in-module,import-error
24+
URLLibInstrumentor,
25+
)
26+
from opentelemetry.sdk.metrics._internal.point import Metric
27+
from opentelemetry.sdk.metrics.export import (
28+
HistogramDataPoint,
29+
NumberDataPoint,
30+
)
31+
from opentelemetry.semconv.metrics import MetricInstruments
32+
from opentelemetry.test.test_base import TestBase
33+
34+
35+
class TestRequestsIntegration(TestBase):
36+
URL = "http://httpbin.org/status/200"
37+
URL_POST = "http://httpbin.org/post"
38+
39+
def setUp(self):
40+
super().setUp()
41+
URLLibInstrumentor().instrument()
42+
httpretty.enable()
43+
httpretty.register_uri(httpretty.GET, self.URL, body=b"Hello!")
44+
httpretty.register_uri(
45+
httpretty.POST, self.URL_POST, body=b"Hello World!"
46+
)
47+
48+
def tearDown(self):
49+
super().tearDown()
50+
URLLibInstrumentor().uninstrument()
51+
httpretty.disable()
52+
53+
def get_sorted_metrics(self):
54+
resource_metrics = (
55+
self.memory_metrics_reader.get_metrics_data().resource_metrics
56+
)
57+
58+
all_metrics = []
59+
for metrics in resource_metrics:
60+
for scope_metrics in metrics.scope_metrics:
61+
all_metrics.extend(scope_metrics.metrics)
62+
63+
return self.sorted_metrics(all_metrics)
64+
65+
@staticmethod
66+
def sorted_metrics(metrics):
67+
"""
68+
Sorts metrics by metric name.
69+
"""
70+
return sorted(
71+
metrics,
72+
key=lambda m: m.name,
73+
)
74+
75+
def assert_metric_expected(
76+
self,
77+
metric: Metric,
78+
expected_value: Union[int, float],
79+
expected_attributes: dict,
80+
est_delta: Optional[float] = None,
81+
):
82+
data_point = next(iter(metric.data.data_points))
83+
84+
if isinstance(data_point, HistogramDataPoint):
85+
self.assertEqual(
86+
data_point.count,
87+
1,
88+
)
89+
if est_delta is None:
90+
self.assertEqual(
91+
data_point.sum,
92+
expected_value,
93+
)
94+
else:
95+
self.assertAlmostEqual(
96+
data_point.sum,
97+
expected_value,
98+
delta=est_delta,
99+
)
100+
elif isinstance(data_point, NumberDataPoint):
101+
self.assertEqual(
102+
data_point.value,
103+
expected_value,
104+
)
105+
106+
self.assertDictEqual(
107+
expected_attributes,
108+
dict(data_point.attributes),
109+
)
110+
111+
def test_basic_metric(self):
112+
start_time = default_timer()
113+
with request.urlopen(self.URL) as result:
114+
client_duration_estimated = (default_timer() - start_time) * 1000
115+
116+
metrics = self.get_sorted_metrics()
117+
self.assertEqual(len(metrics), 3)
118+
119+
(
120+
client_duration,
121+
client_request_size,
122+
client_response_size,
123+
) = metrics[:3]
124+
125+
self.assertEqual(
126+
client_duration.name, MetricInstruments.HTTP_CLIENT_DURATION
127+
)
128+
self.assert_metric_expected(
129+
client_duration,
130+
client_duration_estimated,
131+
{
132+
"http.status_code": str(result.code),
133+
"http.method": "GET",
134+
"http.url": str(result.url),
135+
"http.flavor": "1.1",
136+
},
137+
est_delta=200,
138+
)
139+
140+
# net.peer.name
141+
142+
self.assertEqual(
143+
client_request_size.name,
144+
MetricInstruments.HTTP_CLIENT_REQUEST_SIZE,
145+
)
146+
self.assert_metric_expected(
147+
client_request_size,
148+
0,
149+
{
150+
"http.status_code": str(result.code),
151+
"http.method": "GET",
152+
"http.url": str(result.url),
153+
"http.flavor": "1.1",
154+
},
155+
)
156+
157+
self.assertEqual(
158+
client_response_size.name,
159+
MetricInstruments.HTTP_CLIENT_RESPONSE_SIZE,
160+
)
161+
self.assert_metric_expected(
162+
client_response_size,
163+
result.length,
164+
{
165+
"http.status_code": str(result.code),
166+
"http.method": "GET",
167+
"http.url": str(result.url),
168+
"http.flavor": "1.1",
169+
},
170+
)
171+
172+
def test_basic_metric_request_not_empty(self):
173+
data = {"header1": "value1", "header2": "value2"}
174+
data_encoded = urlencode(data).encode()
175+
176+
start_time = default_timer()
177+
with request.urlopen(self.URL_POST, data=data_encoded) as result:
178+
client_duration_estimated = (default_timer() - start_time) * 1000
179+
180+
metrics = self.get_sorted_metrics()
181+
self.assertEqual(len(metrics), 3)
182+
183+
(
184+
client_duration,
185+
client_request_size,
186+
client_response_size,
187+
) = metrics[:3]
188+
189+
self.assertEqual(
190+
client_duration.name, MetricInstruments.HTTP_CLIENT_DURATION
191+
)
192+
self.assert_metric_expected(
193+
client_duration,
194+
client_duration_estimated,
195+
{
196+
"http.status_code": str(result.code),
197+
"http.method": "POST",
198+
"http.url": str(result.url),
199+
"http.flavor": "1.1",
200+
},
201+
est_delta=200,
202+
)
203+
204+
self.assertEqual(
205+
client_request_size.name,
206+
MetricInstruments.HTTP_CLIENT_REQUEST_SIZE,
207+
)
208+
self.assert_metric_expected(
209+
client_request_size,
210+
len(data_encoded),
211+
{
212+
"http.status_code": str(result.code),
213+
"http.method": "POST",
214+
"http.url": str(result.url),
215+
"http.flavor": "1.1",
216+
},
217+
)
218+
219+
self.assertEqual(
220+
client_response_size.name,
221+
MetricInstruments.HTTP_CLIENT_RESPONSE_SIZE,
222+
)
223+
self.assert_metric_expected(
224+
client_response_size,
225+
result.length,
226+
{
227+
"http.status_code": str(result.code),
228+
"http.method": "POST",
229+
"http.url": str(result.url),
230+
"http.flavor": "1.1",
231+
},
232+
)
233+
234+
def test_metric_uninstrument(self):
235+
with request.urlopen(self.URL):
236+
metrics = self.get_sorted_metrics()
237+
self.assertEqual(len(metrics), 3)
238+
239+
URLLibInstrumentor().uninstrument()
240+
with request.urlopen(self.URL):
241+
metrics = self.get_sorted_metrics()
242+
self.assertEqual(len(metrics), 3)
243+
244+
for metric in metrics:
245+
for point in list(metric.data.data_points):
246+
self.assertEqual(point.count, 1)

0 commit comments

Comments
 (0)