feat: Add exclude urls feature to HTTPX instrumentation

Issue: open-telemetry#539

Author: zqumei0

CHANGELOG.md

@@ -19,6 +19,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Added `exclude urls` feature to HTTPX instrumentation
+  ([#1900](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1900))
 - `opentelemetry-resource-detector-azure` Add resource detectors for Azure App Service and VM
   ([#1901](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1901))
 

instrumentation/opentelemetry-instrumentation-httpx/pyproject.toml

@@ -38,6 +38,7 @@ test = [
   "opentelemetry-instrumentation-httpx[instruments]",
   "opentelemetry-sdk ~= 1.12",
   "opentelemetry-test-utils == 0.42b0.dev",
+  "httpretty ~= 1.0",
 ]
 
 [project.entry-points.opentelemetry_instrumentor]

instrumentation/opentelemetry-instrumentation-httpx/src/opentelemetry/instrumentation/httpx/__init__.py

@@ -176,7 +176,13 @@ def response_hook(span, request, response):
 from opentelemetry.trace import SpanKind, TracerProvider, get_tracer
 from opentelemetry.trace.span import Span
 from opentelemetry.trace.status import Status
+from opentelemetry.utils.http import (
+    ExcludeList,
+    get_excluded_urls,
+    parse_excluded_urls,
+)
 
+_excluded_urls_from_env = get_excluded_urls("HTTPX")
 _logger = logging.getLogger(__name__)
 
 URL = typing.Tuple[bytes, bytes, typing.Optional[int], bytes]
@@ -276,6 +282,7 @@ class SyncOpenTelemetryTransport(httpx.BaseTransport):
             right after the span is created
         response_hook: A hook that receives the span, request, and response
             that is called right before the span ends
+        excluded_urls: List of urls that should be excluded from tracing
     """
 
     def __init__(
@@ -284,6 +291,7 @@ def __init__(
         tracer_provider: typing.Optional[TracerProvider] = None,
         request_hook: typing.Optional[RequestHook] = None,
         response_hook: typing.Optional[ResponseHook] = None,
+        excluded_urls: typing.Optional[ExcludeList] = None,
     ):
         self._transport = transport
         self._tracer = get_tracer(
@@ -293,6 +301,7 @@ def __init__(
         )
         self._request_hook = request_hook
         self._response_hook = response_hook
+        self._excluded_urls = excluded_urls
 
     def __enter__(self) -> "SyncOpenTelemetryTransport":
         self._transport.__enter__()
@@ -317,10 +326,13 @@ def handle_request(
         """Add request info to span."""
         if context.get_value("suppress_instrumentation"):
             return self._transport.handle_request(*args, **kwargs)
-
         method, url, headers, stream, extensions = _extract_parameters(
             args, kwargs
         )
+
+        if self._excluded_urls and self._excluded_urls.url_disabled(url):
+            return self._transport.handle_request(*args, **kwargs)
+
         span_attributes = _prepare_attributes(method, url)
 
         request_info = RequestInfo(method, url, headers, stream, extensions)
@@ -370,6 +382,7 @@ class AsyncOpenTelemetryTransport(httpx.AsyncBaseTransport):
             right after the span is created
         response_hook: A hook that receives the span, request, and response
             that is called right before the span ends
+        excluded_urls: List of urls that should be excluded from tracing
     """
 
     def __init__(
@@ -378,6 +391,7 @@ def __init__(
         tracer_provider: typing.Optional[TracerProvider] = None,
         request_hook: typing.Optional[RequestHook] = None,
         response_hook: typing.Optional[ResponseHook] = None,
+        excluded_urls: typing.Optional[ExcludeList] = None,
     ):
         self._transport = transport
         self._tracer = get_tracer(
@@ -387,6 +401,7 @@ def __init__(
         )
         self._request_hook = request_hook
         self._response_hook = response_hook
+        self._excluded_urls = excluded_urls
 
     async def __aenter__(self) -> "AsyncOpenTelemetryTransport":
         await self._transport.__aenter__()
@@ -407,12 +422,16 @@ async def handle_async_request(
         httpx.Response,
     ]:
         """Add request info to span."""
-        if context.get_value("suppress_instrumentation"):
-            return await self._transport.handle_async_request(*args, **kwargs)
-
         method, url, headers, stream, extensions = _extract_parameters(
             args, kwargs
         )
+
+        if self._excluded_urls and self._excluded_urls.url_disabled(url):
+            return await self._transport.handle_async_request(*args, **kwargs)
+
+        if context.get_value("suppress_instrumentation"):
+            return await self._transport.handle_async_request(*args, **kwargs)
+
         span_attributes = _prepare_attributes(method, url)
 
         span_name = _get_default_span_name(
@@ -459,6 +478,7 @@ class _InstrumentedClient(httpx.Client):
     _tracer_provider = None
     _request_hook = None
     _response_hook = None
+    _excluded_urls = None
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -478,6 +498,7 @@ class _InstrumentedAsyncClient(httpx.AsyncClient):
     _tracer_provider = None
     _request_hook = None
     _response_hook = None
+    _excluded_urls = None
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -513,11 +534,18 @@ def _instrument(self, **kwargs):
                     right after the span is created
                 ``response_hook``: A hook that receives the span, request, and response
                     that is called right before the span ends
+                ``excluded_urls``: A string containing a comma-delimited
+                    list of regexes used to exclude URLs from tracking
         """
         self._original_client = httpx.Client
         self._original_async_client = httpx.AsyncClient
         request_hook = kwargs.get("request_hook")
         response_hook = kwargs.get("response_hook")
+        excluded_urls = kwargs.get("excluded_urls")
+        if  excluded_urls is None:
+            excluded_urls = _excluded_urls_from_env
+        else:
+            excluded_urls = parse_excluded_urls(excluded_urls)
         if callable(request_hook):
             _InstrumentedClient._request_hook = request_hook
             _InstrumentedAsyncClient._request_hook = request_hook
@@ -536,9 +564,11 @@ def _uninstrument(self, **kwargs):
         _InstrumentedClient._tracer_provider = None
         _InstrumentedClient._request_hook = None
         _InstrumentedClient._response_hook = None
+        _InstrumentedClient._excluded_urls = None
         _InstrumentedAsyncClient._tracer_provider = None
         _InstrumentedAsyncClient._request_hook = None
         _InstrumentedAsyncClient._response_hook = None
+        _InstrumentedAsyncClient._excluded_urls = None
 
     @staticmethod
     def instrument_client(

instrumentation/opentelemetry-instrumentation-httpx/tests/test_httpx_integration.py

@@ -17,6 +17,7 @@
 import typing
 from unittest import mock
 
+import httpretty
 import httpx
 import respx
 
@@ -43,10 +44,10 @@
         ResponseHook,
         ResponseInfo,
     )
-    from opentelemetry.sdk.trace.export import SpanExporter
-    from opentelemetry.trace import TracerProvider
-    from opentelemetry.trace.span import Span
-
+from opentelemetry.sdk.trace.export import SpanExporter
+from opentelemetry.trace import TracerProvider
+from opentelemetry.trace.span import Span
+from opentelemetry.util.http import get_excluded_urls
 
 HTTP_RESPONSE_BODY = "http.response.body"
 
@@ -380,8 +381,24 @@ def create_client(
 
         def setUp(self):
             super().setUp()
+
+            self.env_patch = mock.patch.dict(
+                "os.environ",
+                {
+                    "OTEL_PYTHON_HTTPX_EXCLUDED_URLS": "http://localhost/env_excluded_arg/123,env_excluded_noarg"
+                },
+            )
+            self.env_patch.start()
+
+            self.exclude_patch = mock.patch(
+                "opentelemetry.instrumentation.httpx._excluded_urls_from_env",
+                get_excluded_urls("HTTPX"),
+            )
+            self.exclude_patch.start()
+
             HTTPXClientInstrumentor().instrument()
             self.client = self.create_client()
+            self.env_patch.stop()
             HTTPXClientInstrumentor().uninstrument()
 
         def test_custom_tracer_provider(self):
@@ -495,6 +512,36 @@ def test_uninstrument(self):
             self.assertEqual(result.text, "Hello!")
             self.assert_span(num_spans=0)
 
+        def test_excluded_urls_explicit(self):
+            url_404 = "http://mock/status/404"
+            httpretty.register_uri(
+                httpretty.GET,
+                url_404,
+                status=404,
+            )
+
+            HTTPXClientInstrumentor().instrument(excluded_urls=".*/404")
+            client = self.create_client()
+            self.perform_request(self.URL)
+            self.perform_request(url_404)
+
+            self.assert_span(num_spans=1)
+
+        def test_excluded_urls_from_env(self):
+            url = "http://localhost/env_excluded_arg/123"
+            httpretty.register_uri(
+                httpretty.GET,
+                url,
+                status=200,
+            )
+
+            HTTPXClientInstrumentor().instrument()
+            client = self.create_client()
+            self.perform_request(self.URL)
+            self.perform_request(url)
+
+            self.assert_span(num_spans=1)
+
         def test_uninstrument_client(self):
             HTTPXClientInstrumentor().uninstrument_client(self.client)