Added opt-in support to return traceresponse headers for server instrumentations.

This allows users to configure their web apps to inject trace context
as headers in HTTP responses. This is useful when client side apps need
to connect their spans with the server side spans e.g, in RUM products.

Today the most practical way to do this is to use the Server-Timing
header but in near future we might use the traceresponse header as
described here: https://w3c.github.io/trace-context/#trace-context-http-response-headers-format

Added trace response propagation support for:

    Django
    Falcon
    Flask
    Pyramid
    Tornado

Author: owais

.github/workflows/test.yml

@@ -6,7 +6,7 @@ on:
     - 'release/*'
   pull_request:
 env:
-  CORE_REPO_SHA: cad261e5dae1fe986c87e6965664b45cc9ab73c3
+  CORE_REPO_SHA: f6b04c483f6c416e1927f010c07e71a17a5d79d0
 
 jobs:
   build:

CHANGELOG.md

@@ -20,7 +20,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - `opentelemetry-instrumentation-urllib3` Add urllib3 instrumentation
   ([#299](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/299))
 - `opentelemetry-instrumenation-django` now supports request and response hooks.
-  ([#407](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/407))
+  ([#436](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/436))
+- `opentelemetry-instrumenation-tornado` now supports trace response headers.
+  ([#436](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/436))
+- `opentelemetry-instrumenation-pyramid` now supports trace response headers.
+  ([#436](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/436))
+- `opentelemetry-instrumenation-falcon` added trace response headers support to Falcon.
+  ([#436](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/436))
+- `opentelemetry-instrumenation-flask` added support for trace response headers.
+  ([#436](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/436))
 
 ### Removed
 - Remove `http.status_text` from span attributes

instrumentation/opentelemetry-instrumentation-django/src/opentelemetry/instrumentation/django/middleware.py

@@ -20,6 +20,9 @@
 
 from opentelemetry.context import attach, detach
 from opentelemetry.instrumentation.django.version import __version__
+from opentelemetry.instrumentation.propagators import (
+    get_global_back_propagator,
+)
 from opentelemetry.instrumentation.utils import extract_attributes_from_object
 from opentelemetry.instrumentation.wsgi import (
     add_response_attributes,
@@ -179,6 +182,11 @@ def process_response(self, request, response):
                 response,
             )
 
+            propagator = get_global_back_propagator()
+            if propagator:
+                propagator.inject(response)
+
+            # record any exceptions raised while processing the request
             exception = request.META.pop(self._environ_exception_key, None)
             if _DjangoMiddleware._otel_response_hook:
                 _DjangoMiddleware._otel_response_hook(  # pylint: disable=not-callable

instrumentation/opentelemetry-instrumentation-django/tests/test_middleware.py

@@ -26,10 +26,19 @@
     DjangoInstrumentor,
     _DjangoMiddleware,
 )
+from opentelemetry.instrumentation.propagators import (
+    TraceResponsePropagator,
+    set_global_back_propagator,
+)
 from opentelemetry.sdk.trace import Span
 from opentelemetry.test.test_base import TestBase
 from opentelemetry.test.wsgitestutil import WsgiTestBase
-from opentelemetry.trace import SpanKind, StatusCode
+from opentelemetry.trace import (
+    SpanKind,
+    StatusCode,
+    format_span_id,
+    format_trace_id,
+)
 from opentelemetry.util.http import get_excluded_urls, get_traced_request_attrs
 
 # pylint: disable=import-error
@@ -41,6 +50,7 @@
     route_span_name,
     traced,
     traced_template,
+    with_response_header,
 )
 
 DJANGO_2_2 = VERSION >= (2, 2)
@@ -52,6 +62,7 @@
     url(r"^excluded_arg/", excluded),
     url(r"^excluded_noarg/", excluded_noarg),
     url(r"^excluded_noarg2/", excluded_noarg2),
+    url(r"^response_header/", with_response_header),
     url(r"^span_name/([0-9]{4})/$", route_span_name),
 ]
 _django_instrumentor = DjangoInstrumentor()
@@ -309,3 +320,55 @@ def response_hook(span, request, response):
         self.assertIsInstance(response_hook_args[1], HttpRequest)
         self.assertIsInstance(response_hook_args[2], HttpResponse)
         self.assertEqual(response_hook_args[2], response)
+
+    def test_trace_response_headers(self):
+        response = Client().get("/span_name/1234/")
+        self.assertNotIn("Server-Timing", response._headers)
+        self.memory_exporter.clear()
+
+        set_global_back_propagator(TraceResponsePropagator())
+
+        response = Client().get("/span_name/1234/")
+        span = self.memory_exporter.get_finished_spans()[0]
+
+        self.assertIn("traceresponse", response._headers)
+        self.assertEqual(
+            response._headers["access-control-expose-headers"][0],
+            "Access-Control-Expose-Headers",
+        )
+        self.assertEqual(
+            response._headers["access-control-expose-headers"][1],
+            "traceresponse",
+        )
+        self.assertEqual(
+            response._headers["traceresponse"][0], "traceresponse"
+        )
+        self.assertEqual(
+            response._headers["traceresponse"][1],
+            "00-{0}-{1}-01".format(
+                format_trace_id(span.get_span_context().trace_id),
+                format_span_id(span.get_span_context().span_id),
+            ),
+        )
+        self.memory_exporter.clear()
+
+    def test_trace_response_header_pre_existing_header(self):
+        set_global_back_propagator(TraceResponsePropagator())
+
+        response = Client().get("/response_header/")
+        span = self.memory_exporter.get_finished_spans()[0]
+        self.assertIn("traceresponse", response._headers)
+        self.assertEqual(
+            response._headers["access-control-expose-headers"][1],
+            "X-Test-Header, traceresponse",
+        )
+        self.assertEqual(
+            response._headers["traceresponse"][1],
+            "abc; val=1, "
+            + "00-{0}-{1}-01".format(
+                format_trace_id(span.get_span_context().trace_id),
+                format_span_id(span.get_span_context().span_id),
+            ),
+        )
+
+        self.memory_exporter.clear()

instrumentation/opentelemetry-instrumentation-django/tests/views.py

@@ -1,5 +1,9 @@
 from django.http import HttpResponse
 
+from opentelemetry.instrumentation.propagators import (
+    _HTTP_HEADER_ACCESS_CONTROL_EXPOSE_HEADERS,
+)
+
 
 def traced(request):  # pylint: disable=unused-argument
     return HttpResponse()
@@ -29,3 +33,10 @@ def route_span_name(
     request, *args, **kwargs
 ):  # pylint: disable=unused-argument
     return HttpResponse()
+
+
+def with_response_header(request):  # pylint: disable=unused-argument
+    response = HttpResponse()
+    response["traceresponse"] = "abc; val=1"
+    response[_HTTP_HEADER_ACCESS_CONTROL_EXPOSE_HEADERS] = "X-Test-Header"
+    return response

instrumentation/opentelemetry-instrumentation-falcon/src/opentelemetry/instrumentation/falcon/__init__.py

@@ -52,6 +52,10 @@ def on_get(self, req, resp):
 from opentelemetry import context, trace
 from opentelemetry.instrumentation.falcon.version import __version__
 from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
+from opentelemetry.instrumentation.propagators import (
+    FuncSetter,
+    get_global_back_propagator,
+)
 from opentelemetry.instrumentation.utils import (
     extract_attributes_from_object,
     http_status_to_status_code,
@@ -148,6 +152,8 @@ def _start_response(status, response_headers, *args, **kwargs):
 class _TraceMiddleware:
     # pylint:disable=R0201,W0613
 
+    back_propagation_setter = FuncSetter(falcon.api.Response.append_header)
+
     def __init__(self, tracer=None, traced_request_attrs=None):
         self.tracer = tracer
         self._traced_request_attrs = _traced_request_attrs
@@ -209,3 +215,7 @@ def process_response(
                     description=reason,
                 )
             )
+
+        propagator = get_global_back_propagator()
+        if propagator:
+            propagator.inject(resp, setter=self.back_propagation_setter)

instrumentation/opentelemetry-instrumentation-falcon/tests/test_falcon.py

@@ -17,8 +17,13 @@
 from falcon import testing
 
 from opentelemetry.instrumentation.falcon import FalconInstrumentor
+from opentelemetry.instrumentation.propagators import (
+    TraceResponsePropagator,
+    get_global_back_propagator,
+    set_global_back_propagator,
+)
 from opentelemetry.test.test_base import TestBase
-from opentelemetry.trace import StatusCode
+from opentelemetry.trace import StatusCode, format_span_id, format_trace_id
 from opentelemetry.util.http import get_excluded_urls, get_traced_request_attrs
 
 from .app import make_app
@@ -192,6 +197,28 @@ def test_traced_request_attributes(self):
         self.assertEqual(span.attributes["query_string"], "q=abc")
         self.assertNotIn("not_available_attr", span.attributes)
 
+    def test_trace_response(self):
+        orig = get_global_back_propagator()
+        set_global_back_propagator(TraceResponsePropagator())
+
+        response = self.client().simulate_get(path="/hello?q=abc")
+        headers = response.headers
+        span = self.memory_exporter.get_finished_spans()[0]
+
+        self.assertIn("traceresponse", headers)
+        self.assertEqual(
+            headers["access-control-expose-headers"], "traceresponse",
+        )
+        self.assertEqual(
+            headers["traceresponse"],
+            "00-{0}-{1}-01".format(
+                format_trace_id(span.get_span_context().trace_id),
+                format_span_id(span.get_span_context().span_id),
+            ),
+        )
+
+        set_global_back_propagator(orig)
+
     def test_traced_not_recording(self):
         mock_tracer = Mock()
         mock_span = Mock()

instrumentation/opentelemetry-instrumentation-flask/src/opentelemetry/instrumentation/flask/__init__.py

@@ -55,6 +55,9 @@ def hello():
 from opentelemetry import context, trace
 from opentelemetry.instrumentation.flask.version import __version__
 from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
+from opentelemetry.instrumentation.propagators import (
+    get_global_back_propagator,
+)
 from opentelemetry.propagate import extract
 from opentelemetry.util._time import _time_ns
 from opentelemetry.util.http import get_excluded_urls
@@ -91,6 +94,13 @@ def _start_response(status, response_headers, *args, **kwargs):
             if not _excluded_urls.url_disabled(flask.request.url):
                 span = flask.request.environ.get(_ENVIRON_SPAN_KEY)
 
+                propagator = get_global_back_propagator()
+                if propagator:
+                    propagator.inject(
+                        response_headers,
+                        setter=otel_wsgi.default_back_propagation_setter,
+                    )
+
                 if span:
                     otel_wsgi.add_response_attributes(
                         span, status, response_headers

instrumentation/opentelemetry-instrumentation-flask/tests/test_programmatic.py

@@ -18,6 +18,11 @@
 
 from opentelemetry import trace
 from opentelemetry.instrumentation.flask import FlaskInstrumentor
+from opentelemetry.instrumentation.propagators import (
+    TraceResponsePropagator,
+    get_global_back_propagator,
+    set_global_back_propagator,
+)
 from opentelemetry.test.test_base import TestBase
 from opentelemetry.test.wsgitestutil import WsgiTestBase
 from opentelemetry.util.http import get_excluded_urls
@@ -119,6 +124,31 @@ def test_simple(self):
         self.assertEqual(span_list[0].kind, trace.SpanKind.SERVER)
         self.assertEqual(span_list[0].attributes, expected_attrs)
 
+    def test_trace_response(self):
+        orig = get_global_back_propagator()
+
+        set_global_back_propagator(TraceResponsePropagator())
+        response = self.client.get("/hello/123")
+        headers = response.headers
+
+        span_list = self.memory_exporter.get_finished_spans()
+        self.assertEqual(len(span_list), 1)
+        span = span_list[0]
+
+        self.assertIn("traceresponse", headers)
+        self.assertEqual(
+            headers["access-control-expose-headers"], "traceresponse",
+        )
+        self.assertEqual(
+            headers["traceresponse"],
+            "00-{0}-{1}-01".format(
+                trace.format_trace_id(span.get_span_context().trace_id),
+                trace.format_span_id(span.get_span_context().span_id),
+            ),
+        )
+
+        set_global_back_propagator(orig)
+
     def test_not_recording(self):
         mock_tracer = Mock()
         mock_span = Mock()

instrumentation/opentelemetry-instrumentation-pyramid/src/opentelemetry/instrumentation/pyramid/callbacks.py

@@ -21,6 +21,9 @@
 
 import opentelemetry.instrumentation.wsgi as otel_wsgi
 from opentelemetry import context, trace
+from opentelemetry.instrumentation.propagators import (
+    get_global_back_propagator,
+)
 from opentelemetry.instrumentation.pyramid.version import __version__
 from opentelemetry.propagate import extract
 from opentelemetry.util._time import _time_ns
@@ -157,6 +160,10 @@ def trace_tween(request):
                     response_or_exception.headers,
                 )
 
+                propagator = get_global_back_propagator()
+                if propagator:
+                    propagator.inject(response.headers)
+
                 activation = request.environ.get(_ENVIRON_ACTIVATION_KEY)
 
                 if isinstance(response_or_exception, HTTPException):

instrumentation/opentelemetry-instrumentation-pyramid/tests/test_programmatic.py

@@ -17,6 +17,11 @@
 from pyramid.config import Configurator
 
 from opentelemetry import trace
+from opentelemetry.instrumentation.propagators import (
+    TraceResponsePropagator,
+    get_global_back_propagator,
+    set_global_back_propagator,
+)
 from opentelemetry.instrumentation.pyramid import PyramidInstrumentor
 from opentelemetry.test.test_base import TestBase
 from opentelemetry.test.wsgitestutil import WsgiTestBase
@@ -98,6 +103,28 @@ def test_simple(self):
         self.assertEqual(span_list[0].kind, trace.SpanKind.SERVER)
         self.assertEqual(span_list[0].attributes, expected_attrs)
 
+    def test_response_headers(self):
+        orig = get_global_back_propagator()
+        set_global_back_propagator(TraceResponsePropagator())
+
+        response = self.client.get("/hello/500")
+        headers = response.headers
+        span = self.memory_exporter.get_finished_spans()[0]
+
+        self.assertIn("traceresponse", headers)
+        self.assertEqual(
+            headers["access-control-expose-headers"], "traceresponse",
+        )
+        self.assertEqual(
+            headers["traceresponse"],
+            "00-{0}-{1}-01".format(
+                trace.format_trace_id(span.get_span_context().trace_id),
+                trace.format_span_id(span.get_span_context().span_id),
+            ),
+        )
+
+        set_global_back_propagator(orig)
+
     def test_not_recording(self):
         mock_tracer = Mock()
         mock_span = Mock()

instrumentation/opentelemetry-instrumentation-tornado/src/opentelemetry/instrumentation/tornado/__init__.py

@@ -46,6 +46,10 @@ def get(self):
 
 from opentelemetry import context, trace
 from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
+from opentelemetry.instrumentation.propagators import (
+    FuncSetter,
+    get_global_back_propagator,
+)
 from opentelemetry.instrumentation.tornado.version import __version__
 from opentelemetry.instrumentation.utils import (
     extract_attributes_from_object,
@@ -68,6 +72,8 @@ def get(self):
 _excluded_urls = get_excluded_urls("TORNADO")
 _traced_request_attrs = get_traced_request_attrs("TORNADO")
 
+back_propagation_setter = FuncSetter(tornado.web.RequestHandler.add_header)
+
 
 class TornadoInstrumentor(BaseInstrumentor):
     patched_handlers = []
@@ -211,6 +217,14 @@ def _start_span(tracer, handler, start_time) -> _TraceContext:
     activation.__enter__()  # pylint: disable=E1101
     ctx = _TraceContext(activation, span, token)
     setattr(handler, _HANDLER_CONTEXT_KEY, ctx)
+
+    # finish handler is called after the response is sent back to
+    # the client so it is too late to inject trace response headers
+    # there.
+    propagator = get_global_back_propagator()
+    if propagator:
+        propagator.inject(handler, setter=back_propagation_setter)
+
     return ctx