Integrating sql commenter into otel_django_instrumentation

.pylintrc

@@ -81,6 +81,7 @@ disable=missing-docstring,
       invalid-overridden-method, # temp-pylint-upgrade 
       missing-module-docstring, # temp-pylint-upgrade
       import-error, # needed as a workaround as reported here: https://github.com/open-telemetry/opentelemetry-python-contrib/issues/290
+      consider-using-f-string, # needed flexibility for string formatting
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option

CHANGELOG.md

@@ -33,6 +33,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   ([#1111](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1111))
 - Set otlp-proto-grpc as the default metrics exporter for auto-instrumentation
   ([#1127](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1127))
+- Integrated sqlcommenter plugin into opentelemetry-instrumentation-django
+  ([#896](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/896))
 
 
 ## [1.12.0rc1-0.31b0](https://github.com/open-telemetry/opentelemetry-python/releases/tag/v1.12.0rc1-0.31b0) - 2022-05-17

instrumentation/opentelemetry-instrumentation-django/src/opentelemetry/instrumentation/django/__init__.py

@@ -136,7 +136,9 @@ def response_hook(span, request, response):
 from opentelemetry.instrumentation.django.environment_variables import (
     OTEL_PYTHON_DJANGO_INSTRUMENT,
 )
-from opentelemetry.instrumentation.django.middleware import _DjangoMiddleware
+from opentelemetry.instrumentation.django.middleware.otel_middleware import (
+    _DjangoMiddleware,
+)
 from opentelemetry.instrumentation.django.package import _instruments
 from opentelemetry.instrumentation.django.version import __version__
 from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
@@ -166,6 +168,8 @@ class DjangoInstrumentor(BaseInstrumentor):
         [_DjangoMiddleware.__module__, _DjangoMiddleware.__qualname__]
     )
 
+    _sql_commenter_middleware = "opentelemetry.instrumentation.django.middleware.sqlcommenter_middleware.SqlCommenter"
+
     def instrumentation_dependencies(self) -> Collection[str]:
         return _instruments
 
@@ -204,7 +208,13 @@ def _instrument(self, **kwargs):
         if isinstance(settings_middleware, tuple):
             settings_middleware = list(settings_middleware)
 
+        is_sql_commentor_enabled = kwargs.pop("is_sql_commentor_enabled", None)
+
+        if is_sql_commentor_enabled:
+            settings_middleware.insert(0, self._sql_commenter_middleware)
+
         settings_middleware.insert(0, self._opentelemetry_middleware)
+
         setattr(settings, _middleware_setting, settings_middleware)
 
     def _uninstrument(self, **kwargs):

instrumentation/opentelemetry-instrumentation-django/src/opentelemetry/instrumentation/django/middleware/sqlcommenter_middleware.py

@@ -0,0 +1,183 @@
+#!/usr/bin/python
+#
+# Copyright The OpenTelemetry Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import absolute_import
+
+import logging
+import sys
+
+import django
+from django.db import connection
+from django.db.backends.utils import CursorDebugWrapper
+
+if sys.version_info.major <= 2:
+    import urllib
+
+    url_quote_fn = urllib.quote  # pylint: disable=maybe-no-member
+else:
+    import urllib.parse
+
+    url_quote_fn = urllib.parse.quote
+
+try:
+    from opentelemetry.trace.propagation.tracecontext import (
+        TraceContextTextMapPropagator,
+    )
+
+    propagator = TraceContextTextMapPropagator()
+except ImportError:
+    propagator = None
+
+django_version = django.get_version()
+logger = logging.getLogger(__name__)
+
+KEY_VALUE_DELIMITER = ","
+
+
+class SqlCommenter:
+    """
+    Middleware to append a comment to each database query with details about
+    the framework and the execution context.
+    """
+
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        with connection.execute_wrapper(QueryWrapper(request)):
+            return self.get_response(request)
+
+
+class QueryWrapper:
+    def __init__(self, request):
+        self.request = request
+
+    def __call__(self, execute, sql, params, many, context):
+        # pylint: disable-msg=too-many-locals
+        _with_framework = getattr(
+            django.conf.settings, "SQLCOMMENTER_WITH_FRAMEWORK", True
+        )
+        _with_controller = getattr(
+            django.conf.settings, "SQLCOMMENTER_WITH_CONTROLLER", True
+        )
+        _with_route = getattr(
+            django.conf.settings, "SQLCOMMENTER_WITH_ROUTE", True
+        )
+        _with_app_name = getattr(
+            django.conf.settings, "SQLCOMMENTER_WITH_APP_NAME", False
+        )
+        _with_opencensus = getattr(
+            django.conf.settings, "SQLCOMMENTER_WITH_OPENCENSUS", False
+        )
+        _with_opentelemetry = getattr(
+            django.conf.settings, "SQLCOMMENTER_WITH_OPENTELEMETRY", False
+        )
+        _with_db_driver = getattr(
+            django.conf.settings, "SQLCOMMENTER_WITH_DB_DRIVER", False
+        )
+
+        if _with_opencensus and _with_opentelemetry:
+            logger.warning(
+                "SQLCOMMENTER_WITH_OPENCENSUS and SQLCOMMENTER_WITH_OPENTELEMETRY were enabled. "
+                "Only use one to avoid unexpected behavior"
+            )
+
+        _db_driver = context["connection"].settings_dict.get("ENGINE", "")
+        _resolver_match = self.request.resolver_match
+
+        _sql_comment = _generate_sql_comment(
+            # Information about the controller.
+            controller=_resolver_match.view_name
+            if _resolver_match and _with_controller
+            else None,
+            # route is the pattern that matched a request with a controller i.e. the regex
+            # See https://docs.djangoproject.com/en/stable/ref/urlresolvers/#django.urls.ResolverMatch.route
+            # getattr() because the attribute doesn't exist in Django < 2.2.
+            route=getattr(_resolver_match, "route", None)
+            if _resolver_match and _with_route
+            else None,
+            # app_name is the application namespace for the URL pattern that matches the URL.
+            # See https://docs.djangoproject.com/en/stable/ref/urlresolvers/#django.urls.ResolverMatch.app_name
+            app_name=(_resolver_match.app_name or None)
+            if _resolver_match and _with_app_name
+            else None,
+            # Framework centric information.
+            framework=("django:%s" % django_version)
+            if _with_framework
+            else None,
+            # Information about the database and driver.
+            db_driver=_db_driver if _with_db_driver else None,
+            **_get_opentelemetry_values() if _with_opentelemetry else {}
+        )
+
+        # TODO: MySQL truncates logs > 1024B so prepend comments
+        # instead of statements, if the engine is MySQL.
+        # See:
+        #  * https://github.com/basecamp/marginalia/issues/61
+        #  * https://github.com/basecamp/marginalia/pull/80
+        sql += _sql_comment
+
+        # Add the query to the query log if debugging.
+        if context["cursor"].__class__ is CursorDebugWrapper:
+            context["connection"].queries_log.append(sql)
+
+        return execute(sql, params, many, context)
+
+
+def _generate_sql_comment(**meta):
+    """
+    Return a SQL comment with comma delimited key=value pairs created from
+    **meta kwargs.
+    """
+    if not meta:  # No entries added.
+        return ""
+
+    # Sort the keywords to ensure that caching works and that testing is
+    # deterministic. It eases visual inspection as well.
+    return (
+        " /*"
+        + KEY_VALUE_DELIMITER.join(
+            "{}={!r}".format(_url_quote(key), _url_quote(value))
+            for key, value in sorted(meta.items())
+            if value is not None
+        )
+        + "*/"
+    )
+
+
+def _url_quote(value):
+    if not isinstance(value, (str, bytes)):
+        return value
+    _quoted = url_quote_fn(value)
+    # Since SQL uses '%' as a keyword, '%' is a by-product of url quoting
+    # e.g. foo,bar --> foo%2Cbar
+    # thus in our quoting, we need to escape it too to finally give
+    #      foo,bar --> foo%%2Cbar
+    return _quoted.replace("%", "%%")
+
+
+def _get_opentelemetry_values():
+    """
+    Return the OpenTelemetry Trace and Span IDs if Span ID is set in the
+    OpenTelemetry execution context.
+    """
+    # pylint: disable=no-else-return
+    if propagator:
+        # Insert the W3C TraceContext generated
+        _headers = {}
+        propagator.inject(_headers)
+        return _headers
+    else:
+        raise ImportError("OpenTelemetry is not installed.")

instrumentation/opentelemetry-instrumentation-django/tests/test_middleware.py

@@ -103,11 +103,11 @@ def setUp(self):
         )
         self.env_patch.start()
         self.exclude_patch = patch(
-            "opentelemetry.instrumentation.django.middleware._DjangoMiddleware._excluded_urls",
+            "opentelemetry.instrumentation.django.middleware.otel_middleware._DjangoMiddleware._excluded_urls",
             get_excluded_urls("DJANGO"),
         )
         self.traced_patch = patch(
-            "opentelemetry.instrumentation.django.middleware._DjangoMiddleware._traced_request_attrs",
+            "opentelemetry.instrumentation.django.middleware.otel_middleware._DjangoMiddleware._traced_request_attrs",
             get_traced_request_attrs("DJANGO"),
         )
         self.exclude_patch.start()

instrumentation/opentelemetry-instrumentation-django/tests/test_middleware_asgi.py

@@ -104,11 +104,11 @@ def setUp(self):
         )
         self.env_patch.start()
         self.exclude_patch = patch(
-            "opentelemetry.instrumentation.django.middleware._DjangoMiddleware._excluded_urls",
+            "opentelemetry.instrumentation.django.middleware.otel_middleware._DjangoMiddleware._excluded_urls",
             get_excluded_urls("DJANGO"),
         )
         self.traced_patch = patch(
-            "opentelemetry.instrumentation.django.middleware._DjangoMiddleware._traced_request_attrs",
+            "opentelemetry.instrumentation.django.middleware.otel_middleware._DjangoMiddleware._traced_request_attrs",
             get_traced_request_attrs("DJANGO"),
         )
         self.exclude_patch.start()

instrumentation/opentelemetry-instrumentation-django/tests/test_sqlcommenter.py

@@ -0,0 +1,100 @@
+# Copyright The OpenTelemetry Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=E0611
+
+from unittest.mock import MagicMock, patch
+
+import django
+from django import VERSION, conf
+from django.http import HttpResponse
+from django.test.utils import setup_test_environment, teardown_test_environment
+
+from opentelemetry.instrumentation.django import DjangoInstrumentor
+from opentelemetry.instrumentation.django.middleware.sqlcommenter_middleware import (
+    QueryWrapper,
+)
+from opentelemetry.test.test_base import TestBase
+from opentelemetry.test.wsgitestutil import WsgiTestBase
+
+DJANGO_2_0 = VERSION >= (2, 0)
+
+_django_instrumentor = DjangoInstrumentor()
+
+
+class TestMiddleware(TestBase, WsgiTestBase):
+    @classmethod
+    def setUpClass(cls):
+        conf.settings.configure(
+            SQLCOMMENTER_WITH_OPENTELEMETRY=True,
+            SQLCOMMENTER_WITH_FRAMEWORK=False,
+        )
+        super().setUpClass()
+
+    def setUp(self):
+        super().setUp()
+        setup_test_environment()
+        _django_instrumentor.instrument(is_sql_commentor_enabled=True)
+
+    def tearDown(self):
+        super().tearDown()
+        teardown_test_environment()
+        _django_instrumentor.uninstrument()
+
+    @classmethod
+    def tearDownClass(cls):
+        super().tearDownClass()
+        conf.settings = conf.LazySettings()
+
+    @patch(
+        "opentelemetry.instrumentation.django.middleware.sqlcommenter_middleware.SqlCommenter"
+    )
+    def test_middleware_added(self, sqlcommenter_middleware):
+        instance = sqlcommenter_middleware.return_value
+        instance.get_response = HttpResponse()
+        if DJANGO_2_0:
+            middleware = django.conf.settings.MIDDLEWARE
+        else:
+            middleware = django.conf.settings.MIDDLEWARE_CLASSES
+        self.assertTrue(
+            "opentelemetry.instrumentation.django.middleware.sqlcommenter_middleware.SqlCommenter"
+            in middleware
+        )
+
+    @patch(
+        "opentelemetry.instrumentation.django.middleware.sqlcommenter_middleware._get_opentelemetry_values"
+    )
+    def test_query_wrapper(self, trace_capture):
+        requests_mock = MagicMock()
+        requests_mock.resolver_match.view_name = "view"
+        requests_mock.resolver_match.route = "route"
+
+        trace_capture.return_value = {
+            "traceparent": "*traceparent='00-000000000000000000000000deadbeef-000000000000beef-00"
+        }
+        qw_instance = QueryWrapper(requests_mock)
+        execute_mock_obj = MagicMock()
+        qw_instance(
+            execute_mock_obj,
+            "Select 1",
+            MagicMock("test"),
+            MagicMock("test1"),
+            MagicMock(),
+        )
+        output_sql = execute_mock_obj.call_args[0][0]
+        self.assertEqual(
+            output_sql,
+            "Select 1 /*controller='view',route='route',traceparent='%%2Atraceparent%%3D%%2700-0000000"
+            "00000000000000000deadbeef-000000000000beef-00'*/",
+        )