From 0ba92103a556dfd9f01e1b3e984102069ba98d7a Mon Sep 17 00:00:00 2001 From: Hasier Rodriguez Date: Wed, 31 Oct 2018 17:58:25 +0000 Subject: [PATCH 1/5] Add params and functions to allow recording and streaming SQL queries --- aws_xray_sdk/core/recorder.py | 15 ++++++++++++++- aws_xray_sdk/ext/dbapi2.py | 10 ++++++---- aws_xray_sdk/ext/django/apps.py | 1 + aws_xray_sdk/ext/django/conf.py | 1 + tests/ext/psycopg2/test_psycopg2.py | 25 ++++++++++++++++++++++--- 5 files changed, 44 insertions(+), 8 deletions(-) diff --git a/aws_xray_sdk/core/recorder.py b/aws_xray_sdk/core/recorder.py index 514f83eb..a34a8e8e 100644 --- a/aws_xray_sdk/core/recorder.py +++ b/aws_xray_sdk/core/recorder.py @@ -72,6 +72,7 @@ def __init__(self): self._dynamic_naming = None self._aws_metadata = copy.deepcopy(XRAY_META) self._origin = None + self._stream_sql = False if type(self.sampler).__name__ == 'DefaultSampler': self.sampler.load_settings(DaemonConfig(), self.context) @@ -81,7 +82,8 @@ def configure(self, sampling=None, plugins=None, daemon_address=None, service=None, context=None, emitter=None, streaming=None, dynamic_naming=None, streaming_threshold=None, - max_trace_back=None, sampler=None): + max_trace_back=None, sampler=None, + stream_sql=False): """Configure global X-Ray recorder. Configure needs to run before patching thrid party libraries @@ -130,6 +132,7 @@ class to have your own implementation of the streaming process. maximum number of subsegments within a segment. :param int max_trace_back: The maxinum number of stack traces recorded by auto-capture. Lower this if a single document becomes too large. + :param bool stream_sql: Whether SQL query texts should be streamed. Environment variables AWS_XRAY_DAEMON_ADDRESS, AWS_XRAY_CONTEXT_MISSING and AWS_XRAY_TRACING_NAME respectively overrides arguments @@ -159,6 +162,8 @@ class to have your own implementation of the streaming process. self.streaming_threshold = streaming_threshold if max_trace_back: self.max_trace_back = max_trace_back + if stream_sql is not None: + self.stream_sql = stream_sql if plugins: plugin_modules = get_plugin_modules(plugins) @@ -548,3 +553,11 @@ def max_trace_back(self): @max_trace_back.setter def max_trace_back(self, value): self._max_trace_back = value + + @property + def stream_sql(self): + return self._stream_sql + + @stream_sql.setter + def stream_sql(self, value): + self._stream_sql = value diff --git a/aws_xray_sdk/ext/dbapi2.py b/aws_xray_sdk/ext/dbapi2.py index c3ed8241..8ae4df49 100644 --- a/aws_xray_sdk/ext/dbapi2.py +++ b/aws_xray_sdk/ext/dbapi2.py @@ -43,23 +43,23 @@ def __enter__(self): @xray_recorder.capture() def execute(self, query, *args, **kwargs): - add_sql_meta(self._xray_meta) + add_sql_meta(self._xray_meta, query) return self.__wrapped__.execute(query, *args, **kwargs) @xray_recorder.capture() def executemany(self, query, *args, **kwargs): - add_sql_meta(self._xray_meta) + add_sql_meta(self._xray_meta, query) return self.__wrapped__.executemany(query, *args, **kwargs) @xray_recorder.capture() def callproc(self, proc, args): - add_sql_meta(self._xray_meta) + add_sql_meta(self._xray_meta, proc) return self.__wrapped__.callproc(proc, args) -def add_sql_meta(meta): +def add_sql_meta(meta, query): subsegment = xray_recorder.current_subsegment() @@ -72,5 +72,7 @@ def add_sql_meta(meta): sql_meta = copy.copy(meta) if sql_meta.get('name', None): del sql_meta['name'] + if xray_recorder.stream_sql: + sql_meta['sanitized_query'] = query subsegment.set_sql(sql_meta) subsegment.namespace = 'remote' diff --git a/aws_xray_sdk/ext/django/apps.py b/aws_xray_sdk/ext/django/apps.py index 3166fe20..1c1412be 100644 --- a/aws_xray_sdk/ext/django/apps.py +++ b/aws_xray_sdk/ext/django/apps.py @@ -36,6 +36,7 @@ def ready(self): dynamic_naming=settings.DYNAMIC_NAMING, streaming_threshold=settings.STREAMING_THRESHOLD, max_trace_back=settings.MAX_TRACE_BACK, + stream_sql=settings.STREAM_SQL, ) # if turned on subsegment will be generated on diff --git a/aws_xray_sdk/ext/django/conf.py b/aws_xray_sdk/ext/django/conf.py index 5cd77bff..6730f74e 100644 --- a/aws_xray_sdk/ext/django/conf.py +++ b/aws_xray_sdk/ext/django/conf.py @@ -14,6 +14,7 @@ 'DYNAMIC_NAMING': None, 'STREAMING_THRESHOLD': None, 'MAX_TRACE_BACK': None, + 'STREAM_SQL': False, } XRAY_NAMESPACE = 'XRAY_RECORDER' diff --git a/tests/ext/psycopg2/test_psycopg2.py b/tests/ext/psycopg2/test_psycopg2.py index 3d275408..46e242b4 100644 --- a/tests/ext/psycopg2/test_psycopg2.py +++ b/tests/ext/psycopg2/test_psycopg2.py @@ -12,20 +12,34 @@ patch(('psycopg2',)) -@pytest.fixture(autouse=True) -def construct_ctx(): +@pytest.fixture( + autouse=True, + params=[ + False, + True, + ], +) +def construct_ctx(request): """ Clean up context storage on each test run and begin a segment so that later subsegment can be attached. After each test run it cleans up context storage again. """ - xray_recorder.configure(service='test', sampling=False, context=Context()) + xray_recorder.configure(service='test', sampling=False, context=Context(), stream_sql=request.param) xray_recorder.clear_trace_entities() xray_recorder.begin_segment('name') yield xray_recorder.clear_trace_entities() +def _assert_query(sql_meta, query): + if xray_recorder.stream_sql: + assert 'sanitized_query' in sql_meta + assert sql_meta['sanitized_query'] == query + else: + assert 'sanitized_query' not in sql_meta + + def test_execute_dsn_kwargs(): q = 'SELECT 1' with testing.postgresql.Postgresql() as postgresql: @@ -46,6 +60,7 @@ def test_execute_dsn_kwargs(): assert sql['user'] == dsn['user'] assert sql['url'] == url assert sql['database_version'] + _assert_query(sql, q) def test_execute_dsn_kwargs_alt_dbname(): @@ -72,6 +87,7 @@ def test_execute_dsn_kwargs_alt_dbname(): assert sql['user'] == dsn['user'] assert sql['url'] == url assert sql['database_version'] + _assert_query(sql, q) def test_execute_dsn_string(): @@ -94,6 +110,7 @@ def test_execute_dsn_string(): assert sql['user'] == dsn['user'] assert sql['url'] == url assert sql['database_version'] + _assert_query(sql, q) def test_execute_in_pool(): @@ -117,6 +134,7 @@ def test_execute_in_pool(): assert sql['user'] == dsn['user'] assert sql['url'] == url assert sql['database_version'] + _assert_query(sql, q) def test_execute_bad_query(): @@ -145,6 +163,7 @@ def test_execute_bad_query(): exception = subsegment.cause['exceptions'][0] assert exception.type == 'ProgrammingError' + _assert_query(sql, q) def test_register_extensions(): From ea9644f2c257f893712c5ccc1d9722f5fc48bfc7 Mon Sep 17 00:00:00 2001 From: Hasier Rodriguez Date: Thu, 1 Nov 2018 11:57:04 +0000 Subject: [PATCH 2/5] Set general default to True and adapt SQLAlchemy to follow the flag --- aws_xray_sdk/core/recorder.py | 2 +- aws_xray_sdk/ext/sqlalchemy/util/decorators.py | 3 ++- tests/ext/flask_sqlalchemy/test_query.py | 13 +++++++++---- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/aws_xray_sdk/core/recorder.py b/aws_xray_sdk/core/recorder.py index a34a8e8e..16f18012 100644 --- a/aws_xray_sdk/core/recorder.py +++ b/aws_xray_sdk/core/recorder.py @@ -83,7 +83,7 @@ def configure(self, sampling=None, plugins=None, context=None, emitter=None, streaming=None, dynamic_naming=None, streaming_threshold=None, max_trace_back=None, sampler=None, - stream_sql=False): + stream_sql=True): """Configure global X-Ray recorder. Configure needs to run before patching thrid party libraries diff --git a/aws_xray_sdk/ext/sqlalchemy/util/decorators.py b/aws_xray_sdk/ext/sqlalchemy/util/decorators.py index a32a891c..afad1280 100644 --- a/aws_xray_sdk/ext/sqlalchemy/util/decorators.py +++ b/aws_xray_sdk/ext/sqlalchemy/util/decorators.py @@ -47,7 +47,8 @@ def wrapper(*args, **kw): if isinstance(arg, XRayQuery): try: sql = parse_bind(arg.session.bind) - sql['sanitized_query'] = str(arg) + if xray_recorder.stream_sql: + sql['sanitized_query'] = str(arg) except Exception: sql = None if sql is not None: diff --git a/tests/ext/flask_sqlalchemy/test_query.py b/tests/ext/flask_sqlalchemy/test_query.py index efb13aac..69f0bd7a 100644 --- a/tests/ext/flask_sqlalchemy/test_query.py +++ b/tests/ext/flask_sqlalchemy/test_query.py @@ -22,10 +22,15 @@ class User(db.Model): password = db.Column(db.String(255), nullable=False) -@pytest.fixture() -def session(): +@pytest.fixture( + params=[ + False, + True, + ], +) +def session(request): """Test Fixture to Create DataBase Tables and start a trace segment""" - xray_recorder.configure(service='test', sampling=False, context=Context()) + xray_recorder.configure(service='test', sampling=False, context=Context(), stream_sql=request.param) xray_recorder.clear_trace_entities() xray_recorder.begin_segment('SQLAlchemyTest') db.create_all() @@ -41,8 +46,8 @@ def test_all(capsys, session): User.query.all() subsegment = find_subsegment_by_annotation(xray_recorder.current_segment(), 'sqlalchemy', 'sqlalchemy.orm.query.all') assert subsegment['annotations']['sqlalchemy'] == 'sqlalchemy.orm.query.all' - assert subsegment['sql']['sanitized_query'] assert subsegment['sql']['url'] + assert bool(subsegment['sql'].get('sanitized_query', None)) is xray_recorder.stream_sql def test_add(capsys, session): From 1d1783d58cac6b48da7597a51c8542d3eafb21d4 Mon Sep 17 00:00:00 2001 From: Hasier Rodriguez Date: Wed, 7 Nov 2018 17:36:05 +0000 Subject: [PATCH 3/5] Update docs --- CHANGELOG.rst | 12 ++++++++---- README.md | 21 ++++++++++++++++++++- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f83e1adb..21ce3194 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,10 @@ CHANGELOG ========= +unreleased +========== +* feature: Stream dbapi2 SQL queries and add flag to toggle their streaming + 2.2.0 ===== * feature: Added context managers on segment/subsegment capture. `PR97 `_. @@ -32,11 +36,11 @@ CHANGELOG * **Breaking**: The original sampling modules for local defined rules are moved from `models.sampling` to `models.sampling.local`. * **Breaking**: The default behavior of `patch_all` changed to selectively patches libraries to avoid double patching. You can use `patch_all(double_patch=True)` to force it to patch ALL supported libraries. See more details on `ISSUE63 `_ * **Breaking**: The latest `botocore` that has new X-Ray service API `GetSamplingRules` and `GetSamplingTargets` are required. -* **Breaking**: Version 2.x doesn't support pynamodb and aiobotocore as it requires botocore >= 1.11.3 which isn’t currently supported by the pynamodb and aiobotocore libraries. Please continue to use version 1.x if you’re using pynamodb or aiobotocore until those haven been updated to use botocore > = 1.11.3. +* **Breaking**: Version 2.x doesn't support pynamodb and aiobotocore as it requires botocore >= 1.11.3 which isn’t currently supported by the pynamodb and aiobotocore libraries. Please continue to use version 1.x if you’re using pynamodb or aiobotocore until those haven been updated to use botocore > = 1.11.3. * feature: Environment variable `AWS_XRAY_DAEMON_ADDRESS` now takes an additional notation in `tcp:127.0.0.1:2000 udp:127.0.0.2:2001` to set TCP and UDP destination separately. By default it assumes a X-Ray daemon listening to both UDP and TCP traffic on `127.0.0.1:2000`. * feature: Added MongoDB python client support. `PR65 `_. -* bugfix: Support binding connection in sqlalchemy as well as engine. `PR78 `_. -* bugfix: Flask middleware safe request teardown. `ISSUE75 `_. +* bugfix: Support binding connection in sqlalchemy as well as engine. `PR78 `_. +* bugfix: Flask middleware safe request teardown. `ISSUE75 `_. 1.1.2 @@ -68,7 +72,7 @@ CHANGELOG * bugfix: Fixed an issue where arbitrary fields in trace header being dropped when calling downstream. * bugfix: Fixed a compatibility issue between botocore and httplib patcher. `ISSUE48 `_. * bugfix: Fixed a typo in sqlalchemy decorators. `PR50 `_. -* Updated `README` with more usage examples. +* Updated `README` with more usage examples. 0.97 ==== diff --git a/README.md b/README.md index c02e2385..86970710 100644 --- a/README.md +++ b/README.md @@ -251,6 +251,20 @@ with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: pass ``` +### Trace SQL queries +By default, if no other value is provided to `.configure()`, SQL trace streaming is enabled +for all the supported DB engines. Those currently are: +- Any engine attached to the Django ORM. +- Any engine attached to SQLAlchemy. +- SQLite3. + +The behaviour can be toggled by sending the appropriate `stream_sql` value, for example: +```python +from aws_xray_sdk.core import xray_recorder + +xray_recorder.configure(service='fallback_name', stream_sql=True) +``` + ### Patch third-party libraries ```python @@ -260,7 +274,8 @@ libs_to_patch = ('boto3', 'mysql', 'requests') patch(libs_to_patch) ``` -### Add Django middleware +### Django +#### Add middleware In django settings.py, use the following. @@ -275,6 +290,10 @@ MIDDLEWARE = [ # ... other middlewares ] ``` +#### SQL tracing +If Django's ORM is patched - either using the `AUTO_INSTRUMENT = True` in your settings file +or explicitly calling `patch_db()` - the SQL query trace streaming can be enabled or disabled +updating the `STREAM_SQL` variable in your settings file. ### Add Flask middleware From 1bf947205d3daf9bb12b361793dff717c36d7db7 Mon Sep 17 00:00:00 2001 From: Hasier Rodriguez Date: Thu, 27 Dec 2018 09:58:59 +0000 Subject: [PATCH 4/5] Set stream_sql to True by default --- aws_xray_sdk/core/recorder.py | 2 +- aws_xray_sdk/ext/django/conf.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aws_xray_sdk/core/recorder.py b/aws_xray_sdk/core/recorder.py index 16f18012..d0103041 100644 --- a/aws_xray_sdk/core/recorder.py +++ b/aws_xray_sdk/core/recorder.py @@ -72,7 +72,7 @@ def __init__(self): self._dynamic_naming = None self._aws_metadata = copy.deepcopy(XRAY_META) self._origin = None - self._stream_sql = False + self._stream_sql = True if type(self.sampler).__name__ == 'DefaultSampler': self.sampler.load_settings(DaemonConfig(), self.context) diff --git a/aws_xray_sdk/ext/django/conf.py b/aws_xray_sdk/ext/django/conf.py index 6730f74e..d48c5b46 100644 --- a/aws_xray_sdk/ext/django/conf.py +++ b/aws_xray_sdk/ext/django/conf.py @@ -14,7 +14,7 @@ 'DYNAMIC_NAMING': None, 'STREAMING_THRESHOLD': None, 'MAX_TRACE_BACK': None, - 'STREAM_SQL': False, + 'STREAM_SQL': True, } XRAY_NAMESPACE = 'XRAY_RECORDER' From 202cc604e1fb7b863473756b0c6e515f458eb691 Mon Sep 17 00:00:00 2001 From: Hasier Rodriguez Date: Mon, 7 Jan 2019 14:08:40 +0000 Subject: [PATCH 5/5] Unpatch dbapi2, patch use custom cursor for Django and chunked_cursor --- CHANGELOG.rst | 2 +- README.md | 5 +- aws_xray_sdk/ext/dbapi2.py | 10 ++-- aws_xray_sdk/ext/django/db.py | 56 ++++++++++++++++--- tests/ext/django/test_db.py | 87 +++++++++++++++++++++++++++++ tests/ext/psycopg2/test_psycopg2.py | 25 +-------- tox.ini | 1 + 7 files changed, 145 insertions(+), 41 deletions(-) create mode 100644 tests/ext/django/test_db.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 21ce3194..00813e17 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,7 +4,7 @@ CHANGELOG unreleased ========== -* feature: Stream dbapi2 SQL queries and add flag to toggle their streaming +* feature: Stream Django ORM SQL queries and add flag to toggle their streaming 2.2.0 ===== diff --git a/README.md b/README.md index 86970710..13804828 100644 --- a/README.md +++ b/README.md @@ -256,7 +256,6 @@ By default, if no other value is provided to `.configure()`, SQL trace streaming for all the supported DB engines. Those currently are: - Any engine attached to the Django ORM. - Any engine attached to SQLAlchemy. -- SQLite3. The behaviour can be toggled by sending the appropriate `stream_sql` value, for example: ```python @@ -292,8 +291,8 @@ MIDDLEWARE = [ ``` #### SQL tracing If Django's ORM is patched - either using the `AUTO_INSTRUMENT = True` in your settings file -or explicitly calling `patch_db()` - the SQL query trace streaming can be enabled or disabled -updating the `STREAM_SQL` variable in your settings file. +or explicitly calling `patch_db()` - the SQL query trace streaming can then be enabled or +disabled updating the `STREAM_SQL` variable in your settings file. It is enabled by default. ### Add Flask middleware diff --git a/aws_xray_sdk/ext/dbapi2.py b/aws_xray_sdk/ext/dbapi2.py index 8ae4df49..c3ed8241 100644 --- a/aws_xray_sdk/ext/dbapi2.py +++ b/aws_xray_sdk/ext/dbapi2.py @@ -43,23 +43,23 @@ def __enter__(self): @xray_recorder.capture() def execute(self, query, *args, **kwargs): - add_sql_meta(self._xray_meta, query) + add_sql_meta(self._xray_meta) return self.__wrapped__.execute(query, *args, **kwargs) @xray_recorder.capture() def executemany(self, query, *args, **kwargs): - add_sql_meta(self._xray_meta, query) + add_sql_meta(self._xray_meta) return self.__wrapped__.executemany(query, *args, **kwargs) @xray_recorder.capture() def callproc(self, proc, args): - add_sql_meta(self._xray_meta, proc) + add_sql_meta(self._xray_meta) return self.__wrapped__.callproc(proc, args) -def add_sql_meta(meta, query): +def add_sql_meta(meta): subsegment = xray_recorder.current_subsegment() @@ -72,7 +72,5 @@ def add_sql_meta(meta, query): sql_meta = copy.copy(meta) if sql_meta.get('name', None): del sql_meta['name'] - if xray_recorder.stream_sql: - sql_meta['sanitized_query'] = query subsegment.set_sql(sql_meta) subsegment.namespace = 'remote' diff --git a/aws_xray_sdk/ext/django/db.py b/aws_xray_sdk/ext/django/db.py index 0a2c80d6..fdf7e27a 100644 --- a/aws_xray_sdk/ext/django/db.py +++ b/aws_xray_sdk/ext/django/db.py @@ -1,29 +1,62 @@ +import copy import logging import importlib from django.db import connections +from aws_xray_sdk.core import xray_recorder from aws_xray_sdk.ext.dbapi2 import XRayTracedCursor log = logging.getLogger(__name__) def patch_db(): - for conn in connections.all(): module = importlib.import_module(conn.__module__) _patch_conn(getattr(module, conn.__class__.__name__)) -def _patch_conn(conn): - - attr = '_xray_original_cursor' +class DjangoXRayTracedCursor(XRayTracedCursor): + def execute(self, query, *args, **kwargs): + if xray_recorder.stream_sql: + _previous_meta = copy.copy(self._xray_meta) + self._xray_meta['sanitized_query'] = query + result = super(DjangoXRayTracedCursor, self).execute(query, *args, **kwargs) + if xray_recorder.stream_sql: + self._xray_meta = _previous_meta + return result + + def executemany(self, query, *args, **kwargs): + if xray_recorder.stream_sql: + _previous_meta = copy.copy(self._xray_meta) + self._xray_meta['sanitized_query'] = query + result = super(DjangoXRayTracedCursor, self).executemany(query, *args, **kwargs) + if xray_recorder.stream_sql: + self._xray_meta = _previous_meta + return result + + def callproc(self, proc, args): + if xray_recorder.stream_sql: + _previous_meta = copy.copy(self._xray_meta) + self._xray_meta['sanitized_query'] = proc + result = super(DjangoXRayTracedCursor, self).callproc(proc, args) + if xray_recorder.stream_sql: + self._xray_meta = _previous_meta + return result + + +def _patch_cursor(cursor_name, conn): + attr = '_xray_original_{}'.format(cursor_name) if hasattr(conn, attr): - log.debug('django built-in db already patched') + log.debug('django built-in db {} already patched'.format(cursor_name)) + return + + if not hasattr(conn, cursor_name): + log.debug('django built-in db does not have {}'.format(cursor_name)) return - setattr(conn, attr, conn.cursor) + setattr(conn, attr, getattr(conn, cursor_name)) meta = {} @@ -45,7 +78,12 @@ def cursor(self, *args, **kwargs): if user: meta['user'] = user - return XRayTracedCursor( - self._xray_original_cursor(*args, **kwargs), meta) + original_cursor = getattr(self, attr)(*args, **kwargs) + return DjangoXRayTracedCursor(original_cursor, meta) + + setattr(conn, cursor_name, cursor) - conn.cursor = cursor + +def _patch_conn(conn): + _patch_cursor('cursor', conn) + _patch_cursor('chunked_cursor', conn) diff --git a/tests/ext/django/test_db.py b/tests/ext/django/test_db.py new file mode 100644 index 00000000..1c3e5439 --- /dev/null +++ b/tests/ext/django/test_db.py @@ -0,0 +1,87 @@ +import django + +import pytest + +from aws_xray_sdk.core import xray_recorder +from aws_xray_sdk.core.context import Context +from aws_xray_sdk.ext.django.db import patch_db + + +@pytest.fixture(scope='module', autouse=True) +def setup(): + django.setup() + xray_recorder.configure(context=Context(), + context_missing='LOG_ERROR') + patch_db() + + +@pytest.fixture(scope='module') +def user_class(setup): + from django.db import models + from django_fake_model import models as f + + class User(f.FakeModel): + name = models.CharField(max_length=255) + password = models.CharField(max_length=255) + + return User + + +@pytest.fixture( + autouse=True, + params=[ + False, + True, + ] +) +@pytest.mark.django_db +def func_setup(request, user_class): + xray_recorder.stream_sql = request.param + xray_recorder.clear_trace_entities() + xray_recorder.begin_segment('name') + try: + user_class.create_table() + yield + finally: + xray_recorder.clear_trace_entities() + try: + user_class.delete_table() + finally: + xray_recorder.end_segment() + + +def _assert_query(sql_meta): + if xray_recorder.stream_sql: + assert 'sanitized_query' in sql_meta + assert sql_meta['sanitized_query'] + assert sql_meta['sanitized_query'].startswith('SELECT') + else: + if 'sanitized_query' in sql_meta: + assert sql_meta['sanitized_query'] + # Django internally executes queries for table checks, ignore those + assert not sql_meta['sanitized_query'].startswith('SELECT') + + +def test_all(user_class): + """ Test calling all() on get all records. + Verify we run the query and return the SQL as metadata""" + # Materialising the query executes the SQL + list(user_class.objects.all()) + subsegment = xray_recorder.current_segment().subsegments[-1] + sql = subsegment.sql + assert sql['database_type'] == 'sqlite' + _assert_query(sql) + + +def test_filter(user_class): + """ Test calling filter() to get filtered records. + Verify we run the query and return the SQL as metadata""" + # Materialising the query executes the SQL + list(user_class.objects.filter(password='mypassword!').all()) + subsegment = xray_recorder.current_segment().subsegments[-1] + sql = subsegment.sql + assert sql['database_type'] == 'sqlite' + _assert_query(sql) + if xray_recorder.stream_sql: + assert 'mypassword!' not in sql['sanitized_query'] + assert '"password" = %s' in sql['sanitized_query'] diff --git a/tests/ext/psycopg2/test_psycopg2.py b/tests/ext/psycopg2/test_psycopg2.py index 7b833097..c491d706 100644 --- a/tests/ext/psycopg2/test_psycopg2.py +++ b/tests/ext/psycopg2/test_psycopg2.py @@ -12,34 +12,20 @@ patch(('psycopg2',)) -@pytest.fixture( - autouse=True, - params=[ - False, - True, - ], -) -def construct_ctx(request): +@pytest.fixture(autouse=True) +def construct_ctx(): """ Clean up context storage on each test run and begin a segment so that later subsegment can be attached. After each test run it cleans up context storage again. """ - xray_recorder.configure(service='test', sampling=False, context=Context(), stream_sql=request.param) + xray_recorder.configure(service='test', sampling=False, context=Context()) xray_recorder.clear_trace_entities() xray_recorder.begin_segment('name') yield xray_recorder.clear_trace_entities() -def _assert_query(sql_meta, query): - if xray_recorder.stream_sql: - assert 'sanitized_query' in sql_meta - assert sql_meta['sanitized_query'] == query - else: - assert 'sanitized_query' not in sql_meta - - def test_execute_dsn_kwargs(): q = 'SELECT 1' with testing.postgresql.Postgresql() as postgresql: @@ -60,7 +46,6 @@ def test_execute_dsn_kwargs(): assert sql['user'] == dsn['user'] assert sql['url'] == url assert sql['database_version'] - _assert_query(sql, q) def test_execute_dsn_kwargs_alt_dbname(): @@ -87,7 +72,6 @@ def test_execute_dsn_kwargs_alt_dbname(): assert sql['user'] == dsn['user'] assert sql['url'] == url assert sql['database_version'] - _assert_query(sql, q) def test_execute_dsn_string(): @@ -110,7 +94,6 @@ def test_execute_dsn_string(): assert sql['user'] == dsn['user'] assert sql['url'] == url assert sql['database_version'] - _assert_query(sql, q) def test_execute_in_pool(): @@ -134,7 +117,6 @@ def test_execute_in_pool(): assert sql['user'] == dsn['user'] assert sql['url'] == url assert sql['database_version'] - _assert_query(sql, q) def test_execute_bad_query(): @@ -163,7 +145,6 @@ def test_execute_bad_query(): exception = subsegment.cause['exceptions'][0] assert exception.type == 'ProgrammingError' - _assert_query(sql, q) def test_register_extensions(): diff --git a/tox.ini b/tox.ini index 1bd78b23..0dbbe0be 100644 --- a/tox.ini +++ b/tox.ini @@ -18,6 +18,7 @@ deps = future # the sdk doesn't support earlier version of django django >= 1.10, <2.0 + django-fake-model pynamodb >= 3.3.1 psycopg2 pg8000