diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d0b96ef5..6b74990a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## v4.0.0 + +**BREAKING** Version 4 of the agent implements a new wire protocol for communicating with +the APM Server. This format is only supported in APM Server 6.5+. + +Further breaking changes: + + * the undocumented `AsyncioHTTPTransport` has been removed. + * the `flush_interval` and `max_queue_size` settings have been removed. + * new settings introduced: `api_request_time` and `api_request_size`. + * Some settings now require a unit for duration or size. See documentation on + configuration for more information. + ## v3.0.1 [Check the diff](https://github.com/elastic/apm-agent-python/compare/v3.0.0...v3.0.1) diff --git a/docs/configuration.asciidoc b/docs/configuration.asciidoc index c4f7831c4..cac11eb0c 100644 --- a/docs/configuration.asciidoc +++ b/docs/configuration.asciidoc @@ -230,10 +230,11 @@ Transactions that match any of the of the configured patterns will be ignored an [options="header"] |============ | Environment | Django/Flask | Default -| `ELASTIC_APM_SERVER_TIMEOUT` | `SERVER_TIMEOUT` | `5` +| `ELASTIC_APM_SERVER_TIMEOUT` | `SERVER_TIMEOUT` | `"5s"` |============ -A timeout in seconds. +A timeout for requests to the APM Server. +The setting has to be provided in *<>*. If a request to the APM server takes longer than the configured timeout, the request is cancelled and the event (exception or transaction) is discarded. Set to `None` to disable timeouts. @@ -369,21 +370,6 @@ It contains the name of the field, and the name of the uploaded file, if provide WARNING: request bodies often contain sensitive values like passwords, credit card numbers etc. If your service handles data like this, we advise to only enable this feature with care. -[float] -[[config-flush-interval]] -==== `flush_interval` - -|============ -| Environment | Django/Flask | Default -| `ELASTIC_APM_FLUSH_INTERVAL` | `FLUSH_INTERVAL` | `10` -|============ - -Interval with which transactions should be sent to the APM server, in seconds. -A lower value will increase the load on your APM server, -while a higher value can increase the memory pressure on your app. -A higher value also impacts the time until transactions are indexed and searchable in Elasticsearch. - - [float] [[config-transaction-max-spans]] ==== `transaction_max_spans` @@ -402,8 +388,8 @@ Setting an upper limit will prevent overloading the agent and the APM server wit ==== `span_frames_min_duration` |============ -| Environment | Django/Flask | Default -| `ELASTIC_APM_SPAN_FRAMES_MIN_DURATION` | `SPAN_FRAMES_MIN_DURATION` | `5` +| Environment | Django/Flask | Default +| `ELASTIC_APM_SPAN_FRAMES_MIN_DURATION` | `SPAN_FRAMES_MIN_DURATION` | `"5ms"` |============ In its default settings, the APM agent will collect a stack trace with every recorded span. @@ -416,21 +402,42 @@ with durations equal or longer than the given value in milliseconds, e.g. 5 mill To disable stack trace collection for spans completely, set the value to `0`. +Except for the special values `-1` and `0`, +this setting has to be provided in *<>*. + [float] -[[config-max-queue-size]] -==== `max_queue_size` +[[config-api-request-size]] +==== `api_request_size` |============ -| Environment | Django/Flask | Default -| `ELASTIC_APM_MAX_QUEUE_SIZE` | `MAX_QUEUE_SIZE` | `500` +| Environment | Django/Flask | Default +| `ELASTIC_APM_API_REQUEST_SIZE` | `API_REQUEST_TIME` | `"724kb"` +|============ + +Maximum queue length of the request buffer before sending the request to the APM server. +A lower value will increase the load on your APM server, +while a higher value can increase the memory pressure of your app. +A higher value also impacts the time until data is indexed and searchable in Elasticsearch. + +This setting is useful to limit memory consumption if you experience a sudden spike of traffic. +It has to be provided in *<>*. + +[float] +[[config-api-request-time]] +==== `api_request_size` + +|============ +| Environment | Django/Flask | Default +| `ELASTIC_APM_API_REQUEST_SIZE` | `API_REQUEST_SIZE` | `"724kb"` |============ -Maximum queue length of transactions before sending transactions to the APM server. +Maximum queue time of the request buffer before sending the request to the APM server. A lower value will increase the load on your APM server, while a higher value can increase the memory pressure of your app. -A higher value also impacts the time until transactions are indexed and searchable in Elasticsearch. +A higher value also impacts the time until data is indexed and searchable in Elasticsearch. This setting is useful to limit memory consumption if you experience a sudden spike of traffic. +It has to be provided in *<>*. [float] [[config-processors]] @@ -551,3 +558,45 @@ By default, the agent verifies the SSL certificate if you use an HTTPS connectio Verification can be disabled by changing this setting to `False`. NOTE: SSL certificate verification is only available in Python 2.7.9+ and Python 3.4.3+. + + +[float] +[[config-formats]] +=== Configuration formats + +Some options require a unit, either duration or size. +These need to be provided in a specific format. + +[float] +[[config-format-duration]] +==== Duration format + +The _duration_ format is used for options like timeouts. +The unit is provided as suffix directly after the number, without and separation by whitespace. + +*Example*: `5ms` + +*Supported units* + + * `ms` (milliseconds) + * `s` (seconds) + * `m` (minutes) + +[float] +[[config-format-size]] +==== Size format + +The _size_ format is used for options like maximum buffer sizes. +The unit is provided as suffix directly after the number, without and separation by whitespace. + + +*Example*: `10kb` + +*Supported units*: + + * `b` (bytes) + * `kb` (kilobytes) + * `mb` (megabytes) + * `gb` (gigabytes) + +NOTE: we use the power-of-two sizing convention, e.g. `1 kilobyte == 1024 bytes` diff --git a/docs/sanitizing-data.asciidoc b/docs/sanitizing-data.asciidoc index 85bbaf42d..2b319b52d 100644 --- a/docs/sanitizing-data.asciidoc +++ b/docs/sanitizing-data.asciidoc @@ -12,12 +12,19 @@ This is an example of a processor that removes the exception stacktrace from an [source,python] ---- +from elasticapm.conf.constants import ERROR +from elasticapm.processors import for_events + +@for_events(ERROR) def my_processor(client, event): if 'exception' in event and 'stacktrace' in event['exception']: event['exception'].pop('stacktrace') return event ---- +You can use the `@for_events` decorator to limit for which event type the processor should be called. +Possible choices are `ERROR`, `TRANSACTION`, and `SPAN`, all of which are defined in `elasticapm.conf.constants`. + To use this processor, update your `ELASTIC_APM` settings like this: [source,python] diff --git a/docs/tuning.asciidoc b/docs/tuning.asciidoc index 39fd76380..caee60605 100644 --- a/docs/tuning.asciidoc +++ b/docs/tuning.asciidoc @@ -47,11 +47,11 @@ holding on to the transaction data in a queue uses memory. If you notice that using the Python agent results in a large increase of memory use, you can use these settings: - * <> to reduce the time between queue flushes - * <> to reduce the maximum size of the queue + * <> to reduce the time between queue flushes + * <> to reduce the maximum size of the queue -The first setting, `flush_interval`, is helpful if you have a sustained high number of transactions. -The second setting, `max_queue_size`, can help if you experience peaks of transactions +The first setting, `api_request_time`, is helpful if you have a sustained high number of transactions. +The second setting, `api_request_size`, can help if you experience peaks of transactions (a large amount of transactions in a short period of time). Keep in mind that reducing the value of either setting will cause the agent to send more HTTP requests to the APM Server, @@ -78,6 +78,6 @@ Another option to reduce overhead of collecting contextual data for spans is to While this contextual data (specifically, the stack trace) can be very useful to pinpoint where exectly the span is caused in your code, it is less interesting for very short spans. You can define a minimal threshold for span duration in milliseconds, -using the <> setting. +using the <> setting. If a span takes less than this duration, no stack frames will be collected for this span. Other contextual information, like the SQL query, will still be available. diff --git a/elasticapm/base.py b/elasticapm/base.py index 2bf1741b1..2341efdff 100644 --- a/elasticapm/base.py +++ b/elasticapm/base.py @@ -12,63 +12,25 @@ from __future__ import absolute_import import datetime +import inspect import logging import os import platform import socket import sys -import threading -import time -import zlib from copy import deepcopy import elasticapm from elasticapm.conf import Config, constants +from elasticapm.conf.constants import ERROR from elasticapm.traces import TransactionsStore, get_transaction -from elasticapm.transport.base import TransportException -from elasticapm.utils import compat, is_master_process -from elasticapm.utils import json_encoder as json -from elasticapm.utils import stacks, varmap +from elasticapm.utils import compat, is_master_process, stacks, varmap from elasticapm.utils.encoding import keyword_field, shorten, transform from elasticapm.utils.module_import import import_string __all__ = ("Client",) -class ClientState(object): - ONLINE = 1 - ERROR = 0 - - def __init__(self): - self.status = self.ONLINE - self.last_check = None - self.retry_number = 0 - - def should_try(self): - if self.status == self.ONLINE: - return True - - interval = min(self.retry_number, 6) ** 2 - - if time.time() - self.last_check > interval: - return True - - return False - - def set_fail(self): - self.status = self.ERROR - self.retry_number += 1 - self.last_check = time.time() - - def set_success(self): - self.status = self.ONLINE - self.last_check = None - self.retry_number = 0 - - def did_fail(self): - return self.status == self.ERROR - - class Client(object): """ The base ElasticAPM client, which handles communication over the @@ -105,13 +67,10 @@ def __init__(self, config=None, **inline): cls = self.__class__ self.logger = logging.getLogger("%s.%s" % (cls.__module__, cls.__name__)) self.error_logger = logging.getLogger("elasticapm.errors") - self.state = ClientState() self.transaction_store = None self.processors = [] self.filter_exception_types_dict = {} - self._send_timer = None - self._transports = {} self._service_info = None self.config = Config(config, inline_dict=inline) @@ -120,7 +79,25 @@ def __init__(self, config=None, **inline): self.error_logger.error(msg) self.config.disable_send = True - self._transport_class = import_string(self.config.transport_class) + headers = { + "Content-Type": "application/x-ndjson", + "Content-Encoding": "gzip", + "User-Agent": "elasticapm-python/%s" % elasticapm.VERSION, + } + + if self.config.secret_token: + headers["Authorization"] = "Bearer %s" % self.config.secret_token + transport_kwargs = { + "metadata": self._build_metadata(), + "headers": headers, + "verify_server_cert": self.config.verify_server_cert, + "timeout": self.config.server_timeout, + "max_flush_time": self.config.api_request_time / 1000.0, + "max_buffer_size": self.config.api_request_size, + } + self._transport = import_string(self.config.transport_class)( + compat.urlparse.urljoin(self.config.server_url, constants.EVENTS_API_PATH), **transport_kwargs + ) for exc_to_filter in self.config.filter_exception_types or []: exc_to_filter_type = exc_to_filter.split(".")[-1] @@ -136,9 +113,12 @@ def __init__(self, config=None, **inline): else: skip_modules = ("elasticapm.",) - def frames_collector_func(): - return self._get_stack_info_for_trace( - stacks.iter_stack_frames(skip_top_modules=skip_modules), + self.transaction_store = TransactionsStore( + frames_collector_func=lambda: list( + stacks.iter_stack_frames(start_frame=inspect.currentframe(), skip_top_modules=skip_modules) + ), + frames_processing_func=lambda frames: self._get_stack_info_for_trace( + frames, library_frame_context_lines=self.config.source_lines_span_library_frames, in_app_frame_context_lines=self.config.source_lines_span_app_frames, with_locals=self.config.collect_local_variables in ("all", "transactions"), @@ -150,15 +130,11 @@ def frames_collector_func(): ), local_var, ), - ) - - self.transaction_store = TransactionsStore( - frames_collector_func=frames_collector_func, - collect_frequency=self.config.flush_interval, + ), + queue_func=self.queue, sample_rate=self.config.transaction_sample_rate, max_spans=self.config.transaction_max_spans, - span_frames_min_duration=self.config.span_frames_min_duration_ms, - max_queue_size=self.config.max_queue_size, + span_frames_min_duration=self.config.span_frames_min_duration, ignore_patterns=self.config.transactions_ignore_patterns, ) self.include_paths_re = stacks.get_path_regex(self.config.include_paths) if self.config.include_paths else None @@ -180,9 +156,9 @@ def capture(self, event_type, date=None, context=None, custom=None, stack=None, ) if data: - url = self.config.server_url + constants.ERROR_API_PATH - self.send(url, **data) - return data["errors"][0]["id"] + # queue data, and flush the queue if this is an unhandled exception + self.queue(ERROR, data, flush=not handled) + return data["id"] def capture_message(self, message=None, param_message=None, **kwargs): """ @@ -208,46 +184,17 @@ def capture_exception(self, exc_info=None, handled=True, **kwargs): """ return self.capture("Exception", exc_info=exc_info, handled=handled, **kwargs) - def send(self, url, **data): - """ - Encodes and sends data to remote URL using configured transport - :param url: URL of endpoint - :param data: dictionary of data to send - """ - if self.config.disable_send or self._filter_exception_type(data): - return - - payload = self.encode(data) - - headers = { - "Content-Type": "application/json", - "Content-Encoding": "deflate", - "User-Agent": "elasticapm-python/%s" % elasticapm.VERSION, - } - - if self.config.secret_token: - headers["Authorization"] = "Bearer %s" % self.config.secret_token - - if not self.state.should_try(): - message = self._get_log_message(payload) - self.error_logger.error(message) + def queue(self, event_type, data, flush=False): + if self.config.disable_send: return - try: - self._send_remote(url=url, data=payload, headers=headers) - except Exception as e: - self.handle_transport_fail(exception=e) - - def encode(self, data): - """ - Serializes ``data`` into a raw string. - """ - return zlib.compress(json.dumps(data).encode("utf8")) - - def decode(self, data): - """ - Unserializes a string, ``data``. - """ - return json.loads(zlib.decompress(data).decode("utf8")) + # Run the data through processors + for processor in self.processors: + if not hasattr(processor, "event_types") or event_type in processor.event_types: + data = processor(self, data) + if flush and is_master_process(): + # don't flush in uWSGI master process to avoid ending up in an unpredictable threading state + flush = False + self._transport.queue(event_type, data, flush) def begin_transaction(self, transaction_type): """Register the start of a transaction on the client @@ -256,85 +203,10 @@ def begin_transaction(self, transaction_type): def end_transaction(self, name=None, result=""): transaction = self.transaction_store.end_transaction(result, name) - if self.transaction_store.should_collect(): - self._collect_transactions() - if not self._send_timer: - # send first batch of data after config._wait_to_first_send - self._start_send_timer(timeout=min(self.config._wait_to_first_send, self.config.flush_interval)) return transaction def close(self): - self._collect_transactions() - if self._send_timer: - self._stop_send_timer() - for url, transport in list(self._transports.items()): - transport.close() - self._transports.pop(url) - - def handle_transport_success(self, **kwargs): - """ - Success handler called by the transport - """ - if kwargs.get("url"): - self.logger.info("Logged error at " + kwargs["url"]) - self.state.set_success() - - def handle_transport_fail(self, exception=None, **kwargs): - """ - Failure handler called by the transport - """ - if isinstance(exception, TransportException): - message = self._get_log_message(exception.data) - self.error_logger.error(exception.args[0]) - else: - # stdlib exception - message = str(exception) - self.error_logger.error( - "Failed to submit message: %r", message, exc_info=getattr(exception, "print_trace", True) - ) - self.state.set_fail() - - def _collect_transactions(self): - self._stop_send_timer() - transactions = [] - if self.transaction_store: - for transaction in self.transaction_store.get_all(): - for processor in self.processors: - transaction = processor(self, transaction) - transactions.append(transaction) - if not transactions: - return - - data = self._build_msg({"transactions": transactions}) - - api_path = constants.TRANSACTIONS_API_PATH - - self.send(self.config.server_url + api_path, **data) - self._start_send_timer() - - def _start_send_timer(self, timeout=None): - timeout = timeout or self.config.flush_interval - self._send_timer = threading.Timer(timeout, self._collect_transactions) - self._send_timer.start() - - def _stop_send_timer(self): - if self._send_timer and not self._send_timer == threading.current_thread(): - self._send_timer.cancel() - if self._send_timer.is_alive(): - self._send_timer.join() - - def _send_remote(self, url, data, headers=None): - if headers is None: - headers = {} - parsed = compat.urlparse.urlparse(url) - transport = self._get_transport(parsed) - if transport.async_mode: - transport.send_async( - data, headers, success_callback=self.handle_transport_success, fail_callback=self.handle_transport_fail - ) - else: - url = transport.send(data, headers, timeout=self.config.server_timeout) - self.handle_transport_success(url=url) + self._transport.close() def get_service_info(self): if self._service_info: @@ -378,13 +250,12 @@ def get_system_info(self): "platform": platform.system().lower(), } - def _build_msg(self, data=None, **kwargs): - data = data or {} - data["service"] = self.get_service_info() - data["process"] = self.get_process_info() - data["system"] = self.get_system_info() - data.update(**kwargs) - return data + def _build_metadata(self): + return { + "service": self.get_service_info(), + "process": self.get_process_info(), + "system": self.get_system_info(), + } def _build_msg_for_logging( self, event_type, date=None, context=None, custom=None, stack=None, handled=True, **kwargs @@ -471,10 +342,6 @@ def _build_msg_for_logging( else: context["custom"] = custom - # Run the data through processors - for processor in self.processors: - event_data = processor(self, event_data) - # Make sure all data is coerced event_data = transform(event_data) if "exception" in event_data: @@ -486,7 +353,7 @@ def _build_msg_for_logging( if transaction: event_data["transaction"] = {"id": transaction.id} - return self._build_msg({"errors": [event_data]}) + return event_data def _filter_exception_type(self, data): exception = data.get("exception") @@ -509,28 +376,6 @@ def _filter_exception_type(self, data): return True return False - def _get_log_message(self, data): - # decode message so we can show the actual event - try: - data = self.decode(data) - except Exception: - message = "" - else: - message = data.pop("message", "") - return message - - def _get_transport(self, parsed_url): - if hasattr(self._transport_class, "sync_transport") and is_master_process(): - # when in the master process, always use SYNC mode. This avoids - # the danger of being forked into an inconsistent threading state - self.logger.info("Sending message synchronously while in master " "process. PID: %s", os.getpid()) - return self._transport_class.sync_transport(parsed_url) - if parsed_url not in self._transports: - self._transports[parsed_url] = self._transport_class( - parsed_url, verify_server_cert=self.config.verify_server_cert - ) - return self._transports[parsed_url] - def _get_stack_info_for_trace( self, frames, diff --git a/elasticapm/conf/__init__.py b/elasticapm/conf/__init__.py index 14a50cf0f..d33f4c080 100644 --- a/elasticapm/conf/__init__.py +++ b/elasticapm/conf/__init__.py @@ -43,10 +43,10 @@ def __get__(self, instance, owner): return self.default def __set__(self, instance, value): + value = self._validate(instance, value) if value is not None: value = self.type(value) - if self._validate(instance, value): - instance._values[self.dict_key] = value + instance._values[self.dict_key] = value def _validate(self, instance, value): if value is None and self.required: @@ -54,12 +54,10 @@ def _validate(self, instance, value): "Configuration error: value for {} is required.".format(self.dict_key), self.dict_key ) if self.validators and value is not None: - if not all(validator(value) for validator in self.validators): - raise ConfigurationError( - "Configuration error: value {} is not valid for {}.".format(value, self.dict_key), self.dict_key - ) + for validator in self.validators: + value = validator(value, self.dict_key) instance._errors.pop(self.dict_key, None) - return True + return value class _ListConfigValue(_ConfigValue): @@ -92,6 +90,42 @@ def __set__(self, instance, value): instance._values[self.dict_key] = bool(value) +class RegexValidator(object): + def __init__(self, regex, verbose_pattern=None): + self.regex = regex + self.verbose_pattern = verbose_pattern or regex + + def __call__(self, value, field_name): + value = compat.text_type(value) + match = re.match(self.regex, value) + if match: + return value + raise ConfigurationError("{} does not match pattern {}".format(value, self.verbose_pattern), field_name) + + +class UnitValidator(object): + def __init__(self, regex, verbose_pattern, unit_multipliers): + self.regex = regex + self.verbose_pattern = verbose_pattern + self.unit_multipliers = unit_multipliers + + def __call__(self, value, field_name): + value = compat.text_type(value) + match = re.match(self.regex, value, re.IGNORECASE) + if not match: + raise ConfigurationError("{} does not match pattern {}".format(value, self.verbose_pattern), field_name) + val, unit = match.groups() + try: + val = int(val) * self.unit_multipliers[unit] + except KeyError: + raise ConfigurationError("{} is not a supported unit".format(unit), field_name) + return val + + +duration_validator = UnitValidator("^((?:-)?\d+)(ms|s|m)$", "\d+(ms|s|m)", {"ms": 1, "s": 1000, "m": 60000}) +size_validator = UnitValidator("^(\d+)(b|kib|mib)$", "\d+(b|KiB|MiB)", {"b": 1, "kib": 1024, "mib": 1024 * 1024}) + + class _ConfigBase(object): _NO_VALUE = object() # sentinel object @@ -130,9 +164,7 @@ def errors(self): class Config(_ConfigBase): - service_name = _ConfigValue( - "SERVICE_NAME", validators=[lambda val: re.match("^[a-zA-Z0-9 _-]+$", val)], required=True - ) + service_name = _ConfigValue("SERVICE_NAME", validators=[RegexValidator("^[a-zA-Z0-9 _-]+$")], required=True) environment = _ConfigValue("ENVIRONMENT", default=None) secret_token = _ConfigValue("SECRET_TOKEN") debug = _BoolConfigValue("DEBUG", default=False) @@ -141,7 +173,12 @@ class Config(_ConfigBase): include_paths = _ListConfigValue("INCLUDE_PATHS") exclude_paths = _ListConfigValue("EXCLUDE_PATHS", default=compat.get_default_library_patters()) filter_exception_types = _ListConfigValue("FILTER_EXCEPTION_TYPES") - server_timeout = _ConfigValue("SERVER_TIMEOUT", type=float, default=5) + server_timeout = _ConfigValue( + "SERVER_TIMEOUT", + type=float, + validators=[UnitValidator("^((?:-)?\d+)(ms|s|m)?$", "\d+(ms|s|m)", {"ms": 0.001, "s": 1, "m": 60, None: 1000})], + default=5, + ) hostname = _ConfigValue("HOSTNAME", default=socket.gethostname()) auto_log_stacks = _BoolConfigValue("AUTO_LOG_STACKS", default=True) transport_class = _ConfigValue("TRANSPORT_CLASS", default="elasticapm.transport.http.AsyncTransport", required=True) @@ -157,11 +194,16 @@ class Config(_ConfigBase): "elasticapm.processors.sanitize_http_request_body", ], ) - flush_interval = _ConfigValue("FLUSH_INTERVAL", type=int, default=10) + api_request_size = _ConfigValue("API_REQUEST_SIZE", type=int, validators=[size_validator], default=750 * 1024) + api_request_time = _ConfigValue("API_REQUEST_TIME", type=int, validators=[duration_validator], default=10 * 1000) transaction_sample_rate = _ConfigValue("TRANSACTION_SAMPLE_RATE", type=float, default=1.0) transaction_max_spans = _ConfigValue("TRANSACTION_MAX_SPANS", type=int, default=500) - span_frames_min_duration_ms = _ConfigValue("SPAN_FRAMES_MIN_DURATION", default=5, type=int) - max_queue_size = _ConfigValue("MAX_QUEUE_SIZE", type=int, default=500) + span_frames_min_duration = _ConfigValue( + "SPAN_FRAMES_MIN_DURATION", + default=5, + validators=[UnitValidator("^((?:-)?\d+)(ms|s|m)?$", "\d+(ms|s|m)", {"ms": 1, "s": 1000, "m": 60000, None: 1})], + type=int, + ) collect_local_variables = _ConfigValue("COLLECT_LOCAL_VARIABLES", default="errors") source_lines_error_app_frames = _ConfigValue("SOURCE_LINES_ERROR_APP_FRAMES", type=int, default=5) source_lines_error_library_frames = _ConfigValue("SOURCE_LINES_ERROR_LIBRARY_FRAMES", type=int, default=5) @@ -179,9 +221,6 @@ class Config(_ConfigBase): disable_send = _BoolConfigValue("DISABLE_SEND", default=False) instrument = _BoolConfigValue("DISABLE_INSTRUMENTATION", default=True) - # undocumented configuration - _wait_to_first_send = _ConfigValue("_WAIT_TO_FIRST_SEND", type=int, default=5) - def setup_logging(handler, exclude=["elasticapm", "gunicorn", "south", "elasticapm.errors"]): """ diff --git a/elasticapm/conf/constants.py b/elasticapm/conf/constants.py index bb497308f..6ed18028b 100644 --- a/elasticapm/conf/constants.py +++ b/elasticapm/conf/constants.py @@ -1,11 +1,11 @@ -# Error API path -ERROR_API_PATH = "/v1/errors" - -# Transactions API path -TRANSACTIONS_API_PATH = "/v1/transactions" +EVENTS_API_PATH = "/intake/v2/events" TIMESTAMP_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ" KEYWORD_MAX_LENGTH = 1024 HTTP_WITH_BODY = {"POST", "PUT", "PATCH", "DELETE"} + +ERROR = "error" +TRANSACTION = "transaction" +SPAN = "span" diff --git a/elasticapm/context/__init__.py b/elasticapm/context/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/elasticapm/context/contextvars.py b/elasticapm/context/contextvars.py new file mode 100644 index 000000000..f1a13928c --- /dev/null +++ b/elasticapm/context/contextvars.py @@ -0,0 +1,31 @@ +from __future__ import absolute_import + +import contextvars + +elasticapm_transaction_var = contextvars.ContextVar("elasticapm_transaction_var") +elasticapm_span_var = contextvars.ContextVar("elasticapm_span_var") + + +def get_transaction(clear=False): + try: + transaction = elasticapm_transaction_var.get() + if clear: + set_transaction(None) + return transaction + except LookupError: + return None + + +def set_transaction(transaction): + elasticapm_transaction_var.set(transaction) + + +def get_span(): + try: + return elasticapm_span_var.get() + except LookupError: + return None + + +def set_span(span): + elasticapm_span_var.set(span) diff --git a/elasticapm/context/threadlocal.py b/elasticapm/context/threadlocal.py new file mode 100644 index 000000000..41b926c2a --- /dev/null +++ b/elasticapm/context/threadlocal.py @@ -0,0 +1,30 @@ +import threading + +thread_local = threading.local() +thread_local.transaction = None +elasticapm_span_var = None + + +def get_transaction(clear=False): + """ + Get the transaction registered for the current thread. + + :return: + :rtype: Transaction + """ + transaction = getattr(thread_local, "transaction", None) + if clear: + thread_local.transaction = None + return transaction + + +def set_transaction(transaction): + thread_local.transaction = transaction + + +def get_span(): + return getattr(thread_local, "span", None) + + +def set_span(span): + thread_local.span = span diff --git a/elasticapm/contrib/async_worker.py b/elasticapm/contrib/async_worker.py index f6ef30549..7406eebf0 100644 --- a/elasticapm/contrib/async_worker.py +++ b/elasticapm/contrib/async_worker.py @@ -93,6 +93,7 @@ def start(self): if not self._thread: self._thread = Thread(target=self._target) self._thread.setDaemon(True) + self._thread.name = "elasticapm sender thread" self._thread.start() finally: self._lock.release() diff --git a/elasticapm/contrib/asyncio/__init__.py b/elasticapm/contrib/asyncio/__init__.py deleted file mode 100644 index f8cbddc1d..000000000 --- a/elasticapm/contrib/asyncio/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .client import Client # noqa E401 diff --git a/elasticapm/contrib/asyncio/client.py b/elasticapm/contrib/asyncio/client.py deleted file mode 100644 index 8c63ebc9e..000000000 --- a/elasticapm/contrib/asyncio/client.py +++ /dev/null @@ -1,48 +0,0 @@ -import asyncio -import urllib.parse - -from elasticapm.base import Client as BaseClient - - -class Client(BaseClient): - def handle_transport_response(self, task): - try: - url = task.result() - except Exception as exc: - self.handle_transport_fail(exception=exc) - else: - self.handle_transport_success(url=url) - - def _send_remote(self, url, data, headers=None): - if headers is None: - headers = {} - parsed = urllib.parse.urlparse(url) - transport = self._get_transport(parsed) - loop = asyncio.get_event_loop() - task = loop.create_task(transport.send(data, headers, timeout=self.config.server_timeout)) - task.add_done_callback(self.handle_transport_response) - - def _start_send_timer(self, timeout=None): - timeout = timeout or self.config.flush_interval - self._send_timer = AsyncTimer(timeout, self._collect_transactions) - - def _stop_send_timer(self): - if self._send_timer: - self._send_timer.cancel() - - -class AsyncTimer: - def __init__(self, interval, callback): - self.interval = interval - self.callback = callback - self.task = asyncio.ensure_future(self._job()) - self._done = False - - async def _job(self): - await asyncio.sleep(self.interval) - await self._callback() - self._done = True - - def cancel(self): - if not self._done: - self.task.cancel() diff --git a/elasticapm/contrib/django/management/commands/elasticapm.py b/elasticapm/contrib/django/management/commands/elasticapm.py index 06d4f81bc..bc07517f8 100644 --- a/elasticapm/contrib/django/management/commands/elasticapm.py +++ b/elasticapm/contrib/django/management/commands/elasticapm.py @@ -108,8 +108,6 @@ def handle_test(self, command, **options): client = DjangoClient(**config) client.error_logger = ColoredLogger(self.stderr) client.logger = ColoredLogger(self.stderr) - client.state.logger = client.logger - client.state.error_logger = client.error_logger self.write( "Trying to send a test error to APM Server using these settings:\n\n" "SERVICE_NAME:\t%s\n" diff --git a/elasticapm/processors.py b/elasticapm/processors.py index c332f0ef4..c1af99827 100644 --- a/elasticapm/processors.py +++ b/elasticapm/processors.py @@ -12,6 +12,7 @@ import re import warnings +from elasticapm.conf.constants import ERROR, SPAN, TRANSACTION from elasticapm.utils import compat, varmap from elasticapm.utils.encoding import force_text @@ -24,6 +25,22 @@ SANITIZE_VALUE_PATTERNS = [re.compile(r"^[- \d]{16,19}$")] # credit card numbers, with or without spacers +def for_events(*events): + """ + :param events: list of event types + + Only calls wrapped function if given event_type is in list of events + """ + events = set(events) + + def wrap(func): + func.event_types = events + return func + + return wrap + + +@for_events(ERROR, TRANSACTION) def remove_http_request_body(client, event): """ Removes request.body from context @@ -37,6 +54,7 @@ def remove_http_request_body(client, event): return event +@for_events(ERROR, SPAN) def remove_stacktrace_locals(client, event): """ Removes local variables from any frames. @@ -49,6 +67,7 @@ def remove_stacktrace_locals(client, event): return _process_stack_frames(event, func) +@for_events(ERROR, SPAN) def sanitize_stacktrace_locals(client, event): """ Sanitizes local variables in all frames @@ -65,6 +84,7 @@ def func(frame): return _process_stack_frames(event, func) +@for_events(ERROR, TRANSACTION) def sanitize_http_request_cookies(client, event): """ Sanitizes http request cookies @@ -90,6 +110,7 @@ def sanitize_http_request_cookies(client, event): return event +@for_events(ERROR, TRANSACTION) def sanitize_http_response_cookies(client, event): """ Sanitizes the set-cookie header of the response @@ -105,6 +126,7 @@ def sanitize_http_response_cookies(client, event): return event +@for_events(ERROR, TRANSACTION) def sanitize_http_headers(client, event): """ Sanitizes http request/response headers @@ -130,6 +152,7 @@ def sanitize_http_headers(client, event): return event +@for_events(ERROR, TRANSACTION) def sanitize_http_wsgi_env(client, event): """ Sanitizes WSGI environment variables @@ -146,6 +169,7 @@ def sanitize_http_wsgi_env(client, event): return event +@for_events(ERROR, TRANSACTION) def sanitize_http_request_querystring(client, event): """ Sanitizes http request query string @@ -166,6 +190,7 @@ def sanitize_http_request_querystring(client, event): return event +@for_events(ERROR, TRANSACTION) def sanitize_http_request_body(client, event): """ Sanitizes http request body. This only works if the request body @@ -186,9 +211,10 @@ def sanitize_http_request_body(client, event): return event +@for_events(ERROR, SPAN) def mark_in_app_frames(client, event): warnings.warn( - "The mark_in_app_frames processor is deprecated " "and can be removed from your PROCESSORS setting", + "The mark_in_app_frames processor is deprecated and can be removed from your PROCESSORS setting", DeprecationWarning, ) return event @@ -232,12 +258,9 @@ def _sanitize_string(unsanitized, itemsep, kvsep): def _process_stack_frames(event, func): - if "spans" in event: - # every capture_span can have a stack capture_span - for span in event["spans"]: - if "stacktrace" in span: - for frame in span["stacktrace"]: - func(frame) + if "stacktrace" in event: + for frame in event["stacktrace"]: + func(frame) # an error can have two stacktraces, one in "exception", one in "log" if "exception" in event and "stacktrace" in event["exception"]: for frame in event["exception"]["stacktrace"]: diff --git a/elasticapm/traces.py b/elasticapm/traces.py index e39c1dbe8..33378d88d 100644 --- a/elasticapm/traces.py +++ b/elasticapm/traces.py @@ -8,63 +8,39 @@ import uuid from elasticapm.conf import constants +from elasticapm.conf.constants import SPAN, TRANSACTION from elasticapm.utils import compat, encoding, get_name_from_func __all__ = ("capture_span", "tag", "set_transaction_name", "set_custom_context", "set_user_context") error_logger = logging.getLogger("elasticapm.errors") -thread_local = threading.local() -thread_local.transaction = None - - _time_func = timeit.default_timer TAG_RE = re.compile('^[^.*"]+$') -DROPPED_SPAN = object() -IGNORED_SPAN = object() - - -def get_transaction(clear=False): - """ - Get the transaction registered for the current thread. - - :return: - :rtype: Transaction - """ - transaction = getattr(thread_local, "transaction", None) - if clear: - thread_local.transaction = None - return transaction +try: + from elasticapm.context.contextvars import get_transaction, set_transaction, get_span, set_span +except ImportError: + from elasticapm.context.threadlocal import get_transaction, set_transaction, get_span, set_span class Transaction(object): - def __init__( - self, - frames_collector_func, - transaction_type="custom", - is_sampled=True, - max_spans=None, - span_frames_min_duration=None, - ): + def __init__(self, store, transaction_type="custom", is_sampled=True): self.id = str(uuid.uuid4()) + self.trace_id = None # for later use in distributed tracing self.timestamp = datetime.datetime.utcnow() self.start_time = _time_func() self.name = None self.duration = None self.result = None self.transaction_type = transaction_type - self._frames_collector_func = frames_collector_func + self._store = store self.spans = [] - self.span_stack = [] - self.max_spans = max_spans - self.span_frames_min_duration = span_frames_min_duration self.dropped_spans = 0 - self.ignore_subtree = False self.context = {} self.tags = {} @@ -75,72 +51,76 @@ def end_transaction(self, skip_frames=8): self.duration = _time_func() - self.start_time def begin_span(self, name, span_type, context=None, leaf=False): - # If we were already called with `leaf=True`, we'll just push - # a placeholder on the stack. - if self.ignore_subtree: - self.span_stack.append(IGNORED_SPAN) - return None - - if leaf: - self.ignore_subtree = True - - self._span_counter += 1 - - if self.max_spans and self._span_counter > self.max_spans: + parent_span = get_span() + store = self._store + if parent_span and parent_span.leaf: + span = DroppedSpan(parent_span, leaf=True) + elif store.max_spans and self._span_counter > store.max_spans - 1: self.dropped_spans += 1 - self.span_stack.append(DROPPED_SPAN) - return None - - start = _time_func() - self.start_time - span = Span(self._span_counter - 1, name, span_type, start, context) - self.span_stack.append(span) + span = DroppedSpan(parent_span) + self._span_counter += 1 + else: + start = _time_func() - self.start_time + span = Span(self._span_counter, self.id, self.trace_id, name, span_type, start, context, leaf) + span.frames = store.frames_collector_func() + span.parent = parent_span + self._span_counter += 1 + set_span(span) return span def end_span(self, skip_frames): - span = self.span_stack.pop() - if span is IGNORED_SPAN: - return None - - self.ignore_subtree = False - - if span is DROPPED_SPAN: + span = get_span() + if span is None: + raise LookupError() + if isinstance(span, DroppedSpan): + set_span(span.parent) return span.duration = _time_func() - span.start_time - self.start_time - if self.span_stack: - span.parent = self.span_stack[-1].idx - - if not self.span_frames_min_duration or span.duration >= self.span_frames_min_duration: - span.frames = self._frames_collector_func()[skip_frames:] + if not self._store.span_frames_min_duration or span.duration >= self._store.span_frames_min_duration: + span.frames = self._store.frames_processing_func(span.frames)[skip_frames:] + else: + span.frames = None self.spans.append(span) - + set_span(span.parent) + self._store.queue_func(SPAN, span.to_dict()) return span def to_dict(self): self.context["tags"] = self.tags result = { "id": self.id, + "trace_id": self.trace_id, "name": encoding.keyword_field(self.name or ""), "type": encoding.keyword_field(self.transaction_type), "duration": self.duration * 1000, # milliseconds "result": encoding.keyword_field(str(self.result)), "timestamp": self.timestamp.strftime(constants.TIMESTAMP_FORMAT), "sampled": self.is_sampled, + "span_count": {"started": self._span_counter - self.dropped_spans, "dropped": self.dropped_spans}, } if self.is_sampled: - result["spans"] = [span_obj.to_dict() for span_obj in self.spans] result["context"] = self.context - - if self.dropped_spans: - result["span_count"] = {"dropped": {"total": self.dropped_spans}} return result class Span(object): - __slots__ = ("idx", "name", "type", "context", "leaf", "start_time", "duration", "parent", "frames") - - def __init__(self, idx, name, span_type, start_time, context=None, leaf=False): + __slots__ = ( + "idx", + "transaction_id", + "trace_id", + "name", + "type", + "context", + "leaf", + "start_time", + "duration", + "parent", + "frames", + ) + + def __init__(self, idx, transaction_id, trace_id, name, span_type, start_time, context=None, leaf=False): """ Create a new Span @@ -152,6 +132,8 @@ def __init__(self, idx, name, span_type, start_time, context=None, leaf=False): :param leaf: is this transaction a leaf transaction? """ self.idx = idx + self.transaction_id = transaction_id + self.trace_id = trace_id self.name = name self.type = span_type self.context = context @@ -163,12 +145,14 @@ def __init__(self, idx, name, span_type, start_time, context=None, leaf=False): def to_dict(self): result = { - "id": self.idx, + "id": compat.text_type(self.idx), + "transaction_id": self.transaction_id, + "trace_id": self.trace_id, "name": encoding.keyword_field(self.name), "type": encoding.keyword_field(self.type), "start": self.start_time * 1000, # milliseconds "duration": self.duration * 1000, # milliseconds - "parent": self.parent, + "parent": self.parent.idx if self.parent else None, "context": self.context, } if self.frames: @@ -176,22 +160,30 @@ def to_dict(self): return result +class DroppedSpan(object): + __slots__ = ("leaf", "parent") + + def __init__(self, parent, leaf=False): + self.parent = parent + self.leaf = leaf + + class TransactionsStore(object): def __init__( self, frames_collector_func, - collect_frequency, + frames_processing_func, + queue_func, sample_rate=1.0, max_spans=0, - max_queue_size=None, span_frames_min_duration=None, ignore_patterns=None, ): self.cond = threading.Condition() - self.collect_frequency = collect_frequency - self.max_queue_size = max_queue_size self.max_spans = max_spans - self._frames_collector_func = frames_collector_func + self.queue_func = queue_func + self.frames_processing_func = frames_processing_func + self.frames_collector_func = frames_collector_func self._transactions = [] self._last_collect = _time_func() self._ignore_patterns = [re.compile(p) for p in ignore_patterns or []] @@ -216,11 +208,6 @@ def get_all(self, blocking=False): self._last_collect = _time_func() return transactions - def should_collect(self): - return (self.max_queue_size and len(self._transactions) >= self.max_queue_size) or ( - _time_func() - self._last_collect - ) >= self.collect_frequency - def __len__(self): with self.cond: return len(self._transactions) @@ -232,14 +219,8 @@ def begin_transaction(self, transaction_type): :returns the Transaction object """ is_sampled = self._sample_rate == 1.0 or self._sample_rate > random.random() - transaction = Transaction( - self._frames_collector_func, - transaction_type, - max_spans=self.max_spans, - span_frames_min_duration=self.span_frames_min_duration, - is_sampled=is_sampled, - ) - thread_local.transaction = transaction + transaction = Transaction(self, transaction_type, is_sampled=is_sampled) + set_transaction(transaction) return transaction def _should_ignore(self, transaction_name): @@ -258,7 +239,7 @@ def end_transaction(self, result=None, transaction_name=None): return if transaction.result is None: transaction.result = result - self.add_transaction(transaction.to_dict()) + self.queue_func(TRANSACTION, transaction.to_dict()) return transaction @@ -290,7 +271,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): if transaction and transaction.is_sampled: try: transaction.end_span(self.skip_frames) - except IndexError: + except LookupError: error_logger.info("ended non-existing span %s of type %s", self.name, self.type) diff --git a/elasticapm/transport/asyncio.py b/elasticapm/transport/asyncio.py deleted file mode 100644 index 875579f7b..000000000 --- a/elasticapm/transport/asyncio.py +++ /dev/null @@ -1,52 +0,0 @@ -import asyncio - -import aiohttp -import async_timeout - -from .base import TransportException -from .http_base import HTTPTransportBase - - -class AsyncioHTTPTransport(HTTPTransportBase): - """ - HTTP Transport ready for asyncio - """ - - def __init__(self, parsed_url, **kwargs): - super(AsyncioHTTPTransport, self).__init__(parsed_url, **kwargs) - loop = asyncio.get_event_loop() - session_kwargs = {"loop": loop} - if not self._verify_server_cert: - session_kwargs["connector"] = aiohttp.TCPConnector(verify_ssl=False) - self.client = aiohttp.ClientSession(**session_kwargs) - - async def send(self, data, headers, timeout=None): - """Use synchronous interface, because this is a coroutine.""" - - try: - with async_timeout.timeout(timeout): - async with self.client.post(self._url, data=data, headers=headers) as response: - assert response.status == 202 - except asyncio.TimeoutError as e: - print_trace = True - message = "Connection to APM Server timed out " "(url: %s, timeout: %s seconds)" % (self._url, timeout) - raise TransportException(message, data, print_trace=print_trace) from e - except AssertionError as e: - print_trace = True - body = await response.read() - if response.status == 429: - message = "Temporarily rate limited: " - print_trace = False - else: - message = "Unable to reach APM Server: " - message += "%s (url: %s, body: %s)" % (e, self._url, body) - raise TransportException(message, data, print_trace=print_trace) from e - except Exception as e: - print_trace = True - message = "Unable to reach APM Server: %s (url: %s)" % (e, self._url) - raise TransportException(message, data, print_trace=print_trace) from e - else: - return response.headers.get("Location") - - def __del__(self): - self.client.close() diff --git a/elasticapm/transport/base.py b/elasticapm/transport/base.py index 6c89c5efd..da9fe9344 100644 --- a/elasticapm/transport/base.py +++ b/elasticapm/transport/base.py @@ -1,6 +1,14 @@ # -*- coding: utf-8 -*- +import gzip +import logging +import threading +import timeit -from elasticapm.transport.exceptions import InvalidScheme +from elasticapm.contrib.async_worker import AsyncWorker +from elasticapm.utils import json_encoder +from elasticapm.utils.compat import BytesIO + +logger = logging.getLogger("elasticapm.transport") class TransportException(Exception): @@ -18,13 +26,112 @@ class Transport(object): """ async_mode = False - scheme = [] - def check_scheme(self, url): - if url.scheme not in self.scheme: - raise InvalidScheme() + def __init__( + self, + metadata=None, + compress_level=5, + json_serializer=json_encoder.dumps, + max_flush_time=None, + max_buffer_size=None, + **kwargs + ): + """ + Create a new Transport instance + + :param metadata: Metadata object to prepend to every queue + :param compress_level: GZip compress level. If zero, no GZip compression will be used + :param json_serializer: serializer to use for JSON encoding + :param max_flush_time: Maximum time between flushes in seconds + :param max_buffer_size: Maximum size of buffer before flush + :param kwargs: + """ + self.state = TransportState() + self._metadata = metadata if metadata is not None else {} + self._compress_level = compress_level + self._json_serializer = json_serializer + self._max_flush_time = max_flush_time + self._max_buffer_size = max_buffer_size + self._queued_data = None + self._flush_lock = threading.Lock() + self._last_flush = timeit.default_timer() + self._flush_timer = None + + def queue(self, event_type, data, flush=False): + self._queue(self.queued_data, {event_type: data}) + since_last_flush = timeit.default_timer() - self._last_flush + queue_size = self.queued_data_size + if flush: + logger.debug("forced flush") + self.flush() + elif self._max_flush_time and since_last_flush > self._max_flush_time: + logger.debug( + "flushing due to time since last flush %.3fs > max_flush_time %.3fs", + since_last_flush, + self._max_flush_time, + ) + self.flush() + elif self._max_buffer_size and queue_size > self._max_buffer_size: + logger.debug( + "flushing since queue size %d bytes > max_queue_size %d bytes", queue_size, self._max_buffer_size + ) + self.flush() + elif not self._flush_timer: + with self._flush_lock: + self._start_flush_timer() - def send(self, data, headers): + def _queue(self, queue, data): + queue.write((self._json_serializer(data) + "\n").encode("utf-8")) + + @property + def queued_data(self): + if self._queued_data is None: + if self._compress_level: + self._queued_data = gzip.GzipFile(fileobj=BytesIO(), mode="w", compresslevel=self._compress_level) + else: + self._queued_data = BytesIO() + self._queue(self._queued_data, {"metadata": self._metadata}) + return self._queued_data + + @property + def queued_data_size(self): + f = self.queued_data + # return size of the underlying BytesIO object if it is compressed + return f.fileobj.tell() if hasattr(f, "fileobj") else f.tell() + + def flush(self, sync=False, start_flush_timer=True): + """ + Flush the queue + :param sync: if true, flushes the queue synchronously in the current thread + :param start_flush_timer: set to True if the flush timer thread should be restarted at the end of the flush + :return: None + """ + with self._flush_lock: + self._stop_flush_timer() + queued_data, self._queued_data = self._queued_data, None + if queued_data and not self.state.should_try(): + logger.error("dropping flushed data due to transport failure back-off") + elif queued_data: + if self._compress_level: + fileobj = queued_data.fileobj # get a reference to the fileobj before closing the gzip file + queued_data.close() + else: + fileobj = queued_data + # StringIO on Python 2 does not have getbuffer, so we need to fall back to getvalue + data = fileobj.getbuffer() if hasattr(fileobj, "getbuffer") else fileobj.getvalue() + if hasattr(self, "send_async") and not sync: + self.send_async(data) + else: + try: + self.send(data) + self.handle_transport_success() + except Exception as e: + self.handle_transport_fail(e) + self._last_flush = timeit.default_timer() + if start_flush_timer: + self._start_flush_timer() + + def send(self, data): """ You need to override this to do something with the actual data. Usually - this is sending to a server @@ -36,11 +143,92 @@ def close(self): Cleans up resources and closes connection :return: """ - pass + self.flush(sync=True, start_flush_timer=False) + + def handle_transport_success(self, **kwargs): + """ + Success handler called by the transport on successful send + """ + self.state.set_success() + + def handle_transport_fail(self, exception=None, **kwargs): + """ + Failure handler called by the transport on send failure + """ + message = str(exception) + logger.error("Failed to submit message: %r", message, exc_info=getattr(exception, "print_trace", True)) + self.state.set_fail() + + def _start_flush_timer(self, timeout=None): + timeout = timeout or self._max_flush_time + self._flush_timer = threading.Timer(timeout, self.flush) + self._flush_timer.name = "elasticapm flush timer" + self._flush_timer.daemon = True + logger.debug("Starting flush timer") + self._flush_timer.start() + + def _stop_flush_timer(self): + if self._flush_timer: + logger.debug("Cancelling flush timer") + self._flush_timer.cancel() class AsyncTransport(Transport): async_mode = True + sync_transport = Transport - def send_async(self, data, headers, success_callback=None, fail_callback=None): - raise NotImplementedError + def __init__(self, *args, **kwargs): + super(AsyncTransport, self).__init__(*args, **kwargs) + self._worker = None + + @property + def worker(self): + if not self._worker or not self._worker.is_alive(): + self._worker = AsyncWorker() + return self._worker + + def send_sync(self, data=None): + try: + self.sync_transport.send(self, data) + self.handle_transport_success() + except Exception as e: + self.handle_transport_fail(exception=e) + + def send_async(self, data): + self.worker.queue(self.send_sync, {"data": data}) + + def close(self): + super(AsyncTransport, self).close() + if self._worker: + self._worker.main_thread_terminated() + + +class TransportState(object): + ONLINE = 1 + ERROR = 0 + + def __init__(self): + self.status = self.ONLINE + self.last_check = None + self.retry_number = -1 + + def should_try(self): + if self.status == self.ONLINE: + return True + + interval = min(self.retry_number, 6) ** 2 + + return timeit.default_timer() - self.last_check > interval + + def set_fail(self): + self.status = self.ERROR + self.retry_number += 1 + self.last_check = timeit.default_timer() + + def set_success(self): + self.status = self.ONLINE + self.last_check = None + self.retry_number = -1 + + def did_fail(self): + return self.status == self.ERROR diff --git a/elasticapm/transport/http.py b/elasticapm/transport/http.py index 322ee969d..ba50fbdee 100644 --- a/elasticapm/transport/http.py +++ b/elasticapm/transport/http.py @@ -15,11 +15,8 @@ class Transport(HTTPTransportBase): - - scheme = ["http", "https"] - - def __init__(self, parsed_url, **kwargs): - super(Transport, self).__init__(parsed_url, **kwargs) + def __init__(self, url, **kwargs): + super(Transport, self).__init__(url, **kwargs) pool_kwargs = {"cert_reqs": "CERT_REQUIRED", "ca_certs": certifi.where(), "block": True} if not self._verify_server_cert: pool_kwargs["cert_reqs"] = ssl.CERT_NONE @@ -30,18 +27,9 @@ def __init__(self, parsed_url, **kwargs): else: self.http = urllib3.PoolManager(**pool_kwargs) - def send(self, data, headers, timeout=None): + def send(self, data): response = None - # ensure headers are byte strings - headers = { - k.encode("ascii") - if isinstance(k, compat.text_type) - else k: v.encode("ascii") - if isinstance(v, compat.text_type) - else v - for k, v in headers.items() - } if compat.PY2 and isinstance(self._url, compat.text_type): url = self._url.encode("utf-8") else: @@ -49,7 +37,7 @@ def send(self, data, headers, timeout=None): try: try: response = self.http.urlopen( - "POST", url, body=data, headers=headers, timeout=timeout, preload_content=False + "POST", url, body=data, headers=self._headers, timeout=self._timeout, preload_content=False ) logger.info("Sent request, url=%s size=%.2fkb status=%s", url, len(data) / 1024.0, response.status) except Exception as e: @@ -57,7 +45,7 @@ def send(self, data, headers, timeout=None): if isinstance(e, MaxRetryError) and isinstance(e.reason, TimeoutError): message = "Connection to APM Server timed out " "(url: %s, timeout: %s seconds)" % ( self._url, - timeout, + self._timeout, ) print_trace = False else: @@ -80,6 +68,5 @@ def send(self, data, headers, timeout=None): class AsyncTransport(AsyncHTTPTransportBase, Transport): - scheme = ["http", "https"] async_mode = True sync_transport = Transport diff --git a/elasticapm/transport/http_base.py b/elasticapm/transport/http_base.py index 1f4053a2e..82d02afbb 100644 --- a/elasticapm/transport/http_base.py +++ b/elasticapm/transport/http_base.py @@ -1,19 +1,26 @@ # -*- coding: utf-8 -*- -from elasticapm.contrib.async_worker import AsyncWorker from elasticapm.transport.base import AsyncTransport, Transport +from elasticapm.utils import compat class HTTPTransportBase(Transport): - scheme = ["http", "https"] - - def __init__(self, parsed_url, verify_server_cert=True): - self.check_scheme(parsed_url) - - self._parsed_url = parsed_url - self._url = parsed_url.geturl() + def __init__( + self, url, verify_server_cert=True, compress_level=5, metadata=None, headers=None, timeout=None, **kwargs + ): + self._url = url self._verify_server_cert = verify_server_cert + self._timeout = timeout + self._headers = { + k.encode("ascii") + if isinstance(k, compat.text_type) + else k: v.encode("ascii") + if isinstance(v, compat.text_type) + else v + for k, v in (headers if headers is not None else {}).items() + } + super(HTTPTransportBase, self).__init__(metadata=metadata, compress_level=compress_level, **kwargs) - def send(self, data, headers, timeout=None): + def send(self, data): """ Sends a request to a remote webserver using HTTP POST. @@ -23,40 +30,5 @@ def send(self, data, headers, timeout=None): class AsyncHTTPTransportBase(AsyncTransport, HTTPTransportBase): - scheme = ["http", "https"] async_mode = True sync_transport = HTTPTransportBase - - def __init__(self, parsed_url, **kwargs): - super(AsyncHTTPTransportBase, self).__init__(parsed_url, **kwargs) - if self._url.startswith("async+"): - self._url = self._url[6:] - self._worker = None - - @property - def worker(self): - if not self._worker or not self._worker.is_alive(): - self._worker = AsyncWorker() - return self._worker - - def send_sync(self, data=None, headers=None, success_callback=None, fail_callback=None): - try: - url = self.sync_transport.send(self, data, headers) - if callable(success_callback): - success_callback(url=url) - except Exception as e: - if callable(fail_callback): - fail_callback(exception=e) - - def send_async(self, data, headers, success_callback=None, fail_callback=None): - kwargs = { - "data": data, - "headers": headers, - "success_callback": success_callback, - "fail_callback": fail_callback, - } - self.worker.queue(self.send_sync, kwargs) - - def close(self): - if self._worker: - self._worker.main_thread_terminated() diff --git a/elasticapm/utils/stacks.py b/elasticapm/utils/stacks.py index 352301e11..1e54c9543 100644 --- a/elasticapm/utils/stacks.py +++ b/elasticapm/utils/stacks.py @@ -158,7 +158,7 @@ def iter_traceback_frames(tb): tb = tb.tb_next -def iter_stack_frames(frames=None, skip=0, skip_top_modules=()): +def iter_stack_frames(frames=None, start_frame=None, skip=0, skip_top_modules=()): """ Given an optional list of frames (defaults to current stack), iterates over all frames that do not contain the ``__traceback_hide__`` @@ -173,12 +173,13 @@ def iter_stack_frames(frames=None, skip=0, skip_top_modules=()): itself. :param frames: a list of frames, or None + :param start_frame: a Frame object or None :param skip: number of frames to skip from the beginning :param skip_top_modules: tuple of strings """ if not frames: - frame = inspect.currentframe().f_back + frame = start_frame if start_frame is not None else inspect.currentframe().f_back frames = _walk_stack(frame) stop_ignoring = False for i, frame in enumerate(frames): diff --git a/setup.cfg b/setup.cfg index 6c8698bf2..ade57d675 100644 --- a/setup.cfg +++ b/setup.cfg @@ -29,7 +29,7 @@ include_trailing_comma=true known_standard_library=importlib,types,asyncio known_django=django known_first_party=elasticapm,tests -known_third_party=StringIO,__pypy__,aiohttp,async_timeout,boto3,cachetools,cassandra,celery,certifi,elasticsearch,flask,jinja2,jsonschema,logbook,memcache,mock,pkg_resources,psycopg2,psycopg2cffi,pymongo,pytest,pytest_localserver,redis,requests,setuptools,simplejson,twisted,urllib2,urllib3,urllib3_mock,urlparse,uwsgi,webob,werkzeug,zope +known_third_party=StringIO,__pypy__,aiohttp,async_timeout,boto3,cachetools,cassandra,celery,certifi,elasticsearch,flask,jinja2,jsonschema,logbook,memcache,mock,pkg_resources,psycopg2,psycopg2cffi,pymongo,pytest,pytest_localserver,redis,requests,setuptools,simplejson,twisted,urllib2,urllib3,urllib3_mock,urlparse,uwsgi,webob,werkzeug,zope,asynctest default_section=FIRSTPARTY sections=FUTURE,STDLIB,DJANGO,THIRDPARTY,FIRSTPARTY,LOCALFOLDER diff --git a/tests/asyncio/test_asyncio_client.py b/tests/asyncio/test_asyncio_client.py deleted file mode 100644 index c09456d1c..000000000 --- a/tests/asyncio/test_asyncio_client.py +++ /dev/null @@ -1,116 +0,0 @@ -import asyncio -import sys -from urllib.parse import urlparse - -import mock -import pytest - -pytestmark = pytest.mark.skipif(sys.version_info < (3, 5), reason="python3.5+ requried for asyncio") - - -class MockTransport(mock.MagicMock): - async_mode = False - - def __init__(self, url=None, *args, **kwargs): - super().__init__(*args, **kwargs) - self.url = url - - async def send(self, data, headers, timeout): - from elasticapm.transport.base import TransportException - - self.data = data - if self.url == urlparse("http://error"): - raise TransportException("", data, False) - await asyncio.sleep(0.0001) - - -class DummyTransport: - async_mode = False - - def __init__(self, *args, **kwargs): - pass - - async def send(self, data, headers, timeout): - return - - def close(self): - pass - - -@pytest.mark.asyncio -async def test_client_success(): - from elasticapm.contrib.asyncio import Client - - client = Client( - server_url="http://localhost", - service_name="service_name", - secret_token="secret", - transport_class=".".join((__name__, MockTransport.__name__)), - ) - client.send(client.config.server_url, foo="bar") - tasks = asyncio.Task.all_tasks() - task = next(t for t in tasks if t is not asyncio.Task.current_task()) - await task - assert client.state.status == 1 - transport = client._get_transport(urlparse("http://localhost")) - assert transport.data == client.encode({"foo": "bar"}) - - -@pytest.mark.asyncio -async def test_client_failure(): - from elasticapm.contrib.asyncio import Client - from elasticapm.transport.base import TransportException - - client = Client( - server_url="http://error", - service_name="service_name", - secret_token="secret", - transport_class=".".join((__name__, MockTransport.__name__)), - ) - client.send(client.config.server_url, foo="bar") - tasks = asyncio.Task.all_tasks() - task = next(t for t in tasks if t is not asyncio.Task.current_task()) - with pytest.raises(TransportException): - await task - assert client.state.status == 0 - - -@pytest.mark.asyncio -async def test_client_failure_stdlib_exception(mocker): - from elasticapm.contrib.asyncio import Client - from elasticapm.transport.base import TransportException - - client = Client( - server_url="http://elastic.co", - service_name="service_name", - secret_token="secret", - async_mode=False, - transport_class="elasticapm.transport.asyncio.AsyncioHTTPTransport", - ) - mock_client = mocker.Mock() - mock_client.post = mocker.Mock(side_effect=RuntimeError("oops")) - transport = client._get_transport(urlparse("http://elastic.co")) - transport.client = mock_client - client.send(client.config.server_url, foo="bar") - tasks = asyncio.Task.all_tasks() - task = next(t for t in tasks if t is not asyncio.Task.current_task()) - with pytest.raises(TransportException): - await task - assert client.state.status == 0 - - -@pytest.mark.asyncio -async def test_client_send_timer(): - from elasticapm.contrib.asyncio.client import Client, AsyncTimer - - client = Client(transport_class="tests.asyncio.test_asyncio_client.DummyTransport") - - assert client._send_timer is None - - client.begin_transaction("test_type") - client.end_transaction("test") - - assert isinstance(client._send_timer, AsyncTimer) - assert client._send_timer.interval == 5 - - client.close() diff --git a/tests/asyncio/test_asyncio_http.py b/tests/asyncio/test_asyncio_http.py deleted file mode 100644 index 4612c66f8..000000000 --- a/tests/asyncio/test_asyncio_http.py +++ /dev/null @@ -1,108 +0,0 @@ -import asyncio -import sys -from urllib.parse import urlparse - -import pytest - -pytestmark = pytest.mark.skipif(sys.version_info < (3, 5), reason="python3.5+ requried for asyncio") - - -@pytest.fixture -def mock_client(mocker): - mock_client = mocker.MagicMock() - mock_client.config.server_timeout = None - response = mocker.MagicMock() - - async def read(): - return mock_client.body - - response.read = read - - class fake_post: - async def __aenter__(self, *args, **kwargs): - response.status = mock_client.status - response.headers = mock_client.headers - if mock_client.config.server_timeout: - await asyncio.sleep(mock_client.config.server_timeout) - return response - - async def __aexit__(self, *args): - pass - - def __init__(self, *args, **kwargs): - mock_client.args = args - mock_client.kwargs = kwargs - - mock_client.post = mocker.Mock(side_effect=fake_post) - return mock_client - - -@pytest.mark.asyncio -async def test_send(mock_client): - from elasticapm.transport.asyncio import AsyncioHTTPTransport - - transport = AsyncioHTTPTransport(urlparse("http://localhost:9999")) - - mock_client.status = 202 - mock_client.headers = {"Location": "http://example.com/foo"} - transport.client = mock_client - - url = await transport.send(b"data", {"a": "b"}, timeout=2) - assert url == "http://example.com/foo" - assert mock_client.args == ("http://localhost:9999",) - assert mock_client.kwargs == {"headers": {"a": "b"}, "data": b"data"} - - -@pytest.mark.asyncio -async def test_send_not_found(mock_client): - from elasticapm.transport.asyncio import AsyncioHTTPTransport - from elasticapm.transport.base import TransportException - - transport = AsyncioHTTPTransport(urlparse("http://localhost:9999")) - - mock_client.status = 404 - mock_client.headers = {} - mock_client.body = b"Not Found" - transport.client = mock_client - - with pytest.raises(TransportException) as excinfo: - await transport.send(b"data", {}, timeout=2) - assert "Not Found" in str(excinfo.value) - assert excinfo.value.data == b"data" - - -@pytest.mark.asyncio -async def test_send_timeout(mock_client): - from elasticapm.transport.asyncio import AsyncioHTTPTransport - from elasticapm.transport.base import TransportException - - transport = AsyncioHTTPTransport(urlparse("http://localhost:9999")) - - mock_client.config.server_timeout = 0.1 - transport.client = mock_client - - with pytest.raises(TransportException) as excinfo: - await transport.send(b"data", {}, timeout=0.0001) - assert "Connection to APM Server timed out" in str(excinfo.value) - - -@pytest.mark.asyncio -async def test_ssl_verify_fails(waiting_httpsserver): - from elasticapm.transport.asyncio import AsyncioHTTPTransport - from elasticapm.transport.base import TransportException - - waiting_httpsserver.serve_content(code=202, content="", headers={"Location": "http://example.com/foo"}) - transport = AsyncioHTTPTransport(urlparse(waiting_httpsserver.url)) - with pytest.raises(TransportException) as exc_info: - await transport.send(b"x", {}) - assert "CERTIFICATE_VERIFY_FAILED" in str(exc_info) - - -@pytest.mark.asyncio -async def test_ssl_verify_disable(waiting_httpsserver): - from elasticapm.transport.asyncio import AsyncioHTTPTransport - - waiting_httpsserver.serve_content(code=202, content="", headers={"Location": "http://example.com/foo"}) - transport = AsyncioHTTPTransport(urlparse(waiting_httpsserver.url), verify_server_cert=False) - url = await transport.send(b"x", {}) - assert url == "http://example.com/foo" diff --git a/tests/client/client_tests.py b/tests/client/client_tests.py index 7ca8b19ce..24314e708 100644 --- a/tests/client/client_tests.py +++ b/tests/client/client_tests.py @@ -3,60 +3,17 @@ import platform import sys import time +from collections import defaultdict import mock import pytest +from pytest_localserver.http import ContentServer import elasticapm -from elasticapm.base import Client, ClientState -from elasticapm.conf.constants import KEYWORD_MAX_LENGTH -from elasticapm.transport.base import Transport +from elasticapm.base import Client +from elasticapm.conf.constants import ERROR, KEYWORD_MAX_LENGTH, SPAN, TRANSACTION from elasticapm.utils import compat, encoding - - -def test_client_state_should_try_online(): - state = ClientState() - assert state.should_try() is True - - -def test_client_state_should_try_new_error(): - state = ClientState() - state.status = state.ERROR - state.last_check = time.time() - state.retry_number = 1 - assert state.should_try() is False - - -def test_client_state_should_try_time_passed_error(): - state = ClientState() - state.status = state.ERROR - state.last_check = time.time() - 10 - state.retry_number = 1 - assert state.should_try() is True - - -def test_client_state_set_fail(): - state = ClientState() - state.set_fail() - assert state.status == state.ERROR - assert state.last_check is not None - assert state.retry_number == 1 - - -def test_client_state_set_success(): - state = ClientState() - state.status = state.ERROR - state.last_check = "foo" - state.retry_number = 0 - state.set_success() - assert state.status == state.ONLINE - assert state.last_check is None - assert state.retry_number == 0 - - -class DummyTransport(Transport): - def send(self, data, headers): - pass +from tests.fixtures import DummyTransport @pytest.mark.parametrize("elasticapm_client", [{"environment": "production"}], indirect=True) @@ -119,11 +76,9 @@ def __repr__(self): client.close() -@pytest.mark.parametrize( - "elasticapm_client", [{"transport_class": "tests.client.client_tests.DummyTransport"}], indirect=True -) +@pytest.mark.parametrize("elasticapm_client", [{"transport_class": "tests.fixtures.DummyTransport"}], indirect=True) def test_custom_transport(elasticapm_client): - assert elasticapm_client._transport_class == DummyTransport + assert isinstance(elasticapm_client._transport, DummyTransport) @pytest.mark.parametrize("elasticapm_client", [{"processors": []}], indirect=True) @@ -136,31 +91,29 @@ def test_empty_processor_list(elasticapm_client): [{"transport_class": "elasticapm.transport.http.Transport", "async_mode": False}], indirect=True, ) -@mock.patch("elasticapm.base.ClientState.should_try") -def test_send_remote_failover_sync(should_try, sending_elasticapm_client): +@pytest.mark.parametrize("validating_httpserver", [{"app": ContentServer}], indirect=True) +@mock.patch("elasticapm.transport.base.TransportState.should_try") +def test_send_remote_failover_sync(should_try, sending_elasticapm_client, caplog): sending_elasticapm_client.httpserver.code = 400 sending_elasticapm_client.httpserver.content = "go away" should_try.return_value = True - logger = mock.Mock() - sending_elasticapm_client.error_logger.error = logger - # test error - sending_elasticapm_client.send(sending_elasticapm_client.config.server_url, **{"message": "foo"}) - assert sending_elasticapm_client.state.status == sending_elasticapm_client.state.ERROR - assert len(logger.call_args_list) == 2 - assert "go away" in logger.call_args_list[0][0][0] - assert "foo" in logger.call_args_list[1][0][1] + with caplog.at_level("ERROR", "elasticapm.transport"): + sending_elasticapm_client.capture_message("foo", handled=False) + assert sending_elasticapm_client._transport.state.did_fail() + record = caplog.records[0] + assert "go away" in record.message # test recovery sending_elasticapm_client.httpserver.code = 202 - sending_elasticapm_client.send(sending_elasticapm_client.config.server_url, **{"message": "foo"}) - assert sending_elasticapm_client.state.status == sending_elasticapm_client.state.ONLINE + sending_elasticapm_client.capture_message("bar", handled=False) + assert not sending_elasticapm_client._transport.state.did_fail() @mock.patch("elasticapm.transport.http.Transport.send") -@mock.patch("elasticapm.base.ClientState.should_try") -def test_send_remote_failover_sync_stdlib(should_try, http_send): +@mock.patch("elasticapm.transport.base.TransportState.should_try") +def test_send_remote_failover_sync_non_transport_exception_error(should_try, http_send, caplog): should_try.return_value = True client = Client( @@ -169,20 +122,18 @@ def test_send_remote_failover_sync_stdlib(should_try, http_send): secret_token="secret", transport_class="elasticapm.transport.http.Transport", ) - logger = mock.Mock() - client.error_logger.error = logger - # test error http_send.side_effect = ValueError("oopsie") - client.send("http://example.com/api/store", **{"message": "oh no"}) - assert client.state.status == client.state.ERROR - assert len(logger.call_args_list) == 1 - assert "oopsie" in logger.call_args_list[0][0][1] + with caplog.at_level("ERROR", "elasticapm.transport"): + client.capture_message("foo", handled=False) + record = caplog.records[0] + assert client._transport.state.did_fail() + assert "oopsie" in record.message # test recovery http_send.side_effect = None - client.send("http://example.com/api/store", **{"message": "oh no"}) - assert client.state.status == client.state.ONLINE + client.capture_message("foo", handled=False) + assert not client._transport.state.did_fail() client.close() @@ -191,38 +142,35 @@ def test_send_remote_failover_sync_stdlib(should_try, http_send): [{"transport_class": "elasticapm.transport.http.AsyncTransport", "async_mode": True}], indirect=True, ) -@mock.patch("elasticapm.base.ClientState.should_try") -def test_send_remote_failover_async(should_try, sending_elasticapm_client): +@pytest.mark.parametrize("validating_httpserver", [{"app": ContentServer}], indirect=True) +@mock.patch("elasticapm.transport.base.TransportState.should_try") +def test_send_remote_failover_async(should_try, sending_elasticapm_client, caplog): should_try.return_value = True sending_elasticapm_client.httpserver.code = 400 - logger = mock.Mock() - sending_elasticapm_client.error_logger.error = logger # test error - sending_elasticapm_client.send(sending_elasticapm_client.config.server_url, **{"message": "oh no"}) - sending_elasticapm_client.close() - assert sending_elasticapm_client.state.status == sending_elasticapm_client.state.ERROR - assert len(logger.call_args_list) == 2 - assert "400" in logger.call_args_list[0][0][0] - assert "oh no" in logger.call_args_list[1][0][1] + with caplog.at_level("ERROR", "elasticapm.transport"): + sending_elasticapm_client.capture_message("foo", handled=False) + sending_elasticapm_client.close() + assert sending_elasticapm_client._transport.state.did_fail() + assert "400" in caplog.records[0].message # test recovery sending_elasticapm_client.httpserver.code = 202 - sending_elasticapm_client.send(sending_elasticapm_client.config.server_url, **{"message": "yay"}) - sending_elasticapm_client.close() - assert sending_elasticapm_client.state.status == sending_elasticapm_client.state.ONLINE - + with caplog.at_level("ERROR", "elasticapm.transport"): + sending_elasticapm_client.capture_message("bar", handled=False) + sending_elasticapm_client.close() + assert not sending_elasticapm_client._transport.state.did_fail() -@mock.patch("elasticapm.base.time.time") -def test_send(time, sending_elasticapm_client): - time.return_value = 1328055286.51 - sending_elasticapm_client.send(sending_elasticapm_client.config.server_url, foo="bar") +@pytest.mark.parametrize("validating_httpserver", [{"skip_validate": True}], indirect=True) +def test_send(sending_elasticapm_client): + sending_elasticapm_client.queue("x", {}) sending_elasticapm_client.close() request = sending_elasticapm_client.httpserver.requests[0] expected_headers = { - "Content-Type": "application/json", - "Content-Encoding": "deflate", + "Content-Type": "application/x-ndjson", + "Content-Encoding": "gzip", "Authorization": "Bearer %s" % sending_elasticapm_client.config.secret_token, "User-Agent": "elasticapm-python/%s" % elasticapm.VERSION, } @@ -230,15 +178,13 @@ def test_send(time, sending_elasticapm_client): for k, v in expected_headers.items(): assert seen_headers[k] == v - assert request.content_length == 22 + assert 250 < request.content_length < 350 @pytest.mark.parametrize("sending_elasticapm_client", [{"disable_send": True}], indirect=True) -@mock.patch("elasticapm.base.time.time") -def test_send_not_enabled(time, sending_elasticapm_client): - time.return_value = 1328055286.51 +def test_send_not_enabled(sending_elasticapm_client): assert sending_elasticapm_client.config.disable_send - sending_elasticapm_client.send(sending_elasticapm_client.config.server_url, foo="bar") + sending_elasticapm_client.queue("x", {}) sending_elasticapm_client.close() assert len(sending_elasticapm_client.httpserver.requests) == 0 @@ -249,13 +195,10 @@ def test_send_not_enabled(time, sending_elasticapm_client): [{"transport_class": "elasticapm.transport.http.Transport", "async_mode": False}], indirect=True, ) -@mock.patch("elasticapm.base.Client._collect_transactions") -def test_client_shutdown_sync(mock_traces_collect, sending_elasticapm_client): - sending_elasticapm_client.send(sending_elasticapm_client.config.server_url, foo="bar") +def test_client_shutdown_sync(sending_elasticapm_client): + sending_elasticapm_client.capture_message("x") sending_elasticapm_client.close() assert len(sending_elasticapm_client.httpserver.requests) == 1 - assert mock_traces_collect.call_count == 1 - assert len(sending_elasticapm_client._transports) == 0 @pytest.mark.parametrize( @@ -263,20 +206,10 @@ def test_client_shutdown_sync(mock_traces_collect, sending_elasticapm_client): [{"transport_class": "elasticapm.transport.http.AsyncTransport", "async_mode": True}], indirect=True, ) -@mock.patch("elasticapm.base.Client._collect_transactions") -def test_client_shutdown_async(mock_traces_collect, sending_elasticapm_client): - sending_elasticapm_client.send(sending_elasticapm_client.config.server_url, foo="bar") +def test_client_shutdown_async(sending_elasticapm_client): + sending_elasticapm_client.capture_message("x") sending_elasticapm_client.close() - assert mock_traces_collect.call_count == 1 assert len(sending_elasticapm_client.httpserver.requests) == 1 - assert len(sending_elasticapm_client._transports) == 0 - - -def test_encode_decode(elasticapm_client): - data = {"foo": "bar"} - encoded = elasticapm_client.encode(data) - assert isinstance(encoded, compat.binary_type) - assert data == elasticapm_client.decode(encoded) def test_explicit_message_on_exception_event(elasticapm_client): @@ -286,7 +219,7 @@ def test_explicit_message_on_exception_event(elasticapm_client): elasticapm_client.capture("Exception", message="foobar") assert len(elasticapm_client.events) == 1 - event = elasticapm_client.events.pop(0)["errors"][0] + event = elasticapm_client.events[ERROR][0] assert event["exception"]["message"] == "foobar" @@ -305,7 +238,7 @@ def test_exception_event(elasticapm_client): elasticapm_client.capture("Exception") assert len(elasticapm_client.events) == 1 - event = elasticapm_client.events.pop(0)["errors"][0] + event = elasticapm_client.events[ERROR][0] assert "exception" in event exc = event["exception"] assert exc["message"] == "ValueError: foo" @@ -344,7 +277,7 @@ def test_message_event(elasticapm_client): elasticapm_client.capture("Message", message="test") assert len(elasticapm_client.events) == 1 - event = elasticapm_client.events.pop(0)["errors"][0] + event = elasticapm_client.events[ERROR][0] assert event["log"]["message"] == "test" assert "stacktrace" not in event assert "timestamp" in event @@ -366,7 +299,7 @@ def test_param_message_event(elasticapm_client): elasticapm_client.capture("Message", param_message={"message": "test %s %d", "params": ("x", 1)}) assert len(elasticapm_client.events) == 1 - event = elasticapm_client.events.pop(0)["errors"][0] + event = elasticapm_client.events[ERROR][0] assert event["log"]["message"] == "test x 1" assert event["log"]["param_message"] == "test %s %d" @@ -375,7 +308,7 @@ def test_message_with_percent(elasticapm_client): elasticapm_client.capture("Message", message="This works 100% of the time") assert len(elasticapm_client.events) == 1 - event = elasticapm_client.events.pop(0)["errors"][0] + event = elasticapm_client.events[ERROR][0] assert event["log"]["message"] == "This works 100% of the time" assert event["log"]["param_message"] == "This works 100% of the time" @@ -384,65 +317,47 @@ def test_logger(elasticapm_client): elasticapm_client.capture("Message", message="test", logger_name="test") assert len(elasticapm_client.events) == 1 - event = elasticapm_client.events.pop(0)["errors"][0] + event = elasticapm_client.events[ERROR][0] assert event["log"]["logger_name"] == "test" assert "timestamp" in event -@mock.patch("elasticapm.base.TransactionsStore.should_collect") -def test_metrics_collection(should_collect, sending_elasticapm_client): - should_collect.return_value = False - for i in range(7): - sending_elasticapm_client.begin_transaction("transaction.test") - sending_elasticapm_client.end_transaction("test-transaction", 200) - - assert len(sending_elasticapm_client.transaction_store) == 7 - assert len(sending_elasticapm_client.httpserver.requests) == 0 - should_collect.return_value = True - - sending_elasticapm_client.begin_transaction("transaction.test") - sending_elasticapm_client.end_transaction("my-other-transaction", 200) - assert len(sending_elasticapm_client.httpserver.requests) == 1 - - -@mock.patch("elasticapm.base.TransactionsStore.should_collect") -def test_call_end_twice(should_collect, elasticapm_client): - should_collect.return_value = False +def test_call_end_twice(elasticapm_client): elasticapm_client.begin_transaction("celery") elasticapm_client.end_transaction("test-transaction", 200) elasticapm_client.end_transaction("test-transaction", 200) +@mock.patch("elasticapm.transport.base.Transport.queue") @mock.patch("elasticapm.base.is_master_process") -def test_client_uses_sync_mode_when_master_process(is_master_process): - # when in the master process, the client should use the non-async - # HTTP transport, even if async_mode is True +def test_client_doesnt_flush_when_in_master_process(is_master_process, mock_queue): + # when in the master process, the client should not flush the + # HTTP transport is_master_process.return_value = True - client = Client(server_url="http://example.com", service_name="app_name", secret_token="secret", async_mode=True) - transport = client._get_transport(compat.urlparse.urlparse("http://exampe.com")) - assert transport.async_mode is False + client = Client(server_url="http://example.com", service_name="app_name", secret_token="secret") + client.queue("x", {}, flush=True) + assert mock_queue.call_count == 1 + assert mock_queue.call_args[0] == ("x", {}, False) @pytest.mark.parametrize("elasticapm_client", [{"verify_server_cert": False}], indirect=True) def test_client_disables_ssl_verification(elasticapm_client): assert not elasticapm_client.config.verify_server_cert - assert not elasticapm_client._get_transport(compat.urlparse.urlparse("https://example.com"))._verify_server_cert + assert not elasticapm_client._transport._verify_server_cert @pytest.mark.parametrize( "elasticapm_client", [{"transactions_ignore_patterns": ["^OPTIONS", "views.api.v2"]}], indirect=True ) -@mock.patch("elasticapm.base.TransactionsStore.should_collect") -def test_ignore_patterns(should_collect, elasticapm_client): - should_collect.return_value = False +def test_ignore_patterns(elasticapm_client): elasticapm_client.begin_transaction("web") elasticapm_client.end_transaction("OPTIONS views.healthcheck", 200) elasticapm_client.begin_transaction("web") elasticapm_client.end_transaction("GET views.users", 200) - transactions = elasticapm_client.transaction_store.get_all() + transactions = elasticapm_client.events[TRANSACTION] assert len(transactions) == 1 assert transactions[0]["name"] == "GET views.users" @@ -474,30 +389,12 @@ def test_invalid_service_name_disables_send(elasticapm_client): assert elasticapm_client.config.disable_send -@pytest.mark.parametrize( - "elasticapm_client", [{"service_name": "foo", "config": {"TRANSPORT_CLASS": None}}], indirect=True -) -def test_empty_transport_disables_send(elasticapm_client): - assert len(elasticapm_client.config.errors) == 1 - assert "TRANSPORT_CLASS" in elasticapm_client.config.errors - - assert elasticapm_client.config.disable_send - - -@pytest.mark.parametrize("elasticapm_client", [{"flush_interval": 2}], indirect=True) -def test_send_timer(elasticapm_client): - assert elasticapm_client._send_timer is None - assert elasticapm_client.config.flush_interval == 2 - elasticapm_client.begin_transaction("test_type") - elasticapm_client.end_transaction("test") - - assert elasticapm_client._send_timer is not None - assert elasticapm_client._send_timer.interval == 2 - assert elasticapm_client._send_timer.is_alive() +def test_empty_transport_disables_send(): + client = Client(service_name="x", transport_class=None) + assert len(client.config.errors) == 1 + assert "TRANSPORT_CLASS" in client.config.errors - elasticapm_client.close() - - assert not elasticapm_client._send_timer.is_alive() + assert client.config.disable_send @pytest.mark.parametrize( @@ -516,7 +413,7 @@ def test_collect_local_variables_errors(elasticapm_client): 1 / 0 except ZeroDivisionError: elasticapm_client.capture_exception() - event = elasticapm_client.events[0]["errors"][0] + event = elasticapm_client.events[ERROR][0] if mode in ("errors", "all"): assert "vars" in event["exception"]["stacktrace"][0], mode else: @@ -541,7 +438,7 @@ def test_collect_source_errors(elasticapm_client): json.dumps(datetime.datetime.now()) except TypeError: elasticapm_client.capture_exception() - event = elasticapm_client.events[0]["errors"][0] + event = elasticapm_client.events[ERROR][0] in_app_frame = event["exception"]["stacktrace"][0] library_frame = event["exception"]["stacktrace"][1] assert not in_app_frame["library_frame"] @@ -578,9 +475,7 @@ def test_collect_source_errors(elasticapm_client): ], indirect=True, ) -@mock.patch("elasticapm.base.TransactionsStore.should_collect") -def test_collect_local_variables_transactions(should_collect, elasticapm_client): - should_collect.return_value = False +def test_collect_local_variables_transactions(elasticapm_client): mode = elasticapm_client.config.collect_local_variables elasticapm_client.begin_transaction("test") with elasticapm.capture_span("foo"): @@ -589,8 +484,7 @@ def test_collect_local_variables_transactions(should_collect, elasticapm_client) a_long_local_list = list(range(100)) pass elasticapm_client.end_transaction("test", "ok") - transaction = elasticapm_client.transaction_store.get_all()[0] - frame = transaction["spans"][0]["stacktrace"][0] + frame = elasticapm_client.events[SPAN][0]["stacktrace"][0] if mode in ("transactions", "all"): assert "vars" in frame, mode assert frame["vars"]["a_local_var"] == 1 @@ -610,18 +504,16 @@ def test_collect_local_variables_transactions(should_collect, elasticapm_client) ], indirect=True, ) -@mock.patch("elasticapm.base.TransactionsStore.should_collect") -def test_collect_source_transactions(should_collect, elasticapm_client): - should_collect.return_value = False +def test_collect_source_transactions(elasticapm_client): library_frame_context = elasticapm_client.config.source_lines_span_library_frames in_app_frame_context = elasticapm_client.config.source_lines_span_app_frames elasticapm_client.begin_transaction("test") with elasticapm.capture_span("foo"): pass elasticapm_client.end_transaction("test", "ok") - transaction = elasticapm_client.transaction_store.get_all()[0] - in_app_frame = transaction["spans"][0]["stacktrace"][0] - library_frame = transaction["spans"][0]["stacktrace"][1] + span = elasticapm_client.events[SPAN][0] + in_app_frame = span["stacktrace"][0] + library_frame = span["stacktrace"][1] assert not in_app_frame["library_frame"] assert library_frame["library_frame"] if library_frame_context: @@ -653,35 +545,34 @@ def test_transaction_id_is_attached(elasticapm_client): transaction = elasticapm_client.end_transaction("test", "test") elasticapm_client.capture_message("noid") - errors = elasticapm_client.events - assert "transaction" not in errors[0]["errors"][0] - assert errors[1]["errors"][0]["transaction"]["id"] == transaction.id - assert "transaction" not in errors[2]["errors"][0] + errors = elasticapm_client.events[ERROR] + assert "transaction" not in errors[0] + assert errors[1]["transaction"]["id"] == transaction.id + assert "transaction" not in errors[2] @pytest.mark.parametrize("elasticapm_client", [{"transaction_sample_rate": 0.4}], indirect=True) -@mock.patch("elasticapm.base.TransactionsStore.should_collect") -def test_transaction_sampling(should_collect, elasticapm_client, not_so_random): - should_collect.return_value = False +def test_transaction_sampling(elasticapm_client, not_so_random): for i in range(10): elasticapm_client.begin_transaction("test_type") with elasticapm.capture_span("xyz"): pass elasticapm_client.end_transaction("test") - transactions = elasticapm_client.transaction_store.get_all() + transactions = elasticapm_client.events[TRANSACTION] + spans_per_transaction = defaultdict(list) + for span in elasticapm_client.events[SPAN]: + spans_per_transaction[span["transaction_id"]].append(span) # seed is fixed by not_so_random fixture assert len([t for t in transactions if t["sampled"]]) == 5 for transaction in transactions: - assert transaction["sampled"] or not "spans" in transaction + assert transaction["sampled"] or not transaction["id"] in spans_per_transaction assert transaction["sampled"] or not "context" in transaction @pytest.mark.parametrize("elasticapm_client", [{"transaction_max_spans": 5}], indirect=True) -@mock.patch("elasticapm.base.TransactionsStore.should_collect") -def test_transaction_max_spans(should_collect, elasticapm_client): - should_collect.return_value = False +def test_transaction_max_spans(elasticapm_client): elasticapm_client.begin_transaction("test_type") for i in range(5): with elasticapm.capture_span("nodrop"): @@ -691,20 +582,20 @@ def test_transaction_max_spans(should_collect, elasticapm_client): pass transaction_obj = elasticapm_client.end_transaction("test") - transaction = elasticapm_client.transaction_store.get_all()[0] + transaction = elasticapm_client.events[TRANSACTION][0] + spans = elasticapm_client.events[SPAN] + assert all(span["transaction_id"] == transaction["id"] for span in spans) - assert transaction_obj.max_spans == 5 + assert transaction_obj._store.max_spans == 5 assert transaction_obj.dropped_spans == 10 - assert len(transaction["spans"]) == 5 - for span in transaction["spans"]: + assert len(spans) == 5 + for span in spans: assert span["name"] == "nodrop" - assert transaction["span_count"] == {"dropped": {"total": 10}} + assert transaction["span_count"] == {"dropped": 10, "started": 5} -@pytest.mark.parametrize("elasticapm_client", [{"span_frames_min_duration_ms": 20}], indirect=True) -@mock.patch("elasticapm.base.TransactionsStore.should_collect") -def test_transaction_span_frames_min_duration(should_collect, elasticapm_client): - should_collect.return_value = False +@pytest.mark.parametrize("elasticapm_client", [{"span_frames_min_duration": 20}], indirect=True) +def test_transaction_span_frames_min_duration(elasticapm_client): elasticapm_client.begin_transaction("test_type") with elasticapm.capture_span("noframes"): time.sleep(0.001) @@ -712,8 +603,7 @@ def test_transaction_span_frames_min_duration(should_collect, elasticapm_client) time.sleep(0.040) elasticapm_client.end_transaction("test") - transaction = elasticapm_client.transaction_store.get_all()[0] - spans = transaction["spans"] + spans = elasticapm_client.events[SPAN] assert len(spans) == 2 assert spans[0]["name"] == "noframes" @@ -724,9 +614,7 @@ def test_transaction_span_frames_min_duration(should_collect, elasticapm_client) @pytest.mark.parametrize("elasticapm_client", [{"span_frames_min_durarion_ms": -1}], indirect=True) -@mock.patch("elasticapm.base.TransactionsStore.should_collect") -def test_transaction_span_frames_min_duration_no_limit(should_collect, elasticapm_client): - should_collect.return_value = False +def test_transaction_span_frames_min_duration_no_limit(elasticapm_client): elasticapm_client.begin_transaction("test_type") with elasticapm.capture_span("frames"): pass @@ -734,8 +622,7 @@ def test_transaction_span_frames_min_duration_no_limit(should_collect, elasticap time.sleep(0.040) elasticapm_client.end_transaction("test") - transaction = elasticapm_client.transaction_store.get_all()[0] - spans = transaction["spans"] + spans = elasticapm_client.events[SPAN] assert len(spans) == 2 assert spans[0]["name"] == "frames" @@ -746,9 +633,7 @@ def test_transaction_span_frames_min_duration_no_limit(should_collect, elasticap @pytest.mark.parametrize("elasticapm_client", [{"transaction_max_spans": 3}], indirect=True) -@mock.patch("elasticapm.base.TransactionsStore.should_collect") -def test_transaction_max_span_nested(should_collect, elasticapm_client): - should_collect.return_value = False +def test_transaction_max_span_nested(elasticapm_client): elasticapm_client.begin_transaction("test_type") with elasticapm.capture_span("1"): with elasticapm.capture_span("2"): @@ -766,13 +651,14 @@ def test_transaction_max_span_nested(should_collect, elasticapm_client): pass transaction_obj = elasticapm_client.end_transaction("test") - transaction = elasticapm_client.transaction_store.get_all()[0] + transaction = elasticapm_client.events[TRANSACTION][0] + spans = elasticapm_client.events[SPAN] assert transaction_obj.dropped_spans == 6 - assert len(transaction["spans"]) == 3 - for span in transaction["spans"]: + assert len(spans) == 3 + for span in spans: assert span["name"] in ("1", "2", "3") - assert transaction["span_count"] == {"dropped": {"total": 6}} + assert transaction["span_count"] == {"dropped": 6, "started": 3} def test_transaction_context_is_used_in_errors(elasticapm_client): @@ -782,7 +668,7 @@ def test_transaction_context_is_used_in_errors(elasticapm_client): elasticapm.set_user_context(username="foo", email="foo@example.com", user_id=42) elasticapm_client.capture_message("x", custom={"foo": "bar"}) transaction = elasticapm_client.end_transaction("test", "OK") - message = elasticapm_client.events[0]["errors"][0] + message = elasticapm_client.events[ERROR][0] assert message["context"]["custom"] == {"a": "b", "foo": "bar"} assert message["context"]["user"] == {"username": "foo", "email": "foo@example.com", "id": 42} assert message["context"]["tags"] == {"foo": "baz"} @@ -804,8 +690,8 @@ def test_transaction_keyword_truncation(sending_elasticapm_client): sending_elasticapm_client.end_transaction(too_long, too_long) sending_elasticapm_client.close() assert sending_elasticapm_client.httpserver.responses[0]["code"] == 202 - transaction = sending_elasticapm_client.httpserver.payloads[0]["transactions"][0] - span = transaction["spans"][0] + span = sending_elasticapm_client.httpserver.payloads[0][1]["span"] + transaction = sending_elasticapm_client.httpserver.payloads[0][2]["transaction"] assert transaction["name"] == expected assert transaction["type"] == expected @@ -831,8 +717,8 @@ def test_error_keyword_truncation(sending_elasticapm_client): try: raise WayTooLongException() except WayTooLongException: - sending_elasticapm_client.capture_exception() - error = sending_elasticapm_client.httpserver.payloads[0]["errors"][0] + sending_elasticapm_client.capture_exception(handled=False) + error = sending_elasticapm_client.httpserver.payloads[0][1]["error"] assert error["exception"]["type"] == expected.upper() assert error["exception"]["module"] == expected @@ -841,9 +727,11 @@ def test_error_keyword_truncation(sending_elasticapm_client): def test_message_keyword_truncation(sending_elasticapm_client): too_long = "x" * (KEYWORD_MAX_LENGTH + 1) expected = encoding.keyword_field(too_long) - sending_elasticapm_client.capture_message(param_message={"message": too_long, "params": []}, logger_name=too_long) + sending_elasticapm_client.capture_message( + param_message={"message": too_long, "params": []}, logger_name=too_long, handled=False + ) - error = sending_elasticapm_client.httpserver.payloads[0]["errors"][0] + error = sending_elasticapm_client.httpserver.payloads[0][1]["error"] assert error["log"]["param_message"] == expected assert error["log"]["message"] == too_long # message is not truncated @@ -852,20 +740,18 @@ def test_message_keyword_truncation(sending_elasticapm_client): @pytest.mark.parametrize("sending_elasticapm_client", [{"service_name": "*"}], indirect=True) -@mock.patch("elasticapm.base.Client._send_remote") -def test_config_error_stops_error_send(mock_send_remote, sending_elasticapm_client): +@mock.patch("elasticapm.transport.base.Transport.queue") +def test_config_error_stops_error_send(mock_queue, sending_elasticapm_client): assert sending_elasticapm_client.config.disable_send is True - sending_elasticapm_client.capture_message("bla") - assert mock_send_remote.call_count == 0 + sending_elasticapm_client.capture_message("bla", handled=False) + assert mock_queue.call_count == 0 @pytest.mark.parametrize("sending_elasticapm_client", [{"service_name": "*"}], indirect=True) -@mock.patch("elasticapm.base.Client._send_remote") -@mock.patch("elasticapm.base.TransactionsStore.should_collect") -def test_config_error_stops_transaction_send(should_collect, mock_send_remote, sending_elasticapm_client): - should_collect.return_value = False +@mock.patch("elasticapm.transport.base.Transport.queue") +def test_config_error_stops_transaction_send(mock_queue, sending_elasticapm_client): assert sending_elasticapm_client.config.disable_send is True sending_elasticapm_client.begin_transaction("test") sending_elasticapm_client.end_transaction("test", "OK") sending_elasticapm_client.close() - assert mock_send_remote.call_count == 0 + assert mock_queue.call_count == 0 diff --git a/tests/config/tests.py b/tests/config/tests.py index b6d6d5034..de0083427 100644 --- a/tests/config/tests.py +++ b/tests/config/tests.py @@ -4,7 +4,17 @@ import mock -from elasticapm.conf import Config, _BoolConfigValue, _ConfigBase, _ConfigValue, _ListConfigValue, setup_logging +from elasticapm.conf import ( + Config, + RegexValidator, + _BoolConfigValue, + _ConfigBase, + _ConfigValue, + _ListConfigValue, + duration_validator, + setup_logging, + size_validator, +) def test_basic_not_configured(): @@ -33,7 +43,7 @@ def test_config_dict(): "SERVER_URL": "http://example.com:1234", "SERVICE_VERSION": 1, "HOSTNAME": "localhost", - "FLUSH_INTERVAL": "5", + "API_REQUEST_TIME": "5s", } ) @@ -42,7 +52,7 @@ def test_config_dict(): assert config.server_url == "http://example.com:1234" assert config.service_version == "1" assert config.hostname == "localhost" - assert config.flush_interval == 5 + assert config.api_request_time == 5000 def test_config_environment(): @@ -54,7 +64,7 @@ def test_config_environment(): "ELASTIC_APM_SERVER_URL": "http://example.com:1234", "ELASTIC_APM_SERVICE_VERSION": "1", "ELASTIC_APM_HOSTNAME": "localhost", - "ELASTIC_APM_FLUSH_INTERVAL": "5", + "ELASTIC_APM_API_REQUEST_TIME": "5s", "ELASTIC_APM_AUTO_LOG_STACKS": "false", }, ): @@ -65,8 +75,8 @@ def test_config_environment(): assert config.server_url == "http://example.com:1234" assert config.service_version == "1" assert config.hostname == "localhost" - assert config.flush_interval == 5 - assert config.auto_log_stacks == False + assert config.api_request_time == 5000 + assert config.auto_log_stacks is False def test_config_inline_dict(): @@ -77,7 +87,7 @@ def test_config_inline_dict(): "server_url": "http://example.com:1234", "service_version": "1", "hostname": "localhost", - "flush_interval": "5", + "api_request_time": "5s", } ) @@ -86,7 +96,7 @@ def test_config_inline_dict(): assert config.server_url == "http://example.com:1234" assert config.service_version == "1" assert config.hostname == "localhost" - assert config.flush_interval == 5 + assert config.api_request_time == 5000 def test_config_precedence(): @@ -131,3 +141,51 @@ class MyConfig(_ConfigBase): c2 = MyConfig({"MY_BOOL": "nope"}) assert c1.my_bool is not c2.my_bool + + +def test_regex_validation(): + class MyConfig(_ConfigBase): + my_regex = _ConfigValue("MY_REGEX", validators=[RegexValidator("\d+")]) + + c1 = MyConfig({"MY_REGEX": "123"}) + c2 = MyConfig({"MY_REGEX": "abc"}) + assert not c1.errors + assert "MY_REGEX" in c2.errors + + +def test_size_validation(): + class MyConfig(_ConfigBase): + byte = _ConfigValue("BYTE", type=int, validators=[size_validator]) + kbyte = _ConfigValue("KBYTE", type=int, validators=[size_validator]) + mbyte = _ConfigValue("MBYTE", type=int, validators=[size_validator]) + wrong_pattern = _ConfigValue("WRONG_PATTERN", type=int, validators=[size_validator]) + + c = MyConfig({"BYTE": "10b", "KBYTE": "5kib", "MBYTE": "17mib", "WRONG_PATTERN": "5 kb"}) + assert c.byte == 10 + assert c.kbyte == 5 * 1024 + assert c.mbyte == 17 * 1024 * 1024 + assert c.wrong_pattern is None + assert "WRONG_PATTERN" in c.errors + + +def test_duration_validation(): + class MyConfig(_ConfigBase): + millisecond = _ConfigValue("MS", type=int, validators=[duration_validator]) + second = _ConfigValue("S", type=int, validators=[duration_validator]) + minute = _ConfigValue("M", type=int, validators=[duration_validator]) + wrong_pattern = _ConfigValue("WRONG_PATTERN", type=int, validators=[duration_validator]) + + c = MyConfig({"MS": "-10ms", "S": "5s", "M": "17m", "WRONG_PATTERN": "5 ms"}) + assert c.millisecond == -10 + assert c.second == 5 * 1000 + assert c.minute == 17 * 1000 * 60 + assert c.wrong_pattern is None + assert "WRONG_PATTERN" in c.errors + + +def test_chained_validators(): + class MyConfig(_ConfigBase): + chained = _ConfigValue("CHAIN", validators=[lambda val, field: val.upper(), lambda val, field: val * 2]) + + c = MyConfig({"CHAIN": "x"}) + assert c.chained == "XX" diff --git a/tests/contrib/celery/django_tests.py b/tests/contrib/celery/django_tests.py index efbc1a2ad..263fc3bf5 100644 --- a/tests/contrib/celery/django_tests.py +++ b/tests/contrib/celery/django_tests.py @@ -6,6 +6,7 @@ import mock +from elasticapm.conf.constants import ERROR, TRANSACTION from elasticapm.contrib.celery import register_exception_tracking, register_instrumentation from tests.contrib.django.testapp.tasks import failing_task, successful_task @@ -14,17 +15,16 @@ def test_failing_celery_task(django_elasticapm_client): register_exception_tracking(django_elasticapm_client) - with mock.patch("elasticapm.traces.TransactionsStore.should_collect") as should_collect_mock: - should_collect_mock.return_value = True - t = failing_task.delay() + t = failing_task.delay() assert t.state == "FAILURE" - assert len(django_elasticapm_client.events) == 2 - error = django_elasticapm_client.events[0]["errors"][0] + assert len(django_elasticapm_client.events[ERROR]) == 1 + assert len(django_elasticapm_client.events[TRANSACTION]) == 1 + error = django_elasticapm_client.events[ERROR][0] assert error["culprit"] == "tests.contrib.django.testapp.tasks.failing_task" assert error["exception"]["message"] == "ValueError: foo" assert error["exception"]["handled"] is False - transaction = django_elasticapm_client.events[1]["transactions"][0] + transaction = django_elasticapm_client.events[TRANSACTION][0] assert transaction["name"] == "tests.contrib.django.testapp.tasks.failing_task" assert transaction["type"] == "celery" assert transaction["result"] == "FAILURE" @@ -32,12 +32,10 @@ def test_failing_celery_task(django_elasticapm_client): def test_successful_celery_task_instrumentation(django_elasticapm_client): register_instrumentation(django_elasticapm_client) - with mock.patch("elasticapm.traces.TransactionsStore.should_collect") as should_collect_mock: - should_collect_mock.return_value = True - t = successful_task.delay() + t = successful_task.delay() assert t.state == "SUCCESS" - assert len(django_elasticapm_client.events[0]["transactions"]) == 1 - transaction = django_elasticapm_client.events[0]["transactions"][0] + assert len(django_elasticapm_client.events[TRANSACTION]) == 1 + transaction = django_elasticapm_client.events[TRANSACTION][0] assert transaction["name"] == "tests.contrib.django.testapp.tasks.successful_task" assert transaction["type"] == "celery" assert transaction["result"] == "SUCCESS" diff --git a/tests/contrib/celery/flask_tests.py b/tests/contrib/celery/flask_tests.py index acd8a06b7..b60814100 100644 --- a/tests/contrib/celery/flask_tests.py +++ b/tests/contrib/celery/flask_tests.py @@ -5,6 +5,8 @@ import mock +from elasticapm.conf.constants import ERROR, TRANSACTION + pytestmark = pytest.mark.celery @@ -15,17 +17,15 @@ def test_task_failure(flask_celery): def failing_task(): raise ValueError("foo") - with mock.patch("elasticapm.traces.TransactionsStore.should_collect") as should_collect_mock: - should_collect_mock.return_value = True - t = failing_task.delay() + t = failing_task.delay() assert t.status == "FAILURE" - assert len(apm_client.events[0]["errors"]) == 1 - error = apm_client.events[0]["errors"][0] + assert len(apm_client.events[ERROR]) == 1 + error = apm_client.events[ERROR][0] assert error["culprit"] == "tests.contrib.celery.flask_tests.failing_task" assert error["exception"]["message"] == "ValueError: foo" assert error["exception"]["handled"] is False - transaction = apm_client.events[1]["transactions"][0] + transaction = apm_client.events[TRANSACTION][0] assert transaction["name"] == "tests.contrib.celery.flask_tests.failing_task" assert transaction["type"] == "celery" assert transaction["result"] == "FAILURE" @@ -38,13 +38,11 @@ def test_task_instrumentation(flask_celery): def successful_task(): return "OK" - with mock.patch("elasticapm.traces.TransactionsStore.should_collect") as should_collect_mock: - should_collect_mock.return_value = True - t = successful_task.delay() + t = successful_task.delay() assert t.status == "SUCCESS" - assert len(apm_client.events[0]["transactions"]) == 1 - transaction = apm_client.events[0]["transactions"][0] + assert len(apm_client.events[TRANSACTION]) == 1 + transaction = apm_client.events[TRANSACTION][0] assert transaction["name"] == "tests.contrib.celery.flask_tests.successful_task" assert transaction["type"] == "celery" assert transaction["result"] == "SUCCESS" diff --git a/tests/contrib/django/django_tests.py b/tests/contrib/django/django_tests.py index 8384770d4..4116e1814 100644 --- a/tests/contrib/django/django_tests.py +++ b/tests/contrib/django/django_tests.py @@ -25,6 +25,7 @@ import mock from elasticapm.base import Client +from elasticapm.conf.constants import ERROR, SPAN, TRANSACTION from elasticapm.contrib.django.client import client, get_client from elasticapm.contrib.django.handlers import LoggingHandler from elasticapm.contrib.django.middleware.wsgi import ElasticAPM @@ -67,17 +68,15 @@ def test_proxy_responds_as_client(): def test_basic(django_elasticapm_client): config = {"APP_ID": "key", "ORGANIZATION_ID": "org", "SECRET_TOKEN": "99"} - event_count = len(django_elasticapm_client.events) with override_settings(ELASTIC_APM=config): django_elasticapm_client.capture("Message", message="foo") - assert len(django_elasticapm_client.events) == event_count + 1 - django_elasticapm_client.events.pop(0) + assert len(django_elasticapm_client.events[ERROR]) == 1 def test_basic_django(django_elasticapm_client): django_elasticapm_client.capture("Message", message="foo") assert len(django_elasticapm_client.events) == 1 - event = django_elasticapm_client.events.pop(0)["errors"][0] + event = django_elasticapm_client.events[ERROR][0] log = event["log"] assert "message" in log @@ -93,7 +92,7 @@ def test_signal_integration(django_elasticapm_client): got_request_exception.send(sender=None, request=None) assert len(django_elasticapm_client.events) == 1 - event = django_elasticapm_client.events.pop(0)["errors"][0] + event = django_elasticapm_client.events[ERROR][0] assert "exception" in event exc = event["exception"] assert exc["type"] == "ValueError" @@ -107,7 +106,7 @@ def test_view_exception(django_elasticapm_client, client): client.get(reverse("elasticapm-raise-exc")) assert len(django_elasticapm_client.events) == 1 - event = django_elasticapm_client.events.pop(0)["errors"][0] + event = django_elasticapm_client.events[ERROR][0] assert "exception" in event exc = event["exception"] assert exc["type"] == "MyException" @@ -141,8 +140,8 @@ def test_user_info(django_elasticapm_client, client): with pytest.raises(Exception): client.get(reverse("elasticapm-raise-exc")) - assert len(django_elasticapm_client.events) == 1 - event = django_elasticapm_client.events.pop(0)["errors"][0] + assert len(django_elasticapm_client.events[ERROR]) == 1 + event = django_elasticapm_client.events[ERROR][0] assert "user" in event["context"] user_info = event["context"]["user"] assert "is_authenticated" in user_info @@ -155,8 +154,8 @@ def test_user_info(django_elasticapm_client, client): with pytest.raises(Exception): client.get(reverse("elasticapm-raise-exc")) - assert len(django_elasticapm_client.events) == 1 - event = django_elasticapm_client.events.pop(0)["errors"][0] + assert len(django_elasticapm_client.events[ERROR]) == 2 + event = django_elasticapm_client.events[ERROR][1] assert "user" in event["context"] user_info = event["context"]["user"] assert "is_authenticated" in user_info @@ -181,7 +180,7 @@ def test_user_info_raises_database_error(django_elasticapm_client, client): client.get(reverse("elasticapm-raise-exc")) assert len(django_elasticapm_client.events) == 1 - event = django_elasticapm_client.events.pop(0)["errors"][0] + event = django_elasticapm_client.events[ERROR][0] assert "user" in event["context"] user_info = event["context"]["user"] assert user_info == {} @@ -201,7 +200,7 @@ def test_user_info_with_custom_user(django_elasticapm_client, client): client.get(reverse("elasticapm-raise-exc")) assert len(django_elasticapm_client.events) == 1 - event = django_elasticapm_client.events.pop(0)["errors"][0] + event = django_elasticapm_client.events[ERROR][0] assert "user" in event["context"] user_info = event["context"]["user"] assert "is_authenticated" in user_info @@ -224,7 +223,7 @@ def test_user_info_with_non_django_auth(django_elasticapm_client, client): resp = client.get(reverse("elasticapm-raise-exc")) assert len(django_elasticapm_client.events) == 1 - event = django_elasticapm_client.events.pop(0)["errors"][0] + event = django_elasticapm_client.events[ERROR][0] assert event["context"]["user"] == {} @@ -240,7 +239,7 @@ def test_user_info_with_non_django_auth_django_2(django_elasticapm_client, clien resp = client.get(reverse("elasticapm-raise-exc")) assert len(django_elasticapm_client.events) == 1 - event = django_elasticapm_client.events.pop(0)["errors"][0] + event = django_elasticapm_client.events[ERROR][0] assert event["context"]["user"] == {} @@ -254,7 +253,7 @@ def test_user_info_without_auth_middleware(django_elasticapm_client, client): with pytest.raises(Exception): client.get(reverse("elasticapm-raise-exc")) assert len(django_elasticapm_client.events) == 1 - event = django_elasticapm_client.events.pop(0)["errors"][0] + event = django_elasticapm_client.events[ERROR][0] assert event["context"]["user"] == {} @@ -267,7 +266,7 @@ def test_user_info_without_auth_middleware_django_2(django_elasticapm_client, cl with pytest.raises(Exception): client.get(reverse("elasticapm-raise-exc")) assert len(django_elasticapm_client.events) == 1 - event = django_elasticapm_client.events.pop(0)["errors"][0] + event = django_elasticapm_client.events[ERROR][0] assert event["context"]["user"] == {} @@ -279,7 +278,7 @@ def test_request_middleware_exception(django_elasticapm_client, client): client.get(reverse("elasticapm-raise-exc")) assert len(django_elasticapm_client.events) == 1 - event = django_elasticapm_client.events.pop(0)["errors"][0] + event = django_elasticapm_client.events[ERROR][0] assert "exception" in event exc = event["exception"] @@ -299,7 +298,7 @@ def test_response_middlware_exception(django_elasticapm_client, client): client.get(reverse("elasticapm-no-error")) assert len(django_elasticapm_client.events) == 1 - event = django_elasticapm_client.events.pop(0)["errors"][0] + event = django_elasticapm_client.events[ERROR][0] assert "exception" in event exc = event["exception"] @@ -317,8 +316,9 @@ def test_broken_500_handler_with_middleware(django_elasticapm_client, client): with pytest.raises(Exception): client.get(reverse("elasticapm-raise-exc")) - assert len(django_elasticapm_client.events) == 2 - event = django_elasticapm_client.events.pop(0)["errors"][0] + assert len(django_elasticapm_client.events[ERROR]) == 2 + + event = django_elasticapm_client.events[ERROR][0] assert "exception" in event exc = event["exception"] @@ -326,7 +326,7 @@ def test_broken_500_handler_with_middleware(django_elasticapm_client, client): assert exc["message"] == "MyException: view exception" assert event["culprit"] == "tests.contrib.django.testapp.views.raise_exc" - event = django_elasticapm_client.events.pop(0)["errors"][0] + event = django_elasticapm_client.events[ERROR][1] assert "exception" in event exc = event["exception"] @@ -344,7 +344,7 @@ def test_view_middleware_exception(django_elasticapm_client, client): client.get(reverse("elasticapm-raise-exc")) assert len(django_elasticapm_client.events) == 1 - event = django_elasticapm_client.events.pop(0)["errors"][0] + event = django_elasticapm_client.events[ERROR][0] assert "exception" in event exc = event["exception"] @@ -360,7 +360,7 @@ def test_exclude_modules_view(django_elasticapm_client, client): client.get(reverse("elasticapm-raise-exc-decor")) assert len(django_elasticapm_client.events) == 1, django_elasticapm_client.events - event = django_elasticapm_client.events.pop(0)["errors"][0] + event = django_elasticapm_client.events[ERROR][0] assert event["culprit"] == "tests.contrib.django.testapp.views.raise_exc" @@ -372,7 +372,7 @@ def test_include_modules(django_elasticapm_client, client): client.get(reverse("elasticapm-django-exc")) assert len(django_elasticapm_client.events) == 1 - event = django_elasticapm_client.events.pop(0)["errors"][0] + event = django_elasticapm_client.events[ERROR][0] assert event["culprit"] == "django.shortcuts.get_object_or_404" @@ -395,7 +395,7 @@ def test_record_none_exc_info(django_elasticapm_client): handler.emit(record) assert len(django_elasticapm_client.events) == 1 - event = django_elasticapm_client.events.pop(0)["errors"][0] + event = django_elasticapm_client.events[ERROR][0] assert event["log"]["param_message"] == "test" assert event["log"]["logger_name"] == "foo" @@ -411,7 +411,7 @@ def test_404_middleware(django_elasticapm_client, client): assert resp.status_code == 404 assert len(django_elasticapm_client.events) == 1 - event = django_elasticapm_client.events.pop(0)["errors"][0] + event = django_elasticapm_client.events[ERROR][0] assert event["log"]["level"] == "info" assert event["log"]["logger_name"] == "http404" @@ -448,7 +448,7 @@ def test_response_error_id_middleware(django_elasticapm_client, client): headers = dict(resp.items()) assert "X-ElasticAPM-ErrorId" in headers assert len(django_elasticapm_client.events) == 1 - event = django_elasticapm_client.events.pop(0)["errors"][0] + event = django_elasticapm_client.events[ERROR][0] assert event["id"] == headers["X-ElasticAPM-ErrorId"] @@ -476,7 +476,7 @@ def test_raw_post_data_partial_read(django_elasticapm_client): django_elasticapm_client.capture("Message", message="foo", request=request) assert len(django_elasticapm_client.events) == 1 - event = django_elasticapm_client.events.pop(0)["errors"][0] + event = django_elasticapm_client.events[ERROR][0] assert "request" in event["context"] request = event["context"]["request"] @@ -504,7 +504,7 @@ def test_post_data(django_elasticapm_client): django_elasticapm_client.capture("Message", message="foo", request=request) assert len(django_elasticapm_client.events) == 1 - event = django_elasticapm_client.events.pop(0)["errors"][0] + event = django_elasticapm_client.events[ERROR][0] assert "request" in event["context"] request = event["context"]["request"] @@ -536,7 +536,7 @@ def test_post_raw_data(django_elasticapm_client): django_elasticapm_client.capture("Message", message="foo", request=request) assert len(django_elasticapm_client.events) == 1 - event = django_elasticapm_client.events.pop(0)["errors"][0] + event = django_elasticapm_client.events[ERROR][0] assert "request" in event["context"] request = event["context"]["request"] @@ -563,7 +563,7 @@ def test_disallowed_hosts_error_django_19(django_elasticapm_client): with override_settings(ALLOWED_HOSTS=["example.com"]): # this should not raise a DisallowedHost exception django_elasticapm_client.capture("Message", message="foo", request=request) - event = django_elasticapm_client.events.pop(0)["errors"][0] + event = django_elasticapm_client.events[ERROR][0] assert event["context"]["request"]["url"]["full"] == "http://testserver/" @@ -583,7 +583,7 @@ def test_disallowed_hosts_error_django_18(django_elasticapm_client): with override_settings(ALLOWED_HOSTS=["example.com"]): # this should not raise a DisallowedHost exception django_elasticapm_client.capture("Message", message="foo", request=request) - event = django_elasticapm_client.events.pop(0)["errors"][0] + event = django_elasticapm_client.events[ERROR][0] assert event["context"]["request"]["url"] == {"full": "DisallowedHost"} @@ -608,7 +608,7 @@ def test_request_capture(django_elasticapm_client): django_elasticapm_client.capture("Message", message="foo", request=request) assert len(django_elasticapm_client.events) == 1 - event = django_elasticapm_client.events.pop(0)["errors"][0] + event = django_elasticapm_client.events[ERROR][0] assert "request" in event["context"] request = event["context"]["request"] @@ -630,13 +630,11 @@ def test_request_capture(django_elasticapm_client): def test_transaction_request_response_data(django_elasticapm_client, client): client.cookies = SimpleCookie({"foo": "bar"}) - django_elasticapm_client.transaction_store.get_all() with override_settings( **middleware_setting(django.VERSION, ["elasticapm.contrib.django.middleware.TracingMiddleware"]) ): client.get(reverse("elasticapm-no-error")) - assert len(django_elasticapm_client.transaction_store) == 1 - transactions = django_elasticapm_client.transaction_store.get_all() + transactions = django_elasticapm_client.events[TRANSACTION] assert len(transactions) == 1 transaction = transactions[0] assert transaction["result"] == "HTTP 2xx" @@ -657,19 +655,22 @@ def test_transaction_request_response_data(django_elasticapm_client, client): assert "response" in transaction["context"] response = transaction["context"]["response"] assert response["status_code"] == 200 - assert response["headers"]["My-Header"] == "foo" + if "my-header" in response["headers"]: + # Django >= 2 + assert response["headers"]["my-header"] == "foo" + else: + assert response["headers"]["My-Header"] == "foo" def test_transaction_metrics(django_elasticapm_client, client): - django_elasticapm_client.transaction_store.get_all() # clear the store with override_settings( **middleware_setting(django.VERSION, ["elasticapm.contrib.django.middleware.TracingMiddleware"]) ): - assert len(django_elasticapm_client.transaction_store) == 0 + assert len(django_elasticapm_client.events[TRANSACTION]) == 0 client.get(reverse("elasticapm-no-error")) - assert len(django_elasticapm_client.transaction_store) == 1 + assert len(django_elasticapm_client.events[TRANSACTION]) == 1 - transactions = django_elasticapm_client.transaction_store.get_all() + transactions = django_elasticapm_client.events[TRANSACTION] assert len(transactions) == 1 transaction = transactions[0] @@ -682,9 +683,9 @@ def test_transaction_metrics_debug(django_elasticapm_client, client): with override_settings( DEBUG=True, **middleware_setting(django.VERSION, ["elasticapm.contrib.django.middleware.TracingMiddleware"]) ): - assert len(django_elasticapm_client.transaction_store) == 0 + assert len(django_elasticapm_client.events[TRANSACTION]) == 0 client.get(reverse("elasticapm-no-error")) - assert len(django_elasticapm_client.transaction_store) == 0 + assert len(django_elasticapm_client.events[TRANSACTION]) == 0 @pytest.mark.parametrize("django_elasticapm_client", [{"debug": True}], indirect=True) @@ -694,14 +695,12 @@ def test_transaction_metrics_debug_and_client_debug(django_elasticapm_client, cl with override_settings( DEBUG=True, **middleware_setting(django.VERSION, ["elasticapm.contrib.django.middleware.TracingMiddleware"]) ): - assert len(django_elasticapm_client.transaction_store) == 0 + assert len(django_elasticapm_client.events[TRANSACTION]) == 0 client.get(reverse("elasticapm-no-error")) - assert len(django_elasticapm_client.transaction_store) == 1 + assert len(django_elasticapm_client.events[TRANSACTION]) == 1 def test_request_metrics_301_append_slash(django_elasticapm_client, client): - django_elasticapm_client.transaction_store.get_all() # clear the store - # enable middleware wrapping django_elasticapm_client.config.instrument_django_middleware = True @@ -717,7 +716,7 @@ def test_request_metrics_301_append_slash(django_elasticapm_client, client): ) ): client.get(reverse("elasticapm-no-error-slash")[:-1]) - transactions = django_elasticapm_client.transaction_store.get_all() + transactions = django_elasticapm_client.events[TRANSACTION] assert transactions[0]["name"] in ( # django <= 1.8 "GET django.middleware.common.CommonMiddleware.process_request", @@ -728,8 +727,6 @@ def test_request_metrics_301_append_slash(django_elasticapm_client, client): def test_request_metrics_301_prepend_www(django_elasticapm_client, client): - django_elasticapm_client.transaction_store.get_all() # clear the store - # enable middleware wrapping django_elasticapm_client.config.instrument_django_middleware = True @@ -745,15 +742,13 @@ def test_request_metrics_301_prepend_www(django_elasticapm_client, client): ) ): client.get(reverse("elasticapm-no-error")) - transactions = django_elasticapm_client.transaction_store.get_all() + transactions = django_elasticapm_client.events[TRANSACTION] assert transactions[0]["name"] == "GET django.middleware.common.CommonMiddleware.process_request" assert transactions[0]["result"] == "HTTP 3xx" @pytest.mark.django_db def test_request_metrics_contrib_redirect(django_elasticapm_client, client): - django_elasticapm_client.transaction_store.get_all() # clear the store - # enable middleware wrapping django_elasticapm_client.config.instrument_django_middleware = True from elasticapm.contrib.django.middleware import TracingMiddleware @@ -774,7 +769,7 @@ def test_request_metrics_contrib_redirect(django_elasticapm_client, client): ): response = client.get("/redirect/me/") - transactions = django_elasticapm_client.transaction_store.get_all() + transactions = django_elasticapm_client.events[TRANSACTION] assert ( transactions[0]["name"] == "GET django.contrib.redirects.middleware.RedirectFallbackMiddleware.process_response" ) @@ -782,12 +777,11 @@ def test_request_metrics_contrib_redirect(django_elasticapm_client, client): def test_request_metrics_404_resolve_error(django_elasticapm_client, client): - django_elasticapm_client.transaction_store.get_all() # clear the store with override_settings( **middleware_setting(django.VERSION, ["elasticapm.contrib.django.middleware.TracingMiddleware"]) ): client.get("/i-dont-exist/") - transactions = django_elasticapm_client.transaction_store.get_all() + transactions = django_elasticapm_client.events[TRANSACTION] assert transactions[0]["name"] == "" @@ -799,10 +793,12 @@ def test_request_metrics_streaming(django_elasticapm_client, client): resp = client.get(reverse("elasticapm-streaming-view")) assert list(resp.streaming_content) == [b"0", b"1", b"2", b"3", b"4"] resp.close() - transaction = django_elasticapm_client.transaction_store.get_all()[0] + transaction = django_elasticapm_client.events[TRANSACTION][0] assert transaction["result"] == "HTTP 2xx" assert transaction["duration"] >= 50 - assert len(transaction["spans"]) == 5 + + spans = django_elasticapm_client.events[SPAN] + assert len(spans) == 5 def test_request_metrics_name_override(django_elasticapm_client, client): @@ -810,7 +806,7 @@ def test_request_metrics_name_override(django_elasticapm_client, client): **middleware_setting(django.VERSION, ["elasticapm.contrib.django.middleware.TracingMiddleware"]) ): client.get(reverse("elasticapm-name-override")) - transaction = django_elasticapm_client.transaction_store.get_all()[0] + transaction = django_elasticapm_client.events[TRANSACTION][0] assert transaction["name"] == "foo" assert transaction["result"] == "okydoky" @@ -831,7 +827,7 @@ def test_filter_no_match(django_sending_elasticapm_client): try: raise ValueError("foo") except ValueError: - django_sending_elasticapm_client.capture("Exception") + django_sending_elasticapm_client.capture("Exception", handled=False) assert len(django_sending_elasticapm_client.httpserver.requests) == 1 @@ -861,7 +857,7 @@ def test_filter_matches_type_but_not_module(django_sending_elasticapm_client): try: raise FakeException("foo") except FakeException: - django_sending_elasticapm_client.capture("Exception") + django_sending_elasticapm_client.capture("Exception", handled=False) assert len(django_sending_elasticapm_client.httpserver.requests) == 1 @@ -877,7 +873,7 @@ def test_filter_matches_type_and_module(django_sending_elasticapm_client): try: raise FakeException("foo") except FakeException: - django_sending_elasticapm_client.capture("Exception") + django_sending_elasticapm_client.capture("Exception", handled=False) assert len(django_sending_elasticapm_client.httpserver.requests) == 0 @@ -893,7 +889,7 @@ def test_filter_matches_module_only(django_sending_elasticapm_client): try: raise OtherFakeException("foo") except OtherFakeException: - django_sending_elasticapm_client.capture("Exception") + django_sending_elasticapm_client.capture("Exception", handled=False) assert len(django_sending_elasticapm_client.httpserver.requests) == 1 @@ -922,7 +918,7 @@ def test_django_logging_request_kwarg(django_elasticapm_client): ) assert len(django_elasticapm_client.events) == 1 - event = django_elasticapm_client.events.pop(0)["errors"][0] + event = django_elasticapm_client.events[ERROR][0] assert "request" in event["context"] request = event["context"]["request"] assert request["method"] == "POST" @@ -941,7 +937,7 @@ def test_django_logging_middleware(django_elasticapm_client, client): ): client.get(reverse("elasticapm-logging")) assert len(django_elasticapm_client.events) == 1 - event = django_elasticapm_client.events.pop(0)["errors"][0] + event = django_elasticapm_client.events[ERROR][0] assert "request" in event["context"] assert event["context"]["request"]["url"]["pathname"] == reverse("elasticapm-logging") @@ -956,23 +952,21 @@ def test_stacktraces_have_templates(client, django_elasticapm_client): TEMPLATE_DEBUG = django.VERSION < (1, 9) - with mock.patch("elasticapm.traces.TransactionsStore.should_collect") as should_collect: - should_collect.return_value = False - TEMPLATES_copy = deepcopy(settings.TEMPLATES) - TEMPLATES_copy[0]["OPTIONS"]["debug"] = TEMPLATE_DEBUG - with override_settings( - TEMPLATE_DEBUG=TEMPLATE_DEBUG, - TEMPLATES=TEMPLATES_copy, - **middleware_setting(django.VERSION, ["elasticapm.contrib.django.middleware.TracingMiddleware"]) - ): - resp = client.get(reverse("render-heavy-template")) + TEMPLATES_copy = deepcopy(settings.TEMPLATES) + TEMPLATES_copy[0]["OPTIONS"]["debug"] = TEMPLATE_DEBUG + with override_settings( + TEMPLATE_DEBUG=TEMPLATE_DEBUG, + TEMPLATES=TEMPLATES_copy, + **middleware_setting(django.VERSION, ["elasticapm.contrib.django.middleware.TracingMiddleware"]) + ): + resp = client.get(reverse("render-heavy-template")) assert resp.status_code == 200 - transactions = django_elasticapm_client.transaction_store.get_all() + transactions = django_elasticapm_client.events[TRANSACTION] assert len(transactions) == 1 transaction = transactions[0] assert transaction["result"] == "HTTP 2xx" - spans = transaction["spans"] + spans = django_elasticapm_client.events[SPAN] assert len(spans) == 2, [t["name"] for t in spans] expected_names = {"list_users.html", "something_expensive"} @@ -992,17 +986,15 @@ def test_stacktraces_have_templates(client, django_elasticapm_client): def test_stacktrace_filtered_for_elasticapm(client, django_elasticapm_client): - with mock.patch("elasticapm.traces.TransactionsStore.should_collect") as should_collect: - should_collect.return_value = False - with override_settings( - **middleware_setting(django.VERSION, ["elasticapm.contrib.django.middleware.TracingMiddleware"]) - ): - resp = client.get(reverse("render-heavy-template")) + with override_settings( + **middleware_setting(django.VERSION, ["elasticapm.contrib.django.middleware.TracingMiddleware"]) + ): + resp = client.get(reverse("render-heavy-template")) assert resp.status_code == 200 - transactions = django_elasticapm_client.transaction_store.get_all() + transactions = django_elasticapm_client.events[TRANSACTION] assert transactions[0]["result"] == "HTTP 2xx" - spans = transactions[0]["spans"] + spans = django_elasticapm_client.events[SPAN] expected_signatures = ["transaction", "list_users.html", "something_expensive"] @@ -1015,35 +1007,31 @@ def test_stacktrace_filtered_for_elasticapm(client, django_elasticapm_client): @pytest.mark.parametrize("django_elasticapm_client", [{"_wait_to_first_send": 100}], indirect=True) def test_perf_template_render(benchmark, client, django_elasticapm_client): responses = [] - with mock.patch("elasticapm.traces.TransactionsStore.should_collect") as should_collect: - should_collect.return_value = False - with override_settings( - **middleware_setting(django.VERSION, ["elasticapm.contrib.django.middleware.TracingMiddleware"]) - ): - benchmark(lambda: responses.append(client_get(client, reverse("render-heavy-template")))) + with override_settings( + **middleware_setting(django.VERSION, ["elasticapm.contrib.django.middleware.TracingMiddleware"]) + ): + benchmark(lambda: responses.append(client_get(client, reverse("render-heavy-template")))) for resp in responses: assert resp.status_code == 200 - transactions = django_elasticapm_client.transaction_store.get_all() + transactions = django_elasticapm_client.events[TRANSACTION] # If the test falls right at the change from one minute to another # this will have two items. assert len(transactions) == len(responses) for transaction in transactions: - assert len(transaction["spans"]) == 2 + assert len(django_elasticapm_client.spans_for_transaction(transaction)) == 2 assert transaction["result"] == "HTTP 2xx" @pytest.mark.parametrize("django_elasticapm_client", [{"_wait_to_first_send": 100}], indirect=True) def test_perf_template_render_no_middleware(benchmark, client, django_elasticapm_client): responses = [] - with mock.patch("elasticapm.traces.TransactionsStore.should_collect") as should_collect: - should_collect.return_value = False - benchmark(lambda: responses.append(client_get(client, reverse("render-heavy-template")))) + benchmark(lambda: responses.append(client_get(client, reverse("render-heavy-template")))) for resp in responses: assert resp.status_code == 200 - transactions = django_elasticapm_client.transaction_store.get_all() + transactions = django_elasticapm_client.events[TRANSACTION] assert len(transactions) == 0 @@ -1051,100 +1039,53 @@ def test_perf_template_render_no_middleware(benchmark, client, django_elasticapm @pytest.mark.django_db(transaction=True) def test_perf_database_render(benchmark, client, django_elasticapm_client): responses = [] - django_elasticapm_client.transaction_store.get_all() - - with mock.patch("elasticapm.traces.TransactionsStore.should_collect") as should_collect: - should_collect.return_value = False - with override_settings( - **middleware_setting(django.VERSION, ["elasticapm.contrib.django.middleware.TracingMiddleware"]) - ): - benchmark(lambda: responses.append(client_get(client, reverse("render-user-template")))) - for resp in responses: - assert resp.status_code == 200 + with override_settings( + **middleware_setting(django.VERSION, ["elasticapm.contrib.django.middleware.TracingMiddleware"]) + ): + benchmark(lambda: responses.append(client_get(client, reverse("render-user-template")))) + for resp in responses: + assert resp.status_code == 200 - transactions = django_elasticapm_client.transaction_store.get_all() + transactions = django_elasticapm_client.events[TRANSACTION] - assert len(transactions) == len(responses) - for transaction in transactions: - # number of spans per transaction can vary slightly - assert 102 <= len(transaction["spans"]) < 105 + assert len(transactions) == len(responses) + for transaction in transactions: + # number of spans per transaction can vary slightly + assert 102 <= len(django_elasticapm_client.spans_for_transaction(transaction)) < 105 @pytest.mark.django_db @pytest.mark.parametrize("django_elasticapm_client", [{"_wait_to_first_send": 100}], indirect=True) def test_perf_database_render_no_instrumentation(benchmark, django_elasticapm_client): - django_elasticapm_client.transaction_store.get_all() responses = [] - with mock.patch("elasticapm.traces.TransactionsStore.should_collect") as should_collect: - should_collect.return_value = False - - client = _TestClient() - benchmark(lambda: responses.append(client_get(client, reverse("render-user-template")))) - - for resp in responses: - assert resp.status_code == 200 - - transactions = django_elasticapm_client.transaction_store.get_all() - assert len(transactions) == 0 - - -@pytest.mark.django_db -@pytest.mark.parametrize( - "django_elasticapm_client", [{"_wait_to_first_send": 100, "flush_interval": 100}], indirect=True -) -def test_perf_transaction_with_collection(benchmark, django_elasticapm_client): - django_elasticapm_client.transaction_store.get_all() - with mock.patch("elasticapm.traces.TransactionsStore.should_collect") as should_collect: - should_collect.return_value = False - django_elasticapm_client.events = [] - client = _TestClient() + client = _TestClient() + benchmark(lambda: responses.append(client_get(client, reverse("render-user-template")))) - with override_settings( - **middleware_setting(django.VERSION, ["elasticapm.contrib.django.middleware.TracingMiddleware"]) - ): - for i in range(10): - resp = client_get(client, reverse("render-user-template")) - assert resp.status_code == 200 - - assert len(django_elasticapm_client.events) == 0 - - # Force collection on next request - should_collect.return_value = True - - @benchmark - def result(): - # Code to be measured - with override_settings( - **middleware_setting(django.VERSION, ["elasticapm.contrib.django.middleware.TracingMiddleware"]) - ): - return client_get(client, reverse("render-user-template")) + for resp in responses: + assert resp.status_code == 200 - assert result.status_code is 200 - assert len(django_elasticapm_client.events) > 0 + transactions = django_elasticapm_client.events[TRANSACTION] + assert len(transactions) == 0 @pytest.mark.django_db @pytest.mark.parametrize("django_elasticapm_client", [{"_wait_to_first_send": 100}], indirect=True) def test_perf_transaction_without_middleware(benchmark, django_elasticapm_client): - django_elasticapm_client.transaction_store.get_all() - with mock.patch("elasticapm.traces.TransactionsStore.should_collect") as should_collect: - should_collect.return_value = False - client = _TestClient() - django_elasticapm_client.events = [] - for i in range(10): - resp = client_get(client, reverse("render-user-template")) - assert resp.status_code == 200 + client = _TestClient() + for i in range(10): + resp = client_get(client, reverse("render-user-template")) + assert resp.status_code == 200 - assert len(django_elasticapm_client.events) == 0 + assert len(django_elasticapm_client.events) == 0 - @benchmark - def result(): - # Code to be measured - return client_get(client, reverse("render-user-template")) + @benchmark + def result(): + # Code to be measured + return client_get(client, reverse("render-user-template")) - assert len(django_elasticapm_client.events) == 0 + assert len(django_elasticapm_client.events) == 0 @pytest.mark.skipif(django.VERSION > (1, 7), reason="argparse raises CommandError in this case") @@ -1302,7 +1243,7 @@ def test_tracing_middleware_uses_test_client(client, django_elasticapm_client): **middleware_setting(django.VERSION, ["elasticapm.contrib.django.middleware.TracingMiddleware"]) ): client.get("/") - transactions = django_elasticapm_client.transaction_store.get_all() + transactions = django_elasticapm_client.events[TRANSACTION] assert len(transactions) == 1 assert transactions[0]["context"]["request"]["url"]["pathname"] == "/" @@ -1315,11 +1256,11 @@ def test_tracing_middleware_uses_test_client(client, django_elasticapm_client): def test_capture_post_errors_dict(client, django_elasticapm_client): with pytest.raises(MyException): client.post(reverse("elasticapm-raise-exc"), {"username": "john", "password": "smith"}) - error = django_elasticapm_client.events[0] + error = django_elasticapm_client.events[ERROR][0] if django_elasticapm_client.config.capture_body in ("errors", "all"): - assert error["errors"][0]["context"]["request"]["body"] == {"username": "john", "password": "smith"} + assert error["context"]["request"]["body"] == {"username": "john", "password": "smith"} else: - assert error["errors"][0]["context"]["request"]["body"] == "[REDACTED]" + assert error["context"]["request"]["body"] == "[REDACTED]" @pytest.mark.parametrize( @@ -1334,14 +1275,11 @@ def test_capture_post_errors_multivalue_dict(client, django_elasticapm_client): "key=value1&key=value2&test=test&key=value3", content_type="application/x-www-form-urlencoded", ) - error = django_elasticapm_client.events[0] + error = django_elasticapm_client.events[ERROR][0] if django_elasticapm_client.config.capture_body in ("errors", "all"): - assert error["errors"][0]["context"]["request"]["body"] == { - "key": ["value1", "value2", "value3"], - "test": "test", - } + assert error["context"]["request"]["body"] == {"key": ["value1", "value2", "value3"], "test": "test"} else: - assert error["errors"][0]["context"]["request"]["body"] == "[REDACTED]" + assert error["context"]["request"]["body"] == "[REDACTED]" @pytest.mark.parametrize( @@ -1355,11 +1293,11 @@ def test_capture_post_errors_raw(client, django_sending_elasticapm_client): client.post( reverse("elasticapm-raise-exc"), json.dumps({"a": "b"}), content_type="application/json; charset=utf8" ) - error = django_sending_elasticapm_client.httpserver.payloads[0] + error = django_sending_elasticapm_client.httpserver.payloads[0][1]["error"] if django_sending_elasticapm_client.config.capture_body in ("errors", "all"): - assert error["errors"][0]["context"]["request"]["body"] == '{"a": "b"}' + assert error["context"]["request"]["body"] == '{"a": "b"}' else: - assert error["errors"][0]["context"]["request"]["body"] == "[REDACTED]" + assert error["context"]["request"]["body"] == "[REDACTED]" @pytest.mark.parametrize( @@ -1370,9 +1308,9 @@ def test_capture_post_errors_raw(client, django_sending_elasticapm_client): def test_capture_empty_body(client, django_elasticapm_client): with pytest.raises(MyException): client.post(reverse("elasticapm-raise-exc"), data={}) - error = django_elasticapm_client.events[0] + error = django_elasticapm_client.events[ERROR][0] # body should be empty no matter if we capture it or not - assert error["errors"][0]["context"]["request"]["body"] == {} + assert error["context"]["request"]["body"] == {} @pytest.mark.parametrize( @@ -1385,14 +1323,11 @@ def test_capture_files(client, django_elasticapm_client): client.post( reverse("elasticapm-raise-exc"), data={"a": "b", "f1": compat.BytesIO(100 * compat.b("1")), "f2": f} ) - error = django_elasticapm_client.events[0] + error = django_elasticapm_client.events[ERROR][0] if django_elasticapm_client.config.capture_body in ("errors", "all"): - assert error["errors"][0]["context"]["request"]["body"] == { - "a": "b", - "_files": {"f1": "f1", "f2": "django_tests.py"}, - } + assert error["context"]["request"]["body"] == {"a": "b", "_files": {"f1": "f1", "f2": "django_tests.py"}} else: - assert error["errors"][0]["context"]["request"]["body"] == "[REDACTED]" + assert error["context"]["request"]["body"] == "[REDACTED]" @pytest.mark.parametrize("django_elasticapm_client", [{"capture_body": "transactions"}], indirect=True) @@ -1401,5 +1336,5 @@ def test_options_request(client, django_elasticapm_client): **middleware_setting(django.VERSION, ["elasticapm.contrib.django.middleware.TracingMiddleware"]) ): client.options("/") - transactions = django_elasticapm_client.transaction_store.get_all() + transactions = django_elasticapm_client.events[TRANSACTION] assert transactions[0]["context"]["request"]["method"] == "OPTIONS" diff --git a/tests/contrib/django/fixtures.py b/tests/contrib/django/fixtures.py index d6ad75e10..80cd1a008 100644 --- a/tests/contrib/django/fixtures.py +++ b/tests/contrib/django/fixtures.py @@ -2,17 +2,23 @@ import pytest +from elasticapm.conf.constants import SPAN from elasticapm.contrib.django.apps import instrument, register_handlers from elasticapm.contrib.django.client import DjangoClient class TempStoreClient(DjangoClient): - def __init__(self, *args, **kwargs): - self.events = [] - super(TempStoreClient, self).__init__(*args, **kwargs) + def __init__(self, **inline): + inline.setdefault("transport_class", "tests.fixtures.DummyTransport") + super(TempStoreClient, self).__init__(**inline) - def send(self, url, **kwargs): - self.events.append(kwargs) + @property + def events(self): + return self._transport.events + + def spans_for_transaction(self, transaction): + """Test helper method to get all spans of a specific transaction""" + return [span for span in self.events[SPAN] if span["transaction_id"] == transaction["id"]] @pytest.fixture() @@ -20,7 +26,7 @@ def django_elasticapm_client(request): client_config = getattr(request, "param", {}) client_config.setdefault("service_name", "app") client_config.setdefault("secret_token", "secret") - client_config.setdefault("span_frames_min_duration_ms", -1) + client_config.setdefault("span_frames_min_duration", -1) app = apps.get_app_config("elasticapm.contrib.django") old_client = app.client client = TempStoreClient(**client_config) @@ -45,7 +51,7 @@ def django_sending_elasticapm_client(request, validating_httpserver): client_config.setdefault("service_name", "app") client_config.setdefault("secret_token", "secret") client_config.setdefault("transport_class", "elasticapm.transport.http.Transport") - client_config.setdefault("span_frames_min_duration_ms", -1) + client_config.setdefault("span_frames_min_duration", -1) app = apps.get_app_config("elasticapm.contrib.django") old_client = app.client client = DjangoClient(**client_config) diff --git a/tests/contrib/flask/flask_tests.py b/tests/contrib/flask/flask_tests.py index 0bae2cd14..91febe698 100644 --- a/tests/contrib/flask/flask_tests.py +++ b/tests/contrib/flask/flask_tests.py @@ -6,6 +6,7 @@ import mock +from elasticapm.conf.constants import ERROR, TRANSACTION from elasticapm.contrib.flask import ElasticAPM from elasticapm.utils import compat @@ -21,7 +22,7 @@ def test_error_handler(flask_apm_client): assert response.status_code == 500 assert len(flask_apm_client.client.events) == 1 - event = flask_apm_client.client.events.pop(0)["errors"][0] + event = flask_apm_client.client.events[ERROR][0] assert "exception" in event exc = event["exception"] @@ -37,7 +38,7 @@ def test_get(flask_apm_client): assert response.status_code == 500 assert len(flask_apm_client.client.events) == 1 - event = flask_apm_client.client.events.pop(0)["errors"][0] + event = flask_apm_client.client.events[ERROR][0] assert "request" in event["context"] request = event["context"]["request"] @@ -86,9 +87,9 @@ def test_post(flask_apm_client): client = flask_apm_client.app.test_client() response = client.post("/an-error/?biz=baz", data={"foo": "bar"}) assert response.status_code == 500 - assert len(flask_apm_client.client.events) == 1 + assert len(flask_apm_client.client.events[ERROR]) == 1 - event = flask_apm_client.client.events.pop(0)["errors"][0] + event = flask_apm_client.client.events[ERROR][0] assert "request" in event["context"] request = event["context"]["request"] @@ -120,14 +121,12 @@ def test_post(flask_apm_client): indirect=True, ) def test_instrumentation(flask_apm_client): - with mock.patch("elasticapm.traces.TransactionsStore.should_collect") as should_collect: - should_collect.return_value = False - resp = flask_apm_client.app.test_client().post("/users/", data={"foo": "bar"}) - resp.close() + resp = flask_apm_client.app.test_client().post("/users/", data={"foo": "bar"}) + resp.close() assert resp.status_code == 200, resp.response - transactions = flask_apm_client.client.transaction_store.get_all() + transactions = flask_apm_client.client.events[TRANSACTION] assert len(transactions) == 1 transaction = transactions[0] @@ -147,7 +146,7 @@ def test_instrumentation(flask_apm_client): "Content-Length": "78", "Content-Type": "text/html; charset=utf-8", } - spans = transactions[0]["spans"] + spans = flask_apm_client.client.spans_for_transaction(transactions[0]) assert len(spans) == 1, [t["name"] for t in spans] expected_signatures = {"users.html"} @@ -160,47 +159,43 @@ def test_instrumentation(flask_apm_client): def test_instrumentation_debug(flask_apm_client): flask_apm_client.app.debug = True - assert len(flask_apm_client.client.transaction_store) == 0 + assert len(flask_apm_client.client.events[TRANSACTION]) == 0 resp = flask_apm_client.app.test_client().post("/users/", data={"foo": "bar"}) resp.close() - assert len(flask_apm_client.client.transaction_store) == 0 + assert len(flask_apm_client.client.events[TRANSACTION]) == 0 @pytest.mark.parametrize("elasticapm_client", [{"debug": True}], indirect=True) def test_instrumentation_debug_client_debug(flask_apm_client): flask_apm_client.app.debug = True - assert len(flask_apm_client.client.transaction_store) == 0 + assert len(flask_apm_client.client.events[TRANSACTION]) == 0 resp = flask_apm_client.app.test_client().post("/users/", data={"foo": "bar"}) resp.close() - assert len(flask_apm_client.client.transaction_store) == 1 + assert len(flask_apm_client.client.events[TRANSACTION]) == 1 def test_instrumentation_404(flask_apm_client): - with mock.patch("elasticapm.traces.TransactionsStore.should_collect") as should_collect: - should_collect.return_value = False - resp = flask_apm_client.app.test_client().post("/no-such-page/") - resp.close() + resp = flask_apm_client.app.test_client().post("/no-such-page/") + resp.close() assert resp.status_code == 404, resp.response - transactions = flask_apm_client.client.transaction_store.get_all() + transactions = flask_apm_client.client.events[TRANSACTION] assert len(transactions) == 1 - spans = transactions[0]["spans"] + spans = flask_apm_client.client.spans_for_transaction(transactions[0]) assert transactions[0]["result"] == "HTTP 4xx" assert transactions[0]["context"]["response"]["status_code"] == 404 assert len(spans) == 0, [t["signature"] for t in spans] def test_non_standard_http_status(flask_apm_client): - with mock.patch("elasticapm.traces.TransactionsStore.should_collect") as should_collect: - should_collect.return_value = False - resp = flask_apm_client.app.test_client().get("/non-standard-status/") - resp.close() + resp = flask_apm_client.app.test_client().get("/non-standard-status/") + resp.close() assert resp.status == "0 fail", resp.response assert resp.status_code == 0, resp.response - transactions = flask_apm_client.client.transaction_store.get_all() + transactions = flask_apm_client.client.events[TRANSACTION] assert transactions[0]["result"] == "0 fail" # "0" is prepended by Werkzeug BaseResponse assert transactions[0]["context"]["response"]["status_code"] == 0 @@ -228,8 +223,7 @@ def test_post_files(flask_apm_client): assert response.status_code == 500 assert len(flask_apm_client.client.events) == 1 - event = flask_apm_client.client.events.pop(0)["errors"][0] - print(flask_apm_client.client.config.capture_body) + event = flask_apm_client.client.events[ERROR][0] if flask_apm_client.client.config.capture_body in ("errors", "all"): assert event["context"]["request"]["body"] == { "foo": ["bar", "baz"], @@ -243,7 +237,7 @@ def test_post_files(flask_apm_client): def test_options_request(flask_apm_client): resp = flask_apm_client.app.test_client().options("/") resp.close() - transactions = flask_apm_client.client.transaction_store.get_all() + transactions = flask_apm_client.client.events[TRANSACTION] assert transactions[0]["context"]["request"]["method"] == "OPTIONS" @@ -251,9 +245,10 @@ def test_streaming_response(flask_apm_client): resp = flask_apm_client.app.test_client().get("/streaming/") assert resp.data == b"01234" resp.close() - transaction = flask_apm_client.client.transaction_store.get_all()[0] + transaction = flask_apm_client.client.events[TRANSACTION][0] + spans = flask_apm_client.client.spans_for_transaction(transaction) assert transaction["duration"] > 50 - assert len(transaction["spans"]) == 5 + assert len(spans) == 5 def test_response_close_wsgi(flask_wsgi_server): @@ -262,14 +257,15 @@ def test_response_close_wsgi(flask_wsgi_server): url = flask_wsgi_server.url + "/streaming/" response = urlopen(url) response.read() - transaction = elasticapm_client.transaction_store.get_all()[0] + transaction = elasticapm_client.events[TRANSACTION][0] + spans = elasticapm_client.spans_for_transaction(transaction) assert transaction["duration"] > 50 - assert len(transaction["spans"]) == 5 + assert len(spans) == 5 def test_set_transaction_name(flask_apm_client): resp = flask_apm_client.app.test_client().get("/transaction-name/") resp.close() - transaction = flask_apm_client.client.transaction_store.get_all()[0] + transaction = flask_apm_client.client.events[TRANSACTION][0] assert transaction["name"] == "foo" assert transaction["result"] == "okydoky" diff --git a/tests/contrib/twisted/tests.py b/tests/contrib/twisted/tests.py index 92bda7542..fb1c46e48 100644 --- a/tests/contrib/twisted/tests.py +++ b/tests/contrib/twisted/tests.py @@ -4,6 +4,7 @@ from twisted.python.failure import Failure +from elasticapm.conf.constants import ERROR from elasticapm.contrib.twisted import LogObserver pytestmark = pytest.mark.twisted @@ -19,6 +20,6 @@ def test_twisted_log_observer(elasticapm_client): event = dict(log_failure=failure) observer(event) - cli_event = elasticapm_client.events.pop(0)["errors"][0] + cli_event = elasticapm_client.events[ERROR][0] assert cli_event["exception"]["type"] == "ZeroDivisionError" assert cli_event["exception"]["handled"] is False diff --git a/tests/fixtures.py b/tests/fixtures.py index 5f13d93d4..398f1643b 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -1,10 +1,12 @@ import codecs +import gzip import json import os import random import socket import time import zlib +from collections import defaultdict import jsonschema import pytest @@ -13,6 +15,9 @@ import elasticapm from elasticapm.base import Client +from elasticapm.conf.constants import SPAN +from elasticapm.transport.http_base import HTTPTransportBase +from elasticapm.utils import compat try: from urllib.request import pathname2url @@ -22,8 +27,10 @@ cur_dir = os.path.dirname(os.path.realpath(__file__)) -ERRORS_SCHEMA = os.path.join(cur_dir, ".schemacache", "errors", "payload.json") -TRANSACTIONS_SCHEMA = os.path.join(cur_dir, ".schemacache", "transactions", "payload.json") +ERRORS_SCHEMA = os.path.join(cur_dir, ".schemacache", "errors", "v2_error.json") +TRANSACTIONS_SCHEMA = os.path.join(cur_dir, ".schemacache", "transactions", "v2_transaction.json") +SPAN_SCHEMA = os.path.join(cur_dir, ".schemacache", "spans", "v2_span.json") +METADATA_SCHEMA = os.path.join(cur_dir, ".schemacache", "metadata.json") assert os.path.exists(ERRORS_SCHEMA) and os.path.exists( TRANSACTIONS_SCHEMA @@ -32,51 +39,72 @@ with codecs.open(ERRORS_SCHEMA, encoding="utf8") as errors_json, codecs.open( TRANSACTIONS_SCHEMA, encoding="utf8" -) as transactions_json: +) as transactions_json, codecs.open(SPAN_SCHEMA, encoding="utf8") as span_json, codecs.open( + METADATA_SCHEMA, encoding="utf8" +) as metadata_json: VALIDATORS = { - "/v1/errors": jsonschema.Draft4Validator( + "error": jsonschema.Draft4Validator( json.load(errors_json), resolver=jsonschema.RefResolver( base_uri="file:" + pathname2url(ERRORS_SCHEMA), referrer="file:" + pathname2url(ERRORS_SCHEMA) ), ), - "/v1/transactions": jsonschema.Draft4Validator( + "transaction": jsonschema.Draft4Validator( json.load(transactions_json), resolver=jsonschema.RefResolver( base_uri="file:" + pathname2url(TRANSACTIONS_SCHEMA), referrer="file:" + pathname2url(TRANSACTIONS_SCHEMA), ), ), + "span": jsonschema.Draft4Validator( + json.load(span_json), + resolver=jsonschema.RefResolver( + base_uri="file:" + pathname2url(SPAN_SCHEMA), referrer="file:" + pathname2url(SPAN_SCHEMA) + ), + ), + "metadata": jsonschema.Draft4Validator( + json.load(metadata_json), + resolver=jsonschema.RefResolver( + base_uri="file:" + pathname2url(METADATA_SCHEMA), referrer="file:" + pathname2url(METADATA_SCHEMA) + ), + ), } class ValidatingWSGIApp(ContentServer): def __init__(self, **kwargs): + self.skip_validate = kwargs.pop("skip_validate", False) super(ValidatingWSGIApp, self).__init__(**kwargs) self.payloads = [] self.responses = [] - self.skip_validate = False def __call__(self, environ, start_response): - code = self.code content = self.content request = Request(environ) self.requests.append(request) data = request.data if request.content_encoding == "deflate": data = zlib.decompress(data) + elif request.content_encoding == "gzip": + with gzip.GzipFile(fileobj=compat.BytesIO(data)) as f: + data = f.read() data = data.decode(request.charset) - if request.content_type == "application/json": - data = json.loads(data) + if request.content_type == "application/x-ndjson": + data = [json.loads(line) for line in data.split("\n") if line] self.payloads.append(data) - validator = VALIDATORS.get(request.path, None) - if validator and not self.skip_validate: - try: - validator.validate(data) - code = 202 - except jsonschema.ValidationError as e: - code = 400 - content = json.dumps({"status": "error", "message": str(e)}) + code = 202 + success = 0 + fail = 0 + if not self.skip_validate: + for line in data: + item_type, item = list(line.items())[0] + validator = VALIDATORS[item_type] + try: + validator.validate(item) + success += 1 + except jsonschema.ValidationError as e: + fail += 1 + code = 202 if not fail else 400 response = Response(status=code) response.headers.clear() response.headers.extend(self.headers) @@ -91,7 +119,7 @@ def elasticapm_client(request): client_config.setdefault("service_name", "myapp") client_config.setdefault("secret_token", "test_key") client_config.setdefault("include_paths", ("*/tests/*",)) - client_config.setdefault("span_frames_min_duration_ms", -1) + client_config.setdefault("span_frames_min_duration", -1) client = TempStoreClient(**client_config) yield client client.close() @@ -111,7 +139,9 @@ def waiting_httpsserver(httpsserver): @pytest.fixture() def validating_httpserver(request): - server = ValidatingWSGIApp() + config = getattr(request, "param", {}) + app = config.pop("app", ValidatingWSGIApp) + server = app(**config) server.start() wait_for_http_server(server) request.addfinalizer(server.stop) @@ -126,7 +156,7 @@ def sending_elasticapm_client(request, validating_httpserver): client_config.setdefault("service_name", "myapp") client_config.setdefault("secret_token", "test_key") client_config.setdefault("transport_class", "elasticapm.transport.http.Transport") - client_config.setdefault("span_frames_min_duration_ms", -1) + client_config.setdefault("span_frames_min_duration", -1) client_config.setdefault("include_paths", ("*/tests/*",)) client = Client(**client_config) client.httpserver = validating_httpserver @@ -134,13 +164,27 @@ def sending_elasticapm_client(request, validating_httpserver): client.close() +class DummyTransport(HTTPTransportBase): + def __init__(self, url, **kwargs): + super(DummyTransport, self).__init__(url, **kwargs) + self.events = defaultdict(list) + + def queue(self, event_type, data, flush=False): + self.events[event_type].append(data) + + class TempStoreClient(Client): def __init__(self, **inline): - self.events = [] + inline.setdefault("transport_class", "tests.fixtures.DummyTransport") super(TempStoreClient, self).__init__(**inline) - def send(self, url, **kwargs): - self.events.append(kwargs) + @property + def events(self): + return self._transport.events + + def spans_for_transaction(self, transaction): + """Test helper method to get all spans of a specific transaction""" + return [span for span in self.events[SPAN] if span["transaction_id"] == transaction["id"]] @pytest.fixture() diff --git a/tests/handlers/logbook/logbook_tests.py b/tests/handlers/logbook/logbook_tests.py index b76a67237..300584467 100644 --- a/tests/handlers/logbook/logbook_tests.py +++ b/tests/handlers/logbook/logbook_tests.py @@ -2,6 +2,7 @@ import pytest from logbook import LogRecord +from elasticapm.conf.constants import ERROR from elasticapm.handlers.logbook import LogbookHandler @@ -21,7 +22,7 @@ def test_logbook_logger_error_level(logbook_logger, logbook_handler): logbook_logger.error("This is a test error") assert len(logbook_handler.client.events) == 1 - event = logbook_handler.client.events.pop(0)["errors"][0] + event = logbook_handler.client.events[ERROR][0] assert event["log"]["logger_name"] == __name__ assert event["log"]["level"] == "error" assert event["log"]["message"] == "This is a test error" @@ -35,7 +36,7 @@ def test_logger_warning_level(logbook_logger, logbook_handler): with logbook_handler.applicationbound(): logbook_logger.warning("This is a test warning") assert len(logbook_handler.client.events) == 1 - event = logbook_handler.client.events.pop(0)["errors"][0] + event = logbook_handler.client.events[ERROR][0] assert event["log"]["logger_name"] == __name__ assert event["log"]["level"] == "warning" assert event["log"]["message"] == "This is a test warning" @@ -51,7 +52,7 @@ def test_logger_without_stacktrace_config(logbook_logger, logbook_handler): with logbook_handler.applicationbound(): logbook_logger.warning("This is a test warning") - event = logbook_handler.client.events.pop(0)["errors"][0] + event = logbook_handler.client.events[ERROR][0] assert "stacktrace" not in event["log"] @@ -61,7 +62,7 @@ def test_logger_without_stacktrace_stack_false(logbook_logger, logbook_handler): with logbook_handler.applicationbound(): logbook_logger.warning("This is a test warning", stack=False) - event = logbook_handler.client.events.pop(0)["errors"][0] + event = logbook_handler.client.events[ERROR][0] assert "stacktrace" not in event["log"] @@ -69,7 +70,7 @@ def test_logger_with_extra(logbook_logger, logbook_handler): with logbook_handler.applicationbound(): logbook_logger.info("This is a test info with a url", extra=dict(url="http://example.com")) assert len(logbook_handler.client.events) == 1 - event = logbook_handler.client.events.pop(0)["errors"][0] + event = logbook_handler.client.events[ERROR][0] assert event["context"]["custom"]["url"] == "http://example.com" assert "stacktrace" in event["log"] assert "exception" not in event @@ -85,7 +86,7 @@ def test_logger_with_exc_info(logbook_logger, logbook_handler): logbook_logger.info("This is a test info with an exception", exc_info=True) assert len(logbook_handler.client.events) == 1 - event = logbook_handler.client.events.pop(0)["errors"][0] + event = logbook_handler.client.events[ERROR][0] assert event["log"]["message"] == "This is a test info with an exception" assert "stacktrace" in event["log"] @@ -101,7 +102,7 @@ def test_logger_param_message(logbook_logger, logbook_handler): with logbook_handler.applicationbound(): logbook_logger.info("This is a test of %s", "args") assert len(logbook_handler.client.events) == 1 - event = logbook_handler.client.events.pop(0)["errors"][0] + event = logbook_handler.client.events[ERROR][0] assert event["log"]["message"] == "This is a test of args" assert "stacktrace" in event["log"] assert "exception" not in event diff --git a/tests/handlers/logging/logging_tests.py b/tests/handlers/logging/logging_tests.py index 0581f54cc..ecb4ba3ea 100644 --- a/tests/handlers/logging/logging_tests.py +++ b/tests/handlers/logging/logging_tests.py @@ -3,6 +3,7 @@ import pytest +from elasticapm.conf.constants import ERROR from elasticapm.handlers.logging import LoggingHandler from elasticapm.utils.stacks import iter_stack_frames @@ -23,7 +24,7 @@ def test_logger_basic(logger): logger.error("This is a test error") assert len(logger.client.events) == 1 - event = logger.client.events.pop(0)["errors"][0] + event = logger.client.events[ERROR][0] assert event["log"]["logger_name"] == __name__ assert event["log"]["level"] == "error" assert event["log"]["message"] == "This is a test error" @@ -36,7 +37,7 @@ def test_logger_basic(logger): def test_logger_warning(logger): logger.warning("This is a test warning") assert len(logger.client.events) == 1 - event = logger.client.events.pop(0)["errors"][0] + event = logger.client.events[ERROR][0] assert event["log"]["logger_name"] == __name__ assert event["log"]["level"] == "warning" assert "exception" not in event @@ -47,7 +48,7 @@ def test_logger_warning(logger): def test_logger_extra_data(logger): logger.info("This is a test info with a url", extra=dict(data=dict(url="http://example.com"))) assert len(logger.client.events) == 1 - event = logger.client.events.pop(0)["errors"][0] + event = logger.client.events[ERROR][0] assert event["context"]["custom"]["url"] == "http://example.com" assert "stacktrace" in event["log"] assert "exception" not in event @@ -62,7 +63,7 @@ def test_logger_exc_info(logger): logger.info("This is a test info with an exception", exc_info=True) assert len(logger.client.events) == 1 - event = logger.client.events.pop(0)["errors"][0] + event = logger.client.events[ERROR][0] # assert event['message'] == 'This is a test info with an exception' assert "exception" in event @@ -77,7 +78,7 @@ def test_logger_exc_info(logger): def test_message_params(logger): logger.info("This is a test of %s", "args") assert len(logger.client.events) == 1 - event = logger.client.events.pop(0)["errors"][0] + event = logger.client.events[ERROR][0] assert "exception" not in event assert "param_message" in event["log"] assert event["log"]["message"] == "This is a test of args" @@ -87,7 +88,7 @@ def test_message_params(logger): def test_record_stack(logger): logger.info("This is a test of stacks", extra={"stack": True}) assert len(logger.client.events) == 1 - event = logger.client.events.pop(0)["errors"][0] + event = logger.client.events[ERROR][0] frames = event["log"]["stacktrace"] assert len(frames) != 1 frame = frames[0] @@ -102,7 +103,7 @@ def test_record_stack(logger): def test_no_record_stack(logger): logger.info("This is a test of no stacks", extra={"stack": False}) assert len(logger.client.events) == 1 - event = logger.client.events.pop(0)["errors"][0] + event = logger.client.events[ERROR][0] assert event.get("culprit") == None assert event["log"]["message"] == "This is a test of no stacks" assert "stacktrace" not in event["log"] @@ -115,7 +116,7 @@ def test_no_record_stack_via_config(logger): logger.client.config.auto_log_stacks = False logger.info("This is a test of no stacks") assert len(logger.client.events) == 1 - event = logger.client.events.pop(0)["errors"][0] + event = logger.client.events[ERROR][0] assert event.get("culprit") == None assert event["log"]["message"] == "This is a test of no stacks" assert "stacktrace" not in event["log"] @@ -127,7 +128,7 @@ def test_no_record_stack_via_config(logger): def test_explicit_stack(logger): logger.info("This is a test of stacks", extra={"stack": iter_stack_frames()}) assert len(logger.client.events) == 1 - event = logger.client.events.pop(0)["errors"][0] + event = logger.client.events[ERROR][0] assert "culprit" in event, event assert event["culprit"] == "tests.handlers.logging.logging_tests.test_explicit_stack" assert "message" in event["log"], event @@ -141,7 +142,7 @@ def test_explicit_stack(logger): def test_extra_culprit(logger): logger.info("This is a test of stacks", extra={"culprit": "foo.bar"}) assert len(logger.client.events) == 1 - event = logger.client.events.pop(0)["errors"][0] + event = logger.client.events[ERROR][0] assert event["culprit"] == "foo.bar" assert "culprit" not in event["context"]["custom"] @@ -153,7 +154,7 @@ def test_logger_exception(logger): logger.exception("This is a test with an exception", extra={"stack": True}) assert len(logger.client.events) == 1 - event = logger.client.events.pop(0)["errors"][0] + event = logger.client.events[ERROR][0] assert event["log"]["message"] == "This is a test with an exception" assert "stacktrace" in event["log"] diff --git a/tests/instrumentation/base_tests.py b/tests/instrumentation/base_tests.py index 6b04cf4f2..2865d3aa7 100644 --- a/tests/instrumentation/base_tests.py +++ b/tests/instrumentation/base_tests.py @@ -6,6 +6,7 @@ import pytest import elasticapm +from elasticapm.conf.constants import SPAN from elasticapm.instrumentation.packages.base import AbstractInstrumentedModule from elasticapm.utils import compat, wrapt @@ -123,8 +124,9 @@ def test_skip_ignored_frames(elasticapm_client): elasticapm_client.begin_transaction("test") with elasticapm.capture_span("test"): pass - transaction = elasticapm_client.end_transaction("test", "test") - for frame in transaction.spans[0].frames: + elasticapm_client.end_transaction("test", "test") + span = elasticapm_client.events[SPAN][0] + for frame in span["stacktrace"]: assert not frame["module"].startswith("elasticapm") diff --git a/tests/instrumentation/botocore_tests.py b/tests/instrumentation/botocore_tests.py index f845e9181..ef3bce283 100644 --- a/tests/instrumentation/botocore_tests.py +++ b/tests/instrumentation/botocore_tests.py @@ -1,12 +1,12 @@ -import pytest # isort:skip - -pytest.importorskip("boto3") # isort:skip - -import boto3 import mock +import pytest +from elasticapm.conf.constants import SPAN from elasticapm.instrumentation.packages.botocore import BotocoreInstrumentation +boto3 = pytest.importorskip("boto3") + + pytestmark = pytest.mark.boto3 @@ -20,13 +20,13 @@ def test_botocore_instrumentation(mock_make_request, instrument, elasticapm_clie session = boto3.Session(aws_access_key_id="foo", aws_secret_access_key="bar", region_name="us-west-2") ec2 = session.client("ec2") ec2.describe_instances() - transaction = elasticapm_client.end_transaction("MyView") - span = transaction.spans[0] + elasticapm_client.end_transaction("MyView") + span = elasticapm_client.events[SPAN][0] - assert span.name == "ec2:DescribeInstances" - assert span.context["service"] == "ec2" - assert span.context["region"] == "us-west-2" - assert span.context["operation"] == "DescribeInstances" + assert span["name"] == "ec2:DescribeInstances" + assert span["context"]["service"] == "ec2" + assert span["context"]["region"] == "us-west-2" + assert span["context"]["operation"] == "DescribeInstances" def test_nonstandard_endpoint_url(instrument, elasticapm_client): @@ -36,9 +36,9 @@ def test_nonstandard_endpoint_url(instrument, elasticapm_client): instance = mock.Mock(_endpoint=mock.Mock(host="https://example")) instrument.call(module, method, lambda *args, **kwargs: None, instance, ("DescribeInstances",), {}) transaction = elasticapm_client.end_transaction("test", "test") - span = transaction.spans[0] + span = elasticapm_client.events[SPAN][0] - assert span.name == "example:DescribeInstances" - assert span.context["service"] == "example" - assert span.context["region"] is None - assert span.context["operation"] == "DescribeInstances" + assert span["name"] == "example:DescribeInstances" + assert span["context"]["service"] == "example" + assert span["context"]["region"] is None + assert span["context"]["operation"] == "DescribeInstances" diff --git a/tests/instrumentation/cassandra_tests.py b/tests/instrumentation/cassandra_tests.py index 89dba40c7..73d3c8614 100644 --- a/tests/instrumentation/cassandra_tests.py +++ b/tests/instrumentation/cassandra_tests.py @@ -7,6 +7,7 @@ from cassandra.cluster import Cluster from cassandra.query import SimpleStatement +from elasticapm.conf.constants import TRANSACTION from elasticapm.instrumentation.packages.dbapi2 import extract_signature pytestmark = pytest.mark.cassandra @@ -39,8 +40,8 @@ def test_cassandra_connect(instrument, elasticapm_client, cassandra_cluster): sess = cassandra_cluster.connect() elasticapm_client.end_transaction("test") - transactions = elasticapm_client.transaction_store.get_all() - span = transactions[0]["spans"][0] + transactions = elasticapm_client.events[TRANSACTION] + span = elasticapm_client.spans_for_transaction(transactions[0])[0] assert span["type"] == "db.cassandra.connect" assert span["duration"] > 0 @@ -51,8 +52,8 @@ def test_select_query_string(instrument, cassandra_session, elasticapm_client): elasticapm_client.begin_transaction("transaction.test") cassandra_session.execute("SELECT name from users") elasticapm_client.end_transaction("test") - transaction = elasticapm_client.transaction_store.get_all()[0] - span = transaction["spans"][0] + transaction = elasticapm_client.events[TRANSACTION][0] + span = elasticapm_client.spans_for_transaction(transaction)[0] assert span["type"] == "db.cassandra.query" assert span["name"] == "SELECT FROM users" assert span["context"] == {"db": {"statement": "SELECT name from users", "type": "sql"}} @@ -63,8 +64,8 @@ def test_select_simple_statement(instrument, cassandra_session, elasticapm_clien elasticapm_client.begin_transaction("transaction.test") cassandra_session.execute(statement) elasticapm_client.end_transaction("test") - transaction = elasticapm_client.transaction_store.get_all()[0] - span = transaction["spans"][0] + transaction = elasticapm_client.events[TRANSACTION][0] + span = elasticapm_client.spans_for_transaction(transaction)[0] assert span["type"] == "db.cassandra.query" assert span["name"] == "SELECT FROM users" assert span["context"] == {"db": {"statement": "SELECT name from users", "type": "sql"}} @@ -75,8 +76,8 @@ def test_select_prepared_statement(instrument, cassandra_session, elasticapm_cli elasticapm_client.begin_transaction("transaction.test") cassandra_session.execute(prepared_statement) elasticapm_client.end_transaction("test") - transaction = elasticapm_client.transaction_store.get_all()[0] - span = transaction["spans"][0] + transaction = elasticapm_client.events[TRANSACTION][0] + span = elasticapm_client.spans_for_transaction(transaction)[0] assert span["type"] == "db.cassandra.query" assert span["name"] == "SELECT FROM users" assert span["context"] == {"db": {"statement": "SELECT name from users", "type": "sql"}} diff --git a/tests/instrumentation/django_tests/template_tests.py b/tests/instrumentation/django_tests/template_tests.py index 24e795ff2..c09b2c88d 100644 --- a/tests/instrumentation/django_tests/template_tests.py +++ b/tests/instrumentation/django_tests/template_tests.py @@ -11,6 +11,7 @@ import pytest from conftest import BASE_TEMPLATE_DIR +from elasticapm.conf.constants import TRANSACTION from tests.utils.compat import middleware_setting try: @@ -27,9 +28,7 @@ ) -@mock.patch("elasticapm.traces.TransactionsStore.should_collect") -def test_template_rendering(should_collect, instrument, django_elasticapm_client, client): - should_collect.return_value = False +def test_template_rendering(instrument, django_elasticapm_client, client): with override_settings( **middleware_setting(django.VERSION, ["elasticapm.contrib.django.middleware.TracingMiddleware"]) ): @@ -37,10 +36,10 @@ def test_template_rendering(should_collect, instrument, django_elasticapm_client client.get(reverse("render-heavy-template")) client.get(reverse("render-heavy-template")) - transactions = django_elasticapm_client.transaction_store.get_all() + transactions = django_elasticapm_client.events[TRANSACTION] assert len(transactions) == 3 - spans = transactions[0]["spans"] + spans = django_elasticapm_client.spans_for_transaction(transactions[0]) assert len(spans) == 2, [t["name"] for t in spans] kinds = ["code", "template.django"] @@ -56,9 +55,7 @@ def test_template_rendering(should_collect, instrument, django_elasticapm_client @pytest.mark.skipif(django.VERSION < (1, 8), reason="Jinja2 support introduced with Django 1.8") -@mock.patch("elasticapm.traces.TransactionsStore.should_collect") -def test_template_rendering_django18_jinja2(should_collect, instrument, django_elasticapm_client, client): - should_collect.return_value = False +def test_template_rendering_django18_jinja2(instrument, django_elasticapm_client, client): with override_settings( TEMPLATES=TEMPLATES, **middleware_setting(django.VERSION, ["elasticapm.contrib.django.middleware.TracingMiddleware"]) @@ -67,10 +64,10 @@ def test_template_rendering_django18_jinja2(should_collect, instrument, django_e client.get(reverse("render-jinja2-template")) client.get(reverse("render-jinja2-template")) - transactions = django_elasticapm_client.transaction_store.get_all() + transactions = django_elasticapm_client.events[TRANSACTION] assert len(transactions) == 3 - spans = transactions[0]["spans"] + spans = django_elasticapm_client.spans_for_transaction(transactions[0]) assert len(spans) == 1, [t["name"] for t in spans] kinds = ["template.jinja2"] diff --git a/tests/instrumentation/elasticsearch_tests.py b/tests/instrumentation/elasticsearch_tests.py index 3ab9ac554..be04610b4 100644 --- a/tests/instrumentation/elasticsearch_tests.py +++ b/tests/instrumentation/elasticsearch_tests.py @@ -7,6 +7,8 @@ from elasticsearch import VERSION as ES_VERSION from elasticsearch import Elasticsearch +from elasticapm.conf.constants import TRANSACTION + pytestmark = pytest.mark.elasticsearch @@ -22,24 +24,29 @@ def elasticsearch(request): def test_ping(instrument, elasticapm_client, elasticsearch): elasticapm_client.begin_transaction("test") result = elasticsearch.ping() - transaction_obj = elasticapm_client.end_transaction("test", "OK") + elasticapm_client.end_transaction("test", "OK") - assert len(transaction_obj.spans) == 1 - span = transaction_obj.spans[0] - assert span.name == "ES HEAD /" - assert span.type == "db.elasticsearch" + transaction = elasticapm_client.events[TRANSACTION][0] + spans = elasticapm_client.spans_for_transaction(transaction) + assert len(spans) == 1 + span = spans[0] + assert span["name"] == "ES HEAD /" + assert span["type"] == "db.elasticsearch" @pytest.mark.integrationtest def test_info(instrument, elasticapm_client, elasticsearch): elasticapm_client.begin_transaction("test") result = elasticsearch.info() - transaction_obj = elasticapm_client.end_transaction("test", "OK") + elasticapm_client.end_transaction("test", "OK") + + transaction = elasticapm_client.events[TRANSACTION][0] - assert len(transaction_obj.spans) == 1 - span = transaction_obj.spans[0] - assert span.name == "ES GET /" - assert span.type == "db.elasticsearch" + spans = elasticapm_client.spans_for_transaction(transaction) + assert len(spans) == 1 + span = spans[0] + assert span["name"] == "ES GET /" + assert span["type"] == "db.elasticsearch" @pytest.mark.integrationtest @@ -52,18 +59,21 @@ def test_create(instrument, elasticapm_client, elasticsearch): r2 = elasticsearch.create( index="tweets", doc_type="doc", id=2, body={"user": "kimchy", "text": "hola"}, refresh=True ) - transaction_obj = elasticapm_client.end_transaction("test", "OK") + elasticapm_client.end_transaction("test", "OK") + + transaction = elasticapm_client.events[TRANSACTION][0] - assert len(transaction_obj.spans) == 2 + spans = elasticapm_client.spans_for_transaction(transaction) + assert len(spans) == 2 - for i, span in enumerate(transaction_obj.spans): + for i, span in enumerate(spans): if ES_VERSION[0] >= 5: - assert span.name == "ES PUT /tweets/doc/%d/_create" % (i + 1) + assert span["name"] == "ES PUT /tweets/doc/%d/_create" % (i + 1) else: - assert span.name == "ES PUT /tweets/doc/%d" % (i + 1) - assert span.type == "db.elasticsearch" - assert span.context["db"]["type"] == "elasticsearch" - assert "statement" not in span.context["db"] + assert span["name"] == "ES PUT /tweets/doc/%d" % (i + 1) + assert span["type"] == "db.elasticsearch" + assert span["context"]["db"]["type"] == "elasticsearch" + assert "statement" not in span["context"]["db"] @pytest.mark.integrationtest @@ -71,15 +81,18 @@ def test_index(instrument, elasticapm_client, elasticsearch): elasticapm_client.begin_transaction("test") r1 = elasticsearch.index("tweets", "doc", {"user": "kimchy", "text": "hola"}) r2 = elasticsearch.index(index="tweets", doc_type="doc", body={"user": "kimchy", "text": "hola"}, refresh=True) - transaction_obj = elasticapm_client.end_transaction("test", "OK") + elasticapm_client.end_transaction("test", "OK") - assert len(transaction_obj.spans) == 2 + transaction = elasticapm_client.events[TRANSACTION][0] - for span in transaction_obj.spans: - assert span.name == "ES POST /tweets/doc" - assert span.type == "db.elasticsearch" - assert span.context["db"]["type"] == "elasticsearch" - assert "statement" not in span.context["db"] + spans = elasticapm_client.spans_for_transaction(transaction) + assert len(spans) == 2 + + for span in spans: + assert span["name"] == "ES POST /tweets/doc" + assert span["type"] == "db.elasticsearch" + assert span["context"]["db"]["type"] == "elasticsearch" + assert "statement" not in span["context"]["db"] @pytest.mark.integrationtest @@ -87,13 +100,16 @@ def test_exists(instrument, elasticapm_client, elasticsearch): elasticsearch.create(index="tweets", doc_type="doc", id=1, body={"user": "kimchy", "text": "hola"}, refresh=True) elasticapm_client.begin_transaction("test") result = elasticsearch.exists(id=1, index="tweets", doc_type="doc") - transaction_obj = elasticapm_client.end_transaction("test", "OK") + elasticapm_client.end_transaction("test", "OK") + + transaction = elasticapm_client.events[TRANSACTION][0] assert result - assert len(transaction_obj.spans) == 1 - span = transaction_obj.spans[0] - assert span.name == "ES HEAD /tweets/doc/1" - assert span.type == "db.elasticsearch" - assert span.context["db"]["type"] == "elasticsearch" + spans = elasticapm_client.spans_for_transaction(transaction) + assert len(spans) == 1 + span = spans[0] + assert span["name"] == "ES HEAD /tweets/doc/1" + assert span["type"] == "db.elasticsearch" + assert span["context"]["db"]["type"] == "elasticsearch" @pytest.mark.skipif(ES_VERSION[0] < 5, reason="unsupported method") @@ -103,15 +119,18 @@ def test_exists_source(instrument, elasticapm_client, elasticsearch): elasticapm_client.begin_transaction("test") assert elasticsearch.exists_source("tweets", "doc", 1) is True assert elasticsearch.exists_source(index="tweets", doc_type="doc", id=1) is True - transaction_obj = elasticapm_client.end_transaction("test", "OK") + elasticapm_client.end_transaction("test", "OK") - assert len(transaction_obj.spans) == 2 + transaction = elasticapm_client.events[TRANSACTION][0] - for span in transaction_obj.spans: - assert span.name == "ES HEAD /tweets/doc/1/_source" - assert span.type == "db.elasticsearch" - assert span.context["db"]["type"] == "elasticsearch" - assert "statement" not in span.context["db"] + spans = elasticapm_client.spans_for_transaction(transaction) + assert len(spans) == 2 + + for span in spans: + assert span["name"] == "ES HEAD /tweets/doc/1/_source" + assert span["type"] == "db.elasticsearch" + assert span["context"]["db"]["type"] == "elasticsearch" + assert "statement" not in span["context"]["db"] @pytest.mark.integrationtest @@ -123,17 +142,20 @@ def test_get(instrument, elasticapm_client, elasticsearch): else: r1 = elasticsearch.get("tweets", 1, "doc") r2 = elasticsearch.get(index="tweets", doc_type="doc", id=1) - transaction_obj = elasticapm_client.end_transaction("test", "OK") + elasticapm_client.end_transaction("test", "OK") + + transaction = elasticapm_client.events[TRANSACTION][0] for r in (r1, r2): assert r["found"] assert r["_source"] == {"user": "kimchy", "text": "hola"} - assert len(transaction_obj.spans) == 2 + spans = elasticapm_client.spans_for_transaction(transaction) + assert len(spans) == 2 - for span in transaction_obj.spans: - assert span.name == "ES GET /tweets/doc/1" - assert span.type == "db.elasticsearch" - assert span.context["db"]["type"] == "elasticsearch" - assert "statement" not in span.context["db"] + for span in spans: + assert span["name"] == "ES GET /tweets/doc/1" + assert span["type"] == "db.elasticsearch" + assert span["context"]["db"]["type"] == "elasticsearch" + assert "statement" not in span["context"]["db"] @pytest.mark.integrationtest @@ -142,18 +164,21 @@ def test_get_source(instrument, elasticapm_client, elasticsearch): elasticapm_client.begin_transaction("test") r1 = elasticsearch.get_source("tweets", "doc", 1) r2 = elasticsearch.get_source(index="tweets", doc_type="doc", id=1) - transaction_obj = elasticapm_client.end_transaction("test", "OK") + elasticapm_client.end_transaction("test", "OK") + + transaction = elasticapm_client.events[TRANSACTION][0] for r in (r1, r2): assert r == {"user": "kimchy", "text": "hola"} - assert len(transaction_obj.spans) == 2 + spans = elasticapm_client.spans_for_transaction(transaction) + assert len(spans) == 2 - for span in transaction_obj.spans: - assert span.name == "ES GET /tweets/doc/1/_source" - assert span.type == "db.elasticsearch" - assert span.context["db"]["type"] == "elasticsearch" - assert "statement" not in span.context["db"] + for span in spans: + assert span["name"] == "ES GET /tweets/doc/1/_source" + assert span["type"] == "db.elasticsearch" + assert span["context"]["db"]["type"] == "elasticsearch" + assert "statement" not in span["context"]["db"] @pytest.mark.skipif(ES_VERSION[0] < 5, reason="unsupported method") @@ -162,17 +187,20 @@ def test_update_script(instrument, elasticapm_client, elasticsearch): elasticsearch.create(index="tweets", doc_type="doc", id=1, body={"user": "kimchy", "text": "hola"}, refresh=True) elasticapm_client.begin_transaction("test") r1 = elasticsearch.update("tweets", "doc", 1, {"script": "ctx._source.text = 'adios'"}, refresh=True) - transaction_obj = elasticapm_client.end_transaction("test", "OK") + elasticapm_client.end_transaction("test", "OK") + + transaction = elasticapm_client.events[TRANSACTION][0] r2 = elasticsearch.get(index="tweets", doc_type="doc", id=1) assert r1["result"] == "updated" assert r2["_source"] == {"user": "kimchy", "text": "adios"} - assert len(transaction_obj.spans) == 1 + spans = elasticapm_client.spans_for_transaction(transaction) + assert len(spans) == 1 - span = transaction_obj.spans[0] - assert span.name == "ES POST /tweets/doc/1/_update" - assert span.type == "db.elasticsearch" - assert span.context["db"]["type"] == "elasticsearch" - assert span.context["db"]["statement"] == '{"script": "ctx._source.text = \'adios\'"}' + span = spans[0] + assert span["name"] == "ES POST /tweets/doc/1/_update" + assert span["type"] == "db.elasticsearch" + assert span["context"]["db"]["type"] == "elasticsearch" + assert span["context"]["db"]["statement"] == '{"script": "ctx._source.text = \'adios\'"}' @pytest.mark.integrationtest @@ -180,16 +208,19 @@ def test_update_document(instrument, elasticapm_client, elasticsearch): elasticsearch.create(index="tweets", doc_type="doc", id=1, body={"user": "kimchy", "text": "hola"}, refresh=True) elasticapm_client.begin_transaction("test") r1 = elasticsearch.update("tweets", "doc", 1, {"doc": {"text": "adios"}}, refresh=True) - transaction_obj = elasticapm_client.end_transaction("test", "OK") + elasticapm_client.end_transaction("test", "OK") + + transaction = elasticapm_client.events[TRANSACTION][0] r2 = elasticsearch.get(index="tweets", doc_type="doc", id=1) assert r2["_source"] == {"user": "kimchy", "text": "adios"} - assert len(transaction_obj.spans) == 1 + spans = elasticapm_client.spans_for_transaction(transaction) + assert len(spans) == 1 - span = transaction_obj.spans[0] - assert span.name == "ES POST /tweets/doc/1/_update" - assert span.type == "db.elasticsearch" - assert span.context["db"]["type"] == "elasticsearch" - assert "statement" not in span.context["db"] + span = spans[0] + assert span["name"] == "ES POST /tweets/doc/1/_update" + assert span["type"] == "db.elasticsearch" + assert span["context"]["db"]["type"] == "elasticsearch" + assert "statement" not in span["context"]["db"] @pytest.mark.integrationtest @@ -198,14 +229,17 @@ def test_search_body(instrument, elasticapm_client, elasticsearch): elasticapm_client.begin_transaction("test") search_query = {"query": {"term": {"user": "kimchy"}}} result = elasticsearch.search(body=search_query, params=None) - transaction_obj = elasticapm_client.end_transaction("test", "OK") + elasticapm_client.end_transaction("test", "OK") + + transaction = elasticapm_client.events[TRANSACTION][0] assert result["hits"]["hits"][0]["_source"] == {"user": "kimchy", "text": "hola"} - assert len(transaction_obj.spans) == 1 - span = transaction_obj.spans[0] - assert span.name == "ES GET /_search" - assert span.type == "db.elasticsearch" - assert span.context["db"]["type"] == "elasticsearch" - assert span.context["db"]["statement"] == '{"term": {"user": "kimchy"}}' + spans = elasticapm_client.spans_for_transaction(transaction) + assert len(spans) == 1 + span = spans[0] + assert span["name"] == "ES GET /_search" + assert span["type"] == "db.elasticsearch" + assert span["context"]["db"]["type"] == "elasticsearch" + assert span["context"]["db"]["statement"] == '{"term": {"user": "kimchy"}}' @pytest.mark.integrationtest @@ -214,14 +248,17 @@ def test_search_querystring(instrument, elasticapm_client, elasticsearch): elasticapm_client.begin_transaction("test") search_query = "user:kimchy" result = elasticsearch.search(q=search_query, index="tweets") - transaction_obj = elasticapm_client.end_transaction("test", "OK") + elasticapm_client.end_transaction("test", "OK") + + transaction = elasticapm_client.events[TRANSACTION][0] assert result["hits"]["hits"][0]["_source"] == {"user": "kimchy", "text": "hola"} - assert len(transaction_obj.spans) == 1 - span = transaction_obj.spans[0] - assert span.name == "ES GET /tweets/_search" - assert span.type == "db.elasticsearch" - assert span.context["db"]["type"] == "elasticsearch" - assert span.context["db"]["statement"] == "q=user:kimchy" + spans = elasticapm_client.spans_for_transaction(transaction) + assert len(spans) == 1 + span = spans[0] + assert span["name"] == "ES GET /tweets/_search" + assert span["type"] == "db.elasticsearch" + assert span["context"]["db"]["type"] == "elasticsearch" + assert span["context"]["db"]["statement"] == "q=user:kimchy" @pytest.mark.integrationtest @@ -231,15 +268,18 @@ def test_search_both(instrument, elasticapm_client, elasticsearch): search_querystring = "text:hola" search_query = {"query": {"term": {"user": "kimchy"}}} result = elasticsearch.search(body=search_query, q=search_querystring, index="tweets") - transaction_obj = elasticapm_client.end_transaction("test", "OK") + elasticapm_client.end_transaction("test", "OK") + + transaction = elasticapm_client.events[TRANSACTION][0] assert len(result["hits"]["hits"]) == 1 assert result["hits"]["hits"][0]["_source"] == {"user": "kimchy", "text": "hola"} - assert len(transaction_obj.spans) == 1 - span = transaction_obj.spans[0] - assert span.name == "ES GET /tweets/_search" - assert span.type == "db.elasticsearch" - assert span.context["db"]["type"] == "elasticsearch" - assert span.context["db"]["statement"] == 'q=text:hola\n\n{"term": {"user": "kimchy"}}' + spans = elasticapm_client.spans_for_transaction(transaction) + assert len(spans) == 1 + span = spans[0] + assert span["name"] == "ES GET /tweets/_search" + assert span["type"] == "db.elasticsearch" + assert span["context"]["db"]["type"] == "elasticsearch" + assert span["context"]["db"]["statement"] == 'q=text:hola\n\n{"term": {"user": "kimchy"}}' @pytest.mark.integrationtest @@ -248,14 +288,17 @@ def test_count_body(instrument, elasticapm_client, elasticsearch): elasticapm_client.begin_transaction("test") search_query = {"query": {"term": {"user": "kimchy"}}} result = elasticsearch.count(body=search_query) - transaction_obj = elasticapm_client.end_transaction("test", "OK") + elasticapm_client.end_transaction("test", "OK") + + transaction = elasticapm_client.events[TRANSACTION][0] assert result["count"] == 1 - assert len(transaction_obj.spans) == 1 - span = transaction_obj.spans[0] - assert span.name == "ES GET /_count" - assert span.type == "db.elasticsearch" - assert span.context["db"]["type"] == "elasticsearch" - assert span.context["db"]["statement"] == '{"term": {"user": "kimchy"}}' + spans = elasticapm_client.spans_for_transaction(transaction) + assert len(spans) == 1 + span = spans[0] + assert span["name"] == "ES GET /_count" + assert span["type"] == "db.elasticsearch" + assert span["context"]["db"]["type"] == "elasticsearch" + assert span["context"]["db"]["statement"] == '{"term": {"user": "kimchy"}}' @pytest.mark.integrationtest @@ -264,14 +307,17 @@ def test_count_querystring(instrument, elasticapm_client, elasticsearch): elasticapm_client.begin_transaction("test") search_query = "user:kimchy" result = elasticsearch.count(q=search_query, index="tweets") - transaction_obj = elasticapm_client.end_transaction("test", "OK") + elasticapm_client.end_transaction("test", "OK") + + transaction = elasticapm_client.events[TRANSACTION][0] assert result["count"] == 1 - assert len(transaction_obj.spans) == 1 - span = transaction_obj.spans[0] - assert span.name == "ES GET /tweets/_count" - assert span.type == "db.elasticsearch" - assert span.context["db"]["type"] == "elasticsearch" - assert span.context["db"]["statement"] == "q=user:kimchy" + spans = elasticapm_client.spans_for_transaction(transaction) + assert len(spans) == 1 + span = spans[0] + assert span["name"] == "ES GET /tweets/_count" + assert span["type"] == "db.elasticsearch" + assert span["context"]["db"]["type"] == "elasticsearch" + assert span["context"]["db"]["statement"] == "q=user:kimchy" @pytest.mark.integrationtest @@ -279,11 +325,15 @@ def test_delete(instrument, elasticapm_client, elasticsearch): elasticsearch.create(index="tweets", doc_type="doc", id=1, body={"user": "kimchy", "text": "hola"}, refresh=True) elasticapm_client.begin_transaction("test") result = elasticsearch.delete(id=1, index="tweets", doc_type="doc") - transaction_obj = elasticapm_client.end_transaction("test", "OK") - span = transaction_obj.spans[0] - assert span.name == "ES DELETE /tweets/doc/1" - assert span.type == "db.elasticsearch" - assert span.context["db"]["type"] == "elasticsearch" + elasticapm_client.end_transaction("test", "OK") + + transaction = elasticapm_client.events[TRANSACTION][0] + spans = elasticapm_client.spans_for_transaction(transaction) + + span = spans[0] + assert span["name"] == "ES DELETE /tweets/doc/1" + assert span["type"] == "db.elasticsearch" + assert span["context"]["db"]["type"] == "elasticsearch" @pytest.mark.skipif(ES_VERSION[0] < 5, reason="unsupported method") @@ -292,12 +342,16 @@ def test_delete_by_query_body(instrument, elasticapm_client, elasticsearch): elasticsearch.create(index="tweets", doc_type="doc", id=1, body={"user": "kimchy", "text": "hola"}, refresh=True) elasticapm_client.begin_transaction("test") result = elasticsearch.delete_by_query(index="tweets", body={"query": {"term": {"user": "kimchy"}}}) - transaction_obj = elasticapm_client.end_transaction("test", "OK") - span = transaction_obj.spans[0] - assert span.name == "ES POST /tweets/_delete_by_query" - assert span.type == "db.elasticsearch" - assert span.context["db"]["type"] == "elasticsearch" - assert span.context["db"]["statement"] == '{"term": {"user": "kimchy"}}' + elasticapm_client.end_transaction("test", "OK") + + transaction = elasticapm_client.events[TRANSACTION][0] + spans = elasticapm_client.spans_for_transaction(transaction) + + span = spans[0] + assert span["name"] == "ES POST /tweets/_delete_by_query" + assert span["type"] == "db.elasticsearch" + assert span["context"]["db"]["type"] == "elasticsearch" + assert span["context"]["db"]["statement"] == '{"term": {"user": "kimchy"}}' @pytest.mark.integrationtest @@ -306,9 +360,12 @@ def test_multiple_indexes_doctypes(instrument, elasticapm_client, elasticsearch) elasticsearch.create(index="snaps", doc_type="posts", id=1, body={"user": "kimchy", "text": "hola"}, refresh=True) elasticapm_client.begin_transaction("test") result = elasticsearch.search(index=["tweets", "snaps"], doc_type=["users", "posts"], q="user:kimchy") - transaction_obj = elasticapm_client.end_transaction("test", "OK") - assert len(transaction_obj.spans) == 1 - span = transaction_obj.spans[0] - assert span.name == "ES GET /tweets,snaps/users,posts/_search" - assert span.type == "db.elasticsearch" - assert span.context["db"]["type"] == "elasticsearch" + elasticapm_client.end_transaction("test", "OK") + + transaction = elasticapm_client.events[TRANSACTION][0] + spans = elasticapm_client.spans_for_transaction(transaction) + assert len(spans) == 1 + span = spans[0] + assert span["name"] == "ES GET /tweets,snaps/users,posts/_search" + assert span["type"] == "db.elasticsearch" + assert span["context"]["db"]["type"] == "elasticsearch" diff --git a/tests/instrumentation/jinja2_tests/jinja2_tests.py b/tests/instrumentation/jinja2_tests/jinja2_tests.py index e7836d0ac..b6230152b 100644 --- a/tests/instrumentation/jinja2_tests/jinja2_tests.py +++ b/tests/instrumentation/jinja2_tests/jinja2_tests.py @@ -5,6 +5,8 @@ from jinja2 import Environment, FileSystemLoader from jinja2.environment import Template +from elasticapm.conf.constants import TRANSACTION + @pytest.fixture() def jinja_env(): @@ -13,16 +15,14 @@ def jinja_env(): return Environment(loader=loader) -@mock.patch("elasticapm.traces.TransactionsStore.should_collect") -def test_from_file(should_collect, instrument, jinja_env, elasticapm_client): - should_collect.return_value = False +def test_from_file(instrument, jinja_env, elasticapm_client): elasticapm_client.begin_transaction("transaction.test") template = jinja_env.get_template("mytemplate.html") template.render() elasticapm_client.end_transaction("MyView") - transactions = elasticapm_client.transaction_store.get_all() - spans = transactions[0]["spans"] + transactions = elasticapm_client.events[TRANSACTION] + spans = elasticapm_client.spans_for_transaction(transactions[0]) expected_signatures = {"mytemplate.html"} @@ -38,8 +38,8 @@ def test_from_string(instrument, elasticapm_client): template.render() elasticapm_client.end_transaction("test") - transactions = elasticapm_client.transaction_store.get_all() - spans = transactions[0]["spans"] + transactions = elasticapm_client.events[TRANSACTION] + spans = elasticapm_client.spans_for_transaction(transactions[0]) expected_signatures = {"