Merge branch 'develop' of github.com:risenberg-cyberark/aws-lambda-powertools-python into pydantic

Author: Ran Isenberg

CHANGELOG.md

@@ -6,9 +6,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [1.6.1] - 2020-09-23
+
+### Fixed
+- **Utilities**: Fix issue with boolean values in DynamoDB stream event data class.
+
+## [1.6.0] - 2020-09-22
+
 ### Added
-- **Metrics**: Support adding multiple metric values to a metric name
+- **Metrics**: Support adding multiple metric values to a single metric name
 - **Utilities**: Add new `Validator` utility to validate inbound events and responses using JSON Schema
+- **Utilities**: Add new `Event source data classes` utility to easily describe event schema of popular event sources
+- **Docs**: Add new `Testing your code` section to both Logger and Metrics page, and content width is now wider
+- **Tracer**: Support for automatically disable Tracer when running a Chalice app
+
+### Fixed
+- **Docs**: Improve wording on log sampling feature in Logger, and removed duplicate content on main page
+- **Utilities**: Remove DeleteMessageBatch API call when there are no messages to delete
 
 ## [1.5.0] - 2020-09-04
 

README.md

@@ -3,7 +3,7 @@
 ![Build](https://github.com/awslabs/aws-lambda-powertools/workflows/Powertools%20Python/badge.svg?branch=master)
 ![PythonSupport](https://img.shields.io/static/v1?label=python&message=3.6%20|%203.7|%203.8&color=blue?style=flat-square&logo=python) ![PyPI version](https://badge.fury.io/py/aws-lambda-powertools.svg) ![PyPi monthly downloads](https://img.shields.io/pypi/dm/aws-lambda-powertools)
 
-A suite of utilities for AWS Lambda functions that makes tracing with AWS X-Ray, structured logging and creating custom metrics asynchronously easier.
+A suite of utilities for AWS Lambda functions to ease adopting best practices such as tracing, structured logging, custom metrics, and more.
 
 **[📜Documentation](https://awslabs.github.io/aws-lambda-powertools-python/)** | **[API Docs](https://awslabs.github.io/aws-lambda-powertools-python/api/)** | **[🐍PyPi](https://pypi.org/project/aws-lambda-powertools/)** | **[Feature request](https://github.com/awslabs/aws-lambda-powertools-python/issues/new?assignees=&labels=feature-request%2C+triage&template=feature_request.md&title=)** | **[🐛Bug Report](https://github.com/awslabs/aws-lambda-powertools-python/issues/new?assignees=&labels=bug%2C+triage&template=bug_report.md&title=)** | **[Kitchen sink example](https://github.com/awslabs/aws-lambda-powertools-python/tree/develop/example)** | **[Detailed blog post](https://aws.amazon.com/blogs/opensource/simplifying-serverless-best-practices-with-lambda-powertools/)**
 

aws_lambda_powertools/metrics/base.py

@@ -80,7 +80,7 @@ class MetricManager:
 
     def __init__(
         self,
-        metric_set: Dict[str, str] = None,
+        metric_set: Dict[str, Any] = None,
         dimension_set: Dict = None,
         namespace: str = None,
         metadata_set: Dict[str, Any] = None,
@@ -125,7 +125,7 @@ def add_metric(self, name: str, unit: Union[MetricUnit, str], value: float):
             raise MetricValueError(f"{value} is not a valid number")
 
         unit = self.__extract_metric_unit_value(unit=unit)
-        metric = self.metric_set.get(name, defaultdict(list))
+        metric: Dict = self.metric_set.get(name, defaultdict(list))
         metric["Unit"] = unit
         metric["Value"].append(float(value))
         logger.debug(f"Adding metric: {name} with {metric}")
@@ -185,7 +185,7 @@ def serialize_metric_set(self, metrics: Dict = None, dimensions: Dict = None, me
         logger.debug({"details": "Serializing metrics", "metrics": metrics, "dimensions": dimensions})
 
         metric_names_and_units: List[Dict[str, str]] = []  # [ { "Name": "metric_name", "Unit": "Count" } ]
-        metric_names_and_values: Dict[str, str] = {}  # { "metric_name": 1.0 }
+        metric_names_and_values: Dict[str, float] = {}  # { "metric_name": 1.0 }
 
         for metric_name in metrics:
             metric: dict = metrics[metric_name]

aws_lambda_powertools/tracing/tracer.py

@@ -622,14 +622,15 @@ def _is_tracer_disabled() -> bool:
         """
         logger.debug("Verifying whether Tracing has been disabled")
         is_lambda_sam_cli = os.getenv("AWS_SAM_LOCAL")
+        is_chalice_cli = os.getenv("AWS_CHALICE_CLI_MODE")
         env_option = str(os.getenv("POWERTOOLS_TRACE_DISABLED", "false"))
         disabled_env = strtobool(env_option)
 
         if disabled_env:
             logger.debug("Tracing has been disabled via env var POWERTOOLS_TRACE_DISABLED")
             return disabled_env
 
-        if is_lambda_sam_cli:
+        if is_lambda_sam_cli or is_chalice_cli:
             logger.debug("Running under SAM CLI env or not in Lambda env; disabling Tracing")
             return True
 

aws_lambda_powertools/utilities/batch/sqs.py

@@ -115,7 +115,9 @@ def _clean(self):
         queue_url = self._get_queue_url()
         entries_to_remove = self._get_entries_to_clean()
 
-        delete_message_response = self.client.delete_message_batch(QueueUrl=queue_url, Entries=entries_to_remove)
+        delete_message_response = None
+        if entries_to_remove:
+            delete_message_response = self.client.delete_message_batch(QueueUrl=queue_url, Entries=entries_to_remove)
 
         if self.suppress_exception:
             logger.debug(f"{len(self.fail_messages)} records failed processing, but exceptions are suppressed")

aws_lambda_powertools/utilities/data_classes/alb_event.py

@@ -19,7 +19,7 @@ class ALBEvent(BaseProxyEvent):
 
     @property
     def request_context(self) -> ALBEventRequestContext:
-        return ALBEventRequestContext(self)
+        return ALBEventRequestContext(self._data)
 
     @property
     def http_method(self) -> str:

aws_lambda_powertools/utilities/data_classes/api_gateway_proxy_event.py

@@ -231,7 +231,7 @@ def multi_value_query_string_parameters(self) -> Optional[Dict[str, List[str]]]:
 
     @property
     def request_context(self) -> APIGatewayEventRequestContext:
-        return APIGatewayEventRequestContext(self)
+        return APIGatewayEventRequestContext(self._data)
 
     @property
     def path_parameters(self) -> Optional[Dict[str, str]]:
@@ -371,7 +371,7 @@ def cookies(self) -> Optional[List[str]]:
 
     @property
     def request_context(self) -> RequestContextV2:
-        return RequestContextV2(self)
+        return RequestContextV2(self._data)
 
     @property
     def path_parameters(self) -> Optional[Dict[str, str]]:

aws_lambda_powertools/utilities/data_classes/cognito_user_pool_event.py

@@ -51,7 +51,7 @@ def user_name(self) -> str:
     @property
     def caller_context(self) -> CallerContext:
         """The caller context"""
-        return CallerContext(self)
+        return CallerContext(self._data)
 
 
 class PreSignUpTriggerEventRequest(DictWrapper):
@@ -119,11 +119,11 @@ class PreSignUpTriggerEvent(BaseTriggerEvent):
 
     @property
     def request(self) -> PreSignUpTriggerEventRequest:
-        return PreSignUpTriggerEventRequest(self)
+        return PreSignUpTriggerEventRequest(self._data)
 
     @property
     def response(self) -> PreSignUpTriggerEventResponse:
-        return PreSignUpTriggerEventResponse(self)
+        return PreSignUpTriggerEventResponse(self._data)
 
 
 class PostConfirmationTriggerEventRequest(DictWrapper):
@@ -156,7 +156,7 @@ class PostConfirmationTriggerEvent(BaseTriggerEvent):
 
     @property
     def request(self) -> PostConfirmationTriggerEventRequest:
-        return PostConfirmationTriggerEventRequest(self)
+        return PostConfirmationTriggerEventRequest(self._data)
 
 
 class UserMigrationTriggerEventRequest(DictWrapper):
@@ -257,11 +257,11 @@ class UserMigrationTriggerEvent(BaseTriggerEvent):
 
     @property
     def request(self) -> UserMigrationTriggerEventRequest:
-        return UserMigrationTriggerEventRequest(self)
+        return UserMigrationTriggerEventRequest(self._data)
 
     @property
     def response(self) -> UserMigrationTriggerEventResponse:
-        return UserMigrationTriggerEventResponse(self)
+        return UserMigrationTriggerEventResponse(self._data)
 
 
 class CustomMessageTriggerEventRequest(DictWrapper):
@@ -342,11 +342,11 @@ class CustomMessageTriggerEvent(BaseTriggerEvent):
 
     @property
     def request(self) -> CustomMessageTriggerEventRequest:
-        return CustomMessageTriggerEventRequest(self)
+        return CustomMessageTriggerEventRequest(self._data)
 
     @property
     def response(self) -> CustomMessageTriggerEventResponse:
-        return CustomMessageTriggerEventResponse(self)
+        return CustomMessageTriggerEventResponse(self._data)
 
 
 class PreAuthenticationTriggerEventRequest(DictWrapper):
@@ -386,7 +386,7 @@ class PreAuthenticationTriggerEvent(BaseTriggerEvent):
     @property
     def request(self) -> PreAuthenticationTriggerEventRequest:
         """Pre Authentication Request Parameters"""
-        return PreAuthenticationTriggerEventRequest(self)
+        return PreAuthenticationTriggerEventRequest(self._data)
 
 
 class PostAuthenticationTriggerEventRequest(DictWrapper):
@@ -428,7 +428,7 @@ class PostAuthenticationTriggerEvent(BaseTriggerEvent):
     @property
     def request(self) -> PostAuthenticationTriggerEventRequest:
         """Post Authentication Request Parameters"""
-        return PostAuthenticationTriggerEventRequest(self)
+        return PostAuthenticationTriggerEventRequest(self._data)
 
 
 class GroupOverrideDetails(DictWrapper):
@@ -552,9 +552,9 @@ class PreTokenGenerationTriggerEvent(BaseTriggerEvent):
     @property
     def request(self) -> PreTokenGenerationTriggerEventRequest:
         """Pre Token Generation Request Parameters"""
-        return PreTokenGenerationTriggerEventRequest(self)
+        return PreTokenGenerationTriggerEventRequest(self._data)
 
     @property
     def response(self) -> PreTokenGenerationTriggerEventResponse:
         """Pre Token Generation Response Parameters"""
-        return PreTokenGenerationTriggerEventResponse(self)
+        return PreTokenGenerationTriggerEventResponse(self._data)

aws_lambda_powertools/utilities/data_classes/dynamo_db_stream_event.py

@@ -35,7 +35,7 @@ def bool_value(self) -> Optional[bool]:
         Example:
             >>> {"BOOL": True}
         """
-        item = self.get("bool")
+        item = self.get("BOOL")
         return None if item is None else bool(item)
 
     @property

docs/content/core/logger.mdx

@@ -222,21 +222,24 @@ If you ever forget to use `child` param, we will return an existing `Logger` wit
 
 ## Sampling debug logs
 
-You can dynamically set a percentage of your logs to **DEBUG** level using `sample_rate` param or via env var `POWERTOOLS_LOGGER_SAMPLE_RATE`.
+Sampling allows you to set your Logger Log Level as DEBUG based on a percentage of your concurrent/cold start invocations. You can set a sampling value of `0.0` to `1` (100%) using either `sample_rate` parameter or `POWERTOOLS_LOGGER_SAMPLE_RATE` env var.
 
-Sampling calculation happens at the Logger class initialization. This means, when configured it, sampling it's more likely to happen during concurrent requests, or infrequent invocations as [new Lambda execution contexts are created](https://docs.aws.amazon.com/lambda/latest/dg/runtimes-context.html), not reused.
+This is useful when you want to troubleshoot an issue, say a sudden increase in concurrency, and you might not have enough information in your logs as Logger log level was understandably set as INFO.
+
+Sampling decision happens at the Logger class initialization, which only happens during a cold start. This means sampling may happen significantly more or less than you expect if you have a steady low number of invocations and thus few cold starts.
 
 <Note type="info">
-  If you want this logic to happen on every invocation regardless whether Lambda reuses the execution environment or not, then create your Logger inside your Lambda handler.
+  If you want Logger to calculate sampling on every invocation, then please open a feature request.
 </Note><br/>
 
 ```python:title=collect.py
 from aws_lambda_powertools import Logger
 
 # Sample 10% of debug logs e.g. 0.1
-logger = Logger(sample_rate=0.1) # highlight-line
+logger = Logger(sample_rate=0.1, level="INFO") # highlight-line
 
 def handler(event, context):
+   logger.debug("Verifying whether order_id is present")
    if "order_id" in event:
       logger.info("Collecting payment")
       ...
@@ -245,7 +248,21 @@ def handler(event, context):
 <details>
 <summary><strong>Excerpt output in CloudWatch Logs</strong></summary>
 
-```json:title=cloudwatch_logs.json
+```json:title=sampled_log_request_as_debug.json
+{
+   "timestamp": "2020-05-24 18:17:33,774",
+   "level": "DEBUG", // highlight-line
+   "location": "collect.handler:1",
+   "service": "payment",
+   "lambda_function_name": "test",
+   "lambda_function_memory_size": 128,
+   "lambda_function_arn": "arn:aws:lambda:eu-west-1:12345678910:function:test",
+   "lambda_request_id": "52fdfc07-2182-154f-163f-5f0f9a621d72",
+   "cold_start": true,
+   "sampling_rate": 0.1, // highlight-line
+   "message": "Verifying whether order_id is present"
+}
+
 {
    "timestamp": "2020-05-24 18:17:33,774",
    "level": "INFO",
@@ -260,6 +277,7 @@ def handler(event, context):
    "message": "Collecting payment"
 }
 ```
+
 </details>
 
 
@@ -305,7 +323,7 @@ This can be fixed by either ensuring both has the `service` value as `payment`,
 
 You might want to continue to use the same date formatting style, or override `location` to display the `package.function_name:line_number` as you previously had.
 
-Logger allows you to either change the format or suppress the following keys altogether at the initialization: `location`, `timestamp`, `level`, and `datefmt`
+Logger allows you to either change the format or suppress the following keys altogether at the initialization: `location`, `timestamp`, `level`, `xray_trace_id`, and `datefmt`
 
 ```python
 from aws_lambda_powertools import Logger
@@ -317,7 +335,7 @@ logger = Logger(stream=stdout, location="[%(funcName)s] %(module)s", datefmt="fa
 logger = Logger(stream=stdout, location=None) # highlight-line
 ```
 
-Alternatively, you can also change the order of the following log record keys via the `log_record_order` parameter: `level`, `location`, `message`, and `timestamp`
+Alternatively, you can also change the order of the following log record keys via the `log_record_order` parameter: `level`, `location`, `message`, `xray_trace_id`, and `timestamp`
 
 ```python
 from aws_lambda_powertools import Logger
@@ -358,3 +376,27 @@ except Exception:
 }
 ```
 </details>
+
+
+## Testing your code
+
+When unit testing your code that makes use of `inject_lambda_context` decorator, you need to pass a dummy Lambda Context, or else Logger will fail.
+
+This is a Pytest sample that provides the minimum information necessary for Logger to succeed:
+
+```python:title=fake_lambda_context_for_logger.py
+@pytest.fixture
+def lambda_context():
+    lambda_context = {
+        "function_name": "test",
+        "memory_limit_in_mb": 128,
+        "invoked_function_arn": "arn:aws:lambda:eu-west-1:809313241:function:test",
+        "aws_request_id": "52fdfc07-2182-154f-163f-5f0f9a621d72",
+    }
+
+    return namedtuple("LambdaContext", lambda_context.keys())(*lambda_context.values())
+
+def test_lambda_handler(lambda_handler, lambda_context):
+    test_event = {'test': 'event'}
+    lambda_handler(test_event, lambda_context) # this will now have a Context object populated
+```

docs/content/core/metrics.mdx

@@ -251,11 +251,37 @@ This has the advantage of keeping cold start metric separate from your applicati
 
 ## Testing your code
 
+### Environment variables
+
 Use `POWERTOOLS_METRICS_NAMESPACE` and `POWERTOOLS_SERVICE_NAME` env vars when unit testing your code to ensure metric namespace and dimension objects are created, and your code doesn't fail validation.
 
 ```bash:title=pytest_metric_namespace.sh
-
 POWERTOOLS_SERVICE_NAME="Example" POWERTOOLS_METRICS_NAMESPACE="Application" python -m pytest
 ```
 
-You can ignore this if you are explicitly setting namespace/default dimension by passing the `namespace` and `service` parameters when initializing Metrics: `metrics = Metrics(namespace=ApplicationName, service=ServiceName)`.
+If you prefer setting environment variable for specific tests, and are using Pytest, you can use [monkeypatch](https://docs.pytest.org/en/latest/monkeypatch.html) fixture:
+
+```python:title=pytest_env_var.py
+def test_namespace_env_var(monkeypatch):
+    # Set POWERTOOLS_METRICS_NAMESPACE before initializating Metrics
+    monkeypatch.setenv("POWERTOOLS_METRICS_NAMESPACE", namespace)
+
+    metrics = Metrics()
+    ...
+```
+
+> Ignore this, if you are explicitly setting namespace/default dimension via `namespace` and `service` parameters: `metrics = Metrics(namespace=ApplicationName, service=ServiceName)`
+
+### Clearing metrics
+
+`Metrics` keep metrics in memory across multiple instances. If you need to test this behaviour, you can use the following Pytest fixture to ensure metrics are reset incl. cold start:
+
+```python:title=pytest_metrics_reset_fixture.py
+@pytest.fixture(scope="function", autouse=True)
+def reset_metric_set():
+    # Clear out every metric data prior to every test
+    metrics = Metrics()
+    metrics.clear_metrics()
+    metrics_global.is_cold_start = True  # ensure each test has cold start
+    yield
+```