Merge branch 'develop' into pydantic

* develop:
  docs: Fix doc for log sampling (#135)
  fix(logging): Don't include `json_default` in logs (#132)
  chore: bump to 1.4.0
  docs: add Lambda Layer SAR App url and ARN
  fix: upgrade dot-prop, serialize-javascript
  fix heading error due to merge
  formatting for bash script
  add layer to docs and how to use it from SAR
  moved publish step to publish workflow after pypi push
  fix(ssm): Make decrypt an explicit option and refactoring (#123)
  change to eu-west-1 default region
  remove tmp release flag and set trigger to release published
  add overwrite flag for ssm
  add relase tag simulation
  more typos
  fix typo in branch trigger
  fix indent, yaml ...
  line endings

Author: heitorlessa

.github/workflows/publish.yml

@@ -56,6 +56,17 @@ jobs:
       env:
         PYPI_USERNAME: __token__
         PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
+    - name: publish lambda layer in SAR by triggering the internal codepipeline
+      run: |
+        aws ssm put-parameter --name "powertools-python-release-version" --value $RELEASE_TAG_VERSION --overwrite
+        aws codepipeline start-pipeline-execution --name ${{ secrets.CODEPIPELINE_NAME }}
+      env:
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        AWS_DEFAULT_REGION: eu-west-1
+        AWS_DEFAULT_OUTPUT: json
+
+
 
   sync_master:
     needs: upload

CHANGELOG.md

@@ -6,10 +6,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [1.4.0] - 2020-08-25
+
+### Added
+- **All**: Official Lambda Layer via [Serverless Application Repository](https://serverlessrepo.aws.amazon.com/applications/eu-west-1/057560766410/aws-lambda-powertools-python-layer)
+- **Tracer**: `capture_method` and `capture_lambda_handler` now support **capture_response=False** parameter to prevent Tracer to capture response as metadata to allow customers running Tracer with sensitive workloads
+
 ### Fixed
 - **Metrics**: Cold start metric is now completely separate from application metrics dimensions, making it easier and cheaper to visualize.
     - This is a breaking change if you were graphing/alerting on both application metrics with the same name to compensate this previous malfunctioning
     - Marked as bugfix as this is the intended behaviour since the beginning, as you shouldn't have the same application metric with different dimensions
+- **Utilities**: SSMProvider within Parameters utility now have decrypt and recursive parameters correctly defined to support autocompletion
 
 ### Added
 - **Tracer**: capture_lambda_handler and capture_method decorators now support `capture_response` parameter to not include function's response as part of tracing metadata

aws_lambda_powertools/logging/formatter.py

@@ -3,13 +3,13 @@
 from typing import Any
 
 
-def json_formatter(unserialized_value: Any):
-    """JSON custom serializer to cast unserialisable values to strings.
+def json_formatter(unserializable_value: Any):
+    """JSON custom serializer to cast unserializable values to strings.
 
     Example
     -------
 
-    **Serialize unserialisable value to string**
+    **Serialize unserializable value to string**
 
         class X: pass
         value = {"x": X()}
@@ -18,10 +18,10 @@ class X: pass
 
     Parameters
     ----------
-    unserialized_value: Any
+    unserializable_value: Any
         Python object unserializable by JSON
     """
-    return str(unserialized_value)
+    return str(unserializable_value)
 
 
 class JsonFormatter(logging.Formatter):
@@ -39,11 +39,12 @@ def __init__(self, **kwargs):
         """Return a JsonFormatter instance.
 
         The `json_default` kwarg is used to specify a formatter for otherwise
-        unserialisable values.  It must not throw.  Defaults to a function that
+        unserializable values.  It must not throw.  Defaults to a function that
         coerces the value to a string.
 
         Other kwargs are used to specify log field format strings.
         """
+        self.default_json_formatter = kwargs.pop("json_default", json_formatter)
         datefmt = kwargs.pop("datefmt", None)
 
         super(JsonFormatter, self).__init__(datefmt=datefmt)
@@ -54,7 +55,6 @@ def __init__(self, **kwargs):
             "location": "%(funcName)s:%(lineno)d",
         }
         self.format_dict.update(kwargs)
-        self.default_json_formatter = kwargs.pop("json_default", json_formatter)
 
     def update_formatter(self, **kwargs):
         self.format_dict.update(kwargs)
@@ -64,6 +64,7 @@ def format(self, record):  # noqa: A003
         record_dict["asctime"] = self.formatTime(record, self.datefmt)
 
         log_dict = {}
+
         for key, value in self.format_dict.items():
             if value and key in self.reserved_keys:
                 # converts default logging expr to its record value
@@ -84,19 +85,13 @@ def format(self, record):  # noqa: A003
             except (json.decoder.JSONDecodeError, TypeError, ValueError):
                 pass
 
-        if record.exc_info:
+        if record.exc_info and not record.exc_text:
             # Cache the traceback text to avoid converting it multiple times
             # (it's constant anyway)
             # from logging.Formatter:format
-            if not record.exc_text:  # pragma: no cover
-                record.exc_text = self.formatException(record.exc_info)
+            record.exc_text = self.formatException(record.exc_info)
 
         if record.exc_text:
             log_dict["exception"] = record.exc_text
 
-        json_record = json.dumps(log_dict, default=self.default_json_formatter)
-
-        if hasattr(json_record, "decode"):  # pragma: no cover
-            json_record = json_record.decode("utf-8")
-
-        return json_record
+        return json.dumps(log_dict, default=self.default_json_formatter)

aws_lambda_powertools/utilities/parameters/base.py

@@ -7,7 +7,7 @@
 from abc import ABC, abstractmethod
 from collections import namedtuple
 from datetime import datetime, timedelta
-from typing import Dict, Optional, Union
+from typing import Dict, Optional, Tuple, Union
 
 from .exceptions import GetParameterError, TransformParameterError
 
@@ -31,6 +31,9 @@ def __init__(self):
 
         self.store = {}
 
+    def _has_not_expired(self, key: Tuple[str, Optional[str]]) -> bool:
+        return key in self.store and self.store[key].ttl >= datetime.now()
+
     def get(
         self, name: str, max_age: int = DEFAULT_MAX_AGE_SECS, transform: Optional[str] = None, **sdk_options
     ) -> Union[str, list, dict, bytes]:
@@ -70,24 +73,26 @@ def get(
         # an acceptable tradeoff.
         key = (name, transform)
 
-        if key not in self.store or self.store[key].ttl < datetime.now():
-            try:
-                value = self._get(name, **sdk_options)
-            # Encapsulate all errors into a generic GetParameterError
-            except Exception as exc:
-                raise GetParameterError(str(exc))
+        if self._has_not_expired(key):
+            return self.store[key].value
+
+        try:
+            value = self._get(name, **sdk_options)
+        # Encapsulate all errors into a generic GetParameterError
+        except Exception as exc:
+            raise GetParameterError(str(exc))
 
-            if transform is not None:
-                value = transform_value(value, transform)
+        if transform is not None:
+            value = transform_value(value, transform)
 
-            self.store[key] = ExpirableValue(value, datetime.now() + timedelta(seconds=max_age),)
+        self.store[key] = ExpirableValue(value, datetime.now() + timedelta(seconds=max_age),)
 
-        return self.store[key].value
+        return value
 
     @abstractmethod
     def _get(self, name: str, **sdk_options) -> str:
         """
-        Retrieve paramater value from the underlying parameter store
+        Retrieve parameter value from the underlying parameter store
         """
         raise NotImplementedError()
 
@@ -129,29 +134,22 @@ def get_multiple(
 
         key = (path, transform)
 
-        if key not in self.store or self.store[key].ttl < datetime.now():
-            try:
-                values = self._get_multiple(path, **sdk_options)
-            # Encapsulate all errors into a generic GetParameterError
-            except Exception as exc:
-                raise GetParameterError(str(exc))
+        if self._has_not_expired(key):
+            return self.store[key].value
 
-            if transform is not None:
-                new_values = {}
-                for key, value in values.items():
-                    try:
-                        new_values[key] = transform_value(value, transform)
-                    except Exception as exc:
-                        if raise_on_transform_error:
-                            raise exc
-                        else:
-                            new_values[key] = None
+        try:
+            values: Dict[str, Union[str, bytes, dict, None]] = self._get_multiple(path, **sdk_options)
+        # Encapsulate all errors into a generic GetParameterError
+        except Exception as exc:
+            raise GetParameterError(str(exc))
 
-                values = new_values
+        if transform is not None:
+            for (key, value) in values.items():
+                values[key] = transform_value(value, transform, raise_on_transform_error)
 
-            self.store[key] = ExpirableValue(values, datetime.now() + timedelta(seconds=max_age),)
+        self.store[key] = ExpirableValue(values, datetime.now() + timedelta(seconds=max_age),)
 
-        return self.store[key].value
+        return values
 
     @abstractmethod
     def _get_multiple(self, path: str, **sdk_options) -> Dict[str, str]:
@@ -161,16 +159,19 @@ def _get_multiple(self, path: str, **sdk_options) -> Dict[str, str]:
         raise NotImplementedError()
 
 
-def transform_value(value: str, transform: str) -> Union[dict, bytes]:
+def transform_value(value: str, transform: str, raise_on_transform_error: bool = True) -> Union[dict, bytes, None]:
     """
     Apply a transform to a value
 
     Parameters
     ---------
     value: str
-        Parameter alue to transform
+        Parameter value to transform
     transform: str
         Type of transform, supported values are "json" and "binary"
+    raise_on_transform_error: bool, optional
+        Raises an exception if any transform fails, otherwise this will
+        return a None value for each transform that failed
 
     Raises
     ------
@@ -187,4 +188,6 @@ def transform_value(value: str, transform: str) -> Union[dict, bytes]:
             raise ValueError(f"Invalid transform type '{transform}'")
 
     except Exception as exc:
-        raise TransformParameterError(str(exc))
+        if raise_on_transform_error:
+            raise TransformParameterError(str(exc))
+        return None

aws_lambda_powertools/utilities/parameters/secrets.py

@@ -77,7 +77,7 @@ def _get(self, name: str, **sdk_options) -> str:
         ----------
         name: str
             Name of the parameter
-        sdk_options: dict
+        sdk_options: dict, optional
             Dictionary of options that will be passed to the Secrets Manager get_secret_value API call
         """
 

aws_lambda_powertools/utilities/parameters/ssm.py

@@ -8,7 +8,7 @@
 import boto3
 from botocore.config import Config
 
-from .base import DEFAULT_PROVIDERS, BaseProvider
+from .base import DEFAULT_MAX_AGE_SECS, DEFAULT_PROVIDERS, BaseProvider
 
 
 class SSMProvider(BaseProvider):
@@ -86,6 +86,46 @@ def __init__(
 
         super().__init__()
 
+    def get(
+        self,
+        name: str,
+        max_age: int = DEFAULT_MAX_AGE_SECS,
+        transform: Optional[str] = None,
+        decrypt: bool = False,
+        **sdk_options
+    ) -> Union[str, list, dict, bytes]:
+        """
+        Retrieve a parameter value or return the cached value
+
+        Parameters
+        ----------
+        name: str
+            Parameter name
+        max_age: int
+            Maximum age of the cached value
+        transform: str
+            Optional transformation of the parameter value. Supported values
+            are "json" for JSON strings and "binary" for base 64 encoded
+            values.
+        decrypt: bool, optional
+            If the parameter value should be decrypted
+        sdk_options: dict, optional
+            Arguments that will be passed directly to the underlying API call
+
+        Raises
+        ------
+        GetParameterError
+            When the parameter provider fails to retrieve a parameter value for
+            a given name.
+        TransformParameterError
+            When the parameter provider fails to transform a parameter value.
+        """
+
+        # Add to `decrypt` sdk_options to we can have an explicit option for this
+        sdk_options["decrypt"] = decrypt
+
+        return super().get(name, max_age, transform, **sdk_options)
+
     def _get(self, name: str, decrypt: bool = False, **sdk_options) -> str:
         """
         Retrieve a parameter value from AWS Systems Manager Parameter Store
@@ -144,7 +184,9 @@ def _get_multiple(self, path: str, decrypt: bool = False, recursive: bool = Fals
         return parameters
 
 
-def get_parameter(name: str, transform: Optional[str] = None, **sdk_options) -> Union[str, list, dict, bytes]:
+def get_parameter(
+    name: str, transform: Optional[str] = None, decrypt: bool = False, **sdk_options
+) -> Union[str, list, dict, bytes]:
     """
     Retrieve a parameter value from AWS Systems Manager (SSM) Parameter Store
 
@@ -154,6 +196,8 @@ def get_parameter(name: str, transform: Optional[str] = None, **sdk_options) ->
         Name of the parameter
     transform: str, optional
         Transforms the content from a JSON object ('json') or base64 binary string ('binary')
+    decrypt: bool, optional
+        If the parameter values should be decrypted
     sdk_options: dict, optional
         Dictionary of options that will be passed to the Parameter Store get_parameter API call
 
@@ -190,7 +234,10 @@ def get_parameter(name: str, transform: Optional[str] = None, **sdk_options) ->
     if "ssm" not in DEFAULT_PROVIDERS:
         DEFAULT_PROVIDERS["ssm"] = SSMProvider()
 
-    return DEFAULT_PROVIDERS["ssm"].get(name, transform=transform)
+    # Add to `decrypt` sdk_options to we can have an explicit option for this
+    sdk_options["decrypt"] = decrypt
+
+    return DEFAULT_PROVIDERS["ssm"].get(name, transform=transform, **sdk_options)
 
 
 def get_parameters(
@@ -205,10 +252,10 @@ def get_parameters(
         Path to retrieve the parameters
     transform: str, optional
         Transforms the content from a JSON object ('json') or base64 binary string ('binary')
-    decrypt: bool, optional
-        If the parameter values should be decrypted
     recursive: bool, optional
         If this should retrieve the parameter values recursively or not, defaults to True
+    decrypt: bool, optional
+        If the parameter values should be decrypted
     sdk_options: dict, optional
         Dictionary of options that will be passed to the Parameter Store get_parameters_by_path API call
 
@@ -245,4 +292,7 @@ def get_parameters(
     if "ssm" not in DEFAULT_PROVIDERS:
         DEFAULT_PROVIDERS["ssm"] = SSMProvider()
 
-    return DEFAULT_PROVIDERS["ssm"].get_multiple(path, transform=transform, recursive=recursive, decrypt=decrypt)
+    sdk_options["recursive"] = recursive
+    sdk_options["decrypt"] = decrypt
+
+    return DEFAULT_PROVIDERS["ssm"].get_multiple(path, transform=transform, **sdk_options)

docs/content/core/logger.mdx

@@ -55,7 +55,7 @@ Key | Type | Example | Description
 **level** | str | "INFO" | Logging level
 **location** | str | "collect.handler:1" | Source code location where statement was executed
 **service** | str | "payment" | Service name defined. "service_undefined" will be used if unknown
-**sampling_rate** | int |  0.1 | Debug logging sampling rate in percentage e.g. 1% in this case
+**sampling_rate** | int |  0.1 | Debug logging sampling rate in percentage e.g. 10% in this case
 **message** | any |  "Collecting payment" | Log statement value. Unserializable JSON values will be casted to string
 
 ## Capturing Lambda context info
@@ -232,7 +232,7 @@ Sampling calculation happens at the Logger class initialization. This means, whe
 ```python:title=collect.py
 from aws_lambda_powertools import Logger
 
-# Sample 1% of debug logs e.g. 0.1
+# Sample 10% of debug logs e.g. 0.1
 logger = Logger(sample_rate=0.1) # highlight-line
 
 def handler(event, context):