From 5d5b4b8f430098055bf457e136eaac27600b83f9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 11 Oct 2022 12:02:30 +0100 Subject: [PATCH 01/23] chore(deps-dev): bump flake8-builtins from 1.5.3 to 2.0.0 (#1582) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 10 +++++----- pyproject.toml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/poetry.lock b/poetry.lock index 8f13157b12b..e0b7c7e9c37 100644 --- a/poetry.lock +++ b/poetry.lock @@ -335,7 +335,7 @@ dev = ["coverage", "hypothesis", "hypothesmith (>=0.2)", "pre-commit"] [[package]] name = "flake8-builtins" -version = "1.5.3" +version = "2.0.0" description = "Check for python builtins being used as variables or parameters." category = "dev" optional = false @@ -345,7 +345,7 @@ python-versions = "*" flake8 = "*" [package.extras] -test = ["coverage", "coveralls", "mock", "pytest", "pytest-cov"] +test = ["pytest"] [[package]] name = "flake8-comprehensions" @@ -1383,7 +1383,7 @@ pydantic = ["pydantic", "email-validator"] [metadata] lock-version = "1.1" python-versions = "^3.6.2" -content-hash = "b6eba8ccb0bd0673dec8656d0fafa5aac520761f92cc152798c41883e3c92dca" +content-hash = "f7f2c132b9a5803a6dab1095a5fa4c039edbec684bd849e829d8ccbd49e2b547" [metadata.files] atomicwrites = [ @@ -1558,8 +1558,8 @@ flake8-bugbear = [ {file = "flake8_bugbear-22.9.23-py3-none-any.whl", hash = "sha256:cd2779b2b7ada212d7a322814a1e5651f1868ab0d3f24cc9da66169ab8fda474"}, ] flake8-builtins = [ - {file = "flake8-builtins-1.5.3.tar.gz", hash = "sha256:09998853b2405e98e61d2ff3027c47033adbdc17f9fe44ca58443d876eb00f3b"}, - {file = "flake8_builtins-1.5.3-py2.py3-none-any.whl", hash = "sha256:7706babee43879320376861897e5d1468e396a40b8918ed7bccf70e5f90b8687"}, + {file = "flake8-builtins-2.0.0.tar.gz", hash = "sha256:98833fa16139a75cd4913003492a9bd9a61c6f8ac146c3db12a2ebaf420dade3"}, + {file = "flake8_builtins-2.0.0-py3-none-any.whl", hash = "sha256:39bfa3badb5e8d22f92baf4e0ea1b816707245233846932d6b13e81fc6f673e8"}, ] flake8-comprehensions = [ {file = "flake8-comprehensions-3.7.0.tar.gz", hash = "sha256:6b3218b2dde8ac5959c6476cde8f41a79e823c22feb656be2710cd2a3232cef9"}, diff --git a/pyproject.toml b/pyproject.toml index eeb60fd0d4a..72505944c65 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ email-validator = {version = "*", optional = true } coverage = {extras = ["toml"], version = "^6.2"} pytest = "^7.0.1" black = "^22.8" -flake8-builtins = "^1.5.3" +flake8-builtins = "^2.0.0" flake8-comprehensions = "^3.7.0" flake8-debugger = "^4.0.0" flake8-fixme = "^1.1.1" From 154df7c519f0da770fa3b9082ea2eb47558aa65e Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Tue, 11 Oct 2022 16:12:15 +0200 Subject: [PATCH 02/23] docs(homepage): include .NET powertools --- README.md | 7 ++++--- docs/index.md | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c1845f43ce7..fb5fc480f37 100644 --- a/README.md +++ b/README.md @@ -3,10 +3,11 @@ [![Build](https://github.com/awslabs/aws-lambda-powertools-python/actions/workflows/python_build.yml/badge.svg)](https://github.com/awslabs/aws-lambda-powertools-python/actions/workflows/python_build.yml) [![codecov.io](https://codecov.io/github/awslabs/aws-lambda-powertools-python/branch/develop/graphs/badge.svg)](https://app.codecov.io/gh/awslabs/aws-lambda-powertools-python) -![PythonSupport](https://img.shields.io/static/v1?label=python&message=3.6%20|%203.7|%203.8|%203.9&color=blue?style=flat-square&logo=python) ![PyPI version](https://badge.fury.io/py/aws-lambda-powertools.svg) ![PyPi monthly downloads](https://img.shields.io/pypi/dm/aws-lambda-powertools) ![Lambda Layer](https://api.globadge.com/v1/badgen/aws/lambda/layer/latest-version/eu-central-1/017000801446/AWSLambdaPowertoolsPython) -[![Join our Discord](https://dcbadge.vercel.app/api/server/B8zZKbbyET)](https://discord.gg/B8zZKbbyET) +![PythonSupport](https://img.shields.io/static/v1?label=python&message=3.6%20|%203.7|%203.8|%203.9&color=blue?style=flat-square&logo=python) ![PyPI version](https://badge.fury.io/py/aws-lambda-powertools.svg) ![PyPi monthly downloads](https://img.shields.io/pypi/dm/aws-lambda-powertools) [![Join our Discord](https://dcbadge.vercel.app/api/server/B8zZKbbyET)](https://discord.gg/B8zZKbbyET) -A suite of Python utilities for AWS Lambda functions to ease adopting best practices such as tracing, structured logging, custom metrics, and more. (AWS Lambda Powertools [Java](https://github.com/awslabs/aws-lambda-powertools-java) and [Typescript](https://github.com/awslabs/aws-lambda-powertools-typescript) is also available). +A suite of Python utilities for AWS Lambda functions to ease adopting best practices such as tracing, structured logging, custom metrics, and more. + +> Also available in [Java](https://github.com/awslabs/aws-lambda-powertools-java), [Typescript](https://github.com/awslabs/aws-lambda-powertools-typescript), and [.NET](https://awslabs.github.io/aws-lambda-powertools-dotnet/). **[📜Documentation](https://awslabs.github.io/aws-lambda-powertools-python/)** | **[🐍PyPi](https://pypi.org/project/aws-lambda-powertools/)** | **[Roadmap](https://awslabs.github.io/aws-lambda-powertools-python/latest/roadmap/)** | **[Detailed blog post](https://aws.amazon.com/blogs/opensource/simplifying-serverless-best-practices-with-lambda-powertools/)** diff --git a/docs/index.md b/docs/index.md index 3ba70df740c..5a72514c311 100644 --- a/docs/index.md +++ b/docs/index.md @@ -8,7 +8,7 @@ description: AWS Lambda Powertools for Python A suite of utilities for AWS Lambda functions to ease adopting best practices such as tracing, structured logging, custom metrics, idempotency, batching, and more. ???+ note - Lambda Powertools is also available for [Java](https://awslabs.github.io/aws-lambda-powertools-java/){target="_blank"} and [TypeScript](https://awslabs.github.io/aws-lambda-powertools-typescript/latest/){target="_blank"}. + Lambda Powertools is also available for [Java](https://awslabs.github.io/aws-lambda-powertools-java/){target="_blank"}, [TypeScript](https://awslabs.github.io/aws-lambda-powertools-typescript/latest/){target="_blank"}, and [.NET](https://awslabs.github.io/aws-lambda-powertools-dotnet/){target="_blank"}. ## Install From 73865b30c5643bf6ed376f5a870dd369bc9f3004 Mon Sep 17 00:00:00 2001 From: Release bot Date: Tue, 11 Oct 2022 14:12:41 +0000 Subject: [PATCH 03/23] update changelog with latest changes --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef23f9d6c8f..65b2f79fde4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,9 +4,15 @@ # Unreleased +## Documentation + +* **homepage:** include .NET powertools + ## Maintenance +* add dummy v2 sar deploy job * bump layer version to 38 +* **deps-dev:** bump flake8-builtins from 1.5.3 to 2.0.0 ([#1582](https://github.com/awslabs/aws-lambda-powertools-python/issues/1582)) From ea28f04a86ea16a0fe1bc40a02bdac0d917492f7 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 12 Oct 2022 06:50:20 +0900 Subject: [PATCH 04/23] docs(logger): fix typo. (#1587) --- docs/core/logger.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/core/logger.md b/docs/core/logger.md index 4b16a1eeb71..f98962a0f5f 100644 --- a/docs/core/logger.md +++ b/docs/core/logger.md @@ -619,7 +619,7 @@ By default all registered loggers will be modified. You can change this behavior ### How can I add standard library logging attributes to a log record? -The Python standard library log records contains a [large set of atttributes](https://docs.python.org/3/library/logging.html#logrecord-attributes){target="_blank"}, however only a few are included in Powertools Logger log record by default. +The Python standard library log records contains a [large set of attributes](https://docs.python.org/3/library/logging.html#logrecord-attributes){target="_blank"}, however only a few are included in Powertools Logger log record by default. You can include any of these logging attributes as key value arguments (`kwargs`) when instantiating `Logger` or `LambdaPowertoolsFormatter`. From 263ba7b62451ba34d0f466f313fdf426e983a236 Mon Sep 17 00:00:00 2001 From: Release bot Date: Tue, 11 Oct 2022 21:50:47 +0000 Subject: [PATCH 05/23] update changelog with latest changes --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 65b2f79fde4..851ddd81906 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ ## Documentation * **homepage:** include .NET powertools +* **logger:** fix typo. ([#1587](https://github.com/awslabs/aws-lambda-powertools-python/issues/1587)) ## Maintenance From 5d755a7b30e4c8ae416f2018cb2f5db6a59f933e Mon Sep 17 00:00:00 2001 From: Heitor Lessa Date: Wed, 12 Oct 2022 12:02:37 +0200 Subject: [PATCH 06/23] docs(governance): new form to allow customers self-nominate as public reference (#1589) --- .github/ISSUE_TEMPLATE/support_powertools.yml | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/support_powertools.yml diff --git a/.github/ISSUE_TEMPLATE/support_powertools.yml b/.github/ISSUE_TEMPLATE/support_powertools.yml new file mode 100644 index 00000000000..551959d901d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/support_powertools.yml @@ -0,0 +1,64 @@ +name: Support Lambda Powertools (become a reference) +description: Add your organization's name or logo to the Lambda Powertools documentation +title: "[Support Lambda Powertools]: " +labels: ["customer_reference"] +body: + - type: markdown + attributes: + value: | + Thank you for becoming a reference customer. Your support means a lot to us. It also helps new customers to know who's using it. + + If you would like us to also display your organization's logo, please share a link in the `Company logo` field. + - type: input + id: organization + attributes: + label: Organization Name + description: Please share the name of your organization + placeholder: ACME + validations: + required: true + - type: input + id: name + attributes: + label: Your Name + description: Please share your name + validations: + required: true + - type: input + id: job + attributes: + label: Your current position + description: Please share your current position at your company + validations: + required: true + - type: input + id: logo + attributes: + label: (Optional) Company logo + description: Company logo you want us to display. You also allow us to resize for optimal placement in the documentation. + validations: + required: false + - type: textarea + id: use_case + attributes: + label: (Optional) Use case + description: How are you using Lambda Powertools today? *features, etc.* + validations: + required: false + - type: checkboxes + id: other_languages + attributes: + label: Also using other Lambda Powertools languages? + options: + - label: Java + required: false + - label: TypeScript + required: false + - label: .NET + required: false + - type: markdown + attributes: + value: | + *By raising a Support Lambda Powertools issue, you are granting AWS permission to use your company's name (and/or logo) for the limited purpose described here. You are also confirming that you have authority to grant such permission.* + + *You can opt-out at any time by commenting or reopening this issue.* From 99d53a71b5cb35924ba3dac640de60fb4b78e7e2 Mon Sep 17 00:00:00 2001 From: Steven Cook Date: Thu, 13 Oct 2022 02:20:56 +1000 Subject: [PATCH 07/23] docs(idempotency) - Update invalid link target. (#1588) --- docs/utilities/idempotency.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index 7ba61fd3062..90ba916e3d7 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -402,7 +402,7 @@ To prevent against extended failed retries when a [Lambda function times out](ht This means that if an invocation expired during execution, it will be quickly executed again on the next retry. ???+ important - If you are only using the [@idempotent_function decorator](#idempotentfunction-decorator) to guard isolated parts of your code, you must use `register_lambda_context` available in the [idempotency config object](#customizing-the-default-behavior) to benefit from this protection. + If you are only using the [@idempotent_function decorator](#idempotent_function-decorator) to guard isolated parts of your code, you must use `register_lambda_context` available in the [idempotency config object](#customizing-the-default-behavior) to benefit from this protection. Here is an example on how you register the Lambda context in your handler: From 0b9e2d10926bbb366d8b4727f07dfdbd22f9a6ed Mon Sep 17 00:00:00 2001 From: Release bot Date: Wed, 12 Oct 2022 16:21:16 +0000 Subject: [PATCH 08/23] update changelog with latest changes --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 851ddd81906..729659a58a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ ## Documentation +* **governance:** new form to allow customers self-nominate as public reference ([#1589](https://github.com/awslabs/aws-lambda-powertools-python/issues/1589)) * **homepage:** include .NET powertools * **logger:** fix typo. ([#1587](https://github.com/awslabs/aws-lambda-powertools-python/issues/1587)) From c9fd2f1965af3c9dbc2dee85b500973831035ac7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 13 Oct 2022 20:16:08 +0000 Subject: [PATCH 09/23] chore(deps-dev): bump mypy-boto3-ssm from 1.24.81 to 1.24.90 (#1594) Bumps [mypy-boto3-ssm](https://github.com/youtype/mypy_boto3_builder) from 1.24.81 to 1.24.90. - [Release notes](https://github.com/youtype/mypy_boto3_builder/releases) - [Commits](https://github.com/youtype/mypy_boto3_builder/commits) --- updated-dependencies: - dependency-name: mypy-boto3-ssm dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 22 +++++++++++----------- pyproject.toml | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/poetry.lock b/poetry.lock index e0b7c7e9c37..1d3756ad9fa 100644 --- a/poetry.lock +++ b/poetry.lock @@ -18,7 +18,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" dev = ["cloudpickle", "coverage[toml] (>=5.0.2)", "furo", "hypothesis", "mypy", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "six", "sphinx", "sphinx-notfound-page", "zope.interface"] docs = ["furo", "sphinx", "sphinx-notfound-page", "zope.interface"] tests = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "six", "zope.interface"] -tests_no_zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "six"] +tests-no-zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "six"] [[package]] name = "aws-cdk-lib" @@ -159,7 +159,7 @@ optional = false python-versions = ">=3.5.0" [package.extras] -unicode_backport = ["unicodedata2"] +unicode-backport = ["unicodedata2"] [[package]] name = "click" @@ -523,9 +523,9 @@ python-versions = ">=3.6.1,<4.0" [package.extras] colors = ["colorama (>=0.4.3,<0.5.0)"] -pipfile_deprecated_finder = ["pipreqs", "requirementslib"] +pipfile-deprecated-finder = ["pipreqs", "requirementslib"] plugins = ["setuptools"] -requirements_deprecated_finder = ["pip-api", "pipreqs"] +requirements-deprecated-finder = ["pip-api", "pipreqs"] [[package]] name = "jinja2" @@ -596,7 +596,7 @@ python-versions = "*" six = "*" [package.extras] -restructuredText = ["rst2ansi"] +restructuredtext = ["rst2ansi"] [[package]] name = "markdown" @@ -824,8 +824,8 @@ typing-extensions = ">=4.1.0" [[package]] name = "mypy-boto3-ssm" -version = "1.24.81" -description = "Type annotations for boto3.SSM 1.24.81 service generated with mypy-boto3-builder 7.11.9" +version = "1.24.90" +description = "Type annotations for boto3.SSM 1.24.90 service generated with mypy-boto3-builder 7.11.10" category = "dev" optional = false python-versions = ">=3.7" @@ -1186,7 +1186,7 @@ urllib3 = ">=1.21.1,<1.27" [package.extras] socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"] -use_chardet_on_py3 = ["chardet (>=3.0.2,<5)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<5)"] [[package]] name = "retry" @@ -1383,7 +1383,7 @@ pydantic = ["pydantic", "email-validator"] [metadata] lock-version = "1.1" python-versions = "^3.6.2" -content-hash = "f7f2c132b9a5803a6dab1095a5fa4c039edbec684bd849e829d8ccbd49e2b547" +content-hash = "f41f8a4ab4a0734540d6f34d1b99fe926df7f3d40cde90db062e9a0fb0a1baa0" [metadata.files] atomicwrites = [ @@ -1799,8 +1799,8 @@ mypy-boto3-secretsmanager = [ {file = "mypy_boto3_secretsmanager-1.24.83-py3-none-any.whl", hash = "sha256:9ed3ec38a6c05961cb39a2d9fb891441d4cf22c63e34a6998fbd3d28ba290d9a"}, ] mypy-boto3-ssm = [ - {file = "mypy-boto3-ssm-1.24.81.tar.gz", hash = "sha256:2b3167faa868442e43f0c6065fac8549762aafc967e487aae2d9e15c5bad20c3"}, - {file = "mypy_boto3_ssm-1.24.81-py3-none-any.whl", hash = "sha256:a50fe448f3c18f76255e15878e21020001ec04a85b42996db721d9b89770ff11"}, + {file = "mypy-boto3-ssm-1.24.90.tar.gz", hash = "sha256:8fdc65a34958ae89d4ae8ea7748caec46226216b35d75adf87e8ed40a798bf95"}, + {file = "mypy_boto3_ssm-1.24.90-py3-none-any.whl", hash = "sha256:6fc26896e1fb4f84f5bbc04f79ba698e4dd296586ca462c517bc64e78d326fb5"}, ] mypy-boto3-xray = [ {file = "mypy-boto3-xray-1.24.36.post1.tar.gz", hash = "sha256:104f1ecf7f1f6278c582201e71a7ab64843d3a3fdc8f23295cf68788cc77e9bb"}, diff --git a/pyproject.toml b/pyproject.toml index 72505944c65..5b0339d79e5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,7 +63,7 @@ mypy-boto3-dynamodb = { version = "^1.24.74", python = ">=3.7" } mypy-boto3-lambda = { version = "^1.24.0", python = ">=3.7" } mypy-boto3-logs = { version = "^1.24.0", python = ">=3.7" } mypy-boto3-secretsmanager = { version = "^1.24.83", python = ">=3.7" } -mypy-boto3-ssm = { version = "^1.24.81", python = ">=3.7" } +mypy-boto3-ssm = { version = "^1.24.90", python = ">=3.7" } mypy-boto3-s3 = { version = "^1.24.76", python = ">=3.7" } mypy-boto3-xray = { version = "^1.24.0", python = ">=3.7" } types-requests = "^2.28.11" From 74cf2cc15c330155ee1bd4c6c4154f8e6f05e440 Mon Sep 17 00:00:00 2001 From: Sen <63241411+senmm@users.noreply.github.com> Date: Fri, 14 Oct 2022 16:46:52 +0900 Subject: [PATCH 10/23] docs(idempotency): "persisntence" typo (#1596) --- docs/utilities/idempotency.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index 90ba916e3d7..4893bb76ab0 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -842,7 +842,7 @@ This utility provides an abstract base class (ABC), so that you can implement yo You can inherit from the `BasePersistenceLayer` class and implement the abstract methods `_get_record`, `_put_record`, `_update_record` and `_delete_record`. -```python hl_lines="8-13 57 65 74 96 124" title="Excerpt DynamoDB Persisntence Layer implementation for reference" +```python hl_lines="8-13 57 65 74 96 124" title="Excerpt DynamoDB Persistence Layer implementation for reference" import datetime import logging from typing import Any, Dict, Optional From 7dbc6da1ce872c5b1f24b8911e7e52b4e8b69025 Mon Sep 17 00:00:00 2001 From: Release bot Date: Fri, 14 Oct 2022 07:47:14 +0000 Subject: [PATCH 11/23] update changelog with latest changes --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 729659a58a6..a1b58c65376 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,12 +8,14 @@ * **governance:** new form to allow customers self-nominate as public reference ([#1589](https://github.com/awslabs/aws-lambda-powertools-python/issues/1589)) * **homepage:** include .NET powertools +* **idempotency:** "persisntence" typo ([#1596](https://github.com/awslabs/aws-lambda-powertools-python/issues/1596)) * **logger:** fix typo. ([#1587](https://github.com/awslabs/aws-lambda-powertools-python/issues/1587)) ## Maintenance * add dummy v2 sar deploy job * bump layer version to 38 +* **deps-dev:** bump mypy-boto3-ssm from 1.24.81 to 1.24.90 ([#1594](https://github.com/awslabs/aws-lambda-powertools-python/issues/1594)) * **deps-dev:** bump flake8-builtins from 1.5.3 to 2.0.0 ([#1582](https://github.com/awslabs/aws-lambda-powertools-python/issues/1582)) From e2d4744270a25ee74b4b5495735996293a1b2b96 Mon Sep 17 00:00:00 2001 From: Heitor Lessa Date: Fri, 14 Oct 2022 11:12:29 +0200 Subject: [PATCH 12/23] docs(governance): allow community to suggest feature content (#1593) --- .github/ISSUE_TEMPLATE/share_your_work.yml | 56 +++++++++++++++++++ .github/ISSUE_TEMPLATE/support_powertools.yml | 2 +- MAINTAINERS.md | 2 + 3 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 .github/ISSUE_TEMPLATE/share_your_work.yml diff --git a/.github/ISSUE_TEMPLATE/share_your_work.yml b/.github/ISSUE_TEMPLATE/share_your_work.yml new file mode 100644 index 00000000000..974aec87b06 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/share_your_work.yml @@ -0,0 +1,56 @@ +name: I Made This (showcase your work) +description: Share what you did with Powertools 💞💞. Blog post, workshops, presentation, sample apps, etc. +title: "[I Made This]: " +labels: ["community-content"] +body: + - type: markdown + attributes: + value: Thank you for helping spread the word out on Powertools, truly! + - type: input + id: content + attributes: + label: Link to your material + description: | + Please share the original link to your material. + + *Note: Short links will be expanded when added to Powertools documentation* + validations: + required: true + - type: textarea + id: description + attributes: + label: Description + description: Describe in one paragraph what's in it for them (readers) + validations: + required: true + - type: input + id: author + attributes: + label: Preferred contact + description: What's your preferred contact? We'll list it next to this content + validations: + required: true + - type: input + id: author-social + attributes: + label: (Optional) Social Network + description: If different from preferred contact, what's your preferred contact for social interactions? + validations: + required: false + - type: textarea + id: notes + attributes: + label: (Optional) Additional notes + description: | + Any notes you might want to share with us related to this material. + + *Note: These notes are explicitly to Powertools maintainers. It will not be added to the community resources page.* + validations: + required: false + - type: checkboxes + id: acknowledgment + attributes: + label: Acknowledgment + options: + - label: I understand this content may be removed from Powertools documentation if it doesn't conform with the [Code of Conduct](https://aws.github.io/code-of-conduct) + required: true diff --git a/.github/ISSUE_TEMPLATE/support_powertools.yml b/.github/ISSUE_TEMPLATE/support_powertools.yml index 551959d901d..e03b1627044 100644 --- a/.github/ISSUE_TEMPLATE/support_powertools.yml +++ b/.github/ISSUE_TEMPLATE/support_powertools.yml @@ -1,7 +1,7 @@ name: Support Lambda Powertools (become a reference) description: Add your organization's name or logo to the Lambda Powertools documentation title: "[Support Lambda Powertools]: <your organization name>" -labels: ["customer_reference"] +labels: ["customer-reference"] body: - type: markdown attributes: diff --git a/MAINTAINERS.md b/MAINTAINERS.md index fb94090f762..4e78aac2eb4 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -94,6 +94,8 @@ These are the most common labels used by maintainers to triage issues, pull requ | github-actions | Changes in GitHub workflows | PR automation | | github-templates | Changes in GitHub issue/PR templates | PR automation | | internal | Changes in governance, tech debt and chores (linting setup, baseline, etc.) | PR automation | +| customer-reference | Authorization to use company name in our documentation | Public Relations | +| community-content | Suggested content to feature in our documentation | Public Relations | ## Maintainer Responsibilities From abb8043fdcd7a88e3f0964dcf11353786e481505 Mon Sep 17 00:00:00 2001 From: Heitor Lessa <lessa@amazon.com> Date: Fri, 14 Oct 2022 16:26:02 +0200 Subject: [PATCH 13/23] fix(parser): loose validation on SNS fields to support FIFO (#1606) --- .../utilities/parser/models/sns.py | 6 ++--- tests/events/snsSqsFifoEvent.json | 23 +++++++++++++++++++ tests/functional/parser/test_sns.py | 18 ++++----------- 3 files changed, 31 insertions(+), 16 deletions(-) create mode 100644 tests/events/snsSqsFifoEvent.json diff --git a/aws_lambda_powertools/utilities/parser/models/sns.py b/aws_lambda_powertools/utilities/parser/models/sns.py index 1b095fde2c4..4666d1c4ff2 100644 --- a/aws_lambda_powertools/utilities/parser/models/sns.py +++ b/aws_lambda_powertools/utilities/parser/models/sns.py @@ -22,10 +22,10 @@ class SnsNotificationModel(BaseModel): MessageAttributes: Optional[Dict[str, SnsMsgAttributeModel]] Message: Union[str, TypingType[BaseModel]] MessageId: str - SigningCertUrl: HttpUrl - Signature: str + SigningCertUrl: Optional[HttpUrl] # NOTE: FIFO opt-in removes attribute + Signature: Optional[str] # NOTE: FIFO opt-in removes attribute Timestamp: datetime - SignatureVersion: str + SignatureVersion: Optional[str] # NOTE: FIFO opt-in removes attribute @root_validator(pre=True, allow_reuse=True) def check_sqs_protocol(cls, values): diff --git a/tests/events/snsSqsFifoEvent.json b/tests/events/snsSqsFifoEvent.json new file mode 100644 index 00000000000..6c23ef62945 --- /dev/null +++ b/tests/events/snsSqsFifoEvent.json @@ -0,0 +1,23 @@ +{ + "Records": [ + { + "messageId": "69bc4bbd-ed69-4325-a434-85c3b428ceab", + "receiptHandle": "AQEBbfAqjhrgIdW3HGWYPz57mdDatG/dT9LZhRPAsNQ1pJmw495w4esDc8ZSbOwMZuPBol7wtiNWug8U25GpSQDDLY1qv//8/lfmdzXOiprG6xRVeiXSHj0j731rJQ3xo+GPdGjOzjIxI09CrE3HtZ4lpXY9NjjHzP8hdxkCLlbttumc8hDBUR365/Tk+GfV2nNP9qvZtLGEbKCdTm/GYdTSoAr+ML9HnnGrS9T25Md71ASiZMI4DZqptN6g7CYYojFPs1LVM9o1258ferA72zbNoQ==", + "body": "{\n \"Type\" : \"Notification\",\n \"MessageId\" : \"a7c9d2fa-77fa-5184-9de9-89391027cc7d\",\n \"SequenceNumber\" : \"10000000000000004000\",\n \"TopicArn\" : \"arn:aws:sns:eu-west-1:231436140809:Test.fifo\",\n \"Message\" : \"{\\\"message\\\": \\\"hello world\\\", \\\"username\\\": \\\"lessa\\\"}\",\n \"Timestamp\" : \"2022-10-14T13:35:25.419Z\",\n \"UnsubscribeURL\" : \"https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:eu-west-1:231436140809:Test.fifo:bb81d3de-a0f9-46e4-b619-d3152a4d545f\"\n}", + "attributes": { + "ApproximateReceiveCount": "1", + "SentTimestamp": "1665754525442", + "SequenceNumber": "18873177232222703872", + "MessageGroupId": "powertools-test", + "SenderId": "AIDAWYJAWPFU7SUQGUJC6", + "MessageDeduplicationId": "4e0a0f61eed277a4b9e4c01d5722b07b0725e42fe782102abee5711adfac701f", + "ApproximateFirstReceiveTimestamp": "1665754525442" + }, + "messageAttributes": {}, + "md5OfBody": "f3c788e623445e3feb263e80c1bffc0b", + "eventSource": "aws:sqs", + "eventSourceARN": "arn:aws:sqs:eu-west-1:231436140809:Test.fifo", + "awsRegion": "eu-west-1" + } + ] +} \ No newline at end of file diff --git a/tests/functional/parser/test_sns.py b/tests/functional/parser/test_sns.py index 6042322e88a..10674c88ef5 100644 --- a/tests/functional/parser/test_sns.py +++ b/tests/functional/parser/test_sns.py @@ -110,19 +110,6 @@ def test_handle_sns_sqs_trigger_event_json_body(): # noqa: F811 handle_sns_sqs_json_body(event_dict, LambdaContext()) -def test_handle_sns_sqs_trigger_event_json_body_missing_signing_cert_url(): - # GIVEN an event is tampered with a missing SigningCertURL - event_dict = load_event("snsSqsEvent.json") - payload = json.loads(event_dict["Records"][0]["body"]) - payload.pop("SigningCertURL") - event_dict["Records"][0]["body"] = json.dumps(payload) - - # WHEN parsing the payload - # THEN raise a ValidationError error - with pytest.raises(ValidationError): - handle_sns_sqs_json_body(event_dict, LambdaContext()) - - def test_handle_sns_sqs_trigger_event_json_body_missing_unsubscribe_url(): # GIVEN an event is tampered with a missing UnsubscribeURL event_dict = load_event("snsSqsEvent.json") @@ -134,3 +121,8 @@ def test_handle_sns_sqs_trigger_event_json_body_missing_unsubscribe_url(): # THEN raise a ValidationError error with pytest.raises(ValidationError): handle_sns_sqs_json_body(event_dict, LambdaContext()) + + +def test_handle_sns_sqs_fifo_trigger_event_json_body(): + event_dict = load_event("snsSqsFifoEvent.json") + handle_sns_sqs_json_body(event_dict, LambdaContext()) From fb59763e43849324bdebfd73e66cd511af9abac5 Mon Sep 17 00:00:00 2001 From: Release bot <aws-devax-open-source@amazon.com> Date: Fri, 14 Oct 2022 14:37:05 +0000 Subject: [PATCH 14/23] update changelog with latest changes --- CHANGELOG.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a1b58c65376..0d165d610c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,16 @@ <a name="unreleased"></a> # Unreleased + +<a name="v1.31.1"></a> +## [v1.31.1] - 2022-10-14 +## Bug Fixes + +* **parser:** loose validation on SNS fields to support FIFO ([#1606](https://github.com/awslabs/aws-lambda-powertools-python/issues/1606)) + ## Documentation +* **governance:** allow community to suggest feature content ([#1593](https://github.com/awslabs/aws-lambda-powertools-python/issues/1593)) * **governance:** new form to allow customers self-nominate as public reference ([#1589](https://github.com/awslabs/aws-lambda-powertools-python/issues/1589)) * **homepage:** include .NET powertools * **idempotency:** "persisntence" typo ([#1596](https://github.com/awslabs/aws-lambda-powertools-python/issues/1596)) @@ -2418,7 +2426,8 @@ * Merge pull request [#5](https://github.com/awslabs/aws-lambda-powertools-python/issues/5) from jfuss/feat/python38 -[Unreleased]: https://github.com/awslabs/aws-lambda-powertools-python/compare/v1.31.0...HEAD +[Unreleased]: https://github.com/awslabs/aws-lambda-powertools-python/compare/v1.31.1...HEAD +[v1.31.1]: https://github.com/awslabs/aws-lambda-powertools-python/compare/v1.31.0...v1.31.1 [v1.31.0]: https://github.com/awslabs/aws-lambda-powertools-python/compare/v1.30.0...v1.31.0 [v1.30.0]: https://github.com/awslabs/aws-lambda-powertools-python/compare/v1.29.2...v1.30.0 [v1.29.2]: https://github.com/awslabs/aws-lambda-powertools-python/compare/v1.29.1...v1.29.2 From c98e03b598b6e033c984a3d8a954a13e8f340450 Mon Sep 17 00:00:00 2001 From: heitorlessa <lessa@amazon.co.uk> Date: Fri, 14 Oct 2022 16:48:35 +0200 Subject: [PATCH 15/23] chore(layer): bump to 1.31.1 (v39) --- docs/index.md | 60 +++++++++++++++++++++++++------------------------- pyproject.toml | 2 +- 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/docs/index.md b/docs/index.md index 5a72514c311..b0fd3f40ce5 100644 --- a/docs/index.md +++ b/docs/index.md @@ -14,7 +14,7 @@ A suite of utilities for AWS Lambda functions to ease adopting best practices su Powertools is available in the following formats: -* **Lambda Layer**: [**arn:aws:lambda:{region}:017000801446:layer:AWSLambdaPowertoolsPython:38**](#){: .copyMe}:clipboard: +* **Lambda Layer**: [**arn:aws:lambda:{region}:017000801446:layer:AWSLambdaPowertoolsPython:39**](#){: .copyMe}:clipboard: * **PyPi**: **`pip install aws-lambda-powertools`** ???+ hint "Support this project by using Lambda Layers :heart:" @@ -32,28 +32,28 @@ You can include Lambda Powertools Lambda Layer using [AWS Lambda Console](https: | Region | Layer ARN | | ---------------- | -------------------------------------------------------------------------------------------------------- | - | `af-south-1` | [arn:aws:lambda:af-south-1:017000801446:layer:AWSLambdaPowertoolsPython:38](#){: .copyMe}:clipboard: | - | `ap-east-1` | [arn:aws:lambda:ap-east-1:017000801446:layer:AWSLambdaPowertoolsPython:38](#){: .copyMe}:clipboard: | - | `ap-northeast-1` | [arn:aws:lambda:ap-northeast-1:017000801446:layer:AWSLambdaPowertoolsPython:38](#){: .copyMe}:clipboard: | - | `ap-northeast-2` | [arn:aws:lambda:ap-northeast-2:017000801446:layer:AWSLambdaPowertoolsPython:38](#){: .copyMe}:clipboard: | - | `ap-northeast-3` | [arn:aws:lambda:ap-northeast-3:017000801446:layer:AWSLambdaPowertoolsPython:38](#){: .copyMe}:clipboard: | - | `ap-south-1` | [arn:aws:lambda:ap-south-1:017000801446:layer:AWSLambdaPowertoolsPython:38](#){: .copyMe}:clipboard: | - | `ap-southeast-1` | [arn:aws:lambda:ap-southeast-1:017000801446:layer:AWSLambdaPowertoolsPython:38](#){: .copyMe}:clipboard: | - | `ap-southeast-2` | [arn:aws:lambda:ap-southeast-2:017000801446:layer:AWSLambdaPowertoolsPython:38](#){: .copyMe}:clipboard: | - | `ap-southeast-3` | [arn:aws:lambda:ap-southeast-3:017000801446:layer:AWSLambdaPowertoolsPython:38](#){: .copyMe}:clipboard: | - | `ca-central-1` | [arn:aws:lambda:ca-central-1:017000801446:layer:AWSLambdaPowertoolsPython:38](#){: .copyMe}:clipboard: | - | `eu-central-1` | [arn:aws:lambda:eu-central-1:017000801446:layer:AWSLambdaPowertoolsPython:38](#){: .copyMe}:clipboard: | - | `eu-north-1` | [arn:aws:lambda:eu-north-1:017000801446:layer:AWSLambdaPowertoolsPython:38](#){: .copyMe}:clipboard: | - | `eu-south-1` | [arn:aws:lambda:eu-south-1:017000801446:layer:AWSLambdaPowertoolsPython:38](#){: .copyMe}:clipboard: | - | `eu-west-1` | [arn:aws:lambda:eu-west-1:017000801446:layer:AWSLambdaPowertoolsPython:38](#){: .copyMe}:clipboard: | - | `eu-west-2` | [arn:aws:lambda:eu-west-2:017000801446:layer:AWSLambdaPowertoolsPython:38](#){: .copyMe}:clipboard: | - | `eu-west-3` | [arn:aws:lambda:eu-west-3:017000801446:layer:AWSLambdaPowertoolsPython:38](#){: .copyMe}:clipboard: | - | `me-south-1` | [arn:aws:lambda:me-south-1:017000801446:layer:AWSLambdaPowertoolsPython:38](#){: .copyMe}:clipboard: | - | `sa-east-1` | [arn:aws:lambda:sa-east-1:017000801446:layer:AWSLambdaPowertoolsPython:38](#){: .copyMe}:clipboard: | - | `us-east-1` | [arn:aws:lambda:us-east-1:017000801446:layer:AWSLambdaPowertoolsPython:38](#){: .copyMe}:clipboard: | - | `us-east-2` | [arn:aws:lambda:us-east-2:017000801446:layer:AWSLambdaPowertoolsPython:38](#){: .copyMe}:clipboard: | - | `us-west-1` | [arn:aws:lambda:us-west-1:017000801446:layer:AWSLambdaPowertoolsPython:38](#){: .copyMe}:clipboard: | - | `us-west-2` | [arn:aws:lambda:us-west-2:017000801446:layer:AWSLambdaPowertoolsPython:38](#){: .copyMe}:clipboard: | + | `af-south-1` | [arn:aws:lambda:af-south-1:017000801446:layer:AWSLambdaPowertoolsPython:39](#){: .copyMe}:clipboard: | + | `ap-east-1` | [arn:aws:lambda:ap-east-1:017000801446:layer:AWSLambdaPowertoolsPython:39](#){: .copyMe}:clipboard: | + | `ap-northeast-1` | [arn:aws:lambda:ap-northeast-1:017000801446:layer:AWSLambdaPowertoolsPython:39](#){: .copyMe}:clipboard: | + | `ap-northeast-2` | [arn:aws:lambda:ap-northeast-2:017000801446:layer:AWSLambdaPowertoolsPython:39](#){: .copyMe}:clipboard: | + | `ap-northeast-3` | [arn:aws:lambda:ap-northeast-3:017000801446:layer:AWSLambdaPowertoolsPython:39](#){: .copyMe}:clipboard: | + | `ap-south-1` | [arn:aws:lambda:ap-south-1:017000801446:layer:AWSLambdaPowertoolsPython:39](#){: .copyMe}:clipboard: | + | `ap-southeast-1` | [arn:aws:lambda:ap-southeast-1:017000801446:layer:AWSLambdaPowertoolsPython:39](#){: .copyMe}:clipboard: | + | `ap-southeast-2` | [arn:aws:lambda:ap-southeast-2:017000801446:layer:AWSLambdaPowertoolsPython:39](#){: .copyMe}:clipboard: | + | `ap-southeast-3` | [arn:aws:lambda:ap-southeast-3:017000801446:layer:AWSLambdaPowertoolsPython:39](#){: .copyMe}:clipboard: | + | `ca-central-1` | [arn:aws:lambda:ca-central-1:017000801446:layer:AWSLambdaPowertoolsPython:39](#){: .copyMe}:clipboard: | + | `eu-central-1` | [arn:aws:lambda:eu-central-1:017000801446:layer:AWSLambdaPowertoolsPython:39](#){: .copyMe}:clipboard: | + | `eu-north-1` | [arn:aws:lambda:eu-north-1:017000801446:layer:AWSLambdaPowertoolsPython:39](#){: .copyMe}:clipboard: | + | `eu-south-1` | [arn:aws:lambda:eu-south-1:017000801446:layer:AWSLambdaPowertoolsPython:39](#){: .copyMe}:clipboard: | + | `eu-west-1` | [arn:aws:lambda:eu-west-1:017000801446:layer:AWSLambdaPowertoolsPython:39](#){: .copyMe}:clipboard: | + | `eu-west-2` | [arn:aws:lambda:eu-west-2:017000801446:layer:AWSLambdaPowertoolsPython:39](#){: .copyMe}:clipboard: | + | `eu-west-3` | [arn:aws:lambda:eu-west-3:017000801446:layer:AWSLambdaPowertoolsPython:39](#){: .copyMe}:clipboard: | + | `me-south-1` | [arn:aws:lambda:me-south-1:017000801446:layer:AWSLambdaPowertoolsPython:39](#){: .copyMe}:clipboard: | + | `sa-east-1` | [arn:aws:lambda:sa-east-1:017000801446:layer:AWSLambdaPowertoolsPython:39](#){: .copyMe}:clipboard: | + | `us-east-1` | [arn:aws:lambda:us-east-1:017000801446:layer:AWSLambdaPowertoolsPython:39](#){: .copyMe}:clipboard: | + | `us-east-2` | [arn:aws:lambda:us-east-2:017000801446:layer:AWSLambdaPowertoolsPython:39](#){: .copyMe}:clipboard: | + | `us-west-1` | [arn:aws:lambda:us-west-1:017000801446:layer:AWSLambdaPowertoolsPython:39](#){: .copyMe}:clipboard: | + | `us-west-2` | [arn:aws:lambda:us-west-2:017000801446:layer:AWSLambdaPowertoolsPython:39](#){: .copyMe}:clipboard: | ??? question "Can't find our Lambda Layer for your preferred AWS region?" You can use [Serverless Application Repository (SAR)](#sar) method, our [CDK Layer Construct](https://github.com/aws-samples/cdk-lambda-powertools-python-layer){target="_blank"}, or PyPi like you normally would for any other library. @@ -67,7 +67,7 @@ You can include Lambda Powertools Lambda Layer using [AWS Lambda Console](https: Type: AWS::Serverless::Function Properties: Layers: - - !Sub arn:aws:lambda:${AWS::Region}:017000801446:layer:AWSLambdaPowertoolsPython:38 + - !Sub arn:aws:lambda:${AWS::Region}:017000801446:layer:AWSLambdaPowertoolsPython:39 ``` === "Serverless framework" @@ -77,7 +77,7 @@ You can include Lambda Powertools Lambda Layer using [AWS Lambda Console](https: hello: handler: lambda_function.lambda_handler layers: - - arn:aws:lambda:${aws:region}:017000801446:layer:AWSLambdaPowertoolsPython:38 + - arn:aws:lambda:${aws:region}:017000801446:layer:AWSLambdaPowertoolsPython:39 ``` === "CDK" @@ -93,7 +93,7 @@ You can include Lambda Powertools Lambda Layer using [AWS Lambda Console](https: powertools_layer = aws_lambda.LayerVersion.from_layer_version_arn( self, id="lambda-powertools", - layer_version_arn=f"arn:aws:lambda:{env.region}:017000801446:layer:AWSLambdaPowertoolsPython:38" + layer_version_arn=f"arn:aws:lambda:{env.region}:017000801446:layer:AWSLambdaPowertoolsPython:39" ) aws_lambda.Function(self, 'sample-app-lambda', @@ -142,7 +142,7 @@ You can include Lambda Powertools Lambda Layer using [AWS Lambda Console](https: role = aws_iam_role.iam_for_lambda.arn handler = "index.test" runtime = "python3.9" - layers = ["arn:aws:lambda:{region}:017000801446:layer:AWSLambdaPowertoolsPython:38"] + layers = ["arn:aws:lambda:{region}:017000801446:layer:AWSLambdaPowertoolsPython:39"] source_code_hash = filebase64sha256("lambda_function_payload.zip") } @@ -161,7 +161,7 @@ You can include Lambda Powertools Lambda Layer using [AWS Lambda Console](https: ? Do you want to configure advanced settings? Yes ... ? Do you want to enable Lambda layers for this function? Yes - ? Enter up to 5 existing Lambda layer ARNs (comma-separated): arn:aws:lambda:eu-central-1:017000801446:layer:AWSLambdaPowertoolsPython:38 + ? Enter up to 5 existing Lambda layer ARNs (comma-separated): arn:aws:lambda:eu-central-1:017000801446:layer:AWSLambdaPowertoolsPython:39 ❯ amplify push -y @@ -172,7 +172,7 @@ You can include Lambda Powertools Lambda Layer using [AWS Lambda Console](https: - Name: <NAME-OF-FUNCTION> ? Which setting do you want to update? Lambda layers configuration ? Do you want to enable Lambda layers for this function? Yes - ? Enter up to 5 existing Lambda layer ARNs (comma-separated): arn:aws:lambda:eu-central-1:017000801446:layer:AWSLambdaPowertoolsPython:38 + ? Enter up to 5 existing Lambda layer ARNs (comma-separated): arn:aws:lambda:eu-central-1:017000801446:layer:AWSLambdaPowertoolsPython:39 ? Do you want to edit the local lambda function now? No ``` @@ -180,7 +180,7 @@ You can include Lambda Powertools Lambda Layer using [AWS Lambda Console](https: Change {region} to your AWS region, e.g. `eu-west-1` ```bash title="AWS CLI" - aws lambda get-layer-version-by-arn --arn arn:aws:lambda:{region}:017000801446:layer:AWSLambdaPowertoolsPython:38 --region {region} + aws lambda get-layer-version-by-arn --arn arn:aws:lambda:{region}:017000801446:layer:AWSLambdaPowertoolsPython:39 --region {region} ``` The pre-signed URL to download this Lambda Layer will be within `Location` key. diff --git a/pyproject.toml b/pyproject.toml index 5b0339d79e5..612c44d4028 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "aws_lambda_powertools" -version = "1.31.0" +version = "1.31.1" description = "A suite of utilities for AWS Lambda functions to ease adopting best practices such as tracing, structured logging, custom metrics, batching, idempotency, feature flags, and more." authors = ["Amazon Web Services"] include = ["aws_lambda_powertools/py.typed", "THIRD-PARTY-LICENSES"] From ec709c3c57b865d3062acaca5ee443480fa9b105 Mon Sep 17 00:00:00 2001 From: Release bot <aws-devax-open-source@amazon.com> Date: Fri, 14 Oct 2022 14:48:59 +0000 Subject: [PATCH 16/23] update changelog with latest changes --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d165d610c9..005fcb1036b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ <a name="unreleased"></a> # Unreleased +## Maintenance + +* **layer:** bump to 1.31.1 (v39) + <a name="v1.31.1"></a> ## [v1.31.1] - 2022-10-14 From 416ab1b32ebf17a427ac428051acd11432f7cc95 Mon Sep 17 00:00:00 2001 From: ryandeivert <ryandeivert@gmail.com> Date: Mon, 17 Oct 2022 10:30:02 -0700 Subject: [PATCH 17/23] feat(data_classes): add KinesisFirehoseEvent (#1540) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Rúben Fonseca <fonseka@gmail.com> Co-authored-by: Leandro Damascena <leandro.damascena@gmail.com> --- .../utilities/data_classes/__init__.py | 2 + .../data_classes/kinesis_firehose_event.py | 113 ++++++++++++++++++ docs/utilities/data_classes.md | 15 +++ .../src/kinesis_firehose_delivery_stream.py | 25 ++++ tests/events/kinesisFirehosePutEvent.json | 14 +-- tests/functional/test_data_classes.py | 67 +++++++++++ 6 files changed, 229 insertions(+), 7 deletions(-) create mode 100644 aws_lambda_powertools/utilities/data_classes/kinesis_firehose_event.py create mode 100644 examples/event_sources/src/kinesis_firehose_delivery_stream.py diff --git a/aws_lambda_powertools/utilities/data_classes/__init__.py b/aws_lambda_powertools/utilities/data_classes/__init__.py index 8ed77f9f3a3..2aa2021ed1e 100644 --- a/aws_lambda_powertools/utilities/data_classes/__init__.py +++ b/aws_lambda_powertools/utilities/data_classes/__init__.py @@ -13,6 +13,7 @@ from .event_bridge_event import EventBridgeEvent from .event_source import event_source from .kafka_event import KafkaEvent +from .kinesis_firehose_event import KinesisFirehoseEvent from .kinesis_stream_event import KinesisStreamEvent from .lambda_function_url_event import LambdaFunctionUrlEvent from .s3_event import S3Event @@ -32,6 +33,7 @@ "DynamoDBStreamEvent", "EventBridgeEvent", "KafkaEvent", + "KinesisFirehoseEvent", "KinesisStreamEvent", "LambdaFunctionUrlEvent", "S3Event", diff --git a/aws_lambda_powertools/utilities/data_classes/kinesis_firehose_event.py b/aws_lambda_powertools/utilities/data_classes/kinesis_firehose_event.py new file mode 100644 index 00000000000..5683902f9d0 --- /dev/null +++ b/aws_lambda_powertools/utilities/data_classes/kinesis_firehose_event.py @@ -0,0 +1,113 @@ +import base64 +import json +from typing import Iterator, Optional + +from aws_lambda_powertools.utilities.data_classes.common import DictWrapper + + +class KinesisFirehoseRecordMetadata(DictWrapper): + @property + def _metadata(self) -> dict: + """Optional: metadata associated with this record; present only when Kinesis Stream is source""" + return self["kinesisRecordMetadata"] # could raise KeyError + + @property + def shard_id(self) -> str: + """Kinesis stream shard ID; present only when Kinesis Stream is source""" + return self._metadata["shardId"] + + @property + def partition_key(self) -> str: + """Kinesis stream partition key; present only when Kinesis Stream is source""" + return self._metadata["partitionKey"] + + @property + def approximate_arrival_timestamp(self) -> int: + """Kinesis stream approximate arrival ISO timestamp; present only when Kinesis Stream is source""" + return self._metadata["approximateArrivalTimestamp"] + + @property + def sequence_number(self) -> str: + """Kinesis stream sequence number; present only when Kinesis Stream is source""" + return self._metadata["sequenceNumber"] + + @property + def subsequence_number(self) -> str: + """Kinesis stream sub-sequence number; present only when Kinesis Stream is source + + Note: this will only be present for Kinesis streams using record aggregation + """ + return self._metadata["subsequenceNumber"] + + +class KinesisFirehoseRecord(DictWrapper): + @property + def approximate_arrival_timestamp(self) -> int: + """The approximate time that the record was inserted into the delivery stream""" + return self["approximateArrivalTimestamp"] + + @property + def record_id(self) -> str: + """Record ID; uniquely identifies this record within the current batch""" + return self["recordId"] + + @property + def data(self) -> str: + """The data blob, base64-encoded""" + return self["data"] + + @property + def metadata(self) -> Optional[KinesisFirehoseRecordMetadata]: + """Optional: metadata associated with this record; present only when Kinesis Stream is source""" + return KinesisFirehoseRecordMetadata(self._data) if self.get("kinesisRecordMetadata") else None + + @property + def data_as_bytes(self) -> bytes: + """Decoded base64-encoded data as bytes""" + return base64.b64decode(self.data) + + @property + def data_as_text(self) -> str: + """Decoded base64-encoded data as text""" + return self.data_as_bytes.decode("utf-8") + + @property + def data_as_json(self) -> dict: + """Decoded base64-encoded data loaded to json""" + if self._json_data is None: + self._json_data = json.loads(self.data_as_text) + return self._json_data + + +class KinesisFirehoseEvent(DictWrapper): + """Kinesis Data Firehose event + + Documentation: + -------------- + - https://docs.aws.amazon.com/lambda/latest/dg/services-kinesisfirehose.html + """ + + @property + def invocation_id(self) -> str: + """Unique ID for for Lambda invocation""" + return self["invocationId"] + + @property + def delivery_stream_arn(self) -> str: + """ARN of the Firehose Data Firehose Delivery Stream""" + return self["deliveryStreamArn"] + + @property + def source_kinesis_stream_arn(self) -> Optional[str]: + """ARN of the Kinesis Stream; present only when Kinesis Stream is source""" + return self.get("sourceKinesisStreamArn") + + @property + def region(self) -> str: + """AWS region where the event originated eg: us-east-1""" + return self["region"] + + @property + def records(self) -> Iterator[KinesisFirehoseRecord]: + for record in self["records"]: + yield KinesisFirehoseRecord(record) diff --git a/docs/utilities/data_classes.md b/docs/utilities/data_classes.md index 67d821fe04f..509110e0480 100644 --- a/docs/utilities/data_classes.md +++ b/docs/utilities/data_classes.md @@ -77,6 +77,7 @@ Event Source | Data_class [EventBridge](#eventbridge) | `EventBridgeEvent` [Kafka](#kafka) | `KafkaEvent` [Kinesis Data Stream](#kinesis-streams) | `KinesisStreamEvent` +[Kinesis Firehose Delivery Stream](#kinesis-firehose-delivery-stream) | `KinesisFirehoseEvent` [Lambda Function URL](#lambda-function-url) | `LambdaFunctionUrlEvent` [Rabbit MQ](#rabbit-mq) | `RabbitMQEvent` [S3](#s3) | `S3Event` @@ -892,6 +893,20 @@ or plain text, depending on the original payload. do_something_with(data) ``` +### Kinesis Firehose delivery stream + +Kinesis Firehose Data Transformation can use a Lambda Function to modify the records +inline, and re-emit them back to the Delivery Stream. + +Similar to Kinesis Data Streams, the events contain base64 encoded data. You can use the helper +function to access the data either as json or plain text, depending on the original payload. + +=== "app.py" + + ```python + --8<-- "examples/event_sources/src/kinesis_firehose_delivery_stream.py" + ``` + ### Lambda Function URL === "app.py" diff --git a/examples/event_sources/src/kinesis_firehose_delivery_stream.py b/examples/event_sources/src/kinesis_firehose_delivery_stream.py new file mode 100644 index 00000000000..67bf53dfe06 --- /dev/null +++ b/examples/event_sources/src/kinesis_firehose_delivery_stream.py @@ -0,0 +1,25 @@ +import base64 +import json + +from aws_lambda_powertools.utilities.data_classes import KinesisFirehoseEvent, event_source +from aws_lambda_powertools.utilities.typing import LambdaContext + + +@event_source(data_class=KinesisFirehoseEvent) +def lambda_handler(event: KinesisFirehoseEvent, context: LambdaContext): + result = [] + + for record in event.records: + # if data was delivered as json; caches loaded value + data = record.data_as_json + + processed_record = { + "recordId": record.record_id, + "data": base64.b64encode(json.dumps(data).encode("utf-8")), + "result": "Ok", + } + + result.append(processed_record) + + # return transformed records + return {"records": result} diff --git a/tests/events/kinesisFirehosePutEvent.json b/tests/events/kinesisFirehosePutEvent.json index 27aeddd80eb..f3e07190710 100644 --- a/tests/events/kinesisFirehosePutEvent.json +++ b/tests/events/kinesisFirehosePutEvent.json @@ -2,16 +2,16 @@ "invocationId": "2b4d1ad9-2f48-94bd-a088-767c317e994a", "deliveryStreamArn": "arn:aws:firehose:us-east-2:123456789012:deliverystream/delivery-stream-name", "region": "us-east-2", - "records":[ + "records": [ { - "recordId":"record1", - "approximateArrivalTimestamp":1664029185290, - "data":"SGVsbG8gV29ybGQ=" + "recordId": "record1", + "approximateArrivalTimestamp": 1664029185290, + "data": "SGVsbG8gV29ybGQ=" }, { - "recordId":"record2", - "approximateArrivalTimestamp":1664029186945, - "data":"eyJIZWxsbyI6ICJXb3JsZCJ9" + "recordId": "record2", + "approximateArrivalTimestamp": 1664029186945, + "data": "eyJIZWxsbyI6ICJXb3JsZCJ9" } ] } diff --git a/tests/functional/test_data_classes.py b/tests/functional/test_data_classes.py index 1f8c0cef955..235a3f8f8da 100644 --- a/tests/functional/test_data_classes.py +++ b/tests/functional/test_data_classes.py @@ -18,6 +18,7 @@ CodePipelineJobEvent, EventBridgeEvent, KafkaEvent, + KinesisFirehoseEvent, KinesisStreamEvent, S3Event, SESEvent, @@ -1239,6 +1240,72 @@ def test_kafka_self_managed_event(): assert record.get_header_value("HeaderKey", case_sensitive=False) == b"headerValue" +def test_kinesis_firehose_kinesis_event(): + event = KinesisFirehoseEvent(load_event("kinesisFirehoseKinesisEvent.json")) + + assert event.region == "us-east-2" + assert event.invocation_id == "2b4d1ad9-2f48-94bd-a088-767c317e994a" + assert event.delivery_stream_arn == "arn:aws:firehose:us-east-2:123456789012:deliverystream/delivery-stream-name" + assert event.source_kinesis_stream_arn == "arn:aws:kinesis:us-east-1:123456789012:stream/kinesis-source" + + records = list(event.records) + assert len(records) == 2 + record_01, record_02 = records[:] + + assert record_01.approximate_arrival_timestamp == 1664028820148 + assert record_01.record_id == "record1" + assert record_01.data == "SGVsbG8gV29ybGQ=" + assert record_01.data_as_bytes == b"Hello World" + assert record_01.data_as_text == "Hello World" + + assert record_01.metadata.shard_id == "shardId-000000000000" + assert record_01.metadata.partition_key == "4d1ad2b9-24f8-4b9d-a088-76e9947c317a" + assert record_01.metadata.approximate_arrival_timestamp == 1664028820148 + assert record_01.metadata.sequence_number == "49546986683135544286507457936321625675700192471156785154" + assert record_01.metadata.subsequence_number == "" + + assert record_02.approximate_arrival_timestamp == 1664028793294 + assert record_02.record_id == "record2" + assert record_02.data == "eyJIZWxsbyI6ICJXb3JsZCJ9" + assert record_02.data_as_bytes == b'{"Hello": "World"}' + assert record_02.data_as_text == '{"Hello": "World"}' + assert record_02.data_as_json == {"Hello": "World"} + + assert record_02.metadata.shard_id == "shardId-000000000001" + assert record_02.metadata.partition_key == "4d1ad2b9-24f8-4b9d-a088-76e9947c318a" + assert record_02.metadata.approximate_arrival_timestamp == 1664028793294 + assert record_02.metadata.sequence_number == "49546986683135544286507457936321625675700192471156785155" + assert record_02.metadata.subsequence_number == "" + + +def test_kinesis_firehose_put_event(): + event = KinesisFirehoseEvent(load_event("kinesisFirehosePutEvent.json")) + + assert event.region == "us-east-2" + assert event.invocation_id == "2b4d1ad9-2f48-94bd-a088-767c317e994a" + assert event.delivery_stream_arn == "arn:aws:firehose:us-east-2:123456789012:deliverystream/delivery-stream-name" + assert event.source_kinesis_stream_arn is None + + records = list(event.records) + assert len(records) == 2 + record_01, record_02 = records[:] + + assert record_01.approximate_arrival_timestamp == 1664029185290 + assert record_01.record_id == "record1" + assert record_01.data == "SGVsbG8gV29ybGQ=" + assert record_01.data_as_bytes == b"Hello World" + assert record_01.data_as_text == "Hello World" + assert record_01.metadata is None + + assert record_02.approximate_arrival_timestamp == 1664029186945 + assert record_02.record_id == "record2" + assert record_02.data == "eyJIZWxsbyI6ICJXb3JsZCJ9" + assert record_02.data_as_bytes == b'{"Hello": "World"}' + assert record_02.data_as_text == '{"Hello": "World"}' + assert record_02.data_as_json == {"Hello": "World"} + assert record_02.metadata is None + + def test_kinesis_stream_event(): event = KinesisStreamEvent(load_event("kinesisStreamEvent.json")) From fa2260a267c49f364494a3e7371442bada9a194e Mon Sep 17 00:00:00 2001 From: Release bot <aws-devax-open-source@amazon.com> Date: Mon, 17 Oct 2022 17:30:21 +0000 Subject: [PATCH 18/23] update changelog with latest changes --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 005fcb1036b..637206e98e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ <a name="unreleased"></a> # Unreleased +## Features + +* **data_classes:** add KinesisFirehoseEvent ([#1540](https://github.com/awslabs/aws-lambda-powertools-python/issues/1540)) + ## Maintenance * **layer:** bump to 1.31.1 (v39) From 97bc09697f7e1e5acb040a727d7f86cc2e71cac8 Mon Sep 17 00:00:00 2001 From: Ahmed Aboshanab <shanab@hey.com> Date: Mon, 10 Oct 2022 08:01:40 +0000 Subject: [PATCH 19/23] Fix spacing --- aws_lambda_powertools/utilities/data_classes/event_source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_lambda_powertools/utilities/data_classes/event_source.py b/aws_lambda_powertools/utilities/data_classes/event_source.py index 3968f923573..37249d4a99f 100644 --- a/aws_lambda_powertools/utilities/data_classes/event_source.py +++ b/aws_lambda_powertools/utilities/data_classes/event_source.py @@ -34,6 +34,6 @@ def event_source( @event_source(data_class=S3Event) def handler(event: S3Event, context): - return {"key": event.object_key} + return {"key": event.object_key} """ return handler(data_class(event), context) From 68704d61c0f49435bb8defdd3ab19046041dd3ec Mon Sep 17 00:00:00 2001 From: Ahmed Aboshanab <shanab@hey.com> Date: Mon, 10 Oct 2022 10:34:26 +0000 Subject: [PATCH 20/23] feat(data-classes): add DynamoDBImageDeserializer --- .../data_classes/dynamodb/__init__.py | 1 + .../dynamodb/dynamodb_image_deserializer.py | 105 ++++++++++++++++++ tests/functional/test_data_classes.py | 45 ++++++++ 3 files changed, 151 insertions(+) create mode 100644 aws_lambda_powertools/utilities/data_classes/dynamodb/__init__.py create mode 100644 aws_lambda_powertools/utilities/data_classes/dynamodb/dynamodb_image_deserializer.py diff --git a/aws_lambda_powertools/utilities/data_classes/dynamodb/__init__.py b/aws_lambda_powertools/utilities/data_classes/dynamodb/__init__.py new file mode 100644 index 00000000000..0271f476222 --- /dev/null +++ b/aws_lambda_powertools/utilities/data_classes/dynamodb/__init__.py @@ -0,0 +1 @@ +from .dynamodb_image_deserializer import DynamoDBImageDeserializer # noqa: F401 diff --git a/aws_lambda_powertools/utilities/data_classes/dynamodb/dynamodb_image_deserializer.py b/aws_lambda_powertools/utilities/data_classes/dynamodb/dynamodb_image_deserializer.py new file mode 100644 index 00000000000..4b59ebd94d4 --- /dev/null +++ b/aws_lambda_powertools/utilities/data_classes/dynamodb/dynamodb_image_deserializer.py @@ -0,0 +1,105 @@ +from typing import Dict, List, Optional, Union + +from aws_lambda_powertools.utilities.data_classes.dynamo_db_stream_event import ( + AttributeValue, +) + + +class DynamoDBImageDeserializer: + """ + Deserializes DynamoDB StreamRecord's old_image and new_image properties of type Dict[str, AttributeValue] + to Dict with Python types. + + Example + ------- + + from aws_lambda_powertools.utilities.typing import LambdaContext + from aws_lambda_powertools.utilities.data_classes import event_source, DynamoDBStreamEvent + from aws_lambda_powertools.utilities.data_classes.dynamodb import DynamoDBImageDeserializer + + deserializer = DynamoDBImageDeserializer() + + @event_source(data_class=DynamoDBStreamEvent) + def handler(event: DynamoDBStreamEvent, context: LambdaContext): + for record in event.records: + new_image = deserializer.deserialize(record.dynamodb.new_image) + old_image = deserializer.deserialize(record.dynamodb.old_image) + """ + + def deserialize(self, image: Dict[str, AttributeValue]): + """ + Deserializes image to Dict with Python types. + AttributeValue fields are deserialized according to the following table: + + DynamoDB Python + -------- ------ + {'NULL': True} None + {'BOOL': True/False} True/False + {'N': string} string + {'S': string} string + {'B': bytes} bytes + {'NS': [string]} set([string]) + {'SS': [string]} set([string]) + {'BS': [bytes]} set([bytes]) + {'L': list} list + {'M': dict} dict + + Please bear in mind that numbers are represented as strings. + + Parameters + ---------- + image: Dict[str, AttributeValue] + Dictionary representing a StreamRecord's image property + + Returns + ------- + Dict + Dictionary with Python types + + Raises + ------ + TypeError + Raised if an unknown DynamoDB attribute type is encountered + """ + return {k: self._deserialize_attr_value(v) for k, v in image.items()} + + def _deserialize_attr_value( + self, attr_value: AttributeValue + ) -> Union[Optional[bool], Optional[str], Optional[List], Optional[Dict]]: + try: + dynamodb_type = attr_value.dynamodb_type.lower() + deserializer = getattr(self, "_deserialize_%s" % dynamodb_type) + except AttributeError: + raise TypeError("DynamoDB type %s is not supported" % dynamodb_type) + return deserializer(attr_value) + + def _deserialize_null(self, attr_value: AttributeValue): + return attr_value.null_value + + def _deserialize_bool(self, attr_value: AttributeValue): + return attr_value.bool_value + + def _deserialize_n(self, attr_value: AttributeValue): + return attr_value.n_value + + def _deserialize_s(self, attr_value: AttributeValue): + return attr_value.s_value + + def _deserialize_b(self, attr_value: AttributeValue): + return attr_value.b_value + + def _deserialize_ns(self, attr_value: AttributeValue): + return set(attr_value.ns_value) + + def _deserialize_ss(self, attr_value: AttributeValue): + print(attr_value) + return set(attr_value.ss_value) + + def _deserialize_bs(self, attr_value: AttributeValue): + return set(attr_value.bs_value) + + def _deserialize_l(self, attr_value: AttributeValue): + return [self._deserialize_attr_value(v) for v in attr_value.list_value] + + def _deserialize_m(self, attr_value: AttributeValue): + return {k: self._deserialize_attr_value(v) for k, v in attr_value.map_value.items()} diff --git a/tests/functional/test_data_classes.py b/tests/functional/test_data_classes.py index 235a3f8f8da..8ac5eb07d90 100644 --- a/tests/functional/test_data_classes.py +++ b/tests/functional/test_data_classes.py @@ -83,6 +83,9 @@ StreamRecord, StreamViewType, ) +from aws_lambda_powertools.utilities.data_classes.dynamodb import ( + DynamoDBImageDeserializer, +) from aws_lambda_powertools.utilities.data_classes.event_source import event_source from aws_lambda_powertools.utilities.data_classes.s3_object_event import ( S3ObjectLambdaEvent, @@ -656,6 +659,48 @@ def test_stream_record_keys_overrides_dict_wrapper_keys(): assert record.keys != data.keys() +def test_deserialize_stream_record(): + byte_list = [s.encode("utf-8") for s in ["item1", "item2"]] + data = { + "Keys": {"key1": {"attr1": "value1"}}, + "NewImage": { + "Name": {"S": "Joe"}, + "Age": {"N": "35"}, + "TypesMap": { + "M": { + "string": {"S": "value"}, + "number": {"N": "100"}, + "bool": {"BOOL": True}, + "dict": {"M": {"key": {"S": "value"}}}, + "stringSet": {"SS": ["item1", "item2"]}, + "numberSet": {"NS": ["100", "200", "300"]}, + "byteSet": {"BS": byte_list}, + "list": {"L": [{"S": "item1"}, {"N": "3.14159"}, {"BOOL": False}]}, + "null": {"NULL": True}, + }, + }, + }, + } + record = StreamRecord(data) + deserializer = DynamoDBImageDeserializer() + new_image = deserializer.deserialize(record.new_image) + assert new_image == { + "Name": "Joe", + "Age": "35", + "TypesMap": { + "string": "value", + "number": "100", + "bool": True, + "dict": {"key": "value"}, + "stringSet": {"item1", "item2"}, + "numberSet": {"100", "200", "300"}, + "byteSet": set(byte_list), + "list": ["item1", "3.14159", False], + "null": None, + }, + } + + def test_event_bridge_event(): event = EventBridgeEvent(load_event("eventBridgeEvent.json")) From d28ed9fe60ad44096f6cc4f7bcba8158538a0339 Mon Sep 17 00:00:00 2001 From: Ahmed Aboshanab <shanab@hey.com> Date: Mon, 10 Oct 2022 12:54:21 +0000 Subject: [PATCH 21/23] fix(typing): fix mypy errors --- .../dynamodb/dynamodb_image_deserializer.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/aws_lambda_powertools/utilities/data_classes/dynamodb/dynamodb_image_deserializer.py b/aws_lambda_powertools/utilities/data_classes/dynamodb/dynamodb_image_deserializer.py index 4b59ebd94d4..c539b2a996c 100644 --- a/aws_lambda_powertools/utilities/data_classes/dynamodb/dynamodb_image_deserializer.py +++ b/aws_lambda_powertools/utilities/data_classes/dynamodb/dynamodb_image_deserializer.py @@ -89,17 +89,22 @@ def _deserialize_b(self, attr_value: AttributeValue): return attr_value.b_value def _deserialize_ns(self, attr_value: AttributeValue): - return set(attr_value.ns_value) + return set(attr_value.ns_value) if attr_value.ns_value else set() def _deserialize_ss(self, attr_value: AttributeValue): - print(attr_value) - return set(attr_value.ss_value) + return set(attr_value.ss_value) if attr_value.ss_value else set() def _deserialize_bs(self, attr_value: AttributeValue): - return set(attr_value.bs_value) + return set(attr_value.bs_value) if attr_value.bs_value else set() def _deserialize_l(self, attr_value: AttributeValue): - return [self._deserialize_attr_value(v) for v in attr_value.list_value] + if attr_value.list_value: + return [self._deserialize_attr_value(v) for v in attr_value.list_value] + else: + return [] def _deserialize_m(self, attr_value: AttributeValue): - return {k: self._deserialize_attr_value(v) for k, v in attr_value.map_value.items()} + if attr_value.map_value: + return {k: self._deserialize_attr_value(v) for k, v in attr_value.map_value.items()} + else: + return {} From ddfd50eff63ac91b88c6e769f39ddc91699aa38c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 Oct 2022 07:17:42 +0200 Subject: [PATCH 22/23] chore(deps): bump release-drafter/release-drafter from 5.21.0 to 5.21.1 (#1611) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/release-drafter.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/release-drafter.yml index 3007aa0241d..68f3cfb962a 100644 --- a/.github/workflows/release-drafter.yml +++ b/.github/workflows/release-drafter.yml @@ -10,6 +10,6 @@ jobs: update_release_draft: runs-on: ubuntu-latest steps: - - uses: release-drafter/release-drafter@df69d584deac33d8569990cb6413f82447181076 # v5.20.1 + - uses: release-drafter/release-drafter@6df64e4ba4842c203c604c1f45246c5863410adb # v5.20.1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 95700dc253920f6165c356a0a3b9b79c434cd602 Mon Sep 17 00:00:00 2001 From: Ahmed Aboshanab <shanab@hey.com> Date: Tue, 18 Oct 2022 07:41:49 +0000 Subject: [PATCH 23/23] feat(data-classes): Replace AttributeValue with values deserialized by TypeDeserializer --- .../data_classes/dynamo_db_stream_event.py | 234 ++++++------------ .../data_classes/dynamodb/__init__.py | 1 - .../dynamodb/dynamodb_image_deserializer.py | 110 -------- .../src/kinesis_firehose_delivery_stream.py | 5 +- tests/functional/test_data_classes.py | 165 ++---------- tests/functional/test_utilities_batch.py | 2 +- 6 files changed, 99 insertions(+), 418 deletions(-) delete mode 100644 aws_lambda_powertools/utilities/data_classes/dynamodb/__init__.py delete mode 100644 aws_lambda_powertools/utilities/data_classes/dynamodb/dynamodb_image_deserializer.py diff --git a/aws_lambda_powertools/utilities/data_classes/dynamo_db_stream_event.py b/aws_lambda_powertools/utilities/data_classes/dynamo_db_stream_event.py index eb674c86b60..19c8231714e 100644 --- a/aws_lambda_powertools/utilities/data_classes/dynamo_db_stream_event.py +++ b/aws_lambda_powertools/utilities/data_classes/dynamo_db_stream_event.py @@ -1,169 +1,78 @@ from enum import Enum -from typing import Any, Dict, Iterator, List, Optional, Union +from typing import Any, Dict, Iterator, Optional from aws_lambda_powertools.utilities.data_classes.common import DictWrapper -class AttributeValueType(Enum): - Binary = "B" - BinarySet = "BS" - Boolean = "BOOL" - List = "L" - Map = "M" - Number = "N" - NumberSet = "NS" - Null = "NULL" - String = "S" - StringSet = "SS" - - -class AttributeValue(DictWrapper): - """Represents the data for an attribute - - Documentation: - -------------- - - https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_streams_AttributeValue.html - - https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/HowItWorks.NamingRulesDataTypes.html +class TypeDeserializer: + """ + This class deserializes DynamoDB types to Python types. + It based on boto3's DynamoDB TypeDeserializer found here: + https://boto3.amazonaws.com/v1/documentation/api/latest/_modules/boto3/dynamodb/types.html + Except that it deserializes DynamoDB numbers into strings, and does not wrap binary + with a Binary class. """ - def __init__(self, data: Dict[str, Any]): - """AttributeValue constructor - - Parameters - ---------- - data: Dict[str, Any] - Raw lambda event dict - """ - super().__init__(data) - self.dynamodb_type = list(data.keys())[0] - - @property - def b_value(self) -> Optional[str]: - """An attribute of type Base64-encoded binary data object - - Example: - >>> {"B": "dGhpcyB0ZXh0IGlzIGJhc2U2NC1lbmNvZGVk"} - """ - return self.get("B") - - @property - def bs_value(self) -> Optional[List[str]]: - """An attribute of type Array of Base64-encoded binary data objects - - Example: - >>> {"BS": ["U3Vubnk=", "UmFpbnk=", "U25vd3k="]} - """ - return self.get("BS") - - @property - def bool_value(self) -> Optional[bool]: - """An attribute of type Boolean - - Example: - >>> {"BOOL": True} - """ - item = self.get("BOOL") - return None if item is None else bool(item) - - @property - def list_value(self) -> Optional[List["AttributeValue"]]: - """An attribute of type Array of AttributeValue objects - - Example: - >>> {"L": [ {"S": "Cookies"} , {"S": "Coffee"}, {"N": "3.14159"}]} - """ - item = self.get("L") - return None if item is None else [AttributeValue(v) for v in item] - - @property - def map_value(self) -> Optional[Dict[str, "AttributeValue"]]: - """An attribute of type String to AttributeValue object map - - Example: - >>> {"M": {"Name": {"S": "Joe"}, "Age": {"N": "35"}}} - """ - return _attribute_value_dict(self._data, "M") - - @property - def n_value(self) -> Optional[str]: - """An attribute of type Number - - Numbers are sent across the network to DynamoDB as strings, to maximize compatibility across languages - and libraries. However, DynamoDB treats them as number type attributes for mathematical operations. - - Example: - >>> {"N": "123.45"} - """ - return self.get("N") - - @property - def ns_value(self) -> Optional[List[str]]: - """An attribute of type Number Set - - Example: - >>> {"NS": ["42.2", "-19", "7.5", "3.14"]} + def deserialize(self, value): + """The method to deserialize the DynamoDB data types. + + :param value: A DynamoDB value to be deserialized to a pythonic value. + Here are the various conversions: + + DynamoDB Python + -------- ------ + {'NULL': True} None + {'BOOL': True/False} True/False + {'N': str(value)} str(value) + {'S': string} string + {'B': bytes} bytes + {'NS': [str(value)]} set([str(value)]) + {'SS': [string]} set([string]) + {'BS': [bytes]} set([bytes]) + {'L': list} list + {'M': dict} dict + + :returns: The pythonic value of the DynamoDB type. """ - return self.get("NS") - @property - def null_value(self) -> None: - """An attribute of type Null. + if not value: + raise TypeError("Value must be a nonempty dictionary whose key " "is a valid dynamodb type.") + dynamodb_type = list(value.keys())[0] + try: + deserializer = getattr(self, f"_deserialize_{dynamodb_type}".lower()) + except AttributeError: + raise TypeError(f"Dynamodb type {dynamodb_type} is not supported") + return deserializer(value[dynamodb_type]) - Example: - >>> {"NULL": True} - """ + def _deserialize_null(self, value): return None - @property - def s_value(self) -> Optional[str]: - """An attribute of type String + def _deserialize_bool(self, value): + return value - Example: - >>> {"S": "Hello"} - """ - return self.get("S") + def _deserialize_n(self, value): + return value - @property - def ss_value(self) -> Optional[List[str]]: - """An attribute of type Array of strings + def _deserialize_s(self, value): + return value - Example: - >>> {"SS": ["Giraffe", "Hippo" ,"Zebra"]} - """ - return self.get("SS") + def _deserialize_b(self, value): + return value - @property - def get_type(self) -> AttributeValueType: - """Get the attribute value type based on the contained data""" - return AttributeValueType(self.dynamodb_type) + def _deserialize_ns(self, value): + return set(map(self._deserialize_n, value)) - @property - def l_value(self) -> Optional[List["AttributeValue"]]: - """Alias of list_value""" - return self.list_value + def _deserialize_ss(self, value): + return set(map(self._deserialize_s, value)) - @property - def m_value(self) -> Optional[Dict[str, "AttributeValue"]]: - """Alias of map_value""" - return self.map_value - - @property - def get_value(self) -> Union[Optional[bool], Optional[str], Optional[List], Optional[Dict]]: - """Get the attribute value""" - try: - return getattr(self, f"{self.dynamodb_type.lower()}_value") - except AttributeError: - raise TypeError(f"Dynamodb type {self.dynamodb_type} is not supported") + def _deserialize_bs(self, value): + return set(map(self._deserialize_b, value)) + def _deserialize_l(self, value): + return [self.deserialize(v) for v in value] -def _attribute_value_dict(attr_values: Dict[str, dict], key: str) -> Optional[Dict[str, AttributeValue]]: - """A dict of type String to AttributeValue object map - - Example: - >>> {"NewImage": {"Id": {"S": "xxx-xxx"}, "Value": {"N": "35"}}} - """ - attr_values_dict = attr_values.get(key) - return None if attr_values_dict is None else {k: AttributeValue(v) for k, v in attr_values_dict.items()} + def _deserialize_m(self, value): + return {k: self.deserialize(v) for k, v in value.items()} class StreamViewType(Enum): @@ -176,28 +85,43 @@ class StreamViewType(Enum): class StreamRecord(DictWrapper): + def __init__(self, data: Dict[str, Any]): + """StreamRecord constructor + + Parameters + ---------- + data: Dict[str, Any] + Represents the dynamodb dict inside DynamoDBStreamEvent's records + """ + super().__init__(data) + self._deserializer = TypeDeserializer() + + def _deserialize_dynamodb_dict(self, key: str) -> Optional[Dict[str, Any]]: + dynamodb_dict = self._data.get(key) + return ( + None if dynamodb_dict is None else {k: self._deserializer.deserialize(v) for k, v in dynamodb_dict.items()} + ) + @property def approximate_creation_date_time(self) -> Optional[int]: """The approximate date and time when the stream record was created, in UNIX epoch time format.""" item = self.get("ApproximateCreationDateTime") return None if item is None else int(item) - # NOTE: This override breaks the Mapping protocol of DictWrapper, it's left here for backwards compatibility with - # a 'type: ignore' comment. See #1516 for discussion @property - def keys(self) -> Optional[Dict[str, AttributeValue]]: # type: ignore[override] + def keys(self) -> Optional[Dict[str, Any]]: # type: ignore[override] """The primary key attribute(s) for the DynamoDB item that was modified.""" - return _attribute_value_dict(self._data, "Keys") + return self._deserialize_dynamodb_dict("Keys") @property - def new_image(self) -> Optional[Dict[str, AttributeValue]]: + def new_image(self) -> Optional[Dict[str, Any]]: """The item in the DynamoDB table as it appeared after it was modified.""" - return _attribute_value_dict(self._data, "NewImage") + return self._deserialize_dynamodb_dict("NewImage") @property - def old_image(self) -> Optional[Dict[str, AttributeValue]]: + def old_image(self) -> Optional[Dict[str, Any]]: """The item in the DynamoDB table as it appeared before it was modified.""" - return _attribute_value_dict(self._data, "OldImage") + return self._deserialize_dynamodb_dict("OldImage") @property def sequence_number(self) -> Optional[str]: @@ -233,7 +157,7 @@ def aws_region(self) -> Optional[str]: @property def dynamodb(self) -> Optional[StreamRecord]: - """The main body of the stream record, containing all the DynamoDB-specific fields.""" + """The main body of the stream record, containing all the DynamoDB-specific dicts.""" stream_record = self.get("dynamodb") return None if stream_record is None else StreamRecord(stream_record) diff --git a/aws_lambda_powertools/utilities/data_classes/dynamodb/__init__.py b/aws_lambda_powertools/utilities/data_classes/dynamodb/__init__.py deleted file mode 100644 index 0271f476222..00000000000 --- a/aws_lambda_powertools/utilities/data_classes/dynamodb/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .dynamodb_image_deserializer import DynamoDBImageDeserializer # noqa: F401 diff --git a/aws_lambda_powertools/utilities/data_classes/dynamodb/dynamodb_image_deserializer.py b/aws_lambda_powertools/utilities/data_classes/dynamodb/dynamodb_image_deserializer.py deleted file mode 100644 index c539b2a996c..00000000000 --- a/aws_lambda_powertools/utilities/data_classes/dynamodb/dynamodb_image_deserializer.py +++ /dev/null @@ -1,110 +0,0 @@ -from typing import Dict, List, Optional, Union - -from aws_lambda_powertools.utilities.data_classes.dynamo_db_stream_event import ( - AttributeValue, -) - - -class DynamoDBImageDeserializer: - """ - Deserializes DynamoDB StreamRecord's old_image and new_image properties of type Dict[str, AttributeValue] - to Dict with Python types. - - Example - ------- - - from aws_lambda_powertools.utilities.typing import LambdaContext - from aws_lambda_powertools.utilities.data_classes import event_source, DynamoDBStreamEvent - from aws_lambda_powertools.utilities.data_classes.dynamodb import DynamoDBImageDeserializer - - deserializer = DynamoDBImageDeserializer() - - @event_source(data_class=DynamoDBStreamEvent) - def handler(event: DynamoDBStreamEvent, context: LambdaContext): - for record in event.records: - new_image = deserializer.deserialize(record.dynamodb.new_image) - old_image = deserializer.deserialize(record.dynamodb.old_image) - """ - - def deserialize(self, image: Dict[str, AttributeValue]): - """ - Deserializes image to Dict with Python types. - AttributeValue fields are deserialized according to the following table: - - DynamoDB Python - -------- ------ - {'NULL': True} None - {'BOOL': True/False} True/False - {'N': string} string - {'S': string} string - {'B': bytes} bytes - {'NS': [string]} set([string]) - {'SS': [string]} set([string]) - {'BS': [bytes]} set([bytes]) - {'L': list} list - {'M': dict} dict - - Please bear in mind that numbers are represented as strings. - - Parameters - ---------- - image: Dict[str, AttributeValue] - Dictionary representing a StreamRecord's image property - - Returns - ------- - Dict - Dictionary with Python types - - Raises - ------ - TypeError - Raised if an unknown DynamoDB attribute type is encountered - """ - return {k: self._deserialize_attr_value(v) for k, v in image.items()} - - def _deserialize_attr_value( - self, attr_value: AttributeValue - ) -> Union[Optional[bool], Optional[str], Optional[List], Optional[Dict]]: - try: - dynamodb_type = attr_value.dynamodb_type.lower() - deserializer = getattr(self, "_deserialize_%s" % dynamodb_type) - except AttributeError: - raise TypeError("DynamoDB type %s is not supported" % dynamodb_type) - return deserializer(attr_value) - - def _deserialize_null(self, attr_value: AttributeValue): - return attr_value.null_value - - def _deserialize_bool(self, attr_value: AttributeValue): - return attr_value.bool_value - - def _deserialize_n(self, attr_value: AttributeValue): - return attr_value.n_value - - def _deserialize_s(self, attr_value: AttributeValue): - return attr_value.s_value - - def _deserialize_b(self, attr_value: AttributeValue): - return attr_value.b_value - - def _deserialize_ns(self, attr_value: AttributeValue): - return set(attr_value.ns_value) if attr_value.ns_value else set() - - def _deserialize_ss(self, attr_value: AttributeValue): - return set(attr_value.ss_value) if attr_value.ss_value else set() - - def _deserialize_bs(self, attr_value: AttributeValue): - return set(attr_value.bs_value) if attr_value.bs_value else set() - - def _deserialize_l(self, attr_value: AttributeValue): - if attr_value.list_value: - return [self._deserialize_attr_value(v) for v in attr_value.list_value] - else: - return [] - - def _deserialize_m(self, attr_value: AttributeValue): - if attr_value.map_value: - return {k: self._deserialize_attr_value(v) for k, v in attr_value.map_value.items()} - else: - return {} diff --git a/examples/event_sources/src/kinesis_firehose_delivery_stream.py b/examples/event_sources/src/kinesis_firehose_delivery_stream.py index 67bf53dfe06..770bfb1ee63 100644 --- a/examples/event_sources/src/kinesis_firehose_delivery_stream.py +++ b/examples/event_sources/src/kinesis_firehose_delivery_stream.py @@ -1,7 +1,10 @@ import base64 import json -from aws_lambda_powertools.utilities.data_classes import KinesisFirehoseEvent, event_source +from aws_lambda_powertools.utilities.data_classes import ( + KinesisFirehoseEvent, + event_source, +) from aws_lambda_powertools.utilities.typing import LambdaContext diff --git a/tests/functional/test_data_classes.py b/tests/functional/test_data_classes.py index 8ac5eb07d90..b4dbd073382 100644 --- a/tests/functional/test_data_classes.py +++ b/tests/functional/test_data_classes.py @@ -76,16 +76,11 @@ ConnectContactFlowInitiationMethod, ) from aws_lambda_powertools.utilities.data_classes.dynamo_db_stream_event import ( - AttributeValue, - AttributeValueType, DynamoDBRecordEventName, DynamoDBStreamEvent, StreamRecord, StreamViewType, ) -from aws_lambda_powertools.utilities.data_classes.dynamodb import ( - DynamoDBImageDeserializer, -) from aws_lambda_powertools.utilities.data_classes.event_source import event_source from aws_lambda_powertools.utilities.data_classes.s3_object_event import ( S3ObjectLambdaEvent, @@ -506,20 +501,8 @@ def test_dynamo_db_stream_trigger_event(): assert dynamodb.approximate_creation_date_time is None keys = dynamodb.keys assert keys is not None - id_key = keys["Id"] - assert id_key.b_value is None - assert id_key.bs_value is None - assert id_key.bool_value is None - assert id_key.list_value is None - assert id_key.map_value is None - assert id_key.n_value == "101" - assert id_key.ns_value is None - assert id_key.null_value is None - assert id_key.s_value is None - assert id_key.ss_value is None - message_key = dynamodb.new_image["Message"] - assert message_key is not None - assert message_key.s_value == "New item!" + assert keys["Id"] == "101" + assert dynamodb.new_image["Message"] == "New item!" assert dynamodb.old_image is None assert dynamodb.sequence_number == "111" assert dynamodb.size_bytes == 26 @@ -532,134 +515,7 @@ def test_dynamo_db_stream_trigger_event(): assert record.user_identity is None -def test_dynamo_attribute_value_b_value(): - example_attribute_value = {"B": "dGhpcyB0ZXh0IGlzIGJhc2U2NC1lbmNvZGVk"} - - attribute_value = AttributeValue(example_attribute_value) - - assert attribute_value.get_type == AttributeValueType.Binary - assert attribute_value.b_value == attribute_value.get_value - - -def test_dynamo_attribute_value_bs_value(): - example_attribute_value = {"BS": ["U3Vubnk=", "UmFpbnk=", "U25vd3k="]} - - attribute_value = AttributeValue(example_attribute_value) - - assert attribute_value.get_type == AttributeValueType.BinarySet - assert attribute_value.bs_value == attribute_value.get_value - - -def test_dynamo_attribute_value_bool_value(): - example_attribute_value = {"BOOL": True} - - attribute_value = AttributeValue(example_attribute_value) - - assert attribute_value.get_type == AttributeValueType.Boolean - assert attribute_value.bool_value == attribute_value.get_value - - -def test_dynamo_attribute_value_list_value(): - example_attribute_value = {"L": [{"S": "Cookies"}, {"S": "Coffee"}, {"N": "3.14159"}]} - attribute_value = AttributeValue(example_attribute_value) - list_value = attribute_value.list_value - assert list_value is not None - item = list_value[0] - assert item.s_value == "Cookies" - assert attribute_value.get_type == AttributeValueType.List - assert attribute_value.l_value == attribute_value.list_value - assert attribute_value.list_value == attribute_value.get_value - - -def test_dynamo_attribute_value_map_value(): - example_attribute_value = {"M": {"Name": {"S": "Joe"}, "Age": {"N": "35"}}} - - attribute_value = AttributeValue(example_attribute_value) - - map_value = attribute_value.map_value - assert map_value is not None - item = map_value["Name"] - assert item.s_value == "Joe" - assert attribute_value.get_type == AttributeValueType.Map - assert attribute_value.m_value == attribute_value.map_value - assert attribute_value.map_value == attribute_value.get_value - - -def test_dynamo_attribute_value_n_value(): - example_attribute_value = {"N": "123.45"} - - attribute_value = AttributeValue(example_attribute_value) - - assert attribute_value.get_type == AttributeValueType.Number - assert attribute_value.n_value == attribute_value.get_value - - -def test_dynamo_attribute_value_ns_value(): - example_attribute_value = {"NS": ["42.2", "-19", "7.5", "3.14"]} - - attribute_value = AttributeValue(example_attribute_value) - - assert attribute_value.get_type == AttributeValueType.NumberSet - assert attribute_value.ns_value == attribute_value.get_value - - -def test_dynamo_attribute_value_null_value(): - example_attribute_value = {"NULL": True} - - attribute_value = AttributeValue(example_attribute_value) - - assert attribute_value.get_type == AttributeValueType.Null - assert attribute_value.null_value is None - assert attribute_value.null_value == attribute_value.get_value - - -def test_dynamo_attribute_value_s_value(): - example_attribute_value = {"S": "Hello"} - - attribute_value = AttributeValue(example_attribute_value) - - assert attribute_value.get_type == AttributeValueType.String - assert attribute_value.s_value == attribute_value.get_value - - -def test_dynamo_attribute_value_ss_value(): - example_attribute_value = {"SS": ["Giraffe", "Hippo", "Zebra"]} - - attribute_value = AttributeValue(example_attribute_value) - - assert attribute_value.get_type == AttributeValueType.StringSet - assert attribute_value.ss_value == attribute_value.get_value - - -def test_dynamo_attribute_value_type_error(): - example_attribute_value = {"UNSUPPORTED": "'value' should raise a type error"} - - attribute_value = AttributeValue(example_attribute_value) - - with pytest.raises(TypeError): - print(attribute_value.get_value) - with pytest.raises(ValueError): - print(attribute_value.get_type) - - -def test_stream_record_keys_with_valid_keys(): - attribute_value = {"Foo": "Bar"} - record = StreamRecord({"Keys": {"Key1": attribute_value}}) - assert record.keys == {"Key1": AttributeValue(attribute_value)} - - -def test_stream_record_keys_with_no_keys(): - record = StreamRecord({}) - assert record.keys is None - - -def test_stream_record_keys_overrides_dict_wrapper_keys(): - data = {"Keys": {"key1": {"attr1": "value1"}}} - record = StreamRecord(data) - assert record.keys != data.keys() - - -def test_deserialize_stream_record(): +def test_dynamo_stream_record(): byte_list = [s.encode("utf-8") for s in ["item1", "item2"]] data = { "Keys": {"key1": {"attr1": "value1"}}, @@ -682,9 +538,7 @@ def test_deserialize_stream_record(): }, } record = StreamRecord(data) - deserializer = DynamoDBImageDeserializer() - new_image = deserializer.deserialize(record.new_image) - assert new_image == { + assert record.new_image == { "Name": "Joe", "Age": "35", "TypesMap": { @@ -701,6 +555,17 @@ def test_deserialize_stream_record(): } +def test_stream_record_keys_with_no_keys(): + record = StreamRecord({}) + assert record.keys is None + + +def test_stream_record_keys_overrides_dict_wrapper_keys(): + data = {"Keys": {"key1": {"N": "101"}}} + record = StreamRecord(data) + assert record.keys != data.keys() + + def test_event_bridge_event(): event = EventBridgeEvent(load_event("eventBridgeEvent.json")) diff --git a/tests/functional/test_utilities_batch.py b/tests/functional/test_utilities_batch.py index b5489fb7c62..996981924e0 100644 --- a/tests/functional/test_utilities_batch.py +++ b/tests/functional/test_utilities_batch.py @@ -139,7 +139,7 @@ def handler(record: KinesisStreamRecord): @pytest.fixture(scope="module") def dynamodb_record_handler() -> Callable: def handler(record: DynamoDBRecord): - body = record.dynamodb.new_image.get("Message").get_value + body = record.dynamodb.new_image.get("Message") if "fail" in body: raise Exception("Failed to process record.") return body