Skip to content

Commit 373aada

Browse files
Merge branch 'develop' into improv/event_handler_api_gateway_strip_prefixes
2 parents 2a7e2b6 + 0916bc5 commit 373aada

File tree

181 files changed

+6112
-3644
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

181 files changed

+6112
-3644
lines changed

.github/ISSUE_TEMPLATE/bug_report.yml

+1
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ body:
6262
- "3.8"
6363
- "3.9"
6464
- "3.10"
65+
- "3.11"
6566
validations:
6667
required: true
6768
- type: dropdown

.github/ISSUE_TEMPLATE/static_typing.yml

+1
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ body:
2929
- "3.8"
3030
- "3.9"
3131
- "3.10"
32+
- "3.11"
3233
validations:
3334
required: true
3435
- type: input
+82
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
name: "Restore sealed source code"
2+
description: "Restore sealed source code and confirm integrity hash"
3+
4+
# PROCESS
5+
#
6+
# 1. Exports artifact name using Prefix + GitHub Run ID (unique for each release trigger)
7+
# 2. Compress entire source code as tarball OR given files
8+
# 3. Create and export integrity hash for tarball
9+
# 4. Upload artifact
10+
# 5. Remove archive
11+
12+
# USAGE
13+
#
14+
# - name: Seal and upload
15+
# id: seal_source_code
16+
# uses: ./.github/actions/seal
17+
# with:
18+
# artifact_name_prefix: "source"
19+
#
20+
# - name: Restore sealed source code
21+
# uses: ./.github/actions/seal-restore
22+
# with:
23+
# integrity_hash: ${{ needs.seal_source_code.outputs.integrity_hash }}
24+
# artifact_name: ${{ needs.seal_source_code.outputs.artifact_name }}
25+
26+
# NOTES
27+
#
28+
# To be used together with .github/actions/seal
29+
30+
inputs:
31+
integrity_hash:
32+
description: "Integrity hash to verify"
33+
required: true
34+
artifact_name:
35+
description: "Sealed artifact name to restore"
36+
required: true
37+
38+
runs:
39+
using: "composite"
40+
steps:
41+
- id: adjust-path
42+
run: echo "${{ github.action_path }}" >> $GITHUB_PATH
43+
shell: bash
44+
45+
- name: Download artifacts
46+
uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
47+
with:
48+
name: ${{ inputs.artifact_name }}
49+
path: .
50+
51+
- id: integrity_hash
52+
name: Create integrity hash for downloaded artifact
53+
run: |
54+
HASH=$(sha256sum "${ARTIFACT_NAME}.tar" | awk '{print $1}')
55+
56+
echo "current_hash=${HASH}" >> "$GITHUB_OUTPUT"
57+
env:
58+
ARTIFACT_NAME: ${{ inputs.artifact_name }}
59+
shell: bash
60+
61+
- id: verify_hash
62+
name: Verify sealed artifact integrity hash
63+
run: test "${CURRENT_HASH}" = "${PROVIDED_HASH}" || exit 1
64+
env:
65+
ARTIFACT_NAME: ${{ inputs.artifact_name }}
66+
PROVIDED_HASH: ${{ inputs.integrity_hash }}
67+
CURRENT_HASH: ${{ steps.integrity_hash.outputs.current_hash }}
68+
shell: bash
69+
70+
# Restore and overwrite tarball in current directory
71+
- id: overwrite
72+
name: Extract tarball
73+
run: tar -xvf "${ARTIFACT_NAME}".tar
74+
env:
75+
ARTIFACT_NAME: ${{ inputs.artifact_name }}
76+
shell: bash
77+
78+
- name: Remove archive
79+
run: rm -f "${ARTIFACT_NAME}.tar"
80+
env:
81+
ARTIFACT_NAME: ${{ inputs.artifact_name }}
82+
shell: bash

.github/actions/seal/action.yml

+93
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
name: "Seal and hash source code"
2+
description: "Seal and export source code as a tarball artifact along with its integrity hash"
3+
4+
# PROCESS
5+
#
6+
# 1. Exports artifact name using Prefix + GitHub Run ID (unique for each release trigger)
7+
# 2. Compress entire source code as tarball OR given files
8+
# 3. Create and export integrity hash for tarball
9+
# 4. Upload artifact
10+
# 5. Remove archive
11+
12+
# USAGE
13+
#
14+
# - name: Seal and upload
15+
# id: seal_source_code
16+
# uses: ./.github/actions/seal
17+
# with:
18+
# artifact_name_prefix: "source"
19+
20+
inputs:
21+
files:
22+
description: "Files to seal separated by space"
23+
required: false
24+
artifact_name_prefix:
25+
description: "Prefix to use when exporting artifact"
26+
required: true
27+
28+
outputs:
29+
integrity_hash:
30+
description: "Source code integrity hash"
31+
value: ${{ steps.integrity_hash.outputs.integrity_hash }}
32+
artifact_name:
33+
description: "Artifact name containTemporary branch created with staged changed"
34+
value: ${{ steps.export_artifact_name.outputs.artifact_name }}
35+
36+
runs:
37+
using: "composite"
38+
steps:
39+
- id: adjust-path
40+
run: echo "${{ github.action_path }}" >> $GITHUB_PATH
41+
shell: bash
42+
43+
- id: export_artifact_name
44+
name: Export final artifact name
45+
run: echo "artifact_name=${ARTIFACT_PREFIX}-${GITHUB_RUN_ID}" >> "$GITHUB_OUTPUT"
46+
env:
47+
GITHUB_RUN_ID: ${{ github.run_id }}
48+
ARTIFACT_PREFIX: ${{ inputs.artifact_name_prefix }}
49+
shell: bash
50+
51+
# By default, create a tarball of the current directory minus .git
52+
# otherwise it breaks GH Actions when restoring it
53+
- id: compress_all
54+
if: ${{ !inputs.files }}
55+
name: Create tarball for entire source
56+
run: tar --exclude-vcs -cvf "${ARTIFACT_NAME}".tar *
57+
env:
58+
ARTIFACT_NAME: ${{ steps.export_artifact_name.outputs.artifact_name }}
59+
shell: bash
60+
61+
# If a list of files are given, then create a tarball for those only
62+
- id: compress_selected_files
63+
if: ${{ inputs.files }}
64+
name: Create tarball for selected files
65+
run: tar --exclude-vcs -cvf "${ARTIFACT_NAME}".tar "${FILES}"
66+
env:
67+
FILES: ${{ inputs.files }}
68+
ARTIFACT_NAME: ${{ steps.export_artifact_name.outputs.artifact_name }}
69+
shell: bash
70+
71+
- id: integrity_hash
72+
name: Create and export integrity hash for tarball
73+
run: |
74+
HASH=$(sha256sum "${ARTIFACT_NAME}.tar" | awk '{print $1}')
75+
76+
echo "integrity_hash=${HASH}" >> "$GITHUB_OUTPUT"
77+
env:
78+
ARTIFACT_NAME: ${{ steps.export_artifact_name.outputs.artifact_name }}
79+
shell: bash
80+
81+
- name: Upload artifacts
82+
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
83+
with:
84+
if-no-files-found: error
85+
name: ${{ steps.export_artifact_name.outputs.artifact_name }}
86+
path: ${{ steps.export_artifact_name.outputs.artifact_name }}.tar
87+
retention-days: 1
88+
89+
- name: Remove archive
90+
run: rm -f "${ARTEFACT_NAME}.tar"
91+
env:
92+
ARTIFACT_NAME: ${{ steps.export_artifact_name.outputs.artifact_name }}
93+
shell: bash
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
name: "Upload provenance attestation to release"
2+
description: "Download and upload newly generated provenance attestation to latest release."
3+
4+
# PROCESS
5+
#
6+
# 1. Downloads provenance attestation artifact generated earlier in the release pipeline
7+
# 2. Updates latest GitHub draft release pointing to newly git release tag
8+
# 3. Uploads provenance attestation file to latest GitHub draft release
9+
10+
# USAGE
11+
#
12+
# - name: Upload provenance
13+
# id: upload-provenance
14+
# uses: ./.github/actions/upload-release-provenance
15+
# with:
16+
# release_version: ${{ needs.seal.outputs.RELEASE_VERSION }}
17+
# provenance_name: ${{needs.provenance.outputs.provenance-name}}
18+
# github_token: ${{ secrets.GITHUB_TOKEN }}
19+
20+
# NOTES
21+
#
22+
# There are no outputs.
23+
#
24+
25+
inputs:
26+
provenance_name:
27+
description: "Provenance artifact name to download"
28+
required: true
29+
release_version:
30+
description: "Release version (e.g., 2.20.0)"
31+
required: true
32+
github_token:
33+
description: "GitHub token for GitHub CLI"
34+
required: true
35+
36+
runs:
37+
using: "composite"
38+
steps:
39+
- id: adjust-path
40+
run: echo "${{ github.action_path }}" >> $GITHUB_PATH
41+
shell: bash
42+
43+
- id: download-provenance
44+
name: Download newly generated provenance
45+
uses: actions/download-artifact@9782bd6a9848b53b110e712e20e42d89988822b7 # v3.0.1
46+
with:
47+
name: ${{ inputs.provenance_name }}
48+
49+
- id: sync-release-tag
50+
name: Update draft release tag to release commit tag
51+
run: |
52+
CURRENT_DRAFT_RELEASE=$(gh release list | awk '{ if ($2 == "Draft") print $1}')
53+
gh release edit "${CURRENT_DRAFT_RELEASE}" --tag v"${RELEASE_VERSION}"
54+
env:
55+
RELEASE_VERSION: ${{ inputs.release_version }}
56+
GH_TOKEN: ${{ inputs.github_token }}
57+
shell: bash
58+
59+
- id: upload-provenance
60+
name: Upload provenance to release tag
61+
# clobber flag means overwrite release asset if available (eventual consistency, retried failed steps)
62+
run: gh release upload --clobber v"${RELEASE_VERSION}" "${PROVENANCE_FILE}"
63+
env:
64+
RELEASE_VERSION: ${{ inputs.release_version }}
65+
PROVENANCE_FILE: ${{ inputs.provenance_name }}
66+
GH_TOKEN: ${{ inputs.github_token }}
67+
shell: bash
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
#!/bin/bash
2+
set -uo pipefail # prevent accessing unset env vars, prevent masking pipeline errors to the next command
3+
4+
#docs
5+
#title :verify_provenance.sh
6+
#description :This script will download and verify a signed Powertools for AWS Lambda (Python) release build with SLSA Verifier
7+
#author :@heitorlessa
8+
#date :July 1st 2023
9+
#version :0.1
10+
#usage :bash verify_provenance.sh {release version}
11+
#notes :Meant to use in GitHub Actions or locally (MacOS, Linux, WSL).
12+
#os_version :Ubuntu 22.04.2 LTS
13+
#==============================================================================
14+
15+
# Check if RELEASE_VERSION is provided as a command line argument
16+
if [[ $# -eq 1 ]]; then
17+
export readonly RELEASE_VERSION="$1"
18+
else
19+
echo "ERROR: Please provider Powertools release version as a command line argument."
20+
echo "Example: bash verify_provenance.sh 2.20.0"
21+
exit 1
22+
fi
23+
24+
export readonly ARCHITECTURE=$(uname -m | sed 's/x86_64/amd64/g') # arm64, x86_64 ->amd64
25+
export readonly OS_NAME=$(uname -s | tr '[:upper:]' '[:lower:]') # darwin, linux
26+
export readonly SLSA_VERIFIER_VERSION="2.3.0"
27+
export readonly SLSA_VERIFIER_CHECKSUM_FILE="SHA256SUM.md"
28+
export readonly SLSA_VERIFIER_BINARY="./slsa-verifier-${OS_NAME}-${ARCHITECTURE}"
29+
30+
export readonly RELEASE_BINARY="aws_lambda_powertools-${RELEASE_VERSION}-py3-none-any.whl"
31+
export readonly ORG="aws-powertools"
32+
export readonly REPO="powertools-lambda-python"
33+
export readonly PROVENANCE_FILE="multiple.intoto.jsonl"
34+
35+
export readonly FILES=("${SLSA_VERIFIER_BINARY}" "${SLSA_VERIFIER_CHECKSUM_FILE}" "${PROVENANCE_FILE}" "${RELEASE_BINARY}")
36+
37+
function debug() {
38+
TIMESTAMP=$(date -u "+%FT%TZ") # 2023-05-10T07:53:59Z
39+
echo ""${TIMESTAMP}" DEBUG - [*] $1"
40+
}
41+
42+
function error() {
43+
cleanup
44+
TIMESTAMP=$(date -u "+%FT%TZ") # 2023-05-10T07:53:59Z
45+
echo ""${TIMESTAMP}" ERROR - [!] $1"
46+
echo ""${TIMESTAMP}" ERROR - [!] exiting"
47+
exit 1
48+
}
49+
50+
function download_slsa_verifier() {
51+
readonly SLSA_URL="https://github.com/slsa-framework/slsa-verifier/releases/download/v${SLSA_VERIFIER_VERSION}/slsa-verifier-${OS_NAME}-${ARCHITECTURE}"
52+
# debug "Downloading SLSA Verifier for - Binary: slsa-verifier-${OS_NAME}-${ARCHITECTURE}"
53+
debug "Downloading SLSA Verifier binary: ${SLSA_URL}"
54+
curl \
55+
--location \
56+
--fail \
57+
--silent \
58+
-O "${SLSA_URL}" || error "Failed to download SLSA Verifier binary"
59+
60+
readonly SLSA_CHECKSUM_URL="https://raw.githubusercontent.com/slsa-framework/slsa-verifier/f59b55ef2190581d40fc1a5f3b7a51cab2f4a652/${SLSA_VERIFIER_CHECKSUM_FILE}"
61+
debug "Downloading SLSA Verifier checksums"
62+
curl \
63+
--location \
64+
--fail \
65+
--silent \
66+
-O "${SLSA_CHECKSUM_URL}" || error "Failed to download SLSA Verifier binary checksum file"
67+
68+
debug "Verifying SLSA Verifier binary integrity"
69+
CURRENT_HASH=$(sha256sum "${SLSA_VERIFIER_BINARY}" | awk '{print $1}')
70+
if [[ $(grep "${CURRENT_HASH}" "${SLSA_VERIFIER_CHECKSUM_FILE}") ]]; then
71+
debug "SLSA Verifier binary integrity confirmed"
72+
chmod +x "${SLSA_VERIFIER_BINARY}"
73+
else
74+
error "Failed integrity check for SLSA Verifier binary: ${SLSA_VERIFIER_BINARY}"
75+
fi
76+
}
77+
78+
function download_provenance() {
79+
readonly PROVENANCE_URL="https://github.com/${ORG}/${REPO}/releases/download/v${RELEASE_VERSION}/${PROVENANCE_FILE}"
80+
debug "Downloading attestation: ${PROVENANCE_URL}"
81+
82+
curl \
83+
--location \
84+
--fail \
85+
--silent \
86+
-O ${PROVENANCE_URL} || error "Failed to download provenance. Does the release already exist?"
87+
}
88+
89+
function download_release_artifact() {
90+
debug "Downloading ${RELEASE_VERSION} release from PyPi"
91+
python -m pip download \
92+
--only-binary=:all: \
93+
--no-deps \
94+
--quiet \
95+
aws-lambda-powertools=="${RELEASE_VERSION}"
96+
}
97+
98+
function verify_provenance() {
99+
debug "Verifying attestation with slsa-verifier"
100+
"${SLSA_VERIFIER_BINARY}" verify-artifact \
101+
--provenance-path "${PROVENANCE_FILE}" \
102+
--source-uri github.com/${ORG}/${REPO} \
103+
${RELEASE_BINARY}
104+
}
105+
106+
function cleanup() {
107+
debug "Cleaning up previously downloaded files"
108+
rm -f "${SLSA_VERIFIER_BINARY}"
109+
rm -f "${SLSA_VERIFIER_CHECKSUM_FILE}"
110+
rm -f "${PROVENANCE_FILE}"
111+
rm -f "${RELEASE_BINARY}"
112+
echo "${FILES[@]}" | xargs -n1 echo "Removed file: "
113+
}
114+
115+
function main() {
116+
download_slsa_verifier
117+
download_provenance
118+
download_release_artifact
119+
verify_provenance
120+
cleanup
121+
}
122+
123+
main
124+
125+
# Lessons learned
126+
#
127+
# 1. If source doesn't match provenance
128+
#
129+
# FAILED: SLSA verification failed: source used to generate the binary does not match provenance: expected source 'awslabs/aws-lambda-powertools-python', got 'heitorlessa/aws-lambda-powertools-test'
130+
#
131+
# 2. Avoid building deps during download in Test registry endpoints
132+
#
133+
# FAILED: Could not find a version that satisfies the requirement poetry-core>=1.3.2 (from versions: 1.2.0)
134+
#

0 commit comments

Comments
 (0)