From f2c1668ba84ac86ec51bc75d103f55c2d8a6d62e Mon Sep 17 00:00:00 2001 From: Illia Volochii Date: Sat, 5 Aug 2023 14:01:39 +0300 Subject: [PATCH 1/5] Set up CIFuzz to run fuzz targets continuously --- .github/workflows/build.yml | 45 +++++++++++++++++++ ...-08-05-14-01-07.gh-issue-107652.5OxOlT.rst | 2 + 2 files changed, 47 insertions(+) create mode 100644 Misc/NEWS.d/next/Tests/2023-08-05-14-01-07.gh-issue-107652.5OxOlT.rst diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 06551b13219c2a..2a83fd2e045f8f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -520,6 +520,50 @@ jobs: - name: Tests run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu" + # CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/ + cifuzz: + name: CIFuzz + runs-on: ubuntu-latest + timeout-minutes: 60 + needs: check_source + # oss-fuzz maintains a configuration for fuzzing the main branch of + # CPython, so the tests should be run only for code that has been or is + # likely to be merged into the main branch. + # Compatibility with older branches may be broken. + if: (github.ref_name == 'main' || github.base_ref == 'main') && needs.check_source.outputs.run_tests == 'true' + permissions: + security-events: write + strategy: + fail-fast: false + matrix: + sanitizer: [address, undefined, memory] + steps: + - name: Build Fuzzers (${{ matrix.sanitizer }}) + id: build + uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master + with: + oss-fuzz-project-name: cpython3 + sanitizer: ${{ matrix.sanitizer }} + - name: Run Fuzzers (${{ matrix.sanitizer }}) + uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master + with: + fuzz-seconds: 600 + oss-fuzz-project-name: cpython3 + output-sarif: true + sanitizer: ${{ matrix.sanitizer }} + - name: Upload Crash + uses: actions/upload-artifact@v3 + if: failure() && steps.build.outcome == 'success' + with: + name: ${{ matrix.sanitizer }}-artifacts + path: ./out/artifacts + - name: Upload SARIF + if: always() && steps.build.outcome == 'success' + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: cifuzz-sarif/results.sarif + checkout_path: cifuzz-sarif + all-required-green: # This job does nothing and is only used for the branch protection name: All required checks pass if: always() @@ -535,6 +579,7 @@ jobs: - build_ubuntu_ssltests - test_hypothesis - build_asan + - cifuzz runs-on: ubuntu-latest diff --git a/Misc/NEWS.d/next/Tests/2023-08-05-14-01-07.gh-issue-107652.5OxOlT.rst b/Misc/NEWS.d/next/Tests/2023-08-05-14-01-07.gh-issue-107652.5OxOlT.rst new file mode 100644 index 00000000000000..49ec5465b7bcc6 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-08-05-14-01-07.gh-issue-107652.5OxOlT.rst @@ -0,0 +1,2 @@ +Set up CIFuzz to run fuzz targets in GitHub Actions. Patch by Illia +Volochii. From 58d0e57e047def390f92e9bad97df3316989cb35 Mon Sep 17 00:00:00 2001 From: Illia Volochii Date: Sat, 5 Aug 2023 14:48:39 +0300 Subject: [PATCH 2/5] Change names of job steps Co-authored-by: Hugo van Kemenade --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2a83fd2e045f8f..4c7555fe95cbeb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -538,20 +538,20 @@ jobs: matrix: sanitizer: [address, undefined, memory] steps: - - name: Build Fuzzers (${{ matrix.sanitizer }}) + - name: Build fuzzers (${{ matrix.sanitizer }}) id: build uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master with: oss-fuzz-project-name: cpython3 sanitizer: ${{ matrix.sanitizer }} - - name: Run Fuzzers (${{ matrix.sanitizer }}) + - name: Run fuzzers (${{ matrix.sanitizer }}) uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master with: fuzz-seconds: 600 oss-fuzz-project-name: cpython3 output-sarif: true sanitizer: ${{ matrix.sanitizer }} - - name: Upload Crash + - name: Upload crash uses: actions/upload-artifact@v3 if: failure() && steps.build.outcome == 'success' with: From ccae802cbecb7295b45f0d7e9f2b440c4461bb26 Mon Sep 17 00:00:00 2001 From: Illia Volochii Date: Sun, 6 Aug 2023 11:49:52 +0300 Subject: [PATCH 3/5] Change the condition for running CIFuzz --- .github/workflows/build.yml | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4c7555fe95cbeb..8f662e938f223f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -40,6 +40,7 @@ jobs: run-docs: ${{ steps.docs-changes.outputs.run-docs || false }} run_tests: ${{ steps.check.outputs.run_tests }} run_hypothesis: ${{ steps.check.outputs.run_hypothesis }} + run_cifuzz: ${{ steps.check.outputs.run_cifuzz }} config_hash: ${{ steps.config_hash.outputs.hash }} steps: - uses: actions/checkout@v3 @@ -76,6 +77,17 @@ jobs: echo "Run hypothesis tests" echo "run_hypothesis=true" >> $GITHUB_OUTPUT fi + + # oss-fuzz maintains a configuration for fuzzing the main branch of + # CPython, so CIFuzz should be run only for code that is likely to be + # merged into the main branch; compatibility with older branches may + # be broken. + if [ "$GITHUB_BASE_REF" = "main" ]; then + # The tests are pretty slow so they are executed only for PRs + # changing relevant files. + FUZZ_RELEVANT_FILES='(\.c$|\.h$|\.cpp$|^configure$|^\.github/workflows/build\.yml$|^Modules/_xxtestfuzz)' + git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qvE $FUZZ_RELEVANT_FILES && echo "run_cifuzz=true" >> $GITHUB_OUTPUT || true + fi - name: Compute hash for config cache key id: config_hash run: | @@ -526,11 +538,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 60 needs: check_source - # oss-fuzz maintains a configuration for fuzzing the main branch of - # CPython, so the tests should be run only for code that has been or is - # likely to be merged into the main branch. - # Compatibility with older branches may be broken. - if: (github.ref_name == 'main' || github.base_ref == 'main') && needs.check_source.outputs.run_tests == 'true' + if: needs.check_source.outputs.run_cifuzz == 'true' permissions: security-events: write strategy: From 6aedeb8628ee1b6d1ac3cb972b254df131368b95 Mon Sep 17 00:00:00 2001 From: Illia Volochii Date: Sun, 6 Aug 2023 11:50:32 +0300 Subject: [PATCH 4/5] Mention CIFuzz in README for fuzz tests --- Modules/_xxtestfuzz/README.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Modules/_xxtestfuzz/README.rst b/Modules/_xxtestfuzz/README.rst index 42bd02a03cbedd..b951858458c82f 100644 --- a/Modules/_xxtestfuzz/README.rst +++ b/Modules/_xxtestfuzz/README.rst @@ -13,6 +13,9 @@ oss-fuzz will regularly pull from CPython, discover all the tests in automatically be run in oss-fuzz, while also being smoke-tested as part of CPython's test suite. +In addition, the tests are run on GitHub Actions using CIFuzz for PRs to the +main branch changing relevant files. + Adding a new fuzz test ---------------------- From c5cec4ab720d35db79ae50157a8c516c2a041d46 Mon Sep 17 00:00:00 2001 From: Illia Volochii Date: Mon, 9 Oct 2023 15:42:36 +0300 Subject: [PATCH 5/5] Add `cifuzz` to `allowed-failures` and `allowed-skips` --- .github/workflows/build.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 31cc72e6581253..277042dfda1903 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -615,6 +615,7 @@ jobs: build_ubuntu_ssltests, build_win32, build_win_arm64, + cifuzz, test_hypothesis, allowed-skips: >- ${{ @@ -638,6 +639,13 @@ jobs: ' || '' }} + ${{ + !fromJSON(needs.check_source.outputs.run_cifuzz) + && ' + cifuzz, + ' + || '' + }} ${{ !fromJSON(needs.check_source.outputs.run_hypothesis) && '