diff --git a/.github/workflows/desktop.yml b/.github/workflows/desktop.yml index 3d5c5073dd..87e7ed7622 100644 --- a/.github/workflows/desktop.yml +++ b/.github/workflows/desktop.yml @@ -312,6 +312,10 @@ jobs: with: ref: ${{needs.check_and_prepare.outputs.github_ref}} submodules: true + - name: Set env vars + run: | + echo "VCPKG_TRIPLET=x64-linux" >> $GITHUB_ENV + echo "VCPKG_RESPONSE_FILE=external/vcpkg_${{ env.VCPKG_TRIPLET }}_response_file.txt" >> $GITHUB_ENV - name: Cache vcpkg C++ dependencies id: cache_vcpkg uses: actions/cache@v2 @@ -328,12 +332,11 @@ jobs: with: python-version: ${{ fromJson(needs.prepare_matrix.outputs.matrix_python_version)[0] }} - name: Prepare for integration tests - uses: nick-invision/retry@v2 - with: - shell: bash - timeout_minutes: 15 - max_attempts: 3 - command: scripts/gha/install_test_workflow_prereqs.sh -p Desktop -a 'x64' -s 'openssl' + run: | + pip install -r scripts/gha/requirements.txt + - name: Install OpenSSL + run: | + sudo apt install openssl - name: Build integration tests shell: bash env: diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 926aa81352..1f3fd84eb0 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -44,7 +44,6 @@ env: triggerLabelFull: "tests-requested: full" triggerLabelQuick: "tests-requested: quick" pythonVersion: '3.7' - xcodeVersion: '13.3.1' artifactRetentionDays: 2 GITHUB_TOKEN: ${{ github.token }} # All self-hosted ARM Mac runners should have this label. Due to how @@ -159,7 +158,7 @@ jobs: timeout_minutes: 1 max_attempts: 3 shell: bash - command: pip install -r scripts/gha/python_requirements.txt + command: pip install -r scripts/gha/requirements.txt - id: matrix_config run: | if [[ "${{ steps.set_outputs.outputs.requested_tests }}" == "expanded" ]]; then @@ -265,30 +264,93 @@ jobs: ssl_variant: openssl arch: arm64 steps: + - name: setup Xcode version (macos) + if: runner.os == 'macOS' + run: sudo xcode-select -s /Applications/Xcode_${{ fromJson(needs.check_and_prepare.outputs.xcode_version)[0] }}.app/Contents/Developer - uses: actions/checkout@v2 with: ref: ${{needs.check_and_prepare.outputs.github_ref}} submodules: true + - name: Store git credentials for all git commands + # Forces all git commands to use authenticated https, to prevent throttling. + shell: bash + run: | + git config --global credential.helper 'store --file /tmp/git-credentials' + echo 'https://${{ github.token }}@github.com' > /tmp/git-credentials + - name: Enable Git Long-paths Support + if: runner.os == 'Windows' + run: git config --system core.longpaths true + - name: Set env vars (Linux) + if: startsWith(matrix.os, 'ubuntu') + run: echo "VCPKG_TRIPLET=x64-linux" >> $GITHUB_ENV + - name: Set env vars (macOS) + if: startsWith(matrix.os, 'macos') + run: echo "VCPKG_TRIPLET=x64-osx" >> $GITHUB_ENV + - name: Set env vars (Windows) + shell: bash + if: startsWith(matrix.os, 'windows') + run: echo "VCPKG_TRIPLET=x64-windows-static" >> $GITHUB_ENV + - name: Set env vars (all) + shell: bash + run: echo "VCPKG_RESPONSE_FILE=external/vcpkg_${{ env.VCPKG_TRIPLET }}_response_file.txt" >> $GITHUB_ENV + - name: Add msbuild to PATH (Windows) + if: startsWith(matrix.os, 'windows') + uses: microsoft/setup-msbuild@v1.0.2 + - name: Cache vcpkg C++ dependencies + id: cache_vcpkg + uses: actions/cache@v2 + with: + path: external/vcpkg/installed + key: dev-vcpkg-${{ env.VCPKG_TRIPLET }}-${{ hashFiles(format('{0}', env.VCPKG_RESPONSE_FILE)) }}-${{ hashFiles('.git/modules/external/vcpkg/HEAD') }} - name: Setup python uses: actions/setup-python@v2 with: python-version: ${{ env.pythonVersion }} + - name: Update homebrew (avoid bintray errors) + uses: nick-invision/retry@v2 + if: startsWith(matrix.os, 'macos') + with: + timeout_minutes: 10 + max_attempts: 3 + command: | + # Temporarily here until Github runners have updated their version of + # homebrew. This prevents errors arising from the shut down of + # binutils, used by older version of homebrew for hosting packages. + brew update - name: Install Desktop SDK & integration tests prerequisites uses: nick-invision/retry@v2 with: - shell: bash timeout_minutes: 15 max_attempts: 3 - command: scripts/gha/install_test_workflow_prereqs.sh -p Desktop -a '${{ matrix.arch }}' -s '${{ matrix.ssl_variant }}' - - name: Add msbuild to PATH (Windows) - if: startsWith(matrix.os, 'windows') - uses: microsoft/setup-msbuild@v1.0.2 - - name: Cache vcpkg C++ dependencies - id: cache_vcpkg - uses: actions/cache@v2 + command: | + python scripts/gha/install_prereqs_desktop.py --gha_build --arch '${{ matrix.arch }}' --ssl '${{ matrix.ssl_variant }}' + pip install -r scripts/gha/requirements.txt + python scripts/gha/restore_secrets.py --passphrase "${{ secrets.TEST_SECRET }}" + - name: Install OpenSSL (Windows) + if: matrix.ssl_variant == 'openssl' && startsWith(matrix.os, 'windows') + uses: nick-invision/retry@v2 with: - path: external/vcpkg/installed - key: dev-vcpkg-${{ env.VCPKG_TRIPLET }}-${{ hashFiles(format('{0}', env.VCPKG_RESPONSE_FILE)) }}-${{ hashFiles('.git/modules/external/vcpkg/HEAD') }} + timeout_minutes: 4 + max_attempts: 3 + shell: bash + command: choco install openssl -r + - name: Install OpenSSL (macOS) + if: matrix.ssl_variant == 'openssl' && startsWith(matrix.os, 'macos') + uses: nick-invision/retry@v2 + with: + timeout_minutes: 2 + max_attempts: 3 + command: | + brew install openssl + # brew won't overwrite MacOS system default OpenSSL, so force it here. + echo "OPENSSL_ROOT_DIR=/usr/local/opt/openssl@1.1" >> $GITHUB_ENV + - name: Install OpenSSL (Linux) + if: matrix.ssl_variant == 'openssl' && startsWith(matrix.os, 'ubuntu') + uses: nick-invision/retry@v2 + with: + timeout_minutes: 2 + max_attempts: 3 + command: sudo apt install openssl - name: Fetch prebuilt packaged SDK from previous run uses: dawidd6/action-download-artifact@v2 if: ${{ github.event.inputs.test_packaged_sdk != '' }} @@ -301,7 +363,6 @@ jobs: env: CCACHE_DIR: ${{ github.workspace }}/ccache_dir run: | - python scripts/gha/restore_secrets.py --passphrase "${{ secrets.TEST_SECRET }}" declare -a additional_flags if [[ -n "${{ github.event.inputs.test_packaged_sdk }}" ]]; then # Building integration tests against a packaged SDK. @@ -384,10 +445,22 @@ jobs: matrix: os: ${{ fromJson(needs.check_and_prepare.outputs.matrix_os) }} steps: + - name: setup Xcode version (macos) + if: runner.os == 'macOS' + run: sudo xcode-select -s /Applications/Xcode_${{ fromJson(needs.check_and_prepare.outputs.xcode_version)[0] }}.app/Contents/Developer - uses: actions/checkout@v2 with: ref: ${{needs.check_and_prepare.outputs.github_ref}} submodules: true + - name: Store git credentials for all git commands + # Forces all git commands to use authenticated https, to prevent throttling. + shell: bash + run: | + git config --global credential.helper 'store --file /tmp/git-credentials' + echo 'https://${{ github.token }}@github.com' > /tmp/git-credentials + - name: Enable Git Long-paths Support + if: runner.os == 'Windows' + run: git config --system core.longpaths true - name: Add msbuild to PATH (Windows) if: startsWith(matrix.os, 'windows') uses: microsoft/setup-msbuild@v1.0.2 @@ -401,13 +474,29 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ env.pythonVersion }} + - name: Update homebrew (avoid bintray errors) + uses: nick-invision/retry@v2 + if: startsWith(matrix.os, 'macos') + with: + timeout_minutes: 10 + max_attempts: 3 + command: | + # Temporarily here until Github runners have updated their version of + # homebrew. This prevents errors arising from the shut down of + # binutils, used by older version of homebrew for hosting packages. + brew update - name: Install Android SDK & integration tests prerequisites uses: nick-invision/retry@v2 with: - shell: bash timeout_minutes: 10 max_attempts: 3 - command: scripts/gha/install_test_workflow_prereqs.sh -p Android + shell: bash + command: | + build_scripts/android/install_prereqs.sh + echo "NDK_ROOT=/tmp/android-ndk-r21e" >> $GITHUB_ENV + echo "ANDROID_NDK_HOME=/tmp/android-ndk-r21e" >> $GITHUB_ENV + pip install -r scripts/gha/requirements.txt + python scripts/gha/restore_secrets.py --passphrase "${{ secrets.TEST_SECRET }}" - name: Fetch prebuilt packaged SDK from previous run uses: dawidd6/action-download-artifact@v2 if: ${{ github.event.inputs.test_packaged_sdk != '' }} @@ -418,7 +507,6 @@ jobs: - name: Build integration tests shell: bash run: | - python scripts/gha/restore_secrets.py --passphrase "${{ secrets.TEST_SECRET }}" declare -a additional_flags if [[ -n "${{ github.event.inputs.test_packaged_sdk }}" ]]; then # Building integration tests against a packaged SDK. @@ -492,20 +580,43 @@ jobs: matrix: os: [macos-12] steps: + - name: setup Xcode version (macos) + if: runner.os == 'macOS' + run: sudo xcode-select -s /Applications/Xcode_${{ fromJson(needs.check_and_prepare.outputs.xcode_version)[0] }}.app/Contents/Developer - uses: actions/checkout@v2 with: ref: ${{needs.check_and_prepare.outputs.github_ref}} submodules: true + - name: Store git credentials for all git commands + # Forces all git commands to use authenticated https, to prevent throttling. + shell: bash + run: | + git config --global credential.helper 'store --file /tmp/git-credentials' + echo 'https://${{ github.token }}@github.com' > /tmp/git-credentials - name: Setup python uses: actions/setup-python@v2 with: python-version: ${{ env.pythonVersion }} + - name: Update homebrew (avoid bintray errors) + uses: nick-invision/retry@v2 + if: startsWith(matrix.os, 'macos') + with: + timeout_minutes: 10 + max_attempts: 3 + command: | + # Temporarily here until Github runners have updated their version of + # homebrew. This prevents errors arising from the shut down of + # binutils, used by older version of homebrew for hosting packages. + brew update - name: Install iOS SDK & integration tests prerequisites uses: nick-invision/retry@v2 with: - timeout_minutes: 3 + timeout_minutes: 2 max_attempts: 3 - command: scripts/gha/install_test_workflow_prereqs.sh -p iOS + command: | + build_scripts/ios/install_prereqs.sh + pip install -r scripts/gha/requirements.txt + python scripts/gha/restore_secrets.py --passphrase "${{ secrets.TEST_SECRET }}" - name: Fetch prebuilt packaged SDK from previous run uses: dawidd6/action-download-artifact@v2 if: ${{ github.event.inputs.test_packaged_sdk != '' }} @@ -516,7 +627,6 @@ jobs: - name: Build integration tests shell: bash run: | - python scripts/gha/restore_secrets.py --passphrase "${{ secrets.TEST_SECRET }}" declare -a additional_flags if [[ -n "${{ github.event.inputs.test_packaged_sdk }}" ]]; then # Building integration tests against a packaged SDK. @@ -591,20 +701,43 @@ jobs: matrix: os: [macos-12] steps: + - name: setup Xcode version (macos) + if: runner.os == 'macOS' + run: sudo xcode-select -s /Applications/Xcode_${{ fromJson(needs.check_and_prepare.outputs.xcode_version)[0] }}.app/Contents/Developer - uses: actions/checkout@v2 with: ref: ${{needs.check_and_prepare.outputs.github_ref}} submodules: true + - name: Store git credentials for all git commands + # Forces all git commands to use authenticated https, to prevent throttling. + shell: bash + run: | + git config --global credential.helper 'store --file /tmp/git-credentials' + echo 'https://${{ github.token }}@github.com' > /tmp/git-credentials - name: Setup python uses: actions/setup-python@v2 with: python-version: ${{ env.pythonVersion }} + - name: Update homebrew (avoid bintray errors) + uses: nick-invision/retry@v2 + if: startsWith(matrix.os, 'macos') + with: + timeout_minutes: 10 + max_attempts: 3 + command: | + # Temporarily here until Github runners have updated their version of + # homebrew. This prevents errors arising from the shut down of + # binutils, used by older version of homebrew for hosting packages. + brew update - name: Install tvOS SDK & integration tests prerequisites uses: nick-invision/retry@v2 with: - timeout_minutes: 3 + timeout_minutes: 2 max_attempts: 3 - command: scripts/gha/install_test_workflow_prereqs.sh -p tvOS + command: | + build_scripts/tvos/install_prereqs.sh + pip install -r scripts/gha/requirements.txt + python scripts/gha/restore_secrets.py --passphrase "${{ secrets.TEST_SECRET }}" - name: Fetch prebuilt packaged SDK from previous run uses: dawidd6/action-download-artifact@v2 if: ${{ github.event.inputs.test_packaged_sdk != '' }} @@ -615,7 +748,6 @@ jobs: - name: Build integration tests shell: bash run: | - python scripts/gha/restore_secrets.py --passphrase "${{ secrets.TEST_SECRET }}" declare -a additional_flags if [[ -n "${{ github.event.inputs.test_packaged_sdk }}" ]]; then # Building integration tests against a packaged SDK. @@ -715,6 +847,9 @@ jobs: - os: macos-12 arch: arm64 steps: + - name: setup Xcode version (macos) + if: runner.os == 'macOS' + run: sudo xcode-select -s /Applications/Xcode_${{ fromJson(needs.check_and_prepare.outputs.xcode_version)[0] }}.app/Contents/Developer - uses: actions/checkout@v2 with: ref: ${{needs.check_and_prepare.outputs.github_ref}} @@ -727,20 +862,28 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ env.pythonVersion }} - - name: Install prerequisites for testing + - name: install platform support libraries for running uses: nick-invision/retry@v2 with: - shell: bash timeout_minutes: 15 max_attempts: 3 - command: scripts/gha/install_test_workflow_prereqs.sh -p Desktop -t true -a '${{ matrix.arch }}' -s '${{ matrix.ssl_variant }}' + command: python scripts/gha/install_prereqs_desktop.py --gha_build --running_only --arch '${{ matrix.arch }}' --ssl '${{ matrix.ssl_variant }}' + - name: Set up Node (14) uses: actions/setup-node@v2 with: node-version: 14.x - name: Setup Firestore Emulator run: | - npm install -g firebase-tools + npm install -g firebase-tools + - name: Setup integration test deps + uses: nick-invision/retry@v2 + with: + timeout_minutes: 2 + max_attempts: 3 + shell: bash + command: | + pip install -r scripts/gha/requirements.txt - name: Setup java uses: actions/setup-java@v3 with: @@ -965,6 +1108,9 @@ jobs: - android_device: "android_latest" test_type: "uitest" steps: + - name: setup Xcode version (macos) + if: runner.os == 'macOS' + run: sudo xcode-select -s /Applications/Xcode_${{ fromJson(needs.check_and_prepare.outputs.xcode_version)[0] }}.app/Contents/Developer - uses: actions/checkout@v2 with: ref: ${{needs.check_and_prepare.outputs.github_ref}} @@ -977,13 +1123,13 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ env.pythonVersion }} - - name: Install prerequisites for testing + - name: Install python deps uses: nick-invision/retry@v2 with: - shell: bash - timeout_minutes: 5 + timeout_minutes: 1 max_attempts: 3 - command: scripts/gha/install_test_workflow_prereqs.sh -p Android -t true + shell: bash + command: pip install -r scripts/gha/requirements.txt - id: get-device-type run: | echo "::set-output name=device_type::$( python scripts/gha/print_matrix_configuration.py -d -k ${{ matrix.android_device }} )" @@ -1094,6 +1240,9 @@ jobs: test_type: "uitest" build_os: [macos-12] steps: + - name: setup Xcode version (macos) + if: runner.os == 'macOS' + run: sudo xcode-select -s /Applications/Xcode_${{ fromJson(needs.check_and_prepare.outputs.xcode_version)[0] }}.app/Contents/Developer - uses: actions/checkout@v2 with: ref: ${{needs.check_and_prepare.outputs.github_ref}} @@ -1106,12 +1255,12 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ env.pythonVersion }} - - name: Install prerequisites for testing + - name: Install python deps uses: nick-invision/retry@v2 with: - timeout_minutes: 3 + timeout_minutes: 1 max_attempts: 3 - command: scripts/gha/install_test_workflow_prereqs.sh -p iOS -t true + command: pip install -r scripts/gha/requirements.txt - id: get-device-type run: | echo "::set-output name=device_type::$( python scripts/gha/print_matrix_configuration.py -d -k ${{ matrix.ios_device }} )" @@ -1210,6 +1359,9 @@ jobs: tvos_device: ${{ fromJson(needs.check_and_prepare.outputs.tvos_device) }} build_os: [macos-12] steps: + - name: setup Xcode version (macos) + if: runner.os == 'macOS' + run: sudo xcode-select -s /Applications/Xcode_${{ fromJson(needs.check_and_prepare.outputs.xcode_version)[0] }}.app/Contents/Developer - uses: actions/checkout@v2 with: ref: ${{needs.check_and_prepare.outputs.github_ref}} @@ -1222,12 +1374,12 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ env.pythonVersion }} - - name: Install prerequisites for testing + - name: Install python deps uses: nick-invision/retry@v2 with: - timeout_minutes: 3 + timeout_minutes: 1 max_attempts: 3 - command: scripts/gha/install_test_workflow_prereqs.sh -p tvOS -t true + command: pip install -r scripts/gha/requirements.txt - name: Setup java for Firestore emulator uses: actions/setup-java@v3 with: @@ -1299,7 +1451,7 @@ jobs: with: python-version: ${{ env.pythonVersion }} - name: Install python deps - run: pip install -r scripts/gha/python_requirements.txt + run: pip install -r scripts/gha/requirements.txt - name: Download log artifacts uses: actions/download-artifact@v3 with: diff --git a/scripts/gha/install_test_workflow_prereqs.sh b/scripts/gha/install_test_workflow_prereqs.sh deleted file mode 100755 index d197439245..0000000000 --- a/scripts/gha/install_test_workflow_prereqs.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/bin/bash -e - -# Copyright 2022 Google LLC - -PLATFORM='Desktop' -TEST_ONLY='' -ARCH='' -SSL='' -# check options -while getopts ":p:a:t:s:" opt; do - case $opt in - p) - PLATFORM=$OPTARG - ;; - a) - ARCH=$OPTARG - ;; - t) - if [[ $OPTARG == true ]]; then - TEST_ONLY="--running_only" - fi - ;; - s) - SSL=$OPTARG - ;; - *) - echo "unknown parameter" - echo $OPTARG - exit 2 - ;; - esac -done - -echo "PLATFORM: ${PLATFORM}" -echo "ARCH: ${ARCH}" -echo "TEST_ONLY: ${TEST_ONLY}" -echo "SSL: ${SSL}" - -OS='' -case "$OSTYPE" in - darwin*) OS='Mac' ;; - linux*) OS='Linux' ;; - msys*) OS='Windows' ;; - *) echo "unknown: $OSTYPE" ;; -esac -echo "OS: ${OS}" - -pip install -r scripts/gha/python_requirements.txt - -git config --global credential.helper 'store --file /tmp/git-credentials' -echo 'https://${GITHUB_TOKEN}@github.com' > /tmp/git-credentials - -if [[ "${OS}" == "Windows" ]]; then - git config --system core.longpaths true -fi - -if [[ "${OS}" == "Mac" ]]; then - sudo xcode-select -s /Applications/Xcode_${xcodeVersion}.app/Contents/Developer - brew update -fi - -if [[ "${PLATFORM}" == "Desktop" ]]; then - if [[ "${OS}" == "Linux" ]]; then - VCPKG_TRIPLET="x64-linux" - elif [[ "${OS}" == "Mac" ]]; then - VCPKG_TRIPLET="x64-osx" - echo "OPENSSL_ROOT_DIR=/usr/local/opt/openssl@1.1" >> $GITHUB_ENV - elif [[ "${OS}" == "Windows" ]]; then - VCPKG_TRIPLET="x64-windows-static" - fi - echo "VCPKG_TRIPLET=${VCPKG_TRIPLET}" >> $GITHUB_ENV - echo "VCPKG_RESPONSE_FILE=external/vcpkg_${VCPKG_TRIPLET}_response_file.txt" >> $GITHUB_ENV - python scripts/gha/install_prereqs_desktop.py --gha_build --arch "${ARCH}" --ssl "${SSL}" ${TEST_ONLY} -fi - -if [[ "${TEST_ONLY}" == "" ]]; then - if [[ "${PLATFORM}" == "Android" ]]; then - echo "NDK_ROOT=/tmp/android-ndk-r21e" >> $GITHUB_ENV - echo "ANDROID_NDK_HOME=/tmp/android-ndk-r21e" >> $GITHUB_ENV - build_scripts/android/install_prereqs.sh - elif [[ "${PLATFORM}" == "iOS" ]]; then - build_scripts/ios/install_prereqs.sh - elif [[ "${PLATFORM}" == "tvOS" ]]; then - build_scripts/tvos/install_prereqs.sh - fi -fi diff --git a/scripts/gha/python_requirements.txt b/scripts/gha/requirements.txt similarity index 100% rename from scripts/gha/python_requirements.txt rename to scripts/gha/requirements.txt index 6541e322ff..5ec721cd3a 100644 --- a/scripts/gha/python_requirements.txt +++ b/scripts/gha/requirements.txt @@ -1,4 +1,4 @@ -attrs absl-py +attrs pytz requests