From 9cf222c809a2b322e99795b6998b32ccf993334d Mon Sep 17 00:00:00 2001 From: Dustin Ingram Date: Fri, 12 Feb 2021 01:31:13 -0600 Subject: [PATCH 1/7] GitHub Actions --- .github/workflows/ci.yml | 73 ++++++++++++++++++++++++++++++++++++++++ Makefile | 14 ++++++-- 2 files changed, 84 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000000..7a4df6e6bf38 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,73 @@ +name: CI +on: [push, pull_request] +jobs: + test: + strategy: + matrix: + include: + - name: Tests + command: bin/tests --postgresql-host localhost + - name: Lint + command: make lint BINDIR="$(dirname $(which python))" + - name: Documentation + command: make docs BINDIR="$(dirname $(which python))" + - name: Dependencies + command: make github-actions-deps BINDIR="$(dirname $(which python))" + - name: Licenses + command: make licenses BINDIR="$(dirname $(which python))" + - name: Translations + command: make translations BINDIR="$(dirname $(which python))" + - name: Static Tests + command: bin/static_tests + - name: Static Pipeline + command: bin/static_pipeline + runs-on: ubuntu-latest + services: + postgres: + image: postgres:10.1 + ports: + - 5432:5432 + name: ${{ matrix.name }} + steps: + - name: Check out repository + uses: actions/checkout@v2 + - name: Install platform dependencies + run: sudo apt -y install libcurl4-openssl-dev libssl-dev pkg-config + - uses: actions/setup-python@v2 + with: + python-version: 3.8.2 + - name: Cache Python dependencies + uses: actions/cache@v2 + env: + cache-name: warehouse-cache-pip + with: + path: ~/.cache/pip + key: ${{ runner.os }}-${{ github.job }}-${{ env.cache-name }}-${{ hashFiles('requirements.txt', 'requirements/*.txt') }} + restore-keys: | + ${{ runner.os }}-${{ github.job }}-${{ env.cache-name }}- + ${{ runner.os }}-${{ github.job }}- + ${{ runner.os }}- + - name: Install Python dependencies + run: | + pip install -U pip setuptools wheel + pip install -r requirements.txt + pip install -r requirements/dev.txt + - uses: actions/setup-node@v2 + with: + node-version: 14.4.0 + - name: Cache Node dependencies + uses: actions/cache@v2 + env: + cache-name: warehouse-cache-npm + with: + path: ~/.npm + key: ${{ runner.os }}-build-${{ github.job }}-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-build-${{ github.job }}-${{ env.cache-name }}- + ${{ runner.os }}-build-${{ github.job }}- + ${{ runner.os }}-build- + ${{ runner.os }}- + - name: Install Node dependencies + run: npm ci + - name: Run ${{ matrix.name }} + run: ${{ matrix.command }} diff --git a/Makefile b/Makefile index d80b785d2a1b..565841449eeb 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,8 @@ BINDIR = $(PWD)/.state/env/bin TRAVIS := $(shell echo "$${TRAVIS:-false}") PR := $(shell echo "$${TRAVIS_PULL_REQUEST:-false}") BRANCH := $(shell echo "$${TRAVIS_BRANCH:-master}") +GITHUB_ACTIONS := $(shell echo "$${GITHUB_ACTIONS:-false}") +GITHUB_HEAD_REF := $(shell echo "$${GITHUB_HEAD_REF:-false}") DB := example IPYTHON := no LOCALES := $(shell .state/env/bin/python -c "from warehouse.i18n import KNOWN_LOCALES; print(' '.join(set(KNOWN_LOCALES)-{'en'}))") @@ -110,12 +112,12 @@ lint: .state/env/pyvenv.cfg $(BINDIR)/doc8 --allow-long-titles README.rst CONTRIBUTING.rst docs/ --ignore-path docs/_build/ $(BINDIR)/curlylint ./warehouse/templates -ifneq ($(TRAVIS), false) - # We're on Travis, so we can lint static files locally +ifneq ($(filter false,$(TRAVIS) $(GITHUB_ACTIONS)),) + # We're either on Travis or GitHub Actions, so we can lint static files locally ./node_modules/.bin/eslint 'warehouse/static/js/**' '**.js' 'tests/frontend/**' --ignore-pattern 'warehouse/static/js/vendor/**' ./node_modules/.bin/sass-lint --verbose else - # We're not on Travis, so we should lint static files inside the static container + # We're not on Travis or GitHub Actions, so we should lint static files inside the static container docker-compose run --rm static ./node_modules/.bin/eslint 'warehouse/static/js/**' '**.js' 'tests/frontend/**' --ignore-pattern 'warehouse/static/js/vendor/**' docker-compose run --rm static ./node_modules/.bin/sass-lint --verbose endif @@ -139,6 +141,12 @@ deps: .state/env/pyvenv.cfg rm -r $(TMPDIR) $(BINDIR)/pip check +github-actions-deps: +ifneq ($(GITHUB_HEAD_REF), false) + git fetch origin $(GITHUB_HEAD_REF):refs/remotes/origin/$(GITHUB_HEAD_REF) + git diff --name-only $(GITHUB_HEAD_REF) | grep '^requirements/' || exit 0 && $(MAKE) deps +endif + travis-deps: ifneq ($(PR), false) git fetch origin $(BRANCH):refs/remotes/origin/$(BRANCH) From 1aa1b5387dc3659c8206b8de193aad036f79b2c8 Mon Sep 17 00:00:00 2001 From: Dustin Ingram Date: Fri, 12 Feb 2021 12:33:27 -0600 Subject: [PATCH 2/7] Use GITHUB_BASE_REF instead --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 565841449eeb..c938aed78ee6 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ TRAVIS := $(shell echo "$${TRAVIS:-false}") PR := $(shell echo "$${TRAVIS_PULL_REQUEST:-false}") BRANCH := $(shell echo "$${TRAVIS_BRANCH:-master}") GITHUB_ACTIONS := $(shell echo "$${GITHUB_ACTIONS:-false}") -GITHUB_HEAD_REF := $(shell echo "$${GITHUB_HEAD_REF:-false}") +GITHUB_BASE_REF := $(shell echo "$${GITHUB_BASE_REF:-false}") DB := example IPYTHON := no LOCALES := $(shell .state/env/bin/python -c "from warehouse.i18n import KNOWN_LOCALES; print(' '.join(set(KNOWN_LOCALES)-{'en'}))") @@ -142,9 +142,9 @@ deps: .state/env/pyvenv.cfg $(BINDIR)/pip check github-actions-deps: -ifneq ($(GITHUB_HEAD_REF), false) - git fetch origin $(GITHUB_HEAD_REF):refs/remotes/origin/$(GITHUB_HEAD_REF) - git diff --name-only $(GITHUB_HEAD_REF) | grep '^requirements/' || exit 0 && $(MAKE) deps +ifneq ($(GITHUB_BASE_REF), false) + git fetch origin $(GITHUB_BASE_REF):refs/remotes/origin/$(GITHUB_BASE_REF) + git diff --name-only FETCH_HEAD | grep '^requirements/' || exit 0 && $(MAKE) deps endif travis-deps: From f1a9d4557e3d1fa4be651aead7828a15816ed2d7 Mon Sep 17 00:00:00 2001 From: Dustin Ingram Date: Fri, 19 Feb 2021 10:07:37 -0600 Subject: [PATCH 3/7] Add conditionals --- .github/workflows/ci.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7a4df6e6bf38..2e2758935f07 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,20 +7,36 @@ jobs: include: - name: Tests command: bin/tests --postgresql-host localhost + needs-python: ${{ true }} + needs-node: ${{ false }} - name: Lint command: make lint BINDIR="$(dirname $(which python))" + needs-python: ${{ true }} + needs-node: ${{ true }} - name: Documentation command: make docs BINDIR="$(dirname $(which python))" + needs-python: ${{ true }} + needs-node: ${{ false }} - name: Dependencies command: make github-actions-deps BINDIR="$(dirname $(which python))" + needs-python: ${{ true }} + needs-node: ${{ false }} - name: Licenses command: make licenses BINDIR="$(dirname $(which python))" + needs-python: ${{ true }} + needs-node: ${{ false }} - name: Translations command: make translations BINDIR="$(dirname $(which python))" + needs-python: ${{ true }} + needs-node: ${{ false }} - name: Static Tests command: bin/static_tests + needs-python: ${{ false }} + needs-node: ${{ true }} - name: Static Pipeline command: bin/static_pipeline + needs-python: ${{ false }} + needs-node: ${{ true }} runs-on: ubuntu-latest services: postgres: @@ -34,9 +50,11 @@ jobs: - name: Install platform dependencies run: sudo apt -y install libcurl4-openssl-dev libssl-dev pkg-config - uses: actions/setup-python@v2 + if: ${{ matrix.needs-python }} with: python-version: 3.8.2 - name: Cache Python dependencies + if: ${{ matrix.needs-python }} uses: actions/cache@v2 env: cache-name: warehouse-cache-pip @@ -48,14 +66,17 @@ jobs: ${{ runner.os }}-${{ github.job }}- ${{ runner.os }}- - name: Install Python dependencies + if: ${{ matrix.needs-python }} run: | pip install -U pip setuptools wheel pip install -r requirements.txt pip install -r requirements/dev.txt - uses: actions/setup-node@v2 + if: ${{ matrix.needs-node }} with: node-version: 14.4.0 - name: Cache Node dependencies + if: ${{ matrix.needs-node }} uses: actions/cache@v2 env: cache-name: warehouse-cache-npm @@ -68,6 +89,7 @@ jobs: ${{ runner.os }}-build- ${{ runner.os }}- - name: Install Node dependencies + if: ${{ matrix.needs-node }} run: npm ci - name: Run ${{ matrix.name }} run: ${{ matrix.command }} From 24e90f8cca282b7daff6f29122aae50ad63471af Mon Sep 17 00:00:00 2001 From: Dustin Ingram Date: Fri, 19 Feb 2021 22:31:59 -0600 Subject: [PATCH 4/7] Add missing check for CI --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c938aed78ee6..c794b0b8e78f 100644 --- a/Makefile +++ b/Makefile @@ -208,7 +208,7 @@ build-mos: compile-pot done translations: compile-pot -ifneq ($(TRAVIS), false) +ifneq ($(filter false,$(TRAVIS) $(GITHUB_ACTIONS)),) git diff --quiet ./warehouse/locale/messages.pot || (echo "There are outstanding translations, run 'make translations' and commit the changes."; exit 1) else endif From 5bea462dbed95e7d91c004c71eec079b05a7352a Mon Sep 17 00:00:00 2001 From: Dustin Ingram Date: Fri, 19 Feb 2021 22:37:07 -0600 Subject: [PATCH 5/7] Split linting, and lint in container --- .github/workflows/ci.yml | 8 ++++++-- Dockerfile | 2 +- Makefile | 20 ++++---------------- bin/lint | 20 ++++++++++++++++++++ bin/static_lint | 13 +++++++++++++ docker-compose.yml | 1 + 6 files changed, 45 insertions(+), 19 deletions(-) create mode 100755 bin/lint create mode 100755 bin/static_lint diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2e2758935f07..be6957d3d2a8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,9 +10,9 @@ jobs: needs-python: ${{ true }} needs-node: ${{ false }} - name: Lint - command: make lint BINDIR="$(dirname $(which python))" + command: bin/lint needs-python: ${{ true }} - needs-node: ${{ true }} + needs-node: ${{ false }} - name: Documentation command: make docs BINDIR="$(dirname $(which python))" needs-python: ${{ true }} @@ -33,6 +33,10 @@ jobs: command: bin/static_tests needs-python: ${{ false }} needs-node: ${{ true }} + - name: Static Lint + command: bin/static_lint + needs-python: ${{ false }} + needs-node: ${{ true }} - name: Static Pipeline command: bin/static_pipeline needs-python: ${{ false }} diff --git a/Dockerfile b/Dockerfile index 917e01a6b6fc..6afa6c3f237f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -111,7 +111,7 @@ RUN set -x \ install --no-binary hiredis \ -r /tmp/requirements/deploy.txt \ -r /tmp/requirements/main.txt \ - $(if [ "$DEVEL" = "yes" ]; then echo '-r /tmp/requirements/tests.txt'; fi) \ + $(if [ "$DEVEL" = "yes" ]; then echo '-r /tmp/requirements/tests.txt -r /tmp/requirements/lint.txt'; fi) \ $(if [ "$THEME_REPO" != "" ]; then echo '-r /tmp/requirements/theme.txt'; fi) \ && find /opt/warehouse -name '*.pyc' -delete diff --git a/Makefile b/Makefile index c794b0b8e78f..34b4602a8b92 100644 --- a/Makefile +++ b/Makefile @@ -105,22 +105,10 @@ reformat: .state/env/pyvenv.cfg $(BINDIR)/isort *.py warehouse/ tests/ $(BINDIR)/black *.py warehouse/ tests/ -lint: .state/env/pyvenv.cfg - $(BINDIR)/flake8 . - $(BINDIR)/black --check *.py warehouse/ tests/ - $(BINDIR)/isort --check *.py warehouse/ tests/ - $(BINDIR)/doc8 --allow-long-titles README.rst CONTRIBUTING.rst docs/ --ignore-path docs/_build/ - $(BINDIR)/curlylint ./warehouse/templates - -ifneq ($(filter false,$(TRAVIS) $(GITHUB_ACTIONS)),) - # We're either on Travis or GitHub Actions, so we can lint static files locally - ./node_modules/.bin/eslint 'warehouse/static/js/**' '**.js' 'tests/frontend/**' --ignore-pattern 'warehouse/static/js/vendor/**' - ./node_modules/.bin/sass-lint --verbose -else - # We're not on Travis or GitHub Actions, so we should lint static files inside the static container - docker-compose run --rm static ./node_modules/.bin/eslint 'warehouse/static/js/**' '**.js' 'tests/frontend/**' --ignore-pattern 'warehouse/static/js/vendor/**' - docker-compose run --rm static ./node_modules/.bin/sass-lint --verbose -endif +lint: .state/docker-build + docker-compose run --rm web env -i ENCODING="C.UTF-8" \ + PATH="/opt/warehouse/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \ + bin/lint && bin/static_lint docs: .state/env/pyvenv.cfg $(MAKE) -C docs/ doctest SPHINXOPTS="-W" SPHINXBUILD="$(BINDIR)/sphinx-build" diff --git a/bin/lint b/bin/lint new file mode 100755 index 000000000000..9989f8c696b6 --- /dev/null +++ b/bin/lint @@ -0,0 +1,20 @@ +#!/bin/bash +set -e + +# Click requires us to ensure we have a well configured environment to run +# our click commands. So we'll set our environment to ensure our locale is +# correct. +export LC_ALL="${ENCODING:-en_US.UTF-8}" +export LANG="${ENCODING:-en_US.UTF-8}" + +COMMAND_ARGS=$@ + +# Print all the followng commands +set -x + +# Actually run our tests. +python -m flake8 . +python -m black --check *.py warehouse/ tests/ +python -m isort --check *.py warehouse/ tests/ +python -m doc8 --allow-long-titles README.rst CONTRIBUTING.rst docs/ --ignore-path docs/_build/ +python -m curlylint ./warehouse/templates diff --git a/bin/static_lint b/bin/static_lint new file mode 100755 index 000000000000..829dc327b2df --- /dev/null +++ b/bin/static_lint @@ -0,0 +1,13 @@ +#!/bin/bash +set -e +set -x + +# Click requires us to ensure we have a well configured environment to run +# our click commands. So we'll set our environment to ensure our locale is +# correct. +export LC_ALL="${ENCODING:-en_US.UTF-8}" +export LANG="${ENCODING:-en_US.UTF-8}" + +# Actually run our tests. +./node_modules/.bin/eslint 'warehouse/static/js/**' '**.js' 'tests/frontend/**' --ignore-pattern 'warehouse/static/js/vendor/**' +./node_modules/.bin/sass-lint --verbose diff --git a/docker-compose.yml b/docker-compose.yml index 567f5aca8cd8..e066d2c6220a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -85,6 +85,7 @@ services: - ./htmlcov:/opt/warehouse/src/htmlcov:z - .coveragerc:/opt/warehouse/src/.coveragerc:z - packages:/var/opt/warehouse/packages + - ./bin:/opt/warehouse/src/bin:z ports: - "80:8000" From 638e2631d6bfcceabed8245106e0c999bef3a388 Mon Sep 17 00:00:00 2001 From: Dustin Ingram Date: Fri, 19 Feb 2021 23:29:12 -0600 Subject: [PATCH 6/7] Guard against bad diff --- Makefile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Makefile b/Makefile index 34b4602a8b92..c7bb3f60f95c 100644 --- a/Makefile +++ b/Makefile @@ -132,12 +132,18 @@ deps: .state/env/pyvenv.cfg github-actions-deps: ifneq ($(GITHUB_BASE_REF), false) git fetch origin $(GITHUB_BASE_REF):refs/remotes/origin/$(GITHUB_BASE_REF) + # Check that the following diff will exit with 0 or 1 + git diff --name-only FETCH_HEAD || test $? -le 1 || exit 1 + # Make the dependencies if any changed files are requirements files, otherwise exit git diff --name-only FETCH_HEAD | grep '^requirements/' || exit 0 && $(MAKE) deps endif travis-deps: ifneq ($(PR), false) git fetch origin $(BRANCH):refs/remotes/origin/$(BRANCH) + # Check that the following diff will exit with 0 or 1 + git diff --name-only $(BRANCH) || test $? -le 1 || exit 1 + # Make the dependencies if any changed files are requirements files, otherwise exit git diff --name-only $(BRANCH) | grep '^requirements/' || exit 0 && $(MAKE) deps endif From a40a042cfb96563fb2dd40b54a294ea539fd965d Mon Sep 17 00:00:00 2001 From: Dustin Ingram Date: Fri, 19 Feb 2021 23:41:17 -0600 Subject: [PATCH 7/7] Why not do reformat as well while we're at it --- Makefile | 7 ++++--- bin/reformat | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) create mode 100755 bin/reformat diff --git a/Makefile b/Makefile index c7bb3f60f95c..6d31323985c3 100644 --- a/Makefile +++ b/Makefile @@ -101,9 +101,10 @@ static_pipeline: .state/docker-build PATH="/opt/warehouse/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \ bin/static_pipeline $(T) $(TESTARGS) -reformat: .state/env/pyvenv.cfg - $(BINDIR)/isort *.py warehouse/ tests/ - $(BINDIR)/black *.py warehouse/ tests/ +reformat: .state/docker-build + docker-compose run --rm web env -i ENCODING="C.UTF-8" \ + PATH="/opt/warehouse/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \ + bin/reformat lint: .state/docker-build docker-compose run --rm web env -i ENCODING="C.UTF-8" \ diff --git a/bin/reformat b/bin/reformat new file mode 100755 index 000000000000..89098348bada --- /dev/null +++ b/bin/reformat @@ -0,0 +1,14 @@ +#!/bin/bash +set -e + +# Click requires us to ensure we have a well configured environment to run +# our click commands. So we'll set our environment to ensure our locale is +# correct. +export LC_ALL="${ENCODING:-en_US.UTF-8}" +export LANG="${ENCODING:-en_US.UTF-8}" + +# Print all the followng commands +set -x + +python -m isort *.py warehouse/ tests/ +python -m black *.py warehouse/ tests/