diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000000..be6957d3d2a8 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,99 @@ +name: CI +on: [push, pull_request] +jobs: + test: + strategy: + matrix: + include: + - name: Tests + command: bin/tests --postgresql-host localhost + needs-python: ${{ true }} + needs-node: ${{ false }} + - name: Lint + command: bin/lint + needs-python: ${{ true }} + needs-node: ${{ false }} + - name: Documentation + command: make docs BINDIR="$(dirname $(which python))" + needs-python: ${{ true }} + needs-node: ${{ false }} + - name: Dependencies + command: make github-actions-deps BINDIR="$(dirname $(which python))" + needs-python: ${{ true }} + needs-node: ${{ false }} + - name: Licenses + command: make licenses BINDIR="$(dirname $(which python))" + needs-python: ${{ true }} + needs-node: ${{ false }} + - name: Translations + command: make translations BINDIR="$(dirname $(which python))" + needs-python: ${{ true }} + needs-node: ${{ false }} + - name: Static Tests + command: bin/static_tests + needs-python: ${{ false }} + needs-node: ${{ true }} + - name: Static Lint + command: bin/static_lint + needs-python: ${{ false }} + needs-node: ${{ true }} + - name: Static Pipeline + command: bin/static_pipeline + needs-python: ${{ false }} + needs-node: ${{ true }} + runs-on: ubuntu-latest + services: + postgres: + image: postgres:10.1 + ports: + - 5432:5432 + name: ${{ matrix.name }} + steps: + - name: Check out repository + uses: actions/checkout@v2 + - name: Install platform dependencies + run: sudo apt -y install libcurl4-openssl-dev libssl-dev pkg-config + - uses: actions/setup-python@v2 + if: ${{ matrix.needs-python }} + with: + python-version: 3.8.2 + - name: Cache Python dependencies + if: ${{ matrix.needs-python }} + uses: actions/cache@v2 + env: + cache-name: warehouse-cache-pip + with: + path: ~/.cache/pip + key: ${{ runner.os }}-${{ github.job }}-${{ env.cache-name }}-${{ hashFiles('requirements.txt', 'requirements/*.txt') }} + restore-keys: | + ${{ runner.os }}-${{ github.job }}-${{ env.cache-name }}- + ${{ runner.os }}-${{ github.job }}- + ${{ runner.os }}- + - name: Install Python dependencies + if: ${{ matrix.needs-python }} + run: | + pip install -U pip setuptools wheel + pip install -r requirements.txt + pip install -r requirements/dev.txt + - uses: actions/setup-node@v2 + if: ${{ matrix.needs-node }} + with: + node-version: 14.4.0 + - name: Cache Node dependencies + if: ${{ matrix.needs-node }} + uses: actions/cache@v2 + env: + cache-name: warehouse-cache-npm + with: + path: ~/.npm + key: ${{ runner.os }}-build-${{ github.job }}-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-build-${{ github.job }}-${{ env.cache-name }}- + ${{ runner.os }}-build-${{ github.job }}- + ${{ runner.os }}-build- + ${{ runner.os }}- + - name: Install Node dependencies + if: ${{ matrix.needs-node }} + run: npm ci + - name: Run ${{ matrix.name }} + run: ${{ matrix.command }} diff --git a/Dockerfile b/Dockerfile index 917e01a6b6fc..6afa6c3f237f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -111,7 +111,7 @@ RUN set -x \ install --no-binary hiredis \ -r /tmp/requirements/deploy.txt \ -r /tmp/requirements/main.txt \ - $(if [ "$DEVEL" = "yes" ]; then echo '-r /tmp/requirements/tests.txt'; fi) \ + $(if [ "$DEVEL" = "yes" ]; then echo '-r /tmp/requirements/tests.txt -r /tmp/requirements/lint.txt'; fi) \ $(if [ "$THEME_REPO" != "" ]; then echo '-r /tmp/requirements/theme.txt'; fi) \ && find /opt/warehouse -name '*.pyc' -delete diff --git a/Makefile b/Makefile index d80b785d2a1b..6d31323985c3 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,8 @@ BINDIR = $(PWD)/.state/env/bin TRAVIS := $(shell echo "$${TRAVIS:-false}") PR := $(shell echo "$${TRAVIS_PULL_REQUEST:-false}") BRANCH := $(shell echo "$${TRAVIS_BRANCH:-master}") +GITHUB_ACTIONS := $(shell echo "$${GITHUB_ACTIONS:-false}") +GITHUB_BASE_REF := $(shell echo "$${GITHUB_BASE_REF:-false}") DB := example IPYTHON := no LOCALES := $(shell .state/env/bin/python -c "from warehouse.i18n import KNOWN_LOCALES; print(' '.join(set(KNOWN_LOCALES)-{'en'}))") @@ -99,26 +101,15 @@ static_pipeline: .state/docker-build PATH="/opt/warehouse/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \ bin/static_pipeline $(T) $(TESTARGS) -reformat: .state/env/pyvenv.cfg - $(BINDIR)/isort *.py warehouse/ tests/ - $(BINDIR)/black *.py warehouse/ tests/ - -lint: .state/env/pyvenv.cfg - $(BINDIR)/flake8 . - $(BINDIR)/black --check *.py warehouse/ tests/ - $(BINDIR)/isort --check *.py warehouse/ tests/ - $(BINDIR)/doc8 --allow-long-titles README.rst CONTRIBUTING.rst docs/ --ignore-path docs/_build/ - $(BINDIR)/curlylint ./warehouse/templates - -ifneq ($(TRAVIS), false) - # We're on Travis, so we can lint static files locally - ./node_modules/.bin/eslint 'warehouse/static/js/**' '**.js' 'tests/frontend/**' --ignore-pattern 'warehouse/static/js/vendor/**' - ./node_modules/.bin/sass-lint --verbose -else - # We're not on Travis, so we should lint static files inside the static container - docker-compose run --rm static ./node_modules/.bin/eslint 'warehouse/static/js/**' '**.js' 'tests/frontend/**' --ignore-pattern 'warehouse/static/js/vendor/**' - docker-compose run --rm static ./node_modules/.bin/sass-lint --verbose -endif +reformat: .state/docker-build + docker-compose run --rm web env -i ENCODING="C.UTF-8" \ + PATH="/opt/warehouse/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \ + bin/reformat + +lint: .state/docker-build + docker-compose run --rm web env -i ENCODING="C.UTF-8" \ + PATH="/opt/warehouse/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \ + bin/lint && bin/static_lint docs: .state/env/pyvenv.cfg $(MAKE) -C docs/ doctest SPHINXOPTS="-W" SPHINXBUILD="$(BINDIR)/sphinx-build" @@ -139,9 +130,21 @@ deps: .state/env/pyvenv.cfg rm -r $(TMPDIR) $(BINDIR)/pip check +github-actions-deps: +ifneq ($(GITHUB_BASE_REF), false) + git fetch origin $(GITHUB_BASE_REF):refs/remotes/origin/$(GITHUB_BASE_REF) + # Check that the following diff will exit with 0 or 1 + git diff --name-only FETCH_HEAD || test $? -le 1 || exit 1 + # Make the dependencies if any changed files are requirements files, otherwise exit + git diff --name-only FETCH_HEAD | grep '^requirements/' || exit 0 && $(MAKE) deps +endif + travis-deps: ifneq ($(PR), false) git fetch origin $(BRANCH):refs/remotes/origin/$(BRANCH) + # Check that the following diff will exit with 0 or 1 + git diff --name-only $(BRANCH) || test $? -le 1 || exit 1 + # Make the dependencies if any changed files are requirements files, otherwise exit git diff --name-only $(BRANCH) | grep '^requirements/' || exit 0 && $(MAKE) deps endif @@ -200,7 +203,7 @@ build-mos: compile-pot done translations: compile-pot -ifneq ($(TRAVIS), false) +ifneq ($(filter false,$(TRAVIS) $(GITHUB_ACTIONS)),) git diff --quiet ./warehouse/locale/messages.pot || (echo "There are outstanding translations, run 'make translations' and commit the changes."; exit 1) else endif diff --git a/bin/lint b/bin/lint new file mode 100755 index 000000000000..9989f8c696b6 --- /dev/null +++ b/bin/lint @@ -0,0 +1,20 @@ +#!/bin/bash +set -e + +# Click requires us to ensure we have a well configured environment to run +# our click commands. So we'll set our environment to ensure our locale is +# correct. +export LC_ALL="${ENCODING:-en_US.UTF-8}" +export LANG="${ENCODING:-en_US.UTF-8}" + +COMMAND_ARGS=$@ + +# Print all the followng commands +set -x + +# Actually run our tests. +python -m flake8 . +python -m black --check *.py warehouse/ tests/ +python -m isort --check *.py warehouse/ tests/ +python -m doc8 --allow-long-titles README.rst CONTRIBUTING.rst docs/ --ignore-path docs/_build/ +python -m curlylint ./warehouse/templates diff --git a/bin/reformat b/bin/reformat new file mode 100755 index 000000000000..89098348bada --- /dev/null +++ b/bin/reformat @@ -0,0 +1,14 @@ +#!/bin/bash +set -e + +# Click requires us to ensure we have a well configured environment to run +# our click commands. So we'll set our environment to ensure our locale is +# correct. +export LC_ALL="${ENCODING:-en_US.UTF-8}" +export LANG="${ENCODING:-en_US.UTF-8}" + +# Print all the followng commands +set -x + +python -m isort *.py warehouse/ tests/ +python -m black *.py warehouse/ tests/ diff --git a/bin/static_lint b/bin/static_lint new file mode 100755 index 000000000000..829dc327b2df --- /dev/null +++ b/bin/static_lint @@ -0,0 +1,13 @@ +#!/bin/bash +set -e +set -x + +# Click requires us to ensure we have a well configured environment to run +# our click commands. So we'll set our environment to ensure our locale is +# correct. +export LC_ALL="${ENCODING:-en_US.UTF-8}" +export LANG="${ENCODING:-en_US.UTF-8}" + +# Actually run our tests. +./node_modules/.bin/eslint 'warehouse/static/js/**' '**.js' 'tests/frontend/**' --ignore-pattern 'warehouse/static/js/vendor/**' +./node_modules/.bin/sass-lint --verbose diff --git a/docker-compose.yml b/docker-compose.yml index 567f5aca8cd8..e066d2c6220a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -85,6 +85,7 @@ services: - ./htmlcov:/opt/warehouse/src/htmlcov:z - .coveragerc:/opt/warehouse/src/.coveragerc:z - packages:/var/opt/warehouse/packages + - ./bin:/opt/warehouse/src/bin:z ports: - "80:8000"