GitHub Actions

.github/workflows/ci.yml

@@ -0,0 +1,99 @@
+name: CI
+on: [push, pull_request]
+jobs:
+  test:
+    strategy:
+      matrix:
+        include:
+          - name: Tests
+            command: bin/tests --postgresql-host localhost
+            needs-python: ${{ true }}
+            needs-node: ${{ false }}
+          - name: Lint
+            command: bin/lint
+            needs-python: ${{ true }}
+            needs-node: ${{ false }}
+          - name: Documentation
+            command: make docs BINDIR="$(dirname $(which python))"
+            needs-python: ${{ true }}
+            needs-node: ${{ false }}
+          - name: Dependencies
+            command: make github-actions-deps BINDIR="$(dirname $(which python))"
+            needs-python: ${{ true }}
+            needs-node: ${{ false }}
+          - name: Licenses
+            command: make licenses BINDIR="$(dirname $(which python))"
+            needs-python: ${{ true }}
+            needs-node: ${{ false }}
+          - name: Translations
+            command: make translations BINDIR="$(dirname $(which python))"
+            needs-python: ${{ true }}
+            needs-node: ${{ false }}
+          - name: Static Tests
+            command: bin/static_tests
+            needs-python: ${{ false }}
+            needs-node: ${{ true }}
+          - name: Static Lint
+            command: bin/static_lint
+            needs-python: ${{ false }}
+            needs-node: ${{ true }}
+          - name: Static Pipeline
+            command: bin/static_pipeline
+            needs-python: ${{ false }}
+            needs-node: ${{ true }}
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:10.1
+        ports:
+          - 5432:5432
+    name: ${{ matrix.name }}
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v2
+      - name: Install platform dependencies
+        run: sudo apt -y install libcurl4-openssl-dev libssl-dev pkg-config
+      - uses: actions/setup-python@v2
+        if: ${{ matrix.needs-python }}
+        with:
+          python-version: 3.8.2
+      - name: Cache Python dependencies
+        if: ${{ matrix.needs-python }}
+        uses: actions/cache@v2
+        env:
+          cache-name: warehouse-cache-pip
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-${{ github.job }}-${{ env.cache-name }}-${{ hashFiles('requirements.txt', 'requirements/*.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ github.job }}-${{ env.cache-name }}-
+            ${{ runner.os }}-${{ github.job }}-
+            ${{ runner.os }}-
+      - name: Install Python dependencies
+        if: ${{ matrix.needs-python }}
+        run: |
+          pip install -U pip setuptools wheel
+          pip install -r requirements.txt
+          pip install -r requirements/dev.txt
+      - uses: actions/setup-node@v2
+        if: ${{ matrix.needs-node }}
+        with:
+          node-version: 14.4.0
+      - name: Cache Node dependencies
+        if: ${{ matrix.needs-node }}
+        uses: actions/cache@v2
+        env:
+          cache-name: warehouse-cache-npm
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-build-${{ github.job }}-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-build-${{ github.job }}-${{ env.cache-name }}-
+            ${{ runner.os }}-build-${{ github.job }}-
+            ${{ runner.os }}-build-
+            ${{ runner.os }}-
+      - name: Install Node dependencies
+        if: ${{ matrix.needs-node }}
+        run: npm ci
+      - name: Run ${{ matrix.name }}
+        run: ${{ matrix.command }}

Dockerfile

@@ -111,7 +111,7 @@ RUN set -x \
             install --no-binary hiredis \
                     -r /tmp/requirements/deploy.txt \
                     -r /tmp/requirements/main.txt \
-                    $(if [ "$DEVEL" = "yes" ]; then echo '-r /tmp/requirements/tests.txt'; fi) \
+                    $(if [ "$DEVEL" = "yes" ]; then echo '-r /tmp/requirements/tests.txt -r /tmp/requirements/lint.txt'; fi) \
                     $(if [ "$THEME_REPO" != "" ]; then echo '-r /tmp/requirements/theme.txt'; fi) \
     && find /opt/warehouse -name '*.pyc' -delete
 

Makefile

@@ -2,6 +2,8 @@ BINDIR = $(PWD)/.state/env/bin
 TRAVIS := $(shell echo "$${TRAVIS:-false}")
 PR := $(shell echo "$${TRAVIS_PULL_REQUEST:-false}")
 BRANCH := $(shell echo "$${TRAVIS_BRANCH:-master}")
+GITHUB_ACTIONS := $(shell echo "$${GITHUB_ACTIONS:-false}")
+GITHUB_BASE_REF := $(shell echo "$${GITHUB_BASE_REF:-false}")
 DB := example
 IPYTHON := no
 LOCALES := $(shell .state/env/bin/python -c "from warehouse.i18n import KNOWN_LOCALES; print(' '.join(set(KNOWN_LOCALES)-{'en'}))")
@@ -99,26 +101,15 @@ static_pipeline: .state/docker-build
 								  PATH="/opt/warehouse/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \
 								  bin/static_pipeline $(T) $(TESTARGS)
 
-reformat: .state/env/pyvenv.cfg
-	$(BINDIR)/isort *.py warehouse/ tests/
-	$(BINDIR)/black *.py warehouse/ tests/
-
-lint: .state/env/pyvenv.cfg
-	$(BINDIR)/flake8 .
-	$(BINDIR)/black --check *.py warehouse/ tests/
-	$(BINDIR)/isort --check *.py warehouse/ tests/
-	$(BINDIR)/doc8 --allow-long-titles README.rst CONTRIBUTING.rst docs/ --ignore-path docs/_build/
-	$(BINDIR)/curlylint ./warehouse/templates
-
-ifneq ($(TRAVIS), false)
-	# We're on Travis, so we can lint static files locally
-	./node_modules/.bin/eslint 'warehouse/static/js/**' '**.js' 'tests/frontend/**' --ignore-pattern 'warehouse/static/js/vendor/**'
-	./node_modules/.bin/sass-lint --verbose
-else
-	# We're not on Travis, so we should lint static files inside the static container
-	docker-compose run --rm static ./node_modules/.bin/eslint 'warehouse/static/js/**' '**.js' 'tests/frontend/**' --ignore-pattern 'warehouse/static/js/vendor/**'
-	docker-compose run --rm static ./node_modules/.bin/sass-lint --verbose
-endif
+reformat: .state/docker-build
+	docker-compose run --rm web env -i ENCODING="C.UTF-8" \
+								  PATH="/opt/warehouse/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \
+								  bin/reformat
+
+lint: .state/docker-build
+	docker-compose run --rm web env -i ENCODING="C.UTF-8" \
+								  PATH="/opt/warehouse/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \
+								  bin/lint && bin/static_lint
 
 docs: .state/env/pyvenv.cfg
 	$(MAKE) -C docs/ doctest SPHINXOPTS="-W" SPHINXBUILD="$(BINDIR)/sphinx-build"
@@ -139,9 +130,21 @@ deps: .state/env/pyvenv.cfg
 	rm -r $(TMPDIR)
 	$(BINDIR)/pip check
 
+github-actions-deps:
+ifneq ($(GITHUB_BASE_REF), false)
+	git fetch origin $(GITHUB_BASE_REF):refs/remotes/origin/$(GITHUB_BASE_REF)
+	# Check that the following diff will exit with 0 or 1
+	git diff --name-only FETCH_HEAD || test $? -le 1 || exit 1
+	# Make the dependencies if any changed files are requirements files, otherwise exit
+	git diff --name-only FETCH_HEAD | grep '^requirements/' || exit 0 && $(MAKE) deps
+endif
+
 travis-deps:
 ifneq ($(PR), false)
 	git fetch origin $(BRANCH):refs/remotes/origin/$(BRANCH)
+	# Check that the following diff will exit with 0 or 1
+	git diff --name-only $(BRANCH) || test $? -le 1 || exit 1
+	# Make the dependencies if any changed files are requirements files, otherwise exit
 	git diff --name-only $(BRANCH) | grep '^requirements/' || exit 0 && $(MAKE) deps
 endif
 
@@ -200,7 +203,7 @@ build-mos: compile-pot
 		done
 
 translations: compile-pot
-ifneq ($(TRAVIS), false)
+ifneq ($(filter false,$(TRAVIS) $(GITHUB_ACTIONS)),)
 	git diff --quiet ./warehouse/locale/messages.pot || (echo "There are outstanding translations, run 'make translations' and commit the changes."; exit 1)
 else
 endif

bin/lint

@@ -0,0 +1,20 @@
+#!/bin/bash
+set -e
+
+# Click requires us to ensure we have a well configured environment to run
+# our click commands. So we'll set our environment to ensure our locale is
+# correct.
+export LC_ALL="${ENCODING:-en_US.UTF-8}"
+export LANG="${ENCODING:-en_US.UTF-8}"
+
+COMMAND_ARGS=$@
+
+# Print all the followng commands
+set -x
+
+# Actually run our tests.
+python -m flake8 .
+python -m black --check *.py warehouse/ tests/
+python -m isort --check *.py warehouse/ tests/
+python -m doc8 --allow-long-titles README.rst CONTRIBUTING.rst docs/ --ignore-path docs/_build/
+python -m curlylint ./warehouse/templates

bin/reformat

@@ -0,0 +1,14 @@
+#!/bin/bash
+set -e
+
+# Click requires us to ensure we have a well configured environment to run
+# our click commands. So we'll set our environment to ensure our locale is
+# correct.
+export LC_ALL="${ENCODING:-en_US.UTF-8}"
+export LANG="${ENCODING:-en_US.UTF-8}"
+
+# Print all the followng commands
+set -x
+
+python -m isort *.py warehouse/ tests/
+python -m black *.py warehouse/ tests/

bin/static_lint

@@ -0,0 +1,13 @@
+#!/bin/bash
+set -e
+set -x
+
+# Click requires us to ensure we have a well configured environment to run
+# our click commands. So we'll set our environment to ensure our locale is
+# correct.
+export LC_ALL="${ENCODING:-en_US.UTF-8}"
+export LANG="${ENCODING:-en_US.UTF-8}"
+
+# Actually run our tests.
+./node_modules/.bin/eslint 'warehouse/static/js/**' '**.js' 'tests/frontend/**' --ignore-pattern 'warehouse/static/js/vendor/**'
+./node_modules/.bin/sass-lint --verbose

docker-compose.yml

@@ -85,6 +85,7 @@ services:
       - ./htmlcov:/opt/warehouse/src/htmlcov:z
       - .coveragerc:/opt/warehouse/src/.coveragerc:z
       - packages:/var/opt/warehouse/packages
+      - ./bin:/opt/warehouse/src/bin:z
     ports:
       - "80:8000"