Skip to content

Commit 3521a34

Browse files
didi
authored
Run all dev actions in containers (#10803)
* Run docs build in container * Run translations build in container * Run depchecker in container * Run requirements build in container * Run license check in container * Clean up Makefile * Update docs * Remove unused COMMAND_ARGS, use double quotes * Add missing semicolons * Move github-actions-deps out of Makefile * Set ENCODING and PATH in dev/environment * Fix typo with translations build
1 parent b26deed commit 3521a34

File tree

16 files changed

+206
-164
lines changed

16 files changed

+206
-164
lines changed

‎.github/workflows/ci.yml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,19 +21,19 @@ jobs:
2121
needs-python: ${{ true }}
2222
needs-node: ${{ false }}
2323
- name: Documentation
24-
command: make docs BINDIR="$(dirname $(which python))"
24+
command: bin/docs
2525
needs-python: ${{ true }}
2626
needs-node: ${{ false }}
2727
- name: Dependencies
28-
command: make github-actions-deps BINDIR="$(dirname $(which python))"
28+
command: bin/github-actions-deps
2929
needs-python: ${{ true }}
3030
needs-node: ${{ false }}
3131
- name: Licenses
32-
command: make licenses BINDIR="$(dirname $(which python))"
32+
command: bin/licenses
3333
needs-python: ${{ true }}
3434
needs-node: ${{ false }}
3535
- name: Translations
36-
command: make translations BINDIR="$(dirname $(which python))"
36+
command: bin/translations
3737
needs-python: ${{ true }}
3838
needs-node: ${{ false }}
3939
- name: Static Tests

‎Dockerfile

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ RUN set -x \
103103
install --no-deps --no-binary hiredis \
104104
-r /tmp/requirements/deploy.txt \
105105
-r /tmp/requirements/main.txt \
106-
$(if [ "$DEVEL" = "yes" ]; then echo '-r /tmp/requirements/tests.txt -r /tmp/requirements/lint.txt'; fi) \
106+
$(if [ "$DEVEL" = "yes" ]; then echo '-r /tmp/requirements/tests.txt -r /tmp/requirements/lint.txt -r /tmp/requirements/docs.txt'; fi) \
107107
&& find /opt/warehouse -name '*.pyc' -delete
108108

109109

@@ -138,7 +138,7 @@ RUN set -x \
138138
&& apt-get update \
139139
&& apt-get install --no-install-recommends -y \
140140
libpq5 libxml2 libxslt1.1 libcurl4 \
141-
$(if [ "$DEVEL" = "yes" ]; then echo 'bash libjpeg62 postgresql-client'; fi) \
141+
$(if [ "$DEVEL" = "yes" ]; then echo 'bash libjpeg62 postgresql-client build-essential libffi-dev libxml2-dev libxslt-dev libpq-dev libcurl4-openssl-dev libssl-dev'; fi) \
142142
&& apt-get clean \
143143
&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
144144

‎Makefile

Lines changed: 20 additions & 126 deletions
Original file line numberDiff line numberDiff line change
@@ -1,39 +1,13 @@
1-
BINDIR = $(PWD)/.state/env/bin
2-
GITHUB_ACTIONS := $(shell echo "$${GITHUB_ACTIONS:-false}")
31
GITHUB_BASE_REF := $(shell echo "$${GITHUB_BASE_REF:-false}")
42
DB := example
53
IPYTHON := no
6-
LOCALES := $(shell .state/env/bin/python -c "from warehouse.i18n import KNOWN_LOCALES; print(' '.join(set(KNOWN_LOCALES)-{'en'}))")
74

85
# set environment variable WAREHOUSE_IPYTHON_SHELL=1 if IPython
96
# needed in development environment
107
ifeq ($(WAREHOUSE_IPYTHON_SHELL), 1)
118
IPYTHON = yes
129
endif
1310

14-
define DEPCHECKER
15-
import sys
16-
from pip_api import parse_requirements
17-
18-
left, right = sys.argv[1:3]
19-
left_reqs = parse_requirements(left).keys()
20-
right_reqs = parse_requirements(right).keys()
21-
22-
extra_in_left = left_reqs - right_reqs
23-
extra_in_right = right_reqs - left_reqs
24-
25-
if extra_in_left:
26-
for dep in sorted(extra_in_left):
27-
print("- {}".format(dep))
28-
29-
if extra_in_right:
30-
for dep in sorted(extra_in_right):
31-
print("+ {}".format(dep))
32-
33-
if extra_in_left or extra_in_right:
34-
sys.exit(1)
35-
endef
36-
3711
default:
3812
@echo "Call a specific subcommand:"
3913
@echo
@@ -44,24 +18,6 @@ default:
4418
@echo
4519
@exit 1
4620

47-
.state/env/pyvenv.cfg: requirements/dev.txt requirements/docs.txt requirements/lint.txt requirements/ipython.txt
48-
# Create our Python 3.9 virtual environment
49-
rm -rf .state/env
50-
python3.9 -m venv .state/env
51-
52-
# install/upgrade general requirements
53-
.state/env/bin/python -m pip install --upgrade pip setuptools wheel
54-
55-
# install various types of requirements
56-
.state/env/bin/python -m pip install -r requirements/dev.txt
57-
.state/env/bin/python -m pip install -r requirements/docs.txt
58-
.state/env/bin/python -m pip install -r requirements/lint.txt
59-
60-
# install ipython if enabled
61-
ifeq ($(IPYTHON),"yes")
62-
.state/env/bin/python -m pip install -r requirements/ipython.txt
63-
endif
64-
6521
.state/docker-build: Dockerfile package.json package-lock.json requirements/main.txt requirements/deploy.txt
6622
# Build our docker containers for this project.
6723
docker-compose build --build-arg IPYTHON=$(IPYTHON) --force-rm web
@@ -84,60 +40,34 @@ debug: .state/docker-build
8440
docker-compose run --rm --service-ports web
8541

8642
tests: .state/docker-build
87-
docker-compose run --rm web env -i ENCODING="C.UTF-8" \
88-
PATH="/opt/warehouse/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \
89-
bin/tests --postgresql-host db $(T) $(TESTARGS)
43+
docker-compose run --rm web bin/tests --postgresql-host db $(T) $(TESTARGS)
9044

9145
static_tests: .state/docker-build
92-
docker-compose run --rm static env -i ENCODING="C.UTF-8" \
93-
PATH="/opt/warehouse/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \
94-
bin/static_tests $(T) $(TESTARGS)
46+
docker-compose run --rm static bin/static_tests $(T) $(TESTARGS)
9547

9648
static_pipeline: .state/docker-build
97-
docker-compose run --rm static env -i ENCODING="C.UTF-8" \
98-
PATH="/opt/warehouse/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \
99-
bin/static_pipeline $(T) $(TESTARGS)
49+
docker-compose run --rm static bin/static_pipeline $(T) $(TESTARGS)
10050

10151
reformat: .state/docker-build
102-
docker-compose run --rm web env -i ENCODING="C.UTF-8" \
103-
PATH="/opt/warehouse/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \
104-
bin/reformat
52+
docker-compose run --rm web bin/reformat
10553

10654
lint: .state/docker-build
107-
docker-compose run --rm web env -i ENCODING="C.UTF-8" \
108-
PATH="/opt/warehouse/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \
109-
bin/lint && bin/static_lint
110-
111-
docs: .state/env/pyvenv.cfg
112-
$(MAKE) -C docs/ doctest SPHINXOPTS="-W" SPHINXBUILD="$(BINDIR)/sphinx-build"
113-
$(MAKE) -C docs/ html SPHINXOPTS="-W" SPHINXBUILD="$(BINDIR)/sphinx-build"
114-
115-
licenses:
116-
bin/licenses
117-
118-
export DEPCHECKER
119-
deps: .state/env/pyvenv.cfg
120-
$(eval TMPDIR := $(shell mktemp -d))
121-
$(BINDIR)/pip-compile --upgrade --allow-unsafe -o $(TMPDIR)/deploy.txt requirements/deploy.in > /dev/null
122-
$(BINDIR)/pip-compile --upgrade --allow-unsafe -o $(TMPDIR)/main.txt requirements/main.in > /dev/null
123-
$(BINDIR)/pip-compile --upgrade --allow-unsafe -o $(TMPDIR)/lint.txt requirements/lint.in > /dev/null
124-
echo "$$DEPCHECKER" | $(BINDIR)/python - $(TMPDIR)/deploy.txt requirements/deploy.txt
125-
echo "$$DEPCHECKER" | $(BINDIR)/python - $(TMPDIR)/main.txt requirements/main.txt
126-
echo "$$DEPCHECKER" | $(BINDIR)/python - $(TMPDIR)/lint.txt requirements/lint.txt
127-
rm -r $(TMPDIR)
128-
$(BINDIR)/pip check
129-
130-
requirements/%.txt: requirements/%.in .state/env/pyvenv.cfg
131-
$(BINDIR)/pip-compile --allow-unsafe --generate-hashes --output-file=$@ $<
132-
133-
github-actions-deps:
134-
ifneq ($(GITHUB_BASE_REF), false)
135-
git fetch origin $(GITHUB_BASE_REF):refs/remotes/origin/$(GITHUB_BASE_REF)
136-
# Check that the following diff will exit with 0 or 1
137-
git diff --name-only FETCH_HEAD || test $? -le 1 || exit 1
138-
# Make the dependencies if any changed files are requirements files, otherwise exit
139-
git diff --name-only FETCH_HEAD | grep '^requirements/' || exit 0 && $(MAKE) deps
140-
endif
55+
docker-compose run --rm web bin/lint && bin/static_lint
56+
57+
docs: .state/docker-build
58+
docker-compose run --rm web bin/docs
59+
60+
licenses: .state/docker-build
61+
docker-compose run --rm web bin/licenses
62+
63+
deps: .state/docker-build
64+
docker-compose run --rm web bin/deps
65+
66+
translations: .state/docker-build
67+
docker-compose run --rm web bin/translations
68+
69+
requirements/%.txt: requirements/%.in
70+
docker-compose run --rm web bin/pip-compile --allow-unsafe --generate-hashes --output-file=$@ $<
14171

14272
initdb:
14373
docker-compose run --rm web psql -h db -d postgres -U postgres -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname ='warehouse';"
@@ -164,40 +94,4 @@ purge: stop clean
16494
stop:
16595
docker-compose down -v
16696

167-
compile-pot: .state/env/pyvenv.cfg
168-
PYTHONPATH=$(PWD) $(BINDIR)/pybabel extract \
169-
-F babel.cfg \
170-
--omit-header \
171-
--output="warehouse/locale/messages.pot" \
172-
warehouse
173-
174-
init-po: .state/env/pyvenv.cfg
175-
$(BINDIR)/pybabel init \
176-
--input-file="warehouse/locale/messages.pot" \
177-
--output-dir="warehouse/locale/" \
178-
--locale="$(L)"
179-
180-
update-po: .state/env/pyvenv.cfg
181-
$(BINDIR)/pybabel update \
182-
--input-file="warehouse/locale/messages.pot" \
183-
--output-file="warehouse/locale/$(L)/LC_MESSAGES/messages.po" \
184-
--locale="$(L)"
185-
186-
compile-po: .state/env/pyvenv.cfg
187-
$(BINDIR)/pybabel compile \
188-
--input-file="warehouse/locale/$(L)/LC_MESSAGES/messages.po" \
189-
--directory="warehouse/locale/" \
190-
--locale="$(L)"
191-
192-
build-mos: compile-pot
193-
for LOCALE in $(LOCALES) ; do \
194-
L=$$LOCALE $(MAKE) compile-po ; \
195-
done
196-
197-
translations: compile-pot
198-
ifneq ($(GITHUB_ACTIONS), false)
199-
git diff --quiet ./warehouse/locale/messages.pot || (echo "There are outstanding translations, run 'make translations' and commit the changes."; exit 1)
200-
else
201-
endif
202-
20397
.PHONY: default build serve initdb shell tests docs deps clean purge debug stop compile-pot

‎bin/depchecker.py

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
# Licensed under the Apache License, Version 2.0 (the "License");
2+
# you may not use this file except in compliance with the License.
3+
# You may obtain a copy of the License at
4+
#
5+
# http://www.apache.org/licenses/LICENSE-2.0
6+
#
7+
# Unless required by applicable law or agreed to in writing, software
8+
# distributed under the License is distributed on an "AS IS" BASIS,
9+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10+
# See the License for the specific language governing permissions and
11+
# limitations under the License.import sys
12+
13+
import sys
14+
15+
from pip_api import parse_requirements
16+
17+
left, right = sys.argv[1:3]
18+
left_reqs = parse_requirements(left).keys()
19+
right_reqs = parse_requirements(right).keys()
20+
21+
extra_in_left = left_reqs - right_reqs
22+
extra_in_right = right_reqs - left_reqs
23+
24+
if extra_in_left:
25+
for dep in sorted(extra_in_left):
26+
print("- {}".format(dep))
27+
28+
if extra_in_right:
29+
for dep in sorted(extra_in_right):
30+
print("+ {}".format(dep))
31+
32+
if extra_in_left or extra_in_right:
33+
sys.exit(1)

‎bin/deps

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
#!/bin/bash
2+
set -e
3+
4+
# Click requires us to ensure we have a well configured environment to run
5+
# our click commands. So we'll set our environment to ensure our locale is
6+
# correct.
7+
export LC_ALL="${ENCODING:-en_US.UTF-8}"
8+
export LANG="${ENCODING:-en_US.UTF-8}"
9+
10+
# Print all the followng commands
11+
set -x
12+
13+
export TMPDIR=$(mktemp -d)
14+
pip-compile --upgrade --allow-unsafe -o $TMPDIR/deploy.txt requirements/deploy.in > /dev/null
15+
pip-compile --upgrade --allow-unsafe -o $TMPDIR/main.txt requirements/main.in > /dev/null
16+
pip-compile --upgrade --allow-unsafe -o $TMPDIR/lint.txt requirements/lint.in > /dev/null
17+
python bin/depchecker.py $TMPDIR/deploy.txt requirements/deploy.txt
18+
python bin/depchecker.py $TMPDIR/main.txt requirements/main.txt
19+
python bin/depchecker.py $TMPDIR/lint.txt requirements/lint.txt
20+
rm -r $TMPDIR
21+
pip check

‎bin/docs

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/bin/bash
2+
set -e
3+
4+
# Click requires us to ensure we have a well configured environment to run
5+
# our click commands. So we'll set our environment to ensure our locale is
6+
# correct.
7+
export LC_ALL="${ENCODING:-en_US.UTF-8}"
8+
export LANG="${ENCODING:-en_US.UTF-8}"
9+
10+
# Print all the followng commands
11+
set -x
12+
13+
make -C docs/ doctest SPHINXOPTS="-W"
14+
make -C docs/ html SPHINXOPTS="-W"

‎bin/github-actions-deps

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#!/bin/bash
2+
set -e
3+
4+
# Click requires us to ensure we have a well configured environment to run
5+
# our click commands. So we'll set our environment to ensure our locale is
6+
# correct.
7+
export LC_ALL="${ENCODING:-en_US.UTF-8}"
8+
export LANG="${ENCODING:-en_US.UTF-8}"
9+
10+
# Print all the followng commands
11+
set -x
12+
13+
if [[ -z "${GITHUB_BASE_REF}" ]]; then
14+
git fetch origin $(GITHUB_BASE_REF):refs/remotes/origin/$(GITHUB_BASE_REF)
15+
# Check that the following diff will exit with 0 or 1
16+
git diff --name-only FETCH_HEAD || test $? -le 1 || exit 1
17+
# Make the dependencies if any changed files are requirements files, otherwise exit
18+
git diff --name-only FETCH_HEAD | grep '^requirements/' || exit 0 && bin/deps
19+
fi

‎bin/licenses

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,11 @@ for fn in $(find . -type f \( \
1010
! -name "*.min.js" \
1111
-name "*.js" \) \
1212
! -path "./.state*" \
13-
! -path "./node_modules*" \
13+
! -path "./docs*" \
1414
! -path "./htmlcov*" \
15+
! -path "./node_modules*" \
16+
! -path "./warehouse/admin/static/dist*" \
17+
! -path "./warehouse/static/dist*" \
1518
! -path "./warehouse/static/html*" \
1619
! -path "./warehouse/static/js/vendor*"); do
1720
# Check for license in first 5 lines (allows for shebang, encoding and handles comment character in various languages)

‎bin/lint

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,6 @@ set -e
77
export LC_ALL="${ENCODING:-en_US.UTF-8}"
88
export LANG="${ENCODING:-en_US.UTF-8}"
99

10-
COMMAND_ARGS=$@
11-
1210
# Print all the followng commands
1311
set -x
1412

‎bin/pip-compile

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#!/bin/bash
2+
set -e
3+
4+
# Click requires us to ensure we have a well configured environment to run
5+
# our click commands. So we'll set our environment to ensure our locale is
6+
# correct.
7+
export LC_ALL="${ENCODING:-en_US.UTF-8}"
8+
export LANG="${ENCODING:-en_US.UTF-8}"
9+
10+
COMMAND_ARGS="$@"
11+
12+
# Print all the followng commands
13+
set -x
14+
15+
pip-compile $COMMAND_ARGS

‎bin/tests

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ set -e
77
export LC_ALL="${ENCODING:-en_US.UTF-8}"
88
export LANG="${ENCODING:-en_US.UTF-8}"
99

10-
COMMAND_ARGS=$@
10+
COMMAND_ARGS="$@"
1111

1212
# Test the postgres connection
1313
while [ $# -gt 0 ]; do

‎bin/translations

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
#!/bin/bash
2+
set -e
3+
4+
# Click requires us to ensure we have a well configured environment to run
5+
# our click commands. So we'll set our environment to ensure our locale is
6+
# correct.
7+
export LC_ALL="${ENCODING:-en_US.UTF-8}"
8+
export LANG="${ENCODING:-en_US.UTF-8}"
9+
10+
# Print all the followng commands
11+
set -x
12+
13+
make -C warehouse/locale/ translations

‎dev/environment

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
ENCODING=C.UTF-8
2+
PATH=/opt/warehouse/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
3+
14
WAREHOUSE_ENV=development
25
WAREHOUSE_TOKEN=insecuretoken
36

‎docker-compose.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,7 @@ services:
8888
- sponsorlogos:/var/opt/warehouse/sponsorlogos
8989
- simple:/var/opt/warehouse/simple
9090
- ./bin:/opt/warehouse/src/bin:z
91+
- ./requirements:/opt/warehouse/src/requirements:z
9192
ports:
9293
- "${WEB_PORT:-80}:8000"
9394
depends_on:

0 commit comments

Comments
 (0)