diff --git a/Dockerfile.build b/Dockerfile.build new file mode 100644 index 000000000000..1a2c1e969876 --- /dev/null +++ b/Dockerfile.build @@ -0,0 +1,40 @@ +FROM python:3.6.6-slim-stretch as python + +WORKDIR /opt/warehouse/src + +# We create an /opt directory with a virtual environment in it to store our +# application in. +RUN set -x \ + && python3 -m venv /opt/warehouse/src/.state/env + + +# Now that we've created our virtual environment, we'll go ahead and update +# our $PATH to refer to it first. +ENV PATH="/opt/warehouse/src/.state/env/bin:${PATH}" + +# Next, we want to update pip, setuptools, and wheel inside of this virtual +# environment to ensure that we have the latest versions of them. +# TODO: We use --require-hashes in our requirements files, but not here, making +# the ones in the requirements files kind of a moot point. We should +# probably pin these too, and update them as we do anything else. +RUN pip --no-cache-dir --disable-pip-version-check install --upgrade pip setuptools wheel + +# We copy this into the docker container prior to copying in the rest of our +# application so that we can skip installing requirements if the only thing +# that has changed is the Warehouse code itself. +COPY requirements /tmp/requirements + +# Install our development dependencies if we're building a development install +# otherwise this will do nothing. +RUN set -x \ + && pip --no-cache-dir --disable-pip-version-check install -r /tmp/requirements/dev.txt + +RUN set -x \ + && pip --no-cache-dir --disable-pip-version-check install -r /tmp/requirements/lint.txt + +RUN set -x \ + && pip --no-cache-dir --disable-pip-version-check install -r /tmp/requirements/docs.txt + +RUN set -x \ + && if [ "$IPYTHON" = "yes" ]; then pip --no-cache-dir --disable-pip-version-check install -r /tmp/requirements/ipython.txt; fi + diff --git a/Makefile b/Makefile index beb526e09121..e0f3aeab12d8 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,8 @@ PR := $(shell echo "$${TRAVIS_PULL_REQUEST:-false}") BRANCH := $(shell echo "$${TRAVIS_BRANCH:-master}") DB := example IPYTHON := no +RUN_IN_DOCKER = +RUN_IN_DOCKER_DEP = .state/env/pyvenv.cfg # set environment variable WAREHOUSE_IPYTHON_SHELL=1 if IPython # needed in development environment @@ -11,6 +13,13 @@ ifeq ($(WAREHOUSE_IPYTHON_SHELL), 1) IPYTHON = yes endif +ifeq ($(TRAVIS), false) + RUN_IN_DOCKER = docker-compose run --rm build + RUN_IN_DOCKER_DEP = .state/docker-build + PYTHON = $(shell $(RUN_IN_DOCKER) which python) + BINDIR = $(shell dirname $(PYTHON)) +endif + define DEPCHECKER import sys @@ -65,9 +74,9 @@ default: .state/env/bin/python -m pip install -r requirements/lint.txt # install ipython if enabled -ifeq ($(IPYTHON),"yes") - .state/env/bin/python -m pip install -r requirements/ipython.txt -endif + ifeq ($(IPYTHON),"yes") + .state/env/bin/python -m pip install -r requirements/ipython.txt + endif .state/docker-build: Dockerfile package.json package-lock.json requirements/main.txt requirements/deploy.txt # Build our docker containers for this project. @@ -101,16 +110,16 @@ tests: bin/tests --postgresql-host db $(T) $(TESTARGS) -reformat: .state/env/pyvenv.cfg - $(BINDIR)/black warehouse/ tests/ +reformat: $(RUN_IN_DOCKER_DEP) + $(RUN_IN_DOCKER) $(BINDIR)/black warehouse/ tests/ -lint: .state/env/pyvenv.cfg - $(BINDIR)/flake8 . - $(BINDIR)/black --check warehouse/ tests/ - $(BINDIR)/doc8 --allow-long-titles README.rst CONTRIBUTING.rst docs/ --ignore-path docs/_build/ +lint: $(RUN_IN_DOCKER_DEP) + $(RUN_IN_DOCKER) $(BINDIR)/flake8 . + $(RUN_IN_DOCKER) $(BINDIR)/black --check warehouse/ tests/ + $(RUN_IN_DOCKER) $(BINDIR)/doc8 --allow-long-titles README.rst CONTRIBUTING.rst docs/ --ignore-path docs/_build/ # TODO: Figure out a solution to https://github.com/deezer/template-remover/issues/1 # so we can remove extra_whitespace from below. - $(BINDIR)/html_lint.py --printfilename --disable=optional_tag,names,protocol,extra_whitespace,concerns_separation `find ./warehouse/templates -path ./warehouse/templates/legacy -prune -o -name '*.html' -print` + $(RUN_IN_DOCKER) $(BINDIR)/html_lint.py --printfilename --disable=optional_tag,names,protocol,extra_whitespace,concerns_separation `find ./warehouse/templates -path ./warehouse/templates/legacy -prune -o -name '*.html' -print` ifneq ($(TRAVIS), false) # We're on Travis, so we can lint static files locally ./node_modules/.bin/eslint 'warehouse/static/js/**' '**.js' --ignore-pattern 'warehouse/static/js/vendor/**' @@ -121,24 +130,24 @@ else docker-compose run --rm static ./node_modules/.bin/sass-lint --verbose endif -docs: .state/env/pyvenv.cfg - $(MAKE) -C docs/ doctest SPHINXOPTS="-W" SPHINXBUILD="$(BINDIR)/sphinx-build" - $(MAKE) -C docs/ html SPHINXOPTS="-W" SPHINXBUILD="$(BINDIR)/sphinx-build" +docs: $(RUN_IN_DOCKER_DEP) + $(MAKE) -C docs/ doctest SPHINXOPTS="-W" RUN_IN_DOCKER="$(RUN_IN_DOCKER)" SUBDIR=docs BINDIR=$(BINDIR) + $(MAKE) -C docs/ html SPHINXOPTS="-W" RUN_IN_DOCKER="$(RUN_IN_DOCKER)" SUBDIR=docs BINDIR=$(BINDIR) licenses: bin/licenses export DEPCHECKER deps: .state/env/pyvenv.cfg - $(eval TMPDIR := $(shell mktemp -d)) - $(BINDIR)/pip-compile --no-annotate --no-header --upgrade --allow-unsafe -o $(TMPDIR)/deploy.txt requirements/deploy.in > /dev/null - $(BINDIR)/pip-compile --no-annotate --no-header --upgrade --allow-unsafe -o $(TMPDIR)/main.txt requirements/main.in > /dev/null - $(BINDIR)/pip-compile --no-annotate --no-header --upgrade --allow-unsafe -o $(TMPDIR)/lint.txt requirements/lint.in > /dev/null - echo "$$DEPCHECKER" | python - $(TMPDIR)/deploy.txt requirements/deploy.txt - echo "$$DEPCHECKER" | python - $(TMPDIR)/main.txt requirements/main.txt - echo "$$DEPCHECKER" | python - $(TMPDIR)/lint.txt requirements/lint.txt - rm -r $(TMPDIR) - $(BINDIR)/pip check + $(RUN_IN_DOCKER) $(eval TMPDIR := $(shell mktemp -d)) + $(RUN_IN_DOCKER) $(BINDIR)/pip-compile --no-annotate --no-header --upgrade --allow-unsafe -o $(TMPDIR)/deploy.txt requirements/deploy.in > /dev/null + $(RUN_IN_DOCKER) $(BINDIR)/pip-compile --no-annotate --no-header --upgrade --allow-unsafe -o $(TMPDIR)/main.txt requirements/main.in > /dev/null + $(RUN_IN_DOCKER) $(BINDIR)/pip-compile --no-annotate --no-header --upgrade --allow-unsafe -o $(TMPDIR)/lint.txt requirements/lint.in > /dev/null + $(RUN_IN_DOCKER) python $(TMPDIR)/deploy.txt requirements/deploy.txt -c "$$DEPCHECKER" + $(RUN_IN_DOCKER) python - $(TMPDIR)/main.txt requirements/main.txt -c "$$DEPCHECKER" + $(RUN_IN_DOCKER) python - $(TMPDIR)/lint.txt requirements/lint.txt -c "$$DEPCHECKER" + $(RUN_IN_DOCKER) rm -r $(TMPDIR) + $(RUN_IN_DOCKER) $(BINDIR)/pip check travis-deps: ifneq ($(PR), false) diff --git a/docker-compose.yml b/docker-compose.yml index 5adf68bd2048..4a41e4c53585 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -109,6 +109,28 @@ services: - ./Gulpfile.babel.js:/opt/warehouse/src/Gulpfile.babel.js:z - ./.babelrc:/opt/warehouse/src/.babelrc:z - ./.sass-lint.yml:/opt/warehouse/src/.sass-lint.yml:z + - ./tests:/opt/warehouse/src/tests:z + - ./docs:/opt/warehouse/src/docs:z + - ./requirements:/opt/warehouse/src/requirements:z + - ./setup.cfg:/opt/warehouse/src/setup.cfg:z + - ./README.rst:/opt/warehouse/src/README.rst:z + - ./CONTRIBUTING.rst:/opt/warehouse/src/CONTRIBUTING.rst:z + + build: + build: + context: . + dockerfile: Dockerfile.build + volumes: + - ./warehouse:/opt/warehouse/src/warehouse:z + - ./Gulpfile.babel.js:/opt/warehouse/src/Gulpfile.babel.js:z + - ./.babelrc:/opt/warehouse/src/.babelrc:z + - ./.sass-lint.yml:/opt/warehouse/src/.sass-lint.yml:z + - ./tests:/opt/warehouse/src/tests:z + - ./docs:/opt/warehouse/src/docs:z + - ./requirements:/opt/warehouse/src/requirements:z + - ./setup.cfg:/opt/warehouse/src/setup.cfg:z + - ./README.rst:/opt/warehouse/src/README.rst:z + - ./CONTRIBUTING.rst:/opt/warehouse/src/CONTRIBUTING.rst:z smtp: build: diff --git a/docs/Makefile b/docs/Makefile index 8bbe9087bbb5..e2a011e90df4 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -6,16 +6,26 @@ SPHINXOPTS = SPHINXBUILD = sphinx-build PAPER = BUILDDIR = _build +BINDIR = +RUN_IN_DOCKER = # User-friendly check for sphinx-build -ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) +ifeq ($(RUN_IN_DOCKER) $(shell which $(BINDIR)$(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) endif +ifdef RUN_IN_DOCKER + SPHINXBUILD = $(RUN_IN_DOCKER) sphinx-build + SOURCEDIR = $(SUBDIR) +else + SPHINXBUILD = $(BINDIR)/sphinx-build + SOURCEDIR = . +endif + # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) $(SOURCEDIR) # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .