Run "make lint" and "make reformat" inside docker container

Dockerfile.build

@@ -0,0 +1,40 @@
+FROM python:3.6.6-slim-stretch as python
+
+WORKDIR /opt/warehouse/src
+
+# We create an /opt directory with a virtual environment in it to store our
+# application in.
+RUN set -x \
+    && python3 -m venv /opt/warehouse/src/.state/env
+
+
+# Now that we've created our virtual environment, we'll go ahead and update
+# our $PATH to refer to it first.
+ENV PATH="/opt/warehouse/src/.state/env/bin:${PATH}"
+
+# Next, we want to update pip, setuptools, and wheel inside of this virtual
+# environment to ensure that we have the latest versions of them.
+# TODO: We use --require-hashes in our requirements files, but not here, making
+#       the ones in the requirements files kind of a moot point. We should
+#       probably pin these too, and update them as we do anything else.
+RUN pip --no-cache-dir --disable-pip-version-check install --upgrade pip setuptools wheel
+
+# We copy this into the docker container prior to copying in the rest of our
+# application so that we can skip installing requirements if the only thing
+# that has changed is the Warehouse code itself.
+COPY requirements /tmp/requirements
+
+# Install our development dependencies if we're building a development install
+# otherwise this will do nothing.
+RUN set -x \
+    && pip --no-cache-dir --disable-pip-version-check install -r /tmp/requirements/dev.txt
+
+RUN set -x \
+    && pip --no-cache-dir --disable-pip-version-check install -r /tmp/requirements/lint.txt
+
+RUN set -x \
+    && pip --no-cache-dir --disable-pip-version-check install -r /tmp/requirements/docs.txt
+
+RUN set -x \
+    && if [ "$IPYTHON" = "yes" ]; then pip --no-cache-dir --disable-pip-version-check install -r /tmp/requirements/ipython.txt; fi
+

Makefile

@@ -4,13 +4,22 @@ PR := $(shell echo "$${TRAVIS_PULL_REQUEST:-false}")
 BRANCH := $(shell echo "$${TRAVIS_BRANCH:-master}")
 DB := example
 IPYTHON := no
+RUN_IN_DOCKER =
+RUN_IN_DOCKER_DEP = .state/env/pyvenv.cfg
 
 # set environment variable WAREHOUSE_IPYTHON_SHELL=1 if IPython
 # needed in development environment
 ifeq ($(WAREHOUSE_IPYTHON_SHELL), 1)
     IPYTHON = yes
 endif
 
+ifeq ($(TRAVIS), false)
+    RUN_IN_DOCKER = docker-compose run --rm build
+    RUN_IN_DOCKER_DEP = .state/docker-build
+    PYTHON = $(shell $(RUN_IN_DOCKER) which python)
+    BINDIR = $(shell dirname $(PYTHON))
+endif
+
 define DEPCHECKER
 import sys
 
@@ -65,9 +74,9 @@ default:
 	.state/env/bin/python -m pip install -r requirements/lint.txt
 
 	# install ipython if enabled
-ifeq ($(IPYTHON),"yes")
-	.state/env/bin/python -m pip install -r requirements/ipython.txt
-endif
+    ifeq ($(IPYTHON),"yes")
+        .state/env/bin/python -m pip install -r requirements/ipython.txt
+    endif
 
 .state/docker-build: Dockerfile package.json package-lock.json requirements/main.txt requirements/deploy.txt
 	# Build our docker containers for this project.
@@ -101,16 +110,16 @@ tests:
 								  bin/tests --postgresql-host db $(T) $(TESTARGS)
 
 
-reformat: .state/env/pyvenv.cfg
-	$(BINDIR)/black warehouse/ tests/
+reformat: $(RUN_IN_DOCKER_DEP)
+	$(RUN_IN_DOCKER) $(BINDIR)/black warehouse/ tests/
 
-lint: .state/env/pyvenv.cfg
-	$(BINDIR)/flake8 .
-	$(BINDIR)/black --check warehouse/ tests/
-	$(BINDIR)/doc8 --allow-long-titles README.rst CONTRIBUTING.rst docs/ --ignore-path docs/_build/
+lint: $(RUN_IN_DOCKER_DEP)
+	$(RUN_IN_DOCKER) $(BINDIR)/flake8 .
+	$(RUN_IN_DOCKER) $(BINDIR)/black --check warehouse/ tests/
+	$(RUN_IN_DOCKER) $(BINDIR)/doc8 --allow-long-titles README.rst CONTRIBUTING.rst docs/ --ignore-path docs/_build/
 	# TODO: Figure out a solution to https://github.com/deezer/template-remover/issues/1
 	#       so we can remove extra_whitespace from below.
-	$(BINDIR)/html_lint.py --printfilename --disable=optional_tag,names,protocol,extra_whitespace,concerns_separation `find ./warehouse/templates -path ./warehouse/templates/legacy -prune -o -name '*.html' -print`
+	$(RUN_IN_DOCKER) $(BINDIR)/html_lint.py --printfilename --disable=optional_tag,names,protocol,extra_whitespace,concerns_separation `find ./warehouse/templates -path ./warehouse/templates/legacy -prune -o -name '*.html' -print`
 ifneq ($(TRAVIS), false)
 	# We're on Travis, so we can lint static files locally
 	./node_modules/.bin/eslint 'warehouse/static/js/**' '**.js' --ignore-pattern 'warehouse/static/js/vendor/**'
@@ -121,24 +130,24 @@ else
 	docker-compose run --rm static ./node_modules/.bin/sass-lint --verbose
 endif
 
-docs: .state/env/pyvenv.cfg
-	$(MAKE) -C docs/ doctest SPHINXOPTS="-W" SPHINXBUILD="$(BINDIR)/sphinx-build"
-	$(MAKE) -C docs/ html SPHINXOPTS="-W" SPHINXBUILD="$(BINDIR)/sphinx-build"
+docs: $(RUN_IN_DOCKER_DEP)
+	$(MAKE) -C docs/ doctest SPHINXOPTS="-W" RUN_IN_DOCKER="$(RUN_IN_DOCKER)" SUBDIR=docs BINDIR=$(BINDIR)
+	$(MAKE) -C docs/ html SPHINXOPTS="-W" RUN_IN_DOCKER="$(RUN_IN_DOCKER)" SUBDIR=docs BINDIR=$(BINDIR)
 
 licenses:
 	bin/licenses
 
 export DEPCHECKER
 deps: .state/env/pyvenv.cfg
-	$(eval TMPDIR := $(shell mktemp -d))
-	$(BINDIR)/pip-compile --no-annotate --no-header --upgrade --allow-unsafe -o $(TMPDIR)/deploy.txt requirements/deploy.in > /dev/null
-	$(BINDIR)/pip-compile --no-annotate --no-header --upgrade --allow-unsafe -o $(TMPDIR)/main.txt requirements/main.in > /dev/null
-	$(BINDIR)/pip-compile --no-annotate --no-header --upgrade --allow-unsafe -o $(TMPDIR)/lint.txt requirements/lint.in > /dev/null
-	echo "$$DEPCHECKER" | python - $(TMPDIR)/deploy.txt requirements/deploy.txt
-	echo "$$DEPCHECKER" | python - $(TMPDIR)/main.txt requirements/main.txt
-	echo "$$DEPCHECKER" | python - $(TMPDIR)/lint.txt requirements/lint.txt
-	rm -r $(TMPDIR)
-	$(BINDIR)/pip check
+	$(RUN_IN_DOCKER) $(eval TMPDIR := $(shell mktemp -d))
+	$(RUN_IN_DOCKER) $(BINDIR)/pip-compile --no-annotate --no-header --upgrade --allow-unsafe -o $(TMPDIR)/deploy.txt requirements/deploy.in > /dev/null
+	$(RUN_IN_DOCKER) $(BINDIR)/pip-compile --no-annotate --no-header --upgrade --allow-unsafe -o $(TMPDIR)/main.txt requirements/main.in > /dev/null
+	$(RUN_IN_DOCKER) $(BINDIR)/pip-compile --no-annotate --no-header --upgrade --allow-unsafe -o $(TMPDIR)/lint.txt requirements/lint.in > /dev/null
+	$(RUN_IN_DOCKER) python $(TMPDIR)/deploy.txt requirements/deploy.txt -c "$$DEPCHECKER"
+	$(RUN_IN_DOCKER) python - $(TMPDIR)/main.txt requirements/main.txt -c "$$DEPCHECKER"
+	$(RUN_IN_DOCKER) python - $(TMPDIR)/lint.txt requirements/lint.txt -c "$$DEPCHECKER"
+	$(RUN_IN_DOCKER) rm -r $(TMPDIR)
+	$(RUN_IN_DOCKER) $(BINDIR)/pip check
 
 travis-deps:
 ifneq ($(PR), false)

docker-compose.yml

@@ -109,6 +109,28 @@ services:
       - ./Gulpfile.babel.js:/opt/warehouse/src/Gulpfile.babel.js:z
       - ./.babelrc:/opt/warehouse/src/.babelrc:z
       - ./.sass-lint.yml:/opt/warehouse/src/.sass-lint.yml:z
+      - ./tests:/opt/warehouse/src/tests:z
+      - ./docs:/opt/warehouse/src/docs:z
+      - ./requirements:/opt/warehouse/src/requirements:z
+      - ./setup.cfg:/opt/warehouse/src/setup.cfg:z
+      - ./README.rst:/opt/warehouse/src/README.rst:z
+      - ./CONTRIBUTING.rst:/opt/warehouse/src/CONTRIBUTING.rst:z
+
+  build:
+    build:
+      context: .
+      dockerfile: Dockerfile.build
+    volumes:
+      - ./warehouse:/opt/warehouse/src/warehouse:z
+      - ./Gulpfile.babel.js:/opt/warehouse/src/Gulpfile.babel.js:z
+      - ./.babelrc:/opt/warehouse/src/.babelrc:z
+      - ./.sass-lint.yml:/opt/warehouse/src/.sass-lint.yml:z
+      - ./tests:/opt/warehouse/src/tests:z
+      - ./docs:/opt/warehouse/src/docs:z
+      - ./requirements:/opt/warehouse/src/requirements:z
+      - ./setup.cfg:/opt/warehouse/src/setup.cfg:z
+      - ./README.rst:/opt/warehouse/src/README.rst:z
+      - ./CONTRIBUTING.rst:/opt/warehouse/src/CONTRIBUTING.rst:z
 
   smtp:
     build:

docs/Makefile

@@ -6,16 +6,26 @@ SPHINXOPTS    =
 SPHINXBUILD   = sphinx-build
 PAPER         =
 BUILDDIR      = _build
+BINDIR        =
+RUN_IN_DOCKER =
 
 # User-friendly check for sphinx-build
-ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
+ifeq ($(RUN_IN_DOCKER) $(shell which $(BINDIR)$(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
 $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)
 endif
 
+ifdef RUN_IN_DOCKER
+  SPHINXBUILD   = $(RUN_IN_DOCKER)  sphinx-build
+  SOURCEDIR = $(SUBDIR)
+else
+  SPHINXBUILD   = $(BINDIR)/sphinx-build
+  SOURCEDIR = .
+endif
+
 # Internal variables.
 PAPEROPT_a4     = -D latex_paper_size=a4
 PAPEROPT_letter = -D latex_paper_size=letter
-ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) $(SOURCEDIR)
 # the i18n builder cannot share the environment and doctrees with the others
 I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .