[maintenance] Upgrades webserver's requirements (#1444)

Fully upgrade requirements of packages and webserver to latest version possible
Upgraded Dockerfiles for webserver

Author: pcrespov

.gitignore

@@ -129,7 +129,7 @@ services/docker-compose.stack.*.yml
 
 # Any generated output folder (e.g. codegen, build, ...)
 out/
-tmp/
+**/tmp/
 
 # See .vscode-template
 pytest.ini
@@ -149,4 +149,3 @@ prof/
 
 # outputs from make
 .stack-*.yml
-services/web/server/tests/data/static/resource/statics.json

Makefile

@@ -87,12 +87,12 @@ endef
 
 rebuild: build-nc # alias
 build build-nc build-kit build-x: .env ## Builds production images and tags them as 'local/{service-name}:production'. For single target e.g. 'make target=webserver build'
-ifeq ($(target),)	
+ifeq ($(target),)
 	# Compiling front-end
-	
+
 	@$(if $(findstring -kit,$@),export DOCKER_BUILDKIT=1;export COMPOSE_DOCKER_CLI_BUILD=1;,) \
 	$(MAKE_C) services/web/client compile$(if $(findstring -x,$@),-x,)
-	
+
 	# Building services
 	@$(if $(findstring -kit,$@),export DOCKER_BUILDKIT=1;export COMPOSE_DOCKER_CLI_BUILD=1;,) \
 	$(_docker_compose_build)
@@ -126,7 +126,7 @@ endif
 
 
 # TODO: should download cache if any??
-build-cache build-cache-nc build-cache-kit build-cache-x: .env ## Build cache images and tags them as 'local/{service-name}:cache'	
+build-cache build-cache-nc build-cache-kit build-cache-x: .env ## Build cache images and tags them as 'local/{service-name}:cache'
 ifeq ($(target),)
 	# Compiling front-end
 	@$(if $(findstring -kit,$@),export DOCKER_BUILDKIT=1;export COMPOSE_DOCKER_CLI_BUILD=1;,)
@@ -189,13 +189,13 @@ endif
 
 
 up-devel: .stack-simcore-development.yml .init-swarm $(CLIENT_WEB_OUTPUT) ## Deploys local development stack, qx-compile+watch and ops stack (pass 'make ops_disabled=1 up-...' to disable)
-	# Start compile+watch front-end container [front-end]	
+	# Start compile+watch front-end container [front-end]
 	$(MAKE_C) services/web/client compile-dev flags=--watch
 	# Deploy stack $(SWARM_STACK_NAME) [back-end]
 	@docker stack deploy -c $< $(SWARM_STACK_NAME)
 	$(MAKE) .deploy-ops
 	$(MAKE_C) services/web/client follow-dev-logs
-	
+
 
 up-prod: .stack-simcore-production.yml .init-swarm ## Deploys local production stack and ops stack (pass 'make ops_disabled=1 up-...' to disable)
 	# Deploy stack $(SWARM_STACK_NAME)
@@ -297,8 +297,6 @@ push-version: tag-version
 ## PYTHON -------------------------------
 .PHONY: pylint
 
-PY_PIP = $(if $(IS_WIN),cd .venv/Scripts && pip.exe,.venv/bin/pip3)
-
 pylint: ## Runs python linter framework's wide
 	# See exit codes and command line https://pylint.readthedocs.io/en/latest/user_guide/run.html#exit-codes
 	# TODO: NOT windows friendly
@@ -315,13 +313,13 @@ pylint: ## Runs python linter framework's wide
 
 .venv:
 	python3 -m venv $@
-	$@/bin/pip3 install --upgrade \
+	$@/bin/pip3 --quiet install --upgrade \
 		pip \
 		wheel \
 		setuptools
 
 devenv: .venv ## create a python virtual environment with dev tools (e.g. linters, etc)
-	$</bin/pip3 install -r requirements.txt
+	$</bin/pip3 --quiet install -r requirements.txt
 	@echo "To activate the venv, execute 'source .venv/bin/activate'"
 
 devenv-all: devenv ## sets up extra development tools (everything else besides python)
@@ -335,7 +333,7 @@ devenv-all: devenv ## sets up extra development tools (everything else besides p
 
 .PHONY: new-service
 new-service: .venv ## Bakes a new project from cookiecutter-simcore-pyservice and drops it under services/ [UNDER DEV]
-	$</bin/pip3 install cookiecutter
+	$</bin/pip3 --quiet install cookiecutter
 	.venv/bin/cookiecutter gh:itisfoundation/cookiecutter-simcore-pyservice --output-dir $(CURDIR)/services
 
 # TODO: NOT windows friendly
@@ -422,39 +420,40 @@ ifneq ($(SWARM_HOSTS), )
 endif
 
 
-.PHONY: clean clean-images clean-venv clean-all
-
-git_clean_args := -dxf -e .vscode -e TODO.md -e .venv
+.PHONY: clean clean-images clean-venv clean-all clean-ps
 
+_git_clean_args := -dxf -e .vscode -e TODO.md -e .venv
+_running_containers = $(shell docker ps -aq)
 
 .check-clean:
-	@git clean -n $(git_clean_args)
+	@git clean -n $(_git_clean_args)
 	@echo -n "Are you sure? [y/N] " && read ans && [ $${ans:-N} = y ]
 	@echo -n "$(shell whoami), are you REALLY sure? [y/N] " && read ans && [ $${ans:-N} = y ]
 
-clean-venv: ## Purges .venv into original configuration
+clean-venv: devenv ## Purges .venv into original configuration
 	# Cleaning your venv
-	pip-sync $(CURDIR)/requirements.txt
+	.venv/bin/pip-sync --quiet $(CURDIR)/requirements.txt
 	@pip list
 
-clean: .check-clean clean-venv ## cleans all unversioned files in project and temp files create by this makefile
+clean: .check-clean ## cleans all unversioned files in project and temp files create by this makefile
 	# Cleaning unversioned
-	@git clean $(git_clean_args)
+	@git clean $(_git_clean_args)
 	# Cleaning web/client
-	@$(MAKE_C) services/web/client clean
-	# Cleaning postgres maintenance
-	@$(MAKE_C) packages/postgres-database/docker clean
+	@$(MAKE_C) services/web/client clean-files
+
+clean-ps: ## stops and deletes running containers
+	$(if $(_running_containers), docker rm -f $(_running_containers),)
 
 clean-images: ## removes all created images
 	# Cleaning all service images
 	-$(foreach service,$(SERVICES_LIST)\
 		,docker image rm -f $(shell docker images */$(service):* -q);)
 	# Cleaning webclient
-	@$(MAKE_C) services/web/client clean
+	@$(MAKE_C) services/web/client clean-images
 	# Cleaning postgres maintenance
 	@$(MAKE_C) packages/postgres-database/docker clean
 
-clean-all: clean clean-images # Deep clean including .venv and produced images
+clean-all: clean clean-ps clean-images # Deep clean including .venv and produced images
 	-rm -rf .venv
 
 
@@ -463,6 +462,7 @@ postgres-upgrade: ## initalize or upgrade postgres db to latest state
 	@$(MAKE_C) packages/postgres-database/docker build
 	@$(MAKE_C) packages/postgres-database/docker upgrade
 
+
 .PHONY: reset
 reset: ## restart docker daemon (LINUX ONLY)
 	sudo systemctl restart docker

docs/python-dependencies.md

@@ -17,6 +17,36 @@ TODO: finish!
 - The dependencies have to be kept up-to-date regularly (e.g. due to security patches;  new feature in our package might add new direct dependencies to our requirements)
 - Inter-dependent libraries typically have different release cycles creating in time version conflicts (e.g. package A and B strictly depend on different versions of C)
 
+
+- packages uses input requirements as install-requirements (i.e. entry in setup and in requirements/ci.txt) NOT compiled ones
+- services use compiled requirements
+
+
+
+- tests packages with the latest (i.e. compile requirements to lastest version)
+  - if tests fails, then add constraints in input requirements
+    - try adding tests that check inter-library compatibility
+  - if tests succeed, you can use them in services
+  - if at least one service has a problem, we need to decide whether to add a constraint:
+    a) at the package level => will ensure is tested but is constraining all services
+    b) at the service level => only affects service but cannot do isolate tests against latests upgrades
+
+
+### How to purge unused requirements?
+
+
+### Propagation of constraints
+
+- Situation:  upgrading one package the developer finds an issue, e.g.
+```
+coverage==5.0.3 # TODO: Downgraded because of a bug https://github.com/nedbat/coveragepy/issues/716
+
+pytest~=5.3.5  # Bug in pytest-sugar https://github.com/Teemu/pytest-sugar/issues/187
+pytest-aiohttp  # incompatible with pytest-asyncio. See https://github.com/pytest-dev/pytest-asyncio/issues/76
+```
+- Question: how to make sure this is also taken into account in other places?
+
+
 !-->
 
 ## Rationale

packages/postgres-database/Makefile

@@ -6,7 +6,7 @@ include ../../scripts/common.Makefile
 
 .PHONY: requirements
 requirements: ## compiles pip requirements (.in -> .txt)
-	@$(MAKE) --directory requirements reqs
+	@$(MAKE_C) requirements reqs
 
 
 .PHONY: install-dev install-prod install-ci

packages/postgres-database/requirements/_base.in

@@ -2,5 +2,4 @@
 # Specifies third-party dependencies for 'simcore-postgres-database'
 #
 sqlalchemy[postgresql_psycopg2binary]>=1.3.3  # binary + Vulnerability
-
 yarl

packages/postgres-database/requirements/_base.txt

@@ -4,8 +4,8 @@
 #
 #    pip-compile --output-file=_base.txt _base.in
 #
-idna==2.8                 # via yarl
-multidict==4.5.2          # via yarl
-psycopg2-binary==2.8.4    # via sqlalchemy
-sqlalchemy[postgresql_psycopg2binary]==1.3.5  # via -r requirements/_base.in
-yarl==1.4.2               # via -r requirements/_base.in
+idna==2.9                 # via yarl
+multidict==4.7.5          # via yarl
+psycopg2-binary==2.8.5    # via sqlalchemy
+sqlalchemy[postgresql_psycopg2binary]==1.3.16  # via -r _base.in
+yarl==1.4.2               # via -r _base.in

packages/postgres-database/requirements/_migration.txt

@@ -4,22 +4,22 @@
 #
 #    pip-compile --output-file=_migration.txt _migration.in
 #
-alembic==1.4.2            # via -r requirements/_migration.in
-certifi==2019.11.28       # via -r requirements/_migration.in, requests
+alembic==1.4.2            # via -r _migration.in
+certifi==2019.11.28       # via -r _migration.in, requests
 chardet==3.0.4            # via requests
-click==7.1.1              # via -r requirements/_migration.in
-docker==4.2.0             # via -r requirements/_migration.in
-idna==2.8                 # via -r requirements/_base.txt, requests, yarl
-mako==1.0.12              # via alembic
+click==7.1.2              # via -r _migration.in
+docker==4.2.0             # via -r _migration.in
+idna==2.9                 # via -r _base.txt, requests, yarl
+mako==1.1.2               # via alembic
 markupsafe==1.1.1         # via mako
-multidict==4.5.2          # via -r requirements/_base.txt, yarl
-psycopg2-binary==2.8.4    # via -r requirements/_base.txt, sqlalchemy
-python-dateutil==2.8.0    # via alembic
+multidict==4.7.5          # via -r _base.txt, yarl
+psycopg2-binary==2.8.5    # via -r _base.txt, sqlalchemy
+python-dateutil==2.8.1    # via alembic
 python-editor==1.0.4      # via alembic
-requests==2.22.0          # via docker
-six==1.12.0               # via docker, python-dateutil, tenacity, websocket-client
-sqlalchemy[postgresql_psycopg2binary]==1.3.5  # via -r requirements/_base.txt, alembic
-tenacity==6.1.0           # via -r requirements/_migration.in
-urllib3==1.25.8           # via -r requirements/_migration.in, requests
-websocket-client==0.56.0  # via docker
-yarl==1.4.2               # via -r requirements/_base.txt
+requests==2.23.0          # via docker
+six==1.14.0               # via docker, python-dateutil, tenacity, websocket-client
+sqlalchemy[postgresql_psycopg2binary]==1.3.16  # via -r _base.txt, alembic
+tenacity==6.1.0           # via -r _migration.in
+urllib3==1.25.9           # via -r _migration.in, requests
+websocket-client==0.57.0  # via docker
+yarl==1.4.2               # via -r _base.txt

packages/postgres-database/requirements/_test.in

@@ -11,15 +11,14 @@ aiopg[sa]
 faker
 
 # testing
-coverage==5.0.3 # TODO: Downgraded because of a bug https://github.com/nedbat/coveragepy/issues/716
+coverage
 pytest~=5.3.5  # Bug in pytest-sugar https://github.com/Teemu/pytest-sugar/issues/187
 pytest-cov
 pytest-aiohttp  # incompatible with pytest-asyncio. See https://github.com/pytest-dev/pytest-asyncio/issues/76
 pytest-instafail
 pytest-runner
 pytest-docker
 
-# tools
--r ../../../requirements.txt
+# CI
 pylint
 coveralls

packages/postgres-database/requirements/_test.txt

@@ -2,65 +2,58 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile requirements/_test.in
+#    pip-compile --output-file=_test.txt _test.in
 #
 aiohttp==3.6.2            # via pytest-aiohttp
-aiopg[sa]==1.0.0          # via -r requirements/_test.in
-alembic==1.4.2            # via -r requirements/_migration.txt
-appdirs==1.4.3            # via black
-astroid==2.3.3            # via pylint
+aiopg[sa]==1.0.0          # via -r _test.in
+alembic==1.4.2            # via -r _migration.txt
+astroid==2.4.0            # via pylint
 async-timeout==3.0.1      # via aiohttp
-attrs==19.3.0             # via aiohttp, black, pytest, pytest-docker
-black==19.10b0            # via -r requirements/../../../requirements.txt
-bump2version==1.0.0       # via -r requirements/../../../requirements.txt
-certifi==2019.11.28       # via -r requirements/_migration.txt, requests
-chardet==3.0.4            # via -r requirements/_migration.txt, aiohttp, requests
-click==7.1.1              # via -r requirements/_migration.txt, black, pip-tools
-coverage==5.0.3           # via -r requirements/_test.in, coveralls, pytest-cov
-coveralls==1.11.1         # via -r requirements/_test.in
-docker==4.2.0             # via -r requirements/_migration.txt
+attrs==19.3.0             # via aiohttp, pytest, pytest-docker
+certifi==2019.11.28       # via -r _migration.txt, requests
+chardet==3.0.4            # via -r _migration.txt, aiohttp, requests
+click==7.1.2              # via -r _migration.txt
+coverage==5.1             # via -r _test.in, coveralls, pytest-cov
+coveralls==2.0.0          # via -r _test.in
+docker==4.2.0             # via -r _migration.txt
 docopt==0.6.2             # via coveralls
-faker==4.0.2              # via -r requirements/_test.in
+faker==4.0.3              # via -r _test.in
 idna-ssl==1.1.0           # via aiohttp
-idna==2.8                 # via -r requirements/_migration.txt, requests, yarl
+idna==2.9                 # via -r _migration.txt, requests, yarl
 importlib-metadata==1.6.0  # via pluggy, pytest
 isort==4.3.21             # via pylint
 lazy-object-proxy==1.4.3  # via astroid
-mako==1.0.12              # via -r requirements/_migration.txt, alembic
-markupsafe==1.1.1         # via -r requirements/_migration.txt, mako
+mako==1.1.2               # via -r _migration.txt, alembic
+markupsafe==1.1.1         # via -r _migration.txt, mako
 mccabe==0.6.1             # via pylint
 more-itertools==8.2.0     # via pytest
-multidict==4.5.2          # via -r requirements/_migration.txt, aiohttp, yarl
+multidict==4.7.5          # via -r _migration.txt, aiohttp, yarl
 packaging==20.3           # via pytest
-pathspec==0.7.0           # via black
-pip-tools==4.5.1          # via -r requirements/../../../requirements.txt
 pluggy==0.13.1            # via pytest
-psycopg2-binary==2.8.4    # via -r requirements/_migration.txt, aiopg, sqlalchemy
+psycopg2-binary==2.8.5    # via -r _migration.txt, aiopg, sqlalchemy
 py==1.8.1                 # via pytest
-pylint==2.4.4             # via -r requirements/../../../requirements.txt, -r requirements/_test.in
-pyparsing==2.4.6          # via packaging
-pytest-aiohttp==0.3.0     # via -r requirements/_test.in
-pytest-cov==2.8.1         # via -r requirements/_test.in
-pytest-docker==0.7.2      # via -r requirements/_test.in
-pytest-instafail==0.4.1.post0  # via -r requirements/_test.in
-pytest-runner==5.2        # via -r requirements/_test.in
-pytest==5.3.5             # via -r requirements/_test.in, pytest-aiohttp, pytest-cov, pytest-instafail
-python-dateutil==2.8.0    # via -r requirements/_migration.txt, alembic, faker
-python-editor==1.0.4      # via -r requirements/_migration.txt, alembic
-pyyaml==5.3.1             # via -r requirements/_test.in
-regex==2020.2.20          # via black
-requests==2.22.0          # via -r requirements/_migration.txt, coveralls, docker
-rope==0.16.0              # via -r requirements/../../../requirements.txt
-six==1.12.0               # via -r requirements/_migration.txt, astroid, docker, packaging, pip-tools, python-dateutil, tenacity, websocket-client
-sqlalchemy[postgresql_psycopg2binary]==1.3.5  # via -r requirements/_migration.txt, aiopg, alembic
-tenacity==6.1.0           # via -r requirements/_migration.txt
+pylint==2.5.0             # via -r _test.in
+pyparsing==2.4.7          # via packaging
+pytest-aiohttp==0.3.0     # via -r _test.in
+pytest-cov==2.8.1         # via -r _test.in
+pytest-docker==0.7.2      # via -r _test.in
+pytest-instafail==0.4.1.post0  # via -r _test.in
+pytest-runner==5.2        # via -r _test.in
+pytest==5.3.5             # via -r _test.in, pytest-aiohttp, pytest-cov, pytest-instafail
+python-dateutil==2.8.1    # via -r _migration.txt, alembic, faker
+python-editor==1.0.4      # via -r _migration.txt, alembic
+pyyaml==5.3.1             # via -r _test.in
+requests==2.23.0          # via -r _migration.txt, coveralls, docker
+six==1.14.0               # via -r _migration.txt, astroid, docker, packaging, python-dateutil, tenacity, websocket-client
+sqlalchemy[postgresql_psycopg2binary]==1.3.16  # via -r _migration.txt, aiopg, alembic
+tenacity==6.1.0           # via -r _migration.txt
 text-unidecode==1.3       # via faker
-toml==0.10.0              # via black
-typed-ast==1.4.1          # via astroid, black
+toml==0.10.0              # via pylint
+typed-ast==1.4.1          # via astroid
 typing-extensions==3.7.4.2  # via aiohttp
-urllib3==1.25.8           # via -r requirements/_migration.txt, requests
-wcwidth==0.1.8            # via pytest
-websocket-client==0.56.0  # via -r requirements/_migration.txt, docker
-wrapt==1.11.2             # via astroid
-yarl==1.4.2               # via -r requirements/_migration.txt, aiohttp
+urllib3==1.25.9           # via -r _migration.txt, requests
+wcwidth==0.1.9            # via pytest
+websocket-client==0.57.0  # via -r _migration.txt, docker
+wrapt==1.12.1             # via astroid
+yarl==1.4.2               # via -r _migration.txt, aiohttp
 zipp==3.1.0               # via importlib-metadata

packages/postgres-database/requirements/dev.txt

@@ -11,3 +11,6 @@
 
 # current module
 -e .[migration]
+
+# basic dev tools
+-r ../../../requirements.txt