Maintenance/upgrades and tooling (#1546)

* script to create invitations

* ujson 2.0.3 -> 3.0.0 in ALL packages

* Upgrades webserver

* Upgrades packages

* Autoformats web-server

Author: pcrespov

Makefile

@@ -6,11 +6,11 @@
 # 	- In windows, only WSL is supported
 #
 # by sanderegg, pcrespov
+#
 .DEFAULT_GOAL := help
-SHELL := /bin/bash
 
+SHELL := /bin/bash
 
-# TOOLS --------------------------------------
 
 MAKE_C := $(MAKE) --no-print-directory --directory
 
@@ -60,7 +60,9 @@ export SWARM_STACK_NAME ?= simcore
 export DOCKER_IMAGE_TAG ?= latest
 export DOCKER_REGISTRY  ?= itisfoundation
 
+
 .PHONY: help
+
 help: ## help on rule's targets
 ifeq ($(IS_WIN),)
 	@awk --posix 'BEGIN {FS = ":.*?## "} /^[[:alpha:][:space:]_-]+:.*?## / {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
@@ -70,7 +72,7 @@ endif
 
 
 
-## docker BUILD -------------------------------
+## DOCKER BUILD -------------------------------
 #
 # - all builds are inmediatly tagged as 'local/{service}:${BUILD_TARGET}' where BUILD_TARGET='development', 'production', 'cache'
 # - only production and cache images are released (i.e. tagged pushed into registry)
@@ -127,7 +129,6 @@ endif
 endif
 
 
-# TODO: should download cache if any??
 build-cache build-cache-nc build-cache-kit build-cache-x: .env ## Build cache images and tags them as 'local/{service-name}:cache'
 ifeq ($(target),)
 	# Compiling front-end
@@ -152,7 +153,7 @@ shell:
 	docker run -it local/$(target):production /bin/sh
 
 
-## docker SWARM -------------------------------
+## DOCKER SWARM -------------------------------
 #
 # - All resolved configuration are named as .stack-${name}-*.yml to distinguish from docker-compose files which can be parametrized
 #
@@ -179,6 +180,7 @@ docker-compose-configs = $(wildcard services/docker-compose*.yml)
 	# Creating config for ops stack to $@
 	@docker-compose -f services/docker-compose-ops.yml --log-level=ERROR config > $@
 
+
 .PHONY: up-devel up-prod up-version up-latest .deploy-ops
 
 .deploy-ops: .stack-ops.yml
@@ -235,7 +237,7 @@ leave: ## Forces to stop all services, networks, etc by the node leaving the swa
 	$(if $(SWARM_HOSTS),,docker swarm init)
 
 
-## docker TAGS  -------------------------------
+## DOCKER TAGS  -------------------------------
 
 .PHONY: tag-local tag-cache tag-version tag-latest
 
@@ -263,10 +265,10 @@ tag-latest: ## Tags last locally built production images as '${DOCKER_REGISTRY}/
 
 
 
-## docker PULL/PUSH  -------------------------------
+## DOCKER PULL/PUSH  -------------------------------
 #
-# TODO: cannot push modified/untracke
-# TODO: cannot push discetedD
+# TODO: cannot push modified/untracked
+# TODO: cannot push disceted
 #
 .PHONY: pull-cache pull-version
 pull-cache: .env
@@ -296,21 +298,7 @@ push-version: tag-version
 	)
 
 
-## PYTHON -------------------------------
-.PHONY: pylint
-
-pylint: ## Runs python linter framework's wide
-	# See exit codes and command line https://pylint.readthedocs.io/en/latest/user_guide/run.html#exit-codes
-	# TODO: NOT windows friendly
-	/bin/bash -c "pylint --jobs=0 --rcfile=.pylintrc $(strip $(shell find services packages -iname '*.py' \
-											-not -path "*egg*" \
-											-not -path "*migration*" \
-											-not -path "*datcore.py" \
-											-not -path "*sandbox*" \
-											-not -path "*-sdk/python*" \
-											-not -path "*generated_code*" \
-											-not -path "*datcore.py" \
-											-not -path "*web/server*"))"
+## ENVIRONMENT -------------------------------
 
 .PHONY: devenv devenv-all
 
@@ -332,25 +320,42 @@ devenv-all: devenv ## sets up extra development tools (everything else besides p
 	@$(MAKE_C) scripts/json-schema-to-openapi-schema
 
 
-## MISC -------------------------------
-
-.PHONY: new-service
-new-service: .venv ## Bakes a new project from cookiecutter-simcore-pyservice and drops it under services/ [UNDER DEV]
-	$</bin/pip3 --quiet install cookiecutter
-	.venv/bin/cookiecutter gh:itisfoundation/cookiecutter-simcore-pyservice --output-dir $(CURDIR)/services
-
-# TODO: NOT windows friendly
 .env: .env-devel ## creates .env file from defaults in .env-devel
 	$(if $(wildcard $@), \
 	@echo "WARNING #####  $< is newer than $@ ####"; diff -uN $@ $<; false;,\
-	@echo "WARNING ##### $@ does not exist, copying $< ############"; cp $< $@)
+	@echo "WARNING ##### $@ does not exist, cloning $< as $@ ############"; cp $< $@)
+
 
-# TODO: NOT windows friendly
 .vscode/settings.json: .vscode-template/settings.json
 	$(info WARNING: #####  $< is newer than $@ ####)
 	@diff -uN $@ $<
 	@false
 
+
+
+## TOOLS -------------------------------
+
+.PHONY: pylint
+
+pylint: ## Runs python linter framework's wide
+	/bin/bash -c "pylint --jobs=0 --rcfile=.pylintrc $(strip $(shell find services packages -iname '*.py' \
+											-not -path "*egg*" \
+											-not -path "*migration*" \
+											-not -path "*datcore.py" \
+											-not -path "*sandbox*" \
+											-not -path "*-sdk/python*" \
+											-not -path "*generated_code*" \
+											-not -path "*datcore.py" \
+											-not -path "*web/server*"))"
+	# See exit codes and command line https://pylint.readthedocs.io/en/latest/user_guide/run.html#exit-codes
+
+
+.PHONY: new-service
+new-service: .venv ## Bakes a new project from cookiecutter-simcore-pyservice and drops it under services/ [UNDER DEV]
+	$</bin/pip3 --quiet install cookiecutter
+	.venv/bin/cookiecutter gh:itisfoundation/cookiecutter-simcore-pyservice --output-dir $(CURDIR)/services
+
+
 .PHONY: openapi-specs
 openapi-specs: ## bundles and validates openapi specifications and schemas of ALL service's API
 	@$(MAKE_C) services/web/server $@
@@ -366,6 +371,23 @@ code-analysis: .codeclimate.yml ## runs code-climate analysis
 	./scripts/code-climate.bash analyze
 
 
+.PHONY: auto-doc
+auto-doc: .stack-simcore-version.yml ## updates diagrams for README.md
+	# Parsing docker-compose config $< and creating graph
+	@./scripts/docker-compose-viz.bash $<
+	# Updating docs/img
+	@mv --verbose $<.png docs/img/
+
+
+.PHONY: postgres-upgrade
+postgres-upgrade: ## initalize or upgrade postgres db to latest state
+	@$(MAKE_C) packages/postgres-database/docker build
+	@$(MAKE_C) packages/postgres-database/docker upgrade
+
+
+
+## INFO -------------------------------
+
 .PHONY: info info-images info-swarm  info-tools
 info: ## displays setup information
 	# setup info:
@@ -388,7 +410,6 @@ info: ## displays setup information
 	@echo ' python : $(shell python3 --version)'
 
 
-
 define show-meta
 	$(foreach iid,$(shell docker images */$(1):* -q | sort | uniq),\
 		docker image inspect $(iid) | jq '.[0] | .RepoTags, .ContainerConfig.Labels';)
@@ -423,6 +444,9 @@ ifneq ($(SWARM_HOSTS), )
 endif
 
 
+
+## CLEAN -------------------------------
+
 .PHONY: clean clean-images clean-venv clean-all clean-ps
 
 _git_clean_args := -dxf -e .vscode -e TODO.md -e .venv
@@ -460,19 +484,6 @@ clean-all: clean clean-ps clean-images # Deep clean including .venv and produced
 	-rm -rf .venv
 
 
-.PHONY: postgres-upgrade
-postgres-upgrade: ## initalize or upgrade postgres db to latest state
-	@$(MAKE_C) packages/postgres-database/docker build
-	@$(MAKE_C) packages/postgres-database/docker upgrade
-
-
 .PHONY: reset
 reset: ## restart docker daemon (LINUX ONLY)
 	sudo systemctl restart docker
-
-.PHONY: auto-doc
-auto-doc: .stack-simcore-version.yml ## updates diagrams for README.md
-	# Parsing docker-compose config $< and creating graph
-	@./scripts/docker-compose-viz.bash $<
-	# Updating docs/img
-	@mv --verbose $<.png docs/img/

README.md

@@ -1,11 +1,11 @@
 # osparc-simcore platform
 
 <p align="center">
-<img src="https://user-images.githubusercontent.com/32800795/61083844-ff48fb00-a42c-11e9-8e63-fa2d709c8baf.png" width="700">
+<img src="https://user-images.githubusercontent.com/32800795/61083844-ff48fb00-a42c-11e9-8e63-fa2d709c8baf.png" width="500">
 </p>
 
+
 <!-- NOTE: when branched replace `master` in urls -->
-[`master`](https://github.com/itisfoundation/osparc-simcore/tree/master)
 [![Code style: black]](https://github.com/psf/black)
 [![Requires.io]](https://requires.io/github/ITISFoundation/osparc-simcore/requirements/?branch=master "State of third party python dependencies")
 [![travis-ci]](https://travis-ci.org/ITISFoundation/osparc-simcore "State of CI: build, test and pushing images")
@@ -14,19 +14,19 @@
 [![codecov.io]](https://codecov.io/gh/ITISFoundation/osparc-simcore)
 [![github.io]](https://itisfoundation.github.io/)
 [![itis.dockerhub]](https://hub.docker.com/u/itisfoundation)
+[![license]](./LICENSE)
 
 
-
-<!-- ADD HERE ALL BADGE URLS -->
+<!-- ADD HERE ALL BADGE URLS. Use https://shields.io/ -->
 [Code style: black]:https://img.shields.io/badge/code%20style-black-000000.svg
 [Requires.io]:https://img.shields.io/requires/github/ITISFoundation/osparc-simcore.svg
 [travis-ci]:https://travis-ci.org/ITISFoundation/osparc-simcore.svg?branch=master
 [github.io]:https://img.shields.io/website-up-down-green-red/https/itisfoundation.github.io.svg?label=documentation
 [itis.dockerhub]:https://img.shields.io/website/https/hub.docker.com/u/itisfoundation.svg?down_color=red&label=dockerhub%20repos&up_color=green
 [coveralls.io]:https://coveralls.io/repos/github/ITISFoundation/osparc-simcore/badge.svg?branch=master
 [codecov.io]:https://codecov.io/gh/ITISFoundation/osparc-simcore/branch/master/graph/badge.svg
-
-<!---------------------------->
+[license]:https://img.shields.io/github/license/ITISFoundation/osparc-simcore
+<!------------------------------------------------------>
 
 
 The SIM-CORE, named **o<sup>2</sup>S<sup>2</sup>PARC** – **O**pen **O**nline **S**imulations for **S**timulating **P**eripheral **A**ctivity to **R**elieve **C**onditions – is one of the three integrative cores of the SPARC program’s Data Resource Center (DRC).

api/tests/requirements.txt

@@ -11,32 +11,32 @@ chardet==3.0.4            # via aiohttp
 coverage==5.1             # via -r requirements.in, pytest-cov
 idna-ssl==1.1.0           # via aiohttp
 idna==2.9                 # via idna-ssl, yarl
-importlib-metadata==1.6.0  # via jsonschema, pluggy, pytest
+importlib-metadata==1.6.1  # via jsonschema, pluggy, pytest
 isodate==0.6.0            # via openapi-schema-validator
 jsonschema==3.2.0         # via openapi-schema-validator, openapi-spec-validator
-lazy-object-proxy==1.4.3  # via openapi-core
-more-itertools==8.3.0     # via openapi-core, pytest
+lazy-object-proxy==1.5.0  # via openapi-core
+more-itertools==8.4.0     # via openapi-core, pytest
 multidict==4.7.6          # via aiohttp, yarl
 openapi-core==0.13.3      # via -r requirements.in
 openapi-schema-validator==0.1.1  # via openapi-core
 openapi-spec-validator==0.2.8  # via openapi-core
 packaging==20.4           # via pytest, pytest-sugar
 parse==1.15.0             # via openapi-core
 pluggy==0.13.1            # via pytest
-py==1.8.1                 # via pytest
+py==1.8.2                 # via pytest
 pyparsing==2.4.7          # via packaging
 pyrsistent==0.16.0        # via jsonschema
 pytest-aiohttp==0.3.0     # via -r requirements.in
-pytest-cov==2.9.0         # via -r requirements.in
-pytest-instafail==0.4.1.post0  # via -r requirements.in
+pytest-cov==2.10.0        # via -r requirements.in
+pytest-instafail==0.4.2   # via -r requirements.in
 pytest-sugar==0.9.3       # via -r requirements.in
 pytest==5.4.3             # via -r requirements.in, pytest-aiohttp, pytest-cov, pytest-instafail, pytest-sugar
 pyyaml==5.3.1             # via openapi-spec-validator
 six==1.15.0               # via isodate, jsonschema, openapi-core, openapi-schema-validator, openapi-spec-validator, packaging
 strict-rfc3339==0.7       # via openapi-schema-validator
 termcolor==1.1.0          # via pytest-sugar
 typing-extensions==3.7.4.2  # via aiohttp
-wcwidth==0.1.9            # via pytest
+wcwidth==0.2.4            # via pytest
 werkzeug==1.0.1           # via openapi-core
 yarl==1.4.2               # via aiohttp
 zipp==3.1.0               # via importlib-metadata

packages/postgres-database/requirements/_migration.txt

@@ -5,18 +5,18 @@
 #    pip-compile --output-file=requirements/_migration.txt requirements/_migration.in
 #
 alembic==1.4.2            # via -r requirements/_migration.in
-certifi==2020.4.5.1       # via requests
+certifi==2020.6.20        # via requests
 chardet==3.0.4            # via requests
 click==7.1.2              # via -r requirements/_migration.in
 docker==4.2.1             # via -r requirements/_migration.in
 idna==2.9                 # via -r requirements/_base.txt, requests, yarl
-mako==1.1.2               # via alembic
+mako==1.1.3               # via alembic
 markupsafe==1.1.1         # via mako
 multidict==4.7.6          # via -r requirements/_base.txt, yarl
 psycopg2-binary==2.8.5    # via -r requirements/_base.txt, sqlalchemy
 python-dateutil==2.8.1    # via alembic
 python-editor==1.0.4      # via alembic
-requests==2.23.0          # via docker
+requests==2.24.0          # via docker
 six==1.15.0               # via docker, python-dateutil, tenacity, websocket-client
 sqlalchemy[postgresql_psycopg2binary]==1.3.17  # via -r requirements/_base.txt, alembic
 tenacity==6.2.0           # via -r requirements/_migration.in

packages/postgres-database/requirements/_test.in

@@ -20,5 +20,5 @@ pytest-runner
 pytest-docker
 
 # CI
-pylint==2.5.0 # 2.5.3 fails to run in parallel
+pylint
 coveralls

packages/postgres-database/requirements/_test.txt

@@ -7,48 +7,53 @@
 aiohttp==3.6.2            # via pytest-aiohttp
 aiopg[sa]==1.0.0          # via -r requirements/_test.in
 alembic==1.4.2            # via -r requirements/_migration.txt
-astroid==2.4.1            # via pylint
+astroid==2.4.2            # via pylint
 async-timeout==3.0.1      # via aiohttp
 attrs==19.3.0             # via aiohttp, pytest, pytest-docker
-certifi==2020.4.5.1       # via -r requirements/_migration.txt, requests
+certifi==2020.6.20        # via -r requirements/_migration.txt, requests
 chardet==3.0.4            # via -r requirements/_migration.txt, aiohttp, requests
 click==7.1.2              # via -r requirements/_migration.txt
 coverage==5.1             # via -r requirements/_test.in, coveralls, pytest-cov
 coveralls==2.0.0          # via -r requirements/_test.in
 docker==4.2.1             # via -r requirements/_migration.txt
 docopt==0.6.2             # via coveralls
 faker==4.1.1              # via -r requirements/_test.in
+idna-ssl==1.1.0           # via aiohttp
 idna==2.9                 # via -r requirements/_migration.txt, requests, yarl
+importlib-metadata==1.6.1  # via pluggy, pytest
 isort==4.3.21             # via pylint
 lazy-object-proxy==1.4.3  # via astroid
-mako==1.1.2               # via -r requirements/_migration.txt, alembic
+mako==1.1.3               # via -r requirements/_migration.txt, alembic
 markupsafe==1.1.1         # via -r requirements/_migration.txt, mako
 mccabe==0.6.1             # via pylint
-more-itertools==8.3.0     # via pytest
+more-itertools==8.4.0     # via pytest
 multidict==4.7.6          # via -r requirements/_migration.txt, aiohttp, yarl
 packaging==20.4           # via pytest
 pluggy==0.13.1            # via pytest
 psycopg2-binary==2.8.5    # via -r requirements/_migration.txt, aiopg, sqlalchemy
-py==1.8.1                 # via pytest
-pylint==2.5.0             # via -r requirements/_test.in
+py==1.8.2                 # via pytest
+pylint==2.5.3             # via -r requirements/_test.in
 pyparsing==2.4.7          # via packaging
 pytest-aiohttp==0.3.0     # via -r requirements/_test.in
-pytest-cov==2.9.0         # via -r requirements/_test.in
+pytest-cov==2.10.0        # via -r requirements/_test.in
 pytest-docker==0.7.2      # via -r requirements/_test.in
-pytest-instafail==0.4.1.post0  # via -r requirements/_test.in
+pytest-instafail==0.4.2   # via -r requirements/_test.in
 pytest-runner==5.2        # via -r requirements/_test.in
 pytest==5.4.3             # via -r requirements/_test.in, pytest-aiohttp, pytest-cov, pytest-instafail
 python-dateutil==2.8.1    # via -r requirements/_migration.txt, alembic, faker
 python-editor==1.0.4      # via -r requirements/_migration.txt, alembic
 pyyaml==5.3.1             # via -r requirements/_test.in
-requests==2.23.0          # via -r requirements/_migration.txt, coveralls, docker
+requests==2.24.0          # via -r requirements/_migration.txt, coveralls, docker
 six==1.15.0               # via -r requirements/_migration.txt, astroid, docker, packaging, python-dateutil, tenacity, websocket-client
 sqlalchemy[postgresql_psycopg2binary]==1.3.17  # via -r requirements/_migration.txt, aiopg, alembic
 tenacity==6.2.0           # via -r requirements/_migration.txt
 text-unidecode==1.3       # via faker
 toml==0.10.1              # via pylint
+typed-ast==1.4.1          # via astroid
+typing-extensions==3.7.4.2  # via aiohttp
 urllib3==1.25.9           # via -r requirements/_migration.txt, requests
-wcwidth==0.1.9            # via pytest
+wcwidth==0.2.4            # via pytest
 websocket-client==0.57.0  # via -r requirements/_migration.txt, docker
 wrapt==1.12.1             # via astroid
 yarl==1.4.2               # via -r requirements/_migration.txt, aiohttp
+zipp==3.1.0               # via importlib-metadata

packages/pytest-simcore/src/pytest_simcore/helpers/utils_pylint.py

@@ -13,7 +13,7 @@ def assert_pylint_is_passing(pylintrc, package_dir, number_of_jobs: int = AUTODE
     command = f"pylint --jobs={number_of_jobs} --rcfile {pylintrc} -v {package_dir}".split(
         " "
     )
-    pipes = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    pipes = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
     std_out, _ = pipes.communicate()
     if pipes.returncode != 0:
         print(f'>>>> Exit code "{pipes.returncode}"\n{std_out.decode("utf-8")}\n<<<<')