Upgrades storage service dependencies (#1708)

pcrespov · web-flow · commit 27668dd01807 · 2020-08-21T19:31:57.000+02:00
diff --git a/Makefile b/Makefile
@@ -205,10 +205,15 @@ up-devel: .stack-simcore-development.yml .init-swarm $(CLIENT_WEB_OUTPUT) ## Dep
 	$(MAKE_C) services/web/client follow-dev-logs
 
 
-up-prod: .stack-simcore-production.yml .init-swarm ## Deploys local production stack and ops stack (pass 'make ops_disabled=1 up-...' to disable)
+up-prod: .stack-simcore-production.yml .init-swarm ## Deploys local production stack and ops stack (pass 'make ops_disabled=1 up-...' to disable or target=<service-name> to deploy a single service)
+ifeq ($(target),)
 	# Deploy stack $(SWARM_STACK_NAME)
 	@docker stack deploy -c $< $(SWARM_STACK_NAME)
 	$(MAKE) .deploy-ops
+else
+	# deploys ONLY $(target) service
+	docker-compose -f $< up --detach $(target)
+endif
 
 up-version: .stack-simcore-version.yml .init-swarm ## Deploys versioned stack '$(DOCKER_REGISTRY)/{service}:$(DOCKER_IMAGE_TAG)' and ops stack (pass 'make ops_disabled=1 up-...' to disable)
 	# Deploy stack $(SWARM_STACK_NAME)
@@ -390,6 +395,7 @@ postgres-upgrade: ## initalize or upgrade postgres db to latest state
 	@$(MAKE_C) packages/postgres-database/docker build
 	@$(MAKE_C) packages/postgres-database/docker upgrade
 
+
 local_registry=registry
 .PHONY: local-registry
 local-registry: .env ## creates a local docker registry and configure simcore to use it (NOTE: needs admin rights)
diff --git a/packages/postgres-database/requirements/_test.txt b/packages/postgres-database/requirements/_test.txt
@@ -24,7 +24,7 @@ docker-compose==1.26.2    # via pytest-docker
 docker[ssh]==4.3.0        # via -r requirements/_migration.txt, docker-compose
 dockerpty==0.4.1          # via docker-compose
 docopt==0.6.2             # via coveralls, docker-compose
-faker==4.1.1              # via -r requirements/_test.in
+faker==4.1.2              # via -r requirements/_test.in
 idna-ssl==1.1.0           # via aiohttp
 idna==2.10                # via -r requirements/_migration.txt, idna-ssl, requests, yarl
 importlib-metadata==1.7.0  # via jsonschema, pluggy, pytest
diff --git a/scripts/common.Makefile b/scripts/common.Makefile
@@ -148,8 +148,14 @@ shell: ## runs shell in production container
 # SUBTASKS
 #
 
-.PHONY: _check_venv_active
-_check_venv_active:
+.PHONY: _check_python_version _check_venv_active
+
+_check_python_version:
+	# Checking that runs with correct python version
+	@python3 -c "import sys; assert sys.version_info[:2]==(3,6), f'Expected python 3.6, got {sys.version_info}'"
+
+
+_check_venv_active: _check_python_version
 	# checking whether virtual environment was activated
 	@python3 -c "import sys; assert sys.base_prefix!=sys.prefix"
 
diff --git a/scripts/requirements/Makefile b/scripts/requirements/Makefile
@@ -28,9 +28,9 @@ MAKE_C := $(MAKE) --no-print-directory --directory
 # packages/
 _input-requirements = $(shell find ${PACKAGES_DIR} -type f -name _base.in)
 # api tests
-_input-requirements += $(shell find $(REPODIR)/api/tests/ -type f -name *.in)
+_input-requirements += $(shell find $(REPODIR)/api/tests/ -type f -name "*.in")
 # system tests
-_input-requirements += $(shell find $(REPODIR)/tests/ -type f -name *.in)
+_input-requirements += $(shell find $(REPODIR)/tests/ -type f -name "*.in")
 
 $(info Found $(_input-requirements))
 
diff --git a/services/storage/requirements/_base.txt b/services/storage/requirements/_base.txt
@@ -10,10 +10,10 @@ aiofiles==0.5.0           # via -r requirements/_base.in
 aiohttp==3.6.2            # via -r requirements/../../../packages/service-library/requirements/_base.in, -r requirements/_base.in, aiobotocore, aiozipkin
 aioitertools==0.7.0       # via aiobotocore
 aiopg[sa]==1.0.0          # via -r requirements/../../../packages/service-library/requirements/_base.in, -r requirements/_base.in
-aiozipkin==0.6.0          # via -r requirements/../../../packages/service-library/requirements/_base.in
+aiozipkin==0.7.0          # via -r requirements/../../../packages/service-library/requirements/_base.in
 async-timeout==3.0.1      # via aiohttp
 attrs==19.3.0             # via -r requirements/../../../packages/service-library/requirements/_base.in, aiohttp, jsonschema, openapi-core
-blackfynn==3.6.3          # via -r requirements/_base.in
+blackfynn==4.0.0          # via -r requirements/_base.in
 boto3==1.12.32            # via -r requirements/_base.in, blackfynn
 botocore==1.15.32         # via aiobotocore, boto3, s3transfer
 certifi==2020.6.20        # via requests
@@ -24,19 +24,19 @@ docopt==0.6.2             # via blackfynn
 docutils==0.15.2          # via botocore
 future==0.18.2            # via blackfynn
 idna-ssl==1.1.0           # via aiohttp
-idna==2.9                 # via idna-ssl, requests, yarl
-importlib-metadata==1.6.1  # via jsonschema
+idna==2.10                # via idna-ssl, requests, yarl
+importlib-metadata==1.7.0  # via jsonschema
 isodate==0.6.0            # via openapi-core
 jmespath==0.10.0          # via boto3, botocore
 jsonschema==3.2.0         # via -r requirements/../../../packages/service-library/requirements/_base.in, openapi-spec-validator
 lazy-object-proxy==1.4.3  # via -r requirements/../../../packages/service-library/requirements/_base.in, openapi-core
-marshmallow==3.6.1        # via -r requirements/_base.in
+marshmallow==3.7.1        # via -r requirements/_base.in
 multidict==4.7.6          # via aiohttp, yarl
 openapi-core==0.12.0      # via -r requirements/../../../packages/service-library/requirements/_base.in
-openapi-spec-validator==0.2.8  # via openapi-core
+openapi-spec-validator==0.2.9  # via openapi-core
 prometheus-client==0.8.0  # via -r requirements/../../../packages/service-library/requirements/_base.in
-protobuf==3.12.2          # via blackfynn
-psutil==5.7.0             # via blackfynn
+protobuf==3.13.0          # via blackfynn
+psutil==5.7.2             # via blackfynn
 psycopg2-binary==2.8.5    # via -r requirements/../../../packages/service-library/requirements/_base.in, aiopg, sqlalchemy
 pyrsistent==0.16.0        # via jsonschema
 python-dateutil==2.8.1    # via blackfynn, botocore
@@ -52,13 +52,13 @@ strict-rfc3339==0.7       # via openapi-core
 tenacity==6.2.0           # via -r requirements/../../../packages/service-library/requirements/_base.in, -r requirements/_base.in
 trafaret-config==2.0.2    # via -r requirements/_base.in
 trafaret==2.0.2           # via -r requirements/../../../packages/service-library/requirements/_base.in, -r requirements/_base.in, trafaret-config
-typing-extensions==3.7.4.2  # via aiohttp, aioitertools
-ujson==3.0.0              # via -r requirements/../../../packages/service-library/requirements/_base.in
-urllib3==1.25.9           # via botocore, requests
+typing-extensions==3.7.4.2  # via aiohttp, aioitertools, yarl
+ujson==3.1.0              # via -r requirements/../../../packages/service-library/requirements/_base.in
+urllib3==1.25.10          # via botocore, requests
 websocket-client==0.57.0  # via blackfynn
 werkzeug==1.0.1           # via -r requirements/../../../packages/service-library/requirements/_base.in
 wrapt==1.12.1             # via aiobotocore, deprecated
-yarl==1.4.2               # via -r requirements/../../../packages/postgres-database/requirements/_base.in, aiohttp
+yarl==1.5.1               # via -r requirements/../../../packages/postgres-database/requirements/_base.in, aiohttp
 zipp==3.1.0               # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
diff --git a/services/storage/requirements/_test.txt b/services/storage/requirements/_test.txt
@@ -10,82 +10,95 @@ aiofiles==0.5.0           # via -r requirements/_base.txt
 aiohttp==3.6.2            # via -r requirements/_base.txt, aiobotocore, aiozipkin, pytest-aiohttp
 aioitertools==0.7.0       # via -r requirements/_base.txt, aiobotocore
 aiopg[sa]==1.0.0          # via -r requirements/_base.txt
-aiozipkin==0.6.0          # via -r requirements/_base.txt
+aiozipkin==0.7.0          # via -r requirements/_base.txt
 astroid==2.4.2            # via pylint
 async-timeout==3.0.1      # via -r requirements/_base.txt, aiohttp
 attrs==19.3.0             # via -r requirements/_base.txt, aiohttp, jsonschema, openapi-core, pytest, pytest-docker
-blackfynn==3.6.3          # via -r requirements/_base.txt
+bcrypt==3.2.0             # via paramiko
+blackfynn==4.0.0          # via -r requirements/_base.txt
 boto3==1.12.32            # via -r requirements/_base.txt, blackfynn
 botocore==1.15.32         # via -r requirements/_base.txt, aiobotocore, boto3, s3transfer
+cached-property==1.5.1    # via docker-compose
 certifi==2020.6.20        # via -r requirements/_base.txt, requests
+cffi==1.14.2              # via bcrypt, cryptography, pynacl
 chardet==3.0.4            # via -r requirements/_base.txt, aiohttp, requests
-codecov==2.1.7            # via -r requirements/_test.in
+codecov==2.1.8            # via -r requirements/_test.in
 configparser==5.0.0       # via -r requirements/_base.txt, blackfynn
-coverage==5.1             # via -r requirements/_test.in, codecov, coveralls, pytest-cov
-coveralls==2.0.0          # via -r requirements/_test.in
+coverage==5.2.1           # via -r requirements/_test.in, codecov, coveralls, pytest-cov
+coveralls==2.1.2          # via -r requirements/_test.in
+cryptography==3.0         # via paramiko
 deprecated==1.2.10        # via -r requirements/_base.txt, blackfynn
-docopt==0.6.2             # via -r requirements/_base.txt, blackfynn, coveralls
+distro==1.5.0             # via docker-compose
+docker-compose==1.26.2    # via pytest-docker
+docker[ssh]==4.3.1        # via docker-compose
+dockerpty==0.4.1          # via docker-compose
+docopt==0.6.2             # via -r requirements/_base.txt, blackfynn, coveralls, docker-compose
 docutils==0.15.2          # via -r requirements/_base.txt, botocore
 future==0.18.2            # via -r requirements/_base.txt, blackfynn
 idna-ssl==1.1.0           # via -r requirements/_base.txt, aiohttp
-idna==2.9                 # via -r requirements/_base.txt, idna-ssl, requests, yarl
-importlib-metadata==1.6.1  # via -r requirements/_base.txt, jsonschema, pluggy, pytest
+idna==2.10                # via -r requirements/_base.txt, idna-ssl, requests, yarl
+importlib-metadata==1.7.0  # via -r requirements/_base.txt, jsonschema, pluggy, pytest
 isodate==0.6.0            # via -r requirements/_base.txt, openapi-core
-isort==4.3.21             # via pylint
+isort==5.4.2              # via pylint
 jmespath==0.10.0          # via -r requirements/_base.txt, boto3, botocore
-jsonschema==3.2.0         # via -r requirements/_base.txt, openapi-spec-validator
+jsonschema==3.2.0         # via -r requirements/_base.txt, docker-compose, openapi-spec-validator
 lazy-object-proxy==1.4.3  # via -r requirements/_base.txt, astroid, openapi-core
-marshmallow==3.6.1        # via -r requirements/_base.txt
+marshmallow==3.7.1        # via -r requirements/_base.txt
 mccabe==0.6.1             # via pylint
 more-itertools==8.4.0     # via pytest
 multidict==4.7.6          # via -r requirements/_base.txt, aiohttp, yarl
-numpy==1.19.0             # via pandas
+numpy==1.19.1             # via pandas
 openapi-core==0.12.0      # via -r requirements/_base.txt
-openapi-spec-validator==0.2.8  # via -r requirements/_base.txt, openapi-core
+openapi-spec-validator==0.2.9  # via -r requirements/_base.txt, openapi-core
 packaging==20.4           # via pytest, pytest-sugar
-pandas==1.0.5             # via -r requirements/_test.in
+pandas==1.1.1             # via -r requirements/_test.in
+paramiko==2.7.1           # via docker
 pluggy==0.13.1            # via pytest
 prometheus-client==0.8.0  # via -r requirements/_base.txt
-protobuf==3.12.2          # via -r requirements/_base.txt, blackfynn
-psutil==5.7.0             # via -r requirements/_base.txt, blackfynn
+protobuf==3.13.0          # via -r requirements/_base.txt, blackfynn
+psutil==5.7.2             # via -r requirements/_base.txt, blackfynn
 psycopg2-binary==2.8.5    # via -r requirements/_base.txt, aiopg, sqlalchemy
 ptvsd==4.3.2              # via -r requirements/_test.in
 py==1.9.0                 # via pytest
-pylint==2.5.3             # via -r requirements/_test.in
+pycparser==2.20           # via cffi
+pylint==2.6.0             # via -r requirements/_test.in
+pynacl==1.4.0             # via paramiko
 pyparsing==2.4.7          # via packaging
 pyrsistent==0.16.0        # via -r requirements/_base.txt, jsonschema
 pytest-aiohttp==0.3.0     # via -r requirements/_test.in
-pytest-cov==2.10.0        # via -r requirements/_test.in
-pytest-docker==0.7.2      # via -r requirements/_test.in
+pytest-cov==2.10.1        # via -r requirements/_test.in
+pytest-docker==0.8.0      # via -r requirements/_test.in
 pytest-instafail==0.4.2   # via -r requirements/_test.in
-pytest-mock==3.1.1        # via -r requirements/_test.in
+pytest-mock==3.2.0        # via -r requirements/_test.in
 pytest-runner==5.2        # via -r requirements/_test.in
-pytest-sugar==0.9.3       # via -r requirements/_test.in
-pytest==5.4.3             # via -r requirements/_test.in, pytest-aiohttp, pytest-cov, pytest-instafail, pytest-mock, pytest-sugar
+pytest-sugar==0.9.4       # via -r requirements/_test.in
+pytest==5.4.3             # via -r requirements/_test.in, pytest-aiohttp, pytest-cov, pytest-docker, pytest-instafail, pytest-mock, pytest-sugar
 python-dateutil==2.8.1    # via -r requirements/_base.txt, blackfynn, botocore, pandas
+python-dotenv==0.14.0     # via docker-compose
 pytz==2020.1              # via -r requirements/_base.txt, blackfynn, pandas
-pyyaml==5.3.1             # via -r requirements/_base.txt, openapi-spec-validator, trafaret-config
-requests==2.24.0          # via -r requirements/_base.txt, blackfynn, codecov, coveralls
+pyyaml==5.3.1             # via -r requirements/_base.txt, docker-compose, openapi-spec-validator, trafaret-config
+requests==2.24.0          # via -r requirements/_base.txt, blackfynn, codecov, coveralls, docker, docker-compose
 s3transfer==0.3.3         # via -r requirements/_base.txt, boto3
 semantic-version==2.8.5   # via -r requirements/_base.txt
 semver==2.10.2            # via -r requirements/_base.txt, blackfynn
-six==1.15.0               # via -r requirements/_base.txt, astroid, isodate, jsonschema, openapi-core, openapi-spec-validator, packaging, protobuf, pyrsistent, python-dateutil, tenacity, websocket-client
+six==1.15.0               # via -r requirements/_base.txt, astroid, bcrypt, cryptography, docker, docker-compose, dockerpty, isodate, jsonschema, openapi-core, openapi-spec-validator, packaging, protobuf, pynacl, pyrsistent, python-dateutil, tenacity, websocket-client
 sqlalchemy[postgresql_psycopg2binary]==1.3.18  # via -r requirements/_base.txt, aiopg
 strict-rfc3339==0.7       # via -r requirements/_base.txt, openapi-core
 tenacity==6.2.0           # via -r requirements/_base.txt
 termcolor==1.1.0          # via pytest-sugar
+texttable==1.6.2          # via docker-compose
 toml==0.10.1              # via pylint
 trafaret-config==2.0.2    # via -r requirements/_base.txt
 trafaret==2.0.2           # via -r requirements/_base.txt, trafaret-config
 typed-ast==1.4.1          # via astroid
-typing-extensions==3.7.4.2  # via -r requirements/_base.txt, aiohttp, aioitertools
-ujson==3.0.0              # via -r requirements/_base.txt
-urllib3==1.25.9           # via -r requirements/_base.txt, botocore, requests
+typing-extensions==3.7.4.2  # via -r requirements/_base.txt, aiohttp, aioitertools, yarl
+ujson==3.1.0              # via -r requirements/_base.txt
+urllib3==1.25.10          # via -r requirements/_base.txt, botocore, requests
 wcwidth==0.2.5            # via pytest
-websocket-client==0.57.0  # via -r requirements/_base.txt, blackfynn
+websocket-client==0.57.0  # via -r requirements/_base.txt, blackfynn, docker, docker-compose
 werkzeug==1.0.1           # via -r requirements/_base.txt
 wrapt==1.12.1             # via -r requirements/_base.txt, aiobotocore, astroid, deprecated
-yarl==1.4.2               # via -r requirements/_base.txt, aiohttp
+yarl==1.5.1               # via -r requirements/_base.txt, aiohttp
 zipp==3.1.0               # via -r requirements/_base.txt, importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
diff --git a/tests/e2e/README.md b/tests/e2e/README.md
@@ -1,11 +1,32 @@
+
+**WARNING**: Be aware that running these tests in your host might risk changing some
+configuration.
+
 ## To run the tests locally
+
+Setting up the test
+
 ```bash
+/bin/bash ci/github/system-testing/e2e.bash clean_up
+docker volume prune
+/bin/bash ci/github/system-testing/e2e.bash install
+```
+
+Run the test
+```bash
+/bin/bash ci/github/system-testing/e2e.bash test
+# or
 cd tests/e2e
-npm install
 npm test
-# or
 npm run tutorials -- http://127.0.0.1:9081 --demo
+
+```
+
+Trying to cleanup
+```bash
+/bin/bash ci/github/system-testing/e2e.bash clean_up
 ```
+
 ## To debug the tests locally with VSCode
 Add the following configuration to your local ``launch.json``:
 ```json
@@ -27,6 +48,7 @@ Add the following configuration to your local ``launch.json``:
 }
 ```
 Now you can run the tests by clicking on the Play button, using that configuration. It should allow you to insert breakpoints and inspect variables.
+
 ## To run the tutorials
 ```bash
 cd tests/e2e
diff --git a/tests/e2e/requirements/Makefile b/tests/e2e/requirements/Makefile
@@ -6,7 +6,7 @@ VENV_DIR ?= $(realpath $(ROOT_DIR)/.venv)
 .PHONY: reqs
 requirements.txt: requirements.in
 	# pip compiling $<
-	@$(VENV_DIR)/bin/pip-compile --output-file $@ $<
+	@$(VENV_DIR)/bin/pip-compile --upgrade --build-isolation --output-file $@ $<
 
 reqs: requirements.txt ## alias to compile requirements.txt
 
diff --git a/tests/e2e/requirements/requirements.txt b/tests/e2e/requirements/requirements.txt
@@ -4,13 +4,13 @@
 #
 #    pip-compile --output-file=requirements.txt requirements.in
 #
-certifi==2019.11.28       # via requests
+certifi==2020.6.20        # via requests
 chardet==3.0.4            # via requests
-docker==4.2.0             # via -r requirements.in
-idna==2.9                 # via requests
-pyyaml==5.3               # via -r requirements.in
-requests==2.23.0          # via docker
-six==1.14.0               # via docker, tenacity, websocket-client
-tenacity==6.1.0           # via -r requirements.in
-urllib3==1.25.8           # via requests
+docker==4.3.0             # via -r requirements.in
+idna==2.10                # via requests
+pyyaml==5.3.1             # via -r requirements.in
+requests==2.24.0          # via docker
+six==1.15.0               # via docker, tenacity, websocket-client
+tenacity==6.2.0           # via -r requirements.in
+urllib3==1.25.10          # via requests
 websocket-client==0.57.0  # via docker
diff --git a/tests/e2e/utils/auto.js b/tests/e2e/utils/auto.js
@@ -38,7 +38,7 @@ async function logOut(page) {
 
   await page.waitForSelector('[osparc-test-id="userMenuMainBtn"]', {
     visible: true,
-    timeout: 1000
+    timeout: 2000
   });
   await utils.waitAndClick(page, '[osparc-test-id="userMenuMainBtn"]');
   await utils.waitAndClick(page, '[osparc-test-id="userMenuLogoutBtn"]');
diff --git a/tests/e2e/utils/utils.js b/tests/e2e/utils/utils.js
@@ -173,7 +173,9 @@ async function waitForValidOutputFile(page) {
 }
 
 async function waitAndClick(page, id) {
-  await page.waitForSelector(id);
+  await page.waitForSelector(id, {
+    timeout: 30000 // default 30s
+  });
   await page.click(id);
 }
 

Original file line number	Diff line number	Diff line change
`@@ -173,7 +173,9 @@ async function waitForValidOutputFile(page) {`
`173`	`173`	`}`
`174`	`174`
`175`	`175`	`async function waitAndClick(page, id) {`
`176`		`- await page.waitForSelector(id);`
	`176`	`+ await page.waitForSelector(id, {`
	`177`	`+ timeout: 30000 // default 30s`
	`178`	`+ });`
`177`	`179`	`await page.click(id);`
`178`	`180`	`}`
`179`	`181`