diff --git a/.github/workflows/ci-testing-deploy.yml b/.github/workflows/ci-testing-deploy.yml index e576d696dc8..7f4366f5789 100644 --- a/.github/workflows/ci-testing-deploy.yml +++ b/.github/workflows/ci-testing-deploy.yml @@ -1058,6 +1058,8 @@ jobs: ${{ runner.os }}-node- - name: setup images run: ./ci/github/system-testing/e2e.bash setup_images + - name: setup and start swarm + run: ./ci/github/system-testing/e2e.bash setup_and_run_stack - name: setup environment run: ./ci/github/system-testing/e2e.bash setup_environment - name: setup registry @@ -1070,7 +1072,7 @@ jobs: - name: recover docker logs id: logs_recovery if: always() - run: ./ci/github/system-testing/e2e.bash recover_artifacts + run: ./ci/github/system-testing/e2e.bash dump_docker_logs - name: upload docker logs if: always() uses: actions/upload-artifact@v2 diff --git a/Makefile b/Makefile index 41913ad6cd8..97bc438abd8 100644 --- a/Makefile +++ b/Makefile @@ -318,7 +318,7 @@ push-version: tag-version ## ENVIRONMENT ------------------------------- -.PHONY: devenv devenv-all +.PHONY: devenv devenv-all node-env .venv: python3 -m venv $@ @@ -338,6 +338,15 @@ devenv-all: devenv ## sets up extra development tools (everything else besides p @$(MAKE_C) scripts/json-schema-to-openapi-schema +node_modules: package.json + # checking npm installed + @npm --version + # installing package.json + npm install --package-lock + +nodenv: node_modules ## builds node_modules local environ (TODO) + + .env: .env-devel ## creates .env file from defaults in .env-devel $(if $(wildcard $@), \ @echo "WARNING ##### $< is newer than $@ ####"; diff -uN $@ $<; false;,\ diff --git a/api/tests/requirements.txt b/api/tests/requirements.txt index 2aebb922871..057d7e9c2b1 100644 --- a/api/tests/requirements.txt +++ b/api/tests/requirements.txt @@ -6,7 +6,7 @@ # aiohttp==3.6.2 # via -r requirements.in, pytest-aiohttp async-timeout==3.0.1 # via aiohttp -attrs==20.1.0 # via aiohttp, jsonschema, openapi-core, pytest +attrs==20.2.0 # via aiohttp, jsonschema, openapi-core, pytest chardet==3.0.4 # via aiohttp coverage==5.2.1 # via -r requirements.in, pytest-cov idna-ssl==1.1.0 # via aiohttp @@ -16,13 +16,13 @@ iniconfig==1.0.1 # via pytest isodate==0.6.0 # via openapi-core, openapi-schema-validator jsonschema==3.2.0 # via openapi-schema-validator, openapi-spec-validator lazy-object-proxy==1.5.1 # via openapi-core -more-itertools==8.4.0 # via openapi-core, pytest +more-itertools==8.5.0 # via openapi-core, pytest multidict==4.7.6 # via aiohttp, yarl openapi-core==0.13.4 # via -r requirements.in openapi-schema-validator==0.1.1 # via openapi-core openapi-spec-validator==0.2.9 # via openapi-core packaging==20.4 # via pytest, pytest-sugar -parse==1.16.0 # via openapi-core +parse==1.17.0 # via openapi-core pluggy==0.13.1 # via pytest py==1.9.0 # via pytest pyparsing==2.4.7 # via packaging diff --git a/ci/github/system-testing/e2e.bash b/ci/github/system-testing/e2e.bash index 8a36a457d4a..547f1e1e46c 100755 --- a/ci/github/system-testing/e2e.bash +++ b/ci/github/system-testing/e2e.bash @@ -74,13 +74,17 @@ uninstall_insecure_registry() { } setup_images() { - echo "--------------- getting simcore docker images..." + echo "--------------- preparing docker images..." make pull-version || ( (make pull-cache || true) && make build-x tag-version) make info-images +} + +setup_and_run_stack() { # configure simcore for testing with a private registry install_insecure_registry + echo "--------------- starting swarm ..." # start simcore and set log-level export LOG_LEVEL=WARNING make up-version @@ -135,29 +139,32 @@ setup_database() { install() { ## shortcut setup_images + setup_and_run_stack setup_environment setup_registry setup_database } test() { + sleep 5 pushd tests/e2e make test popd - } -recover_artifacts() { +dump_docker_logs() { # all screenshots are in tests/e2e/screenshots if any # get docker logs. - # WARNING: dumping long logs might take hours!! - mkdir simcore_logs - (docker service logs --timestamps --tail=300 --details ${SWARM_STACK_NAME}_webserver >simcore_logs/webserver.log 2>&1) || true - (docker service logs --timestamps --tail=200 --details ${SWARM_STACK_NAME}_director >simcore_logs/director.log 2>&1) || true - (docker service logs --timestamps --tail=200 --details ${SWARM_STACK_NAME}_storage >simcore_logs/storage.log 2>&1) || true - (docker service logs --timestamps --tail=200 --details ${SWARM_STACK_NAME}_sidecar >simcore_logs/sidecar.log 2>&1) || true - (docker service logs --timestamps --tail=200 --details ${SWARM_STACK_NAME}_catalog >simcore_logs/catalog.log 2>&1) || true + # NOTE: dumping logs sometimes hangs. Introducing a timeout + mkdir --parents simcore_logs + (timeout 30 docker service logs --timestamps --tail=300 --details ${SWARM_STACK_NAME}_webserver >simcore_logs/webserver.log 2>&1) || true + (timeout 30 docker service logs --timestamps --tail=200 --details ${SWARM_STACK_NAME}_director >simcore_logs/director.log 2>&1) || true + (timeout 30 docker service logs --timestamps --tail=200 --details ${SWARM_STACK_NAME}_storage >simcore_logs/storage.log 2>&1) || true + (timeout 30 docker service logs --timestamps --tail=200 --details ${SWARM_STACK_NAME}_sidecar >simcore_logs/sidecar.log 2>&1) || true + (timeout 30 docker service logs --timestamps --tail=200 --details ${SWARM_STACK_NAME}_catalog >simcore_logs/catalog.log 2>&1) || true + (timeout 30 docker service logs --timestamps --tail=200 --details ${SWARM_STACK_NAME}_migration >simcore_logs/migration.log 2>&1) || true + (timeout 30 docker service logs --timestamps --tail=200 --details ${SWARM_STACK_NAME}_postgres >simcore_logs/postgres.log 2>&1) || true } clean_up() { diff --git a/docs/img/.stack-simcore-version.yml.png b/docs/img/.stack-simcore-version.yml.png index 26f730a4065..894ba87dc9f 100644 Binary files a/docs/img/.stack-simcore-version.yml.png and b/docs/img/.stack-simcore-version.yml.png differ diff --git a/package-lock.json b/package-lock.json index ec22eaa55e5..a5d7c367d5d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2166,6 +2166,17 @@ "which-module": "^2.0.0", "y18n": "^3.2.1", "yargs-parser": "^9.0.2" + }, + "dependencies": { + "yargs-parser": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-9.0.2.tgz", + "integrity": "sha1-nM9qQ0YP5O1Aqbto9I1DuKaMwHc=", + "dev": true, + "requires": { + "camelcase": "^4.1.0" + } + } } } } @@ -2858,13 +2869,10 @@ } }, "yargs-parser": { - "version": "9.0.2", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-9.0.2.tgz", - "integrity": "sha1-nM9qQ0YP5O1Aqbto9I1DuKaMwHc=", - "dev": true, - "requires": { - "camelcase": "^4.1.0" - } + "version": "19.0.4", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-19.0.4.tgz", + "integrity": "sha512-eXeQm7yXRjPFFyf1voPkZgXQZJjYfjgQUmGPbD2TLtZeIYzvacgWX7sQ5a1HsRgVP+pfKAkRZDNtTGev4h9vhw==", + "dev": true }, "yauzl": { "version": "2.4.1", diff --git a/package.json b/package.json index 19aaa447355..8533c589643 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "eslint-plugin-qx-rules": "^0.1.0", "puppeteer": "^1.19.0", "puppeteer-to-istanbul": "^1.2.2", - "yargs": "^13.3.0" + "yargs": "^13.3.0", + "yargs-parser": ">=13.1.2" } } diff --git a/packages/postgres-database/Makefile b/packages/postgres-database/Makefile index 630ede5bf07..ab469176512 100644 --- a/packages/postgres-database/Makefile +++ b/packages/postgres-database/Makefile @@ -89,7 +89,6 @@ migrate: $(DOT_ENV_FILE) ## basic migration update (use ONLY for development pur - .PHONY: up-pg down-pg up-prod down-prod docker-compose-configs = $(wildcard tests/docker-compose*.yml) up-pg up-prod: $(docker-compose-configs) ## starts pg server @@ -97,3 +96,12 @@ up-pg up-prod: $(docker-compose-configs) ## starts pg server down-pg down-prod: $(docker-compose-configs) ## stops pg server docker-compose -f tests/docker-compose.yml $(if $(findstring -prod,$@),-f tests/docker-compose.prod.yml,) down + + + +.PHONY: auto-doc +auto-doc: install-dev ## Creates entity relationship diagram (ERD) defined under ``simcore_postgres_database.models`` + # installing doc dependencies (install-doc) + pip install eralchemy + # running script + python scripts/create_erd.py diff --git a/packages/postgres-database/doc/img/postgres-database-models.svg b/packages/postgres-database/doc/img/postgres-database-models.svg index b5b82bade70..2cabdf4b41b 100644 --- a/packages/postgres-database/doc/img/postgres-database-models.svg +++ b/packages/postgres-database/doc/img/postgres-database-models.svg @@ -4,11 +4,11 @@ - - + + %3 - + file_meta_data @@ -72,539 +72,542 @@ groups - -groups - -gid - [BIGINT] - -name - [VARCHAR] - -description - [VARCHAR] - -type - [VARCHAR(8)] - -thumbnail - [VARCHAR] - -created - [DATETIME] - -modified - [DATETIME] + +groups + +gid + [BIGINT] + +name + [VARCHAR] + +description + [VARCHAR] + +type + [VARCHAR(8)] + +thumbnail + [VARCHAR] + +inclusion_rules + [JSONB] + +created + [DATETIME] + +modified + [DATETIME] user_to_groups - -user_to_groups - -uid - [BIGINT] - -gid - [BIGINT] - -access_rights - [JSONB] - -created - [DATETIME] - -modified - [DATETIME] + +user_to_groups + +uid + [BIGINT] + +gid + [BIGINT] + +access_rights + [JSONB] + +created + [DATETIME] + +modified + [DATETIME] groups--user_to_groups - -{0,1} -0..N + +{0,1} +0..N users - -users - -id - [BIGINT] - -name - [VARCHAR] - -email - [VARCHAR] - -password_hash - [VARCHAR] - -primary_gid - [BIGINT] - -status - [VARCHAR(20)] - -role - [VARCHAR(9)] - -created_at - [DATETIME] - -modified - [DATETIME] - -created_ip - [VARCHAR] + +users + +id + [BIGINT] + +name + [VARCHAR] + +email + [VARCHAR] + +password_hash + [VARCHAR] + +primary_gid + [BIGINT] + +status + [VARCHAR(20)] + +role + [VARCHAR(9)] + +created_at + [DATETIME] + +modified + [DATETIME] + +created_ip + [VARCHAR] groups--users - -{0,1} -0..N + +{0,1} +0..N group_classifiers - -group_classifiers - -id - [BIGINT] - -bundle - [JSONB] - -created - [DATETIME] - -modified - [DATETIME] - -gid - [BIGINT] + +group_classifiers + +id + [BIGINT] + +bundle + [JSONB] + +created + [DATETIME] + +modified + [DATETIME] + +gid + [BIGINT] groups--group_classifiers - -{0,1} -0..N + +{0,1} +0..N services_meta_data - -services_meta_data - -key - [VARCHAR] - -version - [VARCHAR] - -owner - [BIGINT] - -name - [VARCHAR] - -description - [VARCHAR] - -thumbnail - [VARCHAR] - -classifiers - [VARCHAR[]] - -created - [DATETIME] - -modified - [DATETIME] + +services_meta_data + +key + [VARCHAR] + +version + [VARCHAR] + +owner + [BIGINT] + +name + [VARCHAR] + +description + [VARCHAR] + +thumbnail + [VARCHAR] + +classifiers + [VARCHAR[]] + +created + [DATETIME] + +modified + [DATETIME] groups--services_meta_data - -{0,1} -0..N + +{0,1} +0..N services_access_rights - -services_access_rights - -key - [VARCHAR] - -version - [VARCHAR] - -gid - [BIGINT] - -execute_access - [BOOLEAN] - -write_access - [BOOLEAN] - -created - [DATETIME] - -modified - [DATETIME] + +services_access_rights + +key + [VARCHAR] + +version + [VARCHAR] + +gid + [BIGINT] + +execute_access + [BOOLEAN] + +write_access + [BOOLEAN] + +created + [DATETIME] + +modified + [DATETIME] groups--services_access_rights - -{0,1} -0..N + +{0,1} +0..N users--user_to_groups - -{0,1} -0..N + +{0,1} +0..N projects - -projects - -id - [BIGINT] - -type - [VARCHAR(8)] - -uuid - [VARCHAR] - -name - [VARCHAR] - -description - [VARCHAR] - -thumbnail - [VARCHAR] - -prj_owner - [BIGINT] - -creation_date - [DATETIME] - -last_change_date - [DATETIME] - -access_rights - [JSONB] - -workbench - [Null] - -classifiers - [VARCHAR[]] - -dev - [JSONB] - -published - [BOOLEAN] + +projects + +id + [BIGINT] + +type + [VARCHAR(8)] + +uuid + [VARCHAR] + +name + [VARCHAR] + +description + [VARCHAR] + +thumbnail + [VARCHAR] + +prj_owner + [BIGINT] + +creation_date + [DATETIME] + +last_change_date + [DATETIME] + +access_rights + [JSONB] + +workbench + [Null] + +classifiers + [VARCHAR[]] + +dev + [JSONB] + +published + [BOOLEAN] users--projects - -{0,1} -0..N + +{0,1} +0..N user_to_projects - -user_to_projects - -id - [BIGINT] - -user_id - [BIGINT] - -project_id - [BIGINT] + +user_to_projects + +id + [BIGINT] + +user_id + [BIGINT] + +project_id + [BIGINT] users--user_to_projects - -{0,1} -0..N + +{0,1} +0..N tokens - -tokens - -token_id - [BIGINT] - -user_id - [BIGINT] - -token_service - [VARCHAR] - -token_data - [Null] + +tokens + +token_id + [BIGINT] + +user_id + [BIGINT] + +token_service + [VARCHAR] + +token_data + [Null] users--tokens - -{0,1} -0..N + +{0,1} +0..N api_keys - -api_keys - -id - [BIGINT] - -display_name - [VARCHAR] - -user_id - [BIGINT] - -api_key - [VARCHAR] - -api_secret - [VARCHAR] + +api_keys + +id + [BIGINT] + +display_name + [VARCHAR] + +user_id + [BIGINT] + +api_key + [VARCHAR] + +api_secret + [VARCHAR] users--api_keys - -{0,1} -0..N + +{0,1} +0..N confirmations - -confirmations - -code - [TEXT] - -user_id - [BIGINT] - -action - [VARCHAR(14)] - -data - [TEXT] - -created_at - [DATETIME] + +confirmations + +code + [TEXT] + +user_id + [BIGINT] + +action + [VARCHAR(14)] + +data + [TEXT] + +created_at + [DATETIME] users--confirmations - -{0,1} -0..N + +{0,1} +0..N tags - -tags - -id - [BIGINT] - -user_id - [BIGINT] - -name - [VARCHAR] - -description - [VARCHAR] - -color - [VARCHAR] + +tags + +id + [BIGINT] + +user_id + [BIGINT] + +name + [VARCHAR] + +description + [VARCHAR] + +color + [VARCHAR] users--tags - -{0,1} -0..N + +{0,1} +0..N services_meta_data--services_access_rights - -{0,1} -0..N + +{0,1} +0..N services_meta_data--services_access_rights - -{0,1} -0..N + +{0,1} +0..N study_tags - -study_tags - -study_id - [BIGINT] - -tag_id - [BIGINT] + +study_tags + +study_id + [BIGINT] + +tag_id + [BIGINT] projects--study_tags - -{0,1} -0..N + +{0,1} +0..N projects--user_to_projects - -{0,1} -0..N + +{0,1} +0..N tags--study_tags - -{0,1} -0..N + +{0,1} +0..N comp_pipeline - -comp_pipeline - -project_id - [VARCHAR] - -dag_adjacency_list - [Null] - -state - [VARCHAR] + +comp_pipeline + +project_id + [VARCHAR] + +dag_adjacency_list + [Null] + +state + [VARCHAR] comp_tasks - -comp_tasks - -task_id - [INTEGER] - -project_id - [VARCHAR] - -node_id - [VARCHAR] - -node_class - [VARCHAR(13)] - -job_id - [VARCHAR] - -internal_id - [INTEGER] - -schema - [Null] - -inputs - [Null] - -outputs - [Null] - -image - [Null] - -state - [INTEGER] - -submit - [DATETIME] - -start - [DATETIME] - -end - [DATETIME] + +comp_tasks + +task_id + [INTEGER] + +project_id + [VARCHAR] + +node_id + [VARCHAR] + +node_class + [VARCHAR(13)] + +job_id + [VARCHAR] + +internal_id + [INTEGER] + +schema + [Null] + +inputs + [Null] + +outputs + [Null] + +image + [Null] + +state + [INTEGER] + +submit + [DATETIME] + +start + [DATETIME] + +end + [DATETIME] comp_pipeline--comp_tasks - -{0,1} -0..N + +{0,1} +0..N dags - -dags - -id - [INTEGER] - -key - [VARCHAR] - -version - [VARCHAR] - -name - [VARCHAR] - -description - [VARCHAR] - -contact - [VARCHAR] - -workbench - [Null] + +dags + +id + [INTEGER] + +key + [VARCHAR] + +version + [VARCHAR] + +name + [VARCHAR] + +description + [VARCHAR] + +contact + [VARCHAR] + +workbench + [Null] diff --git a/packages/postgres-database/requirements/_base.txt b/packages/postgres-database/requirements/_base.txt index a1a556ce21b..79c456b019b 100644 --- a/packages/postgres-database/requirements/_base.txt +++ b/packages/postgres-database/requirements/_base.txt @@ -6,7 +6,7 @@ # idna==2.10 # via yarl multidict==4.7.6 # via yarl -psycopg2-binary==2.8.5 # via sqlalchemy +psycopg2-binary==2.8.6 # via sqlalchemy sqlalchemy[postgresql_psycopg2binary]==1.3.19 # via -r requirements/_base.in typing-extensions==3.7.4.3 # via yarl yarl==1.5.1 # via -r requirements/_base.in diff --git a/packages/postgres-database/requirements/_migration.txt b/packages/postgres-database/requirements/_migration.txt index 99ec2763727..58b43461f95 100644 --- a/packages/postgres-database/requirements/_migration.txt +++ b/packages/postgres-database/requirements/_migration.txt @@ -13,7 +13,7 @@ idna==2.10 # via -r requirements/_base.txt, requests, yarl mako==1.1.3 # via alembic markupsafe==1.1.1 # via mako multidict==4.7.6 # via -r requirements/_base.txt, yarl -psycopg2-binary==2.8.5 # via -r requirements/_base.txt, sqlalchemy +psycopg2-binary==2.8.6 # via -r requirements/_base.txt, sqlalchemy python-dateutil==2.8.1 # via alembic python-editor==1.0.4 # via alembic requests==2.24.0 # via docker diff --git a/packages/postgres-database/requirements/_test.txt b/packages/postgres-database/requirements/_test.txt index 192f4f0a0de..0a8a8387fe8 100644 --- a/packages/postgres-database/requirements/_test.txt +++ b/packages/postgres-database/requirements/_test.txt @@ -18,7 +18,7 @@ chardet==3.0.4 # via -r requirements/_migration.txt, aiohttp, request click==7.1.2 # via -r requirements/_migration.txt coverage==5.2.1 # via -r requirements/_test.in, coveralls, pytest-cov coveralls==2.1.2 # via -r requirements/_test.in -cryptography==3.0 # via paramiko +cryptography==3.1 # via paramiko distro==1.5.0 # via docker-compose docker-compose==1.26.2 # via pytest-docker docker[ssh]==4.3.1 # via -r requirements/_migration.txt, docker-compose @@ -28,18 +28,18 @@ faker==4.1.2 # via -r requirements/_test.in idna-ssl==1.1.0 # via aiohttp idna==2.10 # via -r requirements/_migration.txt, idna-ssl, requests, yarl importlib-metadata==1.7.0 # via jsonschema, pluggy, pytest -isort==5.4.2 # via pylint +isort==5.5.1 # via pylint jsonschema==3.2.0 # via docker-compose lazy-object-proxy==1.4.3 # via astroid mako==1.1.3 # via -r requirements/_migration.txt, alembic markupsafe==1.1.1 # via -r requirements/_migration.txt, mako mccabe==0.6.1 # via pylint -more-itertools==8.4.0 # via pytest +more-itertools==8.5.0 # via pytest multidict==4.7.6 # via -r requirements/_migration.txt, aiohttp, yarl packaging==20.4 # via pytest -paramiko==2.7.1 # via docker +paramiko==2.7.2 # via docker pluggy==0.13.1 # via pytest -psycopg2-binary==2.8.5 # via -r requirements/_migration.txt, aiopg, sqlalchemy +psycopg2-binary==2.8.6 # via -r requirements/_migration.txt, aiopg, sqlalchemy py==1.9.0 # via pytest pycparser==2.20 # via cffi pylint==2.6.0 # via -r requirements/_test.in @@ -61,7 +61,7 @@ six==1.15.0 # via -r requirements/_migration.txt, astroid, bcrypt, sqlalchemy[postgresql_psycopg2binary]==1.3.19 # via -r requirements/_migration.txt, aiopg, alembic tenacity==6.2.0 # via -r requirements/_migration.txt text-unidecode==1.3 # via faker -texttable==1.6.2 # via docker-compose +texttable==1.6.3 # via docker-compose toml==0.10.1 # via pylint typed-ast==1.4.1 # via astroid typing-extensions==3.7.4.3 # via -r requirements/_migration.txt, aiohttp, yarl diff --git a/packages/pytest-simcore/src/pytest_simcore/postgres_service.py b/packages/pytest-simcore/src/pytest_simcore/postgres_service.py index 47225792688..0b4f955bfac 100644 --- a/packages/pytest-simcore/src/pytest_simcore/postgres_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/postgres_service.py @@ -58,27 +58,41 @@ def postgres_engine( @pytest.fixture(scope="module") def postgres_db( - postgres_dsn: Dict, postgres_engine: sa.engine.Engine + postgres_dsn: Dict, + postgres_engine: sa.engine.Engine, ) -> sa.engine.Engine: - # migrate the database + + # upgrades database from zero kwargs = postgres_dsn.copy() pg_cli.discover.callback(**kwargs) pg_cli.upgrade.callback("head") + yield postgres_engine + # downgrades database to zero --- + # + # NOTE: This step CANNOT be avoided since it would leave the db in an invalid state + # E.g. 'alembic_version' table is not deleted and keeps head version or routines + # like 'notify_comp_tasks_changed' remain undeleted + # pg_cli.downgrade.callback("base") - pg_cli.clean.callback() + pg_cli.clean.callback() # just cleans discover cache - # FIXME: deletes all because downgrade is not reliable! + # FIXME: migration downgrade fails to remove User types SEE https://github.com/ITISFoundation/osparc-simcore/issues/1776 + # Added drop_all as tmp fix metadata.drop_all(postgres_engine) @pytest.fixture(scope="module") def postgres_session(postgres_db: sa.engine.Engine) -> sa.orm.session.Session: - Session = sessionmaker(postgres_db) - session = Session() + from sqlalchemy.orm.session import Session + + Session_cls = sessionmaker(postgres_db) + session: Session = Session_cls() + yield session - session.close() + + session.close() # pylint: disable=no-member @tenacity.retry( diff --git a/packages/s3wrapper/requirements/_test.txt b/packages/s3wrapper/requirements/_test.txt index c61a4bfae31..44d68ad356c 100644 --- a/packages/s3wrapper/requirements/_test.txt +++ b/packages/s3wrapper/requirements/_test.txt @@ -14,7 +14,7 @@ chardet==3.0.4 # via requests configparser==5.0.0 # via -r requirements/_base.txt, minio coverage==5.2.1 # via -r requirements/_test.in, coveralls, pytest-cov coveralls==2.1.2 # via -r requirements/_test.in -cryptography==3.0 # via paramiko +cryptography==3.1 # via paramiko distro==1.5.0 # via docker-compose docker-compose==1.26.2 # via pytest-docker docker[ssh]==4.3.1 # via docker-compose @@ -22,14 +22,14 @@ dockerpty==0.4.1 # via docker-compose docopt==0.6.2 # via coveralls, docker-compose idna==2.10 # via requests importlib-metadata==1.7.0 # via jsonschema, pluggy, pytest -isort==5.4.2 # via pylint +isort==5.5.1 # via pylint jsonschema==3.2.0 # via docker-compose lazy-object-proxy==1.4.3 # via astroid mccabe==0.6.1 # via pylint minio==6.0.0 # via -r requirements/_base.txt -more-itertools==8.4.0 # via pytest +more-itertools==8.5.0 # via pytest packaging==20.4 # via pytest -paramiko==2.7.1 # via docker +paramiko==2.7.2 # via docker pluggy==0.13.1 # via pytest py==1.9.0 # via pytest pycparser==2.20 # via cffi @@ -47,7 +47,7 @@ pytz==2020.1 # via -r requirements/_base.txt, minio pyyaml==5.3.1 # via docker-compose requests==2.24.0 # via -r requirements/_test.in, coveralls, docker, docker-compose six==1.15.0 # via -r requirements/_base.txt, astroid, bcrypt, cryptography, docker, docker-compose, dockerpty, jsonschema, packaging, pynacl, pyrsistent, python-dateutil, websocket-client -texttable==1.6.2 # via docker-compose +texttable==1.6.3 # via docker-compose toml==0.10.1 # via pylint typed-ast==1.4.1 # via astroid urllib3==1.25.10 # via -r requirements/_base.txt, minio, requests diff --git a/packages/service-library/requirements/_base.txt b/packages/service-library/requirements/_base.txt index 3a40ca6e357..c7d387d9946 100644 --- a/packages/service-library/requirements/_base.txt +++ b/packages/service-library/requirements/_base.txt @@ -21,7 +21,7 @@ multidict==4.7.6 # via aiohttp, yarl openapi-core==0.12.0 # via -r requirements/_base.in openapi-spec-validator==0.2.9 # via openapi-core prometheus-client==0.8.0 # via -r requirements/_base.in -psycopg2-binary==2.8.5 # via -r requirements/_base.in, aiopg, sqlalchemy +psycopg2-binary==2.8.6 # via -r requirements/_base.in, aiopg, sqlalchemy pyrsistent==0.16.0 # via jsonschema pyyaml==5.3.1 # via -r requirements/_base.in, openapi-spec-validator six==1.15.0 # via isodate, jsonschema, openapi-core, openapi-spec-validator, pyrsistent, tenacity diff --git a/packages/service-library/requirements/_test.txt b/packages/service-library/requirements/_test.txt index 9c135badd2e..ef35ba52f19 100644 --- a/packages/service-library/requirements/_test.txt +++ b/packages/service-library/requirements/_test.txt @@ -18,29 +18,29 @@ cffi==1.14.2 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via -r requirements/_base.txt, aiohttp, requests coverage==5.2.1 # via -r requirements/_test.in, coveralls, pytest-cov coveralls==2.1.2 # via -r requirements/_test.in -cryptography==3.0 # via paramiko +cryptography==3.1 # via paramiko distro==1.5.0 # via docker-compose docker-compose==1.26.2 # via pytest-docker docker[ssh]==4.3.1 # via docker-compose dockerpty==0.4.1 # via docker-compose docopt==0.6.2 # via coveralls, docker-compose -idna-ssl==1.1.0 # via -r requirements/_base.txt +idna-ssl==1.1.0 # via -r requirements/_base.txt, aiohttp idna==2.10 # via -r requirements/_base.txt, idna-ssl, requests, yarl -importlib-metadata==1.7.0 # via -r requirements/_base.txt +importlib-metadata==1.7.0 # via -r requirements/_base.txt, jsonschema, pluggy, pytest isodate==0.6.0 # via -r requirements/_base.txt, openapi-core -isort==5.4.2 # via pylint +isort==5.5.1 # via pylint jsonschema==3.2.0 # via -r requirements/_base.txt, docker-compose, openapi-spec-validator lazy-object-proxy==1.4.3 # via -r requirements/_base.txt, astroid, openapi-core mccabe==0.6.1 # via pylint -more-itertools==8.4.0 # via pytest +more-itertools==8.5.0 # via pytest multidict==4.7.6 # via -r requirements/_base.txt, aiohttp, yarl openapi-core==0.12.0 # via -r requirements/_base.txt openapi-spec-validator==0.2.9 # via -r requirements/_base.txt, openapi-core packaging==20.4 # via pytest, pytest-sugar -paramiko==2.7.1 # via docker +paramiko==2.7.2 # via docker pluggy==0.13.1 # via pytest prometheus-client==0.8.0 # via -r requirements/_base.txt -psycopg2-binary==2.8.5 # via -r requirements/_base.txt, aiopg, sqlalchemy +psycopg2-binary==2.8.6 # via -r requirements/_base.txt, aiopg, sqlalchemy py==1.9.0 # via pytest pycparser==2.20 # via cffi pylint==2.6.0 # via -r requirements/_test.in @@ -63,10 +63,11 @@ sqlalchemy[postgresql_psycopg2binary]==1.3.19 # via -r requirements/_base.txt, strict-rfc3339==0.7 # via -r requirements/_base.txt, openapi-core tenacity==6.2.0 # via -r requirements/_base.txt termcolor==1.1.0 # via pytest-sugar -texttable==1.6.2 # via docker-compose +texttable==1.6.3 # via docker-compose toml==0.10.1 # via pylint trafaret==2.0.2 # via -r requirements/_base.txt -typing-extensions==3.7.4.3 # via -r requirements/_base.txt +typed-ast==1.4.1 # via astroid +typing-extensions==3.7.4.3 # via -r requirements/_base.txt, aiohttp, yarl ujson==3.1.0 # via -r requirements/_base.txt urllib3==1.25.10 # via requests wcwidth==0.2.5 # via pytest diff --git a/packages/simcore-sdk/requirements/_base.txt b/packages/simcore-sdk/requirements/_base.txt index 345b0a7d24f..cf80d30ec2c 100644 --- a/packages/simcore-sdk/requirements/_base.txt +++ b/packages/simcore-sdk/requirements/_base.txt @@ -16,7 +16,7 @@ idna-ssl==1.1.0 # via aiohttp idna==2.10 # via idna-ssl, yarl multidict==4.7.6 # via aiohttp, yarl networkx==2.5 # via -r requirements/_base.in -psycopg2-binary==2.8.5 # via -r requirements/_base.in, aiopg, sqlalchemy +psycopg2-binary==2.8.6 # via -r requirements/_base.in, aiopg, sqlalchemy pydantic==1.6.1 # via -r requirements/_base.in pyyaml==5.3.1 # via trafaret-config six==1.15.0 # via tenacity diff --git a/packages/simcore-sdk/requirements/_test.txt b/packages/simcore-sdk/requirements/_test.txt index 303a2b53524..2c9b35b2743 100644 --- a/packages/simcore-sdk/requirements/_test.txt +++ b/packages/simcore-sdk/requirements/_test.txt @@ -17,7 +17,7 @@ cffi==1.14.2 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via -r requirements/_base.txt, aiohttp, requests coverage==5.2.1 # via -r requirements/_test.in, coveralls, pytest-cov coveralls==2.1.2 # via -r requirements/_test.in -cryptography==3.0 # via paramiko +cryptography==3.1 # via paramiko dataclasses==0.7 # via -r requirements/_base.txt, pydantic decorator==4.4.2 # via -r requirements/_base.txt, networkx distro==1.5.0 # via docker-compose @@ -28,18 +28,18 @@ docopt==0.6.2 # via coveralls, docker-compose idna-ssl==1.1.0 # via -r requirements/_base.txt, aiohttp idna==2.10 # via -r requirements/_base.txt, idna-ssl, requests, yarl importlib-metadata==1.7.0 # via jsonschema, pluggy, pytest -isort==5.4.2 # via pylint +isort==5.5.1 # via pylint jsonschema==3.2.0 # via docker-compose lazy-object-proxy==1.4.3 # via astroid mccabe==0.6.1 # via pylint mock==4.0.2 # via -r requirements/_test.in -more-itertools==8.4.0 # via pytest +more-itertools==8.5.0 # via pytest multidict==4.7.6 # via -r requirements/_base.txt, aiohttp, yarl networkx==2.5 # via -r requirements/_base.txt packaging==20.4 # via pytest, pytest-sugar -paramiko==2.7.1 # via docker +paramiko==2.7.2 # via docker pluggy==0.13.1 # via pytest -psycopg2-binary==2.8.5 # via -r requirements/_base.txt, aiopg, sqlalchemy +psycopg2-binary==2.8.6 # via -r requirements/_base.txt, aiopg, sqlalchemy py==1.9.0 # via pytest pycparser==2.20 # via cffi pydantic==1.6.1 # via -r requirements/_base.txt @@ -51,7 +51,7 @@ pytest-aiohttp==0.3.0 # via -r requirements/_test.in pytest-cov==2.10.1 # via -r requirements/_test.in pytest-docker==0.8.0 # via -r requirements/_test.in pytest-instafail==0.4.2 # via -r requirements/_test.in -pytest-mock==3.3.0 # via -r requirements/_test.in +pytest-mock==3.3.1 # via -r requirements/_test.in pytest-runner==5.2 # via -r requirements/_test.in pytest-sugar==0.9.4 # via -r requirements/_test.in pytest==5.4.3 # via -r requirements/_test.in, pytest-aiohttp, pytest-cov, pytest-docker, pytest-instafail, pytest-mock, pytest-sugar @@ -62,7 +62,7 @@ six==1.15.0 # via -r requirements/_base.txt, astroid, bcrypt, cryp sqlalchemy[postgresql_psycopg2binary]==1.3.19 # via -r requirements/_base.txt, aiopg tenacity==6.2.0 # via -r requirements/_base.txt termcolor==1.1.0 # via pytest-sugar -texttable==1.6.2 # via docker-compose +texttable==1.6.3 # via docker-compose toml==0.10.1 # via pylint trafaret-config==2.0.2 # via -r requirements/_base.txt trafaret==2.0.2 # via -r requirements/_base.txt, trafaret-config diff --git a/services/docker-compose-build.yml b/services/docker-compose-build.yml index 15f222079ed..d0f47c1652e 100644 --- a/services/docker-compose-build.yml +++ b/services/docker-compose-build.yml @@ -80,6 +80,10 @@ services: - ${DOCKER_REGISTRY:-itisfoundation}/webserver:cache - ${DOCKER_REGISTRY:-itisfoundation}/webserver:${DOCKER_IMAGE_TAG:-latest} target: ${BUILD_TARGET:?build_target_required} + args: + - BUILD_DATE=${BUILD_DATE} + - VCS_URL=${VCS_URL} + - VCS_REF=${VCS_REF} labels: org.label-schema.schema-version: "1.0" org.label-schema.build-date: "${BUILD_DATE}" diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 5b02049300a..4352764ee0d 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -93,7 +93,6 @@ services: image: ${DOCKER_REGISTRY:-itisfoundation}/webserver:${DOCKER_IMAGE_TAG:-latest} init: true environment: - - BUILD_DATE=${BUILD_DATE:-1970-01-01T01:00:00Z} - DIRECTOR_HOST=${DIRECTOR_HOST:-director} - DIRECTOR_PORT=${DIRECTOR_PORT:-8080} - STORAGE_HOST=${STORAGE_HOST:-storage} diff --git a/services/sidecar/src/simcore_service_sidecar/config.py b/services/sidecar/src/simcore_service_sidecar/config.py index a68ca2e9b78..e18cd5b1e54 100644 --- a/services/sidecar/src/simcore_service_sidecar/config.py +++ b/services/sidecar/src/simcore_service_sidecar/config.py @@ -43,7 +43,7 @@ ) SIDECAR_LOGLEVEL: str = getattr( - logging, os.environ.get("SIDECAR_LOGLEVEL", "WARNING").upper(), logging.WARNING + logging, os.environ.get("SIDECAR_LOGLEVEL", "WARNING").upper(), logging.DEBUG ) DOCKER_REGISTRY: str = os.environ.get("REGISTRY_URL", "masu.speag.com") diff --git a/services/sidecar/src/simcore_service_sidecar/utils.py b/services/sidecar/src/simcore_service_sidecar/utils.py index 6013cc0a961..0828f22cb32 100644 --- a/services/sidecar/src/simcore_service_sidecar/utils.py +++ b/services/sidecar/src/simcore_service_sidecar/utils.py @@ -99,8 +99,12 @@ async def async_is_gpu_node() -> bool: config=spec_config, name=f"sidecar_{uuid.uuid4()}_test_gpu" ) return True - except aiodocker.exceptions.DockerError as e: - logger.warning("is_gpu_node DockerError during check: %s", str(e)) + except aiodocker.exceptions.DockerError as err: + logger.debug( + "is_gpu_node DockerError while check-run %s: %s", + spec_config, + err + ) return False diff --git a/services/web/Dockerfile b/services/web/Dockerfile index 9ee0e27b361..2a603f26b0b 100644 --- a/services/web/Dockerfile +++ b/services/web/Dockerfile @@ -9,6 +9,7 @@ FROM python:${PYTHON_VERSION}-slim-buster as base # REQUIRED: context expected at ``osparc-simcore/`` folder because we need access to osparc-simcore/packages # REQUIRED: client_qx:build image ready + LABEL maintainer=pcrespov RUN set -eux; \ @@ -116,10 +117,17 @@ RUN pip --no-cache-dir --quiet install -r requirements/prod.txt # FROM base as production +ARG BUILD_DATE +ARG VCS_URL +ARG VCS_REF + ENV SC_BUILD_TARGET=production \ SC_BOOT_MODE=production \ SC_HEALTHCHECK_INTERVAL=30 \ - SC_HEALTHCHECK_RETRY=3 + SC_HEALTHCHECK_RETRY=3 \ + SC_BUILD_DATE=${BUILD_DATE} \ + SC_VCS_URL=${VCS_URL} \ + SC_VCS_REF=${VCS_REF} ENV PYTHONOPTIMIZE=TRUE diff --git a/services/web/server/requirements/_base.in b/services/web/server/requirements/_base.in index 932c3002c2f..80b4bd10fad 100644 --- a/services/web/server/requirements/_base.in +++ b/services/web/server/requirements/_base.in @@ -28,3 +28,5 @@ python-socketio semantic_version aiodebug json2html + +pydantic diff --git a/services/web/server/requirements/_base.txt b/services/web/server/requirements/_base.txt index 4efd2190709..6973b50db52 100644 --- a/services/web/server/requirements/_base.txt +++ b/services/web/server/requirements/_base.txt @@ -27,6 +27,7 @@ cffi==1.14.2 # via cryptography change-case==0.5.2 # via -r requirements/_base.in chardet==3.0.4 # via aiohttp cryptography==3.0 # via -r requirements/_base.in, aiohttp-session +dataclasses==0.7 # via pydantic expiringdict==1.2.1 # via -r requirements/_base.in hiredis==1.1.0 # via aioredis idna-ssl==1.1.0 # via aiohttp @@ -49,6 +50,7 @@ passlib==1.7.2 # via -r requirements/_base.in prometheus-client==0.8.0 # via -r requirements/../../../../packages/service-library/requirements/_base.in psycopg2-binary==2.8.5 # via -r requirements/../../../../packages/service-library/requirements/_base.in, aiopg, sqlalchemy pycparser==2.20 # via cffi +pydantic==1.6.1 # via -r requirements/_base.in pyrsistent==0.16.0 # via jsonschema python-engineio==3.13.2 # via python-socketio python-socketio==4.6.0 # via -r requirements/_base.in diff --git a/services/web/server/requirements/_test.txt b/services/web/server/requirements/_test.txt index 2595091615e..ebe451d26c7 100644 --- a/services/web/server/requirements/_test.txt +++ b/services/web/server/requirements/_test.txt @@ -36,6 +36,7 @@ codecov==2.1.8 # via -r requirements/_test.in coverage==5.2.1 # via -r requirements/_test.in, codecov, coveralls, pytest-cov coveralls==2.1.2 # via -r requirements/_test.in cryptography==3.0 # via -r requirements/_base.txt, aiohttp-session, paramiko +dataclasses==0.7 # via -r requirements/_base.txt, pydantic distro==1.5.0 # via docker-compose docker-compose==1.26.2 # via pytest-docker docker[ssh]==4.3.0 # via -r requirements/_test.in, docker-compose @@ -74,6 +75,7 @@ psycopg2-binary==2.8.5 # via -r requirements/_base.txt, aiopg, sqlalchemy ptvsd==4.3.2 # via -r requirements/_test.in py==1.9.0 # via pytest pycparser==2.20 # via -r requirements/_base.txt, cffi +pydantic==1.6.1 # via -r requirements/_base.txt pylint==2.5.0 # via -r requirements/_test.in pynacl==1.4.0 # via paramiko pyparsing==2.4.7 # via packaging diff --git a/services/web/server/src/simcore_service_webserver/__version__.py b/services/web/server/src/simcore_service_webserver/__version__.py index 9267ec38c28..1d6e7152e70 100644 --- a/services/web/server/src/simcore_service_webserver/__version__.py +++ b/services/web/server/src/simcore_service_webserver/__version__.py @@ -8,4 +8,9 @@ version = Version(__version__) -api_version_prefix: str = f"v{version.major}" +app_name: str = __name__.split(".")[0] +api_version: str = __version__ +api_vtag: str = f"v{version.major}" + +# legacy +api_version_prefix: str = api_vtag diff --git a/services/web/server/src/simcore_service_webserver/application.py b/services/web/server/src/simcore_service_webserver/application.py index dd5cfe5f484..9ff9e707332 100644 --- a/services/web/server/src/simcore_service_webserver/application.py +++ b/services/web/server/src/simcore_service_webserver/application.py @@ -31,6 +31,7 @@ from .tags import setup_tags from .tracing import setup_app_tracing from .users import setup_users +from .settings import setup_settings log = logging.getLogger(__name__) @@ -46,6 +47,8 @@ def create_application(config: Dict) -> web.Application: app = create_safe_application(config) + setup_settings(app) + # TODO: create dependency mechanism # and compute setup order https://github.com/ITISFoundation/osparc-simcore/issues/1142 setup_app_tracing(app) diff --git a/services/web/server/src/simcore_service_webserver/computation_comp_tasks_listening_task.py b/services/web/server/src/simcore_service_webserver/computation_comp_tasks_listening_task.py index c5e8563d9e3..0caecfad27f 100644 --- a/services/web/server/src/simcore_service_webserver/computation_comp_tasks_listening_task.py +++ b/services/web/server/src/simcore_service_webserver/computation_comp_tasks_listening_task.py @@ -28,9 +28,12 @@ async def listen(app: web.Application): db_engine: Engine = app[APP_DB_ENGINE_KEY] async with db_engine.acquire() as conn: await conn.execute(listen_query) + while True: msg = await conn.connection.notifies.get() - log.debug("DB comp_tasks.outputs Update: <- %s", msg.payload) + + # Changes on comp_tasks.outputs of non-frontend task + log.debug("DB comp_tasks.outputs updated: <- %s", msg.payload) node = json.loads(msg.payload) node_data = node["data"] task_output = node_data["outputs"] @@ -53,7 +56,7 @@ async def listen(app: web.Application): continue the_project_owner = the_project["prj_owner"] - # update the project + # Update the project try: node_data = await projects_api.update_project_node_outputs( app, the_project_owner, project_id, node_id, data=task_output @@ -70,8 +73,9 @@ async def listen(app: web.Application): project_id, ) continue - # notify the client(s), the owner + any one with read writes - clients = [the_project_owner] + + # Notify the client(s), the owner + any one with read writes + clients_ids = [the_project_owner] for gid, access_rights in the_project["access_rights"].items(): if not access_rights["read"]: continue @@ -79,13 +83,17 @@ async def listen(app: web.Application): async for user in conn.execute( select([user_to_groups.c.uid]).where(user_to_groups.c.gid == gid) ): - clients.append(user["uid"]) + clients_ids.append(user["uid"]) - messages = {"nodeUpdated": {"Node": node_id, "Data": node_data}} - - await logged_gather( - *[post_messages(app, client, messages) for client in clients], False - ) + posts_tasks = [ + post_messages( + app, + uid, + messages={"nodeUpdated": {"Node": node_id, "Data": node_data}}, + ) + for uid in clients_ids + ] + await logged_gather(*posts_tasks, False) async def comp_tasks_listening_task(app: web.Application) -> None: diff --git a/services/web/server/src/simcore_service_webserver/computation_handlers.py b/services/web/server/src/simcore_service_webserver/computation_handlers.py index 9e53a39fd07..cefae76b3d9 100644 --- a/services/web/server/src/simcore_service_webserver/computation_handlers.py +++ b/services/web/server/src/simcore_service_webserver/computation_handlers.py @@ -21,15 +21,15 @@ log = logging.getLogger(__file__) -def get_celery(_app: web.Application): +def get_celery(_app: web.Application) -> Celery: config = _app[APP_CONFIG_KEY][CONFIG_RABBIT_SECTION] rabbit = RabbitConfig(**config) - celery = Celery( + celery_app = Celery( rabbit.name, broker=rabbit.broker_url, backend=rabbit.backend, ) - return celery + return celery_app async def _process_request(request): @@ -73,9 +73,11 @@ async def start_pipeline(request: web.Request) -> web.Response: user_id, project_id = await _process_request(request) + # FIXME: if start is already ongoing. Do not re-start! try: project = await get_project_for_user(request.app, project_id, user_id) await update_pipeline_db(request.app, project_id, project["workbench"]) + except ProjectNotFoundError as exc: raise web.HTTPNotFound(reason=f"Project {project_id} not found") from exc diff --git a/services/web/server/src/simcore_service_webserver/projects/projects_db.py b/services/web/server/src/simcore_service_webserver/projects/projects_db.py index f67d755bbb1..c2009144a01 100644 --- a/services/web/server/src/simcore_service_webserver/projects/projects_db.py +++ b/services/web/server/src/simcore_service_webserver/projects/projects_db.py @@ -6,6 +6,7 @@ """ import logging +import textwrap import uuid as uuidlib from collections import deque from datetime import datetime @@ -170,7 +171,7 @@ async def add_project( force_project_uuid=False, force_as_template=False, ) -> Dict: - """ Inserts a new project in the database and, if a user is specified, it assigns ownership + """Inserts a new project in the database and, if a user is specified, it assigns ownership - A valid uuid is automaticaly assigned to the project except if force_project_uuid=False. In the latter case, invalid uuid will raise an exception. @@ -193,7 +194,10 @@ async def add_project( # TODO: check best rollback design. see transaction.begin... # TODO: check if template, otherwise standard (e.g. template- prefix in uuid) prj.update( - {"creationDate": now_str(), "lastChangeDate": now_str(),} + { + "creationDate": now_str(), + "lastChangeDate": now_str(), + } ) kargs = _convert_to_db_names(prj) kargs.update( @@ -249,13 +253,15 @@ async def load_user_projects(self, user_id: int) -> List[Dict]: log.info("Loading projects for user %s", user_id) async with self.engine.acquire() as conn: user_groups: List[RowProxy] = await self.__load_user_groups(conn, user_id) - query = f""" - SELECT * - FROM projects - WHERE projects.type != 'TEMPLATE' - AND (jsonb_exists_any(projects.access_rights, array[{', '.join(f"'{group.gid}'" for group in user_groups)}]) - OR prj_owner = {user_id}) - """ + query = textwrap.dedent( + f"""\ + SELECT * + FROM projects + WHERE projects.type != 'TEMPLATE' + AND (jsonb_exists_any(projects.access_rights, array[{', '.join(f"'{group.gid}'" for group in user_groups)}]) + OR prj_owner = {user_id}) + """ + ) projects_list = await self.__load_projects( conn, query, user_id, user_groups ) @@ -272,16 +278,20 @@ async def load_template_projects( async with self.engine.acquire() as conn: user_groups: List[RowProxy] = await self.__load_user_groups(conn, user_id) + # NOTE: in order to use specific postgresql function jsonb_exists_any we use raw call here - query = f""" -SELECT * -FROM projects -WHERE projects.type = 'TEMPLATE' -{'AND projects.published ' if only_published else ''} -AND (jsonb_exists_any(projects.access_rights, array[{', '.join(f"'{group.gid}'" for group in user_groups)}]) -OR prj_owner = {user_id}) - """ + query = textwrap.dedent( + f"""\ + SELECT * + FROM projects + WHERE projects.type = 'TEMPLATE' + {'AND projects.published ' if only_published else ''} + AND (jsonb_exists_any(projects.access_rights, array[{', '.join(f"'{group.gid}'" for group in user_groups)}]) + OR prj_owner = {user_id}) + """ + ) db_projects = await self.__load_projects(conn, query, user_id, user_groups) + projects_list.extend(db_projects) return projects_list @@ -337,15 +347,17 @@ async def _get_project( user_groups: List[RowProxy] = await self.__load_user_groups(conn, user_id) # NOTE: in order to use specific postgresql function jsonb_exists_any we use raw call here - query = f""" -SELECT * -FROM projects -WHERE -{"" if include_templates else "projects.type != 'TEMPLATE' AND"} -uuid = '{project_uuid}' -AND (jsonb_exists_any(projects.access_rights, array[{', '.join(f"'{group.gid}'" for group in user_groups)}]) -OR prj_owner = {user_id}) -""" + query = textwrap.dedent( + f"""\ + SELECT * + FROM projects + WHERE + {"" if include_templates else "projects.type != 'TEMPLATE' AND"} + uuid = '{project_uuid}' + AND (jsonb_exists_any(projects.access_rights, array[{', '.join(f"'{group.gid}'" for group in user_groups)}]) + OR prj_owner = {user_id}) + """ + ) result = await conn.execute(query) project_row = await result.first() @@ -392,7 +404,7 @@ async def remove_tag(self, user_id: int, project_uuid: str, tag_id: int) -> Dict return _convert_to_schema_names(project, user_email) async def get_user_project(self, user_id: int, project_uuid: str) -> Dict: - """ Returns all projects *owned* by the user + """Returns all projects *owned* by the user - prj_owner - Notice that a user can have access to a template but he might not onw it @@ -448,16 +460,21 @@ async def get_template_project( return template_prj async def update_user_project( - self, project_data: Dict, user_id: int, project_uuid: str, include_templates: Optional[bool] = False + self, + project_data: Dict, + user_id: int, + project_uuid: str, + include_templates: Optional[bool] = False, ): - """ updates a project from a user - - """ + """updates a project from a user""" log.info("Updating project %s for user %s", project_uuid, user_id) async with self.engine.acquire() as conn: row = await self._get_project( - user_id, project_uuid, exclude_foreign=["tags"], include_templates=include_templates + user_id, + project_uuid, + exclude_foreign=["tags"], + include_templates=include_templates, ) user_groups: List[RowProxy] = await self.__load_user_groups(conn, user_id) _check_project_permissions(row, user_id, user_groups, "write") @@ -505,7 +522,7 @@ async def delete_user_project(self, user_id: int, project_uuid: str): ) async def make_unique_project_uuid(self) -> str: - """ Generates a project identifier still not used in database + """Generates a project identifier still not used in database WARNING: this method does not guarantee always unique id due to possible race condition (i.e. while client gets this uuid and uses it, another client might have used the same id already) diff --git a/services/web/server/src/simcore_service_webserver/projects/projects_handlers.py b/services/web/server/src/simcore_service_webserver/projects/projects_handlers.py index 7efe7fc8749..61b185bf607 100644 --- a/services/web/server/src/simcore_service_webserver/projects/projects_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/projects_handlers.py @@ -143,7 +143,12 @@ async def list_projects(request: web.Request): projects_api.validate_project(request.app, project) validated_projects.append(project) except ValidationError: - log.exception("Skipping invalid project from list") + log.warning( + "Invalid project with id='%s' in database." + "Skipping project from listed response." + "RECOMMENDED db data diagnose and cleanup", + project.get("uuid", "undefined"), + ) continue return {"data": validated_projects} @@ -152,9 +157,7 @@ async def list_projects(request: web.Request): @login_required @permission_required("project.read") async def get_project(request: web.Request): - """ Returns all projects accessible to a user (not necesarly owned) - - """ + """Returns all projects accessible to a user (not necesarly owned)""" # TODO: temporary hidden until get_handlers_from_namespace refactor to seek marked functions instead! user_id = request[RQT_USERID_KEY] from .projects_api import get_project_for_user @@ -167,8 +170,8 @@ async def get_project(request: web.Request): user_id=user_id, include_templates=True, ) - return {"data": project} + except ProjectInvalidRightsError as exc: raise web.HTTPForbidden( reason=f"You do not have sufficient rights to read project {project_uuid}" @@ -180,7 +183,7 @@ async def get_project(request: web.Request): @login_required @permission_required("services.pipeline.*") # due to update_pipeline_db async def replace_project(request: web.Request): - """ Implements PUT /projects + """Implements PUT /projects In a PUT request, the enclosed entity is considered to be a modified version of the resource stored on the origin server, and the client is requesting that the @@ -224,11 +227,14 @@ async def replace_project(request: web.Request): user_id=user_id, include_templates=True, ) + if current_project["accessRights"] != new_project["accessRights"]: await check_permission(request, "project.access_rights.update") + new_project = await db.update_user_project( new_project, user_id, project_uuid, include_templates=True ) + await update_pipeline_db( request.app, project_uuid, new_project["workbench"], replace_pipeline ) @@ -407,7 +413,10 @@ async def state_project(request: web.Request) -> web.Response: # check that project exists await get_project_for_user( - request.app, project_uuid=project_uuid, user_id=user_id, include_templates=True, + request.app, + project_uuid=project_uuid, + user_id=user_id, + include_templates=True, ) with managed_resource(user_id, None, request.app) as rt: users_of_project = await rt.find_users_of_resource("project_id", project_uuid) diff --git a/services/web/server/src/simcore_service_webserver/rest_handlers.py b/services/web/server/src/simcore_service_webserver/rest_handlers.py index 4ad305cb014..63a0ea4f025 100644 --- a/services/web/server/src/simcore_service_webserver/rest_handlers.py +++ b/services/web/server/src/simcore_service_webserver/rest_handlers.py @@ -11,6 +11,7 @@ from servicelib.rest_utils import body_to_dict, extract_and_validate from . import __version__ +from .settings import APP_SETTINGS_KEY log = logging.getLogger(__name__) @@ -53,14 +54,14 @@ async def check_action(request: web.Request): async def get_config(request: web.Request): """ - This entrypoint aims to provide an extra configuration mechanism for - the front-end app. + This entrypoint aims to provide an extra configuration mechanism for + the front-end app. - Some of the server configuration can be forwarded to the front-end here + Some of the server configuration can be forwarded to the front-end here - Example use case: the front-end app is served to the client. Then the user wants to - register but the server has been setup to require an invitation. This option is setup - at runtime and the front-end can only get it upon request to /config + Example use case: the front-end app is served to the client. Then the user wants to + register but the server has been setup to require an invitation. This option is setup + at runtime and the front-end can only get it upon request to /config """ await extract_and_validate(request) @@ -71,5 +72,6 @@ async def get_config(request: web.Request): data = { "invitation_required": login_cfg.get("registration_invitation_required", False) } + data.update(request.app[APP_SETTINGS_KEY].public_dict()) return data diff --git a/services/web/server/src/simcore_service_webserver/settings.py b/services/web/server/src/simcore_service_webserver/settings.py new file mode 100644 index 00000000000..f4f4dca0b16 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/settings.py @@ -0,0 +1,52 @@ +""" +TODO: pydantic settings comming soon and replacing trafaret + +""" +import logging +from typing import Dict, Optional + +from aiohttp import web +from pydantic import BaseSettings + +from .__version__ import app_name, api_version + +APP_SETTINGS_KEY = f"{__name__ }.app_settings" + +log = logging.getLogger(__name__) + + +class ApplicationSettings(BaseSettings): + # settings defined by the code + app_name: str = app_name + api_version: str = api_version + + # settings defined when docker image is built + vcs_url: Optional[str] = None + vcs_ref: Optional[str] = None + build_date: Optional[str] = None + build_target: Optional[str] = None + + boot_mode: Optional[str] = None + user_name: Optional[str] = None + user_id: Optional[int] = None + + heathcheck_retry: Optional[int] = None + heatcheck_inteval: Optional[int] = None + + class Config: + env_prefix = "SC_" + case_sensitive = False + + # --- + + def public_dict(self) -> Dict: + """ Data publicaly available """ + return self.dict( + include={"vcs_url", "vcs_ref", "build_date", "app_name", "api_version"}, + exclude_none=True, + ) + + +def setup_settings(app: web.Application): + app[APP_SETTINGS_KEY] = ApplicationSettings() + log.info("Captured app settings:\n%s", app[APP_SETTINGS_KEY].json(indent=2)) diff --git a/services/web/server/src/simcore_service_webserver/statics.py b/services/web/server/src/simcore_service_webserver/statics.py index de86dd3f659..68f177cd022 100644 --- a/services/web/server/src/simcore_service_webserver/statics.py +++ b/services/web/server/src/simcore_service_webserver/statics.py @@ -19,6 +19,8 @@ from servicelib.application_keys import APP_CONFIG_KEY from servicelib.application_setup import ModuleCategory, app_module_setup +from .settings import APP_SETTINGS_KEY + INDEX_RESOURCE_NAME = "statics.index" TMPDIR_KEY = f"{__name__}.tmpdir" @@ -46,7 +48,7 @@ async def _delete_tmps(app: web.Application): async def index(request: web.Request): """ - Serves boot application under index + Serves boot application under index """ log.debug("index.request:\n %s", request) @@ -55,14 +57,16 @@ async def index(request: web.Request): return web.Response(text=ofh.read(), content_type="text/html") -def write_statics_file(directory: Path) -> None: +def write_statics_file(app: web.Application, directory: Path) -> None: # ensures directory exists os.makedirs(directory, exist_ok=True) # create statics field statics = {} statics["stackName"] = os.environ.get("SWARM_STACK_NAME") - statics["buildDate"] = os.environ.get("BUILD_DATE") + statics["buildDate"] = app[APP_SETTINGS_KEY].build_date + statics.update(app[APP_SETTINGS_KEY].public_dict()) + with open(directory / "statics.json", "wt") as fh: json.dump(statics, fh) @@ -76,7 +80,7 @@ def setup_statics(app: web.Application): outdir: Path = get_client_outdir(app) # Create statics file - write_statics_file(outdir / "resource") + write_statics_file(app, outdir / "resource") required_dirs = ["osparc", "resource", "transpiled"] folders = [x for x in outdir.iterdir() if x.is_dir()] diff --git a/services/web/server/tests/integration/computation/test_computation.py b/services/web/server/tests/integration/computation/test_computation.py index b2d8752a6b7..a9ff0d74517 100644 --- a/services/web/server/tests/integration/computation/test_computation.py +++ b/services/web/server/tests/integration/computation/test_computation.py @@ -42,14 +42,14 @@ ops_services = [ "minio", - # 'adminer', - # 'portainer' -] +] # + ["adminer", "portainer"] @pytest.fixture def client( - loop, aiohttp_client, app_config, ## waits until swarm with *_services are up + loop, + aiohttp_client, + app_config, ## waits until swarm with *_services are up ): assert app_config["rest"]["version"] == API_VERSION diff --git a/services/web/server/tests/integration/computation/test_rabbit.py b/services/web/server/tests/integration/computation/test_rabbit.py index 6dd1db9973a..e8cb64743b2 100644 --- a/services/web/server/tests/integration/computation/test_rabbit.py +++ b/services/web/server/tests/integration/computation/test_rabbit.py @@ -11,8 +11,8 @@ import aio_pika import pytest +import sqlalchemy as sa from mock import call -from pytest_simcore.postgres_service import postgres_db from servicelib.application import create_safe_application from servicelib.application_keys import APP_CONFIG_KEY @@ -36,8 +36,6 @@ ops_services = [] -import sqlalchemy as sa - @pytest.fixture def client( diff --git a/services/web/server/tests/integration/test_project_workflow.py b/services/web/server/tests/integration/test_project_workflow.py index 4293caf93ad..2fd5c9253bf 100644 --- a/services/web/server/tests/integration/test_project_workflow.py +++ b/services/web/server/tests/integration/test_project_workflow.py @@ -41,9 +41,7 @@ "redis", ] -ops_services = [ - # 'adminer' -] +ops_services = [] # + ["adminer"] @pytest.fixture @@ -122,7 +120,7 @@ def fake_project_data(fake_data_dir: Path) -> Dict: @pytest.fixture async def logged_user(client): # , role: UserRole): - """ adds a user in db and logs in with client + """adds a user in db and logs in with client NOTE: role fixture is defined as a parametrization below """ @@ -150,9 +148,9 @@ def computational_system_mock(mocker): @pytest.fixture async def storage_subsystem_mock(loop, mocker): """ - Patches client calls to storage service + Patches client calls to storage service - Patched functions are exposed within projects but call storage subsystem + Patched functions are exposed within projects but call storage subsystem """ # requests storage to copy data mock = mocker.patch( diff --git a/services/web/server/tests/unit/isolated/test_rest.py b/services/web/server/tests/unit/isolated/test_rest.py index ef1a1c9e078..d21a45c9614 100644 --- a/services/web/server/tests/unit/isolated/test_rest.py +++ b/services/web/server/tests/unit/isolated/test_rest.py @@ -16,6 +16,7 @@ from simcore_service_webserver.resources import resources from simcore_service_webserver.rest import setup_rest from simcore_service_webserver.security import setup_security +from simcore_service_webserver.settings import setup_settings @pytest.fixture @@ -43,7 +44,9 @@ async def slow_handler(request: web.Request): "main": server_kwargs, "rest": {"enabled": True, "version": api_version_prefix}, } + # activates only security+restAPI sub-modules + setup_settings(app) setup_security(app) setup_rest(app) diff --git a/services/web/server/tests/unit/isolated/test_settings.py b/services/web/server/tests/unit/isolated/test_settings.py new file mode 100644 index 00000000000..4ff97d01e85 --- /dev/null +++ b/services/web/server/tests/unit/isolated/test_settings.py @@ -0,0 +1,28 @@ +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name +# pylint:disable=no-name-in-module + +from simcore_service_webserver.settings import setup_settings, APP_SETTINGS_KEY + + +def test_settings_constructs(monkeypatch): + monkeypatch.setenv("SC_VCS_URL", "FOO") + app = dict() + + # + setup_settings(app) + + # + assert APP_SETTINGS_KEY in app + settings = app[APP_SETTINGS_KEY] + + assert 'vcs_url' in settings.public_dict() + assert settings.public_dict()['vcs_url'] == "FOO" + + assert 'app_name' in settings.public_dict() + assert 'api_version' in settings.public_dict() + + # avoids display of sensitive info + assert not any( "pass" in key for key in settings.public_dict().keys() ) + assert not any( "token" in key for key in settings.public_dict().keys() ) + assert not any( "secret" in key for key in settings.public_dict().keys() ) diff --git a/services/web/server/tests/unit/with_dbs/fast/test_access_to_studies.py b/services/web/server/tests/unit/with_dbs/fast/test_access_to_studies.py index 79e1937a86f..6dd9d6c24fb 100644 --- a/services/web/server/tests/unit/with_dbs/fast/test_access_to_studies.py +++ b/services/web/server/tests/unit/with_dbs/fast/test_access_to_studies.py @@ -28,6 +28,7 @@ from simcore_service_webserver.rest import setup_rest from simcore_service_webserver.security import setup_security from simcore_service_webserver.session import setup_session +from simcore_service_webserver.settings import setup_settings from simcore_service_webserver.statics import setup_statics from simcore_service_webserver.studies_access import setup_studies_access from simcore_service_webserver.users import setup_users @@ -77,6 +78,7 @@ def client(loop, aiohttp_client, app_cfg, postgres_db, qx_client_outdir, monkeyp app = create_safe_application(cfg) + setup_settings(app) setup_statics(app) setup_db(app) setup_session(app) @@ -98,7 +100,7 @@ def client(loop, aiohttp_client, app_cfg, postgres_db, qx_client_outdir, monkeyp @pytest.fixture async def logged_user(client): # , role: UserRole): - """ adds a user in db and logs in with client + """adds a user in db and logs in with client NOTE: role fixture is defined as a parametrization below """ diff --git a/tests/e2e/Makefile b/tests/e2e/Makefile index 5f81b6b6551..c5940ceeff3 100644 --- a/tests/e2e/Makefile +++ b/tests/e2e/Makefile @@ -41,17 +41,17 @@ registry-down: ## bring the docker registry down wait-for-services: ## wait for simcore services to be up @python utils/wait_for_services.py + .PHONY: transfer-images-to-registry transfer-images-to-registry: ## transfer images to registry + # pushing sleeper image @docker pull itisfoundation/sleeper:1.0.0 @docker tag itisfoundation/sleeper:1.0.0 registry:5000/simcore/services/comp/itis/sleeper:1.0.0 @docker push registry:5000/simcore/services/comp/itis/sleeper:1.0.0 - - # @docker pull itisfoundation/jupyter-base-notebook:2.13.0 - # @docker tag itisfoundation/jupyter-base-notebook:2.13.0 registry:5000/simcore/services/dynamic/jupyter-base-notebook:2.13.0 - # @docker push registry:5000/simcore/services/dynamic/jupyter-base-notebook:2.13.0 # completed transfer of images curl -s registry:5000/v2/_catalog | jq '.repositories' + curl -s http://registry:5000/v2/simcore/services/comp/itis/sleeper/tags/list?n=50 | jq '.' + PUBLISHED_PORT = $(shell docker inspect $(shell docker service ls --format "{{ .Name }}" | grep postgres) --format "{{(index .Endpoint.Ports 0).PublishedPort}}") .PHONY: inject-templates-in-db @@ -64,8 +64,9 @@ inject-templates-in-db: ## inject project templates --dbname simcoredb \ --command "\copy projects from 'tutorials/sleepers_project_template_sql.csv' csv header;"; + .PHONY: test -test: ## test the platform +test: ## test the platformå # tests npm test # tests whether tutorial run diff --git a/tests/e2e/package-lock.json b/tests/e2e/package-lock.json index 756e942efd6..64c9836f196 100644 --- a/tests/e2e/package-lock.json +++ b/tests/e2e/package-lock.json @@ -3987,6 +3987,19 @@ "resolved": "https://registry.npmjs.org/lodash.sortby/-/lodash.sortby-4.7.0.tgz", "integrity": "sha1-7dFMgk4sycHgsKG0K7UhBRakJDg=" }, + "log-prefix": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/log-prefix/-/log-prefix-0.1.1.tgz", + "integrity": "sha512-aP1Lst8OCdZKATqzXDN0JBissNVZuiKLyo6hOXDBxaQ1jHDsaxh2J1i5Pp0zMy6ayTKDWfUlLMXyLaQe1PJ48g==" + }, + "log-timestamp": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/log-timestamp/-/log-timestamp-0.3.0.tgz", + "integrity": "sha512-luRz6soxijd1aJh0GkLXFjKABihxthvTfWTzu3XhCgg5EivG2bsTpSd63QFbUgS+/KmFtL+0RfSpeaD2QvOV8Q==", + "requires": { + "log-prefix": "0.1.1" + } + }, "lolex": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/lolex/-/lolex-5.1.2.tgz", diff --git a/tests/e2e/package.json b/tests/e2e/package.json index 0baba4dcc6b..e2b35d3c9b3 100644 --- a/tests/e2e/package.json +++ b/tests/e2e/package.json @@ -9,7 +9,8 @@ "dependencies": { "jest": "^25.5.4", "jest-puppeteer": "^4.4.0", - "puppeteer": "^5.2.1" + "puppeteer": "^5.2.1", + "log-timestamp": "^0.3.0" }, "devDependencies": {}, "scripts": { diff --git a/tests/e2e/utils/auto.js b/tests/e2e/utils/auto.js index 3934a9fcd84..390baf36c6f 100644 --- a/tests/e2e/utils/auto.js +++ b/tests/e2e/utils/auto.js @@ -1,5 +1,7 @@ const utils = require("./utils") const responses = require('./responsesQueue'); +require('log-timestamp'); + async function register(page, user, pass) { await utils.waitAndClick(page, '[osparc-test-id="loginCreateAccountBtn"]'); diff --git a/tests/swarm-deploy/requirements/requirements.txt b/tests/swarm-deploy/requirements/requirements.txt index 5899b6d9a9f..94f10429bf0 100644 --- a/tests/swarm-deploy/requirements/requirements.txt +++ b/tests/swarm-deploy/requirements/requirements.txt @@ -4,12 +4,12 @@ # # pip-compile --output-file=requirements/requirements.txt requirements/requirements.in # -aio-pika==6.6.1 # via -r requirements/requirements.in +aio-pika==6.7.0 # via -r requirements/requirements.in aiohttp==3.6.2 # via pytest-aiohttp aiormq==3.2.3 # via aio-pika alembic==1.4.2 # via -r requirements/requirements.in async-timeout==3.0.1 # via aiohttp -attrs==20.1.0 # via aiohttp, pytest +attrs==20.2.0 # via aiohttp, pytest certifi==2020.6.20 # via requests chardet==3.0.4 # via aiohttp, requests click==7.1.2 # via -r requirements/requirements.in @@ -21,7 +21,7 @@ importlib-metadata==1.7.0 # via pluggy, pytest iniconfig==1.0.1 # via pytest mako==1.1.3 # via alembic markupsafe==1.1.1 # via mako -more-itertools==8.4.0 # via pytest +more-itertools==8.5.0 # via pytest multidict==4.7.6 # via aiohttp, yarl packaging==20.4 # via pytest, pytest-sugar pamqp==2.3.0 # via aiormq @@ -31,7 +31,7 @@ pyparsing==2.4.7 # via packaging pytest-aiohttp==0.3.0 # via -r requirements/requirements.in pytest-cov==2.10.1 # via -r requirements/requirements.in pytest-instafail==0.4.2 # via -r requirements/requirements.in -pytest-mock==3.3.0 # via -r requirements/requirements.in +pytest-mock==3.3.1 # via -r requirements/requirements.in pytest-runner==5.2 # via -r requirements/requirements.in pytest-sugar==0.9.4 # via -r requirements/requirements.in pytest==6.0.1 # via -r requirements/requirements.in, pytest-aiohttp, pytest-cov, pytest-instafail, pytest-mock, pytest-sugar