diff --git a/.env-devel b/.env-devel index 158197d780b..76ec8a39e23 100644 --- a/.env-devel +++ b/.env-devel @@ -2,6 +2,7 @@ # - Keep it alfphabetical order and grouped by prefix # - To expose: export $(grep -v '^#' .env | xargs -0) # +API_SERVER_DEV_FEATURES_ENABLED=0 BF_API_KEY=none BF_API_SECRET=none diff --git a/.github/workflows/ci-testing-deploy.yml b/.github/workflows/ci-testing-deploy.yml index 68f05c2a1c8..72c580257ea 100644 --- a/.github/workflows/ci-testing-deploy.yml +++ b/.github/workflows/ci-testing-deploy.yml @@ -1176,6 +1176,52 @@ jobs: name: integration_simcoresdk_coverage path: codeclimate.integration_simcoresdk_coverage.json + system-test-public-api: + name: "[sys] public api" + needs: [build-test-images] + runs-on: ${{ matrix.os }} + strategy: + matrix: + python: [3.6] + os: [ubuntu-20.04] + fail-fast: false + steps: + - name: set PR default variables + # only pushes have access to the docker credentials, use a default + if: github.event_name == 'pull_request' + run: | + export TMP_DOCKER_REGISTRY=${GITHUB_REPOSITORY%/*} + echo "DOCKER_REGISTRY=${TMP_DOCKER_REGISTRY,,}" >> $GITHUB_ENV + - uses: actions/checkout@v2 + - name: setup docker + run: | + sudo ./ci/github/helpers/setup_docker_compose.bash + ./ci/github/helpers/setup_docker_experimental.bash + ./ci/github/helpers/setup_docker_buildx.bash + echo "DOCKER_BUILDX=1" >> $GITHUB_ENV + - name: setup python environment + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} + - name: show system version + run: ./ci/helpers/show_system_versions.bash + - uses: actions/cache@v2 + name: getting cached data + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-public-api-${{ hashFiles('tests/public-api/requirements/ci.txt') }} + restore-keys: | + ${{ runner.os }}-pip-public-api- + ${{ runner.os }}-pip- + ${{ runner.os }}- + - name: install + run: ./ci/github/system-testing/public-api.bash install + - name: test + run: ./ci/github/system-testing/public-api.bash test + - name: cleanup + if: always() + run: ./ci/github/system-testing/public-api.bash clean_up + system-test-swarm-deploy: name: "[sys] deploy simcore" needs: [build-test-images] @@ -1183,7 +1229,7 @@ jobs: strategy: matrix: python: [3.6] - os: [ubuntu-16.04, ubuntu-18.04, ubuntu-20.04] + os: [ubuntu-20.04] fail-fast: false steps: - name: set PR default variables @@ -1477,6 +1523,7 @@ jobs: integration-test-director-v2, integration-test-sidecar, integration-test-simcore-sdk, + system-test-public-api, system-test-swarm-deploy, ] runs-on: ubuntu-latest diff --git a/Makefile b/Makefile index 9ec69896d40..aa042b5daf1 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,6 @@ SHELL := /bin/bash - MAKE_C := $(MAKE) --no-print-directory --directory # Operating system @@ -71,8 +70,10 @@ export ETC_HOSTNAME host := $(shell echo $$(hostname) > $(ETC_HOSTNAME)) endif +get_my_ip := $(shell hostname --all-ip-addresses | cut --delimiter=" " --fields=1) + # NOTE: this is only for WSL2 as the WSL2 subsystem IP is changing on each reboot -S3_ENDPOINT = $(shell hostname --all-ip-addresses | cut --delimiter=" " --fields=1):9001 +S3_ENDPOINT := $(get_my_ip):9001 export S3_ENDPOINT @@ -433,7 +434,6 @@ postgres-upgrade: ## initalize or upgrade postgres db to latest state local_registry=registry -get_my_ip := $(shell hostname --all-ip-addresses | cut --delimiter=" " --fields=1) .PHONY: local-registry rm-registry rm-registry: ## remove the registry and changes to host/file diff --git a/ci/github/integration-testing/director-v2.bash b/ci/github/integration-testing/director-v2.bash index 29f81deb3eb..f7ae93967c5 100755 --- a/ci/github/integration-testing/director-v2.bash +++ b/ci/github/integration-testing/director-v2.bash @@ -1,6 +1,8 @@ #!/bin/bash # http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' # in case it's a Pull request, the env are never available, default to itisfoundation to get a maybe not too old version for caching diff --git a/ci/github/integration-testing/sidecar.bash b/ci/github/integration-testing/sidecar.bash index abc9de7cd4a..96e7f271f6b 100755 --- a/ci/github/integration-testing/sidecar.bash +++ b/ci/github/integration-testing/sidecar.bash @@ -1,6 +1,8 @@ #!/bin/bash # http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' # in case it's a Pull request, the env are never available, default to itisfoundation to get a maybe not too old version for caching diff --git a/ci/github/integration-testing/simcore-sdk.bash b/ci/github/integration-testing/simcore-sdk.bash index 6bf19b9a590..dca86b88e88 100755 --- a/ci/github/integration-testing/simcore-sdk.bash +++ b/ci/github/integration-testing/simcore-sdk.bash @@ -1,5 +1,7 @@ #!/bin/bash -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' # in case it's a Pull request, the env are never available, default to itisfoundation to get a maybe not too old version for caching diff --git a/ci/github/integration-testing/webserver.bash b/ci/github/integration-testing/webserver.bash index 452876b4686..0ec68e0fdbb 100755 --- a/ci/github/integration-testing/webserver.bash +++ b/ci/github/integration-testing/webserver.bash @@ -1,6 +1,8 @@ #!/bin/bash # http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' # in case it's a Pull request, the env are never available, default to itisfoundation to get a maybe not too old version for caching diff --git a/ci/github/system-testing/e2e.bash b/ci/github/system-testing/e2e.bash index 71dd5348a77..ac39ca6651f 100755 --- a/ci/github/system-testing/e2e.bash +++ b/ci/github/system-testing/e2e.bash @@ -1,7 +1,9 @@ #!/bin/bash # http://redsymbol.net/articles/unofficial-bash-strict-mode/ # https://github.com/GoogleChrome/puppeteer/blob/master/docs/troubleshooting.md#running-puppeteer-on-travis-ci -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' # in case it's a Pull request, the env are never available, default to itisfoundation to get a maybe not too old version for caching diff --git a/ci/github/system-testing/environment-setup.bash b/ci/github/system-testing/environment-setup.bash index f15af9fbb74..3cffc8a0727 100755 --- a/ci/github/system-testing/environment-setup.bash +++ b/ci/github/system-testing/environment-setup.bash @@ -2,7 +2,9 @@ # # http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' install() { diff --git a/ci/github/system-testing/public-api.bash b/ci/github/system-testing/public-api.bash new file mode 100755 index 00000000000..f120ccf3bf5 --- /dev/null +++ b/ci/github/system-testing/public-api.bash @@ -0,0 +1,47 @@ +#!/bin/bash +# +# This task in the system-testing aims to test some guarantees expected from +# the deployment of osparc-simcore in a cluster (swarm). +# It follows some of the points enumerated in the https://12factor.net/ methodology. +# + +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes +IFS=$'\n\t' + +# in case it's a Pull request, the env are never available, default to itisfoundation to get a maybe not too old version for caching +DOCKER_IMAGE_TAG=$(exec ci/helpers/build_docker_image_tag.bash) +export DOCKER_IMAGE_TAG + +install() { + bash ci/helpers/ensure_python_pip.bash + pushd tests/public-api + pip3 install -r requirements/ci.txt + popd + make pull-version || ( (make pull-cache || true) && make build-x tag-version) + make .env + pip list -v + make info-images +} + +test() { + pytest --color=yes --cov-report=term-missing -v tests/public-api --log-level=DEBUG +} + +clean_up() { + docker images + make down + make leave +} + +# Check if the function exists (bash specific) +if declare -f "$1" >/dev/null; then + # call arguments verbatim + "$@" +else + # Show a helpful error + echo "'$1' is not a known function name" >&2 + exit 1 +fi diff --git a/ci/github/system-testing/swarm-deploy.bash b/ci/github/system-testing/swarm-deploy.bash index 2b3d1599bc6..5dba9d4c75d 100755 --- a/ci/github/system-testing/swarm-deploy.bash +++ b/ci/github/system-testing/swarm-deploy.bash @@ -6,7 +6,9 @@ # # http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' # in case it's a Pull request, the env are never available, default to itisfoundation to get a maybe not too old version for caching diff --git a/ci/github/unit-testing/api-server.bash b/ci/github/unit-testing/api-server.bash index a39ce08c33b..4e8198e8069 100755 --- a/ci/github/unit-testing/api-server.bash +++ b/ci/github/unit-testing/api-server.bash @@ -1,6 +1,8 @@ #!/bin/bash # http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' install() { diff --git a/ci/github/unit-testing/api.bash b/ci/github/unit-testing/api.bash index e795eac06c3..01ffcc0d272 100755 --- a/ci/github/unit-testing/api.bash +++ b/ci/github/unit-testing/api.bash @@ -1,6 +1,8 @@ #!/bin/bash # http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' install() { diff --git a/ci/github/unit-testing/catalog.bash b/ci/github/unit-testing/catalog.bash index 489307df808..1fd32d8044a 100755 --- a/ci/github/unit-testing/catalog.bash +++ b/ci/github/unit-testing/catalog.bash @@ -1,6 +1,8 @@ #!/bin/bash # http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' install() { diff --git a/ci/github/unit-testing/director.bash b/ci/github/unit-testing/director.bash index 460161bbfbb..195afe37416 100755 --- a/ci/github/unit-testing/director.bash +++ b/ci/github/unit-testing/director.bash @@ -1,6 +1,8 @@ #!/bin/bash # http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' install() { diff --git a/ci/github/unit-testing/director_v2.bash b/ci/github/unit-testing/director_v2.bash index 78f7b6385a4..0988d9d80a1 100755 --- a/ci/github/unit-testing/director_v2.bash +++ b/ci/github/unit-testing/director_v2.bash @@ -1,6 +1,8 @@ #!/bin/bash # http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' install() { diff --git a/ci/github/unit-testing/frontend.bash b/ci/github/unit-testing/frontend.bash index 76d03f30bd3..6183d09fb86 100755 --- a/ci/github/unit-testing/frontend.bash +++ b/ci/github/unit-testing/frontend.bash @@ -1,6 +1,8 @@ #!/bin/bash # http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' install() { @@ -39,7 +41,7 @@ test() { popd #TODO: no idea what is this doing... disabled at the moment since travis is supposed to do it as well - + # # prepare documentation site ... # git clone --depth 1 https://github.com/ITISFoundation/itisfoundation.github.io.git # rm -rf itisfoundation.github.io/.git diff --git a/ci/github/unit-testing/models-library.bash b/ci/github/unit-testing/models-library.bash index 268d1f224b7..c50666a8be2 100755 --- a/ci/github/unit-testing/models-library.bash +++ b/ci/github/unit-testing/models-library.bash @@ -1,6 +1,8 @@ #!/bin/bash # http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' install() { diff --git a/ci/github/unit-testing/postgres-database.bash b/ci/github/unit-testing/postgres-database.bash index 0131a28bda3..f0322cb723b 100755 --- a/ci/github/unit-testing/postgres-database.bash +++ b/ci/github/unit-testing/postgres-database.bash @@ -1,6 +1,8 @@ #!/bin/bash # http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' install() { diff --git a/ci/github/unit-testing/python-linting.bash b/ci/github/unit-testing/python-linting.bash index b4e13153ce7..483470cc59a 100755 --- a/ci/github/unit-testing/python-linting.bash +++ b/ci/github/unit-testing/python-linting.bash @@ -1,6 +1,8 @@ #!/bin/bash # http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' install() { diff --git a/ci/github/unit-testing/service-integration.bash b/ci/github/unit-testing/service-integration.bash index 439b166e589..a455b5263ac 100755 --- a/ci/github/unit-testing/service-integration.bash +++ b/ci/github/unit-testing/service-integration.bash @@ -1,6 +1,8 @@ #!/bin/bash # http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' install() { diff --git a/ci/github/unit-testing/service-library.bash b/ci/github/unit-testing/service-library.bash index d88e4d5762d..f6d6b7c1cd7 100755 --- a/ci/github/unit-testing/service-library.bash +++ b/ci/github/unit-testing/service-library.bash @@ -1,6 +1,8 @@ #!/bin/bash # http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' install() { diff --git a/ci/github/unit-testing/sidecar.bash b/ci/github/unit-testing/sidecar.bash index fb3f3eb332f..82d1b3a518b 100755 --- a/ci/github/unit-testing/sidecar.bash +++ b/ci/github/unit-testing/sidecar.bash @@ -1,6 +1,8 @@ #!/bin/bash # http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' install() { diff --git a/ci/github/unit-testing/simcore-sdk.bash b/ci/github/unit-testing/simcore-sdk.bash index 7e6105d82d4..617eb2c086c 100755 --- a/ci/github/unit-testing/simcore-sdk.bash +++ b/ci/github/unit-testing/simcore-sdk.bash @@ -1,5 +1,7 @@ #!/bin/bash -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' # in case it's a Pull request, the env are never available, default to itisfoundation to get a maybe not too old version for caching diff --git a/ci/github/unit-testing/storage.bash b/ci/github/unit-testing/storage.bash index 5499d313d8f..edea9493ff2 100755 --- a/ci/github/unit-testing/storage.bash +++ b/ci/github/unit-testing/storage.bash @@ -1,6 +1,8 @@ #!/bin/bash # http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' install() { diff --git a/ci/github/unit-testing/webserver.bash b/ci/github/unit-testing/webserver.bash index f859a278ba0..8b3745c4290 100755 --- a/ci/github/unit-testing/webserver.bash +++ b/ci/github/unit-testing/webserver.bash @@ -1,6 +1,8 @@ #!/bin/bash # http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' install() { diff --git a/ci/helpers/build_docker_image_tag.bash b/ci/helpers/build_docker_image_tag.bash index 971bcc2eb4f..42bc564f07b 100755 --- a/ci/helpers/build_docker_image_tag.bash +++ b/ci/helpers/build_docker_image_tag.bash @@ -8,7 +8,9 @@ # always adds -testbuild-latest to the image tag to differentiate from the real master/staging builds # http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' default_image_tag="github" diff --git a/ci/helpers/dockerhub_login.bash b/ci/helpers/dockerhub_login.bash index 29972d71d08..f01796f6665 100755 --- a/ci/helpers/dockerhub_login.bash +++ b/ci/helpers/dockerhub_login.bash @@ -1,6 +1,8 @@ #!/bin/bash # http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' # check needed variables are defined diff --git a/ci/helpers/ensure_python_pip.bash b/ci/helpers/ensure_python_pip.bash index daccaca603b..c34d82bf420 100755 --- a/ci/helpers/ensure_python_pip.bash +++ b/ci/helpers/ensure_python_pip.bash @@ -5,7 +5,9 @@ # SEE https://docs.python.org/3/library/ensurepip.html # # http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' # Pin pip version to a compatible release https://www.python.org/dev/peps/pep-0440/#compatible-release diff --git a/ci/helpers/install_pylint.bash b/ci/helpers/install_pylint.bash index 72eec0eb0f4..db453755b28 100755 --- a/ci/helpers/install_pylint.bash +++ b/ci/helpers/install_pylint.bash @@ -4,7 +4,9 @@ # # http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' CURDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" diff --git a/ci/helpers/show_system_versions.bash b/ci/helpers/show_system_versions.bash index c2a3b863b24..0c759de00e1 100755 --- a/ci/helpers/show_system_versions.bash +++ b/ci/helpers/show_system_versions.bash @@ -1,6 +1,8 @@ #!/bin/bash # http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' echo "------------------------------ environs -----------------------------------" diff --git a/ci/helpers/slugify_name.bash b/ci/helpers/slugify_name.bash index dc7dcffb82a..4a16f05dc00 100755 --- a/ci/helpers/slugify_name.bash +++ b/ci/helpers/slugify_name.bash @@ -1,6 +1,8 @@ #!/bin/bash # http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' slugify () { diff --git a/ci/travis/helpers/install-docker-compose.bash b/ci/travis/helpers/install-docker-compose.bash index 0a31d48ac45..3d941ee706c 100755 --- a/ci/travis/helpers/install-docker-compose.bash +++ b/ci/travis/helpers/install-docker-compose.bash @@ -1,6 +1,8 @@ #!/bin/bash # http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' sudo rm /usr/local/bin/docker-compose diff --git a/ci/travis/helpers/test-for-changes.bash b/ci/travis/helpers/test-for-changes.bash index d63f741514d..da164ee910a 100755 --- a/ci/travis/helpers/test-for-changes.bash +++ b/ci/travis/helpers/test-for-changes.bash @@ -1,6 +1,8 @@ #!/bin/bash # http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' # Usage: diff --git a/ci/travis/helpers/update-docker.bash b/ci/travis/helpers/update-docker.bash index 756f334a9b1..d5fde1c1530 100755 --- a/ci/travis/helpers/update-docker.bash +++ b/ci/travis/helpers/update-docker.bash @@ -1,6 +1,8 @@ #!/bin/bash # http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - diff --git a/ci/travis/integration-testing/director-v2.bash b/ci/travis/integration-testing/director-v2.bash index ef44336bfa6..98ea149feaf 100755 --- a/ci/travis/integration-testing/director-v2.bash +++ b/ci/travis/integration-testing/director-v2.bash @@ -1,6 +1,8 @@ #!/bin/bash # http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' # in case it's a Pull request, the env are never available, default to itisfoundation to get a maybe not too old version for caching diff --git a/ci/travis/integration-testing/sidecar.bash b/ci/travis/integration-testing/sidecar.bash index 523eaedc4d3..a067b456952 100755 --- a/ci/travis/integration-testing/sidecar.bash +++ b/ci/travis/integration-testing/sidecar.bash @@ -1,6 +1,8 @@ #!/bin/bash # http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' # in case it's a Pull request, the env are never available, default to itisfoundation to get a maybe not too old version for caching diff --git a/ci/travis/integration-testing/simcore-sdk.bash b/ci/travis/integration-testing/simcore-sdk.bash index ce21b26f275..62d2e134cd7 100755 --- a/ci/travis/integration-testing/simcore-sdk.bash +++ b/ci/travis/integration-testing/simcore-sdk.bash @@ -1,5 +1,7 @@ #!/bin/bash -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' # in case it's a Pull request, the env are never available, default to itisfoundation to get a maybe not too old version for caching diff --git a/ci/travis/integration-testing/webserver.bash b/ci/travis/integration-testing/webserver.bash index a69921d021e..0c237ed2217 100755 --- a/ci/travis/integration-testing/webserver.bash +++ b/ci/travis/integration-testing/webserver.bash @@ -1,6 +1,8 @@ #!/bin/bash # http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' # in case it's a Pull request, the env are never available, default to itisfoundation to get a maybe not too old version for caching diff --git a/ci/travis/system-testing/swarm-deploy.bash b/ci/travis/system-testing/swarm-deploy.bash index 737182d6222..1c00c1a9e7e 100755 --- a/ci/travis/system-testing/swarm-deploy.bash +++ b/ci/travis/system-testing/swarm-deploy.bash @@ -6,7 +6,9 @@ # # http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' # in case it's a Pull request, the env are never available, default to itisfoundation to get a maybe not too old version for caching diff --git a/ci/travis/unit-testing/api-server.bash b/ci/travis/unit-testing/api-server.bash index 7be8367b474..0b253f7efc9 100755 --- a/ci/travis/unit-testing/api-server.bash +++ b/ci/travis/unit-testing/api-server.bash @@ -1,6 +1,8 @@ #!/bin/bash # http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' FOLDER_CHECKS=(api/ api-server packages/ .travis.yml) diff --git a/ci/travis/unit-testing/api.bash b/ci/travis/unit-testing/api.bash index 46ea737fa73..b4785343d6c 100755 --- a/ci/travis/unit-testing/api.bash +++ b/ci/travis/unit-testing/api.bash @@ -1,6 +1,8 @@ #!/bin/bash # http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' FOLDER_CHECKS=(api/ .travis.yml) diff --git a/ci/travis/unit-testing/catalog.bash b/ci/travis/unit-testing/catalog.bash index 0f8cf8c9ca0..96fc6740a76 100755 --- a/ci/travis/unit-testing/catalog.bash +++ b/ci/travis/unit-testing/catalog.bash @@ -1,6 +1,8 @@ #!/bin/bash # http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' FOLDER_CHECKS=(api/ catalog packages/ .travis.yml) diff --git a/ci/travis/unit-testing/director-v2.bash b/ci/travis/unit-testing/director-v2.bash index 8eaea735a6e..2b7612e94a6 100755 --- a/ci/travis/unit-testing/director-v2.bash +++ b/ci/travis/unit-testing/director-v2.bash @@ -1,6 +1,8 @@ #!/bin/bash # http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' FOLDER_CHECKS=(api/ director-v2 packages/ .travis.yml) diff --git a/ci/travis/unit-testing/director.bash b/ci/travis/unit-testing/director.bash index bd50915f0d0..36cda373147 100755 --- a/ci/travis/unit-testing/director.bash +++ b/ci/travis/unit-testing/director.bash @@ -1,6 +1,8 @@ #!/bin/bash # http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' FOLDER_CHECKS=(api/ director packages/ .travis.yml) diff --git a/ci/travis/unit-testing/frontend.bash b/ci/travis/unit-testing/frontend.bash index 572089b3551..6e2fc2644b3 100755 --- a/ci/travis/unit-testing/frontend.bash +++ b/ci/travis/unit-testing/frontend.bash @@ -1,6 +1,8 @@ #!/bin/bash # http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' FOLDER_CHECKS=(js eslintrc json .travis.yml) diff --git a/ci/travis/unit-testing/models-library.bash b/ci/travis/unit-testing/models-library.bash index 5221d674ea0..7fa1d88da60 100755 --- a/ci/travis/unit-testing/models-library.bash +++ b/ci/travis/unit-testing/models-library.bash @@ -1,6 +1,8 @@ #!/bin/bash # http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' FOLDER_CHECKS=(packages/ models-library .travis.yml) diff --git a/ci/travis/unit-testing/python-linting.bash b/ci/travis/unit-testing/python-linting.bash index 61939fd22ac..5762dceab5d 100755 --- a/ci/travis/unit-testing/python-linting.bash +++ b/ci/travis/unit-testing/python-linting.bash @@ -1,6 +1,8 @@ #!/bin/bash # http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' FOLDER_CHECKS=(.py .pylintrc .travis.yml) diff --git a/ci/travis/unit-testing/service-library.bash b/ci/travis/unit-testing/service-library.bash index 1a0934ee808..665bbac7648 100755 --- a/ci/travis/unit-testing/service-library.bash +++ b/ci/travis/unit-testing/service-library.bash @@ -1,6 +1,8 @@ #!/bin/bash # http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' FOLDER_CHECKS=(packages/ service-library .travis.yml) diff --git a/ci/travis/unit-testing/sidecar.bash b/ci/travis/unit-testing/sidecar.bash index 41d089b6e26..7ba42fb1877 100755 --- a/ci/travis/unit-testing/sidecar.bash +++ b/ci/travis/unit-testing/sidecar.bash @@ -1,6 +1,8 @@ #!/bin/bash # http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' FOLDER_CHECKS=(api/ sidecar packages/ .travis.yml) diff --git a/ci/travis/unit-testing/simcore-sdk.bash b/ci/travis/unit-testing/simcore-sdk.bash index a4dea46a734..6d5e893ceaf 100755 --- a/ci/travis/unit-testing/simcore-sdk.bash +++ b/ci/travis/unit-testing/simcore-sdk.bash @@ -1,5 +1,7 @@ #!/bin/bash -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' # in case it's a Pull request, the env are never available, default to itisfoundation to get a maybe not too old version for caching diff --git a/ci/travis/unit-testing/storage.bash b/ci/travis/unit-testing/storage.bash index 45a1e2973cb..b24eed8f877 100755 --- a/ci/travis/unit-testing/storage.bash +++ b/ci/travis/unit-testing/storage.bash @@ -1,6 +1,8 @@ #!/bin/bash # http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' FOLDER_CHECKS=(api/ storage packages/ .travis.yml) diff --git a/ci/travis/unit-testing/webserver.bash b/ci/travis/unit-testing/webserver.bash index beff3cab7df..2803e5b5e2b 100755 --- a/ci/travis/unit-testing/webserver.bash +++ b/ci/travis/unit-testing/webserver.bash @@ -1,6 +1,8 @@ #!/bin/bash # http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' FOLDER_CHECKS=(api/ webserver packages/ services/web .travis.yml) diff --git a/packages/models-library/src/models_library/projects.py b/packages/models-library/src/models_library/projects.py index 3c6c6f7bf7e..30233ba5b39 100644 --- a/packages/models-library/src/models_library/projects.py +++ b/packages/models-library/src/models_library/projects.py @@ -104,7 +104,7 @@ class Config: title = "osparc-simcore project" extra = Extra.forbid - # pylint: disable=no-self-argument + @staticmethod def schema_extra(schema: Dict, _model: "Project"): # pylint: disable=unsubscriptable-object diff --git a/packages/models-library/src/models_library/projects_nodes.py b/packages/models-library/src/models_library/projects_nodes.py index 2df66414e9f..8c8512652e6 100644 --- a/packages/models-library/src/models_library/projects_nodes.py +++ b/packages/models-library/src/models_library/projects_nodes.py @@ -2,6 +2,7 @@ Models Node as a central element in a project's pipeline """ +from copy import deepcopy from typing import Dict, List, Optional, Union from pydantic import ( @@ -87,7 +88,7 @@ class Node(BaseModel): run_hash: Optional[str] = Field( None, description="the hex digest of the resolved inputs +outputs hash at the time when the last outputs were generated", - examples=["a4337bc45a8fc544c03f52dc550cd6e1e87021bc896588bd79e901e2"], + example=["a4337bc45a8fc544c03f52dc550cd6e1e87021bc896588bd79e901e2"], alias="runHash", ) @@ -149,11 +150,13 @@ def convert_old_enum_name(cls, v): class Config: extra = Extra.forbid - # NOTE: exporting without this trick does not make runHash as nullabel. + # NOTE: exporting without this trick does not make runHash as nullable. # It is a Pydantic issue see https://github.com/samuelcolvin/pydantic/issues/1270 @staticmethod - def schema_extra(schema, _): - for prop, value in schema.get("properties", {}).items(): - if prop in ["runHash"]: # Your actual nullable fields go in this list. - was = value["type"] - value["type"] = [was, "null"] + def schema_extra(schema, _model: "Node"): + # NOTE: the variant with anyOf[{type: null}, { other }] is compatible with OpenAPI + # The other as type = [null, other] is only jsonschema compatible + for prop_name in ["runHash"]: + if prop_name in schema.get("properties", {}): + was = deepcopy(schema["properties"][prop_name]) + schema["properties"][prop_name] = {"anyOf": [{"type": "null"}, was]} diff --git a/packages/models-library/src/models_library/services.py b/packages/models-library/src/models_library/services.py index 146cf0d8d66..bb5bb404445 100644 --- a/packages/models-library/src/models_library/services.py +++ b/packages/models-library/src/models_library/services.py @@ -272,7 +272,6 @@ class ServiceDockerData(ServiceKeyVersion, ServiceCommonData): class Config: description = "Description of a simcore node 'class' with input and output" - title = "simcore node" extra = Extra.forbid diff --git a/packages/pytest-simcore/src/pytest_simcore/docker_compose.py b/packages/pytest-simcore/src/pytest_simcore/docker_compose.py index 7a47b26c11e..ffa55cd8b8f 100644 --- a/packages/pytest-simcore/src/pytest_simcore/docker_compose.py +++ b/packages/pytest-simcore/src/pytest_simcore/docker_compose.py @@ -2,11 +2,10 @@ Basically runs `docker-compose config """ -import logging - # pylint:disable=unused-variable # pylint:disable=unused-argument # pylint:disable=redefined-outer-name + import os import shutil import socket @@ -14,7 +13,7 @@ from copy import deepcopy from pathlib import Path from pprint import pformat -from typing import Dict, List +from typing import Dict, Iterator, List import pytest import yaml @@ -22,12 +21,11 @@ from .helpers.utils_docker import run_docker_compose_config, save_docker_infos -log = logging.getLogger(__name__) - @pytest.fixture(scope="session") def devel_environ(env_devel_file: Path) -> Dict[str, str]: - """Loads and extends .env-devel returning + """ + Loads and extends .env-devel returning all environment variables key=value """ @@ -37,6 +35,7 @@ def devel_environ(env_devel_file: Path) -> Dict[str, str]: key: os.environ.get(key, value) for key, value in env_devel_unresolved.items() } + # These are overrides to .env-devel or an extension to them env_devel["LOG_LEVEL"] = "DEBUG" env_devel["REGISTRY_SSL"] = "False" env_devel["REGISTRY_URL"] = "{}:5000".format(_get_ip()) @@ -46,12 +45,15 @@ def devel_environ(env_devel_file: Path) -> Dict[str, str]: env_devel["REGISTRY_AUTH"] = "False" env_devel["SWARM_STACK_NAME"] = "simcore" env_devel["DIRECTOR_REGISTRY_CACHING"] = "False" + env_devel["API_SERVER_DEV_FEATURES_ENABLED"] = "1" return env_devel @pytest.fixture(scope="module") -def env_file(osparc_simcore_root_dir: Path, devel_environ: Dict[str, str]) -> Path: +def env_file( + osparc_simcore_root_dir: Path, devel_environ: Dict[str, str] +) -> Iterator[Path]: """ Creates a .env file from the .env-devel """ @@ -76,6 +78,7 @@ def env_file(osparc_simcore_root_dir: Path, devel_environ: Dict[str, str]) -> Pa @pytest.fixture(scope="module") def make_up_prod_environ(): + # TODO: use monkeypatch for modules as in https://github.com/pytest-dev/pytest/issues/363#issuecomment-289830794 old_env = deepcopy(os.environ) if not "DOCKER_REGISTRY" in os.environ: os.environ["DOCKER_REGISTRY"] = "local" @@ -116,7 +119,7 @@ def simcore_docker_compose( workdir=env_file.parent, destination_path=temp_folder / "simcore_docker_compose.yml", ) - log.debug("simcore docker-compose:\n%s", pformat(config)) + print("simcore docker-compose:\n%s", pformat(config)) return config @@ -143,12 +146,13 @@ def ops_docker_compose( workdir=env_file.parent, destination_path=temp_folder / "ops_docker_compose.yml", ) - log.debug("ops docker-compose:\n%s", pformat(config)) + print("ops docker-compose:\n%s", pformat(config)) return config @pytest.fixture(scope="module") def core_services(request) -> List[str]: + """ Selection of services from the simcore stack """ core_services = getattr(request.module, "core_services", []) assert ( core_services @@ -185,7 +189,6 @@ def ops_docker_compose_file( """Creates a docker-compose config file for every stack of services in 'ops_services' module variable File is created in a temp folder """ - docker_compose_path = Path(temp_folder / "ops_docker_compose.filtered.yml") _filter_services_and_dump(ops_services, ops_docker_compose, docker_compose_path) diff --git a/packages/pytest-simcore/src/pytest_simcore/docker_registry.py b/packages/pytest-simcore/src/pytest_simcore/docker_registry.py index 0f839cb180a..55c995e1231 100644 --- a/packages/pytest-simcore/src/pytest_simcore/docker_registry.py +++ b/packages/pytest-simcore/src/pytest_simcore/docker_registry.py @@ -6,7 +6,7 @@ import os import time from copy import deepcopy -from typing import Dict +from typing import Any, Dict import docker import jsonschema @@ -98,9 +98,11 @@ def wait_till_registry_is_responsive(url: str) -> bool: # ********************************************************* Services *************************************** + + def _pull_push_service( pull_key: str, tag: str, new_registry: str, node_meta_schema: Dict -) -> Dict[str, str]: +) -> Dict[str, Any]: client = docker.from_env() # pull image from original location image = client.images.pull(pull_key, tag=tag) @@ -114,6 +116,7 @@ def _pull_push_service( if key.startswith("io.simcore.") } assert io_simcore_labels + # validate image jsonschema.validate(io_simcore_labels, node_meta_schema) diff --git a/packages/pytest-simcore/src/pytest_simcore/docker_swarm.py b/packages/pytest-simcore/src/pytest_simcore/docker_swarm.py index 06f43e3dea3..01bdd37c4d1 100644 --- a/packages/pytest-simcore/src/pytest_simcore/docker_swarm.py +++ b/packages/pytest-simcore/src/pytest_simcore/docker_swarm.py @@ -33,11 +33,12 @@ def keep_docker_up(request) -> bool: @pytest.fixture(scope="module") def docker_swarm( docker_client: docker.client.DockerClient, keep_docker_up: Iterator[bool] -) -> None: +) -> Iterator[None]: try: docker_client.swarm.reload() print("CAUTION: Already part of a swarm") yield + except docker.errors.APIError: docker_client.swarm.init(advertise_addr=get_ip()) yield diff --git a/packages/pytest-simcore/src/pytest_simcore/simcore_services.py b/packages/pytest-simcore/src/pytest_simcore/simcore_services.py index 1844083de75..0aff93281f0 100644 --- a/packages/pytest-simcore/src/pytest_simcore/simcore_services.py +++ b/packages/pytest-simcore/src/pytest_simcore/simcore_services.py @@ -13,8 +13,9 @@ from .helpers.utils_docker import get_service_published_port -SERVICES_TO_SKIP = ["sidecar", "postgres", "redis", "rabbit"] log = logging.getLogger(__name__) + +SERVICES_TO_SKIP = ["sidecar", "postgres", "redis", "rabbit"] SERVICE_HEALTHCHECK_ENTRYPOINT = {"director-v2": "/"} @@ -34,9 +35,9 @@ def services_endpoint( @pytest.fixture(scope="function") -async def simcore_services( - services_endpoint: Dict[str, URL], monkeypatch -) -> Dict[str, URL]: +async def simcore_services(services_endpoint: Dict[str, URL], monkeypatch) -> None: + + # waits for all services to be responsive wait_tasks = [ wait_till_service_responsive( f"{endpoint}{SERVICE_HEALTHCHECK_ENTRYPOINT.get(service, '/v0/')}" @@ -44,13 +45,12 @@ async def simcore_services( for service, endpoint in services_endpoint.items() ] await asyncio.gather(*wait_tasks, return_exceptions=False) - for service, endpoint in services_endpoint.items(): - monkeypatch.setenv(f"{service.upper().replace('-', '_')}_HOST", endpoint.host) - monkeypatch.setenv( - f"{service.upper().replace('-', '_')}_PORT", str(endpoint.port) - ) - yield + # patches environment variables with host/port per service + for service, endpoint in services_endpoint.items(): + env_prefix = service.upper().replace("-", "_") + monkeypatch.setenv(f"{env_prefix}_HOST", endpoint.host) + monkeypatch.setenv(f"{env_prefix}_PORT", str(endpoint.port)) # HELPERS -- diff --git a/scripts/docker-compose-viz.bash b/scripts/docker-compose-viz.bash index 576451942f1..104dd746537 100755 --- a/scripts/docker-compose-viz.bash +++ b/scripts/docker-compose-viz.bash @@ -5,7 +5,9 @@ # See https://github.com/pmsipilot/docker-compose-viz # -set -euo pipefail +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' USERID=$(stat --format=%u "$PWD") diff --git a/services/api-server/.env-devel b/services/api-server/.env-devel index dc535bb0149..ee3a7e56405 100644 --- a/services/api-server/.env-devel +++ b/services/api-server/.env-devel @@ -1,7 +1,7 @@ # # Environment variables used to configure this service # -FAKE_API_SERVER_ENABLED=0 +API_SERVER_DEV_FEATURES_ENABLED=1 DEBUG=0 # SEE services/api-server/src/simcore_service_api_server/auth_security.py diff --git a/services/api-server/.env-fake-standalone b/services/api-server/.env-fake-standalone deleted file mode 100644 index f0e59ea1788..00000000000 --- a/services/api-server/.env-fake-standalone +++ /dev/null @@ -1,39 +0,0 @@ -# -# Environment variables used to configure this service stand-alone in FAKE mode -# -FAKE_API_SERVER_ENABLED=1 - -# SEE services/api-server/src/simcore_service_api_server/auth_security.py -SECRET_KEY=d0d0397de2c85ad26ffd4a0f9643dfe3a0ca3937f99cf3c2e174e11b5ef79880 - -# SEE services/api-server/src/simcore_service_api_server/settings.py -LOG_LEVEL=DEBUG - -POSTGRES_ENABLED=0 -POSTGRES_USER=test -POSTGRES_PASSWORD=test -POSTGRES_DB=test -POSTGRES_HOST=localhost - -# Enables debug -SC_BOOT_MODE=production - - -# webserver -WEBSERVER_ENABLED=0 -WEBSERVER_HOST=webserver -# Take from general .env-devel -WEBSERVER_SESSION_SECRET_KEY=REPLACE ME with a key of at least length 32. - - -# catalog -CATALOG_ENABLED=0 -CATALOG_HOST=catalog - -# storage -STORAGE_ENABLED=0 -STORAGE_HOST=storage - -# director -DIRECTOR2_ENABLED=0 -DIRECTO2_HOST=director-v2 diff --git a/services/api-server/openapi.json b/services/api-server/openapi.json index 56b27f37ece..ad18add8c79 100644 --- a/services/api-server/openapi.json +++ b/services/api-server/openapi.json @@ -1,8 +1,8 @@ { "openapi": "3.0.2", "info": { - "title": "Public API Server", - "description": "**osparc-simcore Public RESTful API Specifications**\n## Python Library\n- Check the [documentation](https://itisfoundation.github.io/osparc-simcore-python-client)\n- Quick install: ``pip install git+https://github.com/ITISFoundation/osparc-simcore-python-client.git``\n", + "title": "osparc.io web API", + "description": "osparc-simcore public web API specifications", "version": "0.3.0", "x-logo": { "url": "https://raw.githubusercontent.com/ITISFoundation/osparc-manual/b809d93619512eb60c827b7e769c6145758378d0/_media/osparc-logo.svg", @@ -221,6 +221,9 @@ } } }, + "404": { + "description": "File not found" + }, "422": { "description": "Validation Error", "content": { @@ -260,13 +263,24 @@ ], "responses": { "200": { - "description": "Successful Response", + "description": "Returns a arbitrary binary data", "content": { - "application/json": { - "schema": {} + "application/octet-stream": { + "schema": { + "type": "string", + "format": "binary" + } + }, + "text/plain": { + "schema": { + "type": "string" + } } } }, + "404": { + "description": "File not found" + }, "422": { "description": "Validation Error", "content": { @@ -307,7 +321,12 @@ } } } - } + }, + "security": [ + { + "HTTPBasic": [] + } + ] } }, "/v0/solvers/{solver_id}": { @@ -350,7 +369,12 @@ } } } - } + }, + "security": [ + { + "HTTPBasic": [] + } + ] } }, "/v0/solvers/{solver_id}/jobs": { @@ -506,7 +530,12 @@ } } } - } + }, + "security": [ + { + "HTTPBasic": [] + } + ] } }, "/v0/jobs": { @@ -851,7 +880,13 @@ "description": "MD5 hash of the file's content" } }, - "description": "Describes a file stored on the server side " + "description": "Describes a file stored on the server side ", + "example": { + "file_id": "f0e1fb11-208d-3ed2-b5ef-cab7a7398f78", + "filename": "Architecture-of-Scalable-Distributed-ETL-System-whitepaper.pdf", + "content_type": "application/pdf", + "checksum": "de47d0e1229aa2dfb80097389094eadd-1" + } }, "Groups": { "title": "Groups", @@ -1182,8 +1217,28 @@ "title": "Gravatar Id", "maxLength": 40, "type": "string", - "description": "Hash value of email to retrieve an avatar image from https://www.gravatar.com" + "description": "md5 hash value of email to retrieve an avatar image from https://www.gravatar.com" } + }, + "example": { + "first_name": "James", + "last_name": "Maxwell", + "login": "james-maxwell@itis.swiss", + "role": "USER", + "groups": { + "me": { + "gid": "123", + "label": "maxy", + "description": "primary group" + }, + "organizations": [], + "all": { + "gid": "1", + "label": "Everyone", + "description": "all users" + } + }, + "gravatar_id": "9a8930a5b20d7048e37740bac5c1ca4f" } }, "ProfileUpdate": { @@ -1224,11 +1279,7 @@ "title": "Version", "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$", "type": "string", - "description": "semantic version number of the node", - "example": [ - "1.0.0", - "0.0.1" - ] + "description": "semantic version number of the node" }, "id": { "title": "Id", @@ -1261,11 +1312,11 @@ "example": { "name": "simcore/services/comp/isolve", "version": "2.1.1", - "id": "42838344-03de-4ce2-8d93-589a5dcdfd05", + "id": "f7c25b7d-edd6-32a4-9751-6072e4163537", "title": "iSolve", "description": "EM solver", "maintainer": "info@itis.swiss", - "url": "https://api.osparc.io/v0/solvers/42838344-03de-4ce2-8d93-589a5dcdfd05" + "url": "https://api.osparc.io/v0/solvers/f7c25b7d-edd6-32a4-9751-6072e4163537" } }, "TaskStates": { diff --git a/services/api-server/src/simcore_service_api_server/api/root.py b/services/api-server/src/simcore_service_api_server/api/root.py index 065670ade42..05bf80fd3bc 100644 --- a/services/api-server/src/simcore_service_api_server/api/root.py +++ b/services/api-server/src/simcore_service_api_server/api/root.py @@ -12,9 +12,8 @@ def create_router(settings: AppSettings): router.include_router(meta.router, tags=["meta"], prefix="/meta") router.include_router(users.router, tags=["users"], prefix="/me") - if settings.fake_server_enabled: + if settings.dev_features_enabled: router.include_router(files.router, tags=["files"], prefix="/files") - # TODO: still really fake router.include_router(solvers.router, tags=["solvers"], prefix="/solvers") router.include_router(jobs.router, tags=["jobs"], prefix="/jobs") diff --git a/services/api-server/src/simcore_service_api_server/api/routes/files.py b/services/api-server/src/simcore_service_api_server/api/routes/files.py index 4791f13427e..d82bf154476 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/files.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/files.py @@ -1,29 +1,22 @@ import asyncio import json import logging -import re -import shutil -import tempfile from collections import deque from datetime import datetime -from mimetypes import guess_type -from pathlib import Path from textwrap import dedent from typing import List, Optional from uuid import UUID -import aiofiles import httpx from fastapi import APIRouter, Depends, File, Header, UploadFile, status from fastapi.exceptions import HTTPException from fastapi.responses import HTMLResponse from pydantic import ValidationError -from starlette.background import BackgroundTask -from starlette.responses import FileResponse +from starlette.responses import RedirectResponse from ..._meta import api_vtag from ...models.schemas.files import FileMetadata -from ...modules.storage import StorageApi, StorageFileMetaData +from ...modules.storage import StorageApi, StorageFileMetaData, to_file_metadata from ..dependencies.authentication import get_current_user_id from ..dependencies.services import get_api_client from .files_faker import the_fake_impl @@ -39,24 +32,11 @@ # - TODO: extend :search as https://cloud.google.com/apis/design/custom_methods ? # # -FILE_ID_PATTERN = re.compile(r"^api\/(?P[\w-]+)\/(?P.+)$") -def convert_metadata(stored_file_meta: StorageFileMetaData) -> FileMetadata: - # extracts fields from api/{file_id}/{filename} - match = FILE_ID_PATTERN.match(stored_file_meta.file_id or "") - if not match: - raise ValueError(f"Invalid file_id {stored_file_meta.file_id} in file metadata") - - file_id, filename = match.groups() - - meta = FileMetadata( - file_id=file_id, - filename=filename, - content_type=guess_type(filename)[0], - checksum=stored_file_meta.entity_tag, - ) - return meta +common_error_responses = { + 404: {"description": "File not found"}, +} @router.get("", response_model=List[FileMetadata]) @@ -75,7 +55,7 @@ async def list_files( assert stored_file_meta.user_id == user_id # nosec assert stored_file_meta.file_id # nosec - meta = convert_metadata(stored_file_meta) + meta = to_file_metadata(stored_file_meta) except (ValidationError, ValueError, AttributeError) as err: logger.warning( @@ -118,18 +98,10 @@ async def upload_file( ) logger.info("Uploading %s to %s ...", meta, presigned_upload_link) - async with httpx.AsyncClient(timeout=httpx.Timeout(5.0, write=3600)) as client: assert meta.content_type # nosec - # pylint: disable=protected-access - # NOTE: _file attribute is a file-like object of ile.file which is - # a https://docs.python.org/3/library/tempfile.html#tempfile.TemporaryFile - # - resp = await client.put( - presigned_upload_link, - files={"upload-file": (meta.filename, file.file._file, meta.content_type)}, - ) + resp = await client.put(presigned_upload_link, data=await file.read()) resp.raise_for_status() # update checksum @@ -155,7 +127,9 @@ async def save_file(file): return uploaded -@router.get("/{file_id}", response_model=FileMetadata) +@router.get( + "/{file_id}", response_model=FileMetadata, responses={**common_error_responses} +) async def get_file( file_id: UUID, storage_client: StorageApi = Depends(get_api_client(StorageApi)), @@ -175,7 +149,7 @@ async def get_file( assert stored_file_meta.file_id # nosec # Adapts storage API model to API model - meta = convert_metadata(stored_file_meta) + meta = to_file_metadata(stored_file_meta) return meta except (ValueError, ValidationError) as err: @@ -186,12 +160,28 @@ async def get_file( ) from err -@router.get("/{file_id}/content") +@router.get( + "/{file_id}/content", + response_class=RedirectResponse, + responses={ + **common_error_responses, + 200: { + "content": { + "application/octet-stream": { + "schema": {"type": "string", "format": "binary"} + }, + "text/plain": {"schema": {"type": "string"}}, + }, + "description": "Returns a arbitrary binary data", + }, + }, +) async def download_file( file_id: UUID, storage_client: StorageApi = Depends(get_api_client(StorageApi)), user_id: int = Depends(get_current_user_id), ): + # NOTE: application/octet-stream is defined as "arbitrary binary data" in RFC 2046, # gets meta meta: FileMetadata = await get_file(file_id, storage_client, user_id) @@ -201,41 +191,7 @@ async def download_file( ) logger.info("Downloading %s to %s ...", meta, presigned_download_link) - - async def _download_chunk(): - async with httpx.AsyncClient(timeout=httpx.Timeout(5.0, read=3600)) as client: - async with client.stream("GET", presigned_download_link) as resp: - async for chunk in resp.aiter_bytes(): - yield chunk - - resp.raise_for_status() - - def _delete(dirpath): - logger.debug("Deleting %s ...", dirpath) - shutil.rmtree(dirpath, ignore_errors=True) - - async def _download_and_save(): - file_path = Path(tempfile.mkdtemp()) / "dump" - try: - async with aiofiles.open(file_path, mode="wb") as fh: - async for chunk in _download_chunk(): - await fh.write(chunk) - except Exception: - _delete(file_path.parent) - raise - return file_path - - # tmp download here TODO: had some problems with RedirectedResponse(presigned_download_link) - file_path = await _download_and_save() - - task = BackgroundTask(_delete, dirpath=file_path.parent) - - return FileResponse( - str(file_path), - media_type=meta.content_type, - filename=meta.filename, - background=task, - ) + return RedirectResponse(presigned_download_link) async def files_upload_multiple_view(): diff --git a/services/api-server/src/simcore_service_api_server/api/routes/files_faker.py b/services/api-server/src/simcore_service_api_server/api/routes/files_faker.py index 0fdc7453710..880cefcfdee 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/files_faker.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/files_faker.py @@ -7,7 +7,7 @@ and get upload/download links to S3 services and avoid the data traffic to go via the API server - This module should be used ONLY when AppSettings.fake_server_enabled==True + This module should be used ONLY when AppSettings.dev_feature_enabled==True """ diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers.py index 50afd9b8680..ec293b18cb2 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers.py @@ -1,14 +1,23 @@ import logging +from operator import attrgetter from typing import Callable, List from uuid import UUID -from fastapi import APIRouter, Depends, HTTPException -from starlette import status - -from ...models.schemas.solvers import LATEST_VERSION, Solver, SolverName +from fastapi import APIRouter, Depends, HTTPException, status +from httpx import HTTPStatusError +from pydantic import ValidationError + +from ...models.schemas.solvers import ( + LATEST_VERSION, + Solver, + SolverName, + compose_solver_id, +) +from ...modules.catalog import CatalogApi from ..dependencies.application import get_reverse_url_mapper +from ..dependencies.authentication import get_current_user_id +from ..dependencies.services import get_api_client from .jobs import Job, JobInput, create_job_impl, list_jobs_impl -from .solvers_faker import the_fake_impl logger = logging.getLogger(__name__) @@ -16,41 +25,74 @@ ## SOLVERS ----------------------------------------------------------------------------------------- +# +# - TODO: pagination, result ordering, filter field and results fields?? SEE https://cloud.google.com/apis/design/standard_methods#list +# - TODO: :search? SEE https://cloud.google.com/apis/design/custom_methods#common_custom_methods +# - TODO: move more of this logic to catalog service +# - TODO: error handling!!! @router.get("", response_model=List[Solver]) async def list_solvers( + user_id: int = Depends(get_current_user_id), + catalog_client: CatalogApi = Depends(get_api_client(CatalogApi)), url_for: Callable = Depends(get_reverse_url_mapper), ): - def _url_resolver(solver_id: UUID): - return url_for( + assert await catalog_client.is_responsive() # nosec + + solvers: List[Solver] = await catalog_client.list_solvers(user_id) + + for solver in solvers: + solver.url = url_for( "get_solver", - solver_id=solver_id, + solver_id=solver.id, ) - # TODO: Consider sorted(latest_solvers, key=attrgetter("name", "version")) - return list(the_fake_impl.values(_url_resolver)) + # updates id -> (name, version) + catalog_client.ids_cache_map[solver.id] = (solver.name, solver.version) + + return sorted(solvers, key=attrgetter("name", "pep404_version")) @router.get("/{solver_id}", response_model=Solver) async def get_solver( solver_id: UUID, + user_id: int = Depends(get_current_user_id), + catalog_client: CatalogApi = Depends(get_api_client(CatalogApi)), url_for: Callable = Depends(get_reverse_url_mapper), -): +) -> Solver: try: - solver = the_fake_impl.get( - solver_id, - url=url_for( - "get_solver", - solver_id=solver_id, - ), + if solver_id in catalog_client.ids_cache_map: + solver_name, solver_version = catalog_client.ids_cache_map[solver_id] + + solver = await get_solver_by_name_and_version( + solver_name, solver_version, user_id, catalog_client, url_for + ) + + else: + + def _with_id(s: Solver): + return compose_solver_id(s.name, s.version) == solver_id + + solvers: List[Solver] = await catalog_client.list_solvers(user_id, _with_id) + assert len(solvers) <= 1 # nosec + solver = solvers[0] + + solver.url = url_for( + "get_solver", + solver_id=solver.id, ) + assert solver.id == solver_id # nosec + + # updates id -> (name, version) + catalog_client.ids_cache_map[solver.id] = (solver.name, solver.version) + return solver - except KeyError as err: + except (KeyError, HTTPStatusError) as err: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, - detail=f"Solver {solver_id} not found", + detail=f"Solver with id={solver_id} not found", ) from err @@ -81,26 +123,26 @@ async def create_job( async def get_solver_by_name_and_version( solver_name: SolverName, version: str, + user_id: int = Depends(get_current_user_id), + catalog_client: CatalogApi = Depends(get_api_client(CatalogApi)), url_for: Callable = Depends(get_reverse_url_mapper), -): +) -> Solver: try: - print(f"/{solver_name}/{version}", flush=True) - - def _url_resolver(solver_id: UUID): - return url_for( - "get_solver", - solver_id=solver_id, - ) - if version == LATEST_VERSION: - solver = the_fake_impl.get_latest(solver_name, _url_resolver) + solver = await catalog_client.get_latest_solver(user_id, solver_name) else: - solver = the_fake_impl.get_by_name_and_version( - solver_name, version, _url_resolver - ) + solver = await catalog_client.get_solver(user_id, solver_name, version) + + solver.url = url_for( + "get_solver", + solver_id=solver.id, + ) + + # updates id -> (name, version) + catalog_client.ids_cache_map[solver.id] = (solver.name, solver.version) return solver - except KeyError as err: + except (ValueError, IndexError, ValidationError, HTTPStatusError) as err: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail=f"Solver {solver_name}:{version} not found", diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers_faker.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers_faker.py index 6f61d1e74c6..b674f06b6e4 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers_faker.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers_faker.py @@ -5,10 +5,11 @@ import packaging.version import yaml +from fastapi import HTTPException, status from importlib_resources import files from models_library.services import ServiceDockerData -from ...models.schemas.solvers import Solver +from ...models.schemas.solvers import LATEST_VERSION, Solver, SolverName @dataclass @@ -59,3 +60,71 @@ def create_from_mocks(cls) -> "SolversFaker": the_fake_impl = SolversFaker.create_from_mocks() + + +# /files API fake implementations + +# GET /solvers + + +async def list_solvers( + url_for: Callable, +): + def _url_resolver(solver_id: UUID): + return url_for( + "get_solver", + solver_id=solver_id, + ) + + # TODO: Consider sorted(latest_solvers, key=attrgetter("name", "version")) + return list(the_fake_impl.values(_url_resolver)) + + +async def get_solver_by_name_and_version( + solver_name: SolverName, + version: str, + url_for: Callable, +): + try: + print(f"/{solver_name}/{version}", flush=True) + + def _url_resolver(solver_id: UUID): + return url_for( + "get_solver", + solver_id=solver_id, + ) + + if version == LATEST_VERSION: + solver = the_fake_impl.get_latest(solver_name, _url_resolver) + else: + solver = the_fake_impl.get_by_name_and_version( + solver_name, version, _url_resolver + ) + return solver + + except KeyError as err: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Solver {solver_name}:{version} not found", + ) from err + + +async def get_solver( + solver_id: UUID, + url_for: Callable, +): + try: + solver = the_fake_impl.get( + solver_id, + url=url_for( + "get_solver", + solver_id=solver_id, + ), + ) + return solver + + except KeyError as err: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Solver {solver_id} not found", + ) from err diff --git a/services/api-server/src/simcore_service_api_server/core/settings.py b/services/api-server/src/simcore_service_api_server/core/settings.py index 2c660c058e9..15a2982ad41 100644 --- a/services/api-server/src/simcore_service_api_server/core/settings.py +++ b/services/api-server/src/simcore_service_api_server/core/settings.py @@ -144,7 +144,9 @@ def loglevel(self) -> int: debug: bool = False # If True, debug tracebacks should be returned on errors. remote_debug_port: int = 3000 - fake_server_enabled: bool = Field(False, env="FAKE_API_SERVER_ENABLED") + dev_features_enabled: bool = Field( + False, env=["API_SERVER_DEV_FEATURES_ENABLED", "FAKE_API_SERVER_ENABLED"] + ) class Config(_CommonConfig): env_prefix = "" diff --git a/services/api-server/src/simcore_service_api_server/models/schemas/files.py b/services/api-server/src/simcore_service_api_server/models/schemas/files.py index 328e32c6ff3..5fe05c96af2 100644 --- a/services/api-server/src/simcore_service_api_server/models/schemas/files.py +++ b/services/api-server/src/simcore_service_api_server/models/schemas/files.py @@ -25,6 +25,16 @@ class FileMetadata(BaseModel): # SEE https://ant.apache.org/manual/Tasks/checksum.html checksum: str = Field(None, description="MD5 hash of the file's content") + class Config: + schema_extra = { + "example": { + "file_id": "f0e1fb11-208d-3ed2-b5ef-cab7a7398f78", + "filename": "Architecture-of-Scalable-Distributed-ETL-System-whitepaper.pdf", + "content_type": "application/pdf", + "checksum": "de47d0e1229aa2dfb80097389094eadd-1", + } + } + @classmethod async def create_from_path(cls, path: Path) -> "FileMetadata": async with aiofiles.open(path, mode="rb") as file: diff --git a/services/api-server/src/simcore_service_api_server/models/schemas/profiles.py b/services/api-server/src/simcore_service_api_server/models/schemas/profiles.py index 5ab3bdb8184..231b30a1cae 100644 --- a/services/api-server/src/simcore_service_api_server/models/schemas/profiles.py +++ b/services/api-server/src/simcore_service_api_server/models/schemas/profiles.py @@ -30,9 +30,30 @@ class Profile(ProfileCommon): groups: Optional[Groups] = None gravatar_id: Optional[str] = Field( None, - description="Hash value of email to retrieve an avatar image from https://www.gravatar.com", + description="md5 hash value of email to retrieve an avatar image from https://www.gravatar.com", max_length=40, ) class Config: - schema_extra = {} + schema_extra = { + "example": { + "first_name": "James", + "last_name": "Maxwell", + "login": "james-maxwell@itis.swiss", + "role": "USER", + "groups": { + "me": { + "gid": "123", + "label": "maxy", + "description": "primary group", + }, + "organizations": [], + "all": { + "gid": "1", + "label": "Everyone", + "description": "all users", + }, + }, + "gravatar_id": "9a8930a5b20d7048e37740bac5c1ca4f", + } + } diff --git a/services/api-server/src/simcore_service_api_server/models/schemas/solvers.py b/services/api-server/src/simcore_service_api_server/models/schemas/solvers.py index dff9ba914f8..0538c4a2741 100644 --- a/services/api-server/src/simcore_service_api_server/models/schemas/solvers.py +++ b/services/api-server/src/simcore_service_api_server/models/schemas/solvers.py @@ -4,9 +4,11 @@ from typing import Optional, Union from uuid import UUID, uuid3 +import packaging.version from models_library.basic_regex import VERSION_RE from models_library.services import COMPUTATIONAL_SERVICE_KEY_RE, ServiceDockerData -from pydantic import BaseModel, Field, HttpUrl, conint, constr, validator +from packaging.version import LegacyVersion, Version +from pydantic import BaseModel, Extra, Field, HttpUrl, conint, constr, validator LATEST_VERSION = "latest" NAMESPACE_SOLVER_KEY = UUID("ca7bdfc4-08e8-11eb-935a-ac9e17b76a71") @@ -14,7 +16,7 @@ @functools.lru_cache() -def _compose_solver_id(name, version) -> UUID: +def compose_solver_id(name: str, version: str) -> UUID: return uuid3(NAMESPACE_SOLVER_KEY, f"{name}:{version}") @@ -26,7 +28,7 @@ def _compose_job_id(solver_id: UUID, inputs_sha: str, created_at: str) -> UUID: # SOLVER ---------- # -# +VersionStr = constr(strip_whitespace=True, regex=VERSION_RE) SolverName = constr( strip_whitespace=True, regex=COMPUTATIONAL_SERVICE_KEY_RE, @@ -41,11 +43,9 @@ class Solver(BaseModel): ..., description="Unique solver name with path namespaces", ) - version: str = Field( + version: VersionStr = Field( ..., description="semantic version number of the node", - regex=VERSION_RE, - example=["1.0.0", "0.0.1"], ) id: UUID @@ -60,24 +60,32 @@ class Solver(BaseModel): url: Optional[HttpUrl] = Field(..., description="Link to get this resource") class Config: + extra = Extra.ignore schema_extra = { "example": { "name": "simcore/services/comp/isolve", "version": "2.1.1", - "id": "42838344-03de-4ce2-8d93-589a5dcdfd05", + "id": "f7c25b7d-edd6-32a4-9751-6072e4163537", "title": "iSolve", "description": "EM solver", "maintainer": "info@itis.swiss", - "url": "https://api.osparc.io/v0/solvers/42838344-03de-4ce2-8d93-589a5dcdfd05", + "url": "https://api.osparc.io/v0/solvers/f7c25b7d-edd6-32a4-9751-6072e4163537", } } - @validator("id", pre=True) + @validator("id", pre=True, always=True) @classmethod def compose_id_with_name_and_version(cls, v, values): - if v is None: - return _compose_solver_id(values["name"], values["version"]) - return v + try: + sid = compose_solver_id(values["name"], values["version"]) + if v and str(v) != str(sid): + raise ValueError( + f"Invalid id: {v}!={sid} is incompatible with name and version composition" + ) + return sid + except KeyError as err: + # If validation of name or version fails, it is NOT passed as values + raise ValueError(f"Id requires valid {err}") from err @classmethod def create_from_image(cls, image_meta: ServiceDockerData) -> "Solver": @@ -95,6 +103,11 @@ def create_from_image(cls, image_meta: ServiceDockerData) -> "Solver": **data, ) + @property + def pep404_version(self) -> Union[Version, LegacyVersion]: + """ Rich version type that can be used e.g. to compare """ + return packaging.version.parse(self.version) + # JOBS ---------- # @@ -125,14 +138,15 @@ class Config: } } - @validator("id", pre=True) + @validator("id", pre=True, always=True) @classmethod def compose_id_with_solver_and_input(cls, v, values): - if v is None: - return _compose_job_id( - values["solver_id"], values["inputs_checksum"], values["created_at"] - ) - return v + jid = _compose_job_id( + values["solver_id"], values["inputs_checksum"], values["created_at"] + ) + if v and str(v) != str(jid): + raise ValueError(f"Invalid id: {v}!={jid} is incompatible with composition") + return jid @classmethod def create_now(cls, solver_id: UUID, inputs_checksum: str) -> "Job": @@ -214,7 +228,7 @@ class SolverPort(BaseModel): class JobInput(SolverPort): - value: PortValue = None + value: Optional[PortValue] = None # TODO: validate one or the other but not both diff --git a/services/api-server/src/simcore_service_api_server/modules/catalog.py b/services/api-server/src/simcore_service_api_server/modules/catalog.py index e1db1c7e6c7..74cfb275301 100644 --- a/services/api-server/src/simcore_service_api_server/modules/catalog.py +++ b/services/api-server/src/simcore_service_api_server/modules/catalog.py @@ -1,12 +1,21 @@ import logging +import urllib.parse +from dataclasses import dataclass, field +from operator import attrgetter +from typing import Callable, Dict, List, Optional, Tuple +from uuid import UUID import httpx from fastapi import FastAPI +from models_library.services import ServiceDockerData, ServiceType +from pydantic import EmailStr, Extra, ValidationError from ..core.settings import CatalogSettings -from ..utils.client_decorators import JsonDataType, handle_errors, handle_retry +from ..models.schemas.solvers import LATEST_VERSION, Solver, SolverName, VersionStr from ..utils.client_base import BaseServiceClientApi +## from ..utils.client_decorators import JsonDataType, handle_errors, handle_retry + logger = logging.getLogger(__name__) # Module's setup logic --------------------------------------------- @@ -34,15 +43,129 @@ async def on_shutdown() -> None: app.add_event_handler("shutdown", on_shutdown) +SolverNameVersionPair = Tuple[SolverName, str] + + +class TruncatedServiceOut(ServiceDockerData): + """ + This model is used to truncate the response of the catalog, whose schema is + in services/catalog/src/simcore_service_catalog/models/schemas/services.py::ServiceOut + and is a superset of ServiceDockerData. + + We do not use directly ServiceDockerData because it will only consume the exact fields + (it is configured as Extra.forbid). Instead we inherit from it, override this configuration + and add an extra field that we want to capture from ServiceOut. + + Ideally the rest of the response is dropped so here it would + perhaps make more sense to use something like graphql + that asks only what is needed. + """ + + owner: Optional[EmailStr] + + class Config: + extra = Extra.ignore + + # Converters + def to_solver(self) -> Solver: + data = self.dict( + include={"name", "key", "version", "description", "contact", "owner"}, + ) + + return Solver( + name=data.pop("key"), + version=data.pop("version"), + title=data.pop("name"), + maintainer=data.pop("owner") or data.pop("contact"), + url=None, + id=None, # auto-generated + **data, + ) + + # API CLASS --------------------------------------------- +# +# - Error handling: What do we reraise, suppress, transform??? +# +# +# TODO: handlers should not capture outputs +# @handle_errors("catalog", logger, return_json=True) +# @handle_retry(logger) +# async def get(self, path: str, *args, **kwargs) -> JsonDataType: +# return await self.client.get(path, *args, **kwargs) +@dataclass class CatalogApi(BaseServiceClientApi): + """ + This class acts a proxy of the catalog service + It abstracts request to the catalog API service + """ + + ids_cache_map: Dict[UUID, SolverNameVersionPair] = field(default_factory=dict) + + async def list_solvers( + self, + user_id: int, + predicate: Optional[Callable[[Solver], bool]] = None, + ) -> List[Solver]: + resp = await self.client.get( + "/services", + params={"user_id": user_id, "details": False}, + headers={"x-simcore-products-name": "osparc"}, + ) + resp.raise_for_status() + + # TODO: move this sorting down to catalog service? + solvers = [] + for data in resp.json(): + try: + service = TruncatedServiceOut.parse_obj(data) + if service.service_type == ServiceType.COMPUTATIONAL: + solver = service.to_solver() + if predicate is None or predicate(solver): + solvers.append(solver) + + except ValidationError as err: + # NOTE: For the moment, this is necessary because there are no guarantees + # at the image registry. Therefore we exclude and warn + # invalid items instead of returning error + logger.warning( + "Skipping invalid service returned by catalog '%s': %s", + data, + err, + ) + return solvers + + async def get_solver( + self, user_id: int, name: SolverName, version: VersionStr + ) -> Solver: + + assert version != LATEST_VERSION # nosec + + service_key = urllib.parse.quote_plus(name) + service_version = version + + resp = await self.client.get( + f"/services/{service_key}/{service_version}", + params={"user_id": user_id}, + headers={"x-simcore-products-name": "osparc"}, + ) + resp.raise_for_status() + + service = TruncatedServiceOut.parse_obj(resp.json()) + assert ( + service.service_type == ServiceType.COMPUTATIONAL + ), "Expected by SolverName regex" # nosec + + return service.to_solver() + + async def get_latest_solver(self, user_id: int, name: SolverName) -> Solver: + def _this_solver(solver: Solver) -> bool: + return solver.name == name - # OPERATIONS - # TODO: add ping to healthcheck + solvers = await self.list_solvers(user_id, _this_solver) - @handle_errors("catalog", logger, return_json=True) - @handle_retry(logger) - async def get(self, path: str, *args, **kwargs) -> JsonDataType: - return await self.client.get(path, *args, **kwargs) + # raise IndexError if None + latest = sorted(solvers, key=attrgetter("pep404_version"))[-1] + return latest diff --git a/services/api-server/src/simcore_service_api_server/modules/storage.py b/services/api-server/src/simcore_service_api_server/modules/storage.py index ecfc93a20c4..6b375ab56d1 100644 --- a/services/api-server/src/simcore_service_api_server/modules/storage.py +++ b/services/api-server/src/simcore_service_api_server/modules/storage.py @@ -1,5 +1,7 @@ import logging +import re import urllib.parse +from mimetypes import guess_type from typing import List from uuid import UUID @@ -9,6 +11,7 @@ from models_library.api_schemas_storage import FileMetaDataArray, PresignedLink from ..core.settings import StorageSettings +from ..models.schemas.files import FileMetadata from ..utils.client_base import BaseServiceClientApi ## from ..utils.client_decorators import JsonDataType, handle_errors, handle_retry @@ -55,14 +58,6 @@ class StorageApi(BaseServiceClientApi): # async def get(self, path: str, *args, **kwargs) -> JsonDataType: # return await self.client.get(path, *args, **kwargs) - async def is_responsive(self) -> bool: - try: - resp = await self.client.get("/") - resp.raise_for_status() - return True - except httpx.HTTPStatusError: - return False - async def list_files(self, user_id: int) -> List[StorageFileMetaData]: """ Lists metadata of all s3 objects name as api/* from a given user""" resp = await self.client.post( @@ -119,3 +114,26 @@ async def get_upload_link(self, user_id: int, file_id: UUID, file_name: str) -> presigned_link = PresignedLink.parse_obj(resp.json()["data"]) return presigned_link.link + + +FILE_ID_PATTERN = re.compile(r"^api\/(?P[\w-]+)\/(?P.+)$") + + +def to_file_metadata(stored_file_meta: StorageFileMetaData) -> FileMetadata: + # extracts fields from api/{file_id}/{filename} + match = FILE_ID_PATTERN.match(stored_file_meta.file_id or "") + if not match: + raise ValueError(f"Invalid file_id {stored_file_meta.file_id} in file metadata") + + file_id, filename = match.groups() + + meta = FileMetadata( + file_id=file_id, + filename=filename, + # FIXME: UploadFile gets content from the request header while here is + # mimetypes.guess_type used. Sometimes it does not match. + # Add column in meta_data table of storage and stop guessing :-) + content_type=guess_type(filename)[0] or "application/octet-stream", + checksum=stored_file_meta.entity_tag, + ) + return meta diff --git a/services/api-server/src/simcore_service_api_server/utils/client_base.py b/services/api-server/src/simcore_service_api_server/utils/client_base.py index a0ec431a0bd..0c268e0f9f6 100644 --- a/services/api-server/src/simcore_service_api_server/utils/client_base.py +++ b/services/api-server/src/simcore_service_api_server/utils/client_base.py @@ -8,14 +8,16 @@ @dataclass class BaseServiceClientApi: """ - - wrapper around thin-client to simplify service's API + - wrapper around thin-client to simplify service's API calls - sets endspoint upon construction - MIME type: application/json - processes responses, returning data or raising formatted HTTP exception - + - helpers to create a unique client instance per application and service """ + client: httpx.AsyncClient service_name: str = "" + health_check_path: str = "/" @classmethod def create(cls, app: FastAPI, **kwargs): @@ -35,3 +37,11 @@ def get_instance(cls, app: FastAPI) -> Optional["BaseServiceClientApi"]: async def aclose(self): await self.client.aclose() + + async def is_responsive(self) -> bool: + try: + resp = await self.client.get(self.health_check_path) + resp.raise_for_status() + return True + except (httpx.HTTPStatusError, httpx.RequestError): + return False diff --git a/services/api-server/tests/unit/conftest.py b/services/api-server/tests/unit/conftest.py index ffed275b37e..99edfd21077 100644 --- a/services/api-server/tests/unit/conftest.py +++ b/services/api-server/tests/unit/conftest.py @@ -1,18 +1,21 @@ # pylint:disable=unused-variable # pylint:disable=unused-argument # pylint:disable=redefined-outer-name + import os import shutil import subprocess import sys from pathlib import Path -from typing import Callable, Coroutine, Dict, Union +from typing import Any, Callable, Dict, Iterator, List, Type, Union import aiopg.sa +import aiopg.sa.engine as aiopg_sa_engine import pytest import simcore_postgres_database.cli as pg_cli import simcore_service_api_server import sqlalchemy as sa +import sqlalchemy.engine as sa_engine import yaml from _helpers import RWApiKeysRepository, RWUsersRepository from asgi_lifespan import LifespanManager @@ -20,15 +23,19 @@ from fastapi import FastAPI from fastapi.testclient import TestClient from httpx import AsyncClient +from pydantic import BaseModel from simcore_postgres_database.models.base import metadata from simcore_service_api_server.models.domain.api_keys import ApiKeyInDB current_dir = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent +pytestmark = pytest.mark.asyncio + pytest_plugins = [ "pytest_simcore.repository_paths", ] + ## TEST_ENVIRON --- @@ -60,6 +67,9 @@ def project_env_devel_environment(project_env_devel_dict, monkeypatch): for key, value in project_env_devel_dict.items(): monkeypatch.setenv(key, value) + # overrides + monkeypatch.setenv("API_SERVER_DEV_FEATURES_ENABLED", "1") + ## FOLDER LAYOUT --------------------------------------------------------------------- @@ -164,22 +174,29 @@ def is_postgres_responsive() -> bool: def make_engine(postgres_service: Dict) -> Callable: dsn = postgres_service["dsn"] # session scope freezes dsn - def maker(is_async=True) -> Union[Coroutine, Callable]: - return aiopg.sa.create_engine(dsn) if is_async else sa.create_engine(dsn) + def maker(is_async=True) -> Union[aiopg_sa_engine.Engine, sa_engine.Engine]: + if is_async: + return aiopg.sa.create_engine(dsn) + return sa.create_engine(dsn) return maker @pytest.fixture -def apply_migration(postgres_service: Dict, make_engine) -> None: +def apply_migration(postgres_service: Dict, make_engine) -> Iterator[None]: + # NOTE: this is equivalent to packages/pytest-simcore/src/pytest_simcore/postgres_service.py::postgres_db + # but we do override postgres_dsn -> postgres_engine -> postgres_db because we want the latter + # fixture to have local scope + # kwargs = postgres_service.copy() kwargs.pop("dsn") pg_cli.discover.callback(**kwargs) pg_cli.upgrade.callback("head") + yield + pg_cli.downgrade.callback("base") pg_cli.clean.callback() - # FIXME: deletes all because downgrade is not reliable! engine = make_engine(False) metadata.drop_all(engine) @@ -201,13 +218,13 @@ def app(monkeypatch, environment, apply_migration) -> FastAPI: @pytest.fixture -async def initialized_app(app: FastAPI) -> FastAPI: +async def initialized_app(app: FastAPI) -> Iterator[FastAPI]: async with LifespanManager(app): yield app @pytest.fixture -async def client(initialized_app: FastAPI) -> AsyncClient: +async def client(initialized_app: FastAPI) -> Iterator[AsyncClient]: async with AsyncClient( app=initialized_app, base_url="http://api.testserver.io", @@ -249,3 +266,21 @@ async def test_api_key(loop, initialized_app, test_user_id) -> ApiKeyInDB: "test-api-key", api_key="key", api_secret="secret", user_id=test_user_id ) return apikey + + +## PYDANTIC MODELS & SCHEMAS ----------------------------------------------------- + + +@pytest.fixture +def model_cls_examples(model_cls: Type[BaseModel]) -> List[Dict[str, Any]]: + # Extracts examples from pydantic model class + # Use by defining model_cls as test parametrization + # SEE https://pydantic-docs.helpmanual.io/usage/schema/#schema-customization + examples = model_cls.Config.schema_extra.get("examples", []) + example = model_cls.Config.schema_extra.get("example") + if example: + examples.append(example) + + assert model_cls_examples, f"{model_cls} has NO examples. Add them in Config class" + + return examples diff --git a/services/api-server/tests/unit/test_api_user.py b/services/api-server/tests/unit/test_api_user.py index 6f4529914ed..6dd0c58c495 100644 --- a/services/api-server/tests/unit/test_api_user.py +++ b/services/api-server/tests/unit/test_api_user.py @@ -6,11 +6,12 @@ import pytest from httpx import AsyncClient -from starlette import status - from simcore_service_api_server._meta import api_vtag from simcore_service_api_server.models.domain.api_keys import ApiKeyInDB from simcore_service_api_server.models.schemas.profiles import Profile +from starlette import status + +pytestmark = pytest.mark.asyncio @pytest.fixture diff --git a/services/api-server/tests/unit/test_model_schemas_files.py b/services/api-server/tests/unit/test_model_schemas_files.py index 6b6f637d2f1..08de994d71e 100644 --- a/services/api-server/tests/unit/test_model_schemas_files.py +++ b/services/api-server/tests/unit/test_model_schemas_files.py @@ -5,14 +5,15 @@ import hashlib import tempfile from pathlib import Path +from pprint import pprint from uuid import uuid4 import pytest from fastapi import UploadFile from models_library.api_schemas_storage import FileMetaData as StorageFileMetaData from pydantic import ValidationError -from simcore_service_api_server.api.routes.files import convert_metadata from simcore_service_api_server.models.schemas.files import FileMetadata +from simcore_service_api_server.modules.storage import to_file_metadata pytestmark = pytest.mark.asyncio @@ -77,17 +78,25 @@ def test_convert_filemetadata(): **StorageFileMetaData.Config.schema_extra["examples"][1] ) storage_file_meta.file_id = f"api/{uuid4()}/extensionless" - apiserver_file_meta = convert_metadata(storage_file_meta) + apiserver_file_meta = to_file_metadata(storage_file_meta) assert apiserver_file_meta.file_id assert apiserver_file_meta.filename == "extensionless" - assert apiserver_file_meta.content_type is None + assert apiserver_file_meta.content_type == "application/octet-stream" # default assert apiserver_file_meta.checksum == storage_file_meta.entity_tag with pytest.raises(ValueError): storage_file_meta.file_id = f"{uuid4()}/{uuid4()}/foo.txt" - convert_metadata(storage_file_meta) + to_file_metadata(storage_file_meta) with pytest.raises(ValidationError): storage_file_meta.file_id = "api/NOTUUID/foo.txt" - convert_metadata(storage_file_meta) + to_file_metadata(storage_file_meta) + + +@pytest.mark.parametrize("model_cls", (FileMetadata,)) +def test_file_model_examples(model_cls, model_cls_examples): + for example in model_cls_examples: + pprint(example) + model_instance = model_cls(**example) + assert model_instance diff --git a/services/api-server/tests/unit/test_model_schemas_meta.py b/services/api-server/tests/unit/test_model_schemas_meta.py new file mode 100644 index 00000000000..8ec3a747e97 --- /dev/null +++ b/services/api-server/tests/unit/test_model_schemas_meta.py @@ -0,0 +1,15 @@ +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name +from pprint import pprint + +import pytest +from simcore_service_api_server.models.schemas.meta import Meta + + +@pytest.mark.parametrize("model_cls", (Meta,)) +def test_meta_model_examples(model_cls, model_cls_examples): + for example in model_cls_examples: + pprint(example) + model_instance = model_cls(**example) + assert model_instance diff --git a/services/api-server/tests/unit/test_model_schemas_profiles.py b/services/api-server/tests/unit/test_model_schemas_profiles.py new file mode 100644 index 00000000000..14403547c01 --- /dev/null +++ b/services/api-server/tests/unit/test_model_schemas_profiles.py @@ -0,0 +1,15 @@ +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name +from pprint import pprint + +import pytest +from simcore_service_api_server.models.schemas.profiles import Profile + + +@pytest.mark.parametrize("model_cls", (Profile,)) +def test_profiles_model_examples(model_cls, model_cls_examples): + for example in model_cls_examples: + pprint(example) + model_instance = model_cls(**example) + assert model_instance diff --git a/services/api-server/tests/unit/test_model_schemas_solvers.py b/services/api-server/tests/unit/test_model_schemas_solvers.py index bd2f5e97186..101c4f69466 100644 --- a/services/api-server/tests/unit/test_model_schemas_solvers.py +++ b/services/api-server/tests/unit/test_model_schemas_solvers.py @@ -3,6 +3,8 @@ # pylint:disable=redefined-outer-name import sys +from operator import attrgetter +from pprint import pprint from uuid import uuid4 import pytest @@ -12,12 +14,20 @@ JobInput, JobOutput, Solver, + Version, _compose_job_id, ) -def test_create_solver_from_image_metadata(): +@pytest.mark.parametrize("model_cls", (Job, Solver, JobInput, JobOutput)) +def test_solvers_model_examples(model_cls, model_cls_examples): + for example in model_cls_examples: + pprint(example) + model_instance = model_cls(**example) + assert model_instance + +def test_create_solver_from_image_metadata(): for image_metadata in SolversFaker.load_images(): solver = Solver.create_from_image(image_metadata) print(solver.json(indent=2)) @@ -27,7 +37,6 @@ def test_create_solver_from_image_metadata(): def test_create_job_model(): - job = Job.create_now(uuid4(), "12345") print(job.json(indent=2)) @@ -43,10 +52,40 @@ def test_create_job_model(): # v.utc -@pytest.mark.parametrize("model_cls", (Job, Solver, JobInput, JobOutput)) -def test_solvers_model_examples(model_cls): - example = model_cls.Config.schema_extra["example"] - print(example) +def test_solvers_sorting_by_name_and_version(faker): + # SEE https://packaging.pypa.io/en/latest/version.html + + # have a solver + solver0 = Solver(**Solver.Config.schema_extra["example"]) + + assert isinstance(solver0.pep404_version, Version) + major, minor, micro = solver0.pep404_version.release + solver0.version = f"{major}.{minor}.{micro}" + + # and a different version of the same + # NOTE: that id=None so that it can be re-coputed + solver1 = solver0.copy( + update={"version": f"{solver0.version}beta", "id": None}, deep=True + ) + assert solver1.pep404_version.is_prerelease + assert solver1.pep404_version < solver0.pep404_version + assert solver0.id != solver1.id, "changing vesion should automaticaly change id" + + # and yet a completely different solver + other_solver = solver0.copy( + update={"name": f"simcore/services/comp/{faker.name()}", "id": None} + ) + assert ( + solver0.id != other_solver.id + ), "changing vesion should automaticaly change id" + + # let's sort a list of solvers by name and then by version + sorted_solvers = sorted( + [solver0, other_solver, solver1], key=attrgetter("name", "pep404_version") + ) - model_instance = model_cls(**example) - assert model_instance + # dont' really know reference solver name so... + if solver0.name < other_solver.name: + assert sorted_solvers == [solver1, solver0, other_solver] + else: + assert sorted_solvers == [other_solver, solver1, solver0] diff --git a/services/api-server/tests/utils/docker-compose.yml b/services/api-server/tests/utils/docker-compose.yml index ecf59948cfb..52501306a50 100644 --- a/services/api-server/tests/utils/docker-compose.yml +++ b/services/api-server/tests/utils/docker-compose.yml @@ -1,7 +1,7 @@ -version: '3.4' +version: "3.4" services: postgres: - image: postgres:10 + image: postgres:10.11@sha256:2aef165ab4f30fbb109e88959271d8b57489790ea13a77d27c02d8adb8feb20f environment: - POSTGRES_USER=${POSTGRES_USER:-test} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-test} @@ -14,10 +14,14 @@ services: command: [ "postgres", - "-c", "log_connections=true", - "-c", "log_disconnections=true", - "-c", "log_duration=true", - "-c", "log_line_prefix=[%p] [%a] [%c] [%x] " + "-c", + "log_connections=true", + "-c", + "log_disconnections=true", + "-c", + "log_duration=true", + "-c", + "log_line_prefix=[%p] [%a] [%c] [%x] ", ] adminer: image: adminer diff --git a/services/catalog/.env-devel b/services/catalog/.env-devel index 31f78072143..54f692d2a5e 100644 --- a/services/catalog/.env-devel +++ b/services/catalog/.env-devel @@ -21,5 +21,6 @@ REGISTRY_USER=admin DIRECTOR_REGISTRY_CACHING=True DIRECTOR_REGISTRY_CACHING_TTL=10 +BACKGROUND_TASK_REST_TIME=60 SC_BOOT_MODE=debug-ptvsd diff --git a/services/catalog/Makefile b/services/catalog/Makefile index 024f5d42c04..8e3fc590b90 100644 --- a/services/catalog/Makefile +++ b/services/catalog/Makefile @@ -55,7 +55,10 @@ run-prod: .env up-extra # BUILD ##################### .PHONY: openapi-specs openapi.json -openapi-specs: openapi.json .env -openapi.json: +openapi-specs: openapi.json +openapi.json: .env # generating openapi specs file python3 -c "import json; from $(APP_PACKAGE_NAME).__main__ import *; print( json.dumps(the_app.openapi(), indent=2) )" > $@ + # validates OAS file: $@ + @cd $(CURDIR); \ + $(SCRIPTS_DIR)/openapi-generator-cli.bash validate --input-spec /local/$@ diff --git a/services/catalog/docker-compose-extra.yml b/services/catalog/docker-compose-extra.yml index bdc4736f630..df30d9505a5 100644 --- a/services/catalog/docker-compose-extra.yml +++ b/services/catalog/docker-compose-extra.yml @@ -4,7 +4,7 @@ version: "3.7" services: postgres: - image: postgres:10 + image: postgres:10.11@sha256:2aef165ab4f30fbb109e88959271d8b57489790ea13a77d27c02d8adb8feb20f init: true environment: - POSTGRES_USER=${POSTGRES_USER:-test} diff --git a/services/catalog/openapi.json b/services/catalog/openapi.json index 897907492b5..cade3ccef03 100644 --- a/services/catalog/openapi.json +++ b/services/catalog/openapi.json @@ -429,7 +429,7 @@ "required": true, "schema": { "title": "Service Key", - "pattern": "^(simcore)/(services)/(comp|dynamic|frontend)(/[^\\s/]+)+$", + "pattern": "^(simcore)/(services)/(comp|dynamic|frontend)(/[\\w/-]+)+$", "type": "string" }, "name": "service_key", @@ -498,7 +498,7 @@ "required": true, "schema": { "title": "Service Key", - "pattern": "^(simcore)/(services)/(comp|dynamic|frontend)(/[^\\s/]+)+$", + "pattern": "^(simcore)/(services)/(comp|dynamic|frontend)(/[\\w/-]+)+$", "type": "string" }, "name": "service_key", @@ -598,20 +598,12 @@ "title": "Email", "type": "string", "description": "Email address", - "format": "email", - "example": [ - "sun@sense.eight", - "deleen@minbar.bab" - ] + "format": "email" }, "affiliation": { "title": "Affiliation", "type": "string", - "description": "Affiliation of the author", - "example": [ - "Sense8", - "Babylon 5" - ] + "description": "Affiliation of the author" } }, "additionalProperties": false @@ -628,12 +620,7 @@ "name": { "title": "Name", "type": "string", - "description": "Name of the subject", - "example": [ - "travis-ci", - "coverals.io", - "github.io" - ] + "description": "Name of the subject" }, "image": { "title": "Image", @@ -641,12 +628,7 @@ "minLength": 1, "type": "string", "description": "Url to the badge", - "format": "uri", - "example": [ - "https://travis-ci.org/ITISFoundation/osparc-simcore.svg?branch=master", - "https://coveralls.io/repos/github/ITISFoundation/osparc-simcore/badge.svg?branch=master", - "https://img.shields.io/website-up-down-green-red/https/itisfoundation.github.io.svg?label=documentation" - ] + "format": "uri" }, "url": { "title": "Url", @@ -654,12 +636,7 @@ "minLength": 1, "type": "string", "description": "Link to the status", - "format": "uri", - "example": [ - "https://travis-ci.org/ITISFoundation/osparc-simcore 'State of CI: build, test and pushing images'", - "https://coveralls.io/github/ITISFoundation/osparc-simcore?branch=master 'Test coverage'", - "https://itisfoundation.github.io/" - ] + "format": "uri" } }, "additionalProperties": false @@ -675,7 +652,7 @@ "properties": { "key": { "title": "Key", - "pattern": "^(simcore)/(services)/(comp|dynamic|frontend)(/[^\\s/]+)+$", + "pattern": "^(simcore)/(services)/(comp|dynamic|frontend)(/[\\w/-]+)+$", "type": "string", "example": "simcore/services/frontend/nodes-group/macros/1" }, @@ -719,7 +696,7 @@ "properties": { "key": { "title": "Key", - "pattern": "^(simcore)/(services)/(comp|dynamic|frontend)(/[^\\s/]+)+$", + "pattern": "^(simcore)/(services)/(comp|dynamic|frontend)(/[\\w/-]+)+$", "type": "string", "example": "simcore/services/frontend/nodes-group/macros/1" }, @@ -760,8 +737,8 @@ "required": [ "store", "path", - "dataset", - "label" + "label", + "dataset" ], "type": "object", "properties": { @@ -775,20 +752,23 @@ "type": "integer" } ], - "description": "The store identifier, '0' or 0 for simcore S3, '1' or 1 for datcore", - "example": [ - "0", - 1 - ] + "description": "The store identifier, '0' or 0 for simcore S3, '1' or 1 for datcore" }, "path": { "title": "Path", + "pattern": "^.+$", "type": "string", - "description": "The path to the file in the storage provider domain", - "example": [ - "N:package:b05739ef-260c-4038-b47d-0240d04b0599", - "94453a6a-c8d4-52b3-a22d-ccbf81f8d636/d4442ca4-23fd-5b6b-ba6d-0b75f711c109/y_1D.txt" - ] + "description": "The path to the file in the storage provider domain" + }, + "eTag": { + "title": "Etag", + "type": "string", + "description": "Entity tag that uniquely represents the file. The method to generate the tag is not specified (black box)." + }, + "label": { + "title": "Label", + "type": "string", + "description": "The real file name" }, "dataset": { "title": "Dataset", @@ -797,14 +777,27 @@ "example": [ "N:dataset:f9f5ac51-33ea-4861-8e08-5b4faf655041" ] + } + }, + "additionalProperties": false + }, + "DownloadLink": { + "title": "DownloadLink", + "required": [ + "downloadLink" + ], + "type": "object", + "properties": { + "downloadLink": { + "title": "Downloadlink", + "maxLength": 65536, + "minLength": 1, + "type": "string", + "format": "uri" }, "label": { "title": "Label", - "type": "string", - "description": "The real file name (REQUIRED for datcore)", - "example": [ - "MyFile.txt" - ] + "type": "string" } }, "additionalProperties": false @@ -869,7 +862,7 @@ "properties": { "key": { "title": "Key", - "pattern": "^(simcore)/(services)/(comp|dynamic|frontend)(/[^\\s/]+)+$", + "pattern": "^(simcore)/(services)/(comp|dynamic|frontend)(/[\\w/-]+)+$", "type": "string", "description": "distinctive name for the node based on the docker registry path", "example": [ @@ -914,13 +907,27 @@ "https://placeimg.com/171/96/tech/grayscale/?0.jpg" ] }, + "runHash": { + "anyOf": [ + { + "type": "null" + }, + { + "title": "Runhash", + "description": "the hex digest of the resolved inputs +outputs hash at the time when the last outputs were generated", + "example": [ + "a4337bc45a8fc544c03f52dc550cd6e1e87021bc896588bd79e901e2" + ], + "type": "string" + } + ] + }, "inputs": { "title": "Inputs", "type": "object", "description": "values of input properties" }, "inputAccess": { - "title": "Inputaccess", "type": "object", "description": "map with key - access level pairs" }, @@ -929,7 +936,7 @@ "type": "array", "items": { "type": "string", - "format": "uuid4" + "format": "uuid" }, "description": "node IDs of where the node is connected to", "example": [ @@ -939,7 +946,8 @@ }, "outputs": { "title": "Outputs", - "type": "object" + "type": "object", + "description": "values of output properties" }, "outputNode": { "title": "Outputnode", @@ -951,7 +959,7 @@ "type": "array", "items": { "type": "string", - "format": "uuid4" + "format": "uuid" }, "description": "Used in group-nodes. Node IDs of those connected to the output", "example": [ @@ -962,13 +970,22 @@ "parent": { "title": "Parent", "type": "string", - "description": "Parent's (group-nodes') node ID s.", - "format": "uuid4", + "description": "Parent's (group-nodes') node ID s. Used to group", + "format": "uuid", "example": [ "nodeUUid1", "nodeUuid2" ] }, + "state": { + "allOf": [ + { + "$ref": "#/components/schemas/RunningState" + } + ], + "description": "the node's running state", + "default": "NOT_STARTED" + }, "position": { "title": "Position", "allOf": [ @@ -977,9 +994,6 @@ } ], "deprecated": true - }, - "state": { - "$ref": "#/components/schemas/RunningState" } }, "additionalProperties": false @@ -996,14 +1010,14 @@ "title": "Nodeuuid", "type": "string", "description": "The node to get the port output from", - "format": "uuid4", + "format": "uuid", "example": [ "da5068e0-8a8d-4fb9-9516-56e5ddaef15b" ] }, "output": { "title": "Output", - "pattern": "^(simcore)/(services)/(comp|dynamic|frontend)(/[^\\s/]+)+$", + "pattern": "^[-_a-zA-Z0-9]+$", "type": "string", "description": "The port key in the node given by nodeUuid", "example": [ @@ -1050,7 +1064,7 @@ "STARTED", "RETRY", "SUCCESS", - "FAILURE", + "FAILED", "ABORTED" ], "type": "string", @@ -1105,11 +1119,7 @@ "displayOrder": { "title": "Displayorder", "type": "number", - "description": "use this to numerically sort the properties for display", - "example": [ - 1, - -0.2 - ] + "description": "use this to numerically sort the properties for display" }, "label": { "title": "Label", @@ -1127,48 +1137,28 @@ "title": "Type", "pattern": "^(number|integer|boolean|string|data:([^/\\s,]+/[^/\\s,]+|\\[[^/\\s,]+/[^/\\s,]+(,[^/\\s]+/[^/,\\s]+)*\\]))$", "type": "string", - "description": "data type expected on this input glob matching for data type is allowed", - "example": [ - "number", - "boolean", - "data:*/*", - "data:text/*", - "data:[image/jpeg,image/png]", - "data:application/json", - "data:application/json;schema=https://my-schema/not/really/schema.json", - "data:application/vnd.ms-excel", - "data:text/plain", - "data:application/hdf5", - "data:application/edu.ucdavis@ceclancy.xyz" - ] + "description": "data type expected on this input glob matching for data type is allowed" }, "fileToKeyMap": { "title": "Filetokeymap", "type": "object", - "description": "Place the data associated with the named keys in files", - "example": [ - { - "dir/input1.txt": "key_1", - "dir33/input2.txt": "key2" - } - ] + "description": "Place the data associated with the named keys in files" }, "defaultValue": { "title": "Defaultvalue", "anyOf": [ { - "type": "string" + "type": "boolean" + }, + { + "type": "integer" }, { "type": "number" }, { - "type": "integer" + "type": "string" } - ], - "example": [ - "Dog", - true ] }, "widget": { @@ -1210,17 +1200,12 @@ "minLength": 1, "type": "string", "description": "url to the thumbnail", - "format": "uri", - "example": "https://user-images.githubusercontent.com/32800795/61083844-ff48fb00-a42c-11e9-8e63-fa2d709c8baf.png" + "format": "uri" }, "description": { "title": "Description", "type": "string", - "description": "human readable description of the purpose of the node", - "example": [ - "Our best node type", - "The mother of all nodes, makes your numbers shine!" - ] + "description": "human readable description of the purpose of the node" }, "classifiers": { "title": "Classifiers", @@ -1229,6 +1214,11 @@ "type": "string" } }, + "quality": { + "title": "Quality", + "type": "object", + "default": {} + }, "access_rights": { "title": "Access Rights", "type": "object", @@ -1239,33 +1229,29 @@ }, "key": { "title": "Key", - "pattern": "^(simcore)/(services)/(comp|dynamic|frontend)(/[^\\s/]+)+$", + "pattern": "^(simcore)/(services)/(comp|dynamic|frontend)(/[\\w/-]+)+$", "type": "string", - "description": "distinctive name for the node based on the docker registry path", - "example": [ - "simcore/services/comp/itis/sleeper", - "simcore/services/dynamic/3dviewer" - ] + "description": "distinctive name for the node based on the docker registry path" }, "version": { "title": "Version", "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$", "type": "string", - "description": "service version number", - "example": [ - "1.0.0", - "0.0.1" - ] + "description": "service version number" }, "integration-version": { "title": "Integration-Version", "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$", "type": "string", - "description": "integration version number", - "example": "1.0.0" + "description": "integration version number" }, "type": { - "$ref": "#/components/schemas/ServiceType" + "allOf": [ + { + "$ref": "#/components/schemas/ServiceType" + } + ], + "description": "service type" }, "badges": { "title": "Badges", @@ -1286,10 +1272,7 @@ "title": "Contact", "type": "string", "description": "email to correspond to the authors about the node", - "format": "email", - "example": [ - "lab@net.flix" - ] + "format": "email" }, "inputs": { "title": "Inputs", @@ -1307,7 +1290,8 @@ "format": "email" } }, - "additionalProperties": false + "additionalProperties": false, + "description": "Service base schema (used for docker labels on docker images)" }, "ServiceOutput": { "title": "ServiceOutput", @@ -1322,11 +1306,7 @@ "displayOrder": { "title": "Displayorder", "type": "number", - "description": "use this to numerically sort the properties for display", - "example": [ - 1, - -0.2 - ] + "description": "use this to numerically sort the properties for display" }, "label": { "title": "Label", @@ -1344,49 +1324,39 @@ "title": "Type", "pattern": "^(number|integer|boolean|string|data:([^/\\s,]+/[^/\\s,]+|\\[[^/\\s,]+/[^/\\s,]+(,[^/\\s]+/[^/,\\s]+)*\\]))$", "type": "string", - "description": "data type expected on this input glob matching for data type is allowed", - "example": [ - "number", - "boolean", - "data:*/*", - "data:text/*", - "data:[image/jpeg,image/png]", - "data:application/json", - "data:application/json;schema=https://my-schema/not/really/schema.json", - "data:application/vnd.ms-excel", - "data:text/plain", - "data:application/hdf5", - "data:application/edu.ucdavis@ceclancy.xyz" - ] + "description": "data type expected on this input glob matching for data type is allowed" }, "fileToKeyMap": { "title": "Filetokeymap", "type": "object", - "description": "Place the data associated with the named keys in files", - "example": [ - { - "dir/input1.txt": "key_1", - "dir33/input2.txt": "key2" - } - ] + "description": "Place the data associated with the named keys in files" }, "defaultValue": { "title": "Defaultvalue", "anyOf": [ { - "type": "string" + "type": "boolean" + }, + { + "type": "integer" }, { "type": "number" }, { - "type": "integer" + "type": "string" } - ], - "example": [ - "Dog", - true ] + }, + "widget": { + "title": "Widget", + "allOf": [ + { + "$ref": "#/components/schemas/Widget" + } + ], + "description": "custom widget to use instead of the default one determined from the data-type", + "deprecated": true } }, "additionalProperties": false @@ -1394,9 +1364,9 @@ "ServiceType": { "title": "ServiceType", "enum": [ - "frontend", "computational", - "dynamic" + "dynamic", + "frontend" ], "type": "string", "description": "An enumeration." @@ -1434,6 +1404,11 @@ "items": { "type": "string" } + }, + "quality": { + "title": "Quality", + "type": "object", + "default": {} } } }, @@ -1455,20 +1430,23 @@ "type": "integer" } ], - "description": "The store identifier, '0' or 0 for simcore S3, '1' or 1 for datcore", - "example": [ - "0", - 1 - ] + "description": "The store identifier, '0' or 0 for simcore S3, '1' or 1 for datcore" }, "path": { "title": "Path", + "pattern": "^.+$", "type": "string", - "description": "The path to the file in the storage provider domain", - "example": [ - "N:package:b05739ef-260c-4038-b47d-0240d04b0599", - "94453a6a-c8d4-52b3-a22d-ccbf81f8d636/d4442ca4-23fd-5b6b-ba6d-0b75f711c109/y_1D.txt" - ] + "description": "The path to the file in the storage provider domain" + }, + "eTag": { + "title": "Etag", + "type": "string", + "description": "Entity tag that uniquely represents the file. The method to generate the tag is not specified (black box)." + }, + "label": { + "title": "Label", + "type": "string", + "description": "The real file name" } }, "additionalProperties": false @@ -1553,7 +1531,12 @@ "type": "object", "properties": { "type": { - "$ref": "#/components/schemas/WidgetType" + "allOf": [ + { + "$ref": "#/components/schemas/WidgetType" + } + ], + "description": "type of the property" }, "details": { "title": "Details", diff --git a/services/catalog/tests/unit/with_dbs/docker-compose.yml b/services/catalog/tests/unit/with_dbs/docker-compose.yml index 2c19068e55d..81d720f6f94 100644 --- a/services/catalog/tests/unit/with_dbs/docker-compose.yml +++ b/services/catalog/tests/unit/with_dbs/docker-compose.yml @@ -1,7 +1,7 @@ version: "3.7" services: postgres: - image: postgres:10 + image: postgres:10.11@sha256:2aef165ab4f30fbb109e88959271d8b57489790ea13a77d27c02d8adb8feb20f environment: - POSTGRES_USER=${POSTGRES_USER:-test} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-test} diff --git a/services/director-v2/docker-compose-extra.yml b/services/director-v2/docker-compose-extra.yml index 01554db7f64..1fa4dacc3b4 100644 --- a/services/director-v2/docker-compose-extra.yml +++ b/services/director-v2/docker-compose-extra.yml @@ -1,7 +1,7 @@ version: "3.7" services: postgres: - image: postgres:10 + image: postgres:10.11@sha256:2aef165ab4f30fbb109e88959271d8b57489790ea13a77d27c02d8adb8feb20f init: true environment: - POSTGRES_USER=${POSTGRES_USER:-test} diff --git a/services/director-v2/src/simcore_service_director_v2/models/schemas/services.py b/services/director-v2/src/simcore_service_director_v2/models/schemas/services.py index b6e2bb4866d..463cf050833 100644 --- a/services/director-v2/src/simcore_service_director_v2/models/schemas/services.py +++ b/services/director-v2/src/simcore_service_director_v2/models/schemas/services.py @@ -73,7 +73,15 @@ class RunningServiceDetails(BaseModel): ) service_state: ServiceState = Field( ..., - description="the service state * 'pending' - The service is waiting for resources to start * 'pulling' - The service is being pulled from the registry * 'starting' - The service is starting * 'running' - The service is running * 'complete' - The service completed * 'failed' - The service failed to start\n", + description=( + "the service state" + " * 'pending' - The service is waiting for resources to start" + " * 'pulling' - The service is being pulled from the registry" + " * 'starting' - The service is starting" + " * 'running' - The service is running" + " * 'complete' - The service completed" + " * 'failed' - The service failed to start" + ), ) service_message: str = Field(..., description="the service message") diff --git a/services/storage/tests/docker-compose.yml b/services/storage/tests/docker-compose.yml index 2e30dc6bae5..5e347c4b466 100644 --- a/services/storage/tests/docker-compose.yml +++ b/services/storage/tests/docker-compose.yml @@ -1,7 +1,7 @@ version: "3.4" services: postgres: - image: postgres:10 + image: postgres:10.11@sha256:2aef165ab4f30fbb109e88959271d8b57489790ea13a77d27c02d8adb8feb20f restart: always environment: POSTGRES_DB: ${POSTGRES_DB:-aio_login_tests} diff --git a/tests/public-api/Makefile b/tests/public-api/Makefile new file mode 100644 index 00000000000..5d21e8c7ef4 --- /dev/null +++ b/tests/public-api/Makefile @@ -0,0 +1,41 @@ +# +# Targets for DEVELOPMENT of tests/public-api +# +include ../../scripts/common.Makefile +include ../../scripts/common-package.Makefile + +# MAIN ------------------ + +# Redirections to recipes in the main Makefile +.PHONY: leave build +leave build: + $(MAKE_C) $(REPO_BASE_DIR) $@ + + +# LOCAL ------------------ + +.PHONY: requirements +requirements: ## compiles pip requirements (.in -> .txt) + @$(MAKE_C) requirements reqs + + +.PHONY: install-dev install-prod install-ci +install-dev install-prod install-ci: _check_venv_active ## install app in development/production or CI mode + # installing in $(subst install-,,$@) mode + pip-sync requirements/$(subst install-,,$@).txt + + +.PHONY: test-dev +test-dev: ## runs tests with --keep-docker-up, --pdb and --ff + # WARNING: + # - do not forget to build latest changes images + # - this test can be affected by existing docker volumes in your host machine + # + # running unit tests + @pytest --keep-docker-up \ + -vv \ + --color=yes \ + --failed-first \ + --durations=10 \ + --pdb \ + $(CURDIR) diff --git a/tests/public-api/conftest.py b/tests/public-api/conftest.py new file mode 100644 index 00000000000..28ad0ff6caf --- /dev/null +++ b/tests/public-api/conftest.py @@ -0,0 +1,157 @@ +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name + +import logging +import os +import sys +from pathlib import Path +from pprint import pformat +from typing import Any, Dict + +import httpx +import osparc +import pytest +from osparc.configuration import Configuration +from tenacity import Retrying, before_sleep_log, stop_after_attempt, wait_fixed + +current_dir = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent +log = logging.getLogger(__name__) + +pytest_plugins = [ + "pytest_simcore.repository_paths", + "pytest_simcore.docker_compose", + "pytest_simcore.docker_swarm", + "pytest_simcore.docker_registry", + "pytest_simcore.schemas", +] + + +@pytest.fixture(scope="module") +def prepare_all_services( + simcore_docker_compose: Dict, + ops_docker_compose: Dict, + request, +) -> Dict: + + setattr( + request.module, "core_services", list(simcore_docker_compose["services"].keys()) + ) + core_services = getattr(request.module, "core_services", []) + + setattr(request.module, "ops_services", list(ops_docker_compose["services"].keys())) + ops_services = getattr(request.module, "ops_services", []) + + services = {"simcore": simcore_docker_compose, "ops": ops_docker_compose} + return services + + +@pytest.fixture(scope="module") +def services_registry(sleeper_service) -> Dict[str, Any]: + # See other service fixtures in + # packages/pytest-simcore/src/pytest_simcore/docker_registry.py + return { + "sleeper_service": { + "name": sleeper_service["image"]["name"], + "version": sleeper_service["image"]["tag"], + "schema": sleeper_service["schema"], + }, + # add here more + } + + +@pytest.fixture(scope="module") +def make_up_prod( + prepare_all_services: Dict, + simcore_docker_compose: Dict, + ops_docker_compose: Dict, + docker_stack: Dict, + services_registry, +) -> Dict: + + for attempt in Retrying( + wait=wait_fixed(5), + stop=stop_after_attempt(60), + reraise=True, + before_sleep=before_sleep_log(log, logging.INFO), + ): + with attempt: + resp = httpx.get("http://127.0.0.1:9081/v0/") + resp.raise_for_status() + + stack_configs = {"simcore": simcore_docker_compose, "ops": ops_docker_compose} + return stack_configs + + +@pytest.fixture(scope="module") +def registered_user(make_up_prod): + user = { + "email": "first.last@mymail.com", + "password": "my secret", + "api_key": None, + "api_secret": None, + } + + with httpx.Client(base_url="http://127.0.0.1:9081/v0") as client: + # setup user via web-api + resp = client.post( + "/auth/login", + json={ + "email": user["email"], + "password": user["password"], + }, + ) + + if resp.status_code != 200: + resp = client.post( + "/auth/register", + json={ + "email": user["email"], + "password": user["password"], + "confirm": user["password"], + }, + ) + resp.raise_for_status() + + # create a key via web-api + resp = client.post("/auth/api-keys", json={"display_name": "test-public-api"}) + + print(resp.text) + resp.raise_for_status() + + data = resp.json()["data"] + assert data["display_name"] == "test-public-api" + + user.update({"api_key": data["api_key"], "api_secret": data["api_secret"]}) + + yield user + + resp = client.request( + "DELETE", "/auth/api-keys", json={"display_name": "test-public-api"} + ) + + +@pytest.fixture +def api_client(registered_user): + cfg = Configuration( + host=os.environ.get("OSPARC_API_URL", "http://127.0.0.1:8006"), + username=registered_user["api_key"], + password=registered_user["api_secret"], + ) + + def as_dict(obj: object): + return { + attr: getattr(obj, attr) + for attr in obj.__dict__.keys() + if not attr.startswith("_") + } + + print("cfg", pformat(as_dict(cfg))) + + with osparc.ApiClient(cfg) as api_client: + yield api_client + + +@pytest.fixture() +def solvers_api(api_client): + return osparc.SolversApi(api_client) diff --git a/tests/public-api/requirements/Makefile b/tests/public-api/requirements/Makefile new file mode 100644 index 00000000000..3f25442b790 --- /dev/null +++ b/tests/public-api/requirements/Makefile @@ -0,0 +1,6 @@ +# +# Targets to pip-compile requirements +# +include ../../../requirements/base.Makefile + +# Add here any extra explicit dependency: e.g. _migration.txt: _base.txt diff --git a/tests/public-api/requirements/_base.txt b/tests/public-api/requirements/_base.txt new file mode 100644 index 00000000000..0eb14367cec --- /dev/null +++ b/tests/public-api/requirements/_base.txt @@ -0,0 +1,6 @@ +# NOTE: +# This file file is just here as placeholder +# to fulfill dependencies of _tools.txt target in requirements/base.Makefile +# +# This is a pure-tests project and all dependencies are added in _test.in +# diff --git a/tests/public-api/requirements/_test.in b/tests/public-api/requirements/_test.in new file mode 100644 index 00000000000..8c22e8d856e --- /dev/null +++ b/tests/public-api/requirements/_test.in @@ -0,0 +1,16 @@ +-c ../../../requirements/constraints.txt + +pytest +pytest-cov +pytest-asyncio + +osparc @ git+https://github.com/ITISFoundation/osparc-simcore-python-client.git@37641017e1222fe6a6e3adaf822ca009b096dd60 + +python-dotenv +httpx + +# pulled by pytest-simcore +docker +tenacity +jsonschema +pyyaml diff --git a/tests/public-api/requirements/_test.txt b/tests/public-api/requirements/_test.txt new file mode 100644 index 00000000000..e43effb3e46 --- /dev/null +++ b/tests/public-api/requirements/_test.txt @@ -0,0 +1,111 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --output-file=requirements/_test.txt requirements/_test.in +# +attrs==20.3.0 + # via + # jsonschema + # pytest +certifi==2020.12.5 + # via + # httpx + # osparc + # requests +chardet==3.0.4 + # via + # httpx + # requests +contextvars==2.4 + # via sniffio +coverage==5.4 + # via pytest-cov +docker==4.4.1 + # via -r requirements/_test.in +h11==0.9.0 + # via httpcore +httpcore==0.10.2 + # via httpx +httpx==0.14.3 + # via + # -c requirements/../../../requirements/constraints.txt + # -r requirements/_test.in +idna==2.10 + # via + # requests + # rfc3986 +immutables==0.14 + # via contextvars +importlib-metadata==3.4.0 + # via + # jsonschema + # pluggy + # pytest +iniconfig==1.1.1 + # via pytest +jsonschema==3.2.0 + # via -r requirements/_test.in +git+https://github.com/ITISFoundation/osparc-simcore-python-client.git@37641017e1222fe6a6e3adaf822ca009b096dd60 + # via -r requirements/_test.in +packaging==20.9 + # via pytest +pluggy==0.13.1 + # via pytest +py==1.10.0 + # via pytest +pyparsing==2.4.7 + # via packaging +pyrsistent==0.17.3 + # via jsonschema +pytest-asyncio==0.14.0 + # via -r requirements/_test.in +pytest-cov==2.11.1 + # via -r requirements/_test.in +pytest==6.2.2 + # via + # -r requirements/_test.in + # pytest-asyncio + # pytest-cov +python-dateutil==2.8.1 + # via osparc +python-dotenv==0.15.0 + # via -r requirements/_test.in +pyyaml==5.4.1 + # via + # -c requirements/../../../requirements/constraints.txt + # -r requirements/_test.in +requests==2.25.1 + # via docker +rfc3986[idna2008]==1.4.0 + # via httpx +six==1.15.0 + # via + # docker + # jsonschema + # osparc + # python-dateutil + # tenacity + # websocket-client +sniffio==1.2.0 + # via + # httpcore + # httpx +tenacity==6.3.1 + # via -r requirements/_test.in +toml==0.10.2 + # via pytest +typing-extensions==3.7.4.3 + # via importlib-metadata +urllib3==1.26.3 + # via + # -c requirements/../../../requirements/constraints.txt + # osparc + # requests +websocket-client==0.57.0 + # via docker +zipp==3.4.0 + # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/tests/public-api/requirements/_tools.in b/tests/public-api/requirements/_tools.in new file mode 100644 index 00000000000..e5f368f9ce8 --- /dev/null +++ b/tests/public-api/requirements/_tools.in @@ -0,0 +1,4 @@ +-c ../../../requirements/constraints.txt +-c _test.txt + +-r ../../../requirements/devenv.txt diff --git a/tests/public-api/requirements/_tools.txt b/tests/public-api/requirements/_tools.txt new file mode 100644 index 00000000000..2401d381cdb --- /dev/null +++ b/tests/public-api/requirements/_tools.txt @@ -0,0 +1,82 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --output-file=requirements/_tools.txt requirements/_tools.in +# +appdirs==1.4.4 + # via + # black + # virtualenv +black==20.8b1 + # via -r requirements/../../../requirements/devenv.txt +bump2version==1.0.1 + # via -r requirements/../../../requirements/devenv.txt +cfgv==3.2.0 + # via pre-commit +click==7.1.2 + # via + # black + # pip-tools +dataclasses==0.8 + # via black +distlib==0.3.1 + # via virtualenv +filelock==3.0.12 + # via virtualenv +identify==1.5.13 + # via pre-commit +importlib-metadata==3.4.0 + # via + # -c requirements/_test.txt + # pre-commit + # virtualenv +importlib-resources==5.1.0 + # via + # pre-commit + # virtualenv +isort==5.7.0 + # via -r requirements/../../../requirements/devenv.txt +mypy-extensions==0.4.3 + # via black +nodeenv==1.5.0 + # via pre-commit +pathspec==0.8.1 + # via black +pip-tools==5.5.0 + # via -r requirements/../../../requirements/devenv.txt +pre-commit==2.10.1 + # via -r requirements/../../../requirements/devenv.txt +pyyaml==5.4.1 + # via + # -c requirements/../../../requirements/constraints.txt + # -c requirements/_test.txt + # pre-commit +regex==2020.11.13 + # via black +six==1.15.0 + # via + # -c requirements/_test.txt + # virtualenv +toml==0.10.2 + # via + # -c requirements/_test.txt + # black + # pre-commit +typed-ast==1.4.2 + # via black +typing-extensions==3.7.4.3 + # via + # -c requirements/_test.txt + # black + # importlib-metadata +virtualenv==20.4.2 + # via pre-commit +zipp==3.4.0 + # via + # -c requirements/_test.txt + # importlib-metadata + # importlib-resources + +# The following packages are considered to be unsafe in a requirements file: +# pip diff --git a/tests/public-api/requirements/ci.txt b/tests/public-api/requirements/ci.txt new file mode 100644 index 00000000000..ccab455b688 --- /dev/null +++ b/tests/public-api/requirements/ci.txt @@ -0,0 +1,13 @@ +# Shortcut to install all packages for the contigous integration (CI) of 'models-library' +# +# - As ci.txt but w/ tests +# +# Usage: +# pip install -r requirements/ci.txt +# + +# installs base + tests requirements +-r _test.txt + +# installs this repo's packages +../../packages/pytest-simcore/ diff --git a/tests/public-api/requirements/dev.txt b/tests/public-api/requirements/dev.txt new file mode 100644 index 00000000000..73691dc070a --- /dev/null +++ b/tests/public-api/requirements/dev.txt @@ -0,0 +1,14 @@ +# Shortcut to install all packages needed to develop 'models-library' +# +# - As ci.txt but with current and repo packages in develop (edit) mode +# +# Usage: +# pip install -r requirements/dev.txt +# + +# installs base + tests requirements +-r _test.txt +-r _tools.txt + +# installs this repo's packages +-e ../../packages/pytest-simcore/ diff --git a/tests/public-api/test_files_api.py b/tests/public-api/test_files_api.py new file mode 100644 index 00000000000..d785fdeaa75 --- /dev/null +++ b/tests/public-api/test_files_api.py @@ -0,0 +1,61 @@ +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name + +import time +from pathlib import Path +from uuid import UUID + +import osparc +import pytest +from osparc.api.files_api import FilesApi +from osparc.models import FileMetadata + + +@pytest.fixture() +def files_api(api_client): + return FilesApi(api_client) + + +def test_upload_file(files_api: FilesApi, tmpdir): + input_path = Path(tmpdir) / "some-text-file.txt" + input_path.write_text("demo") + + input_file: FileMetadata = files_api.upload_file(file=input_path) + assert isinstance(input_file, FileMetadata) + time.sleep(2) # let time to upload to S3 + + assert input_file.filename == input_path.name + assert input_file.content_type == "text/plain" + assert UUID(input_file.file_id) + assert input_file.checksum + + same_file = files_api.get_file(input_file.file_id) + assert same_file == input_file + + # FIXME: for some reason, S3 takes produces different etags + # for the same file. Are we changing some bytes in the + # intermediate upload? Would it work the same avoiding that step + # and doing direct upload? + same_file = files_api.upload_file(file=input_path) + # FIXME: assert input_file.checksum == same_file.checksum + + +def test_upload_list_and_download(files_api: FilesApi, tmpdir): + input_path = Path(tmpdir) / "some-hdf5-file.h5" + input_path.write_bytes(b"demo but some other stuff as well") + + input_file: FileMetadata = files_api.upload_file(file=input_path) + assert isinstance(input_file, FileMetadata) + time.sleep(2) # let time to upload to S3 + + assert input_file.filename == input_path.name + + myfiles = files_api.list_files() + assert myfiles + assert all(isinstance(f, FileMetadata) for f in myfiles) + assert input_file in myfiles + + download_path: str = files_api.download_file(file_id=input_file.file_id) + print("Downloaded", Path(download_path).read_text()) + assert input_path.read_text() == Path(download_path).read_text() diff --git a/tests/public-api/test_jobs_api.py b/tests/public-api/test_jobs_api.py new file mode 100644 index 00000000000..e6e21f3bd90 --- /dev/null +++ b/tests/public-api/test_jobs_api.py @@ -0,0 +1,105 @@ +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name + +import time +from datetime import timedelta +from typing import Any, Dict, List + +import pytest +from osparc import ApiClient, JobsApi, SolversApi +from osparc.models import Job, JobOutput, JobStatus, Solver +from osparc.rest import ApiException + + +@pytest.fixture() +def jobs_api(api_client: ApiClient): + return JobsApi(api_client) + + +def test_create_job( + solvers_api: SolversApi, jobs_api: JobsApi, services_registry: Dict[str, Any] +): + + sleeper = services_registry["sleeper_service"] + + solver = solvers_api.get_solver_by_name_and_version( + solver_name=sleeper["name"], version=sleeper["version"] + ) + assert isinstance(solver, Solver) + + # requests resources for a job with given inputs + job = solvers_api.create_job(solver.id, job_input=[]) + assert isinstance(job, Job) + + assert job.id + assert job == jobs_api.get_job(job.id) + + # gets jobs granted for user with a given solver + solver_jobs = solvers_api.list_jobs(solver.id) + assert job in solver_jobs + + # I only have jobs from this solver ? + all_jobs = jobs_api.list_all_jobs() + assert len(solver_jobs) <= len(all_jobs) + assert all(job in all_jobs for job in solver_jobs) + + +def test_run_job( + solvers_api: SolversApi, jobs_api: JobsApi, services_registry: Dict[str, Any] +): + + sleeper = services_registry["sleeper_service"] + + solver = solvers_api.get_solver_by_name_and_version( + solver_name=sleeper["name"], version=sleeper["version"] + ) + assert isinstance(solver, Solver) + + # requests resources for a job with given inputs + job = solvers_api.create_job(solver.id, job_input=[]) + assert isinstance(job, Job) + + assert job.id + assert job == jobs_api.get_job(job.id) + + # let's do it! + status: JobStatus = jobs_api.start_job(job.id) + assert isinstance(status, JobStatus) + + assert status.state == "undefined" + assert status.progress == 0 + assert ( + job.created_at < status.submitted_at < (job.created_at + timedelta(seconds=2)) + ) + + # poll stop time-stamp + while not status.stopped_at: + time.sleep(0.5) + status: JobStatus = jobs_api.inspect_job(job.id) + assert isinstance(status, JobStatus) + + print("Solver progress", f"{status.progress}/100", flush=True) + + # done, either successfully or with failures! + assert status.progress == 100 + assert status.state in ["success", "failed"] + assert status.submitted_at < status.started_at + assert status.started_at < status.stopped_at + + # let's get the results + try: + outputs: List[JobOutput] = jobs_api.list_job_outputs(job.id) + assert outputs + + for output in outputs: + print(output) + assert isinstance(output, JobOutput) + + assert output.job_id == job.id + assert output == jobs_api.get_job_output(job.id, output.name) + + except ApiException as err: + assert ( + status.state == "failed" and err.status == 404 + ), f"No outputs if solver run failed {err}" diff --git a/tests/public-api/test_meta_api.py b/tests/public-api/test_meta_api.py new file mode 100644 index 00000000000..760bac5a722 --- /dev/null +++ b/tests/public-api/test_meta_api.py @@ -0,0 +1,24 @@ +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name + +import osparc +import pytest +from osparc.models import Meta + + +@pytest.fixture() +def meta_api(api_client): + return osparc.MetaApi(api_client) + + +def test_get_service_metadata(meta_api): + print("get Service Metadata", "-" * 10) + meta: Meta = meta_api.get_service_metadata() + print(meta) + assert isinstance(meta, Meta) + + meta, status_code, headers = meta_api.get_service_metadata_with_http_info() + + assert isinstance(meta, Meta) + assert status_code == 200 diff --git a/tests/public-api/test_solvers_api.py b/tests/public-api/test_solvers_api.py new file mode 100644 index 00000000000..41af699eb61 --- /dev/null +++ b/tests/public-api/test_solvers_api.py @@ -0,0 +1,70 @@ +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name + + +from http import HTTPStatus +from typing import Any, Dict, List + +import pytest +from osparc.api.solvers_api import SolversApi +from osparc.exceptions import ApiException +from osparc.models import Solver +from packaging.version import parse as parse_version + + +def test_get_latest_solver(solvers_api: SolversApi): + solvers: List[Solver] = solvers_api.list_solvers() + + latest = None + for solver in solvers: + if "sleeper" in solver.name: + assert isinstance(solver, Solver) + + if not latest: + latest = solver + + elif parse_version(latest.version) < parse_version(solver.version): + latest = solvers_api.get_solver(solver.id) + + print(latest) + assert latest + + assert ( + solvers_api.get_solver_by_name_and_version( + solver_name=latest.name, version="latest" + ) + == latest + ) + + +def test_get_solver(solvers_api: SolversApi, services_registry: Dict[str, Any]): + expected_name = services_registry["sleeper_service"]["name"] + expected_version = services_registry["sleeper_service"]["version"] + + solver = solvers_api.get_solver_by_name_and_version( + solver_name=expected_name, version=expected_version + ) + + assert solver.name == expected_name + assert solver.version == expected_version + + same_solver = solvers_api.get_solver(solver.id) + + assert same_solver.id == solver.id + assert same_solver.name == solver.name + assert same_solver.version == solver.version + + # FIXME: same uuid returns different maintener, title and description (probably bug in catalog since it shows "nodetails" tags) + assert solver == same_solver + + +def test_solvers_not_found(solvers_api): + + with pytest.raises(ApiException) as excinfo: + solvers_api.get_solver_by_name_and_version( + solver_name="simcore/services/comp/something-not-in-this-registry", + version="1.4.55", + ) + assert excinfo.value.status == HTTPStatus.NOT_FOUND # 404 + assert "not found" in excinfo.value.reason.lower() diff --git a/tests/public-api/test_users_api.py b/tests/public-api/test_users_api.py new file mode 100644 index 00000000000..e1c9617bc1f --- /dev/null +++ b/tests/public-api/test_users_api.py @@ -0,0 +1,50 @@ +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name + +import hashlib + +import pytest +from osparc import UsersApi +from osparc.models import Profile, ProfileUpdate, UserRoleEnum + + +@pytest.fixture() +def users_api(api_client): + return UsersApi(api_client) + + +@pytest.fixture +def expected_profile(registered_user): + email = registered_user["email"] + name, surname = email.split("@")[0].split(".") + + return { + "first_name": name.capitalize(), + "last_name": name.capitalize(), + "login": email, + "role": UserRoleEnum.USER, + "groups": { + "me": {"gid": "123", "label": "maxy", "description": "primary group"}, + "organizations": [], + "all": {"gid": "1", "label": "Everyone", "description": "all users"}, + }, + "gravatar_id": hashlib.md5(email.encode()).hexdigest(), # nosec + } + + +def test_get_user(users_api: UsersApi, expected_profile): + user: Profile = users_api.get_my_profile() + + # TODO: check all fields automatically + assert user.login == expected_profile["login"] + + +def test_update_user(users_api: UsersApi): + before: Profile = users_api.get_my_profile() + assert before.first_name != "Richard" + + after: Profile = users_api.update_my_profile(ProfileUpdate(first_name="Richard")) + assert after != before + assert after.first_name == "Richard" + assert after == users_api.get_my_profile()