diff --git a/.env-devel b/.env-devel index 2f80c7116ff..01c98f22962 100644 --- a/.env-devel +++ b/.env-devel @@ -14,10 +14,9 @@ POSTGRES_PORT=5432 POSTGRES_USER=scu RABBIT_HOST=rabbit -RABBIT_LOG_CHANNEL=comp.backend.channels.log +RABBIT_CHANNELS={"progress": "comp.backend.channels.progress", "log": "comp.backend.channels.log", "celery": {"result_backend": "rpc://"}} RABBIT_PASSWORD=adminadmin RABBIT_PORT=5672 -RABBIT_PROGRESS_CHANNEL=comp.backend.channels.progress RABBIT_USER=admin REDIS_HOST=redis diff --git a/.github/workflows/ci-release.yml b/.github/workflows/ci-release.yml index a163304fc8d..1f4958fb7e2 100644 --- a/.github/workflows/ci-release.yml +++ b/.github/workflows/ci-release.yml @@ -20,11 +20,13 @@ jobs: env: TAG_PREFIX: staging-github steps: - - uses: actions/checkout@v1 - - name: setup-docker - uses: docker-practice/actions-setup-docker@0.0.1 - - name: setup docker-compose - run: sudo ./ci/github/helpers/setup_docker_compose.bash + - uses: actions/checkout@v2 + - name: setup docker + run: | + sudo ./ci/github/helpers/setup_docker_compose.bash + ./ci/github/helpers/setup_docker_experimental.bash + ./ci/github/helpers/setup_docker_buildx.bash + echo ::set-env name=DOCKER_BUILDX::1 - name: set owner variable run: echo ::set-env name=OWNER::${GITHUB_REPOSITORY%/*} - name: set git tag diff --git a/.github/workflows/ci-testing-deploy.yml b/.github/workflows/ci-testing-deploy.yml index e2576d0f945..4f67b7e893a 100644 --- a/.github/workflows/ci-testing-deploy.yml +++ b/.github/workflows/ci-testing-deploy.yml @@ -32,11 +32,13 @@ jobs: os: [ubuntu-18.04] fail-fast: false steps: - - uses: actions/checkout@v1 - - name: setup-docker - uses: docker-practice/actions-setup-docker@0.0.1 - - name: setup docker-compose - run: sudo ./ci/github/helpers/setup_docker_compose.bash + - uses: actions/checkout@v2 + - name: setup docker + run: | + sudo ./ci/github/helpers/setup_docker_compose.bash + ./ci/github/helpers/setup_docker_experimental.bash + ./ci/github/helpers/setup_docker_buildx.bash + echo ::set-env name=DOCKER_BUILDX::1 - name: setup python environment uses: actions/setup-python@v1.1.1 with: @@ -64,11 +66,13 @@ jobs: os: [ubuntu-18.04] fail-fast: false steps: - - uses: actions/checkout@v1 - - name: setup-docker - uses: docker-practice/actions-setup-docker@0.0.1 - - name: setup docker-compose - run: sudo ./ci/github/helpers/setup_docker_compose.bash + - uses: actions/checkout@v2 + - name: setup docker + run: | + sudo ./ci/github/helpers/setup_docker_compose.bash + ./ci/github/helpers/setup_docker_experimental.bash + ./ci/github/helpers/setup_docker_buildx.bash + echo ::set-env name=DOCKER_BUILDX::1 - name: setup python environment uses: actions/setup-python@v1.1.1 with: @@ -110,11 +114,13 @@ jobs: os: [ubuntu-18.04] fail-fast: false steps: - - uses: actions/checkout@v1 - - name: setup-docker - uses: docker-practice/actions-setup-docker@0.0.1 - - name: setup docker-compose - run: sudo ./ci/github/helpers/setup_docker_compose.bash + - uses: actions/checkout@v2 + - name: setup docker + run: | + sudo ./ci/github/helpers/setup_docker_compose.bash + ./ci/github/helpers/setup_docker_experimental.bash + ./ci/github/helpers/setup_docker_buildx.bash + echo ::set-env name=DOCKER_BUILDX::1 - name: setup python environment uses: actions/setup-python@v1.1.1 with: @@ -156,11 +162,13 @@ jobs: os: [ubuntu-18.04] fail-fast: false steps: - - uses: actions/checkout@v1 - - name: setup-docker - uses: docker-practice/actions-setup-docker@0.0.1 - - name: setup docker-compose - run: sudo ./ci/github/helpers/setup_docker_compose.bash + - uses: actions/checkout@v2 + - name: setup docker + run: | + sudo ./ci/github/helpers/setup_docker_compose.bash + ./ci/github/helpers/setup_docker_experimental.bash + ./ci/github/helpers/setup_docker_buildx.bash + echo ::set-env name=DOCKER_BUILDX::1 - name: setup python environment uses: actions/setup-python@v1.1.1 with: @@ -193,6 +201,53 @@ jobs: name: unit_director_coverage path: codeclimate.unit_director_coverage.json + unit-test-sidecar: + name: Unit-testing sidecar + runs-on: ${{ matrix.os }} + strategy: + matrix: + python: [3.6] + os: [ubuntu-18.04] + fail-fast: false + steps: + - uses: actions/checkout@v2 + - name: setup docker + run: | + sudo ./ci/github/helpers/setup_docker_compose.bash + ./ci/github/helpers/setup_docker_experimental.bash + ./ci/github/helpers/setup_docker_buildx.bash + echo ::set-env name=DOCKER_BUILDX::1 + - name: setup python environment + uses: actions/setup-python@v1.1.1 + with: + python-version: ${{ matrix.python }} + - name: show system version + run: ./ci/helpers/show_system_versions.bash + - uses: actions/cache@v1 + name: getting cached data + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: install + run: ./ci/github/unit-testing/sidecar.bash install + - name: test + run: ./ci/github/unit-testing/sidecar.bash test + - uses: codecov/codecov-action@v1 + with: + token: ${{ env.CODECOV_TOKEN }} #required + flags: unittests #optional + - name: prepare codeclimate coverage file + run: | + curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter + chmod +x ./cc-test-reporter + ./cc-test-reporter format-coverage -t coverage.py -o codeclimate.unit_sidecar_coverage.json coverage.xml + - name: upload codeclimate coverage + uses: actions/upload-artifact@v1 + with: + name: unit_sidecar_coverage + path: codeclimate.unit_sidecar_coverage.json unit-test-frontend: name: Unit-testing frontend runs-on: ${{ matrix.os }} @@ -202,11 +257,13 @@ jobs: os: [ubuntu-18.04] fail-fast: false steps: - - uses: actions/checkout@v1 - - name: setup-docker - uses: docker-practice/actions-setup-docker@0.0.1 - - name: setup docker-compose - run: sudo ./ci/github/helpers/setup_docker_compose.bash + - uses: actions/checkout@v2 + - name: setup docker + run: | + sudo ./ci/github/helpers/setup_docker_compose.bash + ./ci/github/helpers/setup_docker_experimental.bash + ./ci/github/helpers/setup_docker_buildx.bash + echo ::set-env name=DOCKER_BUILDX::1 - uses: actions/setup-node@v1 with: node-version: ${{ matrix.node }} @@ -238,11 +295,13 @@ jobs: os: [ubuntu-18.04] fail-fast: false steps: - - uses: actions/checkout@v1 - - name: setup-docker - uses: docker-practice/actions-setup-docker@0.0.1 - - name: setup docker-compose - run: sudo ./ci/github/helpers/setup_docker_compose.bash + - uses: actions/checkout@v2 + - name: setup docker + run: | + sudo ./ci/github/helpers/setup_docker_compose.bash + ./ci/github/helpers/setup_docker_experimental.bash + ./ci/github/helpers/setup_docker_buildx.bash + echo ::set-env name=DOCKER_BUILDX::1 - name: setup python environment uses: actions/setup-python@v1.1.1 with: @@ -270,11 +329,13 @@ jobs: os: [ubuntu-18.04] fail-fast: false steps: - - uses: actions/checkout@v1 - - name: setup-docker - uses: docker-practice/actions-setup-docker@0.0.1 - - name: setup docker-compose - run: sudo ./ci/github/helpers/setup_docker_compose.bash + - uses: actions/checkout@v2 + - name: setup docker + run: | + sudo ./ci/github/helpers/setup_docker_compose.bash + ./ci/github/helpers/setup_docker_experimental.bash + ./ci/github/helpers/setup_docker_buildx.bash + echo ::set-env name=DOCKER_BUILDX::1 - name: setup python environment uses: actions/setup-python@v1.1.1 with: @@ -316,11 +377,13 @@ jobs: os: [ubuntu-18.04] fail-fast: false steps: - - uses: actions/checkout@v1 - - name: setup-docker - uses: docker-practice/actions-setup-docker@0.0.1 - - name: setup docker-compose - run: sudo ./ci/github/helpers/setup_docker_compose.bash + - uses: actions/checkout@v2 + - name: setup docker + run: | + sudo ./ci/github/helpers/setup_docker_compose.bash + ./ci/github/helpers/setup_docker_experimental.bash + ./ci/github/helpers/setup_docker_buildx.bash + echo ::set-env name=DOCKER_BUILDX::1 - name: setup python environment uses: actions/setup-python@v1.1.1 with: @@ -362,11 +425,13 @@ jobs: os: [ubuntu-18.04] fail-fast: false steps: - - uses: actions/checkout@v1 - - name: setup-docker - uses: docker-practice/actions-setup-docker@0.0.1 - - name: setup docker-compose - run: sudo ./ci/github/helpers/setup_docker_compose.bash + - uses: actions/checkout@v2 + - name: setup docker + run: | + sudo ./ci/github/helpers/setup_docker_compose.bash + ./ci/github/helpers/setup_docker_experimental.bash + ./ci/github/helpers/setup_docker_buildx.bash + echo ::set-env name=DOCKER_BUILDX::1 - name: setup python environment uses: actions/setup-python@v1.1.1 with: @@ -408,11 +473,13 @@ jobs: os: [ubuntu-18.04] fail-fast: false steps: - - uses: actions/checkout@v1 - - name: setup-docker - uses: docker-practice/actions-setup-docker@0.0.1 - - name: setup docker-compose - run: sudo ./ci/github/helpers/setup_docker_compose.bash + - uses: actions/checkout@v2 + - name: setup docker + run: | + sudo ./ci/github/helpers/setup_docker_compose.bash + ./ci/github/helpers/setup_docker_experimental.bash + ./ci/github/helpers/setup_docker_buildx.bash + echo ::set-env name=DOCKER_BUILDX::1 - name: setup python environment uses: actions/setup-python@v1.1.1 with: @@ -446,21 +513,29 @@ jobs: path: codeclimate.unit_webserver_coverage.json build-test-images: + # make PR faster by executing this one straight as PR cannot push to the registry anyway runs-on: ubuntu-18.04 name: build docker test images steps: - - uses: actions/checkout@v1 - - name: setup-docker - uses: docker-practice/actions-setup-docker@0.0.1 - - name: setup docker-compose - run: sudo ./ci/github/helpers/setup_docker_compose.bash + - uses: actions/checkout@v2 + - name: setup docker + if: github.event_name == 'push' + run: | + sudo ./ci/github/helpers/setup_docker_compose.bash + ./ci/github/helpers/setup_docker_experimental.bash + ./ci/github/helpers/setup_docker_buildx.bash + echo ::set-env name=DOCKER_BUILDX::1 - name: show system environs + if: github.event_name == 'push' run: ./ci/helpers/show_system_versions.bash - name: pull images + if: github.event_name == 'push' run: ./ci/build/test-images.bash pull_images - name: build images + if: github.event_name == 'push' run: ./ci/build/test-images.bash build_images - name: set owner variable + if: github.event_name == 'push' run: echo ::set-env name=OWNER::${GITHUB_REPOSITORY%/*} - name: push images # only pushes have access to the docker credentials @@ -483,11 +558,13 @@ jobs: run: | export TMP_DOCKER_REGISTRY=${GITHUB_REPOSITORY%/*} echo ::set-env name=DOCKER_REGISTRY::${TMP_DOCKER_REGISTRY,,} - - uses: actions/checkout@v1 - - name: setup-docker - uses: docker-practice/actions-setup-docker@0.0.1 - - name: setup docker-compose - run: sudo ./ci/github/helpers/setup_docker_compose.bash + - uses: actions/checkout@v2 + - name: setup docker + run: | + sudo ./ci/github/helpers/setup_docker_compose.bash + ./ci/github/helpers/setup_docker_experimental.bash + ./ci/github/helpers/setup_docker_buildx.bash + echo ::set-env name=DOCKER_BUILDX::1 - name: setup python environment uses: actions/setup-python@v1.1.1 with: @@ -523,6 +600,64 @@ jobs: name: integration_webserver_coverage path: codeclimate.integration_webserver_coverage.json + integration-test-sidecar: + name: Integration-testing sidecar + needs: [build-test-images] + runs-on: ${{ matrix.os }} + strategy: + matrix: + python: [3.6] + os: [ubuntu-18.04] + fail-fast: false + steps: + - name: set PR default variables + # only pushes have access to the docker credentials, use a default + if: github.event_name == 'pull_request' + run: | + export TMP_DOCKER_REGISTRY=${GITHUB_REPOSITORY%/*} + echo ::set-env name=DOCKER_REGISTRY::${TMP_DOCKER_REGISTRY,,} + - uses: actions/checkout@v2 + - name: setup docker + run: | + sudo ./ci/github/helpers/setup_docker_compose.bash + ./ci/github/helpers/setup_docker_experimental.bash + ./ci/github/helpers/setup_docker_buildx.bash + echo ::set-env name=DOCKER_BUILDX::1 + - name: setup python environment + uses: actions/setup-python@v1.1.1 + with: + python-version: ${{ matrix.python }} + - name: show system version + run: ./ci/helpers/show_system_versions.bash + - uses: actions/cache@v1 + name: getting cached data + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: install + run: ./ci/github/integration-testing/sidecar.bash install + - name: test + run: ./ci/github/integration-testing/sidecar.bash test + - name: cleanup + if: always() + run: ./ci/github/integration-testing/sidecar.bash clean_up + - uses: codecov/codecov-action@v1 + with: + token: ${{ env.CODECOV_TOKEN }} #required + flags: integrationtests #optional + - name: prepare codeclimate coverage file + run: | + curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter + chmod +x ./cc-test-reporter + ./cc-test-reporter format-coverage -t coverage.py -o codeclimate.integration_sidecar_coverage.json coverage.xml + - name: upload codeclimate coverage + uses: actions/upload-artifact@v1 + with: + name: integration_sidecar_coverage + path: codeclimate.integration_sidecar_coverage.json + integration-test-simcore-sdk: name: Integration-testing simcore-sdk needs: [build-test-images] @@ -539,11 +674,13 @@ jobs: run: | export TMP_DOCKER_REGISTRY=${GITHUB_REPOSITORY%/*} echo ::set-env name=DOCKER_REGISTRY::${TMP_DOCKER_REGISTRY,,} - - uses: actions/checkout@v1 - - name: setup-docker - uses: docker-practice/actions-setup-docker@0.0.1 - - name: setup docker-compose - run: sudo ./ci/github/helpers/setup_docker_compose.bash + - uses: actions/checkout@v2 + - name: setup docker + run: | + sudo ./ci/github/helpers/setup_docker_compose.bash + ./ci/github/helpers/setup_docker_experimental.bash + ./ci/github/helpers/setup_docker_buildx.bash + echo ::set-env name=DOCKER_BUILDX::1 - name: setup python environment uses: actions/setup-python@v1.1.1 with: @@ -595,11 +732,13 @@ jobs: run: | export TMP_DOCKER_REGISTRY=${GITHUB_REPOSITORY%/*} echo ::set-env name=DOCKER_REGISTRY::${TMP_DOCKER_REGISTRY,,} - - uses: actions/checkout@v1 - - name: setup-docker - uses: docker-practice/actions-setup-docker@0.0.1 - - name: setup docker-compose - run: sudo ./ci/github/helpers/setup_docker_compose.bash + - uses: actions/checkout@v2 + - name: setup docker + run: | + sudo ./ci/github/helpers/setup_docker_compose.bash + ./ci/github/helpers/setup_docker_experimental.bash + ./ci/github/helpers/setup_docker_buildx.bash + echo ::set-env name=DOCKER_BUILDX::1 - name: setup python environment uses: actions/setup-python@v1.1.1 with: @@ -638,11 +777,13 @@ jobs: run: | export TMP_DOCKER_REGISTRY=${GITHUB_REPOSITORY%/*} echo ::set-env name=DOCKER_REGISTRY::${TMP_DOCKER_REGISTRY,,} - - uses: actions/checkout@v1 - - name: setup-docker - uses: docker-practice/actions-setup-docker@0.0.1 - - name: setup docker-compose - run: sudo ./ci/github/helpers/setup_docker_compose.bash + - uses: actions/checkout@v2 + - name: setup docker + run: | + sudo ./ci/github/helpers/setup_docker_compose.bash + ./ci/github/helpers/setup_docker_experimental.bash + ./ci/github/helpers/setup_docker_buildx.bash + echo ::set-env name=DOCKER_BUILDX::1 - name: setup python environment uses: actions/setup-python@v1.1.1 with: @@ -693,20 +834,21 @@ jobs: coverage: needs: [ - unit-test-api-gateway, unit-test-catalog, unit-test-director, + unit-test-sidecar, unit-test-service-library, unit-test-simcore-sdk, unit-test-storage, unit-test-webserver, integration-test-webserver, + integration-test-sidecar, integration-test-simcore-sdk, ] name: coverage runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - uses: actions/download-artifact@v1 with: name: unit_api_gateway_coverage @@ -716,6 +858,9 @@ jobs: - uses: actions/download-artifact@v1 with: name: unit_director_coverage + - uses: actions/download-artifact@v1 + with: + name: unit_sidecar_coverage - uses: actions/download-artifact@v1 with: name: unit_servicelib_coverage @@ -731,6 +876,9 @@ jobs: - uses: actions/download-artifact@v1 with: name: integration_webserver_coverage + - uses: actions/download-artifact@v1 + with: + name: integration_sidecar_coverage - uses: actions/download-artifact@v1 with: name: integration_simcoresdk_coverage @@ -741,11 +889,13 @@ jobs: unit_api_gateway_coverage/*.json \ unit_catalog_coverage/*.json \ unit_director_coverage/*.json \ + unit_sidecar_coverage/*.json \ unit_servicelib_coverage/*.json \ unit_simcoresdk_coverage/*.json \ unit_storage_coverage/*.json \ unit_webserver_coverage/*.json \ integration_webserver_coverage/*.json \ + integration_sidecar_coverage/*.json \ integration_simcoresdk_coverage/*.json \ all_coverages/ ls -al all_coverages @@ -753,7 +903,7 @@ jobs: run: | curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter chmod +x ./cc-test-reporter - ./cc-test-reporter sum-coverage all_coverages/codeclimate.*.json --parts 9 + ./cc-test-reporter sum-coverage all_coverages/codeclimate.*.json --parts 11 - name: upload coverages run: | ./cc-test-reporter upload-coverage @@ -764,9 +914,9 @@ jobs: needs: [ unit-test-api, - unit-test-api-gateway, unit-test-catalog, unit-test-director, + unit-test-sidecar, unit-test-frontend, unit-test-python-linting, unit-test-service-library, @@ -774,17 +924,20 @@ jobs: unit-test-storage, unit-test-webserver, integration-test-webserver, + integration-test-sidecar, integration-test-simcore-sdk, system-test-swarm-deploy, system-test-e2e, ] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 - - name: setup-docker - uses: docker-practice/actions-setup-docker@0.0.1 - - name: setup docker-compose - run: sudo ./ci/github/helpers/setup_docker_compose.bash + - uses: actions/checkout@v2 + - name: setup docker + run: | + sudo ./ci/github/helpers/setup_docker_compose.bash + ./ci/github/helpers/setup_docker_experimental.bash + ./ci/github/helpers/setup_docker_buildx.bash + echo ::set-env name=DOCKER_BUILDX::1 - name: set owner variable run: echo ::set-env name=OWNER::${GITHUB_REPOSITORY%/*} - name: deploy master diff --git a/.travis.yml b/.travis.yml index b56afc97e67..8d619454f95 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,8 @@ dist: bionic env: global: - - DOCKER_COMPOSE_VERSION=1.25.0 + - DOCKER_COMPOSE_VERSION=1.25.3 - DOCKER_VERSION=5:19.03.5~3-0~ubuntu-bionic - # - DOCKER_BUILDKIT=1 - # - COMPOSE_DOCKER_CLI_BUILD=1 services: - docker addons: @@ -176,6 +174,27 @@ jobs: after_failure: - unbuffer bash ci/travis/unit-testing/director.bash after_failure + # test python, director ---------------------------------------------------------------------- + - stage: build / unit-testing + name: sidecar + language: python + python: + - "3.6" + sudo: required + cache: pip + before_install: + - sudo bash ci/travis/unit-testing/sidecar.bash before_install + install: + - unbuffer bash ci/travis/unit-testing/sidecar.bash install + before_script: + - unbuffer bash ci/travis/unit-testing/sidecar.bash before_script + script: + - unbuffer bash ci/travis/unit-testing/sidecar.bash script + after_success: + - unbuffer bash ci/travis/unit-testing/sidecar.bash after_success + after_failure: + - unbuffer bash ci/travis/unit-testing/sidecar.bash after_failure + # test python, service-library ---------------------------------------------------------------------- - stage: build / unit-testing name: service-library @@ -303,6 +322,27 @@ jobs: after_failure: - unbuffer bash ci/travis/integration-testing/webserver.bash after_failure + # integrate sidecar in swarm ------------------------------------------------------------- + - stage: integration-testing / system-testing + name: sidecar in swarm + language: python + python: + - "3.6" + sudo: required + cache: pip + before_install: + - sudo bash ci/travis/integration-testing/sidecar.bash before_install + install: + - unbuffer bash ci/travis/integration-testing/sidecar.bash install + before_script: + - unbuffer bash ci/travis/integration-testing/sidecar.bash before_script + script: + - unbuffer bash ci/travis/integration-testing/sidecar.bash script + after_success: + - unbuffer bash ci/travis/integration-testing/sidecar.bash after_success + after_failure: + - unbuffer bash ci/travis/integration-testing/sidecar.bash after_failure + # integrate simcore-sdk in swarm ------------------------------------------------------------- - stage: integration-testing / system-testing name: simcore-sdk in swarm diff --git a/Makefile b/Makefile index c116bd38d81..be2a46a8652 100644 --- a/Makefile +++ b/Makefile @@ -26,6 +26,7 @@ SHELL := /bin/bash # TODO: read from docker-compose file instead $(shell find $(CURDIR)/services -type f -name 'Dockerfile') # or $(notdir $(subst /Dockerfile,,$(wildcard services/*/Dockerfile))) ... SERVICES_LIST := \ + api-gateway \ catalog \ director \ sidecar \ @@ -71,52 +72,67 @@ endif # SWARM_HOSTS = $(shell docker node ls --format="{{.Hostname}}" 2>$(if $(IS_WIN),NUL,/dev/null)) -.PHONY: build build-nc rebuild build-devel build-devel-nc build-cache build-cache-nc +.PHONY: build build-nc rebuild build-devel build-devel-nc build-devel-kit build-devel-x build-cache build-cache-kit build-cache-x build-cache-nc build-kit build-x define _docker_compose_build export BUILD_TARGET=$(if $(findstring -devel,$@),development,$(if $(findstring -cache,$@),cache,production)); \ -docker-compose -f services/docker-compose-build.yml build $(if $(findstring -nc,$@),--no-cache,) +$(if $(findstring -x,$@),\ + pushd services; docker buildx bake --file docker-compose-build.yml; popd;,\ + docker-compose -f services/docker-compose-build.yml build $(if $(findstring -nc,$@),--no-cache,) --parallel;\ +) endef rebuild: build-nc # alias -build build-nc: .env ## Builds production images and tags them as 'local/{service-name}:production'. For single target e.g. 'make target=webserver build' -ifeq ($(target),) +build build-nc build-kit build-x: .env ## Builds production images and tags them as 'local/{service-name}:production'. For single target e.g. 'make target=webserver build' +ifeq ($(target),) # Compiling front-end - @$(MAKE) -C services/web/client compile + + @$(if $(findstring -kit,$@),export DOCKER_BUILDKIT=1;export COMPOSE_DOCKER_CLI_BUILD=1;,) \ + $(MAKE) -C services/web/client compile$(if $(findstring -x,$@),-x,) + # Building services - $(_docker_compose_build) --parallel + @$(if $(findstring -kit,$@),export DOCKER_BUILDKIT=1;export COMPOSE_DOCKER_CLI_BUILD=1;,) \ + $(_docker_compose_build) else ifeq ($(findstring webserver,$(target)),webserver) # Compiling front-end - @$(MAKE) -C services/web/client clean compile + @$(if $(findstring -kit,$@),export DOCKER_BUILDKIT=1;export COMPOSE_DOCKER_CLI_BUILD=1;,) \ + $(MAKE) -C services/web/client clean compile$(if $(findstring -x,$@),-x,) endif # Building service $(target) + @$(if $(findstring -kit,$@),export DOCKER_BUILDKIT=1;export COMPOSE_DOCKER_CLI_BUILD=1;,) \ $(_docker_compose_build) $(target) endif -build-devel build-devel-nc: .env ## Builds development images and tags them as 'local/{service-name}:development'. For single target e.g. 'make target=webserver build-devel' +build-devel build-devel-nc build-devel-kit build-devel-x: .env ## Builds development images and tags them as 'local/{service-name}:development'. For single target e.g. 'make target=webserver build-devel' ifeq ($(target),) # Building services - $(_docker_compose_build) --parallel + @$(if $(findstring -kit,$@),export DOCKER_BUILDKIT=1;export COMPOSE_DOCKER_CLI_BUILD=1;,) \ + $(_docker_compose_build) else ifeq ($(findstring webserver,$(target)),webserver) # Compiling front-end - @$(MAKE) -C services/web/client touch compile-dev + @$(if $(findstring -kit,$@),export DOCKER_BUILDKIT=1;export COMPOSE_DOCKER_CLI_BUILD=1;,) \ + $(MAKE) -C services/web/client touch$(if $(findstring -x,$@),-x,) compile-dev endif # Building service $(target) + @$(if $(findstring -kit,$@),export DOCKER_BUILDKIT=1;export COMPOSE_DOCKER_CLI_BUILD=1;,) \ $(_docker_compose_build) $(target) endif # TODO: should download cache if any?? -build-cache build-cache-nc: .env ## Build cache images and tags them as 'local/{service-name}:cache' +build-cache build-cache-nc build-cache-kit build-cache-x: .env ## Build cache images and tags them as 'local/{service-name}:cache' ifeq ($(target),) # Compiling front-end - @$(MAKE) -C services/web/client compile + @$(if $(findstring -kit,$@),export DOCKER_BUILDKIT=1;export COMPOSE_DOCKER_CLI_BUILD=1;,) + $(MAKE) -C services/web/client compile$(if $(findstring -x,$@),-x,) # Building cache images - $(_docker_compose_build) --parallel + @$(if $(findstring -kit,$@),export DOCKER_BUILDKIT=1;export COMPOSE_DOCKER_CLI_BUILD=1;,) + $(_docker_compose_build) else + @$(if $(findstring -kit,$@),export DOCKER_BUILDKIT=1;export COMPOSE_DOCKER_CLI_BUILD=1;,) $(_docker_compose_build) $(target) endif diff --git a/ci/build/test-images.bash b/ci/build/test-images.bash index b0d570eb7c5..7058fd9095d 100755 --- a/ci/build/test-images.bash +++ b/ci/build/test-images.bash @@ -23,8 +23,13 @@ pull_images() { } build_images() { - make build-cache - make build + if [[ -v DOCKER_BUILDX ]]; then + make build-cache-x + make build-x + else + make build-cache + make build + fi make info-images } diff --git a/ci/github/helpers/setup_docker_buildx.bash b/ci/github/helpers/setup_docker_buildx.bash new file mode 100755 index 00000000000..4906d93eab1 --- /dev/null +++ b/ci/github/helpers/setup_docker_buildx.bash @@ -0,0 +1,9 @@ +#!/bin/bash +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -euo pipefail +IFS=$'\n\t' + +mkdir --parents ~/.docker/cli-plugins/ +DOCKER_BUILDX="0.3.1" +curl --location https://github.com/docker/buildx/releases/download/v${DOCKER_BUILDX}/buildx-v${DOCKER_BUILDX}.linux-amd64 --output ~/.docker/cli-plugins/docker-buildx +chmod a+x ~/.docker/cli-plugins/docker-buildx \ No newline at end of file diff --git a/ci/github/helpers/setup_docker_compose.bash b/ci/github/helpers/setup_docker_compose.bash index bc39997ee5b..62209060d0e 100755 --- a/ci/github/helpers/setup_docker_compose.bash +++ b/ci/github/helpers/setup_docker_compose.bash @@ -3,6 +3,6 @@ set -euo pipefail IFS=$'\n\t' -DOCKER_COMPOSE_VERSION="1.25.0" +DOCKER_COMPOSE_VERSION="1.25.3" curl -L "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose chmod +x /usr/local/bin/docker-compose \ No newline at end of file diff --git a/ci/github/helpers/setup_docker_experimental.bash b/ci/github/helpers/setup_docker_experimental.bash new file mode 100755 index 00000000000..9ee3b9239cd --- /dev/null +++ b/ci/github/helpers/setup_docker_experimental.bash @@ -0,0 +1,7 @@ +#!/bin/bash +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -euo pipefail +IFS=$'\n\t' + +mkdir --parents ~/.docker/ +echo '{"experimental":"enabled"}' > ~/.docker/config.json \ No newline at end of file diff --git a/ci/github/integration-testing/sidecar.bash b/ci/github/integration-testing/sidecar.bash new file mode 100755 index 00000000000..e56531d7b87 --- /dev/null +++ b/ci/github/integration-testing/sidecar.bash @@ -0,0 +1,38 @@ +#!/bin/bash +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -euo pipefail +IFS=$'\n\t' + +# in case it's a Pull request, the env are never available, default to itisfoundation to get a maybe not too old version for caching +DOCKER_IMAGE_TAG=$(exec ci/helpers/build_docker_image_tag.bash) +export DOCKER_IMAGE_TAG + +install() { + bash ci/helpers/ensure_python_pip.bash + pushd services/sidecar; pip3 install -r requirements/ci.txt; popd; + pip list -v + make pull-version || ( (make pull-cache || true) && make build-x tag-version) + make info-images +} + +test() { + pytest --cov=simcore_service_sidecar --durations=10 --cov-append \ + --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \ + -v -m "not travis" services/sidecar/tests/integration +} + +clean_up() { + docker images + make down +} + +# Check if the function exists (bash specific) +if declare -f "$1" > /dev/null +then + # call arguments verbatim + "$@" +else + # Show a helpful error + echo "'$1' is not a known function name" >&2 + exit 1 +fi diff --git a/ci/github/integration-testing/simcore-sdk.bash b/ci/github/integration-testing/simcore-sdk.bash index ab9fd5dd069..c5b76799d54 100755 --- a/ci/github/integration-testing/simcore-sdk.bash +++ b/ci/github/integration-testing/simcore-sdk.bash @@ -11,7 +11,7 @@ install() { pushd packages/simcore-sdk; pip3 install -r requirements/ci.txt; popd; pip list -v # pull the test images if registry is set up, else build the images - make pull-version || ( (make pull-cache || true) && make build tag-version) + make pull-version || ( (make pull-cache || true) && make build-x tag-version) make info-images # pip3 install services/storage/client-sdk/python } diff --git a/ci/github/integration-testing/webserver.bash b/ci/github/integration-testing/webserver.bash index f0f5fc50eee..5e07fe5b870 100755 --- a/ci/github/integration-testing/webserver.bash +++ b/ci/github/integration-testing/webserver.bash @@ -11,7 +11,7 @@ install() { bash ci/helpers/ensure_python_pip.bash pushd services/web/server; pip3 install -r requirements/ci.txt; popd; pip list -v - make pull-version || ( (make pull-cache || true) && make build tag-version) + make pull-version || ( (make pull-cache || true) && make build-x tag-version) make info-images } diff --git a/ci/github/system-testing/e2e.bash b/ci/github/system-testing/e2e.bash index f2d8fd3b635..3749c0db734 100755 --- a/ci/github/system-testing/e2e.bash +++ b/ci/github/system-testing/e2e.bash @@ -12,7 +12,7 @@ install() { echo "--------------- installing psql client..." /bin/bash -c 'sudo apt install -y postgresql-client' echo "--------------- getting simcore docker images..." - make pull-version || ( (make pull-cache || true) && make build tag-version) + make pull-version || ( (make pull-cache || true) && make build-x tag-version) make info-images # configure simcore for testing with a private registry diff --git a/ci/github/system-testing/swarm-deploy.bash b/ci/github/system-testing/swarm-deploy.bash index bf4c509cda7..cd880973e5d 100755 --- a/ci/github/system-testing/swarm-deploy.bash +++ b/ci/github/system-testing/swarm-deploy.bash @@ -15,8 +15,8 @@ export DOCKER_IMAGE_TAG install() { bash ci/helpers/ensure_python_pip.bash - pip3 install -r tests/swarm-deploy/requirements.txt - make pull-version || ( (make pull-cache || true) && make build tag-version) + pushd tests/swarm-deploy; pip3 install -r requirements/ci.txt; popd + make pull-version || ( (make pull-cache || true) && make build-x tag-version) make .env echo WEBSERVER_DB_INITTABLES=1 >> .env pip list -v diff --git a/ci/github/unit-testing/sidecar.bash b/ci/github/unit-testing/sidecar.bash index d2497350c0d..fb3f3eb332f 100755 --- a/ci/github/unit-testing/sidecar.bash +++ b/ci/github/unit-testing/sidecar.bash @@ -10,10 +10,9 @@ install() { } test() { - # TODO: call services/sidecar/tests/unit when integration tests available pytest --cov=simcore_service_sidecar --durations=10 --cov-append \ --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \ - -v -m "not travis" services/sidecar/tests + -v -m "not travis" services/sidecar/tests/unit } # Check if the function exists (bash specific) diff --git a/ci/helpers/show_system_versions.bash b/ci/helpers/show_system_versions.bash index 1e6b175ba7f..c2a3b863b24 100755 --- a/ci/helpers/show_system_versions.bash +++ b/ci/helpers/show_system_versions.bash @@ -22,7 +22,7 @@ fi echo "------------------------------ docker -----------------------------------" if command -v docker; then - docker -v + docker version fi echo "------------------------------ docker-compose -----------------------------------" diff --git a/ci/travis/integration-testing/sidecar.bash b/ci/travis/integration-testing/sidecar.bash new file mode 100755 index 00000000000..523eaedc4d3 --- /dev/null +++ b/ci/travis/integration-testing/sidecar.bash @@ -0,0 +1,55 @@ +#!/bin/bash +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -euo pipefail +IFS=$'\n\t' + +# in case it's a Pull request, the env are never available, default to itisfoundation to get a maybe not too old version for caching +DOCKER_IMAGE_TAG=$(exec ci/helpers/build_docker_image_tag.bash) +export DOCKER_IMAGE_TAG + +if [[ ! -v DOCKER_REGISTRY ]]; then + export DOCKER_REGISTRY="itisfoundation" +fi + +before_install() { + bash ci/travis/helpers/update-docker.bash + bash ci/travis/helpers/install-docker-compose.bash + bash ci/helpers/show_system_versions.bash +} + +install() { + bash ci/helpers/ensure_python_pip.bash + pushd services/sidecar; pip3 install -r requirements/ci.txt; popd; +} + +before_script() { + pip list -v + make pull-version || ( (make pull-cache || true) && make build tag-version) + make info-images +} + +script() { + pytest --cov=simcore_service_sidecar --durations=10 --cov-append \ + --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \ + -v -m "not travis" services/sidecar/tests/integration +} + +after_success() { + coveralls +} + +after_failure() { + docker images + make down +} + +# Check if the function exists (bash specific) +if declare -f "$1" > /dev/null +then + # call arguments verbatim + "$@" +else + # Show a helpful error + echo "'$1' is not a known function name" >&2 + exit 1 +fi diff --git a/ci/travis/integration-testing/simcore-sdk.bash b/ci/travis/integration-testing/simcore-sdk.bash index e0352a31866..ce21b26f275 100755 --- a/ci/travis/integration-testing/simcore-sdk.bash +++ b/ci/travis/integration-testing/simcore-sdk.bash @@ -8,6 +8,11 @@ export DOCKER_IMAGE_TAG FOLDER_CHECKS=(packages/ simcore-sdk storage/ simcore-sdk .travis.yml) +if [[ ! -v DOCKER_REGISTRY ]]; then + export DOCKER_REGISTRY="itisfoundation" +fi + + before_install() { if bash ci/travis/helpers/test-for-changes.bash "${FOLDER_CHECKS[@]}"; then diff --git a/ci/travis/integration-testing/webserver.bash b/ci/travis/integration-testing/webserver.bash index 939dce244d6..a69921d021e 100755 --- a/ci/travis/integration-testing/webserver.bash +++ b/ci/travis/integration-testing/webserver.bash @@ -7,6 +7,11 @@ IFS=$'\n\t' DOCKER_IMAGE_TAG=$(exec ci/helpers/build_docker_image_tag.bash) export DOCKER_IMAGE_TAG +if [[ ! -v DOCKER_REGISTRY ]]; then + export DOCKER_REGISTRY="itisfoundation" +fi + + before_install() { bash ci/travis/helpers/update-docker.bash bash ci/travis/helpers/install-docker-compose.bash diff --git a/ci/travis/system-testing/e2e.bash b/ci/travis/system-testing/e2e.bash index 06655cc3de2..6cf65d4ccf8 100644 --- a/ci/travis/system-testing/e2e.bash +++ b/ci/travis/system-testing/e2e.bash @@ -8,6 +8,11 @@ IFS=$'\n\t' DOCKER_IMAGE_TAG=$(exec ci/helpers/build_docker_image_tag.bash) export DOCKER_IMAGE_TAG +if [[ ! -v DOCKER_REGISTRY ]]; then + export DOCKER_REGISTRY="itisfoundation" +fi + + before_install() { bash ci/travis/helpers/update-docker.bash bash ci/travis/helpers/install-docker-compose.bash diff --git a/ci/travis/system-testing/swarm-deploy.bash b/ci/travis/system-testing/swarm-deploy.bash index 6a4699a8ca5..7ff3e3f3a10 100755 --- a/ci/travis/system-testing/swarm-deploy.bash +++ b/ci/travis/system-testing/swarm-deploy.bash @@ -13,6 +13,11 @@ IFS=$'\n\t' DOCKER_IMAGE_TAG=$(exec ci/helpers/build_docker_image_tag.bash) export DOCKER_IMAGE_TAG +if [[ ! -v DOCKER_REGISTRY ]]; then + export DOCKER_REGISTRY="itisfoundation" +fi + + before_install() { bash ci/travis/helpers/update-docker.bash bash ci/travis/helpers/install-docker-compose.bash @@ -21,7 +26,7 @@ before_install() { install() { bash ci/helpers/ensure_python_pip.bash - pip3 install -r tests/swarm-deploy/requirements.txt + pushd tests/swarm-deploy; pip3 install -r requirements/ci.txt; popd make pull-version || ( (make pull-cache || true) && make build tag-version) make .env echo WEBSERVER_DB_INITTABLES=1 >> .env diff --git a/ci/travis/unit-testing/sidecar.bash b/ci/travis/unit-testing/sidecar.bash index e46b2be3919..41d089b6e26 100755 --- a/ci/travis/unit-testing/sidecar.bash +++ b/ci/travis/unit-testing/sidecar.bash @@ -32,10 +32,9 @@ before_script() { script() { if bash ci/travis/helpers/test-for-changes.bash "${FOLDER_CHECKS[@]}"; then - # TODO: call services/sidecar/tests/unit when integration tests available pytest --cov=simcore_service_sidecar --durations=10 --cov-append \ --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \ - -v -m "not travis" services/sidecar/tests + -v -m "not travis" services/sidecar/tests/unit else echo "No changes detected. Skipping unit-testing of sidecar." fi diff --git a/packages/postgres-database/src/simcore_postgres_database/sidecar_models.py b/packages/postgres-database/src/simcore_postgres_database/sidecar_models.py new file mode 100644 index 00000000000..245005cd874 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/sidecar_models.py @@ -0,0 +1,27 @@ +""" Facade for sidecar service + + Facade to direct access to models in the database by + the sidecar service + +""" +from .models.comp_pipeline import ( + comp_pipeline, + UNKNOWN, + FAILED, + PENDING, + RUNNING, + SUCCESS, +) +from .models.comp_tasks import comp_tasks + + +__all__ = [ + "comp_tasks", + "comp_pipeline", + "comp_pipeline", + "UNKNOWN", + "FAILED", + "PENDING", + "RUNNING", + "SUCCESS", +] diff --git a/packages/pytest-simcore/src/pytest_simcore/docker_compose.py b/packages/pytest-simcore/src/pytest_simcore/docker_compose.py index 7a3b0d35e72..3f942620e3f 100644 --- a/packages/pytest-simcore/src/pytest_simcore/docker_compose.py +++ b/packages/pytest-simcore/src/pytest_simcore/docker_compose.py @@ -77,9 +77,23 @@ def env_file(osparc_simcore_root_dir: Path, devel_environ: Dict[str, str]) -> Pa backup_path.unlink() +@pytest.fixture(scope="module") +def make_up_prod_environ(): + old_env = deepcopy(os.environ) + if not "DOCKER_REGISTRY" in os.environ: + os.environ["DOCKER_REGISTRY"] = "local" + if not "DOCKER_IMAGE_TAG" in os.environ: + os.environ["DOCKER_IMAGE_TAG"] = "production" + yield + os.environ = old_env + + @pytest.fixture("module") def simcore_docker_compose( - osparc_simcore_root_dir: Path, env_file: Path, temp_folder: Path + osparc_simcore_root_dir: Path, + env_file: Path, + temp_folder: Path, + make_up_prod_environ, ) -> Dict: """ Resolves docker-compose for simcore stack in local host diff --git a/packages/pytest-simcore/src/pytest_simcore/docker_registry.py b/packages/pytest-simcore/src/pytest_simcore/docker_registry.py index 24e7249e07f..2f3ec614370 100644 --- a/packages/pytest-simcore/src/pytest_simcore/docker_registry.py +++ b/packages/pytest-simcore/src/pytest_simcore/docker_registry.py @@ -3,6 +3,7 @@ # pylint:disable=redefined-outer-name import json +import logging import os import time from typing import Dict @@ -11,6 +12,8 @@ import pytest import tenacity +log = logging.getLogger(__name__) + @pytest.fixture(scope="session") def docker_registry(keep_docker_up: bool) -> str: @@ -24,7 +27,7 @@ def docker_registry(keep_docker_up: bool) -> str: try: docker_client.login(registry=url, username="simcore") container = docker_client.containers.list({"name": "pytest_registry"})[0] - except Exception: # pylint: disable=broad-except + except Exception: # pylint: disable=broad-except print("Warning: docker registry is already up!") container = docker_client.containers.run( "registry:2", @@ -36,7 +39,7 @@ def docker_registry(keep_docker_up: bool) -> str: ) # Wait until we can connect - assert _wait_till_registry_is_responsive(url) + assert wait_till_registry_is_responsive(url) # get the hello world example from docker hub hello_world_image = docker_client.images.pull("hello-world", "latest") @@ -63,13 +66,19 @@ def docker_registry(keep_docker_up: bool) -> str: if not keep_docker_up: container.stop() + container.remove(force=True) while docker_client.containers.list(filters={"name": container.name}): time.sleep(1) -@tenacity.retry(wait=tenacity.wait_fixed(1), stop=tenacity.stop_after_delay(60)) -def _wait_till_registry_is_responsive(url: str) -> bool: +@tenacity.retry( + wait=tenacity.wait_fixed(2), + stop=tenacity.stop_after_delay(20), + before_sleep=tenacity.before_sleep_log(log, logging.INFO), + reraise=True, +) +def wait_till_registry_is_responsive(url: str) -> bool: docker_client = docker.from_env() docker_client.login(registry=url, username="simcore") return True @@ -98,7 +107,7 @@ def sleeper_service(docker_registry: str) -> Dict[str, str]: for key, value in image_labels.items() if key.startswith("io.simcore.") }, - "image": repo, + "image": {"name": "simcore/services/comp/itis/sleeper", "tag": TAG}, } @@ -133,5 +142,5 @@ def jupyter_service(docker_registry: str) -> Dict[str, str]: for key, value in image_labels.items() if key.startswith("io.simcore.") }, - "image": repo, + "image": {"name": f"simcore/services/dynamic/{image_name}", "tag": f"{tag}"}, } diff --git a/packages/pytest-simcore/src/pytest_simcore/docker_swarm.py b/packages/pytest-simcore/src/pytest_simcore/docker_swarm.py index 1326ba64725..cd092d449c3 100644 --- a/packages/pytest-simcore/src/pytest_simcore/docker_swarm.py +++ b/packages/pytest-simcore/src/pytest_simcore/docker_swarm.py @@ -12,6 +12,7 @@ import pytest import tenacity from servicelib.simcore_service_utils import SimcoreRetryPolicyUponInitialization +from .helpers.utils_docker import get_ip @pytest.fixture(scope="session") @@ -20,21 +21,24 @@ def docker_client() -> docker.client.DockerClient: yield client +@pytest.fixture(scope="session") +def keep_docker_up(request) -> bool: + return request.config.getoption("--keep-docker-up") + + @pytest.fixture(scope="module") -def docker_swarm(docker_client: docker.client.DockerClient) -> None: +def docker_swarm( + docker_client: docker.client.DockerClient, keep_docker_up: bool +) -> None: try: docker_client.swarm.reload() print("CAUTION: Already part of a swarm") yield except docker.errors.APIError: - docker_client.swarm.init() + docker_client.swarm.init(advertise_addr=get_ip()) yield - assert docker_client.swarm.leave(force=True) - - -@pytest.fixture(scope="session") -def keep_docker_up(request) -> bool: - return request.config.getoption("--keep-docker-up") == True + if not keep_docker_up: + assert docker_client.swarm.leave(force=True) @tenacity.retry(**SimcoreRetryPolicyUponInitialization().kwargs) @@ -47,7 +51,7 @@ def _wait_for_services(docker_client: docker.client.DockerClient) -> None: task = service.tasks()[0] if task["Status"]["State"].upper() not in pre_states: if not task["Status"]["State"].upper() == "RUNNING": - raise Exception(f"service {service} not running") + raise Exception(f"service {service.name} not running") def _print_services(docker_client: docker.client.DockerClient, msg: str) -> None: diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/utils_docker.py b/packages/pytest-simcore/src/pytest_simcore/helpers/utils_docker.py index 0818d869d75..f3cfd2c6242 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/utils_docker.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/utils_docker.py @@ -1,5 +1,6 @@ import logging import os +import socket import subprocess import tempfile from pathlib import Path @@ -12,6 +13,19 @@ log = logging.getLogger(__name__) +def get_ip() -> str: + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + try: + # doesn't even have to be reachable + s.connect(("10.255.255.255", 1)) + IP = s.getsockname()[0] + except Exception: # pylint: disable=W0703 + IP = "127.0.0.1" + finally: + s.close() + return IP + + @retry( wait=wait_fixed(2), stop=stop_after_attempt(10), after=after_log(log, logging.WARN) ) diff --git a/packages/pytest-simcore/src/pytest_simcore/minio_service.py b/packages/pytest-simcore/src/pytest_simcore/minio_service.py index 60f74adf700..0f7deec4d59 100644 --- a/packages/pytest-simcore/src/pytest_simcore/minio_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/minio_service.py @@ -34,6 +34,7 @@ def minio_config(docker_stack: Dict, devel_environ: Dict) -> Dict[str, str]: # nodeports takes its configuration from env variables for key, value in config["client"].items(): os.environ[f"S3_{key.upper()}"] = str(value) + os.environ[f"S3_SECURE"] = devel_environ["S3_SECURE"] os.environ[f"S3_BUCKET_NAME"] = devel_environ["S3_BUCKET_NAME"] return config @@ -59,3 +60,12 @@ def wait_till_minio_responsive(minio_config: Dict[str, str]) -> bool: client.remove_bucket("pytest") return True raise Exception(f"Minio not responding to {minio_config}") + + +@pytest.fixture(scope="module") +def bucket(minio_config: Dict[str, str], minio_service: S3Client) -> str: + bucket_name = minio_config["bucket_name"] + minio_service.create_bucket(bucket_name, delete_contents_if_exists=True) + yield bucket_name + + minio_service.remove_bucket(bucket_name, delete_contents=True) diff --git a/packages/pytest-simcore/src/pytest_simcore/postgres_service.py b/packages/pytest-simcore/src/pytest_simcore/postgres_service.py index 31adbb47df3..83d73f74e9b 100644 --- a/packages/pytest-simcore/src/pytest_simcore/postgres_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/postgres_service.py @@ -7,8 +7,8 @@ import pytest import sqlalchemy as sa +import tenacity from sqlalchemy.orm import sessionmaker -from tenacity import Retrying from servicelib.aiopg_utils import DSN, PostgresRetryPolicyUponInitialization from simcore_postgres_database.models.base import metadata @@ -38,15 +38,10 @@ def postgres_dsn(docker_stack: Dict, devel_environ: Dict) -> Dict[str, str]: @pytest.fixture(scope="module") def postgres_db(postgres_dsn: Dict[str, str], docker_stack: Dict) -> sa.engine.Engine: url = DSN.format(**postgres_dsn) - # Attempts until responsive - for attempt in Retrying(**PostgresRetryPolicyUponInitialization().kwargs): - with attempt: - engine = sa.create_engine(url, isolation_level="AUTOCOMMIT") - conn = engine.connect() - conn.close() - + wait_till_postgres_is_responsive(url) # Configures db and initializes tables + engine = sa.create_engine(url, isolation_level="AUTOCOMMIT") metadata.create_all(bind=engine, checkfirst=True) yield engine @@ -61,3 +56,10 @@ def postgres_session(postgres_db: sa.engine.Engine) -> sa.orm.session.Session: session = Session() yield session session.close() + + +@tenacity.retry(**PostgresRetryPolicyUponInitialization().kwargs) +def wait_till_postgres_is_responsive(url: str) -> None: + engine = sa.create_engine(url, isolation_level="AUTOCOMMIT") + conn = engine.connect() + conn.close() \ No newline at end of file diff --git a/packages/pytest-simcore/src/pytest_simcore/rabbit_service.py b/packages/pytest-simcore/src/pytest_simcore/rabbit_service.py index aa1c1cacff6..6570300f65f 100644 --- a/packages/pytest-simcore/src/pytest_simcore/rabbit_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/rabbit_service.py @@ -3,13 +3,15 @@ # pylint:disable=redefined-outer-name import os -from typing import Dict +import socket +from typing import Dict, Optional, Tuple import aio_pika import pytest import tenacity from servicelib.rabbitmq_utils import RabbitMQRetryPolicyUponInitialization +from simcore_sdk.config.rabbit import Config from .helpers.utils_docker import get_service_published_port @@ -30,7 +32,7 @@ def rabbit_config(docker_stack: Dict, devel_environ: Dict) -> Dict: os.environ["RABBIT_PORT"] = config["port"] os.environ["RABBIT_USER"] = devel_environ["RABBIT_USER"] os.environ["RABBIT_PASSWORD"] = devel_environ["RABBIT_PASSWORD"] - os.environ["RABBIT_PROGRESS_CHANNEL"] = devel_environ["RABBIT_PROGRESS_CHANNEL"] + os.environ["RABBIT_CHANNELS"] = devel_environ["RABBIT_CHANNELS"] yield config @@ -38,13 +40,86 @@ def rabbit_config(docker_stack: Dict, devel_environ: Dict) -> Dict: @pytest.fixture(scope="function") async def rabbit_service(rabbit_config: Dict, docker_stack: Dict) -> str: url = "amqp://{user}:{password}@{host}:{port}".format(**rabbit_config) - wait_till_rabbit_responsive(url) + await wait_till_rabbit_responsive(url) yield url +@pytest.fixture(scope="function") +async def rabbit_connection(rabbit_service: str) -> aio_pika.RobustConnection: + def reconnect_callback(): + pytest.fail("rabbit reconnected") + + # create connection + # NOTE: to show the connection name in the rabbitMQ UI see there [https://www.bountysource.com/issues/89342433-setting-custom-connection-name-via-client_properties-doesn-t-work-when-connecting-using-an-amqp-url] + connection = await aio_pika.connect_robust( + rabbit_service + f"?name={__name__}_{id(socket.gethostname())}", + client_properties={"connection_name": "pytest read connection"}, + ) + assert connection + assert not connection.is_closed + connection.add_reconnect_callback(reconnect_callback) + + yield connection + # close connection + await connection.close() + assert connection.is_closed + + +@pytest.fixture(scope="function") +async def rabbit_channel( + rabbit_connection: aio_pika.RobustConnection, +) -> aio_pika.Channel: + def channel_close_callback(exc: Optional[BaseException]): + if exc: + pytest.fail("rabbit channel closed!") + + # create channel + channel = await rabbit_connection.channel() + assert channel + channel.add_close_callback(channel_close_callback) + yield channel + # close channel + await channel.close() + + +@pytest.fixture(scope="function") +async def rabbit_exchange( + rabbit_channel: aio_pika.Channel, +) -> Tuple[aio_pika.Exchange, aio_pika.Exchange]: + rb_config = Config() + + # declare log exchange + LOG_EXCHANGE_NAME: str = rb_config.channels["log"] + logs_exchange = await rabbit_channel.declare_exchange( + LOG_EXCHANGE_NAME, aio_pika.ExchangeType.FANOUT + ) + # declare progress exchange + PROGRESS_EXCHANGE_NAME: str = rb_config.channels["progress"] + progress_exchange = await rabbit_channel.declare_exchange( + PROGRESS_EXCHANGE_NAME, aio_pika.ExchangeType.FANOUT + ) + + yield logs_exchange, progress_exchange + + +@pytest.fixture(scope="function") +async def rabbit_queue( + rabbit_channel: aio_pika.Channel, + rabbit_exchange: Tuple[aio_pika.Exchange, aio_pika.Exchange], +) -> aio_pika.Queue: + (logs_exchange, progress_exchange) = rabbit_exchange + # declare queue + queue = await rabbit_channel.declare_queue(exclusive=True) + # Binding queue to exchange + await queue.bind(logs_exchange) + await queue.bind(progress_exchange) + yield queue + + # HELPERS -- @tenacity.retry(**RabbitMQRetryPolicyUponInitialization().kwargs) async def wait_till_rabbit_responsive(url: str) -> None: - await aio_pika.connect(url) + connection = await aio_pika.connect(url) + await connection.close() diff --git a/packages/pytest-simcore/src/pytest_simcore/simcore_storage_service.py b/packages/pytest-simcore/src/pytest_simcore/simcore_storage_service.py index 588cdc524ae..9a82db35733 100644 --- a/packages/pytest-simcore/src/pytest_simcore/simcore_storage_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/simcore_storage_service.py @@ -32,7 +32,7 @@ def storage_endpoint(docker_stack: Dict, devel_environ: Dict) -> URL: async def storage_service( minio_service: S3Client, storage_endpoint: URL, docker_stack: Dict ) -> URL: - assert await wait_till_storage_responsive(storage_endpoint) + await wait_till_storage_responsive(storage_endpoint) yield storage_endpoint diff --git a/packages/service-library/src/servicelib/utils.py b/packages/service-library/src/servicelib/utils.py index 07763aad9f7..242f5346b82 100644 --- a/packages/service-library/src/servicelib/utils.py +++ b/packages/service-library/src/servicelib/utils.py @@ -35,7 +35,7 @@ def search_osparc_repo_dir(start: Union[str, Path], max_iterations=8) -> Optiona # FUTURES -def fire_and_forget_task(obj: Union[Coroutine, asyncio.Future]) -> None: +def fire_and_forget_task(obj: Union[Coroutine, asyncio.Future]) -> asyncio.Future: future = asyncio.ensure_future(obj) def log_exception_callback(fut: asyncio.Future): @@ -45,6 +45,7 @@ def log_exception_callback(fut: asyncio.Future): logger.exception("Error occured while running task!") future.add_done_callback(log_exception_callback) + return future # // tasks diff --git a/packages/simcore-sdk/Makefile b/packages/simcore-sdk/Makefile index 2e0efa3ad9a..03f44e4caf6 100644 --- a/packages/simcore-sdk/Makefile +++ b/packages/simcore-sdk/Makefile @@ -11,7 +11,7 @@ include ../../scripts/common.Makefile .PHONY: install-dev install-prod install-ci -install-dev install-prod install-ci: openapi-specs _check_venv_active ## install app in development/production or CI mode +install-dev install-prod install-ci: _check_venv_active ## install app in development/production or CI mode # installing in $(subst install-,,$@) mode python -m pip install -r requirements/$(subst install-,,$@).txt diff --git a/packages/simcore-sdk/requirements/_base.in b/packages/simcore-sdk/requirements/_base.in index 56e6fe99934..289c922f100 100644 --- a/packages/simcore-sdk/requirements/_base.in +++ b/packages/simcore-sdk/requirements/_base.in @@ -3,11 +3,12 @@ # pyyaml >=5.3 # Vulnerability +aiofiles aiohttp +aiopg[sa] networkx psycopg2-binary +pydantic +sqlalchemy>=1.3.3 # https://nvd.nist.gov/vuln/detail/CVE-2019-7164 tenacity trafaret-config -aiofiles -sqlalchemy>=1.3.3 # https://nvd.nist.gov/vuln/detail/CVE-2019-7164 -pika diff --git a/packages/simcore-sdk/requirements/_base.txt b/packages/simcore-sdk/requirements/_base.txt index 0d0368e0cee..9a60f8a0766 100644 --- a/packages/simcore-sdk/requirements/_base.txt +++ b/packages/simcore-sdk/requirements/_base.txt @@ -2,23 +2,25 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile --build-isolation _base.in +# pip-compile --output-file=_base.txt _base.in # aiofiles==0.4.0 # via -r _base.in aiohttp==3.6.2 # via -r _base.in +aiopg[sa]==1.0.0 # via -r _base.in async-timeout==3.0.1 # via aiohttp -attrs==19.1.0 # via aiohttp +attrs==19.3.0 # via aiohttp chardet==3.0.4 # via aiohttp +dataclasses==0.7 # via pydantic decorator==4.4.0 # via networkx idna-ssl==1.1.0 # via aiohttp idna==2.8 # via idna-ssl, yarl multidict==4.5.2 # via aiohttp, yarl networkx==2.3 # via -r _base.in -pika==1.0.1 # via -r _base.in -psycopg2-binary==2.8.4 # via -r _base.in +psycopg2-binary==2.8.4 # via -r _base.in, aiopg, sqlalchemy +pydantic==1.4 # via -r _base.in pyyaml==5.3 # via -r _base.in, trafaret-config six==1.12.0 # via tenacity -sqlalchemy==1.3.3 # via -r _base.in +sqlalchemy[postgresql_psycopg2binary]==1.3.3 # via -r _base.in, aiopg tenacity==6.0.0 # via -r _base.in trafaret-config==2.0.2 # via -r _base.in trafaret==2.0.2 # via trafaret-config diff --git a/packages/simcore-sdk/requirements/_test.txt b/packages/simcore-sdk/requirements/_test.txt index c5262d30cf7..4af68911986 100644 --- a/packages/simcore-sdk/requirements/_test.txt +++ b/packages/simcore-sdk/requirements/_test.txt @@ -2,17 +2,19 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile --build-isolation _test.in +# pip-compile --output-file=_test.txt _test.in # aiofiles==0.4.0 # via -r _base.txt aiohttp==3.6.2 # via -r _base.txt, pytest-aiohttp +aiopg[sa]==1.0.0 # via -r _base.txt astroid==2.3.3 # via pylint async-timeout==3.0.1 # via -r _base.txt, aiohttp -attrs==19.1.0 # via -r _base.txt, aiohttp, pytest, pytest-docker +attrs==19.3.0 # via -r _base.txt, aiohttp, pytest, pytest-docker certifi==2019.11.28 # via requests chardet==3.0.4 # via -r _base.txt, aiohttp, requests coverage==4.5.1 # via -r _test.in, coveralls, pytest-cov coveralls==1.11.1 # via -r _test.in +dataclasses==0.7 # via -r _base.txt, pydantic decorator==4.4.0 # via -r _base.txt, networkx docker==4.2.0 # via -r _test.in docopt==0.6.2 # via coveralls @@ -27,10 +29,10 @@ more-itertools==8.2.0 # via pytest multidict==4.5.2 # via -r _base.txt, aiohttp, yarl networkx==2.3 # via -r _base.txt packaging==20.3 # via pytest, pytest-sugar -pika==1.0.1 # via -r _base.txt pluggy==0.13.1 # via pytest -psycopg2-binary==2.8.4 # via -r _base.txt +psycopg2-binary==2.8.4 # via -r _base.txt, aiopg, sqlalchemy py==1.8.1 # via pytest +pydantic==1.4 # via -r _base.txt pylint==2.4.4 # via -r _test.in pyparsing==2.4.6 # via packaging pytest-aiohttp==0.3.0 # via -r _test.in @@ -44,7 +46,7 @@ pytest==5.3.5 # via -r _test.in, pytest-aiohttp, pytest-cov, pytest- pyyaml==5.3 # via -r _base.txt, trafaret-config requests==2.23.0 # via -r _test.in, coveralls, docker six==1.12.0 # via -r _base.txt, astroid, docker, packaging, tenacity, websocket-client -sqlalchemy==1.3.3 # via -r _base.txt +sqlalchemy[postgresql_psycopg2binary]==1.3.3 # via -r _base.txt, aiopg tenacity==6.0.0 # via -r _base.txt termcolor==1.1.0 # via pytest-sugar trafaret-config==2.0.2 # via -r _base.txt @@ -52,7 +54,7 @@ trafaret==2.0.2 # via -r _base.txt, trafaret-config typed-ast==1.4.1 # via astroid typing-extensions==3.7.2 # via -r _base.txt, aiohttp urllib3==1.25.8 # via requests -wcwidth==0.1.8 # via pytest +wcwidth==0.1.9 # via pytest websocket-client==0.57.0 # via docker wrapt==1.11.2 # via astroid yarl==1.3.0 # via -r _base.txt, aiohttp diff --git a/packages/simcore-sdk/requirements/ci.txt b/packages/simcore-sdk/requirements/ci.txt index 238465fb0d4..62fbc579dc0 100644 --- a/packages/simcore-sdk/requirements/ci.txt +++ b/packages/simcore-sdk/requirements/ci.txt @@ -13,6 +13,7 @@ ../postgres-database/ ../s3wrapper/ ../service-library/ +../pytest-simcore/ ../../services/storage/client-sdk/python/ # Needed ONLY for testing diff --git a/packages/simcore-sdk/requirements/dev.txt b/packages/simcore-sdk/requirements/dev.txt index d124bc03eed..a51583f4fb4 100644 --- a/packages/simcore-sdk/requirements/dev.txt +++ b/packages/simcore-sdk/requirements/dev.txt @@ -13,6 +13,7 @@ -e ../postgres-database/ -e ../s3wrapper/ -e ../service-library/ +-e ../pytest-simcore/ ../../services/storage/client-sdk/python/ # Needed ONLY for testing # installs current package diff --git a/packages/simcore-sdk/src/simcore_sdk/config/docker.py b/packages/simcore-sdk/src/simcore_sdk/config/docker.py deleted file mode 100644 index 01a75d3e776..00000000000 --- a/packages/simcore-sdk/src/simcore_sdk/config/docker.py +++ /dev/null @@ -1,41 +0,0 @@ -""" Basic configuration file for docker registry - -""" -from os import environ as env - -import trafaret as T - -# TODO: adapt all data below! -CONFIG_SCHEMA = T.Dict({ - "user": T.String(), - "password": T.String(), - "registry": T.String() -}) - - -class Config(): - # TODO: uniform config classes . see server.config file - def __init__(self): - REGISTRY = env.get("REGISTRY_URL", "masu.speag.com") - USER = env.get("REGISTRY_USER", "z43") - PWD = env.get("REGISTRY_PW", "z43") - - self._registry = REGISTRY - self._user = USER - self._pwd = PWD - - @property - def registry(self): - return self._registry + "/v2" - - @property - def registry_name(self): - return self._registry - - @property - def user(self): - return self._user - - @property - def pwd(self): - return self._pwd diff --git a/packages/simcore-sdk/src/simcore_sdk/config/rabbit.py b/packages/simcore-sdk/src/simcore_sdk/config/rabbit.py index c176807d029..d682fdbb83b 100644 --- a/packages/simcore-sdk/src/simcore_sdk/config/rabbit.py +++ b/packages/simcore-sdk/src/simcore_sdk/config/rabbit.py @@ -2,130 +2,60 @@ """ -from os import environ as env +from typing import Dict, Union -import pika -import yaml import trafaret as T - +from pydantic import BaseSettings # TODO: adapt all data below! -# TODO: can use venv as defaults? e.g. $RABBIT_LOG_CHANNEL -CONFIG_SCHEMA = T.Dict({ - T.Key("name", default="tasks", optional=True): T.String(), - T.Key("enabled", default=True, optional=True): T.Bool(), - T.Key("host", default='rabbit', optional=True): T.String(), - T.Key("port", default=5672, optional=True): T.ToInt(), - "user": T.String(), - "password": T.String(), - "channels": T.Dict({ - "progress": T.String(), - "log": T.String(), - T.Key("celery", default=dict(result_backend="rpc://"), optional=True): T.Dict({ - T.Key("result_backend", default="${CELERY_RESULT_BACKEND}", optional=True): T.String() - }) - }) -}) - - -CONFIG_EXAMPLES = map(yaml.safe_load,[ -""" - user: simcore - password: simcore - channels: - log: comp.backend.channels.log - progress: comp.backend.channels.progress -""", -""" - host: rabbito - port: 1234 - user: foo - password: secret - channels: - log: comp.backend.channels.log - progress: comp.backend.channels.progress -""", -""" - user: bar - password: secret - channels: - log: comp.backend.channels.log - progress: comp.backend.channels.progress - celery: - result_backend: 'rpc://' -"""]) - - -def eval_broker(config): - """ - Raises trafaret.DataError if config validation fails - """ - CONFIG_SCHEMA.check(config) # raise exception - url = 'amqp://{user}:{password}@{host}:{port}'.format(**config) - return url - - -# TODO: deprecate! ----------------------------------------------------------------------------- -# TODO: uniform config classes . see server.config file - -class Config: - def __init__(self, config=None): - if config is not None: - CONFIG_SCHEMA.check(config) # raise exception - else: - config = {} - - RABBIT_USER = env.get('RABBIT_USER','simcore') - RABBIT_PASSWORD = env.get('RABBIT_PASSWORD','simcore') - RABBIT_HOST=env.get('RABBIT_HOST','rabbit') - RABBIT_PORT=int(env.get('RABBIT_PORT', 5672)) - RABBIT_LOG_CHANNEL = env.get('RABBIT_LOG_CHANNEL','comp.backend.channels.log') - RABBIT_PROGRESS_CHANNEL = env.get('RABBIT_PROGRESS_CHANNEL','comp.backend.channels.progress') - CELERY_RESULT_BACKEND=env.get('CELERY_RESULT_BACKEND','rpc://') - # FIXME: get variables via config.get('') or - # rabbit - - try: - self._broker_url = eval_broker(config) - except: # pylint: disable=W0702 - self._broker_url = 'amqp://{user}:{pw}@{url}:{port}'.format(user=RABBIT_USER, pw=RABBIT_PASSWORD, url=RABBIT_HOST, port=RABBIT_PORT) - - self._result_backend = config.get("celery", {}).get("result_backend") or CELERY_RESULT_BACKEND - self._module_name = config.get("name") or "tasks" - - # pika - self._pika_credentials = pika.PlainCredentials( - config.get("user") or RABBIT_USER, - config.get("password") or RABBIT_PASSWORD) - self._pika_parameters = pika.ConnectionParameters( - host=config.get("host") or RABBIT_HOST, - port=config.get("port") or RABBIT_PORT, - credentials=self._pika_credentials, - connection_attempts=100) - - self._log_channel = config.get("celery", {}).get("result_backend") or RABBIT_LOG_CHANNEL - self._progress_channel = config.get("celery", {}).get("result_backend") or RABBIT_PROGRESS_CHANNEL - - @property - def parameters(self): - return self._pika_parameters +CONFIG_SCHEMA = T.Dict( + { + T.Key("name", default="tasks", optional=True): T.String(), + T.Key("enabled", default=True, optional=True): T.Bool(), + T.Key("host", default="rabbit", optional=True): T.String(), + T.Key("port", default=5672, optional=True): T.Int(), + "user": T.String(), + "password": T.String(), + "channels": T.Dict( + { + "progress": T.String(), + "log": T.String(), + T.Key( + "celery", default=dict(result_backend="rpc://"), optional=True + ): T.Dict( + { + T.Key( + "result_backend", + default="${CELERY_RESULT_BACKEND}", + optional=True, + ): T.String() + } + ), + } + ), + } +) +# TODO: use BaseSettings instead of BaseModel and remove trafaret ! ----------------------------------------------------------------------------- +class Config(BaseSettings): + name: str = "tasks" + enabled: bool = True + user: str = "simcore" + password: str = "simcore" + host: str = "rabbit" + port: int = 5672 + channels: Dict[str, Union[str, Dict]] = { + "progress": "comp.backend.channels.progress", + "log": "comp.backend.channels.log", + "celery": {"result_backend": "rpc://"}, + } + + class Config: + env_prefix = "RABBIT_" @property - def log_channel(self): - return self._log_channel - - @property - def progress_channel(self): - return self._progress_channel - - @property - def broker(self): - return self._broker_url + def broker_url(self): + return f"amqp://{self.user}:{self.password}@{self.host}:{self.port}" @property def backend(self): - return self._result_backend - - @property - def name(self): - return self._module_name + return self.channels["celery"]["result_backend"] diff --git a/packages/simcore-sdk/src/simcore_sdk/models/pipeline_models.py b/packages/simcore-sdk/src/simcore_sdk/models/pipeline_models.py index 0842b35cbed..a14bfa4a60e 100644 --- a/packages/simcore-sdk/src/simcore_sdk/models/pipeline_models.py +++ b/packages/simcore-sdk/src/simcore_sdk/models/pipeline_models.py @@ -2,10 +2,14 @@ import networkx as nx from sqlalchemy.orm import mapper -from simcore_postgres_database.models.comp_pipeline import (FAILED, PENDING, - RUNNING, SUCCESS, - UNKNOWN, - comp_pipeline) +from simcore_postgres_database.models.comp_pipeline import ( + FAILED, + PENDING, + RUNNING, + SUCCESS, + UNKNOWN, + comp_pipeline, +) from simcore_postgres_database.models.comp_tasks import comp_tasks from .base import metadata @@ -17,14 +21,14 @@ class Base: class ComputationalPipeline: - #pylint: disable=no-member + # pylint: disable=no-member def __init__(self, **kargs): for key, value in kargs.items(): if key in ComputationalPipeline._sa_class_manager.keys(): setattr(self, key, value) @property - def execution_graph(self): + def execution_graph(self) -> nx.DiGraph: d = self.dag_adjacency_list G = nx.DiGraph() @@ -37,16 +41,14 @@ def execution_graph(self): return G def __repr__(self): - return ''.format(self.id) - -mapper(ComputationalPipeline, comp_pipeline) - + return "".format(self.id) +mapper(ComputationalPipeline, comp_pipeline) class ComputationalTask: - #pylint: disable=no-member + # pylint: disable=no-member def __init__(self, **kargs): for key, value in kargs.items(): if key in ComputationalTask._sa_class_manager.keys(): @@ -60,5 +62,9 @@ def __init__(self, **kargs): "metadata", "ComputationalPipeline", "ComputationalTask", - "UNKNOWN", "PENDING", "RUNNING", "SUCCESS", "FAILED" + "UNKNOWN", + "PENDING", + "RUNNING", + "SUCCESS", + "FAILED", ] diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports/_data_items_list.py b/packages/simcore-sdk/src/simcore_sdk/node_ports/_data_items_list.py index 33f1862e9f8..184d3753e09 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports/_data_items_list.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports/_data_items_list.py @@ -2,7 +2,7 @@ # pylint: disable=too-many-ancestors import logging -from collections import MutableMapping +from collections.abc import MutableMapping from typing import Dict from . import exceptions @@ -10,10 +10,11 @@ log = logging.getLogger(__name__) + class DataItemsList(MutableMapping): """This class contains a list of Data Items.""" - def __init__(self, data:Dict[str, DataItem]=None): + def __init__(self, data: Dict[str, DataItem] = None): log.debug("Creating DataItemsList with %s", data) if data is None: data = {} @@ -25,7 +26,7 @@ def __setitem__(self, key, value: DataItem): raise TypeError if isinstance(key, int): key = self._store.keys()[key] - + self._store[key] = value def __getitem__(self, key): @@ -36,7 +37,7 @@ def __getitem__(self, key): if not key in self._store: raise exceptions.UnboundPortError(key) return self._store[key] - + def __iter__(self): return iter(self._store) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports/_item.py b/packages/simcore-sdk/src/simcore_sdk/node_ports/_item.py index a537a957c59..736aa31194f 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports/_item.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports/_item.py @@ -1,6 +1,7 @@ import logging import shutil from pathlib import Path +from typing import Dict from . import config, data_items_utils, exceptions, filemanager from ._data_item import DataItem @@ -9,24 +10,29 @@ log = logging.getLogger(__name__) - def _check_type(item_type, value): if not value: return if data_items_utils.is_value_link(value): return - if isinstance(value, (int, float)): if item_type in ("number", "integer"): return - possible_types = [key for key, key_type in config.TYPE_TO_PYTHON_TYPE_MAP.items() if isinstance(value, key_type["type"])] + possible_types = [ + key + for key, key_type in config.TYPE_TO_PYTHON_TYPE_MAP.items() + if isinstance(value, key_type["type"]) + ] if not item_type in possible_types: - if data_items_utils.is_file_type(item_type) and data_items_utils.is_value_on_store(value): + if data_items_utils.is_file_type( + item_type + ) and data_items_utils.is_value_on_store(value): return raise exceptions.InvalidItemTypeError(item_type, value) + class Item: """An item contains data (DataItem) and an schema (SchemaItem) @@ -36,6 +42,7 @@ class Item: :raises exceptions.InvalidItemTypeError: [description] :raises exceptions.NodeportsException: [description] """ + def __init__(self, schema: SchemaItem, data: DataItem): if not schema: raise exceptions.InvalidProtocolError(None, msg="empty schema or payload") @@ -68,7 +75,10 @@ async def get(self): :return: the converted value or None if no value is defined """ log.debug("Getting item %s", self.key) - if self.type not in config.TYPE_TO_PYTHON_TYPE_MAP and not data_items_utils.is_file_type(self.type): + if ( + self.type not in config.TYPE_TO_PYTHON_TYPE_MAP + and not data_items_utils.is_file_type(self.type) + ): raise exceptions.InvalidProtocolError(self.type) if self.value is None: log.debug("Got empty data item") @@ -101,7 +111,9 @@ async def get(self): if data_items_utils.is_value_on_store(self.value): return await self.__get_value_from_store(self.value) # the value is not a link, let's directly convert it to the right type - return config.TYPE_TO_PYTHON_TYPE_MAP[self.type]["type"](config.TYPE_TO_PYTHON_TYPE_MAP[self.type]["converter"](self.value)) + return config.TYPE_TO_PYTHON_TYPE_MAP[self.type]["type"]( + config.TYPE_TO_PYTHON_TYPE_MAP[self.type]["converter"](self.value) + ) async def set(self, value): """ sets the data to the underlying port @@ -114,10 +126,16 @@ async def set(self, value): log.info("Setting data item with value %s", value) # try to guess the type and check the type set fits this (there can be more than one possibility, e.g. string) - possible_types = [key for key, key_type in config.TYPE_TO_PYTHON_TYPE_MAP.items() if isinstance(value, key_type["type"])] + possible_types = [ + key + for key, key_type in config.TYPE_TO_PYTHON_TYPE_MAP.items() + if isinstance(value, key_type["type"]) + ] log.debug("possible types are for value %s are %s", value, possible_types) if not self.type in possible_types: - if not data_items_utils.is_file_type(self.type) or not isinstance(value, (Path, str)): + if not data_items_utils.is_file_type(self.type) or not isinstance( + value, (Path, str) + ): raise exceptions.InvalidItemTypeError(self.type, value) # upload to S3 if file @@ -126,8 +144,12 @@ async def set(self, value): if not file_path.exists() or not file_path.is_file(): raise exceptions.InvalidItemTypeError(self.type, value) log.debug("file path %s will be uploaded to s3", value) - s3_object = data_items_utils.encode_file_id(file_path, project_id=config.PROJECT_ID, node_id=config.NODE_UUID) - store_id = await filemanager.upload_file(store_name=config.STORE, s3_object=s3_object, local_file_path=file_path) + s3_object = data_items_utils.encode_file_id( + file_path, project_id=config.PROJECT_ID, node_id=config.NODE_UUID + ) + store_id = await filemanager.upload_file( + store_name=config.STORE, s3_object=s3_object, local_file_path=file_path + ) log.debug("file path %s uploaded", value) value = data_items_utils.encode_store(store_id, s3_object) @@ -136,26 +158,34 @@ async def set(self, value): new_data = DataItem(key=self.key, value=value) if self.new_data_cb: log.debug("calling new data callback to update database") - self.new_data_cb(new_data) #pylint: disable=not-callable + await self.new_data_cb(new_data) # pylint: disable=not-callable log.debug("database updated") - async def __get_value_from_link(self, value): # pylint: disable=R1710 + async def __get_value_from_link( + self, value: Dict[str, str] + ): # pylint: disable=R1710 log.debug("Getting value %s", value) node_uuid, port_key = data_items_utils.decode_link(value) if not self.get_node_from_uuid_cb: - raise exceptions.NodeportsException("callback to get other node information is not set") + raise exceptions.NodeportsException( + "callback to get other node information is not set" + ) # create a node ports for the other node - other_nodeports = self.get_node_from_uuid_cb(node_uuid) #pylint: disable=not-callable + other_nodeports = await self.get_node_from_uuid_cb( # pylint: disable=not-callable + node_uuid + ) # get the port value through that guy log.debug("Received node from DB %s, now returning value", other_nodeports) return await other_nodeports.get(port_key) - async def __get_value_from_store(self, value) -> Path: + async def __get_value_from_store(self, value: Dict[str, str]) -> Path: log.debug("Getting value from storage %s", value) store_id, s3_path = data_items_utils.decode_store(value) # do not make any assumption about s3_path, it is a str containing stuff that can be anything depending on the store local_path = data_items_utils.create_folder_path(self.key) - downloaded_file = await filemanager.download_file(store_id=store_id, s3_object=s3_path, local_folder=local_path) + downloaded_file = await filemanager.download_file( + store_id=store_id, s3_object=s3_path, local_folder=local_path + ) # if a file alias is present use it to rename the file accordingly if self._schema.fileToKeyMap: renamed_file = local_path / next(iter(self._schema.fileToKeyMap)) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports/_items_list.py b/packages/simcore-sdk/src/simcore_sdk/node_ports/_items_list.py index 1741501b329..c6a97c4f9cb 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports/_items_list.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports/_items_list.py @@ -1,5 +1,5 @@ import logging -from collections import Sequence +from collections.abc import Sequence from . import exceptions from ._data_items_list import DataItemsList @@ -8,18 +8,22 @@ log = logging.getLogger(__name__) -class ItemsList(Sequence): - def __init__(self, schemas: SchemaItemsList, payloads: DataItemsList, - change_cb=None, - get_node_from_node_uuid_cb=None): +class ItemsList(Sequence): + def __init__( + self, + schemas: SchemaItemsList, + payloads: DataItemsList, + change_cb=None, + get_node_from_node_uuid_cb=None, + ): self._schemas = schemas self._payloads = payloads self._change_notifier = change_cb self._get_node_from_node_uuid_cb = get_node_from_node_uuid_cb - def __getitem__(self, key)->Item: + def __getitem__(self, key) -> Item: schema = self._schemas[key] try: payload = self._payloads[schema.key] @@ -55,14 +59,14 @@ def get_node_from_node_uuid_cb(self): def get_node_from_node_uuid_cb(self, value): self._get_node_from_node_uuid_cb = value - def _item_value_updated_cb(self, new_data_item): + async def _item_value_updated_cb(self, new_data_item): # a new item shall replace the current one self.__replace_item(new_data_item) - self._notify_client() + await self._notify_client() def __replace_item(self, new_data_item): self._payloads[new_data_item.key] = new_data_item - def _notify_client(self): + async def _notify_client(self): if self.change_notifier and callable(self.change_notifier): - self.change_notifier() #pylint: disable=not-callable + await self.change_notifier() # pylint: disable=not-callable diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports/_schema_items_list.py b/packages/simcore-sdk/src/simcore_sdk/node_ports/_schema_items_list.py index 285ffc0b76d..96aec910775 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports/_schema_items_list.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports/_schema_items_list.py @@ -1,5 +1,5 @@ import logging -from collections import Mapping +from collections.abc import Mapping from typing import Dict from . import exceptions @@ -7,14 +7,15 @@ log = logging.getLogger(__name__) + class SchemaItemsList(Mapping): - def __init__(self, data:Dict[str, SchemaItem]=None): + def __init__(self, data: Dict[str, SchemaItem] = None): log.debug("creating SchemaItemsList with %s", data) if not data: data = {} self._store = data - def __getitem__(self, key)->SchemaItem: + def __getitem__(self, key) -> SchemaItem: if isinstance(key, int): if key < len(self._store): key = list(self._store.keys())[key] diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports/config.py b/packages/simcore-sdk/src/simcore_sdk/node_ports/config.py index 902c4394d9e..70af3933c7b 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports/config.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports/config.py @@ -4,48 +4,46 @@ # required configurations +PROJECT_ID = os.environ.get("SIMCORE_PROJECT_ID", default="undefined") NODE_UUID = os.environ.get("SIMCORE_NODE_UUID", default="undefined") USER_ID = os.environ.get("SIMCORE_USER_ID", default="undefined") STORAGE_ENDPOINT = os.environ.get("STORAGE_ENDPOINT", default="undefined") STORAGE_VERSION = "v0" + +POSTGRES_ENDPOINT: str = os.environ.get("POSTGRES_ENDPOINT", "postgres:5432") +POSTGRES_DB: str = os.environ.get("POSTGRES_DB", "simcoredb") +POSTGRES_PW: str = os.environ.get("POSTGRES_PASSWORD", "simcore") +POSTGRES_USER: str = os.environ.get("POSTGRES_USER", "simcore") + # overridable required configurations STORE = os.environ.get("STORAGE_STORE_LOCATION_NAME", default="simcore.s3") BUCKET = os.environ.get("S3_BUCKET_NAME", default="simcore") # ------------------------------------------------------------------------- -# internals -PROJECT_ID = os.environ.get("SIMCORE_PROJECT_ID", default="undefined") -USER_ID = os.environ.get("SIMCORE_USER_ID", default="undefined") - STORAGE_ENDPOINT = os.environ.get("STORAGE_ENDPOINT", default="undefined") STORAGE_VERSION = "v0" -STORE = "simcore.s3" -BUCKET = "simcore" - - -NODE_KEYS = {"version":True, -"schema":True, -"inputs":True, -"outputs":True} +NODE_KEYS = {"version": True, "schema": True, "inputs": True, "outputs": True} -DATA_ITEM_KEYS = {"key":True, - "value":True} +DATA_ITEM_KEYS = {"key": True, "value": True} # True if required, defined by JSON schema -SCHEMA_ITEM_KEYS = {"key":True, - "label":True, - "description":True, - "type":True, - "displayOrder":True, - "fileToKeyMap":False, - "defaultValue":False, - "widget":False} +SCHEMA_ITEM_KEYS = { + "key": True, + "label": True, + "description": True, + "type": True, + "displayOrder": True, + "fileToKeyMap": False, + "defaultValue": False, + "widget": False, +} # allowed types -TYPE_TO_PYTHON_TYPE_MAP = {"integer":{"type":int, "converter":int}, - "number":{"type":float, "converter":float}, - "boolean":{"type":bool, "converter":bool}, - "string":{"type":str, "converter":str} - } +TYPE_TO_PYTHON_TYPE_MAP = { + "integer": {"type": int, "converter": int}, + "number": {"type": float, "converter": float}, + "boolean": {"type": bool, "converter": bool}, + "string": {"type": str, "converter": str}, +} FILE_TYPE_PREFIX = "data:" diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports/dbmanager.py b/packages/simcore-sdk/src/simcore_sdk/node_ports/dbmanager.py index 4dc40d99f36..14a2fc54ea8 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports/dbmanager.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports/dbmanager.py @@ -1,114 +1,133 @@ import json import logging -from contextlib import contextmanager +import socket +from typing import Optional -import sqlalchemy -from sqlalchemy import create_engine, and_ -from sqlalchemy.orm import exc, sessionmaker -from sqlalchemy.orm.attributes import flag_modified +import aiopg.sa +import tenacity +from sqlalchemy import and_ -from simcore_sdk.config.db import Config as db_config -from simcore_sdk.models.pipeline_models import ComputationalTask as NodeModel +from servicelib.aiopg_utils import ( + DataSourceName, + PostgresRetryPolicyUponInitialization, + create_pg_engine, + is_postgres_responsive, +) +from simcore_postgres_database.models.comp_tasks import comp_tasks from . import config log = logging.getLogger(__name__) -@contextmanager -def session_scope(session_factory): - """Provide a transactional scope around a series of operations - - """ - session = session_factory() - try: - yield session - except: - session.rollback() - raise - finally: - session.close() - -class DbSettings: - def __init__(self): - self._db_settings_config = db_config() - # FIXME: this is a SYNCRONOUS engine! And not disposed!? - self.db = create_engine( - self._db_settings_config.endpoint + f"?application_name={__name__}_{id(self)}", - pool_pre_ping=True, - client_encoding='utf8') - self.Session = sessionmaker(self.db) - # self.session = self.Session() - - -class _NodeModelEncoder(json.JSONEncoder): - def default(self, o): # pylint: disable=E0202 - log.debug("Encoding object: %s", o) - if isinstance(o, NodeModel): - log.debug("Encoding Node object %s", o) - return { - "version": "0.1", - "schema": o.schema, - "inputs": o.inputs, - "outputs": o.outputs - } +async def _get_node_from_db( + node_uuid: str, connection: aiopg.sa.SAConnection +) -> comp_tasks: + log.debug( + "Reading from comp_tasks table for node uuid %s, project %s", + node_uuid, + config.PROJECT_ID, + ) + result = await connection.execute( + comp_tasks.select( + and_( + comp_tasks.c.node_id == node_uuid, + comp_tasks.c.project_id == config.PROJECT_ID, + ) + ) + ) + if result.rowcount > 1: + log.error("the node id %s is not unique", node_uuid) + node = await result.fetchone() + if not node: + log.error("the node id %s was not found", node_uuid) + return node + + +@tenacity.retry(**PostgresRetryPolicyUponInitialization().kwargs) +async def wait_till_postgres_responsive(dsn: DataSourceName) -> None: + if not is_postgres_responsive(dsn): + raise Exception + + +class DBContextManager: + def __init__(self, db_engine: Optional[aiopg.sa.Engine] = None): + self._db_engine: aiopg.sa.Engine = db_engine + self._db_engine_created: bool = False + + async def _create_db_engine(self) -> aiopg.sa.Engine: + dsn = DataSourceName( + application_name=f"{__name__}_{id(socket.gethostname())}", + database=config.POSTGRES_DB, + user=config.POSTGRES_USER, + password=config.POSTGRES_PW, + host=config.POSTGRES_ENDPOINT.split(":")[0], + port=config.POSTGRES_ENDPOINT.split(":")[1], + ) + await wait_till_postgres_responsive(dsn) + engine = await create_pg_engine(dsn, minsize=1, maxsize=4) + return engine + + async def __aenter__(self): + if not self._db_engine: + self._db_engine = await self._create_db_engine() + self._db_engine_created = True + return self._db_engine + + async def __aexit__(self, exc_type, exc, tb): + if self._db_engine_created: + self._db_engine.close() + await self._db_engine.wait_closed() + log.debug( + "engine '%s' after shutdown: closed=%s, size=%d", + self._db_engine.dsn, + self._db_engine.closed, + self._db_engine.size, + ) - log.debug("Encoding object using defaults") - return json.JSONEncoder.default(self, o) - -def _get_node_from_db(node_uuid: str, session: sqlalchemy.orm.session.Session) -> NodeModel: - log.debug("Reading from database for node uuid %s", node_uuid) - try: - # project id should be also defined but was not the case before - # pylint: disable=no-member - criteria = (NodeModel.node_id == node_uuid if config.PROJECT_ID == 'undefined' else and_(NodeModel.node_id == node_uuid, NodeModel.project_id == config.PROJECT_ID)) - return session.query(NodeModel).filter(criteria).one() - except exc.NoResultFound: - log.exception("the node id %s was not found", node_uuid) - except exc.MultipleResultsFound: - log.exception("the node id %s is not unique", node_uuid) class DBManager: - def __init__(self): - self._db_settings = DbSettings() - with session_scope(self._db_settings.Session) as session: - # pylint: disable=no-member - # project id should be also defined but was not the case before - criteria = (NodeModel.node_id == config.NODE_UUID - if config.PROJECT_ID == 'undefined' - else and_(NodeModel.node_id == config.NODE_UUID, NodeModel.project_id == config.PROJECT_ID) - ) - node = session.query(NodeModel).filter(criteria).one() - if config.PROJECT_ID == 'undefined': - config.PROJECT_ID = node.project_id + def __init__(self, db_engine: Optional[aiopg.sa.Engine] = None): + self._db_engine = db_engine - def write_ports_configuration(self, json_configuration: str, node_uuid: str): - log.debug("Writing ports configuration") - log.debug("Writing to database") + async def write_ports_configuration(self, json_configuration: str, node_uuid: str): + log.debug("Writing ports configuration to database") node_configuration = json.loads(json_configuration) - with session_scope(self._db_settings.Session) as session: - # pylint: disable=no-member - updated_node = NodeModel(schema=node_configuration["schema"], inputs=node_configuration["inputs"], outputs=node_configuration["outputs"]) - node = _get_node_from_db(node_uuid=node_uuid, session=session) - - if node.schema != updated_node.schema: - node.schema = updated_node.schema - flag_modified(node, "schema") - if node.inputs != updated_node.inputs: - node.inputs = updated_node.inputs - flag_modified(node, "inputs") - if node.outputs != updated_node.outputs: - node.outputs = updated_node.outputs - flag_modified(node, "outputs") - - session.commit() - - def get_ports_configuration_from_node_uuid(self, node_uuid:str) -> str: - log.debug("Getting ports configuration of node %s", node_uuid) - log.debug("Reading from database") - with session_scope(self._db_settings.Session) as session: - node = _get_node_from_db(node_uuid, session) - node_json_config = json.dumps(node, cls=_NodeModelEncoder) + async with DBContextManager(self._db_engine) as engine: + async with engine.acquire() as connection: + # update the necessary parts + await connection.execute( + # FIXME: E1120:No value for argument 'dml' in method call + # pylint: disable=E1120 + comp_tasks.update() + .where( + and_( + comp_tasks.c.node_id == node_uuid, + comp_tasks.c.project_id == config.PROJECT_ID, + ) + ) + .values( + schema=node_configuration["schema"], + inputs=node_configuration["inputs"], + outputs=node_configuration["outputs"], + ) + ) + + async def get_ports_configuration_from_node_uuid(self, node_uuid: str) -> str: + log.debug( + "Getting ports configuration of node %s from comp_tasks table", node_uuid + ) + async with DBContextManager(self._db_engine) as engine: + async with engine.acquire() as connection: + node = await _get_node_from_db(node_uuid, connection) + node_json_config = json.dumps( + { + "version": "0.1", + "schema": node.schema, + "inputs": node.inputs, + "outputs": node.outputs, + } + ) log.debug("Found and converted to json") return node_json_config diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports/nodeports.py b/packages/simcore-sdk/src/simcore_sdk/node_ports/nodeports.py index 9fdc5549c74..5cc21371c5f 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports/nodeports.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports/nodeports.py @@ -17,21 +17,29 @@ # pylint: disable=too-many-instance-attributes # pylint: disable=too-many-arguments + class Nodeports: """Allows the client to access the inputs and outputs assigned to the node """ - _version = "0.1" - def __init__( self, - version: str, - input_schemas: SchemaItemsList = None, - output_schemas: SchemaItemsList = None, - input_payloads: DataItemsList = None, - outputs_payloads: DataItemsList = None): + _version = "0.1" - log.debug("Initialising Nodeports object with version %s, inputs %s and outputs %s", - version, input_payloads, outputs_payloads) + def __init__( + self, + version: str, + input_schemas: SchemaItemsList = None, + output_schemas: SchemaItemsList = None, + input_payloads: DataItemsList = None, + outputs_payloads: DataItemsList = None, + ): + + log.debug( + "Initialising Nodeports object with version %s, inputs %s and outputs %s", + version, + input_payloads, + outputs_payloads, + ) if self._version != version: raise exceptions.WrongProtocolVersionError(self._version, version) @@ -45,35 +53,50 @@ def __init__( self, outputs_payloads = DataItemsList() self._copy_schemas_payloads( - input_schemas, output_schemas, input_payloads, outputs_payloads) + input_schemas, output_schemas, input_payloads, outputs_payloads + ) self.db_mgr = None self.autoread = False self.autowrite = False - log.debug("Initialised Nodeports object with version %s, inputs %s and outputs %s", - version, input_payloads, outputs_payloads) - - def _copy_schemas_payloads( self, - input_schemas: SchemaItemsList, - output_schemas: SchemaItemsList, - input_payloads: DataItemsList, - outputs_payloads: DataItemsList): + log.debug( + "Initialised Nodeports object with version %s, inputs %s and outputs %s", + version, + input_payloads, + outputs_payloads, + ) + + def _copy_schemas_payloads( + self, + input_schemas: SchemaItemsList, + output_schemas: SchemaItemsList, + input_payloads: DataItemsList, + outputs_payloads: DataItemsList, + ): self._input_schemas = input_schemas self._output_schemas = output_schemas self._inputs_payloads = input_payloads self._outputs_payloads = outputs_payloads - self._inputs = ItemsList(self._input_schemas, self._inputs_payloads, - change_cb=self._save_to_json, get_node_from_node_uuid_cb=self._get_node_from_node_uuid) - self._outputs = ItemsList(self._output_schemas, self._outputs_payloads, - change_cb=self._save_to_json, get_node_from_node_uuid_cb=self._get_node_from_node_uuid) + self._inputs = ItemsList( + self._input_schemas, + self._inputs_payloads, + change_cb=self._save_to_json, + get_node_from_node_uuid_cb=self._get_node_from_node_uuid, + ) + self._outputs = ItemsList( + self._output_schemas, + self._outputs_payloads, + change_cb=self._save_to_json, + get_node_from_node_uuid_cb=self._get_node_from_node_uuid, + ) @property - def inputs(self) -> ItemsList: + async def inputs(self) -> ItemsList: log.debug("Getting inputs with autoread: %s", self.autoread) if self.autoread: - self._update_from_json() + await self._update_from_json() return self._inputs @inputs.setter @@ -83,10 +106,10 @@ def inputs(self, value): raise exceptions.ReadOnlyError(self._inputs) @property - def outputs(self) -> ItemsList: + async def outputs(self) -> ItemsList: log.debug("Getting outputs with autoread: %s", self.autoread) if self.autoread: - self._update_from_json() + await self._update_from_json() return self._outputs @outputs.setter @@ -97,58 +120,64 @@ def outputs(self, value): async def get(self, item_key: str): try: - return await self.inputs[item_key].get() + return await (await self.inputs)[item_key].get() except exceptions.UnboundPortError: # not available try outputs pass # if this fails it will raise an exception - return await self.outputs[item_key].get() + return await (await self.outputs)[item_key].get() async def set(self, item_key: str, item_value): try: - await self.inputs[item_key].set(item_value) + await (await self.inputs)[item_key].set(item_value) except exceptions.UnboundPortError: # not available try outputs pass # if this fails it will raise an exception - return await self.outputs[item_key].set(item_value) + return await (await self.outputs)[item_key].set(item_value) async def set_file_by_keymap(self, item_value: Path): - for output in self.outputs: + for output in await self.outputs: if data_items_utils.is_file_type(output.type): if output.fileToKeyMap: if item_value.name in output.fileToKeyMap: await output.set(item_value) return raise exceptions.PortNotFound( - msg="output port for item {item} not found".format(item=str(item_value))) + msg="output port for item {item} not found".format(item=str(item_value)) + ) - def _update_from_json(self): + async def _update_from_json(self): # pylint: disable=protected-access log.debug("Updating json configuration") if not self.db_mgr: - raise exceptions.NodeportsException( - "db manager is not initialised") - upd_node = serialization.create_from_json(self.db_mgr) + raise exceptions.NodeportsException("db manager is not initialised") + upd_node = await serialization.create_from_json(self.db_mgr) # copy from updated nodeports - self._copy_schemas_payloads(upd_node._input_schemas, upd_node._output_schemas, - upd_node._inputs_payloads, upd_node._outputs_payloads) + self._copy_schemas_payloads( + upd_node._input_schemas, + upd_node._output_schemas, + upd_node._inputs_payloads, + upd_node._outputs_payloads, + ) log.debug("Updated json configuration") - def _save_to_json(self): + async def _save_to_json(self): log.info("Saving Nodeports object to json") - serialization.save_to_json(self) + await serialization.save_to_json(self) - def _get_node_from_node_uuid(self, node_uuid): + async def _get_node_from_node_uuid(self, node_uuid: str): if not self.db_mgr: raise exceptions.NodeportsException("db manager is not initialised") - return serialization.create_nodeports_from_uuid(self.db_mgr, node_uuid) + return await serialization.create_nodeports_from_uuid(self.db_mgr, node_uuid) -def ports(db_manager: Optional[dbmanager.DBManager]=None) -> Nodeports: +async def ports(db_manager: Optional[dbmanager.DBManager] = None) -> Nodeports: # FIXME: warning every dbmanager create a new db engine! - if db_manager is None: # NOTE: keeps backwards compatibility + if db_manager is None: # NOTE: keeps backwards compatibility db_manager = dbmanager.DBManager() # create initial Simcore object - return serialization.create_from_json(db_manager, auto_read=True, auto_write=True) + return await serialization.create_from_json( + db_manager, auto_read=True, auto_write=True + ) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports/serialization.py b/packages/simcore-sdk/src/simcore_sdk/node_ports/serialization.py index 106c86b9c39..1033e9e730f 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports/serialization.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports/serialization.py @@ -15,7 +15,10 @@ log = logging.getLogger(__name__) -def create_from_json(db_mgr: DBManager, auto_read: bool=False, auto_write: bool=False): + +async def create_from_json( + db_mgr: DBManager, auto_read: bool = False, auto_write: bool = False +): """ creates a Nodeports object provided a json configuration in form of a callback function :param db_mgr: interface object to connect to nodeports description @@ -27,10 +30,17 @@ def create_from_json(db_mgr: DBManager, auto_read: bool=False, auto_write: bool= :return: the Nodeports object """ - log.debug("Creating Nodeports object with io object: %s, auto read %s and auto write %s", db_mgr, auto_read, auto_write) + log.debug( + "Creating Nodeports object with io object: %s, auto read %s and auto write %s", + db_mgr, + auto_read, + auto_write, + ) if not db_mgr: raise exceptions.NodeportsException("io object empty, this is not allowed") - nodeports_dict = json.loads(db_mgr.get_ports_configuration_from_node_uuid(config.NODE_UUID)) + nodeports_dict = json.loads( + await db_mgr.get_ports_configuration_from_node_uuid(config.NODE_UUID) + ) nodeports_obj = __decodeNodePorts(nodeports_dict) nodeports_obj.db_mgr = db_mgr nodeports_obj.autoread = auto_read @@ -38,16 +48,22 @@ def create_from_json(db_mgr: DBManager, auto_read: bool=False, auto_write: bool= log.debug("Created Nodeports object") return nodeports_obj -def create_nodeports_from_uuid(db_mgr: DBManager, node_uuid: str): + +async def create_nodeports_from_uuid(db_mgr: DBManager, node_uuid: str): log.debug("Creating Nodeports object from node uuid: %s", node_uuid) if not db_mgr: - raise exceptions.NodeportsException("Invalid call to create nodeports from uuid") - nodeports_dict = json.loads(db_mgr.get_ports_configuration_from_node_uuid(node_uuid)) + raise exceptions.NodeportsException( + "Invalid call to create nodeports from uuid" + ) + nodeports_dict = json.loads( + await db_mgr.get_ports_configuration_from_node_uuid(node_uuid) + ) nodeports_obj = __decodeNodePorts(nodeports_dict) log.debug("Created Nodeports object") return nodeports_obj -def save_to_json(nodeports_obj): + +async def save_to_json(nodeports_obj) -> None: """ Encodes a Nodeports object to json and calls a linked writer if available. :param nodeports_obj: the object to encode @@ -63,36 +79,44 @@ def save_to_json(nodeports_obj): nodeports_obj.autoread = auto_update_state if nodeports_obj.autowrite: - nodeports_obj.db_mgr.write_ports_configuration(nodeports_json, config.NODE_UUID) + await nodeports_obj.db_mgr.write_ports_configuration( + nodeports_json, config.NODE_UUID + ) log.debug("Saved Nodeports object to json: %s", nodeports_json) + class _NodeportsEncoder(json.JSONEncoder): # SAN: looks like pylint is having an issue here - def default(self, o): # pylint: disable=E0202 + def default(self, o): # pylint: disable=E0202 log.debug("Encoding object: %s", o) if isinstance(o, nodeports.Nodeports): log.debug("Encoding Nodeports object") return { - "version": o._version, # pylint: disable=W0212 - "schema": {"inputs": o._input_schemas, "outputs": o._output_schemas}, # pylint: disable=W0212 - "inputs": o._inputs_payloads, # pylint: disable=W0212 - "outputs": o._outputs_payloads # pylint: disable=W0212 + # pylint: disable=W0212 + "version": o._version, + "schema": {"inputs": o._input_schemas, "outputs": o._output_schemas,}, + "inputs": o._inputs_payloads, + "outputs": o._outputs_payloads, } if isinstance(o, SchemaItemsList): log.debug("Encoding SchemaItemsList object") items = { - key:{ - item_key:item_value for item_key, item_value in item._asdict().items() if item_key != "key" - } for key, item in o.items() + key: { + item_key: item_value + for item_key, item_value in item._asdict().items() + if item_key != "key" + } + for key, item in o.items() } return items if isinstance(o, DataItemsList): log.debug("Encoding DataItemsList object") - items = {key:item.value for key, item in o.items()} + items = {key: item.value for key, item in o.items()} return items log.debug("Encoding object using defaults") return json.JSONEncoder.default(self, o) + def __decodeNodePorts(dct: Dict): if not all(k in dct for k in config.NODE_KEYS.keys()): raise exceptions.InvalidProtocolError(dct) @@ -100,14 +124,24 @@ def __decodeNodePorts(dct: Dict): schema = dct["schema"] if not all(k in schema for k in ("inputs", "outputs")): raise exceptions.InvalidProtocolError(dct, "invalid schemas") - decoded_input_schema = SchemaItemsList({key:SchemaItem(key=key, **value) for key, value in schema["inputs"].items()}) - decoded_output_schema = SchemaItemsList({key:SchemaItem(key=key, **value) for key, value in schema["outputs"].items()}) + decoded_input_schema = SchemaItemsList( + {key: SchemaItem(key=key, **value) for key, value in schema["inputs"].items()} + ) + decoded_output_schema = SchemaItemsList( + {key: SchemaItem(key=key, **value) for key, value in schema["outputs"].items()} + ) # decode payload - decoded_input_payload = DataItemsList({key:DataItem(key=key, value=value) for key, value in dct["inputs"].items()}) - decoded_output_payload = DataItemsList({key:DataItem(key=key, value=value) for key, value in dct["outputs"].items()}) - - return nodeports.Nodeports(dct["version"], - SchemaItemsList(decoded_input_schema), - SchemaItemsList(decoded_output_schema), - DataItemsList(decoded_input_payload), - DataItemsList(decoded_output_payload)) + decoded_input_payload = DataItemsList( + {key: DataItem(key=key, value=value) for key, value in dct["inputs"].items()} + ) + decoded_output_payload = DataItemsList( + {key: DataItem(key=key, value=value) for key, value in dct["outputs"].items()} + ) + + return nodeports.Nodeports( + dct["version"], + SchemaItemsList(decoded_input_schema), + SchemaItemsList(decoded_output_schema), + DataItemsList(decoded_input_payload), + DataItemsList(decoded_output_payload), + ) diff --git a/packages/simcore-sdk/tests/helpers/utils_futures.py b/packages/simcore-sdk/tests/helpers/utils_futures.py new file mode 100644 index 00000000000..fa3f4461b2c --- /dev/null +++ b/packages/simcore-sdk/tests/helpers/utils_futures.py @@ -0,0 +1,7 @@ +from asyncio import Future + + +def future_with_result(result) -> Future: + f = Future() + f.set_result(result) + return f diff --git a/packages/simcore-sdk/tests/integration/conftest.py b/packages/simcore-sdk/tests/integration/conftest.py index 5c0ce844b01..b13adc57ea5 100644 --- a/packages/simcore-sdk/tests/integration/conftest.py +++ b/packages/simcore-sdk/tests/integration/conftest.py @@ -12,10 +12,14 @@ from typing import Any, Dict, List, Tuple import pytest +from yarl import URL import np_helpers -from simcore_sdk.models.pipeline_models import (Base, ComputationalPipeline, - ComputationalTask) +from simcore_sdk.models.pipeline_models import ( + Base, + ComputationalPipeline, + ComputationalTask, +) from simcore_sdk.node_ports import node_config from utils_docker import get_service_published_port @@ -23,67 +27,101 @@ # FIXTURES pytest_plugins = [ - "fixtures.docker_compose", - "fixtures.docker_swarm", - "fixtures.postgres_service", - "shared_fixtures.minio_fix", + "pytest_simcore.environs", + "pytest_simcore.docker_compose", + "pytest_simcore.docker_swarm", + "pytest_simcore.postgres_service", + "pytest_simcore.minio_service", + "pytest_simcore.simcore_storage_service", ] + +@pytest.fixture +def nodeports_config( + postgres_dsn: Dict[str, str], minio_config: Dict[str, str] +) -> None: + node_config.POSTGRES_DB = postgres_dsn["database"] + node_config.POSTGRES_ENDPOINT = f"{postgres_dsn['host']}:{postgres_dsn['port']}" + node_config.POSTGRES_USER = postgres_dsn["user"] + node_config.POSTGRES_PW = postgres_dsn["password"] + node_config.BUCKET = minio_config["bucket_name"] + + @pytest.fixture -def user_id()->int: +def user_id() -> int: # see fixtures/postgres.py yield 1258 + @pytest.fixture -def s3_simcore_location() ->str: +def s3_simcore_location() -> str: yield np_helpers.SIMCORE_STORE + @pytest.fixture -def filemanager_cfg(docker_stack: Dict, devel_environ: Dict, user_id: str, bucket: str, - postgres_db # waits for db and initializes it - ): - assert "simcore_storage" in docker_stack["services"] - storage_port = devel_environ['STORAGE_ENDPOINT'].split(':')[1] - node_config.STORAGE_ENDPOINT = f"127.0.0.1:{get_service_published_port('storage', storage_port)}" +async def filemanager_cfg( + loop, + storage_service: URL, + devel_environ: Dict, + user_id: str, + bucket: str, + postgres_db, # waits for db and initializes it +): + node_config.STORAGE_ENDPOINT = f"{storage_service.host}:{storage_service.port}" node_config.USER_ID = user_id node_config.BUCKET = bucket yield + @pytest.fixture -def project_id()->str: +def project_id() -> str: return str(uuid.uuid4()) + @pytest.fixture -def node_uuid()->str: +def node_uuid() -> str: return str(uuid.uuid4()) + @pytest.fixture -def file_uuid(project_id, node_uuid)->str: - def create(file_path:Path, project:str=None, node:str=None): +def file_uuid(project_id, node_uuid) -> str: + def create(file_path: Path, project: str = None, node: str = None): if project is None: project = project_id if node is None: node = node_uuid return np_helpers.file_uuid(file_path, project, node) + yield create + @pytest.fixture def default_configuration_file(): return current_dir / "mock" / "default_config.json" + @pytest.fixture def empty_configuration_file(): return current_dir / "mock" / "empty_config.json" -@pytest.fixture(scope='module') + +@pytest.fixture(scope="module") def postgres(postgres_db, postgres_session): # prepare database with default configuration Base.metadata.create_all(postgres_db) yield postgres_session Base.metadata.drop_all(postgres_db) + @pytest.fixture() -def default_configuration(bucket, postgres, default_configuration_file, project_id, node_uuid): +def default_configuration( + nodeports_config, + bucket, + postgres, + default_configuration_file, + project_id, + node_uuid, +): # prepare database with default configuration json_configuration = default_configuration_file.read_text() @@ -98,54 +136,90 @@ def default_configuration(bucket, postgres, default_configuration_file, project_ postgres.query(ComputationalPipeline).delete() postgres.commit() + @pytest.fixture() def node_link(): - def create_node_link(key:str): - return {"nodeUuid":"TEST_NODE_UUID", "output":key} + def create_node_link(key: str): + return {"nodeUuid": "TEST_NODE_UUID", "output": key} + yield create_node_link + @pytest.fixture() -def store_link(s3_client, bucket, file_uuid, s3_simcore_location): - def create_store_link(file_path:Path, project_id:str=None, node_id:str=None): +def store_link(minio_service, bucket, file_uuid, s3_simcore_location): + def create_store_link(file_path: Path, project_id: str = None, node_id: str = None): # upload the file to S3 assert Path(file_path).exists() file_id = file_uuid(file_path, project_id, node_id) # using the s3 client the path must be adapted - #TODO: use the storage sdk instead + # TODO: use the storage sdk instead s3_object = Path(project_id, node_id, Path(file_path).name).as_posix() - s3_client.upload_file(bucket, s3_object, str(file_path)) - return {"store":s3_simcore_location, "path":file_id} + minio_service.upload_file(bucket, s3_object, str(file_path)) + return {"store": s3_simcore_location, "path": file_id} + yield create_store_link + @pytest.fixture(scope="function") -def special_configuration(bucket, postgres, empty_configuration_file: Path, project_id, node_uuid): - def create_config(inputs: List[Tuple[str, str, Any]] =None, outputs: List[Tuple[str, str, Any]] =None, project_id:str =project_id, node_id:str = node_uuid): +def special_configuration( + nodeports_config, + bucket, + postgres, + empty_configuration_file: Path, + project_id, + node_uuid, +): + def create_config( + inputs: List[Tuple[str, str, Any]] = None, + outputs: List[Tuple[str, str, Any]] = None, + project_id: str = project_id, + node_id: str = node_uuid, + ): config_dict = json.loads(empty_configuration_file.read_text()) _assign_config(config_dict, "inputs", inputs) _assign_config(config_dict, "outputs", outputs) project_id = _create_new_pipeline(postgres, project_id) - node_uuid = _set_configuration(postgres, project_id, node_id, json.dumps(config_dict)) + node_uuid = _set_configuration( + postgres, project_id, node_id, json.dumps(config_dict) + ) node_config.NODE_UUID = str(node_uuid) node_config.PROJECT_ID = str(project_id) return config_dict, project_id, node_uuid + yield create_config # teardown postgres.query(ComputationalTask).delete() postgres.query(ComputationalPipeline).delete() postgres.commit() + @pytest.fixture(scope="function") -def special_2nodes_configuration(bucket, postgres, empty_configuration_file: Path, project_id, node_uuid): - def create_config(prev_node_inputs: List[Tuple[str, str, Any]] =None, prev_node_outputs: List[Tuple[str, str, Any]] =None, - inputs: List[Tuple[str, str, Any]] =None, outputs: List[Tuple[str, str, Any]] =None, - project_id:str =project_id, previous_node_id:str = node_uuid, node_id:str = "asdasdadsa"): +def special_2nodes_configuration( + nodeports_config, + bucket, + postgres, + empty_configuration_file: Path, + project_id, + node_uuid, +): + def create_config( + prev_node_inputs: List[Tuple[str, str, Any]] = None, + prev_node_outputs: List[Tuple[str, str, Any]] = None, + inputs: List[Tuple[str, str, Any]] = None, + outputs: List[Tuple[str, str, Any]] = None, + project_id: str = project_id, + previous_node_id: str = node_uuid, + node_id: str = "asdasdadsa", + ): _create_new_pipeline(postgres, project_id) # create previous node previous_config_dict = json.loads(empty_configuration_file.read_text()) _assign_config(previous_config_dict, "inputs", prev_node_inputs) _assign_config(previous_config_dict, "outputs", prev_node_outputs) - previous_node_uuid = _set_configuration(postgres, project_id, previous_node_id, json.dumps(previous_config_dict)) + previous_node_uuid = _set_configuration( + postgres, project_id, previous_node_id, json.dumps(previous_config_dict) + ) # create current node config_dict = json.loads(empty_configuration_file.read_text()) @@ -159,42 +233,54 @@ def create_config(prev_node_inputs: List[Tuple[str, str, Any]] =None, prev_node_ node_config.NODE_UUID = str(node_uuid) node_config.PROJECT_ID = str(project_id) return config_dict, project_id, node_uuid + yield create_config # teardown postgres.query(ComputationalTask).delete() postgres.query(ComputationalPipeline).delete() postgres.commit() -def _create_new_pipeline(session, project:str)->str: + +def _create_new_pipeline(session, project: str) -> str: # pylint: disable=no-member new_Pipeline = ComputationalPipeline(project_id=project) session.add(new_Pipeline) session.commit() return new_Pipeline.project_id -def _set_configuration(session, project_id: str, node_id:str, json_configuration: str): + +def _set_configuration(session, project_id: str, node_id: str, json_configuration: str): node_uuid = node_id json_configuration = json_configuration.replace("SIMCORE_NODE_UUID", str(node_uuid)) configuration = json.loads(json_configuration) - new_Node = ComputationalTask(project_id=project_id, node_id=node_uuid, schema=configuration["schema"], inputs=configuration["inputs"], outputs=configuration["outputs"]) + new_Node = ComputationalTask( + project_id=project_id, + node_id=node_uuid, + schema=configuration["schema"], + inputs=configuration["inputs"], + outputs=configuration["outputs"], + ) session.add(new_Node) session.commit() return node_uuid -def _assign_config(config_dict:dict, port_type:str, entries: List[Tuple[str, str, Any]]): + +def _assign_config( + config_dict: dict, port_type: str, entries: List[Tuple[str, str, Any]] +): if entries is None: return for entry in entries: - config_dict["schema"][port_type].update({ - entry[0]:{ - "label":"some label", - "description": "some description", - "displayOrder":2, - "type": entry[1] + config_dict["schema"][port_type].update( + { + entry[0]: { + "label": "some label", + "description": "some description", + "displayOrder": 2, + "type": entry[1], + } } - }) + ) if not entry[2] is None: - config_dict[port_type].update({ - entry[0]:entry[2] - }) + config_dict[port_type].update({entry[0]: entry[2]}) diff --git a/packages/simcore-sdk/tests/integration/fixtures/__init__.py b/packages/simcore-sdk/tests/integration/fixtures/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/packages/simcore-sdk/tests/integration/fixtures/docker_compose.py b/packages/simcore-sdk/tests/integration/fixtures/docker_compose.py deleted file mode 100644 index 8fb15e68b36..00000000000 --- a/packages/simcore-sdk/tests/integration/fixtures/docker_compose.py +++ /dev/null @@ -1,198 +0,0 @@ -""" - - Main Makefile produces a set of docker-compose configuration files - Here we can find fixtures of most of these configurations - -""" -# pylint:disable=unused-variable -# pylint:disable=unused-argument -# pylint:disable=redefined-outer-name - -import os -import re -import shutil -import socket -import sys -from copy import deepcopy -from pathlib import Path -from typing import Dict, List - -import pytest -import yaml -from utils_docker import run_docker_compose_config - - -@pytest.fixture("session") -def devel_environ(env_devel_file: Path) -> Dict[str, str]: - """ Loads and extends .env-devel - - """ - PATTERN_ENVIRON_EQUAL= re.compile(r"^(\w+)=(.*)$") - env_devel = {} - with env_devel_file.open() as f: - for line in f: - m = PATTERN_ENVIRON_EQUAL.match(line) - if m: - key, value = m.groups() - env_devel[key] = str(value) - - # Customized EXTENSION: change some of the environ to accomodate the test case ---- - if 'REGISTRY_SSL' in env_devel: - env_devel['REGISTRY_SSL'] = 'False' - if 'REGISTRY_URL' in env_devel: - env_devel['REGISTRY_URL'] = "{}:5000".format(_get_ip()) - if 'REGISTRY_USER' in env_devel: - env_devel['REGISTRY_USER'] = "simcore" - if 'REGISTRY_PW' in env_devel: - env_devel['REGISTRY_PW'] = "" - if 'REGISTRY_AUTH' in env_devel: - env_devel['REGISTRY_AUTH'] = False - - if 'SWARM_STACK_NAME' not in os.environ: - env_devel['SWARM_STACK_NAME'] = "simcore" - - return env_devel - - -@pytest.fixture(scope="module") -def temp_folder(request, tmpdir_factory) -> Path: - tmp = Path(tmpdir_factory.mktemp("docker_compose_{}".format(request.module.__name__))) - yield tmp - - -@pytest.fixture(scope="module") -def env_file(osparc_simcore_root_dir: Path, devel_environ: Dict[str, str]) -> Path: - """ - Creates a .env file from the .env-devel - """ - # preserves .env at git_root_dir after test if already exists - env_path = osparc_simcore_root_dir / ".env" - backup_path = osparc_simcore_root_dir / ".env.bak" - if env_path.exists(): - shutil.copy(env_path, backup_path) - - with env_path.open('wt') as fh: - print(f"# TEMPORARY .env auto-generated from env_path in {__file__}") - for key, value in devel_environ.items(): - print(f"{key}={value}", file=fh) - - yield env_path - - env_path.unlink() - if backup_path.exists(): - shutil.copy(backup_path, env_path) - backup_path.unlink() - - - -@pytest.fixture("module") -def simcore_docker_compose(osparc_simcore_root_dir: Path, env_file: Path, temp_folder: Path) -> Dict: - """ Resolves docker-compose for simcore stack in local host - - Produces same as `make .stack-simcore-version.yml` in a temporary folder - """ - COMPOSE_FILENAMES = [ - "docker-compose.yml", - "docker-compose.local.yml" - ] - - # ensures .env at git_root_dir - assert env_file.exists() - assert env_file.parent == osparc_simcore_root_dir - - # target docker-compose path - docker_compose_paths = [osparc_simcore_root_dir / "services" / filename - for filename in COMPOSE_FILENAMES] - assert all(docker_compose_path.exists() for docker_compose_path in docker_compose_paths) - - config = run_docker_compose_config(docker_compose_paths, - workdir=env_file.parent, - destination_path=temp_folder / "simcore_docker_compose.yml") - - return config - -@pytest.fixture("module") -def ops_docker_compose(osparc_simcore_root_dir: Path, env_file: Path, temp_folder: Path) -> Dict: - """ Filters only services in docker-compose-ops.yml and returns yaml data - - Produces same as `make .stack-ops.yml` in a temporary folder - """ - # ensures .env at git_root_dir, which will be used as current directory - assert env_file.exists() - assert env_file.parent == osparc_simcore_root_dir - - # target docker-compose path - docker_compose_path = osparc_simcore_root_dir / "services" / "docker-compose-ops.yml" - assert docker_compose_path.exists() - - config = run_docker_compose_config(docker_compose_path, - workdir=env_file.parent, - destination_path=temp_folder / "ops_docker_compose.yml") - return config - - -@pytest.fixture(scope='module') -def core_services_config_file(request, temp_folder, simcore_docker_compose): - """ Creates a docker-compose config file for every stack of services in'core_services' module variable - File is created in a temp folder - """ - core_services = getattr(request.module, 'core_services', []) # TODO: PC->SAN could also be defined as a fixture instead of a single variable (as with docker_compose) - assert core_services, f"Expected at least one service in 'core_services' within '{request.module.__name__}'" - - docker_compose_path = Path(temp_folder / 'simcore_docker_compose.filtered.yml') - - _filter_services_and_dump(core_services, simcore_docker_compose, docker_compose_path) - - return docker_compose_path - -@pytest.fixture(scope='module') -def ops_services_config_file(request, temp_folder, ops_docker_compose): - """ Creates a docker-compose config file for every stack of services in 'ops_services' module variable - File is created in a temp folder - """ - ops_services = getattr(request.module, 'ops_services', []) - docker_compose_path = Path(temp_folder / 'ops_docker_compose.filtered.yml') - - _filter_services_and_dump(ops_services, ops_docker_compose, docker_compose_path) - - return docker_compose_path - -# HELPERS --------------------------------------------- -def _get_ip()->str: - s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - try: - # doesn't even have to be reachable - s.connect(('10.255.255.255', 1)) - IP = s.getsockname()[0] - except Exception: #pylint: disable=W0703 - IP = '127.0.0.1' - finally: - s.close() - return IP - - -def _filter_services_and_dump(include: List, services_compose: Dict, docker_compose_path: Path): - content = deepcopy(services_compose) - - # filters services - remove = [name for name in content['services'] if name not in include] - for name in remove: - content['services'].pop(name, None) - - for name in include: - service = content['services'][name] - # removes builds (No more) - if "build" in service: - service.pop("build", None) - - # updates current docker-compose (also versioned ... do not change by hand) - with docker_compose_path.open('wt') as fh: - if 'TRAVIS' in os.environ: - # in travis we do not have access to file - print("{:-^100}".format(str(docker_compose_path))) - yaml.dump(content, sys.stdout, default_flow_style=False) - print("-"*100) - else: - # locally we have access to file - print(f"Saving config to '{docker_compose_path}'") - yaml.dump(content, fh, default_flow_style=False) diff --git a/packages/simcore-sdk/tests/integration/fixtures/docker_swarm.py b/packages/simcore-sdk/tests/integration/fixtures/docker_swarm.py deleted file mode 100644 index ddc39034e49..00000000000 --- a/packages/simcore-sdk/tests/integration/fixtures/docker_swarm.py +++ /dev/null @@ -1,123 +0,0 @@ -# pylint:disable=unused-variable -# pylint:disable=unused-argument -# pylint:disable=redefined-outer-name - -import socket -import subprocess -import time -from pathlib import Path - -import docker -import pytest - - -def _get_ip()->str: - s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - try: - # doesn't even have to be reachable - s.connect(('10.255.255.255', 1)) - IP = s.getsockname()[0] - except Exception: #pylint: disable=W0703 - IP = '127.0.0.1' - finally: - s.close() - return IP - -@pytest.fixture(scope='session') -def docker_client(): - client = docker.from_env() - yield client - -@pytest.fixture(scope='module') -def docker_swarm(docker_client): - docker_client.swarm.init(advertise_addr=_get_ip()) - yield - # teardown - assert docker_client.swarm.leave(force=True) - - -@pytest.fixture(scope='module') -def docker_stack(docker_swarm, docker_client, core_services_config_file: Path, ops_services_config_file: Path): - stacks = { - 'simcore': core_services_config_file, - 'ops': ops_services_config_file - } - - # make up-version - stacks_up = [] - for stack_name, stack_config_file in stacks.items(): - subprocess.run( f"docker stack deploy -c {stack_config_file.name} {stack_name}", - shell=True, check=True, - cwd=stack_config_file.parent) - stacks_up.append(stack_name) - - # wait for the stack to come up - def _wait_for_services(retry_count, max_wait_time_s): - pre_states = [ - "NEW", - "PENDING", - "ASSIGNED", - "PREPARING", - "STARTING" - ] - services = docker_client.services.list() - WAIT_TIME_BEFORE_RETRY = 5 - start_time = time.time() - for service in services: - for n in range(retry_count): - assert (time.time() - start_time) < max_wait_time_s - if service.tasks(): - task = service.tasks()[0] - if task["Status"]["State"].upper() not in pre_states: - assert task["Status"]["State"].upper() == "RUNNING" - break - print(f"Waiting for {service.name}...") - time.sleep(WAIT_TIME_BEFORE_RETRY) - - - def _print_services(msg): - from pprint import pprint - print("{:*^100}".format("docker services running " + msg)) - for service in docker_client.services.list(): - pprint(service.attrs) - print("-"*100) - RETRY_COUNT = 12 - WAIT_TIME_BEFORE_FAILING = 60 - _wait_for_services(RETRY_COUNT, WAIT_TIME_BEFORE_FAILING) - _print_services("[BEFORE TEST]") - - yield { - 'stacks': stacks_up, - 'services': [service.name for service in docker_client.services.list()] - } - - _print_services("[AFTER TEST]") - - # clean up. Guarantees that all services are down before creating a new stack! - # - # WORKAROUND https://github.com/moby/moby/issues/30942#issue-207070098 - # - # docker stack rm services - - # until [ -z "$(docker service ls --filter label=com.docker.stack.namespace=services -q)" ] || [ "$limit" -lt 0 ]; do - # sleep 1; - # done - - # until [ -z "$(docker network ls --filter label=com.docker.stack.namespace=services -q)" ] || [ "$limit" -lt 0 ]; do - # sleep 1; - # done - - # make down - # NOTE: remove them in reverse order since stacks share common networks - WAIT_BEFORE_RETRY_SECS = 1 - stacks_up.reverse() - for stack in stacks_up: - subprocess.run(f"docker stack rm {stack}", shell=True, check=True) - - while docker_client.services.list(filters={"label":f"com.docker.stack.namespace={stack}"}): - time.sleep(WAIT_BEFORE_RETRY_SECS) - - while docker_client.networks.list(filters={"label":f"com.docker.stack.namespace={stack}"}): - time.sleep(WAIT_BEFORE_RETRY_SECS) - - _print_services("[AFTER REMOVED]") diff --git a/packages/simcore-sdk/tests/integration/fixtures/postgres_service.py b/packages/simcore-sdk/tests/integration/fixtures/postgres_service.py deleted file mode 100644 index ec1adeaedcf..00000000000 --- a/packages/simcore-sdk/tests/integration/fixtures/postgres_service.py +++ /dev/null @@ -1,65 +0,0 @@ -# pylint:disable=unused-variable -# pylint:disable=unused-argument -# pylint:disable=redefined-outer-name - -import os -from typing import Dict - -import pytest -import sqlalchemy as sa -import tenacity -from sqlalchemy.orm import sessionmaker - -from servicelib.aiopg_utils import PostgresRetryPolicyUponInitialization -from simcore_postgres_database.models.base import metadata -from utils_docker import get_service_published_port - - -@pytest.fixture(scope='module') -def postgres_dsn(docker_stack: Dict, devel_environ: Dict) -> Dict: - assert "simcore_postgres" in docker_stack["services"] - - DSN = { - "user": devel_environ["POSTGRES_USER"], - "password": devel_environ["POSTGRES_PASSWORD"], - "dbname": devel_environ["POSTGRES_DB"], - "host": "127.0.0.1", - "port": get_service_published_port("postgres", devel_environ["POSTGRES_PORT"]) - } - # nodeports takes its configuration from env variables - os.environ["POSTGRES_ENDPOINT"]=f"127.0.0.1:{DSN['port']}" - os.environ["POSTGRES_USER"]=devel_environ["POSTGRES_USER"] - os.environ["POSTGRES_PASSWORD"]=devel_environ["POSTGRES_PASSWORD"] - os.environ["POSTGRES_DB"]=devel_environ["POSTGRES_DB"] - return DSN - -@pytest.fixture(scope='module') -def postgres_db(postgres_dsn: Dict, docker_stack): - url = 'postgresql://{user}:{password}@{host}:{port}/{dbname}'.format(**postgres_dsn) - # NOTE: Comment this to avoid postgres_service - assert wait_till_postgres_responsive(url) - - # Configures db and initializes tables - # Uses syncrounous engine for that - engine = sa.create_engine(url, isolation_level="AUTOCOMMIT") - metadata.create_all(bind=engine, checkfirst=True) - - yield engine - - metadata.drop_all(engine) - engine.dispose() - -@pytest.fixture(scope='module') -def postgres_session(postgres_db): - Session = sessionmaker(postgres_db) - session = Session() - yield session - session.close() - -@tenacity.retry(**PostgresRetryPolicyUponInitialization().kwargs) -def wait_till_postgres_responsive(url): - """Check if something responds to ``url`` """ - engine = sa.create_engine(url) - conn = engine.connect() - conn.close() - return True diff --git a/packages/simcore-sdk/tests/integration/fixtures/redis_service.py b/packages/simcore-sdk/tests/integration/fixtures/redis_service.py deleted file mode 100644 index 2f3ab61f373..00000000000 --- a/packages/simcore-sdk/tests/integration/fixtures/redis_service.py +++ /dev/null @@ -1,42 +0,0 @@ -# pylint:disable=wildcard-import -# pylint:disable=unused-import -# pylint:disable=unused-variable -# pylint:disable=unused-argument -# pylint:disable=redefined-outer-name - -from copy import deepcopy - -import aioredis -import pytest -import tenacity -from yarl import URL - - -@pytest.fixture(scope='module') -async def redis_service(loop, _webserver_dev_config, webserver_environ, docker_stack): - cfg = deepcopy(_webserver_dev_config["resource_manager"]["redis"]) - - host = cfg["host"] - port = cfg["port"] - url = URL(f"redis://{host}:{port}") - - assert await wait_till_redis_responsive(url) - - yield url - - -@tenacity.retry(wait=tenacity.wait_fixed(0.1), stop=tenacity.stop_after_delay(60)) -async def wait_till_redis_responsive(redis_url: URL) -> bool: - client = await aioredis.create_redis_pool(str(redis_url), encoding="utf-8") - client.close() - await client.wait_closed() - return True - -@pytest.fixture(scope='module') -async def redis_client(loop, redis_service): - client = await aioredis.create_redis_pool(str(redis_service), encoding="utf-8") - yield client - - await client.flushall() - client.close() - await client.wait_closed() diff --git a/packages/simcore-sdk/tests/integration/test_dbmanager.py b/packages/simcore-sdk/tests/integration/test_dbmanager.py index f0af1919db6..e3846c10678 100644 --- a/packages/simcore-sdk/tests/integration/test_dbmanager.py +++ b/packages/simcore-sdk/tests/integration/test_dbmanager.py @@ -1,40 +1,45 @@ -# pylint:disable=wildcard-import -# pylint:disable=unused-import # pylint:disable=unused-variable # pylint:disable=unused-argument # pylint:disable=redefined-outer-name -# pylint:disable=too-many-arguments import json +from pathlib import Path +from typing import Dict from simcore_sdk.node_ports import config from simcore_sdk.node_ports.dbmanager import DBManager - core_services = [ - 'postgres', + "postgres", ] -ops_services = [ -# 'adminer' -] +ops_services = ["minio"] -def test_db_manager_read_config(default_configuration): - config_dict = default_configuration + +async def test_db_manager_read_config( + loop, nodeports_config, default_configuration: Dict +): db_manager = DBManager() - ports_configuration_str = db_manager.get_ports_configuration_from_node_uuid(config.NODE_UUID) + ports_configuration_str = await db_manager.get_ports_configuration_from_node_uuid( + config.NODE_UUID + ) loaded_config_specs = json.loads(ports_configuration_str) - assert loaded_config_specs == config_dict + assert loaded_config_specs == default_configuration + -def test_db_manager_write_config(special_configuration, default_configuration_file): +async def test_db_manager_write_config( + loop, nodeports_config, special_configuration, default_configuration_file: Path +): # create an empty config special_configuration() # read the default config json_configuration = default_configuration_file.read_text() # write the default config to the database db_manager = DBManager() - db_manager.write_ports_configuration(json_configuration, config.NODE_UUID) + await db_manager.write_ports_configuration(json_configuration, config.NODE_UUID) - ports_configuration_str = db_manager.get_ports_configuration_from_node_uuid(config.NODE_UUID) + ports_configuration_str = await db_manager.get_ports_configuration_from_node_uuid( + config.NODE_UUID + ) assert json.loads(ports_configuration_str) == json.loads(json_configuration) diff --git a/packages/simcore-sdk/tests/integration/test_filemanager.py b/packages/simcore-sdk/tests/integration/test_filemanager.py index 4d6b7d57c26..1e2d55f136c 100644 --- a/packages/simcore-sdk/tests/integration/test_filemanager.py +++ b/packages/simcore-sdk/tests/integration/test_filemanager.py @@ -11,66 +11,87 @@ import pytest from simcore_sdk.node_ports import exceptions, filemanager -core_services = [ - 'postgres', - 'storage' -] +core_services = ["postgres", "storage"] -ops_services = [ - # 'minio' -] +ops_services = ["minio"] -async def test_valid_upload_download(tmpdir, bucket, filemanager_cfg, user_id, file_uuid, s3_simcore_location): + +async def test_valid_upload_download( + tmpdir, bucket, filemanager_cfg, user_id, file_uuid, s3_simcore_location +): file_path = Path(tmpdir) / "test.test" file_path.write_text("I am a test file") assert file_path.exists() file_id = file_uuid(file_path) store = s3_simcore_location - await filemanager.upload_file(store_id=store, s3_object=file_id, local_file_path=file_path) + await filemanager.upload_file( + store_id=store, s3_object=file_id, local_file_path=file_path + ) download_folder = Path(tmpdir) / "downloads" - download_file_path = await filemanager.download_file(store_id=store, s3_object=file_id, local_folder=download_folder) + download_file_path = await filemanager.download_file( + store_id=store, s3_object=file_id, local_folder=download_folder + ) assert download_file_path.exists() assert download_file_path.name == "test.test" assert filecmp.cmp(download_file_path, file_path) -async def test_invalid_file_path(tmpdir, bucket, filemanager_cfg, user_id, file_uuid, s3_simcore_location): +async def test_invalid_file_path( + tmpdir, bucket, filemanager_cfg, user_id, file_uuid, s3_simcore_location +): file_path = Path(tmpdir) / "test.test" file_path.write_text("I am a test file") assert file_path.exists() - file_id = file_uuid(file_path) store = s3_simcore_location with pytest.raises(FileNotFoundError): - await filemanager.upload_file(store_id=store, s3_object=file_id, local_file_path=Path(tmpdir)/"some other file.txt") + await filemanager.upload_file( + store_id=store, + s3_object=file_id, + local_file_path=Path(tmpdir) / "some other file.txt", + ) download_folder = Path(tmpdir) / "downloads" with pytest.raises(exceptions.S3InvalidPathError): - await filemanager.download_file(store_id=store, s3_object=file_id, local_folder=download_folder) + await filemanager.download_file( + store_id=store, s3_object=file_id, local_folder=download_folder + ) -async def test_invalid_fileid(tmpdir, bucket, filemanager_cfg, user_id, s3_simcore_location): +async def test_invalid_fileid( + tmpdir, bucket, filemanager_cfg, user_id, s3_simcore_location +): file_path = Path(tmpdir) / "test.test" file_path.write_text("I am a test file") assert file_path.exists() store = s3_simcore_location with pytest.raises(exceptions.StorageInvalidCall): - await filemanager.upload_file(store_id=store, s3_object="", local_file_path=file_path) + await filemanager.upload_file( + store_id=store, s3_object="", local_file_path=file_path + ) with pytest.raises(exceptions.StorageServerIssue): - await filemanager.upload_file(store_id=store, s3_object="file_id", local_file_path=file_path) + await filemanager.upload_file( + store_id=store, s3_object="file_id", local_file_path=file_path + ) download_folder = Path(tmpdir) / "downloads" with pytest.raises(exceptions.StorageInvalidCall): - await filemanager.download_file(store_id=store, s3_object="", local_folder=download_folder) + await filemanager.download_file( + store_id=store, s3_object="", local_folder=download_folder + ) with pytest.raises(exceptions.S3InvalidPathError): - await filemanager.download_file(store_id=store, s3_object="file_id", local_folder=download_folder) + await filemanager.download_file( + store_id=store, s3_object="file_id", local_folder=download_folder + ) -async def test_invalid_store(tmpdir, bucket, filemanager_cfg, user_id, file_uuid, s3_simcore_location): +async def test_invalid_store( + tmpdir, bucket, filemanager_cfg, user_id, file_uuid, s3_simcore_location +): file_path = Path(tmpdir) / "test.test" file_path.write_text("I am a test file") assert file_path.exists() @@ -78,8 +99,12 @@ async def test_invalid_store(tmpdir, bucket, filemanager_cfg, user_id, file_uuid file_id = file_uuid(file_path) store = "somefunkystore" with pytest.raises(exceptions.S3InvalidStore): - await filemanager.upload_file(store_name=store, s3_object=file_id, local_file_path=file_path) + await filemanager.upload_file( + store_name=store, s3_object=file_id, local_file_path=file_path + ) download_folder = Path(tmpdir) / "downloads" with pytest.raises(exceptions.S3InvalidStore): - await filemanager.download_file(store_name=store, s3_object=file_id, local_folder=download_folder) + await filemanager.download_file( + store_name=store, s3_object=file_id, local_folder=download_folder + ) diff --git a/packages/simcore-sdk/tests/integration/test_nodeports.py b/packages/simcore-sdk/tests/integration/test_nodeports.py index 80b8cd68784..ed918490eef 100644 --- a/packages/simcore-sdk/tests/integration/test_nodeports.py +++ b/packages/simcore-sdk/tests/integration/test_nodeports.py @@ -16,241 +16,353 @@ import np_helpers # pylint: disable=no-name-in-module -core_services = [ - 'postgres', - 'storage' -] +core_services = ["postgres", "storage"] -ops_services = [ - # 'minio' -] +ops_services = ["minio"] -def _check_port_valid(ports, config_dict: dict, port_type:str, key_name: str, key): - assert getattr(ports, port_type)[key].key == key_name + +async def _check_port_valid( + ports, config_dict: dict, port_type: str, key_name: str, key +): + assert (await getattr(ports, port_type))[key].key == key_name # check required values - assert getattr(ports, port_type)[key].label == config_dict["schema"][port_type][key_name]["label"] - assert getattr(ports, port_type)[key].description == config_dict["schema"][port_type][key_name]["description"] - assert getattr(ports, port_type)[key].type == config_dict["schema"][port_type][key_name]["type"] - assert getattr(ports, port_type)[key].displayOrder == config_dict["schema"][port_type][key_name]["displayOrder"] + assert (await getattr(ports, port_type))[key].label == config_dict["schema"][ + port_type + ][key_name]["label"] + assert (await getattr(ports, port_type))[key].description == config_dict["schema"][ + port_type + ][key_name]["description"] + assert (await getattr(ports, port_type))[key].type == config_dict["schema"][ + port_type + ][key_name]["type"] + assert (await getattr(ports, port_type))[key].displayOrder == config_dict["schema"][ + port_type + ][key_name]["displayOrder"] # check optional values if "defaultValue" in config_dict["schema"][port_type][key_name]: - assert getattr(ports, port_type)[key].defaultValue == config_dict["schema"][port_type][key_name]["defaultValue"] + assert (await getattr(ports, port_type))[key].defaultValue == config_dict[ + "schema" + ][port_type][key_name]["defaultValue"] else: - assert getattr(ports, port_type)[key].defaultValue == None + assert (await getattr(ports, port_type))[key].defaultValue == None if "fileToKeyMap" in config_dict["schema"][port_type][key_name]: - assert getattr(ports, port_type)[key].fileToKeyMap == config_dict["schema"][port_type][key_name]["fileToKeyMap"] + assert (await getattr(ports, port_type))[key].fileToKeyMap == config_dict[ + "schema" + ][port_type][key_name]["fileToKeyMap"] else: - assert getattr(ports, port_type)[key].fileToKeyMap == None + assert (await getattr(ports, port_type))[key].fileToKeyMap == None if "widget" in config_dict["schema"][port_type][key_name]: - assert getattr(ports, port_type)[key].widget == config_dict["schema"][port_type][key_name]["widget"] + assert (await getattr(ports, port_type))[key].widget == config_dict["schema"][ + port_type + ][key_name]["widget"] else: - assert getattr(ports, port_type)[key].widget == None + assert (await getattr(ports, port_type))[key].widget == None # check payload values if key_name in config_dict[port_type]: - assert getattr(ports, port_type)[key].value == config_dict[port_type][key_name] + assert (await getattr(ports, port_type))[key].value == config_dict[port_type][ + key_name + ] elif "defaultValue" in config_dict["schema"][port_type][key_name]: - assert getattr(ports, port_type)[key].value == config_dict["schema"][port_type][key_name]["defaultValue"] + assert (await getattr(ports, port_type))[key].value == config_dict["schema"][ + port_type + ][key_name]["defaultValue"] else: - assert getattr(ports, port_type)[key].value == None + assert (await getattr(ports, port_type))[key].value == None -def _check_ports_valid(ports, config_dict: dict, port_type:str): +async def _check_ports_valid(ports, config_dict: dict, port_type: str): for key in config_dict["schema"][port_type].keys(): # test using "key" name - _check_port_valid(ports, config_dict, port_type, key, key) + await _check_port_valid(ports, config_dict, port_type, key, key) # test using index key_index = list(config_dict["schema"][port_type].keys()).index(key) - _check_port_valid(ports, config_dict, port_type, key, key_index) + await _check_port_valid(ports, config_dict, port_type, key, key_index) -def check_config_valid(ports, config_dict: dict): - _check_ports_valid(ports, config_dict, "inputs") - _check_ports_valid(ports, config_dict, "outputs") +async def check_config_valid(ports, config_dict: dict): + await _check_ports_valid(ports, config_dict, "inputs") + await _check_ports_valid(ports, config_dict, "outputs") -def test_default_configuration(default_configuration): # pylint: disable=W0613, W0621 +async def test_default_configuration( + loop, default_configuration +): # pylint: disable=W0613, W0621 config_dict = default_configuration - check_config_valid(node_ports.ports(), config_dict) + await check_config_valid(await node_ports.ports(), config_dict) -def test_invalid_ports(special_configuration): +async def test_invalid_ports(loop, special_configuration): config_dict, _, _ = special_configuration() - PORTS = node_ports.ports() - check_config_valid(PORTS, config_dict) + PORTS = await node_ports.ports() + await check_config_valid(PORTS, config_dict) - assert not PORTS.inputs - assert not PORTS.outputs + assert not (await PORTS.inputs) + assert not (await PORTS.outputs) with pytest.raises(exceptions.UnboundPortError): - PORTS.inputs[0] + (await PORTS.inputs)[0] with pytest.raises(exceptions.UnboundPortError): - PORTS.outputs[0] - - -@pytest.mark.parametrize("item_type, item_value, item_pytype", [ - ("integer", 26, int), - ("integer", 0, int), - ("integer", -52, int), - ("number", -746.4748, float), - ("number", 0.0, float), - ("number", 4566.11235, float), - ("boolean", False, bool), - ("boolean", True, bool), - ("string", "test-string", str), - ("string", "", str) -]) -async def test_port_value_accessors(special_configuration, item_type, item_value, item_pytype): # pylint: disable=W0613, W0621 + (await PORTS.outputs)[0] + + +@pytest.mark.parametrize( + "item_type, item_value, item_pytype", + [ + ("integer", 26, int), + ("integer", 0, int), + ("integer", -52, int), + ("number", -746.4748, float), + ("number", 0.0, float), + ("number", 4566.11235, float), + ("boolean", False, bool), + ("boolean", True, bool), + ("string", "test-string", str), + ("string", "", str), + ], +) +async def test_port_value_accessors( + special_configuration, item_type, item_value, item_pytype +): # pylint: disable=W0613, W0621 item_key = "some key" - config_dict, _, _ = special_configuration(inputs=[(item_key, item_type, item_value)], outputs=[(item_key, item_type, None)]) - PORTS = node_ports.ports() - check_config_valid(PORTS, config_dict) + config_dict, _, _ = special_configuration( + inputs=[(item_key, item_type, item_value)], + outputs=[(item_key, item_type, None)], + ) + PORTS = await node_ports.ports() + await check_config_valid(PORTS, config_dict) - assert isinstance(await PORTS.inputs[item_key].get(), item_pytype) - assert await PORTS.inputs[item_key].get() == item_value - assert await PORTS.outputs[item_key].get() is None + assert isinstance(await (await PORTS.inputs)[item_key].get(), item_pytype) + assert await (await PORTS.inputs)[item_key].get() == item_value + assert await (await PORTS.outputs)[item_key].get() is None assert isinstance(await PORTS.get(item_key), item_pytype) assert await PORTS.get(item_key) == item_value - await PORTS.outputs[item_key].set(item_value) - assert PORTS.outputs[item_key].value == item_value - assert isinstance(await PORTS.outputs[item_key].get(), item_pytype) - assert await PORTS.outputs[item_key].get() == item_value - - -@pytest.mark.parametrize("item_type, item_value, item_pytype, config_value", [ - ("data:*/*", __file__, Path, {"store":"0", "path":__file__}), - ("data:text/*", __file__, Path, {"store":"0", "path":__file__}), - ("data:text/py", __file__, Path, {"store":"0", "path":__file__}), -]) -async def test_port_file_accessors(special_configuration, filemanager_cfg, s3_simcore_location, bucket, item_type, item_value, item_pytype, config_value): # pylint: disable=W0613, W0621 - config_dict, project_id, node_uuid = special_configuration(inputs=[("in_1", item_type, config_value)], outputs=[("out_34", item_type, None)]) - PORTS = node_ports.ports() - check_config_valid(PORTS, config_dict) - assert await PORTS.outputs["out_34"].get() is None # check emptyness + await (await PORTS.outputs)[item_key].set(item_value) + assert (await PORTS.outputs)[item_key].value == item_value + assert isinstance(await (await PORTS.outputs)[item_key].get(), item_pytype) + assert await (await PORTS.outputs)[item_key].get() == item_value + + +@pytest.mark.parametrize( + "item_type, item_value, item_pytype, config_value", + [ + ("data:*/*", __file__, Path, {"store": "0", "path": __file__}), + ("data:text/*", __file__, Path, {"store": "0", "path": __file__}), + ("data:text/py", __file__, Path, {"store": "0", "path": __file__}), + ], +) +async def test_port_file_accessors( + special_configuration, + filemanager_cfg, + s3_simcore_location, + bucket, + item_type, + item_value, + item_pytype, + config_value, +): # pylint: disable=W0613, W0621 + config_dict, project_id, node_uuid = special_configuration( + inputs=[("in_1", item_type, config_value)], + outputs=[("out_34", item_type, None)], + ) + PORTS = await node_ports.ports() + await check_config_valid(PORTS, config_dict) + assert await (await PORTS.outputs)["out_34"].get() is None # check emptyness # with pytest.raises(exceptions.S3InvalidPathError): # await PORTS.inputs["in_1"].get() # this triggers an upload to S3 + configuration change - await PORTS.outputs["out_34"].set(item_value) + await (await PORTS.outputs)["out_34"].set(item_value) # this is the link to S3 storage - assert PORTS.outputs["out_34"].value == {"store":s3_simcore_location, "path":Path(str(project_id), str(node_uuid), Path(item_value).name).as_posix()} + assert (await PORTS.outputs)["out_34"].value == { + "store": s3_simcore_location, + "path": Path(str(project_id), str(node_uuid), Path(item_value).name).as_posix(), + } # this triggers a download from S3 to a location in /tempdir/simcorefiles/item_key - assert isinstance(await PORTS.outputs["out_34"].get(), item_pytype) - assert (await PORTS.outputs["out_34"].get()).exists() - assert str(await PORTS.outputs["out_34"].get()).startswith(str(Path(tempfile.gettempdir(), "simcorefiles", "out_34"))) + assert isinstance(await (await PORTS.outputs)["out_34"].get(), item_pytype) + assert (await (await PORTS.outputs)["out_34"].get()).exists() + assert str(await (await PORTS.outputs)["out_34"].get()).startswith( + str(Path(tempfile.gettempdir(), "simcorefiles", "out_34")) + ) filecmp.clear_cache() - assert filecmp.cmp(item_value, await PORTS.outputs["out_34"].get()) + assert filecmp.cmp(item_value, await (await PORTS.outputs)["out_34"].get()) -def test_adding_new_ports(special_configuration, postgres_session): +async def test_adding_new_ports(special_configuration, postgres_session): config_dict, project_id, node_uuid = special_configuration() - PORTS = node_ports.ports() - check_config_valid(PORTS, config_dict) + PORTS = await node_ports.ports() + await check_config_valid(PORTS, config_dict) # check empty configuration - assert not PORTS.inputs - assert not PORTS.outputs + assert not (await PORTS.inputs) + assert not (await PORTS.outputs) # replace the configuration now, add an input - config_dict["schema"]["inputs"].update({ - "in_15":{ - "label": "additional data", - "description": "here some additional data", - "displayOrder":2, - "type": "integer"}}) - config_dict["inputs"].update({"in_15":15}) - np_helpers.update_configuration(postgres_session, project_id, node_uuid, config_dict) #pylint: disable=E1101 - check_config_valid(PORTS, config_dict) + config_dict["schema"]["inputs"].update( + { + "in_15": { + "label": "additional data", + "description": "here some additional data", + "displayOrder": 2, + "type": "integer", + } + } + ) + config_dict["inputs"].update({"in_15": 15}) + np_helpers.update_configuration( + postgres_session, project_id, node_uuid, config_dict + ) # pylint: disable=E1101 + await check_config_valid(PORTS, config_dict) # # replace the configuration now, add an output - config_dict["schema"]["outputs"].update({ - "out_15":{ - "label": "output data", - "description": "a cool output", - "displayOrder":2, - "type": "boolean"}}) - np_helpers.update_configuration(postgres_session, project_id, node_uuid, config_dict) #pylint: disable=E1101 - check_config_valid(PORTS, config_dict) - - -def test_removing_ports(special_configuration, postgres_session): - config_dict, project_id, node_uuid = special_configuration(inputs=[("in_14", "integer", 15), - ("in_17", "boolean", False)], - outputs=[("out_123", "string", "blahblah"), - ("out_2", "number", -12.3)]) #pylint: disable=W0612 - PORTS = node_ports.ports() - check_config_valid(PORTS, config_dict) + config_dict["schema"]["outputs"].update( + { + "out_15": { + "label": "output data", + "description": "a cool output", + "displayOrder": 2, + "type": "boolean", + } + } + ) + np_helpers.update_configuration( + postgres_session, project_id, node_uuid, config_dict + ) # pylint: disable=E1101 + await check_config_valid(PORTS, config_dict) + + +async def test_removing_ports(special_configuration, postgres_session): + config_dict, project_id, node_uuid = special_configuration( + inputs=[("in_14", "integer", 15), ("in_17", "boolean", False)], + outputs=[("out_123", "string", "blahblah"), ("out_2", "number", -12.3)], + ) # pylint: disable=W0612 + PORTS = await node_ports.ports() + await check_config_valid(PORTS, config_dict) # let's remove the first input del config_dict["schema"]["inputs"]["in_14"] del config_dict["inputs"]["in_14"] - np_helpers.update_configuration(postgres_session, project_id, node_uuid, config_dict) #pylint: disable=E1101 - check_config_valid(PORTS, config_dict) + np_helpers.update_configuration( + postgres_session, project_id, node_uuid, config_dict + ) # pylint: disable=E1101 + await check_config_valid(PORTS, config_dict) # let's do the same for the second output del config_dict["schema"]["outputs"]["out_2"] del config_dict["outputs"]["out_2"] - np_helpers.update_configuration(postgres_session, project_id, node_uuid, config_dict) #pylint: disable=E1101 - check_config_valid(PORTS, config_dict) - - -@pytest.mark.parametrize("item_type, item_value, item_pytype", [ - ("integer", 26, int), - ("integer", 0, int), - ("integer", -52, int), - ("number", -746.4748, float), - ("number", 0.0, float), - ("number", 4566.11235, float), - ("boolean", False, bool), - ("boolean", True, bool), - ("string", "test-string", str), - ("string", "", str), -]) -async def test_get_value_from_previous_node(special_2nodes_configuration, node_link, item_type, item_value, item_pytype): - config_dict, _, _ = special_2nodes_configuration(prev_node_outputs=[("output_123", item_type, item_value)], - inputs=[("in_15", item_type, node_link("output_123"))]) - PORTS = node_ports.ports() - - check_config_valid(PORTS, config_dict) - input_value = await PORTS.inputs["in_15"].get() + np_helpers.update_configuration( + postgres_session, project_id, node_uuid, config_dict + ) # pylint: disable=E1101 + await check_config_valid(PORTS, config_dict) + + +@pytest.mark.parametrize( + "item_type, item_value, item_pytype", + [ + ("integer", 26, int), + ("integer", 0, int), + ("integer", -52, int), + ("number", -746.4748, float), + ("number", 0.0, float), + ("number", 4566.11235, float), + ("boolean", False, bool), + ("boolean", True, bool), + ("string", "test-string", str), + ("string", "", str), + ], +) +async def test_get_value_from_previous_node( + special_2nodes_configuration, node_link, item_type, item_value, item_pytype +): + config_dict, _, _ = special_2nodes_configuration( + prev_node_outputs=[("output_123", item_type, item_value)], + inputs=[("in_15", item_type, node_link("output_123"))], + ) + PORTS = await node_ports.ports() + + await check_config_valid(PORTS, config_dict) + input_value = await (await PORTS.inputs)["in_15"].get() assert isinstance(input_value, item_pytype) - assert await PORTS.inputs["in_15"].get() == item_value - - -@pytest.mark.parametrize("item_type, item_value, item_pytype", [ - ("data:*/*", __file__, Path), - ("data:text/*", __file__, Path), - ("data:text/py", __file__, Path), -]) -async def test_get_file_from_previous_node(special_2nodes_configuration, project_id, node_uuid, filemanager_cfg, node_link, store_link, item_type, item_value, item_pytype): - config_dict, _, _ = special_2nodes_configuration(prev_node_outputs=[("output_123", item_type, store_link(item_value, project_id, node_uuid))], - inputs=[("in_15", item_type, node_link("output_123"))], - project_id=project_id, previous_node_id=node_uuid) - PORTS = node_ports.ports() - check_config_valid(PORTS, config_dict) - file_path = await PORTS.inputs["in_15"].get() + assert await (await PORTS.inputs)["in_15"].get() == item_value + + +@pytest.mark.parametrize( + "item_type, item_value, item_pytype", + [ + ("data:*/*", __file__, Path), + ("data:text/*", __file__, Path), + ("data:text/py", __file__, Path), + ], +) +async def test_get_file_from_previous_node( + special_2nodes_configuration, + project_id, + node_uuid, + filemanager_cfg, + node_link, + store_link, + item_type, + item_value, + item_pytype, +): + config_dict, _, _ = special_2nodes_configuration( + prev_node_outputs=[ + ("output_123", item_type, store_link(item_value, project_id, node_uuid)) + ], + inputs=[("in_15", item_type, node_link("output_123"))], + project_id=project_id, + previous_node_id=node_uuid, + ) + PORTS = await node_ports.ports() + await check_config_valid(PORTS, config_dict) + file_path = await (await PORTS.inputs)["in_15"].get() assert isinstance(file_path, item_pytype) - assert file_path == Path(tempfile.gettempdir(), "simcorefiles", "in_15", Path(item_value).name) + assert file_path == Path( + tempfile.gettempdir(), "simcorefiles", "in_15", Path(item_value).name + ) assert file_path.exists() filecmp.clear_cache() assert filecmp.cmp(file_path, item_value) -@pytest.mark.parametrize("item_type, item_value, item_alias, item_pytype", [ - ("data:*/*", __file__, Path(__file__).name, Path), - ("data:*/*", __file__, "some funky name.txt", Path), - ("data:text/*", __file__, "some funky name without extension", Path), - ("data:text/py", __file__, "öä$äö2-34 name without extension", Path), -]) -async def test_get_file_from_previous_node_with_mapping_of_same_key_name(special_2nodes_configuration, project_id, node_uuid, filemanager_cfg, node_link, store_link, postgres_session, item_type, item_value, item_alias, item_pytype): - config_dict, _, this_node_uuid = special_2nodes_configuration(prev_node_outputs=[("in_15", item_type, store_link(item_value, project_id, node_uuid))], - inputs=[("in_15", item_type, node_link("in_15"))], - project_id=project_id, previous_node_id=node_uuid) - PORTS = node_ports.ports() - check_config_valid(PORTS, config_dict) + +@pytest.mark.parametrize( + "item_type, item_value, item_alias, item_pytype", + [ + ("data:*/*", __file__, Path(__file__).name, Path), + ("data:*/*", __file__, "some funky name.txt", Path), + ("data:text/*", __file__, "some funky name without extension", Path), + ("data:text/py", __file__, "öä$äö2-34 name without extension", Path), + ], +) +async def test_get_file_from_previous_node_with_mapping_of_same_key_name( + special_2nodes_configuration, + project_id, + node_uuid, + filemanager_cfg, + node_link, + store_link, + postgres_session, + item_type, + item_value, + item_alias, + item_pytype, +): + config_dict, _, this_node_uuid = special_2nodes_configuration( + prev_node_outputs=[ + ("in_15", item_type, store_link(item_value, project_id, node_uuid)) + ], + inputs=[("in_15", item_type, node_link("in_15"))], + project_id=project_id, + previous_node_id=node_uuid, + ) + PORTS = await node_ports.ports() + await check_config_valid(PORTS, config_dict) # add a filetokeymap - config_dict["schema"]["inputs"]["in_15"]["fileToKeyMap"] = {item_alias:"in_15"} - np_helpers.update_configuration(postgres_session, project_id, this_node_uuid, config_dict) #pylint: disable=E1101 - check_config_valid(PORTS, config_dict) - file_path = await PORTS.inputs["in_15"].get() + config_dict["schema"]["inputs"]["in_15"]["fileToKeyMap"] = {item_alias: "in_15"} + np_helpers.update_configuration( + postgres_session, project_id, this_node_uuid, config_dict + ) # pylint: disable=E1101 + await check_config_valid(PORTS, config_dict) + file_path = await (await PORTS.inputs)["in_15"].get() assert isinstance(file_path, item_pytype) assert file_path == Path(tempfile.gettempdir(), "simcorefiles", "in_15", item_alias) assert file_path.exists() @@ -258,31 +370,50 @@ async def test_get_file_from_previous_node_with_mapping_of_same_key_name(special assert filecmp.cmp(file_path, item_value) -@pytest.mark.parametrize("item_type, item_value, item_alias, item_pytype", [ - ("data:*/*", __file__, Path(__file__).name, Path), - ("data:*/*", __file__, "some funky name.txt", Path), - ("data:text/*", __file__, "some funky name without extension", Path), - ("data:text/py", __file__, "öä$äö2-34 name without extension", Path), -]) -async def test_file_mapping(special_configuration, project_id, node_uuid, filemanager_cfg, s3_simcore_location, bucket, store_link, postgres_session, item_type, item_value, item_alias, item_pytype): +@pytest.mark.parametrize( + "item_type, item_value, item_alias, item_pytype", + [ + ("data:*/*", __file__, Path(__file__).name, Path), + ("data:*/*", __file__, "some funky name.txt", Path), + ("data:text/*", __file__, "some funky name without extension", Path), + ("data:text/py", __file__, "öä$äö2-34 name without extension", Path), + ], +) +async def test_file_mapping( + special_configuration, + project_id, + node_uuid, + filemanager_cfg, + s3_simcore_location, + bucket, + store_link, + postgres_session, + item_type, + item_value, + item_alias, + item_pytype, +): config_dict, project_id, node_uuid = special_configuration( - inputs=[("in_1", item_type, store_link(item_value, project_id, node_uuid))], - outputs=[("out_1", item_type, None)], - project_id=project_id, - node_id=node_uuid) - PORTS = node_ports.ports() - check_config_valid(PORTS, config_dict) + inputs=[("in_1", item_type, store_link(item_value, project_id, node_uuid))], + outputs=[("out_1", item_type, None)], + project_id=project_id, + node_id=node_uuid, + ) + PORTS = await node_ports.ports() + await check_config_valid(PORTS, config_dict) # add a filetokeymap - config_dict["schema"]["inputs"]["in_1"]["fileToKeyMap"] = {item_alias:"in_1"} - config_dict["schema"]["outputs"]["out_1"]["fileToKeyMap"] = {item_alias:"out_1"} - np_helpers.update_configuration(postgres_session, project_id, node_uuid, config_dict) #pylint: disable=E1101 - check_config_valid(PORTS, config_dict) - file_path = await PORTS.inputs["in_1"].get() + config_dict["schema"]["inputs"]["in_1"]["fileToKeyMap"] = {item_alias: "in_1"} + config_dict["schema"]["outputs"]["out_1"]["fileToKeyMap"] = {item_alias: "out_1"} + np_helpers.update_configuration( + postgres_session, project_id, node_uuid, config_dict + ) # pylint: disable=E1101 + await check_config_valid(PORTS, config_dict) + file_path = await (await PORTS.inputs)["in_1"].get() assert isinstance(file_path, item_pytype) assert file_path == Path(tempfile.gettempdir(), "simcorefiles", "in_1", item_alias) # let's get it a second time to see if replacing works - file_path = await PORTS.inputs["in_1"].get() + file_path = await (await PORTS.inputs)["in_1"].get() assert isinstance(file_path, item_pytype) assert file_path == Path(tempfile.gettempdir(), "simcorefiles", "in_1", item_alias) @@ -293,4 +424,7 @@ async def test_file_mapping(special_configuration, project_id, node_uuid, filema await PORTS.set_file_by_keymap(file_path) file_id = np_helpers.file_uuid(file_path, project_id, node_uuid) - assert PORTS.outputs["out_1"].value == {"store":s3_simcore_location, "path": file_id} + assert (await PORTS.outputs)["out_1"].value == { + "store": s3_simcore_location, + "path": file_id, + } diff --git a/packages/simcore-sdk/tests/shared_fixtures/__init__.py b/packages/simcore-sdk/tests/shared_fixtures/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/packages/simcore-sdk/tests/shared_fixtures/minio_fix.py b/packages/simcore-sdk/tests/shared_fixtures/minio_fix.py deleted file mode 100644 index a3e714415d1..00000000000 --- a/packages/simcore-sdk/tests/shared_fixtures/minio_fix.py +++ /dev/null @@ -1,110 +0,0 @@ -# pylint: disable=redefined-outer-name - -import logging -import os -import socket -import sys -from pathlib import Path -from typing import Dict - -import docker -import pytest -import requests -import tenacity -import yaml -from s3wrapper.s3_client import S3Client - -log = logging.getLogger(__name__) -here = Path(sys.argv[0] if __name__=="__main__" else __file__ ).resolve().parent - -@tenacity.retry(wait=tenacity.wait_fixed(2), stop=tenacity.stop_after_delay(10)) -def _minio_is_responsive(url:str, code:int=403) ->bool: - """Check if something responds to ``url`` syncronously""" - try: - response = requests.get(url) - if response.status_code == code: - log.info("minio is up and running") - return True - except requests.exceptions.RequestException as _e: - pass - - return False - -def _get_ip()->str: - s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - try: - # doesn't even have to be reachable - s.connect(('10.255.255.255', 1)) - IP = s.getsockname()[0] - except Exception: #pylint: disable=W0703 - IP = '127.0.0.1' - finally: - s.close() - log.info("minio is set up to run on IP %s", IP) - return IP - - -@pytest.fixture(scope="session") -def repo_folder_path(): - MAX_ITERATIONS = 7 - - repo_path, n = here.parent, 0 - while not any(repo_path.glob(".git")) and nDict: - client = docker.from_env() - minio_config = {"host":_get_ip(), "port":9001, "s3access":"s3access", "s3secret":"s3secret"} - container = client.containers.run(minio_image_name, command="server /data", - environment=["".join(["MINIO_ACCESS_KEY=", minio_config["s3access"]]), - "".join(["MINIO_SECRET_KEY=", minio_config["s3secret"]])], - ports={'9000':minio_config["port"]}, - detach=True) - url = "http://{}:{}".format(minio_config["host"], minio_config["port"]) - _minio_is_responsive(url) - - # set up env variables - os.environ["S3_ENDPOINT"] = "{}:{}".format(minio_config["host"], minio_config["port"]) - os.environ["S3_ACCESS_KEY"] = minio_config["s3access"] - os.environ["S3_SECRET_KEY"] = minio_config["s3secret"] - log.info("env variables for accessing S3 set") - - # return the host, port to minio - yield minio_config - # tear down - log.info("tearing down minio container") - container.remove(force=True) - -@pytest.fixture(scope="module") -def s3_client(external_minio:Dict)->S3Client: # pylint:disable=redefined-outer-name - s3_endpoint = "{}:{}".format(external_minio["host"], external_minio["port"]) - yield S3Client(s3_endpoint, external_minio["s3access"], external_minio["s3secret"], False) - # tear down - -@pytest.fixture(scope="module") -def bucket(s3_client:S3Client)->str: # pylint: disable=W0621 - bucket_name = "simcore-test" - s3_client.create_bucket(bucket_name, delete_contents_if_exists=True) - # set env variables - os.environ["S3_BUCKET_NAME"] = bucket_name - yield bucket_name - - s3_client.remove_bucket(bucket_name, delete_contents=True) diff --git a/packages/simcore-sdk/tests/unit/conftest.py b/packages/simcore-sdk/tests/unit/conftest.py index ccd08fd3f27..7be424f4a36 100644 --- a/packages/simcore-sdk/tests/unit/conftest.py +++ b/packages/simcore-sdk/tests/unit/conftest.py @@ -23,22 +23,21 @@ current_dir = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent # FIXTURES -pytest_plugins = [ - "shared_fixtures.minio_fix", -] +pytest_plugins = [] -@pytest.fixture(scope='session') + +@pytest.fixture(scope="session") def docker_compose_file(): """ Overrides pytest-docker fixture """ old = os.environ.copy() # docker-compose reads these environs - os.environ['TEST_POSTGRES_DB']="test" - os.environ['TEST_POSTGRES_USER']="test" - os.environ['TEST_POSTGRES_PASSWORD']="test" + os.environ["TEST_POSTGRES_DB"] = "test" + os.environ["TEST_POSTGRES_USER"] = "test" + os.environ["TEST_POSTGRES_PASSWORD"] = "test" - dc_path = current_dir / 'docker-compose.yml' + dc_path = current_dir / "docker-compose.yml" assert dc_path.exists() yield str(dc_path) @@ -46,11 +45,11 @@ def docker_compose_file(): os.environ = old -@pytest.fixture(scope='session') +@pytest.fixture(scope="session") def postgres_service(docker_services, docker_ip): cfg = { "host": docker_ip, - "port": docker_services.port_for('postgres', 5432), + "port": docker_services.port_for("postgres", 5432), "user": "test", "password": "test", "database": "test", @@ -60,13 +59,12 @@ def postgres_service(docker_services, docker_ip): # Wait until service is responsive. docker_services.wait_until_responsive( - check=lambda: is_postgres_responsive(url), - timeout=30.0, - pause=0.1, + check=lambda: is_postgres_responsive(url), timeout=30.0, pause=0.1, ) return url + @pytest.fixture def postgres_db(postgres_service): url = postgres_service @@ -81,6 +79,7 @@ def postgres_db(postgres_service): # metadata.drop_all(engine) engine.dispose() + @pytest.fixture def postgres_session(postgres_db): Session = sessionmaker(postgres_db) @@ -88,6 +87,7 @@ def postgres_session(postgres_db): yield session session.close() + # helpers --------------- def is_postgres_responsive(url): """Check if something responds to ``url`` """ diff --git a/packages/simcore-sdk/tests/unit/test_item.py b/packages/simcore-sdk/tests/unit/test_item.py index 75c172be8fe..e8ce0510cba 100644 --- a/packages/simcore-sdk/tests/unit/test_item.py +++ b/packages/simcore-sdk/tests/unit/test_item.py @@ -7,23 +7,28 @@ from pathlib import Path -import mock import pytest from simcore_sdk.node_ports import config, exceptions from simcore_sdk.node_ports._data_item import DataItem from simcore_sdk.node_ports._item import Item from simcore_sdk.node_ports._schema_item import SchemaItem +from utils_futures import future_with_result def create_item(item_type, item_value): key = "some key" - return Item(SchemaItem(key=key, - label="a label", - description="a description", - type=item_type, - displayOrder=2), DataItem(key=key, - value=item_value)) + return Item( + SchemaItem( + key=key, + label="a label", + description="a description", + type=item_type, + displayOrder=2, + ), + DataItem(key=key, value=item_value), + ) + def test_default_item(): with pytest.raises(exceptions.InvalidProtocolError): @@ -38,8 +43,16 @@ async def test_item(loop): item_value = True display_order = 2 - item = Item(SchemaItem(key=key, label=label, description=description, type=item_type, displayOrder=display_order), - DataItem(key=key, value=item_value)) + item = Item( + SchemaItem( + key=key, + label=label, + description=description, + type=item_type, + displayOrder=display_order, + ), + DataItem(key=key, value=item_value), + ) assert item.key == key assert item.label == label @@ -66,35 +79,45 @@ async def test_invalid_type(): async def test_invalid_value_type(): - #pylint: disable=W0612 + # pylint: disable=W0612 with pytest.raises(exceptions.InvalidItemTypeError) as excinfo: create_item("integer", "not an integer") -@pytest.mark.parametrize("item_type, item_value_to_set, expected_value", [ - ("integer", 26, 26), - ("number", -746.4748, -746.4748), -# ("data:*/*", __file__, {"store":"s3-z43", "path":"undefined/undefined/{filename}".format(filename=Path(__file__).name)}), - ("boolean", False, False), - ("string", "test-string", "test-string") -]) -async def test_set_new_value(bucket, item_type, item_value_to_set, expected_value): # pylint: disable=W0613 - mock_method = mock.Mock() +@pytest.mark.parametrize( + "item_type, item_value_to_set, expected_value", + [ + ("integer", 26, 26), + ("number", -746.4748, -746.4748), + # ("data:*/*", __file__, {"store":"s3-z43", "path":"undefined/undefined/{filename}".format(filename=Path(__file__).name)}), + ("boolean", False, False), + ("string", "test-string", "test-string"), + ], +) +async def test_set_new_value( + item_type, item_value_to_set, expected_value, mocker +): # pylint: disable=W0613 + mock_method = mocker.Mock(return_value=future_with_result("")) item = create_item(item_type, None) item.new_data_cb = mock_method assert await item.get() is None await item.set(item_value_to_set) mock_method.assert_called_with(DataItem(key=item.key, value=expected_value)) -@pytest.mark.parametrize("item_type, item_value_to_set", [ - ("integer", -746.4748), - ("number", "a string"), - ("data:*/*", str(Path(__file__).parent)), - ("boolean", 123), - ("string", True) -]) -async def test_set_new_invalid_value(bucket, item_type, item_value_to_set): # pylint: disable=W0613 +@pytest.mark.parametrize( + "item_type, item_value_to_set", + [ + ("integer", -746.4748), + ("number", "a string"), + ("data:*/*", str(Path(__file__).parent)), + ("boolean", 123), + ("string", True), + ], +) +async def test_set_new_invalid_value( + item_type, item_value_to_set +): # pylint: disable=W0613 item = create_item(item_type, None) assert await item.get() is None with pytest.raises(exceptions.InvalidItemTypeError) as excinfo: diff --git a/packages/simcore-sdk/tests/unit/test_itemstlist.py b/packages/simcore-sdk/tests/unit/test_itemstlist.py index ded98759ced..861465ad649 100644 --- a/packages/simcore-sdk/tests/unit/test_itemstlist.py +++ b/packages/simcore-sdk/tests/unit/test_itemstlist.py @@ -5,7 +5,6 @@ # pylint:disable=redefined-outer-name # pylint:disable=no-member -import mock import pytest from simcore_sdk.node_ports._data_item import DataItem @@ -14,20 +13,41 @@ from simcore_sdk.node_ports._items_list import ItemsList from simcore_sdk.node_ports._schema_item import SchemaItem from simcore_sdk.node_ports._schema_items_list import SchemaItemsList +from utils_futures import future_with_result def create_item(key, item_type, item_value): - return Item(SchemaItem(key=key, - label="a label", - description="a description", - type=item_type, - displayOrder=2), - DataItem(key=key, - value=item_value)) + return Item( + SchemaItem( + key=key, + label="a label", + description="a description", + type=item_type, + displayOrder=2, + ), + DataItem(key=key, value=item_value), + ) + def create_items_list(key_item_value_tuples): - schemas = SchemaItemsList({key:SchemaItem(key=key, label="a label", description="a description", type=item_type, displayOrder=2) for (key, item_type, _) in key_item_value_tuples}) - payloads = DataItemsList({key:DataItem(key=key, value=item_value) for key,_,item_value in key_item_value_tuples}) + schemas = SchemaItemsList( + { + key: SchemaItem( + key=key, + label="a label", + description="a description", + type=item_type, + displayOrder=2, + ) + for (key, item_type, _) in key_item_value_tuples + } + ) + payloads = DataItemsList( + { + key: DataItem(key=key, value=item_value) + for key, _, item_value in key_item_value_tuples + } + ) return ItemsList(schemas, payloads) @@ -38,12 +58,18 @@ def test_default_list(): assert not itemslist.change_notifier assert not itemslist.get_node_from_node_uuid_cb + def test_creating_list(): - itemslist = create_items_list([("1", "integer", 333), ("2", "integer", 333), ("3", "integer", 333)]) + itemslist = create_items_list( + [("1", "integer", 333), ("2", "integer", 333), ("3", "integer", 333)] + ) assert len(itemslist) == 3 + def test_accessing_by_key(): - itemslist = create_items_list([("1", "integer", 333), ("2", "integer", 333), ("3", "integer", 333)]) + itemslist = create_items_list( + [("1", "integer", 333), ("2", "integer", 333), ("3", "integer", 333)] + ) assert itemslist[0].key == "1" assert itemslist["1"].key == "1" assert itemslist[1].key == "2" @@ -51,17 +77,23 @@ def test_accessing_by_key(): assert itemslist[2].key == "3" assert itemslist["3"].key == "3" + def test_access_by_wrong_key(): from simcore_sdk.node_ports import exceptions - itemslist = create_items_list([("1", "integer", 333), ("2", "integer", 333), ("3", "integer", 333)]) + + itemslist = create_items_list( + [("1", "integer", 333), ("2", "integer", 333), ("3", "integer", 333)] + ) with pytest.raises(exceptions.UnboundPortError): print(itemslist["fdoiht"]) -async def test_modifying_items_triggers_cb(): #pylint: disable=C0103 - mock_method = mock.Mock() +async def test_modifying_items_triggers_cb(mocker): # pylint: disable=C0103 + mock_method = mocker.Mock(return_value=future_with_result("")) - itemslist = create_items_list([("1", "integer", 333), ("2", "integer", 333), ("3", "integer", 333)]) + itemslist = create_items_list( + [("1", "integer", 333), ("2", "integer", 333), ("3", "integer", 333)] + ) itemslist.change_notifier = mock_method await itemslist[0].set(-123) mock_method.assert_called_once() diff --git a/services/api-gateway/Dockerfile b/services/api-gateway/Dockerfile index e38029d5619..42bd8b20907 100644 --- a/services/api-gateway/Dockerfile +++ b/services/api-gateway/Dockerfile @@ -17,12 +17,12 @@ ENV SC_USER_ID=8004 \ SC_BOOT_MODE=default RUN adduser \ - --uid ${SC_USER_ID} \ - --disabled-password \ - --gecos "" \ - --shell /bin/sh \ - --home /home/${SC_USER_NAME} \ - ${SC_USER_NAME} + --uid ${SC_USER_ID} \ + --disabled-password \ + --gecos "" \ + --shell /bin/sh \ + --home /home/${SC_USER_NAME} \ + ${SC_USER_NAME} # Sets utf-8 encoding for Python et al @@ -34,7 +34,7 @@ ENV PYTHONDONTWRITEBYTECODE=1 \ # Ensures that the python and pip executables used in the image will be # those from our virtualenv. -ENV PATH="/${VIRTUAL_ENV}/bin:$PATH" +ENV PATH="${VIRTUAL_ENV}/bin:$PATH" EXPOSE 8000 EXPOSE 3000 @@ -50,15 +50,15 @@ ENV SC_BUILD_TARGET=build RUN apt-get update RUN apt-get install -y --no-install-recommends \ - build-essential \ - gcc + build-essential \ + gcc RUN python -m venv ${VIRTUAL_ENV} RUN pip install --upgrade --no-cache-dir \ - pip~=20.0.2 \ - wheel \ - setuptools + pip~=20.0.2 \ + wheel \ + setuptools WORKDIR /build @@ -108,10 +108,10 @@ COPY --chown=scu:scu services/api-gateway/docker services/api-gateway/docker HEALTHCHECK --interval=30s \ - --timeout=20s \ - --start-period=30s \ - --retries=3 \ - CMD ["python3", "services/api-gateway/docker/healthcheck.py", "http://localhost:8000/"] + --timeout=20s \ + --start-period=30s \ + --retries=3 \ + CMD ["python3", "services/api-gateway/docker/healthcheck.py", "http://localhost:8000/"] ENTRYPOINT [ "/bin/sh", "services/api-gateway/docker/entrypoint.sh" ] CMD ["/bin/sh", "services/api-gateway/docker/boot.sh"] diff --git a/services/api-gateway/docker/entrypoint.sh b/services/api-gateway/docker/entrypoint.sh index 756f6cfbd5b..47b831e0fcc 100755 --- a/services/api-gateway/docker/entrypoint.sh +++ b/services/api-gateway/docker/entrypoint.sh @@ -82,4 +82,4 @@ echo "$INFO Starting $* ..." echo " $SC_USER_NAME rights : $(id "$SC_USER_NAME")" echo " local dir : $(ls -al)" -su --preserve-environment --command "export PATH=${PATH}; $*" "$SC_USER_NAME" +su --command "$*" "$SC_USER_NAME" diff --git a/services/director/Dockerfile b/services/director/Dockerfile index 412da6e1408..3ac0a4dd031 100644 --- a/services/director/Dockerfile +++ b/services/director/Dockerfile @@ -1,4 +1,5 @@ -FROM python:3.6-alpine as base +ARG PYTHON_VERSION="3.6.10" +FROM python:${PYTHON_VERSION}-slim as base # # USAGE: # cd sercices/director @@ -10,27 +11,37 @@ FROM python:3.6-alpine as base LABEL maintainer=sanderegg # simcore-user uid=8004(scu) gid=8004(scu) groups=8004(scu) -RUN adduser -D -u 8004 -s /bin/sh -h /home/scu scu - -RUN apk add --no-cache \ - su-exec - -ENV PATH "/home/scu/.local/bin:$PATH" - -# All SC_ variables are customized -ENV SC_PIP pip3 --no-cache-dir -ENV SC_BUILD_TARGET base -ENV SC_BOOT_MODE default - - -ENV REGISTRY_AUTH = '' \ - REGISTRY_USER = '' \ - REGISTRY_PW = '' \ - REGISTRY_URL = '' \ - REGISTRY_VERSION = 'v2' \ - PUBLISHED_HOST_NAME='' \ - SIMCORE_SERVICES_NETWORK_NAME = '' \ - EXTRA_HOSTS_SUFFIX = 'undefined' +ENV SC_USER_ID=8004 \ + SC_USER_NAME=scu \ + SC_BUILD_TARGET=base \ + SC_BOOT_MODE=default + +RUN adduser \ + --uid ${SC_USER_ID} \ + --disabled-password \ + --gecos "" \ + --shell /bin/sh \ + --home /home/${SC_USER_NAME} \ + ${SC_USER_NAME} + +# Sets utf-8 encoding for Python et al +ENV LANG=C.UTF-8 +# Turns off writing .pyc files; superfluous on an ephemeral container. +ENV PYTHONDONTWRITEBYTECODE=1 \ + VIRTUAL_ENV=/home/scu/.venv +# Ensures that the python and pip executables used +# in the image will be those from our virtualenv. +ENV PATH="${VIRTUAL_ENV}/bin:$PATH" + +# environment variables +ENV REGISTRY_AUTH='' \ + REGISTRY_USER='' \ + REGISTRY_PW='' \ + REGISTRY_URL='' \ + REGISTRY_VERSION='v2' \ + PUBLISHED_HOST_NAME='' \ + SIMCORE_SERVICES_NETWORK_NAME='' \ + EXTRA_HOSTS_SUFFIX='undefined' EXPOSE 8080 @@ -43,45 +54,33 @@ EXPOSE 8080 FROM base as build -ENV SC_BUILD_TARGET build - -# Installing client libraries and any other package you need -# -# libpq: client library for PostgreSQL https://www.postgresql.org/docs/9.5/libpq.html -# libstdc++: needed in ujson https://github.com/kohlschutter/junixsocket/issues/33 -# -RUN apk update && \ - apk add --no-cache \ - libpq \ - libstdc++ - - +ENV SC_BUILD_TARGET=build +RUN apt-get update &&\ + apt-get install -y --no-install-recommends \ + build-essential \ + git -# Installing build dependencies (will be deleted in production) -RUN apk add --virtual .build-deps \ - git \ - g++ \ - libc-dev \ - python-dev \ - musl-dev \ - libffi-dev \ - postgresql-dev +# NOTE: python virtualenv is used here such that installed packages may be moved to production image easily by copying the venv +RUN python -m venv ${VIRTUAL_ENV} - -RUN $SC_PIP install --upgrade \ +RUN pip --no-cache-dir install --upgrade \ pip~=20.0.2 \ wheel \ setuptools -WORKDIR /build +# copy director and dependencies +COPY --chown=scu:scu packages /build/packages +COPY --chown=scu:scu services/director /build/services/director -# install base 3rd party dependencies -COPY --chown=scu:scu services/director/requirements/*.txt \ - tmp/director/requirements/ +# install base 3rd party dependencies (NOTE: this speeds up devel mode) +RUN pip --no-cache-dir install -r /build/services/director/requirements/_base.txt -RUN $SC_PIP install \ - -r tmp/director/requirements/_base.txt +# FIXME: +# necessary to prevent duplicated files. +# Will be removed when director is refactored using cookiecutter as this will not be necessary anymore +COPY --chown=scu:scu api/specs/common/schemas/node-meta-v0.0.1.json \ + /build/services/director/src/simcore_service_director/api/v0/oas-parts/schemas/node-meta-v0.0.1.json #--------------------------Cache stage ------------------- # CI in master buils & pushes this target to speed-up image build @@ -91,24 +90,9 @@ RUN $SC_PIP install \ # FROM build as cache -ENV SC_BUILD_TARGET cache - - -COPY --chown=scu:scu packages/service-library /build/packages/service-library -COPY --chown=scu:scu services/director /build/services/director - -# FIXME: -# necessary to prevent duplicated files. -# Will be removed when director is refactored using cookiecutter as this will not be necessary anymore -COPY --chown=scu:scu api/specs/common/schemas/node-meta-v0.0.1.json \ - /build/services/director/src/simcore_service_director/api/v0/oas-parts/schemas/node-meta-v0.0.1.json - WORKDIR /build/services/director - - -RUN $SC_PIP install -r requirements/prod.txt &&\ - $SC_PIP list -v - +ENV SC_BUILD_TARGET=cache +RUN pip --no-cache-dir install -r requirements/prod.txt # --------------------------Production stage ------------------- # Final cleanup up to reduce image size and startup setup @@ -117,26 +101,24 @@ RUN $SC_PIP install -r requirements/prod.txt &&\ # + /home/scu $HOME = WORKDIR # + services/director [scu:scu] # -FROM cache as production +FROM base as production - -ENV SC_BUILD_TARGET production +ENV SC_BUILD_TARGET=production \ + SC_BOOT_MODE=production ENV PYTHONOPTIMIZE=TRUE WORKDIR /home/scu -RUN mkdir -p services/director &&\ - chown scu:scu services/director &&\ - mv /build/services/director/docker services/director/docker &&\ - rm -rf /build - -RUN apk del --no-cache .build-deps +# bring installed package without build tools +COPY --from=cache --chown=scu:scu ${VIRTUAL_ENV} ${VIRTUAL_ENV} +# copy docker entrypoint and boot scripts +COPY --chown=scu:scu services/director/docker services/director/docker HEALTHCHECK --interval=30s \ - --timeout=120s \ - --start-period=30s \ - --retries=3 \ - CMD ["python3", "/home/scu/services/director/docker/healthcheck.py", "http://localhost:8080/v0/"] + --timeout=120s \ + --start-period=30s \ + --retries=3 \ + CMD ["python3", "/home/scu/services/director/docker/healthcheck.py", "http://localhost:8080/v0/"] ENTRYPOINT [ "/bin/sh", "services/director/docker/entrypoint.sh" ] CMD ["/bin/sh", "services/director/docker/boot.sh"] @@ -152,14 +134,8 @@ CMD ["/bin/sh", "services/director/docker/boot.sh"] # FROM build as development -ENV SC_BUILD_TARGET development - - +ENV SC_BUILD_TARGET=development +ENV NODE_SCHEMA_LOCATION=../../../api/specs/common/schemas/node-meta-v0.0.1.json WORKDIR /devel -VOLUME /devel/packages -VOLUME /devel/services/director/ -VOLUME /devel/services/api -ENV NODE_SCHEMA_LOCATION ../../../api/specs/common/schemas/node-meta-v0.0.1.json - ENTRYPOINT [ "/bin/sh", "services/director/docker/entrypoint.sh" ] CMD ["/bin/sh", "services/director/docker/boot.sh"] diff --git a/services/director/docker/boot.sh b/services/director/docker/boot.sh index 86d3e5b7763..dc8d3f98eb5 100755 --- a/services/director/docker/boot.sh +++ b/services/director/docker/boot.sh @@ -1,5 +1,9 @@ #!/bin/sh -# +set -o errexit +set -o nounset + +IFS=$(printf '\n\t') + INFO="INFO: [$(basename "$0")] " # BOOTING application --------------------------------------------- @@ -7,38 +11,23 @@ echo "$INFO" "Booting in ${SC_BOOT_MODE} mode ..." echo " User :$(id "$(whoami)")" echo " Workdir :$(pwd)" -LOG_LEVEL=info -entrypoint='' if [ "${SC_BUILD_TARGET}" = "development" ] then echo "$INFO" "Environment :" printenv | sed 's/=/: /' | sed 's/^/ /' | sort - #-------------------- - LOG_LEVEL=debug - - cd services/director || exit - $SC_PIP install --user -r requirements/dev.txt - cd /devel || exit - - #-------------------- echo "$INFO" "Python :" python --version | sed 's/^/ /' command -v python | sed 's/^/ /' echo "$INFO" "PIP :" - $SC_PIP list | sed 's/^/ /' - - #------------ - echo "$INFO" "setting entrypoint to use watchmedo autorestart..." - entrypoint='watchmedo auto-restart --recursive --pattern="*.py" --' + pip list | sed 's/^/ /' fi # RUNNING application ---------------------------------------- if [ "${SC_BOOT_MODE}" = "debug-ptvsd" ] then - echo - echo "$INFO" "PTVSD Debugger initializing in port 3004" - eval "$entrypoint" python3 -m ptvsd --host 0.0.0.0 --port 3000 -m \ - simcore_service_director --loglevel=$LOG_LEVEL + watchmedo auto-restart --recursive --pattern="*.py" -- \ + python3 -m ptvsd --host 0.0.0.0 --port 3000 -m \ + simcore_service_director --loglevel=${LOGLEVEL} else - exec simcore-service-director --loglevel=$LOG_LEVEL + exec simcore-service-director --loglevel=${LOGLEVEL} fi diff --git a/services/director/docker/entrypoint.sh b/services/director/docker/entrypoint.sh index 71b22d1d7c1..17345332813 100755 --- a/services/director/docker/entrypoint.sh +++ b/services/director/docker/entrypoint.sh @@ -1,6 +1,11 @@ #!/bin/sh -# +set -o errexit +set -o nounset + +IFS=$(printf '\n\t') + INFO="INFO: [$(basename "$0")] " +WARNING="WARNING: [$(basename "$0")] " ERROR="ERROR: [$(basename "$0")] " # This entrypoint script: @@ -12,8 +17,12 @@ ERROR="ERROR: [$(basename "$0")] " echo "$INFO" "Entrypoint for stage ${SC_BUILD_TARGET} ..." echo "$INFO" "User :$(id "$(whoami)")" echo "$INFO" "Workdir :$(pwd)" +echo scuUser :"$(id scu)" +USERNAME=scu +GROUPNAME=scu + if [ "${SC_BUILD_TARGET}" = "development" ] then # NOTE: expects docker run ... -v $(pwd):/devel/services/director @@ -22,26 +31,43 @@ then stat $DEVEL_MOUNT > /dev/null 2>&1 || \ (echo "$ERROR" "You must mount '$DEVEL_MOUNT' to deduce user and group ids" && exit 1) # FIXME: exit does not stop script - USERID=$(stat -c %u $DEVEL_MOUNT) - GROUPID=$(stat -c %g $DEVEL_MOUNT) - GROUPNAME=$(getent group "${GROUPID}" | cut -d: -f1) + USERID=$(stat --format=%u $DEVEL_MOUNT) + GROUPID=$(stat --format=%g $DEVEL_MOUNT) + GROUPNAME=$(getent group "${GROUPID}" | cut --delimiter=: --fields=1) if [ "$USERID" -eq 0 ] then - addgroup scu root + echo "$WARNING" Folder mounted owned by root user... adding "$SC_USER_NAME" to root... + adduser "${SC_USER_NAME}" root else - # take host's credentials + # take host's credentials in scu if [ -z "$GROUPNAME" ] then - GROUPNAME=host_group - addgroup -g "$GROUPID" $GROUPNAME + echo "$INFO" mounted folder from "$USERID", creating new group my"${SC_USER_NAME}" + GROUPNAME=my"${SC_USER_NAME}" + addgroup --gid "$GROUPID" "$GROUPNAME" + # change group property of files already around + find / -path /proc -prune -group "$SC_USER_ID" -exec chgrp --no-dereference "$GROUPNAME" {} \; else - addgroup scu $GROUPNAME + echo "$INFO" "mounted folder from $USERID, adding ${SC_USER_NAME} to $GROUPNAME..." + adduser "$SC_USER_NAME" "$GROUPNAME" fi - deluser scu > /dev/null 2>&1 - adduser -u "$USERID" -G $GROUPNAME -D -s /bin/sh scu + echo "$INFO changing $SC_USER_NAME $SC_USER_ID:$SC_USER_ID to $USERID:$GROUPID" + deluser "${SC_USER_NAME}" > /dev/null 2>&1 + if [ "$SC_USER_NAME" = "$GROUPNAME" ] + then + addgroup --gid "$GROUPID" "$GROUPNAME" + fi + adduser --disabled-password --gecos "" --uid "$USERID" --gid "$GROUPID" --shell /bin/sh "$SC_USER_NAME" --no-create-home + # change user property of files already around + find / -path /proc -prune -user "$SC_USER_ID" -exec chown --no-dereference "$SC_USER_NAME" {} \; fi + + echo "$INFO installing python dependencies..." + cd services/director || exit 1 + pip install --no-cache-dir -r requirements/dev.txt + cd - || exit 1 fi @@ -53,20 +79,24 @@ fi # Appends docker group if socket is mounted DOCKER_MOUNT=/var/run/docker.sock - - if stat $DOCKER_MOUNT > /dev/null 2>&1 then - GROUPID=$(stat -c %g $DOCKER_MOUNT) - GROUPNAME=docker + echo "$INFO detected docker socket is mounted, adding user to group..." + GROUPID=$(stat --format=%g $DOCKER_MOUNT) + GROUPNAME=scdocker - - if ! addgroup -g "$GROUPID" $GROUPNAME > /dev/null 2>&1 + if ! addgroup --gid "$GROUPID" $GROUPNAME > /dev/null 2>&1 then + echo "$WARNING docker group with $GROUPID already exists, getting group name..." # if group already exists in container, then reuse name - GROUPNAME=$(getent group "${GROUPID}" | cut -d: -f1) + GROUPNAME=$(getent group "${GROUPID}" | cut --delimiters=: --fields=1) + echo "$WARNING docker group with $GROUPID has name $GROUPNAME" fi - addgroup scu "$GROUPNAME" + adduser "$SC_USER_NAME" "$GROUPNAME" fi -exec su-exec scu "$@" +echo "$INFO Starting $* ..." +echo " $SC_USER_NAME rights : $(id "$SC_USER_NAME")" +echo " local dir : $(ls -al)" + +su --command "$*" "$SC_USER_NAME" diff --git a/services/director/requirements/ci.txt b/services/director/requirements/ci.txt index 078f0a0922b..84983de50b2 100644 --- a/services/director/requirements/ci.txt +++ b/services/director/requirements/ci.txt @@ -8,7 +8,7 @@ # installs base + tests requirements -r _test.txt - +../../packages/pytest-simcore/ # installs this repo's packages ../../packages/service-library/ diff --git a/services/director/requirements/dev.txt b/services/director/requirements/dev.txt index 377b6beb787..9c29d60c458 100644 --- a/services/director/requirements/dev.txt +++ b/services/director/requirements/dev.txt @@ -15,6 +15,7 @@ watchdog[watchmedo] # installs this repo's packages -e ../../packages/service-library/ +-e ../../packages/pytest-simcore/ # installs current package -e . diff --git a/services/director/tests/conftest.py b/services/director/tests/conftest.py index d69695bf3df..d35d2d1707a 100644 --- a/services/director/tests/conftest.py +++ b/services/director/tests/conftest.py @@ -15,20 +15,15 @@ from simcore_service_director import config, resources pytest_plugins = [ - "fixtures.docker_registry", - "fixtures.docker_swarm", + "pytest_simcore.environs", + "pytest_simcore.docker_compose", + "pytest_simcore.docker_swarm", + "pytest_simcore.docker_registry", "fixtures.fake_services" ] current_dir = Path(sys.argv[0] if __name__ == "__main__" else __file__).parent.absolute() -@pytest.fixture(scope='session') -def osparc_simcore_root_dir(): - root_dir = current_dir.parent.parent.parent.resolve() - assert root_dir.exists(), "Is this service within osparc-simcore repo?" - assert any(root_dir.glob("services/web/server")), "%s not look like rootdir" % root_dir - return root_dir - @pytest.fixture(scope='session') def common_schemas_specs_dir(osparc_simcore_root_dir): specs_dir = osparc_simcore_root_dir/ "api" / "specs" / "common" / "schemas" diff --git a/services/director/tests/fixtures/docker_registry.py b/services/director/tests/fixtures/docker_registry.py deleted file mode 100644 index 4a2106d0405..00000000000 --- a/services/director/tests/fixtures/docker_registry.py +++ /dev/null @@ -1,60 +0,0 @@ -# pylint:disable=unused-import -# pylint:disable=unused-argument -# pylint:disable=redefined-outer-name - -import logging -import time - -import docker -import pytest -import tenacity - -log = logging.getLogger(__name__) - - -@pytest.fixture(scope="session") -def docker_registry(): - # run the registry outside of the stack - docker_client = docker.from_env() - container = docker_client.containers.run("registry:2", - ports={"5000":"5000"}, - environment=["REGISTRY_STORAGE_DELETE_ENABLED=true"], - restart_policy={"Name":"always"}, - detach=True - ) - host = "127.0.0.1" - port = 5000 - url = "{host}:{port}".format(host=host, port=port) - # Wait until we can connect - assert _wait_till_registry_is_responsive(url) - - # test the registry - docker_client = docker.from_env() - # get the hello world example from docker hub - hello_world_image = docker_client.images.pull("hello-world","latest") - # login to private registry - docker_client.login(registry=url, username="simcore") - # tag the image - repo = url + "/hello-world:dev" - assert hello_world_image.tag(repo) == True - # push the image to the private registry - docker_client.images.push(repo) - # wipe the images - docker_client.images.remove(image="hello-world:latest") - docker_client.images.remove(image=hello_world_image.id) - # pull the image from the private registry - private_image = docker_client.images.pull(repo) - docker_client.images.remove(image=private_image.id) - - yield url - - container.stop() - - while docker_client.containers.list(filters={"name": container.name}): - time.sleep(1) - -@tenacity.retry(wait=tenacity.wait_fixed(1), stop=tenacity.stop_after_delay(60)) -def _wait_till_registry_is_responsive(url): - docker_client = docker.from_env() - docker_client.login(registry=url, username="simcore") - return True diff --git a/services/director/tests/fixtures/docker_swarm.py b/services/director/tests/fixtures/docker_swarm.py deleted file mode 100644 index e336c8c0653..00000000000 --- a/services/director/tests/fixtures/docker_swarm.py +++ /dev/null @@ -1,13 +0,0 @@ -import docker -import pytest - -@pytest.fixture(scope="module") -def docker_swarm(): #pylint: disable=W0613, W0621 - client = docker.from_env() - assert client is not None - client.swarm.init() - - yield client - - # teardown - assert client.swarm.leave(force=True) == True diff --git a/services/director/tests/test_producer.py b/services/director/tests/test_producer.py index 22743cb0868..d39581cc545 100644 --- a/services/director/tests/test_producer.py +++ b/services/director/tests/test_producer.py @@ -154,9 +154,8 @@ async def test_interactive_service_published_port(run_services): @pytest.fixture -def docker_network(docker_swarm) -> docker.models.networks.Network: - client = docker_swarm - network = client.networks.create("test_network", driver="overlay", scope="swarm") +def docker_network(docker_client: docker.client.DockerClient, docker_swarm: None) -> docker.models.networks.Network: + network = docker_client.networks.create("test_network", driver="overlay", scope="swarm") config.SIMCORE_SERVICES_NETWORK_NAME = network.name yield network @@ -165,7 +164,7 @@ def docker_network(docker_swarm) -> docker.models.networks.Network: config.SIMCORE_SERVICES_NETWORK_NAME = None -async def test_interactive_service_in_correct_network(docker_network, run_services): +async def test_interactive_service_in_correct_network(docker_network: docker.models.networks.Network, run_services): running_dynamic_services = await run_services(number_comp=0, number_dyn=2, dependant=False) assert len(running_dynamic_services) == 2 diff --git a/services/docker-compose.devel.yml b/services/docker-compose.devel.yml index 50d1e703359..3fc1118a882 100644 --- a/services/docker-compose.devel.yml +++ b/services/docker-compose.devel.yml @@ -18,9 +18,10 @@ services: director: environment: - SC_BOOT_MODE=debug-ptvsd + - LOGLEVEL=debug volumes: - ./director:/devel/services/director - - ../packages/service-library:/devel/packages/service-library + - ../packages:/devel/packages - ../api:/devel/services/api webserver: diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 72a2c11de1e..9e8f1ffb452 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -42,6 +42,7 @@ services: - SIMCORE_SERVICES_NETWORK_NAME=interactive_services_subnet - TRACING_ENABLED=${TRACING_ENABLED:-True} - TRACING_ZIPKIN_ENDPOINT=${TRACING_ZIPKIN_ENDPOINT:-http://jaeger:9411} + - LOGLEVEL=${LOG_LEVEL:-WARNING} volumes: - "/var/run/docker.sock:/var/run/docker.sock" deploy: @@ -103,8 +104,7 @@ services: - RABBIT_PORT=${RABBIT_PORT} - RABBIT_USER=${RABBIT_USER} - RABBIT_PASSWORD=${RABBIT_PASSWORD} - - RABBIT_LOG_CHANNEL=${RABBIT_LOG_CHANNEL} - - RABBIT_PROGRESS_CHANNEL=${RABBIT_PROGRESS_CHANNEL} + - RABBIT_CHANNELS=${RABBIT_CHANNELS} - POSTGRES_ENDPOINT=${POSTGRES_ENDPOINT} - POSTGRES_USER=${POSTGRES_USER} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} diff --git a/services/sidecar/Dockerfile b/services/sidecar/Dockerfile index 71de3080257..d5a94f2550a 100644 --- a/services/sidecar/Dockerfile +++ b/services/sidecar/Dockerfile @@ -1,4 +1,5 @@ -FROM python:3.6-alpine as base +ARG PYTHON_VERSION="3.6.10" +FROM python:${PYTHON_VERSION}-slim as base # # USAGE: # cd sercices/sidecar @@ -10,60 +11,67 @@ FROM python:3.6-alpine as base LABEL maintainer=mguidon # simcore-user uid=8004(scu) gid=8004(scu) groups=8004(scu) -RUN adduser -D -u 8004 -s /bin/sh -h /home/scu scu - -RUN apk add --no-cache \ - su-exec - -ENV PATH "/home/scu/.local/bin:$PATH" - -# All SC_ variables are customized -ENV SC_PIP pip3 --no-cache-dir -ENV SC_BUILD_TARGET base -ENV SC_BOOT_MODE default +ENV SC_USER_ID=8004 \ + SC_USER_NAME=scu \ + SC_BUILD_TARGET=base \ + SC_BOOT_MODE=default + +RUN adduser \ + --uid ${SC_USER_ID} \ + --disabled-password \ + --gecos "" \ + --shell /bin/sh \ + --home /home/${SC_USER_NAME} \ + ${SC_USER_NAME} + +# Sets utf-8 encoding for Python et al +ENV LANG=C.UTF-8 +# Turns off writing .pyc files; superfluous on an ephemeral container. +ENV PYTHONDONTWRITEBYTECODE=1 \ + VIRTUAL_ENV=/home/scu/.venv +# Ensures that the python and pip executables used +# in the image will be those from our virtualenv. +ENV PATH="${VIRTUAL_ENV}/bin:$PATH" # environment variables -ENV SWARM_STACK_NAME = '' \ - SIDECAR_SERVICES_MAX_NANO_CPUS = '4000000000' \ - SIDECAR_SERVICES_MAX_MEMORY_BYTES = '2147483648' \ - SIDECAR_SERVICES_TIMEOUT_SECONDS = '1200' +ENV SWARM_STACK_NAME=""\ + SIDECAR_SERVICES_MAX_NANO_CPUS=4000000000 \ + SIDECAR_SERVICES_MAX_MEMORY_BYTES=2147483648 \ + SIDECAR_SERVICES_TIMEOUT_SECONDS=1200 \ + SIDECAR_INPUT_FOLDER=/home/scu/input \ + SIDECAR_OUTPUT_FOLDER=/home/scu/output \ + SIDECAR_LOG_FOLDER=/home/scu/log EXPOSE 8080 -VOLUME /home/scu/input -VOLUME /home/scu/output -VOLUME /home/scu/log - # -------------------------- Build stage ------------------- # Installs build/package management tools and third party dependencies # # + /build WORKDIR # - FROM base as build -ENV SC_BUILD_TARGET build - - -RUN apk add --no-cache \ - postgresql-dev \ - gcc \ - libc-dev +ENV SC_BUILD_TARGET=build -RUN $SC_PIP install --upgrade \ - pip~=20.0.2 \ - wheel \ - setuptools +RUN apt-get update &&\ + apt-get install -y --no-install-recommends \ + build-essential -WORKDIR /build +# NOTE: python virtualenv is used here such that installed packages may be moved to production image easily by copying the venv +RUN python -m venv ${VIRTUAL_ENV} -# install base 3rd party dependencies -COPY --chown=scu:scu services/sidecar/requirements/*.txt \ - tmp/sidecar/requirements/ +RUN pip --no-cache-dir install --upgrade \ + pip~=20.0.2 \ + wheel \ + setuptools -RUN $SC_PIP install \ - -r tmp/sidecar/requirements/_base.txt +# copy sidecar and dependencies +COPY --chown=scu:scu packages /build/packages +COPY --chown=scu:scu services/sidecar /build/services/sidecar +COPY --chown=scu:scu services/storage/client-sdk /build/services/storage/client-sdk +# install base 3rd party dependencies (NOTE: this speeds up devel mode) +RUN pip --no-cache-dir install -r /build/services/sidecar/requirements/_base.txt # --------------------------Cache stage ------------------- # CI in master buils & pushes this target to speed-up image build @@ -73,20 +81,9 @@ RUN $SC_PIP install \ # FROM build as cache -ENV SC_BUILD_TARGET cache - - -COPY --chown=scu:scu packages /build/packages -COPY --chown=scu:scu services/sidecar /build/services/sidecar -COPY --chown=scu:scu services/storage/client-sdk /build/services/storage/client-sdk - WORKDIR /build/services/sidecar - - -RUN $SC_PIP install -r requirements/prod.txt &&\ - $SC_PIP list -v - - +ENV SC_BUILD_TARGET=cache +RUN pip --no-cache-dir install -r requirements/prod.txt # --------------------------Production stage ------------------- # Final cleanup up to reduce image size and startup setup @@ -95,22 +92,25 @@ RUN $SC_PIP install -r requirements/prod.txt &&\ # + /home/scu $HOME = WORKDIR # + services/sidecar [scu:scu] # -FROM cache as production +FROM base as production - -ENV SC_BUILD_TARGET production +ENV SC_BUILD_TARGET=production \ + SC_BOOT_MODE=production ENV PYTHONOPTIMIZE=TRUE WORKDIR /home/scu -RUN mkdir -p services/sidecar &&\ - chown scu:scu services/sidecar &&\ - mv /build/services/sidecar/docker services/sidecar/docker &&\ - rm -rf /build - -RUN apk del --no-cache\ - gcc +# bring installed package without build tools +COPY --from=cache --chown=scu:scu ${VIRTUAL_ENV} ${VIRTUAL_ENV} +# copy docker entrypoint and boot scripts +COPY --chown=scu:scu services/sidecar/docker services/sidecar/docker +# NOTE: when the sidecar gets an API, or maybe through rabbitMQ +# HEALTHCHECK --interval=30s \ +# --timeout=20s \ +# --start-period=30s \ +# --retries=3 \ +# CMD ["python3", "services/api-gateway/docker/healthcheck.py", "http://localhost:8000/"] ENTRYPOINT [ "/bin/sh", "services/sidecar/docker/entrypoint.sh" ] CMD ["/bin/sh", "services/sidecar/docker/boot.sh"] @@ -126,17 +126,8 @@ CMD ["/bin/sh", "services/sidecar/docker/boot.sh"] # FROM build as development -ENV SC_BUILD_TARGET development - - -# TODO: consider installing tooling in a separate virtualenv? -# TODO: really necesary??? -WORKDIR /build - +ENV SC_BUILD_TARGET=development WORKDIR /devel -VOLUME /devel/packages -VOLUME /devel/services/sidecar/ - ENTRYPOINT [ "/bin/sh", "services/sidecar/docker/entrypoint.sh" ] CMD ["/bin/sh", "services/sidecar/docker/boot.sh"] diff --git a/services/sidecar/Makefile b/services/sidecar/Makefile new file mode 100644 index 00000000000..e3fc19cac4c --- /dev/null +++ b/services/sidecar/Makefile @@ -0,0 +1,45 @@ +# +# Targets for DEVELOPMENT of Components Catalog Service +# +include ../../scripts/common.Makefile + + +APP_NAME := sidecar +APP_CLI_NAME := simcore-service-sidecar +export APP_VERSION = $(shell cat VERSION) + +REPO_BASE_DIR = $(abspath $(CURDIR)/../../) +VENV_DIR ?= $(abspath $(REPO_BASE_DIR)/.venv) + + +.PHONY: requirements +requirements: ## compiles pip requirements (.in -> .txt) + @$(MAKE) --directory requirements reqs + + +.PHONY: install-dev install-prod install-ci +install-dev install-prod install-ci: requirements _check_venv_active ## install app in development/production or CI mode + # installing in $(subst install-,,$@) mode + # FIXME: pip-sync does not manage to install storage-sdk + # pip-sync requirements/$(subst install-,,$@).txt + pip install -r requirements/$(subst install-,,$@).txt + + +.PHONY: tests-unit tests-integration tests +tests: tests-unit tests-integration + +tests-unit: ## runs unit tests + # running unit tests + @pytest -vv --exitfirst --failed-first --durations=10 --pdb $(CURDIR)/tests/unit + +tests-integration: ## runs integration tests against local+production images + # running integration tests local/(service):production images ... + @export DOCKER_REGISTRY=local; \ + export DOCKER_IMAGE_TAG=production; \ + pytest -vv --exitfirst --failed-first --durations=10 --pdb $(CURDIR)/tests/integration + + +.PHONY: build build-nc build-devel build-devel-nc build-cache build-cache-nc +build build-nc build-devel build-devel-nc build-cache build-cache-nc: ## docker image build in many flavours + # building ${APP_NAME} ... + @$(MAKE) --directory ${REPO_BASE_DIR} $@ target=${APP_NAME} \ No newline at end of file diff --git a/services/sidecar/VERSION b/services/sidecar/VERSION new file mode 100644 index 00000000000..8acdd82b765 --- /dev/null +++ b/services/sidecar/VERSION @@ -0,0 +1 @@ +0.0.1 diff --git a/services/sidecar/docker/boot.sh b/services/sidecar/docker/boot.sh index bffd2976490..0220f0dae33 100755 --- a/services/sidecar/docker/boot.sh +++ b/services/sidecar/docker/boot.sh @@ -1,5 +1,9 @@ #!/bin/sh -# +set -o errexit +set -o nounset + +IFS=$(printf '\n\t') + INFO="INFO: [$(basename "$0")] " # BOOTING application --------------------------------------------- @@ -11,18 +15,11 @@ if [ "${SC_BUILD_TARGET}" = "development" ] then echo "$INFO" "Environment :" printenv | sed 's/=/: /' | sed 's/^/ /' | sort - #-------------------- - - cd services/sidecar || exit - $SC_PIP install --user -r requirements/dev.txt - cd /devel || exit - - #-------------------- echo "$INFO" "Python :" python --version | sed 's/^/ /' command -v python | sed 's/^/ /' echo "$INFO" "PIP :" - $SC_PIP list | sed 's/^/ /' + pip list | sed 's/^/ /' fi # RUNNING application ---------------------------------------- @@ -38,13 +35,13 @@ then POOL=solo watchmedo auto-restart --recursive --pattern="*.py" -- \ celery worker \ - --app sidecar.celery:app \ + --app simcore_service_sidecar.celery:app \ --concurrency ${CONCURRENCY} \ --loglevel="${SIDECAR_LOGLEVEL-WARNING}" \ --pool=${POOL} else exec celery worker \ - --app sidecar.celery:app \ + --app simcore_service_sidecar.celery:app \ --concurrency ${CONCURRENCY} \ --loglevel="${SIDECAR_LOGLEVEL-WARNING}" \ --pool=${POOL} diff --git a/services/sidecar/docker/entrypoint.sh b/services/sidecar/docker/entrypoint.sh index 3d8435f58f3..bbc17806b00 100755 --- a/services/sidecar/docker/entrypoint.sh +++ b/services/sidecar/docker/entrypoint.sh @@ -1,6 +1,11 @@ #!/bin/sh -# +set -o errexit +set -o nounset + +IFS=$(printf '\n\t') + INFO="INFO: [$(basename "$0")] " +WARNING="WARNING: [$(basename "$0")] " ERROR="ERROR: [$(basename "$0")] " # This entrypoint script: @@ -10,9 +15,9 @@ ERROR="ERROR: [$(basename "$0")] " # *runs* as non-root user [scu] # echo "$INFO" "Entrypoint for stage ${SC_BUILD_TARGET} ..." -echo " User :$(id "$(whoami)")" -echo " Workdir :$(pwd)" -echo " scuUser :$(id scu)" +echo User :"$(id "$(whoami)")" +echo Workdir :"$(pwd)" +echo scuUser :"$(id scu)" USERNAME=scu @@ -25,56 +30,84 @@ then DEVEL_MOUNT=/devel/services/sidecar stat $DEVEL_MOUNT > /dev/null 2>&1 || \ - (echo "$ERROR" "You must mount '$DEVEL_MOUNT' to deduce user and group ids" && exit 1) # FIXME: exit does not stop script + (echo "$ERROR" "You must mount '$DEVEL_MOUNT' to deduce user and group ids" && exit 1) - USERID=$(stat -c %u $DEVEL_MOUNT) - GROUPID=$(stat -c %g $DEVEL_MOUNT) - GROUPNAME=$(getent group "${GROUPID}" | cut -d: -f1) + USERID=$(stat --format=%u $DEVEL_MOUNT) + GROUPID=$(stat --format=%g $DEVEL_MOUNT) + GROUPNAME=$(getent group "${GROUPID}" | cut --delimiter=: --fields=1) if [ "$USERID" -eq 0 ] then - echo "$INFO" "mounted folder from root, adding scu to root..." - addgroup scu root + echo "$WARNING" Folder mounted owned by root user... adding "$SC_USER_NAME" to root... + adduser "${SC_USER_NAME}" root else # take host's credentials in scu if [ -z "$GROUPNAME" ] then - echo "$INFO" "mounted folder from $USERID, creating new group..." - GROUPNAME=host_group - addgroup -g "$GROUPID" $GROUPNAME + echo "$INFO" mounted folder from "$USERID", creating new group my"${SC_USER_NAME}" + GROUPNAME=my"${SC_USER_NAME}" + addgroup --gid "$GROUPID" "$GROUPNAME" + # change group property of files already around + find / -path /proc -prune -group "$SC_USER_ID" -exec chgrp --no-dereference "$GROUPNAME" {} \; else - echo "$INFO" "mounted folder from $USERID, adding scu to $GROUPNAME..." - addgroup scu $GROUPNAME + echo "$INFO" "mounted folder from $USERID, adding ${SC_USER_NAME} to $GROUPNAME..." + adduser "$SC_USER_NAME" "$GROUPNAME" fi - deluser scu > /dev/null 2>&1 - adduser -u "$USERID" -G $GROUPNAME -D -s /bin/sh scu + echo "$INFO changing $SC_USER_NAME $SC_USER_ID:$SC_USER_ID to $USERID:$GROUPID" + deluser "${SC_USER_NAME}" > /dev/null 2>&1 + if [ "$SC_USER_NAME" = "$GROUPNAME" ] + then + addgroup --gid "$GROUPID" "$GROUPNAME" + fi + adduser --disabled-password --gecos "" --uid "$USERID" --gid "$GROUPID" --shell /bin/sh "$SC_USER_NAME" --no-create-home + # change user property of files already around + find / -path /proc -prune -user "$SC_USER_ID" -exec chown --no-dereference "$SC_USER_NAME" {} \; fi + + echo "$INFO installing python dependencies..." + cd services/sidecar || exit 1 + pip install --no-cache-dir -r requirements/dev.txt + cd - || exit 1 fi +if [ "${SC_BOOT_MODE}" = "debug-ptvsd" ] +then + # NOTE: production does NOT pre-installs ptvsd + pip install --no-cache-dir ptvsd +fi + # Appends docker group if socket is mounted DOCKER_MOUNT=/var/run/docker.sock - - if stat $DOCKER_MOUNT > /dev/null 2>&1 then - GROUPID=$(stat -c %g $DOCKER_MOUNT) + echo "$INFO detected docker socket is mounted, adding user to group..." + GROUPID=$(stat --format=%g $DOCKER_MOUNT) GROUPNAME=scdocker - - if ! addgroup -g "$GROUPID" $GROUPNAME > /dev/null 2>&1 + if ! addgroup --gid "$GROUPID" $GROUPNAME > /dev/null 2>&1 then + echo "$WARNING docker group with $GROUPID already exists, getting group name..." # if group already exists in container, then reuse name - GROUPNAME=$(getent group "${GROUPID}" | cut -d: -f1) + GROUPNAME=$(getent group "${GROUPID}" | cut --delimiters=: --fields=1) + echo "$WARNING docker group with $GROUPID has name $GROUPNAME" fi - addgroup scu "$GROUPNAME" + adduser "$SC_USER_NAME" "$GROUPNAME" fi -echo "$INFO" "Starting boot ..." -chown -R $USERNAME:"$GROUPNAME" /home/scu/input -chown -R $USERNAME:"$GROUPNAME" /home/scu/output -chown -R $USERNAME:"$GROUPNAME" /home/scu/log +echo "$INFO ensuring write rights on folders ..." +chown -R $USERNAME:"$GROUPNAME" "${SIDECAR_INPUT_FOLDER}" +chown -R $USERNAME:"$GROUPNAME" "${SIDECAR_OUTPUT_FOLDER}" +chown -R $USERNAME:"$GROUPNAME" "${SIDECAR_LOG_FOLDER}" + + +echo "$INFO Starting $* ..." +echo " $SC_USER_NAME rights : $(id "$SC_USER_NAME")" +echo " local dir : $(ls -al)" +echo " input dir : $(ls -al "${SIDECAR_INPUT_FOLDER}")" +echo " output dir : $(ls -al "${SIDECAR_OUTPUT_FOLDER}")" +echo " log dir : $(ls -al "${SIDECAR_LOG_FOLDER}")" -exec su-exec scu "$@" +su --command "$*" "$SC_USER_NAME" diff --git a/services/sidecar/requirements/_base.in b/services/sidecar/requirements/_base.in index 7f488cfa53b..dacaa801792 100644 --- a/services/sidecar/requirements/_base.in +++ b/services/sidecar/requirements/_base.in @@ -3,13 +3,15 @@ # urllib3>=1.25.8 # Vulnerability +aio-pika +aiodocker +aiofiles +aiopg +click sqlalchemy>=1.3.3 # https://nvd.nist.gov/vuln/detail/CVE-2019-7164 -psycopg2-binary # enforces binary version - http://initd.org/psycopg/docs/install.html#binary-install-from-pypi celery -docker kombu -minio networkx -pika +pydantic tenacity diff --git a/services/sidecar/requirements/_base.txt b/services/sidecar/requirements/_base.txt index f01e03a3474..3b09138280e 100644 --- a/services/sidecar/requirements/_base.txt +++ b/services/sidecar/requirements/_base.txt @@ -2,28 +2,38 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile --output-file=_base.txt _base.in +# pip-compile --build-isolation --output-file=_base.txt _base.in # -amqp==2.4.2 # via kombu -billiard==3.6.0.0 # via celery -celery==4.3.0 # via -r _base.in -certifi==2019.3.9 # via minio, requests -chardet==3.0.4 # via requests -decorator==4.4.0 # via networkx -docker-pycreds==0.4.0 # via docker -docker==3.7.2 # via -r _base.in -idna==2.8 # via requests -kombu==4.5.0 # via -r _base.in, celery -minio==4.0.16 # via -r _base.in -networkx==2.3 # via -r _base.in -pika==1.0.1 # via -r _base.in -psycopg2-binary==2.8.4 # via -r _base.in -python-dateutil==2.8.0 # via minio -pytz==2019.1 # via celery, minio -requests==2.22.0 # via docker -six==1.12.0 # via docker, docker-pycreds, python-dateutil, tenacity, websocket-client -sqlalchemy==1.3.3 # via -r _base.in -tenacity==6.0.0 # via -r _base.in -urllib3==1.25.8 # via -r _base.in, minio, requests +aio-pika==6.6.0 # via -r _base.in +aiodocker==0.17.0 # via -r _base.in +aiofiles==0.4.0 # via -r _base.in +aiohttp==3.6.2 # via aiodocker +aiopg==1.0.0 # via -r _base.in +aiormq==3.2.1 # via aio-pika +amqp==2.5.2 # via kombu +async-timeout==3.0.1 # via aiohttp +attrs==19.3.0 # via aiohttp +billiard==3.6.3.0 # via celery +celery==4.4.2 # via -r _base.in +chardet==3.0.4 # via aiohttp +click==7.1.1 # via -r _base.in +dataclasses==0.7 # via pydantic +decorator==4.4.2 # via networkx +idna-ssl==1.1.0 # via aiohttp +idna==2.9 # via yarl +importlib-metadata==1.5.0 # via kombu +kombu==4.6.8 # via -r _base.in, celery +multidict==4.7.5 # via aiohttp, yarl +networkx==2.4 # via -r _base.in +pamqp==2.3.0 # via aiormq +psycopg2-binary==2.8.4 # via aiopg +pydantic==1.4 # via -r _base.in +pytz==2019.3 # via celery +six==1.14.0 # via tenacity +sqlalchemy==1.3.15 # via -r _base.in +tenacity==6.1.0 # via -r _base.in +typing-extensions==3.7.4.1 # via aiohttp +urllib3==1.25.8 # via -r _base.in vine==1.3.0 # via amqp, celery -websocket-client==0.56.0 # via docker +yarl==1.4.2 # via aio-pika, aiohttp, aiormq +zipp==3.1.0 # via importlib-metadata diff --git a/services/sidecar/requirements/_test.in b/services/sidecar/requirements/_test.in index 9899ee525e9..efb72802f3c 100644 --- a/services/sidecar/requirements/_test.in +++ b/services/sidecar/requirements/_test.in @@ -8,7 +8,7 @@ # testing coverage==4.5.1 # TODO: Downgraded because of a bug https://github.com/nedbat/coveragepy/issues/716 pytest~=5.3.5 # Bug in pytest-sugar https://github.com/Teemu/pytest-sugar/issues/187 -pytest-docker +pytest-aiohttp # incompatible with pytest-asyncio. See https://github.com/pytest-dev/pytest-asyncio/issues/76 pytest-cov pytest-instafail pytest-mock @@ -16,6 +16,7 @@ pytest-sugar # fixtures aiopg +docker # tools for CI pylint diff --git a/services/sidecar/requirements/_test.txt b/services/sidecar/requirements/_test.txt index d5b53d403fc..1b4e0b1c240 100644 --- a/services/sidecar/requirements/_test.txt +++ b/services/sidecar/requirements/_test.txt @@ -4,54 +4,64 @@ # # pip-compile --build-isolation --output-file=_test.txt _test.in # -aiopg==1.0.0 # via -r _test.in -amqp==2.4.2 # via -r _base.txt, kombu +aio-pika==6.6.0 # via -r _base.txt +aiodocker==0.17.0 # via -r _base.txt +aiofiles==0.4.0 # via -r _base.txt +aiohttp==3.6.2 # via -r _base.txt, aiodocker, pytest-aiohttp +aiopg==1.0.0 # via -r _base.txt, -r _test.in +aiormq==3.2.1 # via -r _base.txt, aio-pika +amqp==2.5.2 # via -r _base.txt, kombu astroid==2.3.3 # via pylint -attrs==19.3.0 # via pytest, pytest-docker -billiard==3.6.0.0 # via -r _base.txt, celery -celery==4.3.0 # via -r _base.txt -certifi==2019.3.9 # via -r _base.txt, minio, requests -chardet==3.0.4 # via -r _base.txt, requests +async-timeout==3.0.1 # via -r _base.txt, aiohttp +attrs==19.3.0 # via -r _base.txt, aiohttp, pytest +billiard==3.6.3.0 # via -r _base.txt, celery +celery==4.4.2 # via -r _base.txt +certifi==2019.11.28 # via requests +chardet==3.0.4 # via -r _base.txt, aiohttp, requests +click==7.1.1 # via -r _base.txt coverage==4.5.1 # via -r _test.in, coveralls, pytest-cov coveralls==1.11.1 # via -r _test.in -decorator==4.4.0 # via -r _base.txt, networkx -docker-pycreds==0.4.0 # via -r _base.txt, docker -docker==3.7.2 # via -r _base.txt +dataclasses==0.7 # via -r _base.txt, pydantic +decorator==4.4.2 # via -r _base.txt, networkx +docker==4.2.0 # via -r _test.in docopt==0.6.2 # via coveralls -idna==2.8 # via -r _base.txt, requests -importlib-metadata==1.5.0 # via pluggy, pytest +idna-ssl==1.1.0 # via -r _base.txt, aiohttp +idna==2.9 # via -r _base.txt, idna-ssl, requests, yarl +importlib-metadata==1.5.0 # via -r _base.txt, kombu, pluggy, pytest isort==4.3.21 # via pylint -kombu==4.5.0 # via -r _base.txt, celery +kombu==4.6.8 # via -r _base.txt, celery lazy-object-proxy==1.4.3 # via astroid mccabe==0.6.1 # via pylint -minio==4.0.16 # via -r _base.txt more-itertools==8.2.0 # via pytest -networkx==2.3 # via -r _base.txt +multidict==4.7.5 # via -r _base.txt, aiohttp, yarl +networkx==2.4 # via -r _base.txt packaging==20.3 # via pytest, pytest-sugar -pika==1.0.1 # via -r _base.txt +pamqp==2.3.0 # via -r _base.txt, aiormq pluggy==0.13.1 # via pytest psycopg2-binary==2.8.4 # via -r _base.txt, aiopg ptvsd==4.3.2 # via -r _test.in py==1.8.1 # via pytest +pydantic==1.4 # via -r _base.txt pylint==2.4.4 # via -r _test.in pyparsing==2.4.6 # via packaging +pytest-aiohttp==0.3.0 # via -r _test.in pytest-cov==2.8.1 # via -r _test.in -pytest-docker==0.7.2 # via -r _test.in pytest-instafail==0.4.1.post0 # via -r _test.in pytest-mock==2.0.0 # via -r _test.in pytest-sugar==0.9.2 # via -r _test.in -pytest==5.3.5 # via -r _test.in, pytest-cov, pytest-instafail, pytest-mock, pytest-sugar -python-dateutil==2.8.0 # via -r _base.txt, minio -pytz==2019.1 # via -r _base.txt, celery, minio -requests==2.22.0 # via -r _base.txt, coveralls, docker -six==1.12.0 # via -r _base.txt, astroid, docker, docker-pycreds, packaging, python-dateutil, tenacity, websocket-client -sqlalchemy==1.3.3 # via -r _base.txt -tenacity==6.0.0 # via -r _base.txt +pytest==5.3.5 # via -r _test.in, pytest-aiohttp, pytest-cov, pytest-instafail, pytest-mock, pytest-sugar +pytz==2019.3 # via -r _base.txt, celery +requests==2.23.0 # via coveralls, docker +six==1.14.0 # via -r _base.txt, astroid, docker, packaging, tenacity, websocket-client +sqlalchemy==1.3.15 # via -r _base.txt +tenacity==6.1.0 # via -r _base.txt termcolor==1.1.0 # via pytest-sugar typed-ast==1.4.1 # via astroid -urllib3==1.25.8 # via -r _base.txt, minio, requests +typing-extensions==3.7.4.1 # via -r _base.txt, aiohttp +urllib3==1.25.8 # via -r _base.txt, requests vine==1.3.0 # via -r _base.txt, amqp, celery -wcwidth==0.1.8 # via pytest -websocket-client==0.56.0 # via -r _base.txt, docker +wcwidth==0.1.9 # via pytest +websocket-client==0.57.0 # via docker wrapt==1.11.2 # via astroid -zipp==3.1.0 # via importlib-metadata +yarl==1.4.2 # via -r _base.txt, aio-pika, aiohttp, aiormq +zipp==3.1.0 # via -r _base.txt, importlib-metadata diff --git a/services/sidecar/requirements/ci.txt b/services/sidecar/requirements/ci.txt index 86d132dc3c0..89bfd577ff4 100644 --- a/services/sidecar/requirements/ci.txt +++ b/services/sidecar/requirements/ci.txt @@ -12,7 +12,10 @@ # installs this repo's packages ../../services/storage/client-sdk/python/ ../../packages/s3wrapper/ +../../packages/postgres-database/ ../../packages/simcore-sdk/ +../../packages/service-library/ +../../packages/pytest-simcore/ # installs current package . diff --git a/services/sidecar/requirements/dev.txt b/services/sidecar/requirements/dev.txt index 7508c989fb4..20c6521380f 100644 --- a/services/sidecar/requirements/dev.txt +++ b/services/sidecar/requirements/dev.txt @@ -16,6 +16,8 @@ watchdog[watchmedo] -e ../../packages/s3wrapper/ -e ../../packages/postgres-database/ -e ../../packages/simcore-sdk/ +-e ../../packages/service-library/ +-e ../../packages/pytest-simcore/ # installs current package -e . diff --git a/services/sidecar/requirements/prod.txt b/services/sidecar/requirements/prod.txt index c26a540f09b..4217b5bd2ef 100644 --- a/services/sidecar/requirements/prod.txt +++ b/services/sidecar/requirements/prod.txt @@ -14,6 +14,7 @@ ../../packages/s3wrapper/ ../../packages/postgres-database/ ../../packages/simcore-sdk/ +../../packages/service-library/ # installs current package . diff --git a/services/sidecar/setup.cfg b/services/sidecar/setup.cfg new file mode 100644 index 00000000000..be7478de091 --- /dev/null +++ b/services/sidecar/setup.cfg @@ -0,0 +1,11 @@ +[bumpversion] +current_version = 0.0.2 +commit = True +message = sidecar api version: {current_version} → {new_version} +tag = False + +[bumpversion:file:setup.py] +search = "{current_version}" +replace = "{new_version}" + +[bumpversion:file:VERSION] diff --git a/services/sidecar/setup.py b/services/sidecar/setup.py index 4edb4b59025..3a6de7a3755 100644 --- a/services/sidecar/setup.py +++ b/services/sidecar/setup.py @@ -4,29 +4,57 @@ from setuptools import find_packages, setup -here = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent +if sys.version_info.major != 3 and sys.version_info.minor != 6: + raise RuntimeError( + "Expected ~=3.6, got %s. Did you forget to activate virtualenv?" + % str(sys.version_info) + ) +current_dir = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent -def read_reqs( reqs_path: Path): - return re.findall(r'(^[^#-][\w]+[-~>=<.\w]+)', reqs_path.read_text(), re.MULTILINE) +def read_reqs(reqs_path: Path): + return re.findall(r"(^[^#-][\w]+[-~>=<.\w]+)", reqs_path.read_text(), re.MULTILINE) -install_requirements = read_reqs( here / "requirements" / "_base.txt" ) + [ + +readme = (current_dir / "README.md").read_text() +version = (current_dir / "VERSION").read_text().strip() + +install_requirements = read_reqs(current_dir / "requirements" / "_base.txt") + [ "s3wrapper==0.1.0", + "simcore-postgres-database", "simcore-sdk==0.1.0", - "simcore-service-storage-sdk==0.1.0" + "simcore-service-storage-sdk==0.1.0", + "simcore-service-library", ] -test_requirements = read_reqs( here / "requirements" / "_test.txt" ) +test_requirements = read_reqs(current_dir / "requirements" / "_test.txt") setup( - name='simcore-service-sidecar', - version='0.0.1', - packages=find_packages(where='src'), - package_dir={'': 'src'}, + name="simcore-service-sidecar", + version=version, + author="Sylvain Anderegg (sanderegg)", + description="Platform's sidecar", + classifiers={ + "Development Status :: 1 - Planning", + "License :: OSI Approved :: MIT License", + "Natural Language :: English", + "Programming Language :: Python :: 3.6", + }, + long_description=readme, + license="MIT license", + python_requires="~=3.6", + packages=find_packages(where="src"), + package_dir={"": "src",}, + include_package_data=True, install_requires=install_requirements, - python_requires='>=3.6', - test_suite='tests', - tests_require=test_requirements + test_suite="tests", + tests_require=test_requirements, + extras_require={"test": test_requirements}, + entry_points={ + "console_scripts": [ + "simcore-service-sidecar = simcore_service_sidecar.cli:main", + ], + }, ) diff --git a/services/sidecar/src/sidecar/__init__.py b/services/sidecar/src/sidecar/__init__.py deleted file mode 100644 index a365dfaa39c..00000000000 --- a/services/sidecar/src/sidecar/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from . import config -from . import tasks diff --git a/services/sidecar/src/sidecar/_deprecated.py b/services/sidecar/src/sidecar/_deprecated.py deleted file mode 100644 index d48e91fd2ec..00000000000 --- a/services/sidecar/src/sidecar/_deprecated.py +++ /dev/null @@ -1,479 +0,0 @@ -# TODO: PC->MaG, please check if something missing and delete - -import json -import logging -import os -import time -from pathlib import Path - -import docker -import pika -from celery import Celery -from celery.states import SUCCESS as CSUCCESS -from celery.utils.log import get_task_logger -from sqlalchemy import and_, exc -from sqlalchemy.orm.attributes import flag_modified - -from simcore_sdk.config.rabbit import Config as rabbit_config -from simcore_sdk.models.pipeline_models import (RUNNING, SUCCESS, - ComputationalPipeline, - ComputationalTask) - -from .utils import (DbSettings, DockerSettings, ExecutorSettings, - RabbitSettings, S3Settings, delete_contents, - find_entry_point, is_node_ready) - -rabbit_config = rabbit_config() -celery= Celery(rabbit_config.name, broker=rabbit_config.broker, backend=rabbit_config.backend) - -# TODO: configure via command line or config file -#log = logging.getLogger(__name__) -log = get_task_logger(__name__) - - - -class Sidecar: - def __init__(self): - # publish subscribe config - self._pika = RabbitSettings() - - # docker client config - self._docker = DockerSettings() - - # object storage config - self._s3 = S3Settings() - - # db config - self._db = DbSettings() - - # current task - self._task = None - - # executor options - self._executor = ExecutorSettings() - - def _create_shared_folders(self): - for folder in [self._executor.in_dir, self._executor.log_dir, self._executor.out_dir]: - if not os.path.exists(folder): - os.makedirs(folder) - else: - delete_contents(folder) - - def _process_task_input(self, port, input_ports): - # pylint: disable=too-many-branches - - port_name = port['key'] - port_value = port['value'] - log.debug("PROCESSING %s %s", port_name, port_value) - log.debug(type(port_value)) - if isinstance(port_value, str) and port_value.startswith("link."): - if port['type'] == 'file-url': - log.debug('Fetch S3 %s', port_value) - #parse the link assuming it is link.id.file.ending - _parts = port_value.split(".") - object_name = os.path.join(str(self._task.project_id), _parts[1], ".".join(_parts[2:])) - input_file = os.path.join(self._executor.in_dir, port_name) - log.debug('Downloading from S3 %s/%s', self._s3.bucket, object_name) - success = False - ntry = 3 - trial = 0 - while not success and trial < ntry: - log.debug('Downloading to %s trial %s from %s', input_file, trial, ntry) - success = self._s3.client.download_file(self._s3.bucket, object_name, input_file) - trial = trial + 1 - if success: - input_ports[port_name] = port_name - log.debug("DONWLOAD successfull %s", port_name) - else: - log.debug("ERROR, input port %s not found in S3", object_name) - input_ports[port_name] = None - else: - log.debug('Fetch DB %s', port_value) - other_node_id = port_value.split(".")[1] - other_output_port_id = port_value.split(".")[2] - other_task = None - _session = self._db.Session() - try: - # pylint: disable=no-member - other_task =_session.query(ComputationalTask).filter( - and_( ComputationalTask.node_id==other_node_id, - ComputationalTask.project_id==self._task.project_id ) - ).one() - except exc.SQLAlchemyError: - log.exception("Could not find other task") - _session.rollback() - finally: - _session.close() - - if other_task is None: - log.debug("ERROR, input port %s not found in db", port_value) - else: - for oport in other_task.output: - if oport['key'] == other_output_port_id: - input_ports[port_name] = oport['value'] - else: - log.debug('Non link data %s : %s', port_name, port_value) - input_ports[port_name] = port_value - - def _process_task_inputs(self): - """ Writes input key-value pairs into a dictionary - - if the value of any port starts with 'link.' the corresponding - output ports a fetched or files dowloaded --> @ jsonld - - The dictionary is dumped to input.json, files are dumped - as port['key']. Both end up in /input/ of the container - """ - _input = self._task.input - log.debug('Input parsing for %s and node %s from container', self._task.project_id, self._task.internal_id) - log.debug(_input) - - input_ports = dict() - for port in _input: - log.debug(port) - self._process_task_input(port, input_ports) - - log.debug('DUMPING json') - #dump json file - if input_ports: - file_name = os.path.join(self._executor.in_dir, 'input.json') - with open(file_name, 'w') as f: - json.dump(input_ports, f) - - log.debug('DUMPING DONE') - - def _pull_image(self): - log.debug('PULLING IMAGE') - log.debug('reg %s user %s pwd %s', self._docker.registry, self._docker.registry_user,self._docker.registry_pwd ) - - try: - self._docker.client.login(registry=self._docker.registry, - username=self._docker.registry_user, password=self._docker.registry_pwd) - log.debug('img %s tag %s', self._docker.image_name, self._docker.image_tag) - self._docker.client.images.pull(self._docker.image_name, tag=self._docker.image_tag) - except docker.errors.APIError: - log.exception("Pulling image failed") - raise docker.errors.APIError - - - def _log(self, channel, msg): - log_data = {"Channel" : "Log", "Node": self._task.node_id, "Message" : msg} - log_body = json.dumps(log_data) - channel.basic_publish(exchange=self._pika.log_channel, routing_key='', body=log_body) - - def _progress(self, channel, progress): - prog_data = {"Channel" : "Progress", "Node": self._task.node_id, "Progress" : progress} - prog_body = json.dumps(prog_data) - channel.basic_publish(exchange=self._pika.progress_channel, routing_key='', body=prog_body) - - def _bg_job(self, log_file): - connection = pika.BlockingConnection(self._pika.parameters) - - channel = connection.channel() - channel.exchange_declare(exchange=self._pika.log_channel, exchange_type='fanout', auto_delete=True) - channel.exchange_declare(exchange=self._pika.progress_channel, exchange_type='fanout', auto_delete=True) - - with open(log_file) as file_: - # Go to the end of file - file_.seek(0,2) - while self._executor.run_pool: - curr_position = file_.tell() - line = file_.readline() - if not line: - file_.seek(curr_position) - time.sleep(1) - else: - clean_line = line.strip() - # TODO: This should be 'settings', a regex for every service - if clean_line.lower().startswith("[progress]"): - progress = clean_line.lower().lstrip("[progress]").rstrip("%").strip() - self._progress(channel, progress) - log.debug('PROGRESS %s', progress) - elif "percent done" in clean_line.lower(): - progress = clean_line.lower().rstrip("percent done") - try: - float_progress = float(progress) / 100.0 - progress = str(float_progress) - self._progress(channel, progress) - log.debug('PROGRESS %s', progress) - except ValueError: - log.exception("Could not extract progress from solver") - self._log(channel, clean_line) - else: - self._log(channel, clean_line) - log.debug('LOG %s', clean_line) - - # set progress to 1.0 at the end, ignore failures - progress = "1.0" - self._progress(channel, progress) - connection.close() - - def _process_task_output(self): - # pylint: disable=too-many-branches - - """ There will be some files in the /output - - - Maybe a output.json (should contain key value for simple things) - - other files: should be named by the key in the output port - - Files will be pushed to S3 with reference in db. output.json will be parsed - and the db updated - """ - directory = self._executor.out_dir - if not os.path.exists(directory): - return - try: - for root, _dirs, files in os.walk(directory): - for name in files: - filepath = os.path.join(root, name) - # the name should match what is in the db! - - if name == 'output.json': - log.debug("POSTRO FOUND output.json") - # parse and compare/update with the tasks output ports from db - output_ports = dict() - with open(filepath) as f: - output_ports = json.load(f) - task_outputs = self._task.output - for to in task_outputs: - if to['key'] in output_ports.keys(): - to['value'] = output_ports[to['key']] - log.debug("POSTRPO to['value]' becomes %s", output_ports[to['key']]) - flag_modified(self._task, "output") - _session = self._db.Session() - try: - _session.commit() - except exc.SQLAlchemyError as e: - log.debug(e) - _session.rollback() - finally: - _session.close() - else: - # we want to keep the folder structure - if not root == directory: - rel_name = os.path.relpath(root, directory) - name = rel_name + "/" + name - - object_name = str(self._task.project_id) + "/" + self._task.node_id + "/" + name - success = False - ntry = 3 - trial = 0 - while not success and trial < ntry: - log.debug("POSTRO pushes to S3 %s try %s from %s", object_name, trial, ntry) - success = self._s3.client.upload_file(self._s3.bucket, object_name, filepath) - trial = trial + 1 - - except (OSError, IOError) as _e: - logging.exception("Could not process output") - - def _process_task_log(self): - """ There will be some files in the /log - - - put them all into S3 /log - """ - directory = self._executor.log_dir - if os.path.exists(directory): - for root, _dirs, files in os.walk(directory): - for name in files: - filepath = os.path.join(root, name) - object_name = str(self._task.project_id) + "/" + self._task.node_id + "/log/" + name - if not self._s3.client.upload_file(self._s3.bucket, object_name, filepath): - log.error("Error uploading file to S3") - - def initialize(self, task): - self._task = task - - self._docker.image_name = self._docker.registry_name + "/" + task.image['name'] - self._docker.image_tag = task.image['tag'] - self._executor.in_dir = os.path.join("/", "input", task.job_id) - self._executor.out_dir = os.path.join("/", "output", task.job_id) - self._executor.log_dir = os.path.join("/", "log", task.job_id) - - self._docker.env = ["INPUT_FOLDER=" + self._executor.in_dir, - "OUTPUT_FOLDER=" + self._executor.out_dir, - "LOG_FOLDER=" + self._executor.log_dir] - - - def preprocess(self): - log.debug('Pre-Processing Pipeline %s and node %s from container', self._task.project_id, self._task.internal_id) - self._create_shared_folders() - self._process_task_inputs() - self._pull_image() - - def process(self): - log.debug('Processing Pipeline %s and node %s from container', self._task.project_id, self._task.internal_id) - - self._executor.run_pool = True - - # touch output file - log_file = os.path.join(self._executor.log_dir, "log.dat") - - Path(log_file).touch() - fut = self._executor.pool.submit(self._bg_job, log_file) - - try: - docker_image = self._docker.image_name + ":" + self._docker.image_tag - self._docker.client.containers.run(docker_image, "run", - detach=False, remove=True, - volumes = {'services_input' : {'bind' : '/input'}, - 'services_output' : {'bind' : '/output'}, - 'services_log' : {'bind' : '/log'}}, - environment=self._docker.env) - except docker.errors.ContainerError as _e: - log.error("Run container returned non zero exit code") - except docker.errors.ImageNotFound as _e: - log.error("Run container: Image not found") - except docker.errors.APIError as _e: - log.error("Run Container: Server returns error") - - - time.sleep(1) - self._executor.run_pool = False - while not fut.done(): - time.sleep(0.1) - - log.debug('DONE Processing Pipeline %s and node %s from container', self._task.project_id, self._task.internal_id) - - def run(self): - connection = pika.BlockingConnection(self._pika.parameters) - - channel = connection.channel() - channel.exchange_declare(exchange=self._pika.log_channel, exchange_type='fanout', auto_delete=True) - - msg = "Preprocessing start..." - self._log(channel, msg) - self.preprocess() - msg = "...preprocessing end" - self._log(channel, msg) - - msg = "Processing start..." - self._log(channel, msg) - self.process() - msg = "...processing end" - self._log(channel, msg) - - msg = "Postprocessing start..." - self._log(channel, msg) - self.postprocess() - msg = "...postprocessing end" - self._log(channel, msg) - connection.close() - - - def postprocess(self): - #log.debug('Post-Processing Pipeline %s and node %s from container', self._task.project_id, self._task.internal_id) - - self._process_task_output() - self._process_task_log() - - self._task.state = SUCCESS - _session = self._db.Session() - try: - _session.add(self._task) - _session.commit() - # log.debug('DONE Post-Processing Pipeline %s and node %s from container', self._task.project_id, self._task.internal_id) - - except exc.SQLAlchemyError: - log.exception("Could not update job from postprocessing") - _session.rollback() - finally: - _session.close() - - def inspect(self, celery_task, project_id, node_id): - log.debug("ENTERING inspect pipeline:node %s: %s", project_id, node_id) - - next_task_nodes = [] - do_run = False - - try: - _session = self._db.Session() - _pipeline =_session.query(ComputationalPipeline).filter_by(project_id=project_id).one() - - graph = _pipeline.execution_graph - if node_id: - do_process = True - # find the for the current node_id, skip if there is already a job_id around - # pylint: disable=assignment-from-no-return - # pylint: disable=no-member - query =_session.query(ComputationalTask).filter( - and_( - ComputationalTask.node_id==node_id, - ComputationalTask.project_id==project_id, - ComputationalTask.job_id==None) - ) - # Use SELECT FOR UPDATE TO lock the row - query.with_for_update() - task = query.one() - - if task == None: - return next_task_nodes - - # already done or running and happy - if task.job_id and (task.state == SUCCESS or task.state == RUNNING): - log.debug("TASK %s ALREADY DONE OR RUNNING", task.internal_id) - do_process = False - - # Check if node's dependecies are there - if not is_node_ready(task, graph, _session, log): - log.debug("TASK %s NOT YET READY", task.internal_id) - do_process = False - - if do_process: - task.job_id = celery_task.request.id - _session.add(task) - _session.commit() - - task =_session.query(ComputationalTask).filter( - and_(ComputationalTask.node_id==node_id,ComputationalTask.project_id==project_id)).one() - - if task.job_id != celery_task.request.id: - # somebody else was faster - # return next_task_nodes - pass - else: - task.state = RUNNING - _session.add(task) - _session.commit() - - self.initialize(task) - - do_run = True - else: - log.debug("NODE id was zero") - log.debug("graph looks like this %s", graph) - - next_task_nodes = find_entry_point(graph) - log.debug("Next task nodes %s", next_task_nodes) - - celery_task.update_state(state=CSUCCESS) - - except exc.SQLAlchemyError: - log.exception("DB error") - _session.rollback() - - finally: - _session.close() - - # now proceed actually running the task (we do that after the db session has been closed) - if do_run: - # try to run the task, return empyt list of next nodes if anything goes wrong - self.run() - next_task_nodes = list(graph.successors(node_id)) - - return next_task_nodes - - -# FIXME: this should be moved into tasks.py and need a main.py as well! -SIDECAR = Sidecar() -@celery.task(name='comp.task', bind=True) -def pipeline(self, project_id, node_id=None): - log.debug("ENTERING run") - next_task_nodes = [] - try: - next_task_nodes = SIDECAR.inspect(self, project_id, node_id) - #pylint:disable=broad-except - except Exception: - log.exception("Uncaught exception") - - for _node_id in next_task_nodes: - _task = celery.send_task('comp.task', args=(project_id, _node_id), kwargs={}) diff --git a/services/sidecar/src/sidecar/celery.py b/services/sidecar/src/sidecar/celery.py deleted file mode 100644 index 81021df6110..00000000000 --- a/services/sidecar/src/sidecar/celery.py +++ /dev/null @@ -1,22 +0,0 @@ -from celery import Celery -from simcore_sdk.config.rabbit import Config as RabbitConfig - -from .celery_log_setup import get_task_logger -from .remote_debug import setup_remote_debugging - -log = get_task_logger(__name__) -log.info("Inititalizing celery app ...") - -rabbit_config = RabbitConfig() - -setup_remote_debugging() - -# TODO: make it a singleton? -app= Celery(rabbit_config.name, - broker=rabbit_config.broker, - backend=rabbit_config.backend) - -__all__ = [ - "rabbit_config", - "app" -] diff --git a/services/sidecar/src/sidecar/config.py b/services/sidecar/src/sidecar/config.py deleted file mode 100644 index 8d319e73182..00000000000 --- a/services/sidecar/src/sidecar/config.py +++ /dev/null @@ -1,18 +0,0 @@ -import logging -import multiprocessing -import os - - -SERVICES_MAX_NANO_CPUS = min(multiprocessing.cpu_count(), os.environ.get("SIDECAR_SERVICES_MAX_NANO_CPUS", 4 * pow(10, 9))) -SERVICES_MAX_MEMORY_BYTES = os.environ.get("SIDECAR_SERVICES_MAX_MEMORY_BYTES", 2 * pow(1024, 3)) -SERVICES_TIMEOUT_SECONDS = os.environ.get("SIDECAR_SERVICES_TIMEOUT_SECONDS", 20*60) -SWARM_STACK_NAME = os.environ["SWARM_STACK_NAME"] - -SIDECAR_LOGLEVEL = getattr( - logging, - os.environ.get("SIDECAR_LOGLEVEL", "WARNING").upper(), - logging.WARNING) - -logging.basicConfig(level=SIDECAR_LOGLEVEL) -logging.getLogger('sqlalchemy.engine').setLevel(SIDECAR_LOGLEVEL) -logging.getLogger('sqlalchemy.pool').setLevel(SIDECAR_LOGLEVEL) diff --git a/services/sidecar/src/sidecar/core.py b/services/sidecar/src/sidecar/core.py deleted file mode 100644 index e82cd8209d4..00000000000 --- a/services/sidecar/src/sidecar/core.py +++ /dev/null @@ -1,512 +0,0 @@ -import json -import logging -import os -import shutil -import time -from contextlib import contextmanager -from datetime import datetime -from pathlib import Path -from typing import Dict, List, Union - -import docker -import requests -from celery.states import SUCCESS as CSUCCESS -from celery.utils.log import get_task_logger - -import pika -from simcore_sdk import node_ports, node_data -from simcore_sdk.models.pipeline_models import (RUNNING, SUCCESS, - ComputationalPipeline, - ComputationalTask) -from simcore_sdk.node_ports import log as node_port_log -from simcore_sdk.node_ports.dbmanager import DBManager -from sqlalchemy import and_, exc - -from . import config -from .utils import (DbSettings, DockerSettings, ExecutorSettings, - RabbitSettings, S3Settings, delete_contents, - find_entry_point, is_node_ready, safe_channel, - wrap_async_call) - -log = get_task_logger(__name__) -log.setLevel(config.SIDECAR_LOGLEVEL) - -node_port_log.setLevel(config.SIDECAR_LOGLEVEL) - -@contextmanager -def session_scope(session_factory): - """Provide a transactional scope around a series of operations - - """ - session = session_factory() - try: - yield session - except: # pylint: disable=W0702 - log.exception("DB access error, rolling back") - session.rollback() - finally: - session.close() - -class Sidecar: # pylint: disable=too-many-instance-attributes - def __init__(self): - # publish subscribe config - self._pika = RabbitSettings() - - # docker client config - self._docker = DockerSettings() - - # object storage config - self._s3 = S3Settings() - - # db config - self._db = DbSettings() # keeps single db engine: sidecar.utils_{id} - self._db_manager = None # lazy init because still not configured. SEE _get_node_ports - - # current task - self._task = None - - # current user id - self._user_id = None - - # stack name - self._stack_name = None - - # executor options - self._executor = ExecutorSettings() - - def _get_node_ports(self): - if self._db_manager is None: - self._db_manager = DBManager() # Keeps single db engine: simcore_sdk.node_ports.dbmanager_{id} - return node_ports.ports(self._db_manager) - - def _create_shared_folders(self): - for folder in [self._executor.in_dir, self._executor.log_dir, self._executor.out_dir]: - if not os.path.exists(folder): - os.makedirs(folder) - else: - delete_contents(folder) - - def _process_task_input(self, port:node_ports.Port, input_ports:Dict): - # pylint: disable=too-many-branches - port_name = port.key - port_value = wrap_async_call(port.get()) - log.debug("PROCESSING %s %s", port_name, port_value) - log.debug(type(port_value)) - if str(port.type).startswith("data:"): - path = port_value - if not path is None: - # the filename is not necessarily the name of the port, might be mapped - mapped_filename = Path(path).name - input_ports[port_name] = str(port_value) - final_path = Path(self._executor.in_dir, mapped_filename) - shutil.copy(str(path), str(final_path)) - log.debug("DOWNLOAD successfull from %s to %s via %s" , str(port_name), str(final_path), str(path)) - else: - input_ports[port_name] = port_value - else: - input_ports[port_name] = port_value - - def _process_task_inputs(self): - """ Writes input key-value pairs into a dictionary - - if the value of any port starts with 'link.' the corresponding - output ports a fetched or files dowloaded --> @ jsonld - - The dictionary is dumped to input.json, files are dumped - as port['key']. Both end up in /input/ of the container - """ - log.debug('Input parsing for %s and node %s from container', self._task.project_id, self._task.internal_id) - - input_ports = dict() - PORTS = self._get_node_ports() - for port in PORTS.inputs: - log.debug(port) - self._process_task_input(port, input_ports) - - log.debug('DUMPING json') - #dump json file - if input_ports: - file_name = os.path.join(self._executor.in_dir, 'input.json') - with open(file_name, 'w') as f: - json.dump(input_ports, f) - - log.debug('DUMPING DONE') - - def _pull_image(self): - log.debug('PULLING IMAGE') - log.debug('reg %s user %s pwd %s', self._docker.registry, self._docker.registry_user,self._docker.registry_pwd ) - - try: - self._docker.client.login( - registry=self._docker.registry, - username=self._docker.registry_user, - password=self._docker.registry_pwd) - log.debug('img %s tag %s', self._docker.image_name, self._docker.image_tag) - - self._docker.client.images.pull(self._docker.image_name, tag=self._docker.image_tag) - except docker.errors.APIError: - msg = f"Failed to pull image '{self._docker.image_name}:{self._docker.image_tag}' from {self._docker.registry,}" - log.exception(msg) - raise docker.errors.APIError(msg) - - def _post_log(self, channel: pika.channel.Channel, msg: Union[str, List[str]]): - log_data = { - "Channel" : "Log", - "Node": self._task.node_id, - "user_id": self._user_id, - "project_id": self._task.project_id, - "Messages" : msg if isinstance(msg, list) else [msg] - } - log_body = json.dumps(log_data) - channel.basic_publish(exchange=self._pika.log_channel, routing_key='', body=log_body) - - def _post_progress(self, channel, progress): - prog_data = { - "Channel" : "Progress", - "Node": self._task.node_id, - "user_id": self._user_id, - "project_id": self._task.project_id, - "Progress" : progress - } - prog_body = json.dumps(prog_data) - channel.basic_publish(exchange=self._pika.progress_channel, routing_key='', body=prog_body) - - def _bg_job(self, log_file): - log.debug('Bck job started %s:node %s:internal id %s from container', self._task.project_id, self._task.node_id, self._task.internal_id) - with safe_channel(self._pika) as (channel, blocking_connection): - - def _follow(thefile): - thefile.seek(0,2) - while self._executor.run_pool: - line = thefile.readline() - if not line: - time.sleep(1) - blocking_connection.process_data_events() - continue - yield line - - def _parse_progress(line: str): - # TODO: This should be 'settings', a regex for every service - if line.lower().startswith("[progress]"): - progress = line.lower().lstrip("[progress]").rstrip("%").strip() - self._post_progress(channel, progress) - log.debug('PROGRESS %s', progress) - elif "percent done" in line.lower(): - progress = line.lower().rstrip("percent done") - try: - float_progress = float(progress) / 100.0 - progress = str(float_progress) - self._post_progress(channel, progress) - log.debug('PROGRESS %s', progress) - except ValueError: - log.exception("Could not extract progress from solver") - self._post_log(channel, line) - - def _log_accumulated_logs(new_log: str, acc_logs: List[str], time_logs_sent: float): - # do not overload broker with messages, we log once every 1sec - TIME_BETWEEN_LOGS_S = 2.0 - acc_logs.append(new_log) - now = time.monotonic() - if (now - time_logs_sent) > TIME_BETWEEN_LOGS_S: - self._post_log(channel, acc_logs) - log.debug('LOG %s', acc_logs) - # empty the logs - acc_logs = [] - time_logs_sent = now - return acc_logs,time_logs_sent - - - acc_logs = [] - time_logs_sent = time.monotonic() - file_path = Path(log_file) - with file_path.open() as fp: - for line in _follow(fp): - if not self._executor.run_pool: - break - _parse_progress(line) - acc_logs, time_logs_sent = _log_accumulated_logs(line, acc_logs, time_logs_sent) - if acc_logs: - # send the remaining logs - self._post_log(channel, acc_logs) - log.debug('LOG %s', acc_logs) - - # set progress to 1.0 at the end, ignore failures - progress = "1.0" - self._post_progress(channel, progress) - log.debug('Bck job completed %s:node %s:internal id %s from container', self._task.project_id, self._task.node_id, self._task.internal_id) - - def _process_task_output(self): - # pylint: disable=too-many-branches - - """ There will be some files in the /output - - - Maybe a output.json (should contain key value for simple things) - - other files: should be named by the key in the output port - - Files will be pushed to S3 with reference in db. output.json will be parsed - and the db updated - """ - log.debug('Processing task outputs %s:node %s:internal id %s from container', self._task.project_id, self._task.node_id, self._task.internal_id) - PORTS = self._get_node_ports() - directory = self._executor.out_dir - if not os.path.exists(directory): - return - try: - for root, _dirs, files in os.walk(directory): - for name in files: - filepath = os.path.join(root, name) - # the name should match what is in the db! - if name == 'output.json': - log.debug("POSTRO FOUND output.json") - # parse and compare/update with the tasks output ports from db - output_ports = dict() - with open(filepath) as f: - output_ports = json.load(f) - task_outputs = PORTS.outputs - for to in task_outputs: - if to.key in output_ports.keys(): - wrap_async_call(to.set(output_ports[to.key])) - else: - wrap_async_call(PORTS.set_file_by_keymap(Path(filepath))) - - except (OSError, IOError) as _e: - logging.exception("Could not process output") - log.debug('Processing task outputs DONE %s:node %s:internal id %s from container', self._task.project_id, self._task.node_id, self._task.internal_id) - - - # pylint: disable=no-self-use - def _process_task_log(self): - log.debug('Processing Logs %s:node %s:internal id %s from container', self._task.project_id, self._task.node_id, self._task.internal_id) - directory = Path(self._executor.log_dir) - - if directory.exists(): - wrap_async_call(node_data.data_manager.push(directory, rename_to="logs")) - log.debug('Processing Logs DONE %s:node %s:internal id %s from container', self._task.project_id, self._task.node_id, self._task.internal_id) - - def initialize(self, task, user_id): - log.debug("TASK %s of user %s FOUND, initializing...", task.internal_id, user_id) - self._task = task - self._user_id = user_id - - HOMEDIR = str(Path.home()) - - self._docker.image_name = self._docker.registry_name + "/" + task.image['name'] - self._docker.image_tag = task.image['tag'] - self._docker.env = [] - - tails = dict( (name, Path(name, task.job_id).as_posix()) for name in ("input", "output", "log") ) - - # volume paths for side-car container - self._executor.in_dir = os.path.join(HOMEDIR, tails['input']) - self._executor.out_dir = os.path.join(HOMEDIR, tails['output']) - self._executor.log_dir = os.path.join(HOMEDIR, tails['log']) - - # volume paths for car container (w/o prefix) - self._docker.env = ["{}_FOLDER=/{}".format(name.upper(), tail) for name, tail in tails.items()] - - # stack name, should throw if not set - self._stack_name = config.SWARM_STACK_NAME - - # config nodeports - node_ports.node_config.USER_ID = user_id - node_ports.node_config.NODE_UUID = task.node_id - node_ports.node_config.PROJECT_ID = task.project_id - log.debug("TASK %s of user %s FOUND, initializing DONE", task.internal_id, user_id) - - - - def preprocess(self): - log.debug('Pre-Processing Pipeline %s:node %s:internal id %s from container', self._task.project_id, self._task.node_id, self._task.internal_id) - self._create_shared_folders() - self._process_task_inputs() - self._pull_image() - log.debug('Pre-Processing Pipeline DONE %s:node %s:internal id %s from container', self._task.project_id, self._task.node_id, self._task.internal_id) - - - def process(self): - log.debug('Processing Pipeline %s:node %s:internal id %s from container', self._task.project_id, self._task.node_id, self._task.internal_id) - - self._executor.run_pool = True - - # touch output file - log_file = os.path.join(self._executor.log_dir, "log.dat") - - Path(log_file).touch() - fut = self._executor.pool.submit(self._bg_job, log_file) - - start_time = time.perf_counter() - container = None - try: - docker_image = self._docker.image_name + ":" + self._docker.image_tag - container = self._docker.client.containers.run(docker_image, "run", - init=True, - detach=True, remove=False, - volumes = {'{}_input'.format(self._stack_name) : {'bind' : '/input'}, - '{}_output'.format(self._stack_name) : {'bind' : '/output'}, - '{}_log'.format(self._stack_name) : {'bind' : '/log'}}, - environment=self._docker.env, - nano_cpus=config.SERVICES_MAX_NANO_CPUS, - mem_limit=config.SERVICES_MAX_MEMORY_BYTES, - labels={ - 'user_id': str(self._user_id), - 'study_id': str(self._task.project_id), - 'node_id': str(self._task.node_id), - 'nano_cpus_limit': str(config.SERVICES_MAX_NANO_CPUS), - 'mem_limit': str(config.SERVICES_MAX_MEMORY_BYTES) - }) - except docker.errors.ImageNotFound: - log.exception("Run container: Image not found") - except docker.errors.APIError: - log.exception("Run Container: Server returns error") - - if container: - try: - wait_arguments = {} - if config.SERVICES_TIMEOUT_SECONDS > 0: - wait_arguments["timeout"] = int(config.SERVICES_TIMEOUT_SECONDS) - response = container.wait(**wait_arguments) - log.info("container completed with response %s\nlogs: %s", response, container.logs()) - except requests.exceptions.ConnectionError: - log.exception("Running container timed-out after %ss and will be killed now\nlogs: %s", config.SERVICES_TIMEOUT_SECONDS, container.logs()) - except docker.errors.APIError: - log.exception("Run Container: Server returns error") - finally: - stop_time = time.perf_counter() - log.info("Running %s took %sseconds", docker_image, stop_time-start_time) - container.remove(force=True) - - time.sleep(1) - self._executor.run_pool = False - while not fut.done(): - time.sleep(0.1) - - log.debug('DONE Processing Pipeline %s:node %s:internal id %s from container', self._task.project_id, self._task.node_id, self._task.internal_id) - - def run(self): - log.debug('Running Pipeline %s:node %s:internal id %s from container', self._task.project_id, self._task.node_id, self._task.internal_id) - #NOTE: the rabbit has a timeout of 60seconds so blocking this channel for more is a no go. - - with safe_channel(self._pika) as (channel,_): - self._post_log(channel, msg = "Preprocessing start...") - - self.preprocess() - - with safe_channel(self._pika) as (channel,_): - self._post_log(channel, msg = "...preprocessing end") - self._post_log(channel, msg = "Processing start...") - self.process() - - with safe_channel(self._pika) as (channel,_): - self._post_log(channel, msg = "...processing end") - self._post_log(channel, msg = "Postprocessing start...") - self.postprocess() - - with safe_channel(self._pika) as (channel,_): - self._post_log(channel, msg = "...postprocessing end") - - log.debug('Running Pipeline DONE %s:node %s:internal id %s from container', self._task.project_id, self._task.node_id, self._task.internal_id) - - def postprocess(self): - log.debug('Post-Processing Pipeline %s:node %s:internal id %s from container', self._task.project_id, self._task.node_id, self._task.internal_id) - - self._process_task_output() - self._process_task_log() - - self._task.state = SUCCESS - self._task.end = datetime.utcnow() - _session = self._db.Session() - try: - _session.add(self._task) - _session.commit() - log.debug('Post-Processing Pipeline DONE %s:node %s:internal id %s from container', self._task.project_id, self._task.node_id, self._task.internal_id) - - except exc.SQLAlchemyError: - log.exception("Could not update job from postprocessing") - _session.rollback() - finally: - _session.close() - - - def inspect(self, celery_task, user_id, project_id, node_id): - log.debug("ENTERING inspect with user %s pipeline:node %s: %s", user_id, project_id, node_id) - - next_task_nodes = [] - do_run = False - - with session_scope(self._db.Session) as _session: - _pipeline =_session.query(ComputationalPipeline).filter_by(project_id=project_id).one() - - graph = _pipeline.execution_graph - if node_id: - do_process = True - # find the for the current node_id, skip if there is already a job_id around - # pylint: disable=assignment-from-no-return - # pylint: disable=no-member - query =_session.query(ComputationalTask).filter( - and_( ComputationalTask.node_id==node_id, - ComputationalTask.project_id==project_id, - ComputationalTask.job_id==None ) - ) - # Use SELECT FOR UPDATE TO lock the row - query.with_for_update() - task = query.one_or_none() - - if task == None: - log.debug("No task found") - return next_task_nodes - - # already done or running and happy - if task.job_id and (task.state == SUCCESS or task.state == RUNNING): - log.debug("TASK %s ALREADY DONE OR RUNNING", task.internal_id) - do_process = False - - # Check if node's dependecies are there - if not is_node_ready(task, graph, _session, log): - log.debug("TASK %s NOT YET READY", task.internal_id) - do_process = False - - if do_process: - task.job_id = celery_task.request.id - _session.add(task) - _session.commit() - - task =_session.query(ComputationalTask).filter( - and_(ComputationalTask.node_id==node_id,ComputationalTask.project_id==project_id)).one() - - if task.job_id != celery_task.request.id: - # somebody else was faster - # return next_task_nodes - pass - else: - task.state = RUNNING - task.start = datetime.utcnow() - _session.add(task) - _session.commit() - - self.initialize(task, user_id) - - do_run = True - else: - log.debug("NODE id was zero") - log.debug("graph looks like this %s", graph) - - next_task_nodes = find_entry_point(graph) - log.debug("Next task nodes %s", next_task_nodes) - - celery_task.update_state(state=CSUCCESS) - - # now proceed actually running the task (we do that after the db session has been closed) - if do_run: - # try to run the task, return empyt list of next nodes if anything goes wrong - self.run() - next_task_nodes = list(graph.successors(node_id)) - - return next_task_nodes - - -# TODO: if a singleton, then use -SIDECAR = Sidecar() - -__all__ = [ - "SIDECAR" -] diff --git a/services/sidecar/src/sidecar/tasks.py b/services/sidecar/src/sidecar/tasks.py deleted file mode 100644 index 452a6974cd8..00000000000 --- a/services/sidecar/src/sidecar/tasks.py +++ /dev/null @@ -1,20 +0,0 @@ -import logging - -from .celery import app - -log = logging.getLogger(__name__) - -@app.task(name='comp.task', bind=True) -def pipeline(self, user_id, project_id, node_id=None): - from .core import SIDECAR - - log.debug("STARTING task processing") - next_task_nodes = [] - try: - next_task_nodes = SIDECAR.inspect(self, user_id, project_id, node_id) - #pylint:disable=broad-except - except Exception: - log.exception("Uncaught exception") - - for _node_id in next_task_nodes: - _task = app.send_task('comp.task', args=(user_id, project_id, _node_id), kwargs={}) diff --git a/services/sidecar/src/sidecar/utils.py b/services/sidecar/src/sidecar/utils.py deleted file mode 100644 index 90a5c1bc945..00000000000 --- a/services/sidecar/src/sidecar/utils.py +++ /dev/null @@ -1,127 +0,0 @@ -import asyncio -import logging -import os -import shutil -from concurrent.futures import ThreadPoolExecutor -from contextlib import contextmanager -from typing import Tuple - -import docker -import pika -import tenacity -from sqlalchemy import and_, create_engine -from sqlalchemy.orm import sessionmaker - -from s3wrapper.s3_client import S3Client -from simcore_sdk.config.db import Config as db_config -from simcore_sdk.config.docker import Config as docker_config -from simcore_sdk.config.rabbit import Config as rabbit_config -from simcore_sdk.config.s3 import Config as s3_config -from simcore_sdk.models.pipeline_models import SUCCESS, ComputationalTask - - -def wrap_async_call(fct: asyncio.coroutine): - return asyncio.get_event_loop().run_until_complete(fct) - -def delete_contents(folder): - for _fname in os.listdir(folder): - file_path = os.path.join(folder, _fname) - try: - if os.path.isfile(file_path): - os.unlink(file_path) - elif os.path.isdir(file_path): - shutil.rmtree(file_path) - except (OSError, IOError): - logging.exception("Could not delete files") - -def find_entry_point(g): - result = [] - for node in g.nodes: - if len(list(g.predecessors(node))) == 0: - result.append(node) - return result - -def is_node_ready(task, graph, _session, _logger): - #pylint: disable=no-member - tasks = _session.query(ComputationalTask).filter(and_( - ComputationalTask.node_id.in_(list(graph.predecessors(task.node_id))), - ComputationalTask.project_id==task.project_id)).all() - - _logger.debug("TASK %s ready? Checking ..", task.internal_id) - for dep_task in tasks: - job_id = dep_task.job_id - if not job_id: - return False - _logger.debug("TASK %s DEPENDS ON %s with stat %s", task.internal_id, dep_task.internal_id,dep_task.state) - if not dep_task.state == SUCCESS: - return False - _logger.debug("TASK %s is ready", task.internal_id) - return True - -class DockerSettings: - # pylint: disable=too-many-instance-attributes - def __init__(self): - self._config = docker_config() - self.client = docker.from_env() - self.registry = self._config.registry - self.registry_name = self._config.registry_name - self.registry_user = self._config.user - self.registry_pwd = self._config.pwd - self.image_name = "" - self.image_tag = "" - self.env = [] - - -class S3Settings: - def __init__(self): - self._config = s3_config() - self.client = S3Client(endpoint=self._config.endpoint, - access_key=self._config.access_key, secret_key=self._config.secret_key, secure=self._config.secure) - self.bucket = self._config.bucket_name - - #self.__create_bucket() - - - @tenacity.retry(wait=tenacity.wait_fixed(2), stop=tenacity.stop_after_attempt(15)) - def __create_bucket(self): - self.client.create_bucket(self.bucket) - - -class RabbitSettings: - def __init__(self): - self._pika = rabbit_config() - self.parameters = self._pika.parameters - self.log_channel = self._pika.log_channel - self.progress_channel = self._pika.progress_channel - -class DbSettings: - def __init__(self): - self._db_config = db_config() - self.db = create_engine( - self._db_config.endpoint + f"?application_name={__name__}_{id(self)}", - client_encoding='utf8', - pool_pre_ping=True) - self.Session = sessionmaker(self.db, expire_on_commit=False) - #self.session = self.Session() - -class ExecutorSettings: - def __init__(self): - # Pool - self.pool = ThreadPoolExecutor(1) - self.run_pool = False - - # shared folders - self.in_dir = "" - self.out_dir = "" - self.log_dir = "" - -@contextmanager -def safe_channel(rabbit_settings: RabbitSettings) -> Tuple[pika.channel.Channel, pika.adapters.BlockingConnection]: - try: - connection = pika.BlockingConnection(rabbit_settings.parameters) - channel = connection.channel() - channel.exchange_declare(exchange=rabbit_settings.log_channel, exchange_type='fanout') - channel.exchange_declare(exchange=rabbit_settings.progress_channel, exchange_type='fanout') - yield channel, connection - finally: - connection.close() diff --git a/services/sidecar/src/simcore_service_sidecar/__init__.py b/services/sidecar/src/simcore_service_sidecar/__init__.py new file mode 100644 index 00000000000..bfab70de5d7 --- /dev/null +++ b/services/sidecar/src/simcore_service_sidecar/__init__.py @@ -0,0 +1,3 @@ +""" Python package for the simcore_service_sidecar. +""" +from .__version__ import __version__ \ No newline at end of file diff --git a/services/sidecar/src/simcore_service_sidecar/__version__.py b/services/sidecar/src/simcore_service_sidecar/__version__.py new file mode 100644 index 00000000000..b1fa554fb17 --- /dev/null +++ b/services/sidecar/src/simcore_service_sidecar/__version__.py @@ -0,0 +1,10 @@ +""" Current version of the simcore_service_api_gateway application +""" +import pkg_resources + +__version__ = pkg_resources.get_distribution("simcore_service_sidecar").version + +major, minor, patch = __version__.split(".") + +api_version = __version__ +api_version_prefix: str = f"v{major}" \ No newline at end of file diff --git a/services/sidecar/src/simcore_service_sidecar/celery.py b/services/sidecar/src/simcore_service_sidecar/celery.py new file mode 100644 index 00000000000..54075f12339 --- /dev/null +++ b/services/sidecar/src/simcore_service_sidecar/celery.py @@ -0,0 +1,41 @@ +from celery import Celery, states + +from simcore_sdk.config.rabbit import Config as RabbitConfig + +from .celery_log_setup import get_task_logger +from .cli import run_sidecar +from .remote_debug import setup_remote_debugging +from .utils import wrap_async_call + +log = get_task_logger(__name__) +log.info("Inititalizing celery app ...") + +rabbit_config = RabbitConfig() + +setup_remote_debugging() + +# TODO: make it a singleton? +app = Celery( + rabbit_config.name, broker=rabbit_config.broker_url, backend=rabbit_config.backend +) + + +@app.task(name="comp.task", bind=True) +def pipeline(self, user_id: str, project_id: str, node_id: str = None): + next_task_nodes = [] + try: + next_task_nodes = wrap_async_call( + run_sidecar(self.request.id, user_id, project_id, node_id) + ) + self.update_state(state=states.SUCCESS) + except Exception: # pylint: disable=broad-except + self.update_state(state=states.FAILURE) + log.exception("Uncaught exception") + + for _node_id in next_task_nodes: + _task = app.send_task( + "comp.task", args=(user_id, project_id, _node_id), kwargs={} + ) + + +__all__ = ["rabbit_config", "app"] diff --git a/services/sidecar/src/sidecar/celery_log_setup.py b/services/sidecar/src/simcore_service_sidecar/celery_log_setup.py similarity index 100% rename from services/sidecar/src/sidecar/celery_log_setup.py rename to services/sidecar/src/simcore_service_sidecar/celery_log_setup.py diff --git a/services/sidecar/src/simcore_service_sidecar/cli.py b/services/sidecar/src/simcore_service_sidecar/cli.py new file mode 100644 index 00000000000..724dbdcff41 --- /dev/null +++ b/services/sidecar/src/simcore_service_sidecar/cli.py @@ -0,0 +1,57 @@ +import logging +from typing import List + +import click + +from .core import inspect +from .db import DBContextManager +from .rabbitmq import RabbitMQContextManager +from .utils import wrap_async_call + +log = logging.getLogger(__name__) + + +@click.command() +@click.option("--job_id", default=0, type=int, help="The job ID") +@click.option("--user_id", default=0, type=int, help="The user ID") +@click.option("--project_id", default="0", help="The project ID") +@click.option("--node_id", default=None, help="The node ID or nothing") +def main(job_id: str, user_id: str, project_id: str, node_id: str) -> List[str]: + + log.info( + "STARTING task processing for user %s, project %s, node %s", + user_id, + project_id, + node_id, + ) + try: + next_task_nodes = wrap_async_call( + run_sidecar(job_id, user_id, project_id, node_id=node_id) + ) + log.info( + "COMPLETED task processing for user %s, project %s, node %s", + user_id, + project_id, + node_id, + ) + return next_task_nodes + except Exception: # pylint: disable=broad-except + log.exception("Uncaught exception") + + +async def run_sidecar( + job_id: str, user_id: str, project_id: str, node_id: str +) -> List[str]: + + async with DBContextManager() as db_engine: + async with RabbitMQContextManager() as rabbit_mq: + next_task_nodes = await inspect( + db_engine, rabbit_mq, job_id, user_id, project_id, node_id=node_id + ) + log.info( + "COMPLETED task processing for user %s, project %s, node %s", + user_id, + project_id, + node_id, + ) + return next_task_nodes diff --git a/services/sidecar/src/simcore_service_sidecar/config.py b/services/sidecar/src/simcore_service_sidecar/config.py new file mode 100644 index 00000000000..9d564b9df8f --- /dev/null +++ b/services/sidecar/src/simcore_service_sidecar/config.py @@ -0,0 +1,42 @@ +import logging +import multiprocessing +import os + + +SERVICES_MAX_NANO_CPUS: int = min( + multiprocessing.cpu_count() * pow(10, 9), + int(os.environ.get("SIDECAR_SERVICES_MAX_NANO_CPUS", 4 * pow(10, 9))), +) +SERVICES_MAX_MEMORY_BYTES: int = int( + os.environ.get("SIDECAR_SERVICES_MAX_MEMORY_BYTES", 2 * pow(1024, 3)) +) +SERVICES_TIMEOUT_SECONDS: int = int( + os.environ.get("SIDECAR_SERVICES_TIMEOUT_SECONDS", 20 * 60) +) +SWARM_STACK_NAME: str = os.environ.get("SWARM_STACK_NAME", "simcore") + +SIDECAR_DOCKER_VOLUME_INPUT: str = os.environ.get( + "SIDECAR_DOCKER_VOLUME_INPUT", f"{SWARM_STACK_NAME}_input" +) +SIDECAR_DOCKER_VOLUME_OUTPUT: str = os.environ.get( + "SIDECAR_DOCKER_VOLUME_OUTPUT", f"{SWARM_STACK_NAME}_output" +) +SIDECAR_DOCKER_VOLUME_LOG: str = os.environ.get( + "SIDECAR_DOCKER_VOLUME_LOG", f"{SWARM_STACK_NAME}_log" +) +SIDECAR_LOGLEVEL: str = getattr( + logging, os.environ.get("SIDECAR_LOGLEVEL", "WARNING").upper(), logging.WARNING +) + +DOCKER_REGISTRY: str = os.environ.get("REGISTRY_URL", "masu.speag.com") +DOCKER_USER: str = os.environ.get("REGISTRY_USER", "z43") +DOCKER_PASSWORD: str = os.environ.get("REGISTRY_PW", "z43") + +POSTGRES_ENDPOINT: str = os.environ.get("POSTGRES_ENDPOINT", "postgres:5432") +POSTGRES_DB: str = os.environ.get("POSTGRES_DB", "simcoredb") +POSTGRES_PW: str = os.environ.get("POSTGRES_PASSWORD", "simcore") +POSTGRES_USER: str = os.environ.get("POSTGRES_USER", "simcore") + +logging.basicConfig(level=SIDECAR_LOGLEVEL) +logging.getLogger("sqlalchemy.engine").setLevel(SIDECAR_LOGLEVEL) +logging.getLogger("sqlalchemy.pool").setLevel(SIDECAR_LOGLEVEL) diff --git a/services/sidecar/src/simcore_service_sidecar/core.py b/services/sidecar/src/simcore_service_sidecar/core.py new file mode 100644 index 00000000000..842ef2c7ee9 --- /dev/null +++ b/services/sidecar/src/simcore_service_sidecar/core.py @@ -0,0 +1,565 @@ +# pylint: disable=no-member +import json +import logging +import shutil +import time +from datetime import datetime +from pathlib import Path +from typing import Dict, List, Optional + +import aiodocker +import aiopg +import attr +import networkx as nx +from celery.utils.log import get_task_logger +from sqlalchemy import and_, literal_column + +from servicelib.utils import fire_and_forget_task, logged_gather +from simcore_postgres_database.sidecar_models import ( # PENDING, + FAILED, + RUNNING, + SUCCESS, + UNKNOWN, + comp_pipeline, + comp_tasks, +) +from simcore_sdk import node_data, node_ports +from simcore_sdk.node_ports import log as node_port_log +from simcore_sdk.node_ports.dbmanager import DBManager + +from . import config, exceptions +from .log_parser import LogType, monitor_logs_task +from .rabbitmq import RabbitMQ +from .utils import execution_graph, find_entry_point, is_node_ready + +log = get_task_logger(__name__) +log.setLevel(config.SIDECAR_LOGLEVEL) + +node_port_log.setLevel(config.SIDECAR_LOGLEVEL) + + +@attr.s(auto_attribs=True) +class TaskSharedVolumes: + input_folder: Path = None + output_folder: Path = None + log_folder: Path = None + + @classmethod + def from_task(cls, task: aiopg.sa.result.RowProxy): + return cls( + Path.home() / f"input/{task.job_id}", + Path.home() / f"output/{task.job_id}", + Path.home() / f"log/{task.job_id}", + ) + + def create(self) -> None: + for folder in [ + self.input_folder, + self.output_folder, + self.log_folder, + ]: + if folder.exists(): + shutil.rmtree(folder) + folder.mkdir(parents=True, exist_ok=True) + + +@attr.s(auto_attribs=True) +class Sidecar: + db_engine: aiopg.sa.Engine = None + db_manager: DBManager = None + rabbit_mq: RabbitMQ = None + task: aiopg.sa.result.RowProxy = None + user_id: str = None + stack_name: str = config.SWARM_STACK_NAME + shared_folders: TaskSharedVolumes = None + + async def _get_node_ports(self): + if self.db_manager is None: + # Keeps single db engine: simcore_sdk.node_ports.dbmanager_{id} + self.db_manager = DBManager(self.db_engine) + return await node_ports.ports(self.db_manager) + + async def _process_task_input(self, port: node_ports.Port, input_ports: Dict): + # pylint: disable=too-many-branches + port_name = port.key + port_value = await port.get() + log.debug("PROCESSING %s %s:%s", port_name, type(port_value), port_value) + if str(port.type).startswith("data:"): + path = port_value + if not path is None: + # the filename is not necessarily the name of the port, might be mapped + mapped_filename = Path(path).name + input_ports[port_name] = str(port_value) + final_path = Path(self.shared_folders.input_folder, mapped_filename) + shutil.copy(str(path), str(final_path)) + log.debug( + "DOWNLOAD successfull from %s to %s via %s", + str(port_name), + str(final_path), + str(path), + ) + else: + input_ports[port_name] = port_value + else: + input_ports[port_name] = port_value + + async def _process_task_inputs(self): + """ Writes input key-value pairs into a dictionary + + if the value of any port starts with 'link.' the corresponding + output ports a fetched or files dowloaded --> @ jsonld + + The dictionary is dumped to input.json, files are dumped + as port['key']. Both end up in /input/ of the container + """ + log.debug( + "Input parsing for %s and node %s from container", + self.task.project_id, + self.task.internal_id, + ) + + input_ports = dict() + PORTS = await self._get_node_ports() + + await logged_gather( + *[ + self._process_task_input(port, input_ports) + for port in (await PORTS.inputs) + ] + ) + + log.debug("DUMPING json") + if input_ports: + file_name = self.shared_folders.input_folder / "input.json" + with file_name.open("w") as fp: + json.dump(input_ports, fp) + log.debug("DUMPING DONE") + + async def _pull_image(self): + docker_image = f"{config.DOCKER_REGISTRY}/{self.task.image['name']}:{self.task.image['tag']}" + log.debug( + "PULLING IMAGE %s as %s with pwd %s", + docker_image, + config.DOCKER_USER, + config.DOCKER_PASSWORD, + ) + try: + docker_client: aiodocker.Docker = aiodocker.Docker() + await docker_client.images.pull( + docker_image, + auth={ + "username": config.DOCKER_USER, + "password": config.DOCKER_PASSWORD, + }, + ) + except aiodocker.exceptions.DockerError: + msg = f"Failed to pull image '{docker_image}'" + log.exception(msg) + raise + + async def _process_task_output(self): + # pylint: disable=too-many-branches + + """ There will be some files in the /output + + - Maybe a output.json (should contain key value for simple things) + - other files: should be named by the key in the output port + + Files will be pushed to S3 with reference in db. output.json will be parsed + and the db updated + """ + log.debug( + "Processing task outputs %s:node %s:internal id %s from container", + self.task.project_id, + self.task.node_id, + self.task.internal_id, + ) + PORTS = await self._get_node_ports() + directory = self.shared_folders.output_folder + if not directory.exists(): + return + try: + for file_path in directory.rglob("*"): + if file_path.name == "output.json": + log.debug("POSTRO FOUND output.json") + # parse and compare/update with the tasks output ports from db + with file_path.open() as fp: + output_ports = json.load(fp) + task_outputs = await PORTS.outputs + for port in task_outputs: + if port.key in output_ports.keys(): + await port.set(output_ports[port.key]) + else: + log.debug("Uploading %s", file_path) + await PORTS.set_file_by_keymap(file_path) + except json.JSONDecodeError: + logging.exception("Error occured while decoding output.json") + except node_ports.exceptions.NodeportsException: + logging.exception("Error occured while setting port") + except (OSError, IOError): + logging.exception("Could not process output") + log.debug( + "Processing task outputs DONE %s:node %s:internal id %s from container", + self.task.project_id, + self.task.node_id, + self.task.internal_id, + ) + + async def _process_task_log(self): + log.debug( + "Processing Logs %s:node %s:internal id %s from container", + self.task.project_id, + self.task.node_id, + self.task.internal_id, + ) + directory = self.shared_folders.log_folder + if directory.exists(): + await node_data.data_manager.push(directory, rename_to="logs") + log.debug( + "Processing Logs DONE %s:node %s:internal id %s from container", + self.task.project_id, + self.task.node_id, + self.task.internal_id, + ) + + async def preprocess(self): + log.debug( + "Pre-Processing Pipeline %s:node %s:internal id %s from container", + self.task.project_id, + self.task.node_id, + self.task.internal_id, + ) + self.shared_folders.create() + await logged_gather(self._process_task_inputs(), self._pull_image()) + log.debug( + "Pre-Processing Pipeline DONE %s:node %s:internal id %s from container", + self.task.project_id, + self.task.node_id, + self.task.internal_id, + ) + + async def post_messages(self, log_type: LogType, message: str): + if log_type == LogType.LOG: + await self.rabbit_mq.post_log_message( + self.user_id, self.task.project_id, self.task.node_id, message, + ) + elif log_type == LogType.PROGRESS: + await self.rabbit_mq.post_progress_message( + self.user_id, self.task.project_id, self.task.node_id, message, + ) + + async def process(self): + log.debug( + "Processing Pipeline %s:node %s:internal id %s from container", + self.task.project_id, + self.task.node_id, + self.task.internal_id, + ) + + # touch output file, so it's ready for the container (v0) + log_file = self.shared_folders.log_folder / "log.dat" + log_file.touch() + + log_processor_task = fire_and_forget_task( + monitor_logs_task(log_file, self.post_messages) + ) + + start_time = time.perf_counter() + container = None + docker_image = f"{config.DOCKER_REGISTRY}/{self.task.image['name']}:{self.task.image['tag']}" + + docker_container_config = { + "Env": [ + f"{name.upper()}_FOLDER=/{name}/{self.task.job_id}" + for name in ["input", "output", "log"] + ], + "Cmd": "run", + "Image": docker_image, + "Labels": { + "user_id": str(self.user_id), + "study_id": str(self.task.project_id), + "node_id": str(self.task.node_id), + "nano_cpus_limit": str(config.SERVICES_MAX_NANO_CPUS), + "mem_limit": str(config.SERVICES_MAX_MEMORY_BYTES), + }, + "HostConfig": { + "Memory": config.SERVICES_MAX_MEMORY_BYTES, + "NanoCPUs": config.SERVICES_MAX_NANO_CPUS, + "Init": True, + "AutoRemove": False, + "Binds": [ + f"{config.SIDECAR_DOCKER_VOLUME_INPUT}:/input", + f"{config.SIDECAR_DOCKER_VOLUME_OUTPUT}:/output", + f"{config.SIDECAR_DOCKER_VOLUME_LOG}:/log", + ], + }, + } + + # volume paths for car container (w/o prefix) + container = None + try: + docker_client: aiodocker.Docker = aiodocker.Docker() + container = await docker_client.containers.run( + config=docker_container_config + ) + + container_data = await container.show() + while container_data["State"]["Running"]: + # reload container data + container_data = await container.show() + if ( + (time.perf_counter() - start_time) > config.SERVICES_TIMEOUT_SECONDS + and config.SERVICES_TIMEOUT_SECONDS > 0 + ): + log.error( + "Running container timed-out after %ss and will be stopped now\nlogs: %s", + config.SERVICES_TIMEOUT_SECONDS, + await container.logs(stdout=True, stderr=True), + ) + await container.stop() + break + + # reload container data + container_data = await container.show() + if container_data["State"]["ExitCode"] > 0: + log.error( + "%s completed with error code %s: %s", + docker_image, + container_data["State"]["ExitCode"], + container_data["State"]["Error"], + ) + else: + log.info("%s completed with successfully!", docker_image) + except aiodocker.exceptions.DockerContainerError: + log.exception( + "Error while running %s with parameters %s", + docker_image, + docker_container_config, + ) + except aiodocker.exceptions.DockerError: + log.exception( + "Unknown error while trying to run %s with parameters %s", + docker_image, + docker_container_config, + ) + finally: + stop_time = time.perf_counter() + log.info("Running %s took %sseconds", docker_image, stop_time - start_time) + if container: + await container.delete(force=True) + # stop monitoring logs now + log_processor_task.cancel() + await log_processor_task + + log.debug( + "DONE Processing Pipeline %s:node %s:internal id %s from container", + self.task.project_id, + self.task.node_id, + self.task.internal_id, + ) + + async def run(self): + log.debug( + "Running Pipeline %s:node %s:internal id %s from container", + self.task.project_id, + self.task.node_id, + self.task.internal_id, + ) + await self.rabbit_mq.post_log_message( + self.user_id, + self.task.project_id, + self.task.node_id, + "Preprocessing start...", + ) + await self.preprocess() + await self.rabbit_mq.post_log_message( + self.user_id, + self.task.project_id, + self.task.node_id, + "...preprocessing end", + ) + + await self.rabbit_mq.post_log_message( + self.user_id, self.task.project_id, self.task.node_id, "Processing start..." + ) + await self.process() + await self.rabbit_mq.post_log_message( + self.user_id, self.task.project_id, self.task.node_id, "...processing end" + ) + + await self.rabbit_mq.post_log_message( + self.user_id, + self.task.project_id, + self.task.node_id, + "Postprocessing start...", + ) + await self.postprocess() + await self.rabbit_mq.post_log_message( + self.user_id, + self.task.project_id, + self.task.node_id, + "...postprocessing end", + ) + + log.debug( + "Running Pipeline DONE %s:node %s:internal id %s from container", + self.task.project_id, + self.task.node_id, + self.task.internal_id, + ) + + async def postprocess(self): + log.debug( + "Post-Processing Pipeline %s:node %s:internal id %s from container", + self.task.project_id, + self.task.node_id, + self.task.internal_id, + ) + + await self._process_task_output() + await self._process_task_log() + + +async def _try_get_task_from_db( + db_connection: aiopg.sa.SAConnection, + graph: nx.DiGraph, + job_request_id: int, + project_id: str, + node_id: str, +) -> Optional[aiopg.sa.result.RowProxy]: + task: aiopg.sa.result.RowProxy = None + # Use SELECT FOR UPDATE TO lock the row + result = await db_connection.execute( + query=comp_tasks.select(for_update=True).where( + and_( + comp_tasks.c.node_id == node_id, + comp_tasks.c.project_id == project_id, + comp_tasks.c.job_id == None, + comp_tasks.c.state == UNKNOWN, + ) + ) + ) + task = await result.fetchone() + + if not task: + log.debug("No task found") + return + + # Check if node's dependecies are there + if not await is_node_ready(task, graph, db_connection, log): + log.debug("TASK %s NOT YET READY", task.internal_id) + return + + # the task is ready! + result = await db_connection.execute( + # FIXME: E1120:No value for argument 'dml' in method call + # pylint: disable=E1120 + comp_tasks.update() + .where( + and_( + comp_tasks.c.node_id == node_id, comp_tasks.c.project_id == project_id, + ) + ) + .values(job_id=job_request_id, state=RUNNING, start=datetime.utcnow()) + .returning(literal_column("*")) + ) + task = await result.fetchone() + log.debug( + "Task %s taken for project:node %s:%s", + task.job_id, + task.project_id, + task.node_id, + ) + return task + + +async def _get_pipeline_from_db( + db_connection: aiopg.sa.SAConnection, project_id: str, +) -> aiopg.sa.result.RowProxy: + pipeline: aiopg.sa.result.RowProxy = None + # get the pipeline + result = await db_connection.execute( + comp_pipeline.select().where(comp_pipeline.c.project_id == project_id) + ) + if result.rowcount > 1: + raise exceptions.DatabaseError( + f"Pipeline {result.rowcount} found instead of only one for project_id {project_id}" + ) + + pipeline = await result.first() + if not pipeline: + raise exceptions.DatabaseError(f"Pipeline {project_id} not found") + log.debug("found pipeline %s", pipeline) + return pipeline + + +async def inspect( + # pylint: disable=too-many-arguments + db_engine: aiopg.sa.Engine, + rabbit_mq: RabbitMQ, + job_request_id: int, + user_id: str, + project_id: str, + node_id: str, +) -> Optional[List[str]]: + log.debug( + "ENTERING inspect with user %s pipeline:node %s: %s", + user_id, + project_id, + node_id, + ) + + pipeline: aiopg.sa.result.RowProxy = None + task: aiopg.sa.result.RowProxy = None + graph: nx.DiGraph = None + async with db_engine.acquire() as connection: + pipeline = await _get_pipeline_from_db(connection, project_id) + graph = execution_graph(pipeline) + if not node_id: + log.debug("NODE id was zero, this was the entry node id") + return find_entry_point(graph) + task = await _try_get_task_from_db( + connection, graph, job_request_id, project_id, node_id + ) + + if not task: + log.debug("no task at hand, let's rest...") + return + + # config nodeports + node_ports.node_config.USER_ID = user_id + node_ports.node_config.NODE_UUID = task.node_id + node_ports.node_config.PROJECT_ID = task.project_id + + # now proceed actually running the task (we do that after the db session has been closed) + # try to run the task, return empyt list of next nodes if anything goes wrong + run_result = SUCCESS + next_task_nodes = [] + try: + sidecar = Sidecar( + db_engine=db_engine, + rabbit_mq=rabbit_mq, + task=task, + user_id=user_id, + shared_folders=TaskSharedVolumes.from_task(task), + ) + await sidecar.run() + next_task_nodes = list(graph.successors(node_id)) + except exceptions.SidecarException: + run_result = FAILED + finally: + async with db_engine.acquire() as connection: + await connection.execute( + # FIXME: E1120:No value for argument 'dml' in method call + # pylint: disable=E1120 + comp_tasks.update() + .where( + and_( + comp_tasks.c.node_id == node_id, + comp_tasks.c.project_id == project_id, + ) + ) + .values(state=run_result, end=datetime.utcnow()) + ) + + return next_task_nodes diff --git a/services/sidecar/src/simcore_service_sidecar/db.py b/services/sidecar/src/simcore_service_sidecar/db.py new file mode 100644 index 00000000000..8ad0b3853b4 --- /dev/null +++ b/services/sidecar/src/simcore_service_sidecar/db.py @@ -0,0 +1,57 @@ +""" database submodule associated to the postgres uservice + +""" + +import logging +import socket + +import tenacity +from aiopg.sa import Engine + +from servicelib.aiopg_utils import ( + DataSourceName, + PostgresRetryPolicyUponInitialization, + create_pg_engine, + is_postgres_responsive, +) + +from .config import POSTGRES_DB, POSTGRES_ENDPOINT, POSTGRES_PW, POSTGRES_USER + +log = logging.getLogger(__name__) + + +@tenacity.retry(**PostgresRetryPolicyUponInitialization().kwargs) +async def wait_till_postgres_responsive(dsn: DataSourceName) -> None: + if not is_postgres_responsive(dsn): + raise Exception + + +class DBContextManager: + def __init__(self): + self._db_engine: Engine = None + + async def __aenter__(self): + dsn = DataSourceName( + application_name=f"{__name__}_{id(socket.gethostname())}", + database=POSTGRES_DB, + user=POSTGRES_USER, + password=POSTGRES_PW, + host=POSTGRES_ENDPOINT.split(":")[0], + port=POSTGRES_ENDPOINT.split(":")[1], + ) + + log.info("Creating pg engine for %s", dsn) + await wait_till_postgres_responsive(dsn) + engine = await create_pg_engine(dsn, minsize=1, maxsize=4) + self._db_engine = engine + return self._db_engine + + async def __aexit__(self, exc_type, exc, tb): + self._db_engine.close() + await self._db_engine.wait_closed() + log.debug( + "engine '%s' after shutdown: closed=%s, size=%d", + self._db_engine.dsn, + self._db_engine.closed, + self._db_engine.size, + ) diff --git a/services/sidecar/src/simcore_service_sidecar/exceptions.py b/services/sidecar/src/simcore_service_sidecar/exceptions.py new file mode 100644 index 00000000000..48bd3d40bf3 --- /dev/null +++ b/services/sidecar/src/simcore_service_sidecar/exceptions.py @@ -0,0 +1,17 @@ +from typing import Optional + + +class SidecarException(Exception): + """Basic exception for errors raised with sidecar""" + + def __init__(self, msg: Optional[str] = None): + if msg is None: + msg = "Unexpected error occured in director subpackage" + super(SidecarException, self).__init__(msg) + + +class DatabaseError(SidecarException): + """Service was not found in swarm""" + + def __init__(self, msg: str): + super(DatabaseError, self).__init__(msg) diff --git a/services/sidecar/src/simcore_service_sidecar/log_parser.py b/services/sidecar/src/simcore_service_sidecar/log_parser.py new file mode 100644 index 00000000000..d9af512ddfe --- /dev/null +++ b/services/sidecar/src/simcore_service_sidecar/log_parser.py @@ -0,0 +1,59 @@ +import asyncio +import logging +from enum import Enum +from pathlib import Path +from typing import Callable, Tuple, Awaitable + +import aiofiles + +log = logging.getLogger(__name__) + + +class LogType(Enum): + LOG = 1 + PROGRESS = 2 + + +async def parse_line(line: str) -> Tuple[LogType, str]: + # TODO: This should be 'settings', a regex for every service + if line.lower().startswith("[progress]"): + return (LogType.PROGRESS, line.lower().lstrip("[progress]").rstrip("%").strip()) + + if "percent done" in line.lower(): + progress = line.lower().rstrip("percent done") + try: + return (LogType.PROGRESS, str(float(progress) / 100.0)) + except ValueError: + log.exception("Could not extract progress from log line %s", line) + # default return as log + return (LogType.LOG, line) + + +async def monitor_logs_task( + log_file: Path, log_cb: Awaitable[Callable[[LogType, str], None]] +) -> None: + try: + log.debug("start monitoring log in %s", log_file) + async with aiofiles.open(log_file, mode="r") as fp: + log.debug("log monitoring: opened %s", log_file) + await fp.seek(0, 2) + await monitor_logs(fp, log_cb) + + except asyncio.CancelledError: + # user cancels + log.debug("stop monitoring log in %s", log_file) + + +async def monitor_logs( + file_pointer, log_cb: Awaitable[Callable[[LogType, str], None]] +) -> None: + while True: + # try to read line + line = await file_pointer.readline() + if not line: + asyncio.sleep(1) + continue + log.debug("log monitoring: found log %s", line) + log_type, parsed_line = await parse_line(line) + + await log_cb(log_type, parsed_line) diff --git a/services/sidecar/src/simcore_service_sidecar/rabbitmq.py b/services/sidecar/src/simcore_service_sidecar/rabbitmq.py new file mode 100644 index 00000000000..ae852c4c556 --- /dev/null +++ b/services/sidecar/src/simcore_service_sidecar/rabbitmq.py @@ -0,0 +1,122 @@ +import json +import logging +import socket +from typing import Dict, List, Optional, Union + +import aio_pika +import tenacity +from pydantic import BaseModel # pylint: disable=no-name-in-module + +from servicelib.rabbitmq_utils import RabbitMQRetryPolicyUponInitialization +from simcore_sdk.config.rabbit import Config as RabbitConfig + +log = logging.getLogger(__file__) + + +def reconnect_callback(): + log.error("Rabbit reconnected") + + +def channel_close_callback(exc: Optional[BaseException]): + if exc: + log.error("Rabbit channel closed: %s", exc) + + +class RabbitMQ(BaseModel): + config: RabbitConfig = RabbitConfig() + connection: aio_pika.RobustConnection = None + channel: aio_pika.Channel = None + logs_exchange: aio_pika.Exchange = None + progress_exchange: aio_pika.Exchange = None + + class Config: + # see https://pydantic-docs.helpmanual.io/usage/types/#arbitrary-types-allowed + arbitrary_types_allowed = True + + async def connect(self): + url = self.config.broker_url + log.debug("Connecting to %s", url) + await wait_till_rabbit_responsive(url) + + # NOTE: to show the connection name in the rabbitMQ UI see there [https://www.bountysource.com/issues/89342433-setting-custom-connection-name-via-client_properties-doesn-t-work-when-connecting-using-an-amqp-url] + self.connection = await aio_pika.connect_robust( + url + f"?name={__name__}_{id(socket.gethostname())}", + client_properties={"connection_name": "sidecar connection"}, + ) + self.connection.add_reconnect_callback(reconnect_callback) + + log.debug("Creating channel") + self.channel = await self.connection.channel(publisher_confirms=False) + self.channel.add_close_callback(channel_close_callback) + + log.debug("Declaring %s exchange", self.config.channels["log"]) + self.logs_exchange = await self.channel.declare_exchange( + self.config.channels["log"], aio_pika.ExchangeType.FANOUT + ) + log.debug("Declaring %s exchange", self.config.channels["progress"]) + self.progress_exchange = await self.channel.declare_exchange( + self.config.channels["progress"], aio_pika.ExchangeType.FANOUT, + ) + + async def close(self): + log.debug("Closing channel...") + await self.channel.close() + log.debug("Closing connection...") + await self.connection.close() + + async def _post_message(self, exchange: aio_pika.Exchange, data: Dict[str, str]): + await exchange.publish( + aio_pika.Message(body=json.dumps(data).encode()), routing_key="" + ) + + async def post_log_message( + self, + user_id: str, + project_id: str, + node_id: str, + log_msg: Union[str, List[str]], + ): + await self._post_message( + self.logs_exchange, + data={ + "Channel": "Log", + "Node": node_id, + "user_id": user_id, + "project_id": project_id, + "Messages": log_msg if isinstance(log_msg, list) else [log_msg], + }, + ) + + async def post_progress_message( + self, user_id: str, project_id: str, node_id: str, progress_msg: str + ): + await self._post_message( + self.logs_exchange, + data={ + "Channel": "Progress", + "Node": node_id, + "user_id": user_id, + "project_id": project_id, + "Progress": progress_msg, + }, + ) + + +@tenacity.retry(**RabbitMQRetryPolicyUponInitialization().kwargs) +async def wait_till_rabbit_responsive(url: str): + connection = await aio_pika.connect(url) + await connection.close() + return True + + +class RabbitMQContextManager: + def __init__(self): + self._rabbit_mq: RabbitMQ = None + + async def __aenter__(self): + self._rabbit_mq = RabbitMQ() + await self._rabbit_mq.connect() + return self._rabbit_mq + + async def __aexit__(self, exc_type, exc, tb): + await self._rabbit_mq.close() diff --git a/services/sidecar/src/sidecar/remote_debug.py b/services/sidecar/src/simcore_service_sidecar/remote_debug.py similarity index 59% rename from services/sidecar/src/sidecar/remote_debug.py rename to services/sidecar/src/simcore_service_sidecar/remote_debug.py index 8205b76e5d8..1a3029c55a3 100644 --- a/services/sidecar/src/sidecar/remote_debug.py +++ b/services/sidecar/src/simcore_service_sidecar/remote_debug.py @@ -5,17 +5,20 @@ from .celery_log_setup import get_task_logger +REMOTE_DEBUG_PORT = 3000 + log = get_task_logger(__name__) -def setup_remote_debugging(force_enabled=False): +def setup_remote_debugging(force_enabled: bool = False, *, boot_mode=None) -> None: """ Programaticaly enables remote debugging if SC_BOOT_MODE==debug-ptvsd """ if "SC_BOOT_MODE" not in os.environ: - raise ValueError("Remote debugging only available when running in a container") + log.warning("Remote debugging only available when running in a container") + return - boot_mode = os.environ["SC_BOOT_MODE"] + boot_mode = boot_mode or os.environ.get("SC_BOOT_MODE") if boot_mode == "debug-ptvsd" or force_enabled: try: @@ -24,17 +27,18 @@ def setup_remote_debugging(force_enabled=False): # SEE https://github.com/microsoft/ptvsd#enabling-debugging # import ptvsd - ptvsd.enable_attach(address=('0.0.0.0', 3000), redirect_output=True) # nosec + + ptvsd.enable_attach( + address=("0.0.0.0", REMOTE_DEBUG_PORT), redirect_output=True + ) # nosec except ImportError: log.exception("Unable to use remote debugging. ptvsd is not installed") else: - log.info("Remote debugging enabled") + log.info("Remote debugging enabled: listening port %s", REMOTE_DEBUG_PORT) else: log.debug("Booting without remote debugging since SC_BOOT_MODE=%s", boot_mode) -__all__ = [ - 'setup_remote_debugging' -] +__all__ = ["setup_remote_debugging"] diff --git a/services/sidecar/src/simcore_service_sidecar/utils.py b/services/sidecar/src/simcore_service_sidecar/utils.py new file mode 100644 index 00000000000..e60ce11c2c8 --- /dev/null +++ b/services/sidecar/src/simcore_service_sidecar/utils.py @@ -0,0 +1,65 @@ +import asyncio +import logging +from typing import List + +import aiopg +import networkx as nx +from simcore_postgres_database.sidecar_models import SUCCESS, comp_pipeline, comp_tasks +from sqlalchemy import and_ + + +def wrap_async_call(fct: asyncio.coroutine): + return asyncio.get_event_loop().run_until_complete(fct) + + +def find_entry_point(g: nx.DiGraph) -> List: + result = [] + for node in g.nodes: + if len(list(g.predecessors(node))) == 0: + result.append(node) + return result + + +async def is_node_ready( + task: comp_tasks, + graph: nx.DiGraph, + db_connection: aiopg.sa.SAConnection, + _logger: logging.Logger, +) -> bool: + query = comp_tasks.select().where( + and_( + comp_tasks.c.node_id.in_(list(graph.predecessors(task.node_id))), + comp_tasks.c.project_id == task.project_id, + ) + ) + result = await db_connection.execute(query) + tasks = await result.fetchall() + + _logger.debug("TASK %s ready? Checking ..", task.internal_id) + for dep_task in tasks: + job_id = dep_task.job_id + if not job_id: + return False + _logger.debug( + "TASK %s DEPENDS ON %s with stat %s", + task.internal_id, + dep_task.internal_id, + dep_task.state, + ) + if not dep_task.state == SUCCESS: + return False + _logger.debug("TASK %s is ready", task.internal_id) + return True + + +def execution_graph(pipeline: comp_pipeline) -> nx.DiGraph: + d = pipeline.dag_adjacency_list + G = nx.DiGraph() + + for node in d.keys(): + nodes = d[node] + if len(nodes) == 0: + G.add_node(node) + continue + G.add_edges_from([(node, n) for n in nodes]) + return G diff --git a/services/sidecar/tests/THESE_TESTS_ARE_DISABLED b/services/sidecar/tests/THESE_TESTS_ARE_DISABLED deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/services/sidecar/tests/conftest.py b/services/sidecar/tests/conftest.py index 2ca9a680a75..ab35f42022e 100644 --- a/services/sidecar/tests/conftest.py +++ b/services/sidecar/tests/conftest.py @@ -1,160 +1,29 @@ -# pylint: disable=unused-argument -# pylint: disable=unused-import -# pylint: disable=no-name-in-module -# pylint: disable=W0621 -import asyncio -import os -import subprocess +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name + import sys -import uuid -from collections import namedtuple -from concurrent.futures import ThreadPoolExecutor from pathlib import Path -from random import randrange import pytest -from aiopg.sa import create_engine - -import sidecar -# from simcore_service_storage.datcore_wrapper import DatcoreWrapper -# from simcore_service_storage.dsm import DataStorageManager -# from simcore_service_storage.models import FileMetaData - -import utils -from utils import (ACCESS_KEY, BUCKET_NAME, DATABASE, PASS, RABBIT_PWD, - RABBIT_USER, SECRET_KEY, USER) - -# fixtures ------------------------------------------------------- - -@pytest.fixture(scope='session') -def here(): - return Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent - -@pytest.fixture(scope='session') -def docker_compose_file(here): - """ Overrides pytest-docker fixture - """ - old = os.environ.copy() - - # docker-compose reads these environs - os.environ['POSTGRES_DB']=DATABASE - os.environ['POSTGRES_USER']=USER - os.environ['POSTGRES_PASSWORD']=PASS - os.environ['POSTGRES_ENDPOINT']="FOO" # TODO: update config schema!! - os.environ['MINIO_ACCESS_KEY']=ACCESS_KEY - os.environ['MINIO_SECRET_KEY']=SECRET_KEY - os.environ['RABBIT_USER']=RABBIT_USER - os.environ['RABBIT_PASSWORD']=RABBIT_PWD - - - dc_path = here / 'docker-compose.yml' - - assert dc_path.exists() - yield str(dc_path) - - os.environ = old - -@pytest.fixture(scope='session') -def postgres_service(docker_services, docker_ip): - url = 'postgresql://{user}:{password}@{host}:{port}/{database}'.format( - user = USER, - password = PASS, - database = DATABASE, - host=docker_ip, - port=docker_services.port_for('postgres', 5432), - ) - - # Wait until service is responsive. - docker_services.wait_until_responsive( - check=lambda: utils.is_postgres_responsive(url), - timeout=30.0, - pause=0.1, - ) - - postgres_service = { - 'user' : USER, - 'password' : PASS, - 'database' : DATABASE, - 'host' : docker_ip, - 'port' : docker_services.port_for('postgres', 5432) - } - # FIXME: use monkeypatch.setenv("POSTGRES_ENDPOINT", ...) instead - - # set env var here that is explicitly used from sidecar - os.environ['POSTGRES_ENDPOINT'] = "{host}:{port}".format(host=docker_ip, port=docker_services.port_for('postgres', 5432)) - return postgres_service - -@pytest.fixture(scope='session') -def rabbit_service(docker_services, docker_ip): - # set env var here that is explicitly used from sidecar - # FIXME: use monkeypatch.setenv("RABBIT_HOST", ...) instead - os.environ['RABBIT_HOST'] = "{host}".format(host=docker_ip) - os.environ['RABBIT_PORT'] = "{port}".format(port=docker_services.port_for('rabbit', 5672)) - - rabbit_service = "dummy" - return rabbit_service - -@pytest.fixture(scope='session') -def postgres_service_url(postgres_service, docker_services, docker_ip): - postgres_service_url = 'postgresql://{user}:{password}@{host}:{port}/{database}'.format( - user = USER, - password = PASS, - database = DATABASE, - host=docker_ip, - port=docker_services.port_for('postgres', 5432), - ) - - return postgres_service_url - -@pytest.fixture(scope='function') -async def postgres_engine(loop, postgres_service_url): - postgres_engine = await create_engine(postgres_service_url) - - yield postgres_engine - - if postgres_engine: - postgres_engine.close() - await postgres_engine.wait_closed() - - -@pytest.fixture(scope='session') -def minio_service(docker_services, docker_ip): - - # Build URL to service listening on random port. - url = 'http://%s:%d/' % ( - docker_ip, - docker_services.port_for('minio', 9000), - ) - - # Wait until service is responsive. - docker_services.wait_until_responsive( - check=lambda: utils.is_responsive(url, 403), - timeout=30.0, - pause=0.1, - ) - # FIXME: use monkeypatch.setenv("S3_BUCKET_NAME", ...) instead - os.environ['S3_BUCKET_NAME'] = "simcore-testing" - os.environ['S3_ENDPOINT'] = '{ip}:{port}'.format(ip=docker_ip, port=docker_services.port_for('minio', 9000)) - os.environ['S3_ACCESS_KEY'] = ACCESS_KEY - os.environ['S3_SECRET_KEY'] = SECRET_KEY +import simcore_service_sidecar - return { - 'endpoint': '{ip}:{port}'.format(ip=docker_ip, port=docker_services.port_for('minio', 9000)), - 'access_key': ACCESS_KEY, - 'secret_key' : SECRET_KEY, - } +pytest_plugins = ["pytest_simcore.environs"] -@pytest.fixture(scope="module") -def s3_client(minio_service): - from s3wrapper.s3_client import S3Client +current_dir = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent - s3_client = S3Client(**minio_service) - return s3_client +@pytest.fixture(scope="session") +def project_slug_dir(): + folder = current_dir.parent + assert folder.exists() + assert any(folder.glob("src/simcore_service_sidecar")) + return folder -@pytest.fixture(scope="module") -def sidecar_platform_fixture(s3_client, postgres_service_url, rabbit_service): - sidecar_platform_fixture = 1 - return sidecar_platform_fixture +@pytest.fixture(scope="session") +def package_dir(): + dirpath = Path(simcore_service_sidecar.__file__).resolve().parent + assert dirpath.exists() + return dirpath diff --git a/services/sidecar/tests/docker-compose.yml b/services/sidecar/tests/docker-compose.yml deleted file mode 100644 index 768781b89aa..00000000000 --- a/services/sidecar/tests/docker-compose.yml +++ /dev/null @@ -1,44 +0,0 @@ -version: '3.4' -services: - postgres: - image: postgres:10 - restart: always - environment: - POSTGRES_DB: ${POSTGRES_DB:-sidecar_test} - POSTGRES_USER: ${POSTGRES_USER:-scu} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-scu} - ports: - - '5432:5432' - # NOTES: this is not yet compatible with portainer deployment but could work also for other containers - # works with Docker 19.03 and not yet with Portainer 1.23.0 (see https://github.com/portainer/portainer/issues/3551) - # in the meantime postgres allows to set a configuration through CLI. - # sysctls: - # # NOTES: these values are needed here because docker swarm kills long running idle - # # connections by default after 15 minutes see https://github.com/moby/moby/issues/31208 - # # info about these values are here https://tldp.org/HOWTO/TCP-Keepalive-HOWTO/usingkeepalive.html - # - net.ipv4.tcp_keepalive_intvl=600 - # - net.ipv4.tcp_keepalive_probes=9 - # - net.ipv4.tcp_keepalive_time=600 - command: postgres -c tcp_keepalives_idle=600 -c tcp_keepalives_interval=600 -c tcp_keepalives_count=5 - adminer: - image: adminer - restart: always - ports: - - 18080:8080 - depends_on: - - postgres - minio: - image: minio/minio - environment: - - MINIO_ACCESS_KEY=${MINIO_ACCESS_KEY:-12345678} - - MINIO_SECRET_KEY=${MINIO_SECRET_KEY:-12345678} - ports: - - "9001:9000" - command: server /data - rabbit: - image: rabbitmq:3-management - environment: - - RABBIT_DEFAULT_USER=${RABBIT_USER:-rabbit} - - RABBIT_DEFAULT_PASS=${RABBIT_PASSWORD:-carrot} - ports: - - "15672:15672" diff --git a/services/sidecar/tests/integration/conftest.py b/services/sidecar/tests/integration/conftest.py new file mode 100644 index 00000000000..6d46725d03d --- /dev/null +++ b/services/sidecar/tests/integration/conftest.py @@ -0,0 +1,18 @@ +import logging +import sys +from pathlib import Path + +current_dir = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent + +# imports the fixtures for the integration tests +pytest_plugins = [ + "pytest_simcore.environs", + "pytest_simcore.docker_compose", + "pytest_simcore.docker_swarm", + "pytest_simcore.docker_registry", + "pytest_simcore.rabbit_service", + "pytest_simcore.postgres_service", + "pytest_simcore.minio_service", + "pytest_simcore.simcore_storage_service", +] +log = logging.getLogger(__name__) diff --git a/services/sidecar/tests/integration/fixtures/docker_registry.py b/services/sidecar/tests/integration/fixtures/docker_registry.py deleted file mode 100644 index bea2ab0412c..00000000000 --- a/services/sidecar/tests/integration/fixtures/docker_registry.py +++ /dev/null @@ -1,100 +0,0 @@ -# pylint:disable=wildcard-import -# pylint:disable=unused-import -# pylint:disable=unused-variable -# pylint:disable=unused-argument -# pylint:disable=redefined-outer-name - -import docker -import pytest -import tenacity -import time - -@pytest.fixture(scope="session") -def docker_registry(): - # run the registry outside of the stack - docker_client = docker.from_env() - container = docker_client.containers.run("registry:2", - ports={"5000":"5000"}, - environment=["REGISTRY_STORAGE_DELETE_ENABLED=true"], - restart_policy={"Name":"always"}, - detach=True - ) - host = "127.0.0.1" - port = 5000 - url = "{host}:{port}".format(host=host, port=port) - # Wait until we can connect - assert _wait_till_registry_is_responsive(url) - - # test the registry - docker_client = docker.from_env() - # get the hello world example from docker hub - hello_world_image = docker_client.images.pull("hello-world","latest") - # login to private registry - docker_client.login(registry=url, username="simcore") - # tag the image - repo = url + "/hello-world:dev" - assert hello_world_image.tag(repo) == True - # push the image to the private registry - docker_client.images.push(repo) - # wipe the images - docker_client.images.remove(image="hello-world:latest") - docker_client.images.remove(image=hello_world_image.id) - # pull the image from the private registry - private_image = docker_client.images.pull(repo) - docker_client.images.remove(image=private_image.id) - - yield url - - container.stop() - - while docker_client.containers.list(filters={"name": container.name}): - time.sleep(1) - -@tenacity.retry(wait=tenacity.wait_fixed(1), stop=tenacity.stop_after_delay(60)) -def _wait_till_registry_is_responsive(url): - docker_client = docker.from_env() - docker_client.login(registry=url, username="simcore") - return True - - -#pull from itisfoundation/sleeper and push into local registry -@pytest.fixture(scope="session") -def sleeper_service(docker_registry) -> str: - """ Adds a itisfoundation/sleeper in docker registry - - """ - client = docker.from_env() - image = client.images.pull("itisfoundation/sleeper", tag="1.0.0") - assert not image is None - repo = "{}/simcore/services/comp/itis/sleeper:1.0.0".format(docker_registry) - assert image.tag(repo) == True - client.images.push(repo) - image = client.images.pull(repo) - assert image - yield repo - -@pytest.fixture(scope="session") -def jupyter_service(docker_registry) -> str: - """ Adds a itisfoundation/jupyter-base-notebook in docker registry - - """ - client = docker.from_env() - - # TODO: cleanup - - # pull from dockerhub - reponame, tag = "itisfoundation/jupyter-base-notebook:2.13.0".split(":") - image = client.images.pull(reponame, tag=tag) - assert not image is None - - # push to fixture registry (services/{dynamic|comp}) - image_name = reponame.split("/")[-1] - repo = f"{docker_registry}/simcore/services/dynamic/{image_name}:{tag}" - assert image.tag(repo) == True - client.images.push(repo) - - # check - image = client.images.pull(repo) - assert image - - yield repo diff --git a/services/sidecar/tests/integration/test_sidecar.py b/services/sidecar/tests/integration/test_sidecar.py new file mode 100644 index 00000000000..e968ad47207 --- /dev/null +++ b/services/sidecar/tests/integration/test_sidecar.py @@ -0,0 +1,136 @@ +# pylint: disable=unused-argument +# pylint: disable=unused-import +# pylint: disable=redefined-outer-name +# pylint: disable=too-many-arguments + +import json +import os +from pathlib import Path +from typing import Any, Dict +from uuid import uuid4 + +import aio_pika +import pytest +import sqlalchemy as sa +from yarl import URL + +from simcore_sdk.models.pipeline_models import ComputationalPipeline, ComputationalTask +from simcore_service_sidecar import config + +# Selection of core and tool services started in this swarm fixture (integration) +core_services = ["storage", "postgres", "rabbit"] + +ops_services = ["minio", "adminer"] + + +@pytest.fixture +def project_id() -> str: + return str(uuid4()) + + +@pytest.fixture +def user_id() -> int: + return 1 + + +@pytest.fixture +def create_pipeline(postgres_session: sa.orm.session.Session, project_id: str): + def create(tasks: Dict[str, Any], dag: Dict) -> ComputationalPipeline: + # set the pipeline + pipeline = ComputationalPipeline(project_id=project_id, dag_adjacency_list=dag) + postgres_session.add(pipeline) + postgres_session.commit() + # now create the tasks + for node_uuid, service in tasks.items(): + comp_task = ComputationalTask( + project_id=project_id, + node_id=node_uuid, + schema=service["schema"], + image=service["image"], + inputs={}, + outputs={}, + ) + postgres_session.add(comp_task) + postgres_session.commit() + return pipeline + + yield create + + +@pytest.fixture +def sidecar_config(postgres_dsn: Dict[str, str], docker_registry: str) -> None: + # NOTE: in integration tests the sidecar runs bare-metal which means docker volume cannot be used. + config.SIDECAR_DOCKER_VOLUME_INPUT = Path.home() / f"input" + config.SIDECAR_DOCKER_VOLUME_OUTPUT = Path.home() / f"output" + config.SIDECAR_DOCKER_VOLUME_LOG = Path.home() / f"log" + + config.DOCKER_REGISTRY = docker_registry + config.DOCKER_USER = "simcore" + config.DOCKER_PASSWORD = "" + + config.POSTGRES_DB = postgres_dsn["database"] + config.POSTGRES_ENDPOINT = f"{postgres_dsn['host']}:{postgres_dsn['port']}" + config.POSTGRES_USER = postgres_dsn["user"] + config.POSTGRES_PW = postgres_dsn["password"] + + +async def test_run_sleepers( + loop, + postgres_session: sa.orm.session.Session, + rabbit_queue, + storage_service: URL, + sleeper_service: Dict[str, str], + sidecar_config: None, + create_pipeline, + user_id: int, + mocker, +): + from simcore_service_sidecar import cli + + incoming_data = [] + + async def rabbit_message_handler(message: aio_pika.IncomingMessage): + data = json.loads(message.body) + incoming_data.append(data) + + await rabbit_queue.consume(rabbit_message_handler, exclusive=True, no_ack=True) + + job_id = 1 + + pipeline = create_pipeline( + tasks={ + "node_1": sleeper_service, + "node_2": sleeper_service, + "node_3": sleeper_service, + "node_4": sleeper_service, + }, + dag={ + "node_1": ["node_2", "node_3"], + "node_2": ["node_4"], + "node_3": ["node_4"], + "node_4": [], + }, + ) + + import asyncio + + next_task_nodes = await cli.run_sidecar(job_id, user_id, pipeline.project_id, None) + await asyncio.sleep(5) + assert not incoming_data + # async with rabbit_queue.iterator() as queue_iter: + # async for message in queue_iter: + # async with message.process(): + + # incoming_data.append(json.loads(message.body)) + + assert len(next_task_nodes) == 1 + assert next_task_nodes[0] == "node_1" + + for node_id in next_task_nodes: + job_id += 1 + next_tasks = await cli.run_sidecar( + job_id, user_id, pipeline.project_id, node_id + ) + if next_tasks: + next_task_nodes.extend(next_tasks) + assert next_task_nodes == ["node_1", "node_2", "node_3", "node_4", "node_4"] diff --git a/services/sidecar/tests/test_sidecar.py b/services/sidecar/tests/test_sidecar.py deleted file mode 100644 index 3a8b876023e..00000000000 --- a/services/sidecar/tests/test_sidecar.py +++ /dev/null @@ -1,42 +0,0 @@ -#pylint: disable=unused-argument, unused-import, no-name-in-module -import pytest - -from utils import create_tables, setup_sleepers - -class FakeRequest(): - def __init__(self): - self.id = "1" - -class FakeTask(): - def __init__(self): - self.request = FakeRequest() - self.state = 0 - - def update_state(self, state): - self.state = state - - -def run(task, user_id, pipeline_id, node_id=None): - next_task_nodes = [] - try: - from sidecar.core import SIDECAR - next_task_nodes = SIDECAR.inspect(task, user_id, pipeline_id, node_id) - #pylint:disable=broad-except - except Exception: - assert False - - for _node_id in next_task_nodes: - task.request.id = str(int(task.request.id) + 1) - run(task, user_id, pipeline_id, _node_id) - - -def test_sleeper(sidecar_platform_fixture, postgres_service_url): - # create database tables - create_tables(postgres_service_url) - - # create entry for sleeper - - pipeline_id = setup_sleepers(postgres_service_url) - task = FakeTask() - user_id = "fakeuser" - run(task, user_id, pipeline_id, node_id=None) diff --git a/services/sidecar/tests/unit/test_code_syntax.py b/services/sidecar/tests/unit/test_code_syntax.py new file mode 100644 index 00000000000..de951423e9c --- /dev/null +++ b/services/sidecar/tests/unit/test_code_syntax.py @@ -0,0 +1,36 @@ +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name + +import os +import re +import subprocess +from pathlib import Path + +import pytest + + +@pytest.fixture +def pylintrc(osparc_simcore_root_dir): + pylintrc_path = osparc_simcore_root_dir / ".pylintrc" + assert pylintrc_path.exists() + return pylintrc_path + + +def test_run_pylint(pylintrc, package_dir): + cmd = "pylint --jobs 0 --rcfile {} -v {}".format(pylintrc, package_dir) + proc: subprocess.CompletedProcess = subprocess.run(cmd.split(), check=True) + assert proc.returncode == 0, f"pylint error: {proc.stdout}" + + +def test_no_pdbs_in_place(package_dir): + MATCH = re.compile(r"pdb.set_trace()") + EXCLUDE = ["__pycache__", ".git"] + for root, dirs, files in os.walk(package_dir): + for name in files: + if name.endswith(".py"): + pypth = Path(root) / name + code = pypth.read_text() + found = MATCH.findall(code) + assert not found, "pbd.set_trace found in %s" % pypth + dirs[:] = [d for d in dirs if d not in EXCLUDE] \ No newline at end of file diff --git a/services/sidecar/tests/utils.py b/services/sidecar/tests/utils.py deleted file mode 100644 index b831dccade9..00000000000 --- a/services/sidecar/tests/utils.py +++ /dev/null @@ -1,163 +0,0 @@ -import datetime - -import requests -import sqlalchemy as sa - -from simcore_sdk.models.pipeline_models import (Base, ComputationalPipeline, - ComputationalTask) - -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker - -DATABASE = 'sidecar_test' -USER = 'scu' -PASS = 'scu' - -ACCESS_KEY = '12345678' -SECRET_KEY = '12345678' - -BUCKET_NAME ="simcore-testing" - -RABBIT_USER = "rabbit" -RABBIT_PWD = "carrot" - -def is_responsive(url, code=200): - """Check if something responds to ``url`` syncronously""" - try: - response = requests.get(url) - if response.status_code == code: - return True - except requests.exceptions.RequestException as _e: - pass - - return False - -def is_postgres_responsive(url): - """Check if something responds to ``url`` """ - try: - engine = sa.create_engine(url) - conn = engine.connect() - conn.close() - except sa.exc.OperationalError: - return False - return True - -def create_tables(url, engine=None): - if not engine: - engine = create_engine(url, - client_encoding="utf8", - connect_args={"connect_timeout": 30}, - pool_pre_ping=True) - Base.metadata.create_all(engine) - engine.dispose() - else: - Base.metadata.create_all(engine) - - -def drop_tables(url, engine=None): - is_owned = not engine - if is_owned: - engine = create_engine(url, - client_encoding="utf8", - connect_args={"connect_timeout": 30}, - pool_pre_ping=True) - - Base.metadata.drop_tables(engine) - if is_owned: - engine.dispose() - -def setup_sleepers(url): - db_engine = create_engine(url, - client_encoding="utf8", - connect_args={"connect_timeout": 30}, - pool_pre_ping=True) - - DatabaseSession = sessionmaker(db_engine) - db_session = DatabaseSession() - - - dag_adjacency_list = {"e609a68c-d743-4a12-9745-f31734d1b911": ["3e5103b3-8930-4025-846b-b8995460379e"], "3e5103b3-8930-4025-846b-b8995460379e": []} - - pipeline = ComputationalPipeline(dag_adjacency_list=dag_adjacency_list, state=0) - db_session.add(pipeline) - db_session.flush() - # pylint: disable=no-member - project_id = pipeline.project_id - - node_id_1 = "e609a68c-d743-4a12-9745-f31734d1b911" - internal_id_1 = 1 - - node_schema = { - "inputs":{ - "in_1":{ - "label": "Number of seconds to sleep", - "description": "Number of seconds to sleep", - "displayOrder":0, - "type": "data:*/*" - }, - "in_2": { - "label": "Number of seconds to sleep", - "description": "Number of seconds to sleep", - "displayOrder":1, - "type": "integer", - "defaultValue": 4 - } - }, - "outputs":{ - "out_1":{ - "label": "Number of seconds to sleep", - "description": "Number of seconds to sleep", - "displayOrder":0, - "type": "data:*/*" - }, - "out_2": { - "label": "Number of seconds to sleep", - "description": "Number of seconds to sleep", - "displayOrder":1, - "type": "integer" - } - } - } - node_inputs_1 = {} - node_outputs_1 = {"out_2":1} - - node_key = "simcore/services/comp/itis/sleeper" - node_version = "0.0.1" - # create the task - comp_task_1 = ComputationalTask( - project_id=project_id, - node_id=node_id_1, - internal_id=internal_id_1, - schema=node_schema, - image={"name":node_key, "tag":node_version}, - inputs=node_inputs_1, - outputs=node_outputs_1, - submit=datetime.datetime.utcnow() - ) - db_session.add(comp_task_1) - db_session.commit() - - node_id_2 = "3e5103b3-8930-4025-846b-b8995460379e" - internal_id_2 = 2 - - node_inputs_2 = {"in_1":{"nodeUuid": "e609a68c-d743-4a12-9745-f31734d1b911", "output":"out_1"}, "in_2":{"nodeUuid": "e609a68c-d743-4a12-9745-f31734d1b911", "output":"out_2"}} - node_outputs_2 = {"out_2":5} - - node_key = "simcore/services/comp/itis/sleeper" - node_version = "0.0.1" - # create the task - comp_task_2 = ComputationalTask( - project_id=project_id, - node_id=node_id_2, - internal_id=internal_id_2, - schema=node_schema, - image={"name":node_key, "tag":node_version}, - inputs=node_inputs_2, - outputs=node_outputs_2, - submit=datetime.datetime.utcnow() - ) - - db_session.add(comp_task_2) - db_session.commit() - - return project_id diff --git a/services/web/client/Makefile b/services/web/client/Makefile index 3f4c2642aea..d3850e0c701 100644 --- a/services/web/client/Makefile +++ b/services/web/client/Makefile @@ -29,9 +29,9 @@ compile-dev: qx_packages ## qx compiles host' 'source' -> host's 'source-output' --set-env osparc.vcsOriginUrl="${VCS_URL}" .PHONY: compile touch upgrade -compile: ## qx compiles host' 'source' -> image's 'build-output' +compile compile-x: ## qx compiles host' 'source' -> image's 'build-output' # qx compile 'source' within $(docker_image) image [itisfoundation/qooxdoo-kit:${QOOXDOO_KIT_TAG}] - @docker build --file $(docker_file) --tag $(docker_image) \ + @docker $(if $(findstring -x,$@),buildx,) build --file $(docker_file) --tag $(docker_image) \ --build-arg tag=${QOOXDOO_KIT_TAG} \ --build-arg VCS_REF=${VCS_REF} \ --build-arg VCS_REF_CLIENT=${VCS_REF_CLIENT} \ @@ -39,9 +39,9 @@ compile: ## qx compiles host' 'source' -> image's 'build-output' --build-arg VCS_URL=${VCS_URL} \ --target=build-client . -touch: ## minimal image build with /project/output-build inside +touch touch-x: ## minimal image build with /project/output-build inside # touch /project/output-build such that multi-stage 'services/web/Dockerfile' can build development target (fixes #1097) - @docker build --file $(docker_file) --tag $(docker_image) --build-arg tag=${QOOXDOO_KIT_TAG} --target=touch . + @docker $(if $(findstring -x,$@),buildx,) build --file $(docker_file) --tag $(docker_image) --build-arg tag=${QOOXDOO_KIT_TAG} --target=touch . upgrade: ## upgrade to official version of the tool # upgrading to ${QOOXDOO_KIT_TAG} diff --git a/services/web/server/src/simcore_service_webserver/computation_handlers.py b/services/web/server/src/simcore_service_webserver/computation_handlers.py index ae7f4b41680..81a56c8d242 100644 --- a/services/web/server/src/simcore_service_webserver/computation_handlers.py +++ b/services/web/server/src/simcore_service_webserver/computation_handlers.py @@ -9,7 +9,7 @@ from servicelib.application_keys import APP_CONFIG_KEY from servicelib.request_keys import RQT_USERID_KEY -from simcore_sdk.config.rabbit import Config as rabbit_config +from simcore_sdk.config.rabbit import Config as RabbitConfig from .computation_api import update_pipeline_db from .computation_config import CONFIG_SECTION_NAME as CONFIG_RABBIT_SECTION @@ -25,8 +25,8 @@ def get_celery(_app: web.Application): config = _app[APP_CONFIG_KEY][CONFIG_RABBIT_SECTION] - rabbit = rabbit_config(config=config) - celery = Celery(rabbit.name, broker=rabbit.broker, backend=rabbit.backend) + rabbit = RabbitConfig(**config) + celery = Celery(rabbit.name, broker=rabbit.broker_url, backend=rabbit.backend,) return celery @@ -57,7 +57,6 @@ async def update_pipeline(request: web.Request) -> web.Response: await update_pipeline_db(request.app, project_id, project["workbench"]) except ProjectNotFoundError: raise web.HTTPNotFound(reason=f"Project {project_id} not found") - raise web.HTTPNoContent() diff --git a/services/web/server/src/simcore_service_webserver/computation_subscribe.py b/services/web/server/src/simcore_service_webserver/computation_subscribe.py index af952421019..372b9e8d88a 100644 --- a/services/web/server/src/simcore_service_webserver/computation_subscribe.py +++ b/services/web/server/src/simcore_service_webserver/computation_subscribe.py @@ -11,17 +11,19 @@ from servicelib.application_keys import APP_CONFIG_KEY from servicelib.rabbitmq_utils import RabbitMQRetryPolicyUponInitialization -from simcore_sdk.config.rabbit import eval_broker +from simcore_sdk.config.rabbit import Config as RabbitConfig -from .computation_config import (APP_CLIENT_RABBIT_DECORATED_HANDLERS_KEY, - CONFIG_SECTION_NAME) +from .computation_config import ( + APP_CLIENT_RABBIT_DECORATED_HANDLERS_KEY, + CONFIG_SECTION_NAME, +) from .projects import projects_api -from .projects.projects_exceptions import (NodeNotFoundError, - ProjectNotFoundError) +from .projects.projects_exceptions import NodeNotFoundError, ProjectNotFoundError from .socketio.events import post_messages log = logging.getLogger(__file__) + def rabbit_adapter(app: web.Application) -> Callable: """this decorator allows passing additional paramters to python-socketio compatible handlers. I.e. aiopika handler expect functions of type `async def function(message)` @@ -78,12 +80,13 @@ async def subscribe(app: web.Application) -> None: # This exception is catch and pika persists ... WARNING:pika.connection:Could not connect, 5 attempts l rb_config: Dict = app[APP_CONFIG_KEY][CONFIG_SECTION_NAME] - rabbit_broker = eval_broker(rb_config) + rabbit_broker = RabbitConfig(**rb_config).broker_url log.info("Creating pika connection for %s", rabbit_broker) await wait_till_rabbitmq_responsive(rabbit_broker) + # NOTE: to show the connection name in the rabbitMQ UI see there [https://www.bountysource.com/issues/89342433-setting-custom-connection-name-via-client_properties-doesn-t-work-when-connecting-using-an-amqp-url] connection = await aio_pika.connect_robust( - rabbit_broker, + rabbit_broker + f"?name={__name__}_{id(app)}", client_properties={"connection_name": "webserver read connection"}, ) diff --git a/services/web/server/src/simcore_service_webserver/config/server-docker-dev.yaml b/services/web/server/src/simcore_service_webserver/config/server-docker-dev.yaml index 296a8845fa0..be2ef9acf1e 100644 --- a/services/web/server/src/simcore_service_webserver/config/server-docker-dev.yaml +++ b/services/web/server/src/simcore_service_webserver/config/server-docker-dev.yaml @@ -46,8 +46,8 @@ rabbit: user: ${RABBIT_USER} password: ${RABBIT_PASSWORD} channels: - progress: ${RABBIT_PROGRESS_CHANNEL} - log: ${RABBIT_LOG_CHANNEL} + progress: "comp.backend.channels.progress" + log: "comp.backend.channels.log" activity: enabled: True prometheus_host: ${WEBSERVER_PROMETHEUS_HOST} diff --git a/services/web/server/src/simcore_service_webserver/config/server-docker-prod.yaml b/services/web/server/src/simcore_service_webserver/config/server-docker-prod.yaml index a4ab13ab9a4..4e62a974ee6 100644 --- a/services/web/server/src/simcore_service_webserver/config/server-docker-prod.yaml +++ b/services/web/server/src/simcore_service_webserver/config/server-docker-prod.yaml @@ -47,8 +47,8 @@ rabbit: user: ${RABBIT_USER} password: ${RABBIT_PASSWORD} channels: - progress: ${RABBIT_PROGRESS_CHANNEL} - log: ${RABBIT_LOG_CHANNEL} + progress: "comp.backend.channels.progress" + log: "comp.backend.channels.log" login: enabled: True registration_invitation_required: ${WEBSERVER_LOGIN_REGISTRATION_INVITATION_REQUIRED} diff --git a/services/web/server/tests/integration/computation/test_computation.py b/services/web/server/tests/integration/computation/test_computation.py index aa612a94cf5..76cc0bbcc20 100644 --- a/services/web/server/tests/integration/computation/test_computation.py +++ b/services/web/server/tests/integration/computation/test_computation.py @@ -171,7 +171,7 @@ async def test_check_health(docker_stack, client): async def test_start_pipeline( client, postgres_session, - celery_service, + rabbit_service, sleeper_service, logged_user, user_project, diff --git a/services/web/server/tests/integration/computation/test_rabbit.py b/services/web/server/tests/integration/computation/test_rabbit.py index be2fdf92000..aa0718966e5 100644 --- a/services/web/server/tests/integration/computation/test_rabbit.py +++ b/services/web/server/tests/integration/computation/test_rabbit.py @@ -14,7 +14,7 @@ from servicelib.application import create_safe_application from servicelib.application_keys import APP_CONFIG_KEY -from simcore_sdk.config.rabbit import eval_broker +from simcore_sdk.config.rabbit import Config from simcore_service_webserver.computation import setup_computation from simcore_service_webserver.computation_config import CONFIG_SECTION_NAME from simcore_service_webserver.db import setup_db @@ -73,19 +73,19 @@ def client( @pytest.fixture -def rabbit_config(app_config): +def rabbit_config(app_config) -> Dict: rb_config = app_config[CONFIG_SECTION_NAME] yield rb_config @pytest.fixture -def rabbit_broker(rabbit_config): - rabbit_broker = eval_broker(rabbit_config) +def rabbit_broker(rabbit_config: Dict) -> str: + rabbit_broker = Config(**rabbit_config).broker_url yield rabbit_broker @pytest.fixture -async def pika_connection(loop, rabbit_broker): +async def pika_connection(loop, rabbit_broker: str): connection = await aio_pika.connect( rabbit_broker, ssl=True, connection_attempts=100 ) diff --git a/tests/swarm-deploy/Makefile b/tests/swarm-deploy/Makefile index c3f436f0fdd..365a27f5d6d 100644 --- a/tests/swarm-deploy/Makefile +++ b/tests/swarm-deploy/Makefile @@ -6,17 +6,15 @@ include ../../scripts/common.Makefile ROOT_DIR = $(abspath $(CURDIR)/../../) VENV_DIR ?= $(abspath $(ROOT_DIR)/.venv) -.PHONY: reqs -requirements.txt: requirements.in - # pip compiling $< - @$(VENV_DIR)/bin/pip-compile --output-file $@ $< +.PHONY: requirements +requirements: ## compiles pip requirements (.in -> .txt) + @$(MAKE) --directory requirements reqs -reqs: requirements.txt ## alias to compile requirements.txt .PHONY: install -install: $(VENV_DIR) requirements.txt ## installs dependencies +install: $(VENV_DIR) requirements ## installs dependencies # installing requirements - @$str: - s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - try: - # doesn't even have to be reachable - s.connect(('10.255.255.255', 1)) - IP = s.getsockname()[0] - except Exception: #pylint: disable=W0703 - IP = '127.0.0.1' - finally: - s.close() - return IP - - -@pytest.fixture(scope='session') -def osparc_simcore_root_dir() -> Path: - WILDCARD = "services/web/server" - - root_dir = Path(current_dir) - while not any(root_dir.glob(WILDCARD)) and root_dir != Path("/"): - root_dir = root_dir.parent - - msg = f"'{root_dir}' does not look like the git root directory of osparc-simcore" - assert root_dir.exists(), msg - assert any(root_dir.glob(WILDCARD)), msg - assert any(root_dir.glob(".git")), msg - - return root_dir - - -@pytest.fixture(scope='session') -def docker_client() -> DockerClient: - client = docker.from_env() - yield client - - -@pytest.fixture(scope='session') -def docker_swarm_node(docker_client: DockerClient) -> None: - # SAME node along ALL session - docker_client.swarm.init(advertise_addr=_get_ip()) - yield #-------------------- - assert docker_client.swarm.leave(force=True) - - -@pytest.fixture(scope='module') -def osparc_deploy( osparc_simcore_root_dir: Path, - docker_client: DockerClient, - docker_swarm_node) -> Dict: - - environ = dict(os.environ) - if "TRAVIS" not in environ and "GITHUB_ACTIONS" not in environ: - environ["DOCKER_REGISTRY"] = "local" - environ["DOCKER_IMAGE_TAG"] = "production" - - subprocess.run( - "make info up-version info-swarm", - shell=True, check=True, env=environ, - cwd=osparc_simcore_root_dir - ) - - with open( osparc_simcore_root_dir / ".stack-simcore-version.yml" ) as fh: - simcore_config = yaml.safe_load(fh) - - with open( osparc_simcore_root_dir / ".stack-ops.yml" ) as fh: - ops_config = yaml.safe_load(fh) - - stack_configs = { - 'simcore': simcore_config, - 'ops': ops_config - } - - yield stack_configs #------------------------------------------------- - - WAIT_BEFORE_RETRY_SECS = 1 - - subprocess.run( - "make down", - shell=True, check=True, env=environ, - cwd=osparc_simcore_root_dir - ) - - subprocess.run(f"docker network prune -f", shell=True, check=False) - - for stack in stack_configs.keys(): - while True: - online = docker_client.services.list(filters={"label":f"com.docker.stack.namespace={stack}"}) - if online: - print(f"Waiting until {len(online)} services stop: {[s.name for s in online]}") - time.sleep(WAIT_BEFORE_RETRY_SECS) - else: - break - - while True: - networks = docker_client.networks.list(filters={"label":f"com.docker.stack.namespace={stack}"}) - if networks: - print(f"Waiting until {len(networks)} networks stop: {[n.name for n in networks]}") - time.sleep(WAIT_BEFORE_RETRY_SECS) - else: - break - - # make down might delete these files - def _safe_unlink(file_path: Path): - # TODO: in py 3.8 it will simply be file_path.unlink(missing_ok=True) - try: - file_path.unlink() - except FileNotFoundError: - pass - - _safe_unlink(osparc_simcore_root_dir / ".stack-simcore-version.yml") - _safe_unlink(osparc_simcore_root_dir / ".stack-ops.yml") +current_dir = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent + +pytest_plugins = [ + "pytest_simcore.environs", + "pytest_simcore.docker_compose", + "pytest_simcore.docker_swarm", + "pytest_simcore.docker_registry", + "pytest_simcore.rabbit_service", + "pytest_simcore.postgres_service", + "pytest_simcore.minio_service", + # "pytest_simcore.simcore_storage_service", +] +log = logging.getLogger(__name__) + + +@pytest.fixture(scope="module") +def prepare_all_services( + simcore_docker_compose: Dict, ops_docker_compose: Dict, request +) -> Dict: + services = [] + for service in simcore_docker_compose["services"].keys(): + services.append(service) + setattr(request.module, "core_services", services) + core_services = getattr(request.module, "core_services", []) + + services = [] + for service in ops_docker_compose["services"].keys(): + services.append(service) + setattr(request.module, "ops_services", services) + ops_services = getattr(request.module, "ops_services", []) + + services = {"simcore": simcore_docker_compose, "ops": ops_docker_compose} + return services + + +@pytest.fixture(scope="module") +def make_up_prod( + prepare_all_services: Dict, + simcore_docker_compose: Dict, + ops_docker_compose: Dict, + docker_stack: Dict, +) -> Dict: + stack_configs = {"simcore": simcore_docker_compose, "ops": ops_docker_compose} + return stack_configs diff --git a/tests/swarm-deploy/requirements.txt b/tests/swarm-deploy/requirements.txt deleted file mode 100644 index 8369f9c0e01..00000000000 --- a/tests/swarm-deploy/requirements.txt +++ /dev/null @@ -1,40 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile --output-file=requirements.txt requirements.in -# -aiohttp==3.6.2 # via pytest-aiohttp -async-timeout==3.0.1 # via aiohttp -attrs==19.3.0 # via aiohttp, pytest -certifi==2019.11.28 # via requests -chardet==3.0.4 # via aiohttp, requests -coverage==4.5.1 -docker==4.2.0 -idna-ssl==1.1.0 # via aiohttp -idna==2.9 # via idna-ssl, requests, yarl -importlib-metadata==1.5.0 # via pluggy, pytest -more-itertools==8.2.0 # via pytest -multidict==4.7.5 # via aiohttp, yarl -packaging==20.3 # via pytest, pytest-sugar -pluggy==0.13.1 # via pytest -py==1.8.1 # via pytest -pyparsing==2.4.6 # via packaging -pytest-aiohttp==0.3.0 -pytest-cov==2.8.1 -pytest-instafail==0.4.1.post0 -pytest-mock==2.0.0 -pytest-runner==5.2 -pytest-sugar==0.9.2 -pytest==5.3.5 -pyyaml==5.3 -requests==2.23.0 # via docker -six==1.14.0 # via docker, packaging, tenacity, websocket-client -tenacity==6.1.0 -termcolor==1.1.0 # via pytest-sugar -typing-extensions==3.7.4.1 # via aiohttp -urllib3==1.25.8 # via requests -wcwidth==0.1.8 # via pytest -websocket-client==0.57.0 # via docker -yarl==1.4.2 # via aiohttp -zipp==3.1.0 # via importlib-metadata diff --git a/tests/swarm-deploy/requirements/Makefile b/tests/swarm-deploy/requirements/Makefile new file mode 100644 index 00000000000..e3d6b8e230d --- /dev/null +++ b/tests/swarm-deploy/requirements/Makefile @@ -0,0 +1,6 @@ +# +# Targets to pip-compile requirements +# +include ../../../scripts/requirements.Makefile + +# Add here any extra explicit dependency: e.g. _migration.txt: _base.txt diff --git a/tests/swarm-deploy/requirements/ci.txt b/tests/swarm-deploy/requirements/ci.txt new file mode 100644 index 00000000000..745bc8ad80b --- /dev/null +++ b/tests/swarm-deploy/requirements/ci.txt @@ -0,0 +1,18 @@ +# Shortcut to install all packages for the contigous integration (CI) of 'services/web/server' +# +# - As ci.txt but w/ tests +# +# Usage: +# pip install -r requirements/ci.txt +# + +# installs base + tests requirements +-r requirements.txt + +# installs this repo's packages +../../packages/s3wrapper/ +../../packages/postgres-database/ +../../packages/simcore-sdk/ +../../packages/service-library/ +../../packages/pytest-simcore/ + diff --git a/tests/swarm-deploy/requirements/dev.txt b/tests/swarm-deploy/requirements/dev.txt new file mode 100644 index 00000000000..9c26e59e2fb --- /dev/null +++ b/tests/swarm-deploy/requirements/dev.txt @@ -0,0 +1,21 @@ +# Shortcut to install all packages needed to develop 'services/web/server' +# +# - As ci.txt but with current and repo packages in develop (edit) mode +# +# Usage: +# pip install -r requirements/dev.txt +# + +# tools +bump2version +watchdog[watchmedo] + +# installs base + tests requirements +-r requirements.txt + +# installs this repo's packages +-e ../../packages/s3wrapper/ +-e ../../packages/postgres-database/ +-e ../../packages/simcore-sdk/ +-e ../../packages/service-library/ +-e ../../packages/pytest-simcore/ diff --git a/tests/swarm-deploy/requirements.in b/tests/swarm-deploy/requirements/requirements.in similarity index 93% rename from tests/swarm-deploy/requirements.in rename to tests/swarm-deploy/requirements/requirements.in index 4589b14789e..a4dfbb9c0dd 100644 --- a/tests/swarm-deploy/requirements.in +++ b/tests/swarm-deploy/requirements/requirements.in @@ -1,5 +1,5 @@ +aio-pika coverage==4.5.1 # TODO: Downgraded because of a bug https://github.com/nedbat/coveragepy/issues/716 - pytest~=5.3.5 # Bug in pytest-sugar https://github.com/Teemu/pytest-sugar/issues/187 pytest-aiohttp pytest-cov @@ -7,7 +7,6 @@ pytest-instafail pytest-mock pytest-runner pytest-sugar - docker tenacity pyyaml>=5.3 # Vulnerable diff --git a/tests/swarm-deploy/requirements/requirements.txt b/tests/swarm-deploy/requirements/requirements.txt new file mode 100644 index 00000000000..eb073f4c8ac --- /dev/null +++ b/tests/swarm-deploy/requirements/requirements.txt @@ -0,0 +1,43 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --build-isolation --output-file=requirements.txt requirements.in +# +aio-pika==6.6.0 # via -r requirements.in +aiohttp==3.6.2 # via pytest-aiohttp +aiormq==3.2.1 # via aio-pika +async-timeout==3.0.1 # via aiohttp +attrs==19.3.0 # via aiohttp, pytest +certifi==2019.11.28 # via requests +chardet==3.0.4 # via aiohttp, requests +coverage==4.5.1 # via -r requirements.in, pytest-cov +docker==4.2.0 # via -r requirements.in +idna-ssl==1.1.0 # via aiohttp +idna==2.9 # via requests, yarl +importlib-metadata==1.5.2 # via pluggy, pytest +more-itertools==8.2.0 # via pytest +multidict==4.7.5 # via aiohttp, yarl +packaging==20.3 # via pytest, pytest-sugar +pamqp==2.3.0 # via aiormq +pluggy==0.13.1 # via pytest +py==1.8.1 # via pytest +pyparsing==2.4.6 # via packaging +pytest-aiohttp==0.3.0 # via -r requirements.in +pytest-cov==2.8.1 # via -r requirements.in +pytest-instafail==0.4.1.post0 # via -r requirements.in +pytest-mock==2.0.0 # via -r requirements.in +pytest-runner==5.2 # via -r requirements.in +pytest-sugar==0.9.2 # via -r requirements.in +pytest==5.3.5 # via -r requirements.in, pytest-aiohttp, pytest-cov, pytest-instafail, pytest-mock, pytest-sugar +pyyaml==5.3.1 # via -r requirements.in +requests==2.23.0 # via docker +six==1.14.0 # via docker, packaging, tenacity, websocket-client +tenacity==6.1.0 # via -r requirements.in +termcolor==1.1.0 # via pytest-sugar +typing-extensions==3.7.4.1 # via aiohttp +urllib3==1.25.8 # via requests +wcwidth==0.1.9 # via pytest +websocket-client==0.57.0 # via docker +yarl==1.4.2 # via aio-pika, aiohttp, aiormq +zipp==3.1.0 # via importlib-metadata diff --git a/tests/swarm-deploy/test_service_images.py b/tests/swarm-deploy/test_service_images.py index dd5d97c62de..759f2ff2970 100644 --- a/tests/swarm-deploy/test_service_images.py +++ b/tests/swarm-deploy/test_service_images.py @@ -4,23 +4,37 @@ import subprocess from typing import Dict + import pytest +from yarl import URL # search ujson in all _base.txt and add here all services that contains it -@pytest.mark.parametrize("service", [ - 'director', - 'webserver', - 'storage', - 'catalog' -]) -def test_ujson_installation(service:str, osparc_deploy: Dict): +@pytest.mark.parametrize( + "service,type", + [ + ("director", "debian"), + ("webserver", "alpine"), + ("storage", "alpine"), + ("catalog", "alpine"), + ], +) +async def test_ujson_installation( + loop, service: str, type: str, simcore_docker_compose: Dict, +): # tets failing installation undetected # and fixed in PR https://github.com/ITISFoundation/osparc-simcore/pull/1353 - image_name = osparc_deploy['simcore']['services'][service]['image'] + image_name = simcore_docker_compose["services"][service]["image"] - assert subprocess.run( - f'docker run -t --rm {image_name} python -c "import ujson; print(ujson.__version__)"', - shell=True, - check=True, - ) + if type == "debian": + assert subprocess.run( + f"docker run -t --rm {image_name} \"python -c 'import ujson; print(ujson.__version__)'\"", + shell=True, + check=True, + ) + else: + assert subprocess.run( + f"docker run -t --rm {image_name} python -c 'import ujson; print(ujson.__version__)'", + shell=True, + check=True, + ) diff --git a/tests/swarm-deploy/test_service_restart.py b/tests/swarm-deploy/test_service_restart.py index 0b353e1d9a2..fe830fc718f 100644 --- a/tests/swarm-deploy/test_service_restart.py +++ b/tests/swarm-deploy/test_service_restart.py @@ -28,13 +28,13 @@ @pytest.fixture("module") -def deployed_simcore_stack(osparc_deploy: Dict, docker_client: DockerClient) -> List[Service]: +def deployed_simcore_stack(make_up_prod: Dict, docker_client: DockerClient) -> List[Service]: # NOTE: the goal here is NOT to test time-to-deplopy but # rather guaranteing that the framework is fully deployed before starting # tests. Obviously in a critical state in which the frameworks has a problem # the fixture will fail STACK_NAME = 'simcore' - assert STACK_NAME in osparc_deploy + assert STACK_NAME in make_up_prod @retry( wait=wait_fixed(MAX_TIME_TO_DEPLOY_SECS), stop=stop_after_attempt(5), @@ -70,7 +70,7 @@ def ensure_deployed(): def test_graceful_restart_services( service_name: str, deployed_simcore_stack: List[Service], - osparc_deploy: Dict): + make_up_prod: Dict): """ NOTE: loop fixture makes this test async NOTE: needs to run AFTER test_core_service_running diff --git a/tests/swarm-deploy/test_swarm_runs.py b/tests/swarm-deploy/test_swarm_runs.py index e99a7f14c10..921412b8d00 100644 --- a/tests/swarm-deploy/test_swarm_runs.py +++ b/tests/swarm-deploy/test_swarm_runs.py @@ -6,6 +6,7 @@ import logging import os import sys +import time import urllib from pathlib import Path from pprint import pformat @@ -17,39 +18,34 @@ logger = logging.getLogger(__name__) -current_dir = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent +current_dir = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent WAIT_TIME_SECS = 20 RETRY_COUNT = 7 -MAX_WAIT_TIME=240 +MAX_WAIT_TIME = 240 docker_compose_service_names = [ - 'catalog', - 'director', - 'sidecar', - 'storage', - 'webserver', - 'rabbit', - 'postgres', - 'redis' + "catalog", + "director", + "sidecar", + "storage", + "webserver", + "rabbit", + "postgres", + "redis", ] -stack_name = os.environ.get("SWARM_STACK_NAME", 'simcore') +stack_name = os.environ.get("SWARM_STACK_NAME", "simcore") -stack_service_names = sorted([ f"{stack_name}_{name}" - for name in docker_compose_service_names ]) +stack_service_names = sorted( + [f"{stack_name}_{name}" for name in docker_compose_service_names] +) # wait if running pre-state # https://docs.docker.com/engine/swarm/how-swarm-mode-works/swarm-task-states/ -pre_states = [ - "NEW", - "PENDING", - "ASSIGNED", - "PREPARING", - "STARTING" -] +pre_states = ["NEW", "PENDING", "ASSIGNED", "PREPARING", "STARTING"] failed_states = [ "COMPLETE", @@ -58,7 +54,7 @@ "REJECTED", "ORPHANED", "REMOVE", - "CREATED" + "CREATED", ] @@ -78,32 +74,33 @@ def core_services_running(docker_client: DockerClient) -> List[Service]: # for a stack named 'mystack' # maps service names in docker-compose with actual services - running_services = [ s for s in docker_client.services.list() - if s.name.startswith(stack_name) ] + running_services = [ + s for s in docker_client.services.list() if s.name.startswith(stack_name) + ] return running_services -def test_all_services_up(core_services_running: str, osparc_deploy:Dict): - running_services = sorted( [s.name for s in core_services_running] ) - assert running_services == stack_service_names +def test_all_services_up(core_services_running: str, make_up_prod: Dict): + running_services = sorted([s.name for s in core_services_running]) + assert running_services == stack_service_names - expected = [ f'{stack_name}_{service_name}' - for service_name in osparc_deploy[stack_name]['services'].keys() + expected = [ + f"{stack_name}_{service_name}" + for service_name in make_up_prod[stack_name]["services"].keys() ] assert running_services == sorted(expected) -async def test_core_service_running( +def test_core_service_running( core_service_name: str, core_services_running: List[Service], docker_client: DockerClient, - loop: asyncio.BaseEventLoop, - osparc_deploy: Dict ): - """ - NOTE: loop fixture makes this test async - """ + make_up_prod: Dict, +): # find core_service_name - running_service = next( s for s in core_services_running if s.name == core_service_name ) + running_service = next( + s for s in core_services_running if s.name == core_service_name + ) # Every service in the fixture runs a number of tasks, but they might have failed! # @@ -112,31 +109,41 @@ async def test_core_service_running( # puiaevvmtbs1 simcore_storage.1 simcore_storage:latest crespo-wkstn Running Running 18 minutes ago # j5xtlrnn684y \_ simcore_storage.1 simcore_storage:latest crespo-wkstn Shutdown Failed 18 minutes ago "task: non-zero exit (1)" tasks = running_service.tasks() - service_config = osparc_deploy['simcore']['services'][core_service_name.split(sep="_")[1]] + service_config = make_up_prod["simcore"]["services"][ + core_service_name.split(sep="_")[1] + ] num_tasks = get_replicas(service_config) - assert len(tasks) == num_tasks, f"Expected a {num_tasks} task(s) for '{0}',"\ - " got:\n{1}\n{2}".format(core_service_name, - get_tasks_summary(tasks), - get_failed_tasks_logs(running_service, docker_client)) - + assert len(tasks) == num_tasks, ( + f"Expected a {num_tasks} task(s) for '{0}'," + " got:\n{1}\n{2}".format( + core_service_name, + get_tasks_summary(tasks), + get_failed_tasks_logs(running_service, docker_client), + ) + ) for i in range(num_tasks): for n in range(RETRY_COUNT): task = running_service.tasks()[i] - if task['Status']['State'].upper() in pre_states: - print("Waiting [{}/{}] ...\n{}".format(n, RETRY_COUNT, get_tasks_summary(tasks))) - await asyncio.sleep(WAIT_TIME_SECS) + if task["Status"]["State"].upper() in pre_states: + print( + "Waiting [{}/{}] ...\n{}".format( + n, RETRY_COUNT, get_tasks_summary(tasks) + ) + ) + time.sleep(WAIT_TIME_SECS) else: break # should be running - assert task['Status']['State'].upper() == "RUNNING", \ - "Expected running, got \n{}\n{}".format( - pformat(task), - get_failed_tasks_logs(running_service, docker_client)) + assert ( + task["Status"]["State"].upper() == "RUNNING" + ), "Expected running, got \n{}\n{}".format( + pformat(task), get_failed_tasks_logs(running_service, docker_client) + ) -async def test_check_serve_root(osparc_deploy: Dict): +def test_check_serve_root(make_up_prod: Dict): req = urllib.request.Request("http://127.0.0.1:9081/") try: resp = urllib.request.urlopen(req) @@ -147,15 +154,16 @@ async def test_check_serve_root(osparc_deploy: Dict): if content.find(search) < 0: pytest.fail("{} not found in main index.html".format(search)) except urllib.error.HTTPError as err: - pytest.fail("The server could not fulfill the request.\nError code {}".format(err.code)) + pytest.fail( + "The server could not fulfill the request.\nError code {}".format(err.code) + ) except urllib.error.URLError as err: pytest.fail("Failed reaching the server..\nError reason {}".format(err.reason)) - - # UTILS -------------------------------- + def get_replicas(service: Dict) -> int: replicas = 1 if "deploy" in service: @@ -167,25 +175,28 @@ def get_replicas(service: Dict) -> int: def get_tasks_summary(tasks): msg = "" for t in tasks: - t["Status"].setdefault("Err", '') + t["Status"].setdefault("Err", "") msg += "- task ID:{ID}, STATE: {Status[State]}, ERROR: '{Status[Err]}' \n".format( - **t) + **t + ) return msg def get_failed_tasks_logs(service, docker_client): failed_logs = "" for t in service.tasks(): - if t['Status']['State'].upper() in failed_states: - cid = t['Status']['ContainerStatus']['ContainerID'] + if t["Status"]["State"].upper() in failed_states: + cid = t["Status"]["ContainerStatus"]["ContainerID"] failed_logs += "{2} {0} - {1} BEGIN {2}\n".format( - service.name, t['ID'], "="*10) + service.name, t["ID"], "=" * 10 + ) if cid: container = docker_client.containers.get(cid) - failed_logs += container.logs().decode('utf-8') + failed_logs += container.logs().decode("utf-8") else: failed_logs += " log unavailable. container does not exists\n" failed_logs += "{2} {0} - {1} END {2}\n".format( - service.name, t['ID'], "="*10) + service.name, t["ID"], "=" * 10 + ) return failed_logs