Is1865/catalog solvers api

.env-devel

@@ -2,6 +2,7 @@
 # - Keep it alfphabetical order and grouped by prefix
 # - To expose:  export $(grep -v '^#' .env | xargs -0)
 #
+API_SERVER_DEV_FEATURES_ENABLED=0
 
 BF_API_KEY=none
 BF_API_SECRET=none

.github/workflows/ci-testing-deploy.yml

@@ -1176,14 +1176,60 @@ jobs:
           name: integration_simcoresdk_coverage
           path: codeclimate.integration_simcoresdk_coverage.json
 
+  system-test-public-api:
+    name: "[sys] public api"
+    needs: [build-test-images]
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python: [3.6]
+        os: [ubuntu-20.04]
+      fail-fast: false
+    steps:
+      - name: set PR default variables
+        # only pushes have access to the docker credentials, use a default
+        if: github.event_name == 'pull_request'
+        run: |
+          export TMP_DOCKER_REGISTRY=${GITHUB_REPOSITORY%/*}
+          echo "DOCKER_REGISTRY=${TMP_DOCKER_REGISTRY,,}" >> $GITHUB_ENV
+      - uses: actions/checkout@v2
+      - name: setup docker
+        run: |
+          sudo ./ci/github/helpers/setup_docker_compose.bash
+          ./ci/github/helpers/setup_docker_experimental.bash
+          ./ci/github/helpers/setup_docker_buildx.bash
+          echo "DOCKER_BUILDX=1" >> $GITHUB_ENV
+      - name: setup python environment
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python }}
+      - name: show system version
+        run: ./ci/helpers/show_system_versions.bash
+      - uses: actions/cache@v2
+        name: getting cached data
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-public-api-${{ hashFiles('tests/public-api/requirements/ci.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-public-api-
+            ${{ runner.os }}-pip-
+            ${{ runner.os }}-
+      - name: install
+        run: ./ci/github/system-testing/public-api.bash install
+      - name: test
+        run: ./ci/github/system-testing/public-api.bash test
+      - name: cleanup
+        if: always()
+        run: ./ci/github/system-testing/public-api.bash clean_up
+
   system-test-swarm-deploy:
     name: "[sys] deploy simcore"
     needs: [build-test-images]
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
         python: [3.6]
-        os: [ubuntu-16.04, ubuntu-18.04, ubuntu-20.04]
+        os: [ubuntu-20.04]
       fail-fast: false
     steps:
       - name: set PR default variables
@@ -1477,6 +1523,7 @@ jobs:
         integration-test-director-v2,
         integration-test-sidecar,
         integration-test-simcore-sdk,
+        system-test-public-api,
         system-test-swarm-deploy,
       ]
     runs-on: ubuntu-latest

Makefile

@@ -11,7 +11,6 @@
 
 SHELL := /bin/bash
 
-
 MAKE_C := $(MAKE) --no-print-directory --directory
 
 # Operating system
@@ -71,8 +70,10 @@ export ETC_HOSTNAME
 host := $(shell echo $$(hostname) > $(ETC_HOSTNAME))
 endif
 
+get_my_ip := $(shell hostname --all-ip-addresses | cut --delimiter=" " --fields=1)
+
 # NOTE: this is only for WSL2 as the WSL2 subsystem IP is changing on each reboot
-S3_ENDPOINT = $(shell hostname --all-ip-addresses | cut --delimiter=" " --fields=1):9001
+S3_ENDPOINT := $(get_my_ip):9001
 export S3_ENDPOINT
 
 
@@ -433,7 +434,6 @@ postgres-upgrade: ## initalize or upgrade postgres db to latest state
 
 
 local_registry=registry
-get_my_ip := $(shell hostname --all-ip-addresses | cut --delimiter=" " --fields=1)
 .PHONY: local-registry rm-registry
 
 rm-registry: ## remove the registry and changes to host/file

ci/github/integration-testing/director-v2.bash

@@ -1,6 +1,8 @@
 #!/bin/bash
 # http://redsymbol.net/articles/unofficial-bash-strict-mode/
-set -euo pipefail
+set -o errexit   # abort on nonzero exitstatus
+set -o nounset   # abort on unbound variable
+set -o pipefail  # don't hide errors within pipes
 IFS=$'\n\t'
 
 # in case it's a Pull request, the env are never available, default to itisfoundation to get a maybe not too old version for caching

ci/github/integration-testing/sidecar.bash

@@ -1,6 +1,8 @@
 #!/bin/bash
 # http://redsymbol.net/articles/unofficial-bash-strict-mode/
-set -euo pipefail
+set -o errexit   # abort on nonzero exitstatus
+set -o nounset   # abort on unbound variable
+set -o pipefail  # don't hide errors within pipes
 IFS=$'\n\t'
 
 # in case it's a Pull request, the env are never available, default to itisfoundation to get a maybe not too old version for caching

ci/github/integration-testing/simcore-sdk.bash

@@ -1,5 +1,7 @@
 #!/bin/bash
-set -euo pipefail
+set -o errexit   # abort on nonzero exitstatus
+set -o nounset   # abort on unbound variable
+set -o pipefail  # don't hide errors within pipes
 IFS=$'\n\t'
 
 # in case it's a Pull request, the env are never available, default to itisfoundation to get a maybe not too old version for caching

ci/github/integration-testing/webserver.bash

@@ -1,6 +1,8 @@
 #!/bin/bash
 # http://redsymbol.net/articles/unofficial-bash-strict-mode/
-set -euo pipefail
+set -o errexit   # abort on nonzero exitstatus
+set -o nounset   # abort on unbound variable
+set -o pipefail  # don't hide errors within pipes
 IFS=$'\n\t'
 
 # in case it's a Pull request, the env are never available, default to itisfoundation to get a maybe not too old version for caching

ci/github/system-testing/e2e.bash

@@ -1,7 +1,9 @@
 #!/bin/bash
 # http://redsymbol.net/articles/unofficial-bash-strict-mode/
 # https://github.com/GoogleChrome/puppeteer/blob/master/docs/troubleshooting.md#running-puppeteer-on-travis-ci
-set -euo pipefail
+set -o errexit   # abort on nonzero exitstatus
+set -o nounset   # abort on unbound variable
+set -o pipefail  # don't hide errors within pipes
 IFS=$'\n\t'
 
 # in case it's a Pull request, the env are never available, default to itisfoundation to get a maybe not too old version for caching

ci/github/system-testing/environment-setup.bash

@@ -2,7 +2,9 @@
 #
 
 # http://redsymbol.net/articles/unofficial-bash-strict-mode/
-set -euo pipefail
+set -o errexit   # abort on nonzero exitstatus
+set -o nounset   # abort on unbound variable
+set -o pipefail  # don't hide errors within pipes
 IFS=$'\n\t'
 
 install() {

ci/github/system-testing/public-api.bash

@@ -0,0 +1,47 @@
+#!/bin/bash
+#
+#  This task in the system-testing aims to test some guarantees expected from
+#  the deployment of osparc-simcore in a cluster (swarm).
+#  It follows some of the points enumerated in the https://12factor.net/  methodology.
+#
+
+# http://redsymbol.net/articles/unofficial-bash-strict-mode/
+set -o errexit   # abort on nonzero exitstatus
+set -o nounset   # abort on unbound variable
+set -o pipefail  # don't hide errors within pipes
+IFS=$'\n\t'
+
+# in case it's a Pull request, the env are never available, default to itisfoundation to get a maybe not too old version for caching
+DOCKER_IMAGE_TAG=$(exec ci/helpers/build_docker_image_tag.bash)
+export DOCKER_IMAGE_TAG
+
+install() {
+  bash ci/helpers/ensure_python_pip.bash
+  pushd tests/public-api
+  pip3 install -r requirements/ci.txt
+  popd
+  make pull-version || ( (make pull-cache || true) && make build-x tag-version)
+  make .env
+  pip list -v
+  make info-images
+}
+
+test() {
+  pytest --color=yes --cov-report=term-missing -v tests/public-api --log-level=DEBUG
+}
+
+clean_up() {
+  docker images
+  make down
+  make leave
+}
+
+# Check if the function exists (bash specific)
+if declare -f "$1" >/dev/null; then
+  # call arguments verbatim
+  "$@"
+else
+  # Show a helpful error
+  echo "'$1' is not a known function name" >&2
+  exit 1
+fi

ci/github/system-testing/swarm-deploy.bash

@@ -6,7 +6,9 @@
 #
 
 # http://redsymbol.net/articles/unofficial-bash-strict-mode/
-set -euo pipefail
+set -o errexit   # abort on nonzero exitstatus
+set -o nounset   # abort on unbound variable
+set -o pipefail  # don't hide errors within pipes
 IFS=$'\n\t'
 
 # in case it's a Pull request, the env are never available, default to itisfoundation to get a maybe not too old version for caching

ci/github/unit-testing/api-server.bash

@@ -1,6 +1,8 @@
 #!/bin/bash
 # http://redsymbol.net/articles/unofficial-bash-strict-mode/
-set -euo pipefail
+set -o errexit   # abort on nonzero exitstatus
+set -o nounset   # abort on unbound variable
+set -o pipefail  # don't hide errors within pipes
 IFS=$'\n\t'
 
 install() {

ci/github/unit-testing/api.bash

@@ -1,6 +1,8 @@
 #!/bin/bash
 # http://redsymbol.net/articles/unofficial-bash-strict-mode/
-set -euo pipefail
+set -o errexit   # abort on nonzero exitstatus
+set -o nounset   # abort on unbound variable
+set -o pipefail  # don't hide errors within pipes
 IFS=$'\n\t'
 
 install() {

ci/github/unit-testing/catalog.bash

@@ -1,6 +1,8 @@
 #!/bin/bash
 # http://redsymbol.net/articles/unofficial-bash-strict-mode/
-set -euo pipefail
+set -o errexit   # abort on nonzero exitstatus
+set -o nounset   # abort on unbound variable
+set -o pipefail  # don't hide errors within pipes
 IFS=$'\n\t'
 
 install() {

ci/github/unit-testing/director.bash

@@ -1,6 +1,8 @@
 #!/bin/bash
 # http://redsymbol.net/articles/unofficial-bash-strict-mode/
-set -euo pipefail
+set -o errexit   # abort on nonzero exitstatus
+set -o nounset   # abort on unbound variable
+set -o pipefail  # don't hide errors within pipes
 IFS=$'\n\t'
 
 install() {

ci/github/unit-testing/director_v2.bash

@@ -1,6 +1,8 @@
 #!/bin/bash
 # http://redsymbol.net/articles/unofficial-bash-strict-mode/
-set -euo pipefail
+set -o errexit   # abort on nonzero exitstatus
+set -o nounset   # abort on unbound variable
+set -o pipefail  # don't hide errors within pipes
 IFS=$'\n\t'
 
 install() {

ci/github/unit-testing/frontend.bash

@@ -1,6 +1,8 @@
 #!/bin/bash
 # http://redsymbol.net/articles/unofficial-bash-strict-mode/
-set -euo pipefail
+set -o errexit   # abort on nonzero exitstatus
+set -o nounset   # abort on unbound variable
+set -o pipefail  # don't hide errors within pipes
 IFS=$'\n\t'
 
 install() {
@@ -39,7 +41,7 @@ test() {
     popd
 
     #TODO: no idea what is this doing... disabled at the moment since travis is supposed to do it as well
-    
+
     # # prepare documentation site ...
     # git clone --depth 1 https://github.com/ITISFoundation/itisfoundation.github.io.git
     # rm -rf itisfoundation.github.io/.git

ci/github/unit-testing/models-library.bash

@@ -1,6 +1,8 @@
 #!/bin/bash
 # http://redsymbol.net/articles/unofficial-bash-strict-mode/
-set -euo pipefail
+set -o errexit   # abort on nonzero exitstatus
+set -o nounset   # abort on unbound variable
+set -o pipefail  # don't hide errors within pipes
 IFS=$'\n\t'
 
 install() {

ci/github/unit-testing/postgres-database.bash

@@ -1,6 +1,8 @@
 #!/bin/bash
 # http://redsymbol.net/articles/unofficial-bash-strict-mode/
-set -euo pipefail
+set -o errexit   # abort on nonzero exitstatus
+set -o nounset   # abort on unbound variable
+set -o pipefail  # don't hide errors within pipes
 IFS=$'\n\t'
 
 install() {

ci/github/unit-testing/python-linting.bash

@@ -1,6 +1,8 @@
 #!/bin/bash
 # http://redsymbol.net/articles/unofficial-bash-strict-mode/
-set -euo pipefail
+set -o errexit   # abort on nonzero exitstatus
+set -o nounset   # abort on unbound variable
+set -o pipefail  # don't hide errors within pipes
 IFS=$'\n\t'
 
 install() {

ci/github/unit-testing/service-integration.bash

@@ -1,6 +1,8 @@
 #!/bin/bash
 # http://redsymbol.net/articles/unofficial-bash-strict-mode/
-set -euo pipefail
+set -o errexit   # abort on nonzero exitstatus
+set -o nounset   # abort on unbound variable
+set -o pipefail  # don't hide errors within pipes
 IFS=$'\n\t'
 
 install() {

ci/github/unit-testing/service-library.bash

@@ -1,6 +1,8 @@
 #!/bin/bash
 # http://redsymbol.net/articles/unofficial-bash-strict-mode/
-set -euo pipefail
+set -o errexit   # abort on nonzero exitstatus
+set -o nounset   # abort on unbound variable
+set -o pipefail  # don't hide errors within pipes
 IFS=$'\n\t'
 
 install() {

ci/github/unit-testing/sidecar.bash

@@ -1,6 +1,8 @@
 #!/bin/bash
 # http://redsymbol.net/articles/unofficial-bash-strict-mode/
-set -euo pipefail
+set -o errexit   # abort on nonzero exitstatus
+set -o nounset   # abort on unbound variable
+set -o pipefail  # don't hide errors within pipes
 IFS=$'\n\t'
 
 install() {

ci/github/unit-testing/simcore-sdk.bash

@@ -1,5 +1,7 @@
 #!/bin/bash
-set -euo pipefail
+set -o errexit   # abort on nonzero exitstatus
+set -o nounset   # abort on unbound variable
+set -o pipefail  # don't hide errors within pipes
 IFS=$'\n\t'
 
 # in case it's a Pull request, the env are never available, default to itisfoundation to get a maybe not too old version for caching

ci/github/unit-testing/storage.bash

@@ -1,6 +1,8 @@
 #!/bin/bash
 # http://redsymbol.net/articles/unofficial-bash-strict-mode/
-set -euo pipefail
+set -o errexit   # abort on nonzero exitstatus
+set -o nounset   # abort on unbound variable
+set -o pipefail  # don't hide errors within pipes
 IFS=$'\n\t'
 
 install() {

ci/github/unit-testing/webserver.bash

@@ -1,6 +1,8 @@
 #!/bin/bash
 # http://redsymbol.net/articles/unofficial-bash-strict-mode/
-set -euo pipefail
+set -o errexit   # abort on nonzero exitstatus
+set -o nounset   # abort on unbound variable
+set -o pipefail  # don't hide errors within pipes
 IFS=$'\n\t'
 
 install() {

ci/helpers/build_docker_image_tag.bash

@@ -8,7 +8,9 @@
 #   always adds -testbuild-latest to the image tag to differentiate from the real master/staging builds
 
 # http://redsymbol.net/articles/unofficial-bash-strict-mode/
-set -euo pipefail
+set -o errexit   # abort on nonzero exitstatus
+set -o nounset   # abort on unbound variable
+set -o pipefail  # don't hide errors within pipes
 IFS=$'\n\t'
 
 default_image_tag="github"