Skip to content

Commit 731807d

Browse files
pcrespovpcrespov
authored
API extension: solver's catalog (#2112)
Implementing case #1865 * Connects ``api-server`` with catalog service - can list, get released solver (i.e. cataloged computational services ) - this solution is temporary. Next iteration will push to the catalog service most of the logic * Changes in ``catalog`` service: - Fixes OAS: nullable in openapi shall be introduced as an ``anyOf`` alternative. * NEW ``tests/public-api`` system testing - deploys simcore stack, ops stack and registry with a sleeper - registers a user and create API tokens - run tests using osparc python API client (which is installed from repository) * All actions test on ubuntu 20 * Froze master commit in osparc
1 parent 3e051b5 commit 731807d

File tree

106 files changed

+1846
-493
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

106 files changed

+1846
-493
lines changed

‎.env-devel

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
# - Keep it alfphabetical order and grouped by prefix
33
# - To expose: export $(grep -v '^#' .env | xargs -0)
44
#
5+
API_SERVER_DEV_FEATURES_ENABLED=0
56

67
BF_API_KEY=none
78
BF_API_SECRET=none

‎.github/workflows/ci-testing-deploy.yml

Lines changed: 48 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1176,14 +1176,60 @@ jobs:
11761176
name: integration_simcoresdk_coverage
11771177
path: codeclimate.integration_simcoresdk_coverage.json
11781178

1179+
system-test-public-api:
1180+
name: "[sys] public api"
1181+
needs: [build-test-images]
1182+
runs-on: ${{ matrix.os }}
1183+
strategy:
1184+
matrix:
1185+
python: [3.6]
1186+
os: [ubuntu-20.04]
1187+
fail-fast: false
1188+
steps:
1189+
- name: set PR default variables
1190+
# only pushes have access to the docker credentials, use a default
1191+
if: github.event_name == 'pull_request'
1192+
run: |
1193+
export TMP_DOCKER_REGISTRY=${GITHUB_REPOSITORY%/*}
1194+
echo "DOCKER_REGISTRY=${TMP_DOCKER_REGISTRY,,}" >> $GITHUB_ENV
1195+
- uses: actions/checkout@v2
1196+
- name: setup docker
1197+
run: |
1198+
sudo ./ci/github/helpers/setup_docker_compose.bash
1199+
./ci/github/helpers/setup_docker_experimental.bash
1200+
./ci/github/helpers/setup_docker_buildx.bash
1201+
echo "DOCKER_BUILDX=1" >> $GITHUB_ENV
1202+
- name: setup python environment
1203+
uses: actions/setup-python@v2
1204+
with:
1205+
python-version: ${{ matrix.python }}
1206+
- name: show system version
1207+
run: ./ci/helpers/show_system_versions.bash
1208+
- uses: actions/cache@v2
1209+
name: getting cached data
1210+
with:
1211+
path: ~/.cache/pip
1212+
key: ${{ runner.os }}-pip-public-api-${{ hashFiles('tests/public-api/requirements/ci.txt') }}
1213+
restore-keys: |
1214+
${{ runner.os }}-pip-public-api-
1215+
${{ runner.os }}-pip-
1216+
${{ runner.os }}-
1217+
- name: install
1218+
run: ./ci/github/system-testing/public-api.bash install
1219+
- name: test
1220+
run: ./ci/github/system-testing/public-api.bash test
1221+
- name: cleanup
1222+
if: always()
1223+
run: ./ci/github/system-testing/public-api.bash clean_up
1224+
11791225
system-test-swarm-deploy:
11801226
name: "[sys] deploy simcore"
11811227
needs: [build-test-images]
11821228
runs-on: ${{ matrix.os }}
11831229
strategy:
11841230
matrix:
11851231
python: [3.6]
1186-
os: [ubuntu-16.04, ubuntu-18.04, ubuntu-20.04]
1232+
os: [ubuntu-20.04]
11871233
fail-fast: false
11881234
steps:
11891235
- name: set PR default variables
@@ -1477,6 +1523,7 @@ jobs:
14771523
integration-test-director-v2,
14781524
integration-test-sidecar,
14791525
integration-test-simcore-sdk,
1526+
system-test-public-api,
14801527
system-test-swarm-deploy,
14811528
]
14821529
runs-on: ubuntu-latest

‎Makefile

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@
1111

1212
SHELL := /bin/bash
1313

14-
1514
MAKE_C := $(MAKE) --no-print-directory --directory
1615

1716
# Operating system
@@ -71,8 +70,10 @@ export ETC_HOSTNAME
7170
host := $(shell echo $$(hostname) > $(ETC_HOSTNAME))
7271
endif
7372

73+
get_my_ip := $(shell hostname --all-ip-addresses | cut --delimiter=" " --fields=1)
74+
7475
# NOTE: this is only for WSL2 as the WSL2 subsystem IP is changing on each reboot
75-
S3_ENDPOINT = $(shell hostname --all-ip-addresses | cut --delimiter=" " --fields=1):9001
76+
S3_ENDPOINT := $(get_my_ip):9001
7677
export S3_ENDPOINT
7778

7879

@@ -433,7 +434,6 @@ postgres-upgrade: ## initalize or upgrade postgres db to latest state
433434

434435

435436
local_registry=registry
436-
get_my_ip := $(shell hostname --all-ip-addresses | cut --delimiter=" " --fields=1)
437437
.PHONY: local-registry rm-registry
438438

439439
rm-registry: ## remove the registry and changes to host/file

‎ci/github/integration-testing/director-v2.bash

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
#!/bin/bash
22
# http://redsymbol.net/articles/unofficial-bash-strict-mode/
3-
set -euo pipefail
3+
set -o errexit # abort on nonzero exitstatus
4+
set -o nounset # abort on unbound variable
5+
set -o pipefail # don't hide errors within pipes
46
IFS=$'\n\t'
57

68
# in case it's a Pull request, the env are never available, default to itisfoundation to get a maybe not too old version for caching

‎ci/github/integration-testing/sidecar.bash

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
#!/bin/bash
22
# http://redsymbol.net/articles/unofficial-bash-strict-mode/
3-
set -euo pipefail
3+
set -o errexit # abort on nonzero exitstatus
4+
set -o nounset # abort on unbound variable
5+
set -o pipefail # don't hide errors within pipes
46
IFS=$'\n\t'
57

68
# in case it's a Pull request, the env are never available, default to itisfoundation to get a maybe not too old version for caching

‎ci/github/integration-testing/simcore-sdk.bash

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
#!/bin/bash
2-
set -euo pipefail
2+
set -o errexit # abort on nonzero exitstatus
3+
set -o nounset # abort on unbound variable
4+
set -o pipefail # don't hide errors within pipes
35
IFS=$'\n\t'
46

57
# in case it's a Pull request, the env are never available, default to itisfoundation to get a maybe not too old version for caching

‎ci/github/integration-testing/webserver.bash

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
#!/bin/bash
22
# http://redsymbol.net/articles/unofficial-bash-strict-mode/
3-
set -euo pipefail
3+
set -o errexit # abort on nonzero exitstatus
4+
set -o nounset # abort on unbound variable
5+
set -o pipefail # don't hide errors within pipes
46
IFS=$'\n\t'
57

68
# in case it's a Pull request, the env are never available, default to itisfoundation to get a maybe not too old version for caching

‎ci/github/system-testing/e2e.bash

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
#!/bin/bash
22
# http://redsymbol.net/articles/unofficial-bash-strict-mode/
33
# https://github.com/GoogleChrome/puppeteer/blob/master/docs/troubleshooting.md#running-puppeteer-on-travis-ci
4-
set -euo pipefail
4+
set -o errexit # abort on nonzero exitstatus
5+
set -o nounset # abort on unbound variable
6+
set -o pipefail # don't hide errors within pipes
57
IFS=$'\n\t'
68

79
# in case it's a Pull request, the env are never available, default to itisfoundation to get a maybe not too old version for caching

‎ci/github/system-testing/environment-setup.bash

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,9 @@
22
#
33

44
# http://redsymbol.net/articles/unofficial-bash-strict-mode/
5-
set -euo pipefail
5+
set -o errexit # abort on nonzero exitstatus
6+
set -o nounset # abort on unbound variable
7+
set -o pipefail # don't hide errors within pipes
68
IFS=$'\n\t'
79

810
install() {

‎ci/github/system-testing/public-api.bash

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
#!/bin/bash
2+
#
3+
# This task in the system-testing aims to test some guarantees expected from
4+
# the deployment of osparc-simcore in a cluster (swarm).
5+
# It follows some of the points enumerated in the https://12factor.net/ methodology.
6+
#
7+
8+
# http://redsymbol.net/articles/unofficial-bash-strict-mode/
9+
set -o errexit # abort on nonzero exitstatus
10+
set -o nounset # abort on unbound variable
11+
set -o pipefail # don't hide errors within pipes
12+
IFS=$'\n\t'
13+
14+
# in case it's a Pull request, the env are never available, default to itisfoundation to get a maybe not too old version for caching
15+
DOCKER_IMAGE_TAG=$(exec ci/helpers/build_docker_image_tag.bash)
16+
export DOCKER_IMAGE_TAG
17+
18+
install() {
19+
bash ci/helpers/ensure_python_pip.bash
20+
pushd tests/public-api
21+
pip3 install -r requirements/ci.txt
22+
popd
23+
make pull-version || ( (make pull-cache || true) && make build-x tag-version)
24+
make .env
25+
pip list -v
26+
make info-images
27+
}
28+
29+
test() {
30+
pytest --color=yes --cov-report=term-missing -v tests/public-api --log-level=DEBUG
31+
}
32+
33+
clean_up() {
34+
docker images
35+
make down
36+
make leave
37+
}
38+
39+
# Check if the function exists (bash specific)
40+
if declare -f "$1" >/dev/null; then
41+
# call arguments verbatim
42+
"$@"
43+
else
44+
# Show a helpful error
45+
echo "'$1' is not a known function name" >&2
46+
exit 1
47+
fi

‎ci/github/system-testing/swarm-deploy.bash

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,9 @@
66
#
77

88
# http://redsymbol.net/articles/unofficial-bash-strict-mode/
9-
set -euo pipefail
9+
set -o errexit # abort on nonzero exitstatus
10+
set -o nounset # abort on unbound variable
11+
set -o pipefail # don't hide errors within pipes
1012
IFS=$'\n\t'
1113

1214
# in case it's a Pull request, the env are never available, default to itisfoundation to get a maybe not too old version for caching

‎ci/github/unit-testing/api-server.bash

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
#!/bin/bash
22
# http://redsymbol.net/articles/unofficial-bash-strict-mode/
3-
set -euo pipefail
3+
set -o errexit # abort on nonzero exitstatus
4+
set -o nounset # abort on unbound variable
5+
set -o pipefail # don't hide errors within pipes
46
IFS=$'\n\t'
57

68
install() {

‎ci/github/unit-testing/api.bash

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
#!/bin/bash
22
# http://redsymbol.net/articles/unofficial-bash-strict-mode/
3-
set -euo pipefail
3+
set -o errexit # abort on nonzero exitstatus
4+
set -o nounset # abort on unbound variable
5+
set -o pipefail # don't hide errors within pipes
46
IFS=$'\n\t'
57

68
install() {

‎ci/github/unit-testing/catalog.bash

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
#!/bin/bash
22
# http://redsymbol.net/articles/unofficial-bash-strict-mode/
3-
set -euo pipefail
3+
set -o errexit # abort on nonzero exitstatus
4+
set -o nounset # abort on unbound variable
5+
set -o pipefail # don't hide errors within pipes
46
IFS=$'\n\t'
57

68
install() {

‎ci/github/unit-testing/director.bash

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
#!/bin/bash
22
# http://redsymbol.net/articles/unofficial-bash-strict-mode/
3-
set -euo pipefail
3+
set -o errexit # abort on nonzero exitstatus
4+
set -o nounset # abort on unbound variable
5+
set -o pipefail # don't hide errors within pipes
46
IFS=$'\n\t'
57

68
install() {

‎ci/github/unit-testing/director_v2.bash

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
#!/bin/bash
22
# http://redsymbol.net/articles/unofficial-bash-strict-mode/
3-
set -euo pipefail
3+
set -o errexit # abort on nonzero exitstatus
4+
set -o nounset # abort on unbound variable
5+
set -o pipefail # don't hide errors within pipes
46
IFS=$'\n\t'
57

68
install() {

‎ci/github/unit-testing/frontend.bash

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
#!/bin/bash
22
# http://redsymbol.net/articles/unofficial-bash-strict-mode/
3-
set -euo pipefail
3+
set -o errexit # abort on nonzero exitstatus
4+
set -o nounset # abort on unbound variable
5+
set -o pipefail # don't hide errors within pipes
46
IFS=$'\n\t'
57

68
install() {
@@ -39,7 +41,7 @@ test() {
3941
popd
4042

4143
#TODO: no idea what is this doing... disabled at the moment since travis is supposed to do it as well
42-
44+
4345
# # prepare documentation site ...
4446
# git clone --depth 1 https://github.com/ITISFoundation/itisfoundation.github.io.git
4547
# rm -rf itisfoundation.github.io/.git

‎ci/github/unit-testing/models-library.bash

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
#!/bin/bash
22
# http://redsymbol.net/articles/unofficial-bash-strict-mode/
3-
set -euo pipefail
3+
set -o errexit # abort on nonzero exitstatus
4+
set -o nounset # abort on unbound variable
5+
set -o pipefail # don't hide errors within pipes
46
IFS=$'\n\t'
57

68
install() {

‎ci/github/unit-testing/postgres-database.bash

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
#!/bin/bash
22
# http://redsymbol.net/articles/unofficial-bash-strict-mode/
3-
set -euo pipefail
3+
set -o errexit # abort on nonzero exitstatus
4+
set -o nounset # abort on unbound variable
5+
set -o pipefail # don't hide errors within pipes
46
IFS=$'\n\t'
57

68
install() {

‎ci/github/unit-testing/python-linting.bash

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
#!/bin/bash
22
# http://redsymbol.net/articles/unofficial-bash-strict-mode/
3-
set -euo pipefail
3+
set -o errexit # abort on nonzero exitstatus
4+
set -o nounset # abort on unbound variable
5+
set -o pipefail # don't hide errors within pipes
46
IFS=$'\n\t'
57

68
install() {

‎ci/github/unit-testing/service-integration.bash

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
#!/bin/bash
22
# http://redsymbol.net/articles/unofficial-bash-strict-mode/
3-
set -euo pipefail
3+
set -o errexit # abort on nonzero exitstatus
4+
set -o nounset # abort on unbound variable
5+
set -o pipefail # don't hide errors within pipes
46
IFS=$'\n\t'
57

68
install() {

‎ci/github/unit-testing/service-library.bash

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
#!/bin/bash
22
# http://redsymbol.net/articles/unofficial-bash-strict-mode/
3-
set -euo pipefail
3+
set -o errexit # abort on nonzero exitstatus
4+
set -o nounset # abort on unbound variable
5+
set -o pipefail # don't hide errors within pipes
46
IFS=$'\n\t'
57

68
install() {

‎ci/github/unit-testing/sidecar.bash

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
#!/bin/bash
22
# http://redsymbol.net/articles/unofficial-bash-strict-mode/
3-
set -euo pipefail
3+
set -o errexit # abort on nonzero exitstatus
4+
set -o nounset # abort on unbound variable
5+
set -o pipefail # don't hide errors within pipes
46
IFS=$'\n\t'
57

68
install() {

‎ci/github/unit-testing/simcore-sdk.bash

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
#!/bin/bash
2-
set -euo pipefail
2+
set -o errexit # abort on nonzero exitstatus
3+
set -o nounset # abort on unbound variable
4+
set -o pipefail # don't hide errors within pipes
35
IFS=$'\n\t'
46

57
# in case it's a Pull request, the env are never available, default to itisfoundation to get a maybe not too old version for caching

‎ci/github/unit-testing/storage.bash

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
#!/bin/bash
22
# http://redsymbol.net/articles/unofficial-bash-strict-mode/
3-
set -euo pipefail
3+
set -o errexit # abort on nonzero exitstatus
4+
set -o nounset # abort on unbound variable
5+
set -o pipefail # don't hide errors within pipes
46
IFS=$'\n\t'
57

68
install() {

‎ci/github/unit-testing/webserver.bash

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
#!/bin/bash
22
# http://redsymbol.net/articles/unofficial-bash-strict-mode/
3-
set -euo pipefail
3+
set -o errexit # abort on nonzero exitstatus
4+
set -o nounset # abort on unbound variable
5+
set -o pipefail # don't hide errors within pipes
46
IFS=$'\n\t'
57

68
install() {

‎ci/helpers/build_docker_image_tag.bash

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,9 @@
88
# always adds -testbuild-latest to the image tag to differentiate from the real master/staging builds
99

1010
# http://redsymbol.net/articles/unofficial-bash-strict-mode/
11-
set -euo pipefail
11+
set -o errexit # abort on nonzero exitstatus
12+
set -o nounset # abort on unbound variable
13+
set -o pipefail # don't hide errors within pipes
1214
IFS=$'\n\t'
1315

1416
default_image_tag="github"

0 commit comments

Comments
 (0)