From 2c89c5a3073cf17934f7c15f9e61071eaa9e84a2 Mon Sep 17 00:00:00 2001 From: Pedro Crespo Date: Mon, 9 Mar 2020 11:54:45 +0100 Subject: [PATCH 1/8] Fixes ujson missing dependency --- services/director/Dockerfile | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/services/director/Dockerfile b/services/director/Dockerfile index f6a13b59b3b..9887c8ff969 100644 --- a/services/director/Dockerfile +++ b/services/director/Dockerfile @@ -51,12 +51,16 @@ ENV SC_BUILD_TARGET build # RUN apk update && \ apk add --no-cache \ - libpq + libpq \ + libstdc++ + +# libstdc++: needed in ujson https://github.com/kohlschutter/junixsocket/issues/33 + # Installing build dependencies (will be deleted in production) RUN apk add --virtual .build-deps \ git \ - gcc \ + g++ \ libc-dev \ python-dev \ musl-dev \ @@ -150,9 +154,6 @@ FROM build as development ENV SC_BUILD_TARGET development -# WORKDIR /build -# NOTE: can copy from /build if necessary - WORKDIR /devel VOLUME /devel/packages VOLUME /devel/services/director/ From a4742b90a6a8406ad6e1b1165ce60752a6d1edee Mon Sep 17 00:00:00 2001 From: Pedro Crespo Date: Mon, 9 Mar 2020 11:55:41 +0100 Subject: [PATCH 2/8] Adds shell access for manual testing --- Makefile | 5 +++++ services/director/Makefile | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/Makefile b/Makefile index 383c32d848d..ba32da65096 100644 --- a/Makefile +++ b/Makefile @@ -126,6 +126,11 @@ $(CLIENT_WEB_OUTPUT): -mkdir $(if $(IS_WIN),,-p) $(CLIENT_WEB_OUTPUT) +.PHONY: shell +shell: + docker run -it local/$(target):production /bin/sh + + ## docker SWARM ------------------------------- # # - All resolved configuration are named as .stack-${name}-*.yml to distinguish from docker-compose files which can be parametrized diff --git a/services/director/Makefile b/services/director/Makefile index c4b164283eb..722f41e5018 100644 --- a/services/director/Makefile +++ b/services/director/Makefile @@ -34,6 +34,10 @@ build: openapi-specs ## builds docker image (using main services/docker-compose- @$(MAKE) -C ${REPO_BASE_DIR} target=${APP_NAME} $@ +.PHONY: shell +shell: ## runs shell in production container + @$(MAKE) --directory ${REPO_BASE_DIR} $@ target=${APP_NAME} + .PHONY: version-patch version-minor version-patch version-minor: ## commits version as patch (bug fixes not affecting the API), minor/minor (backwards-compatible/INcompatible API addition or changes) # upgrades as $(subst version-,,$@) version, commits and tags From 85dd0de4471994ba7d7fe848017f9e1d1be26f22 Mon Sep 17 00:00:00 2001 From: Pedro Crespo Date: Mon, 9 Mar 2020 12:30:56 +0100 Subject: [PATCH 3/8] WIP: Creating tests for ujson --- services/director/Dockerfile | 5 +++-- services/director/tests/test_ujson.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 services/director/tests/test_ujson.py diff --git a/services/director/Dockerfile b/services/director/Dockerfile index 9887c8ff969..5a1991bc860 100644 --- a/services/director/Dockerfile +++ b/services/director/Dockerfile @@ -47,14 +47,15 @@ ENV SC_BUILD_TARGET build # Installing client libraries and any other package you need # -# - client library for PostgreSQL https://www.postgresql.org/docs/9.5/libpq.html +# libpq: client library for PostgreSQL https://www.postgresql.org/docs/9.5/libpq.html +# libstdc++: needed in ujson https://github.com/kohlschutter/junixsocket/issues/33 # RUN apk update && \ apk add --no-cache \ libpq \ libstdc++ -# libstdc++: needed in ujson https://github.com/kohlschutter/junixsocket/issues/33 + # Installing build dependencies (will be deleted in production) diff --git a/services/director/tests/test_ujson.py b/services/director/tests/test_ujson.py new file mode 100644 index 00000000000..bff0c4341d0 --- /dev/null +++ b/services/director/tests/test_ujson.py @@ -0,0 +1,15 @@ +# +# https://github.com/kohlschutter/junixsocket/issues/33 +# +import sys +from pathlib import Path + +current_dir = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent + +def test_ujson_installation(): + import ujson + assert ujson.__version__ + + with open( current_dir / "fixtures/dummy_service_description-v1.json" ) as fh: + obj = ujson.load(fh) + assert ujson.loads(ujson.dumps(obj)) == obj From e1dd36b8af861f3c85450fc9f1f9d87a70481b6a Mon Sep 17 00:00:00 2001 From: Pedro Crespo Date: Mon, 9 Mar 2020 13:53:05 +0100 Subject: [PATCH 4/8] Adds minimal testing to avoid ujson to fail again --- services/director/tests/test_ujson.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/services/director/tests/test_ujson.py b/services/director/tests/test_ujson.py index bff0c4341d0..7551eed2a6c 100644 --- a/services/director/tests/test_ujson.py +++ b/services/director/tests/test_ujson.py @@ -1,15 +1,32 @@ # -# https://github.com/kohlschutter/junixsocket/issues/33 +# Tests instalation issues related to ujson inside or outside the container +# https://github.com/kohlschutter/junixsocket/issues/33 # +import subprocess import sys +import os from pathlib import Path current_dir = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent + def test_ujson_installation(): + # pylint: disable=c-extension-no-member import ujson + assert ujson.__version__ - with open( current_dir / "fixtures/dummy_service_description-v1.json" ) as fh: + with open(current_dir / "fixtures/dummy_service_description-v1.json") as fh: obj = ujson.load(fh) assert ujson.loads(ujson.dumps(obj)) == obj + + +def test_docker_installation(): + registry = os.environ.get("DOCKER_REGISTRY", "local") + tag = os.environ.get("DOCKER_IMAGE_TAG", "production") + + assert subprocess.run( + f'docker run -t --rm {registry}/director:{tag} python -c "import ujson; print(ujson.__version__)"', + shell=True, + check=True, + ) From 8342104e55fd26ef7cf802433dad48110f7846e5 Mon Sep 17 00:00:00 2001 From: Pedro Crespo Date: Mon, 9 Mar 2020 14:29:55 +0100 Subject: [PATCH 5/8] Moved docker test to system --- services/director/tests/test_ujson.py | 32 ----------------------- tests/swarm-deploy/test_service_images.py | 14 ++++++++++ 2 files changed, 14 insertions(+), 32 deletions(-) delete mode 100644 services/director/tests/test_ujson.py create mode 100644 tests/swarm-deploy/test_service_images.py diff --git a/services/director/tests/test_ujson.py b/services/director/tests/test_ujson.py deleted file mode 100644 index 7551eed2a6c..00000000000 --- a/services/director/tests/test_ujson.py +++ /dev/null @@ -1,32 +0,0 @@ -# -# Tests instalation issues related to ujson inside or outside the container -# https://github.com/kohlschutter/junixsocket/issues/33 -# -import subprocess -import sys -import os -from pathlib import Path - -current_dir = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent - - -def test_ujson_installation(): - # pylint: disable=c-extension-no-member - import ujson - - assert ujson.__version__ - - with open(current_dir / "fixtures/dummy_service_description-v1.json") as fh: - obj = ujson.load(fh) - assert ujson.loads(ujson.dumps(obj)) == obj - - -def test_docker_installation(): - registry = os.environ.get("DOCKER_REGISTRY", "local") - tag = os.environ.get("DOCKER_IMAGE_TAG", "production") - - assert subprocess.run( - f'docker run -t --rm {registry}/director:{tag} python -c "import ujson; print(ujson.__version__)"', - shell=True, - check=True, - ) diff --git a/tests/swarm-deploy/test_service_images.py b/tests/swarm-deploy/test_service_images.py new file mode 100644 index 00000000000..cec7322c0d0 --- /dev/null +++ b/tests/swarm-deploy/test_service_images.py @@ -0,0 +1,14 @@ +import subprocess +import os + + + +def test_ujson_installation_in_director(): + registry = os.environ.get("DOCKER_REGISTRY", "local") + tag = os.environ.get("DOCKER_IMAGE_TAG", "production") + + assert subprocess.run( + f'docker run -t --rm {registry}/director:{tag} python -c "import ujson; print(ujson.__version__)"', + shell=True, + check=True, + ) From 5c0146d0fe57fd68b74fd9da13aa1b67f3566b38 Mon Sep 17 00:00:00 2001 From: Pedro Crespo Date: Mon, 9 Mar 2020 15:14:52 +0100 Subject: [PATCH 6/8] Refactor test_service_images Uses osparc_deploy fixture --- tests/swarm-deploy/test_service_images.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/tests/swarm-deploy/test_service_images.py b/tests/swarm-deploy/test_service_images.py index cec7322c0d0..e334610230b 100644 --- a/tests/swarm-deploy/test_service_images.py +++ b/tests/swarm-deploy/test_service_images.py @@ -1,14 +1,21 @@ -import subprocess -import os +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name +import subprocess +from typing import Dict +import pytest -def test_ujson_installation_in_director(): - registry = os.environ.get("DOCKER_REGISTRY", "local") - tag = os.environ.get("DOCKER_IMAGE_TAG", "production") +@pytest.mark.parametrize("service", [ + 'director', + 'webserver', +]) +def test_ujson_installation(service:str, osparc_deploy: Dict): + image_name = osparc_deploy['simcore']['services'][service]['image'] assert subprocess.run( - f'docker run -t --rm {registry}/director:{tag} python -c "import ujson; print(ujson.__version__)"', + f'docker run -t --rm {image_name} python -c "import ujson; print(ujson.__version__)"', shell=True, check=True, ) From ba23a3756af9ec8f725eb2275bfcc2a97abd229b Mon Sep 17 00:00:00 2001 From: Pedro Crespo Date: Mon, 9 Mar 2020 15:32:40 +0100 Subject: [PATCH 7/8] Refined test_ujson_installation --- tests/swarm-deploy/test_service_images.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/swarm-deploy/test_service_images.py b/tests/swarm-deploy/test_service_images.py index e334610230b..dd5d97c62de 100644 --- a/tests/swarm-deploy/test_service_images.py +++ b/tests/swarm-deploy/test_service_images.py @@ -7,11 +7,16 @@ import pytest +# search ujson in all _base.txt and add here all services that contains it @pytest.mark.parametrize("service", [ 'director', 'webserver', + 'storage', + 'catalog' ]) def test_ujson_installation(service:str, osparc_deploy: Dict): + # tets failing installation undetected + # and fixed in PR https://github.com/ITISFoundation/osparc-simcore/pull/1353 image_name = osparc_deploy['simcore']['services'][service]['image'] assert subprocess.run( From 2d6b96174b3f23db3c7099f9dda82ae6bc355811 Mon Sep 17 00:00:00 2001 From: Pedro Crespo Date: Mon, 9 Mar 2020 15:50:28 +0100 Subject: [PATCH 8/8] Shall trigger CI again ... --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 94b1570c637..b3e7af8eaab 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ [![storage]](https://microbadger.com/images/itisfoundation/storage "More on itisfoundation/storage:staging-latest image") --> + [Requires.io]:https://img.shields.io/requires/github/ITISFoundation/osparc-simcore.svg [travis-ci]:https://travis-ci.org/ITISFoundation/osparc-simcore.svg?branch=master