Export group

ci/github/system-testing/e2e.bash

@@ -14,10 +14,12 @@ install() {
     echo "--------------- getting simcore docker images..."
     make pull-version || ( (make pull-cache || true) && make build tag-version)
     make info-images
+
     # configure simcore for testing with a private registry
-    bash tests/e2e/setup_env_insecure_registry
-    # start simcore
-    make up-version
+    bash tests/e2e/scripts/setup_env_insecure_registry.bash
+
+    # start simcore and set log-level to debug
+    export LOG_LEVEL=INFO; make up-version
 
     echo "-------------- installing test framework..."
     # create a python venv and activate
@@ -34,6 +36,7 @@ install() {
     echo "--------------- transfering the images to the local registry..."
     make transfer-images-to-registry
     echo "--------------- injecting templates in postgres db..."
+    make pg-db-tables
     make inject-templates-in-db
     popd
 }
@@ -53,6 +56,9 @@ recover_artifacts() {
     (docker service logs --timestamps simcore_storage > simcore_logs/storage.log) || true
     (docker service logs --timestamps simcore_sidecar > simcore_logs/sidecar.log) || true
     (docker service logs --timestamps simcore_catalog > simcore_logs/catalog.log) || true
+
+    # stack config
+    (cp .stack-simcore-version.yml simcore_logs/) || true
 }
 
 clean_up() {

ci/travis/system-testing/e2e.bash

@@ -24,13 +24,15 @@ before_script() {
     make pull-version || ( (make pull-cache || true) && make build tag-version)
     make info-images
     # configure simcore for testing with a private registry
-    bash tests/e2e/setup_env_insecure_registry
-    # start simcore
-    make up-version
+    bash tests/e2e/scripts/setup_env_insecure_registry.bash
+
+    # start simcore and set log-level to debug
+    export LOG_LEVEL=INFO; make up-version
 
     echo "-------------- installing test framework..."
     # create a python venv and activate
     make .venv
+     # shellcheck disable=SC1091
     source .venv/bin/activate
     bash ci/helpers/ensure_python_pip.bash
     pushd tests/e2e;
@@ -42,6 +44,7 @@ before_script() {
     echo "--------------- transfering the images to the local registry..."
     make transfer-images-to-registry
     echo "--------------- injecting templates in postgres db..."
+    make pg-db-tables
     make inject-templates-in-db
     popd
 }

packages/postgres-database/requirements/prod.txt

@@ -0,0 +1,11 @@
+# Shortcut to install 'simcore-postgres-database'
+#
+# Usage:
+#   pip install -r requirements/prod.txt
+#
+
+# installs requirements first
+-r _base.txt
+-r _migration.txt
+
+.[migration]

services/catalog/src/simcore_service_catalog/db.py

@@ -35,6 +35,7 @@ async def teardown_engine() -> None:
 
 
 async def create_tables(conn: SAConnection):
+    # FIXME: this is dangerous since it enforces an empty table
     await conn.execute(f"DROP TABLE IF EXISTS {DAG.__tablename__}")
     await conn.execute(CreateTable(dags))
 

services/director/tests/fixtures/fake_services.py

@@ -13,26 +13,58 @@
 
 _logger = logging.getLogger(__name__)
 
+
 @pytest.fixture(scope="function")
 def push_services(loop, docker_registry, tmpdir):
     registry_url = docker_registry
     tmp_dir = Path(tmpdir)
 
     list_of_pushed_images_tags = []
     dependent_images = []
-    def build_push_images(number_of_computational_services, number_of_interactive_services, inter_dependent_services=False, bad_json_format=False, version="1.0."):
+
+    def build_push_images(
+        number_of_computational_services,
+        number_of_interactive_services,
+        inter_dependent_services=False,
+        bad_json_format=False,
+        version="1.0.",
+    ):
         try:
             dependent_image = None
             if inter_dependent_services:
-                dependent_image = _build_push_image(tmp_dir, registry_url, "computational", "dependency", "10.52.999999", None, bad_json_format=bad_json_format)
+                dependent_image = _build_push_image(
+                    tmp_dir,
+                    registry_url,
+                    "computational",
+                    "dependency",
+                    "10.52.999999",
+                    None,
+                    bad_json_format=bad_json_format,
+                )
                 dependent_images.append(dependent_image)
 
             for image_index in range(0, number_of_computational_services):
-                image = _build_push_image(tmp_dir, registry_url, "computational", "test", version + str(image_index), dependent_image, bad_json_format=bad_json_format)
+                image = _build_push_image(
+                    tmp_dir,
+                    registry_url,
+                    "computational",
+                    "test",
+                    version + str(image_index),
+                    dependent_image,
+                    bad_json_format=bad_json_format,
+                )
                 list_of_pushed_images_tags.append(image)
 
             for image_index in range(0, number_of_interactive_services):
-                image = _build_push_image(tmp_dir, registry_url, "dynamic", "test", version + str(image_index), dependent_image, bad_json_format=bad_json_format)
+                image = _build_push_image(
+                    tmp_dir,
+                    registry_url,
+                    "dynamic",
+                    "test",
+                    version + str(image_index),
+                    dependent_image,
+                    bad_json_format=bad_json_format,
+                )
                 list_of_pushed_images_tags.append(image)
         except docker.errors.APIError:
             _logger.exception("Unexpected docker API error")
@@ -45,70 +77,110 @@ def build_push_images(number_of_computational_services, number_of_interactive_se
     _clean_registry(registry_url, list_of_pushed_images_tags)
     _clean_registry(registry_url, dependent_images)
 
-def _build_push_image(docker_dir, registry_url, service_type, name, tag, dependent_image=None, *, bad_json_format=False): # pylint: disable=R0913
+
+def _build_push_image(
+    docker_dir,
+    registry_url,
+    service_type,
+    name,
+    tag,
+    dependent_image=None,
+    *,
+    bad_json_format=False
+):  # pylint: disable=R0913
     docker_client = docker.from_env()
     # crate image
     service_description = _create_service_description(service_type, name, tag)
     docker_labels = _create_docker_labels(service_description, bad_json_format)
-    additional_docker_labels = [{"name": "constraints", "type": "string", "value": ["node.role==manager"]}]
+    additional_docker_labels = [
+        {"name": "constraints", "type": "string", "value": ["node.role==manager"]}
+    ]
     internal_port = None
-    entry_point = ''
+    entry_point = ""
     if service_type == "dynamic":
         internal_port = random.randint(1, 65535)
-        additional_docker_labels.append({"name": "ports", "type": "int", "value": internal_port})
+        additional_docker_labels.append(
+            {"name": "ports", "type": "int", "value": internal_port}
+        )
         entry_point = "/test/entry_point"
-        docker_labels["simcore.service.bootsettings"] = json.dumps([{"name": "entry_point", "type": "string", "value": entry_point}])
+        docker_labels["simcore.service.bootsettings"] = json.dumps(
+            [{"name": "entry_point", "type": "string", "value": entry_point}]
+        )
     docker_labels["simcore.service.settings"] = json.dumps(additional_docker_labels)
     if bad_json_format:
-        docker_labels["simcore.service.settings"] = "'fjks" + docker_labels["simcore.service.settings"]
+        docker_labels["simcore.service.settings"] = (
+            "'fjks" + docker_labels["simcore.service.settings"]
+        )
 
     if dependent_image is not None:
         dependent_description = dependent_image["service_description"]
-        dependency_docker_labels = [{"key":dependent_description["key"], "tag":dependent_description["version"]}]
-        docker_labels["simcore.service.dependencies"] = json.dumps(dependency_docker_labels)
+        dependency_docker_labels = [
+            {
+                "key": dependent_description["key"],
+                "tag": dependent_description["version"],
+            }
+        ]
+        docker_labels["simcore.service.dependencies"] = json.dumps(
+            dependency_docker_labels
+        )
         if bad_json_format:
-            docker_labels["simcore.service.dependencies"] = "'fjks" + docker_labels["simcore.service.dependencies"]
+            docker_labels["simcore.service.dependencies"] = (
+                "'fjks" + docker_labels["simcore.service.dependencies"]
+            )
 
     image = _create_base_image(docker_dir, docker_labels)
     # tag image
-    image_tag = registry_url + "/{key}:{version}".format(key=service_description["key"], version=tag)
+    image_tag = registry_url + "/{key}:{version}".format(
+        key=service_description["key"], version=tag
+    )
     assert image.tag(image_tag) is True
     # push image to registry
     docker_client.images.push(image_tag)
     # remove image from host
     docker_client.images.remove(image_tag)
     return {
-        "service_description":service_description,
-        "docker_labels":docker_labels,
-        "image_path":image_tag,
-        "internal_port":internal_port,
-        "entry_point": entry_point
-        }
+        "service_description": service_description,
+        "docker_labels": docker_labels,
+        "image_path": image_tag,
+        "internal_port": internal_port,
+        "entry_point": entry_point,
+    }
+
 
 def _clean_registry(registry_url, list_of_images):
-    request_headers = {'accept': "application/vnd.docker.distribution.manifest.v2+json"}
+    request_headers = {"accept": "application/vnd.docker.distribution.manifest.v2+json"}
     for image in list_of_images:
         service_description = image["service_description"]
         # get the image digest
         tag = service_description["version"]
-        url = "http://{host}/v2/{name}/manifests/{tag}".format(host=registry_url, name=service_description["key"], tag=tag)
+        url = "http://{host}/v2/{name}/manifests/{tag}".format(
+            host=registry_url, name=service_description["key"], tag=tag
+        )
         response = requests.get(url, headers=request_headers)
         docker_content_digest = response.headers["Docker-Content-Digest"]
         # remove the image from the registry
-        url = "http://{host}/v2/{name}/manifests/{digest}".format(host=registry_url, name=service_description["key"], digest=docker_content_digest)
+        url = "http://{host}/v2/{name}/manifests/{digest}".format(
+            host=registry_url,
+            name=service_description["key"],
+            digest=docker_content_digest,
+        )
         response = requests.delete(url, headers=request_headers)
 
+
 def _create_base_image(base_dir, labels):
     # create a basic dockerfile
     docker_file = base_dir / "Dockerfile"
     with docker_file.open("w") as file_pointer:
-        file_pointer.write('FROM alpine\nCMD while true; do sleep 10; done\n')
+        file_pointer.write("FROM alpine\nCMD while true; do sleep 10; done\n")
     assert docker_file.exists() == True
     # build docker base image
     docker_client = docker.from_env()
-    base_docker_image = docker_client.images.build(path=str(base_dir), rm=True, labels=labels)
+    base_docker_image = docker_client.images.build(
+        path=str(base_dir), rm=True, labels=labels
+    )
     return base_docker_image[0]
 
+
 def _create_service_description(service_type, name, tag):
     file_name = "dummy_service_description-v1.json"
     dummy_description_path = Path(__file__).parent / file_name
@@ -125,10 +197,13 @@ def _create_service_description(service_type, name, tag):
 
     return service_desc
 
+
 def _create_docker_labels(service_description, bad_json_format):
     docker_labels = {}
     for key, value in service_description.items():
-        docker_labels[".".join(["io", "simcore", key])] = json.dumps({key:value})
+        docker_labels[".".join(["io", "simcore", key])] = json.dumps({key: value})
         if bad_json_format:
-            docker_labels[".".join(["io", "simcore", key])] = "d32;'" + docker_labels[".".join(["io", "simcore", key])]
+            docker_labels[".".join(["io", "simcore", key])] = (
+                "d32;'" + docker_labels[".".join(["io", "simcore", key])]
+            )
     return docker_labels

services/director/tests/test_registry_cache_task.py

@@ -4,22 +4,28 @@
 
 import pytest
 
-from simcore_service_director import (config, main, registry_cache_task,
-                                      registry_proxy)
+from simcore_service_director import config, main, registry_cache_task, registry_proxy
 
 
 @pytest.fixture
-def client(loop, aiohttp_client, aiohttp_unused_port, configure_schemas_location, configure_registry_access):
+def client(
+    loop,
+    aiohttp_client,
+    aiohttp_unused_port,
+    configure_schemas_location,
+    configure_registry_access,
+):
     config.DIRECTOR_REGISTRY_CACHING = True
     config.DIRECTOR_REGISTRY_CACHING_TTL = 5
     # config.DIRECTOR_REGISTRY_CACHING_TTL = 5
     app = main.setup_app()
-    server_kwargs={'port': aiohttp_unused_port(), 'host': 'localhost'}
+    server_kwargs = {"port": aiohttp_unused_port(), "host": "localhost"}
 
     registry_cache_task.setup(app)
 
     yield loop.run_until_complete(aiohttp_client(app, server_kwargs=server_kwargs))
 
+
 async def test_registry_caching_task(loop, client, push_services):
     app = client.app
     assert app
@@ -30,17 +36,29 @@ async def test_registry_caching_task(loop, client, push_services):
     assert registry_cache_task.APP_REGISTRY_CACHE_DATA_KEY in app
 
     # check we do not get any repository
-    list_of_services = await registry_proxy.list_services(app, registry_proxy.ServiceType.ALL)
+    list_of_services = await registry_proxy.list_services(
+        app, registry_proxy.ServiceType.ALL
+    )
     assert not list_of_services
     assert app[registry_cache_task.APP_REGISTRY_CACHE_DATA_KEY] != {}
     # create services in the registry
-    pushed_services = push_services(1,1)
+    pushed_services = push_services(
+        number_of_computational_services=1, number_of_interactive_services=1
+    )
     # the services shall be updated
     await sleep(config.DIRECTOR_REGISTRY_CACHING_TTL)
-    list_of_services = await registry_proxy.list_services(app, registry_proxy.ServiceType.ALL)
+    list_of_services = await registry_proxy.list_services(
+        app, registry_proxy.ServiceType.ALL
+    )
     assert len(list_of_services) == 2
     # add more
-    pushed_services = push_services(2,2, version="2.0.")
-    await sleep(config.DIRECTOR_REGISTRY_CACHING_TTL)
-    list_of_services = await registry_proxy.list_services(app, registry_proxy.ServiceType.ALL)
-    assert len(list_of_services) == len(pushed_services)
+    pushed_services = push_services(
+        number_of_computational_services=2,
+        number_of_interactive_services=2,
+        version="2.0.",
+    )
+    await sleep(config.DIRECTOR_REGISTRY_CACHING_TTL * 1.1) # NOTE: this sometimes takes a bit more. Sleep increased a 10%.
+    list_of_services = await registry_proxy.list_services(
+        app, registry_proxy.ServiceType.ALL
+    )
+    assert len(list_of_services) == len(pushed_services)

services/docker-compose.yml

@@ -10,7 +10,7 @@ services:
       - POSTGRES_HOST=${POSTGRES_HOST}
       - POSTGRES_PORT=${POSTGRES_PORT}
       - TESTING=false
-      - LOGLEVEL=WARNING
+      - LOGLEVEL=${LOG_LEVEL:-WARNING}
     depends_on:
       - postgres
     networks:
@@ -64,7 +64,7 @@ services:
       - STORAGE_PORT=8080
       - SWARM_STACK_NAME=${SWARM_STACK_NAME:-simcore}
       - WEBSERVER_MONITORING_ENABLED=1
-      - WEBSERVER_LOGLEVEL=WARNING
+      - WEBSERVER_LOGLEVEL=${LOG_LEVEL:-WARNING}
     env_file:
       - ../.env
     depends_on:
@@ -118,7 +118,7 @@ services:
       - REGISTRY_USER=${REGISTRY_USER}
       - REGISTRY_PW=${REGISTRY_PW}
       - SWARM_STACK_NAME=${SWARM_STACK_NAME:-simcore}
-      - SIDECAR_LOGLEVEL=WARNING
+      - SIDECAR_LOGLEVEL=${LOG_LEVEL:-WARNING}
     depends_on:
       - rabbit
       - postgres
@@ -135,7 +135,7 @@ services:
       - POSTGRES_DB=${POSTGRES_DB}
       - POSTGRES_HOST=${POSTGRES_HOST}
       - POSTGRES_PORT=${POSTGRES_PORT}
-      - STORAGE_LOGLEVEL=WARNING
+      - STORAGE_LOGLEVEL=${LOG_LEVEL:-WARNING}
       - STORAGE_MONITORING_ENABLED=1
       - S3_ENDPOINT=${S3_ENDPOINT}
       - S3_ACCESS_KEY=${S3_ACCESS_KEY}