Is1304/add internal traefik instance for reverse proxy

.env-devel

@@ -45,8 +45,10 @@ STORAGE_ENDPOINT=storage:8080
 TRACING_ENABLED=1
 TRACING_ZIPKIN_ENDPOINT=http://jaeger:9411
 
-WEBSERVER_LOGIN_REGISTRATION_CONFIRMATION_REQUIRED=1
-WEBSERVER_LOGIN_REGISTRATION_INVITATION_REQUIRED=1
+TRAEFIK_SIMCORE_ZONE=internal_simcore_stack
+
+WEBSERVER_LOGIN_REGISTRATION_CONFIRMATION_REQUIRED=0
+WEBSERVER_LOGIN_REGISTRATION_INVITATION_REQUIRED=0
 # python3 -c "from cryptography.fernet import Fernet; print(Fernet.generate_key())"
 WEBSERVER_SESSION_SECRET_KEY=REPLACE ME with a key of at least length 32.
 WEBSERVER_STUDIES_ACCESS_ENABLED=0

.github/workflows/ci-testing-deploy.yml

@@ -16,7 +16,6 @@ env:
   DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }}
   DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
   DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
-  CODECOV_TOKEN: 0e81ae5f-1909-4035-921c-571693fdafd1
   CC_TEST_REPORTER_ID: 21a72eb30476c870140b1576258873a41be6692f71bd9aebe812174b7d8f4b4e
   #enable buildkit
   DOCKER_BUILDKIT: 1
@@ -92,7 +91,6 @@ jobs:
         run: ./ci/github/unit-testing/api-gateway.bash test
       - uses: codecov/codecov-action@v1
         with:
-          token: ${{ env.CODECOV_TOKEN }} #required
           flags: unittests #optional
       - name: prepare codeclimate coverage file
         run: |
@@ -140,7 +138,6 @@ jobs:
         run: ./ci/github/unit-testing/catalog.bash test
       - uses: codecov/codecov-action@v1
         with:
-          token: ${{ env.CODECOV_TOKEN }} #required
           flags: unittests #optional
       - name: prepare codeclimate coverage file
         run: |
@@ -188,7 +185,6 @@ jobs:
         run: ./ci/github/unit-testing/director.bash test
       - uses: codecov/codecov-action@v1
         with:
-          token: ${{ env.CODECOV_TOKEN }} #required
           flags: unittests #optional
       - name: prepare codeclimate coverage file
         run: |
@@ -236,7 +232,6 @@ jobs:
         run: ./ci/github/unit-testing/sidecar.bash test
       - uses: codecov/codecov-action@v1
         with:
-          token: ${{ env.CODECOV_TOKEN }} #required
           flags: unittests #optional
       - name: prepare codeclimate coverage file
         run: |
@@ -283,7 +278,6 @@ jobs:
     # no coverage here??
     # - uses: codecov/codecov-action@v1
     #   with:
-    #     token: ${{ env.CODECOV_TOKEN }} #required
     #     flags: unittests #optional
 
   unit-test-python-linting:
@@ -355,7 +349,6 @@ jobs:
         run: ./ci/github/unit-testing/service-library.bash test
       - uses: codecov/codecov-action@v1
         with:
-          token: ${{ env.CODECOV_TOKEN }} #required
           flags: unittests #optional
       - name: prepare codeclimate coverage file
         run: |
@@ -403,7 +396,6 @@ jobs:
         run: ./ci/github/unit-testing/simcore-sdk.bash test
       - uses: codecov/codecov-action@v1
         with:
-          token: ${{ env.CODECOV_TOKEN }} #required
           flags: unittests #optional
       - name: prepare codeclimate coverage file
         run: |
@@ -451,7 +443,6 @@ jobs:
         run: ./ci/github/unit-testing/storage.bash test
       - uses: codecov/codecov-action@v1
         with:
-          token: ${{ env.CODECOV_TOKEN }} #required
           flags: unittests #optional
       - name: prepare codeclimate coverage file
         run: |
@@ -499,7 +490,6 @@ jobs:
         run: ./ci/github/unit-testing/webserver.bash test
       - uses: codecov/codecov-action@v1
         with:
-          token: ${{ env.CODECOV_TOKEN }} #required
           flags: unittests #optional
       - name: prepare codeclimate coverage file
         run: |
@@ -587,7 +577,6 @@ jobs:
         run: ./ci/github/integration-testing/webserver.bash clean_up
       - uses: codecov/codecov-action@v1
         with:
-          token: ${{ env.CODECOV_TOKEN }} #required
           flags: integrationtests #optional
       - name: prepare codeclimate coverage file
         run: |
@@ -645,7 +634,6 @@ jobs:
         run: ./ci/github/integration-testing/sidecar.bash clean_up
       - uses: codecov/codecov-action@v1
         with:
-          token: ${{ env.CODECOV_TOKEN }} #required
           flags: integrationtests #optional
       - name: prepare codeclimate coverage file
         run: |
@@ -703,7 +691,6 @@ jobs:
         run: ./ci/github/integration-testing/simcore-sdk.bash clean_up
       - uses: codecov/codecov-action@v1
         with:
-          token: ${{ env.CODECOV_TOKEN }} #required
           flags: integrationtests
       - name: prepare codeclimate coverage file
         run: |

.vscode-template/launch.json

@@ -4,6 +4,19 @@
   // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
   "version": "0.2.0",
   "configurations": [
+    {
+      "name": "Python: Remote Attach director",
+      "type": "python",
+      "request": "attach",
+      "port": 3004,
+      "host": "127.0.0.1",
+      "pathMappings": [
+        {
+          "localRoot": "${workspaceFolder}",
+          "remoteRoot": "/devel"
+        }
+      ]
+    },
     {
       "name": "Python: Remote Attach webserver",
       "type": "python",
@@ -17,19 +30,19 @@
         }
       ]
     },
-      {
-        "name": "Python: Remote Attach sidecar",
-        "type": "python",
-        "request": "attach",
-        "port": 3002,
-        "host": "127.0.0.1",
-        "pathMappings": [
-          {
-            "localRoot": "${workspaceFolder}",
-            "remoteRoot": "/devel"
-          }
-        ]
-      },
+    {
+      "name": "Python: Remote Attach sidecar",
+      "type": "python",
+      "request": "attach",
+      "port": 3002,
+      "host": "127.0.0.1",
+      "pathMappings": [
+        {
+          "localRoot": "${workspaceFolder}",
+          "remoteRoot": "/devel"
+        }
+      ]
+    },
     {
       "name": "Python: Remote Attach storage",
       "type": "python",
@@ -57,4 +70,4 @@
       ]
     }
   ]
-}
+}

.vscode-template/settings.json

@@ -29,8 +29,9 @@
   },
   "python.testing.pyTestEnabled": true,
   "autoDocstring.docstringFormat": "sphinx",
+  "hadolint.hadolintPath": "${workspaceFolder}/scripts/hadolint.bash",
   "shellcheck.executablePath": "${workspaceFolder}/scripts/shellcheck.bash",
   "shellcheck.run": "onSave",
   "shellcheck.enableQuickFix": true,
   "python.formatting.provider": "black"
-}
+}

packages/postgres-database/docker/Dockerfile

@@ -18,7 +18,7 @@ RUN apk add --no-cache \
     bash \
     postgresql
 
-COPY entrypoint.sh /home/entrypoint.sh
+COPY entrypoint.bash /home/entrypoint.bash
 
 
 ENV POSTGRES_USER=scu \
@@ -27,5 +27,5 @@ ENV POSTGRES_USER=scu \
     POSTGRES_PORT=5432 \
     POSTGRES_DB=simcoredb
 
-ENTRYPOINT [ "/bin/sh", "/home/entrypoint.sh" ]
+ENTRYPOINT [ "/bin/sh", "/home/entrypoint.bash" ]
 CMD [ "sc-pg", "upgrade" ]

packages/pytest-simcore/src/pytest_simcore/traefik_service.py

@@ -0,0 +1,49 @@
+# pylint:disable=unused-variable
+# pylint:disable=unused-argument
+# pylint:disable=redefined-outer-name
+
+from typing import Dict, Tuple
+
+import aiohttp
+import pytest
+import tenacity
+from yarl import URL
+
+from servicelib.minio_utils import MinioRetryPolicyUponInitialization
+
+from .helpers.utils_docker import get_service_published_port
+
+
+@pytest.fixture(scope="module")
+def traefik_endpoints(docker_stack: Dict, devel_environ: Dict) -> Tuple[URL, URL]:
+    """get the endpoint for the given simcore_service.
+    NOTE: simcore_service defined as a parametrization
+    """
+    assert f"simcore_traefik" in docker_stack["services"]
+    api_endpoint = f"127.0.0.1:{get_service_published_port('traefik', 8080)}"
+    webserver_endpoint = f"127.0.0.1:{get_service_published_port('traefik', 80)}"
+    return (URL(f"http://{api_endpoint}"), URL(f"http://{webserver_endpoint}"))
+
+
+@pytest.fixture(scope="function")
+async def traefik_service(
+    loop, traefik_endpoints: Tuple[URL, URL], docker_stack: Dict
+) -> URL:
+    api_endpoint, webserver_endpoint = traefik_endpoints
+    await wait_till_traefik_responsive(api_endpoint)
+    yield traefik_endpoints
+
+
+# HELPERS --
+
+# TODO: this can be used by ANY of the simcore services!
+@tenacity.retry(**MinioRetryPolicyUponInitialization().kwargs)
+async def wait_till_traefik_responsive(api_endpoint: URL):
+    async with aiohttp.ClientSession() as session:
+        async with session.get(api_endpoint.with_path("/api/http/routers")) as resp:
+            assert resp.status == 200
+            data = await resp.json()
+            for proxied_service in data:
+                assert "service" in proxied_service
+                if "webserver" in proxied_service["service"]:
+                    assert proxied_service["status"] == "enabled"

packages/service-library/src/servicelib/minio_utils.py

@@ -11,7 +11,7 @@ class MinioRetryPolicyUponInitialization:
     """
 
     WAIT_SECS = 2
-    ATTEMPTS_COUNT = 20
+    ATTEMPTS_COUNT = 40
 
     def __init__(self, logger: Optional[logging.Logger] = None):
         logger = logger or log

scripts/hadolint.bash

@@ -0,0 +1,4 @@
+#!/bin/bash
+# dockerfile linter tool
+# - https://github.com/hadolint/hadolint
+exec docker run --rm --interactive hadolint/hadolint < "$@"

scripts/shellcheck.bash

@@ -5,4 +5,4 @@
 # - VS extension: https://github.com/timonwong/vscode-shellcheck
 #
 
-exec docker run --rm -i -v "$PWD:/mnt:ro" koalaman/shellcheck:v0.7.0 "$@"
+exec docker run --rm --interactive --volume "$PWD:/mnt:ro" koalaman/shellcheck:v0.7.0 "$@"

services/director/src/simcore_service_director/config.py

@@ -11,14 +11,35 @@
 
 logging.basicConfig(
     level=logging.DEBUG if DEBUG_MODE else logging.INFO,
-    format='%(levelname)s:%(name)s-%(lineno)d: %(message)s'
-    )
+    format="%(levelname)s:%(name)s-%(lineno)d: %(message)s",
+)
 
 API_VERSION: str = "v0"
 API_ROOT: str = "api"
 
-DIRECTOR_REGISTRY_CACHING: bool = os.environ.get("DIRECTOR_REGISTRY_CACHING", True) in ["true", "True", True]
-DIRECTOR_REGISTRY_CACHING_TTL: int = int(os.environ.get("DIRECTOR_REGISTRY_CACHING_TTL", 15*60))
+DIRECTOR_REGISTRY_CACHING: bool = os.environ.get("DIRECTOR_REGISTRY_CACHING", True) in [
+    "true",
+    "True",
+    True,
+]
+DIRECTOR_REGISTRY_CACHING_TTL: int = int(
+    os.environ.get("DIRECTOR_REGISTRY_CACHING_TTL", 15 * 60)
+)
+
+# for passing self-signed certificate to spawned services
+DIRECTOR_SELF_SIGNED_SSL_SECRET_ID: str = os.environ.get(
+    "DIRECTOR_SELF_SIGNED_SSL_SECRET_ID", ""
+)
+DIRECTOR_SELF_SIGNED_SSL_SECRET_NAME: str = os.environ.get(
+    "DIRECTOR_SELF_SIGNED_SSL_SECRET_NAME", ""
+)
+DIRECTOR_SELF_SIGNED_SSL_FILENAME: str = os.environ.get(
+    "DIRECTOR_SELF_SIGNED_SSL_FILENAME", ""
+)
+
+TRAEFIK_SIMCORE_ZONE: str = os.environ.get(
+    "TRAEFIK_SIMCORE_ZONE", "internal_simcore_stack"
+)
 APP_REGISTRY_CACHE_DATA_KEY: str = __name__ + "_registry_cache_data"
 
 REGISTRY_AUTH: bool = os.environ.get("REGISTRY_AUTH", False) in ["true", "True", True]
@@ -31,28 +52,45 @@
 
 # these are the envs passed to the dynamic services by default
 SERVICES_DEFAULT_ENVS: Dict[str, str] = {
-    "POSTGRES_ENDPOINT": os.environ.get("POSTGRES_ENDPOINT", "undefined postgres endpoint"),
+    "POSTGRES_ENDPOINT": os.environ.get(
+        "POSTGRES_ENDPOINT", "undefined postgres endpoint"
+    ),
     "POSTGRES_USER": os.environ.get("POSTGRES_USER", "undefined postgres user"),
-    "POSTGRES_PASSWORD": os.environ.get("POSTGRES_PASSWORD", "undefined postgres password"),
+    "POSTGRES_PASSWORD": os.environ.get(
+        "POSTGRES_PASSWORD", "undefined postgres password"
+    ),
     "POSTGRES_DB": os.environ.get("POSTGRES_DB", "undefined postgres db"),
-    "STORAGE_ENDPOINT": os.environ.get("STORAGE_ENDPOINT", "undefined storage endpoint")
+    "STORAGE_ENDPOINT": os.environ.get(
+        "STORAGE_ENDPOINT", "undefined storage endpoint"
+    ),
 }
 
 # some services need to know the published host to be functional (paraview)
 # TODO: please review if needed
 PUBLISHED_HOST_NAME: str = os.environ.get("PUBLISHED_HOST_NAME", "")
+
+
 # used when in devel mode vs release mode
-NODE_SCHEMA_LOCATION: str = os.environ.get("NODE_SCHEMA_LOCATION",
-    f"{API_ROOT}/{API_VERSION}/schemas/node-meta-v0.0.1.json")
+NODE_SCHEMA_LOCATION: str = os.environ.get(
+    "NODE_SCHEMA_LOCATION", f"{API_ROOT}/{API_VERSION}/schemas/node-meta-v0.0.1.json"
+)
 # used to find the right network name
-SIMCORE_SERVICES_NETWORK_NAME: Optional[str] = os.environ.get("SIMCORE_SERVICES_NETWORK_NAME")
+SIMCORE_SERVICES_NETWORK_NAME: Optional[str] = os.environ.get(
+    "SIMCORE_SERVICES_NETWORK_NAME"
+)
 # useful when developing with an alternative registry namespace
-SIMCORE_SERVICES_PREFIX: str = os.environ.get("SIMCORE_SERVICES_PREFIX", "simcore/services")
+SIMCORE_SERVICES_PREFIX: str = os.environ.get(
+    "SIMCORE_SERVICES_PREFIX", "simcore/services"
+)
 
 # tracing
-TRACING_ENABLED: bool = os.environ.get("TRACING_ENABLED", True) in ["true", "True", True]
-TRACING_ZIPKIN_ENDPOINT: str = os.environ.get("TRACING_ZIPKIN_ENDPOINT", "http://jaeger:9411")
-
-__all__ = [
-    'APP_CLIENT_SESSION_KEY'
+TRACING_ENABLED: bool = os.environ.get("TRACING_ENABLED", True) in [
+    "true",
+    "True",
+    True,
 ]
+TRACING_ZIPKIN_ENDPOINT: str = os.environ.get(
+    "TRACING_ZIPKIN_ENDPOINT", "http://jaeger:9411"
+)
+
+__all__ = ["APP_CLIENT_SESSION_KEY"]