✨Run dynamic services via dynamic-sidecar (🏗️ OPS + CI action)

.env-devel

@@ -18,6 +18,8 @@ CATALOG_DEV_FEATURES_ENABLED=0
 
 DASK_SCHEDULER_HOST=dask-scheduler
 
+DYNAMIC_SIDECAR_IMAGE=${DOCKER_REGISTRY:-itisfoundation}/dynamic-sidecar:${DOCKER_IMAGE_TAG:-latest}
+
 DIRECTOR_REGISTRY_CACHING_TTL=900
 DIRECTOR_REGISTRY_CACHING=True
 

.github/workflows/ci-testing-deploy.yml

@@ -1824,9 +1824,9 @@ jobs:
           name: codeclimate-${{ github.job }}-coverage
           path: codeclimate.${{ github.job }}_coverage.json
 
-  integration-test-director-v2:
+  integration-test-director-v2-01:
     timeout-minutes: 30 # if this timeout gets too small, then split the tests
-    name: "[int] director-v2"
+    name: "[int] director-v2 01"
     needs: [build-test-images]
     runs-on: ${{ matrix.os }}
     strategy:
@@ -1873,7 +1873,81 @@ jobs:
       - name: install
         run: ./ci/github/integration-testing/director-v2.bash install
       - name: test
-        run: ./ci/github/integration-testing/director-v2.bash test
+        run: ./ci/github/integration-testing/director-v2.bash test 01
+      - name: upload failed tests logs
+        if: failure()
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{ github.job }}_docker_logs
+          path: ./services/director-v2/test_failures
+      - name: cleanup
+        if: always()
+        run: ./ci/github/integration-testing/director-v2.bash clean_up
+      - uses: codecov/codecov-action@v1
+        with:
+          flags: integrationtests #optional
+      - name: prepare codeclimate coverage file
+        run: |
+          curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-0.7.0-linux-amd64 > ./cc-test-reporter
+          chmod +x ./cc-test-reporter && ./cc-test-reporter --version
+          ./cc-test-reporter format-coverage -t coverage.py -o codeclimate.${{ github.job }}_coverage.json coverage.xml
+      - name: upload codeclimate coverage
+        uses: actions/upload-artifact@v2
+        with:
+          name: codeclimate-${{ github.job }}-coverage
+          path: codeclimate.${{ github.job }}_coverage.json
+
+  integration-test-director-v2-02:
+    timeout-minutes: 20 # if this timeout gets too small, then split the tests
+    name: "[int] director-v2 02"
+    needs: [build-test-images]
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python: [3.8]
+        os: [ubuntu-20.04]
+        docker_buildx: [v0.5.1]
+        docker_compose: [1.29.1]
+        include:
+          - docker_compose: 1.29.1
+            docker_compose_sha: 8097769d32e34314125847333593c8edb0dfc4a5b350e4839bef8c2fe8d09de7
+      fail-fast: false
+    steps:
+      - name: set PR default variables
+        # only pushes have access to the docker credentials, use a default
+        if: github.event_name == 'pull_request'
+        run: |
+          export TMP_DOCKER_REGISTRY=${GITHUB_REPOSITORY%/*}
+          echo "DOCKER_REGISTRY=${TMP_DOCKER_REGISTRY,,}" >> $GITHUB_ENV
+      - uses: actions/checkout@v2
+      - name: setup docker buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          version: ${{ matrix.docker_buildx }}
+          driver: docker
+
+      - name: setup docker-compose
+        run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }}
+      - name: setup python environment
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python }}
+      - name: show system version
+        run: ./ci/helpers/show_system_versions.bash
+      - uses: actions/cache@v2
+        name: getting cached data
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-director-v2-${{ hashFiles('services/director-v2/requirements/ci.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-director-v2-
+            ${{ runner.os }}-pip-
+            ${{ runner.os }}-
+      - name: install
+        run: ./ci/github/integration-testing/director-v2.bash install
+      - name: test
+        run: ./ci/github/integration-testing/director-v2.bash test 02
       - name: upload failed tests logs
         if: failure()
         uses: actions/upload-artifact@v2
@@ -2044,7 +2118,7 @@ jobs:
           path: codeclimate.${{ github.job }}_coverage.json
 
   system-test-public-api:
-    timeout-minutes: 25 # if this timeout gets too small, then split the tests
+    timeout-minutes: 30 # if this timeout gets too small, then split the tests
     name: "[sys] public api"
     needs: [build-test-images]
     runs-on: ${{ matrix.os }}
@@ -2320,7 +2394,8 @@ jobs:
         unit-test-webserver-10,
         integration-test-webserver-01,
         integration-test-webserver-02,
-        integration-test-director-v2,
+        integration-test-director-v2-01,
+        integration-test-director-v2-02,
         integration-test-sidecar,
         integration-test-simcore-sdk,
       ]
@@ -2380,7 +2455,8 @@ jobs:
         unit-test-webserver-10,
         integration-test-webserver-01,
         integration-test-webserver-02,
-        integration-test-director-v2,
+        integration-test-director-v2-01,
+        integration-test-director-v2-02,
         integration-test-sidecar,
         integration-test-simcore-sdk,
         system-test-public-api,

Makefile

@@ -195,17 +195,19 @@ define _show_endpoints
 set -o allexport; \
 source $(CURDIR)/.env; \
 set +o allexport; \
-separator=------------------------------------------;\
+separator=------------------------------------------------------------------------------------;\
 separator=$${separator}$${separator}$${separator};\
-rows="%-22s | %80s | %12s | %12s\n";\
+rows="%-22s | %90s | %12s | %12s\n";\
 TableWidth=140;\
-printf "%22s | %80s | %12s | %12s\n" Name Endpoint User Password;\
+printf "%22s | %90s | %12s | %12s\n" Name Endpoint User Password;\
 printf "%.$${TableWidth}s\n" "$$separator";\
-printf "$$rows" 'oSparc platform' "http://$(if $(IS_WSL2),$(get_my_ip),127.0.0.1):9081";\
-printf "$$rows" 'Postgres DB' "http://$(if $(IS_WSL2),$(get_my_ip),127.0.0.1):18080/?pgsql=postgres&username=$${POSTGRES_USER}&db=$${POSTGRES_DB}&ns=public" $${POSTGRES_USER} $${POSTGRES_PASSWORD};\
-printf "$$rows" Portainer "http://$(if $(IS_WSL2),$(get_my_ip),127.0.0.1):9000" admin adminadmin;\
-printf "$$rows" Redis-commander "http://$(if $(IS_WSL2),$(get_my_ip),127.0.0.1):18081";\
-printf "$$rows" "Docker Registry" "$${REGISTRY_URL}" $${REGISTRY_USER} $${REGISTRY_PW}
+printf "$$rows" 'oSparc platform' 'http://$(get_my_ip).nip.io:9081';\
+printf "$$rows" 'Postgres DB' 'http://$(get_my_ip).nip.io:18080/?pgsql=postgres&username='$${POSTGRES_USER}'&db='$${POSTGRES_DB}'&ns=public' $${POSTGRES_USER} $${POSTGRES_PASSWORD};\
+printf "$$rows" Portainer 'http://$(get_my_ip).nip.io:9000' admin adminadmin;\
+printf "$$rows" Redis 'http://$(get_my_ip).nip.io:18081';\
+printf "$$rows" 'Docker Registry' $${REGISTRY_URL} $${REGISTRY_USER} $${REGISTRY_PW};\
+echo "⚠️ if a DNS is not used (as displayed above), the interactive services started via dynamic-sidecar"
+echo "⚠️ will not be shown. The frontend accesses them via the uuid.services.YOUR_IP.nip.io:9081"
 endef
 
 show-endpoints:

README.md

@@ -52,16 +52,17 @@ This is the common workflow to build and deploy locally:
   make info-swarm
 
   # open front-end in the browser
-  #  localhost:9081 - simcore front-end site
+  #  127.0.0.1.nip.io:9081 - simcore front-end site
   #
-  xdg-open http://localhost:9081/
+  xdg-open http://127.0.0.1.nip.io:9081/
 
   # stops
   make down
 ```
 
-Services are deployed in two stacks:``simcore-stack`` comprises all core-services in the framework
-and ``ops-stack`` is a subset of services from [ITISFoundation/osparc-ops](https://github.com/ITISFoundation/osparc-ops) used
+Some routes can only be reached via DNS such as `UUID.services.DNS`. Since `UUID.services.127.0.0.1` is **not a valid DNS**, the solution is to use [nip.io](https://nip.io/). A service that maps ``<anything>[.-]<IP Address>.nip.io`` in "dot", "dash" or "hexadecimal" notation to the corresponding ``<IP Address>``.
+
+Services are deployed in two stacks:``simcore-stack`` comprises all core-services in the framework and ``ops-stack`` is a subset of services from [ITISFoundation/osparc-ops](https://github.com/ITISFoundation/osparc-ops) used
 for operations during development. This is a representation of ``simcore-stack``:
 
 ![](docs/img/.stack-simcore-version.yml.png)
@@ -101,6 +102,26 @@ In **windows**, it works under [WSL] (windows subsystem for linux). Some details
 
 In **MacOS**, [replacing the MacOS utilities with GNU utils](https://apple.stackexchange.com/a/69332) might be required.
 
+#### Upgrading services requirements
+
+Updates are upgraded using a docker container and pip-sync.
+Build and start the container:
+
+    cd requirements/tools
+    make build
+    make shell
+
+Once inside the container navigate to the service's requirements directory.
+
+To upgrade all requirements run:
+
+    make reqs
+
+To upgrade a single requirement named `fastapi`run:
+
+    make reqs upgrade=fastapi
+
+
 ## Releases
 
 **WARNING** This application is **still under development**.

ci/github/integration-testing/director-v2.bash

@@ -20,9 +20,10 @@ install() {
 }
 
 test() {
+  echo "testing in services/director-v2/tests/integration/$1"
   pytest --cov=simcore_service_director_v2 --durations=10 --cov-append \
     --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \
-    -v -m "not travis" services/director-v2/tests/integration --log-level=DEBUG
+    -v -m "not travis" "services/director-v2/tests/integration/$1" --log-level=DEBUG
 }
 
 clean_up() {

packages/models-library/src/models_library/service_settings_labels.py

@@ -144,7 +144,8 @@ def needs_dynamic_sidecar(self) -> bool:
 
     @validator("container_http_entry", always=True)
     @classmethod
-    def compose_spec_requires_container_http_entry(cls, v, values):
+    def compose_spec_requires_container_http_entry(cls, v, values) -> Optional[str]:
+        v = None if v == "" else v
         if v is None and values.get("compose_spec") is not None:
             raise ValueError(
                 "Field `container_http_entry` must be defined but is missing"
@@ -186,6 +187,7 @@ class SimcoreServiceLabels(DynamicSidecarServiceLabels):
     )
 
     class Config(_BaseConfig):
+        extra = Extra.allow
         schema_extra = {
             "examples": [
                 # legacy service

packages/models-library/src/models_library/settings/docker_registry.py

@@ -0,0 +1,26 @@
+from typing import Optional
+
+from pydantic import Field, SecretStr
+
+from .base import BaseCustomSettings
+
+
+class RegistrySettings(BaseCustomSettings):
+
+    REGISTRY_AUTH: bool = Field(..., description="do registry authentication")
+    REGISTRY_PATH: Optional[str] = Field(
+        None, description="development mode only, in case a local registry is used"
+    )
+    REGISTRY_URL: str = Field("", description="url to the docker registry")
+
+    REGISTRY_USER: str = Field(
+        ..., description="username to access the docker registry"
+    )
+    REGISTRY_PW: SecretStr = Field(
+        ..., description="password to access the docker registry"
+    )
+    REGISTRY_SSL: bool = Field(..., description="access to registry through ssl")
+
+    @property
+    def resolved_registry_url(self) -> str:
+        return self.REGISTRY_PATH or self.REGISTRY_URL

packages/models-library/src/models_library/settings/services_common.py

@@ -1,27 +1,30 @@
 from pydantic import BaseSettings, Field, PositiveInt
 
-_BASE_TIMEOUT_FOR_STOPPING_SERVICES = 60 * 60
+_MINUTE = 60
+_HOUR = 60 * _MINUTE
 
 
 class ServicesCommonSettings(BaseSettings):
     # set this interval to 1 hour
     director_dynamic_service_save_timeout: PositiveInt = Field(
-        _BASE_TIMEOUT_FOR_STOPPING_SERVICES,
+        _HOUR,
         description=(
             "When stopping a dynamic service, if it has "
             "big payloads it is important to have longer timeouts."
         ),
     )
     webserver_director_stop_service_timeout: PositiveInt = Field(
-        _BASE_TIMEOUT_FOR_STOPPING_SERVICES + 10,
+        _HOUR + 10,
         description=(
-            "When the webserver invokes the director API to stop "
-            "a service which has a very long timeout, it also "
-            "requires to wait that amount plus some extra padding."
+            "The below will try to help explaining what is happening: "
+            "webserver -(stop_service)-> director-v* -(save_state)-> service_x"
+            "- webserver requests stop_service and uses a 01:00:10 timeout"
+            "- director-v* requests save_state and uses a 01:00:00 timeout"
+            "The +10 seconds is used to make sure the director replies"
         ),
     )
     storage_service_upload_download_timeout: PositiveInt = Field(
-        60 * 60,
+        _HOUR,
         description=(
             "When dynamic services upload and download data from storage, "
             "sometimes very big payloads are involved. In order to handle "

packages/pytest-simcore/src/pytest_simcore/docker_registry.py

@@ -225,3 +225,54 @@ def jupyter_service(docker_registry: str, node_meta_schema: Dict) -> Dict[str, s
         docker_registry,
         node_meta_schema,
     )
+
+
+DY_STATIC_FILE_SERVER_VERSION = "1.0.5"
+
+
+@pytest.fixture(scope="session")
+def dy_static_file_server_service(
+    docker_registry: str, node_meta_schema: Dict
+) -> Dict[str, str]:
+    """
+    Adds the below service in docker registry
+    itisfoundation/dy-static-file-server
+    """
+    return _pull_push_service(
+        "itisfoundation/dy-static-file-server",
+        DY_STATIC_FILE_SERVER_VERSION,
+        docker_registry,
+        node_meta_schema,
+    )
+
+
+@pytest.fixture(scope="session")
+def dy_static_file_server_dynamic_sidecar_service(
+    docker_registry: str, node_meta_schema: Dict
+) -> Dict[str, str]:
+    """
+    Adds the below service in docker registry
+    itisfoundation/dy-static-file-server-dynamic-sidecar
+    """
+    return _pull_push_service(
+        "itisfoundation/dy-static-file-server-dynamic-sidecar",
+        DY_STATIC_FILE_SERVER_VERSION,
+        docker_registry,
+        node_meta_schema,
+    )
+
+
+@pytest.fixture(scope="session")
+def dy_static_file_server_dynamic_sidecar_compose_spec_service(
+    docker_registry: str, node_meta_schema: Dict
+) -> Dict[str, str]:
+    """
+    Adds the below service in docker registry
+    itisfoundation/dy-static-file-server-dynamic-sidecar-compose-spec
+    """
+    return _pull_push_service(
+        "itisfoundation/dy-static-file-server-dynamic-sidecar-compose-spec",
+        DY_STATIC_FILE_SERVER_VERSION,
+        docker_registry,
+        node_meta_schema,
+    )

packages/settings-library/src/settings_library/docker_registry.py

@@ -1,7 +1,7 @@
 from functools import cached_property
 from typing import Optional
 
-from pydantic import Field, SecretStr
+from pydantic import Field, SecretStr, validator
 
 from .base import BaseCustomSettings
 
@@ -23,6 +23,11 @@ class RegistrySettings(BaseCustomSettings):
     )
     REGISTRY_SSL: bool = Field(..., description="access to registry through ssl")
 
+    @validator("REGISTRY_PATH", pre=True)
+    @classmethod
+    def escape_none_string(cls, v) -> Optional[str]:
+        return None if v == "None" else v
+
     @cached_property
     def resolved_registry_url(self) -> str:
         return self.REGISTRY_PATH or self.REGISTRY_URL