storage/tests/test_config passes

- Adds workspace variables in config-files
- Update config files with new section
- Adds APIHUB_HOST, _PORT in storage docker-compose
- Fixes duplication of docker-compose storage service

Author: Pedro Crespo

services/docker-compose.yml

@@ -148,36 +148,8 @@ services:
     ports:
       - "11111:8080"
     environment:
-      - POSTGRES_ENDPOINT=${POSTGRES_ENDPOINT}
-      - POSTGRES_USER=${POSTGRES_USER}
-      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
-      - POSTGRES_DB=${POSTGRES_DB}
-      - POSTGRES_HOST=${POSTGRES_HOST}
-      - POSTGRES_PORT=${POSTGRES_PORT}
-      - S3_ENDPOINT=${S3_ENDPOINT}
-      - S3_ACCESS_KEY=${S3_ACCESS_KEY}
-      - S3_SECRET_KEY=${S3_SECRET_KEY}
-      - S3_BUCKET_NAME=${S3_BUCKET_NAME}
-      - STORAGE_ENDPOINT=${STORAGE_ENDPOINT}
-      - RUN_DOCKER_ENGINE_ROOT=${RUN_DOCKER_ENGINE_ROOT}
-      - BF_API_SECRET=${BF_API_SECRET}
-      - BF_API_KEY=${BF_API_KEY}
-    depends_on:
-      - minio
-      - postgres
-  #--------------------------------------------------------------------
-  storage:
-    build:
-      # the context for the build is the git repo root directory, this allows to copy
-      # the packages directory into any docker image
-      context: ../
-      dockerfile: services/storage/Dockerfile
-      args:
-        - DOCKER_GID_ARG=${DOCKER_GID:?Undefined docker gid in host}
-      target: production
-    ports:
-      - "11111:8080"
-    environment:
+      - APIHUB_HOST=apihub
+      - APIHUB_PORT=8043
       - POSTGRES_ENDPOINT=${POSTGRES_ENDPOINT}
       - POSTGRES_USER=${POSTGRES_USER}
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}

services/storage/src/simcore_service_storage/cli.py

@@ -15,12 +15,36 @@
 import argparse
 import logging
 import sys
+import os
 
 from . import application, cli_config
+from servicelib.utils import search_osparc_repo_dir
+
 
 log = logging.getLogger(__name__)
 
 
+def create_environ(skip_system_environ=False):
+    """
+        Build environment of substitutable variables
+
+    """
+    # system's environment variables
+    environ = {} if skip_system_environ else dict(os.environ)
+
+    # project-related environment variables
+    here = os.path.dirname(__file__)
+    environ['THIS_PACKAGE_DIR'] = here
+
+    rootdir = search_osparc_repo_dir(start=here)
+    if rootdir is not None:
+        environ['OSPARC_SIMCORE_REPO_ROOTDIR'] = str(rootdir)
+
+
+    return environ
+
+
+
 def setup(_parser):
     cli_config.add_cli_options(_parser)
     return _parser
@@ -33,7 +57,7 @@ def parse(args, _parser):
 
     # ignore unknown options
     options, _ = _parser.parse_known_args(args)
-    config = cli_config.config_from_options(options)
+    config = cli_config.config_from_options(options, vars=create_environ())
 
     # TODO: check whether extra options can be added to the config?!
     return config

services/storage/src/simcore_service_storage/data/docker-dev-config.yaml

@@ -1,5 +1,5 @@
+version: '1.0'
 main:
-#  disable_services: ['postgres', 's3']
   host: 0.0.0.0
   log_level: INFO
   port: 8080
@@ -21,4 +21,6 @@ s3:
   access_key: ${S3_ACCESS_KEY}
   secret_key: ${S3_SECRET_KEY}
   bucket_name: ${S3_BUCKET_NAME}
-version: '1.0'
+rest:
+  oas_repo: ${OSPARC_SIMCORE_REPO_ROOTDIR}/api/specs
+  #oas_repo: http://0.0.0.0:8043/api/specs

services/storage/src/simcore_service_storage/data/docker-prod-config.yaml

@@ -1,5 +1,5 @@
+version: '1.0'
 main:
-#  disable_services: ['postgres', 's3']
   host: 0.0.0.0
   log_level: INFO
   port: 8080
@@ -20,4 +20,5 @@ s3:
   access_key: ${S3_ACCESS_KEY}
   secret_key: ${S3_SECRET_KEY}
   bucket_name: ${S3_BUCKET_NAME}
-version: '1.0'
+rest:
+  oas_repo: http://${APIHUB_HOST}:${APIHUB_PORT}/api/specs

services/storage/src/simcore_service_storage/data/host-dev-config.yaml

@@ -1,3 +1,4 @@
+version: '1.0'
 main:
   disable_services: ['postgres', 's3']
   host: 127.0.0.1
@@ -22,4 +23,6 @@ s3:
   bucket_name: simcore
   endpoint: minio:9000
   secret_key: '12345678'
-version: '1.0'
+rest:
+  oas_repo: ${OSPARC_SIMCORE_REPO_ROOTDIR}/api/specs
+  #oas_repo: http://localhost:8043/api/specs

services/storage/tests/test_configs.py

@@ -41,6 +41,7 @@ def devel_environ(env_devel_file):
                 env_devel[key] = value
     return env_devel
 
+from simcore_service_storage.cli import create_environ
 
 @pytest.fixture("session")
 def container_environ(services_docker_compose_file, devel_environ):
@@ -51,11 +52,12 @@ def container_environ(services_docker_compose_file, devel_environ):
     with services_docker_compose_file.open() as f:
         dc = yaml.safe_load(f)
 
-    container_environ = {
+    container_environ = create_environ(skip_system_environ=True)
+    container_environ.update({
         'VENV2': '/home/scu/.venv27/' # defined in Dockerfile
-    }
+    })
 
-    environ_items =dc["services"][THIS_SERVICE].get("environment", list())
+    environ_items = dc["services"][THIS_SERVICE].get("environment", list())
     MATCH = re.compile(r'\$\{(\w+)+')
 
     for item in environ_items: