Feature/#215 adapt platform to new definition of nodes (#230)

* added label descriptors as json\nadded utils functions for validating nodes

* disabled conversion from old to new schema

* git

* added validation of json schema
fixed teardown of fixtures

* temporaily skip test that should be modified with issue #222

* refactored tests

* add handlers tests

* pylint

* added pytest-asyncio for async testing

* fixed issue while testing
pylint fixes
requirements

* missing include to aiohttp-apiset in testing env

* fix includes in travis

* fixed default values in config
made base image wait for 10secs for testing purposes
added tests for handler

* pylint

* tests done through handler

* fixes tests

* moved init registry fixture to conftest

* moved docker swarm fixture to conftest

* added test producer unit test
docker registry fixture allow for long running tasks in docker
fixed env variables set up in director
renamed is_service_up to get_service_details

* bad merge

* using pkg_resources

* separated fixtures in more files
added test to check dummmy service validity

* refactoring

* added test to check openapi interaface (currently does not convert the json schema part)

* fix default sleep time

* code cleanup

* get service logs when an issue arises

* onlyt start computational service

* try getting logs out

* travis

* fix bad image

* set dynamic services back in

* fix container under linux

* check if part of swarm

* revert last change

* try updating docker in travis

* switch conversion back on as node description are again going to change after PM2

* fix test with conversion back on

* added force flag

Author: sanderegg

.travis.yml

@@ -27,6 +27,10 @@ matrix:
         - python --version
         - uname -a
         - lsb_release -a
+        - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+        - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+        - sudo apt-get update
+        - sudo apt-get -y install docker-ce
         - docker --version
         - docker-compose --version
         # shutdown postgres because sometimes is is alrady up ???
@@ -37,8 +41,8 @@ matrix:
       install:
         - pip install --upgrade pip wheel setuptools && pip3 --version
         - pip3 install packages/s3wrapper[test]
-        - pip3 install packages/simcore-sdk[test]
-        - pip3 install services/director[test]
+        - pip3 install packages/simcore-sdk[test]        
+        - pushd services/director; pip3 install -r requirements/ci.txt; popd
         - pip3 install packages/director-sdk/python
         - pushd services/web/server; pip3 install -r requirements/ci.txt; popd
 

scripts/docker/remove-all-images.sh

@@ -1,2 +1,2 @@
 #!/bin/bash
-docker rmi $(docker images -q)
+docker rmi $(docker images -q) -f

services/director/requirements/ci.txt

@@ -0,0 +1,2 @@
+.[test]
+git+https://github.com/ITISFoundation/aiohttp_apiset.git@fixes_4_osparc#egg=aiohttp_apiset

services/director/requirements/test.txt

@@ -1,5 +1,8 @@
 coveralls~=1.3
+jsonschema~=2.6
+openapi_spec_validator
 pytest-aiohttp
 pytest~=3.6
 pytest-cov~=2.5
-pytest-docker~=0.6
+pytest-docker~=0.6
+pytest-asyncio~=0.9.0

services/director/src/simcore_service_director/config.py

@@ -1,31 +1,31 @@
+"""Director service configuration
+"""
+
 import logging
 import os
-from pathlib import Path
 
 logging.basicConfig(
     level=logging.DEBUG,
     format='%(levelname)s:%(name)s-%(lineno)d: %(message)s'
     )
 
-# FIXME: adapt to new config. Issue #195
 CONVERT_OLD_API = True
-
-OPEN_API_BASE_FOLDER = Path(__file__).parent / "oas3/v1"
-OPEN_API_SPEC_FILE = "openapi.yaml"
-JSON_SCHEMA_BASE_FOLDER = Path(__file__).parent / "oas3/v1/schemas"
-NODE_JSON_SCHEMA_FILE = "node-meta-v0.0.1.json"
+API_VERSION = "v1"
 
 REGISTRY_AUTH = os.environ.get("REGISTRY_AUTH", False) in ["true", "True"]
 REGISTRY_USER = os.environ.get("REGISTRY_USER", "")
 REGISTRY_PW = os.environ.get("REGISTRY_PW", "")
 REGISTRY_URL = os.environ.get("REGISTRY_URL", "")
+REGISTRY_SSL = os.environ.get("REGISTRY_SSL", True)
 
 POSTGRES_ENDPOINT = os.environ.get("POSTGRES_ENDPOINT", "")
+POSTGRES_HOST = os.environ.get("POSTGRES_HOST", "")
+POSTGRES_PORT = os.environ.get("POSTGRES_PORT", "")
 POSTGRES_USER = os.environ.get("POSTGRES_USER", "")
 POSTGRES_PASSWORD = os.environ.get("POSTGRES_PASSWORD", "")
 POSTGRES_DB = os.environ.get("POSTGRES_DB", "")
 
-S3_ENDPOINT = os.environ.get("S3_ENDPOINT")
-S3_ACCESS_KEY = os.environ.get("S3_ACCESS_KEY")
-S3_SECRET_KEY = os.environ.get("S3_SECRET_KEY")
-S3_BUCKET_NAME = os.environ.get("S3_BUCKET_NAME")
+S3_ENDPOINT = os.environ.get("S3_ENDPOINT", "")
+S3_ACCESS_KEY = os.environ.get("S3_ACCESS_KEY", "")
+S3_SECRET_KEY = os.environ.get("S3_SECRET_KEY", "")
+S3_BUCKET_NAME = os.environ.get("S3_BUCKET_NAME", "")

services/director/src/simcore_service_director/main.py

@@ -3,8 +3,7 @@
 import logging
 
 from aiohttp import web
-
-from simcore_service_director import config, registry_proxy
+from simcore_service_director import registry_proxy, resources
 from simcore_service_director.rest import routing
 
 log = logging.getLogger(__name__)
@@ -14,7 +13,8 @@ def main():
     registry_proxy.setup_registry_connection()
 
     # create web app and serve
-    app = routing.create_web_app(config.OPEN_API_BASE_FOLDER, config.OPEN_API_SPEC_FILE)
+    api_spec_path = resources.get_path(resources.RESOURCE_OPEN_API)
+    app = routing.create_web_app(api_spec_path.parent, api_spec_path.name)
     web.run_app(app, port=8001)
 
 if __name__ == "__main__":

services/director/src/simcore_service_director/producer.py

@@ -138,6 +138,8 @@ def __add_network_to_service_runtime_params(docker_service_runtime_parameters, d
 def __add_env_variables_to_service_runtime_params(docker_service_runtime_parameters, service_uuid):
     variables = [
         "POSTGRES_ENDPOINT=" + config.POSTGRES_ENDPOINT,
+        "POSTGRES_HOST=" + config.POSTGRES_HOST,
+        "POSTGRES_PORT=" + config.POSTGRES_PORT,
         "POSTGRES_USER=" + config.POSTGRES_USER,
         "POSTGRES_PASSWORD=" + config.POSTGRES_PASSWORD,
         "POSTGRES_DB=" + config.POSTGRES_DB,
@@ -240,6 +242,7 @@ def __wait_until_service_running_or_failed(service_id, service_name, service_uui
             if task_state == "running":
                 break
             elif task_state in ("failed", "rejected"):
+                log.error("Error while waiting for service")               
                 raise exceptions.ServiceStartTimeoutError(service_name, service_uuid)
         # TODO: all these functions should be async and here one could use await sleep which
         # would allow dealing with other events instead of wasting time here
@@ -288,7 +291,7 @@ def __prepare_runtime_parameters(docker_image_path, tag, service_uuid, docker_cl
     __add_uuid_label_to_service_runtime_params(docker_service_runtime_parameters, service_uuid)
     __add_env_variables_to_service_runtime_params(docker_service_runtime_parameters, service_uuid)
     __set_service_name(docker_service_runtime_parameters,
-        registry_proxy.get_interactive_service_sub_name(docker_image_path),
+        registry_proxy.get_service_last_names(docker_image_path),
         service_uuid)
     return docker_service_runtime_parameters
 
@@ -373,12 +376,12 @@ def start_service(service_key, service_tag, service_uuid):
 
     # create services
     __login_docker_registry(docker_client)
-    service_name = registry_proxy.get_service_name(service_key, registry_proxy.INTERACTIVE_SERVICES_PREFIX)
+    service_name = registry_proxy.get_service_first_name(service_key)
     containers_meta_data = __create_services(docker_client, list_of_images, service_name, service_tag, service_uuid)
     # we return only the info of the main service
     return containers_meta_data[0]
 
-def is_service_up(service_uuid):
+def get_service_details(service_uuid):
     # get the docker client
     docker_client = __get_docker_client()
     __login_docker_registry(docker_client)

services/director/src/simcore_service_director/registry_proxy.py

@@ -12,20 +12,20 @@
 INTERACTIVE_SERVICES_PREFIX = 'simcore/services/dynamic/'
 COMPUTATIONAL_SERVICES_PREFIX = 'simcore/services/comp/'
 _SESSION = Session()
-log = logging.getLogger(__name__)
+_logger = logging.getLogger(__name__)
 
 def setup_registry_connection():
-    log.debug("Setup registry connection started...%s", config.REGISTRY_AUTH)
+    _logger.debug("Setup registry connection started...%s", config.REGISTRY_AUTH)
 
     # get authentication state or set default value
     if config.REGISTRY_AUTH:
-        log.debug("Authentifying registry...")
+        _logger.debug("Authentifying registry...")
         if not config.REGISTRY_USER:
             raise exceptions.DirectorException("User to access to registry is not defined")
         if not config.REGISTRY_PW:
             raise exceptions.DirectorException("PW to access to registry is not defined")
         _SESSION.auth = (config.REGISTRY_USER, config.REGISTRY_PW)
-        log.debug("Session authorization complete")
+        _logger.debug("Session authorization complete")
 
 def list_computational_services():
     return __list_services(COMPUTATIONAL_SERVICES_PREFIX)
@@ -34,12 +34,12 @@ def list_interactive_services():
     return __list_services(INTERACTIVE_SERVICES_PREFIX)
 
 def get_service_details(service_key, service_version):
-    return _get_repo_version_details(service_key, service_version)
+    return __get_repo_version_details(service_key, service_version)
 
 def retrieve_list_of_images_in_repo(repository_name):
     request_result = __registry_request(repository_name + '/tags/list')
     result_json = request_result.json()
-    log.info("retrieved list of images in %s: %s",repository_name, result_json)
+    _logger.info("retrieved list of images in %s: %s",repository_name, result_json)
     return result_json
 
 def list_interactive_service_dependencies(service_key):
@@ -54,9 +54,9 @@ def list_interactive_service_dependencies(service_key):
     # ok let's get the dependencies
     dependencies = []
     repos = __retrieve_list_of_repositories()
-    service_name = get_service_name(service_key, prefix)
+    service_name = get_service_first_name(service_key)
     for repo in repos:
-        if get_service_name(repo, prefix) == service_name:
+        if get_service_first_name(repo) == service_name:
             if repo == service_key:
                 continue
             dependencies.append(repo)
@@ -67,64 +67,64 @@ def retrieve_labels_of_image(image, tag):
     result_json = request_result.json()
     labels = json.loads(result_json["history"][0]["v1Compatibility"])[
         "container_config"]["Labels"]
-    log.info("retrieved labels of image %s:%s: %s", image, tag, result_json)
+    _logger.info("retrieved labels of image %s:%s: %s", image, tag, result_json)
     return labels
 
-def get_service_name(repository_name, service_prefix):
-    service_name_suffixes = str(repository_name)[len(service_prefix):]
-    log.info("retrieved service name from repo %s : %s", repository_name, service_name_suffixes)
+def get_service_first_name(repository_name):
+    if str(repository_name).startswith(INTERACTIVE_SERVICES_PREFIX):
+        service_name_suffixes = str(repository_name)[len(INTERACTIVE_SERVICES_PREFIX):]
+    elif str(repository_name).startswith(COMPUTATIONAL_SERVICES_PREFIX):
+        service_name_suffixes = str(repository_name)[len(COMPUTATIONAL_SERVICES_PREFIX):]
+    else:
+        return "invalid service"
+
+    _logger.info("retrieved service name from repo %s : %s", repository_name, service_name_suffixes)
     return service_name_suffixes.split('/')[0]
 
-def get_interactive_service_sub_name(repository_name):
-    return __get_service_sub_name(repository_name, INTERACTIVE_SERVICES_PREFIX)
+def get_service_last_names(repository_name):
+    if str(repository_name).startswith(INTERACTIVE_SERVICES_PREFIX):
+        service_name_suffixes = str(repository_name)[len(INTERACTIVE_SERVICES_PREFIX):]
+    elif str(repository_name).startswith(COMPUTATIONAL_SERVICES_PREFIX):
+        service_name_suffixes = str(repository_name)[len(COMPUTATIONAL_SERVICES_PREFIX):]
+    else:
+        return "invalid service"
+    service_last_name = str(service_name_suffixes).replace("/", "_")
+    _logger.info("retrieved service last name from repo %s : %s", repository_name, service_last_name)
+    return service_last_name
 
 def __registry_request(path, method="GET"):
     if not config.REGISTRY_URL:
         raise exceptions.DirectorException("URL to registry is not defined")
-    # TODO: is is always ssh?
-    api_url = 'https://' + config.REGISTRY_URL + '/v2/' + path
+
+    if config.REGISTRY_SSL:
+        api_url = 'https://' + config.REGISTRY_URL + '/v2/' + path
+    else:
+        api_url = 'http://' + config.REGISTRY_URL + '/v2/' + path
 
     try:
         # r = s.get(api_url, verify=False) #getattr(s, method.lower())(api_url)
         request_result = getattr(_SESSION, method.lower())(api_url)
-        log.info("Request status: %s",request_result.status_code)
-        if request_result.status_code > 399:
+        _logger.info("Request status: %s",request_result.status_code)
+        if request_result.status_code > 399:            
             request_result.raise_for_status()
 
         return request_result
     except HTTPError as err:
-        log.exception("HTTP error returned while accessing registry")
+        _logger.exception("HTTP error returned while accessing registry")
         if err.response.status_code == 404:
             raise exceptions.ServiceNotAvailableError(path, None) from err
         raise exceptions.RegistryConnectionError("Error while accessing docker registry" ,err) from err
     except RequestException as err:
-        log.exception("Error while connecting to docker registry")
+        _logger.exception("Error while connecting to docker registry")
         raise exceptions.DirectorException(str(err)) from err
 
 def __retrieve_list_of_repositories():
     request_result = __registry_request('_catalog')
     result_json = request_result.json()['repositories']
-    log.info("retrieved list of repos: %s", result_json)
+    _logger.info("retrieved list of repos: %s", result_json)
     return result_json
 
-
-def __has_service_sub_name(repository_name, service_prefix):
-    try:
-        __get_service_sub_name(repository_name, service_prefix)
-        return True
-    except exceptions.ServiceNotAvailableError:
-        return False
-
-def __get_service_sub_name(repository_name, service_prefix):
-    service_name_suffixes = str(repository_name)[len(service_prefix):]
-    list_of_suffixes = service_name_suffixes.split('/')
-    last_suffix_index = len(list_of_suffixes) - 1
-    if last_suffix_index < 0:
-        raise exceptions.ServiceNotAvailableError(repository_name, None)
-    log.info("retrieved service sub name from repo %s : %s", repository_name, list_of_suffixes)
-    return list_of_suffixes[last_suffix_index]
-
-def _get_repo_version_details(repo_key, repo_tag):
+def __get_repo_version_details(repo_key, repo_tag):
     image_tags = {}
     label_request = __registry_request(repo_key + '/manifests/' + repo_tag)
     label_data = label_request.json()
@@ -137,31 +137,32 @@ def _get_repo_version_details(repo_key, repo_tag):
                     image_tags[label_key] = label_data[label_key]
     return image_tags
 
-def _get_repo_details(repo):
+def __get_repo_details(repo):
     #pylint: disable=too-many-nested-blocks
     current_repo = []
     if "/comp/" in repo or "/dynamic/" in repo:
         # get list of repo versions
         req_images = __registry_request(repo + '/tags/list')
         im_data = req_images.json()
         tags = im_data['tags']
-        for tag in tags:
-            image_tags = _get_repo_version_details(repo, tag)
-            if image_tags:
-                current_repo.append(image_tags)
+        if tags:
+            for tag in tags:
+                image_tags = __get_repo_version_details(repo, tag)
+                if image_tags:
+                    current_repo.append(image_tags)
 
     return current_repo
 
 def __list_services(service_prefix):
-    log.info("getting list of computational services")
+    _logger.info("getting list of computational services")
     list_all_repos = __retrieve_list_of_repositories()
     # get the services repos
     list_of_specific_repos = [repo for repo in list_all_repos if str(repo).startswith(service_prefix)]
-    log.info("retrieved list of computational repos : %s", list_of_specific_repos)
+    _logger.info("retrieved list of computational repos : %s", list_of_specific_repos)
     repositories = []
     # or each repo get all tags details
     for repo in list_of_specific_repos:
-        details = _get_repo_details(repo)
+        details = __get_repo_details(repo)
         for repo_detail in details:
             repositories.append(repo_detail)
 

services/director/src/simcore_service_director/resources.py

@@ -0,0 +1,35 @@
+import functools
+from pathlib import Path
+
+import pkg_resources
+from simcore_service_director import config
+
+RESOURCE_OPENAPI_ROOT = "oas3"
+RESOURCE_OPEN_API = "{root}/{version}/openapi.yaml".format(root=RESOURCE_OPENAPI_ROOT, version=config.API_VERSION)
+RESOURCE_NODE_SCHEMA = "{root}/{version}/schemas/node-meta-v0.0.1.json".format(root=RESOURCE_OPENAPI_ROOT, version=config.API_VERSION)
+
+"""
+ List of pkg_resources functions *bound* to current package with the following signature
+
+   function(resource_name)
+
+ Note that resource names must be /-separated paths and
+ cannot be absolute (i.e. no leading /) or contain relative names like "..".
+ Do not use os.path routines to manipulate resource paths, as they are not filesystem paths.
+
+ Resources are read/only files/folders
+"""
+exists  = functools.partial(pkg_resources.resource_exists, __name__)
+stream  = functools.partial(pkg_resources.resource_stream, __name__)
+listdir = functools.partial(pkg_resources.resource_listdir, __name__)
+isdir = functools.partial(pkg_resources.resource_isdir, __name__)
+
+
+def get_path(resource_name):
+    """ Returns a path to a resource
+
+        WARNING: existence of file is not guaranteed. Use resources.exists
+        WARNING: resource files are supposed to be used as read-only!
+    """
+    resource_path = Path( pkg_resources.resource_filename(__name__, resource_name) )
+    return resource_path