WIP: Registry api (#115)

* Add rest endpoint to get list of available computational services

Author: mguidon

Makefile

@@ -36,6 +36,14 @@ down-swarm:
 	docker-compose -f services/docker-compose.yml -f services/docker-compose.deploy.yml down
 	docker swarm leave -f
 
+stack-up:
+	docker swarm init
+	POSTGRES_USER=simcore POSTGRES_PASSWORD=simcore POSTGRES_DB=simcoredb RABBITMQ_USER=simcore RABBITMQ_PASSWORD=simcore RABBITMQ_PROGRESS_CHANNEL=comp.backend.channels.progress RABBITMQ_LOG_CHANNEL=comp.backend.channels.log S3_ENDPOINT=minio:9000 S3_ACCESS_KEY=12345678  S3_SECRET_KEY=12345678 S3_BUCKET_NAME=simcore docker stack deploy -c services/docker-compose.yml -c services/docker-compose.deploy.yml osparc
+
+stack-down:
+	docker stack rm osparc
+	docker swarm leave -f
+
 pylint:
 	# See exit codes and command line https://pylint.readthedocs.io/en/latest/user_guide/run.html#exit-codes
 	/bin/bash -c "pylint --rcfile=.pylintrc $(PY_FILES)"

packages/s3wrapper/src/s3wrapper/s3_client.py

@@ -130,6 +130,14 @@ def list_objects(self, bucket_name, recursive=False):
 
         return []
 
+    def list_objects_v2(self, bucket_name, recursive=False):
+        try:
+            return self.client.list_objects_v2(bucket_name, recursive=recursive)
+        except ResponseError as _err:
+            logging.exception("Could not list objects")
+
+        return []
+
     def remove_objects(self, bucket_name, objects):
         try:
             for del_err in self.client.remove_objects(bucket_name, objects):

packages/simcore-sdk/src/simcore_sdk/config/docker.py

@@ -7,15 +7,19 @@ def __init__(self):
         self._registry = "masu.speag.com"
         self._user = "z43"
         self._pwd = "z43"
-    
+
     @property
     def registry(self):
         return self._registry + "/v2"
-    
+
+    @property
+    def registry_name(self):
+        return self._registry
+
     @property
     def user(self):
         return self._user
 
     @property
     def pwd(self):
-        return self._pwd
+        return self._pwd

packages/simcore-sdk/src/simcore_sdk/config/s3.py

@@ -3,30 +3,30 @@
 """
 from os import environ as env
 
-S3_ENDPOINT = env.get("S3_ENDPOINT", "")
-S3_ACCESS_KEY = env.get("S3_ACCESS_KEY", "")
-S3_SECRET_KEY = env.get("S3_SECRET_KEY", "")
-S3_BUCKET_NAME = env.get("S3_BUCKET_NAME", "")
+S3_ENDPOINT = env.get("S3_ENDPOINT", "minio:9000")
+S3_ACCESS_KEY = env.get("S3_ACCESS_KEY", "12345678")
+S3_SECRET_KEY = env.get("S3_SECRET_KEY", "12345678")
+S3_BUCKET_NAME = env.get("S3_BUCKET_NAME", "simcore")
 
 class Config():
     def __init__(self):
         self._endpoint = S3_ENDPOINT
         self._access_key = S3_ACCESS_KEY
         self._secret_key = S3_SECRET_KEY
         self._bucket_name = S3_BUCKET_NAME
-    
+
     @property
     def endpoint(self):
         return self._endpoint
-    
+
     @property
     def access_key(self):
         return self._access_key
-    
+
     @property
     def secret_key(self):
         return self._secret_key
 
     @property
     def bucket_name(self):
-        return self._bucket_name
+        return self._bucket_name

services/director/source/director.py

@@ -102,6 +102,11 @@ def stop_service():
         _LOGGER.exception("Failed to stop service")
         abort(500)
 
+@APP.route('/list_repositories', methods=['GET'])
+def list_repositories():
+    repos = registry_proxy.get_repo_details()
+
+    return json.dumps(repos)
 
 if __name__ == "__main__":
     APP.run(host='0.0.0.0', debug=False, port=8001, threaded=True)

services/director/source/registry_proxy.py

@@ -86,3 +86,41 @@ def get_service_sub_name(repository_name):
     if last_suffix_index < 0:
         raise Exception('Invalid service name: ' + repository_name)
     return list_of_suffixes[last_suffix_index]
+
+def _get_repo_details(repo):
+    #pylint: disable=too-many-nested-blocks
+    current_repo = []
+    if "/comp/" in repo:
+        req_images = registry_request(repo + '/tags/list')
+        im_data = req_images.json()
+        tags = im_data['tags']
+        for tag in tags:
+            image_tags = {}
+            label_request = registry_request(repo + '/manifests/' + tag)
+            label_data = label_request.json()
+            labels = json.loads(label_data["history"][0]["v1Compatibility"])["container_config"]["Labels"]
+            if labels:
+                for key in labels.keys():
+                    if key.startswith("io.simcore."):
+                        label_data = json.loads(labels[key])
+                        for label_key in label_data.keys():
+                            image_tags[label_key] = label_data[label_key]
+            if image_tags:
+                current_repo.append(image_tags)
+
+    return current_repo
+
+def get_repo_details():
+    request_result = registry_request('_catalog')
+
+    repos = request_result.json()['repositories']
+    repositories = {}
+    for repo in repos:
+        details = _get_repo_details(repo)
+        if details:
+            repositories[repo] = details
+
+    result_json = json.dumps(repositories)
+
+
+    return result_json

services/docker-compose.deploy.yml

@@ -23,3 +23,5 @@ services:
         constraints:
           - node.platform.os == linux
           - node.role == manager
+  sidecar:
+    image: services_sidecar:latest

services/docker-compose.devel.yml

@@ -22,7 +22,7 @@ services:
     volumes:
       - ./web/server:/home/scu/server
       - ./web/client/source-output:/home/scu/client
-      - ../packages/simcore-sdk/src:/home/scu/packages
+      - ../packages:/home/scu/packages
     depends_on:
       - webclient
   #--------------------------------------------------------------------

services/sidecar/src/sidecar/sidecar.py

@@ -59,6 +59,7 @@ def _create_shared_folders(self):
     def _process_task_input(self, port, input_ports):
         port_name = port['key']
         port_value = port['value']
+        _LOGGER.debug("PROCESSING %s %s", port_name, port_value)
         _LOGGER.debug(type(port_value))
         if isinstance(port_value, str) and port_value.startswith("link."):
             if port['type'] == 'file-url':
@@ -91,6 +92,9 @@ def _process_task_input(self, port, input_ports):
                     for oport in other_task.output:
                         if oport['key'] == other_output_port_id:
                             input_ports[port_name] = oport['value']
+        else:
+            _LOGGER.debug('Non link data %s : %s', port_name, port_value)
+            input_ports[port_name] = port_value
 
     def _process_task_inputs(self):
         """ Writes input key-value pairs into a dictionary
@@ -110,16 +114,25 @@ def _process_task_inputs(self):
             _LOGGER.debug(port)
             self._process_task_input(port, input_ports)
 
+        _LOGGER.debug('DUMPING json')
         #dump json file
         if input_ports:
             file_name = os.path.join(self._executor.in_dir, 'input.json')
             with open(file_name, 'w') as f:
                 json.dump(input_ports, f)
 
+        _LOGGER.debug('DUMPING DONE')
+
     def _pull_image(self):
+        _LOGGER.debug('PULLING IMAGE')
+        _LOGGER.debug('reg %s user %s pwd %s', self._docker.registry, self._docker.registry_user,self._docker.registry_pwd )
+
+
         self._docker.client.login(registry=self._docker.registry,
             username=self._docker.registry_user, password=self._docker.registry_pwd)
 
+        _LOGGER.debug('img %s tag %s', self._docker.image_name, self._docker.image_tag)
+
         self._docker.client.images.pull(self._docker.image_name, tag=self._docker.image_tag)
 
     def _bg_job(self, task, log_file):
@@ -213,7 +226,7 @@ def _process_task_log(self):
 
     def initialize(self, task):
         self._task = task
-        self._docker.image_name = task.image['name']
+        self._docker.image_name = self._docker.registry_name + "/" + task.image['name']
         self._docker.image_tag = task.image['tag']
         self._executor.in_dir = os.path.join("/", "input", task.job_id)
         self._executor.out_dir = os.path.join("/", "output", task.job_id)

services/sidecar/src/sidecar/sidecar_utils.py

@@ -38,6 +38,7 @@ def __init__(self):
         self._config = docker_config()
         self.client = docker.from_env(version='auto')
         self.registry = self._config.registry
+        self.registry_name = self._config.registry_name
         self.registry_user = self._config.user
         self.registry_pwd = self._config.pwd
         self.image_name = ""

services/web/Dockerfile

@@ -41,8 +41,7 @@ RUN apk add --no-cache \
 
 ENV HOME /home/scu
 ENV SIMCORE_WEB_OUTDIR $HOME/client
-ENV PYTHONPATH "$HOME/server/src:$HOME/packages"
-
+ENV PYTHONPATH "$HOME/server/src:$HOME/packages/simcore-sdk/src:$HOME/packages/s3wrapper/src"
 
 WORKDIR /home/scu
 
@@ -91,7 +90,7 @@ RUN pip3 install --no-cache-dir -r requirements/prod.txt && \
     rm -rf requirements
 
 # 2. install 2nd party packages
-COPY --chown=scu:scu packages/simcore-sdk/src packages
+COPY --chown=scu:scu packages packages
 
 # 3. install client
 COPY --from=services_webclient:build --chown=scu:scu /home/scu/client/build-output client

services/web/client/source/class/qxapp/components/form/Auto.js

@@ -45,6 +45,8 @@
  *
  */
 
+/* eslint no-warning-comments: "off" */
+
 qx.Class.define("qxapp.components.form.Auto", {
   extend : qx.ui.form.Form,
   include : [qx.locale.MTranslation],
@@ -366,6 +368,10 @@ qx.Class.define("qxapp.components.form.Auto", {
       );
     },
     __addField: function(s) {
+      // FIXME: OM why null?
+      if (s === null) {
+        return;
+      }
       let option = {
         exposable: s.exposable
       }; // for passing info into the form renderer
@@ -384,12 +390,21 @@ qx.Class.define("qxapp.components.form.Auto", {
         }
         s.set.value = s.defaultValue;
       }
+      // FIXME: This should go away
+      if (s.value) {
+        if (!s.set) {
+          s.set = {};
+        }
+        s.set.value = s.value;
+      }
       if (!s.widget) {
         s.widget = {
           string: "text",
           integer: "spinner",
+          number: "spinner",
           bool: "checkBox",
-          fileUrl: "fileButton"
+          "file-url": "fileButton",
+          "folder-url": "fileButton"
         }[s.type];
       }
       let control;
@@ -405,6 +420,10 @@ qx.Class.define("qxapp.components.form.Auto", {
           break;
         case "spinner":
           control = new qx.ui.form.Spinner();
+          control.set({
+            maximum: 10000,
+            minimum: -10000
+          });
           setup = this.__setupSpinner;
           break;
         case "password":

services/web/client/source/class/qxapp/components/workbench/NodeBase.js

@@ -238,7 +238,7 @@ qx.Class.define("qxapp.components.workbench.NodeBase", {
       label.isInput = isInput;
       label.portType = portData.type;
 
-      label.ui = new qx.ui.basic.Label(portData.label).set({
+      label.ui = new qx.ui.basic.Label(portData.key).set({
         height: 16,
         draggable: true,
         droppable: true

services/web/client/source/class/qxapp/components/workbench/Workbench.js

@@ -208,7 +208,7 @@ qx.Class.define("qxapp.components.workbench.Workbench", {
             var node = d["Node"];
             var msg = d["Message"];
             this.__updateLogger(node, msg);
-          });
+          }, this);
         }
         socket.emit("logger");
 
@@ -220,7 +220,7 @@ qx.Class.define("qxapp.components.workbench.Workbench", {
             var node = d["Node"];
             var progress = 100*Number.parseFloat(d["Progress"]).toFixed(4);
             this.updateProgress(node, progress);
-          });
+          }, this);
         }
 
         if (this.getCanStart()) {
@@ -811,9 +811,10 @@ qx.Class.define("qxapp.components.workbench.Workbench", {
       // post pipeline
       let currentPipeline = this.__serializeData();
       console.log(currentPipeline);
-      var req = new qx.io.request.Xhr();
-      var data = {};
-      data["pipeline_mockup_id"] = currentPipeline;
+      let req = new qx.io.request.Xhr();
+      let data = {};
+      data = currentPipeline;
+      data["pipeline_mockup_id"] = qxapp.utils.Utils.uuidv4();
       req.set({
         url: "/start_pipeline",
         method: "POST",
@@ -832,7 +833,7 @@ qx.Class.define("qxapp.components.workbench.Workbench", {
     },
 
     __onPipelinesubmitted: function(e) {
-      var req = e.getTarget();
+      let req = e.getTarget();
       console.debug("Everything went fine!!");
       console.debug("status  : ", req.getStatus());
       console.debug("phase   : ", req.getPhase());