Feature/uploader (#112)

* smaller arrow

* More separation node-link

* Remove colon from logger message sender

* Upload Manager added

* minio added to server

* File Manager added to client. File upload working

* UploadMgr not needed

* Fill tree with data coming from s3

* File Manager selects file and filePath is written in output

* Move hardcoded stuff to data/Fake

* Update server to list objects in bucket

* retrieve_url_for_file works

* get objects list works

* Progress bar added to file upload

* S3 configuration variables moved to source/config.py

Author: odeimaiz

services/web/client/source/class/qxapp/components/workbench/SvgWidget.js

@@ -37,7 +37,7 @@ qx.Class.define("qxapp.components.workbench.SvgWidget", {
     __linksCanvas: null,
 
     __getControls: function(x1, y1, x2, y2) {
-      const offset = 50;
+      const offset = 60;
       return [{
         x: x1,
         y: y1

services/web/client/source/class/qxapp/components/workbench/Workbench.js

@@ -327,7 +327,35 @@ qx.Class.define("qxapp.components.workbench.Workbench", {
       }, this);
 
       node.addListener("dblclick", function(e) {
-        this.fireDataEvent("NodeDoubleClicked", node);
+        if (node.getMetadata().key === "FileManager") {
+          let win = new qx.ui.window.Window(node.getMetadata().name).set({
+            width: 800,
+            height: 600,
+            minWidth: 400,
+            minHeight: 400,
+            modal: true,
+            showMinimize: false,
+            layout: new qx.ui.layout.Canvas()
+          });
+          win.moveTo(50, 50);
+
+          let fileManager = new qxapp.components.workbench.widgets.FileManager();
+          win.add(fileManager, {
+            left: 0,
+            top: 0,
+            right: 0,
+            bottom: 0
+          });
+          fileManager.addListener("FileSelected", function(data) {
+            node.getMetadata().outputs[0].value = data.getData().filePath;
+            node.setProgress(100);
+            win.close();
+          }, this);
+
+          this.addWindowToDesktop(win);
+        } else {
+          this.fireDataEvent("NodeDoubleClicked", node);
+        }
         e.stopPropagation();
       }, this);
 

services/web/client/source/class/qxapp/components/workbench/logger/LoggerView.js

@@ -185,7 +185,7 @@ qx.Class.define("qxapp.components.workbench.logger.LoggerView", {
         this.__messengerColors.add([who, whoColor]);
       }
 
-      return ("<font color=" + whoColor +">" + who + ": </font>");
+      return ("<font color=" + whoColor +">" + who + "</font>");
     },
 
     __addLevelColorTag: function(what, logLevel) {

services/web/client/source/class/qxapp/components/workbench/widgets/FileManager.js

@@ -0,0 +1,206 @@
+/* global XMLHttpRequest */
+qx.Class.define("qxapp.components.workbench.widgets.FileManager", {
+  extend: qx.ui.container.Composite,
+
+  construct: function() {
+    this.base(arguments, new qx.ui.layout.VBox(10));
+
+
+    // Create a button
+    let input = new qx.html.Input("file", {
+      display: "none"
+    }, {
+      multiple: true,
+      accept: "image/*"
+    });
+
+    this.getContentElement().add(input);
+
+    let pick = new qx.ui.form.Button(this.tr("Add file"));
+    this.add(pick);
+
+    // Add an event listener
+    pick.addListener("execute", function(e) {
+      input.getDomElement().click();
+    });
+
+    input.addListener("change", function(e) {
+      let files = input.getDomElement().files;
+      for (let i=0; i<files.length; i++) {
+        this.__retrieveURLAndUpload(files[i]);
+      }
+    }, this);
+
+    let tree = this.__mainTree = new qx.ui.tree.Tree();
+    this.add(tree, {
+      flex: 1
+    });
+    tree.addListener("changeSelection", this.__selectionChanged, this);
+
+    this.__selectFileBtn = new qx.ui.form.Button(this.tr("Select File"));
+    this.__selectFileBtn.setEnabled(false);
+    this.add(this.__selectFileBtn);
+    this.__selectFileBtn.addListener("execute", function() {
+      this.__fileSelected();
+    }, this);
+
+    this.__reloadTree();
+  },
+
+  events: {
+    "FileSelected": "qx.event.type.Data"
+  },
+
+  members: {
+    __mainTree: null,
+    __publicTree: null,
+    __userTree: null,
+    __selectFileBtn: null,
+
+    __reloadTree: function() {
+      this.__mainTree.resetRoot();
+
+      let root = this.__configureTreeItem(new qx.ui.tree.TreeFolder(), this.tr("Available Files"));
+      root.setOpen(true);
+      this.__mainTree.setRoot(root);
+
+      let tree1 = this.__publicTree = this.__configureTreeItem(new qx.ui.tree.TreeFolder(), this.tr("Public Files"));
+      tree1.setOpen(true);
+      root.add(tree1);
+
+      let tree2 = this.__userTree = this.__configureTreeItem(new qx.ui.tree.TreeFolder(), this.tr("User Files"));
+      tree2.setOpen(true);
+      root.add(tree2);
+
+      const username = qxapp.data.Fake.getUsername();
+      this.__getObjLists(username);
+    },
+
+    __getObjLists: function(bucketName) {
+      let socket = qxapp.wrappers.WebSocket.getInstance();
+
+      socket.removeSlot("listObjectsPub");
+      socket.on("listObjectsPub", function(data) {
+        let treeItem = this.__addTreeItem(this.__publicTree, data);
+        const publicBucket = qxapp.data.Fake.getS3PublicBucketName();
+        treeItem.path = publicBucket + "/" + data.name;
+      }, this);
+
+      socket.removeSlot("listObjectsUser");
+      socket.on("listObjectsUser", function(data) {
+        let treeItem = this.__addTreeItem(this.__userTree, data);
+        const username = qxapp.data.Fake.getUsername();
+        treeItem.path = username + "/" + data.name;
+      }, this);
+
+      socket.emit("listObjects", bucketName);
+    },
+
+    // Request to the server an upload URL.
+    __retrieveURLAndUpload: function(file) {
+      let socket = qxapp.wrappers.WebSocket.getInstance();
+
+      socket.removeSlot("presignedUrl");
+      socket.on("presignedUrl", function(data) {
+        const url = data["url"];
+        this.__uploadFile(file, url);
+      }, this);
+      const data = {
+        bucketName: qxapp.data.Fake.getUsername(),
+        fileName: file.name
+      };
+      socket.emit("presignedUrl", data);
+    },
+
+    // Use XMLHttpRequest to upload the file to S3.
+    __uploadFile: function(file, url) {
+      let hBox = new qx.ui.container.Composite(new qx.ui.layout.HBox());
+      let label = new qx.ui.basic.Label(file.name);
+      let progressBar = new qx.ui.indicator.ProgressBar();
+      hBox.add(label, {
+        width: "15%"
+      });
+      hBox.add(progressBar, {
+        width: "85%"
+      });
+      this.addAt(hBox, 1);
+      let xhr = new XMLHttpRequest();
+      xhr.upload.addEventListener("progress", function(e) {
+        if (e.lengthComputable) {
+          const percentComplete = e.loaded / e.total * 100;
+          progressBar.setValue(percentComplete);
+        } else {
+          console.log("Unable to compute progress information since the total size is unknown");
+        }
+      }, false);
+      xhr.open("PUT", url, true);
+      xhr.send(file);
+      xhr.onload = () => {
+        if (xhr.status == 200) {
+          console.log("Uploaded", file.name);
+          this.remove(hBox);
+          this.__reloadTree();
+        }
+      };
+    },
+
+    __selectionChanged: function() {
+      let selectedItem = this.__mainTree.getSelection();
+      this.__selectFileBtn.setEnabled("path" in selectedItem[0]);
+    },
+
+    __fileSelected: function() {
+      let selectedItem = this.__mainTree.getSelection();
+      if ("path" in selectedItem[0]) {
+        const data = {
+          filePath: selectedItem[0].path
+        };
+        this.fireDataEvent("FileSelected", data);
+      }
+    },
+
+    __addTreeItem: function(tree, data) {
+      let treeItem = this.__configureTreeItem(new qx.ui.tree.TreeFile(), data.name, data);
+      tree.add(treeItem);
+
+      return treeItem;
+    },
+
+    __configureTreeItem: function(treeItem, label, extraInfo) {
+      // A left-justified icon
+      treeItem.addWidget(new qx.ui.core.Spacer(16, 16));
+
+      // Here's our indentation and tree-lines
+      treeItem.addSpacer();
+
+      if (treeItem instanceof qx.ui.tree.TreeFolder) {
+        treeItem.addOpenButton();
+      }
+
+      // The standard tree icon follows
+      treeItem.addIcon();
+
+      // The label
+      treeItem.addLabel(label);
+
+      // All else should be right justified
+      treeItem.addWidget(new qx.ui.core.Spacer(), {
+        flex: 1
+      });
+
+      if (treeItem instanceof qx.ui.tree.TreeFile) {
+        // Add a file size, date and mode
+        const formattedSize = qxapp.utils.Utils.formatBytes(extraInfo.size);
+        let text = new qx.ui.basic.Label(formattedSize);
+        text.setWidth(80);
+        treeItem.addWidget(text);
+
+        text = new qx.ui.basic.Label((new Date(extraInfo.lastModified)).toUTCString());
+        text.setWidth(200);
+        treeItem.addWidget(text);
+      }
+
+      return treeItem;
+    }
+  }
+});

services/web/client/source/class/qxapp/data/Fake.js

@@ -22,6 +22,14 @@ qx.Class.define("qxapp.data.Fake", {
       prjId: null
     }),
 
+    getUsername: function() {
+      return "bizzy";
+    },
+
+    getS3PublicBucketName: function() {
+      return "simcore";
+    },
+
     /**
      * Returns a qx array with projects associated to a user
      */
@@ -614,23 +622,11 @@ qx.Class.define("qxapp.data.Fake", {
           "port": null
         }
       }, {
-        "key": "FileReader",
+        "key": "FileManager",
         "tag": "1.0",
-        "name": "File Reader",
-        "desc": "File Reader",
-        "inputs": [{
-          "key": "in_1",
-          "label": "File",
-          "desc": "File",
-          "type": "string",
-          "defaultValue": null
-        }, {
-          "key": "in_2",
-          "label": "File type",
-          "desc": "File type",
-          "type": "string",
-          "defaultValue": ""
-        }],
+        "name": "File Manager",
+        "desc": "File Manager",
+        "inputs": [],
         "outputs": [{
           "key": "out_1",
           "label": "File-url",

services/web/client/source/class/qxapp/utils/Utils.js

@@ -34,6 +34,18 @@ qx.Class.define("qxapp.utils.Utils", {
 
     getKeyByValue(object, value) {
       return Object.keys(object).find(key => object[key] === value);
+    },
+
+    formatBytes: function(bytes) {
+      const precision = 2;
+      if (bytes < 1024) {
+        return bytes + " B";
+      } else if (bytes < (1024*1024)) {
+        return (bytes / 1024).toFixed(precision) + " KB";
+      } else if (bytes < (1024*1024*1024)) {
+        return (bytes / (1024*1024)).toFixed(precision) + " MB";
+      }
+      return (bytes / (1024*1024*1024)).toFixed(precision) + " GB";
     }
   }
 });

services/web/client/source/class/qxapp/wrappers/SvgWrapper.js

@@ -72,7 +72,7 @@ qx.Class.define("qxapp.wrappers.SvgWrapper", {
       });
       path.marker("start", marker1);
 
-      const arrowSize = 5;
+      const arrowSize = 4;
       let marker2 = draw.marker(arrowSize, arrowSize, function(add) {
         add.path("M 0 0 V 4 L 2 2 Z")
           .fill(linkColor)

services/web/client/source/class/qxapp/wrappers/WebSocket.js

@@ -6,8 +6,6 @@
 
 /* global window */
 /* global io */
-/* eslint valid-jsdoc: "error" */
-/* eslint-env es6 */
 
 qx.Class.define("qxapp.wrappers.WebSocket", {
   extend: qx.core.Object,
@@ -212,6 +210,14 @@ qx.Class.define("qxapp.wrappers.WebSocket", {
         }
       }
       return false;
+    },
+
+    removeSlot: function(name) {
+      var index = this.__name.indexOf(name);
+      if (index > -1) {
+        this.getSocket().removeAllListeners(this.__name[index]);
+        this.__name.splice(index, 1);
+      }
     }
   },
 

services/web/server/requirements/common.txt

@@ -7,3 +7,4 @@ psycopg2==2.7.4
 python-socketio==1.9.0
 requests==2.19.0
 sqlalchemy==1.2.8
+minio==4.0.1