Skip to content

Commit 401938e

Browse files
odeimaizpcrespov
odeimaiz
authored and
pcrespov
committed
Multi plot Dashboard (#621)
- Creates a container-node-widget able to display the image outputs of its inner services in a grid-dashboard layout. - When double-clicking a cell, the edition mode of its graph will be opened. - Improves the raw-graphs service: - More input ports (5). - One output port. Every time the graph in the frontend is modified it is saved and also pushed to S3 when sync function (input-retriever) is called. - closes #630 - Added a diagram that shows the frontend UI workflow - child of #4
1 parent ef9e8ca commit 401938e

31 files changed

+1403
-297
lines changed

‎services/dy-raw-graphs/Dockerfile

+1
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ ENV SIMCORE_NODE_UUID="-1" \
4141
SIMCORE_NODE_BASEPATH="/raw" \
4242
STORAGE_ENDPOINT="=1" \
4343
RAWGRAPHS_INPUT_PATH="../inputs" \
44+
RAWGRAPHS_OUTPUT_PATH="../outputs" \
4445
S3_ENDPOINT="=1" \
4546
S3_ACCESS_KEY="-1" \
4647
S3_SECRET_KEY="-1" \

‎services/dy-raw-graphs/docker-compose.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ services:
1313
io.simcore.description: '{"description": "2D plots powered by RAW Graphs"}'
1414
io.simcore.authors: '{"authors": [{"name": "odeimaiz", "email": "[email protected]", "affiliation": "ITIS Foundation"}]}'
1515
io.simcore.contact: '{"contact": "[email protected]"}'
16-
io.simcore.inputs: '{"inputs": {"input_1": {"label": "input 1", "displayOrder": 0, "description": "Input 1", "type": "data:*/*"}}, {"input_2": {"label": "input 2", "displayOrder": 1, "description": "Input 2", "type": "data:*/*"}}, {"input_3": {"label": "input 3", "displayOrder": 2, "description": "Input 3", "type": "data:*/*"}}, {"input_4": {"label": "input 4", "displayOrder": 3, "description": "Input 4", "type": "data:*/*"}}, {"input_5": {"label": "input 5", "displayOrder": 4, "description": "Input 5", "type": "data:*/*"}}}'
16+
io.simcore.inputs: '{"inputs": {"input_1": {"label": "input 1", "displayOrder": 0, "description": "Input 1", "type": "data:*/*"}, "input_2": {"label": "input 2", "displayOrder": 1, "description": "Input 2", "type": "data:*/*"}, "input_3": {"label": "input 3", "displayOrder": 2, "description": "Input 3", "type": "data:*/*"}, "input_4": {"label": "input 4", "displayOrder": 3, "description": "Input 4", "type": "data:*/*"}, "input_5": {"label": "input 5", "displayOrder": 4, "description": "Input 5", "type": "data:*/*"}}}'
1717
io.simcore.outputs: '{"outputs": {"output_1":{"label": "Output Graph", "displayOrder":0, "description": "Output Graph", "type": "data:image/svg+xml"}}}'
1818
simcore.service.settings: '[{"name": "resources", "type": "Resources", "value": {"mem_limit":17179869184, "cpu_limit": 4000000000}}, {"name": "ports", "type": "int", "value": 4000}, {"name": "constraints", "type": "string", "value": ["node.platform.os == linux"]}]'
1919
org.label-schema.schema-version: "1.0"

‎services/dy-raw-graphs/server/input-retriever.py

+108-41
Original file line numberDiff line numberDiff line change
@@ -1,59 +1,126 @@
11
import asyncio
2-
32
import logging
43
import os
54
import shutil
6-
import time
5+
import sys
6+
import tarfile
7+
import tempfile
78
import zipfile
89
from pathlib import Path
910

1011
from simcore_sdk import node_ports
1112

12-
log = logging.getLogger(__name__)
13+
logger = logging.getLogger(__name__)
1314

14-
_INPUT_PATH = Path(os.environ.get("RAWGRAPHS_INPUT_PATH"))
15+
_INPUTS_FOLDER = Path(os.environ.get("RAWGRAPHS_INPUT_PATH"))
16+
_OUTPUTS_FOLDER = Path(os.environ.get("RAWGRAPHS_OUTPUT_PATH"))
17+
_FILE_TYPE_PREFIX = "data:"
18+
_KEY_VALUE_FILE_NAME = "key_values.json"
1519

1620
# clean the directory
17-
shutil.rmtree(str(_INPUT_PATH), ignore_errors=True)
21+
shutil.rmtree(str(_INPUTS_FOLDER), ignore_errors=True)
22+
23+
if not _INPUTS_FOLDER.exists():
24+
_INPUTS_FOLDER.mkdir()
25+
logger.debug("Created input folder at %s", _INPUTS_FOLDER)
26+
27+
if not _OUTPUTS_FOLDER.exists():
28+
_OUTPUTS_FOLDER.mkdir()
29+
logger.debug("Created output folder at %s", _OUTPUTS_FOLDER)
30+
31+
def _no_relative_path_tar(members: tarfile.TarFile):
32+
for tarinfo in members:
33+
path = Path(tarinfo.name)
34+
if path.is_absolute():
35+
# absolute path are not allowed
36+
continue
37+
if path.match("/../"):
38+
# relative paths are not allowed
39+
continue
40+
yield tarinfo
1841

19-
if not _INPUT_PATH.exists():
20-
_INPUT_PATH.mkdir()
21-
log.debug("Created input folder at %s", _INPUT_PATH)
42+
def _no_relative_path_zip(members: zipfile.ZipFile):
43+
for zipinfo in members.infolist():
44+
path = Path(zipinfo.filename)
45+
if path.is_absolute():
46+
# absolute path are not allowed
47+
continue
48+
if path.match("/../"):
49+
# relative paths are not allowed
50+
continue
51+
yield zipinfo
2252

23-
async def retrieve_data():
24-
log.debug("retrieving data...")
25-
print("retrieving data...")
53+
async def download_data():
54+
logger.info("retrieving data from simcore...")
55+
print("retrieving data from simcore...")
2656

2757
# get all files in the local system and copy them to the input folder
28-
start_time = time.time()
2958
PORTS = node_ports.ports()
30-
download_tasks = []
31-
for node_input in PORTS.inputs:
32-
if not node_input or node_input.value is None:
59+
for port in PORTS.inputs:
60+
if not port or port.value is None:
3361
continue
34-
35-
# collect coroutines
36-
download_tasks.append(node_input.get())
37-
if download_tasks:
38-
downloaded_files = await asyncio.gather(*download_tasks)
39-
print("downloaded {} files /tmp <br>".format(len(download_tasks)))
40-
for local_path in downloaded_files:
41-
if local_path is None:
42-
continue
43-
# log.debug("Completed download of %s in local path %s", node_input.value, local_path)
44-
if local_path.exists():
45-
if zipfile.is_zipfile(str(local_path)):
46-
zip_ref = zipfile.ZipFile(str(local_path), 'r')
47-
zip_ref.extractall(str(_INPUT_PATH))
48-
zip_ref.close()
49-
log.debug("Unzipped")
50-
print("unzipped {file} to {path}<br>".format(file=str(local_path), path=str(_INPUT_PATH)))
51-
else:
52-
log.debug("Start moving %s to input path %s", local_path, _INPUT_PATH)
53-
shutil.move(str(local_path), str(_INPUT_PATH / local_path.name))
54-
log.debug("Move completed")
55-
print("moved {file} to {path}<br>".format(file=str(local_path), path=str(_INPUT_PATH)))
56-
end_time = time.time()
57-
print("time to download: {} seconds".format(end_time - start_time))
58-
59-
asyncio.get_event_loop().run_until_complete(retrieve_data())
62+
63+
local_path = await port.get()
64+
dest_path = _INPUTS_FOLDER / port.key
65+
dest_path.mkdir(exist_ok=True, parents=True)
66+
67+
# clean up destination directory
68+
for path in dest_path.iterdir():
69+
if path.is_file():
70+
path.unlink()
71+
elif path.is_dir():
72+
shutil.rmtree(path)
73+
# check if local_path is a compressed file
74+
if tarfile.is_tarfile(local_path):
75+
with tarfile.open(local_path) as tar_file:
76+
tar_file.extractall(dest_path, members=_no_relative_path_tar(tar_file))
77+
elif zipfile.is_zipfile(local_path):
78+
with zipfile.ZipFile(local_path) as zip_file:
79+
zip_file.extractall(dest_path, members=_no_relative_path_zip(zip_file))
80+
else:
81+
dest_path_name = _INPUTS_FOLDER / (port.key + ":" + Path(local_path).name)
82+
shutil.move(local_path, dest_path_name)
83+
shutil.rmtree(Path(local_path).parents[0])
84+
85+
async def upload_data():
86+
logger.info("uploading data to simcore...")
87+
PORTS = node_ports.ports()
88+
outputs_path = Path(_OUTPUTS_FOLDER).expanduser()
89+
for port in PORTS.outputs:
90+
logger.debug("uploading data to port '%s' with value '%s'...", port.key, port.value)
91+
src_folder = outputs_path / port.key
92+
list_files = list(src_folder.glob("*"))
93+
if len(list_files) == 1:
94+
# special case, direct upload
95+
await port.set(list_files[0])
96+
continue
97+
# generic case let's create an archive
98+
if len(list_files) > 1:
99+
temp_file = tempfile.NamedTemporaryFile(suffix=".tgz")
100+
temp_file.close()
101+
for _file in list_files:
102+
with tarfile.open(temp_file.name, mode='w:gz') as tar_ptr:
103+
for file_path in list_files:
104+
tar_ptr.add(file_path, arcname=file_path.name, recursive=False)
105+
try:
106+
await port.set(temp_file.name)
107+
finally:
108+
#clean up
109+
Path(temp_file.name).unlink()
110+
111+
logger.info("all data uploaded to simcore")
112+
113+
async def sync_data():
114+
try:
115+
await download_data()
116+
await upload_data()
117+
# self.set_status(200)
118+
except node_ports.exceptions.NodeportsException as exc:
119+
# self.set_status(500, reason=str(exc))
120+
logger.error("error when syncing '%s'", str(exc))
121+
sys.exit(1)
122+
finally:
123+
# self.finish('completed retrieve!')
124+
logger.info("download and upload finished")
125+
126+
asyncio.get_event_loop().run_until_complete(sync_data())

‎services/dy-raw-graphs/server/routes.js

+91-45
Original file line numberDiff line numberDiff line change
@@ -22,51 +22,14 @@ appRouter.get('/', function (request, response) {
2222
response.sendFile(path.resolve(config.APP_PATH, 'index.html'));
2323
});
2424

25+
appRouter.get('/retrieve', callInputRetriever);
2526
appRouter.get('/input', getInputFile);
26-
2727
appRouter.get('/inputs', getInputFiles);
28-
29-
appRouter.get('/retrieve', callInputRetriever);
30-
3128
appRouter.get('/output', getOutput);
3229
appRouter.put('/output', setOutput);
3330

3431
module.exports = appRouter;
3532

36-
function getInputFile(request, response) {
37-
const inputsDir = '../inputs/';
38-
const fileName = inputsDir + request.query["fileName"];
39-
console.log('getInputFile', fileName);
40-
fs.readFile(fileName, (err, data) => {
41-
if (err) {
42-
console.error(err);
43-
response.sendStatus("500");
44-
return;
45-
}
46-
response.send(data);
47-
});
48-
}
49-
50-
function getInputFiles(request, response) {
51-
console.log('getInputFiles');
52-
const inputsDir = '../inputs/';
53-
fs.readdir(inputsDir, (err, files) => {
54-
if (err) {
55-
console.error(err);
56-
response.sendStatus("500");
57-
return;
58-
}
59-
let metadata = [];
60-
for (let i=0; i<files.length; i++) {
61-
metadata.push({
62-
title: files[i],
63-
type: 'Other',
64-
url: files[i]
65-
});
66-
}
67-
response.send(metadata);
68-
});
69-
}
7033

7134
function callInputRetriever(request, response) {
7235
console.log('Received a call to retrieve the data on input ports from ' + request.ip);
@@ -100,14 +63,96 @@ function callInputRetriever(request, response) {
10063
});
10164
}
10265

103-
function setOutput(request, response) {
104-
console.log('setOutput');
66+
function getInputDir() {
67+
const inputsDir = '../inputs/';
68+
if (!fs.existsSync(inputsDir)) {
69+
fs.mkdirSync(inputsDir);
70+
}
71+
return inputsDir;
72+
}
73+
74+
function getOutputDir() {
10575
const outputsDir = '../outputs/';
10676
if (!fs.existsSync(outputsDir)) {
10777
fs.mkdirSync(outputsDir);
10878
}
109-
const outputFileName = outputsDir + "output.svg";
110-
const svgCode = request.body.svgCode;
79+
const port = "output_1/";
80+
const outputsDirPort = outputsDir + port;
81+
if (!fs.existsSync(outputsDirPort)) {
82+
fs.mkdirSync(outputsDirPort);
83+
}
84+
return outputsDirPort;
85+
}
86+
87+
function getInputFile(request, response) {
88+
const inputsDir = getInputDir();
89+
const fileName = inputsDir + request.query["fileName"];
90+
console.log('getInputFile', fileName);
91+
fs.readFile(fileName, (err, data) => {
92+
if (err) {
93+
console.error(err);
94+
response.sendStatus("500");
95+
return;
96+
}
97+
response.send(data);
98+
});
99+
}
100+
101+
function getInputFiles(request, response) {
102+
console.log('getInputFiles');
103+
const inputsDir = getInputDir();
104+
fs.readdir(inputsDir, (err, files) => {
105+
if (err) {
106+
console.error(err);
107+
response.sendStatus("500");
108+
return;
109+
}
110+
let metadata = [];
111+
for (let i=0; i<files.length; i++) {
112+
if (fs.lstatSync(inputsDir+files[i]).isFile()) {
113+
metadata.push({
114+
title: files[i],
115+
type: 'Other',
116+
url: files[i]
117+
});
118+
}
119+
}
120+
response.send(metadata);
121+
});
122+
}
123+
124+
function addViewBoxAttr(svgCode) {
125+
// get width value and replace it by 'auto'
126+
let width = svgCode.match(/"(.*?)"/);
127+
if (width) {
128+
width = width[1];
129+
svgCode = svgCode.replace(/width="(.*?)"/,"width='auto'");
130+
}
131+
132+
// get height value and replace it by 'auto'
133+
let height = svgCode.match(/"(.*?)"/);
134+
if (height) {
135+
for (let i=0; i<height.length; i++) {
136+
console.log("height_"+i, height[i]);
137+
}
138+
height = height[1];
139+
svgCode = svgCode.replace(/height="(.*?)"/,"height='auto'");
140+
}
141+
142+
// add viewBox attribute right after svg tag
143+
const viewBoxStr = " viewBox='0 0 "+width+" " +height+ "'";
144+
console.log(viewBoxStr);
145+
svgCode = svgCode.slice(0, 4) + viewBoxStr + svgCode.slice(4);
146+
147+
return svgCode;
148+
}
149+
150+
function setOutput(request, response) {
151+
console.log('setOutput');
152+
const outputsDirPort = getOutputDir();
153+
const outputFileName = outputsDirPort + "output.svg";
154+
let svgCode = request.body.svgCode;
155+
svgCode = addViewBoxAttr(svgCode);
111156
fs.writeFile(outputFileName, svgCode, err => {
112157
if (err) {
113158
console.log(err);
@@ -123,15 +168,15 @@ function setOutput(request, response) {
123168

124169
function getOutput(request, response) {
125170
console.log('getOutput');
126-
const outputsDir = '../outputs/';
127-
fs.readdir(outputsDir, (err, files) => {
171+
const outputsDirPort = getOutputDir();
172+
fs.readdir(outputsDirPort, (err, files) => {
128173
if (err) {
129174
console.log(err);
130175
response.sendStatus("500");
131176
return;
132177
}
133178
if (files.length > 0) {
134-
const fileName = outputsDir + files[0];
179+
const fileName = outputsDirPort + files[0];
135180
fs.readFile(fileName, (err, data) => {
136181
if (err) {
137182
console.log(err);
@@ -143,6 +188,7 @@ function getOutput(request, response) {
143188
});
144189
} else {
145190
console.log('outdir is empty');
191+
response.sendStatus("204");
146192
}
147193
});
148194
}

‎services/dy-raw-graphs/server/server.js

+9-2
Original file line numberDiff line numberDiff line change
@@ -8,15 +8,22 @@
88
const express = require('express');
99

1010
const app = express();
11-
let server = require('http').createServer(app);
12-
let routes = require('./routes');
11+
const server = require('http').createServer(app);
12+
const routes = require('./routes');
1313
const config = require('./config');
1414

1515
console.log(`received basepath: ${config.BASEPATH}`);
1616
// serve static assets normally
1717
console.log('Serving static : ' + config.APP_PATH);
1818
app.use(`${config.BASEPATH}`, express.static(config.APP_PATH));
1919

20+
const bodyParser = require('body-parser');
21+
app.use(bodyParser.json({
22+
limit: "5MB"
23+
}))
24+
app.use(bodyParser.urlencoded({
25+
limit: "5MB"
26+
}))
2027

2128
// init route for retrieving port inputs
2229
app.use(`${config.BASEPATH}`, routes);

0 commit comments

Comments
 (0)