fixes ITISFoundation#455

Author: sanderegg

packages/simcore-sdk/src/simcore_sdk/node_data/__init__.py

@@ -0,0 +1 @@
+from . import data_manager

packages/simcore-sdk/src/simcore_sdk/node_data/data_manager.py

@@ -1,17 +1,19 @@
 import logging
 from pathlib import Path
-from shutil import make_archive, unpack_archive, move
+from shutil import make_archive, move, unpack_archive
 from tempfile import TemporaryDirectory
+from typing import Optional, Union
 
 from simcore_sdk.node_ports import config, filemanager
 
 log = logging.getLogger(__name__)
 
-def _create_s3_object(file_path: Path) -> str:
-    return "{}/{}/{}".format(config.PROJECT_ID, config.NODE_UUID, file_path.name)
+def _create_s3_object(file_path: Union[Path, str]) -> str:
+    file_name = file_path.name if isinstance(file_path, Path) else file_path
+    return f"{config.PROJECT_ID}/{config.NODE_UUID}/{file_name}"
 
 
-async def _push_file(file_path: Path):
+async def _push_file(file_path: Path, rename_to: Optional[str]):
     store_id = 0 # this is for simcore.s3
     s3_object = _create_s3_object(file_path)
     log.info("uploading %s to S3 to %s...", file_path.name, s3_object)
@@ -20,16 +22,16 @@ async def _push_file(file_path: Path):
                                 local_file_path=file_path)
     log.info("%s successfuly uploaded", file_path)
 
-async def push(file_or_folder: Path):
+async def push(file_or_folder: Path, rename_to: Optional[str] = None):
     if file_or_folder.is_file():
-        return await _push_file(file_or_folder)
+        return await _push_file(file_or_folder, rename_to)
     # we have a folder, so we create a compressed file
     with TemporaryDirectory() as tmp_dir_name:
         log.info("compressing %s into %s...", file_or_folder.name, tmp_dir_name)
         # compress the files
-        compressed_file_wo_ext = Path(tmp_dir_name) / file_or_folder.stem
+        compressed_file_wo_ext = Path(tmp_dir_name) / (rename_to if rename_to else file_or_folder.stem)
         archive_file = Path(make_archive(str(compressed_file_wo_ext), 'zip', root_dir=file_or_folder)) #, base_dir=folder))
-        return await _push_file(archive_file)
+        return await _push_file(archive_file, rename_to)
 
 async def _pull_file(file_path: Path):
     s3_object = _create_s3_object(file_path)

services/sidecar/src/sidecar/core.py

@@ -14,7 +14,7 @@
 from celery.utils.log import get_task_logger
 
 import pika
-from simcore_sdk import node_ports
+from simcore_sdk import node_ports, node_data
 from simcore_sdk.models.pipeline_models import (RUNNING, SUCCESS,
                                                 ComputationalPipeline,
                                                 ComputationalTask)
@@ -276,9 +276,10 @@ def _process_task_log(self):
 
                 - put them all into S3 /logg
         """
-        return
-        #directory = self._executor.log_dir
-        #if os.path.exists(directory):
+        directory = Path(self._executor.log_dir)
+
+        if directory.exists():
+            wrap_async_call(node_data.data_manager.push(directory, rename_to="logs"))
         #    for root, _dirs, files in os.walk(directory):
         #        for name in files:
         #            filepath = os.path.join(root, name)