Faking the data

Makefile

@@ -158,7 +158,7 @@ test:
 	make run_test
 	make after_test
 
-PLATFORM_VERSION=3.27
+PLATFORM_VERSION=3.28
 DOCKER_REGISTRY=masu.speag.com
 #DOCKER_REGISTRY=registry.osparc.io
 

packages/simcore-sdk/src/simcore_sdk/node_ports/filemanager.py

@@ -14,6 +14,7 @@
 
 log = logging.getLogger(__name__)
 
+CHUNK_SIZE = 1*1024*1024
 
 
 @contextmanager
@@ -82,24 +83,31 @@ async def _download_link_to_file(session:aiohttp.ClientSession, url:URL, file_pa
             raise exceptions.S3TransferError("Error when downloading {} from {} using {}".format(s3_object, store, url))
         file_path.parent.mkdir(parents=True, exist_ok=True)
         async with aiofiles.open(file_path, 'wb') as file_pointer:
-            chunk = await response.content.read(1024)
+            # await file_pointer.write(await response.read())
+            chunk = await response.content.read(CHUNK_SIZE)
             while chunk:
                 await file_pointer.write(chunk)
-                chunk = await response.content.read(1024)
+                chunk = await response.content.read(CHUNK_SIZE)
         log.debug("Download complete")
         return await response.release()
 
 async def _file_sender(file_path:Path):
     # with async_timeout.timeout(10):
     async with aiofiles.open(file_path, 'rb') as file_pointer:
-        chunk = await file_pointer.read(1024)
+        chunk = await file_pointer.read(CHUNK_SIZE)
         while chunk:
             yield chunk
-            chunk = await file_pointer.read(1024)
+            chunk = await file_pointer.read(CHUNK_SIZE)
 
 async def _upload_file_to_link(session: aiohttp.ClientSession, url: URL, file_path: Path):
     log.debug("Uploading from %s to %s", file_path, url)
+    # with aiohttp.MultipartWriter() as writer:
+    #     writer.append(await aiofiles.open(file_path, 'rb'))
 
+    #     async with session.put(url, data=writer) as resp:
+    #         if resp.status > 299:
+    #             response_text = await resp.text()
+    #             raise exceptions.S3TransferError("Could not upload file {}:{}".format(file_path, response_text))    
     async with session.put(url, data=file_path.open('rb')) as resp:
         if resp.status > 299:
             response_text = await resp.text()

scripts/dy_services_helpers/platform_initialiser.py

@@ -1,3 +1,4 @@
+#pylint: disable=unused-argument
 import argparse
 import asyncio
 import json
@@ -43,7 +44,7 @@ def init_s3():
     s3 = S3Settings()
     return s3
 
-async def _initialise_platform(port_configuration_path: Path, file_generator):
+async def _initialise_platform(port_configuration_path: Path, file_generator, delete_file):
 
 
     with port_configuration_path.open() as file_pointer:
@@ -80,20 +81,22 @@ async def _initialise_platform(port_configuration_path: Path, file_generator):
     file_index = 0
     for key, input_item in configuration["schema"]["inputs"].items():
         if str(input_item["type"]).startswith("data:"):
-            file_to_upload = file_generator(file_index)
+            file_to_upload = file_generator(file_index, input_item["type"])
             if file_to_upload is not None:
                 # upload to S3
                 await PORTS.inputs[key].set(Path(file_to_upload))
                 file_index += 1
+                if delete_file:
+                    Path(file_to_upload).unlink()
 
 
     # print the node uuid so that it can be set as env variable from outside
     print("{pipelineid},{nodeuuid}".format(pipelineid=str(new_Node.project_id), nodeuuid=node_uuid))
 
-def main(port_configuration_path: Path, file_generator):
+def main(port_configuration_path: Path, file_generator, delete_file=False):
 
     loop = asyncio.get_event_loop()
-    loop.run_until_complete(_initialise_platform(port_configuration_path, file_generator))
+    loop.run_until_complete(_initialise_platform(port_configuration_path, file_generator, delete_file))
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Initialise an oSparc database/S3 with user data for development.")
@@ -105,14 +108,14 @@ def main(port_configuration_path: Path, file_generator):
     options = parser.parse_args(args)
     #print("options %s", options)
     if options.files is not None:
-        def _file_generator(file_index: int):
+        def _file_generator(file_index: int, file_type: str):
             if file_index < len(options.files):
                 return options.files[file_index]
             return None
         main(port_configuration_path=options.portconfig, file_generator=_file_generator)
 
     if options.folder is not None:
-        def _file_generator(file_index: int):
+        def _file_generator(file_index: int, file_type: str):
             files = [x for x in options.folder.iterdir() if x.is_file()]
             if file_index < len(files):
                 return files[file_index]

scripts/dy_services_helpers/platform_initialiser_csv_files.py

@@ -1,5 +1,7 @@
+#pylint: disable=unused-argument
 import argparse
 import sys
+import tarfile
 import tempfile
 from pathlib import Path
 
@@ -15,6 +17,15 @@ def _create_dummy_table(number_of_rows, number_of_columns):
     df = pd.DataFrame(fullmatrix)
     return df
 
+def _generate_one_file(rows, columns, separator)->str:
+    # on Windows you need to close the file to be sure to re-open it to get a name
+    temp_file = tempfile.NamedTemporaryFile(suffix=".csv")
+    temp_file.close()
+    df = _create_dummy_table(rows, columns)
+    with open(temp_file.name, "w") as file_pointer:
+        df.to_csv(path_or_buf=file_pointer, sep=separator, header=False, index=False)        
+        return temp_file.name    
+
 def main():
     parser = argparse.ArgumentParser(description="Initialise an oSparc database/S3 with fake data for development.")
     parser.add_argument("portconfig", help="The path to the port configuration file (json format)", type=Path)
@@ -32,17 +43,20 @@ def main():
         separator = options.separator    
 
 
-    def _file_generator(file_index: int): # pylint: disable=W0613
-        # on Windows you need to close the file to be sure to re-open it to get a name
-        temp_file = tempfile.NamedTemporaryFile(suffix=".csv")
-        temp_file.close()
-        df = _create_dummy_table(options.rows, options.columns)
-        with open(temp_file.name, "w") as file_pointer:
-            df.to_csv(path_or_buf=file_pointer, sep=separator, header=False, index=False)        
+    def _file_generator(file_index: int, file_type: str): # pylint: disable=W0613
+        if "zip" in file_type:
+            temp_file = tempfile.NamedTemporaryFile(suffix=".tgz")
+            temp_file.close()
+            with tarfile.open(temp_file.name, mode='w:gz') as tar_ptr:
+                for index in range(options.files):
+                    table_file = _generate_one_file(options.rows, options.columns, separator)
+                    file_name = "{}.dat".format(str(index))
+                    tar_ptr.add(table_file, arcname=file_name, recursive=False)
+                    Path(table_file).unlink()
             return temp_file.name
-        Path(temp_file.name).unlink()
+        return _generate_one_file(options.rows, options.columns, separator)
 
-    init_platform(port_configuration_path=options.portconfig, file_generator=_file_generator)
+    init_platform(port_configuration_path=options.portconfig, file_generator=_file_generator, delete_file=True)
 
 if __name__ == "__main__":
     main()

services/dy-2Dgraph/use-cases/.env-devel

@@ -9,3 +9,7 @@ S3_ENDPOINT=minio:9000
 S3_ACCESS_KEY=12345678
 S3_SECRET_KEY=12345678
 S3_BUCKET_NAME=simcore
+S3_SECURE=0
+APIHUB_HOST=apihub
+APIHUB_PORT=8043
+

services/dy-2Dgraph/use-cases/Makefile

@@ -11,7 +11,7 @@ endif
 export SERVICES_VERSION=1.2.0
 export VCS_REF=$(shell git rev-parse --short HEAD) 
 export BUILD_DATE=$(shell date -u +"%Y-%m-%dT%H:%M:%SZ")
-export BASE_IMAGE=masu.speag.com/simcore/services/dynamic/jupyter-base-notebook:1.6.0
+export BASE_IMAGE=masu.speag.com/simcore/services/dynamic/jupyter-base-notebook:1.7.0
 
 all:
 	@echo 'run `make build-devel` to build your dev environment'

services/dy-2Dgraph/use-cases/cc/cc-twod/notebook/cc_2d.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 6,
    "metadata": {
     "hideoutput": true,
     "init_cell": true
@@ -29,7 +29,7 @@
        "<IPython.core.display.HTML object>"
       ]
      },
-     "execution_count": 5,
+     "execution_count": 6,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -95,7 +95,9 @@
     "import matplotlib.animation as animation\n",
     "import pandas as pd\n",
     "import numpy as np\n",
-    "import tqdm"
+    "import tqdm\n",
+    "import tarfile\n",
+    "import tempfile"
    ]
   },
   {
@@ -104,14 +106,19 @@
    "metadata": {
     "extensions": {},
     "hide_input": true,
-    "init_cell": true
+    "init_cell": true,
+    "scrolled": true
    },
    "outputs": [],
    "source": [
-    "data_path = await PORTS.inputs[0].get()\n",
+    "temp_folder = tempfile.mkdtemp()\n",
+    "compressed_data = await PORTS.inputs[0].get()\n",
+    "if tarfile.is_tarfile(compressed_data):\n",
+    "    with tarfile.open(compressed_data) as tar_file:\n",
+    "        tar_file.extractall(temp_folder)\n",
     "\n",
     "# get the list of files\n",
-    "dat_files = sorted([os.path.join(data_path, x) for x in os.listdir(data_path) if x.endswith(\".dat\")], key=lambda f: int(''.join(filter(str.isdigit, f))))\n",
+    "dat_files = sorted([os.path.join(temp_folder, x) for x in os.listdir(temp_folder) if x.endswith(\".dat\")], key=lambda f: int(''.join(filter(str.isdigit, f))))\n",
     "out_images_path = tempfile.gettempdir()\n"
    ]
   },

services/dy-2Dgraph/use-cases/docker-compose.devel.yml

@@ -100,6 +100,11 @@ services:
       - postgres
       - minio
   #--------------------------------------------------------------------
+  apihub:
+    image: services_apihub:latest
+    ports:
+      - 8043:8043
+  #--------------------------------------------------------------------
   storage:
     image: services_storage:latest
     ports:
@@ -115,12 +120,17 @@ services:
       - S3_ACCESS_KEY=${S3_ACCESS_KEY}
       - S3_SECRET_KEY=${S3_SECRET_KEY}
       - S3_BUCKET_NAME=${S3_BUCKET_NAME}
+      - S3_SECURE=${S3_SECURE}
       - RUN_DOCKER_ENGINE_ROOT=1
       - BF_API_SECRET=none
       - BF_API_KEY=none
+      - APIHUB_HOST=${APIHUB_HOST}
+      - APIHUB_PORT=${APIHUB_PORT}
+
     restart: always
     depends_on: 
       - postgres
+      - apihub
   #--------------------------------------------------------------------
   postgres:
     image: postgres:10

services/dy-2Dgraph/use-cases/docker/boot.sh

@@ -2,7 +2,6 @@
 
 if test "${CREATE_DUMMY_TABLE}" = "1"
 then
-    pip install -r /home/jovyan/devel/requirements.txt
     pushd /packages/simcore-sdk; pip install -r requirements-dev.txt; popd
     pushd /packages/s3wrapper; pip install -r requirements-dev.txt; popd
 

services/dy-jupyter/docker-compose.devel.yml

@@ -53,6 +53,7 @@ services:
       - S3_ACCESS_KEY=${S3_ACCESS_KEY}
       - S3_SECRET_KEY=${S3_SECRET_KEY}
       - S3_BUCKET_NAME=${S3_BUCKET_NAME}
+      - S3_SECURE=${S3_SECURE}
       - RUN_DOCKER_ENGINE_ROOT=1
       - BF_API_SECRET=none
       - BF_API_KEY=none