diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports/filemanager.py b/packages/simcore-sdk/src/simcore_sdk/node_ports/filemanager.py index 908a16caa26..5272d2b68e8 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports/filemanager.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports/filemanager.py @@ -14,6 +14,7 @@ log = logging.getLogger(__name__) +CHUNK_SIZE = 1*1024*1024 @contextmanager @@ -82,24 +83,31 @@ async def _download_link_to_file(session:aiohttp.ClientSession, url:URL, file_pa raise exceptions.S3TransferError("Error when downloading {} from {} using {}".format(s3_object, store, url)) file_path.parent.mkdir(parents=True, exist_ok=True) async with aiofiles.open(file_path, 'wb') as file_pointer: - chunk = await response.content.read(1024) + # await file_pointer.write(await response.read()) + chunk = await response.content.read(CHUNK_SIZE) while chunk: await file_pointer.write(chunk) - chunk = await response.content.read(1024) + chunk = await response.content.read(CHUNK_SIZE) log.debug("Download complete") return await response.release() async def _file_sender(file_path:Path): # with async_timeout.timeout(10): async with aiofiles.open(file_path, 'rb') as file_pointer: - chunk = await file_pointer.read(1024) + chunk = await file_pointer.read(CHUNK_SIZE) while chunk: yield chunk - chunk = await file_pointer.read(1024) + chunk = await file_pointer.read(CHUNK_SIZE) async def _upload_file_to_link(session: aiohttp.ClientSession, url: URL, file_path: Path): log.debug("Uploading from %s to %s", file_path, url) + # with aiohttp.MultipartWriter() as writer: + # writer.append(await aiofiles.open(file_path, 'rb')) + # async with session.put(url, data=writer) as resp: + # if resp.status > 299: + # response_text = await resp.text() + # raise exceptions.S3TransferError("Could not upload file {}:{}".format(file_path, response_text)) async with session.put(url, data=file_path.open('rb')) as resp: if resp.status > 299: response_text = await resp.text() diff --git a/scripts/dy_services_helpers/platform_initialiser.py b/scripts/dy_services_helpers/platform_initialiser.py index ebb272a3f11..fc5fff909f8 100644 --- a/scripts/dy_services_helpers/platform_initialiser.py +++ b/scripts/dy_services_helpers/platform_initialiser.py @@ -1,3 +1,4 @@ +#pylint: disable=unused-argument import argparse import asyncio import json @@ -43,7 +44,7 @@ def init_s3(): s3 = S3Settings() return s3 -async def _initialise_platform(port_configuration_path: Path, file_generator): +async def _initialise_platform(port_configuration_path: Path, file_generator, delete_file): with port_configuration_path.open() as file_pointer: @@ -80,20 +81,22 @@ async def _initialise_platform(port_configuration_path: Path, file_generator): file_index = 0 for key, input_item in configuration["schema"]["inputs"].items(): if str(input_item["type"]).startswith("data:"): - file_to_upload = file_generator(file_index) + file_to_upload = file_generator(file_index, input_item["type"]) if file_to_upload is not None: # upload to S3 await PORTS.inputs[key].set(Path(file_to_upload)) file_index += 1 + if delete_file: + Path(file_to_upload).unlink() # print the node uuid so that it can be set as env variable from outside print("{pipelineid},{nodeuuid}".format(pipelineid=str(new_Node.project_id), nodeuuid=node_uuid)) -def main(port_configuration_path: Path, file_generator): +def main(port_configuration_path: Path, file_generator, delete_file=False): loop = asyncio.get_event_loop() - loop.run_until_complete(_initialise_platform(port_configuration_path, file_generator)) + loop.run_until_complete(_initialise_platform(port_configuration_path, file_generator, delete_file)) if __name__ == "__main__": parser = argparse.ArgumentParser(description="Initialise an oSparc database/S3 with user data for development.") @@ -105,14 +108,14 @@ def main(port_configuration_path: Path, file_generator): options = parser.parse_args(args) #print("options %s", options) if options.files is not None: - def _file_generator(file_index: int): + def _file_generator(file_index: int, file_type: str): if file_index < len(options.files): return options.files[file_index] return None main(port_configuration_path=options.portconfig, file_generator=_file_generator) if options.folder is not None: - def _file_generator(file_index: int): + def _file_generator(file_index: int, file_type: str): files = [x for x in options.folder.iterdir() if x.is_file()] if file_index < len(files): return files[file_index] diff --git a/scripts/dy_services_helpers/platform_initialiser_csv_files.py b/scripts/dy_services_helpers/platform_initialiser_csv_files.py index 1650c998bb1..608a1993f4e 100644 --- a/scripts/dy_services_helpers/platform_initialiser_csv_files.py +++ b/scripts/dy_services_helpers/platform_initialiser_csv_files.py @@ -1,5 +1,7 @@ +#pylint: disable=unused-argument import argparse import sys +import tarfile import tempfile from pathlib import Path @@ -15,6 +17,15 @@ def _create_dummy_table(number_of_rows, number_of_columns): df = pd.DataFrame(fullmatrix) return df +def _generate_one_file(rows, columns, separator)->str: + # on Windows you need to close the file to be sure to re-open it to get a name + temp_file = tempfile.NamedTemporaryFile(suffix=".csv") + temp_file.close() + df = _create_dummy_table(rows, columns) + with open(temp_file.name, "w") as file_pointer: + df.to_csv(path_or_buf=file_pointer, sep=separator, header=False, index=False) + return temp_file.name + def main(): parser = argparse.ArgumentParser(description="Initialise an oSparc database/S3 with fake data for development.") parser.add_argument("portconfig", help="The path to the port configuration file (json format)", type=Path) @@ -32,17 +43,20 @@ def main(): separator = options.separator - def _file_generator(file_index: int): # pylint: disable=W0613 - # on Windows you need to close the file to be sure to re-open it to get a name - temp_file = tempfile.NamedTemporaryFile(suffix=".csv") - temp_file.close() - df = _create_dummy_table(options.rows, options.columns) - with open(temp_file.name, "w") as file_pointer: - df.to_csv(path_or_buf=file_pointer, sep=separator, header=False, index=False) + def _file_generator(file_index: int, file_type: str): # pylint: disable=W0613 + if "zip" in file_type: + temp_file = tempfile.NamedTemporaryFile(suffix=".tgz") + temp_file.close() + with tarfile.open(temp_file.name, mode='w:gz') as tar_ptr: + for index in range(options.files): + table_file = _generate_one_file(options.rows, options.columns, separator) + file_name = "{}.dat".format(str(index)) + tar_ptr.add(table_file, arcname=file_name, recursive=False) + Path(table_file).unlink() return temp_file.name - Path(temp_file.name).unlink() + return _generate_one_file(options.rows, options.columns, separator) - init_platform(port_configuration_path=options.portconfig, file_generator=_file_generator) + init_platform(port_configuration_path=options.portconfig, file_generator=_file_generator, delete_file=True) if __name__ == "__main__": main() diff --git a/services/dy-2Dgraph/use-cases/.env-devel b/services/dy-2Dgraph/use-cases/.env-devel index 70199db6f45..4659a6da56f 100644 --- a/services/dy-2Dgraph/use-cases/.env-devel +++ b/services/dy-2Dgraph/use-cases/.env-devel @@ -9,3 +9,7 @@ S3_ENDPOINT=minio:9000 S3_ACCESS_KEY=12345678 S3_SECRET_KEY=12345678 S3_BUCKET_NAME=simcore +S3_SECURE=0 +APIHUB_HOST=apihub +APIHUB_PORT=8043 + diff --git a/services/dy-2Dgraph/use-cases/Makefile b/services/dy-2Dgraph/use-cases/Makefile index 7564a1a3177..1315aaa0e20 100644 --- a/services/dy-2Dgraph/use-cases/Makefile +++ b/services/dy-2Dgraph/use-cases/Makefile @@ -11,7 +11,7 @@ endif export SERVICES_VERSION=1.2.0 export VCS_REF=$(shell git rev-parse --short HEAD) export BUILD_DATE=$(shell date -u +"%Y-%m-%dT%H:%M:%SZ") -export BASE_IMAGE=masu.speag.com/simcore/services/dynamic/jupyter-base-notebook:1.6.0 +export BASE_IMAGE=masu.speag.com/simcore/services/dynamic/jupyter-base-notebook:1.7.0 all: @echo 'run `make build-devel` to build your dev environment' diff --git a/services/dy-2Dgraph/use-cases/cc/cc-twod/notebook/cc_2d.ipynb b/services/dy-2Dgraph/use-cases/cc/cc-twod/notebook/cc_2d.ipynb index eb4c890158c..68bf2d2e7a6 100644 --- a/services/dy-2Dgraph/use-cases/cc/cc-twod/notebook/cc_2d.ipynb +++ b/services/dy-2Dgraph/use-cases/cc/cc-twod/notebook/cc_2d.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "metadata": { "hideoutput": true, "init_cell": true @@ -29,7 +29,7 @@ "" ] }, - "execution_count": 5, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } @@ -95,7 +95,9 @@ "import matplotlib.animation as animation\n", "import pandas as pd\n", "import numpy as np\n", - "import tqdm" + "import tqdm\n", + "import tarfile\n", + "import tempfile" ] }, { @@ -104,14 +106,19 @@ "metadata": { "extensions": {}, "hide_input": true, - "init_cell": true + "init_cell": true, + "scrolled": true }, "outputs": [], "source": [ - "data_path = await PORTS.inputs[0].get()\n", + "temp_folder = tempfile.mkdtemp()\n", + "compressed_data = await PORTS.inputs[0].get()\n", + "if tarfile.is_tarfile(compressed_data):\n", + " with tarfile.open(compressed_data) as tar_file:\n", + " tar_file.extractall(temp_folder)\n", "\n", "# get the list of files\n", - "dat_files = sorted([os.path.join(data_path, x) for x in os.listdir(data_path) if x.endswith(\".dat\")], key=lambda f: int(''.join(filter(str.isdigit, f))))\n", + "dat_files = sorted([os.path.join(temp_folder, x) for x in os.listdir(temp_folder) if x.endswith(\".dat\")], key=lambda f: int(''.join(filter(str.isdigit, f))))\n", "out_images_path = tempfile.gettempdir()\n" ] }, diff --git a/services/dy-2Dgraph/use-cases/docker-compose.devel.yml b/services/dy-2Dgraph/use-cases/docker-compose.devel.yml index eac585c1ecc..44653147a85 100644 --- a/services/dy-2Dgraph/use-cases/docker-compose.devel.yml +++ b/services/dy-2Dgraph/use-cases/docker-compose.devel.yml @@ -100,6 +100,11 @@ services: - postgres - minio #-------------------------------------------------------------------- + apihub: + image: services_apihub:latest + ports: + - 8043:8043 + #-------------------------------------------------------------------- storage: image: services_storage:latest ports: @@ -115,12 +120,17 @@ services: - S3_ACCESS_KEY=${S3_ACCESS_KEY} - S3_SECRET_KEY=${S3_SECRET_KEY} - S3_BUCKET_NAME=${S3_BUCKET_NAME} + - S3_SECURE=${S3_SECURE} - RUN_DOCKER_ENGINE_ROOT=1 - BF_API_SECRET=none - BF_API_KEY=none + - APIHUB_HOST=${APIHUB_HOST} + - APIHUB_PORT=${APIHUB_PORT} + restart: always depends_on: - postgres + - apihub #-------------------------------------------------------------------- postgres: image: postgres:10 diff --git a/services/dy-2Dgraph/use-cases/docker/boot.sh b/services/dy-2Dgraph/use-cases/docker/boot.sh index 2df9283ff2e..5cb6c94eabc 100644 --- a/services/dy-2Dgraph/use-cases/docker/boot.sh +++ b/services/dy-2Dgraph/use-cases/docker/boot.sh @@ -2,7 +2,6 @@ if test "${CREATE_DUMMY_TABLE}" = "1" then - pip install -r /home/jovyan/devel/requirements.txt pushd /packages/simcore-sdk; pip install -r requirements-dev.txt; popd pushd /packages/s3wrapper; pip install -r requirements-dev.txt; popd diff --git a/services/dy-jupyter/.env-devel b/services/dy-jupyter/.env-devel index d0e80a41572..8df9c349581 100644 --- a/services/dy-jupyter/.env-devel +++ b/services/dy-jupyter/.env-devel @@ -9,5 +9,6 @@ S3_ENDPOINT=minio:9000 S3_ACCESS_KEY=12345678 S3_SECRET_KEY=12345678 S3_BUCKET_NAME=simcore +S3_SECURE=0 APIHUB_HOST=apihub APIHUB_PORT=8043 \ No newline at end of file diff --git a/services/dy-jupyter/docker-compose.devel.yml b/services/dy-jupyter/docker-compose.devel.yml index 6fefe6f0572..bf5e5a4e692 100644 --- a/services/dy-jupyter/docker-compose.devel.yml +++ b/services/dy-jupyter/docker-compose.devel.yml @@ -53,6 +53,7 @@ services: - S3_ACCESS_KEY=${S3_ACCESS_KEY} - S3_SECRET_KEY=${S3_SECRET_KEY} - S3_BUCKET_NAME=${S3_BUCKET_NAME} + - S3_SECURE=${S3_SECURE} - RUN_DOCKER_ENGINE_ROOT=1 - BF_API_SECRET=none - BF_API_KEY=none