From 27418349c35aab9062585702867ee5a97005dc89 Mon Sep 17 00:00:00 2001 From: Pedro Crespo Date: Fri, 6 Jul 2018 14:34:58 +0200 Subject: [PATCH 01/10] Fixes typos in client after bad merge --- .../web/client/source/class/qxapp/components/login/Form.js | 2 +- services/web/client/source/class/qxapp/dev/fake/Data.js | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/services/web/client/source/class/qxapp/components/login/Form.js b/services/web/client/source/class/qxapp/components/login/Form.js index c1138004cc1..b62cb7e5e15 100644 --- a/services/web/client/source/class/qxapp/components/login/Form.js +++ b/services/web/client/source/class/qxapp/components/login/Form.js @@ -8,6 +8,7 @@ qx.Class.define("qxapp.components.login.Form", { extend: qx.ui.form.Form, + include: [qx.locale.MTranslation], construct: function() { this.base(arguments); @@ -29,7 +30,6 @@ qx.Class.define("qxapp.components.login.Form", { placeholder: this.tr("Your password"), tabIndex: username.getTabIndex()+1 }); - password.setPlaceholder(); this.add(password, "", null, "password", null); // TODO: diff --git a/services/web/client/source/class/qxapp/dev/fake/Data.js b/services/web/client/source/class/qxapp/dev/fake/Data.js index 5ce5d0cea81..e9f0fbcfa22 100644 --- a/services/web/client/source/class/qxapp/dev/fake/Data.js +++ b/services/web/client/source/class/qxapp/dev/fake/Data.js @@ -728,9 +728,9 @@ qx.Class.define("qxapp.dev.fake.Data", { getServices: function() { let fakeServices = []; - Array.prototype.push.apply(fakeServices, qxapp.qxapp.dev.fake.Data.getProducers()); - Array.prototype.push.apply(fakeServices, qxapp.qxapp.dev.fake.Data.getComputationals()); - Array.prototype.push.apply(fakeServices, qxapp.qxapp.dev.fake.Data.getAnalyses()); + Array.prototype.push.apply(fakeServices, qxapp.dev.fake.Data.getProducers()); + Array.prototype.push.apply(fakeServices, qxapp.dev.fake.Data.getComputationals()); + Array.prototype.push.apply(fakeServices, qxapp.dev.fake.Data.getAnalyses()); return fakeServices; }, From df42c63063fa1675f17db1b376443ef38c7268d9 Mon Sep 17 00:00:00 2001 From: Pedro Crespo Date: Fri, 23 Nov 2018 19:32:41 +0100 Subject: [PATCH 02/10] Moved oas3/v0 -> api/specs/storage/v0 Removed reference to oas3 in setup.py Added config rest section to set the remote or local location of apispecs repo rest.py loads oas doc, validates and proceeds with setup as before Oas3 is not anymore a resource. Removed all references as a resource Cleanup - removed cyclic dependency in settings - renamed settings_config as config_schema --- .../storage}/v0/components/schemas/error.yaml | 0 .../storage}/v0/components/schemas/fake.yaml | 0 .../v0/components/schemas/file_meta_data.yaml | 0 .../schemas/file_meta_data_array.yaml | 0 .../v0/components/schemas/health_check.yaml | 0 .../v0/components/schemas/location.yaml | 0 .../v0/components/schemas/location_array.yaml | 0 .../v0/components/schemas/log_message.yaml | 0 .../v0/components/schemas/presigned_link.yaml | 0 .../specs/storage}/v0/openapi.yaml | 0 services/storage/setup.py | 7 +- .../simcore_service_storage/__version__.py | 8 +- .../src/simcore_service_storage/cli_config.py | 9 ++- .../{settings_schema.py => config_schema.py} | 13 ++-- .../data/config-schema-v1.json | 2 - .../src/simcore_service_storage/resources.py | 19 +---- .../src/simcore_service_storage/rest.py | 77 ++++++++++--------- .../simcore_service_storage/rest_config.py | 16 ++++ .../simcore_service_storage/rest_routes.py | 14 +--- .../src/simcore_service_storage/settings.py | 76 ++++++++---------- .../src/simcore_service_storage/utils.py | 9 --- services/storage/tests/test_resources.py | 5 +- 22 files changed, 113 insertions(+), 142 deletions(-) rename {services/storage/src/simcore_service_storage/oas3 => api/specs/storage}/v0/components/schemas/error.yaml (100%) rename {services/storage/src/simcore_service_storage/oas3 => api/specs/storage}/v0/components/schemas/fake.yaml (100%) rename {services/storage/src/simcore_service_storage/oas3 => api/specs/storage}/v0/components/schemas/file_meta_data.yaml (100%) rename {services/storage/src/simcore_service_storage/oas3 => api/specs/storage}/v0/components/schemas/file_meta_data_array.yaml (100%) rename {services/storage/src/simcore_service_storage/oas3 => api/specs/storage}/v0/components/schemas/health_check.yaml (100%) rename {services/storage/src/simcore_service_storage/oas3 => api/specs/storage}/v0/components/schemas/location.yaml (100%) rename {services/storage/src/simcore_service_storage/oas3 => api/specs/storage}/v0/components/schemas/location_array.yaml (100%) rename {services/storage/src/simcore_service_storage/oas3 => api/specs/storage}/v0/components/schemas/log_message.yaml (100%) rename {services/storage/src/simcore_service_storage/oas3 => api/specs/storage}/v0/components/schemas/presigned_link.yaml (100%) rename {services/storage/src/simcore_service_storage/oas3 => api/specs/storage}/v0/openapi.yaml (100%) rename services/storage/src/simcore_service_storage/{settings_schema.py => config_schema.py} (78%) delete mode 100644 services/storage/src/simcore_service_storage/data/config-schema-v1.json create mode 100644 services/storage/src/simcore_service_storage/rest_config.py delete mode 100644 services/storage/src/simcore_service_storage/utils.py diff --git a/services/storage/src/simcore_service_storage/oas3/v0/components/schemas/error.yaml b/api/specs/storage/v0/components/schemas/error.yaml similarity index 100% rename from services/storage/src/simcore_service_storage/oas3/v0/components/schemas/error.yaml rename to api/specs/storage/v0/components/schemas/error.yaml diff --git a/services/storage/src/simcore_service_storage/oas3/v0/components/schemas/fake.yaml b/api/specs/storage/v0/components/schemas/fake.yaml similarity index 100% rename from services/storage/src/simcore_service_storage/oas3/v0/components/schemas/fake.yaml rename to api/specs/storage/v0/components/schemas/fake.yaml diff --git a/services/storage/src/simcore_service_storage/oas3/v0/components/schemas/file_meta_data.yaml b/api/specs/storage/v0/components/schemas/file_meta_data.yaml similarity index 100% rename from services/storage/src/simcore_service_storage/oas3/v0/components/schemas/file_meta_data.yaml rename to api/specs/storage/v0/components/schemas/file_meta_data.yaml diff --git a/services/storage/src/simcore_service_storage/oas3/v0/components/schemas/file_meta_data_array.yaml b/api/specs/storage/v0/components/schemas/file_meta_data_array.yaml similarity index 100% rename from services/storage/src/simcore_service_storage/oas3/v0/components/schemas/file_meta_data_array.yaml rename to api/specs/storage/v0/components/schemas/file_meta_data_array.yaml diff --git a/services/storage/src/simcore_service_storage/oas3/v0/components/schemas/health_check.yaml b/api/specs/storage/v0/components/schemas/health_check.yaml similarity index 100% rename from services/storage/src/simcore_service_storage/oas3/v0/components/schemas/health_check.yaml rename to api/specs/storage/v0/components/schemas/health_check.yaml diff --git a/services/storage/src/simcore_service_storage/oas3/v0/components/schemas/location.yaml b/api/specs/storage/v0/components/schemas/location.yaml similarity index 100% rename from services/storage/src/simcore_service_storage/oas3/v0/components/schemas/location.yaml rename to api/specs/storage/v0/components/schemas/location.yaml diff --git a/services/storage/src/simcore_service_storage/oas3/v0/components/schemas/location_array.yaml b/api/specs/storage/v0/components/schemas/location_array.yaml similarity index 100% rename from services/storage/src/simcore_service_storage/oas3/v0/components/schemas/location_array.yaml rename to api/specs/storage/v0/components/schemas/location_array.yaml diff --git a/services/storage/src/simcore_service_storage/oas3/v0/components/schemas/log_message.yaml b/api/specs/storage/v0/components/schemas/log_message.yaml similarity index 100% rename from services/storage/src/simcore_service_storage/oas3/v0/components/schemas/log_message.yaml rename to api/specs/storage/v0/components/schemas/log_message.yaml diff --git a/services/storage/src/simcore_service_storage/oas3/v0/components/schemas/presigned_link.yaml b/api/specs/storage/v0/components/schemas/presigned_link.yaml similarity index 100% rename from services/storage/src/simcore_service_storage/oas3/v0/components/schemas/presigned_link.yaml rename to api/specs/storage/v0/components/schemas/presigned_link.yaml diff --git a/services/storage/src/simcore_service_storage/oas3/v0/openapi.yaml b/api/specs/storage/v0/openapi.yaml similarity index 100% rename from services/storage/src/simcore_service_storage/oas3/v0/openapi.yaml rename to api/specs/storage/v0/openapi.yaml diff --git a/services/storage/setup.py b/services/storage/setup.py index ec8579c8896..3a9f501fa97 100644 --- a/services/storage/setup.py +++ b/services/storage/setup.py @@ -60,12 +60,7 @@ def list_packages(*parts): '': [ 'data/*.json', 'data/*.yml', - 'data/*.yaml', - 'oas3/**/*.yaml', - 'oas3/**/*.yml', - 'oas3/**/components/schemas/*.json', - 'oas3/**/components/schemas/*.yaml', - 'oas3/**/components/schemas/*.yml' + 'data/*.yaml' ], }, entry_points={ diff --git a/services/storage/src/simcore_service_storage/__version__.py b/services/storage/src/simcore_service_storage/__version__.py index 174b62febda..6001a45c222 100644 --- a/services/storage/src/simcore_service_storage/__version__.py +++ b/services/storage/src/simcore_service_storage/__version__.py @@ -6,10 +6,10 @@ -Major versions introduce significant changes to the API, and backwards -compatibility is not guaranteed. +Major versions introduce significant changes to the API, and backwards +compatibility is not guaranteed. -Minor versions are for new features and other backwards-compatible changes to the API. +Minor versions are for new features and other backwards-compatible changes to the API. Patch versions are for bug fixes and internal code changes that do not affect the API. @@ -37,5 +37,3 @@ def get_version_object(): return semantic_version.Version(__version__) - - diff --git a/services/storage/src/simcore_service_storage/cli_config.py b/services/storage/src/simcore_service_storage/cli_config.py index 5d7b8ff63cd..3479dcb5b0e 100644 --- a/services/storage/src/simcore_service_storage/cli_config.py +++ b/services/storage/src/simcore_service_storage/cli_config.py @@ -7,7 +7,8 @@ import trafaret_config.commandline as commandline from .resources import RSC_CONFIG_DIR_KEY, resources -from .settings import CONFIG_SCHEMA, DEFAULT_CONFIG +from .config_schema import schema +from .settings import DEFAULT_CONFIG log = logging.getLogger(__name__) @@ -48,14 +49,14 @@ def config_from_options(options, vars=None): # pylint: disable=W0622 log.debug("loading %s", options.config) - return commandline.config_from_options(options, trafaret=CONFIG_SCHEMA, vars=vars) + return commandline.config_from_options(options, trafaret=schema, vars=vars) def read_and_validate(filepath, vars=None): # pylint: disable=W0622 if vars is None: vars = os.environ # NOTE: vars=os.environ in signature freezes default to os.environ before it gets # Cannot user functools.partial because os.environ gets then frozen - return trafaret_config.read_and_validate(filepath, trafaret=CONFIG_SCHEMA, vars=vars) + return trafaret_config.read_and_validate(filepath, trafaret=schema, vars=vars) def config_from_file(filepath) -> dict: @@ -65,5 +66,5 @@ def config_from_file(filepath) -> dict: Raises trafaret_config.ConfigError """ - config = trafaret_config.read_and_validate(filepath, CONFIG_SCHEMA, vars=os.environ) + config = trafaret_config.read_and_validate(filepath, schema, vars=os.environ) return config diff --git a/services/storage/src/simcore_service_storage/settings_schema.py b/services/storage/src/simcore_service_storage/config_schema.py similarity index 78% rename from services/storage/src/simcore_service_storage/settings_schema.py rename to services/storage/src/simcore_service_storage/config_schema.py index e4e86eb2c48..716badfd6c6 100644 --- a/services/storage/src/simcore_service_storage/settings_schema.py +++ b/services/storage/src/simcore_service_storage/config_schema.py @@ -2,9 +2,9 @@ from simcore_sdk.config import db, s3 -## Config file schema -# FIXME: load from json schema instead! -_APP_SCHEMA = T.Dict({ +from . import rest_config + +app_schema = T.Dict({ T.Key("host", default="0.0.0.0"): T.IP, "port": T.Int(), "log_level": T.Enum("DEBUG", "WARNING", "INFO", "ERROR", "CRITICAL", "FATAL", "NOTSET"), @@ -18,11 +18,12 @@ T.Key("disable_services", default=[], optional=True): T.List(T.String()) }) -CONFIG_SCHEMA = T.Dict({ +schema = T.Dict({ "version": T.String(), - T.Key("main"): _APP_SCHEMA, + T.Key("main"): app_schema, T.Key("postgres"): db.CONFIG_SCHEMA, - T.Key("s3"): s3.CONFIG_SCHEMA + T.Key("s3"): s3.CONFIG_SCHEMA, + T.Key(rest_config.CONFIG_SECTION_NAME): rest_config.schema }) diff --git a/services/storage/src/simcore_service_storage/data/config-schema-v1.json b/services/storage/src/simcore_service_storage/data/config-schema-v1.json deleted file mode 100644 index 934e403f7e1..00000000000 --- a/services/storage/src/simcore_service_storage/data/config-schema-v1.json +++ /dev/null @@ -1,2 +0,0 @@ -# Issue #195 -# TODO: jsonschema compatible with openapi diff --git a/services/storage/src/simcore_service_storage/resources.py b/services/storage/src/simcore_service_storage/resources.py index f374f71a0e7..0180cb01a80 100644 --- a/services/storage/src/simcore_service_storage/resources.py +++ b/services/storage/src/simcore_service_storage/resources.py @@ -1,30 +1,17 @@ """ Access to data resources installed with this package """ -from pathlib import Path - from servicelib.resources import ResourcesFacade - -from .settings import (OAS_ROOT_FILE, # pylint: disable=unused-import - RSC_CONFIG_DIR_KEY, RSC_OPENAPI_DIR_KEY) +from .settings import RSC_CONFIG_DIR_KEY # pylint: disable=unused-import resources = ResourcesFacade( package_name=__name__, distribution_name="simcore-service-storage", - config_folder='etc/', + config_folder=RSC_CONFIG_DIR_KEY, ) -def openapi_path() -> Path: - """ Returns path to the roots's oas file - Notice that the specs can be split in multiple files. Thisone - is the root file and it is normally named as `opeapi.yaml` - """ - return resources.get_path(OAS_ROOT_FILE) - - __all__ = ( 'resources', - 'RSC_CONFIG_DIR_KEY', - 'RSC_OPENAPI_DIR_KEY' + 'RSC_CONFIG_DIR_KEY' ) diff --git a/services/storage/src/simcore_service_storage/rest.py b/services/storage/src/simcore_service_storage/rest.py index 7952005c3f2..70b13c44395 100644 --- a/services/storage/src/simcore_service_storage/rest.py +++ b/services/storage/src/simcore_service_storage/rest.py @@ -1,21 +1,21 @@ """ RESTful API for simcore_service_storage """ +import asyncio import copy import logging +from pprint import pformat from typing import Dict from aiohttp import web from servicelib import openapi +from servicelib.openapi import create_openapi_specs, get_base_path from servicelib.rest_middlewares import append_rest_middlewares - from . import rest_routes -from .resources import resources -from .settings import (APP_CONFIG_KEY, APP_OPENAPI_SPECS_KEY, - RSC_OPENAPI_ROOTFILE_KEY) - +from .rest_config import CONFIG_SECTION_NAME +from .settings import API_VERSION_TAG, APP_CONFIG_KEY, APP_OPENAPI_SPECS_KEY log = logging.getLogger(__name__) @@ -63,54 +63,59 @@ def _setup_servers_specs(specs: openapi.Spec, app_config: Dict) -> openapi.Spec: return specs -def create_apispecs(app_config: Dict) -> openapi.Spec: +# def create_apispecs(app_config: Dict) -> openapi.Spec: +# # TODO: What if many specs to expose? v0, v1, v2 ... +# openapi_path = resources.get_path(RSC_OPENAPI_ROOTFILE_KEY) - # TODO: What if many specs to expose? v0, v1, v2 ... - openapi_path = resources.get_path(RSC_OPENAPI_ROOTFILE_KEY) +# try: +# specs = openapi.create_specs(openapi_path) +# specs = _setup_servers_specs(specs, app_config) - try: - specs = openapi.create_specs(openapi_path) - specs = _setup_servers_specs(specs, app_config) +# except openapi.OpenAPIError: +# # TODO: protocol when some parts are unavailable because of failure +# # Define whether it is critical or this server can still +# # continue working offering partial services +# log.exception("Invalid rest API specs. Rest API is DISABLED") +# specs = None +# return specs - except openapi.OpenAPIError: - # TODO: protocol when some parts are unavailable because of failure - # Define whether it is critical or this server can still - # continue working offering partial services - log.exception("Invalid rest API specs. Rest API is DISABLED") - specs = None - return specs +def setup(app: web.Application): + """Setup the rest API module in the application in aiohttp fashion. -def get_base_path(specs: openapi.Spec) ->str: - # TODO: guarantee this convention is true - return '/v' + specs.info.version.split('.')[0] + - users "rest" section of configuration (see schema in rest_config.py) + - loads and validate openapi specs from a remote (e.g. apihub) or local location + - connects openapi specs paths to handlers (see rest_routes.py) + - enables error, validation and envelope middlewares on API routes -def setup(app: web.Application): - """Setup the rest API module in the application in aiohttp fashion. """ + IMPORTANT: this is a critical subsystem. Any failure should stop + the system startup. It CANNOT be simply disabled & continue + """ log.debug("Setting up %s ...", __name__) - app_config = app[APP_CONFIG_KEY]['main'] # TODO: define appconfig key based on config schema + cfg = app[APP_CONFIG_KEY][CONFIG_SECTION_NAME] + + # app_config = app[APP_CONFIG_KEY]['main'] # TODO: define appconfig key based on config schema + # api_specs = create_apispecs(app_config) - api_specs = create_apispecs(app_config) + loop = asyncio.get_event_loop() + location = "{}/storage/{}/openapi.yaml".format(cfg["oas_repo"], API_VERSION_TAG) + api_specs = loop.run_until_complete( create_openapi_specs(location) ) - if not api_specs: - log.error("%s service disabled. Invalid specs", __name__) - return + # validated openapi specs + app[APP_OPENAPI_SPECS_KEY] = api_specs - # NOTE: after setup app-keys are all defined, but they might be set to None when they cannot - # be initialized - # TODO: What if many specs to expose? v0, v1, v2 ... perhaps a dict instead? - # TODO: should freeze specs here?? - app[APP_OPENAPI_SPECS_KEY] = api_specs # validated openapi specs + # Connects handlers + routes = rest_routes.create(api_specs) + app.router.add_routes(routes) - #Injects rest middlewares in the application + log.debug("routes: %s", pformat(routes)) + # Enable error, validation and envelop middleware on API routes base_path = get_base_path(api_specs) append_rest_middlewares(app, base_path) - rest_routes.setup(app) - # alias setup_rest = setup diff --git a/services/storage/src/simcore_service_storage/rest_config.py b/services/storage/src/simcore_service_storage/rest_config.py new file mode 100644 index 00000000000..9773decd7f9 --- /dev/null +++ b/services/storage/src/simcore_service_storage/rest_config.py @@ -0,0 +1,16 @@ +""" rest subsystem's configuration + + - config-file schema + - settings +""" +import trafaret as T + +from .settings import APP_OPENAPI_SPECS_KEY + +APP_OPENAPI_SPECS_KEY = APP_OPENAPI_SPECS_KEY + +CONFIG_SECTION_NAME = 'rest' + +schema = T.Dict({ + T.Key("oas_repo"): T.Or(T.String, T.URL), # either path or url should contain version in it +}) diff --git a/services/storage/src/simcore_service_storage/rest_routes.py b/services/storage/src/simcore_service_storage/rest_routes.py index c0bbfa7b505..2c6c17b38f1 100644 --- a/services/storage/src/simcore_service_storage/rest_routes.py +++ b/services/storage/src/simcore_service_storage/rest_routes.py @@ -8,15 +8,14 @@ from aiohttp import web -from servicelib import openapi +from servicelib.openapi import OpenApiSpec from . import handlers -from .settings import APP_OPENAPI_SPECS_KEY log = logging.getLogger(__name__) -def create(specs: openapi.Spec) -> List[web.RouteDef]: +def create(specs: OpenApiSpec) -> List[web.RouteDef]: # TODO: consider the case in which server creates routes for both v0 and v1!!! # TODO: should this be taken from servers instead? BASEPATH = '/v' + specs.info.version.split('.')[0] @@ -67,12 +66,3 @@ def create(specs: openapi.Spec) -> List[web.RouteDef]: return routes - - -def setup(app: web.Application): - valid_specs = app[APP_OPENAPI_SPECS_KEY] - - assert valid_specs, "No API specs in app[%s]. Skipping setup %s "% (APP_OPENAPI_SPECS_KEY, __name__) - - routes = create(valid_specs) - app.router.add_routes(routes) diff --git a/services/storage/src/simcore_service_storage/settings.py b/services/storage/src/simcore_service_storage/settings.py index c2afc918b7a..f3020b26437 100644 --- a/services/storage/src/simcore_service_storage/settings.py +++ b/services/storage/src/simcore_service_storage/settings.py @@ -4,68 +4,58 @@ stages of the development workflow. This submodule gives access to all of them. -""" -import logging -from servicelib import application_keys +Naming convention: -from .__version__ import get_version_object -from .settings_schema import CONFIG_SCHEMA # pylint: disable=W0611 +APP_*_KEY: is a key in app-storage +RQT_*_KEY: is a key in request-storage +RSP_*_KEY: is a key in response-storage -log = logging.getLogger(__name__) +See https://docs.aiohttp.org/en/stable/web_advanced.html#data-sharing-aka-no-singletons-please +""" -## CONSTANTS-------------------- -TIMEOUT_IN_SECS = 2 -RESOURCE_KEY_OPENAPI = "oas3/v0" +import logging +from servicelib import application_keys -DEFAULT_CONFIG='docker-prod-config.yaml' +# IMPORTANT: lowest level module +# I order to avoid cyclic dependences, please +# DO NOT IMPORT ANYTHING from . (except for __version__) +from .__version__ import get_version_object -## BUILD ------------------------ -# - Settings revealed at build/installation time -# - Only known after some setup or build step is completed -PACKAGE_VERSION = get_version_object() -API_MAJOR_VERSION = PACKAGE_VERSION.major -API_URL_VERSION = "v{:.0f}".format(API_MAJOR_VERSION) +log = logging.getLogger(__name__) -## KEYS ------------------------- -# TODO: test no key collisions -# Keys used in different scopes. Common naming format: -# -# $(SCOPE)_$(NAME)_KEY -# +## CONSTANTS-------------------- +RETRY_WAIT_SECS = 2 +RETRY_COUNT = 20 +CONNECT_TIMEOUT_SECS = 30 -# APP=application -APP_CONFIG_KEY = application_keys.APP_CONFIG_KEY -APP_OPENAPI_SPECS_KEY = application_keys.APP_OPENAPI_SPECS_KEY +## VERSION----------------------------- +service_version = get_version_object() -APP_DB_ENGINE_KEY = 'db_engine' -APP_DB_SESSION_KEY = 'db_session' -APP_DSM_THREADPOOL = "dsm_threadpool" +## CONFIGURATION FILES------------------ +DEFAULT_CONFIG='docker-prod-config.yaml' -# CFG=configuration +APP_CONFIG_KEY = application_keys.APP_CONFIG_KEY # app-storage-key for config object +RSC_CONFIG_DIR_KEY = "data" # resource folder -# RSC=resource -RSC_OPENAPI_DIR_KEY = "oas3/{}".format(API_URL_VERSION) -RSC_OPENAPI_ROOTFILE_KEY = "{}/openapi.yaml".format(RSC_OPENAPI_DIR_KEY) -RSC_CONFIG_DIR_KEY = "data" -RSC_CONFIG_SCHEMA_KEY = RSC_CONFIG_DIR_KEY + "/config-schema-v1.json" -# RQT=request -RQT_DSM_KEY = "DSM" +# REST API ---------------------------- +API_MAJOR_VERSION = service_version.major # NOTE: syncs with service key +API_VERSION_TAG = "v{:.0f}".format(API_MAJOR_VERSION) -# RSP=response +APP_OPENAPI_SPECS_KEY = application_keys.APP_OPENAPI_SPECS_KEY # app-storage-key for openapi specs object -## Settings revealed at runtime: only known when the application starts -# - via the config file passed to the cli +# DATABASE ---------------------------- +APP_DB_ENGINE_KEY = __name__ + '.db_engine' +APP_DB_SESSION_KEY = __name__ + '.db_session' -OAS_ROOT_FILE = "{}/openapi.yaml".format(RSC_OPENAPI_DIR_KEY) # TODO: delete +# DATA STORAGE MANAGER ---------------------------------- +APP_DSM_THREADPOOL = __name__ + '.dsm_threadpool' -__all__ = ( - 'CONFIG_SCHEMA', # TODO: fill with proper values -) +RQT_DSM_KEY = "DSM" # request-storage-key diff --git a/services/storage/src/simcore_service_storage/utils.py b/services/storage/src/simcore_service_storage/utils.py deleted file mode 100644 index e99707fd217..00000000000 --- a/services/storage/src/simcore_service_storage/utils.py +++ /dev/null @@ -1,9 +0,0 @@ -import yaml - -from .resources import resources -from .settings import OAS_ROOT_FILE - - -def api_version() -> str: - specs = yaml.load(resources.stream(OAS_ROOT_FILE)) - return specs['info']['version'] diff --git a/services/storage/tests/test_resources.py b/services/storage/tests/test_resources.py index 3e20ad1e1f8..337d934896f 100644 --- a/services/storage/tests/test_resources.py +++ b/services/storage/tests/test_resources.py @@ -10,15 +10,14 @@ # under test from simcore_service_storage.resources import resources -from simcore_service_storage.settings import (RSC_CONFIG_DIR_KEY, - RSC_OPENAPI_DIR_KEY) +from simcore_service_storage.settings import RSC_CONFIG_DIR_KEY log = logging.getLogger(__name__) @pytest.fixture def app_resources(package_dir): resource_names = [] - for name in (RSC_CONFIG_DIR_KEY, RSC_OPENAPI_DIR_KEY): + for name in (RSC_CONFIG_DIR_KEY, ): folder = package_dir / name resource_names += [ str(p.relative_to(package_dir)) for p in folder.rglob("*.y*ml") ] From 552e60109adbdbf6aab0a948348abbae3e5a6e32 Mon Sep 17 00:00:00 2001 From: Pedro Crespo Date: Fri, 23 Nov 2018 20:33:14 +0100 Subject: [PATCH 03/10] servicelib: Added some utils to find repo dirs --- .../service-library/src/servicelib/utils.py | 29 +++++++++++++++++++ .../service-library/tests/test_package.py | 13 +++++++++ 2 files changed, 42 insertions(+) create mode 100644 packages/service-library/src/servicelib/utils.py diff --git a/packages/service-library/src/servicelib/utils.py b/packages/service-library/src/servicelib/utils.py new file mode 100644 index 00000000000..7fbbc3e04a0 --- /dev/null +++ b/packages/service-library/src/servicelib/utils.py @@ -0,0 +1,29 @@ +""" General utils + +IMPORTANT: lowest level module + I order to avoid cyclic dependences, please + DO NOT IMPORT ANYTHING from . +""" +from pathlib import Path + + +def is_osparc_repo_dir(path: Path) -> bool: + # TODO: implement with git cli + expected = (".github", "packages", "services") + got = [p.name for p in path.iterdir() if p.is_dir()] + return all(d in got for d in expected) + + +def search_osparc_repo_dir(start, max_iterations=8): + """ Returns path to root repo dir or None if it does not exists + + NOTE: assumes starts is a path within repo + """ + max_iterations = max(max_iterations, 1) + root_dir = Path(start) + it = 1 + while not is_osparc_repo_dir(root_dir) and it Date: Fri, 23 Nov 2018 20:36:12 +0100 Subject: [PATCH 04/10] storage/tests/test_config passes - Adds workspace variables in config-files - Update config files with new section - Adds APIHUB_HOST, _PORT in storage docker-compose - Fixes duplication of docker-compose storage service --- services/docker-compose.yml | 32 ++----------------- .../src/simcore_service_storage/cli.py | 26 ++++++++++++++- .../data/docker-dev-config.yaml | 6 ++-- .../data/docker-prod-config.yaml | 5 +-- .../data/host-dev-config.yaml | 5 ++- services/storage/tests/test_configs.py | 8 +++-- 6 files changed, 43 insertions(+), 39 deletions(-) diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 2a2aa8e4dd8..e1d4ca9e817 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -148,36 +148,8 @@ services: ports: - "11111:8080" environment: - - POSTGRES_ENDPOINT=${POSTGRES_ENDPOINT} - - POSTGRES_USER=${POSTGRES_USER} - - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - - POSTGRES_DB=${POSTGRES_DB} - - POSTGRES_HOST=${POSTGRES_HOST} - - POSTGRES_PORT=${POSTGRES_PORT} - - S3_ENDPOINT=${S3_ENDPOINT} - - S3_ACCESS_KEY=${S3_ACCESS_KEY} - - S3_SECRET_KEY=${S3_SECRET_KEY} - - S3_BUCKET_NAME=${S3_BUCKET_NAME} - - STORAGE_ENDPOINT=${STORAGE_ENDPOINT} - - RUN_DOCKER_ENGINE_ROOT=${RUN_DOCKER_ENGINE_ROOT} - - BF_API_SECRET=${BF_API_SECRET} - - BF_API_KEY=${BF_API_KEY} - depends_on: - - minio - - postgres - #-------------------------------------------------------------------- - storage: - build: - # the context for the build is the git repo root directory, this allows to copy - # the packages directory into any docker image - context: ../ - dockerfile: services/storage/Dockerfile - args: - - DOCKER_GID_ARG=${DOCKER_GID:?Undefined docker gid in host} - target: production - ports: - - "11111:8080" - environment: + - APIHUB_HOST=apihub + - APIHUB_PORT=8043 - POSTGRES_ENDPOINT=${POSTGRES_ENDPOINT} - POSTGRES_USER=${POSTGRES_USER} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} diff --git a/services/storage/src/simcore_service_storage/cli.py b/services/storage/src/simcore_service_storage/cli.py index 4cb5d481133..d27cda3ec6c 100644 --- a/services/storage/src/simcore_service_storage/cli.py +++ b/services/storage/src/simcore_service_storage/cli.py @@ -15,12 +15,36 @@ import argparse import logging import sys +import os from . import application, cli_config +from servicelib.utils import search_osparc_repo_dir + log = logging.getLogger(__name__) +def create_environ(skip_system_environ=False): + """ + Build environment of substitutable variables + + """ + # system's environment variables + environ = {} if skip_system_environ else dict(os.environ) + + # project-related environment variables + here = os.path.dirname(__file__) + environ['THIS_PACKAGE_DIR'] = here + + rootdir = search_osparc_repo_dir(start=here) + if rootdir is not None: + environ['OSPARC_SIMCORE_REPO_ROOTDIR'] = str(rootdir) + + + return environ + + + def setup(_parser): cli_config.add_cli_options(_parser) return _parser @@ -33,7 +57,7 @@ def parse(args, _parser): # ignore unknown options options, _ = _parser.parse_known_args(args) - config = cli_config.config_from_options(options) + config = cli_config.config_from_options(options, vars=create_environ()) # TODO: check whether extra options can be added to the config?! return config diff --git a/services/storage/src/simcore_service_storage/data/docker-dev-config.yaml b/services/storage/src/simcore_service_storage/data/docker-dev-config.yaml index 8f52ce388a8..b3d656c8bde 100644 --- a/services/storage/src/simcore_service_storage/data/docker-dev-config.yaml +++ b/services/storage/src/simcore_service_storage/data/docker-dev-config.yaml @@ -1,5 +1,5 @@ +version: '1.0' main: -# disable_services: ['postgres', 's3'] host: 0.0.0.0 log_level: INFO port: 8080 @@ -21,4 +21,6 @@ s3: access_key: ${S3_ACCESS_KEY} secret_key: ${S3_SECRET_KEY} bucket_name: ${S3_BUCKET_NAME} -version: '1.0' +rest: + oas_repo: ${OSPARC_SIMCORE_REPO_ROOTDIR}/api/specs + #oas_repo: http://0.0.0.0:8043/api/specs diff --git a/services/storage/src/simcore_service_storage/data/docker-prod-config.yaml b/services/storage/src/simcore_service_storage/data/docker-prod-config.yaml index aac152f2d12..96c8a251887 100644 --- a/services/storage/src/simcore_service_storage/data/docker-prod-config.yaml +++ b/services/storage/src/simcore_service_storage/data/docker-prod-config.yaml @@ -1,5 +1,5 @@ +version: '1.0' main: -# disable_services: ['postgres', 's3'] host: 0.0.0.0 log_level: INFO port: 8080 @@ -20,4 +20,5 @@ s3: access_key: ${S3_ACCESS_KEY} secret_key: ${S3_SECRET_KEY} bucket_name: ${S3_BUCKET_NAME} -version: '1.0' +rest: + oas_repo: http://${APIHUB_HOST}:${APIHUB_PORT}/api/specs diff --git a/services/storage/src/simcore_service_storage/data/host-dev-config.yaml b/services/storage/src/simcore_service_storage/data/host-dev-config.yaml index 2d80e5f77f7..e338c057d81 100644 --- a/services/storage/src/simcore_service_storage/data/host-dev-config.yaml +++ b/services/storage/src/simcore_service_storage/data/host-dev-config.yaml @@ -1,3 +1,4 @@ +version: '1.0' main: disable_services: ['postgres', 's3'] host: 127.0.0.1 @@ -22,4 +23,6 @@ s3: bucket_name: simcore endpoint: minio:9000 secret_key: '12345678' -version: '1.0' +rest: + oas_repo: ${OSPARC_SIMCORE_REPO_ROOTDIR}/api/specs + #oas_repo: http://localhost:8043/api/specs diff --git a/services/storage/tests/test_configs.py b/services/storage/tests/test_configs.py index 3dbf8b654da..650109fbd0e 100644 --- a/services/storage/tests/test_configs.py +++ b/services/storage/tests/test_configs.py @@ -41,6 +41,7 @@ def devel_environ(env_devel_file): env_devel[key] = value return env_devel +from simcore_service_storage.cli import create_environ @pytest.fixture("session") def container_environ(services_docker_compose_file, devel_environ): @@ -51,11 +52,12 @@ def container_environ(services_docker_compose_file, devel_environ): with services_docker_compose_file.open() as f: dc = yaml.safe_load(f) - container_environ = { + container_environ = create_environ(skip_system_environ=True) + container_environ.update({ 'VENV2': '/home/scu/.venv27/' # defined in Dockerfile - } + }) - environ_items =dc["services"][THIS_SERVICE].get("environment", list()) + environ_items = dc["services"][THIS_SERVICE].get("environment", list()) MATCH = re.compile(r'\$\{(\w+)+') for item in environ_items: From 07f0154ff47be6f0e1da058a94fb7d88f6b31a3d Mon Sep 17 00:00:00 2001 From: Pedro Crespo Date: Fri, 23 Nov 2018 20:42:04 +0100 Subject: [PATCH 05/10] Fixes pytest-asyncio loop issue in storage Copied changes in storage files from PR #355 - commit ff814d9d03547ccbbafec06e64a3ac27b8496b67 --- .../src/simcore_service_storage/handlers.py | 1 + services/storage/tests/conftest.py | 137 ++++++++++-------- services/storage/tests/requirements.txt | 4 +- services/storage/tests/test_dsm.py | 27 ++-- 4 files changed, 88 insertions(+), 81 deletions(-) diff --git a/services/storage/src/simcore_service_storage/handlers.py b/services/storage/src/simcore_service_storage/handlers.py index a6329943306..da8f2c40d0a 100644 --- a/services/storage/src/simcore_service_storage/handlers.py +++ b/services/storage/src/simcore_service_storage/handlers.py @@ -68,6 +68,7 @@ async def check_action(request: web.Request): } return output + async def get_storage_locations(request: web.Request): log.info("CHECK LOCATION PATH %s %s",request.path, request.url) diff --git a/services/storage/tests/conftest.py b/services/storage/tests/conftest.py index 9b72a40419d..84a7d4c551c 100644 --- a/services/storage/tests/conftest.py +++ b/services/storage/tests/conftest.py @@ -1,9 +1,9 @@ -# TODO: W0611:Unused import ... -# pylint: disable=W0611 -# TODO: W0613:Unused argument ... -# pylint: disable=W0613 -# -# pylint: disable=W0621 +# pylint:disable=wildcard-import +# pylint:disable=unused-import +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name + import asyncio import os import subprocess @@ -26,24 +26,27 @@ SIMCORE_S3_STR) from utils import ACCESS_KEY, BUCKET_NAME, DATABASE, PASS, SECRET_KEY, USER -# fixtures ------------------------------------------------------- + @pytest.fixture(scope='session') def here(): return Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent + @pytest.fixture(scope='session') def package_dir(here): dirpath = Path(simcore_service_storage.__file__).parent assert dirpath.exists() return dirpath + @pytest.fixture(scope='session') def osparc_simcore_root_dir(here): root_dir = here.parent.parent.parent assert root_dir.exists(), "Is this service within osparc-simcore repo?" return root_dir + @pytest.fixture(scope='session') def python27_exec(osparc_simcore_root_dir, tmpdir_factory, here): # Assumes already created with make .venv27 @@ -52,16 +55,18 @@ def python27_exec(osparc_simcore_root_dir, tmpdir_factory, here): if not venv27.exists(): # create its own virtualenv venv27 = tmpdir_factory.mktemp("virtualenv") / ".venv27" - cmd = "virtualenv --python=python2 %s"%(venv27) # TODO: how to split in command safely? - assert subprocess.check_call(cmd.split()) == 0, "Unable to run %s" %cmd + # TODO: how to split in command safely? + cmd = "virtualenv --python=python2 %s" % (venv27) + assert subprocess.check_call( + cmd.split()) == 0, "Unable to run %s" % cmd # installs python2 requirements pip_exec = venv27 / "bin" / "pip" assert pip_exec.exists() requirements_py2 = here.parent / "requirements/py27.txt" cmd = "{} install -r {}".format(pip_exec, requirements_py2) - assert subprocess.check_call(cmd.split()) == 0, "Unable to run %s" %cmd - + assert subprocess.check_call( + cmd.split()) == 0, "Unable to run %s" % cmd python27_exec = venv27 / "bin" / "python2.7" assert python27_exec.exists() @@ -73,6 +78,7 @@ def python27_path(python27_exec): return Path(python27_exec).parent.parent # Assumes already created with make .venv27 + @pytest.fixture(scope='session') def docker_compose_file(here): """ Overrides pytest-docker fixture @@ -80,12 +86,12 @@ def docker_compose_file(here): old = os.environ.copy() # docker-compose reads these environs - os.environ['POSTGRES_DB']=DATABASE - os.environ['POSTGRES_USER']=USER - os.environ['POSTGRES_PASSWORD']=PASS - os.environ['POSTGRES_ENDPOINT']="FOO" # TODO: update config schema!! - os.environ['MINIO_ACCESS_KEY']=ACCESS_KEY - os.environ['MINIO_SECRET_KEY']=SECRET_KEY + os.environ['POSTGRES_DB'] = DATABASE + os.environ['POSTGRES_USER'] = USER + os.environ['POSTGRES_PASSWORD'] = PASS + os.environ['POSTGRES_ENDPOINT'] = "FOO" # TODO: update config schema!! + os.environ['MINIO_ACCESS_KEY'] = ACCESS_KEY + os.environ['MINIO_SECRET_KEY'] = SECRET_KEY dc_path = here / 'docker-compose.yml' @@ -94,12 +100,13 @@ def docker_compose_file(here): os.environ = old + @pytest.fixture(scope='session') def postgres_service(docker_services, docker_ip): url = 'postgresql://{user}:{password}@{host}:{port}/{database}'.format( - user = USER, - password = PASS, - database = DATABASE, + user=USER, + password=PASS, + database=DATABASE, host=docker_ip, port=docker_services.port_for('postgres', 5432), ) @@ -112,27 +119,29 @@ def postgres_service(docker_services, docker_ip): ) postgres_service = { - 'user' : USER, - 'password' : PASS, - 'database' : DATABASE, - 'host' : docker_ip, - 'port' : docker_services.port_for('postgres', 5432) + 'user': USER, + 'password': PASS, + 'database': DATABASE, + 'host': docker_ip, + 'port': docker_services.port_for('postgres', 5432) } return postgres_service + @pytest.fixture(scope='session') def postgres_service_url(postgres_service, docker_services, docker_ip): postgres_service_url = 'postgresql://{user}:{password}@{host}:{port}/{database}'.format( - user = USER, - password = PASS, - database = DATABASE, + user=USER, + password=PASS, + database=DATABASE, host=docker_ip, port=docker_services.port_for('postgres', 5432), ) return postgres_service_url + @pytest.fixture(scope='function') async def postgres_engine(loop, postgres_service_url): postgres_engine = await create_engine(postgres_service_url) @@ -163,24 +172,28 @@ def minio_service(docker_services, docker_ip): return { 'endpoint': '{ip}:{port}'.format(ip=docker_ip, port=docker_services.port_for('minio', 9000)), 'access_key': ACCESS_KEY, - 'secret_key' : SECRET_KEY, - 'bucket_name' : BUCKET_NAME, - } + 'secret_key': SECRET_KEY, + 'bucket_name': BUCKET_NAME, + } + @pytest.fixture(scope="module") def s3_client(minio_service): from s3wrapper.s3_client import S3Client - s3_client = S3Client(endpoint=minio_service['endpoint'],access_key=minio_service["access_key"], secret_key=minio_service["secret_key"]) + s3_client = S3Client( + endpoint=minio_service['endpoint'], access_key=minio_service["access_key"], secret_key=minio_service["secret_key"]) return s3_client + @pytest.fixture(scope="function") def mock_files_factory(tmpdir_factory): def _create_files(count): filepaths = [] for _i in range(count): name = str(uuid.uuid4()) - filepath = os.path.normpath(str(tmpdir_factory.mktemp('data').join(name + ".txt"))) + filepath = os.path.normpath( + str(tmpdir_factory.mktemp('data').join(name + ".txt"))) with open(filepath, 'w') as fout: fout.write("Hello world\n") filepaths.append(filepath) @@ -198,10 +211,11 @@ def dsm_mockup_db(postgres_service_url, s3_client, mock_files_factory): bucket_name = BUCKET_NAME s3_client.create_bucket(bucket_name, delete_contents_if_exists=True) - #TODO: use pip install Faker - users = [ 'alice', 'bob', 'chuck', 'dennis'] + # TODO: use pip install Faker + users = ['alice', 'bob', 'chuck', 'dennis'] - projects = ['astronomy', 'biology', 'chemistry', 'dermatology', 'economics', 'futurology', 'geology'] + projects = ['astronomy', 'biology', 'chemistry', + 'dermatology', 'economics', 'futurology', 'geology'] location = SIMCORE_S3_STR nodes = ['alpha', 'beta', 'gamma', 'delta'] @@ -214,41 +228,43 @@ def dsm_mockup_db(postgres_service_url, s3_client, mock_files_factory): idx = randrange(len(users)) user_name = users[idx] user_id = idx + 10 - idx = randrange(len(projects)) + idx = randrange(len(projects)) project_name = projects[idx] project_id = idx + 100 - idx = randrange(len(nodes)) + idx = randrange(len(nodes)) node = nodes[idx] node_id = idx + 10000 file_name = str(counter) - object_name = Path(str(project_id), str(node_id), str(counter)).as_posix() + object_name = Path(str(project_id), str( + node_id), str(counter)).as_posix() file_uuid = Path(object_name).as_posix() assert s3_client.upload_file(bucket_name, object_name, _file) - d = { 'file_uuid' : file_uuid, - 'location_id' : "0", - 'location' : location, - 'bucket_name' : bucket_name, - 'object_name' : object_name, - 'project_id' : str(project_id), - 'project_name' : project_name, - 'node_id' : str(node_id), - 'node_name' : node, - 'file_name' : file_name, - 'user_id' : str(user_id), - 'user_name' : user_name - } + d = {'file_uuid': file_uuid, + 'location_id': "0", + 'location': location, + 'bucket_name': bucket_name, + 'object_name': object_name, + 'project_id': str(project_id), + 'project_name': project_name, + 'node_id': str(node_id), + 'node_name': node, + 'file_name': file_name, + 'user_id': str(user_id), + 'user_name': user_name + } counter = counter + 1 data[object_name] = FileMetaData(**d) - utils.insert_metadata(postgres_service_url, data[object_name]) #pylint: disable=no-member - + # pylint: disable=no-member + utils.insert_metadata(postgres_service_url, + data[object_name]) total_count = 0 - for _obj in s3_client.list_objects_v2(bucket_name, recursive = True): + for _obj in s3_client.list_objects_v2(bucket_name, recursive=True): total_count = total_count + 1 assert total_count == N @@ -260,10 +276,6 @@ def dsm_mockup_db(postgres_service_url, s3_client, mock_files_factory): # db utils.drop_tables(url=postgres_service_url) -# This is weird, somehow the default loop gives problems with pytest asyncio, so lets override it -@pytest.fixture -def loop(event_loop): - return event_loop @pytest.fixture(scope="function") async def datcore_testbucket(loop, python27_exec, mock_files_factory): @@ -282,19 +294,20 @@ async def datcore_testbucket(loop, python27_exec, mock_files_factory): ready = False counter = 0 - while not ready and counter<5: + while not ready and counter < 5: data = await dcw.list_files() ready = len(data) == 2 await asyncio.sleep(10) counter = counter + 1 - yield BUCKET_NAME await dcw.delete_test_dataset(BUCKET_NAME) + @pytest.fixture(scope="function") def dsm_fixture(s3_client, python27_exec, postgres_engine, loop): pool = ThreadPoolExecutor(3) - dsm_fixture = DataStorageManager(s3_client, python27_exec, postgres_engine, loop, pool, BUCKET_NAME) + dsm_fixture = DataStorageManager( + s3_client, python27_exec, postgres_engine, loop, pool, BUCKET_NAME) return dsm_fixture diff --git a/services/storage/tests/requirements.txt b/services/storage/tests/requirements.txt index 8f86b464b12..6161e06b3ea 100644 --- a/services/storage/tests/requirements.txt +++ b/services/storage/tests/requirements.txt @@ -5,9 +5,11 @@ coveralls pytest pytest-aiohttp -pytest-asyncio pytest-cov pytest-docker openapi_spec_validator pyyaml virtualenv + +# NOTE: pytest-aiohttp and pytest-asyncio incompatible +# https://github.com/pytest-dev/pytest-asyncio/issues/76 diff --git a/services/storage/tests/test_dsm.py b/services/storage/tests/test_dsm.py index 15b168854be..7194e477b0c 100644 --- a/services/storage/tests/test_dsm.py +++ b/services/storage/tests/test_dsm.py @@ -1,7 +1,9 @@ -# TODO: W0611:Unused import ... -# pylint: disable=W0611 -# TODO: W0613:Unused argument ... -# pylint: disable=W0613 +# pylint:disable=wildcard-import +# pylint:disable=unused-import +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name +# pylint: disable=too-many-arguments import filecmp import io @@ -10,9 +12,8 @@ import pdb import urllib import uuid -from pprint import pprint - from pathlib import Path +from pprint import pprint import attr import pytest @@ -20,7 +21,8 @@ import utils from simcore_service_storage.dsm import DataStorageManager from simcore_service_storage.models import FileMetaData -from simcore_service_storage.s3 import DATCORE_STR, SIMCORE_S3_STR, SIMCORE_S3_ID +from simcore_service_storage.s3 import (DATCORE_STR, SIMCORE_S3_ID, + SIMCORE_S3_STR) from utils import BUCKET_NAME @@ -176,7 +178,6 @@ async def test_copy_s3_s3(postgres_service_url, s3_client, mock_files_factory, d assert len(data) == 2 #NOTE: Below tests directly access the datcore platform, use with care! - @pytest.mark.travis def test_datcore_fixture(datcore_testbucket): print(datcore_testbucket) @@ -198,8 +199,6 @@ async def test_dsm_datcore(postgres_service_url, dsm_fixture, datcore_testbucket data = await dsm.list_files(user_id=user_id, location=DATCORE_STR) assert len(data) == 1 -# pylint: disable=R0913 -# Too many arguments @pytest.mark.travis async def test_dsm_s3_to_datcore(postgres_service_url, s3_client, mock_files_factory, dsm_fixture, datcore_testbucket): utils.create_tables(url=postgres_service_url) @@ -230,8 +229,6 @@ async def test_dsm_s3_to_datcore(postgres_service_url, s3_client, mock_files_fac # there should now be 3 files assert len(data) == 3 -# pylint: disable=R0913 -# Too many arguments @pytest.mark.travis async def test_dsm_datcore_to_local(postgres_service_url, dsm_fixture, mock_files_factory, datcore_testbucket): utils.create_tables(url=postgres_service_url) @@ -250,8 +247,6 @@ async def test_dsm_datcore_to_local(postgres_service_url, dsm_fixture, mock_file assert filecmp.cmp(tmp_file2, tmp_file) -# pylint: disable=R0913 -# Too many arguments @pytest.mark.travis async def test_dsm_datcore_to_S3(postgres_service_url, s3_client, dsm_fixture, mock_files_factory, datcore_testbucket): utils.create_tables(url=postgres_service_url) @@ -287,10 +282,6 @@ async def test_dsm_datcore_to_S3(postgres_service_url, s3_client, dsm_fixture, m assert filecmp.cmp(tmp_file1, tmp_file2) - - -# pylint: disable=R0913 -# Too many arguments @pytest.mark.travis async def test_copy_datcore(postgres_service_url, s3_client, dsm_fixture, mock_files_factory, datcore_testbucket): utils.create_tables(url=postgres_service_url) From e6dedafc4a1ba537b6b45046a4df5e80715f45ee Mon Sep 17 00:00:00 2001 From: Pedro Crespo Date: Fri, 23 Nov 2018 21:31:08 +0100 Subject: [PATCH 06/10] Fixes openapi servers in specs adapting tests: - Adds new environ variables in tests fixtures - adds new config section --- api/specs/storage/v0/openapi.yaml | 13 ++-- .../src/simcore_service_storage/handlers.py | 1 - .../src/simcore_service_storage/rest.py | 66 +------------------ services/storage/tests/conftest.py | 7 +- services/storage/tests/test_openapi.py | 36 ---------- services/storage/tests/test_rest.py | 36 ++++++---- 6 files changed, 37 insertions(+), 122 deletions(-) delete mode 100644 services/storage/tests/test_openapi.py diff --git a/api/specs/storage/v0/openapi.yaml b/api/specs/storage/v0/openapi.yaml index 3db8c83b135..598ec503c16 100644 --- a/api/specs/storage/v0/openapi.yaml +++ b/api/specs/storage/v0/openapi.yaml @@ -10,20 +10,19 @@ info: name: MIT url: https://github.com/ITISFoundation/osparc-simcore/blob/master/LICENSE servers: + - description: API server + url: '/v0' - description: Development server - url: 'http://{host}:{port}/{basePath}' + url: http://{host}:{port}/{basePath} variables: host: default: 'localhost' port: default: '11111' basePath: - default: 'v0' - - description: Production server - url: 'http://storage:{port}/v0' - variables: - port: - default: '8080' + enum: + - v0 + default: v0 tags: - name: admins description: Secured Admin-only calls diff --git a/services/storage/src/simcore_service_storage/handlers.py b/services/storage/src/simcore_service_storage/handlers.py index da8f2c40d0a..3575bd7bb6b 100644 --- a/services/storage/src/simcore_service_storage/handlers.py +++ b/services/storage/src/simcore_service_storage/handlers.py @@ -25,7 +25,6 @@ async def check_health(request: web.Request): log.info("CHECK HEALTH INCOMING PATH %s",request.path) - params, query, body = await extract_and_validate(request) assert not params diff --git a/services/storage/src/simcore_service_storage/rest.py b/services/storage/src/simcore_service_storage/rest.py index 70b13c44395..c8a83555e1e 100644 --- a/services/storage/src/simcore_service_storage/rest.py +++ b/services/storage/src/simcore_service_storage/rest.py @@ -2,14 +2,10 @@ """ import asyncio -import copy import logging -from pprint import pformat -from typing import Dict from aiohttp import web -from servicelib import openapi from servicelib.openapi import create_openapi_specs, get_base_path from servicelib.rest_middlewares import append_rest_middlewares @@ -20,66 +16,6 @@ log = logging.getLogger(__name__) - -#TODO: move to servicelib -def _get_server(servers, url): - # Development server: http://{host}:{port}/{basePath} - for server in servers: - if server.url == url: - return server - raise ValueError("Cannot find server %s" % url) - -def _setup_servers_specs(specs: openapi.Spec, app_config: Dict) -> openapi.Spec: - # TODO: temporary solution. Move to servicelib. Modifying dynamically servers does not seem like - # the best solution! - - if app_config.get('testing', True): - # FIXME: host/port in host side! - # - server running inside container. use environ set by container to find port maps maps (see portainer) - # - server running in host - - devserver = _get_server(specs.servers, "http://{host}:{port}/{basePath}") - host, port = app_config['host'], app_config['port'] - - devserver.variables['host'].default = host - devserver.variables['port'].default = port - - # Extends server specs to locahosts - for host in {'127.0.0.1', 'localhost', host}: - for port in {port, 11111, 8080}: - log.info("Extending to server %s:%s", host, port) - new_server = copy.deepcopy(devserver) - new_server.variables['host'].default = host - new_server.variables['port'].default = port - specs.servers.append(new_server) - - for s in specs.servers: - if 'host' in s.variables.keys(): - log.info("SERVER SPEC %s:%s", s.variables['host'].default, s.variables['port'].default) - else: - log.info("SERVER SPEC storage :%s", s.variables['port'].default) - - - return specs - - -# def create_apispecs(app_config: Dict) -> openapi.Spec: -# # TODO: What if many specs to expose? v0, v1, v2 ... -# openapi_path = resources.get_path(RSC_OPENAPI_ROOTFILE_KEY) - -# try: -# specs = openapi.create_specs(openapi_path) -# specs = _setup_servers_specs(specs, app_config) - -# except openapi.OpenAPIError: -# # TODO: protocol when some parts are unavailable because of failure -# # Define whether it is critical or this server can still -# # continue working offering partial services -# log.exception("Invalid rest API specs. Rest API is DISABLED") -# specs = None -# return specs - - def setup(app: web.Application): """Setup the rest API module in the application in aiohttp fashion. @@ -110,7 +46,7 @@ def setup(app: web.Application): routes = rest_routes.create(api_specs) app.router.add_routes(routes) - log.debug("routes: %s", pformat(routes)) + log.debug("routes:\n\t%s", "\n\t".join(map(str, routes)) ) # Enable error, validation and envelop middleware on API routes base_path = get_base_path(api_specs) diff --git a/services/storage/tests/conftest.py b/services/storage/tests/conftest.py index 84a7d4c551c..1a4701a39aa 100644 --- a/services/storage/tests/conftest.py +++ b/services/storage/tests/conftest.py @@ -43,9 +43,14 @@ def package_dir(here): @pytest.fixture(scope='session') def osparc_simcore_root_dir(here): root_dir = here.parent.parent.parent - assert root_dir.exists(), "Is this service within osparc-simcore repo?" + assert root_dir.exists() and any(root_dir.glob("services")), "Is this service within osparc-simcore repo?" return root_dir +@pytest.fixture(scope='session') +def osparc_api_specs_dir(osparc_simcore_root_dir): + dirpath = osparc_simcore_root_dir / "api" / "specs" + assert dirpath.exists() + return dirpath @pytest.fixture(scope='session') def python27_exec(osparc_simcore_root_dir, tmpdir_factory, here): diff --git a/services/storage/tests/test_openapi.py b/services/storage/tests/test_openapi.py deleted file mode 100644 index 42ca997245e..00000000000 --- a/services/storage/tests/test_openapi.py +++ /dev/null @@ -1,36 +0,0 @@ -# W0621:Redefining name 'spec_basepath' from outer scope (line 14) -# pylint: disable=W0621 - -from pathlib import Path - -import pkg_resources -import pytest -import yaml -from openapi_spec_validator import validate_spec -from openapi_spec_validator.exceptions import OpenAPIValidationError - -import simcore_service_storage - -API_VERSIONS = ('v0', ) - -@pytest.fixture -def spec_basepath(): - basepath = Path(pkg_resources.resource_filename(simcore_service_storage.__name__, 'oas3')) - assert basepath.exists() - return basepath - - -@pytest.mark.parametrize('version', API_VERSIONS) -def test_specifications(spec_basepath, version): - - spec_path = spec_basepath / "{}/openapi.yaml".format(version) - - with spec_path.open() as fh: - specs = yaml.load(fh) - try: - validate_spec(specs, spec_url=spec_path.as_uri()) - except OpenAPIValidationError as err: - pytest.fail(err.message) - - -# TODO: test that all response schemas are enveloped diff --git a/services/storage/tests/test_rest.py b/services/storage/tests/test_rest.py index ff5b4b9eca3..c723d340bbe 100644 --- a/services/storage/tests/test_rest.py +++ b/services/storage/tests/test_rest.py @@ -29,20 +29,30 @@ def parse_db(dsm_mockup_db): return id_file_count, id_name_map @pytest.fixture -def client(loop, aiohttp_unused_port, aiohttp_client, python27_path, postgres_service, minio_service): +def client(loop, aiohttp_unused_port, aiohttp_client, python27_path, postgres_service, minio_service, osparc_api_specs_dir): app = web.Application() - max_workers = 4 - - server_kwargs={'port': aiohttp_unused_port(), 'host': 'localhost', 'python2' : python27_path, "max_workers" : max_workers } - - postgres_kwargs = postgres_service - - s3_kwargs = minio_service + main_cfg = { + 'port': aiohttp_unused_port(), + 'host': 'localhost', + 'python2': python27_path, + "max_workers" : 4 + } + rest_cfg = { + 'oas_repo': str(osparc_api_specs_dir), #'${OSPARC_SIMCORE_REPO_ROOTDIR}/api/specs', + #oas_repo: http://localhost:8043/api/specs + } + postgres_cfg = postgres_service + s3_cfg = minio_service - # fake main - app[APP_CONFIG_KEY] = { 'main': server_kwargs, 'postgres' : postgres_kwargs, "s3" : s3_kwargs } # Fake config + # fake config + app[APP_CONFIG_KEY] = { + 'main': main_cfg, + 'postgres' : postgres_cfg, + 's3' : s3_cfg, + 'rest': rest_cfg + } app.middlewares.append(dsm_middleware) @@ -53,12 +63,14 @@ def client(loop, aiohttp_unused_port, aiohttp_client, python27_path, postgres_se assert "SECRET_KEY" in app[APP_CONFIG_KEY] - cli = loop.run_until_complete( aiohttp_client(app, server_kwargs=server_kwargs) ) + cli = loop.run_until_complete( aiohttp_client(app, server_kwargs=main_cfg) ) return cli async def test_health_check(client): resp = await client.get("/v0/") - assert resp.status == 200 + text = await resp.text() + + assert resp.status == 200, text payload = await resp.json() data, error = tuple( payload.get(k) for k in ('data', 'error') ) From 58c180d94fcd413658bdca87ea196687a1caaff6 Mon Sep 17 00:00:00 2001 From: Pedro Crespo Date: Mon, 26 Nov 2018 17:15:02 +0100 Subject: [PATCH 07/10] tests cleanup: with pytest-aiohttp no need to mark async tests --- .../simcore-sdk/tests/node_ports/test_item.py | 26 ++++----- .../tests/node_ports/test_itemstlist.py | 20 +++---- .../tests/node_ports/test_nodeports.py | 58 +++++++++++-------- .../director/tests/test_registry_proxy.py | 26 ++++----- 4 files changed, 70 insertions(+), 60 deletions(-) diff --git a/packages/simcore-sdk/tests/node_ports/test_item.py b/packages/simcore-sdk/tests/node_ports/test_item.py index 88d0f5b25c3..9fed02319b7 100644 --- a/packages/simcore-sdk/tests/node_ports/test_item.py +++ b/packages/simcore-sdk/tests/node_ports/test_item.py @@ -12,18 +12,18 @@ def create_item(item_type, item_value): key = "some key" - return Item(SchemaItem(key=key, - label="a label", - description="a description", - type=item_type, - displayOrder=2), DataItem(key=key, + return Item(SchemaItem(key=key, + label="a label", + description="a description", + type=item_type, + displayOrder=2), DataItem(key=key, value=item_value)) def test_default_item(): with pytest.raises(exceptions.InvalidProtocolError, message="Expecting InvalidProtocolError"): Item(None, None) -@pytest.mark.asyncio + async def test_item(): key = "my key" label = "my label" @@ -45,20 +45,20 @@ async def test_item(): assert await item.get() == item_value -@pytest.mark.asyncio + async def test_valid_type(): for item_type in config.TYPE_TO_PYTHON_TYPE_MAP: item = create_item(item_type, None) assert await item.get() is None -@pytest.mark.asyncio + async def test_invalid_type(): item = create_item("some wrong type", None) with pytest.raises(exceptions.InvalidProtocolError, message="Expecting InvalidProtocolError") as excinfo: await item.get() assert "Invalid protocol used" in str(excinfo.value) -@pytest.mark.asyncio + async def test_invalid_value_type(): #pylint: disable=W0612 with pytest.raises(exceptions.InvalidItemTypeError, message="Expecting InvalidItemTypeError") as excinfo: @@ -68,11 +68,11 @@ async def test_invalid_value_type(): ("integer", 26, 26), ("number", -746.4748, -746.4748), # ("data:*/*", __file__, {"store":"s3-z43", "path":"undefined/undefined/{filename}".format(filename=Path(__file__).name)}), - ("boolean", False, False), + ("boolean", False, False), ("string", "test-string", "test-string") ]) -@pytest.mark.asyncio -async def test_set_new_value(bucket, item_type, item_value_to_set, expected_value): # pylint: disable=W0613 + +async def test_set_new_value(bucket, item_type, item_value_to_set, expected_value): # pylint: disable=W0613 mock_method = mock.Mock() item = create_item(item_type, None) item.new_data_cb = mock_method @@ -87,7 +87,7 @@ async def test_set_new_value(bucket, item_type, item_value_to_set, expected_valu ("boolean", 123), ("string", True) ]) -@pytest.mark.asyncio + async def test_set_new_invalid_value(bucket, item_type, item_value_to_set): # pylint: disable=W0613 item = create_item(item_type, None) assert await item.get() is None diff --git a/packages/simcore-sdk/tests/node_ports/test_itemstlist.py b/packages/simcore-sdk/tests/node_ports/test_itemstlist.py index 9c6966ee36c..80e0af63022 100644 --- a/packages/simcore-sdk/tests/node_ports/test_itemstlist.py +++ b/packages/simcore-sdk/tests/node_ports/test_itemstlist.py @@ -11,24 +11,24 @@ def create_item(key, item_type, item_value): - return Item(SchemaItem(key=key, - label="a label", - description="a description", - type=item_type, - displayOrder=2), - DataItem(key=key, + return Item(SchemaItem(key=key, + label="a label", + description="a description", + type=item_type, + displayOrder=2), + DataItem(key=key, value=item_value)) def create_items_list(key_item_value_tuples): schemas = SchemaItemsList({key:SchemaItem(key=key, label="a label", description="a description", type=item_type, displayOrder=2) for (key, item_type, _) in key_item_value_tuples}) payloads = DataItemsList({key:DataItem(key=key, value=item_value) for key,_,item_value in key_item_value_tuples}) return ItemsList(schemas, payloads) - + def test_default_list(): itemslist = ItemsList(SchemaItemsList(), DataItemsList()) - assert not itemslist + assert not itemslist assert not itemslist.change_notifier assert not itemslist.get_node_from_node_uuid_cb @@ -47,11 +47,11 @@ def test_accessing_by_key(): def test_access_by_wrong_key(): from simcore_sdk.node_ports import exceptions - itemslist = create_items_list([("1", "integer", 333), ("2", "integer", 333), ("3", "integer", 333)]) + itemslist = create_items_list([("1", "integer", 333), ("2", "integer", 333), ("3", "integer", 333)]) with pytest.raises(exceptions.UnboundPortError, message="Expecting UnboundPortError"): print(itemslist["fdoiht"]) -@pytest.mark.asyncio + async def test_modifying_items_triggers_cb(): #pylint: disable=C0103 mock_method = mock.Mock() diff --git a/packages/simcore-sdk/tests/node_ports/test_nodeports.py b/packages/simcore-sdk/tests/node_ports/test_nodeports.py index a2b90b8c30b..7622c22b8c2 100644 --- a/packages/simcore-sdk/tests/node_ports/test_nodeports.py +++ b/packages/simcore-sdk/tests/node_ports/test_nodeports.py @@ -1,26 +1,27 @@ - -import filecmp -import tempfile -from pathlib import Path - #pylint: disable=W0212 #pylint: disable=C0111 #pylint: disable=R0913 #pylint: disable=W0104 #pylint: disable=unused-argument + +import filecmp +import tempfile +from pathlib import Path + import pytest -from helpers import helpers #pylint: disable=no-name-in-module +from helpers import helpers # pylint: disable=no-name-in-module from simcore_sdk import node_ports from simcore_sdk.node_ports import exceptions + def check_port_valid(ports, config_dict: dict, port_type:str, key_name: str, key): assert getattr(ports, port_type)[key].key == key_name # check required values assert getattr(ports, port_type)[key].label == config_dict["schema"][port_type][key_name]["label"] assert getattr(ports, port_type)[key].description == config_dict["schema"][port_type][key_name]["description"] - assert getattr(ports, port_type)[key].type == config_dict["schema"][port_type][key_name]["type"] + assert getattr(ports, port_type)[key].type == config_dict["schema"][port_type][key_name]["type"] assert getattr(ports, port_type)[key].displayOrder == config_dict["schema"][port_type][key_name]["displayOrder"] # check optional values if "defaultValue" in config_dict["schema"][port_type][key_name]: @@ -43,22 +44,26 @@ def check_port_valid(ports, config_dict: dict, port_type:str, key_name: str, key else: assert getattr(ports, port_type)[key].value == None + def check_ports_valid(ports, config_dict: dict, port_type:str): - for key in config_dict["schema"][port_type].keys(): + for key in config_dict["schema"][port_type].keys(): # test using "key" name check_port_valid(ports, config_dict, port_type, key, key) # test using index key_index = list(config_dict["schema"][port_type].keys()).index(key) check_port_valid(ports, config_dict, port_type, key, key_index) + def check_config_valid(ports, config_dict: dict): check_ports_valid(ports, config_dict, "inputs") check_ports_valid(ports, config_dict, "outputs") -def test_default_configuration(default_configuration): # pylint: disable=W0613, W0621 - config_dict = default_configuration + +def test_default_configuration(default_configuration): # pylint: disable=W0613, W0621 + config_dict = default_configuration check_config_valid(node_ports.ports(), config_dict) + def test_invalid_ports(special_configuration): config_dict, _, _ = special_configuration() PORTS = node_ports.ports() @@ -81,12 +86,11 @@ def test_invalid_ports(special_configuration): ("number", -746.4748, float), ("number", 0.0, float), ("number", 4566.11235, float), - ("boolean", False, bool), + ("boolean", False, bool), ("boolean", True, bool), ("string", "test-string", str), ("string", "", str) ]) -@pytest.mark.asyncio async def test_port_value_accessors(special_configuration, item_type, item_value, item_pytype): # pylint: disable=W0613, W0621 item_key = "some key" config_dict, _, _ = special_configuration(inputs=[(item_key, item_type, item_value)], outputs=[(item_key, item_type, None)]) @@ -105,12 +109,12 @@ async def test_port_value_accessors(special_configuration, item_type, item_value assert isinstance(await PORTS.outputs[item_key].get(), item_pytype) assert await PORTS.outputs[item_key].get() == item_value + @pytest.mark.parametrize("item_type, item_value, item_pytype, config_value", [ ("data:*/*", __file__, Path, {"store":"0", "path":__file__}), ("data:text/*", __file__, Path, {"store":"0", "path":__file__}), ("data:text/py", __file__, Path, {"store":"0", "path":__file__}), ]) -@pytest.mark.asyncio async def test_port_file_accessors(special_configuration, storage, filemanager_cfg, s3_simcore_location, bucket, item_type, item_value, item_pytype, config_value): # pylint: disable=W0613, W0621 config_dict, project_id, node_uuid = special_configuration(inputs=[("in_1", item_type, config_value)], outputs=[("out_34", item_type, None)]) PORTS = node_ports.ports() @@ -122,7 +126,7 @@ async def test_port_file_accessors(special_configuration, storage, filemanager_c # this triggers an upload to S3 + configuration change await PORTS.outputs["out_34"].set(item_value) # this is the link to S3 storage - assert PORTS.outputs["out_34"].value == {"store":s3_simcore_location, "path":Path(str(project_id), str(node_uuid), Path(item_value).name).as_posix()} + assert PORTS.outputs["out_34"].value == {"store":s3_simcore_location, "path":Path(str(project_id), str(node_uuid), Path(item_value).name).as_posix()} # this triggers a download from S3 to a location in /tempdir/simcorefiles/item_key assert isinstance(await PORTS.outputs["out_34"].get(), item_pytype) assert (await PORTS.outputs["out_34"].get()).exists() @@ -130,6 +134,7 @@ async def test_port_file_accessors(special_configuration, storage, filemanager_c filecmp.clear_cache() assert filecmp.cmp(item_value, await PORTS.outputs["out_34"].get()) + def test_adding_new_ports(special_configuration, session): config_dict, project_id, node_uuid = special_configuration() PORTS = node_ports.ports() @@ -155,12 +160,13 @@ def test_adding_new_ports(special_configuration, session): "label": "output data", "description": "a cool output", "displayOrder":2, - "type": "boolean"}}) + "type": "boolean"}}) helpers.update_configuration(session, project_id, node_uuid, config_dict) #pylint: disable=E1101 check_config_valid(PORTS, config_dict) + def test_removing_ports(special_configuration, session): - config_dict, project_id, node_uuid = special_configuration(inputs=[("in_14", "integer", 15), + config_dict, project_id, node_uuid = special_configuration(inputs=[("in_14", "integer", 15), ("in_17", "boolean", False)], outputs=[("out_123", "string", "blahblah"), ("out_2", "number", -12.3)]) #pylint: disable=W0612 @@ -177,6 +183,7 @@ def test_removing_ports(special_configuration, session): helpers.update_configuration(session, project_id, node_uuid, config_dict) #pylint: disable=E1101 check_config_valid(PORTS, config_dict) + @pytest.mark.parametrize("item_type, item_value, item_pytype", [ ("integer", 26, int), ("integer", 0, int), @@ -184,31 +191,30 @@ def test_removing_ports(special_configuration, session): ("number", -746.4748, float), ("number", 0.0, float), ("number", 4566.11235, float), - ("boolean", False, bool), + ("boolean", False, bool), ("boolean", True, bool), ("string", "test-string", str), ("string", "", str), ]) -@pytest.mark.asyncio async def test_get_value_from_previous_node(special_2nodes_configuration, node_link, item_type, item_value, item_pytype): config_dict, _, _ = special_2nodes_configuration(prev_node_outputs=[("output_123", item_type, item_value)], inputs=[("in_15", item_type, node_link("output_123"))]) PORTS = node_ports.ports() - + check_config_valid(PORTS, config_dict) input_value = await PORTS.inputs["in_15"].get() assert isinstance(input_value, item_pytype) assert await PORTS.inputs["in_15"].get() == item_value + @pytest.mark.parametrize("item_type, item_value, item_pytype", [ ("data:*/*", __file__, Path), ("data:text/*", __file__, Path), ("data:text/py", __file__, Path), ]) -@pytest.mark.asyncio async def test_get_file_from_previous_node(special_2nodes_configuration, project_id, node_uuid, filemanager_cfg, node_link, store_link, item_type, item_value, item_pytype): config_dict, _, _ = special_2nodes_configuration(prev_node_outputs=[("output_123", item_type, store_link(item_value, project_id, node_uuid))], - inputs=[("in_15", item_type, node_link("output_123"))], + inputs=[("in_15", item_type, node_link("output_123"))], project_id=project_id, previous_node_id=node_uuid) PORTS = node_ports.ports() check_config_valid(PORTS, config_dict) @@ -219,14 +225,18 @@ async def test_get_file_from_previous_node(special_2nodes_configuration, project filecmp.clear_cache() assert filecmp.cmp(file_path, item_value) + @pytest.mark.parametrize("item_type, item_value, item_alias, item_pytype", [ ("data:*/*", __file__, "some funky name.txt", Path), ("data:text/*", __file__, "some funky name without extension", Path), ("data:text/py", __file__, "öä$äö2-34 name without extension", Path), ]) -@pytest.mark.asyncio async def test_file_mapping(special_configuration, project_id, node_uuid, filemanager_cfg, s3_simcore_location, bucket, store_link, session, item_type, item_value, item_alias, item_pytype): - config_dict, project_id, node_uuid = special_configuration(inputs=[("in_1", item_type, store_link(item_value, project_id, node_uuid))], outputs=[("out_1", item_type, None)], project_id=project_id, node_id=node_uuid) + config_dict, project_id, node_uuid = special_configuration( + inputs=[("in_1", item_type, store_link(item_value, project_id, node_uuid))], + outputs=[("out_1", item_type, None)], + project_id=project_id, + node_id=node_uuid) PORTS = node_ports.ports() check_config_valid(PORTS, config_dict) # add a filetokeymap @@ -241,7 +251,7 @@ async def test_file_mapping(special_configuration, project_id, node_uuid, filema invalid_alias = Path("invalid_alias.fjfj") with pytest.raises(exceptions.PortNotFound, message="Expecting PortNotFound"): await PORTS.set_file_by_keymap(invalid_alias) - + await PORTS.set_file_by_keymap(file_path) file_id = helpers.file_uuid(file_path, project_id, node_uuid) assert PORTS.outputs["out_1"].value == {"store":s3_simcore_location, "path": file_id} diff --git a/services/director/tests/test_registry_proxy.py b/services/director/tests/test_registry_proxy.py index 5d83bd3fa11..a58c55b1d92 100644 --- a/services/director/tests/test_registry_proxy.py +++ b/services/director/tests/test_registry_proxy.py @@ -7,26 +7,26 @@ from simcore_service_director import registry_proxy -@pytest.mark.asyncio -async def test_list_no_services_available(docker_registry, configure_registry_access): + +async def test_list_no_services_available(docker_registry, configure_registry_access): computational_services = await registry_proxy.list_computational_services() assert (not computational_services) # it's empty interactive_services = await registry_proxy.list_interactive_services() assert (not interactive_services) -@pytest.mark.asyncio + async def test_list_computational_services(docker_registry, push_services, configure_registry_access): - push_services(6, 3) + push_services(6, 3) computational_services = await registry_proxy.list_computational_services() assert len(computational_services) == 6 -@pytest.mark.asyncio + async def test_list_interactive_services(docker_registry, push_services, configure_registry_access): - push_services(5, 4) + push_services(5, 4) interactive_services = await registry_proxy.list_interactive_services() assert len(interactive_services) == 4 -@pytest.mark.asyncio + async def test_retrieve_list_of_images_in_repo(docker_registry, push_services, configure_registry_access): images = push_services(5, 3) image_number = {} @@ -36,12 +36,12 @@ async def test_retrieve_list_of_images_in_repo(docker_registry, push_services, c if key not in image_number: image_number[key] = 0 image_number[key] = image_number[key]+1 - + for key, number in image_number.items(): list_of_images = await registry_proxy.retrieve_list_of_images_in_repo(key) assert len(list_of_images["tags"]) == number -@pytest.mark.asyncio + async def test_list_interactive_service_dependencies(docker_registry, push_services, configure_registry_access): images = push_services(2,2, inter_dependent_services=True) for image in images: @@ -56,9 +56,9 @@ async def test_list_interactive_service_dependencies(docker_registry, push_servi assert image_dependencies[0]["tag"] == docker_dependencies[0]["tag"] -@pytest.mark.asyncio + async def test_retrieve_labels_of_image(docker_registry, push_services, configure_registry_access): - images = push_services(1, 1) + images = push_services(1, 1) for image in images: service_description = image["service_description"] labels = await registry_proxy.retrieve_labels_of_image(service_description["key"], service_description["version"]) @@ -108,9 +108,9 @@ def test_get_service_last_namess(): repo = "services/dynamic/modeler" assert registry_proxy.get_service_last_names(repo) == "invalid service" -@pytest.mark.asyncio + async def test_get_service_details(push_services, configure_registry_access): - images = push_services(1, 1) + images = push_services(1, 1) for image in images: service_description = image["service_description"] details = await registry_proxy.get_service_details(service_description["key"], service_description["version"]) From ccb5904ca058661ec3f82c2e98c212f0fd87d535 Mon Sep 17 00:00:00 2001 From: Pedro Crespo Date: Mon, 26 Nov 2018 17:31:29 +0100 Subject: [PATCH 08/10] Fixes node_ports tests. Adds apihub+storage in the fixtures --- .travis.yml | 6 +++--- .../simcore-sdk/tests/fixtures/storage.py | 14 +++++++------ .../tests/node_ports/docker-compose.yml | 21 ++++++++++--------- services/docker-compose.yml | 1 + 4 files changed, 23 insertions(+), 19 deletions(-) diff --git a/.travis.yml b/.travis.yml index d9392dd1662..b2f9177b3e9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,12 +44,12 @@ matrix: - chmod +x docker-compose - sudo mv docker-compose /usr/local/bin - docker-compose --version - # shutdown postgres because sometimes is is alrady up ??? + # shutdown postgres because sometimes is is already up ??? - sudo service postgresql stop # wait for postgresql to shutdown - while sudo lsof -Pi :5432 -sTCP:LISTEN -t; do sleep 1; done - install: + install: - pip install --upgrade pip wheel setuptools && pip3 --version - pip3 install pytest-travis-fold - pip3 install packages/s3wrapper[test] @@ -67,7 +67,7 @@ matrix: script: - export DOCKER_GID=1042 - - docker-compose -f services/docker-compose.yml build storage + - docker-compose -f services/docker-compose.yml build storage apihub - make pylint - make test diff --git a/packages/simcore-sdk/tests/fixtures/storage.py b/packages/simcore-sdk/tests/fixtures/storage.py index 6ecb0322691..596968adac7 100644 --- a/packages/simcore-sdk/tests/fixtures/storage.py +++ b/packages/simcore-sdk/tests/fixtures/storage.py @@ -8,8 +8,10 @@ log = logging.getLogger(__name__) +API_VERSION = 'v0' + def _fake_logger_while_building_storage(): - print("Hey Travis I'm still alive...") + print("Hey Travis I'm still alive ... don't give up on me :-)") def _is_responsive(url, code=200): try: @@ -22,20 +24,20 @@ def _is_responsive(url, code=200): return False @pytest.fixture(scope="module") -def storage(bucket, engine, docker_ip, docker_services): +def storage(bucket, engine, docker_ip, docker_services): host = docker_ip port = docker_services.port_for('storage', 8080) - endpoint = "http://{}:{}".format(host, port) + endpoint = "http://{}:{}/{}/".format(host, port, API_VERSION) + # Wait until we can connect keep_alive_timer = threading.Timer(interval=60.0, function=_fake_logger_while_building_storage) keep_alive_timer.start() docker_services.wait_until_responsive( - check=lambda: _is_responsive(endpoint, 404), + check=lambda: _is_responsive(endpoint, 200), timeout=30.0, pause=1.0, ) keep_alive_timer.cancel() - + yield endpoint # cleanup - diff --git a/packages/simcore-sdk/tests/node_ports/docker-compose.yml b/packages/simcore-sdk/tests/node_ports/docker-compose.yml index bf18be01a56..1da5e9adc83 100644 --- a/packages/simcore-sdk/tests/node_ports/docker-compose.yml +++ b/packages/simcore-sdk/tests/node_ports/docker-compose.yml @@ -1,16 +1,12 @@ version: '3.6' services: storage: - # build: - # context: ../../../../ - # dockerfile: services/storage/Dockerfile - # args: - # - DOCKER_GID_ARG=1500 - # target: production image: services_storage:latest ports: - 11111:8080 environment: + - APIHUB_HOST=apihub + - APIHUB_PORT=8043 - POSTGRES_ENDPOINT=postgres:5432 - POSTGRES_USER=user - POSTGRES_PASSWORD=pwd @@ -25,9 +21,10 @@ services: - BF_API_SECRET="none" - BF_API_KEY="none" restart: always - depends_on: + depends_on: - postgres - # - minio + - apihub + # DEPENDENCIES ---------------------------------------------------- postgres: restart: always image: postgres:10 @@ -37,10 +34,14 @@ services: - POSTGRES_DB=test ports: - "5432:5432" - # monitors databases + apihub: + image: services_apihub:latest + ports: + - '8043:8043' + # TOOLS ----------------------------------------------------------- adminer: image: adminer ports: - 18080:8080 depends_on: - - postgres \ No newline at end of file + - postgres diff --git a/services/docker-compose.yml b/services/docker-compose.yml index e1d4ca9e817..0aefeb80579 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -166,6 +166,7 @@ services: depends_on: - minio - postgres + - apihub #-------------------------------------------------------------------- minio: image: minio/minio From 96f37a2ed0a39be7429350c0908d954e5afd1279 Mon Sep 17 00:00:00 2001 From: Pedro Crespo Date: Mon, 26 Nov 2018 18:04:53 +0100 Subject: [PATCH 09/10] Fixed pylint errors --- services/director/tests/test_registry_proxy.py | 2 -- .../server/src/simcore_service_webserver/reverse_proxy/abc.py | 2 -- 2 files changed, 4 deletions(-) diff --git a/services/director/tests/test_registry_proxy.py b/services/director/tests/test_registry_proxy.py index a58c55b1d92..8b798419668 100644 --- a/services/director/tests/test_registry_proxy.py +++ b/services/director/tests/test_registry_proxy.py @@ -2,8 +2,6 @@ import json -import pytest - from simcore_service_director import registry_proxy diff --git a/services/web/server/src/simcore_service_webserver/reverse_proxy/abc.py b/services/web/server/src/simcore_service_webserver/reverse_proxy/abc.py index 72b985df43a..b365532ea5e 100644 --- a/services/web/server/src/simcore_service_webserver/reverse_proxy/abc.py +++ b/services/web/server/src/simcore_service_webserver/reverse_proxy/abc.py @@ -18,7 +18,6 @@ async def get_image_name(self, service_identifier: str) -> str: Identifies a type of service. This normally corresponds to the name of the docker image """ - pass @abc.abstractmethod async def find_url(self, service_identifier: str) -> URL: @@ -30,7 +29,6 @@ async def find_url(self, service_identifier: str) -> URL: E.g. 'http://127.0.0.1:58873/x/ae1q8/' """ - pass # TODO: on_closed signal to notify sub-system that the service # has closed and can raise HTTPServiceAnavailable From 3ff7e1214f209482a6afb5f2627b4e5b29740093 Mon Sep 17 00:00:00 2001 From: Pedro Crespo Date: Tue, 27 Nov 2018 08:04:19 +0100 Subject: [PATCH 10/10] Cleanup director/tests - Added loop dependency (might help in windows version) - cleanup tests_registry_proxy.py --- .../tests/fixtures/docker_registry.py | 10 ++++-- .../director/tests/fixtures/fake_services.py | 35 +++++++++++-------- .../director/tests/test_registry_proxy.py | 21 +++++++---- 3 files changed, 41 insertions(+), 25 deletions(-) diff --git a/services/director/tests/fixtures/docker_registry.py b/services/director/tests/fixtures/docker_registry.py index 4637d58349f..ee6ff2c2f93 100644 --- a/services/director/tests/fixtures/docker_registry.py +++ b/services/director/tests/fixtures/docker_registry.py @@ -1,8 +1,12 @@ +# pylint:disable=unused-import +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name + import logging import docker import pytest -from pytest_docker import docker_ip, docker_services # pylint:disable=W0611 +from pytest_docker import docker_ip, docker_services log = logging.getLogger(__name__) @@ -15,9 +19,9 @@ def is_responsive(url): return False return True -# pylint:disable=redefined-outer-name + @pytest.fixture(scope="session") -def docker_registry(docker_ip, docker_services): +def docker_registry(docker_ip, docker_services): host = docker_ip port = docker_services.port_for('registry', 5000) url = "{host}:{port}".format(host=host, port=port) diff --git a/services/director/tests/fixtures/fake_services.py b/services/director/tests/fixtures/fake_services.py index 15f57a42930..442152c81a5 100644 --- a/services/director/tests/fixtures/fake_services.py +++ b/services/director/tests/fixtures/fake_services.py @@ -1,28 +1,33 @@ -from pathlib import Path +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name + + +import json import logging +from pathlib import Path + import docker import pytest -import json _logger = logging.getLogger(__name__) @pytest.fixture(scope="function") -def push_services(docker_registry, tmpdir): +def push_services(loop, docker_registry, tmpdir): registry_url = docker_registry tmp_dir = Path(tmpdir) list_of_pushed_images_tags = [] def build_push_images(number_of_computational_services, number_of_interactive_services, inter_dependent_services=False, sleep_time_s=60): - try: + try: version = "1.0." dependent_image = None - for image_index in range(0, number_of_computational_services): + for image_index in range(0, number_of_computational_services): image = _build_push_image(tmp_dir, registry_url, "computational", "test", version + str(image_index), sleep_time_s, dependent_image) if inter_dependent_services and image_index == 0: dependent_image = image list_of_pushed_images_tags.append(image) dependent_image = None - for image_index in range(0, number_of_interactive_services): + for image_index in range(0, number_of_interactive_services): image = _build_push_image(tmp_dir, registry_url, "dynamic", "test", version + str(image_index), sleep_time_s, dependent_image) if inter_dependent_services and image_index == 0: dependent_image = image @@ -44,16 +49,16 @@ def push_v0_schema_services(docker_registry, tmpdir): schema_version = "v0" list_of_pushed_images_tags = [] - def build_push_images(number_of_computational_services, number_of_interactive_services, sleep_time_s=10): - try: + def build_push_images(number_of_computational_services, number_of_interactive_services, sleep_time_s=10): + try: version = "0.0." - + dependent_image = None - for image_index in range(0, number_of_computational_services): + for image_index in range(0, number_of_computational_services): image = _build_push_image(tmp_dir, registry_url, "computational", "test", version + str(image_index), sleep_time_s, dependent_image, schema_version) list_of_pushed_images_tags.append(image) dependent_image = None - for image_index in range(0, number_of_interactive_services): + for image_index in range(0, number_of_interactive_services): image = _build_push_image(tmp_dir, registry_url, "dynamic", "test", version + str(image_index), sleep_time_s, dependent_image, schema_version) list_of_pushed_images_tags.append(image) except docker.errors.APIError: @@ -106,7 +111,7 @@ def _clean_registry(registry_url, list_of_images, schema_version="v1"): else: tag = service_description["version"] url = "http://{host}/v2/{name}/manifests/{tag}".format(host=registry_url, name=service_description["key"], tag=tag) - response = requests.request("GET", url, headers=request_headers) + response = requests.request("GET", url, headers=request_headers) docker_content_digest = response.headers["Docker-Content-Digest"] # remove the image from the registry url = "http://{host}/v2/{name}/manifests/{digest}".format(host=registry_url, name=service_description["key"], digest=docker_content_digest) @@ -115,7 +120,7 @@ def _clean_registry(registry_url, list_of_images, schema_version="v1"): def _create_base_image(base_dir, labels, sleep_time_s): # create a basic dockerfile docker_file = base_dir / "Dockerfile" - with docker_file.open("w") as file_pointer: + with docker_file.open("w") as file_pointer: file_pointer.write('FROM alpine\nCMD sleep %s\n' % (sleep_time_s)) assert docker_file.exists() == True # build docker base image @@ -133,13 +138,13 @@ def _create_service_description(service_type, name, tag, schema_version): service_key_type = "comp" elif service_type == "dynamic": service_key_type = "dynamic" - service_desc["key"] = "simcore/services/" + service_key_type + "/" + name + service_desc["key"] = "simcore/services/" + service_key_type + "/" + name # version 0 had no validation, no version, no type if schema_version == "v0": service_desc["tag"] = tag elif schema_version == "v1": service_desc["version"] = tag - service_desc["type"] = service_type + service_desc["type"] = service_type else: raise Exception("invalid version!!") diff --git a/services/director/tests/test_registry_proxy.py b/services/director/tests/test_registry_proxy.py index 8b798419668..57510562aa6 100644 --- a/services/director/tests/test_registry_proxy.py +++ b/services/director/tests/test_registry_proxy.py @@ -5,7 +5,6 @@ from simcore_service_director import registry_proxy - async def test_list_no_services_available(docker_registry, configure_registry_access): computational_services = await registry_proxy.list_computational_services() assert (not computational_services) # it's empty @@ -14,19 +13,23 @@ async def test_list_no_services_available(docker_registry, configure_registry_ac async def test_list_computational_services(docker_registry, push_services, configure_registry_access): - push_services(6, 3) + push_services( number_of_computational_services=6, + number_of_interactive_services=3) + computational_services = await registry_proxy.list_computational_services() assert len(computational_services) == 6 async def test_list_interactive_services(docker_registry, push_services, configure_registry_access): - push_services(5, 4) + push_services( number_of_computational_services=5, + number_of_interactive_services=4) interactive_services = await registry_proxy.list_interactive_services() assert len(interactive_services) == 4 async def test_retrieve_list_of_images_in_repo(docker_registry, push_services, configure_registry_access): - images = push_services(5, 3) + images = push_services( number_of_computational_services=5, + number_of_interactive_services=3) image_number = {} for image in images: service_description = image["service_description"] @@ -41,7 +44,9 @@ async def test_retrieve_list_of_images_in_repo(docker_registry, push_services, c async def test_list_interactive_service_dependencies(docker_registry, push_services, configure_registry_access): - images = push_services(2,2, inter_dependent_services=True) + images = push_services( number_of_computational_services=2, + number_of_interactive_services=2, + inter_dependent_services=True) for image in images: service_description = image["service_description"] docker_labels = image["docker_labels"] @@ -56,7 +61,8 @@ async def test_list_interactive_service_dependencies(docker_registry, push_servi async def test_retrieve_labels_of_image(docker_registry, push_services, configure_registry_access): - images = push_services(1, 1) + images = push_services(number_of_computational_services=1, + number_of_interactive_services=1) for image in images: service_description = image["service_description"] labels = await registry_proxy.retrieve_labels_of_image(service_description["key"], service_description["version"]) @@ -108,7 +114,8 @@ def test_get_service_last_namess(): async def test_get_service_details(push_services, configure_registry_access): - images = push_services(1, 1) + images = push_services(number_of_computational_services=1, + number_of_interactive_services=1) for image in images: service_description = image["service_description"] details = await registry_proxy.get_service_details(service_description["key"], service_description["version"])