Maintenance/upgrade trafaret 2

packages/service-library/requirements/_base.txt

@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile --build-isolation _base.in
+#    pip-compile --build-isolation requirements/_base.in
 #
 aiodebug==1.1.2           # via -r requirements/_base.in
 aiohttp==3.6.2            # via -r requirements/_base.in, aiozipkin
@@ -11,7 +11,9 @@ aiozipkin==0.6.0          # via -r requirements/_base.in
 async-timeout==3.0.1      # via aiohttp
 attrs==19.3.0             # via -r requirements/_base.in, aiohttp, jsonschema, openapi-core
 chardet==3.0.4            # via aiohttp
-idna==2.8                 # via yarl
+idna-ssl==1.1.0           # via aiohttp
+idna==2.8                 # via idna-ssl, yarl
+importlib-metadata==1.5.2  # via jsonschema
 isodate==0.6.0            # via openapi-core
 jsonschema==3.2.0         # via -r requirements/_base.in, openapi-spec-validator
 lazy-object-proxy==1.4.1  # via openapi-core
@@ -27,9 +29,11 @@ sqlalchemy[postgresql_psycopg2binary]==1.3.4  # via -r requirements/_base.in, ai
 strict-rfc3339==0.7       # via openapi-core
 tenacity==6.1.0           # via -r requirements/_base.in
 trafaret==2.0.2           # via -r requirements/_base.in
+typing-extensions==3.7.4.1  # via aiohttp
 ujson==2.0.3              # via -r requirements/_base.in
 werkzeug==1.0.0           # via -r requirements/_base.in
 yarl==1.3.0               # via aiohttp
+zipp==3.1.0               # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools

packages/service-library/requirements/_test.txt

@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile --build-isolation _test.in
+#    pip-compile --build-isolation requirements/_test.in
 #
 aiodebug==1.1.2           # via -r requirements/_base.txt
 aiohttp==3.6.2            # via -r requirements/_base.txt, aiozipkin, pytest-aiohttp
@@ -16,7 +16,9 @@ chardet==3.0.4            # via -r requirements/_base.txt, aiohttp, requests
 coverage==5.0.3           # via -r requirements/_test.in, coveralls, pytest-cov
 coveralls==1.11.1         # via -r requirements/_test.in
 docopt==0.6.2             # via coveralls
-idna==2.8                 # via -r requirements/_base.txt, requests, yarl
+idna-ssl==1.1.0           # via -r requirements/_base.txt, aiohttp
+idna==2.8                 # via -r requirements/_base.txt, idna-ssl, requests, yarl
+importlib-metadata==1.5.2  # via -r requirements/_base.txt, jsonschema, pluggy, pytest
 isodate==0.6.0            # via -r requirements/_base.txt, openapi-core
 isort==4.3.21             # via pylint
 jsonschema==3.2.0         # via -r requirements/_base.txt, openapi-spec-validator
@@ -50,12 +52,15 @@ strict-rfc3339==0.7       # via -r requirements/_base.txt, openapi-core
 tenacity==6.1.0           # via -r requirements/_base.txt
 termcolor==1.1.0          # via pytest-sugar
 trafaret==2.0.2           # via -r requirements/_base.txt
+typed-ast==1.4.1          # via astroid
+typing-extensions==3.7.4.1  # via -r requirements/_base.txt, aiohttp
 ujson==2.0.3              # via -r requirements/_base.txt
 urllib3==1.25.8           # via requests
 wcwidth==0.1.8            # via pytest
 werkzeug==1.0.0           # via -r requirements/_base.txt
 wrapt==1.11.2             # via astroid
 yarl==1.3.0               # via -r requirements/_base.txt, aiohttp
+zipp==3.1.0               # via -r requirements/_base.txt, importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools

packages/service-library/src/servicelib/tracing.py

@@ -31,7 +31,7 @@ def setup_tracing(
 
 schema = T.Dict(
     {
-        T.Key("enabled", default=True, optional=True): T.Or(T.Bool(), T.Int),
+        T.Key("enabled", default=True, optional=True): T.Or(T.Bool(), T.ToInt),
         T.Key("zipkin_endpoint", default="http://jaeger:9411"): T.String(),
     }
 )

packages/simcore-sdk/requirements/_base.txt

@@ -2,25 +2,25 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile --output-file=_base.txt _base.in
+#    pip-compile --build-isolation _base.in
 #
-aiofiles==0.4.0
-aiohttp==3.6.2
+aiofiles==0.4.0           # via -r _base.in
+aiohttp==3.6.2            # via -r _base.in
 async-timeout==3.0.1      # via aiohttp
 attrs==19.1.0             # via aiohttp
 chardet==3.0.4            # via aiohttp
 decorator==4.4.0          # via networkx
 idna-ssl==1.1.0           # via aiohttp
 idna==2.8                 # via idna-ssl, yarl
 multidict==4.5.2          # via aiohttp, yarl
-networkx==2.3
-pika==1.0.1
-psycopg2-binary==2.8.4
-pyyaml==5.3
+networkx==2.3             # via -r _base.in
+pika==1.0.1               # via -r _base.in
+psycopg2-binary==2.8.4    # via -r _base.in
+pyyaml==5.3               # via -r _base.in, trafaret-config
 six==1.12.0               # via tenacity
-sqlalchemy==1.3.3
-tenacity==6.0.0
-trafaret-config==2.0.2
-trafaret==1.2.0           # via trafaret-config
+sqlalchemy==1.3.3         # via -r _base.in
+tenacity==6.0.0           # via -r _base.in
+trafaret-config==2.0.2    # via -r _base.in
+trafaret==2.0.2           # via trafaret-config
 typing-extensions==3.7.2  # via aiohttp
 yarl==1.3.0               # via aiohttp

packages/simcore-sdk/requirements/_test.txt

@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile --build-isolation --output-file=_test.txt _test.in
+#    pip-compile --build-isolation _test.in
 #
 aiofiles==0.4.0           # via -r _base.txt
 aiohttp==3.6.2            # via -r _base.txt, pytest-aiohttp
@@ -48,7 +48,7 @@ sqlalchemy==1.3.3         # via -r _base.txt
 tenacity==6.0.0           # via -r _base.txt
 termcolor==1.1.0          # via pytest-sugar
 trafaret-config==2.0.2    # via -r _base.txt
-trafaret==1.2.0           # via -r _base.txt, trafaret-config
+trafaret==2.0.2           # via -r _base.txt, trafaret-config
 typed-ast==1.4.1          # via astroid
 typing-extensions==3.7.2  # via -r _base.txt, aiohttp
 urllib3==1.25.8           # via requests

packages/simcore-sdk/src/simcore_sdk/config/db.py

@@ -9,10 +9,10 @@
     "database": T.String(),
     "user": T.String(),
     "password": T.String(),
-    T.Key("minsize", default=1 ,optional=True): T.Int(),
-    T.Key("maxsize", default=4, optional=True): T.Int(),
+    T.Key("minsize", default=1 ,optional=True): T.ToInt(),
+    T.Key("maxsize", default=4, optional=True): T.ToInt(),
     "host": T.Or( T.String, T.Null),
-    "port": T.Or( T.Int, T.Null),
+    "port": T.Or( T.ToInt, T.Null),
     "endpoint": T.Or( T.String, T.Null)
 })
 

packages/simcore-sdk/src/simcore_sdk/config/rabbit.py

@@ -15,7 +15,7 @@
     T.Key("name", default="tasks", optional=True): T.String(),
     T.Key("enabled", default=True, optional=True): T.Bool(),
     T.Key("host", default='rabbit', optional=True): T.String(),
-    T.Key("port", default=5672, optional=True): T.Int(),
+    T.Key("port", default=5672, optional=True): T.ToInt(),
     "user": T.String(),
     "password": T.String(),
     "channels": T.Dict({

packages/simcore-sdk/src/simcore_sdk/config/s3.py

@@ -14,7 +14,7 @@
     "access_key": T.String(),
     "secret_key": T.String(),
     "bucket_name": T.String(),
-    T.Key("secure", default=0): T.Int(),
+    T.Key("secure", default=0): T.ToInt(),
 })
 
 

services/storage/requirements/_base.txt

@@ -57,7 +57,7 @@ sqlalchemy[postgresql_psycopg2binary]==1.3.3  # via -r ../../../packages/postgre
 strict-rfc3339==0.7       # via openapi-core
 tenacity==6.0.0           # via -r ../../../packages/service-library/requirements/_base.in, -r _base.in
 trafaret-config==2.0.2    # via -r _base.in
-trafaret==1.2.0           # via -r ../../../packages/service-library/requirements/_base.in, -r _base.in, trafaret-config
+trafaret==2.0.2           # via -r ../../../packages/service-library/requirements/_base.in, -r _base.in, trafaret-config
 typing-extensions==3.7.2  # via aiohttp
 ujson==1.35               # via -r ../../../packages/service-library/requirements/_base.in
 urllib3==1.25.8           # via -r _base.in, botocore, requests

services/storage/requirements/_test.txt

@@ -82,7 +82,7 @@ strict-rfc3339==0.7       # via -r _base.txt, openapi-core
 tenacity==6.0.0           # via -r _base.txt
 termcolor==1.1.0          # via pytest-sugar
 trafaret-config==2.0.2    # via -r _base.txt
-trafaret==1.2.0           # via -r _base.txt, trafaret-config
+trafaret==2.0.2           # via -r _base.txt, trafaret-config
 typed-ast==1.4.1          # via astroid
 typing-extensions==3.7.2  # via -r _base.txt, aiohttp
 ujson==1.35               # via -r _base.txt

services/storage/src/simcore_service_storage/config_schema.py

@@ -1,21 +1,28 @@
+import os
+
 import trafaret as T
+
 from servicelib.config_schema_utils import addon_section
 from servicelib.tracing import schema as tracing_schema
 from simcore_sdk.config import db, s3
 
 from . import rest_config
 
+runs_in_container = "SC_BUILD_TARGET" in os.environ
+
 app_schema = T.Dict(
     {
-        T.Key("host", default="0.0.0.0"): T.IP,
-        "port": T.Int(),
+        T.Key(
+            "host", default="0.0.0.0" if runs_in_container else "127.0.0.1"  # nosec
+        ): T.IP,
+        "port": T.ToInt(),
         "log_level": T.Enum(
             "DEBUG", "WARNING", "INFO", "ERROR", "CRITICAL", "FATAL", "NOTSET"
         ),
         "testing": T.Bool(),
-        T.Key("max_workers", default=8, optional=True): T.Int(),
+        T.Key("max_workers", default=8, optional=True): T.ToInt(),
         T.Key("monitoring_enabled", default=False): T.Or(
-            T.Bool(), T.Int
+            T.Bool(), T.ToInt
         ),  # Int added to use environs
         T.Key("test_datcore", optional=True): T.Dict(
             {"token_key": T.String(), "token_secret": T.String()}

services/storage/src/simcore_service_storage/datcore.py

@@ -14,8 +14,7 @@
 
 from blackfynn import Blackfynn
 from blackfynn.models import BaseCollection, Collection, DataPackage
-from simcore_service_storage.models import (DatasetMetaData, FileMetaData,
-                                            FileMetaDataEx)
+from simcore_service_storage.models import DatasetMetaData, FileMetaData, FileMetaDataEx
 from simcore_service_storage.settings import DATCORE_ID, DATCORE_STR
 
 logger = logging.getLogger(__name__)

services/storage/src/simcore_service_storage/dsm.py

@@ -144,10 +144,8 @@ class DataStorageManager:
     # TODO: perhaps can be used a cache? add a lifetime?
 
     def _get_datcore_tokens(self, user_id: str) -> Tuple[str, str]:
-          # pylint: disable=no-member
-        token = self.datcore_tokens.get(
-            user_id, DatCoreApiToken()
-        )
+        # pylint: disable=no-member
+        token = self.datcore_tokens.get(user_id, DatCoreApiToken())
         return token.to_tuple()
 
     async def locations(self, user_id: str):

services/storage/src/simcore_service_storage/handlers.py

@@ -305,9 +305,9 @@ async def create_folders_from_project(request: web.Request):
     nodes_map = body.get("nodes_map", {})
 
     assert set(nodes_map.keys()) == set(source_project["workbench"].keys())  # nosec
-    assert set(nodes_map.values()) == set(       # nosec
+    assert set(nodes_map.values()) == set(  # nosec
         destination_project["workbench"].keys()  # nosec
-    )                                            # nosec
+    )  # nosec
 
     # TODO: validate project with jsonschema instead??
     params = {"location_id": SIMCORE_S3_ID}

services/storage/tests/test_configs.py

@@ -47,15 +47,16 @@ def devel_environ(env_devel_file):
 
 variable_expansion_pattern = re.compile(r"\$\{*(\w+)+[:-]*(\w+)*\}")
 
+
 @pytest.mark.parametrize(
     "sample,expected_match",
     [
         (r"${varname:-default}", ("varname", "default")),
         (r"${varname}", ("varname", None)),
         (r"33", None),
         (r"${VAR_name:-33}", ("VAR_name", "33")),
-        (r"${varname-default}", ("varname", "default")), # this is not standard!
-        (r"${varname:default}", ("varname", "default")), # this is not standard!
+        (r"${varname-default}", ("varname", "default")),  # this is not standard!
+        (r"${varname:default}", ("varname", "default")),  # this is not standard!
     ],
 )
 def test_variable_expansions(sample, expected_match):

services/web/server/requirements/_base.txt

@@ -2,71 +2,71 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile --output-file=_base.txt _base.in
+#    pip-compile --build-isolation _base.in
 #
-aio-pika==6.5.2           # via -r _base.in (line 12)
-aiodebug==1.1.2           # via -r _base.in (line 34)
-aiohttp-jinja2==1.1.1     # via -r _base.in (line 14)
-aiohttp-security==0.4.0   # via -r _base.in (line 16)
-aiohttp-session[secure]==2.7.0  # via -r _base.in (line 15)
-aiohttp-swagger[performance]==1.0.14  # via -r _base.in (line 17)
-aiohttp==3.6.2            # via -r ../../../../packages/service-library/requirements/_base.in (line 10), -r _base.in (line 13), aiohttp-jinja2, aiohttp-security, aiohttp-session, aiohttp-swagger, aiozipkin
-aiopg[sa]==1.0.0          # via -r ../../../../packages/service-library/requirements/_base.in (line 11), -r _base.in (line 18)
-aioredis==1.3.0           # via -r _base.in (line 19)
+aio-pika==6.5.2           # via -r _base.in
+aiodebug==1.1.2           # via -r ../../../../packages/service-library/requirements/_base.in, -r _base.in
+aiohttp-jinja2==1.1.1     # via -r _base.in
+aiohttp-security==0.4.0   # via -r _base.in
+aiohttp-session[secure]==2.7.0  # via -r _base.in
+aiohttp-swagger[performance]==1.0.14  # via -r _base.in
+aiohttp==3.6.2            # via -r ../../../../packages/service-library/requirements/_base.in, -r _base.in, aiohttp-jinja2, aiohttp-security, aiohttp-session, aiohttp-swagger, aiozipkin
+aiopg[sa]==1.0.0          # via -r ../../../../packages/service-library/requirements/_base.in, -r _base.in
+aioredis==1.3.0           # via -r _base.in
 aiormq==3.2.1             # via aio-pika
-aiosmtplib==1.0.5         # via -r _base.in (line 20)
-aiozipkin==0.6.0          # via -r ../../../../packages/service-library/requirements/_base.in (line 12)
+aiosmtplib==1.0.5         # via -r _base.in
+aiozipkin==0.6.0          # via -r ../../../../packages/service-library/requirements/_base.in
 amqp==2.4.2               # via kombu
 asn1crypto==0.24.0        # via cryptography
 async-timeout==3.0.1      # via aiohttp, aioredis
-asyncpg==0.18.3           # via -r _base.in (line 21)
-attrs==19.1.0             # via -r ../../../../packages/service-library/requirements/_base.in (line 19), aiohttp, jsonschema, openapi-core
+asyncpg==0.18.3           # via -r _base.in
+attrs==19.1.0             # via -r ../../../../packages/service-library/requirements/_base.in, aiohttp, jsonschema, openapi-core
 billiard==3.6.0.0         # via celery
-celery==4.3.0             # via -r _base.in (line 22)
+celery==4.3.0             # via -r _base.in
 cffi==1.12.3              # via cryptography
-change-case==0.5.2        # via -r _base.in (line 24)
+change-case==0.5.2        # via -r _base.in
 chardet==3.0.4            # via aiohttp
-cryptography==2.6.1       # via -r _base.in (line 23), aiohttp-session
-expiringdict==1.2.0       # via -r _base.in (line 25)
+cryptography==2.6.1       # via -r _base.in, aiohttp-session
+expiringdict==1.2.0       # via -r _base.in
 hiredis==1.0.1            # via aioredis
 idna-ssl==1.1.0           # via aiohttp
 idna==2.8                 # via idna-ssl, yarl
 importlib-metadata==0.23  # via jsonschema
 isodate==0.6.0            # via openapi-core
-jinja-app-loader==1.0.2   # via -r _base.in (line 26)
+jinja-app-loader==1.0.2   # via -r _base.in
 jinja2==2.10.1            # via aiohttp-jinja2, aiohttp-swagger
-jsondiff==1.1.2           # via -r _base.in (line 27)
-jsonschema==3.2.0         # via -r ../../../../packages/service-library/requirements/_base.in (line 15), openapi-spec-validator
+jsondiff==1.1.2           # via -r _base.in
+jsonschema==3.2.0         # via -r ../../../../packages/service-library/requirements/_base.in, openapi-spec-validator
 kombu==4.5.0              # via celery
 lazy-object-proxy==1.4.3  # via openapi-core
 markupsafe==1.1.1         # via jinja2
 more-itertools==7.2.0     # via zipp
 multidict==4.5.2          # via aiohttp, yarl
-openapi-core==0.12.0      # via -r ../../../../packages/service-library/requirements/_base.in (line 16)
+openapi-core==0.12.0      # via -r ../../../../packages/service-library/requirements/_base.in
 openapi-spec-validator==0.2.8  # via openapi-core
 pamqp==2.3.0              # via aiormq
-passlib==1.7.1            # via -r _base.in (line 28)
-prometheus-client==0.7.1  # via -r ../../../../packages/service-library/requirements/_base.in (line 17)
-psycopg2-binary==2.8.4    # via -r ../../../../packages/service-library/requirements/_base.in (line 7), -r _base.in (line 8), aiopg, sqlalchemy
+passlib==1.7.1            # via -r _base.in
+prometheus-client==0.7.1  # via -r ../../../../packages/service-library/requirements/_base.in
+psycopg2-binary==2.8.4    # via -r ../../../../packages/service-library/requirements/_base.in, -r _base.in, aiopg, sqlalchemy
 pycparser==2.19           # via cffi
 pyrsistent==0.15.6        # via jsonschema
 python-engineio==3.9.3    # via python-socketio
-python-socketio==4.3.1    # via -r _base.in (line 29)
+python-socketio==4.3.1    # via -r _base.in
 pytz==2019.1              # via celery
-pyyaml==5.3               # via -r ../../../../packages/service-library/requirements/_base.in (line 6), aiohttp-swagger, openapi-spec-validator, trafaret-config
-semantic-version==2.6.0   # via -r _base.in (line 30)
+pyyaml==5.3               # via -r ../../../../packages/service-library/requirements/_base.in, aiohttp-swagger, openapi-spec-validator, trafaret-config
+semantic-version==2.6.0   # via -r _base.in
 six==1.12.0               # via cryptography, isodate, jsonschema, openapi-core, openapi-spec-validator, pyrsistent, python-engineio, python-socketio, tenacity
-sqlalchemy[postgresql_psycopg2binary]==1.3.4  # via -r ../../../../packages/postgres-database/requirements/_base.in (line 7), -r ../../../../packages/service-library/requirements/_base.in (line 5), -r _base.in (line 9), aiopg
+sqlalchemy[postgresql_psycopg2binary]==1.3.4  # via -r ../../../../packages/postgres-database/requirements/_base.in, -r ../../../../packages/service-library/requirements/_base.in, -r _base.in, aiopg
 strict-rfc3339==0.7       # via openapi-core
-tenacity==6.0.0           # via -r ../../../../packages/service-library/requirements/_base.in (line 18), -r _base.in (line 31)
-trafaret-config==2.0.2    # via -r _base.in (line 33)
-trafaret==1.2.0           # via -r ../../../../packages/service-library/requirements/_base.in (line 20), -r _base.in (line 32), trafaret-config
+tenacity==6.0.0           # via -r ../../../../packages/service-library/requirements/_base.in, -r _base.in
+trafaret-config==2.0.2    # via -r _base.in
+trafaret==2.0.2           # via -r ../../../../packages/service-library/requirements/_base.in, -r _base.in, trafaret-config
 typing-extensions==3.7.2  # via aiohttp
 typing==3.7.4.1           # via expiringdict
-ujson==1.35               # via -r ../../../../packages/service-library/requirements/_base.in (line 13), aiohttp-swagger
+ujson==1.35               # via -r ../../../../packages/service-library/requirements/_base.in, aiohttp-swagger
 vine==1.3.0               # via amqp, celery
-werkzeug==0.16.0          # via -r ../../../../packages/service-library/requirements/_base.in (line 14)
-yarl==1.3.0               # via -r ../../../../packages/postgres-database/requirements/_base.in (line 9), aio-pika, aiohttp, aiormq
+werkzeug==0.16.0          # via -r ../../../../packages/service-library/requirements/_base.in
+yarl==1.3.0               # via -r ../../../../packages/postgres-database/requirements/_base.in, aio-pika, aiohttp, aiormq
 zipp==1.0.0               # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:

services/web/server/requirements/_test.txt

@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile --build-isolation --output-file=_test.txt _test.in
+#    pip-compile --build-isolation _test.in
 #
 aio-pika==6.5.2           # via -r _base.txt
 aiodebug==1.1.2           # via -r _base.txt
@@ -90,7 +90,7 @@ tenacity==6.0.0           # via -r _base.txt, -r _test.in
 termcolor==1.1.0          # via pytest-sugar
 text-unidecode==1.3       # via faker
 trafaret-config==2.0.2    # via -r _base.txt
-trafaret==1.2.0           # via -r _base.txt, trafaret-config
+trafaret==2.0.2           # via -r _base.txt, trafaret-config
 typed-ast==1.4.1          # via astroid
 typing-extensions==3.7.2  # via -r _base.txt, aiohttp
 typing==3.7.4.1           # via -r _base.txt, expiringdict