Minor+Patch upgrades in python packages and system+api testing

api/tests/requirements.txt

@@ -15,12 +15,12 @@ importlib-metadata==1.6.0  # via jsonschema, pluggy, pytest
 isodate==0.6.0            # via openapi-schema-validator
 jsonschema==3.2.0         # via openapi-schema-validator, openapi-spec-validator
 lazy-object-proxy==1.4.3  # via openapi-core
-more-itertools==8.2.0     # via openapi-core, pytest
-multidict==4.7.5          # via aiohttp, yarl
+more-itertools==8.3.0     # via openapi-core, pytest
+multidict==4.7.6          # via aiohttp, yarl
 openapi-core==0.13.3      # via -r requirements.in
 openapi-schema-validator==0.1.1  # via openapi-core
 openapi-spec-validator==0.2.8  # via openapi-core
-packaging==20.3           # via pytest, pytest-sugar
+packaging==20.4           # via pytest, pytest-sugar
 parse==1.15.0             # via openapi-core
 pluggy==0.13.1            # via pytest
 py==1.8.1                 # via pytest

api/tests/test_individual_openapi_schemas.py

@@ -1,5 +1,3 @@
-# pylint:disable=wildcard-import
-# pylint:disable=unused-import
 # pylint:disable=unused-variable
 # pylint:disable=unused-argument
 # pylint:disable=redefined-outer-name
@@ -13,39 +11,41 @@
 import yaml
 from openapi_spec_validator import validate_spec
 from openapi_spec_validator.exceptions import OpenAPIValidationError
-
-from utils import (dump_specs, is_json_schema, is_openapi_schema,
-                   list_files_in_api_specs, load_specs, specs_folder)
+from utils import (
+    dump_specs,
+    is_json_schema,
+    is_openapi_schema,
+    list_files_in_api_specs,
+    load_specs,
+    specs_folder,
+)
 
 # Conventions
 _REQUIRED_FIELDS = ["error", "data"]
 CONVERTED_SUFFIX = "-converted.yaml"
 _FAKE_SCHEMA_NAME = "FakeSchema"
 
 _FAKE_OPEN_API_HEADERS = {
-        "openapi": "3.0.0",
-        "info":{
-            "title": "An include file to define sortable attributes",
-            "version": "1.0.0"
-        },
-        "paths": {},
-        "components": {
-            "parameters":{},
-            "schemas":{}
-        }
-    }
+    "openapi": "3.0.0",
+    "info": {
+        "title": "An include file to define sortable attributes",
+        "version": "1.0.0",
+    },
+    "paths": {},
+    "components": {"parameters": {}, "schemas": {}},
+}
+
 
 def add_namespace_for_converted_schemas(schema_specs: dict):
     # schemas converted from jsonschema do not have an overarching namespace.
     # the openapi validator does not like this
     # we use the jsonschema title to create a fake namespace
-    fake_schema_specs = {
-        _FAKE_SCHEMA_NAME: schema_specs
-        }
+    fake_schema_specs = {_FAKE_SCHEMA_NAME: schema_specs}
     return fake_schema_specs
 
+
 def change_references_to_schemas(filepath: Path, specs: dict):
-    from os.path import relpath, isabs, join, abspath , exists
+    from os.path import relpath, isabs, join, abspath, exists
 
     filedir = filepath.parent
 
@@ -54,36 +54,44 @@ def change_references_to_schemas(filepath: Path, specs: dict):
             # navigate specs
             change_references_to_schemas(filepath, value)
 
-        elif key in ("allOf", "oneOf", "anyOf"): # navigates allOf, oneOf, anyOf
+        elif key in ("allOf", "oneOf", "anyOf"):  # navigates allOf, oneOf, anyOf
             for item in value:
                 change_references_to_schemas(filepath, item)
 
-        elif key=="$ref":
+        elif key == "$ref":
             # Ensures value = "file_ref#section_ref"
             value = str(value)
-            if value.startswith('#'):
+            if value.startswith("#"):
                 value = str(filepath) + value
-            elif '#' not in value:
+            elif "#" not in value:
                 value = value + "# "
 
-
             file_ref, section_ref = value.split("#")
 
             if not isabs(file_ref):
-                file_ref =  str(filedir / file_ref)
+                file_ref = str(filedir / file_ref)
 
-            file_ref = abspath(file_ref) # resolves
+            file_ref = abspath(file_ref)  # resolves
             assert exists(file_ref), file_ref
 
-            if 'schemas' in file_ref: # reference to a schema file (i.e. inside a schemas folder)
-                if not section_ref.startswith('/components/schemas/'): # not updated!
-                    section_ref = "/components/schemas/" + section_ref.lstrip('/').strip()
-                    if file_ref.endswith(CONVERTED_SUFFIX): # fake name used in converted schemas
+            if (
+                "schemas" in file_ref
+            ):  # reference to a schema file (i.e. inside a schemas folder)
+                if not section_ref.startswith("/components/schemas/"):  # not updated!
+                    section_ref = (
+                        "/components/schemas/" + section_ref.lstrip("/").strip()
+                    )
+                    if file_ref.endswith(
+                        CONVERTED_SUFFIX
+                    ):  # fake name used in converted schemas
                         section_ref += _FAKE_SCHEMA_NAME
 
-                    file_ref = "./" + relpath(file_ref, filedir) if not filepath.samefile(file_ref) else ""
-                    specs[key] =  file_ref + "#" + section_ref
-
+                    file_ref = (
+                        "./" + relpath(file_ref, filedir)
+                        if not filepath.samefile(file_ref)
+                        else ""
+                    )
+                    specs[key] = file_ref + "#" + section_ref
 
 
 @pytest.fixture("session")
@@ -98,20 +106,22 @@ def converted_specs_testdir(api_specs_dir, all_api_specs_tails, tmpdir_factory):
 
     """
     basedir = api_specs_dir
-    testdir = Path( tmpdir_factory.mktemp("converted-specs") )
+    testdir = Path(tmpdir_factory.mktemp("converted-specs"))
 
     print(testdir)
 
     for tail in all_api_specs_tails:
 
         # directory with converted specs
-        os.makedirs(testdir/tail.parent, exist_ok=True)
+        os.makedirs(testdir / tail.parent, exist_ok=True)
 
-        specs = load_specs(basedir/tail)
+        specs = load_specs(basedir / tail)
 
-        if "schemas" in str(tail) and  \
-           not is_openapi_schema(specs) and \
-           not is_json_schema(specs):
+        if (
+            "schemas" in str(tail)
+            and not is_openapi_schema(specs)
+            and not is_json_schema(specs)
+        ):
 
             # convert to valid openapi
             if tail.name.endswith(CONVERTED_SUFFIX):
@@ -121,16 +131,16 @@ def converted_specs_testdir(api_specs_dir, all_api_specs_tails, tmpdir_factory):
             new_specs["components"]["schemas"] = specs
 
             # change references
-            change_references_to_schemas(basedir/tail, new_specs)
-            dump_specs(new_specs, testdir/tail)
+            change_references_to_schemas(basedir / tail, new_specs)
+            dump_specs(new_specs, testdir / tail)
 
         elif is_openapi_schema(specs):
             new_specs = specs
             # change references
-            change_references_to_schemas(basedir/tail, new_specs)
-            dump_specs(new_specs, testdir/tail)
+            change_references_to_schemas(basedir / tail, new_specs)
+            dump_specs(new_specs, testdir / tail)
         else:
-            shutil.copy2(basedir/tail, testdir/tail)
+            shutil.copy2(basedir / tail, testdir / tail)
 
     return testdir
 

packages/postgres-database/requirements/_base.txt

@@ -5,7 +5,7 @@
 #    pip-compile --output-file=requirements/_base.txt requirements/_base.in
 #
 idna==2.9                 # via yarl
-multidict==4.7.5          # via yarl
+multidict==4.7.6          # via yarl
 psycopg2-binary==2.8.5    # via sqlalchemy
-sqlalchemy[postgresql_psycopg2binary]==1.3.16  # via -r requirements/_base.in
+sqlalchemy[postgresql_psycopg2binary]==1.3.17  # via -r requirements/_base.in
 yarl==1.4.2               # via -r requirements/_base.in

packages/postgres-database/requirements/_migration.txt

@@ -12,13 +12,13 @@ docker==4.2.0             # via -r requirements/_migration.in
 idna==2.9                 # via -r requirements/_base.txt, requests, yarl
 mako==1.1.2               # via alembic
 markupsafe==1.1.1         # via mako
-multidict==4.7.5          # via -r requirements/_base.txt, yarl
+multidict==4.7.6          # via -r requirements/_base.txt, yarl
 psycopg2-binary==2.8.5    # via -r requirements/_base.txt, sqlalchemy
 python-dateutil==2.8.1    # via alembic
 python-editor==1.0.4      # via alembic
 requests==2.23.0          # via docker
 six==1.14.0               # via docker, python-dateutil, tenacity, websocket-client
-sqlalchemy[postgresql_psycopg2binary]==1.3.16  # via -r requirements/_base.txt, alembic
+sqlalchemy[postgresql_psycopg2binary]==1.3.17  # via -r requirements/_base.txt, alembic
 tenacity==6.2.0           # via -r requirements/_migration.in
 urllib3==1.25.9           # via -r requirements/_migration.in, requests
 websocket-client==0.57.0  # via docker

packages/postgres-database/requirements/_test.txt

@@ -26,9 +26,9 @@ lazy-object-proxy==1.4.3  # via astroid
 mako==1.1.2               # via -r requirements/_migration.txt, alembic
 markupsafe==1.1.1         # via -r requirements/_migration.txt, mako
 mccabe==0.6.1             # via pylint
-more-itertools==8.2.0     # via pytest
-multidict==4.7.5          # via -r requirements/_migration.txt, aiohttp, yarl
-packaging==20.3           # via pytest
+more-itertools==8.3.0     # via pytest
+multidict==4.7.6          # via -r requirements/_migration.txt, aiohttp, yarl
+packaging==20.4           # via pytest
 pluggy==0.13.1            # via pytest
 psycopg2-binary==2.8.5    # via -r requirements/_migration.txt, aiopg, sqlalchemy
 py==1.8.1                 # via pytest
@@ -45,10 +45,10 @@ python-editor==1.0.4      # via -r requirements/_migration.txt, alembic
 pyyaml==5.3.1             # via -r requirements/_test.in
 requests==2.23.0          # via -r requirements/_migration.txt, coveralls, docker
 six==1.14.0               # via -r requirements/_migration.txt, astroid, docker, packaging, python-dateutil, tenacity, websocket-client
-sqlalchemy[postgresql_psycopg2binary]==1.3.16  # via -r requirements/_migration.txt, aiopg, alembic
+sqlalchemy[postgresql_psycopg2binary]==1.3.17  # via -r requirements/_migration.txt, aiopg, alembic
 tenacity==6.2.0           # via -r requirements/_migration.txt
 text-unidecode==1.3       # via faker
-toml==0.10.0              # via pylint
+toml==0.10.1              # via pylint
 typed-ast==1.4.1          # via astroid
 typing-extensions==3.7.4.2  # via aiohttp
 urllib3==1.25.9           # via -r requirements/_migration.txt, requests

packages/postgres-database/src/simcore_postgres_database/cli.py

@@ -2,6 +2,7 @@
 
 """
 # pylint: disable=broad-except
+# pylint: disable=wildcard-import,unused-wildcard-import
 
 import json
 import logging
@@ -17,7 +18,7 @@
 from alembic import __version__ as __alembic_version__
 from alembic.config import Config as AlembicConfig
 
-from simcore_postgres_database.models import *  # pylint: disable=wildcard-import,unused-wildcard-import
+from simcore_postgres_database.models import *
 from simcore_postgres_database.utils import build_url, raise_if_not_responsive
 
 alembic_version = tuple([int(v) for v in __alembic_version__.split(".")[0:3]])
@@ -109,8 +110,6 @@ def main():
     """ Simplified CLI for database migration with alembic """
 
 
-
-
 @main.command()
 @click.option("--user", "-u")
 @click.option("--password", "-p")
@@ -207,6 +206,7 @@ def clean():
 
 # Bypasses alembic CLI into a reduced version  ------------
 
+
 @main.command()
 @click.option("-m", "message")
 def review(message):

packages/postgres-database/tests/test_delete_projects_and_users.py

@@ -2,7 +2,6 @@
 # pylint:disable=unused-variable
 # pylint:disable=unused-argument
 # pylint:disable=redefined-outer-name
-# pylint:disable=wildcard-import
 
 from typing import List
 from uuid import uuid4
@@ -67,7 +66,9 @@ async def start():
             await conn.execute(projects.insert().values(**random_project(prj_owner=2)))
             await conn.execute(projects.insert().values(**random_project(prj_owner=3)))
             with pytest.raises(ForeignKeyViolation):
-                await conn.execute(projects.insert().values(**random_project(prj_owner=4)))
+                await conn.execute(
+                    projects.insert().values(**random_project(prj_owner=4))
+                )
 
             await conn.execute(
                 user_to_projects.insert().values(user_id=1, project_id=1)

packages/postgres-database/tests/test_uniqueness_in_comp_tasks.py

@@ -2,7 +2,6 @@
 # pylint:disable=unused-variable
 # pylint:disable=unused-argument
 # pylint:disable=redefined-outer-name
-# pylint:disable=wildcard-import
 
 import json
 import random
@@ -13,8 +12,6 @@
 import faker
 import pytest
 import sqlalchemy as sa
-
-# from aiopg.sa.result import ResultProxy, RowProxy
 from psycopg2.errors import UniqueViolation  # pylint: disable=no-name-in-module
 
 from simcore_postgres_database.models.base import metadata

packages/s3wrapper/requirements/_test.txt

@@ -18,8 +18,8 @@ isort==4.3.21             # via pylint
 lazy-object-proxy==1.4.3  # via astroid
 mccabe==0.6.1             # via pylint
 minio==5.0.10             # via -r requirements/_base.txt
-more-itertools==8.2.0     # via pytest
-packaging==20.3           # via pytest
+more-itertools==8.3.0     # via pytest
+packaging==20.4           # via pytest
 pluggy==0.13.1            # via pytest
 py==1.8.1                 # via pytest
 pylint==2.5.0             # via -r requirements/_test.in
@@ -32,7 +32,7 @@ python-dateutil==2.8.1    # via -r requirements/_base.txt, minio
 pytz==2020.1              # via -r requirements/_base.txt, minio
 requests==2.23.0          # via -r requirements/_test.in, coveralls
 six==1.14.0               # via -r requirements/_base.txt, astroid, packaging, python-dateutil
-toml==0.10.0              # via pylint
+toml==0.10.1              # via pylint
 typed-ast==1.4.1          # via astroid
 urllib3==1.25.9           # via -r requirements/_base.txt, minio, requests
 wcwidth==0.1.9            # via pytest

packages/service-library/requirements/_base.txt

@@ -17,15 +17,15 @@ importlib-metadata==1.6.0  # via jsonschema
 isodate==0.6.0            # via openapi-core
 jsonschema==3.2.0         # via -r requirements/_base.in, openapi-spec-validator
 lazy-object-proxy==1.4.3  # via openapi-core
-multidict==4.7.5          # via aiohttp, yarl
+multidict==4.7.6          # via aiohttp, yarl
 openapi-core==0.12.0      # via -r requirements/_base.in
 openapi-spec-validator==0.2.8  # via openapi-core
 prometheus-client==0.7.1  # via -r requirements/_base.in
 psycopg2-binary==2.8.5    # via -r requirements/_base.in, aiopg, sqlalchemy
 pyrsistent==0.16.0        # via jsonschema
 pyyaml==5.3.1             # via -r requirements/_base.in, openapi-spec-validator
 six==1.14.0               # via isodate, jsonschema, openapi-core, openapi-spec-validator, pyrsistent, tenacity
-sqlalchemy[postgresql_psycopg2binary]==1.3.16  # via -r requirements/_base.in, aiopg
+sqlalchemy[postgresql_psycopg2binary]==1.3.17  # via -r requirements/_base.in, aiopg
 strict-rfc3339==0.7       # via openapi-core
 tenacity==6.2.0           # via -r requirements/_base.in
 trafaret==2.0.2           # via -r requirements/_base.in

packages/service-library/requirements/_test.txt

@@ -24,11 +24,11 @@ isort==4.3.21             # via pylint
 jsonschema==3.2.0         # via -r requirements/_base.txt, openapi-spec-validator
 lazy-object-proxy==1.4.3  # via -r requirements/_base.txt, astroid, openapi-core
 mccabe==0.6.1             # via pylint
-more-itertools==8.2.0     # via pytest
-multidict==4.7.5          # via -r requirements/_base.txt, aiohttp, yarl
+more-itertools==8.3.0     # via pytest
+multidict==4.7.6          # via -r requirements/_base.txt, aiohttp, yarl
 openapi-core==0.12.0      # via -r requirements/_base.txt
 openapi-spec-validator==0.2.8  # via -r requirements/_base.txt, openapi-core
-packaging==20.3           # via pytest, pytest-sugar
+packaging==20.4           # via pytest, pytest-sugar
 pluggy==0.13.1            # via pytest
 prometheus-client==0.7.1  # via -r requirements/_base.txt
 psycopg2-binary==2.8.5    # via -r requirements/_base.txt, aiopg, sqlalchemy
@@ -47,11 +47,11 @@ pytest==5.4.2             # via -r requirements/_test.in, pytest-aiohttp, pytest
 pyyaml==5.3.1             # via -r requirements/_base.txt, openapi-spec-validator
 requests==2.23.0          # via coveralls
 six==1.14.0               # via -r requirements/_base.txt, astroid, isodate, jsonschema, openapi-core, openapi-spec-validator, packaging, pyrsistent, tenacity
-sqlalchemy[postgresql_psycopg2binary]==1.3.16  # via -r requirements/_base.txt, aiopg
+sqlalchemy[postgresql_psycopg2binary]==1.3.17  # via -r requirements/_base.txt, aiopg
 strict-rfc3339==0.7       # via -r requirements/_base.txt, openapi-core
 tenacity==6.2.0           # via -r requirements/_base.txt
 termcolor==1.1.0          # via pytest-sugar
-toml==0.10.0              # via pylint
+toml==0.10.1              # via pylint
 trafaret==2.0.2           # via -r requirements/_base.txt
 typed-ast==1.4.1          # via astroid
 typing-extensions==3.7.4.2  # via -r requirements/_base.txt, aiohttp

packages/service-library/src/servicelib/monitor_slow_callbacks.py

@@ -1,4 +1,3 @@
-
 import asyncio.events
 import time
 from asyncio.base_events import _format_handle
@@ -25,7 +24,7 @@ def instrumented(self):
         dt = time.monotonic() - t0
         if dt >= slow_duration_secs:
             task_info = _format_handle(self)
-            incidents.append( SlowCallback(msg=task_info, delay_secs=dt) )
+            incidents.append(SlowCallback(msg=task_info, delay_secs=dt))
             logger.warning("Executing %s took %.3f seconds", task_info, dt)
         return retval