Skip to content

Commit 358f3b7

Browse files
authored
Minor+Patch upgrades in python packages and system+api testing (#1513)
- Minor or Patch upgrades for libraries within python packages or system/api testing requirements: - sqlalchemy patch 1.3.17 - multidict - packaging and more-itertools (for testing via pytest) - Enhance scripts/requirements/Makefile to automate this job - Prepares for pylint upgrade: cleanup of ignore rules in tests - Autoformat cleanup - Removes old deprectaed code web/server/tests/sandbox
1 parent 1e6ff86 commit 358f3b7

File tree

80 files changed

+820
-1114
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

80 files changed

+820
-1114
lines changed

api/tests/requirements.txt

+3-3
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,12 @@ importlib-metadata==1.6.0 # via jsonschema, pluggy, pytest
1515
isodate==0.6.0 # via openapi-schema-validator
1616
jsonschema==3.2.0 # via openapi-schema-validator, openapi-spec-validator
1717
lazy-object-proxy==1.4.3 # via openapi-core
18-
more-itertools==8.2.0 # via openapi-core, pytest
19-
multidict==4.7.5 # via aiohttp, yarl
18+
more-itertools==8.3.0 # via openapi-core, pytest
19+
multidict==4.7.6 # via aiohttp, yarl
2020
openapi-core==0.13.3 # via -r requirements.in
2121
openapi-schema-validator==0.1.1 # via openapi-core
2222
openapi-spec-validator==0.2.8 # via openapi-core
23-
packaging==20.3 # via pytest, pytest-sugar
23+
packaging==20.4 # via pytest, pytest-sugar
2424
parse==1.15.0 # via openapi-core
2525
pluggy==0.13.1 # via pytest
2626
py==1.8.1 # via pytest

api/tests/test_individual_openapi_schemas.py

+55-45
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
1-
# pylint:disable=wildcard-import
2-
# pylint:disable=unused-import
31
# pylint:disable=unused-variable
42
# pylint:disable=unused-argument
53
# pylint:disable=redefined-outer-name
@@ -13,39 +11,41 @@
1311
import yaml
1412
from openapi_spec_validator import validate_spec
1513
from openapi_spec_validator.exceptions import OpenAPIValidationError
16-
17-
from utils import (dump_specs, is_json_schema, is_openapi_schema,
18-
list_files_in_api_specs, load_specs, specs_folder)
14+
from utils import (
15+
dump_specs,
16+
is_json_schema,
17+
is_openapi_schema,
18+
list_files_in_api_specs,
19+
load_specs,
20+
specs_folder,
21+
)
1922

2023
# Conventions
2124
_REQUIRED_FIELDS = ["error", "data"]
2225
CONVERTED_SUFFIX = "-converted.yaml"
2326
_FAKE_SCHEMA_NAME = "FakeSchema"
2427

2528
_FAKE_OPEN_API_HEADERS = {
26-
"openapi": "3.0.0",
27-
"info":{
28-
"title": "An include file to define sortable attributes",
29-
"version": "1.0.0"
30-
},
31-
"paths": {},
32-
"components": {
33-
"parameters":{},
34-
"schemas":{}
35-
}
36-
}
29+
"openapi": "3.0.0",
30+
"info": {
31+
"title": "An include file to define sortable attributes",
32+
"version": "1.0.0",
33+
},
34+
"paths": {},
35+
"components": {"parameters": {}, "schemas": {}},
36+
}
37+
3738

3839
def add_namespace_for_converted_schemas(schema_specs: dict):
3940
# schemas converted from jsonschema do not have an overarching namespace.
4041
# the openapi validator does not like this
4142
# we use the jsonschema title to create a fake namespace
42-
fake_schema_specs = {
43-
_FAKE_SCHEMA_NAME: schema_specs
44-
}
43+
fake_schema_specs = {_FAKE_SCHEMA_NAME: schema_specs}
4544
return fake_schema_specs
4645

46+
4747
def change_references_to_schemas(filepath: Path, specs: dict):
48-
from os.path import relpath, isabs, join, abspath , exists
48+
from os.path import relpath, isabs, join, abspath, exists
4949

5050
filedir = filepath.parent
5151

@@ -54,36 +54,44 @@ def change_references_to_schemas(filepath: Path, specs: dict):
5454
# navigate specs
5555
change_references_to_schemas(filepath, value)
5656

57-
elif key in ("allOf", "oneOf", "anyOf"): # navigates allOf, oneOf, anyOf
57+
elif key in ("allOf", "oneOf", "anyOf"): # navigates allOf, oneOf, anyOf
5858
for item in value:
5959
change_references_to_schemas(filepath, item)
6060

61-
elif key=="$ref":
61+
elif key == "$ref":
6262
# Ensures value = "file_ref#section_ref"
6363
value = str(value)
64-
if value.startswith('#'):
64+
if value.startswith("#"):
6565
value = str(filepath) + value
66-
elif '#' not in value:
66+
elif "#" not in value:
6767
value = value + "# "
6868

69-
7069
file_ref, section_ref = value.split("#")
7170

7271
if not isabs(file_ref):
73-
file_ref = str(filedir / file_ref)
72+
file_ref = str(filedir / file_ref)
7473

75-
file_ref = abspath(file_ref) # resolves
74+
file_ref = abspath(file_ref) # resolves
7675
assert exists(file_ref), file_ref
7776

78-
if 'schemas' in file_ref: # reference to a schema file (i.e. inside a schemas folder)
79-
if not section_ref.startswith('/components/schemas/'): # not updated!
80-
section_ref = "/components/schemas/" + section_ref.lstrip('/').strip()
81-
if file_ref.endswith(CONVERTED_SUFFIX): # fake name used in converted schemas
77+
if (
78+
"schemas" in file_ref
79+
): # reference to a schema file (i.e. inside a schemas folder)
80+
if not section_ref.startswith("/components/schemas/"): # not updated!
81+
section_ref = (
82+
"/components/schemas/" + section_ref.lstrip("/").strip()
83+
)
84+
if file_ref.endswith(
85+
CONVERTED_SUFFIX
86+
): # fake name used in converted schemas
8287
section_ref += _FAKE_SCHEMA_NAME
8388

84-
file_ref = "./" + relpath(file_ref, filedir) if not filepath.samefile(file_ref) else ""
85-
specs[key] = file_ref + "#" + section_ref
86-
89+
file_ref = (
90+
"./" + relpath(file_ref, filedir)
91+
if not filepath.samefile(file_ref)
92+
else ""
93+
)
94+
specs[key] = file_ref + "#" + section_ref
8795

8896

8997
@pytest.fixture("session")
@@ -98,20 +106,22 @@ def converted_specs_testdir(api_specs_dir, all_api_specs_tails, tmpdir_factory):
98106
99107
"""
100108
basedir = api_specs_dir
101-
testdir = Path( tmpdir_factory.mktemp("converted-specs") )
109+
testdir = Path(tmpdir_factory.mktemp("converted-specs"))
102110

103111
print(testdir)
104112

105113
for tail in all_api_specs_tails:
106114

107115
# directory with converted specs
108-
os.makedirs(testdir/tail.parent, exist_ok=True)
116+
os.makedirs(testdir / tail.parent, exist_ok=True)
109117

110-
specs = load_specs(basedir/tail)
118+
specs = load_specs(basedir / tail)
111119

112-
if "schemas" in str(tail) and \
113-
not is_openapi_schema(specs) and \
114-
not is_json_schema(specs):
120+
if (
121+
"schemas" in str(tail)
122+
and not is_openapi_schema(specs)
123+
and not is_json_schema(specs)
124+
):
115125

116126
# convert to valid openapi
117127
if tail.name.endswith(CONVERTED_SUFFIX):
@@ -121,16 +131,16 @@ def converted_specs_testdir(api_specs_dir, all_api_specs_tails, tmpdir_factory):
121131
new_specs["components"]["schemas"] = specs
122132

123133
# change references
124-
change_references_to_schemas(basedir/tail, new_specs)
125-
dump_specs(new_specs, testdir/tail)
134+
change_references_to_schemas(basedir / tail, new_specs)
135+
dump_specs(new_specs, testdir / tail)
126136

127137
elif is_openapi_schema(specs):
128138
new_specs = specs
129139
# change references
130-
change_references_to_schemas(basedir/tail, new_specs)
131-
dump_specs(new_specs, testdir/tail)
140+
change_references_to_schemas(basedir / tail, new_specs)
141+
dump_specs(new_specs, testdir / tail)
132142
else:
133-
shutil.copy2(basedir/tail, testdir/tail)
143+
shutil.copy2(basedir / tail, testdir / tail)
134144

135145
return testdir
136146

packages/postgres-database/requirements/_base.txt

+2-2
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
# pip-compile --output-file=requirements/_base.txt requirements/_base.in
66
#
77
idna==2.9 # via yarl
8-
multidict==4.7.5 # via yarl
8+
multidict==4.7.6 # via yarl
99
psycopg2-binary==2.8.5 # via sqlalchemy
10-
sqlalchemy[postgresql_psycopg2binary]==1.3.16 # via -r requirements/_base.in
10+
sqlalchemy[postgresql_psycopg2binary]==1.3.17 # via -r requirements/_base.in
1111
yarl==1.4.2 # via -r requirements/_base.in

packages/postgres-database/requirements/_migration.txt

+2-2
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,13 @@ docker==4.2.0 # via -r requirements/_migration.in
1212
idna==2.9 # via -r requirements/_base.txt, requests, yarl
1313
mako==1.1.2 # via alembic
1414
markupsafe==1.1.1 # via mako
15-
multidict==4.7.5 # via -r requirements/_base.txt, yarl
15+
multidict==4.7.6 # via -r requirements/_base.txt, yarl
1616
psycopg2-binary==2.8.5 # via -r requirements/_base.txt, sqlalchemy
1717
python-dateutil==2.8.1 # via alembic
1818
python-editor==1.0.4 # via alembic
1919
requests==2.23.0 # via docker
2020
six==1.14.0 # via docker, python-dateutil, tenacity, websocket-client
21-
sqlalchemy[postgresql_psycopg2binary]==1.3.16 # via -r requirements/_base.txt, alembic
21+
sqlalchemy[postgresql_psycopg2binary]==1.3.17 # via -r requirements/_base.txt, alembic
2222
tenacity==6.2.0 # via -r requirements/_migration.in
2323
urllib3==1.25.9 # via -r requirements/_migration.in, requests
2424
websocket-client==0.57.0 # via docker

packages/postgres-database/requirements/_test.txt

+5-5
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,9 @@ lazy-object-proxy==1.4.3 # via astroid
2626
mako==1.1.2 # via -r requirements/_migration.txt, alembic
2727
markupsafe==1.1.1 # via -r requirements/_migration.txt, mako
2828
mccabe==0.6.1 # via pylint
29-
more-itertools==8.2.0 # via pytest
30-
multidict==4.7.5 # via -r requirements/_migration.txt, aiohttp, yarl
31-
packaging==20.3 # via pytest
29+
more-itertools==8.3.0 # via pytest
30+
multidict==4.7.6 # via -r requirements/_migration.txt, aiohttp, yarl
31+
packaging==20.4 # via pytest
3232
pluggy==0.13.1 # via pytest
3333
psycopg2-binary==2.8.5 # via -r requirements/_migration.txt, aiopg, sqlalchemy
3434
py==1.8.1 # via pytest
@@ -45,10 +45,10 @@ python-editor==1.0.4 # via -r requirements/_migration.txt, alembic
4545
pyyaml==5.3.1 # via -r requirements/_test.in
4646
requests==2.23.0 # via -r requirements/_migration.txt, coveralls, docker
4747
six==1.14.0 # via -r requirements/_migration.txt, astroid, docker, packaging, python-dateutil, tenacity, websocket-client
48-
sqlalchemy[postgresql_psycopg2binary]==1.3.16 # via -r requirements/_migration.txt, aiopg, alembic
48+
sqlalchemy[postgresql_psycopg2binary]==1.3.17 # via -r requirements/_migration.txt, aiopg, alembic
4949
tenacity==6.2.0 # via -r requirements/_migration.txt
5050
text-unidecode==1.3 # via faker
51-
toml==0.10.0 # via pylint
51+
toml==0.10.1 # via pylint
5252
typed-ast==1.4.1 # via astroid
5353
typing-extensions==3.7.4.2 # via aiohttp
5454
urllib3==1.25.9 # via -r requirements/_migration.txt, requests

packages/postgres-database/src/simcore_postgres_database/cli.py

+3-3
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
33
"""
44
# pylint: disable=broad-except
5+
# pylint: disable=wildcard-import,unused-wildcard-import
56

67
import json
78
import logging
@@ -17,7 +18,7 @@
1718
from alembic import __version__ as __alembic_version__
1819
from alembic.config import Config as AlembicConfig
1920

20-
from simcore_postgres_database.models import * # pylint: disable=wildcard-import,unused-wildcard-import
21+
from simcore_postgres_database.models import *
2122
from simcore_postgres_database.utils import build_url, raise_if_not_responsive
2223

2324
alembic_version = tuple([int(v) for v in __alembic_version__.split(".")[0:3]])
@@ -109,8 +110,6 @@ def main():
109110
""" Simplified CLI for database migration with alembic """
110111

111112

112-
113-
114113
@main.command()
115114
@click.option("--user", "-u")
116115
@click.option("--password", "-p")
@@ -207,6 +206,7 @@ def clean():
207206

208207
# Bypasses alembic CLI into a reduced version ------------
209208

209+
210210
@main.command()
211211
@click.option("-m", "message")
212212
def review(message):

packages/postgres-database/tests/test_delete_projects_and_users.py

+3-2
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22
# pylint:disable=unused-variable
33
# pylint:disable=unused-argument
44
# pylint:disable=redefined-outer-name
5-
# pylint:disable=wildcard-import
65

76
from typing import List
87
from uuid import uuid4
@@ -67,7 +66,9 @@ async def start():
6766
await conn.execute(projects.insert().values(**random_project(prj_owner=2)))
6867
await conn.execute(projects.insert().values(**random_project(prj_owner=3)))
6968
with pytest.raises(ForeignKeyViolation):
70-
await conn.execute(projects.insert().values(**random_project(prj_owner=4)))
69+
await conn.execute(
70+
projects.insert().values(**random_project(prj_owner=4))
71+
)
7172

7273
await conn.execute(
7374
user_to_projects.insert().values(user_id=1, project_id=1)

packages/postgres-database/tests/test_uniqueness_in_comp_tasks.py

-3
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22
# pylint:disable=unused-variable
33
# pylint:disable=unused-argument
44
# pylint:disable=redefined-outer-name
5-
# pylint:disable=wildcard-import
65

76
import json
87
import random
@@ -13,8 +12,6 @@
1312
import faker
1413
import pytest
1514
import sqlalchemy as sa
16-
17-
# from aiopg.sa.result import ResultProxy, RowProxy
1815
from psycopg2.errors import UniqueViolation # pylint: disable=no-name-in-module
1916

2017
from simcore_postgres_database.models.base import metadata

packages/s3wrapper/requirements/_test.txt

+3-3
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@ isort==4.3.21 # via pylint
1818
lazy-object-proxy==1.4.3 # via astroid
1919
mccabe==0.6.1 # via pylint
2020
minio==5.0.10 # via -r requirements/_base.txt
21-
more-itertools==8.2.0 # via pytest
22-
packaging==20.3 # via pytest
21+
more-itertools==8.3.0 # via pytest
22+
packaging==20.4 # via pytest
2323
pluggy==0.13.1 # via pytest
2424
py==1.8.1 # via pytest
2525
pylint==2.5.0 # via -r requirements/_test.in
@@ -32,7 +32,7 @@ python-dateutil==2.8.1 # via -r requirements/_base.txt, minio
3232
pytz==2020.1 # via -r requirements/_base.txt, minio
3333
requests==2.23.0 # via -r requirements/_test.in, coveralls
3434
six==1.14.0 # via -r requirements/_base.txt, astroid, packaging, python-dateutil
35-
toml==0.10.0 # via pylint
35+
toml==0.10.1 # via pylint
3636
typed-ast==1.4.1 # via astroid
3737
urllib3==1.25.9 # via -r requirements/_base.txt, minio, requests
3838
wcwidth==0.1.9 # via pytest

packages/service-library/requirements/_base.txt

+2-2
Original file line numberDiff line numberDiff line change
@@ -17,15 +17,15 @@ importlib-metadata==1.6.0 # via jsonschema
1717
isodate==0.6.0 # via openapi-core
1818
jsonschema==3.2.0 # via -r requirements/_base.in, openapi-spec-validator
1919
lazy-object-proxy==1.4.3 # via openapi-core
20-
multidict==4.7.5 # via aiohttp, yarl
20+
multidict==4.7.6 # via aiohttp, yarl
2121
openapi-core==0.12.0 # via -r requirements/_base.in
2222
openapi-spec-validator==0.2.8 # via openapi-core
2323
prometheus-client==0.7.1 # via -r requirements/_base.in
2424
psycopg2-binary==2.8.5 # via -r requirements/_base.in, aiopg, sqlalchemy
2525
pyrsistent==0.16.0 # via jsonschema
2626
pyyaml==5.3.1 # via -r requirements/_base.in, openapi-spec-validator
2727
six==1.14.0 # via isodate, jsonschema, openapi-core, openapi-spec-validator, pyrsistent, tenacity
28-
sqlalchemy[postgresql_psycopg2binary]==1.3.16 # via -r requirements/_base.in, aiopg
28+
sqlalchemy[postgresql_psycopg2binary]==1.3.17 # via -r requirements/_base.in, aiopg
2929
strict-rfc3339==0.7 # via openapi-core
3030
tenacity==6.2.0 # via -r requirements/_base.in
3131
trafaret==2.0.2 # via -r requirements/_base.in

packages/service-library/requirements/_test.txt

+5-5
Original file line numberDiff line numberDiff line change
@@ -24,11 +24,11 @@ isort==4.3.21 # via pylint
2424
jsonschema==3.2.0 # via -r requirements/_base.txt, openapi-spec-validator
2525
lazy-object-proxy==1.4.3 # via -r requirements/_base.txt, astroid, openapi-core
2626
mccabe==0.6.1 # via pylint
27-
more-itertools==8.2.0 # via pytest
28-
multidict==4.7.5 # via -r requirements/_base.txt, aiohttp, yarl
27+
more-itertools==8.3.0 # via pytest
28+
multidict==4.7.6 # via -r requirements/_base.txt, aiohttp, yarl
2929
openapi-core==0.12.0 # via -r requirements/_base.txt
3030
openapi-spec-validator==0.2.8 # via -r requirements/_base.txt, openapi-core
31-
packaging==20.3 # via pytest, pytest-sugar
31+
packaging==20.4 # via pytest, pytest-sugar
3232
pluggy==0.13.1 # via pytest
3333
prometheus-client==0.7.1 # via -r requirements/_base.txt
3434
psycopg2-binary==2.8.5 # via -r requirements/_base.txt, aiopg, sqlalchemy
@@ -47,11 +47,11 @@ pytest==5.4.2 # via -r requirements/_test.in, pytest-aiohttp, pytest
4747
pyyaml==5.3.1 # via -r requirements/_base.txt, openapi-spec-validator
4848
requests==2.23.0 # via coveralls
4949
six==1.14.0 # via -r requirements/_base.txt, astroid, isodate, jsonschema, openapi-core, openapi-spec-validator, packaging, pyrsistent, tenacity
50-
sqlalchemy[postgresql_psycopg2binary]==1.3.16 # via -r requirements/_base.txt, aiopg
50+
sqlalchemy[postgresql_psycopg2binary]==1.3.17 # via -r requirements/_base.txt, aiopg
5151
strict-rfc3339==0.7 # via -r requirements/_base.txt, openapi-core
5252
tenacity==6.2.0 # via -r requirements/_base.txt
5353
termcolor==1.1.0 # via pytest-sugar
54-
toml==0.10.0 # via pylint
54+
toml==0.10.1 # via pylint
5555
trafaret==2.0.2 # via -r requirements/_base.txt
5656
typed-ast==1.4.1 # via astroid
5757
typing-extensions==3.7.4.2 # via -r requirements/_base.txt, aiohttp

packages/service-library/src/servicelib/monitor_slow_callbacks.py

+1-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
21
import asyncio.events
32
import time
43
from asyncio.base_events import _format_handle
@@ -25,7 +24,7 @@ def instrumented(self):
2524
dt = time.monotonic() - t0
2625
if dt >= slow_duration_secs:
2726
task_info = _format_handle(self)
28-
incidents.append( SlowCallback(msg=task_info, delay_secs=dt) )
27+
incidents.append(SlowCallback(msg=task_info, delay_secs=dt))
2928
logger.warning("Executing %s took %.3f seconds", task_info, dt)
3029
return retval
3130

0 commit comments

Comments
 (0)