Skip to content

Commit 39817ee

Browse files
pcrespovpcrespov
authored
webserver delayed diagnostics, upgrades reqs and other enhancements (#1741)
- upgrades package requirements - ADDS postgres-database CI testing - FIXES new linter error: raise-missing-from and super-with-arguments - CHANGE: review and cleanup of fixtures in webserver tests, reduce redundancy (reuse helpers from pytest-simcore), classify fixtures and share common fixtures - CHANGE :enhance servicelib.logged_gathered and added testing - CHANGE: webserver delayed activation of diagnosis. This will avoid restarting due to delays produced upon startup (default delay set to 1 min)
1 parent 0308ebb commit 39817ee

File tree

81 files changed

+886
-533
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

81 files changed

+886
-533
lines changed

‎.github/workflows/ci-testing-deploy.yml

+48
Original file line numberDiff line numberDiff line change
@@ -249,6 +249,7 @@ jobs:
249249
with:
250250
name: unit_sidecar_coverage
251251
path: codeclimate.unit_sidecar_coverage.json
252+
252253
unit-test-frontend:
253254
name: Unit-testing frontend
254255
runs-on: ${{ matrix.os }}
@@ -320,6 +321,53 @@ jobs:
320321
- name: test
321322
run: ./ci/github/unit-testing/python-linting.bash test
322323

324+
unit-test-postgres-database:
325+
name: Unit-testing postgres-database
326+
runs-on: ${{ matrix.os }}
327+
strategy:
328+
matrix:
329+
python: [3.6]
330+
os: [ubuntu-20.04]
331+
fail-fast: false
332+
steps:
333+
- uses: actions/checkout@v2
334+
- name: setup docker
335+
run: |
336+
sudo ./ci/github/helpers/setup_docker_compose.bash
337+
./ci/github/helpers/setup_docker_experimental.bash
338+
./ci/github/helpers/setup_docker_buildx.bash
339+
echo ::set-env name=DOCKER_BUILDX::1
340+
- name: setup python environment
341+
uses: actions/[email protected]
342+
with:
343+
python-version: ${{ matrix.python }}
344+
- name: show system version
345+
run: ./ci/helpers/show_system_versions.bash
346+
- uses: actions/cache@v1
347+
name: getting cached data
348+
with:
349+
path: ~/.cache/pip
350+
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
351+
restore-keys: |
352+
${{ runner.os }}-pip-
353+
- name: install
354+
run: ./ci/github/unit-testing/postgres-database.bash install
355+
- name: test
356+
run: ./ci/github/unit-testing/postgres-database.bash test
357+
- uses: codecov/codecov-action@v1
358+
with:
359+
flags: unittests #optional
360+
- name: prepare codeclimate coverage file
361+
run: |
362+
curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
363+
chmod +x ./cc-test-reporter
364+
./cc-test-reporter format-coverage -t coverage.py -o codeclimate.unit_postgresdb_coverage.json coverage.xml
365+
- name: upload codeclimate coverage
366+
uses: actions/upload-artifact@v2
367+
with:
368+
name: unit_postgresdb_coverage
369+
path: codeclimate.unit_postgresdb_coverage.json
370+
323371
unit-test-service-library:
324372
name: Unit-testing service-library
325373
runs-on: ${{ matrix.os }}

‎api/tests/requirements.txt

+2-2
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
#
77
aiohttp==3.6.2 # via -r requirements.in, pytest-aiohttp
88
async-timeout==3.0.1 # via aiohttp
9-
attrs==19.3.0 # via aiohttp, jsonschema, openapi-core, pytest
9+
attrs==20.1.0 # via aiohttp, jsonschema, openapi-core, pytest
1010
chardet==3.0.4 # via aiohttp
1111
coverage==5.2.1 # via -r requirements.in, pytest-cov
1212
idna-ssl==1.1.0 # via aiohttp
@@ -37,7 +37,7 @@ six==1.15.0 # via isodate, jsonschema, openapi-core, openapi-schem
3737
strict-rfc3339==0.7 # via openapi-schema-validator
3838
termcolor==1.1.0 # via pytest-sugar
3939
toml==0.10.1 # via pytest
40-
typing-extensions==3.7.4.2 # via aiohttp, yarl
40+
typing-extensions==3.7.4.3 # via aiohttp, yarl
4141
werkzeug==1.0.1 # via openapi-core
4242
yarl==1.5.1 # via aiohttp
4343
zipp==3.1.0 # via importlib-metadata

‎ci/github/unit-testing/postgres-database.bash

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
#!/bin/bash
2+
# http://redsymbol.net/articles/unofficial-bash-strict-mode/
3+
set -euo pipefail
4+
IFS=$'\n\t'
5+
6+
install() {
7+
bash ci/helpers/ensure_python_pip.bash
8+
pushd packages/postgres-database; pip3 install -r requirements/ci.txt; popd;
9+
pip list -v
10+
}
11+
12+
test() {
13+
pytest \
14+
--color=yes \
15+
--durations=10 \
16+
--cov=simcore_postgres_database \
17+
--cov-append \
18+
--cov-report=term-missing \
19+
--cov-report=xml \
20+
--cov-config=.coveragerc \
21+
--verbose \
22+
packages/postgres-database/tests
23+
}
24+
25+
# Check if the function exists (bash specific)
26+
if declare -f "$1" > /dev/null
27+
then
28+
# call arguments verbatim
29+
"$@"
30+
else
31+
# Show a helpful error
32+
echo "'$1' is not a known function name" >&2
33+
exit 1
34+
fi

‎packages/postgres-database/requirements/_base.txt

+2-2
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,6 @@
77
idna==2.10 # via yarl
88
multidict==4.7.6 # via yarl
99
psycopg2-binary==2.8.5 # via sqlalchemy
10-
sqlalchemy[postgresql_psycopg2binary]==1.3.18 # via -r requirements/_base.in
11-
typing-extensions==3.7.4.2 # via yarl
10+
sqlalchemy[postgresql_psycopg2binary]==1.3.19 # via -r requirements/_base.in
11+
typing-extensions==3.7.4.3 # via yarl
1212
yarl==1.5.1 # via -r requirements/_base.in

‎packages/postgres-database/requirements/_migration.txt

+2-2
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,9 @@ python-dateutil==2.8.1 # via alembic
1818
python-editor==1.0.4 # via alembic
1919
requests==2.24.0 # via docker
2020
six==1.15.0 # via docker, python-dateutil, tenacity, websocket-client
21-
sqlalchemy[postgresql_psycopg2binary]==1.3.18 # via -r requirements/_base.txt, alembic
21+
sqlalchemy[postgresql_psycopg2binary]==1.3.19 # via -r requirements/_base.txt, alembic
2222
tenacity==6.2.0 # via -r requirements/_migration.in
23-
typing-extensions==3.7.4.2 # via -r requirements/_base.txt
23+
typing-extensions==3.7.4.3 # via -r requirements/_base.txt, yarl
2424
urllib3==1.25.10 # via -r requirements/_migration.in, requests
2525
websocket-client==0.57.0 # via docker
2626
yarl==1.5.1 # via -r requirements/_base.txt

‎packages/postgres-database/requirements/_test.txt

+9-5
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,10 @@ docker[ssh]==4.3.1 # via -r requirements/_migration.txt, docker-compose
2525
dockerpty==0.4.1 # via docker-compose
2626
docopt==0.6.2 # via coveralls, docker-compose
2727
faker==4.1.2 # via -r requirements/_test.in
28-
idna==2.10 # via -r requirements/_migration.txt, requests, yarl
29-
isort==4.3.21 # via pylint
28+
idna-ssl==1.1.0 # via aiohttp
29+
idna==2.10 # via -r requirements/_migration.txt, idna-ssl, requests, yarl
30+
importlib-metadata==1.7.0 # via jsonschema, pluggy, pytest
31+
isort==5.4.2 # via pylint
3032
jsonschema==3.2.0 # via docker-compose
3133
lazy-object-proxy==1.4.3 # via astroid
3234
mako==1.1.3 # via -r requirements/_migration.txt, alembic
@@ -40,7 +42,7 @@ pluggy==0.13.1 # via pytest
4042
psycopg2-binary==2.8.5 # via -r requirements/_migration.txt, aiopg, sqlalchemy
4143
py==1.9.0 # via pytest
4244
pycparser==2.20 # via cffi
43-
pylint==2.5.3 # via -r requirements/_test.in
45+
pylint==2.6.0 # via -r requirements/_test.in
4446
pynacl==1.4.0 # via paramiko
4547
pyparsing==2.4.7 # via packaging
4648
pyrsistent==0.16.0 # via jsonschema
@@ -56,17 +58,19 @@ python-editor==1.0.4 # via -r requirements/_migration.txt, alembic
5658
pyyaml==5.3.1 # via -r requirements/_test.in, docker-compose
5759
requests==2.24.0 # via -r requirements/_migration.txt, coveralls, docker, docker-compose
5860
six==1.15.0 # via -r requirements/_migration.txt, astroid, bcrypt, cryptography, docker, docker-compose, dockerpty, jsonschema, packaging, pynacl, pyrsistent, python-dateutil, tenacity, websocket-client
59-
sqlalchemy[postgresql_psycopg2binary]==1.3.18 # via -r requirements/_migration.txt, aiopg, alembic
61+
sqlalchemy[postgresql_psycopg2binary]==1.3.19 # via -r requirements/_migration.txt, aiopg, alembic
6062
tenacity==6.2.0 # via -r requirements/_migration.txt
6163
text-unidecode==1.3 # via faker
6264
texttable==1.6.2 # via docker-compose
6365
toml==0.10.1 # via pylint
64-
typing-extensions==3.7.4.2 # via -r requirements/_migration.txt
66+
typed-ast==1.4.1 # via astroid
67+
typing-extensions==3.7.4.3 # via -r requirements/_migration.txt, aiohttp, yarl
6568
urllib3==1.25.10 # via -r requirements/_migration.txt, requests
6669
wcwidth==0.2.5 # via pytest
6770
websocket-client==0.57.0 # via -r requirements/_migration.txt, docker, docker-compose
6871
wrapt==1.12.1 # via astroid
6972
yarl==1.5.1 # via -r requirements/_migration.txt, aiohttp
73+
zipp==3.1.0 # via importlib-metadata
7074

7175
# The following packages are considered to be unsafe in a requirements file:
7276
# setuptools

‎packages/postgres-database/requirements/ci.txt

+4-1
Original file line numberDiff line numberDiff line change
@@ -9,5 +9,8 @@
99
# installs base + tests requirements
1010
-r _test.txt
1111

12+
# installs this repo's packages
13+
-e ../../packages/pytest-simcore/
14+
1215
# current module
13-
.
16+
.[migration]

‎packages/postgres-database/tests/test_delete_projects_and_users.py

+21-23
Original file line numberDiff line numberDiff line change
@@ -16,29 +16,27 @@
1616

1717

1818
@pytest.fixture
19-
def engine(make_engine, loop):
20-
async def start():
21-
engine = await make_engine()
22-
sync_engine = make_engine(False)
23-
metadata.drop_all(sync_engine)
24-
metadata.create_all(sync_engine)
25-
26-
async with engine.acquire() as conn:
27-
await conn.execute(users.insert().values(**random_user(name="A")))
28-
await conn.execute(users.insert().values(**random_user()))
29-
await conn.execute(users.insert().values(**random_user()))
30-
31-
await conn.execute(projects.insert().values(**random_project(prj_owner=1)))
32-
await conn.execute(projects.insert().values(**random_project(prj_owner=2)))
33-
await conn.execute(projects.insert().values(**random_project(prj_owner=3)))
34-
with pytest.raises(ForeignKeyViolation):
35-
await conn.execute(
36-
projects.insert().values(**random_project(prj_owner=4))
37-
)
38-
39-
return engine
40-
41-
return loop.run_until_complete(start())
19+
async def engine(make_engine, loop):
20+
engine = await make_engine()
21+
sync_engine = make_engine(False)
22+
metadata.drop_all(sync_engine)
23+
metadata.create_all(sync_engine)
24+
25+
async with engine.acquire() as conn:
26+
await conn.execute(users.insert().values(**random_user(name="A")))
27+
await conn.execute(users.insert().values(**random_user()))
28+
await conn.execute(users.insert().values(**random_user()))
29+
30+
await conn.execute(projects.insert().values(**random_project(prj_owner=1)))
31+
await conn.execute(projects.insert().values(**random_project(prj_owner=2)))
32+
await conn.execute(projects.insert().values(**random_project(prj_owner=3)))
33+
with pytest.raises(ForeignKeyViolation):
34+
await conn.execute(projects.insert().values(**random_project(prj_owner=4)))
35+
36+
yield engine
37+
38+
engine.close()
39+
await engine.wait_closed()
4240

4341

4442
@pytest.mark.skip(reason="sandbox for dev purposes")

‎packages/postgres-database/tests/test_uniqueness_in_comp_tasks.py

+18-16
Original file line numberDiff line numberDiff line change
@@ -15,24 +15,26 @@
1515

1616

1717
@pytest.fixture
18-
def engine(make_engine, loop):
19-
async def start():
20-
engine = await make_engine()
21-
sync_engine = make_engine(False)
22-
metadata.drop_all(sync_engine)
23-
metadata.create_all(sync_engine)
24-
25-
async with engine.acquire() as conn:
26-
await conn.execute(
27-
comp_pipeline.insert().values(**fake_pipeline(project_id="PA"))
28-
)
29-
await conn.execute(
30-
comp_pipeline.insert().values(**fake_pipeline(project_id="PB"))
31-
)
18+
async def engine(loop, make_engine):
19+
20+
engine = await make_engine()
21+
sync_engine = make_engine(False)
22+
metadata.drop_all(sync_engine)
23+
metadata.create_all(sync_engine)
24+
25+
async with engine.acquire() as conn:
26+
await conn.execute(
27+
comp_pipeline.insert().values(**fake_pipeline(project_id="PA"))
28+
)
29+
await conn.execute(
30+
comp_pipeline.insert().values(**fake_pipeline(project_id="PB"))
31+
)
32+
33+
yield engine
3234

33-
return engine
35+
engine.close()
36+
await engine.wait_closed()
3437

35-
return loop.run_until_complete(start())
3638

3739

3840
async def test_unique_project_node_pairs(engine):

‎packages/s3wrapper/requirements/_test.txt

+3-3
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,12 @@ coveralls==2.1.2 # via -r requirements/_test.in
1717
cryptography==3.0 # via paramiko
1818
distro==1.5.0 # via docker-compose
1919
docker-compose==1.26.2 # via pytest-docker
20-
docker[ssh]==4.3.0 # via docker-compose
20+
docker[ssh]==4.3.1 # via docker-compose
2121
dockerpty==0.4.1 # via docker-compose
2222
docopt==0.6.2 # via coveralls, docker-compose
2323
idna==2.10 # via requests
2424
importlib-metadata==1.7.0 # via jsonschema, pluggy, pytest
25-
isort==4.3.21 # via pylint
25+
isort==5.4.2 # via pylint
2626
jsonschema==3.2.0 # via docker-compose
2727
lazy-object-proxy==1.4.3 # via astroid
2828
mccabe==0.6.1 # via pylint
@@ -33,7 +33,7 @@ paramiko==2.7.1 # via docker
3333
pluggy==0.13.1 # via pytest
3434
py==1.9.0 # via pytest
3535
pycparser==2.20 # via cffi
36-
pylint==2.5.3 # via -r requirements/_test.in
36+
pylint==2.6.0 # via -r requirements/_test.in
3737
pynacl==1.4.0 # via paramiko
3838
pyparsing==2.4.7 # via packaging
3939
pyrsistent==0.16.0 # via jsonschema

‎packages/service-library/requirements/_base.in

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,6 @@ werkzeug
1616
jsonschema
1717
prometheus_client
1818
tenacity
19-
attrs
19+
attrs<20,>=19 # from pytest-docker==0.8.0
2020
trafaret
2121
aiodebug

‎packages/service-library/requirements/_base.txt

+2-2
Original file line numberDiff line numberDiff line change
@@ -25,11 +25,11 @@ psycopg2-binary==2.8.5 # via -r requirements/_base.in, aiopg, sqlalchemy
2525
pyrsistent==0.16.0 # via jsonschema
2626
pyyaml==5.3.1 # via -r requirements/_base.in, openapi-spec-validator
2727
six==1.15.0 # via isodate, jsonschema, openapi-core, openapi-spec-validator, pyrsistent, tenacity
28-
sqlalchemy[postgresql_psycopg2binary]==1.3.18 # via -r requirements/_base.in, aiopg
28+
sqlalchemy[postgresql_psycopg2binary]==1.3.19 # via -r requirements/_base.in, aiopg
2929
strict-rfc3339==0.7 # via openapi-core
3030
tenacity==6.2.0 # via -r requirements/_base.in
3131
trafaret==2.0.2 # via -r requirements/_base.in
32-
typing-extensions==3.7.4.2 # via aiohttp, yarl
32+
typing-extensions==3.7.4.3 # via aiohttp, yarl
3333
ujson==3.1.0 # via -r requirements/_base.in
3434
werkzeug==1.0.1 # via -r requirements/_base.in
3535
yarl==1.5.1 # via aiohttp

‎packages/service-library/requirements/_test.txt

+6-6
Original file line numberDiff line numberDiff line change
@@ -21,14 +21,14 @@ coveralls==2.1.2 # via -r requirements/_test.in
2121
cryptography==3.0 # via paramiko
2222
distro==1.5.0 # via docker-compose
2323
docker-compose==1.26.2 # via pytest-docker
24-
docker[ssh]==4.3.0 # via docker-compose
24+
docker[ssh]==4.3.1 # via docker-compose
2525
dockerpty==0.4.1 # via docker-compose
2626
docopt==0.6.2 # via coveralls, docker-compose
2727
idna-ssl==1.1.0 # via -r requirements/_base.txt, aiohttp
2828
idna==2.10 # via -r requirements/_base.txt, idna-ssl, requests, yarl
2929
importlib-metadata==1.7.0 # via -r requirements/_base.txt, jsonschema, pluggy, pytest
3030
isodate==0.6.0 # via -r requirements/_base.txt, openapi-core
31-
isort==4.3.21 # via pylint
31+
isort==5.4.2 # via pylint
3232
jsonschema==3.2.0 # via -r requirements/_base.txt, docker-compose, openapi-spec-validator
3333
lazy-object-proxy==1.4.3 # via -r requirements/_base.txt, astroid, openapi-core
3434
mccabe==0.6.1 # via pylint
@@ -43,31 +43,31 @@ prometheus-client==0.8.0 # via -r requirements/_base.txt
4343
psycopg2-binary==2.8.5 # via -r requirements/_base.txt, aiopg, sqlalchemy
4444
py==1.9.0 # via pytest
4545
pycparser==2.20 # via cffi
46-
pylint==2.5.3 # via -r requirements/_test.in
46+
pylint==2.6.0 # via -r requirements/_test.in
4747
pynacl==1.4.0 # via paramiko
4848
pyparsing==2.4.7 # via packaging
4949
pyrsistent==0.16.0 # via -r requirements/_base.txt, jsonschema
5050
pytest-aiohttp==0.3.0 # via -r requirements/_test.in
5151
pytest-cov==2.10.1 # via -r requirements/_test.in
5252
pytest-docker==0.8.0 # via -r requirements/_test.in
5353
pytest-instafail==0.4.2 # via -r requirements/_test.in
54-
pytest-mock==3.2.0 # via -r requirements/_test.in
54+
pytest-mock==3.3.0 # via -r requirements/_test.in
5555
pytest-runner==5.2 # via -r requirements/_test.in
5656
pytest-sugar==0.9.4 # via -r requirements/_test.in
5757
pytest==5.4.3 # via -r requirements/_test.in, pytest-aiohttp, pytest-cov, pytest-docker, pytest-instafail, pytest-mock, pytest-sugar
5858
python-dotenv==0.14.0 # via docker-compose
5959
pyyaml==5.3.1 # via -r requirements/_base.txt, docker-compose, openapi-spec-validator
6060
requests==2.24.0 # via coveralls, docker, docker-compose
6161
six==1.15.0 # via -r requirements/_base.txt, astroid, bcrypt, cryptography, docker, docker-compose, dockerpty, isodate, jsonschema, openapi-core, openapi-spec-validator, packaging, pynacl, pyrsistent, tenacity, websocket-client
62-
sqlalchemy[postgresql_psycopg2binary]==1.3.18 # via -r requirements/_base.txt, aiopg
62+
sqlalchemy[postgresql_psycopg2binary]==1.3.19 # via -r requirements/_base.txt, aiopg
6363
strict-rfc3339==0.7 # via -r requirements/_base.txt, openapi-core
6464
tenacity==6.2.0 # via -r requirements/_base.txt
6565
termcolor==1.1.0 # via pytest-sugar
6666
texttable==1.6.2 # via docker-compose
6767
toml==0.10.1 # via pylint
6868
trafaret==2.0.2 # via -r requirements/_base.txt
6969
typed-ast==1.4.1 # via astroid
70-
typing-extensions==3.7.4.2 # via -r requirements/_base.txt, aiohttp, yarl
70+
typing-extensions==3.7.4.3 # via -r requirements/_base.txt, aiohttp, yarl
7171
ujson==3.1.0 # via -r requirements/_base.txt
7272
urllib3==1.25.10 # via requests
7373
wcwidth==0.2.5 # via pytest

0 commit comments

Comments
 (0)