Delayed diagnostics startup and maintenance upgrades

.github/workflows/ci-testing-deploy.yml

@@ -249,6 +249,7 @@ jobs:
         with:
           name: unit_sidecar_coverage
           path: codeclimate.unit_sidecar_coverage.json
+
   unit-test-frontend:
     name: Unit-testing frontend
     runs-on: ${{ matrix.os }}
@@ -320,6 +321,53 @@ jobs:
       - name: test
         run: ./ci/github/unit-testing/python-linting.bash test
 
+  unit-test-postgres-database:
+    name: Unit-testing postgres-database
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python: [3.6]
+        os: [ubuntu-20.04]
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v2
+      - name: setup docker
+        run: |
+          sudo ./ci/github/helpers/setup_docker_compose.bash
+          ./ci/github/helpers/setup_docker_experimental.bash
+          ./ci/github/helpers/setup_docker_buildx.bash
+          echo ::set-env name=DOCKER_BUILDX::1
+      - name: setup python environment
+        uses: actions/[email protected]
+        with:
+          python-version: ${{ matrix.python }}
+      - name: show system version
+        run: ./ci/helpers/show_system_versions.bash
+      - uses: actions/cache@v1
+        name: getting cached data
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: install
+        run: ./ci/github/unit-testing/postgres-database.bash install
+      - name: test
+        run: ./ci/github/unit-testing/postgres-database.bash test
+      - uses: codecov/codecov-action@v1
+        with:
+          flags: unittests #optional
+      - name: prepare codeclimate coverage file
+        run: |
+          curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
+          chmod +x ./cc-test-reporter
+          ./cc-test-reporter format-coverage -t coverage.py -o codeclimate.unit_postgresdb_coverage.json coverage.xml
+      - name: upload codeclimate coverage
+        uses: actions/upload-artifact@v2
+        with:
+          name: unit_postgresdb_coverage
+          path: codeclimate.unit_postgresdb_coverage.json
+
   unit-test-service-library:
     name: Unit-testing service-library
     runs-on: ${{ matrix.os }}

api/tests/requirements.txt

@@ -6,7 +6,7 @@
 #
 aiohttp==3.6.2            # via -r requirements.in, pytest-aiohttp
 async-timeout==3.0.1      # via aiohttp
-attrs==19.3.0             # via aiohttp, jsonschema, openapi-core, pytest
+attrs==20.1.0             # via aiohttp, jsonschema, openapi-core, pytest
 chardet==3.0.4            # via aiohttp
 coverage==5.2.1           # via -r requirements.in, pytest-cov
 idna-ssl==1.1.0           # via aiohttp
@@ -37,7 +37,7 @@ six==1.15.0               # via isodate, jsonschema, openapi-core, openapi-schem
 strict-rfc3339==0.7       # via openapi-schema-validator
 termcolor==1.1.0          # via pytest-sugar
 toml==0.10.1              # via pytest
-typing-extensions==3.7.4.2  # via aiohttp, yarl
+typing-extensions==3.7.4.3  # via aiohttp, yarl
 werkzeug==1.0.1           # via openapi-core
 yarl==1.5.1               # via aiohttp
 zipp==3.1.0               # via importlib-metadata

ci/github/unit-testing/postgres-database.bash

@@ -0,0 +1,34 @@
+#!/bin/bash
+# http://redsymbol.net/articles/unofficial-bash-strict-mode/
+set -euo pipefail
+IFS=$'\n\t'
+
+install() {
+    bash ci/helpers/ensure_python_pip.bash
+    pushd packages/postgres-database; pip3 install -r requirements/ci.txt; popd;
+    pip list -v
+}
+
+test() {
+    pytest \
+      --color=yes \
+      --durations=10 \
+      --cov=simcore_postgres_database \
+      --cov-append \
+      --cov-report=term-missing \
+      --cov-report=xml \
+      --cov-config=.coveragerc \
+      --verbose \
+      packages/postgres-database/tests
+}
+
+# Check if the function exists (bash specific)
+if declare -f "$1" > /dev/null
+then
+  # call arguments verbatim
+  "$@"
+else
+  # Show a helpful error
+  echo "'$1' is not a known function name" >&2
+  exit 1
+fi

packages/postgres-database/requirements/_base.txt

@@ -7,6 +7,6 @@
 idna==2.10                # via yarl
 multidict==4.7.6          # via yarl
 psycopg2-binary==2.8.5    # via sqlalchemy
-sqlalchemy[postgresql_psycopg2binary]==1.3.18  # via -r requirements/_base.in
-typing-extensions==3.7.4.2  # via yarl
+sqlalchemy[postgresql_psycopg2binary]==1.3.19  # via -r requirements/_base.in
+typing-extensions==3.7.4.3  # via yarl
 yarl==1.5.1               # via -r requirements/_base.in

packages/postgres-database/requirements/_migration.txt

@@ -18,9 +18,9 @@ python-dateutil==2.8.1    # via alembic
 python-editor==1.0.4      # via alembic
 requests==2.24.0          # via docker
 six==1.15.0               # via docker, python-dateutil, tenacity, websocket-client
-sqlalchemy[postgresql_psycopg2binary]==1.3.18  # via -r requirements/_base.txt, alembic
+sqlalchemy[postgresql_psycopg2binary]==1.3.19  # via -r requirements/_base.txt, alembic
 tenacity==6.2.0           # via -r requirements/_migration.in
-typing-extensions==3.7.4.2  # via -r requirements/_base.txt
+typing-extensions==3.7.4.3  # via -r requirements/_base.txt, yarl
 urllib3==1.25.10          # via -r requirements/_migration.in, requests
 websocket-client==0.57.0  # via docker
 yarl==1.5.1               # via -r requirements/_base.txt

packages/postgres-database/requirements/_test.txt

@@ -25,8 +25,10 @@ docker[ssh]==4.3.1        # via -r requirements/_migration.txt, docker-compose
 dockerpty==0.4.1          # via docker-compose
 docopt==0.6.2             # via coveralls, docker-compose
 faker==4.1.2              # via -r requirements/_test.in
-idna==2.10                # via -r requirements/_migration.txt, requests, yarl
-isort==4.3.21             # via pylint
+idna-ssl==1.1.0           # via aiohttp
+idna==2.10                # via -r requirements/_migration.txt, idna-ssl, requests, yarl
+importlib-metadata==1.7.0  # via jsonschema, pluggy, pytest
+isort==5.4.2              # via pylint
 jsonschema==3.2.0         # via docker-compose
 lazy-object-proxy==1.4.3  # via astroid
 mako==1.1.3               # via -r requirements/_migration.txt, alembic
@@ -40,7 +42,7 @@ pluggy==0.13.1            # via pytest
 psycopg2-binary==2.8.5    # via -r requirements/_migration.txt, aiopg, sqlalchemy
 py==1.9.0                 # via pytest
 pycparser==2.20           # via cffi
-pylint==2.5.3             # via -r requirements/_test.in
+pylint==2.6.0             # via -r requirements/_test.in
 pynacl==1.4.0             # via paramiko
 pyparsing==2.4.7          # via packaging
 pyrsistent==0.16.0        # via jsonschema
@@ -56,17 +58,19 @@ python-editor==1.0.4      # via -r requirements/_migration.txt, alembic
 pyyaml==5.3.1             # via -r requirements/_test.in, docker-compose
 requests==2.24.0          # via -r requirements/_migration.txt, coveralls, docker, docker-compose
 six==1.15.0               # via -r requirements/_migration.txt, astroid, bcrypt, cryptography, docker, docker-compose, dockerpty, jsonschema, packaging, pynacl, pyrsistent, python-dateutil, tenacity, websocket-client
-sqlalchemy[postgresql_psycopg2binary]==1.3.18  # via -r requirements/_migration.txt, aiopg, alembic
+sqlalchemy[postgresql_psycopg2binary]==1.3.19  # via -r requirements/_migration.txt, aiopg, alembic
 tenacity==6.2.0           # via -r requirements/_migration.txt
 text-unidecode==1.3       # via faker
 texttable==1.6.2          # via docker-compose
 toml==0.10.1              # via pylint
-typing-extensions==3.7.4.2  # via -r requirements/_migration.txt
+typed-ast==1.4.1          # via astroid
+typing-extensions==3.7.4.3  # via -r requirements/_migration.txt, aiohttp, yarl
 urllib3==1.25.10          # via -r requirements/_migration.txt, requests
 wcwidth==0.2.5            # via pytest
 websocket-client==0.57.0  # via -r requirements/_migration.txt, docker, docker-compose
 wrapt==1.12.1             # via astroid
 yarl==1.5.1               # via -r requirements/_migration.txt, aiohttp
+zipp==3.1.0               # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools

packages/postgres-database/requirements/ci.txt

@@ -9,5 +9,8 @@
 # installs base + tests requirements
 -r _test.txt
 
+# installs this repo's packages
+-e ../../packages/pytest-simcore/
+
 # current module
-.
+.[migration]

packages/postgres-database/tests/test_delete_projects_and_users.py

@@ -16,29 +16,27 @@
 
 
 @pytest.fixture
-def engine(make_engine, loop):
-    async def start():
-        engine = await make_engine()
-        sync_engine = make_engine(False)
-        metadata.drop_all(sync_engine)
-        metadata.create_all(sync_engine)
-
-        async with engine.acquire() as conn:
-            await conn.execute(users.insert().values(**random_user(name="A")))
-            await conn.execute(users.insert().values(**random_user()))
-            await conn.execute(users.insert().values(**random_user()))
-
-            await conn.execute(projects.insert().values(**random_project(prj_owner=1)))
-            await conn.execute(projects.insert().values(**random_project(prj_owner=2)))
-            await conn.execute(projects.insert().values(**random_project(prj_owner=3)))
-            with pytest.raises(ForeignKeyViolation):
-                await conn.execute(
-                    projects.insert().values(**random_project(prj_owner=4))
-                )
-
-        return engine
-
-    return loop.run_until_complete(start())
+async def engine(make_engine, loop):
+    engine = await make_engine()
+    sync_engine = make_engine(False)
+    metadata.drop_all(sync_engine)
+    metadata.create_all(sync_engine)
+
+    async with engine.acquire() as conn:
+        await conn.execute(users.insert().values(**random_user(name="A")))
+        await conn.execute(users.insert().values(**random_user()))
+        await conn.execute(users.insert().values(**random_user()))
+
+        await conn.execute(projects.insert().values(**random_project(prj_owner=1)))
+        await conn.execute(projects.insert().values(**random_project(prj_owner=2)))
+        await conn.execute(projects.insert().values(**random_project(prj_owner=3)))
+        with pytest.raises(ForeignKeyViolation):
+            await conn.execute(projects.insert().values(**random_project(prj_owner=4)))
+
+    yield engine
+
+    engine.close()
+    await engine.wait_closed()
 
 
 @pytest.mark.skip(reason="sandbox for dev purposes")

packages/postgres-database/tests/test_uniqueness_in_comp_tasks.py

@@ -15,24 +15,26 @@
 
 
 @pytest.fixture
-def engine(make_engine, loop):
-    async def start():
-        engine = await make_engine()
-        sync_engine = make_engine(False)
-        metadata.drop_all(sync_engine)
-        metadata.create_all(sync_engine)
-
-        async with engine.acquire() as conn:
-            await conn.execute(
-                comp_pipeline.insert().values(**fake_pipeline(project_id="PA"))
-            )
-            await conn.execute(
-                comp_pipeline.insert().values(**fake_pipeline(project_id="PB"))
-            )
+async def engine(loop, make_engine):
+
+    engine = await make_engine()
+    sync_engine = make_engine(False)
+    metadata.drop_all(sync_engine)
+    metadata.create_all(sync_engine)
+
+    async with engine.acquire() as conn:
+        await conn.execute(
+            comp_pipeline.insert().values(**fake_pipeline(project_id="PA"))
+        )
+        await conn.execute(
+            comp_pipeline.insert().values(**fake_pipeline(project_id="PB"))
+        )
+
+    yield engine
 
-        return engine
+    engine.close()
+    await engine.wait_closed()
 
-    return loop.run_until_complete(start())
 
 
 async def test_unique_project_node_pairs(engine):

packages/s3wrapper/requirements/_test.txt

@@ -17,12 +17,12 @@ coveralls==2.1.2          # via -r requirements/_test.in
 cryptography==3.0         # via paramiko
 distro==1.5.0             # via docker-compose
 docker-compose==1.26.2    # via pytest-docker
-docker[ssh]==4.3.0        # via docker-compose
+docker[ssh]==4.3.1        # via docker-compose
 dockerpty==0.4.1          # via docker-compose
 docopt==0.6.2             # via coveralls, docker-compose
 idna==2.10                # via requests
 importlib-metadata==1.7.0  # via jsonschema, pluggy, pytest
-isort==4.3.21             # via pylint
+isort==5.4.2              # via pylint
 jsonschema==3.2.0         # via docker-compose
 lazy-object-proxy==1.4.3  # via astroid
 mccabe==0.6.1             # via pylint
@@ -33,7 +33,7 @@ paramiko==2.7.1           # via docker
 pluggy==0.13.1            # via pytest
 py==1.9.0                 # via pytest
 pycparser==2.20           # via cffi
-pylint==2.5.3             # via -r requirements/_test.in
+pylint==2.6.0             # via -r requirements/_test.in
 pynacl==1.4.0             # via paramiko
 pyparsing==2.4.7          # via packaging
 pyrsistent==0.16.0        # via jsonschema

packages/service-library/requirements/_base.in

@@ -16,6 +16,6 @@ werkzeug
 jsonschema
 prometheus_client
 tenacity
-attrs
+attrs<20,>=19   # from pytest-docker==0.8.0
 trafaret
 aiodebug

packages/service-library/requirements/_base.txt

@@ -25,11 +25,11 @@ psycopg2-binary==2.8.5    # via -r requirements/_base.in, aiopg, sqlalchemy
 pyrsistent==0.16.0        # via jsonschema
 pyyaml==5.3.1             # via -r requirements/_base.in, openapi-spec-validator
 six==1.15.0               # via isodate, jsonschema, openapi-core, openapi-spec-validator, pyrsistent, tenacity
-sqlalchemy[postgresql_psycopg2binary]==1.3.18  # via -r requirements/_base.in, aiopg
+sqlalchemy[postgresql_psycopg2binary]==1.3.19  # via -r requirements/_base.in, aiopg
 strict-rfc3339==0.7       # via openapi-core
 tenacity==6.2.0           # via -r requirements/_base.in
 trafaret==2.0.2           # via -r requirements/_base.in
-typing-extensions==3.7.4.2  # via aiohttp, yarl
+typing-extensions==3.7.4.3  # via aiohttp, yarl
 ujson==3.1.0              # via -r requirements/_base.in
 werkzeug==1.0.1           # via -r requirements/_base.in
 yarl==1.5.1               # via aiohttp

packages/service-library/requirements/_test.txt

@@ -21,14 +21,14 @@ coveralls==2.1.2          # via -r requirements/_test.in
 cryptography==3.0         # via paramiko
 distro==1.5.0             # via docker-compose
 docker-compose==1.26.2    # via pytest-docker
-docker[ssh]==4.3.0        # via docker-compose
+docker[ssh]==4.3.1        # via docker-compose
 dockerpty==0.4.1          # via docker-compose
 docopt==0.6.2             # via coveralls, docker-compose
 idna-ssl==1.1.0           # via -r requirements/_base.txt, aiohttp
 idna==2.10                # via -r requirements/_base.txt, idna-ssl, requests, yarl
 importlib-metadata==1.7.0  # via -r requirements/_base.txt, jsonschema, pluggy, pytest
 isodate==0.6.0            # via -r requirements/_base.txt, openapi-core
-isort==4.3.21             # via pylint
+isort==5.4.2              # via pylint
 jsonschema==3.2.0         # via -r requirements/_base.txt, docker-compose, openapi-spec-validator
 lazy-object-proxy==1.4.3  # via -r requirements/_base.txt, astroid, openapi-core
 mccabe==0.6.1             # via pylint
@@ -43,31 +43,31 @@ prometheus-client==0.8.0  # via -r requirements/_base.txt
 psycopg2-binary==2.8.5    # via -r requirements/_base.txt, aiopg, sqlalchemy
 py==1.9.0                 # via pytest
 pycparser==2.20           # via cffi
-pylint==2.5.3             # via -r requirements/_test.in
+pylint==2.6.0             # via -r requirements/_test.in
 pynacl==1.4.0             # via paramiko
 pyparsing==2.4.7          # via packaging
 pyrsistent==0.16.0        # via -r requirements/_base.txt, jsonschema
 pytest-aiohttp==0.3.0     # via -r requirements/_test.in
 pytest-cov==2.10.1        # via -r requirements/_test.in
 pytest-docker==0.8.0      # via -r requirements/_test.in
 pytest-instafail==0.4.2   # via -r requirements/_test.in
-pytest-mock==3.2.0        # via -r requirements/_test.in
+pytest-mock==3.3.0        # via -r requirements/_test.in
 pytest-runner==5.2        # via -r requirements/_test.in
 pytest-sugar==0.9.4       # via -r requirements/_test.in
 pytest==5.4.3             # via -r requirements/_test.in, pytest-aiohttp, pytest-cov, pytest-docker, pytest-instafail, pytest-mock, pytest-sugar
 python-dotenv==0.14.0     # via docker-compose
 pyyaml==5.3.1             # via -r requirements/_base.txt, docker-compose, openapi-spec-validator
 requests==2.24.0          # via coveralls, docker, docker-compose
 six==1.15.0               # via -r requirements/_base.txt, astroid, bcrypt, cryptography, docker, docker-compose, dockerpty, isodate, jsonschema, openapi-core, openapi-spec-validator, packaging, pynacl, pyrsistent, tenacity, websocket-client
-sqlalchemy[postgresql_psycopg2binary]==1.3.18  # via -r requirements/_base.txt, aiopg
+sqlalchemy[postgresql_psycopg2binary]==1.3.19  # via -r requirements/_base.txt, aiopg
 strict-rfc3339==0.7       # via -r requirements/_base.txt, openapi-core
 tenacity==6.2.0           # via -r requirements/_base.txt
 termcolor==1.1.0          # via pytest-sugar
 texttable==1.6.2          # via docker-compose
 toml==0.10.1              # via pylint
 trafaret==2.0.2           # via -r requirements/_base.txt
 typed-ast==1.4.1          # via astroid
-typing-extensions==3.7.4.2  # via -r requirements/_base.txt, aiohttp, yarl
+typing-extensions==3.7.4.3  # via -r requirements/_base.txt, aiohttp, yarl
 ujson==3.1.0              # via -r requirements/_base.txt
 urllib3==1.25.10          # via requests
 wcwidth==0.2.5            # via pytest