[enhancement] Check task exists in db and requirements upgrade

.travis.yml

@@ -153,28 +153,6 @@ jobs:
       after_failure:
         - unbuffer bash ci/travis/unit-testing/director.bash after_failure
 
-    # test python, sidecar ----------------------------------------------------------------------
-    # TODO: activate when sidecar tests are READY!
-    # - stage: build / unit-testing
-    #   name: sidecar
-    #   language: python
-    #   python:
-    #     - "3.6"
-    #   sudo: required
-    #   cache: pip
-    #   before_install:
-    #     - sudo bash ci/travis/unit-testing/sidecar.bash before_install
-    #   install:
-    #     - bash ci/travis/unit-testing/sidecar.bash install
-    #   before_script:
-    #     - bash ci/travis/unit-testing/sidecar.bash before_script
-    #   script:
-    #     - bash ci/travis/unit-testing/sidecar.bash script
-    #   after_success:
-    #     - bash ci/travis/unit-testing/sidecar.bash after_success
-    #   after_failure:
-    #     - bash ci/travis/unit-testing/sidecar.bash after_failure
-
     # test python, service-library ----------------------------------------------------------------------
     - stage: build / unit-testing
       name: service-library

api/tests/Makefile

@@ -6,7 +6,7 @@ VENV_DIR ?= $(abspath $(ROOT_DIR)/.venv)
 .PHONY: all
 all: install tests
 
-%.txt: %.in
+requirements.txt: requirements.in
 	# pip compiling $<
 	@$(VENV_DIR)/bin/pip-compile --output-file $@ $<
 

api/tests/requirements.txt

@@ -10,16 +10,18 @@ attrs==19.3.0             # via aiohttp, jsonschema, openapi-core, pytest
 chardet==3.0.4            # via aiohttp
 coverage==4.5.1
 idna-ssl==1.1.0           # via aiohttp
-idna==2.8                 # via idna-ssl, yarl
-importlib-metadata==1.3.0  # via jsonschema, pluggy, pytest
-isodate==0.6.0            # via openapi-core
-jsonschema==3.2.0         # via openapi-spec-validator
+idna==2.9                 # via idna-ssl, yarl
+importlib-metadata==1.5.0  # via jsonschema, pluggy, pytest
+isodate==0.6.0            # via openapi-schema-validator
+jsonschema==3.2.0         # via openapi-schema-validator, openapi-spec-validator
 lazy-object-proxy==1.4.3  # via openapi-core
-more-itertools==8.0.2     # via pytest, zipp
-multidict==4.7.3          # via aiohttp, yarl
-openapi-core==0.12.0
+more-itertools==8.2.0     # via openapi-core, pytest
+multidict==4.7.5          # via aiohttp, yarl
+openapi-core==0.13.3
+openapi-schema-validator==0.1.1  # via openapi-core
 openapi-spec-validator==0.2.8  # via openapi-core
-packaging==20.0           # via pytest, pytest-sugar
+packaging==20.3           # via pytest, pytest-sugar
+parse==1.15.0             # via openapi-core
 pluggy==0.13.1            # via pytest
 py==1.8.1                 # via pytest
 pyparsing==2.4.6          # via packaging
@@ -28,15 +30,16 @@ pytest-aiohttp==0.3.0
 pytest-cov==2.8.1
 pytest-instafail==0.4.1.post0
 pytest-sugar==0.9.2
-pytest==5.3.2
+pytest==5.3.5
 pyyaml==5.3               # via openapi-spec-validator
-six==1.13.0               # via isodate, jsonschema, openapi-core, openapi-spec-validator, packaging, pyrsistent
-strict-rfc3339==0.7       # via openapi-core
+six==1.14.0               # via isodate, jsonschema, openapi-core, openapi-schema-validator, openapi-spec-validator, packaging, pyrsistent
+strict-rfc3339==0.7       # via openapi-schema-validator
 termcolor==1.1.0          # via pytest-sugar
 typing-extensions==3.7.4.1  # via aiohttp
 wcwidth==0.1.8            # via pytest
+werkzeug==1.0.0           # via openapi-core
 yarl==1.4.2               # via aiohttp
-zipp==0.6.0               # via importlib-metadata
+zipp==3.1.0               # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools

packages/postgres-database/Makefile

@@ -34,8 +34,20 @@ setup-commit: install-dev up-pg ## sets up a database to create a new commit int
 	@echo "To add new commit, sc-pg review -m \"Some message\" "
 
 
-.PHONY: up-pg down-pg
+.PHONY: migrate
+DOT_ENV_FILE = $(abspath $(REPO_BASE_DIR)/.env)
+
+migrate: $(DOT_ENV_FILE) ## basic migration update (use ONLY for development purposes)
+	# Discovering postpgres server
+	@export $(shell grep -v '^#' $< | xargs -0); sc-pg discover
+	# Upgraded to head
+	@sc-pg upgrade
+	@sc-pg clean
+
 
+
+
+.PHONY: up-pg down-pg
 DOCKER_COMPOSE_CONFIG := tests/docker-compose.yml
 
 up-pg: $(DOCKER_COMPOSE_CONFIG) ## starts pg server

packages/postgres-database/requirements/_test.txt

@@ -17,51 +17,51 @@ certifi==2019.6.16
 chardet==3.0.4
 click==7.0
 coverage==4.5.1
-coveralls==1.10.0
+coveralls==1.11.1
 docker==4.0.2
 docopt==0.6.2             # via coveralls
 faker==4.0.1
 idna-ssl==1.1.0           # via aiohttp
 idna==2.8
-importlib-metadata==1.3.0  # via pluggy, pytest
+importlib-metadata==1.5.0  # via pluggy, pytest
 isort==4.3.21             # via pylint
 lazy-object-proxy==1.4.3  # via astroid
 mako==1.0.12
 markupsafe==1.1.1
 mccabe==0.6.1             # via pylint
-more-itertools==8.0.2     # via pytest, zipp
+more-itertools==8.2.0     # via pytest
 multidict==4.5.2
-packaging==20.0           # via pytest
+packaging==20.3           # via pytest
 pathspec==0.7.0           # via black
-pip-tools==4.4.1
+pip-tools==4.5.1
 pluggy==0.13.1            # via pytest
 psycopg2-binary==2.8.4
 py==1.8.1                 # via pytest
-pyaml==19.12.0
+pyaml==20.3.1
 pylint==2.4.4
 pyparsing==2.4.6          # via packaging
 pytest-aiohttp==0.3.0
 pytest-cov==2.8.1
 pytest-docker==0.7.2
 pytest-instafail==0.4.1.post0
 pytest-runner==5.2
-pytest==5.3.2
+pytest==5.3.5
 python-dateutil==2.8.0
 python-editor==1.0.4
 pyyaml==5.3               # via pyaml
-regex==2020.2.18          # via black
+regex==2020.2.20          # via black
 requests==2.22.0
 rope==0.16.0
 six==1.12.0
 sqlalchemy[postgresql_psycopg2binary]==1.3.5
 tenacity==6.0.0
 text-unidecode==1.3       # via faker
 toml==0.10.0              # via black
-typed-ast==1.4.0          # via astroid, black
+typed-ast==1.4.1          # via astroid, black
 typing-extensions==3.7.4.1  # via aiohttp
 urllib3==1.25.3
 wcwidth==0.1.8            # via pytest
 websocket-client==0.56.0
 wrapt==1.11.2             # via astroid
 yarl==1.3.0
-zipp==0.6.0               # via importlib-metadata
+zipp==3.1.0               # via importlib-metadata

packages/postgres-database/src/simcore_postgres_database/cli.py

@@ -109,28 +109,30 @@ def main():
     """ Simplified CLI for database migration with alembic """
 
 
+
+
 @main.command()
 @click.option("--user", "-u")
 @click.option("--password", "-p")
 @click.option("--host")
 @click.option("--port", type=int)
 @click.option("--database", "-d")
 def discover(**cli_inputs):
-    """ Discovers databases and stores configs in ~/.simcore_postgres_database.json """
+    """ Discovers databases and caches configs in ~/.simcore_postgres_database.json (except if --no-cache)"""
     # NOTE: Do not add defaults to user, password so we get a chance to ping urls
     # TODO: if multiple candidates online, then query user to select
 
     click.echo(f"Discovering database ...")
     cli_cfg = {key: value for key, value in cli_inputs.items() if value is not None}
 
-    def test_cached():
+    def _test_cached():
         """Tests cached configuration """
         cfg = _load_cache() or {}
         if cfg:
             cfg.update(cli_cfg)  # overrides
         return cfg
 
-    def test_env():
+    def _test_env():
         """Tests environ variables """
         cfg = {
             "user": os.getenv("POSTGRES_USER"),
@@ -142,15 +144,15 @@ def test_env():
         cfg.update(cli_cfg)
         return cfg
 
-    def test_swarm():
+    def _test_swarm():
         """Tests published port in swarm from host """
-        cfg = deepcopy(cli_cfg)
+        cfg = _test_env()
         cfg["host"] = "127.0.0.1"
         cfg["port"] = _get_service_published_port(cli_cfg.get("host", DEFAULT_HOST))
         cfg.setdefault("database", DEFAULT_DB)
         return cfg
 
-    for test in [test_cached, test_env, test_swarm]:
+    for test in [_test_cached, _test_env, _test_swarm]:
         try:
             click.echo("-> {0.__name__}: {0.__doc__}".format(test))
 
@@ -161,14 +163,13 @@ def test_swarm():
             click.echo(" ping {0.__name__}: {1} ...".format(test, url))
 
             raise_if_not_responsive(url)
-
             with open(discovered_cache, "w") as fh:
                 json.dump(cfg, fh, sort_keys=True, indent=4)
 
             click.echo(f"Saved config at{discovered_cache}: {cfg}")
             click.secho(
                 f"{test.__name__} succeeded: {url} is online",
-                blink=True,
+                blink=False,
                 bold=True,
                 fg="green",
             )
@@ -180,7 +181,7 @@ def test_swarm():
             click.echo("<- {0.__name__} failed : {1}".format(test, inline_msg))
 
     _reset_cache()
-    click.secho("Sorry, database not found !!", blink=True, bold=True, fg="red")
+    click.secho("Sorry, database not found !!", blink=False, bold=True, fg="red")
 
 
 @main.command()
@@ -205,8 +206,6 @@ def clean():
 
 
 # Bypasses alembic CLI into a reduced version  ------------
-# TODO: systematic bypass??
-
 
 @main.command()
 @click.option("-m", "message")

packages/postgres-database/tests/test_delete_projects_and_users.py

@@ -9,15 +9,13 @@
 
 import faker
 import pytest
+import sqlalchemy as sa
 from aiopg.sa.result import ResultProxy, RowProxy
 
 from simcore_postgres_database.models.base import metadata
-from simcore_postgres_database.webserver_models import (
-    UserStatus,
-    projects,
-    user_to_projects,
-    users,
-)
+from simcore_postgres_database.webserver_models import (UserStatus, projects,
+                                                        user_to_projects,
+                                                        users)
 
 fake = faker.Faker()
 
@@ -79,6 +77,23 @@ async def start():
     return loop.run_until_complete(start())
 
 
+
+async def test_count_users(engine):
+    async with engine.acquire() as conn:
+        users_count = await conn.scalar(users.count())
+        assert users_count == 3
+
+        users_count = await conn.scalar(
+            sa.select([sa.func.count()]).where(users.c.name == "A")
+        )
+        assert users_count == 1
+
+        users_count = await conn.scalar(
+            sa.select([sa.func.count()]).where(users.c.name == "UNKNOWN NAME")
+        )
+        assert users_count == 0
+
+
 @pytest.mark.skip(reason="UNDER DEV")
 async def test_view(engine):
     async with engine.acquire() as conn:

packages/s3wrapper/requirements/_test.txt

@@ -9,30 +9,30 @@ attrs==19.3.0             # via pytest, pytest-docker
 certifi==2019.3.9
 chardet==3.0.4            # via requests
 coverage==4.5.1
-coveralls==1.10.0
+coveralls==1.11.1
 docopt==0.6.2             # via coveralls
-idna==2.8                 # via requests
-importlib-metadata==1.3.0  # via pluggy, pytest
+idna==2.9                 # via requests
+importlib-metadata==1.5.0  # via pluggy, pytest
 isort==4.3.21             # via pylint
 lazy-object-proxy==1.4.3  # via astroid
 mccabe==0.6.1             # via pylint
 minio==4.0.16
-more-itertools==8.0.2     # via pytest, zipp
-packaging==20.0           # via pytest
+more-itertools==8.2.0     # via pytest
+packaging==20.3           # via pytest
 pluggy==0.13.1            # via pytest
 py==1.8.1                 # via pytest
 pylint==2.4.4
 pyparsing==2.4.6          # via packaging
 pytest-cov==2.8.1
-pytest-docker==0.6.1
+pytest-docker==0.7.2
 pytest-runner==5.2
-pytest==5.3.2
+pytest==5.3.5
 python-dateutil==2.8.0
 pytz==2019.1
-requests==2.22.0
+requests==2.23.0
 six==1.12.0
-typed-ast==1.4.0          # via astroid
+typed-ast==1.4.1          # via astroid
 urllib3==1.25.2
 wcwidth==0.1.8            # via pytest
 wrapt==1.11.2             # via astroid
-zipp==0.6.0               # via importlib-metadata
+zipp==3.1.0               # via importlib-metadata