[maintenance] black-formatted packages/services and new common.Makefile

.vscode-template/settings.json

@@ -8,7 +8,8 @@
     ".env*": "ini",
     "Dockerfile*": "dockerfile",
     "**/requirements/*.txt": "pip-requirements",
-    "**/requirements/*.in": "pip-requirements"
+    "**/requirements/*.in": "pip-requirements",
+    "*Makefile": "makefile"
   },
   "files.eol": "\n",
   "files.insertFinalNewline": true,
@@ -28,7 +29,8 @@
   },
   "python.testing.pyTestEnabled": true,
   "autoDocstring.docstringFormat": "sphinx",
-  "shellcheck.executablePath": "${workspaceFolder}/scripts/shellcheck",
+  "shellcheck.executablePath": "${workspaceFolder}/scripts/shellcheck.bash",
   "shellcheck.run": "onSave",
-  "shellcheck.enableQuickFix": true
+  "shellcheck.enableQuickFix": true,
+  "python.formatting.provider": "black"
 }

Makefile

@@ -340,9 +340,9 @@ openapi-specs: ## bundles and validates openapi specifications and schemas of AL
 .PHONY: code-analysis
 code-analysis: .codeclimate.yml ## runs code-climate analysis
 	# Validates $<
-	./scripts/code-climate.sh validate-config
+	./scripts/code-climate.bash validate-config
 	# Running analysis
-	./scripts/code-climate.sh analyze
+	./scripts/code-climate.bash analyze
 
 
 .PHONY: info info-images info-swarm  info-tools

packages/postgres-database/Makefile

@@ -1,26 +1,17 @@
 #
-# TODO: under development
+# Targets for DEVELOPMENT of postgres-database
 #
-.DEFAULT_GOAL := help
+include ../../scripts/common.Makefile
 
 REPO_BASE_DIR  = $(abspath $(CURDIR)/../../)
 
-.PHONY: devenv
-devenv: ## build development environment (using main services/docker-compose-build.yml)
-	@$(MAKE) --directory ${REPO_BASE_DIR} $@
-
-
 .PHONY: requirements
 requirements: ## compiles pip requirements (.in -> .txt)
 	@$(MAKE) --directory requirements all
 
 
-.check-venv-active:
-	# checking whether virtual environment was activated
-	@python3 -c "import sys; assert sys.base_prefix!=sys.prefix"
-
 .PHONY: install-dev install-prod install-ci
-install-dev install-prod install-ci: requirements .check-venv-active ## install app in development/production or CI mode
+install-dev install-prod install-ci: requirements _check_venv_active ## install app in development/production or CI mode
 	# installing in $(subst install-,,$@) mode
 	pip-sync requirements/$(subst install-,,$@).txt
 
@@ -31,20 +22,6 @@ tests: ## runs unit tests
 	@pytest -vv --exitfirst --failed-first --durations=10 --pdb $(CURDIR)/tests
 
 
-.PHONY: version-patch version-minor
-version-patch version-minor version-major: ## commits version as patch (bug fixes not affecting the API), minor/minor (backwards-compatible/INcompatible API addition or changes)
-	# upgrades as $(subst version-,,$@) version, commits and tags
-	@bump2version --verbose --list $(subst version-,,$@)
-
-
-.PHONY: info
-info: ## displays
-	# installed
-	@pip list
-	# version
-	@cat setup.py | grep version=
-
-
 .PHONY: setup-commit
 setup-commit: install-dev up-pg ## sets up a database to create a new commit into migration history
 	# discovering
@@ -57,38 +34,12 @@ setup-commit: install-dev up-pg ## sets up a database to create a new commit int
 	@echo "To add new commit, sc-pg review -m \"Some message\" "
 
 
-_docker_compose_config := tests/docker-compose.yml
-
 .PHONY: up-pg down-pg
 
-up-pg: ## starts pg server
-	docker-compose -f $(_docker_compose_config) up -d
-
-down-pg: ## stops pg server
-	docker-compose -f $(_docker_compose_config) down
-
-
-.PHONY: autoformat
-autoformat: ## runs black python formatter on this service's code [https://black.readthedocs.io/en/stable/]
-	# auto formatting with black
-	@python3 -m black --verbose \
-		--exclude "/(\.eggs|\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.venv|\.svn|_build|buck-out|build|dist|migration)/" \
-		$(CURDIR)
-
-
-.PHONY: clean
-clean: down ## cleans all unversioned files in project and temp files create by this makefile
-	# Cleaning unversioned
-	@git clean -ndxf
-	@echo -n "Are you sure? [y/N] " && read ans && [ $${ans:-N} = y ]
-	@echo -n "$(shell whoami), are you REALLY sure? [y/N] " && read ans && [ $${ans:-N} = y ]
-	@git clean -dxf
+DOCKER_COMPOSE_CONFIG := tests/docker-compose.yml
 
+up-pg: $(DOCKER_COMPOSE_CONFIG) ## starts pg server
+	docker-compose -f $< up -d
 
-.PHONY: help
-# thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
-help: ## this colorful help
-	@echo "Recipes for '$(notdir $(CURDIR))':"
-	@echo ""
-	@awk --posix 'BEGIN {FS = ":.*?## "} /^[[:alpha:][:space:]_-]+:.*?## / {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
-	@echo ""
+down-pg: $(DOCKER_COMPOSE_CONFIG) ## stops pg server
+	docker-compose -f $< down

packages/postgres-database/docker/Makefile

@@ -53,9 +53,12 @@ upgrade: ## migrate the postgres database
 down: ## stop migration service
 	@docker service rm $(APP_NAME)
 
+
+DOCKER_IMAGES = $(shell docker images */$(APP_NAME):* -q)
+
 .PHONY: clean
 clean: ## clean all created images
-	-@docker image rm -f $(shell docker images */$(APP_NAME):* -q)
+	$(if $(DOCKER_IMAGES),@docker image rm -f $(DOCKER_IMAGES),$(info No image to delete))
 
 .PHONY: help
 # thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html

packages/postgres-database/src/simcore_postgres_database/models/comp_tasks.py

@@ -10,9 +10,9 @@
 
 
 class NodeClass(enum.Enum):
-    COMPUTATIONAL="COMPUTATIONAL"
-    INTERACTIVE="INTERACTIVE"
-    FRONTEND="FRONTEND"
+    COMPUTATIONAL = "COMPUTATIONAL"
+    INTERACTIVE = "INTERACTIVE"
+    FRONTEND = "FRONTEND"
 
 
 comp_tasks = sa.Table(
@@ -37,6 +37,5 @@ class NodeClass(enum.Enum):
     sa.Column("submit", sa.DateTime),
     sa.Column("start", sa.DateTime),
     sa.Column("end", sa.DateTime),
-
-    sa.UniqueConstraint('project_id', 'node_id', name='project_node_uniqueness'),
+    sa.UniqueConstraint("project_id", "node_id", name="project_node_uniqueness"),
 )

packages/postgres-database/src/simcore_postgres_database/models/user_to_projects.py

@@ -15,12 +15,8 @@
         nullable=False,
     ),
     sa.Column(
-        "project_id",
-        sa.BigInteger,
-        sa.ForeignKey(projects.c.id),
-        nullable=False,
+        "project_id", sa.BigInteger, sa.ForeignKey(projects.c.id), nullable=False,
     ),
-
     # TODO: do not ondelete=cascase for project_id or it will delete SHARED PROJECT
     # add instead sa.UniqueConstraint('user_id', 'project_id', name='user_project_uniqueness'),
     #

packages/postgres-database/tests/conftest.py

@@ -10,40 +10,42 @@
 import aiopg.sa
 
 
-@pytest.fixture(scope='session')
+@pytest.fixture(scope="session")
 def postgres_service(docker_services, docker_ip, docker_compose_file) -> str:
     # container environment
     with open(docker_compose_file) as fh:
         config = yaml.safe_load(fh)
-    environ = config['services']['postgres']['environment']
+    environ = config["services"]["postgres"]["environment"]
 
     dsn = "postgresql://{user}:{password}@{host}:{port}/{database}".format(
-        user=environ['POSTGRES_USER'],
-        password=environ['POSTGRES_PASSWORD'],
+        user=environ["POSTGRES_USER"],
+        password=environ["POSTGRES_PASSWORD"],
         host=docker_ip,
-        port=docker_services.port_for('postgres', 5432),
-        database=environ['POSTGRES_DB'],
+        port=docker_services.port_for("postgres", 5432),
+        database=environ["POSTGRES_DB"],
     )
 
     # Wait until service is responsive.
     docker_services.wait_until_responsive(
-        check=lambda: is_postgres_responsive(dsn),
-        timeout=30.0,
-        pause=0.1,
+        check=lambda: is_postgres_responsive(dsn), timeout=30.0, pause=0.1,
     )
     return dsn
 
+
 from typing import Union, Coroutine, Callable
 
+
 @pytest.fixture
 def make_engine(postgres_service):
     dsn = postgres_service
+
     def maker(is_async=True) -> Union[Coroutine, Callable]:
         return aiopg.sa.create_engine(dsn) if is_async else sa.create_engine(dsn)
+
     return maker
 
 
-def is_postgres_responsive(dsn)-> bool:
+def is_postgres_responsive(dsn) -> bool:
     """Check if something responds to ``url`` """
     try:
         engine = sa.create_engine(dsn)

packages/postgres-database/tests/test_delete_projects_and_users.py

@@ -12,31 +12,36 @@
 from aiopg.sa.result import ResultProxy, RowProxy
 
 from simcore_postgres_database.models.base import metadata
-from simcore_postgres_database.webserver_models import (UserStatus, projects,
-                                                        user_to_projects,
-                                                        users)
+from simcore_postgres_database.webserver_models import (
+    UserStatus,
+    projects,
+    user_to_projects,
+    users,
+)
 
 fake = faker.Faker()
 
+
 def random_user(**overrides):
     data = dict(
-        name = fake.name(),
-        email = fake.email(),
-        password_hash = fake.numerify(text='#'*5),
-        status = UserStatus.ACTIVE,
-        created_ip=fake.ipv4()
+        name=fake.name(),
+        email=fake.email(),
+        password_hash=fake.numerify(text="#" * 5),
+        status=UserStatus.ACTIVE,
+        created_ip=fake.ipv4(),
     )
     data.update(overrides)
     return data
 
+
 def random_project(**overrides):
     data = dict(
-        uuid = uuid4(),
-        name = fake.word(),
-        description= fake.sentence(),
-        prj_owner = fake.email(),
-        workbench = {},
-        published = False
+        uuid=uuid4(),
+        name=fake.word(),
+        description=fake.sentence(),
+        prj_owner=fake.email(),
+        workbench={},
+        published=False,
     )
     data.update(overrides)
     return data
@@ -59,9 +64,15 @@ async def start():
             await conn.execute(projects.insert().values(**random_project()))
             await conn.execute(projects.insert().values(**random_project()))
 
-            await conn.execute(user_to_projects.insert().values(user_id=1, project_id=1))
-            await conn.execute(user_to_projects.insert().values(user_id=1, project_id=2))
-            await conn.execute(user_to_projects.insert().values(user_id=2, project_id=3))
+            await conn.execute(
+                user_to_projects.insert().values(user_id=1, project_id=1)
+            )
+            await conn.execute(
+                user_to_projects.insert().values(user_id=1, project_id=2)
+            )
+            await conn.execute(
+                user_to_projects.insert().values(user_id=2, project_id=3)
+            )
 
         return engine
 
@@ -92,8 +103,8 @@ async def test_view(engine):
         assert len(rows) == 3
 
         # effect of cascade is that relation deletes as well
-        res =  await conn.execute(user_to_projects.select())
+        res = await conn.execute(user_to_projects.select())
         rows = await res.fetchall()
 
         assert len(rows) == 1
-        assert not any( row[user_to_projects.c.user_id]==1 for row in rows )
+        assert not any(row[user_to_projects.c.user_id] == 1 for row in rows)

packages/service-library/Makefile

@@ -1,54 +1,18 @@
 #
-# TODO: under development
+# Targets for DEVELOPMENT of Service Library
 #
-.DEFAULT_GOAL := help
+include ../../scripts/common.Makefile
 
 REPO_BASE_DIR  = $(abspath $(CURDIR)/../../)
 
-.PHONY: devenv
-devenv: ## build development environment (using main services/docker-compose-build.yml)
-	@$(MAKE) --directory ${REPO_BASE_DIR} $@
-
 
 .PHONY: install-dev install-prod install-ci
-install-dev install-prod install-ci: ## install app in development/production or CI mode
+install-dev install-prod install-ci: _check_venv_active ## install app in development/production or CI mode
 	# installing in $(subst install-,,$@) mode
-	@pip3 install -r requirements/$(subst install-,,$@).txt
+	python -m pip install -r requirements/$(subst install-,,$@).txt
 
 
 .PHONY: tests
 tests: ## runs unit tests
 	# running unit tests
 	@pytest -vv --exitfirst --failed-first --durations=10 --pdb $(CURDIR)/tests
-
-
-.PHONY: version-patch version-minor
-version-patch version-minor version-major: ## commits version as patch (bug fixes not affecting the API), minor/minor (backwards-compatible/INcompatible API addition or changes)
-	# upgrades as $(subst version-,,$@) version, commits and tags
-	@bump2version --verbose --list $(subst version-,,$@)
-
-
-.PHONY: info
-info: ## displays
-	# installed
-	@pip list
-	# version
-	@cat setup.py | grep version=
-
-
-.PHONY: clean
-clean: ## cleans all unversioned files in project and temp files create by this makefile
-	# Cleaning unversioned
-	@git clean -ndxf
-	@echo -n "Are you sure? [y/N] " && read ans && [ $${ans:-N} = y ]
-	@echo -n "$(shell whoami), are you REALLY sure? [y/N] " && read ans && [ $${ans:-N} = y ]
-	@git clean -dxf
-
-
-.PHONY: help
-# thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
-help: ## this colorful help
-	@echo "Recipes for '$(notdir $(CURDIR))':"
-	@echo ""
-	@awk --posix 'BEGIN {FS = ":.*?## "} /^[[:alpha:][:space:]_-]+:.*?## / {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
-	@echo ""