Skip to content

Commit d4410e6

Browse files
pcrespovpcrespov
authored
[maintenance] black-formatted packages/services and new common.Makefile (#1342)
- Black-formatted ALL python: - added auto-formatter in project makefiles - formatted every relevant python package or service - scripts/common.Makefile collects common recipes: see how services and packages Makefiles are now simplified - Fixed noisy pprint in the logs - Added extensions to bash scripts
1 parent ac51c73 commit d4410e6

File tree

245 files changed

+6833
-4866
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

245 files changed

+6833
-4866
lines changed

‎.vscode-template/settings.json

+5-3
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,8 @@
88
".env*": "ini",
99
"Dockerfile*": "dockerfile",
1010
"**/requirements/*.txt": "pip-requirements",
11-
"**/requirements/*.in": "pip-requirements"
11+
"**/requirements/*.in": "pip-requirements",
12+
"*Makefile": "makefile"
1213
},
1314
"files.eol": "\n",
1415
"files.insertFinalNewline": true,
@@ -28,7 +29,8 @@
2829
},
2930
"python.testing.pyTestEnabled": true,
3031
"autoDocstring.docstringFormat": "sphinx",
31-
"shellcheck.executablePath": "${workspaceFolder}/scripts/shellcheck",
32+
"shellcheck.executablePath": "${workspaceFolder}/scripts/shellcheck.bash",
3233
"shellcheck.run": "onSave",
33-
"shellcheck.enableQuickFix": true
34+
"shellcheck.enableQuickFix": true,
35+
"python.formatting.provider": "black"
3436
}

‎Makefile

+2-2
Original file line numberDiff line numberDiff line change
@@ -340,9 +340,9 @@ openapi-specs: ## bundles and validates openapi specifications and schemas of AL
340340
.PHONY: code-analysis
341341
code-analysis: .codeclimate.yml ## runs code-climate analysis
342342
# Validates $<
343-
./scripts/code-climate.sh validate-config
343+
./scripts/code-climate.bash validate-config
344344
# Running analysis
345-
./scripts/code-climate.sh analyze
345+
./scripts/code-climate.bash analyze
346346

347347

348348
.PHONY: info info-images info-swarm info-tools

‎packages/postgres-database/Makefile

+8-57
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,17 @@
11
#
2-
# TODO: under development
2+
# Targets for DEVELOPMENT of postgres-database
33
#
4-
.DEFAULT_GOAL := help
4+
include ../../scripts/common.Makefile
55

66
REPO_BASE_DIR = $(abspath $(CURDIR)/../../)
77

8-
.PHONY: devenv
9-
devenv: ## build development environment (using main services/docker-compose-build.yml)
10-
@$(MAKE) --directory ${REPO_BASE_DIR} $@
11-
12-
138
.PHONY: requirements
149
requirements: ## compiles pip requirements (.in -> .txt)
1510
@$(MAKE) --directory requirements all
1611

1712

18-
.check-venv-active:
19-
# checking whether virtual environment was activated
20-
@python3 -c "import sys; assert sys.base_prefix!=sys.prefix"
21-
2213
.PHONY: install-dev install-prod install-ci
23-
install-dev install-prod install-ci: requirements .check-venv-active ## install app in development/production or CI mode
14+
install-dev install-prod install-ci: requirements _check_venv_active ## install app in development/production or CI mode
2415
# installing in $(subst install-,,$@) mode
2516
pip-sync requirements/$(subst install-,,$@).txt
2617

@@ -31,20 +22,6 @@ tests: ## runs unit tests
3122
@pytest -vv --exitfirst --failed-first --durations=10 --pdb $(CURDIR)/tests
3223

3324

34-
.PHONY: version-patch version-minor
35-
version-patch version-minor version-major: ## commits version as patch (bug fixes not affecting the API), minor/minor (backwards-compatible/INcompatible API addition or changes)
36-
# upgrades as $(subst version-,,$@) version, commits and tags
37-
@bump2version --verbose --list $(subst version-,,$@)
38-
39-
40-
.PHONY: info
41-
info: ## displays
42-
# installed
43-
@pip list
44-
# version
45-
@cat setup.py | grep version=
46-
47-
4825
.PHONY: setup-commit
4926
setup-commit: install-dev up-pg ## sets up a database to create a new commit into migration history
5027
# discovering
@@ -57,38 +34,12 @@ setup-commit: install-dev up-pg ## sets up a database to create a new commit int
5734
@echo "To add new commit, sc-pg review -m \"Some message\" "
5835

5936

60-
_docker_compose_config := tests/docker-compose.yml
61-
6237
.PHONY: up-pg down-pg
6338

64-
up-pg: ## starts pg server
65-
docker-compose -f $(_docker_compose_config) up -d
66-
67-
down-pg: ## stops pg server
68-
docker-compose -f $(_docker_compose_config) down
69-
70-
71-
.PHONY: autoformat
72-
autoformat: ## runs black python formatter on this service's code [https://black.readthedocs.io/en/stable/]
73-
# auto formatting with black
74-
@python3 -m black --verbose \
75-
--exclude "/(\.eggs|\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.venv|\.svn|_build|buck-out|build|dist|migration)/" \
76-
$(CURDIR)
77-
78-
79-
.PHONY: clean
80-
clean: down ## cleans all unversioned files in project and temp files create by this makefile
81-
# Cleaning unversioned
82-
@git clean -ndxf
83-
@echo -n "Are you sure? [y/N] " && read ans && [ $${ans:-N} = y ]
84-
@echo -n "$(shell whoami), are you REALLY sure? [y/N] " && read ans && [ $${ans:-N} = y ]
85-
@git clean -dxf
39+
DOCKER_COMPOSE_CONFIG := tests/docker-compose.yml
8640

41+
up-pg: $(DOCKER_COMPOSE_CONFIG) ## starts pg server
42+
docker-compose -f $< up -d
8743

88-
.PHONY: help
89-
# thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
90-
help: ## this colorful help
91-
@echo "Recipes for '$(notdir $(CURDIR))':"
92-
@echo ""
93-
@awk --posix 'BEGIN {FS = ":.*?## "} /^[[:alpha:][:space:]_-]+:.*?## / {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
94-
@echo ""
44+
down-pg: $(DOCKER_COMPOSE_CONFIG) ## stops pg server
45+
docker-compose -f $< down

‎packages/postgres-database/docker/Makefile

+4-1
Original file line numberDiff line numberDiff line change
@@ -53,9 +53,12 @@ upgrade: ## migrate the postgres database
5353
down: ## stop migration service
5454
@docker service rm $(APP_NAME)
5555

56+
57+
DOCKER_IMAGES = $(shell docker images */$(APP_NAME):* -q)
58+
5659
.PHONY: clean
5760
clean: ## clean all created images
58-
-@docker image rm -f $(shell docker images */$(APP_NAME):* -q)
61+
$(if $(DOCKER_IMAGES),@docker image rm -f $(DOCKER_IMAGES),$(info No image to delete))
5962

6063
.PHONY: help
6164
# thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html

‎packages/postgres-database/src/simcore_postgres_database/models/comp_tasks.py

+4-5
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,9 @@
1010

1111

1212
class NodeClass(enum.Enum):
13-
COMPUTATIONAL="COMPUTATIONAL"
14-
INTERACTIVE="INTERACTIVE"
15-
FRONTEND="FRONTEND"
13+
COMPUTATIONAL = "COMPUTATIONAL"
14+
INTERACTIVE = "INTERACTIVE"
15+
FRONTEND = "FRONTEND"
1616

1717

1818
comp_tasks = sa.Table(
@@ -37,6 +37,5 @@ class NodeClass(enum.Enum):
3737
sa.Column("submit", sa.DateTime),
3838
sa.Column("start", sa.DateTime),
3939
sa.Column("end", sa.DateTime),
40-
41-
sa.UniqueConstraint('project_id', 'node_id', name='project_node_uniqueness'),
40+
sa.UniqueConstraint("project_id", "node_id", name="project_node_uniqueness"),
4241
)

‎packages/postgres-database/src/simcore_postgres_database/models/user_to_projects.py

+1-5
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,8 @@
1515
nullable=False,
1616
),
1717
sa.Column(
18-
"project_id",
19-
sa.BigInteger,
20-
sa.ForeignKey(projects.c.id),
21-
nullable=False,
18+
"project_id", sa.BigInteger, sa.ForeignKey(projects.c.id), nullable=False,
2219
),
23-
2420
# TODO: do not ondelete=cascase for project_id or it will delete SHARED PROJECT
2521
# add instead sa.UniqueConstraint('user_id', 'project_id', name='user_project_uniqueness'),
2622
#

‎packages/postgres-database/tests/conftest.py

+12-10
Original file line numberDiff line numberDiff line change
@@ -10,40 +10,42 @@
1010
import aiopg.sa
1111

1212

13-
@pytest.fixture(scope='session')
13+
@pytest.fixture(scope="session")
1414
def postgres_service(docker_services, docker_ip, docker_compose_file) -> str:
1515
# container environment
1616
with open(docker_compose_file) as fh:
1717
config = yaml.safe_load(fh)
18-
environ = config['services']['postgres']['environment']
18+
environ = config["services"]["postgres"]["environment"]
1919

2020
dsn = "postgresql://{user}:{password}@{host}:{port}/{database}".format(
21-
user=environ['POSTGRES_USER'],
22-
password=environ['POSTGRES_PASSWORD'],
21+
user=environ["POSTGRES_USER"],
22+
password=environ["POSTGRES_PASSWORD"],
2323
host=docker_ip,
24-
port=docker_services.port_for('postgres', 5432),
25-
database=environ['POSTGRES_DB'],
24+
port=docker_services.port_for("postgres", 5432),
25+
database=environ["POSTGRES_DB"],
2626
)
2727

2828
# Wait until service is responsive.
2929
docker_services.wait_until_responsive(
30-
check=lambda: is_postgres_responsive(dsn),
31-
timeout=30.0,
32-
pause=0.1,
30+
check=lambda: is_postgres_responsive(dsn), timeout=30.0, pause=0.1,
3331
)
3432
return dsn
3533

34+
3635
from typing import Union, Coroutine, Callable
3736

37+
3838
@pytest.fixture
3939
def make_engine(postgres_service):
4040
dsn = postgres_service
41+
4142
def maker(is_async=True) -> Union[Coroutine, Callable]:
4243
return aiopg.sa.create_engine(dsn) if is_async else sa.create_engine(dsn)
44+
4345
return maker
4446

4547

46-
def is_postgres_responsive(dsn)-> bool:
48+
def is_postgres_responsive(dsn) -> bool:
4749
"""Check if something responds to ``url`` """
4850
try:
4951
engine = sa.create_engine(dsn)

‎packages/postgres-database/tests/test_delete_projects_and_users.py

+30-19
Original file line numberDiff line numberDiff line change
@@ -12,31 +12,36 @@
1212
from aiopg.sa.result import ResultProxy, RowProxy
1313

1414
from simcore_postgres_database.models.base import metadata
15-
from simcore_postgres_database.webserver_models import (UserStatus, projects,
16-
user_to_projects,
17-
users)
15+
from simcore_postgres_database.webserver_models import (
16+
UserStatus,
17+
projects,
18+
user_to_projects,
19+
users,
20+
)
1821

1922
fake = faker.Faker()
2023

24+
2125
def random_user(**overrides):
2226
data = dict(
23-
name = fake.name(),
24-
email = fake.email(),
25-
password_hash = fake.numerify(text='#'*5),
26-
status = UserStatus.ACTIVE,
27-
created_ip=fake.ipv4()
27+
name=fake.name(),
28+
email=fake.email(),
29+
password_hash=fake.numerify(text="#" * 5),
30+
status=UserStatus.ACTIVE,
31+
created_ip=fake.ipv4(),
2832
)
2933
data.update(overrides)
3034
return data
3135

36+
3237
def random_project(**overrides):
3338
data = dict(
34-
uuid = uuid4(),
35-
name = fake.word(),
36-
description= fake.sentence(),
37-
prj_owner = fake.email(),
38-
workbench = {},
39-
published = False
39+
uuid=uuid4(),
40+
name=fake.word(),
41+
description=fake.sentence(),
42+
prj_owner=fake.email(),
43+
workbench={},
44+
published=False,
4045
)
4146
data.update(overrides)
4247
return data
@@ -59,9 +64,15 @@ async def start():
5964
await conn.execute(projects.insert().values(**random_project()))
6065
await conn.execute(projects.insert().values(**random_project()))
6166

62-
await conn.execute(user_to_projects.insert().values(user_id=1, project_id=1))
63-
await conn.execute(user_to_projects.insert().values(user_id=1, project_id=2))
64-
await conn.execute(user_to_projects.insert().values(user_id=2, project_id=3))
67+
await conn.execute(
68+
user_to_projects.insert().values(user_id=1, project_id=1)
69+
)
70+
await conn.execute(
71+
user_to_projects.insert().values(user_id=1, project_id=2)
72+
)
73+
await conn.execute(
74+
user_to_projects.insert().values(user_id=2, project_id=3)
75+
)
6576

6677
return engine
6778

@@ -92,8 +103,8 @@ async def test_view(engine):
92103
assert len(rows) == 3
93104

94105
# effect of cascade is that relation deletes as well
95-
res = await conn.execute(user_to_projects.select())
106+
res = await conn.execute(user_to_projects.select())
96107
rows = await res.fetchall()
97108

98109
assert len(rows) == 1
99-
assert not any( row[user_to_projects.c.user_id]==1 for row in rows )
110+
assert not any(row[user_to_projects.c.user_id] == 1 for row in rows)

‎packages/service-library/Makefile

+4-40
Original file line numberDiff line numberDiff line change
@@ -1,54 +1,18 @@
11
#
2-
# TODO: under development
2+
# Targets for DEVELOPMENT of Service Library
33
#
4-
.DEFAULT_GOAL := help
4+
include ../../scripts/common.Makefile
55

66
REPO_BASE_DIR = $(abspath $(CURDIR)/../../)
77

8-
.PHONY: devenv
9-
devenv: ## build development environment (using main services/docker-compose-build.yml)
10-
@$(MAKE) --directory ${REPO_BASE_DIR} $@
11-
128

139
.PHONY: install-dev install-prod install-ci
14-
install-dev install-prod install-ci: ## install app in development/production or CI mode
10+
install-dev install-prod install-ci: _check_venv_active ## install app in development/production or CI mode
1511
# installing in $(subst install-,,$@) mode
16-
@pip3 install -r requirements/$(subst install-,,$@).txt
12+
python -m pip install -r requirements/$(subst install-,,$@).txt
1713

1814

1915
.PHONY: tests
2016
tests: ## runs unit tests
2117
# running unit tests
2218
@pytest -vv --exitfirst --failed-first --durations=10 --pdb $(CURDIR)/tests
23-
24-
25-
.PHONY: version-patch version-minor
26-
version-patch version-minor version-major: ## commits version as patch (bug fixes not affecting the API), minor/minor (backwards-compatible/INcompatible API addition or changes)
27-
# upgrades as $(subst version-,,$@) version, commits and tags
28-
@bump2version --verbose --list $(subst version-,,$@)
29-
30-
31-
.PHONY: info
32-
info: ## displays
33-
# installed
34-
@pip list
35-
# version
36-
@cat setup.py | grep version=
37-
38-
39-
.PHONY: clean
40-
clean: ## cleans all unversioned files in project and temp files create by this makefile
41-
# Cleaning unversioned
42-
@git clean -ndxf
43-
@echo -n "Are you sure? [y/N] " && read ans && [ $${ans:-N} = y ]
44-
@echo -n "$(shell whoami), are you REALLY sure? [y/N] " && read ans && [ $${ans:-N} = y ]
45-
@git clean -dxf
46-
47-
48-
.PHONY: help
49-
# thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
50-
help: ## this colorful help
51-
@echo "Recipes for '$(notdir $(CURDIR))':"
52-
@echo ""
53-
@awk --posix 'BEGIN {FS = ":.*?## "} /^[[:alpha:][:space:]_-]+:.*?## / {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
54-
@echo ""

0 commit comments

Comments
 (0)