Skip to content

Commit 186167a

Browse files
pcrespovpcrespov
authored
Merge branch 'master' into maintenance/refactor-servicelib
2 parents 1db00aa + 7382c4e commit 186167a

18 files changed

+2332
-721
lines changed

‎.dockerignore

+9
Original file line numberDiff line numberDiff line change
@@ -19,3 +19,12 @@ ops/
1919

2020
# coverage files
2121
.coverage*
22+
23+
# pytest-fixture-tools output
24+
artifacts
25+
26+
# dask artifacts in devel mode
27+
dask*-space/
28+
29+
# produced when mounting volumes on docker and pip installing
30+
.local/

‎.gitignore

+3
Original file line numberDiff line numberDiff line change
@@ -166,3 +166,6 @@ artifacts
166166

167167
# dask artifacts in devel mode
168168
dask*-space/
169+
170+
# produced when mounting volumes on docker and pip installing
171+
.local/

‎docs/img/.stack-simcore-version.yml.png

49.2 KB
Loading

‎packages/postgres-database/Makefile

+2-5
Original file line numberDiff line numberDiff line change
@@ -102,8 +102,5 @@ down-pg down-prod: $(docker-compose-configs) ## stops pg server
102102

103103

104104
.PHONY: auto-doc
105-
auto-doc: install-dev ## Creates entity relationship diagram (ERD) defined under ``simcore_postgres_database.models``
106-
# installing doc dependencies (install-doc)
107-
pip install eralchemy
108-
# running script
109-
python scripts/create_erd.py
105+
auto-doc: ## Creates entity relationship diagram (ERD) defined under ``simcore_postgres_database.models``
106+
$(MAKE) --directory=scripts/erd run

‎packages/postgres-database/doc/img/postgres-database-models.svg

+884-675
Loading

‎packages/postgres-database/scripts/create_erd.py

-29
This file was deleted.

‎packages/postgres-database/scripts/erd/Dockerfile

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
ARG PYTHON_VERSION="3.8.10"
2+
FROM python:${PYTHON_VERSION}-slim-buster as base
3+
4+
RUN apt-get update \
5+
&& apt-get -y install --no-install-recommends\
6+
make \
7+
libc-dev \
8+
graphviz-dev \
9+
git \
10+
gcc \
11+
gawk \
12+
graphviz \
13+
&& rm -rf /var/lib/apt/lists/* \
14+
&& apt-get clean
15+
16+
17+
RUN pip --no-cache-dir install --upgrade \
18+
pip~=21.2.3 \
19+
wheel \
20+
setuptools
21+
22+
23+
# devenv
24+
RUN pip install --no-cache-dir \
25+
pyparsing \
26+
pydot \
27+
eralchemy \
28+
sqlalchemy_schemadisplay

‎packages/postgres-database/scripts/erd/Makefile

+62
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
#
2+
# ERD (Entity Relationship Diagram) is used to visualize these relationships
3+
#
4+
.DEFAULT_GOAL := help
5+
6+
PYTHON_VERSION=3.8.10
7+
8+
# locations
9+
REPODIR := $(shell git rev-parse --show-toplevel)
10+
PACKAGES_DIR := $(abspath $(REPODIR)/packages)
11+
SERVICES_DIR := $(abspath $(REPODIR)/services)
12+
PG_DIR := $(abspath $(PACKAGES_DIR)/postgres-database)
13+
14+
# tools
15+
MAKE_C := $(MAKE) --directory
16+
17+
18+
IMAGE_NAME:=local/postgres-database-scripts-erd:${PYTHON_VERSION}
19+
20+
# SEE https://medium.com/faun/set-current-host-user-for-docker-container-4e521cef9ffc
21+
.PHONY: build
22+
build build-nc: ## builds tooling image ${IMAGE_NAME}
23+
docker build $(if $(findstring -nc,$@),--no-cache,) \
24+
--build-arg PYTHON_VERSION="${PYTHON_VERSION}" \
25+
--tag ${IMAGE_NAME} .
26+
27+
28+
.PHONY: shell
29+
shell: build ## Opens shell in ${IMAGE_NAME}
30+
docker run -it \
31+
--workdir="/home/$(USER)" \
32+
--volume="/etc/group:/etc/group:ro" \
33+
--volume="/etc/passwd:/etc/passwd:ro" \
34+
--volume="/etc/shadow:/etc/shadow:ro" \
35+
--volume=$(PG_DIR):/home/$(USER) \
36+
--user=$(shell id -u):$(shell id -g) \
37+
--entrypoint=/bin/bash \
38+
${IMAGE_NAME}
39+
40+
41+
.PHONY: run
42+
run: build ## Runs upgrade in a container [WARNING! UNDER DEV. USE CAREFULY]
43+
docker run -it \
44+
--workdir="/home/$(USER)" \
45+
--volume="/etc/group:/etc/group:ro" \
46+
--volume="/etc/passwd:/etc/passwd:ro" \
47+
--volume="/etc/shadow:/etc/shadow:ro" \
48+
--volume=$(PG_DIR):/home/$(USER) \
49+
--user=$(shell id -u):$(shell id -g) \
50+
--entrypoint=/bin/bash \
51+
${IMAGE_NAME} \
52+
-c "pip install -e .; python scripts/erd/main.py"
53+
54+
55+
56+
.PHONY: help
57+
# thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
58+
help: ## this colorful help
59+
@echo "Recipes for '$(notdir $(CURDIR))':"
60+
@echo ""
61+
@awk --posix 'BEGIN {FS = ":.*?## "} /^[[:alpha:][:space:]_-]+:.*?## / {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
62+
@echo ""

‎packages/postgres-database/scripts/erd/main.py

+50
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
#
2+
# ERD (Entity Relationship Diagram) is used to visualize these relationships
3+
#
4+
5+
6+
# pylint: disable=wildcard-import
7+
# pylint: disable=unused-wildcard-import
8+
#
9+
import sys
10+
from pathlib import Path
11+
12+
from simcore_postgres_database.models import * # registers all schemas in metadata
13+
from simcore_postgres_database.models.base import metadata
14+
15+
CURRENT_DIR = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent
16+
17+
18+
def create_with_sqlalchemy_schemadisplay(svg_path: Path):
19+
# SEE https://github.com/sqlalchemy/sqlalchemy/wiki/SchemaDisplay
20+
21+
from sqlalchemy_schemadisplay import create_schema_graph
22+
23+
# create the pydot graph object by autoloading all tables via a bound metadata object
24+
25+
graph = create_schema_graph(
26+
metadata=metadata,
27+
show_datatypes=True, # The image would get nasty big if we'd show the datatypes
28+
show_indexes=False, # ditto for indexes
29+
rankdir="LR", # From left to right (instead of top to bottom)
30+
concentrate=False, # Don't try to join the relation lines together
31+
)
32+
graph.write_svg(str(svg_path)) # write out the file
33+
34+
35+
def create_with_eralchemy(svg_path: Path):
36+
# SEE https://github.com/Alexis-benoist/eralchemy
37+
from eralchemy import render_er
38+
39+
render_er(metadata, str(svg_path))
40+
41+
42+
if __name__ == "__main__":
43+
44+
output_dir = (CURRENT_DIR / "../../doc/img").resolve()
45+
output_dir.mkdir(parents=True, exist_ok=True)
46+
47+
# FIXME: sqlalchemy_schemadisplay failes with json columns
48+
# create_with_sqlalchemy_schemadisplay( output_dir / "postgres-database-models.svg")
49+
50+
create_with_eralchemy(output_dir / "postgres-database-models.svg")

‎packages/postgres-database/src/simcore_postgres_database/migration/versions/0208f6b32f32_adds_version_control_tables.py

+170
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,170 @@
1+
"""Adds version control tables
2+
3+
Revision ID: 0208f6b32f32
4+
Revises: d10c53a5bea6
5+
Create Date: 2021-09-06 14:19:42.599645+00:00
6+
7+
"""
8+
import sqlalchemy as sa
9+
from alembic import op
10+
from sqlalchemy.dialects import postgresql
11+
12+
# revision identifiers, used by Alembic.
13+
revision = "0208f6b32f32"
14+
down_revision = "d10c53a5bea6"
15+
branch_labels = None
16+
depends_on = None
17+
18+
19+
def upgrade():
20+
# ### commands auto generated by Alembic - please adjust! ###
21+
op.create_table(
22+
"projects_vc_snapshots",
23+
sa.Column("checksum", sa.String(), nullable=False),
24+
sa.Column(
25+
"content",
26+
postgresql.JSONB(astext_type=sa.Text()),
27+
server_default=sa.text("'{}'::jsonb"),
28+
nullable=False,
29+
),
30+
sa.PrimaryKeyConstraint("checksum"),
31+
)
32+
op.create_table(
33+
"projects_vc_repos",
34+
sa.Column("id", sa.BigInteger(), nullable=False),
35+
sa.Column("project_uuid", sa.String(), nullable=False),
36+
sa.Column("project_checksum", sa.String(), nullable=True),
37+
sa.Column(
38+
"created", sa.DateTime(), server_default=sa.text("now()"), nullable=False
39+
),
40+
sa.Column(
41+
"modified", sa.DateTime(), server_default=sa.text("now()"), nullable=False
42+
),
43+
sa.ForeignKeyConstraint(
44+
["project_uuid"],
45+
["projects.uuid"],
46+
name="fk_projects_vc_repos_project_uuid",
47+
),
48+
sa.PrimaryKeyConstraint("id"),
49+
sa.UniqueConstraint("project_uuid"),
50+
)
51+
op.create_table(
52+
"projects_vc_commits",
53+
sa.Column("id", sa.BigInteger(), nullable=False),
54+
sa.Column("repo_id", sa.BigInteger(), nullable=False),
55+
sa.Column("parent_commit_id", sa.BigInteger(), nullable=True),
56+
sa.Column("snapshot_checksum", sa.String(), nullable=False),
57+
sa.Column("message", sa.String(), nullable=True),
58+
sa.Column(
59+
"created", sa.DateTime(), server_default=sa.text("now()"), nullable=False
60+
),
61+
sa.ForeignKeyConstraint(
62+
["parent_commit_id"],
63+
["projects_vc_commits.id"],
64+
name="fk_projects_vc_commits_parent_commit_id",
65+
onupdate="CASCADE",
66+
),
67+
sa.ForeignKeyConstraint(
68+
["repo_id"],
69+
["projects_vc_repos.id"],
70+
name="fk_projects_vc_commits_repo_id",
71+
ondelete="CASCADE",
72+
),
73+
sa.ForeignKeyConstraint(
74+
["snapshot_checksum"],
75+
["projects_vc_snapshots.checksum"],
76+
name="fk_projects_vc_commits_snapshot_checksum",
77+
ondelete="RESTRICT",
78+
),
79+
sa.PrimaryKeyConstraint("id"),
80+
)
81+
op.create_table(
82+
"projects_vc_branches",
83+
sa.Column("id", sa.BigInteger(), nullable=False),
84+
sa.Column("repo_id", sa.BigInteger(), nullable=False),
85+
sa.Column("head_commit_id", sa.BigInteger(), nullable=True),
86+
sa.Column("name", sa.String(), nullable=True),
87+
sa.Column(
88+
"created", sa.DateTime(), server_default=sa.text("now()"), nullable=False
89+
),
90+
sa.Column(
91+
"modified", sa.DateTime(), server_default=sa.text("now()"), nullable=False
92+
),
93+
sa.ForeignKeyConstraint(
94+
["head_commit_id"],
95+
["projects_vc_commits.id"],
96+
name="fk_projects_vc_branches_head_commit_id",
97+
ondelete="RESTRICT",
98+
),
99+
sa.ForeignKeyConstraint(
100+
["repo_id"],
101+
["projects_vc_repos.id"],
102+
name="projects_vc_branches_repo_id",
103+
ondelete="CASCADE",
104+
),
105+
sa.PrimaryKeyConstraint("id"),
106+
sa.UniqueConstraint("name", "repo_id", name="repo_branch_uniqueness"),
107+
)
108+
op.create_table(
109+
"projects_vc_tags",
110+
sa.Column("id", sa.BigInteger(), nullable=False),
111+
sa.Column("repo_id", sa.BigInteger(), nullable=False),
112+
sa.Column("commit_id", sa.BigInteger(), nullable=False),
113+
sa.Column("name", sa.String(), nullable=True),
114+
sa.Column("message", sa.String(), nullable=True),
115+
sa.Column("hidden", sa.Boolean(), nullable=True),
116+
sa.Column(
117+
"created", sa.DateTime(), server_default=sa.text("now()"), nullable=False
118+
),
119+
sa.Column(
120+
"modified", sa.DateTime(), server_default=sa.text("now()"), nullable=False
121+
),
122+
sa.ForeignKeyConstraint(
123+
["commit_id"],
124+
["projects_vc_commits.id"],
125+
name="fk_projects_vc_tags_commit_id",
126+
ondelete="CASCADE",
127+
),
128+
sa.ForeignKeyConstraint(
129+
["repo_id"],
130+
["projects_vc_repos.id"],
131+
name="fk_projects_vc_tags_repo_id",
132+
ondelete="CASCADE",
133+
),
134+
sa.PrimaryKeyConstraint("id"),
135+
sa.UniqueConstraint("name", "repo_id", name="repo_tag_uniqueness"),
136+
)
137+
op.create_table(
138+
"projects_vc_heads",
139+
sa.Column("repo_id", sa.BigInteger(), nullable=False),
140+
sa.Column("head_branch_id", sa.BigInteger(), nullable=True),
141+
sa.Column(
142+
"modified", sa.DateTime(), server_default=sa.text("now()"), nullable=False
143+
),
144+
sa.ForeignKeyConstraint(
145+
["head_branch_id"],
146+
["projects_vc_branches.id"],
147+
name="fk_projects_vc_heads_head_branch_id",
148+
ondelete="CASCADE",
149+
),
150+
sa.ForeignKeyConstraint(
151+
["repo_id"],
152+
["projects_vc_repos.id"],
153+
name="projects_vc_branches_repo_id",
154+
ondelete="CASCADE",
155+
),
156+
sa.PrimaryKeyConstraint("repo_id"),
157+
sa.UniqueConstraint("head_branch_id"),
158+
)
159+
# ### end Alembic commands ###
160+
161+
162+
def downgrade():
163+
# ### commands auto generated by Alembic - please adjust! ###
164+
op.drop_table("projects_vc_heads")
165+
op.drop_table("projects_vc_tags")
166+
op.drop_table("projects_vc_branches")
167+
op.drop_table("projects_vc_commits")
168+
op.drop_table("projects_vc_repos")
169+
op.drop_table("projects_vc_snapshots")
170+
# ### end Alembic commands ###

‎packages/postgres-database/src/simcore_postgres_database/models/projects.py

-3
Original file line numberDiff line numberDiff line change
@@ -4,16 +4,13 @@
44
55
"""
66
import enum
7-
import logging
87

98
import sqlalchemy as sa
109
from sqlalchemy.dialects.postgresql import ARRAY, JSONB
1110
from sqlalchemy.sql import func
1211

1312
from .base import metadata
1413

15-
log = logging.getLogger(__name__)
16-
1714

1815
class ProjectType(enum.Enum):
1916
"""

0 commit comments

Comments
 (0)