Skip to content

Commit 3c55bf3

Browse files
author
Pedro Crespo
committed
Setting up boot for web service and forgot configs
1 parent 8003744 commit 3c55bf3

File tree

10 files changed

+408
-12
lines changed

10 files changed

+408
-12
lines changed

‎services/web/Dockerfile

+12-10
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ RUN pip3 install --no-cache-dir -r requirements/base.txt
5656

5757
EXPOSE 8080
5858

59-
ENTRYPOINT ["python3", "-m", "aiohttp.web"]
59+
#ENTRYPOINT ["python3", "-m", "aiohttp.web"]
6060

6161
# ------------------------------------------------------------------------------------------
6262
FROM base-stage as development
@@ -80,9 +80,13 @@ VOLUME /home/scu/client/
8080
VOLUME /home/scu/packages
8181

8282
# TODO: add watch functionality in dev mode
83-
CMD ["-H", "0.0.0.0", \
84-
"-P", "8080", \
85-
"server.main:init_app"]
83+
#CMD ["-H", "0.0.0.0", \
84+
# "-P", "8080", \
85+
# "server.main:init_app"]
86+
87+
ENV DEBUG 1
88+
CMD ./boot.sh
89+
8690

8791

8892
# ------------------------------------------------------------------------------------------
@@ -103,9 +107,8 @@ USER scu
103107

104108
WORKDIR $HOME/server/src
105109

106-
CMD ["-H", "0.0.0.0", \
107-
"-P", "8080", \
108-
"server.main:init_app"]
110+
ENV DEBUG 0
111+
CMD ./boot.sh
109112

110113

111114
# ------------------------------------------------------------------------------------------
@@ -131,6 +134,5 @@ USER scu
131134

132135
WORKDIR $HOME/server/src
133136

134-
CMD ["-H", "0.0.0.0", \
135-
"-P", "8080", \
136-
"server.main:init_app"]
137+
ENV DEBUG 0
138+
CMD ./boot.sh

‎services/web/server/boot.sh

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
#!/bin/sh
2+
#echo "activating python virtual env..."
3+
#source /home/scu/venv/bin/activate
4+
5+
if [[ ${DEBUG} == "1" ]]
6+
then
7+
echo "Booting in development mode ..."
8+
service-web-server --config server-docker-dev.yaml
9+
else
10+
echo "Booting in production mode ..."
11+
service-web-server --config server-docker-prod.yaml
12+
fi

‎services/web/server/src/server/.config/init_db.py

+170
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,170 @@
1+
""" Initializes tables in database and adds some sample data for testing
2+
3+
FIXME: this place does not fill right... see how this script is called
4+
FIXME: check https://github.com/aio-libs/aiohttp_admin/blob/master/demos/blog/aiohttpdemo_blog/generate_data.py
5+
FIXME: rename as server.dev.generate_data.py and set dev as an optional sub-package as server[dev]
6+
7+
8+
Example of usage
9+
10+
cd services/web/server/tests/mock
11+
docker-compose up
12+
13+
cd ../../config
14+
python init_db.py
15+
16+
References:
17+
[1]:https://github.com/aio-libs/aiohttp-demos/blob/master/docs/preparations.rst#environment
18+
"""
19+
import logging
20+
import pathlib
21+
import sys
22+
23+
from passlib.hash import sha256_crypt
24+
from sqlalchemy import (
25+
MetaData,
26+
create_engine
27+
)
28+
from tenacity import (
29+
retry,
30+
stop_after_attempt,
31+
wait_fixed
32+
)
33+
34+
from server.db.model import (
35+
permissions,
36+
users
37+
)
38+
39+
from server.settings import (
40+
config_from_file
41+
)
42+
43+
44+
CURRENT_DIR = pathlib.Path(sys.argv[0] if __name__ == "__main__" else __file__).parent.parent
45+
CONFIG_DIR = CURRENT_DIR.parent / "config"
46+
47+
logging.basicConfig(level=logging.DEBUG)
48+
_LOGGER = logging.getLogger(__name__)
49+
50+
51+
DSN = "postgresql://{user}:{password}@{host}:{port}/{database}"
52+
53+
54+
USER_CONFIG_PATH = CONFIG_DIR / "server.yaml"
55+
USER_CONFIG = config_from_file(USER_CONFIG_PATH.as_posix())
56+
USER_DB_URL = DSN.format(**USER_CONFIG["postgres"])
57+
user_engine = create_engine(USER_DB_URL)
58+
59+
TEST_CONFIG_PATH = CONFIG_DIR / "server-test.yaml"
60+
TEST_CONFIG = config_from_file(TEST_CONFIG_PATH.as_posix())
61+
TEST_DB_URL = DSN.format(**TEST_CONFIG["postgres"])
62+
test_engine = create_engine(TEST_DB_URL)
63+
64+
# FIXME: admin user/passwords and in sync with other host/port configs
65+
ADMIN_DB_URL = DSN.format(
66+
user="postgres",
67+
password="postgres",
68+
database="postgres",
69+
host=USER_CONFIG["postgres"]["host"],
70+
port=5432
71+
)
72+
73+
# TODO: what is isolation_level?
74+
admin_engine = create_engine(ADMIN_DB_URL, isolation_level="AUTOCOMMIT")
75+
76+
77+
def setup_db(config):
78+
db_name = config["database"]
79+
db_user = config["user"]
80+
db_pass = config["password"]
81+
82+
# TODO: compose using query semantics. Clarify pros/cons vs string cli?
83+
conn = admin_engine.connect()
84+
conn.execute("DROP DATABASE IF EXISTS %s" % db_name)
85+
conn.execute("DROP ROLE IF EXISTS %s" % db_user)
86+
conn.execute("CREATE USER %s WITH PASSWORD '%s'" % (db_user, db_pass))
87+
conn.execute("CREATE DATABASE %s ENCODING 'UTF8'" % db_name)
88+
conn.execute("GRANT ALL PRIVILEGES ON DATABASE %s TO %s" %
89+
(db_name, db_user))
90+
conn.close()
91+
92+
93+
def teardown_db(config):
94+
db_name = config["database"]
95+
db_user = config["user"]
96+
97+
conn = admin_engine.connect()
98+
conn.execute("""
99+
SELECT pg_terminate_backend(pg_stat_activity.pid)
100+
FROM pg_stat_activity
101+
WHERE pg_stat_activity.datname = '%s'
102+
AND pid <> pg_backend_pid();""" % db_name)
103+
conn.execute("DROP DATABASE IF EXISTS %s" % db_name)
104+
conn.execute("DROP ROLE IF EXISTS %s" % db_user)
105+
conn.close()
106+
107+
108+
def create_tables(engine=test_engine):
109+
meta = MetaData()
110+
meta.create_all(bind=engine, tables=[users, permissions])
111+
112+
113+
def drop_tables(engine=test_engine):
114+
meta = MetaData()
115+
meta.drop_all(bind=engine, tables=[users, permissions])
116+
117+
118+
def sample_data(engine=test_engine):
119+
generate_password_hash = sha256_crypt.hash
120+
121+
#TODO: use fake to populate database
122+
# pylint:disable=E1120
123+
conn = engine.connect()
124+
conn.execute(users.insert(), [
125+
{"login": "[email protected]",
126+
"passwd": generate_password_hash("z43"),
127+
"is_superuser": False,
128+
"disabled": False},
129+
{"login": "[email protected]",
130+
"passwd": generate_password_hash("123"),
131+
"is_superuser": True,
132+
"disabled": False},
133+
{"login": "[email protected]",
134+
"passwd": generate_password_hash("345"),
135+
"is_superuser": True,
136+
"disabled": True}
137+
])
138+
139+
conn.execute(permissions.insert(), [
140+
{"user_id": 1,
141+
"perm_name": "tester"},
142+
{"user_id": 2,
143+
"perm_name": "admin"}
144+
])
145+
146+
conn.close()
147+
148+
149+
@retry(stop=stop_after_attempt(5), wait=wait_fixed(2))
150+
def main():
151+
152+
config = USER_CONFIG["postgres"]
153+
engine = user_engine
154+
155+
_LOGGER.info("Setting up db ...")
156+
setup_db(config)
157+
_LOGGER.info("")
158+
159+
_LOGGER.info("Creating tables ...")
160+
create_tables(engine=engine)
161+
162+
_LOGGER.info("Adding sample data ...")
163+
sample_data(engine=engine)
164+
165+
drop_tables()
166+
teardown_db(config)
167+
168+
169+
if __name__ == "__main__":
170+
main()

‎services/web/server/src/server/.config/server-defaults.yaml

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
version: '1.0'
2+
app:
3+
client_outdir: /usr/src/app/client
4+
host: 127.0.0.1
5+
log_level: DEBUG
6+
port: 8080
7+
testing: true
8+
cs_s4l:
9+
# cs_s4l might not be necessary anymore!
10+
host: 172.16.9.89
11+
app:
12+
port: 9095
13+
modeler:
14+
port: 9096
15+
director:
16+
host: director
17+
port: 8001
18+
postgres:
19+
database: simcoredb
20+
endpoint: postgres:5432
21+
host: postgres
22+
maxsize: 5
23+
minsize: 1
24+
password: simcore
25+
port: 5432
26+
user: simcore
27+
rabbit:
28+
channels:
29+
log: comp.backend.channels.log
30+
progress: comp.backend.channels.progress
31+
password: simcore
32+
user: simcore
33+
s3:
34+
access_key: 'Q3AM3UQ867SPQQA43P2F'
35+
bucket_name: simcore
36+
endpoint: play.minio.io:9000
37+
secret_key: 'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG'

‎services/web/server/src/server/.config/server-docker-prod.yaml

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
# Runs insides a container
2+
---
3+
version: "1.0"
4+
app:
5+
host: 127.0.0.1
6+
port: 8080
7+
client_outdir: ${SIMCORE_WEB_OUTDIR}
8+
log_level: INFO
9+
testing: False
10+
director:
11+
host: ${DIRECTOR_HOST}
12+
port: ${DIRECTOR_PORT}
13+
postgres:
14+
database: ${POSTGRES_DB}
15+
endpoint: ${POSTGRES_ENDPOINT}
16+
user: ${POSTGRES_USER}
17+
password: ${POSTGRES_PASSWORD}
18+
host: null
19+
port: null
20+
rabbit:
21+
user: ${RABBITMQ_USER}
22+
password: ${RABBITMQ_PASSWORD}
23+
channels:
24+
progress: ${RABBITMQ_PROGRESS_CHANNEL}
25+
log: ${RABBITMQ_LOG_CHANNEL}
26+
s3:
27+
endpoint: ${S3_ENDPOINT}
28+
access_key: ${S3_ACCESS_KEY}
29+
secret_key: ${S3_SECRET_KEY}
30+
bucket_name: ${S3_BUCKET_NAME}
31+
cs_s4l:
32+
host: ${CS_S4L_HOSTNAME}
33+
app:
34+
port: ${CS_S4L_PORT_APP}
35+
modeler:
36+
port: ${CS_S4L_PORT_MOD}
37+
...

‎services/web/server/src/server/.config/server-host-test.yaml

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
---
2+
version: "1.0"
3+
app:
4+
host: 127.0.0.1
5+
port: 8080
6+
client_outdir: ../../../client/source-output
7+
log_level: DEBUG
8+
testing: True
9+
director:
10+
host: localhost
11+
port: 8001
12+
postgres:
13+
database: test_db
14+
user: test_user
15+
password: test_pass
16+
host: localhost
17+
port: ${POSTGRES_PORT}
18+
# DEPRECATE OR add postgresql+psycopg2:// otherwise will fail sqlalchemy.exc.ArgumentError: Could not parse rfc1738 URL from string 'localhost:5432'
19+
endpoint: localhost:5432
20+
rabbit:
21+
host: ${RABBIT_HOST}
22+
password: simcore
23+
user: simcore
24+
channels:
25+
log: comp.backend.channels.log
26+
progress: comp.backend.channels.progress
27+
s3:
28+
access_key: '12345678'
29+
bucket_name: simcore
30+
endpoint: localhost:9000
31+
secret_key: '12345678'
32+
cs_s4l:
33+
host: localhost
34+
app:
35+
port: 9095
36+
modeler:
37+
port: 9096
38+
...

‎services/web/server/src/server/.config/server-template.yaml

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
---
2+
version: "1.0"
3+
app:
4+
host: 127.0.0.1
5+
port: 8080
6+
client_outdir: ${SIMCORE_WEB_OUTDIR}
7+
log_level: DEBUG
8+
testing: True
9+
director:
10+
host: ${DIRECTOR_HOST}
11+
port: ${DIRECTOR_PORT}
12+
postgres:
13+
database: ${POSTGRES_DB}
14+
user: ${POSTGRES_USER}
15+
password: ${POSTGRES_PASSWORD}
16+
host: localhost
17+
port: 5432
18+
minsize: 1
19+
maxsize: 5
20+
endpoint: ${POSTGRES_ENDPOINT}
21+
rabbit:
22+
user: ${RABBITMQ_USER}
23+
password: ${RABBITMQ_PASSWORD}
24+
channels:
25+
progress: ${RABBITMQ_PROGRESS_CHANNEL}
26+
log: ${RABBITMQ_LOG_CHANNEL}
27+
s3:
28+
endpoint: ${S3_ENDPOINT}
29+
access_key: ${S3_ACCESS_KEY}
30+
secret_key: ${S3_SECRET_KEY}
31+
bucket_name: ${S3_BUCKET_NAME}
32+
cs_s4l:
33+
host: ${CS_S4L_HOSTNAME}
34+
app:
35+
port: ${CS_S4L_PORT_APP}
36+
modeler:
37+
port: ${CS_S4L_PORT_MOD}
38+
...

‎services/web/server/src/server/__main__.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ def main(argv=None):
1919
config = settings.config_from_options(options)
2020

2121
log_level = config.get("app",{}).get("log_level", "DEBUG")
22-
logging.basicConfig( level=getattr("logging", log_level) )
22+
logging.basicConfig( level=getattr(logging, log_level) )
2323

2424
run(config)
2525

0 commit comments

Comments
 (0)