Skip to content

Commit 65e959b

Browse files
mguidonmguidon
authored
Comp backend (#110)
* Initial import of comp. services * Improvements on Dockerfiles and organization of devel/production/deploy configurations
1 parent 2eae9cd commit 65e959b

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+1873
-220
lines changed

‎.env-devel

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
POSTGRES_USER=simcore
2+
POSTGRES_PASSWORD=simcore
3+
POSTGRES_DB=simcoredb
4+
RABBITMQ_USER=simcore
5+
RABBITMQ_PASSWORD=simcore
6+
RABBITMQ_PROGRESS_CHANNEL=comp.backend.channels.progress
7+
RABBITMQ_LOG_CHANNEL=comp.backend.channels.log
8+
S3_ENDPOINT=minio:9000
9+
S3_ACCESS_KEY=12345678
10+
S3_SECRET_KEY=12345678
11+
S3_BUCKET_NAME=simcore

‎.gitignore

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,3 +115,8 @@ services/docker-compose.override.yml
115115

116116
# mac
117117
.DS_Store
118+
119+
120+
# key-words in filename to ignore them
121+
*secret*
122+
*ignore*

‎.vscode-template/settings.json

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,5 +16,9 @@
1616
"eslint.alwaysShowStatus": true,
1717
"python.formatting.autopep8Args": [
1818
"-max-line-length", "140"
19-
]
19+
],
20+
"[python]":{
21+
"editor.detectIndentation" : false,
22+
"editor.tabSize" : 4
23+
}
2024
}

‎Makefile

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,17 +6,33 @@ PY_FILES = $(strip $(shell find services packages -iname '*.py'))
66

77
export PYTHONPATH=${PWD}/packages/s3wrapper/src
88

9+
build-devel:
10+
docker-compose -f services/docker-compose.yml -f services/docker-compose.devel.yml build
11+
12+
rebuild-devel:
13+
docker-compose -f services/docker-compose.yml -f services/docker-compose.devel.yml build --no-cache
14+
15+
up-devel:
16+
docker-compose -f services/docker-compose.yml -f services/docker-compose.devel.yml up
17+
918
build:
1019
docker-compose -f services/docker-compose.yml build
1120

1221
rebuild:
1322
docker-compose -f services/docker-compose.yml build --no-cache
1423

1524
up:
25+
docker-compose -f services/docker-compose.yml up
26+
27+
up-swarm:
1628
docker swarm init
1729
docker-compose -f services/docker-compose.yml -f services/docker-compose.deploy.yml up
1830

1931
down:
32+
docker-compose -f services/docker-compose.yml down
33+
docker-compose -f services/docker-compose.yml -f services/docker-compose.devel.yml down
34+
35+
down-swarm:
2036
docker-compose -f services/docker-compose.yml -f services/docker-compose.deploy.yml down
2137
docker swarm leave -f
2238

@@ -33,8 +49,8 @@ before_test:
3349
docker-compose -f packages/simcore-sdk/tests/docker-compose.yml build
3450

3551
run_test:
36-
pytest -v packages/pytest_docker/
37-
pytest -v packages/s3wrapper/
52+
pytest --cov=pytest_docker -v packages/pytest_docker/
53+
pytest --cov=s3wrapper -v packages/s3wrapper/
3854
pytest -v packages/simcore-sdk/
3955

4056
after_test:

‎services/README.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,3 +85,6 @@ To deploy the application in a single-node swarm
8585
docker swarm init
8686
docker-compose -f docker-compose.yml -f docker-compose.deploy.yml up
8787
```
88+
## Credentials
89+
90+
Rename `.env-devel` to `.env` in order to get the stack up and running.

‎services/adminer/README.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# Adminer
2+
3+
Database management tool

‎services/director/Dockerfile

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,8 @@ ENV FLASK_DEBUG=1
4545

4646
VOLUME /home/app/source
4747

48+
USER root
49+
4850
CMD ["/bin/sh", "./boot.sh"]
4951

5052
# --------------------------Production stage -------------------

‎services/docker-compose.debug.yml

Lines changed: 0 additions & 38 deletions
This file was deleted.

‎services/docker-compose.devel.yml

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
# USAGE: docker-compose -f docker-compose.yml -f docker-compose.devel.yml ...
2+
version: '3.4'
3+
services:
4+
director:
5+
image: services_director:dev
6+
build:
7+
target: development
8+
volumes:
9+
- './director/source:/home/app/source'
10+
#--------------------------------------------------------------------
11+
webclient:
12+
image: services_webclient:dev
13+
build:
14+
target: development
15+
volumes:
16+
- './web/client:/home/scu/client'
17+
command: compile --watch
18+
webserver:
19+
image: services_webserver:dev
20+
build:
21+
target: development
22+
volumes:
23+
- ./web/server:/home/scu/server
24+
- ./web/client/source-output:/home/scu/client
25+
- ../packages/simcore-sdk/src:/home/scu/packages
26+
depends_on:
27+
- webclient
28+
#--------------------------------------------------------------------
29+
sidecar:
30+
image: services_sidecar:dev
31+
build:
32+
target: development
33+
volumes:
34+
- ./sidecar/src/sidecar:/work/sidecar
35+
- ../packages:/work/packages

‎services/docker-compose.yml

Lines changed: 86 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -17,18 +17,96 @@ services:
1717
volumes:
1818
- '/var/run/docker.sock:/var/run/docker.sock'
1919
#--------------------------------------------------------------------
20+
webclient:
21+
image: services_webclient:build
22+
build:
23+
context: ./web/client/
24+
dockerfile: Dockerfile
25+
target: build
26+
command: /bin/bash -c "Echo "built with qx" && cat build-output/version.txt"
2027
webserver:
2128
build:
22-
context: .
23-
dockerfile: web/server/Dockerfile
24-
target: production
25-
args:
26-
client_dir: web/client
27-
server_dir: web/server
29+
context: ../
30+
dockerfile: services/web/Dockerfile
31+
target: ci
2832
ports:
2933
- '9081:8080'
3034
environment:
3135
- DIRECTOR_HOST=director
3236
- DIRECTOR_PORT=8001
33-
- SIMCORE_WEB_CONFIG=production
34-
#command: ["python", "server.py"]
37+
depends_on:
38+
- webclient
39+
#--------------------------------------------------------------------
40+
rabbit:
41+
image: rabbitmq:3-management
42+
environment:
43+
- RABBITMQ_DEFAULT_USER=${RABBITMQ_USER}
44+
- RABBITMQ_DEFAULT_PASS=${RABBITMQ_PASSWORD}
45+
ports:
46+
- "15672:15672"
47+
#--------------------------------------------------------------------
48+
# flower:
49+
# image: ondrejit/flower:latest
50+
# command: --broker=amqp://${RABBITMQ_USER}:${RABBITMQ_PASSWORD}@rabbit:5672
51+
# ports:
52+
# - 5555:5555
53+
# depends_on:
54+
# - rabbit
55+
#--------------------------------------------------------------------
56+
postgres:
57+
image: postgres:10
58+
environment:
59+
- POSTGRES_USER=${POSTGRES_USER}
60+
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
61+
- POSTGRES_DB=${POSTGRES_DB}
62+
volumes:
63+
- postgres:/var/lib/postgresql/data
64+
ports:
65+
- "5432:5432"
66+
#--------------------------------------------------------------------
67+
adminer:
68+
image: adminer
69+
ports:
70+
- 18080:8080
71+
depends_on:
72+
- postgres
73+
#--------------------------------------------------------------------
74+
sidecar:
75+
build:
76+
# the context for the build is the git repo root directory, this allows to copy
77+
# the packages directory into any docker image
78+
context: ../
79+
dockerfile: services/sidecar/Dockerfile
80+
target: production
81+
volumes:
82+
- input:/input
83+
- output:/output
84+
- log:/log
85+
- /var/run/docker.sock:/var/run/docker.sock
86+
ports:
87+
- "8000:8000"
88+
environment:
89+
- PYTHONPATH=/work/packages/simcore-sdk/src:/work/packages/s3wrapper/src
90+
- RABBITMQ_USER=${RABBITMQ_USER}
91+
- RABBITMQ_PASSWORD=${RABBITMQ_PASSWORD}
92+
- S3_ENDPOINT=${S3_ENDPOINT}
93+
- S3_ACCESS_KEY=${S3_ACCESS_KEY}
94+
- S3_SECRET_KEY=${S3_SECRET_KEY}
95+
- S3_BUCKET_NAME=${S3_BUCKET_NAME}
96+
depends_on:
97+
- rabbit
98+
#--------------------------------------------------------------------
99+
minio:
100+
image: minio/minio
101+
environment:
102+
- MINIO_ACCESS_KEY=12345678
103+
- MINIO_SECRET_KEY=12345678
104+
ports:
105+
- "9001:9000"
106+
command: server /data
107+
#--------------------------------------------------------------------
108+
volumes:
109+
input:
110+
output:
111+
log:
112+
postgres:

‎services/flower/README.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# Flower
2+
3+
Celery management tool

‎services/minio/README.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# Minio
2+
3+
S3 object store (for devel)

‎services/postgres/README.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# Db
2+
3+
Posgtres database

‎services/rabbit/README.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# Rabbit
2+
3+
Message queue for celery and pub/sub

‎services/sidecar/Dockerfile

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
FROM python:3.6-alpine as common
2+
3+
LABEL maintainer="Manuel Guidon <[email protected]"
4+
5+
WORKDIR /work/sidecar
6+
7+
RUN apk add --no-cache postgresql-dev gcc libc-dev
8+
9+
COPY services/sidecar/requirements.txt requirements.txt
10+
11+
RUN pip install --upgrade pip \
12+
&& pip install -r requirements.txt \
13+
&& pip list --format=columns
14+
15+
EXPOSE 8000
16+
17+
FROM common as development
18+
19+
VOLUME /work/sidecar
20+
VOLUME /work/packages
21+
22+
# NO clue why this does not work without explicitly specifying
23+
ENTRYPOINT celery -A sidecar worker -c 2 --loglevel=info
24+
25+
FROM common as production
26+
27+
# the context for the build is the git repo root directory
28+
COPY services/sidecar/src /work
29+
COPY packages /work/packages
30+
31+
# NO clue why this does not work without explicitly specifying
32+
ENV PYTHONPATH="/work/packages/simcore-sdk/src:/work/packages/s3wrapper/src"
33+
ENTRYPOINT celery -A sidecar worker -c 2 --loglevel=info

‎services/sidecar/README.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# Sidecar
2+
3+
Use sidecar container to control computational service.

‎services/sidecar/requirements.txt

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
celery==4.2.0
2+
docker==3.3.0
3+
minio==4.0.0
4+
networkx==2.1
5+
pika==0.11.2
6+
psycopg2==2.7.4
7+
sqlalchemy==1.2.8

‎services/computation/README.md renamed to ‎services/sidecar/src/sidecar/__init__.py

File renamed without changes.

0 commit comments

Comments
 (0)