Skip to content

Commit 14cbec2

Browse files
author
Pedro Crespo
committed
Merge remote-tracking branch 'upstream/master'
2 parents 3f4eed8 + b9788ce commit 14cbec2

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

71 files changed

+2370
-495
lines changed

‎.travis.yml

+5-9
Original file line numberDiff line numberDiff line change
@@ -35,18 +35,14 @@ matrix:
3535
- while sudo lsof -Pi :5432 -sTCP:LISTEN -t; do sleep 1; done
3636

3737
install:
38-
- pip3 install pylint
39-
- pip3 install pytest-cov
40-
- pip3 install coveralls
41-
- pip3 install -r packages/pytest_docker/requirements.txt
42-
- pip3 install -r packages/s3wrapper/requirements.txt
43-
- pip3 install -r packages/simcore-sdk/requirements.txt
44-
- pip3 --version
38+
- pip install --upgrade pip wheel setuptools && pip3 --version
39+
- pip3 install packages/s3wrapper[test]
40+
- pip3 install packages/simcore-sdk[test]
41+
- pushd services/web/server; pip3 install -r requirements/ci.txt; popd
4542

4643
before_script:
4744
- pylint --version
48-
# change python path for 2nd party libraries
49-
- export PYTHONPATH=${PYTHONPATH}:${TRAVIS_BUILD_DIR}/packages/s3wrapper/src
45+
- pip3 list
5046

5147
script:
5248
- make pylint

‎Makefile

+12-5
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,9 @@ all:
2020
@echo 'run `make up-devel` to start your dev environment.'
2121
@echo 'see Makefile for further targets'
2222

23+
clean:
24+
@git clean -dxf -e .vscode/
25+
2326
build-devel:
2427
${DOCKER_COMPOSE} -f services/docker-compose.yml -f services/docker-compose.devel.yml build
2528

@@ -76,9 +79,10 @@ before_test:
7679
${DOCKER_COMPOSE} -f packages/simcore-sdk/tests/docker-compose.yml build
7780

7881
run_test:
79-
pytest --cov=pytest_docker -v packages/pytest_docker/
80-
pytest --cov=s3wrapper -v packages/s3wrapper/
81-
pytest -v packages/simcore-sdk/
82+
pytest --cov=pytest_docker -v packages/pytest_docker/tests
83+
pytest --cov=s3wrapper -v packages/s3wrapper/tests
84+
pytest --cov=simcore_sdk -v packages/simcore-sdk/tests
85+
pytest --cov=server -v services/web/server/tests
8286

8387
after_test:
8488
# leave a clean slate (not sure whether this is actually needed)
@@ -114,7 +118,10 @@ push_platform_images:
114118
@diff -uN $@ $<
115119
@false
116120

117-
venv:
121+
.venv:
118122
python3 -m venv .venv
119123
.venv/bin/pip3 install --upgrade pip wheel setuptools
120-
echo "To activate the venv, execute 'source .venv/bin/activate'"
124+
@echo "To activate the venv, execute 'source .venv/bin/activate' or '.venv/bin/activate.bat' (WIN)"
125+
126+
127+
.PHONY: all clean build-devel rebuild-devel up-devel build up down test after_test push_platform_images

‎packages/s3wrapper/requirements-dev.txt

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# develop mode
2+
-e . ".[test]"
3+
4+
autopep8>=1.3.5

‎packages/s3wrapper/requirements.txt

-1
This file was deleted.

‎packages/s3wrapper/setup.py

+23
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,29 @@
11
from setuptools import setup
22

3+
install_requires = [
4+
'minio==4.0.0',
5+
]
6+
7+
tests_require = [
8+
'coveralls~=1.3',
9+
'mock~=2.0',
10+
'pylint~=2.0',
11+
'pytest~=3.6',
12+
'pytest-cov~=2.5',
13+
'pytest-docker~=0.6',
14+
'requests~=2.19'
15+
]
16+
317
setup(
418
name='s3wrapper',
519
version='0.1.0',
20+
package_dir={'': 'src'},
21+
packages=['s3wrapper'],
22+
python_requires='>=3.6',
23+
install_requires=install_requires,
24+
tests_require=tests_require,
25+
extras_require= {
26+
'test': tests_require
27+
},
28+
setup_requires=['pytest-runner']
629
)

‎packages/simcore-sdk/requirements-dev.txt

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# develop mode
2+
-e . ".[test]"
3+
-e ../s3wrapper/
4+
5+
autopep8>=1.3.5

‎packages/simcore-sdk/requirements.txt

-5
This file was deleted.

‎packages/simcore-sdk/setup.py

+27
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,33 @@
11
from setuptools import setup
22

3+
INSTALL_REQUIRES = [
4+
'networkx==2.1',
5+
'psycopg2==2.7.4',
6+
'sqlalchemy==1.2.8',
7+
'tenacity==4.12.0'
8+
'trafaret-config==2.0.1'
9+
]
10+
11+
TEST_REQUIRE = [
12+
'coveralls~=1.3',
13+
'mock~=2.0',
14+
'pylint~=2.0',
15+
'pytest~=3.6',
16+
'pytest-cov~=2.5',
17+
'pytest-docker~=0.6',
18+
'requests~=2.19'
19+
]
20+
321
setup(
422
name='simcore-sdk',
523
version='0.1.0',
24+
package_dir={'': 'src'},
25+
packages=['simcore_sdk'],
26+
python_requires='>=3.6',
27+
INSTALL_REQUIRES=INSTALL_REQUIRES,
28+
TEST_REQUIRE=TEST_REQUIRE,
29+
extras_require= {
30+
'test': TEST_REQUIRE
31+
},
32+
setup_requires=['pytest-runner']
633
)

‎packages/simcore-sdk/src/simcore_sdk/config/db.py

+18-3
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,27 @@
1-
""" Basic configuration file for postgres
1+
""" Basic configuration file for postgres service
22
33
"""
44
from os import environ as env
55

6+
import trafaret as T
67

8+
CONFIG_SCHEMA = T.Dict({
9+
"database": T.String(),
10+
"user": T.String(),
11+
"password": T.String(),
12+
T.Key("minsize", default=1 ,optional=True): T.Int(),
13+
T.Key("maxsize", default=4, optional=True): T.Int(),
14+
"host": T.Or( T.String, T.Null),
15+
"port": T.Or( T.Int, T.Null),
16+
"endpoint": T.Or( T.String, T.Null)
17+
})
718

19+
20+
# TODO: deprecate!
821
class Config():
22+
923
def __init__(self):
24+
# TODO: uniform config classes . see server.config file
1025
POSTGRES_URL = env.get("POSTGRES_ENDPOINT", "postgres:5432")
1126
POSTGRES_USER = env.get("POSTGRES_USER", "simcore")
1227
POSTGRES_PW = env.get("POSTGRES_PASSWORD", "simcore")
@@ -18,7 +33,7 @@ def __init__(self):
1833
self._db = POSTGRES_DB
1934
self._endpoint = 'postgresql+psycopg2://{user}:{pw}@{url}/{db}'.format(
2035
user=self._user, pw=self._pwd, url=self._url, db=self._db)
21-
36+
2237
@property
2338
def endpoint(self):
24-
return self._endpoint
39+
return self._endpoint

‎packages/simcore-sdk/src/simcore_sdk/config/docker.py

+10
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,18 @@
11
""" Basic configuration file for docker registry
22
33
"""
4+
import trafaret as T
5+
6+
# TODO: adapt all data below!
7+
CONFIG_SCHEMA = T.Dict({
8+
"user": T.String(),
9+
"password": T.String(),
10+
"registry": T.String()
11+
})
12+
413

514
class Config():
15+
# TODO: uniform config classes . see server.config file
616
def __init__(self):
717
self._registry = "masu.speag.com"
818
self._user = "z43"

‎packages/simcore-sdk/src/simcore_sdk/config/rabbit.py

+86-11
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,66 @@
55
from os import environ as env
66

77
import pika
8+
import yaml
9+
import trafaret as T
810

11+
12+
# TODO: adapt all data below!
13+
# TODO: can use venv as defaults? e.g. $RABBITMQ_LOG_CHANNEL
14+
CONFIG_SCHEMA = T.Dict({
15+
T.Key("host", default='rabbit', optional=True): T.String(),
16+
T.Key("port", default=5672, optional=True): T.Int(),
17+
"user": T.String(),
18+
"password": T.String(),
19+
"channels": T.Dict({
20+
"progress": T.String(),
21+
"log": T.String(),
22+
T.Key("celery", default=dict(result_backend="rpc://"), optional=True): T.Dict({
23+
T.Key("result_backend", default="${CELERY_RESULT_BACKEND}", optional=True): T.String()
24+
})
25+
})
26+
})
27+
28+
29+
CONFIG_EXAMPLES = map(yaml.load,[
30+
"""
31+
user: simcore
32+
password: simcore
33+
channels:
34+
log: comp.backend.channels.log
35+
progress: comp.backend.channels.progress
36+
""",
37+
"""
38+
host: rabbito
39+
port: 1234
40+
user: foo
41+
password: secret
42+
channels:
43+
log: comp.backend.channels.log
44+
progress: comp.backend.channels.progress
45+
""",
46+
"""
47+
user: bar
48+
password: secret
49+
channels:
50+
log: comp.backend.channels.log
51+
progress: comp.backend.channels.progress
52+
celery:
53+
result_backend: 'rpc://'
54+
"""])
55+
56+
57+
def eval_broker(config):
58+
"""
59+
Raises trafaret.DataError if config validation fails
60+
"""
61+
CONFIG_SCHEMA.check(config) # raise exception
62+
url = 'amqp://{user}:{password}@{host}:{port}'.format(**config)
63+
return url
64+
65+
66+
# TODO: deprecate! -----------------------------------------------------------------------------
67+
# TODO: uniform config classes . see server.config file
968
RABBITMQ_USER = env.get('RABBITMQ_USER','simcore')
1069
RABBITMQ_PASSWORD = env.get('RABBITMQ_PASSWORD','simcore')
1170
RABBITMQ_HOST="rabbit"
@@ -17,21 +76,37 @@
1776

1877
CELERY_BROKER_URL = AMQ_URL
1978
CELERY_RESULT_BACKEND=env.get('CELERY_RESULT_BACKEND','rpc://')
79+
# -------------------------------------
2080

21-
class Config():
22-
def __init__(self):
81+
class Config:
82+
def __init__(self, config=None):
83+
if config is not None:
84+
CONFIG_SCHEMA.check(config) # raise exception
85+
else:
86+
config = {}
87+
88+
# FIXME: get variables via config.get('') or
2389
# rabbit
24-
self._broker_url = CELERY_BROKER_URL
25-
self._result_backend = CELERY_RESULT_BACKEND
90+
try:
91+
self._broker_url = eval_broker(config)
92+
except: # pylint: disable=W0702
93+
self._broker_url = CELERY_BROKER_URL
94+
95+
self._result_backend = config.get("celery", {}).get("result_backend") or CELERY_RESULT_BACKEND
2696
self._module_name = "tasks"
2797

2898
# pika
29-
self._pika_credentials = pika.PlainCredentials(RABBITMQ_USER, RABBITMQ_PASSWORD)
30-
self._pika_parameters = pika.ConnectionParameters(host=RABBITMQ_HOST,
31-
port=RABBITMQ_PORT, credentials=self._pika_credentials, connection_attempts=100)
99+
self._pika_credentials = pika.PlainCredentials(
100+
config.get("user") or RABBITMQ_USER,
101+
config.get("password") or RABBITMQ_PASSWORD)
102+
self._pika_parameters = pika.ConnectionParameters(
103+
host=config.get("host") or RABBITMQ_HOST,
104+
port=config.get("port") or RABBITMQ_PORT,
105+
credentials=self._pika_credentials,
106+
connection_attempts=100)
32107

33-
self._log_channel = RABBITMQ_LOG_CHANNEL
34-
self._progress_channel = RABBITMQ_PROGRESS_CHANNEL
108+
self._log_channel = config.get("celery", {}).get("result_backend") or RABBITMQ_LOG_CHANNEL
109+
self._progress_channel = config.get("celery", {}).get("result_backend") or RABBITMQ_PROGRESS_CHANNEL
35110

36111
@property
37112
def parameters(self):
@@ -44,7 +119,7 @@ def log_channel(self):
44119
@property
45120
def progress_channel(self):
46121
return self._progress_channel
47-
122+
48123
@property
49124
def broker(self):
50125
return self._broker_url
@@ -55,4 +130,4 @@ def backend(self):
55130

56131
@property
57132
def name(self):
58-
return self._module_name
133+
return self._module_name

‎packages/simcore-sdk/src/simcore_sdk/config/s3.py

+12
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,23 @@
44
from os import environ as env
55
import logging
66

7+
import trafaret as T
78

89
_LOGGER = logging.getLogger(__name__)
910

11+
12+
CONFIG_SCHEMA = T.Dict({
13+
"endpoint": T.String(),
14+
"access_key": T.String(),
15+
"secret_key": T.String(),
16+
"bucket_name": T.String(),
17+
})
18+
19+
20+
# TODO: deprecate!
1021
class Config():
1122
def __init__(self):
23+
# TODO: uniform config classes . see server.config file
1224
S3_ENDPOINT = env.get("S3_ENDPOINT", "minio:9000")
1325
S3_ACCESS_KEY = env.get("S3_ACCESS_KEY", "12345678")
1426
S3_SECRET_KEY = env.get("S3_SECRET_KEY", "12345678")

‎packages/simcore-sdk/src/simcore_sdk/models/pipeline_models.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -53,4 +53,4 @@ class ComputationalTask(Base):
5353
# utc timestamps for submission/start/end
5454
submit = Column(DateTime)
5555
start = Column(DateTime)
56-
end = Column(DateTime)
56+
end = Column(DateTime)

‎services/director/Dockerfile

+2-2
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,8 @@ COPY --chown=app:app requirements.txt requirements.txt
2929
RUN python -m venv venv \
3030
&& chown -R app:app venv
3131

32-
RUN venv/bin/pip install --upgrade pip \
33-
&& venv/bin/pip install -r requirements.txt
32+
RUN venv/bin/pip3 install --upgrade pip wheel setuptools\
33+
&& venv/bin/pip3 install -r requirements.txt
3434

3535
COPY --chown=app:app boot.sh boot.sh
3636
RUN chmod +x boot.sh

‎services/docker-compose.yml

+4
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,10 @@ services:
4848
- S3_ACCESS_KEY=${S3_ACCESS_KEY}
4949
- S3_SECRET_KEY=${S3_SECRET_KEY}
5050
- S3_BUCKET_NAME=${S3_BUCKET_NAME}
51+
- POSTGRES_ENDPOINT=${POSTGRES_ENDPOINT}
52+
- POSTGRES_USER=${POSTGRES_USER}
53+
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
54+
- POSTGRES_DB=${POSTGRES_DB}
5155
depends_on:
5256
- webclient
5357
#--------------------------------------------------------------------

0 commit comments

Comments
 (0)