Skip to content

Commit 0ff6c01

Browse files
author
Pedro Crespo
committed
Split sockets subsystem into setup and handlers
Added s3 subsystem
1 parent 5098583 commit 0ff6c01

File tree

7 files changed

+193
-102
lines changed

7 files changed

+193
-102
lines changed

‎services/web/server/src/simcore_service_webserver/application.py

+6-2
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,10 @@
1414
from .rest import setup_rest
1515
from .security import setup_security
1616
from .session import setup_session
17-
from .sockets import setup_sio
17+
from .sockets import setup_sockets
1818
from .statics import setup_statics
19+
from .director import setup_director
20+
from .s3 import setup_s3
1921

2022
log = logging.getLogger(__name__)
2123

@@ -41,9 +43,11 @@ def create_application(config: dict):
4143
setup_email(app)
4244
setup_computation(app)
4345
setup_statics(app)
44-
setup_sio(app)
46+
setup_sockets(app)
4547
setup_rest(app)
4648
setup_login(app)
49+
setup_director(app)
50+
setup_s3(app)
4751

4852
return app
4953

‎services/web/server/src/simcore_service_webserver/application_config.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,6 @@
2929
log = logging.getLogger(__name__)
3030

3131

32-
33-
3432
def create_schema():
3533
"""
3634
Build schema for the configuration's file
@@ -52,6 +50,8 @@ def create_schema():
5250
rest_config.CONFIG_SECTION_NAME: rest_config.schema,
5351
email_config.CONFIG_SECTION_NAME: email_config.schema,
5452
computation_config.CONFIG_SECTION_NAME: computation_config.schema,
53+
#s3_config.CONFIG_SECTION_NAME: s3_config.schema
54+
#TODO: enable when sockets are refactored
5555
})
5656

5757

‎services/web/server/src/simcore_service_webserver/director.py

+12
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,24 @@
11
from aiohttp import web
22

3+
import logging
34
from .application_keys import APP_CONFIG_KEY
45
from .director_config import CONFIG_SECTION_NAME
56

7+
logger = logging.getLogger(__name__)
68

79
def setup(app: web.Application):
10+
log.debug("Setting up %s ...", __name__)
811

912
assert CONFIG_SECTION_NAME in app[APP_CONFIG_KEY]
1013

1114

1215
# TODO: implement!!!
16+
17+
18+
19+
# alias
20+
setup_director = setup
21+
22+
__all__ = (
23+
'setup_director'
24+
)

‎services/web/server/src/simcore_service_webserver/s3.py

+39
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
""" s3 subsystem
2+
3+
Provides a client-sdk to interact with minio services
4+
"""
5+
import logging
6+
7+
from aiohttp import web
8+
9+
from .application_keys import APP_CONFIG_KEY
10+
from .s3_config import CONFIG_SECTION_NAME
11+
12+
#from s3wrapper.s3_client import S3Client
13+
14+
logger = logging.getLogger(__name__)
15+
16+
def setup(app: web.Application):
17+
logger.debug("Setting up %s ...", __name__)
18+
19+
assert CONFIG_SECTION_NAME not in app[APP_CONFIG_KEY], "Temporarily disabled"
20+
21+
# TODO: implement!!!
22+
23+
# TODO: enable when sockets are refactored
24+
#cfg = app[APP_CONFIG_KEY][CONFIG_SECTION_NAME]
25+
#
26+
# client = S3Client(
27+
# endpoint=cfg['endpoint'],
28+
# access_key=cfg['access_key'],
29+
# secret_key=cfg['secret_key'])
30+
31+
# app["s3.client"] = client
32+
33+
34+
# alias
35+
setup_s3 = setup
36+
37+
__all__ = (
38+
'setup_s3'
39+
)

‎services/web/server/src/simcore_service_webserver/s3_config.py

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
""" s3 subsystem's configuration
2+
3+
- config-file schema
4+
- settings
5+
"""
6+
#import trafaret as T
7+
from simcore_sdk.config.s3 import CONFIG_SCHEMA as _S3_SCHEMA
8+
9+
CONFIG_SECTION_NAME = 's3'
10+
11+
schema = _S3_SCHEMA

‎services/web/server/src/simcore_service_webserver/sockets.py

+13-98
Original file line numberDiff line numberDiff line change
@@ -1,112 +1,27 @@
1-
""" Defines **async** handlers for socket.io server
1+
""" socket io subsystem
22
3-
SEE https://pypi.python.org/pypi/python-socketio
4-
SEE http://python-socketio.readthedocs.io/en/latest/
5-
"""
6-
# pylint: disable=C0111
7-
# pylint: disable=W0703
83
4+
"""
95
import logging
10-
import socketio
11-
12-
from s3wrapper.s3_client import S3Client
13-
from simcore_sdk.config.s3 import Config as s3_config
14-
15-
from . import interactive_services_manager
16-
17-
log = logging.getLogger(__file__)
18-
19-
# TODO: separate API from server application!
20-
SIO = socketio.AsyncServer(async_mode="aiohttp", logging=log)
216

7+
from aiohttp import web
228

23-
@SIO.on("connect")
24-
def connect(sid, environ):
25-
# pylint: disable=W0613
26-
# environ = WSGI evnironment dictionary
27-
log.debug("client %s connects", sid)
28-
interactive_services_manager.session_connect(sid)
29-
return True
9+
from .sockets_handlers import sio
3010

31-
@SIO.on("startDynamic")
32-
async def start_dynamic_service(sid, data):
33-
log.debug("client %s starts dynamic service %s", sid, data)
34-
try:
35-
service_key = data["serviceKey"]
36-
service_version = "latest"
37-
# if "serviceVersion" in data:
38-
# service_version = data["serviceVersion"]
39-
node_id = data["nodeId"]
40-
result = await interactive_services_manager.start_service(sid, service_key, node_id, service_version)
41-
await SIO.emit("startDynamic", data=result, room=sid)
42-
except IOError:
43-
log.exception("Error emitting results")
44-
except Exception:
45-
log.exception("Error while starting service")
4611

47-
@SIO.on("stopDynamic")
48-
async def stop_dynamic_service(sid, data):
49-
log.debug("client %s stops dynamic service %s", sid, data)
50-
try:
51-
node_id = data["nodeId"]
52-
await interactive_services_manager.stop_service(sid, node_id)
53-
except Exception:
54-
log.exception("Error while stopping service")
12+
log = logging.getLogger(__name__)
5513

56-
@SIO.on("presignedUrl")
57-
async def retrieve_url_for_file(sid, data):
58-
log.debug("client %s requests S3 url for %s", sid, data)
59-
_config = s3_config()
60-
log.debug("S3 endpoint %s", _config.endpoint)
6114

15+
def setup(app: web.Application):
16+
log.debug("Setting up %s ...", __name__)
6217

63-
s3_client = S3Client(endpoint=_config.endpoint,
64-
access_key=_config.access_key, secret_key=_config.secret_key)
65-
url = s3_client.create_presigned_put_url(_config.bucket_name, data["fileName"])
66-
#result = minioClient.presigned_put_object(data["bucketName"], data["fileName"])
67-
# Response error is still possible since internally presigned does get
68-
# bucket location.
69-
data_out = {}
70-
data_out["url"] = url
71-
try:
72-
await SIO.emit("presignedUrl", data=data_out, room=sid)
73-
except IOError:
74-
log.exception("Error emitting results")
75-
76-
@SIO.on("listObjects")
77-
async def list_S3_objects(sid, data):
78-
log.debug("client %s requests objects in storage. Extra argument %s", sid, data)
79-
_config = s3_config()
80-
81-
s3_client = S3Client(endpoint=_config.endpoint,
82-
access_key=_config.access_key, secret_key=_config.secret_key)
83-
84-
objects = s3_client.list_objects_v2(_config.bucket_name)
85-
data_out = []
86-
location = "simcore.sandbox"
87-
for obj in objects:
88-
obj_info = {}
89-
obj_info["file_uuid"] = obj.bucket_name + "/" + obj.object_name
90-
obj_info["location"] = location
91-
obj_info["bucket_name"] = obj.bucket_name
92-
obj_info["object_name"] = obj.object_name
93-
obj_info["size"] = obj.size
94-
data_out.append(obj_info)
95-
try:
96-
await SIO.emit("listObjects", data=data_out, room=sid)
97-
except IOError:
98-
log.exception("Error emitting results")
18+
sio.attach(app)
9919

100-
@SIO.on("disconnect")
101-
async def disconnect(sid):
102-
log.debug("client %s disconnected", sid)
103-
try:
104-
await interactive_services_manager.session_disconnected(sid)
105-
except Exception:
106-
log.exception("Error while disconnecting client")
10720

21+
# alias
22+
setup_sockets = setup
10823

109-
def setup_sio(app):
110-
log.debug("Setting up %s ...", __name__)
11124

112-
SIO.attach(app)
25+
__all__ = (
26+
"setup_sockets"
27+
)

‎services/web/server/src/simcore_service_webserver/sockets_handlers.py

+110
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
""" Defines **async** handlers for socket.io server
2+
3+
SEE https://pypi.python.org/pypi/python-socketio
4+
SEE http://python-socketio.readthedocs.io/en/latest/
5+
"""
6+
# pylint: disable=C0111
7+
# pylint: disable=W0703
8+
9+
import logging
10+
11+
import socketio
12+
13+
from s3wrapper.s3_client import S3Client
14+
from simcore_sdk.config.s3 import Config as s3_config
15+
# TODO: this is the only config that is not part of the schema
16+
# At first sight, adding it would require refactorin how socketio
17+
# is setup and avoid sio as a singleton!
18+
19+
from . import interactive_services_manager
20+
21+
log = logging.getLogger(__file__)
22+
23+
# TODO: separate API from server application!
24+
sio = socketio.AsyncServer(async_mode="aiohttp", logging=log)
25+
26+
27+
@sio.on("connect")
28+
def connect(sid, environ):
29+
# pylint: disable=W0613
30+
# environ = WSGI evnironment dictionary
31+
log.debug("client %s connects", sid)
32+
interactive_services_manager.session_connect(sid)
33+
return True
34+
35+
@sio.on("startDynamic")
36+
async def start_dynamic_service(sid, data):
37+
log.debug("client %s starts dynamic service %s", sid, data)
38+
try:
39+
service_key = data["serviceKey"]
40+
service_version = "latest"
41+
# if "serviceVersion" in data:
42+
# service_version = data["serviceVersion"]
43+
node_id = data["nodeId"]
44+
result = await interactive_services_manager.start_service(sid, service_key, node_id, service_version)
45+
await sio.emit("startDynamic", data=result, room=sid)
46+
except IOError:
47+
log.exception("Error emitting results")
48+
except Exception:
49+
log.exception("Error while starting service")
50+
51+
@sio.on("stopDynamic")
52+
async def stop_dynamic_service(sid, data):
53+
log.debug("client %s stops dynamic service %s", sid, data)
54+
try:
55+
node_id = data["nodeId"]
56+
await interactive_services_manager.stop_service(sid, node_id)
57+
except Exception:
58+
log.exception("Error while stopping service")
59+
60+
@sio.on("presignedUrl")
61+
async def retrieve_url_for_file(sid, data):
62+
log.debug("client %s requests S3 url for %s", sid, data)
63+
_config = s3_config()
64+
log.debug("S3 endpoint %s", _config.endpoint)
65+
66+
67+
s3_client = S3Client(endpoint=_config.endpoint,
68+
access_key=_config.access_key, secret_key=_config.secret_key)
69+
url = s3_client.create_presigned_put_url(_config.bucket_name, data["fileName"])
70+
#result = minioClient.presigned_put_object(data["bucketName"], data["fileName"])
71+
# Response error is still possible since internally presigned does get
72+
# bucket location.
73+
data_out = {}
74+
data_out["url"] = url
75+
try:
76+
await sio.emit("presignedUrl", data=data_out, room=sid)
77+
except IOError:
78+
log.exception("Error emitting results")
79+
80+
@sio.on("listObjects")
81+
async def list_S3_objects(sid, data):
82+
log.debug("client %s requests objects in storage. Extra argument %s", sid, data)
83+
_config = s3_config()
84+
85+
s3_client = S3Client(endpoint=_config.endpoint,
86+
access_key=_config.access_key, secret_key=_config.secret_key)
87+
88+
objects = s3_client.list_objects_v2(_config.bucket_name)
89+
data_out = []
90+
location = "simcore.sandbox"
91+
for obj in objects:
92+
obj_info = {}
93+
obj_info["file_uuid"] = obj.bucket_name + "/" + obj.object_name
94+
obj_info["location"] = location
95+
obj_info["bucket_name"] = obj.bucket_name
96+
obj_info["object_name"] = obj.object_name
97+
obj_info["size"] = obj.size
98+
data_out.append(obj_info)
99+
try:
100+
await sio.emit("listObjects", data=data_out, room=sid)
101+
except IOError:
102+
log.exception("Error emitting results")
103+
104+
@sio.on("disconnect")
105+
async def disconnect(sid):
106+
log.debug("client %s disconnected", sid)
107+
try:
108+
await interactive_services_manager.session_disconnected(sid)
109+
except Exception:
110+
log.exception("Error while disconnecting client")

0 commit comments

Comments
 (0)