11import logging
2+ from pdb import Pdb
23import sys
34import time
45from pathlib import Path
1011
1112logger = logging .getLogger (__name__ )
1213
13- current_dir = Path (sys .argv [0 ] if __name__ == "__main__" else __file__ ).resolve ().parent
14+ current_dir = Path (sys .argv [0 ] if __name__ == "__main__" else __file__ ).resolve ().parent
1415
1516WAIT_TIME_SECS = 20
1617RETRY_COUNT = 7
17- MAX_WAIT_TIME = 240
18+ MAX_WAIT_TIME = 240
1819
1920# https://docs.docker.com/engine/swarm/how-swarm-mode-works/swarm-task-states/
20- pre_states = [
21- "NEW" ,
22- "PENDING" ,
23- "ASSIGNED" ,
24- "PREPARING" ,
25- "STARTING"
26- ]
21+ pre_states = ["NEW" , "PENDING" , "ASSIGNED" , "PREPARING" , "STARTING" ]
2722
2823failed_states = [
2924 "COMPLETE" ,
3227 "REJECTED" ,
3328 "ORPHANED" ,
3429 "REMOVE" ,
35- "CREATED"
30+ "CREATED" ,
3631]
3732# UTILS --------------------------------
3833
34+
3935def get_tasks_summary (tasks ):
4036 msg = ""
4137 for t in tasks :
42- t ["Status" ].setdefault ("Err" , '' )
38+ t ["Status" ].setdefault ("Err" , "" )
4339 msg += "- task ID:{ID}, STATE: {Status[State]}, ERROR: '{Status[Err]}' \n " .format (
44- ** t )
40+ ** t
41+ )
4542 return msg
4643
4744
4845def get_failed_tasks_logs (service , docker_client ):
4946 failed_logs = ""
5047 for t in service .tasks ():
51- if t [' Status' ][ ' State' upper () in failed_states :
52- cid = t [' Status' ][ ' ContainerStatus' ][ ' ContainerID'
48+ if t [" Status" ][ " State" upper () in failed_states :
49+ cid = t [" Status" ][ " ContainerStatus" ][ " ContainerID"
5350 failed_logs += "{2} {0} - {1} BEGIN {2}\n " .format (
54- service .name , t ['ID' ], "=" * 10 )
51+ service .name , t ["ID" ], "=" * 10
52+ )
5553 if cid :
5654 container = docker_client .containers .get (cid )
57- failed_logs += container .logs ().decode (' utf-8'
55+ failed_logs += container .logs ().decode (" utf-8"
5856 else :
5957 failed_logs += " log unavailable. container does not exists\n "
6058 failed_logs += "{2} {0} - {1} END {2}\n " .format (
61- service .name , t ['ID' ], "=" * 10 )
59+ service .name , t ["ID" ], "=" * 10
60+ )
6261
6362 return failed_logs
63+
64+
6465# --------------------------------------------------------------------------------
6566
67+
6668def osparc_simcore_root_dir () -> Path :
6769 WILDCARD = "services/web/server"
6870
@@ -81,46 +83,66 @@ def osparc_simcore_root_dir() -> Path:
8183def core_docker_compose_file () -> Path :
8284 return osparc_simcore_root_dir () / ".stack-simcore-version.yml"
8385
86+
8487def core_services () -> List [str ]:
8588 with core_docker_compose_file ().open () as fp :
8689 dc_specs = yaml .safe_load (fp )
8790 return [x for x in dc_specs ["services" ].keys ()]
8891
92+
8993def ops_docker_compose_file () -> Path :
9094 return osparc_simcore_root_dir () / ".stack-ops.yml"
9195
96+
9297def ops_services () -> List [str ]:
9398 with ops_docker_compose_file ().open () as fp :
9499 dc_specs = yaml .safe_load (fp )
95100 return [x for x in dc_specs ["services" ].keys ()]
96101
97- def wait_for_services () -> bool :
102+
103+ def wait_for_services () -> None :
98104 # get all services
99105 services = core_services () + ops_services ()
100106
101107 client = docker .from_env ()
102- running_services = [x for x in client .services .list () if x .name .split ("_" )[1 ] in services ]
108+ running_services = [
109+ x for x in client .services .list () if x .name .split ("_" )[- 1 ] in services
110+ ]
111+
103112 # check all services are in
104113 assert len (running_services ), "no services started!"
105- assert len (services ) == len (running_services ), "Some services are missing"
114+ assert len (services ) == len (
115+ running_services
116+ ), f"Some services are missing:\n expected: { services } \n got: { running_services }
106117 # now check they are in running mode
107118 for service in running_services :
119+ task = None
108120 for n in range (RETRY_COUNT ):
109- task = service .tasks ()[0 ]
110- if task ['Status' ]['State' ].upper () in pre_states :
111- print ("Waiting [{}/{}] for {}...\n {}" .format (n , RETRY_COUNT , service .name , get_tasks_summary (service .tasks ())))
121+ # get last updated task
122+ sorted_tasks = sorted (service .tasks (), key = lambda task : task ["UpdatedAt" ])
123+ task = sorted_tasks [- 1 ]
124+
125+ if task ["Status" ]["State" ].upper () in pre_states :
126+ print (
127+ "Waiting [{}/{}] for {}...\n {}" .format (
128+ n , RETRY_COUNT , service .name , get_tasks_summary (service .tasks ())
129+ )
130+ )
112131 time .sleep (WAIT_TIME_SECS )
113- elif task ['Status' ]['State' ].upper () in failed_states :
114- print (f"Waiting [{ n } { RETRY_COUNT } { service .name } \n { get_tasks_summary (service .tasks ())} )
132+ elif task ["Status" ]["State" ].upper () in failed_states :
133+ print (
134+ f"Waiting [{ n } { RETRY_COUNT } { service .name } \n { get_tasks_summary (service .tasks ())}
135+ )
115136 time .sleep (WAIT_TIME_SECS )
116137 else :
117138 break
118- assert task ['Status' ]['State' ].upper () == "RUNNING" ,\
119- "Expected running, got \n {}\n {}" .format (
120- pformat (task ),
121- get_tasks_summary (service .tasks ()))
122- # get_failed_tasks_logs(service, client))
123-
139+ assert task
140+ assert (
141+ task ["Status" ]["State" ].upper () == "RUNNING"
142+ ), "Expected running, got \n {}\n {}" .format (
143+ pformat (task ), get_tasks_summary (service .tasks ())
144+ )
145+ # get_failed_tasks_logs(service, client))
124146
125147
126148if __name__ == "__main__" :
0 commit comments