File tree 1 file changed +7
-6
lines changed 1 file changed +7
-6
lines changed Original file line number Diff line number Diff line change @@ -119,10 +119,9 @@ def collective_rpc(self,
119
119
timeout = dequeue_timeout )
120
120
121
121
if status != WorkerProc .ResponseStatus .SUCCESS :
122
- if isinstance (result , Exception ):
123
- raise result
124
- else :
125
- raise RuntimeError ("Worker failed" )
122
+ raise RuntimeError (
123
+ "Worker failed with error %s, please check the"
124
+ " stack trace above for the root cause" , result )
126
125
127
126
responses [w .rank ] = result
128
127
@@ -378,9 +377,11 @@ def worker_busy_loop(self):
378
377
# Notes have been introduced in python 3.11
379
378
if hasattr (e , "add_note" ):
380
379
e .add_note (traceback .format_exc ())
381
- self .worker_response_mq .enqueue (
382
- (WorkerProc .ResponseStatus .FAILURE , e ))
383
380
logger .exception ("WorkerProc hit an exception: %s" , exc_info = e )
381
+ # exception might not be serializable, so we convert it to
382
+ # string, only for logging purpose.
383
+ self .worker_response_mq .enqueue (
384
+ (WorkerProc .ResponseStatus .FAILURE , str (e )))
384
385
continue
385
386
386
387
self .worker_response_mq .enqueue (
You can’t perform that action at this time.
0 commit comments