Making services more debuggable (#1690)

* fixes an issue with missing logs from container

* updated error message

* maked debugging easier when container fails

* triggering ci agian

* fix previous

Co-authored-by: Andrei Neagu <[email protected]>
Co-authored-by: Odei Maiz <[email protected]>

Author: 3 people

services/sidecar/src/simcore_service_sidecar/executor.py

@@ -74,8 +74,8 @@ async def run(self):
             await self._post_messages(
                 LogType.LOG, "[sidecar]...task completed successfully."
             )
-        except exceptions.SidecarException:
-            await self._post_messages(LogType.LOG, "[sidecar]...task failed.")
+        except exceptions.SidecarException as e:
+            await self._post_messages(LogType.LOG, f"[sidecar]...task failed: {str(e)}")
             raise
 
     async def preprocess(self):

services/sidecar/src/simcore_service_sidecar/log_parser.py

@@ -3,6 +3,7 @@
 import re
 from enum import Enum
 from pathlib import Path
+import aiodocker
 from typing import Awaitable, Callable, Tuple, Union
 
 import aiofiles
@@ -78,8 +79,14 @@ async def monitor_logs_task(
 async def _monitor_docker_container(
     container: DockerContainer, log_cb: Awaitable[Callable[[LogType, str], None]]
 ) -> None:
-    async for line in container.log(stdout=True, stderr=True, follow=True):
-        log_type, parsed_line = await parse_line(line)
+    try:
+        async for line in container.log(stdout=True, stderr=True, follow=True):
+            log_type, parsed_line = await parse_line(line)
+    except aiodocker.exceptions.DockerError as e:
+        log_type, parsed_line = await parse_line(
+            f"Could not recover logs because: {str(e)}"
+        )
+    finally:
         await log_cb(log_type, parsed_line)