Minor changes in sidecar

Author: pcrespov

services/sidecar/src/simcore_service_sidecar/config.py

@@ -43,7 +43,7 @@
 )
 
 SIDECAR_LOGLEVEL: str = getattr(
-    logging, os.environ.get("SIDECAR_LOGLEVEL", "WARNING").upper(), logging.WARNING
+    logging, os.environ.get("SIDECAR_LOGLEVEL", "WARNING").upper(), logging.DEBUG
 )
 
 DOCKER_REGISTRY: str = os.environ.get("REGISTRY_URL", "masu.speag.com")

services/sidecar/src/simcore_service_sidecar/executor.py

@@ -12,7 +12,7 @@
 from aiodocker.containers import DockerContainer
 from aiodocker.exceptions import DockerContainerError, DockerError
 from packaging import version
-from tenacity import retry, stop_after_attempt
+from tenacity import after_log, retry, stop_after_attempt
 
 from celery.utils.log import get_task_logger
 from servicelib.utils import fire_and_forget_task, logged_gather
@@ -117,7 +117,9 @@ async def preprocess(self):
         self.shared_folders.create()
         host_name = config.SIDECAR_HOST_HOSTNAME_PATH.read_text()
         await self._post_messages(LogType.LOG, f"[sidecar]Running on {host_name}")
-        results = await logged_gather(self._process_task_inputs(), self._pull_image())
+        results = await logged_gather(
+            self._process_task_inputs(), self._pull_image(), reraise=True
+        )
         await self._write_input_file(results[0])
         log.debug("Pre-Processing Pipeline DONE")
 
@@ -207,7 +209,9 @@ async def _write_input_file(self, inputs: Dict) -> None:
             file_name.write_text(json.dumps(inputs))
             log.debug("Writing input file DONE")
 
-    @retry(reraise=True, stop=stop_after_attempt(3))
+    @retry(
+        reraise=True, stop=stop_after_attempt(3), after=after_log(log, logging.WARNING)
+    )
     async def _pull_image(self):
         docker_image = f"{config.DOCKER_REGISTRY}/{self.task.image['name']}:{self.task.image['tag']}"
         log.debug(
@@ -216,36 +220,33 @@ async def _pull_image(self):
             config.DOCKER_USER,
             config.DOCKER_PASSWORD,
         )
-        try:
-            async with Docker() as docker_client:
-                await self._post_messages(
-                    LogType.LOG,
-                    f"[sidecar]Pulling {self.task.image['name']}:{self.task.image['tag']}...",
-                )
-                await docker_client.images.pull(
-                    docker_image,
-                    auth={
-                        "username": config.DOCKER_USER,
-                        "password": config.DOCKER_PASSWORD,
-                    },
-                )
 
-                # get integration version
-                image_cfg = await docker_client.images.inspect(docker_image)
-                # NOTE: old services did not have that label
-                if "io.simcore.integration-version" in image_cfg["Config"]["Labels"]:
-                    self.integration_version = version.parse(
-                        json.loads(
-                            image_cfg["Config"]["Labels"][
-                                "io.simcore.integration-version"
-                            ]
-                        )["integration-version"]
-                    )
+        async with Docker() as docker_client:
+            await self._post_messages(
+                LogType.LOG,
+                f"[sidecar]Pulling {self.task.image['name']}:{self.task.image['tag']}...",
+            )
+
+            await docker_client.images.pull(
+                docker_image,
+                auth={
+                    "username": config.DOCKER_USER,
+                    "password": config.DOCKER_PASSWORD,
+                },
+            )
+
+            # get integration version
+            image_cfg = await docker_client.images.inspect(docker_image)
+            # NOTE: old services did not have that label
+            if "io.simcore.integration-version" in image_cfg["Config"]["Labels"]:
+                self.integration_version = version.parse(
+                    json.loads(
+                        image_cfg["Config"]["Labels"][
+                            "io.simcore.integration-version"
+                        ]
+                    )["integration-version"]
+                )
 
-        except DockerError:
-            msg = f"Failed to pull image '{docker_image}'"
-            log.exception(msg)
-            raise
 
     async def _create_container_config(self, docker_image: str) -> Dict:
         # NOTE: Env/Binds for log folder is only necessary for integraion "0"

services/sidecar/src/simcore_service_sidecar/utils.py

@@ -98,8 +98,12 @@ async def async_is_gpu_node() -> bool:
                     config=spec_config, name=f"sidecar_{uuid.uuid4()}_test_gpu"
                 )
                 return True
-            except aiodocker.exceptions.DockerError as e:
-                logger.warning("is_gpu_node DockerError during check: %s", str(e))
+            except aiodocker.exceptions.DockerError as err:
+                logger.debug(
+                    "is_gpu_node DockerError while check-run %s: %s",
+                    spec_config,
+                    err
+                )
 
             return False