Merge branch 'master' of https://github.com/iterative/cml into runner-faster-termination

Author: DavidGOrtega

bin/cml/runner.js

@@ -14,6 +14,7 @@ let RUNNER;
 let RUNNER_ID;
 let RUNNER_JOBS_RUNNING = [];
 let RUNNER_SHUTTING_DOWN = false;
+let RUNNER_TIMER = 0;
 const GH_5_MIN_TIMEOUT = (72 * 60 - 5) * 60 * 1000;
 
 const shutdown = async (opts) => {
@@ -236,6 +237,7 @@ const runLocal = async (opts) => {
       RUNNER_JOBS_RUNNING = RUNNER_JOBS_RUNNING.filter(
         (job) => job.id !== jobId
       );
+      RUNNER_TIMER = 0;
 
       if (single && cml.driver === 'bitbucket') {
         await shutdown({ ...opts, reason: 'single job' });
@@ -264,47 +266,55 @@ const runLocal = async (opts) => {
   RUNNER = proc;
   ({ id: RUNNER_ID } = await cml.runnerByName({ name }));
 
-  if (parseInt(idleTimeout) > 0) {
+  if (idleTimeout > 0) {
     const watcher = setInterval(async () => {
       let idle = RUNNER_JOBS_RUNNING.length === 0;
 
-      try {
-        if (cml.driver === 'github') {
-          const job = await cml.runnerJob({ runnerId: RUNNER_ID });
-
-          if (!job && !idle) {
-            winston.error(
-              `Runner is idle as per the GitHub API but busy as per CML internal state. Resetting jobs. Retrying in ${idleTimeout} seconds...`
-            );
-            winston.warn(`CML GitHub driver response: ${JSON.stringify(job)}`);
-            winston.warn(
-              `CML internal state: ${JSON.stringify(RUNNER_JOBS_RUNNING)}`
-            );
-
-            RUNNER_JOBS_RUNNING = [];
+      if (RUNNER_TIMER >= idleTimeout) {
+        try {
+          if (cml.driver === 'github') {
+            const job = await cml.runnerJob({ runnerId: RUNNER_ID });
+
+            if (!job && !idle) {
+              winston.error(
+                `Runner is idle as per the GitHub API but busy as per CML internal state. Resetting jobs. Retrying in ${idleTimeout} seconds...`
+              );
+              winston.warn(
+                `CML GitHub driver response: ${JSON.stringify(job)}`
+              );
+              winston.warn(
+                `CML internal state: ${JSON.stringify(RUNNER_JOBS_RUNNING)}`
+              );
+
+              RUNNER_JOBS_RUNNING = [];
+            }
+
+            if (job && idle) {
+              winston.error(
+                `Runner is busy as per the GitHub API but idle as per CML internal state. Retrying in ${idleTimeout} seconds...`
+              );
+
+              idle = false;
+            }
           }
+        } catch (err) {
+          winston.error(
+            `Error connecting the SCM: ${err.message}. Will try again in ${idleTimeout} secs`
+          );
 
-          if (job && idle) {
-            winston.error(
-              `Runner is busy as per the GitHub API but idle as per CML internal state. Retrying in ${idleTimeout} seconds...`
-            );
-
-            idle = false;
-          }
+          idle = false;
         }
-      } catch (err) {
-        winston.error(
-          `Error connecting the SCM: ${err.message}. Will try again in ${idleTimeout} secs`
-        );
 
-        idle = false;
+        if (idle) {
+          shutdown({ ...opts, reason: `timeout:${idleTimeout}` });
+          clearInterval(watcher);
+        } else {
+          RUNNER_TIMER = 0;
+        }
       }
 
-      if (idle) {
-        shutdown({ ...opts, reason: `timeout:${idleTimeout}` });
-        clearInterval(watcher);
-      }
-    }, idleTimeout * 1000);
+      RUNNER_TIMER++;
+    }, 1000);
   }
 
   if (!noRetry) {

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "@dvcorg/cml",
-  "version": "0.15.2",
+  "version": "0.16.0",
   "description": "<p align=\"center\"> <img src=\"https://static.iterative.ai/img/cml/title_strip_trim.png\" width=400> </p>",
   "author": {
     "name": "Iterative Inc",