optional telemetry (#1092)

* Telemetry

* appdirs

* analytics tested

* remove analytics

* deterministic check

* remove fake code

* fix isCI

* cml

* integrated

* endpoint

* defensive

* json spec

* Telemetry (#1092) suggestion: don't repeat ourselves (#1094)

* Telemetry (#1092) suggestion: don't repeat ourselves

* Fix manual merging mistakes

* Obliterate the last process.exit on business logic

* dryer cml

* telemetrySend

* naming consistency

* read non json id

* gh check repoOptions

* one error handler

* fix tensorboard

* fix tensorboard and tests

* cleanup yargs fail handler

* unused

* RUNNER_SHUTTING_DOWN

* remove comments

* golantic

* getDriver

* fix telemetry

* fix getter

* remove report

* remove console

* fix os release

* windows release

Co-authored-by: Helio Machado <[email protected]>

Author: DavidGOrtega

Diff for: bin/cml.js

@@ -7,7 +7,39 @@ const which = require('which');
 const winston = require('winston');
 const yargs = require('yargs');
 
-const configureLogger = (level) => {
+const CML = require('../src/cml').default;
+const { jitsuEventPayload } = require('../src/analytics');
+let OPTS;
+
+const setupOpts = (opts) => {
+  const legacyEnvironmentVariables = {
+    TB_CREDENTIALS: 'CML_TENSORBOARD_DEV_CREDENTIALS',
+    DOCKER_MACHINE: 'CML_RUNNER_DOCKER_MACHINE',
+    RUNNER_IDLE_TIMEOUT: 'CML_RUNNER_IDLE_TIMEOUT',
+    RUNNER_LABELS: 'CML_RUNNER_LABELS',
+    RUNNER_SINGLE: 'CML_RUNNER_SINGLE',
+    RUNNER_REUSE: 'CML_RUNNER_REUSE',
+    RUNNER_NO_RETRY: 'CML_RUNNER_NO_RETRY',
+    RUNNER_DRIVER: 'CML_RUNNER_DRIVER',
+    RUNNER_REPO: 'CML_RUNNER_REPO',
+    RUNNER_PATH: 'CML_RUNNER_PATH'
+  };
+
+  for (const [oldName, newName] of Object.entries(legacyEnvironmentVariables)) {
+    if (process.env[oldName]) process.env[newName] = process.env[oldName];
+  }
+
+  const { markdownfile } = opts;
+  opts.markdownFile = markdownfile;
+  opts.cmlCommand = opts._[0];
+  opts.cml = new CML(opts);
+
+  OPTS = opts;
+};
+
+const setupLogger = (opts) => {
+  const { log: level } = opts;
+
   winston.configure({
     format: process.stdout.isTTY
       ? winston.format.combine(
@@ -29,59 +61,52 @@ const configureLogger = (level) => {
   });
 };
 
+const setupTelemetry = async (opts) => {
+  const { cml, cmlCommand: action } = opts;
+  opts.telemetryEvent = await jitsuEventPayload({ action, cml });
+};
+
 const runPlugin = async ({ $0: executable, command }) => {
-  try {
-    if (command === undefined) throw new Error('no command');
-    const path = which.sync(`${basename(executable)}-${command}`);
-    const parameters = process.argv.slice(process.argv.indexOf(command) + 1); // HACK
-    process.exit(await pseudoexec(path, parameters));
-  } catch (error) {
-    yargs.showHelp();
-    winston.debug(error);
-  }
+  if (command === undefined) throw new Error('no command');
+  const path = which.sync(`${basename(executable)}-${command}`);
+  const parameters = process.argv.slice(process.argv.indexOf(command) + 1); // HACK
+  await pseudoexec(path, parameters);
 };
 
-const handleError = (message, error) => {
+const handleError = async (message, error, yargs) => {
   if (error) {
-    winston.error(error);
-  } else {
-    yargs.showHelp();
-    console.error('\n' + message);
+    const { telemetryEvent, cml } = OPTS;
+    const event = { ...telemetryEvent, error: error.message };
+    await cml.telemetrySend({ event });
+    return;
   }
-  process.exit(1);
-};
 
-const options = {
-  log: {
-    type: 'string',
-    description: 'Maximum log level',
-    coerce: (value) => configureLogger(value) && value,
-    choices: ['error', 'warn', 'info', 'debug'],
-    default: 'info'
-  }
+  yargs.showHelp();
+  console.error('\n' + message);
 };
 
-const legacyEnvironmentVariables = {
-  TB_CREDENTIALS: 'CML_TENSORBOARD_DEV_CREDENTIALS',
-  DOCKER_MACHINE: 'CML_RUNNER_DOCKER_MACHINE',
-  RUNNER_IDLE_TIMEOUT: 'CML_RUNNER_IDLE_TIMEOUT',
-  RUNNER_LABELS: 'CML_RUNNER_LABELS',
-  RUNNER_SINGLE: 'CML_RUNNER_SINGLE',
-  RUNNER_REUSE: 'CML_RUNNER_REUSE',
-  RUNNER_NO_RETRY: 'CML_RUNNER_NO_RETRY',
-  RUNNER_DRIVER: 'CML_RUNNER_DRIVER',
-  RUNNER_REPO: 'CML_RUNNER_REPO',
-  RUNNER_PATH: 'CML_RUNNER_PATH'
-};
+process.on('uncaughtException', async (err) => {
+  await handleError('', err, yargs);
+});
 
-for (const [oldName, newName] of Object.entries(legacyEnvironmentVariables)) {
-  if (process.env[oldName]) process.env[newName] = process.env[oldName];
-}
+process.on('unhandledRejection', async (reason) => {
+  await handleError('', new Error(reason), yargs);
+});
 
 yargs
-  .fail(handleError)
   .env('CML')
-  .options(options)
+  .options({
+    log: {
+      type: 'string',
+      description: 'Maximum log level',
+      choices: ['error', 'warn', 'info', 'debug'],
+      default: 'info'
+    }
+  })
+  .fail(handleError)
+  .middleware(setupOpts)
+  .middleware(setupLogger)
+  .middleware(setupTelemetry)
   .commandDir('./cml', { exclude: /\.test\.js$/ })
   .command('$0 <command>', false, (builder) => builder.strict(false), runPlugin)
   .recommendCommands()

Diff for: bin/cml/ci.js

@@ -1,19 +1,20 @@
 const kebabcaseKeys = require('kebabcase-keys');
 
-const { GIT_USER_NAME, GIT_USER_EMAIL } = require('../../src/cml');
-const CML = require('../../src/cml').default;
+const { GIT_USER_NAME, GIT_USER_EMAIL, repoOptions } = require('../../src/cml');
 
 exports.command = 'ci';
 exports.description = 'Fixes specific CI setups';
 
 exports.handler = async (opts) => {
-  const cml = new CML(opts);
-  console.log((await cml.ci(opts)) || '');
+  const { cml, telemetryEvent: event } = opts;
+  await cml.ci(opts);
+  await cml.telemetrySend({ event });
 };
 
 exports.builder = (yargs) =>
   yargs.env('CML_CI').options(
     kebabcaseKeys({
+      ...repoOptions,
       unshallow: {
         type: 'boolean',
         description:
@@ -28,21 +29,6 @@ exports.builder = (yargs) =>
         type: 'string',
         default: GIT_USER_NAME,
         description: 'Set Git user name.'
-      },
-      repo: {
-        type: 'string',
-        description:
-          'Set repository to be used. If unspecified, inferred from the environment.'
-      },
-      token: {
-        type: 'string',
-        description:
-          'Personal access token to be used. If unspecified, inferred from the environment.'
-      },
-      driver: {
-        type: 'string',
-        choices: ['github', 'gitlab', 'bitbucket'],
-        description: 'If unspecified, inferred from the environment.'
       }
     })
   );

Diff for: bin/cml/ci.test.js

@@ -14,16 +14,16 @@ describe('CML e2e', () => {
         --version     Show version number                                    [boolean]
         --log         Maximum log level
                 [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"]
+        --repo        Specifies the repo to be used. If not specified is extracted
+                      from the CI ENV.                                        [string]
+        --token       Personal access token to be used. If not specified is extracted
+                      from ENV REPO_TOKEN.                                    [string]
+        --driver      If not specify it infers it from the ENV.
+                                   [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"]
         --unshallow   Fetch as much as possible, converting a shallow repository to a
                       complete one.                                          [boolean]
         --user-email  Set Git user email.    [string] [default: \\"[email protected]\\"]
-        --user-name   Set Git user name.             [string] [default: \\"Olivaw[bot]\\"]
-        --repo        Set repository to be used. If unspecified, inferred from the
-                      environment.                                            [string]
-        --token       Personal access token to be used. If unspecified, inferred from
-                      the environment.                                        [string]
-        --driver      If unspecified, inferred from the environment.
-                                   [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"]"
+        --user-name   Set Git user name.             [string] [default: \\"Olivaw[bot]\\"]"
     `);
   });
 });

Diff for: bin/cml/pr.js

@@ -1,20 +1,26 @@
 const kebabcaseKeys = require('kebabcase-keys');
 
-const { GIT_REMOTE, GIT_USER_NAME, GIT_USER_EMAIL } = require('../../src/cml');
-const CML = require('../../src/cml').default;
+const {
+  GIT_REMOTE,
+  GIT_USER_NAME,
+  GIT_USER_EMAIL,
+  repoOptions
+} = require('../../src/cml');
 
 exports.command = 'pr <glob path...>';
 exports.description = 'Create a pull request with the specified files';
 
 exports.handler = async (opts) => {
-  const cml = new CML(opts);
-  const link = await cml.prCreate({ ...opts, globs: opts.globpath });
-  if (link) console.log(link);
+  const { cml, telemetryEvent: event, globpath: globs } = opts;
+  const link = await cml.prCreate({ ...opts, globs });
+  console.log(link);
+  await cml.telemetrySend({ event });
 };
 
 exports.builder = (yargs) =>
   yargs.env('CML_PR').options(
     kebabcaseKeys({
+      ...repoOptions,
       md: {
         type: 'boolean',
         description: 'Output in markdown format [](url).'
@@ -69,21 +75,6 @@ exports.builder = (yargs) =>
         type: 'string',
         default: GIT_USER_NAME,
         description: 'Sets git user name.'
-      },
-      repo: {
-        type: 'string',
-        description:
-          'Specifies the repo to be used. If not specified is extracted from the CI ENV.'
-      },
-      token: {
-        type: 'string',
-        description:
-          'Personal access token to be used. If not specified in extracted from ENV REPO_TOKEN.'
-      },
-      driver: {
-        type: 'string',
-        choices: ['github', 'gitlab', 'bitbucket'],
-        description: 'If not specify it infers it from the ENV.'
       }
     })
   );

Diff for: bin/cml/pr.test.js

@@ -14,6 +14,12 @@ describe('CML e2e', () => {
         --version              Show version number                           [boolean]
         --log                  Maximum log level
                 [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"]
+        --repo                 Specifies the repo to be used. If not specified is
+                               extracted from the CI ENV.                     [string]
+        --token                Personal access token to be used. If not specified is
+                               extracted from ENV REPO_TOKEN.                 [string]
+        --driver               If not specify it infers it from the ENV.
+                                   [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"]
         --md                   Output in markdown format [](url).            [boolean]
         --skip-ci              Force skip CI for the created commit (if any) [boolean]
         --merge, --auto-merge  Try to merge the pull request upon creation.  [boolean]
@@ -28,13 +34,7 @@ describe('CML e2e', () => {
         --remote               Sets git remote.           [string] [default: \\"origin\\"]
         --user-email           Sets git user email.
                                              [string] [default: \\"[email protected]\\"]
-        --user-name            Sets git user name.   [string] [default: \\"Olivaw[bot]\\"]
-        --repo                 Specifies the repo to be used. If not specified is
-                               extracted from the CI ENV.                     [string]
-        --token                Personal access token to be used. If not specified in
-                               extracted from ENV REPO_TOKEN.                 [string]
-        --driver               If not specify it infers it from the ENV.
-                                   [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"]"
+        --user-name            Sets git user name.   [string] [default: \\"Olivaw[bot]\\"]"
     `);
   });
 });

Diff for: bin/cml/publish.js

@@ -2,7 +2,7 @@ const fs = require('fs').promises;
 const kebabcaseKeys = require('kebabcase-keys');
 const winston = require('winston');
 
-const CML = require('../../src/cml').default;
+const { CML, repoOptions } = require('../../src/cml');
 
 exports.command = 'publish <asset>';
 exports.description = 'Upload an image to build a report';
@@ -15,23 +15,20 @@ exports.handler = async (opts) => {
     opts.native = true;
   }
 
-  const { file, repo, native } = opts;
-
-  const path = opts.asset;
+  const { file, repo, native, asset: path, telemetryEvent: event } = opts;
   const cml = new CML({ ...opts, repo: native ? repo : 'cml' });
-
-  const output = await cml.publish({
-    ...opts,
-    path
-  });
+  const output = await cml.publish({ ...opts, path });
 
   if (!file) console.log(output);
   else await fs.writeFile(file, output);
+
+  await cml.telemetrySend({ event });
 };
 
 exports.builder = (yargs) =>
   yargs.env('CML_PUBLISH').options(
     kebabcaseKeys({
+      ...repoOptions,
       url: {
         type: 'string',
         description: 'Self-Hosted URL',
@@ -75,16 +72,6 @@ exports.builder = (yargs) =>
         type: 'string',
         description:
           'Specifies the repo to be used. If not specified is extracted from the CI ENV.'
-      },
-      token: {
-        type: 'string',
-        description:
-          'Personal access token to be used. If not specified, extracted from ENV REPO_TOKEN, GITLAB_TOKEN, GITHUB_TOKEN, or BITBUCKET_TOKEN.'
-      },
-      driver: {
-        type: 'string',
-        choices: ['github', 'gitlab', 'bitbucket'],
-        description: 'If not specify it infers it from the ENV.'
       }
     })
   );

Diff for: bin/cml/publish.test.js

@@ -15,20 +15,19 @@ describe('CML e2e', () => {
             --version       Show version number                              [boolean]
             --log           Maximum log level
                 [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"]
+            --repo          Specifies the repo to be used. If not specified is
+                            extracted from the CI ENV.                        [string]
+            --token         Personal access token to be used. If not specified is
+                            extracted from ENV REPO_TOKEN.                    [string]
+            --driver        If not specify it infers it from the ENV.
+                                   [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"]
             --md            Output in markdown format [title || name](url).  [boolean]
         -t, --title         Markdown title [title](url) or ![](url title).    [string]
             --native        Uses driver's native capabilities to upload assets instead
                             of CML's storage. Not available on GitHub.       [boolean]
             --rm-watermark  Avoid CML watermark.                             [boolean]
             --mime-type     Specifies the mime-type. If not set guess it from the
-                            content.                                          [string]
-            --repo          Specifies the repo to be used. If not specified is
-                            extracted from the CI ENV.                        [string]
-            --token         Personal access token to be used. If not specified,
-                            extracted from ENV REPO_TOKEN, GITLAB_TOKEN, GITHUB_TOKEN,
-                            or BITBUCKET_TOKEN.                               [string]
-            --driver        If not specify it infers it from the ENV.
-                                   [string] [choices: \\"github\\", \\"gitlab\\", \\"bitbucket\\"]"
+                            content.                                          [string]"
     `);
   });