Make use of multi-language and indirect tracing

Author: edoardopirovano

.github/workflows/pr-checks.yml

@@ -207,10 +207,9 @@ jobs:
 
       - name: Build code
         shell: powershell
-        # Note we want to make sure that the .win32env file is read correctly, so we unset the CODEQL_EXTRACTOR_CSHARP_ROOT from the .sh file.
         run: |
           cat ./codeql-runner/codeql-env.sh | Invoke-Expression
-          $Env:CODEQL_EXTRACTOR_CSHARP_ROOT = ""
+          $Env:CODEQL_EXTRACTOR_CSHARP_ROOT = "" # Unset an environment variable to make sure the tracer resists this
           & $Env:CODEQL_RUNNER dotnet build /p:UseSharedCompilation=false
 
       - name: Upload tracer logs

lib/codeql.js

@@ -12,7 +12,7 @@ import * as api from "./api-client";
 import { PackWithVersion } from "./config-utils";
 import * as defaults from "./defaults.json"; // Referenced from codeql-action-sync-tool!
 import { errorMatchers } from "./error-matcher";
-import { Language } from "./languages";
+import { isTracedLanguage, Language } from "./languages";
 import { Logger } from "./logging";
 import * as toolcache from "./toolcache";
 import { toolrunnerErrorCatcher } from "./toolrunner-error-catcher";
@@ -75,6 +75,16 @@ export interface CodeQL {
     language: Language,
     sourceRoot: string
   ): Promise<void>;
+  /**
+   * Run 'codeql database init --db-cluster'.
+   */
+  databaseInitCluster(
+    databasePath: string,
+    languages: Language[],
+    sourceRoot: string,
+    processName: string | undefined,
+    processLevel: number | undefined
+  ): Promise<void>;
   /**
    * Runs the autobuilder for the given language.
    */
@@ -198,6 +208,7 @@ const CODEQL_VERSION_DIAGNOSTICS = "2.5.6";
 const CODEQL_VERSION_METRICS = "2.5.5";
 const CODEQL_VERSION_GROUP_RULES = "2.5.5";
 const CODEQL_VERSION_SARIF_GROUP = "2.5.3";
+export const CODEQL_VERSION_NEW_TRACING = "2.6.0"; // Use multi-language (>= 2.5.6) and indirect (>= 2.6.0) tracing.
 
 function getCodeQLBundleName(): string {
   let platform: string;
@@ -528,6 +539,7 @@ export function setCodeQL(partialCodeql: Partial<CodeQL>): CodeQL {
     printVersion: resolveFunction(partialCodeql, "printVersion"),
     getTracerEnv: resolveFunction(partialCodeql, "getTracerEnv"),
     databaseInit: resolveFunction(partialCodeql, "databaseInit"),
+    databaseInitCluster: resolveFunction(partialCodeql, "databaseInitCluster"),
     runAutobuild: resolveFunction(partialCodeql, "runAutobuild"),
     extractScannedLanguage: resolveFunction(
       partialCodeql,
@@ -643,6 +655,32 @@ async function getCodeQLForCmd(
         ...getExtraOptionsFromEnv(["database", "init"]),
       ]);
     },
+    async databaseInitCluster(
+      databasePath: string,
+      languages: Language[],
+      sourceRoot: string,
+      processName: string | undefined,
+      processLevel: number | undefined
+    ) {
+      const extraArgs = languages.map((language) => `--language=${language}`);
+      if (languages.filter(isTracedLanguage).length > 0) {
+        extraArgs.push("--begin-tracing");
+        if (processName !== undefined) {
+          extraArgs.push(`--trace-process-name=${processName}`);
+        } else {
+          extraArgs.push(`--trace-process-level=${processLevel || 3}`);
+        }
+      }
+      await runTool(cmd, [
+        "database",
+        "init",
+        "--db-cluster",
+        databasePath,
+        `--source-root=${sourceRoot}`,
+        ...extraArgs,
+        ...getExtraOptionsFromEnv(["database", "init"]),
+      ]);
+    },
     async runAutobuild(language: Language) {
       const cmdName =
         process.platform === "win32" ? "autobuild.cmd" : "autobuild.sh";

lib/codeql.js.map

@@ -12,7 +12,7 @@ import {
   StatusReportBase,
   validateWorkflow,
 } from "./actions-util";
-import { CodeQL } from "./codeql";
+import { CodeQL, CODEQL_VERSION_NEW_TRACING } from "./codeql";
 import * as configUtils from "./config-utils";
 import {
   initCodeQL,
@@ -30,6 +30,7 @@ import {
   Mode,
   checkGitHubVersionInRange,
   getGitHubVersion,
+  codeQlVersionAbove,
 } from "./util";
 
 // eslint-disable-next-line import/no-commonjs
@@ -210,13 +211,22 @@ async function run() {
       getOptionalInput("source-root") || ""
     );
 
-    const tracerConfig = await runInit(codeql, config, sourceRoot);
+    const tracerConfig = await runInit(
+      codeql,
+      config,
+      sourceRoot,
+      "Runner.Worker.exe",
+      undefined
+    );
     if (tracerConfig !== undefined) {
       for (const [key, value] of Object.entries(tracerConfig.env)) {
         core.exportVariable(key, value);
       }
 
-      if (process.platform === "win32") {
+      if (
+        process.platform === "win32" &&
+        !(await codeQlVersionAbove(codeql, CODEQL_VERSION_NEW_TRACING))
+      ) {
         await injectWindowsTracer(
           "Runner.Worker.exe",
           undefined,

lib/init-action.js

@@ -6,12 +6,13 @@ import * as safeWhich from "@chrisgavin/safe-which";
 
 import * as analysisPaths from "./analysis-paths";
 import { GitHubApiCombinedDetails, GitHubApiDetails } from "./api-client";
-import { CodeQL, setupCodeQL } from "./codeql";
+import { CodeQL, CODEQL_VERSION_NEW_TRACING, setupCodeQL } from "./codeql";
 import * as configUtils from "./config-utils";
 import { Logger } from "./logging";
 import { RepositoryNwo } from "./repository";
 import { TracerConfig, getCombinedTracerConfig } from "./tracer-config";
 import * as util from "./util";
+import { codeQlVersionAbove } from "./util";
 
 export async function initCodeQL(
   codeqlURL: string | undefined,
@@ -75,18 +76,30 @@ export async function initConfig(
 export async function runInit(
   codeql: CodeQL,
   config: configUtils.Config,
-  sourceRoot: string
+  sourceRoot: string,
+  processName: string | undefined,
+  processLevel: number | undefined
 ): Promise<TracerConfig | undefined> {
   fs.mkdirSync(config.dbLocation, { recursive: true });
 
-  // TODO: replace this code once CodeQL supports multi-language tracing
-  for (const language of config.languages) {
-    // Init language database
-    await codeql.databaseInit(
-      util.getCodeQLDatabasePath(config, language),
-      language,
-      sourceRoot
+  if (await codeQlVersionAbove(codeql, CODEQL_VERSION_NEW_TRACING)) {
+    // Init a database cluster
+    await codeql.databaseInitCluster(
+      config.dbLocation,
+      config.languages,
+      sourceRoot,
+      processName,
+      processLevel
     );
+  } else {
+    for (const language of config.languages) {
+      // Init language database
+      await codeql.databaseInit(
+        util.getCodeQLDatabasePath(config, language),
+        language,
+        sourceRoot
+      );
+    }
   }
 
   return await getCombinedTracerConfig(config, codeql);

lib/init-action.js.map

@@ -6,7 +6,7 @@ import { Command } from "commander";
 
 import { runFinalize, runQueries } from "./analyze";
 import { determineAutobuildLanguage, runAutobuild } from "./autobuild";
-import { CodeQL, getCodeQL } from "./codeql";
+import { CodeQL, CODEQL_VERSION_NEW_TRACING, getCodeQL } from "./codeql";
 import { Config, getConfig } from "./config-utils";
 import { initCodeQL, initConfig, injectWindowsTracer, runInit } from "./init";
 import { Language, parseLanguage } from "./languages";
@@ -23,6 +23,7 @@ import {
   getGitHubAuth,
   initializeEnvironment,
   Mode,
+  codeQlVersionAbove,
 } from "./util";
 
 // eslint-disable-next-line import/no-commonjs
@@ -226,12 +227,21 @@ program
       );
 
       const sourceRoot = checkoutPath;
-      const tracerConfig = await runInit(codeql, config, sourceRoot);
+      const tracerConfig = await runInit(
+        codeql,
+        config,
+        sourceRoot,
+        parseTraceProcessName(),
+        parseTraceProcessLevel()
+      );
       if (tracerConfig === undefined) {
         return;
       }
 
-      if (process.platform === "win32") {
+      if (
+        process.platform === "win32" &&
+        !(await codeQlVersionAbove(codeql, CODEQL_VERSION_NEW_TRACING))
+      ) {
         await injectWindowsTracer(
           parseTraceProcessName(),
           parseTraceProcessLevel(),

lib/init.js

@@ -1,10 +1,11 @@
 import * as fs from "fs";
 import * as path from "path";
 
-import { CodeQL } from "./codeql";
+import { CodeQL, CODEQL_VERSION_NEW_TRACING } from "./codeql";
 import * as configUtils from "./config-utils";
 import { Language, isTracedLanguage } from "./languages";
 import * as util from "./util";
+import { codeQlVersionAbove } from "./util";
 
 export type TracerConfig = {
   spec: string;
@@ -19,6 +20,24 @@ const CRITICAL_TRACER_VARS = new Set([
   "SEMMLE_JAVA_TOOL_OPTIONS",
 ]);
 
+export async function getTracerConfigForCluster(
+  config: configUtils.Config
+): Promise<TracerConfig> {
+  const tracingEnvVariables = JSON.parse(
+    fs.readFileSync(
+      path.resolve(
+        config.dbLocation,
+        "temp/tracingEnvironment/start-tracing.json"
+      ),
+      "utf8"
+    )
+  );
+  return {
+    spec: tracingEnvVariables["ODASA_TRACER_CONFIGURATION"],
+    env: tracingEnvVariables,
+  };
+}
+
 export async function getTracerConfigForLanguage(
   codeql: CodeQL,
   config: configUtils.Config,
@@ -179,16 +198,21 @@ export async function getCombinedTracerConfig(
     return undefined;
   }
 
-  // Get all the tracer configs and combine them together
-  const tracedLanguageConfigs: { [lang: string]: TracerConfig } = {};
-  for (const language of tracedLanguages) {
-    tracedLanguageConfigs[language] = await getTracerConfigForLanguage(
-      codeql,
-      config,
-      language
-    );
+  let mainTracerConfig: TracerConfig;
+  if (await codeQlVersionAbove(codeql, CODEQL_VERSION_NEW_TRACING)) {
+    mainTracerConfig = await getTracerConfigForCluster(config);
+  } else {
+    // Get all the tracer configs and combine them together
+    const tracedLanguageConfigs: { [lang: string]: TracerConfig } = {};
+    for (const language of tracedLanguages) {
+      tracedLanguageConfigs[language] = await getTracerConfigForLanguage(
+        codeql,
+        config,
+        language
+      );
+    }
+    mainTracerConfig = concatTracerConfigs(tracedLanguageConfigs, config);
   }
-  const mainTracerConfig = concatTracerConfigs(tracedLanguageConfigs, config);
 
   // Add a couple more variables
   mainTracerConfig.env["ODASA_TRACER_CONFIGURATION"] = mainTracerConfig.spec;

lib/init.js.map

@@ -422,19 +422,6 @@ enum EnvVar {
    * not the cli.
    */
   FEATURE_WILL_UPLOAD = "CODEQL_ACTION_FEATURE_WILL_UPLOAD",
-
-  /**
-   * If set to the "true" string, then the codeql-action is using its
-   * own deprecated and non-standard way of scanning for multiple
-   * languages.
-   */
-  FEATURE_MULTI_LANGUAGE = "CODEQL_ACTION_FEATURE_MULTI_LANGUAGE",
-
-  /**
-   * If set to the "true" string, then the codeql-action is using its
-   * own sandwiched workflow mechanism
-   */
-  FEATURE_SANDWICH = "CODEQL_ACTION_FEATURE_SANDWICH",
 }
 
 export function initializeEnvironment(mode: Mode, version: string) {
@@ -450,8 +437,6 @@ export function initializeEnvironment(mode: Mode, version: string) {
   exportVar(EnvVar.VERSION, version);
   exportVar(EnvVar.FEATURE_SARIF_COMBINE, "true");
   exportVar(EnvVar.FEATURE_WILL_UPLOAD, "true");
-  exportVar(EnvVar.FEATURE_MULTI_LANGUAGE, "true");
-  exportVar(EnvVar.FEATURE_SANDWICH, "true");
 }
 
 export function getMode(): Mode {