Add RAM and threads options to init action

Author: cklin

.github/workflows/__extractor-ram-threads.yml

@@ -393,3 +393,42 @@ jobs:
           # Deliberately don't use TEST_MODE here. This is specifically testing
           # the compatibility with the API.
           runner/dist/codeql-runner-linux upload --sarif-file src/testdata/empty-sarif.sarif --repository $GITHUB_REPOSITORY --commit $GITHUB_SHA --ref $GITHUB_REF --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
+
+  runner-extractor-ram-threads-options:
+    name: Runner ubuntu extractor RAM and threads options
+    needs: [check-js, check-node-modules]
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Build runner
+        run: |
+          cd runner
+          npm install
+          npm run build-runner
+
+      - name: Run init
+        run: |
+          runner/dist/codeql-runner-linux init --ram=230 --threads=1 --repository $GITHUB_REPOSITORY --languages java --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
+
+      - name: Assert Results
+        shell: bash
+        run: |
+          . ./codeql-runner/codeql-env.sh
+          if [ "${CODEQL_RAM}" != "230" ]; then
+            echo "CODEQL_RAM is '${CODEQL_RAM}' instead of 230"
+            exit 1
+          fi
+          if [ "${CODEQL_EXTRACTOR_JAVA_RAM}" != "230" ]; then
+            echo "CODEQL_EXTRACTOR_JAVA_RAM is '${CODEQL_EXTRACTOR_JAVA_RAM}' instead of 230"
+            exit 1
+          fi
+          if [ "${CODEQL_THREADS}" != "1" ]; then
+            echo "CODEQL_THREADS is '${CODEQL_THREADS}' instead of 1"
+            exit 1
+          fi
+          if [ "${CODEQL_EXTRACTOR_JAVA_THREADS}" != "1" ]; then
+            echo "CODEQL_EXTRACTOR_JAVA_THREADS is '${CODEQL_EXTRACTOR_JAVA_THREADS}' instead of 1"
+            exit 1
+          fi

.github/workflows/pr-checks.yml

@@ -2,7 +2,7 @@
 
 ## [UNRELEASED]
 
-No user facing changes.
+- The `init` step of the Action now supports `ram` and `threads` inputs to limit resource use of CodeQL extractors. These inputs also serve as defaults to the subsequent `analyze` step, which finalizes the database and executes queries. [#738](https://github.com/github/codeql-action/pull/738)
 
 ## 1.0.21 - 28 Oct 2021
 

CHANGELOG.md

@@ -18,7 +18,12 @@ inputs:
     required: false
     default: "brutal"
   ram:
-    description: Override the amount of memory in MB to be used by CodeQL. By default, almost all the memory of the machine is used.
+    description: >-
+      The amount of memory in MB that can be used by CodeQL for database finalization and query execution.
+      By default, this action will use the same amount of memory as previously set in the "init" action.
+      If the "init" action also does not have an explicit "ram" input, this action will use most of the
+      memory available in the system (which for GitHub-hosted runners is 6GB for Linux, 5.5GB for Windows,
+      and 13GB for macOS).
     required: false
   add-snippets:
     description: Specify whether or not to add code snippets to the output sarif file.
@@ -29,7 +34,12 @@ inputs:
     required: false
     default: "false"
   threads:
-    description: The number of threads to be used by CodeQL.
+    description: >-
+      The number of threads that can be used by CodeQL for database finalization and query execution.
+      By default, this action will use the same number of threads as previously set in the "init" action.
+      If the "init" action also does not have an explicit "threads" input, this action will use all the
+      hardware threads available in the system (which for GitHub-hosted runners is 2 for Linux and Windows
+      and 3 for macOS).
     required: false
   checkout_path:
     description: "The path at which the analyzed repository was checked out. Used to relativize any absolute paths in the uploaded SARIF file."

analyze/action.yml

@@ -41,6 +41,20 @@ inputs:
   source-root:
     description: Path of the root source code directory, relative to $GITHUB_WORKSPACE.
     required: false
+  ram:
+    description: >-
+      The amount of memory in MB that can be used by CodeQL extractors.
+      By default, CodeQL extractors will use most of the memory available in the system
+      (which for GitHub-hosted runners is 6GB for Linux, 5.5GB for Windows, and 13GB for macOS).
+      This input also sets the amount of memory that can later be used by the "analyze" action.
+    required: false
+  threads:
+    description: >-
+      The number of threads that can be used by CodeQL extractors.
+      By default, CodeQL extractors will use all the hardware threads available in the system
+      (which for GitHub-hosted runners is 2 for Linux and Windows and 3 for macOS).
+      This input also sets the number of threads that can later be used by the "analyze" action.
+    required: false
 outputs:
   codeql-path:
     description: The path of the CodeQL binary used for analysis

init/action.yml

@@ -0,0 +1,29 @@
+name: "Extractor ram and threads options test"
+description: "Tests passing RAM and threads limits to extractors"
+versions: ["latest"]
+os: ["ubuntu-latest"]
+steps:
+  - uses: ./../action/init
+    with:
+      languages: java
+      ram: 230
+      threads: 1
+  - name: Assert Results
+    shell: bash
+    run: |
+      if [ "${CODEQL_RAM}" != "230" ]; then
+        echo "CODEQL_RAM is '${CODEQL_RAM}' instead of 230"
+        exit 1
+      fi
+      if [ "${CODEQL_EXTRACTOR_JAVA_RAM}" != "230" ]; then
+        echo "CODEQL_EXTRACTOR_JAVA_RAM is '${CODEQL_EXTRACTOR_JAVA_RAM}' instead of 230"
+        exit 1
+      fi
+      if [ "${CODEQL_THREADS}" != "1" ]; then
+        echo "CODEQL_THREADS is '${CODEQL_THREADS}' instead of 1"
+        exit 1
+      fi
+      if [ "${CODEQL_EXTRACTOR_JAVA_THREADS}" != "1" ]; then
+        echo "CODEQL_EXTRACTOR_JAVA_THREADS is '${CODEQL_EXTRACTOR_JAVA_THREADS}' instead of 1"
+        exit 1
+      fi

lib/analyze-action-env.test.js

@@ -47,5 +47,7 @@ predicate areNotEquivalent(YAMLValue x, YAMLValue y) {
 from ActionDeclaration actionA, ActionDeclaration actionB, string inputName
 where actionA.getName() < actionB.getName() // prevent duplicates which are permutations of the names
   and areNotEquivalent(actionA.getInput(inputName), actionB.getInput(inputName))
+  // ram and threads inputs in different actions are supposed to have different description
+  and inputName != "ram" and inputName != "threads"
 select actionA, "Action $@ and action $@ both declare input $@, however their definitions are not identical. This may be confusing to users.",
   actionA, actionA.getName(), actionB, actionB.getName(), inputName, inputName

lib/analyze-action-env.test.js.map

@@ -0,0 +1,58 @@
+import test from "ava";
+import * as sinon from "sinon";
+
+import * as actionsUtil from "./actions-util";
+import * as analyze from "./analyze";
+import * as configUtils from "./config-utils";
+import { setupTests, setupActionsVars } from "./testing-utils";
+import * as util from "./util";
+
+setupTests(test);
+
+// This test needs to be in its own file so that ava would run it in its own
+// nodejs process. The code being tested is in analyze-action.ts, which runs
+// immediately on load. So the file needs to be loaded during part of the test,
+// and that can happen only once per nodejs process. If multiple such tests are
+// in the same test file, ava would run them in the same nodejs process, and all
+// but the first test would fail.
+
+test("analyze action with RAM & threads from environment variables", async (t) => {
+  await util.withTmpDir(async (tmpDir) => {
+    process.env["GITHUB_SERVER_URL"] = "fake-server-url";
+    process.env["GITHUB_REPOSITORY"] = "fake/repository";
+    sinon
+      .stub(actionsUtil, "createStatusReportBase")
+      .resolves({} as actionsUtil.StatusReportBase);
+    sinon.stub(actionsUtil, "sendStatusReport").resolves(true);
+    sinon.stub(configUtils, "getConfig").resolves({
+      languages: [],
+    } as unknown as configUtils.Config);
+    const requiredInputStub = sinon.stub(actionsUtil, "getRequiredInput");
+    requiredInputStub.withArgs("token").returns("fake-token");
+    requiredInputStub.withArgs("upload-database").returns("false");
+    const optionalInputStub = sinon.stub(actionsUtil, "getOptionalInput");
+    optionalInputStub.withArgs("cleanup-level").returns("none");
+    setupActionsVars(tmpDir, tmpDir);
+
+    // When there are no action inputs for RAM and threads, the action uses
+    // environment variables (passed down from the init action) to set RAM and
+    // threads usage.
+    process.env["CODEQL_THREADS"] = "-1";
+    process.env["CODEQL_RAM"] = "4992";
+
+    const runFinalizeStub = sinon.stub(analyze, "runFinalize");
+    const runQueriesStub = sinon.stub(analyze, "runQueries");
+    const analyzeAction = require("./analyze-action");
+
+    // When analyze-action.ts loads, it runs an async function from the top
+    // level but does not wait for it to finish. To ensure that calls to
+    // runFinalize and runQueries are correctly captured by spies, we explicitly
+    // wait for the action promise to complete before starting verification.
+    await analyzeAction.runPromise;
+
+    t.deepEqual(runFinalizeStub.firstCall.args[1], "--threads=-1");
+    t.deepEqual(runFinalizeStub.firstCall.args[2], "--ram=4992");
+    t.deepEqual(runQueriesStub.firstCall.args[3], "--threads=-1");
+    t.deepEqual(runQueriesStub.firstCall.args[1], "--ram=4992");
+  });
+});

lib/analyze-action-input.test.js

@@ -0,0 +1,59 @@
+import test from "ava";
+import * as sinon from "sinon";
+
+import * as actionsUtil from "./actions-util";
+import * as analyze from "./analyze";
+import * as configUtils from "./config-utils";
+import { setupTests, setupActionsVars } from "./testing-utils";
+import * as util from "./util";
+
+setupTests(test);
+
+// This test needs to be in its own file so that ava would run it in its own
+// nodejs process. The code being tested is in analyze-action.ts, which runs
+// immediately on load. So the file needs to be loaded during part of the test,
+// and that can happen only once per nodejs process. If multiple such tests are
+// in the same test file, ava would run them in the same nodejs process, and all
+// but the first test would fail.
+
+test("analyze action with RAM & threads from action inputs", async (t) => {
+  await util.withTmpDir(async (tmpDir) => {
+    process.env["GITHUB_SERVER_URL"] = "fake-server-url";
+    process.env["GITHUB_REPOSITORY"] = "fake/repository";
+    sinon
+      .stub(actionsUtil, "createStatusReportBase")
+      .resolves({} as actionsUtil.StatusReportBase);
+    sinon.stub(actionsUtil, "sendStatusReport").resolves(true);
+    sinon.stub(configUtils, "getConfig").resolves({
+      languages: [],
+    } as unknown as configUtils.Config);
+    const requiredInputStub = sinon.stub(actionsUtil, "getRequiredInput");
+    requiredInputStub.withArgs("token").returns("fake-token");
+    requiredInputStub.withArgs("upload-database").returns("false");
+    const optionalInputStub = sinon.stub(actionsUtil, "getOptionalInput");
+    optionalInputStub.withArgs("cleanup-level").returns("none");
+    setupActionsVars(tmpDir, tmpDir);
+
+    process.env["CODEQL_THREADS"] = "1";
+    process.env["CODEQL_RAM"] = "4992";
+
+    // Action inputs have precedence over environment variables.
+    optionalInputStub.withArgs("threads").returns("-1");
+    optionalInputStub.withArgs("ram").returns("3012");
+
+    const runFinalizeStub = sinon.stub(analyze, "runFinalize");
+    const runQueriesStub = sinon.stub(analyze, "runQueries");
+    const analyzeAction = require("./analyze-action");
+
+    // When analyze-action.ts loads, it runs an async function from the top
+    // level but does not wait for it to finish. To ensure that calls to
+    // runFinalize and runQueries are correctly captured by spies, we explicitly
+    // wait for the action promise to complete before starting verification.
+    await analyzeAction.runPromise;
+
+    t.deepEqual(runFinalizeStub.firstCall.args[1], "--threads=-1");
+    t.deepEqual(runFinalizeStub.firstCall.args[2], "--ram=3012");
+    t.deepEqual(runQueriesStub.firstCall.args[3], "--threads=-1");
+    t.deepEqual(runQueriesStub.firstCall.args[1], "--ram=3012");
+  });
+});

lib/analyze-action-input.test.js.map

@@ -31,7 +31,7 @@ interface FinishStatusReport
   extends actionsUtil.StatusReportBase,
     AnalysisStatusReport {}
 
-async function sendStatusReport(
+export async function sendStatusReport(
   startedAt: Date,
   stats: AnalysisStatusReport | undefined,
   error?: Error
@@ -91,10 +91,12 @@ async function run() {
     };
     const outputDir = actionsUtil.getRequiredInput("output");
     const threads = util.getThreadsFlag(
-      actionsUtil.getOptionalInput("threads"),
+      actionsUtil.getOptionalInput("threads") || process.env["CODEQL_THREADS"],
       logger
     );
-    const memory = util.getMemoryFlag(actionsUtil.getOptionalInput("ram"));
+    const memory = util.getMemoryFlag(
+      actionsUtil.getOptionalInput("ram") || process.env["CODEQL_RAM"]
+    );
     await runFinalize(outputDir, threads, memory, config, logger);
     if (actionsUtil.getRequiredInput("skip-queries") !== "true") {
       runStats = await runQueries(
@@ -188,9 +190,11 @@ async function run() {
   }
 }
 
+export const runPromise = run();
+
 async function runWrapper() {
   try {
-    await run();
+    await runPromise;
   } catch (error) {
     core.setFailed(`analyze action failed: ${error}`);
     console.log(error);

lib/analyze-action.js

@@ -32,6 +32,8 @@ import {
   getGitHubVersion,
   codeQlVersionAbove,
   enrichEnvironment,
+  getMemoryFlagValue,
+  getThreadsFlagValue,
 } from "./util";
 
 // eslint-disable-next-line import/no-commonjs
@@ -204,9 +206,20 @@ async function run() {
       );
     }
 
-    // Setup CODEQL_RAM flag (todo improve this https://github.com/github/dsp-code-scanning/issues/935)
-    const codeqlRam = process.env["CODEQL_RAM"] || "6500";
-    core.exportVariable("CODEQL_RAM", codeqlRam);
+    // Limit RAM and threads for extractors. When running extractors, the CodeQL CLI obeys the
+    // CODEQL_RAM and CODEQL_THREADS environment variables to decide how much RAM and how many
+    // threads it would ask extractors to use. See help text for the "--ram" and "--threads"
+    // options at https://codeql.github.com/docs/codeql-cli/manual/database-trace-command/
+    // for details.
+    core.exportVariable(
+      "CODEQL_RAM",
+      process.env["CODEQL_RAM"] ||
+        getMemoryFlagValue(getOptionalInput("ram")).toString()
+    );
+    core.exportVariable(
+      "CODEQL_THREADS",
+      getThreadsFlagValue(getOptionalInput("threads"), logger).toString()
+    );
 
     const sourceRoot = path.resolve(
       getRequiredEnvParam("GITHUB_WORKSPACE"),

lib/analyze-action.js.map

@@ -25,6 +25,8 @@ import {
   Mode,
   codeQlVersionAbove,
   enrichEnvironment,
+  getMemoryFlagValue,
+  getThreadsFlagValue,
 } from "./util";
 
 // eslint-disable-next-line import/no-commonjs
@@ -53,11 +55,15 @@ function getToolsDir(userInput: string | undefined): string {
 
 const codeqlEnvJsonFilename = "codeql-env.json";
 
+function loadTracerEnvironment(config: Config): { [name: string]: string } {
+  const jsonEnvFile = path.join(config.tempDir, codeqlEnvJsonFilename);
+  return JSON.parse(fs.readFileSync(jsonEnvFile).toString("utf-8"));
+}
+
 // Imports the environment from codeqlEnvJsonFilename if not already present
 function importTracerEnvironment(config: Config) {
   if (!("ODASA_TRACER_CONFIGURATION" in process.env)) {
-    const jsonEnvFile = path.join(config.tempDir, codeqlEnvJsonFilename);
-    const env = JSON.parse(fs.readFileSync(jsonEnvFile).toString("utf-8"));
+    const env = loadTracerEnvironment(config);
     for (const key of Object.keys(env)) {
       process.env[key] = env[key];
     }
@@ -109,6 +115,8 @@ interface InitArgs {
   githubAuth: string;
   githubAuthStdin: boolean;
   debug: boolean;
+  ram: string | undefined;
+  threads: string | undefined;
 }
 
 program
@@ -167,6 +175,18 @@ program
     "--trace-process-level <number>",
     "(Advanced, windows-only) Inject a windows tracer of this process into a parent process <number> levels up."
   )
+  .option(
+    "--ram <number>",
+    "The amount of memory in MB that can be used by CodeQL extractors. " +
+      "By default, CodeQL extractors will use most of the memory available in the system. " +
+      'This input also sets the amount of memory that can later be used by the "analyze" command.'
+  )
+  .option(
+    "--threads <number>",
+    "The number of threads that can be used by CodeQL extractors. " +
+      "By default, CodeQL extractors will use all the hardware threads available in the system. " +
+      'This input also sets the number of threads that can later be used by the "analyze" command.'
+  )
   .action(async (cmd: InitArgs) => {
     const logger = getRunnerLogger(cmd.debug);
 
@@ -195,6 +215,17 @@ program
       const gitHubVersion = await getGitHubVersion(apiDetails);
       checkGitHubVersionInRange(gitHubVersion, logger, Mode.runner);
 
+      // Limit RAM and threads for extractors. When running extractors, the CodeQL CLI obeys the
+      // CODEQL_RAM and CODEQL_THREADS environment variables to decide how much RAM and how many
+      // threads it would ask extractors to use. See help text for the "--ram" and "--threads"
+      // options at https://codeql.github.com/docs/codeql-cli/manual/database-trace-command/
+      // for details.
+      process.env["CODEQL_RAM"] = getMemoryFlagValue(cmd.ram).toString();
+      process.env["CODEQL_THREADS"] = getThreadsFlagValue(
+        cmd.threads,
+        logger
+      ).toString();
+
       let codeql: CodeQL;
       if (cmd.codeqlPath !== undefined) {
         codeql = await getCodeQL(cmd.codeqlPath);
@@ -402,16 +433,21 @@ program
   )
   .option(
     "--ram <ram>",
-    "Amount of memory to use when running queries. Default is to use all available memory."
+    "The amount of memory in MB that can be used by CodeQL for database finalization and query execution. " +
+      'By default, this command will use the same amount of memory as previously set in the "init" command. ' +
+      'If the "init" command also does not have an explicit "ram" flag, this command will use most of the ' +
+      "memory available in the system."
   )
   .option(
     "--no-add-snippets",
     "Specify whether to include code snippets in the sarif output."
   )
   .option(
     "--threads <threads>",
-    "Number of threads to use when running queries. " +
-      "Default is to use all available cores."
+    "The number of threads that can be used by CodeQL for database finalization and query execution. " +
+      'By default, this command will use the same number of threads as previously set in the "init" command. ' +
+      'If the "init" command also does not have an explicit "threads" flag, this command will use all the ' +
+      "hardware threads available in the system."
   )
   .option(
     "--temp-dir <dir>",
@@ -447,8 +483,17 @@ program
 
       const outputDir =
         cmd.outputDir || path.join(config.tempDir, "codeql-sarif");
-      const threads = getThreadsFlag(cmd.threads, logger);
-      const memory = getMemoryFlag(cmd.ram);
+      let initEnv: { [name: string]: string } = {};
+      try {
+        initEnv = loadTracerEnvironment(config);
+      } catch (err) {
+        // The init command did not generate a tracer environment file
+      }
+      const threads = getThreadsFlag(
+        cmd.threads || initEnv["CODEQL_THREADS"],
+        logger
+      );
+      const memory = getMemoryFlag(cmd.ram || initEnv["CODEQL_RAM"]);
       await runFinalize(outputDir, threads, memory, config, logger);
       await runQueries(
         outputDir,

lib/init-action.js

@@ -85,13 +85,13 @@ function getSystemReservedMemoryMegaBytes(): number {
 }
 
 /**
- * Get the codeql `--ram` flag as configured by the `ram` input. If no value was
- * specified, the total available memory will be used minus a threshold
- * reserved for the OS.
+ * Get the value of the codeql `--ram` flag as configured by the `ram` input.
+ * If no value was specified, the total available memory will be used minus a
+ * threshold reserved for the OS.
  *
- * @returns string
+ * @returns {number} the amount of RAM to use, in megabytes
  */
-export function getMemoryFlag(userInput: string | undefined): string {
+export function getMemoryFlagValue(userInput: string | undefined): number {
   let memoryToUseMegaBytes: number;
   if (userInput) {
     memoryToUseMegaBytes = Number(userInput);
@@ -104,7 +104,18 @@ export function getMemoryFlag(userInput: string | undefined): string {
     const reservedMemoryMegaBytes = getSystemReservedMemoryMegaBytes();
     memoryToUseMegaBytes = totalMemoryMegaBytes - reservedMemoryMegaBytes;
   }
-  return `--ram=${Math.floor(memoryToUseMegaBytes)}`;
+  return Math.floor(memoryToUseMegaBytes);
+}
+
+/**
+ * Get the codeql `--ram` flag as configured by the `ram` input. If no value was
+ * specified, the total available memory will be used minus a threshold
+ * reserved for the OS.
+ *
+ * @returns string
+ */
+export function getMemoryFlag(userInput: string | undefined): string {
+  return `--ram=${getMemoryFlagValue(userInput)}`;
 }
 
 /**
@@ -123,17 +134,17 @@ export function getAddSnippetsFlag(
 }
 
 /**
- * Get the codeql `--threads` value specified for the `threads` input.
- * If no value was specified, all available threads will be used.
+ * Get the value of the codeql `--threads` flag specified for the `threads`
+ * input. If no value was specified, all available threads will be used.
  *
  * The value will be capped to the number of available CPUs.
  *
- * @returns string
+ * @returns {number}
  */
-export function getThreadsFlag(
+export function getThreadsFlagValue(
   userInput: string | undefined,
   logger: Logger
-): string {
+): number {
   let numThreads: number;
   const maxThreads = os.cpus().length;
   if (userInput) {
@@ -158,7 +169,22 @@ export function getThreadsFlag(
     // Default to using all threads
     numThreads = maxThreads;
   }
-  return `--threads=${numThreads}`;
+  return numThreads;
+}
+
+/**
+ * Get the codeql `--threads` flag specified for the `threads` input.
+ * If no value was specified, all available threads will be used.
+ *
+ * The value will be capped to the number of available CPUs.
+ *
+ * @returns string
+ */
+export function getThreadsFlag(
+  userInput: string | undefined,
+  logger: Logger
+): string {
+  return `--threads=${getThreadsFlagValue(userInput, logger)}`;
 }
 
 /**