Merge pull request #1004 from marcnjaramillo/fix-large-sarif-handling

Fix large sarif handling

Author: aeisenberg

extensions/ql-vscode/CHANGELOG.md

@@ -2,7 +2,8 @@
 
 ## [UNRELEASED]
 
-- Fix the _CodeQL: Open Referenced File_ command for Windows systems. [#979](https://github.com/github/vscode-codeql/pull/979)
+- Fix the _CodeQL: Open Referenced File_ command for Windows systems. [#979](https://github.com/github/vscode-codeql/pull/979) 
+- Support large SARIF results files (>4GB) without crashing VS Code. [#1004](https://github.com/github/vscode-codeql/pull/1004)
 - Fix a bug that shows 'Set current database' when hovering over the currently selected database in the databases view. [#976](https://github.com/github/vscode-codeql/pull/976)
 - Fix a bug with importing large databases. Databases over 4GB can now be imported directly from LGTM or from a zip file. This functionality is only available when using CodeQL CLI version 2.6.0 or later. [#971](https://github.com/github/vscode-codeql/pull/971)
 - Replace certain control codes (`U+0000` - `U+001F`) with their corresponding control labels (`U+2400` - `U+241F`) in the results view. [#963](https://github.com/github/vscode-codeql/pull/963)

extensions/ql-vscode/package-lock.json

@@ -992,6 +992,9 @@
     "react": "^16.8.6",
     "react-dom": "^16.8.6",
     "semver": "~7.3.2",
+    "stream": "^0.0.2",
+    "stream-chain": "~2.2.4",
+    "stream-json": "~1.7.3",
     "tmp": "^0.1.0",
     "tmp-promise": "~3.0.2",
     "tree-kill": "~1.2.2",
@@ -1026,6 +1029,8 @@
     "@types/semver": "~7.2.0",
     "@types/sinon": "~7.5.2",
     "@types/sinon-chai": "~3.2.3",
+    "@types/stream-chain": "~2.0.1",
+    "@types/stream-json": "~1.7.1",
     "@types/through2": "^2.0.36",
     "@types/tmp": "^0.1.0",
     "@types/unzipper": "~0.10.1",

extensions/ql-vscode/package.json

@@ -1,6 +1,5 @@
 import * as cpp from 'child-process-promise';
 import * as child_process from 'child_process';
-import * as fs from 'fs-extra';
 import * as path from 'path';
 import * as sarif from 'sarif';
 import { SemVer } from 'semver';
@@ -17,6 +16,7 @@ import { assertNever } from './pure/helpers-pure';
 import { QueryMetadata, SortDirection } from './pure/interface-types';
 import { Logger, ProgressReporter } from './logging';
 import { CompilationMessage } from './pure/messages';
+import { sarifParser } from './sarif-parser';
 import { dbSchemeToLanguage } from './helpers';
 
 /**
@@ -696,22 +696,7 @@ export class CodeQLCliServer implements Disposable {
 
   async interpretBqrs(metadata: QueryMetadata, resultsPath: string, interpretedResultsPath: string, sourceInfo?: SourceInfo): Promise<sarif.Log> {
     await this.runInterpretCommand(SARIF_FORMAT, metadata, resultsPath, interpretedResultsPath, sourceInfo);
-
-    let output: string;
-    try {
-      output = await fs.readFile(interpretedResultsPath, 'utf8');
-    } catch (e) {
-      const rawMessage = e.stderr || e.message;
-      const errorMessage = rawMessage.startsWith('Cannot create a string')
-        ? `SARIF too large. ${rawMessage}`
-        : rawMessage;
-      throw new Error(`Reading output of interpretation failed: ${errorMessage}`);
-    }
-    try {
-      return JSON.parse(output) as sarif.Log;
-    } catch (err) {
-      throw new Error(`Parsing output of interpretation failed: ${err.stderr || err}`);
-    }
+    return await sarifParser(interpretedResultsPath);
   }
 
   async generateResultsCsv(metadata: QueryMetadata, resultsPath: string, csvPath: string, sourceInfo?: SourceInfo): Promise<void> {
@@ -1157,7 +1142,7 @@ export class CliVersionConstraint {
 
   /**
    * CLI version where database registration was introduced
-   */
+  */
   public static CLI_VERSION_WITH_DB_REGISTRATION = new SemVer('2.4.1');
 
   /**

extensions/ql-vscode/src/cli.ts

@@ -0,0 +1,48 @@
+import * as Sarif from 'sarif';
+import * as fs from 'fs-extra';
+import { parser } from 'stream-json';
+import { pick } from 'stream-json/filters/Pick';
+import Assembler = require('stream-json/Assembler');
+import { chain } from 'stream-chain';
+
+const DUMMY_TOOL : Sarif.Tool = {driver: {name: ''}};
+
+export async function sarifParser(interpretedResultsPath: string) : Promise<Sarif.Log> {
+  try {
+    // Parse the SARIF file into token streams, filtering out only the results array.
+    const p = parser();
+    const pipeline = chain([
+      fs.createReadStream(interpretedResultsPath),
+      p,
+      pick({filter: 'runs.0.results'})
+    ]);
+
+    // Creates JavaScript objects from the token stream
+    const asm = Assembler.connectTo(pipeline);
+
+    // Returns a constructed Log object with the results or an empty array if no results were found.
+    // If the parser fails for any reason, it will reject the promise.
+    return await new Promise((resolve, reject) => {
+      pipeline.on('error', (error) => {
+        reject(error);
+      });
+      
+      asm.on('done', (asm) => {
+
+        const log : Sarif.Log = {
+          version: '2.1.0', 
+          runs: [
+            { 
+              tool: DUMMY_TOOL, 
+              results: asm.current ?? []
+            }
+          ]
+        };
+        
+        resolve(log);
+      });
+    });
+  } catch (err) {
+    throw new Error(`Parsing output of interpretation failed: ${err.stderr || err}`);
+  }
+}

extensions/ql-vscode/src/sarif-parser.ts

@@ -0,0 +1,34 @@
+{
+  "version": "2.1.0",
+  "$schema": "http://json.schemastore.org/sarif-2.1.0-rtm.4",
+  "runs": [
+    {
+      "tool": {
+        "driver": {
+          "name": "ESLint",
+          "informationUri": "https://eslint.org",
+          "rules": [
+            {
+              "id": "no-unused-vars",
+              "shortDescription": {
+                "text": "disallow unused variables"
+              },
+              "helpUri": "https://eslint.org/docs/rules/no-unused-vars",
+              "properties": {
+                "category": "Variables"
+              }
+            }
+          ]
+        }
+      },
+      "artifacts": [
+        {
+          "location": {
+            "uri": "file:///C:/dev/sarif/sarif-tutorials/samples/Introduction/simple-example.js"
+          }
+        }
+      ],
+      "results": []
+    }
+  ]
+}

extensions/ql-vscode/src/vscode-tests/no-workspace/data/sarif/emptyResultsSarif.sarif

@@ -0,0 +1,33 @@
+{
+  "version": "2.1.0",
+  "$schema": "http://json.schemastore.org/sarif-2.1.0-rtm.4",
+  "runs": [
+    {
+      "tool": {
+        "driver": {
+          "name": "ESLint",
+          "informationUri": "https://eslint.org",
+          "rules": [
+            {
+              "id": "no-unused-vars",
+              "shortDescription": {
+                "text": "disallow unused variables"
+              },
+              "helpUri": "https://eslint.org/docs/rules/no-unused-vars",
+              "properties": {
+                "category": "Variables"
+              }
+            }
+          ]
+        }
+      },
+      "artifacts": [
+        {
+          "location": {
+            "uri": "file:///C:/dev/sarif/sarif-tutorials/samples/Introduction/simple-example.js"
+          }
+        }
+      ]
+    }
+  ]
+}