Implement source maps for optimized builds.

Author: int3

emcc

@@ -1477,6 +1477,7 @@ try:
     open(final, 'w').write(src)
     if DEBUG: save_intermediate('bind')
 
+  # TODO: support source maps with js_transform
   # Apply a source code transformation, if requested
   if js_transform:
     shutil.copyfile(final, final + '.tr.js')
@@ -1486,6 +1487,8 @@ try:
     execute(shlex.split(js_transform, posix=posix) + [os.path.abspath(final)])
     if DEBUG: save_intermediate('transformed')
 
+  js_transform_tempfiles = [final]
+
   # It is useful to run several js optimizer passes together, to save on unneeded unparsing/reparsing
   js_optimizer_queue = []
   def flush_js_optimizer_queue():
@@ -1497,6 +1500,7 @@ try:
         logging.debug('applying js optimization passes: %s', js_optimizer_queue)
         final = shared.Building.js_optimizer(final, js_optimizer_queue, jcache,
             keep_llvm_debug and keep_js_debug)
+        js_transform_tempfiles.append(final)
         if DEBUG: save_intermediate('js_opts')
       else:
         for name in js_optimizer_queue:
@@ -1506,6 +1510,7 @@ try:
           logging.debug('applying js optimization pass: %s', passes)
           final = shared.Building.js_optimizer(final, passes, jcache,
             keep_llvm_debug and keep_js_debug)
+          js_transform_tempfiles.append(final)
           save_intermediate(name)
       js_optimizer_queue = []
 
@@ -1514,7 +1519,8 @@ try:
 
     if DEBUG == '2':
       # Clean up the syntax a bit
-      final = shared.Building.js_optimizer(final, [], jcache)
+      final = shared.Building.js_optimizer(final, [], jcache,
+            keep_llvm_debug and keep_js_debug)
       if DEBUG: save_intermediate('pretty')
 
     def get_eliminate():
@@ -1532,6 +1538,8 @@ try:
     flush_js_optimizer_queue()
 
     logging.debug('running closure')
+    # no need to add this to js_transform_tempfiles, because closure and
+    # keep_js_debug are never simultaneously true
     final = shared.Building.closure_compiler(final)
     if DEBUG: save_intermediate('closure')
 
@@ -1579,16 +1587,20 @@ try:
       src = re.sub('/\* memory initializer \*/ allocate\(([\d,\.concat\(\)\[\]\\n ]+)"i8", ALLOC_NONE, Runtime\.GLOBAL_BASE\)', repl, src, count=1)
       open(final + '.mem.js', 'w').write(src)
       final += '.mem.js'
+      js_transform_tempfiles[-1] = final # simple text substitution preserves comment line number mappings
       if DEBUG:
         if os.path.exists(memfile):
           save_intermediate('meminit')
           logging.debug('wrote memory initialization to %s' % memfile)
         else:
           logging.debug('did not see memory initialization')
 
-  def generate_source_map(filename, map_file_base_name, offset=0):
+  def generate_source_map(map_file_base_name, offset=0):
     jsrun.run_js(shared.path_from_root('tools', 'source-maps', 'sourcemapper.js'),
-      shared.NODE_JS, [filename, os.getcwd(), map_file_base_name, str(offset)])
+      shared.NODE_JS, js_transform_tempfiles +
+        ['--sourceRoot', os.getcwd(),
+         '--mapFileBaseName', map_file_base_name,
+         '--offset', str(offset)])
 
   # If we were asked to also generate HTML, do that
   if final_suffix == 'html':
@@ -1601,7 +1613,7 @@ try:
             re.DOTALL)
         if match is None:
           raise RuntimeError('Could not find script insertion point')
-        generate_source_map(final, target, match.group().count('\n'))
+        generate_source_map(target, match.group().count('\n'))
       html.write(shell.replace('{{{ SCRIPT_CODE }}}', open(final).read()))
     else:
       # Compress the main code
@@ -1672,7 +1684,7 @@ try:
         from tools.split import split_javascript_file
         split_javascript_file(final, unsuffixed(target), split_js_file)
     else:
-        if keep_llvm_debug and keep_js_debug: generate_source_map(final, target)
+        if keep_llvm_debug and keep_js_debug: generate_source_map(target)
 
         # copy final JS to output
         shutil.move(final, target)

tests/runner.py

@@ -9585,21 +9585,21 @@ def process(filename):
         assert 'Assertion failed' in str(e), str(e)
 
     def test_source_map(self):
+      if Settings.USE_TYPED_ARRAYS != 2: return self.skip("doesn't pass without typed arrays")
       if '-g' not in Building.COMPILER_TEST_OPTS: Building.COMPILER_TEST_OPTS.append('-g')
-      if self.emcc_args is not None:
-        if '-O1' in self.emcc_args or '-O2' in self.emcc_args: return self.skip('optimizations remove LLVM debug info')
 
       src = '''
         #include <stdio.h>
         #include <assert.h>
 
-        int foo() {
-          return 1; // line 6
+        __attribute__((noinline)) int foo() {
+          printf("hi"); // line 6
+          return 1; // line 7
         }
 
         int main() {
-          int i = foo(); // line 10
-          return 0; // line 11
+          printf("%d", foo()); // line 11
+          return 0; // line 12
         }
       '''
 
@@ -9621,7 +9621,7 @@ def post(filename):
           self.assertIdentical(src_filename, m['source'])
           seen_lines.add(m['originalLine'])
         # ensure that all the 'meaningful' lines in the original code get mapped
-        assert seen_lines.issuperset([6, 10, 11])
+        assert seen_lines.issuperset([6, 7, 11, 12])
 
       self.build(src, dirname, src_filename, post_build=(None,post))
 

tools/eliminator/node_modules/uglify-js/lib/process.js

@@ -2,6 +2,9 @@
 
 "use strict";
 
+var fs = require('fs');
+var path = require('path');
+
 function countLines(s) {
   var count = 0;
   for (var i = 0, l = s.length; i < l; i ++) {
@@ -63,21 +66,39 @@ function extractComments(source, commentHandler) {
   }
 }
 
-function generateMap(fileName, sourceRoot, mapFileBaseName, generatedLineOffset) {
-  var fs = require('fs');
-  var path = require('path');
+function getMappings(source) {
+  // generatedLineNumber -> { originalLineNumber, originalFileName }
+  var mappings = {};
+  extractComments(source, function(content, generatedLineNumber) {
+    var matches = /@line (\d+)(?: "([^"]*)")?/.exec(content);
+    if (matches === null) return;
+    var originalFileName = matches[2];
+    mappings[generatedLineNumber] = {
+      originalLineNumber: parseInt(matches[1], 10),
+      originalFileName: originalFileName
+    }
+  });
+  return mappings;
+}
+
+function generateMap(mappings, sourceRoot, mapFileBaseName, generatedLineOffset) {
   var SourceMapGenerator = require('source-map').SourceMapGenerator;
 
   var generator = new SourceMapGenerator({ file: mapFileBaseName });
-  var generatedSource = fs.readFileSync(fileName, 'utf-8');
   var seenFiles = Object.create(null);
 
-  extractComments(generatedSource, function(content, generatedLineNumber) {
-    var matches = /@line (\d+) "([^"]*)"/.exec(content);
-    if (matches === null) return;
-    var originalLineNumber = parseInt(matches[1], 10);
-    var originalFileName = matches[2];
+  for (var generatedLineNumber in mappings) {
+    var generatedLineNumber = parseInt(generatedLineNumber, 10);
+    var mapping = mappings[generatedLineNumber];
+    var originalFileName = mapping.originalFileName;
+    generator.addMapping({
+      generated: { line: generatedLineNumber + generatedLineOffset, column: 0 },
+      original: { line: mapping.originalLineNumber, column: 0 },
+      source: originalFileName
+    });
 
+    // we could call setSourceContent repeatedly, but readFileSync is slow, so
+    // avoid doing it unnecessarily
     if (!(originalFileName in seenFiles)) {
       seenFiles[originalFileName] = true;
       var rootedPath = originalFileName[0] === path.sep ?
@@ -89,33 +110,64 @@ function generateMap(fileName, sourceRoot, mapFileBaseName, generatedLineOffset)
           " at " + rootedPath);
       }
     }
+  }
 
-    generator.addMapping({
-      generated: { line: generatedLineNumber + generatedLineOffset, column: 0 },
-      original: { line: originalLineNumber, column: 0 },
-      source: originalFileName
-    });
-  });
-
-  var mapFileName = mapFileBaseName + '.map';
-  fs.writeFileSync(mapFileName, generator.toString());
+  fs.writeFileSync(mapFileBaseName + '.map', generator.toString());
+}
 
-  var lastLine = generatedSource.slice(generatedSource.lastIndexOf('\n'));
+function appendMappingURL(fileName, source, mapFileName) {
+  var lastLine = source.slice(source.lastIndexOf('\n'));
   if (!/sourceMappingURL/.test(lastLine))
     fs.appendFileSync(fileName, '//@ sourceMappingURL=' + path.basename(mapFileName));
 }
 
+function parseArgs(args) {
+  var rv = { _: [] }; // unflagged args go into `_`; similar to the optimist library
+  for (var i = 0; i < args.length; i++) {
+    if (/^--/.test(args[i])) rv[args[i].slice(2)] = args[++i];
+    else rv._.push(args[i]);
+  }
+  return rv;
+}
+
 if (require.main === module) {
   if (process.argv.length < 3) {
-    console.log('Usage: ./sourcemapper.js <filename> <source root (default: .)> ' +
-                '<map file basename (default: filename)>' +
-                '<generated line offset (default: 0)>');
+    console.log('Usage: ./sourcemapper.js <original js> <optimized js file ...> \\\n' +
+                '\t--sourceRoot <default "."> \\\n' +
+                '\t--mapFileBaseName <default `filename`> \\\n' +
+                '\t--offset <default 0>');
     process.exit(1);
   } else {
-    var sourceRoot = process.argv.length > 3 ? process.argv[3] : ".";
-    var mapFileBaseName = process.argv.length > 4 ? process.argv[4] : process.argv[2];
-    var generatedLineOffset = process.argv.length > 5 ?
-        parseInt(process.argv[5], 10) : 0;
-    generateMap(process.argv[2], sourceRoot, mapFileBaseName, generatedLineOffset);
+    var opts = parseArgs(process.argv.slice(2));
+    var fileName = opts._[0];
+    var sourceRoot = opts.sourceRoot ? opts.sourceRoot : ".";
+    var mapFileBaseName = opts.mapFileBaseName ? opts.mapFileBaseName : fileName;
+    var generatedLineOffset = opts.offset ? parseInt(opts.offset, 10) : 0;
+
+    var generatedSource = fs.readFileSync(fileName, 'utf-8');
+    var source = generatedSource;
+    var mappings = getMappings(generatedSource);
+    for (var i = 1, l = opts._.length; i < l; i ++) {
+      var optimizedSource = fs.readFileSync(opts._[i], 'utf-8')
+      var optimizedMappings = getMappings(optimizedSource);
+      var newMappings = {};
+      // uglify processes the code between EMSCRIPTEN_START_FUNCS and
+      // EMSCRIPTEN_END_FUNCS, so its line number maps are relative to those
+      // markers. we correct for that here.
+      var startFuncsLineNumber = countLines(
+        source.slice(0, source.indexOf('// EMSCRIPTEN_START_FUNCS'))) + 2;
+      for (var line in optimizedMappings) {
+        var originalLineNumber = optimizedMappings[line].originalLineNumber + startFuncsLineNumber;
+        if (originalLineNumber in mappings) {
+          newMappings[line] = mappings[originalLineNumber];
+        }
+      }
+      mappings = newMappings;
+      source = optimizedSource;
+    }
+
+    generateMap(mappings, sourceRoot, mapFileBaseName, generatedLineOffset);
+    appendMappingURL(opts._[opts._.length - 1], generatedSource,
+                     opts.mapFileBaseName + '.map');
   }
 }