Re-enable preprocessing of pre/post JS file. NFC

This change got reverted in emscripten-core#19006 so this time we make the
preprocessing optional.

This is useful as it allows things like `{{{ POINTER_SIZE }}}`
and `{{{ makeGetValue(..) }}}` to be used in pre/post JS files, just
like they can be in JS library files.

This change allows threadprofiler.js to be fixed such that it works
under wasm64.

Fixes: emscripten-core#21226

Author: sbc100

ChangeLog.md

@@ -20,11 +20,16 @@ See docs/process.md for more on how version tagging works.
 
 3.1.54 (in development)
 -----------------------
-- Added `--use-port` option to `emcc`.  This option allows ports to be enabled 
-  by name and is designed to replace all existing `-sUSE_XXX` settings for 
-  ports. You can use `--show-ports` to get the list of available ports that 
+- Added `--use-port` option to `emcc`.  This option allows ports to be enabled
+  by name and is designed to replace all existing `-sUSE_XXX` settings for
+  ports. You can use `--show-ports` to get the list of available ports that
   can be used with this new option. (#21214)
- 
+- `--pre-js` and `--post-js` files can now opt into being run through the JS
+  preprocessor. This change was originally landed in  #18525, but it got
+  reverted in #19006.  Now it requires explicit opt-in by adding `#preprocess` to
+  the top of the JS file.  This is useful as it allows things like `{{{
+  POINTER_SIZE }}}` and `{{{ makeGetValue(..) }}}` to be used in pre/post JS
+  files, just like they can be in JS library files. (#21227)
 
 3.1.53 - 01/29/24
 -----------------

src/generated_struct_info32.json

@@ -223,6 +223,7 @@
         "EM_PROMISE_MATCH": 1,
         "EM_PROMISE_MATCH_RELEASE": 2,
         "EM_PROMISE_REJECT": 3,
+        "EM_THREAD_STATUS_NUMFIELDS": 7,
         "EM_TIMING_RAF": 1,
         "EM_TIMING_SETIMMEDIATE": 2,
         "EM_TIMING_SETTIMEOUT": 0,

src/generated_struct_info64.json

@@ -223,6 +223,7 @@
         "EM_PROMISE_MATCH": 1,
         "EM_PROMISE_MATCH_RELEASE": 2,
         "EM_PROMISE_REJECT": 3,
+        "EM_THREAD_STATUS_NUMFIELDS": 7,
         "EM_TIMING_RAF": 1,
         "EM_TIMING_SETIMMEDIATE": 2,
         "EM_TIMING_SETTIMEOUT": 0,

src/jsifier.js

@@ -97,6 +97,22 @@ function getTransitiveDeps(symbol) {
   return Array.from(transitiveDeps);
 }
 
+function shouldPreprocess(fileName) {
+  return read(fileName).trim().startsWith('#preprocess\n');
+}
+
+function preJS() {
+  let result = '';
+  for (const fileName of PRE_JS_FILES) {
+    if (shouldPreprocess(fileName)) {
+      result += processMacros(preprocess(fileName));
+    } else {
+      result += read(fileName);
+    }
+  }
+  return result;
+}
+
 function runJSify() {
   const libraryItems = [];
   const symbolDeps = {};
@@ -583,15 +599,13 @@ function(${args}) {
     libraryItems.push(JS);
   }
 
-  function includeFile(fileName) {
+  function includeFile(fileName, needsPreprocess = true) {
     print(`// include: ${fileName}`);
-    print(processMacros(preprocess(fileName)));
-    print(`// end include: ${fileName}`);
-  }
-
-  function includeFileRaw(fileName) {
-    print(`// include: ${fileName}`);
-    print(read(fileName));
+    if (needsPreprocess) {
+      print(processMacros(preprocess(fileName)));
+    } else {
+      print(read(fileName));
+    }
     print(`// end include: ${fileName}`);
   }
 
@@ -653,7 +667,7 @@ var proxiedFunctionTable = [
     includeFile(postFile);
 
     for (const fileName of POST_JS_FILES) {
-      includeFileRaw(fileName);
+      includeFile(fileName, shouldPreprocess(fileName));
     }
 
     print('//FORWARDED_DATA:' + JSON.stringify({

src/parseTools.js

@@ -137,6 +137,8 @@ function preprocess(filename) {
           if (showCurrentLine()) {
             error(`${filename}:${i + 1}: #error ${trimmed.substring(trimmed.indexOf(' ')).trim()}`);
           }
+        } else if (first === '#preprocess') {
+          // Do nothing
         } else {
           error(`${filename}:${i + 1}: Unknown preprocessor directive ${first}`);
         }
@@ -1022,14 +1024,6 @@ function getEntryFunction() {
   return `_${entryFunction}`;
 }
 
-function preJS() {
-  let result = '';
-  for (const fileName of PRE_JS_FILES) {
-    result += read(fileName);
-  }
-  return result;
-}
-
 function formattedMinNodeVersion() {
   var major = MIN_NODE_VERSION / 10000
   var minor = (MIN_NODE_VERSION / 100) % 100

src/struct_info_internal.json

@@ -24,6 +24,12 @@
           "SIGCANCEL"
         ]
     },
+    {
+        "file": "threading_internal.h",
+        "defines": [
+            "EM_THREAD_STATUS_NUMFIELDS"
+        ]
+    },
     {
         "file": "dynlink.h",
         "structs": {

src/threadprofiler.js

@@ -1,3 +1,5 @@
+#preprocess
+
 /**
  * @license
  * Copyright 2015 The Emscripten Authors
@@ -41,7 +43,6 @@ var emscriptenThreadProfiler = {
     }
     for (var i = 0; i < threads.length; ++i) {
       var threadPtr = threads[i];
-      var profilerBlock = Atomics.load(HEAPU32, (threadPtr + 8 /* {{{ C_STRUCTS.pthread.profilerBlock }}}*/) >> 2);
       var threadName = PThread.getThreadName(threadPtr);
       if (threadName) {
         threadName = `"${threadName}" (${ptrToString(threadPtr)})`;
@@ -69,7 +70,10 @@ var emscriptenThreadProfiler = {
 
     for (var i = 0; i < threads.length; ++i) {
       var threadPtr = threads[i];
-      var profilerBlock = Atomics.load(HEAPU32, (threadPtr + 8 /* {{{ C_STRUCTS.pthread.profilerBlock }}}*/) >> 2);
+      var profilerBlock = Atomics.load({{{ getHeapForType('*') }}}, {{{ getHeapOffset('threadPtr + ' + C_STRUCTS.pthread.profilerBlock, '*') }}});
+#if MEMORY64
+      profilerBlock = Number(profilerBlock);
+#endif
       var threadName = PThread.getThreadName(threadPtr);
       if (threadName) {
         threadName = `"${threadName}" (${ptrToString(threadPtr)})`;
@@ -81,11 +85,11 @@ var emscriptenThreadProfiler = {
 
       var threadTimesInStatus = [];
       var totalTime = 0;
-      var offset = profilerBlock + 16/*C_STRUCTS.thread_profiler_block.timeSpentInStatus*/;
-      for (var j = 0; j < 7/*EM_THREAD_STATUS_NUMFIELDS*/; ++j, offset += 8) {
-        threadTimesInStatus.push(Number(getValue(offset, 'double')));
+      var offset = profilerBlock + {{{ C_STRUCTS.thread_profiler_block.timeSpentInStatus }}};
+      for (var j = 0; j < {{{ cDefs.EM_THREAD_STATUS_NUMFIELDS }}}; ++j, offset += 8) {
+        threadTimesInStatus.push({{{ makeGetValue('offset', 0, 'double') }}});
         totalTime += threadTimesInStatus[j];
-        setValue(offset, 0, 'double');
+        {{{ makeSetValue('offset', 0, 0, 'double') }}};
       }
       var recent = '';
       if (threadTimesInStatus[1] > 0) recent += (threadTimesInStatus[1] / totalTime * 100.0).toFixed(1) + '% running. ';

test/test_other.py

@@ -14463,3 +14463,21 @@ def test_uuid(self):
   def test_wasm64_no_asan(self):
     err = self.expect_fail([EMCC, test_file('hello_world.c'), '-sMEMORY64', '-fsanitize=address'])
     self.assertContained('error: MEMORY64 does not yet work with ASAN', err)
+
+  def test_js_preprocess_pre_post(self):
+    create_file('pre.js', '''
+    #preprocess
+    #if ASSERTIONS
+    console.log('assertions enabled')
+    #else
+    console.log('assertions disabled')
+    #endif
+    ''')
+    create_file('post.js', '''
+    #preprocess
+    console.log({{{ POINTER_SIZE }}});
+    ''')
+    self.emcc_args += ['--pre-js', 'pre.js', '--post-js', 'post.js']
+    self.do_runf(test_file('hello_world.c'), 'assertions enabled\n4', emcc_args=['-sASSERTIONS=1'])
+    self.do_runf(test_file('hello_world.c'), 'assertions disabled\n4', emcc_args=['-sASSERTIONS=0'])
+    self.assertNotContained('#preprocess', read_file('hello_world.js'))