[wasm2js] Remove all handling for external memory file (#21217)

We have a lot of support code and complexity in emscripten for handling
of external memory files.  However, its only ever used in wasm2js mode
which is a legacy mode.  The only reason we continue to support it here
IIUC is because its slightly more space efficient than embedding the
data as base64.

For small programs like hello_world this is an over codesize win.  For
larger programs there is a regression in overall size in proportion to
the amount of static data in the program.

Author: sbc100

ChangeLog.md

@@ -25,6 +25,10 @@ See docs/process.md for more on how version tagging works.
   only exported on demand.  This means that they must be added to
   `EXPORTED_RUNTIME_METHODS` for them to appear on the `Module` object.  For
   now, this only effects users of `STRICT` mode. (#21439)
+- Emscripten no longer supports `--memory-init-file` (i.e. extracting static
+  data into an external .mem file).  This feature was only available under
+  wasm2js (`-sWASM=0`) anyway so this change will only affect users of this
+  setting. (#21217)
 
 3.1.54 - 02/15/24
 -----------------
@@ -62,7 +66,6 @@ See docs/process.md for more on how version tagging works.
 - Allow comments in response files. Any line starting with `#` is now ignored.
   This is useful when listing exported symbols. (#21330)
 
-
 3.1.53 - 01/29/24
 -----------------
 - The llvm version that emscripten uses was updated to 19.0.0 trunk. (#21165)

docs/emcc.txt

@@ -493,49 +493,6 @@ Options that are modified or new in *emcc* are listed below:
    Ports repos. After this operation is complete, this process will
    exit.
 
-"--memory-init-file 0|1"
-   [link] Specifies whether to emit a separate memory initialization
-   file.
-
-      Note:
-
-        Note that this is only relevant when *not* emitting Wasm, as
-        Wasm embeds the memory init data in the Wasm binary.
-
-   Possible values are:
-
-      * "0": Do not emit a separate memory initialization file.
-        Instead keep the static initialization inside the generated
-        JavaScript as text. This is the default setting if compiling
-        with -O0 or -O1 link-time optimization flags.
-
-      * "1": Emit a separate memory initialization file in binary
-        format. This is more efficient than storing it as text inside
-        JavaScript, but does mean you have another file to publish.
-        The binary file will also be loaded asynchronously, which
-        means "main()" will not be called until the file is downloaded
-        and applied; you cannot call any C functions until it arrives.
-        This is the default setting when compiling with -O2 or higher.
-
-        Note:
-
-          The safest way to ensure that it is safe to call C functions
-          (the initialisation file has loaded) is to call a notifier
-          function from "main()".
-
-        Note:
-
-          If you assign a network request to
-          "Module.memoryInitializerRequest" (before the script runs),
-          then it will use that request instead of automatically
-          starting a download for you. This is beneficial in that you
-          can, in your HTML, fire off a request for the memory init
-          file before the script actually arrives. For this to work,
-          the network request should be an XMLHttpRequest with
-          responseType set to "'arraybuffer'". (You can also put any
-          other object here, all it must provide is a ".response"
-          property containing an ArrayBuffer.)
-
 "-Wwarn-absolute-paths"
    [compile+link] Enables warnings about the use of absolute paths in
    "-I" and "-L" command line directives. This is used to warn against
@@ -604,11 +561,6 @@ Options that are modified or new in *emcc* are listed below:
    These rules only apply when linking.  When compiling to object code
    (See *-c* below) the name of the output file is irrelevant.
 
-   Note:
-
-     If "--memory-init-file" is used, a **.mem** file will be created
-     in addition to the generated **.js** and/or **.html** file.
-
 "-c"
    [compile] Tells *emcc* to emit an object file which can then be
    linked with other object files to produce an executable.

emcc.py

@@ -81,7 +81,7 @@
     '--bind', '--closure', '--cpuprofiler', '--embed-file',
     '--emit-symbol-map', '--emrun', '--exclude-file', '--extern-post-js',
     '--extern-pre-js', '--ignore-dynamic-linking', '--js-library',
-    '--js-transform', '--memory-init-file', '--oformat', '--output_eol',
+    '--js-transform', '--oformat', '--output_eol',
     '--post-js', '--pre-js', '--preload-file', '--profiling-funcs',
     '--proxy-to-worker', '--shell-file', '--source-map-base',
     '--threadprofiler', '--use-preload-plugins'
@@ -145,7 +145,6 @@ def __init__(self):
     self.emrun = False
     self.cpu_profiler = False
     self.memory_profiler = False
-    self.memory_init_file = None
     self.use_preload_cache = False
     self.use_preload_plugins = False
     self.valid_abspaths = []
@@ -1316,7 +1315,7 @@ def consume_arg_file():
       ports.show_ports()
       should_exit = True
     elif check_arg('--memory-init-file'):
-      options.memory_init_file = int(consume_arg())
+      exit_with_error('--memory-init-file is no longer supported')
     elif check_flag('--proxy-to-worker'):
       settings_changes.append('PROXY_TO_WORKER=1')
     elif check_arg('--valid-abspath'):

site/source/docs/compiling/Building-Projects.rst

@@ -92,7 +92,6 @@ which files are produced under which conditions:
 - ``emcc ... -o output.wasm`` omits generating either JavaScript or HTML launcher file, and produces a single Wasm file built in standalone mode as if the ``-sSTANDALONE_WASM`` setting had been used. The resulting file expects to be run with the `WASI ABI <https://github.com/WebAssembly/WASI/blob/4712d490fd7662f689af6faa5d718e042f014931/legacy/application-abi.md>`_ - in particular, as soon as you initialize the module you must manually invoke either the ``_start`` export or (in the case of ``--no-entry``) the ``_initialize`` export before doing anything else with it.
 - ``emcc ... -o output.{html,js} -sWASM=0`` causes the compiler to target JavaScript, and therefore a ``.wasm`` file is not produced.
 - ``emcc ... -o output.{html,js} --emit-symbol-map`` produces a file ``output.{html,js}.symbols`` if WebAssembly is being targeted (``-sWASM=0`` not specified), or if JavaScript is being targeted and ``-Os``, ``-Oz`` or ``-O2`` or higher is specified, but debug level setting is ``-g1`` or lower (i.e. if symbols minification did occur).
-- ``emcc ... -o output.{html,js} -sWASM=0 --memory-init-file 1`` causes the generation of ``output.{html,js}.mem`` memory initializer file. Passing ``-O2``, ``-Os`` or ``-Oz`` also implies ``--memory-init-file 1``.
 - ``emcc ... -o output.{html,js} -gsource-map`` generates a source map file ``output.wasm.map``. If targeting JavaScript with ``-sWASM=0``, the filename is ``output.{html,js}.map``.
 - ``emcc ... -o output.{html,js} --preload-file xxx`` directive generates a preloaded MEMFS filesystem file ``output.data``.
 - ``emcc ... -o output.{html,js} -sWASM={0,1} -sSINGLE_FILE`` merges JavaScript and WebAssembly code in the single output file ``output.{html,js}`` (in base64) to produce only one file for deployment. (If paired with ``--preload-file``, the preloaded ``.data`` file still exists as a separate file)

site/source/docs/getting_started/FAQ.rst

@@ -308,12 +308,11 @@ function `x` called before runtime initialization``, which is a check enabled in
 ``ASSERTIONS`` builds.)
 
 Calling a compiled function before a page has fully loaded can result in an
-error, if the function relies on files that may not be present (for example the
-:ref:`.mem <emcc-memory-init-file>` file and :ref:`preloaded
-<emcc-preload-file>` files are loaded asynchronously, and therefore if you just
-place some JS that calls compiled code in a ``--post-js``, that code will be
-called synchronously at the end of the combined JS file, potentially before the
-asynchronous event happens, which is bad).
+error, if the function relies on files that may not be present (for example
+:ref:`preloaded <emcc-preload-file>` files are loaded asynchronously, and
+therefore if you just place some JS that calls compiled code in a ``--post-js``,
+that code will be called synchronously at the end of the combined JS file,
+potentially before the asynchronous event happens, which is bad).
 
 The easiest way to find out when loading is complete is to add a ``main()``
 function, and within it call a JavaScript function to notify your code that

site/source/docs/tools_reference/emcc.rst

@@ -487,24 +487,6 @@ Options that are modified or new in *emcc* are listed below:
   [general]
   Shows the list of available projects in the Emscripten Ports repos. After this operation is complete, this process will exit.
 
-.. _emcc-memory-init-file:
-
-``--memory-init-file 0|1``
-  [link]
-  Specifies whether to emit a separate memory initialization file.
-
-      .. note:: Note that this is only relevant when *not* emitting Wasm, as Wasm embeds the memory init data in the Wasm binary.
-
-  Possible values are:
-
-    - ``0``: Do not emit a separate memory initialization file. Instead keep the static initialization inside the generated JavaScript as text. This is the default setting if compiling with -O0 or -O1 link-time optimization flags.
-    - ``1``: Emit a separate memory initialization file in binary format. This is more efficient than storing it as text inside JavaScript, but does mean you have another file to publish. The binary file will also be loaded asynchronously, which means ``main()`` will not be called until the file is downloaded and applied; you cannot call any C functions until it arrives. This is the default setting when compiling with -O2 or higher.
-
-      .. note:: The :ref:`safest way <faq-when-safe-to-call-compiled-functions>` to ensure that it is safe to call C functions (the initialisation file has loaded) is to call a notifier function from ``main()``.
-
-      .. note:: If you assign a network request to ``Module.memoryInitializerRequest`` (before the script runs), then it will use that request instead of automatically starting a download for you. This is beneficial in that you can, in your HTML, fire off a request for the memory init file before the script actually arrives. For this to work, the network request should be an XMLHttpRequest with responseType set to ``'arraybuffer'``. (You can also put any other object here, all it must provide is a ``.response`` property containing an ArrayBuffer.)
-
-
 ``-Wwarn-absolute-paths``
   [compile+link]
   Enables warnings about the use of absolute paths in ``-I`` and ``-L`` command line directives. This is used to warn against unintentional use of absolute paths, which is sometimes dangerous when referring to nonportable local system headers.
@@ -562,8 +544,6 @@ Options that are modified or new in *emcc* are listed below:
   These rules only apply when linking.  When compiling to object code (See `-c`
   below) the name of the output file is irrelevant.
 
-  .. note:: If ``--memory-init-file`` is used, a **.mem** file will be created in addition to the generated **.js** and/or **.html** file.
-
 .. _emcc-c:
 
 ``-c``

src/postamble_minimal.js

@@ -221,13 +221,6 @@ WebAssembly.instantiate(Module['wasm'], imports).then((output) => {
   updateMemoryViews();
 #endif
 
-#if !MEM_INIT_IN_WASM && !SINGLE_FILE
-#if ASSERTIONS
-  if (!Module['mem']) throw 'Must load memory initializer as an ArrayBuffer in to variable Module.mem before adding compiled output .js script to the DOM';
-#endif
-  HEAPU8.set(new Uint8Array(Module['mem']), {{{ GLOBAL_BASE }}});
-#endif
-
   initRuntime(wasmExports);
 #if PTHREADS
   // Export Wasm module for pthread creation to access.

src/preamble.js

@@ -973,9 +973,6 @@ function createWasm() {
 #endif
     updateMemoryViews();
 #endif
-#if !MEM_INIT_IN_WASM
-    runMemoryInitializer();
-#endif
 
 #if '$wasmTable' in addedLibraryItems && !RELOCATABLE
     wasmTable = wasmExports['__indirect_function_table'];
@@ -1123,88 +1120,6 @@ function getCompilerSetting(name) {
 }
 #endif // RETAIN_COMPILER_SETTINGS
 
-#if !MEM_INIT_IN_WASM
-var memoryInitializer = <<< MEM_INITIALIZER >>>;
-
-function runMemoryInitializer() {
-#if PTHREADS
-  if (ENVIRONMENT_IS_PTHREAD) return;
-#endif
-  if (!isDataURI(memoryInitializer)) {
-    memoryInitializer = locateFile(memoryInitializer);
-  }
-  if (ENVIRONMENT_IS_NODE || ENVIRONMENT_IS_SHELL) {
-    var data = readBinary(memoryInitializer);
-    HEAPU8.set(data, {{{ GLOBAL_BASE }}});
-  } else {
-    addRunDependency('memory initializer');
-    var applyMemoryInitializer = (data) => {
-      if (data.byteLength) data = new Uint8Array(data);
-#if ASSERTIONS
-      for (var i = 0; i < data.length; i++) {
-        assert(HEAPU8[{{{ GLOBAL_BASE }}} + i] === 0, "area for memory initializer should not have been touched before it's loaded");
-      }
-#endif
-      HEAPU8.set(data, {{{ GLOBAL_BASE }}});
-      // Delete the typed array that contains the large blob of the memory initializer request response so that
-      // we won't keep unnecessary memory lying around. However, keep the XHR object itself alive so that e.g.
-      // its .status field can still be accessed later.
-      if (Module['memoryInitializerRequest']) delete Module['memoryInitializerRequest'].response;
-      removeRunDependency('memory initializer');
-    };
-    var doBrowserLoad = () => {
-      readAsync(memoryInitializer, applyMemoryInitializer, () => {
-        var e = new Error('could not load memory initializer ' + memoryInitializer);
-#if MODULARIZE
-          readyPromiseReject(e);
-#else
-          throw e;
-#endif
-      });
-    };
-#if SUPPORT_BASE64_EMBEDDING
-    var memoryInitializerBytes = tryParseAsDataURI(memoryInitializer);
-    if (memoryInitializerBytes) {
-      applyMemoryInitializer(memoryInitializerBytes.buffer);
-    } else
-#endif
-    if (Module['memoryInitializerRequest']) {
-      // a network request has already been created, just use that
-      var useRequest = () => {
-        var request = Module['memoryInitializerRequest'];
-        var response = request.response;
-        if (request.status !== 200 && request.status !== 0) {
-#if SUPPORT_BASE64_EMBEDDING
-          var data = tryParseAsDataURI(Module['memoryInitializerRequestURL']);
-          if (data) {
-            response = data.buffer;
-          } else {
-#endif
-            // If you see this warning, the issue may be that you are using locateFile and defining it in JS. That
-            // means that the HTML file doesn't know about it, and when it tries to create the mem init request early, does it to the wrong place.
-            // Look in your browser's devtools network console to see what's going on.
-            console.warn('a problem seems to have happened with Module.memoryInitializerRequest, status: ' + request.status + ', retrying ' + memoryInitializer);
-            doBrowserLoad();
-            return;
-#if SUPPORT_BASE64_EMBEDDING
-          }
-#endif
-        }
-        applyMemoryInitializer(response);
-      };
-      if (Module['memoryInitializerRequest'].response) {
-        setTimeout(useRequest, 0); // it's already here; but, apply it asynchronously
-      } else {
-        Module['memoryInitializerRequest'].addEventListener('load', useRequest); // wait for it
-      }
-    } else {
-      // fetch it from the network ourselves
-      doBrowserLoad();
-    }
-  }
-}
-#endif // MEM_INIT_IN_WASM == 0
-
 #if MAIN_MODULE && ASYNCIFY
 // With MAIN_MODULE + ASYNCIFY the normal method of placing stub functions in
 // wasmImports for as-yet-undefined symbols doesn't work since ASYNCIFY then

src/settings.js

@@ -959,7 +959,7 @@ var INCOMING_MODULE_JS_API = [
   'onCustomMessage', 'onExit', 'onFree', 'onFullScreen', 'onMalloc',
   'onRealloc', 'onRuntimeInitialized', 'postMainLoop', 'postRun', 'preInit',
   'preMainLoop', 'preRun',
-  'preinitializedWebGLContext', 'memoryInitializerRequest', 'preloadPlugins',
+  'preinitializedWebGLContext', 'preloadPlugins',
   'print', 'printErr', 'quit', 'setStatus', 'statusMessage', 'stderr',
   'stdin', 'stdout', 'thisProgram', 'wasm', 'wasmBinary', 'websocket'
 ];

src/settings_internal.js

@@ -143,11 +143,6 @@ var AUDIO_WORKLET_FILE = '';
 // Base URL the source mapfile, if relevant
 var SOURCE_MAP_BASE = '';
 
-// When this is false we use an external memory init file
-// See --memory-init-file.  When not using wasm2js this flag is ignored, and
-// this setting will always be true.
-var MEM_INIT_IN_WASM = true;
-
 // If set to 1, src/base64Utils.js will be included in the bundle.
 // This is set internally when needed (SINGLE_FILE)
 var SUPPORT_BASE64_EMBEDDING = false;

src/shell.js

@@ -439,7 +439,7 @@ if (ENVIRONMENT_IS_NODE) {
 // Merge back in the overrides
 Object.assign(Module, moduleOverrides);
 // Free the object hierarchy contained in the overrides, this lets the GC
-// reclaim data used e.g. in memoryInitializerRequest, which is a large typed array.
+// reclaim data used.
 moduleOverrides = null;
 #if ASSERTIONS
 checkIncomingModuleAPI();

src/shell_minimal.js

@@ -82,7 +82,7 @@ if (ENVIRONMENT_IS_NODE && ENVIRONMENT_IS_SHELL) {
 #endif
 
 #if !SINGLE_FILE
-#if ENVIRONMENT_MAY_BE_NODE && ((WASM == 1 && (!WASM2JS || !MEM_INIT_IN_WASM)) || WASM == 2)
+#if ENVIRONMENT_MAY_BE_NODE && ((WASM == 1 && !WASM2JS) || WASM == 2)
 // Wasm or Wasm2JS loading:
 
 if (ENVIRONMENT_IS_NODE) {
@@ -95,13 +95,10 @@ if (ENVIRONMENT_IS_NODE) {
   Module['wasm'] = fs.readFileSync(__dirname + '/{{{ TARGET_BASENAME }}}.wasm');
 #endif
 #endif
-#if !MEM_INIT_IN_WASM
-  Module['mem'] = fs.readFileSync(__dirname + '/{{{ TARGET_BASENAME }}}.mem');
-#endif
 }
 #endif
 
-#if ENVIRONMENT_MAY_BE_SHELL && ((WASM == 1 && (!WASM2JS || !MEM_INIT_IN_WASM)) || WASM == 2)
+#if ENVIRONMENT_MAY_BE_SHELL && ((WASM == 1 && !WASM2JS) || WASM == 2)
 if (ENVIRONMENT_IS_SHELL) {
 #if WASM == 2
   if (typeof WebAssembly != 'undefined') Module['wasm'] = read('{{{ TARGET_BASENAME }}}.wasm', 'binary');
@@ -111,9 +108,6 @@ if (ENVIRONMENT_IS_SHELL) {
   Module['wasm'] = read('{{{ TARGET_BASENAME }}}.wasm', 'binary');
 #endif
 #endif
-#if !MEM_INIT_IN_WASM
-  Module['mem'] = read('{{{ TARGET_BASENAME }}}.mem', 'binary');
-#endif
 }
 #endif
 

test/code_size/hello_webgl2_wasm2js.json

@@ -1,10 +1,8 @@
 {
-  "a.html": 567,
-  "a.html.gz": 379,
-  "a.js": 17800,
-  "a.js.gz": 7981,
-  "a.mem": 3123,
-  "a.mem.gz": 2693,
-  "total": 21490,
-  "total_gz": 11053
+  "a.html": 354,
+  "a.html.gz": 266,
+  "a.js": 22323,
+  "a.js.gz": 11632,
+  "total": 22677,
+  "total_gz": 11898
 }

test/code_size/hello_webgl_wasm2js.json

@@ -1,10 +1,8 @@
 {
-  "a.html": 567,
-  "a.html.gz": 379,
-  "a.js": 17272,
-  "a.js.gz": 7814,
-  "a.mem": 3123,
-  "a.mem.gz": 2693,
-  "total": 20962,
-  "total_gz": 10886
+  "a.html": 354,
+  "a.html.gz": 266,
+  "a.js": 21794,
+  "a.js.gz": 11450,
+  "total": 22148,
+  "total_gz": 11716
 }