Instantiate nested modules for module linking (#2447)

This commit implements the interpretation necessary of the instance
section of the module linking proposal. Instantiating a module which
itself has nested instantiated instances will now instantiate the nested
instances properly. This isn't all that useful without the ability to
alias exports off the result, but we can at least observe the side
effects of instantiation through the `start` function.

cc #2094

Author: alexcrichton

build.rs

@@ -31,6 +31,7 @@ fn main() -> anyhow::Result<()> {
             test_directory_module(out, "tests/misc_testsuite/bulk-memory-operations", strategy)?;
             test_directory_module(out, "tests/misc_testsuite/reference-types", strategy)?;
             test_directory_module(out, "tests/misc_testsuite/multi-memory", strategy)?;
+            test_directory_module(out, "tests/misc_testsuite/module-linking", strategy)?;
             Ok(())
         })?;
 

cranelift/wasm/src/environ/spec.rs

@@ -8,8 +8,8 @@
 
 use crate::state::FuncTranslationState;
 use crate::translation_utils::{
-    DataIndex, ElemIndex, EntityType, Event, EventIndex, FuncIndex, Global, GlobalIndex, Memory,
-    MemoryIndex, Table, TableIndex, TypeIndex,
+    DataIndex, ElemIndex, EntityIndex, EntityType, Event, EventIndex, FuncIndex, Global,
+    GlobalIndex, Memory, MemoryIndex, ModuleIndex, Table, TableIndex, TypeIndex,
 };
 use core::convert::From;
 use core::convert::TryFrom;
@@ -22,6 +22,7 @@ use cranelift_frontend::FunctionBuilder;
 use serde::{Deserialize, Serialize};
 use std::boxed::Box;
 use std::string::ToString;
+use std::vec::Vec;
 use thiserror::Error;
 use wasmparser::ValidatorResources;
 use wasmparser::{BinaryReaderError, FuncValidator, FunctionBody, Operator, WasmFeatures};
@@ -969,4 +970,16 @@ pub trait ModuleEnvironment<'data>: TargetEnvironment {
     fn module_end(&mut self, index: usize) {
         drop(index);
     }
+
+    /// Indicates that this module will have `amount` instances.
+    fn reserve_instances(&mut self, amount: u32) {
+        drop(amount);
+    }
+
+    /// Declares a new instance which this module will instantiate before it's
+    /// instantiated.
+    fn declare_instance(&mut self, module: ModuleIndex, args: Vec<EntityIndex>) -> WasmResult<()> {
+        drop((module, args));
+        Err(WasmError::Unsupported("wasm instance".to_string()))
+    }
 }

cranelift/wasm/src/module_translator.rs

@@ -3,8 +3,9 @@
 use crate::environ::{ModuleEnvironment, WasmResult};
 use crate::sections_translator::{
     parse_data_section, parse_element_section, parse_event_section, parse_export_section,
-    parse_function_section, parse_global_section, parse_import_section, parse_memory_section,
-    parse_name_section, parse_start_section, parse_table_section, parse_type_section,
+    parse_function_section, parse_global_section, parse_import_section, parse_instance_section,
+    parse_memory_section, parse_name_section, parse_start_section, parse_table_section,
+    parse_type_section,
 };
 use crate::state::ModuleTranslationState;
 use cranelift_codegen::timing;
@@ -116,7 +117,7 @@ pub fn translate_module<'data>(
             }
             Payload::InstanceSection(s) => {
                 validator.instance_section(&s)?;
-                unimplemented!("module linking not implemented yet")
+                parse_instance_section(s, environ)?;
             }
             Payload::AliasSection(s) => {
                 validator.alias_section(&s)?;

cranelift/wasm/src/sections_translator.rs

@@ -10,9 +10,9 @@
 use crate::environ::{ModuleEnvironment, WasmError, WasmResult};
 use crate::state::ModuleTranslationState;
 use crate::translation_utils::{
-    tabletype_to_type, type_to_type, DataIndex, ElemIndex, EntityType, Event, EventIndex,
-    FuncIndex, Global, GlobalIndex, GlobalInit, Memory, MemoryIndex, Table, TableElementType,
-    TableIndex, TypeIndex,
+    tabletype_to_type, type_to_type, DataIndex, ElemIndex, EntityIndex, EntityType, Event,
+    EventIndex, FuncIndex, Global, GlobalIndex, GlobalInit, InstanceIndex, Memory, MemoryIndex,
+    ModuleIndex, Table, TableElementType, TableIndex, TypeIndex,
 };
 use crate::wasm_unsupported;
 use core::convert::TryFrom;
@@ -475,3 +475,37 @@ pub fn parse_name_section<'data>(
     }
     Ok(())
 }
+
+/// Parses the Instance section of the wasm module.
+pub fn parse_instance_section<'data>(
+    section: wasmparser::InstanceSectionReader<'data>,
+    environ: &mut dyn ModuleEnvironment<'data>,
+) -> WasmResult<()> {
+    environ.reserve_types(section.get_count())?;
+
+    for instance in section {
+        let instance = instance?;
+        let module = ModuleIndex::from_u32(instance.module());
+        let args = instance
+            .args()?
+            .into_iter()
+            .map(|result| {
+                let (kind, idx) = result?;
+                Ok(match kind {
+                    ExternalKind::Function => EntityIndex::Function(FuncIndex::from_u32(idx)),
+                    ExternalKind::Table => EntityIndex::Table(TableIndex::from_u32(idx)),
+                    ExternalKind::Memory => EntityIndex::Memory(MemoryIndex::from_u32(idx)),
+                    ExternalKind::Global => EntityIndex::Global(GlobalIndex::from_u32(idx)),
+                    ExternalKind::Module => EntityIndex::Module(ModuleIndex::from_u32(idx)),
+                    ExternalKind::Instance => EntityIndex::Instance(InstanceIndex::from_u32(idx)),
+                    ExternalKind::Event => unimplemented!(),
+
+                    // this won't pass validation
+                    ExternalKind::Type => unreachable!(),
+                })
+            })
+            .collect::<WasmResult<Vec<_>>>()?;
+        environ.declare_instance(module, args)?;
+    }
+    Ok(())
+}

crates/environ/src/module_environ.rs

@@ -1,13 +1,13 @@
-use crate::module::{MemoryPlan, Module, ModuleType, TableElements, TablePlan};
+use crate::module::{Instance, MemoryPlan, Module, ModuleType, TableElements, TablePlan};
 use crate::tunables::Tunables;
 use cranelift_codegen::ir;
 use cranelift_codegen::ir::{AbiParam, ArgumentPurpose};
 use cranelift_codegen::isa::TargetFrontendConfig;
 use cranelift_entity::PrimaryMap;
 use cranelift_wasm::{
     self, translate_module, DataIndex, DefinedFuncIndex, ElemIndex, EntityIndex, EntityType,
-    FuncIndex, Global, GlobalIndex, Memory, MemoryIndex, SignatureIndex, Table, TableIndex,
-    TargetEnvironment, TypeIndex, WasmError, WasmFuncType, WasmResult,
+    FuncIndex, Global, GlobalIndex, Memory, MemoryIndex, ModuleIndex, SignatureIndex, Table,
+    TableIndex, TargetEnvironment, TypeIndex, WasmError, WasmFuncType, WasmResult,
 };
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
@@ -60,7 +60,7 @@ pub struct ModuleTranslation<'data> {
 
     /// Indexes into the returned list of translations that are submodules of
     /// this module.
-    pub submodules: Vec<usize>,
+    pub submodules: PrimaryMap<ModuleIndex, usize>,
 
     code_index: u32,
 }
@@ -649,6 +649,18 @@ and for re-adding support for interface types you can see this issue:
         self.result.submodules.push(self.results.len());
         self.results.push(finished);
     }
+
+    fn reserve_instances(&mut self, amt: u32) {
+        self.result.module.instances.reserve(amt as usize);
+    }
+
+    fn declare_instance(&mut self, module: ModuleIndex, args: Vec<EntityIndex>) -> WasmResult<()> {
+        self.result
+            .module
+            .instances
+            .push(Instance::Instantiate { module, args });
+        Ok(())
+    }
 }
 
 /// Add environment-specific function parameters.

crates/jit/src/instantiate.rs

@@ -17,7 +17,7 @@ use thiserror::Error;
 use wasmtime_debug::create_gdbjit_image;
 use wasmtime_environ::entity::PrimaryMap;
 use wasmtime_environ::isa::TargetIsa;
-use wasmtime_environ::wasm::{DefinedFuncIndex, SignatureIndex};
+use wasmtime_environ::wasm::{DefinedFuncIndex, ModuleIndex, SignatureIndex};
 use wasmtime_environ::{
     CompileError, DataInitializer, DataInitializerLocation, FunctionAddressMap, Module,
     ModuleEnvironment, ModuleTranslation, StackMapInformation, TrapInformation,
@@ -71,6 +71,10 @@ pub struct CompilationArtifacts {
 
     /// Debug info presence flags.
     debug_info: bool,
+
+    /// Where to find this module's submodule code in the top-level list of
+    /// modules.
+    submodules: PrimaryMap<ModuleIndex, usize>,
 }
 
 impl CompilationArtifacts {
@@ -98,6 +102,7 @@ impl CompilationArtifacts {
                 let ModuleTranslation {
                     module,
                     data_initializers,
+                    submodules,
                     ..
                 } = translation;
 
@@ -118,6 +123,7 @@ impl CompilationArtifacts {
                     obj: obj.into_boxed_slice(),
                     unwind_info: unwind_info.into_boxed_slice(),
                     data_initializers,
+                    submodules,
                     funcs: funcs
                         .into_iter()
                         .map(|(_, func)| FunctionInfo {
@@ -336,6 +342,12 @@ impl CompiledModule {
     pub fn code(&self) -> &Arc<ModuleCode> {
         &self.code
     }
+
+    /// Returns where the specified submodule lives in this module's
+    /// array-of-modules (store at the top-level)
+    pub fn submodule_idx(&self, idx: ModuleIndex) -> usize {
+        self.artifacts.submodules[idx]
+    }
 }
 
 /// Similar to `DataInitializer`, but owns its own copy of the data rather

crates/wasmtime/src/externals.rs

@@ -115,15 +115,6 @@ impl Extern {
         };
         Store::same(my_store, store)
     }
-
-    pub(crate) fn desc(&self) -> &'static str {
-        match self {
-            Extern::Func(_) => "function",
-            Extern::Table(_) => "table",
-            Extern::Memory(_) => "memory",
-            Extern::Global(_) => "global",
-        }
-    }
 }
 
 impl From<Func> for Extern {