Encode component types in canonical ABI module custom sections (#331)

* Remove support for handles and resources

This commit removes all support for the `resource` and `Handle` types
from the AST of `wit-parser` and all related support in all code
generators. The motivation for this commit is that `wit-bindgen` is on
the cusp of actually being able to work with components: producing a
component from guest output and consuming components in host generators.
More detail about this is in #314. With components as an intermediate
format, however, there is no way to encode resources since they are not
part of the component model proposal yet.

All is not lost for handles and resources, though. The official design
for handles and resources is being worked on upstream in the component
model repository itself at this time and once added all of this support
will be re-added to `wit-bindgen`. In the meantime though I personally
think that the best way forward is to remove the interim support for a
few reasons:

* Primarily it unblocks progress at this time towards fully integrating
  components and the `wit-bindgen` generators. The requirement to run
  existing tests that use handles would mean that no host generator
  could actually switch to components and/or modes for today's
  core-wasm-lookalike would need to be preserved.

* Otherwise though the semantics of the current handles are basically
  invented out of thin air by myself and were never really formally
  specified, debated, or designed deliberately. I grafted `witx`-style
  handles into `wit-component` and added features as necessary over
  time, but it seems highly unlikely that the handles designed as part
  of the component model will be the ones that `wit-bindgen` currently
  supports. This inevitably means that a new system would need new code
  anyway and would likely result in removal regardless.

As usual git always has the history of handles and this all may come
back in one shape or another if only slightly tweaked. I'm confident in
our history spelunking abilities, though, so I don't feel that keeping
support in the repository is necessary for this purpose.

* Remove resources from the demo

* smuggle wit information in custom sections

* move transcoder to the crate, and make it available in the cli

* gen-guest-rust can emit custom component-type section

* custom section takes a pub static, not a const

* ComponentEncoder needs to own its Interfaces

so that I can use the Interfaces decoded from the module's custom
section

* make ComponentEncoder always transcode component-type info from custom sections

* flavorful tests: types and functions are actually the same namespace

theyre not in wit, but thats a bug we need to fix, because they are in
component types

* test-helpers: build rust guests with wasm32-unknown-unknown and assert they encode as components

except for "invalid" and "handles" which are not gonna work

* refactor

* gen-guest-rust: now infallible

Co-authored-by: Alex Crichton <[email protected]>

Author: pchickey

Cargo.lock

@@ -11,6 +11,7 @@ doctest = false
 [dependencies]
 wit-bindgen-core = { workspace = true }
 wit-bindgen-gen-rust-lib = { workspace = true }
+wit-component = { workspace = true }
 heck = { workspace = true }
 clap = { workspace = true, optional = true }
 

crates/gen-guest-rust/Cargo.toml

@@ -468,6 +468,28 @@ impl Generator for RustWasm {
             ));
         }
 
+        let component_type = wit_component::InterfaceEncoder::new(iface)
+            .encode()
+            .expect(&format!(
+                "encoding interface {} as a component type",
+                iface.name
+            ));
+        let direction = match dir {
+            Direction::Import => "import",
+            Direction::Export => "export",
+        };
+        let iface_name = &iface.name;
+
+        self.src.push_str("#[cfg(target_arch = \"wasm32\")]\n");
+        self.src.push_str(&format!(
+            "#[link_section = \"component-type:{direction}:{iface_name}\"]\n"
+        ));
+        self.src.push_str(&format!(
+            "pub static __WIT_BINDGEN_COMPONENT_TYPE: [u8; {}] = ",
+            component_type.len()
+        ));
+        self.src.push_str(&format!("{:?};\n", component_type));
+
         // For standalone generation, close the export! macro
         if self.opts.standalone && dir == Direction::Export {
             self.src.push_str("});\n");

crates/gen-guest-rust/src/lib.rs

@@ -3,6 +3,7 @@ use std::fs;
 use std::path::PathBuf;
 use std::process::Command;
 use wit_bindgen_core::{wit_parser::Interface, Direction, Generator};
+use wit_component::ComponentEncoder;
 
 fn main() {
     let out_dir = PathBuf::from(std::env::var_os("OUT_DIR").unwrap());
@@ -13,14 +14,20 @@ fn main() {
         let mut cmd = Command::new("cargo");
         cmd.arg("build")
             .current_dir("../test-rust-wasm")
-            .arg("--target=wasm32-wasi")
+            // TODO: this should go back to wasm32-wasi once we have an adapter
+            // for snapshot 1 to a component
+            .arg("--target=wasm32-unknown-unknown")
             .env("CARGO_TARGET_DIR", &out_dir)
             .env("CARGO_PROFILE_DEV_DEBUG", "1")
             .env("RUSTFLAGS", "-Clink-args=--export-table")
             .env_remove("CARGO_ENCODED_RUSTFLAGS");
         let status = cmd.status().unwrap();
         assert!(status.success());
-        for file in out_dir.join("wasm32-wasi/debug").read_dir().unwrap() {
+        for file in out_dir
+            .join("wasm32-unknown-unknown/debug")
+            .read_dir()
+            .unwrap()
+        {
             let file = file.unwrap().path();
             if file.extension().and_then(|s| s.to_str()) != Some("wasm") {
                 continue;
@@ -31,6 +38,22 @@ fn main() {
                 file.to_str().unwrap().to_string(),
             ));
 
+            // The "invalid" test doesn't actually use the rust-guest macro
+            // and doesn't put the custom sections in, so component translation
+            // will fail.
+            if file.file_stem().unwrap().to_str().unwrap() != "invalid" {
+                // Validate that the module can be translated to a component, using
+                // the component-type custom sections. We don't yet consume this component
+                // anywhere.
+                let module = fs::read(&file).expect("failed to read wasm file");
+                ComponentEncoder::default()
+                    .module(module.as_slice())
+                    .expect("pull custom sections from module")
+                    .validate(true)
+                    .encode()
+                    .expect("module can be translated to a component");
+            }
+
             let dep_file = file.with_extension("d");
             let deps = fs::read_to_string(&dep_file).expect("failed to read dep file");
             for dep in deps

crates/test-helpers/build.rs

@@ -139,7 +139,7 @@ impl WitComponentApp {
             .with_context(|| format!("failed to parse module `{}`", self.module.display()))?;
 
         let mut encoder = ComponentEncoder::default()
-            .module(&module)
+            .module(&module)?
             .imports(&self.imports)
             .exports(&self.exports)
             .validate(!self.skip_validation);

crates/wit-component/src/cli.rs

@@ -53,6 +53,7 @@
 //! component model.
 
 use crate::{
+    decode_interface_component,
     validation::{
         expected_export_name, validate_adapter_module, validate_module, ValidatedAdapter,
         ValidatedModule,
@@ -2097,19 +2098,53 @@ impl<'a> ImportEncoder<'a> {
 pub struct ComponentEncoder<'a> {
     module: &'a [u8],
     encoding: StringEncoding,
-    interface: Option<&'a Interface>,
-    imports: &'a [Interface],
-    exports: &'a [Interface],
+    interface: Option<Interface>,
+    imports: Vec<Interface>,
+    exports: Vec<Interface>,
     validate: bool,
     types_only: bool,
     adapters: IndexMap<&'a str, (&'a [u8], &'a Interface)>,
 }
 
 impl<'a> ComponentEncoder<'a> {
     /// Set the core module to encode as a component.
-    pub fn module(mut self, module: &'a [u8]) -> Self {
+    /// This method will also parse any component type information stored in custom sections
+    /// inside the module, and add them as the interface, imports, and exports.
+    pub fn module(mut self, module: &'a [u8]) -> Result<Self> {
+        for payload in wasmparser::Parser::new(0).parse_all(&module) {
+            match payload.context("decoding item in module")? {
+                wasmparser::Payload::CustomSection(cs) => {
+                    if let Some(export) = cs.name().strip_prefix("component-type:export:") {
+                        let mut i = decode_interface_component(cs.data()).with_context(|| {
+                            format!("decoding component-type in export section {}", export)
+                        })?;
+                        i.name = export.to_owned();
+                        self.interface = Some(i);
+                    } else if let Some(import) = cs.name().strip_prefix("component-type:import:") {
+                        let mut i = decode_interface_component(cs.data()).with_context(|| {
+                            format!("decoding component-type in import section {}", import)
+                        })?;
+                        i.name = import.to_owned();
+                        self.imports.push(i);
+                    } else if let Some(export_instance) =
+                        cs.name().strip_prefix("component-type:export-instance:")
+                    {
+                        let mut i = decode_interface_component(cs.data()).with_context(|| {
+                            format!(
+                                "decoding component-type in export-instance section {}",
+                                export_instance
+                            )
+                        })?;
+                        i.name = export_instance.to_owned();
+                        self.exports.push(i);
+                    }
+                }
+                _ => {}
+            }
+        }
+
         self.module = module;
-        self
+        Ok(self)
     }
 
     /// Set the string encoding expected by the core module.
@@ -2126,19 +2161,23 @@ impl<'a> ComponentEncoder<'a> {
 
     /// Set the default interface exported by the component.
     pub fn interface(mut self, interface: &'a Interface) -> Self {
-        self.interface = Some(interface);
+        self.interface = Some(interface.clone());
         self
     }
 
     /// Set the interfaces the component imports.
     pub fn imports(mut self, imports: &'a [Interface]) -> Self {
-        self.imports = imports;
+        for i in imports {
+            self.imports.push(i.clone())
+        }
         self
     }
 
     /// Set the interfaces the component exports.
     pub fn exports(mut self, exports: &'a [Interface]) -> Self {
-        self.exports = exports;
+        for e in exports {
+            self.exports.push(e.clone())
+        }
         self
     }
 
@@ -2171,8 +2210,8 @@ impl<'a> ComponentEncoder<'a> {
             validate_module(
                 self.module,
                 &self.interface,
-                self.imports,
-                self.exports,
+                &self.imports,
+                &self.exports,
                 &adapters,
             )?
         } else {
@@ -2182,15 +2221,14 @@ impl<'a> ComponentEncoder<'a> {
         let exports = self
             .interface
             .iter()
-            .copied()
             .map(|i| (i, true))
             .chain(self.exports.iter().map(|i| (i, false)));
 
         let mut state = EncodingState::default();
         let mut types = TypeEncoder::default();
         let mut imports = ImportEncoder::default();
         types.encode_func_types(exports.clone(), false)?;
-        types.encode_instance_imports(self.imports, &info, &mut imports)?;
+        types.encode_instance_imports(&self.imports, &info, &mut imports)?;
 
         if self.types_only {
             if !self.module.is_empty() {

crates/wit-component/src/encoding.rs

@@ -86,7 +86,7 @@ pub struct ValidatedModule<'a> {
 /// for this module.
 pub fn validate_module<'a>(
     bytes: &'a [u8],
-    interface: &Option<&Interface>,
+    interface: &Option<Interface>,
     imports: &[Interface],
     exports: &[Interface],
     adapters: &IndexSet<&str>,

crates/wit-component/src/validation.rs

@@ -1,7 +1,7 @@
 use anyhow::{bail, Context, Result};
 use pretty_assertions::assert_eq;
 use std::{fs, path::Path};
-use wit_component::ComponentEncoder;
+use wit_component::{ComponentEncoder, InterfaceEncoder};
 use wit_parser::Interface;
 
 fn read_interface(path: &Path) -> Result<Interface> {
@@ -78,7 +78,7 @@ fn read_adapters(dir: &Path) -> Result<Vec<(String, Vec<u8>, Interface)>> {
 /// Run the test with the environment variable `BLESS` set to update
 /// either `component.wat` or `error.txt` depending on the outcome of the encoding.
 #[test]
-fn component_encoding() -> Result<()> {
+fn component_encoding_via_flags() -> Result<()> {
     drop(env_logger::try_init());
 
     for entry in fs::read_dir("tests/components")? {
@@ -105,7 +105,7 @@ fn component_encoding() -> Result<()> {
         let adapters = read_adapters(&path)?;
 
         let mut encoder = ComponentEncoder::default()
-            .module(&module)
+            .module(&module)?
             .imports(&imports)
             .exports(&exports)
             .validate(true);
@@ -119,6 +119,112 @@ fn component_encoding() -> Result<()> {
         }
 
         let r = encoder.encode();
+
+        let (output, baseline_path) = if error_path.is_file() {
+            match r {
+                Ok(_) => bail!("encoding should fail for test case `{}`", test_case),
+                Err(e) => (e.to_string(), &error_path),
+            }
+        } else {
+            (
+                wasmprinter::print_bytes(
+                    &r.with_context(|| format!("failed to encode for test case `{}`", test_case))?,
+                )
+                .with_context(|| {
+                    format!(
+                        "failed to print component bytes for test case `{}`",
+                        test_case
+                    )
+                })?,
+                &component_path,
+            )
+        };
+
+        if std::env::var_os("BLESS").is_some() {
+            fs::write(&baseline_path, output)?;
+        } else {
+            assert_eq!(
+                fs::read_to_string(&baseline_path)?.replace("\r\n", "\n"),
+                output,
+                "failed baseline comparison for test case `{}` ({})",
+                test_case,
+                baseline_path.display(),
+            );
+        }
+    }
+
+    Ok(())
+}
+
+/// Tests the encoding of components.
+///
+/// This test looks in the `components/` directory for test cases. It parses
+/// the inputs to the test out of that directly exactly like
+/// `component_encoding_via_flags` does in this same file.
+///
+/// Rather than pass the default interface, imports, and exports directly to
+/// the `ComponentEncoder`, this test encodes those Interfaces as component
+/// types in custom sections of the wasm Module.
+///
+/// This simulates the flow that toolchains which don't yet know how to
+/// emit a Component will emit a canonical ABI Module containing these custom sections,
+/// and those will then be translated by wit-component to a Component without
+/// needing the wit files passed in as well.
+#[test]
+fn component_encoding_via_custom_sections() -> Result<()> {
+    use wasm_encoder::{Encode, Section};
+
+    for entry in fs::read_dir("tests/components")? {
+        let path = entry?.path();
+        if !path.is_dir() {
+            continue;
+        }
+
+        let test_case = path.file_stem().unwrap().to_str().unwrap();
+
+        let module_path = path.join("module.wat");
+        let interface_path = path.join("default.wit");
+        let component_path = path.join("component.wat");
+        let error_path = path.join("error.txt");
+
+        let mut module = wat::parse_file(&module_path)
+            .with_context(|| format!("expected file `{}`", module_path.display()))?;
+
+        fn encode_interface(i: &Interface, module: &mut Vec<u8>, kind: &str) -> Result<()> {
+            let name = &format!("component-type:{}:{}", kind, i.name);
+            let contents = InterfaceEncoder::new(&i).validate(true).encode()?;
+            let section = wasm_encoder::CustomSection {
+                name,
+                data: &contents,
+            };
+            module.push(section.id());
+            section.encode(module);
+            Ok(())
+        }
+
+        // Encode the interface, exports, and imports into the module, instead of
+        // passing them to the ComponentEncoder explicitly.
+        if interface_path.is_file() {
+            let i = read_interface(&interface_path)?;
+            encode_interface(&i, &mut module, "export")?;
+        }
+        for i in read_interfaces(&path, "import-*.wit")? {
+            encode_interface(&i, &mut module, "import")?;
+        }
+        for i in read_interfaces(&path, "export-*.wit")? {
+            encode_interface(&i, &mut module, "export-instance")?;
+        }
+        //
+        let adapters = read_adapters(&path)?;
+
+        let mut encoder = ComponentEncoder::default().module(&module)?.validate(true);
+
+        for (name, wasm, interface) in adapters.iter() {
+            encoder = encoder.adapter(name, wasm, interface);
+        }
+
+        let r = encoder.encode();
+
         let (output, baseline_path) = if error_path.is_file() {
             match r {
                 Ok(_) => bail!("encoding should fail for test case `{}`", test_case),