RFC: Externally loadable syntax extensions

src/compiletest/header.rs

@@ -90,10 +90,14 @@ pub fn is_test_ignored(config: &config, testfile: &Path) -> bool {
     fn xfail_target(config: &config) -> ~str {
         ~"xfail-" + util::get_os(config.target)
     }
+    fn xfail_stage(config: &config) -> ~str {
+        ~"xfail-" + config.stage_id.split('-').next().unwrap()
+    }
 
     let val = iter_header(testfile, |ln| {
         if parse_name_directive(ln, "xfail-test") { false }
         else if parse_name_directive(ln, xfail_target(config)) { false }
+        else if parse_name_directive(ln, xfail_stage(config)) { false }
         else if config.mode == common::mode_pretty &&
             parse_name_directive(ln, "xfail-pretty") { false }
         else { true }

src/librustc/driver/driver.rs

@@ -20,6 +20,7 @@ use lib::llvm::{ContextRef, ModuleRef};
 use metadata::common::LinkMeta;
 use metadata::{creader, filesearch};
 use metadata::cstore::CStore;
+use metadata::creader::Loader;
 use metadata;
 use middle::{trans, freevars, kind, ty, typeck, lint, astencode, reachable};
 use middle;
@@ -41,6 +42,7 @@ use syntax::attr;
 use syntax::attr::{AttrMetaMethods};
 use syntax::codemap;
 use syntax::diagnostic;
+use syntax::ext::base::CrateLoader;
 use syntax::parse;
 use syntax::parse::token;
 use syntax::print::{pp, pprust};
@@ -163,6 +165,7 @@ pub fn phase_1_parse_input(sess: Session, cfg: ast::CrateConfig, input: &input)
 /// standard library and prelude.
 pub fn phase_2_configure_and_expand(sess: Session,
                                     cfg: ast::CrateConfig,
+                                    loader: &mut CrateLoader,
                                     mut crate: ast::Crate)
                                     -> (ast::Crate, syntax::ast_map::Map) {
     let time_passes = sess.time_passes();
@@ -188,9 +191,14 @@ pub fn phase_2_configure_and_expand(sess: Session,
     crate = time(time_passes, "configuration 1", crate, |crate|
                  front::config::strip_unconfigured_items(crate));
 
-    crate = time(time_passes, "expansion", crate, |crate|
-                 syntax::ext::expand::expand_crate(sess.parse_sess, cfg.clone(),
-                                                   crate));
+    crate = time(time_passes, "expansion", crate, |crate| {
+        syntax::ext::expand::expand_crate(sess.parse_sess,
+                                          loader,
+                                          cfg.clone(),
+                                          crate)
+    });
+    // dump the syntax-time crates
+    sess.cstore.reset();
 
     // strip again, in case expansion added anything with a #[cfg].
     crate = time(time_passes, "configuration 2", crate, |crate|
@@ -248,6 +256,11 @@ pub fn phase_3_run_analysis_passes(sess: Session,
     time(time_passes, "looking for entry point", (),
          |_| middle::entry::find_entry_point(sess, crate, ast_map));
 
+    sess.macro_registrar_fn.with_mut(|r| *r =
+        time(time_passes, "looking for macro registrar", (), |_|
+            syntax::ext::registrar::find_macro_registrar(
+                sess.span_diagnostic, crate)));
+
     let freevars = time(time_passes, "freevar finding", (), |_|
                         freevars::annotate_freevars(def_map, crate));
 
@@ -491,7 +504,8 @@ pub fn compile_input(sess: Session, cfg: ast::CrateConfig, input: &input,
         let (expanded_crate, ast_map) = {
             let crate = phase_1_parse_input(sess, cfg.clone(), input);
             if stop_after_phase_1(sess) { return; }
-            phase_2_configure_and_expand(sess, cfg, crate)
+            let loader = &mut Loader::new(sess);
+            phase_2_configure_and_expand(sess, cfg, loader, crate)
         };
         let outputs = build_output_filenames(input, outdir, output,
                                              expanded_crate.attrs, sess);
@@ -579,7 +593,8 @@ pub fn pretty_print_input(sess: Session,
 
     let (crate, ast_map, is_expanded) = match ppm {
         PpmExpanded | PpmExpandedIdentified | PpmTyped => {
-            let (crate, ast_map) = phase_2_configure_and_expand(sess, cfg, crate);
+            let loader = &mut Loader::new(sess);
+            let (crate, ast_map) = phase_2_configure_and_expand(sess, cfg, loader, crate);
             (crate, Some(ast_map), true)
         }
         _ => (crate, None, false)
@@ -912,6 +927,7 @@ pub fn build_session_(sopts: @session::options,
         // For a library crate, this is always none
         entry_fn: RefCell::new(None),
         entry_type: Cell::new(None),
+        macro_registrar_fn: RefCell::new(None),
         span_diagnostic: span_diagnostic_handler,
         filesearch: filesearch,
         building_library: Cell::new(false),

src/librustc/driver/session.rs

@@ -210,6 +210,7 @@ pub struct Session_ {
     entry_fn: RefCell<Option<(NodeId, codemap::Span)>>,
     entry_type: Cell<Option<EntryFnType>>,
     span_diagnostic: @diagnostic::SpanHandler,
+    macro_registrar_fn: RefCell<Option<ast::DefId>>,
     filesearch: @filesearch::FileSearch,
     building_library: Cell<bool>,
     working_dir: Path,

src/librustc/front/feature_gate.rs

@@ -43,6 +43,8 @@ static KNOWN_FEATURES: &'static [(&'static str, Status)] = &[
     ("non_ascii_idents", Active),
     ("thread_local", Active),
     ("link_args", Active),
+    ("phase", Active),
+    ("macro_registrar", Active),
 
     // These are used to test this portion of the compiler, they don't actually
     // mean anything
@@ -114,7 +116,15 @@ impl Visitor<()> for Context {
                     }
                 }
             }
-            _ => {}
+            ast::ViewItemExternMod(..) => {
+                for attr in i.attrs.iter() {
+                    if "phase" == attr.name() {
+                        self.gate_feature("phase", attr.span,
+                                          "compile time crate loading is \
+                                           experimental and possibly buggy");
+                    }
+                }
+            }
         }
         visit::walk_view_item(self, i, ())
     }
@@ -151,6 +161,14 @@ impl Visitor<()> for Context {
                 }
             }
 
+            ast::ItemFn(..) => {
+                if attr::contains_name(i.attrs, "macro_registrar") {
+                    self.gate_feature("macro_registrar", i.span,
+                                      "cross-crate macro exports are \
+                                       experimental and possibly buggy");
+                }
+            }
+
             _ => {}
         }
 

src/librustc/front/test.rs

@@ -14,6 +14,7 @@
 use driver::session;
 use front::config;
 use front::std_inject::with_version;
+use metadata::creader::Loader;
 
 use std::cell::RefCell;
 use std::vec;
@@ -38,10 +39,10 @@ struct Test {
     should_fail: bool
 }
 
-struct TestCtxt {
+struct TestCtxt<'a> {
     sess: session::Session,
     path: RefCell<~[ast::Ident]>,
-    ext_cx: ExtCtxt,
+    ext_cx: ExtCtxt<'a>,
     testfns: RefCell<~[Test]>,
     is_extra: bool,
     config: ast::CrateConfig,
@@ -63,11 +64,11 @@ pub fn modify_for_testing(sess: session::Session,
     }
 }
 
-struct TestHarnessGenerator {
-    cx: TestCtxt,
+struct TestHarnessGenerator<'a> {
+    cx: TestCtxt<'a>,
 }
 
-impl fold::Folder for TestHarnessGenerator {
+impl<'a> fold::Folder for TestHarnessGenerator<'a> {
     fn fold_crate(&mut self, c: ast::Crate) -> ast::Crate {
         let folded = fold::noop_fold_crate(c, self);
 
@@ -155,9 +156,10 @@ impl fold::Folder for TestHarnessGenerator {
 
 fn generate_test_harness(sess: session::Session, crate: ast::Crate)
                          -> ast::Crate {
+    let loader = &mut Loader::new(sess);
     let mut cx: TestCtxt = TestCtxt {
         sess: sess,
-        ext_cx: ExtCtxt::new(sess.parse_sess, sess.opts.cfg.clone()),
+        ext_cx: ExtCtxt::new(sess.parse_sess, sess.opts.cfg.clone(), loader),
         path: RefCell::new(~[]),
         testfns: RefCell::new(~[]),
         is_extra: is_extra(&crate),

src/librustc/metadata/common.rs

@@ -204,6 +204,10 @@ pub static tag_native_libraries_lib: uint = 0x104;
 pub static tag_native_libraries_name: uint = 0x105;
 pub static tag_native_libraries_kind: uint = 0x106;
 
+pub static tag_macro_registrar_fn: uint = 0x110;
+pub static tag_exported_macros: uint = 0x111;
+pub static tag_macro_def: uint = 0x112;
+
 #[deriving(Clone)]
 pub struct LinkMeta {
     crateid: CrateId,

src/librustc/metadata/creader.rs

@@ -10,10 +10,13 @@
 
 //! Validates all used crates and extern libraries and loads their metadata
 
+use driver::{driver, session};
 use driver::session::Session;
+use metadata::csearch;
 use metadata::cstore;
 use metadata::decoder;
 use metadata::loader;
+use metadata::loader::Os;
 
 use std::cell::RefCell;
 use std::hashmap::HashMap;
@@ -23,6 +26,7 @@ use syntax::attr;
 use syntax::attr::AttrMetaMethods;
 use syntax::codemap::{Span, DUMMY_SP};
 use syntax::diagnostic::SpanHandler;
+use syntax::ext::base::{CrateLoader, MacroCrate};
 use syntax::parse::token;
 use syntax::parse::token::IdentInterner;
 use syntax::crateid::CrateId;
@@ -131,37 +135,65 @@ fn visit_crate(e: &Env, c: &ast::Crate) {
 }
 
 fn visit_view_item(e: &mut Env, i: &ast::ViewItem) {
+    let should_load = i.attrs.iter().all(|attr| {
+        "phase" != attr.name() ||
+            attr.meta_item_list().map_or(false, |phases| {
+                attr::contains_name(phases, "link")
+            })
+    });
+
+    if !should_load {
+        return;
+    }
+
+    match extract_crate_info(i) {
+        Some(info) => {
+            let cnum = resolve_crate(e, info.ident, info.name, info.version,
+                                     @"", i.span);
+            e.sess.cstore.add_extern_mod_stmt_cnum(info.id, cnum);
+        }
+        None => ()
+    }
+}
+
+struct CrateInfo {
+    ident: @str,
+    name: @str,
+    version: @str,
+    id: ast::NodeId,
+}
+
+fn extract_crate_info(i: &ast::ViewItem) -> Option<CrateInfo> {
     match i.node {
-      ast::ViewItemExternMod(ident, path_opt, id) => {
-          let ident = token::ident_to_str(&ident);
-          debug!("resolving extern mod stmt. ident: {:?} path_opt: {:?}",
-                 ident, path_opt);
-          let (name, version) = match path_opt {
-              Some((path_str, _)) => {
-                  let crateid: Option<CrateId> = from_str(path_str);
-                  match crateid {
-                      None => (@"", @""),
-                      Some(crateid) => {
-                          let version = match crateid.version {
-                              None => @"",
-                              Some(ref ver) => ver.to_managed(),
-                          };
-                          (crateid.name.to_managed(), version)
-                      }
-                  }
-              }
-              None => (ident, @""),
-          };
-          let cnum = resolve_crate(e,
-                                   ident,
-                                   name,
-                                   version,
-                                   @"",
-                                   i.span);
-          e.sess.cstore.add_extern_mod_stmt_cnum(id, cnum);
-      }
-      _ => ()
-  }
+        ast::ViewItemExternMod(ident, path_opt, id) => {
+            let ident = token::ident_to_str(&ident);
+            debug!("resolving extern mod stmt. ident: {:?} path_opt: {:?}",
+                   ident, path_opt);
+            let (name, version) = match path_opt {
+                Some((path_str, _)) => {
+                    let crateid: Option<CrateId> = from_str(path_str);
+                    match crateid {
+                        None => (@"", @""),
+                        Some(crateid) => {
+                            let version = match crateid.version {
+                                None => @"",
+                                Some(ref ver) => ver.to_managed(),
+                            };
+                            (crateid.name.to_managed(), version)
+                        }
+                    }
+                }
+                None => (ident, @""),
+            };
+            Some(CrateInfo {
+                  ident: ident,
+                  name: name,
+                  version: version,
+                  id: id,
+            })
+        }
+        _ => None
+    }
 }
 
 fn visit_item(e: &Env, i: &ast::Item) {
@@ -355,3 +387,46 @@ fn resolve_crate_deps(e: &mut Env, cdata: &[u8]) -> cstore::cnum_map {
     }
     return @RefCell::new(cnum_map);
 }
+
+pub struct Loader {
+    priv env: Env,
+}
+
+impl Loader {
+    pub fn new(sess: Session) -> Loader {
+        let os = driver::get_os(driver::host_triple()).unwrap();
+        let os = session::sess_os_to_meta_os(os);
+        Loader {
+            env: Env {
+                sess: sess,
+                os: os,
+                crate_cache: @RefCell::new(~[]),
+                next_crate_num: 1,
+                intr: token::get_ident_interner(),
+            }
+        }
+    }
+}
+
+impl CrateLoader for Loader {
+    fn load_crate(&mut self, crate: &ast::ViewItem) -> MacroCrate {
+        let info = extract_crate_info(crate).unwrap();
+        let cnum = resolve_crate(&mut self.env, info.ident, info.name,
+                                 info.version, @"", crate.span);
+        let library = self.env.sess.cstore.get_used_crate_source(cnum).unwrap();
+        MacroCrate {
+            lib: library.dylib,
+            cnum: cnum
+        }
+    }
+
+    fn get_exported_macros(&mut self, cnum: ast::CrateNum) -> ~[@ast::Item] {
+        csearch::get_exported_macros(self.env.sess.cstore, cnum)
+    }
+
+    fn get_registrar_symbol(&mut self, cnum: ast::CrateNum) -> Option<~str> {
+        let cstore = self.env.sess.cstore;
+        csearch::get_macro_registrar_fn(cstore, cnum)
+            .map(|did| csearch::get_symbol(cstore, did))
+    }
+}

src/librustc/metadata/csearch.rs

@@ -301,3 +301,16 @@ pub fn get_trait_of_method(cstore: @cstore::CStore,
     decoder::get_trait_of_method(cdata, def_id.node, tcx)
 }
 
+pub fn get_macro_registrar_fn(cstore: @cstore::CStore,
+                              crate_num: ast::CrateNum)
+                              -> Option<ast::DefId> {
+    let cdata = cstore.get_crate_data(crate_num);
+    decoder::get_macro_registrar_fn(cdata)
+}
+
+pub fn get_exported_macros(cstore: @cstore::CStore,
+                           crate_num: ast::CrateNum)
+                           -> ~[@ast::Item] {
+    let cdata = cstore.get_crate_data(crate_num);
+    decoder::get_exported_macros(cdata)
+}