Reformat metadata for exported macros

Instead of copy-pasting the whole macro_rules! item from the original .rs file,
we serialize a separate name, attributes list, and body, the latter as
pretty-printed TTs.  The compilation of macro_rules! macros is decoupled
somewhat from the expansion of macros in item position.

This filters out comments, and facilitates selective imports.

src/librustc/metadata/common.rs

@@ -206,8 +206,8 @@ pub const tag_native_libraries_name: uint = 0x89;
 pub const tag_native_libraries_kind: uint = 0x8a;
 
 pub const tag_plugin_registrar_fn: uint = 0x8b;
-pub const tag_exported_macros: uint = 0x8c;
+
-pub const tag_macro_def: uint = 0x8d;
+// GAP 0x8c, 0x8d
 
 pub const tag_method_argument_names: uint = 0x8e;
 pub const tag_method_argument_name: uint = 0x8f;
@@ -261,3 +261,7 @@ pub const tag_associated_type_names: uint = 0xb2;
 pub const tag_associated_type_name: uint = 0xb3;
 
 pub const tag_polarity: uint = 0xb4;
+
+pub const tag_macro_defs: uint = 0xb5;
+pub const tag_macro_def: uint = 0xb6;
+pub const tag_macro_def_body: uint = 0xb7;

src/librustc/metadata/creader.rs

@@ -29,8 +29,9 @@ use syntax::ast;
 use syntax::abi;
 use syntax::attr;
 use syntax::attr::AttrMetaMethods;
-use syntax::codemap::{Span};
+use syntax::codemap::{Span, mk_sp};
 use syntax::diagnostic::SpanHandler;
+use syntax::parse;
 use syntax::parse::token::InternedString;
 use syntax::parse::token;
 use syntax::visit;
@@ -491,7 +492,36 @@ impl<'a> CrateReader<'a> {
             }
             None => { load_ctxt.report_load_errs(); unreachable!() },
         };
-        let macros = decoder::get_exported_macros(library.metadata.as_slice());
+
+        // Read exported macros
+        let imported_from = Some(token::intern(info.ident[]).ident());
+        let source_name = format!("<{} macros>", info.ident[]);
+        let mut macros = vec![];
+        decoder::each_exported_macro(library.metadata.as_slice(), &*self.sess.cstore.intr,
+            |name, attrs, body| {
+                // NB: Don't use parse::parse_tts_from_source_str because it parses with
+                // quote_depth > 0.
+                let mut p = parse::new_parser_from_source_str(&self.sess.parse_sess,
+                                                              self.sess.opts.cfg.clone(),
+                                                              source_name.clone(),
+                                                              body);
+                let lo = p.span.lo;
+                let body = p.parse_all_token_trees();
+                let span = mk_sp(lo, p.last_span.hi);
+                p.abort_if_errors();
+                macros.push(ast::MacroDef {
+                    ident: name.ident(),
+                    attrs: attrs,
+                    id: ast::DUMMY_NODE_ID,
+                    span: span,
+                    imported_from: imported_from,
+                    body: body,
+                });
+                true
+            }
+        );
+
+        // Look for a plugin registrar
         let registrar = decoder::get_plugin_registrar_fn(library.metadata.as_slice()).map(|id| {
             decoder::get_symbol(library.metadata.as_slice(), id)
         });
@@ -504,9 +534,11 @@ impl<'a> CrateReader<'a> {
             // empty dylib.
         }
         let pc = PluginMetadata {
-            lib: library.dylib.clone(),
             macros: macros,
-            registrar_symbol: registrar,
+            registrar: match (library.dylib.as_ref(), registrar) {
+                (Some(dylib), Some(reg)) => Some((dylib.clone(), reg)),
+                _ => None,
+            },
         };
         if should_link && self.existing_match(info.name[], None).is_none() {
             // register crate now to avoid double-reading metadata

src/librustc/metadata/decoder.rs

@@ -1353,15 +1353,16 @@ pub fn get_plugin_registrar_fn(data: &[u8]) -> Option<ast::NodeId> {
         .map(|doc| FromPrimitive::from_u32(reader::doc_as_u32(doc)).unwrap())
 }
 
-pub fn get_exported_macros(data: &[u8]) -> Vec<String> {
+pub fn each_exported_macro<F>(data: &[u8], intr: &IdentInterner, mut f: F) where
-    let macros = reader::get_doc(rbml::Doc::new(data),
+    F: FnMut(ast::Name, Vec<ast::Attribute>, String) -> bool,
-                                 tag_exported_macros);
+{
-    let mut result = Vec::new();
+    let macros = reader::get_doc(rbml::Doc::new(data), tag_macro_defs);
     reader::tagged_docs(macros, tag_macro_def, |macro_doc| {
-        result.push(macro_doc.as_str().to_string());
+        let name = item_name(intr, macro_doc);
-        true
+        let attrs = get_attributes(macro_doc);
+        let body = reader::get_doc(macro_doc, tag_macro_def_body);
+        f(name, attrs, body.as_str().to_string())
     });
-    result
 }
 
 pub fn get_dylib_dependency_formats(cdata: Cmd)

src/librustc/metadata/encoder.rs

@@ -42,6 +42,7 @@ use syntax::attr::AttrMetaMethods;
 use syntax::diagnostic::SpanHandler;
 use syntax::parse::token::special_idents;
 use syntax::parse::token;
+use syntax::print::pprust;
 use syntax::ptr::P;
 use syntax::visit::Visitor;
 use syntax::visit;
@@ -1818,25 +1819,21 @@ fn encode_plugin_registrar_fn(ecx: &EncodeContext, rbml_w: &mut Encoder) {
     }
 }
 
-/// Given a span, write the text of that span into the output stream
-/// as an exported macro
-fn encode_macro_def(ecx: &EncodeContext,
-                    rbml_w: &mut Encoder,
-                    span: &syntax::codemap::Span) {
-    let def = ecx.tcx.sess.codemap().span_to_snippet(*span)
-        .expect("Unable to find source for macro");
-    rbml_w.start_tag(tag_macro_def);
-    rbml_w.wr_str(def[]);
-    rbml_w.end_tag();
-}
-
 /// Serialize the text of the exported macros
-fn encode_macro_defs(ecx: &EncodeContext,
+fn encode_macro_defs(rbml_w: &mut Encoder,
-                     krate: &ast::Crate,
+                     krate: &ast::Crate) {
-                     rbml_w: &mut Encoder) {
+    rbml_w.start_tag(tag_macro_defs);
-    rbml_w.start_tag(tag_exported_macros);
+    for def in krate.exported_macros.iter() {
-    for item in krate.exported_macros.iter() {
+        rbml_w.start_tag(tag_macro_def);
-        encode_macro_def(ecx, rbml_w, &item.span);
+
+        encode_name(rbml_w, def.ident.name);
+        encode_attributes(rbml_w, def.attrs[]);
+
+        rbml_w.start_tag(tag_macro_def_body);
+        rbml_w.wr_str(pprust::tts_to_string(def.body[])[]);
+        rbml_w.end_tag();
+
+        rbml_w.end_tag();
     }
     rbml_w.end_tag();
 }
@@ -2154,7 +2151,7 @@ fn encode_metadata_inner(wr: &mut SeekableMemWriter,
 
     // Encode macro definitions
     i = rbml_w.writer.tell().unwrap();
-    encode_macro_defs(&ecx, krate, &mut rbml_w);
+    encode_macro_defs(&mut rbml_w, krate);
     stats.macro_defs_bytes = rbml_w.writer.tell().unwrap() - i;
 
     // Encode the types of all unboxed closures in this crate.

src/librustc/plugin/load.rs

@@ -8,7 +8,7 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-//! Used by `rustc` when loading a plugin.
+//! Used by `rustc` when loading a plugin, or a crate with exported macros.
 
 use session::Session;
 use metadata::creader::CrateReader;
@@ -21,17 +21,14 @@ use syntax::ast;
 use syntax::attr;
 use syntax::visit;
 use syntax::visit::Visitor;
-use syntax::ext::expand::ExportedMacros;
 use syntax::attr::AttrMetaMethods;
 
-/// Plugin-related crate metadata.
+/// Metadata for a single plugin crate.
 pub struct PluginMetadata {
-    /// Source code of macros exported by the crate.
+    /// Macros exported by the crate.
-    pub macros: Vec<String>,
+    pub macros: Vec<ast::MacroDef>,
-    /// Path to the shared library file.
+    /// Path to the shared library file, and registrar function symbol.
-    pub lib: Option<Path>,
+    pub registrar: Option<(Path, String)>,
-    /// Symbol name of the plugin registrar function.
-    pub registrar_symbol: Option<String>,
 }
 
 /// Pointer to a registrar function.
@@ -40,8 +37,8 @@ pub type PluginRegistrarFun =
 
 /// Information about loaded plugins.
 pub struct Plugins {
-    /// Source code of exported macros.
+    /// Imported macros.
-    pub macros: Vec<ExportedMacros>,
+    pub macros: Vec<ast::MacroDef>,
     /// Registrars, as function pointers.
     pub registrars: Vec<PluginRegistrarFun>,
 }
@@ -90,7 +87,7 @@ pub fn load_plugins(sess: &Session, krate: &ast::Crate,
 impl<'a, 'v> Visitor<'v> for PluginLoader<'a> {
     fn visit_view_item(&mut self, vi: &ast::ViewItem) {
         match vi.node {
-            ast::ViewItemExternCrate(name, _, _) => {
+            ast::ViewItemExternCrate(_, _, _) => {
                 let mut plugin_phase = false;
 
                 for attr in vi.attrs.iter().filter(|a| a.check_name("phase")) {
@@ -107,17 +104,13 @@ impl<'a, 'v> Visitor<'v> for PluginLoader<'a> {
 
                 if !plugin_phase { return; }
 
-                let PluginMetadata { macros, lib, registrar_symbol } =
+                let PluginMetadata { macros, registrar } =
                     self.reader.read_plugin_metadata(vi);
 
-                self.plugins.macros.push(ExportedMacros {
+                self.plugins.macros.extend(macros.into_iter());
-                    crate_name: name,
-                    macros: macros,
-                });
 
-                match (lib, registrar_symbol) {
+                match registrar {
-                    (Some(lib), Some(symbol))
+                    Some((lib, symbol)) => self.dylink_registrar(vi, lib, symbol),
-                        => self.dylink_registrar(vi, lib, symbol),
                     _ => (),
                 }
             }

src/librustdoc/visit_ast.rs

@@ -73,7 +73,7 @@ impl<'a, 'tcx> RustdocVisitor<'a, 'tcx> {
                                               None);
         // attach the crate's exported macros to the top-level module:
         self.module.macros = krate.exported_macros.iter()
-            .map(|it| self.visit_macro(&**it)).collect();
+            .map(|def| self.visit_macro(def)).collect();
         self.module.is_crate = true;
     }
 
@@ -363,13 +363,13 @@ impl<'a, 'tcx> RustdocVisitor<'a, 'tcx> {
     }
 
     // convert each exported_macro into a doc item
-    fn visit_macro(&self, item: &ast::Item) -> Macro {
+    fn visit_macro(&self, def: &ast::MacroDef) -> Macro {
         Macro {
-            id: item.id,
+            id: def.id,
-            attrs: item.attrs.clone(),
+            attrs: def.attrs.clone(),
-            name: item.ident,
+            name: def.ident,
-            whence: item.span,
+            whence: def.span,
-            stab: self.stability(item.id),
+            stab: self.stability(def.id),
         }
     }
 }

src/libsyntax/ast.rs

@@ -476,7 +476,7 @@ pub struct Crate {
     pub attrs: Vec<Attribute>,
     pub config: CrateConfig,
     pub span: Span,
-    pub exported_macros: Vec<P<Item>>
+    pub exported_macros: Vec<MacroDef>,
 }
 
 pub type MetaItem = Spanned<MetaItem_>;
@@ -1698,6 +1698,19 @@ pub enum InlinedItem {
     IIForeign(P<ForeignItem>),
 }
 
+/// A macro definition, in this crate or imported from another.
+///
+/// Not parsed directly, but created on macro import or `macro_rules!` expansion.
+#[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Show)]
+pub struct MacroDef {
+    pub ident: Ident,
+    pub attrs: Vec<Attribute>,
+    pub id: NodeId,
+    pub span: Span,
+    pub imported_from: Option<Ident>,
+    pub body: Vec<TokenTree>,
+}
+
 #[cfg(test)]
 mod test {
     use serialize::json;

src/libsyntax/ext/base.rs

@@ -28,19 +28,6 @@ use fold::Folder;
 use std::collections::HashMap;
 use std::rc::Rc;
 
-// new-style macro! tt code:
-//
-//    MacResult, NormalTT, IdentTT
-//
-// also note that ast::Mac used to have a bunch of extraneous cases and
-// is now probably a redundant AST node, can be merged with
-// ast::MacInvocTT.
-
-pub struct MacroDef {
-    pub name: String,
-    pub ext: SyntaxExtension
-}
-
 pub trait ItemDecorator {
     fn expand(&self,
               ecx: &mut ExtCtxt,
@@ -140,13 +127,6 @@ impl<F> IdentMacroExpander for F
 /// methods are spliced into the AST at the callsite of the macro (or
 /// just into the compiler's internal macro table, for `make_def`).
 pub trait MacResult {
-    /// Attempt to define a new macro.
-    // this should go away; the idea that a macro might expand into
-    // either a macro definition or an expression, depending on what
-    // the context wants, is kind of silly.
-    fn make_def(&mut self) -> Option<MacroDef> {
-        None
-    }
     /// Create an expression.
     fn make_expr(self: Box<Self>) -> Option<P<ast::Expr>> {
         None
@@ -469,7 +449,7 @@ pub struct ExtCtxt<'a> {
 
     pub mod_path: Vec<ast::Ident> ,
     pub trace_mac: bool,
-    pub exported_macros: Vec<P<ast::Item>>,
+    pub exported_macros: Vec<ast::MacroDef>,
 
     pub syntax_env: SyntaxEnv,
     pub recursion_count: uint,

src/libsyntax/ext/expand.rs

@@ -432,7 +432,7 @@ pub fn expand_item(it: P<ast::Item>, fld: &mut MacroExpander)
     }
 
     let mut new_items = match it.node {
-        ast::ItemMac(..) => expand_item_mac(it, None, fld),
+        ast::ItemMac(..) => expand_item_mac(it, fld),
         ast::ItemMod(_) | ast::ItemForeignMod(_) => {
             let valid_ident =
                 it.ident.name != parse::token::special_idents::invalid.name;
@@ -549,7 +549,6 @@ fn contains_macro_use(fld: &mut MacroExpander, attrs: &[ast::Attribute]) -> bool
 // Support for item-position macro invocations, exactly the same
 // logic as for expression-position macro invocations.
 pub fn expand_item_mac(it: P<ast::Item>,
-                       imported_from: Option<ast::Ident>,
                        fld: &mut MacroExpander) -> SmallVector<P<ast::Item>> {
     let (extname, path_span, tts) = match it.node {
         ItemMac(codemap::Spanned {
@@ -630,18 +629,20 @@ pub fn expand_item_mac(it: P<ast::Item>,
                         }
                     });
                     // DON'T mark before expansion.
-                    let MacroDef { name, ext }
-                        = macro_rules::add_new_extension(fld.cx, it.span, it.ident,
-                                                         imported_from, tts);
 
-                    fld.cx.syntax_env.insert(intern(name.as_slice()), ext);
+                    let def = ast::MacroDef {
+                        ident: it.ident,
+                        attrs: it.attrs.clone(),
+                        id: ast::DUMMY_NODE_ID,
+                        span: it.span,
+                        imported_from: None,
+                        body: tts,
+                    };
+                    let ext = macro_rules::compile(fld.cx, &def);
+                    fld.cx.syntax_env.insert(def.ident.name, ext);
 
-                    if match imported_from {
+                    if attr::contains_name(def.attrs.as_slice(), "macro_export") {
-                        None => attr::contains_name(it.attrs.as_slice(), "macro_export"),
+                        fld.cx.exported_macros.push(def);
-                        Some(_) => fld.cx.ecfg.reexported_macros.iter()
-                                       .any(|e| e.as_slice() == name.as_slice()),
-                    } {
-                        fld.cx.exported_macros.push(it);
                     }
 
                     // macro_rules! has a side effect but expands to nothing.
@@ -680,9 +681,6 @@ pub fn expand_item_mac(it: P<ast::Item>,
 }
 
 /// Expand a stmt
-//
-// I don't understand why this returns a vector... it looks like we're
-// half done adding machinery to allow macros to expand into multiple statements.
 fn expand_stmt(s: Stmt, fld: &mut MacroExpander) -> SmallVector<P<Stmt>> {
     let (mac, style) = match s.node {
         StmtMac(mac, style) => (mac, style),
@@ -1195,30 +1193,23 @@ impl ExpansionConfig {
     }
 }
 
-pub struct ExportedMacros {
-    pub crate_name: Ident,
-    pub macros: Vec<String>,
-}
-
 pub fn expand_crate(parse_sess: &parse::ParseSess,
                     cfg: ExpansionConfig,
                     // these are the macros being imported to this crate:
-                    imported_macros: Vec<ExportedMacros>,
+                    imported_macros: Vec<ast::MacroDef>,
                     user_exts: Vec<NamedSyntaxExtension>,
                     c: Crate) -> Crate {
     let mut cx = ExtCtxt::new(parse_sess, c.config.clone(), cfg);
     let mut expander = MacroExpander::new(&mut cx);
 
-    for ExportedMacros { crate_name, macros } in imported_macros.into_iter() {
+    for def in imported_macros.iter() {
-        let name = format!("<{} macros>", token::get_ident(crate_name));
+        let ext = macro_rules::compile(expander.cx, def);
+        expander.cx.syntax_env.insert(def.ident.name, ext);
 
-        for source in macros.into_iter() {
+        if expander.cx.ecfg.reexported_macros.iter()
-            let item = parse::parse_item_from_source_str(name.clone(),
+            .any(|e| e[] == token::get_ident(def.ident).get()) {
-                                                         source,
+
-                                                         expander.cx.cfg(),
+            expander.cx.exported_macros.push(def.clone());
-                                                         expander.cx.parse_sess())
-                    .expect("expected a serialized item");
-            expand_item_mac(item, Some(crate_name), &mut expander);
         }
     }
 

src/libsyntax/ext/tt/macro_rules.rs

@@ -11,7 +11,7 @@
 use ast::{Ident, TtDelimited, TtSequence, TtToken};
 use ast;
 use codemap::{Span, DUMMY_SP};
-use ext::base::{ExtCtxt, MacResult, MacroDef};
+use ext::base::{ExtCtxt, MacResult, SyntaxExtension};
 use ext::base::{NormalTT, TTMacroExpander};
 use ext::tt::macro_parser::{Success, Error, Failure};
 use ext::tt::macro_parser::{NamedMatch, MatchedSeq, MatchedNonterminal};
@@ -208,15 +208,9 @@ fn generic_extension<'cx>(cx: &'cx ExtCtxt,
 //
 // Holy self-referential!
 
-/// This procedure performs the expansion of the
+/// Converts a `macro_rules!` invocation into a syntax extension.
-/// macro_rules! macro. It parses the RHS and adds
+pub fn compile<'cx>(cx: &'cx mut ExtCtxt,
-/// an extension to the current context.
+                    def: &ast::MacroDef) -> SyntaxExtension {
-pub fn add_new_extension<'cx>(cx: &'cx mut ExtCtxt,
-                              sp: Span,
-                              name: Ident,
-                              imported_from: Option<Ident>,
-                              arg: Vec<ast::TokenTree> )
-                              -> MacroDef {
 
     let lhs_nm =  gensym_ident("lhs");
     let rhs_nm =  gensym_ident("rhs");
@@ -254,7 +248,7 @@ pub fn add_new_extension<'cx>(cx: &'cx mut ExtCtxt,
     let arg_reader = new_tt_reader(&cx.parse_sess().span_diagnostic,
                                    None,
                                    None,
-                                   arg.clone());
+                                   def.body.clone());
     let argument_map = parse_or_else(cx.parse_sess(),
                                      cx.cfg(),
                                      arg_reader,
@@ -263,23 +257,20 @@ pub fn add_new_extension<'cx>(cx: &'cx mut ExtCtxt,
     // Extract the arguments:
     let lhses = match *argument_map[lhs_nm] {
         MatchedSeq(ref s, _) => /* FIXME (#2543) */ (*s).clone(),
-        _ => cx.span_bug(sp, "wrong-structured lhs")
+        _ => cx.span_bug(def.span, "wrong-structured lhs")
     };
 
     let rhses = match *argument_map[rhs_nm] {
         MatchedSeq(ref s, _) => /* FIXME (#2543) */ (*s).clone(),
-        _ => cx.span_bug(sp, "wrong-structured rhs")
+        _ => cx.span_bug(def.span, "wrong-structured rhs")
     };
 
     let exp = box MacroRulesMacroExpander {
-        name: name,
+        name: def.ident,
-        imported_from: imported_from,
+        imported_from: def.imported_from,
         lhses: lhses,
         rhses: rhses,
     };
 
-    MacroDef {
+    NormalTT(exp, Some(def.span))
-        name: token::get_ident(name).to_string(),
-        ext: NormalTT(exp, Some(sp))
-    }
 }

src/libsyntax/fold.rs

@@ -1115,7 +1115,7 @@ pub fn noop_fold_mod<T: Folder>(Mod {inner, view_items, items}: Mod, folder: &mu
     }
 }
 
-pub fn noop_fold_crate<T: Folder>(Crate {module, attrs, config, exported_macros, span}: Crate,
+pub fn noop_fold_crate<T: Folder>(Crate {module, attrs, config, mut exported_macros, span}: Crate,
                                   folder: &mut T) -> Crate {
     let config = folder.fold_meta_items(config);
 
@@ -1146,6 +1146,10 @@ pub fn noop_fold_crate<T: Folder>(Crate {module, attrs, config, exported_macros,
         }, Vec::new(), span)
     };
 
+    for def in exported_macros.iter_mut() {
+        def.id = folder.new_id(def.id);
+    }
+
     Crate {
         module: module,
         attrs: attrs,

src/libsyntax/parse/mod.rs

@@ -169,6 +169,8 @@ pub fn parse_stmt_from_source_str(name: String,
 
 // Note: keep in sync with `with_hygiene::parse_tts_from_source_str`
 // until #16472 is resolved.
+//
+// Warning: This parses with quote_depth > 0, which is not the default.
 pub fn parse_tts_from_source_str(name: String,
                                  source: String,
                                  cfg: ast::CrateConfig,
@@ -310,6 +312,8 @@ pub mod with_hygiene {
 
     // Note: keep this in sync with `super::parse_tts_from_source_str` until
     // #16472 is resolved.
+    //
+    // Warning: This parses with quote_depth > 0, which is not the default.
     pub fn parse_tts_from_source_str(name: String,
                                      source: String,
                                      cfg: ast::CrateConfig,