bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Reformat metadata for exported macros

Browse source 
Instead of copy-pasting the whole macro_rules! item from the original .rs file,
we serialize a separate name, attributes list, and body, the latter as
pretty-printed TTs.  The compilation of macro_rules! macros is decoupled
somewhat from the expansion of macros in item position.

This filters out comments, and facilitates selective imports.
This commit is contained in:
Keegan McAllister 2014-12-30 19:10:46 -08:00
parent 24aa7f0e38
commit 677b7cad3d
12 changed files with 141 additions and 131 deletions
Download patch file Download diff file Expand all files Collapse all files

8
src/librustc/metadata/common.rs
View file

@ -206,8 +206,8 @@ pub const tag_native_libraries_name: uint = 0x89;
pub const tag_native_libraries_kind: uint = 0x8a;
pub const tag_plugin_registrar_fn: uint = 0x8b;
pub const tag_exported_macros: uint = 0x8c;
pub const tag_macro_def: uint = 0x8d;
// GAP 0x8c, 0x8d
pub const tag_method_argument_names: uint = 0x8e;
pub const tag_method_argument_name: uint = 0x8f;
@ -261,3 +261,7 @@ pub const tag_associated_type_names: uint = 0xb2;
pub const tag_associated_type_name: uint = 0xb3;
pub const tag_polarity: uint = 0xb4;
pub const tag_macro_defs: uint = 0xb5;
pub const tag_macro_def: uint = 0xb6;
pub const tag_macro_def_body: uint = 0xb7;

40
src/librustc/metadata/creader.rs
View file

@ -29,8 +29,9 @@ use syntax::ast;
use syntax::abi;
use syntax::attr;
use syntax::attr::AttrMetaMethods;
use syntax::codemap::{Span};
use syntax::codemap::{Span, mk_sp};
use syntax::diagnostic::SpanHandler;
use syntax::parse;
use syntax::parse::token::InternedString;
use syntax::parse::token;
use syntax::visit;
@ -491,7 +492,36 @@ impl<'a> CrateReader<'a> {
}
None => { load_ctxt.report_load_errs(); unreachable!() },
};
let macros = decoder::get_exported_macros(library.metadata.as_slice());
// Read exported macros
let imported_from = Some(token::intern(info.ident[]).ident());
let source_name = format!("<{} macros>", info.ident[]);
let mut macros = vec![];
decoder::each_exported_macro(library.metadata.as_slice(), &*self.sess.cstore.intr,
|name, attrs, body| {
// NB: Don't use parse::parse_tts_from_source_str because it parses with
// quote_depth > 0.
let mut p = parse::new_parser_from_source_str(&self.sess.parse_sess,
self.sess.opts.cfg.clone(),
source_name.clone(),
body);
let lo = p.span.lo;
let body = p.parse_all_token_trees();
let span = mk_sp(lo, p.last_span.hi);
p.abort_if_errors();
macros.push(ast::MacroDef {
ident: name.ident(),
attrs: attrs,
id: ast::DUMMY_NODE_ID,
span: span,
imported_from: imported_from,
body: body,
});
true
}
);
// Look for a plugin registrar
let registrar = decoder::get_plugin_registrar_fn(library.metadata.as_slice()).map(|id| {
decoder::get_symbol(library.metadata.as_slice(), id)
});
@ -504,9 +534,11 @@ impl<'a> CrateReader<'a> {
// empty dylib.
}
let pc = PluginMetadata {
lib: library.dylib.clone(),
macros: macros,
registrar_symbol: registrar,
registrar: match (library.dylib.as_ref(), registrar) {
(Some(dylib), Some(reg)) => Some((dylib.clone(), reg)),
_ => None,
},
};
if should_link && self.existing_match(info.name[], None).is_none() {
// register crate now to avoid double-reading metadata

15
src/librustc/metadata/decoder.rs
View file

@ -1353,15 +1353,16 @@ pub fn get_plugin_registrar_fn(data: &[u8]) -> Option<ast::NodeId> {
.map(|doc| FromPrimitive::from_u32(reader::doc_as_u32(doc)).unwrap())
}
pub fn get_exported_macros(data: &[u8]) -> Vec<String> {
let macros = reader::get_doc(rbml::Doc::new(data),
tag_exported_macros);
let mut result = Vec::new();
pub fn each_exported_macro<F>(data: &[u8], intr: &IdentInterner, mut f: F) where
F: FnMut(ast::Name, Vec<ast::Attribute>, String) -> bool,
{
let macros = reader::get_doc(rbml::Doc::new(data), tag_macro_defs);
reader::tagged_docs(macros, tag_macro_def, |macro_doc| {
result.push(macro_doc.as_str().to_string());
true
let name = item_name(intr, macro_doc);
let attrs = get_attributes(macro_doc);
let body = reader::get_doc(macro_doc, tag_macro_def_body);
f(name, attrs, body.as_str().to_string())
});
result
}
pub fn get_dylib_dependency_formats(cdata: Cmd)

35
src/librustc/metadata/encoder.rs
View file

@ -42,6 +42,7 @@ use syntax::attr::AttrMetaMethods;
use syntax::diagnostic::SpanHandler;
use syntax::parse::token::special_idents;
use syntax::parse::token;
use syntax::print::pprust;
use syntax::ptr::P;
use syntax::visit::Visitor;
use syntax::visit;
@ -1818,25 +1819,21 @@ fn encode_plugin_registrar_fn(ecx: &EncodeContext, rbml_w: &mut Encoder) {
}
}
/// Given a span, write the text of that span into the output stream
/// as an exported macro
fn encode_macro_def(ecx: &EncodeContext,
rbml_w: &mut Encoder,
span: &syntax::codemap::Span) {
let def = ecx.tcx.sess.codemap().span_to_snippet(*span)
.expect("Unable to find source for macro");
rbml_w.start_tag(tag_macro_def);
rbml_w.wr_str(def[]);
rbml_w.end_tag();
}
/// Serialize the text of the exported macros
fn encode_macro_defs(ecx: &EncodeContext,
krate: &ast::Crate,
rbml_w: &mut Encoder) {
rbml_w.start_tag(tag_exported_macros);
for item in krate.exported_macros.iter() {
encode_macro_def(ecx, rbml_w, &item.span);
fn encode_macro_defs(rbml_w: &mut Encoder,
krate: &ast::Crate) {
rbml_w.start_tag(tag_macro_defs);
for def in krate.exported_macros.iter() {
rbml_w.start_tag(tag_macro_def);
encode_name(rbml_w, def.ident.name);
encode_attributes(rbml_w, def.attrs[]);
rbml_w.start_tag(tag_macro_def_body);
rbml_w.wr_str(pprust::tts_to_string(def.body[])[]);
rbml_w.end_tag();
rbml_w.end_tag();
}
rbml_w.end_tag();
}
@ -2154,7 +2151,7 @@ fn encode_metadata_inner(wr: &mut SeekableMemWriter,
// Encode macro definitions
i = rbml_w.writer.tell().unwrap();
encode_macro_defs(&ecx, krate, &mut rbml_w);
encode_macro_defs(&mut rbml_w, krate);
stats.macro_defs_bytes = rbml_w.writer.tell().unwrap() - i;
// Encode the types of all unboxed closures in this crate.

33
src/librustc/plugin/load.rs
View file

@ -8,7 +8,7 @@
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//! Used by `rustc` when loading a plugin.
//! Used by `rustc` when loading a plugin, or a crate with exported macros.
use session::Session;
use metadata::creader::CrateReader;
@ -21,17 +21,14 @@ use syntax::ast;
use syntax::attr;
use syntax::visit;
use syntax::visit::Visitor;
use syntax::ext::expand::ExportedMacros;
use syntax::attr::AttrMetaMethods;
/// Plugin-related crate metadata.
/// Metadata for a single plugin crate.
pub struct PluginMetadata {
/// Source code of macros exported by the crate.
pub macros: Vec<String>,
/// Path to the shared library file.
pub lib: Option<Path>,
/// Symbol name of the plugin registrar function.
pub registrar_symbol: Option<String>,
/// Macros exported by the crate.
pub macros: Vec<ast::MacroDef>,
/// Path to the shared library file, and registrar function symbol.
pub registrar: Option<(Path, String)>,
}
/// Pointer to a registrar function.
@ -40,8 +37,8 @@ pub type PluginRegistrarFun =
/// Information about loaded plugins.
pub struct Plugins {
/// Source code of exported macros.
pub macros: Vec<ExportedMacros>,
/// Imported macros.
pub macros: Vec<ast::MacroDef>,
/// Registrars, as function pointers.
pub registrars: Vec<PluginRegistrarFun>,
}
@ -90,7 +87,7 @@ pub fn load_plugins(sess: &Session, krate: &ast::Crate,
impl<'a, 'v> Visitor<'v> for PluginLoader<'a> {
fn visit_view_item(&mut self, vi: &ast::ViewItem) {
match vi.node {
ast::ViewItemExternCrate(name, _, _) => {
ast::ViewItemExternCrate(_, _, _) => {
let mut plugin_phase = false;
for attr in vi.attrs.iter().filter(|a| a.check_name("phase")) {
@ -107,17 +104,13 @@ impl<'a, 'v> Visitor<'v> for PluginLoader<'a> {
if !plugin_phase { return; }
let PluginMetadata { macros, lib, registrar_symbol } =
let PluginMetadata { macros, registrar } =
self.reader.read_plugin_metadata(vi);
self.plugins.macros.push(ExportedMacros {
crate_name: name,
macros: macros,
});
self.plugins.macros.extend(macros.into_iter());
match (lib, registrar_symbol) {
(Some(lib), Some(symbol))
=> self.dylink_registrar(vi, lib, symbol),
match registrar {
Some((lib, symbol)) => self.dylink_registrar(vi, lib, symbol),
_ => (),
}
}

14
src/librustdoc/visit_ast.rs
View file

@ -73,7 +73,7 @@ impl<'a, 'tcx> RustdocVisitor<'a, 'tcx> {
None);
// attach the crate's exported macros to the top-level module:
self.module.macros = krate.exported_macros.iter()
.map(|it| self.visit_macro(&**it)).collect();
.map(|def| self.visit_macro(def)).collect();
self.module.is_crate = true;
}
@ -363,13 +363,13 @@ impl<'a, 'tcx> RustdocVisitor<'a, 'tcx> {
}
// convert each exported_macro into a doc item
fn visit_macro(&self, item: &ast::Item) -> Macro {
fn visit_macro(&self, def: &ast::MacroDef) -> Macro {
Macro {
id: item.id,
attrs: item.attrs.clone(),
name: item.ident,
whence: item.span,
stab: self.stability(item.id),
id: def.id,
attrs: def.attrs.clone(),
name: def.ident,
whence: def.span,
stab: self.stability(def.id),
}
}
}

15
src/libsyntax/ast.rs
View file

@ -476,7 +476,7 @@ pub struct Crate {
pub attrs: Vec<Attribute>,
pub config: CrateConfig,
pub span: Span,
pub exported_macros: Vec<P<Item>>
pub exported_macros: Vec<MacroDef>,
}
pub type MetaItem = Spanned<MetaItem_>;
@ -1698,6 +1698,19 @@ pub enum InlinedItem {
IIForeign(P<ForeignItem>),
}
/// A macro definition, in this crate or imported from another.
///
/// Not parsed directly, but created on macro import or `macro_rules!` expansion.
#[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Show)]
pub struct MacroDef {
pub ident: Ident,
pub attrs: Vec<Attribute>,
pub id: NodeId,
pub span: Span,
pub imported_from: Option<Ident>,
pub body: Vec<TokenTree>,
}
#[cfg(test)]
mod test {
use serialize::json;

22
src/libsyntax/ext/base.rs
View file

@ -28,19 +28,6 @@ use fold::Folder;
use std::collections::HashMap;
use std::rc::Rc;
// new-style macro! tt code:
//
// MacResult, NormalTT, IdentTT
//
// also note that ast::Mac used to have a bunch of extraneous cases and
// is now probably a redundant AST node, can be merged with
// ast::MacInvocTT.
pub struct MacroDef {
pub name: String,
pub ext: SyntaxExtension
}
pub trait ItemDecorator {
fn expand(&self,
ecx: &mut ExtCtxt,
@ -140,13 +127,6 @@ impl<F> IdentMacroExpander for F
/// methods are spliced into the AST at the callsite of the macro (or
/// just into the compiler's internal macro table, for `make_def`).
pub trait MacResult {
/// Attempt to define a new macro.
// this should go away; the idea that a macro might expand into
// either a macro definition or an expression, depending on what
// the context wants, is kind of silly.
fn make_def(&mut self) -> Option<MacroDef> {
None
}
/// Create an expression.
fn make_expr(self: Box<Self>) -> Option<P<ast::Expr>> {
None
@ -469,7 +449,7 @@ pub struct ExtCtxt<'a> {
pub mod_path: Vec<ast::Ident> ,
pub trace_mac: bool,
pub exported_macros: Vec<P<ast::Item>>,
pub exported_macros: Vec<ast::MacroDef>,
pub syntax_env: SyntaxEnv,
pub recursion_count: uint,

51
src/libsyntax/ext/expand.rs
View file

@ -432,7 +432,7 @@ pub fn expand_item(it: P<ast::Item>, fld: &mut MacroExpander)
}
let mut new_items = match it.node {
ast::ItemMac(..) => expand_item_mac(it, None, fld),
ast::ItemMac(..) => expand_item_mac(it, fld),
ast::ItemMod(_) | ast::ItemForeignMod(_) => {
let valid_ident =
it.ident.name != parse::token::special_idents::invalid.name;
@ -549,7 +549,6 @@ fn contains_macro_use(fld: &mut MacroExpander, attrs: &[ast::Attribute]) -> bool
// Support for item-position macro invocations, exactly the same
// logic as for expression-position macro invocations.
pub fn expand_item_mac(it: P<ast::Item>,
imported_from: Option<ast::Ident>,
fld: &mut MacroExpander) -> SmallVector<P<ast::Item>> {
let (extname, path_span, tts) = match it.node {
ItemMac(codemap::Spanned {
@ -630,18 +629,20 @@ pub fn expand_item_mac(it: P<ast::Item>,
}
});
// DON'T mark before expansion.
let MacroDef { name, ext }
= macro_rules::add_new_extension(fld.cx, it.span, it.ident,
imported_from, tts);
fld.cx.syntax_env.insert(intern(name.as_slice()), ext);
let def = ast::MacroDef {
ident: it.ident,
attrs: it.attrs.clone(),
id: ast::DUMMY_NODE_ID,
span: it.span,
imported_from: None,
body: tts,
};
let ext = macro_rules::compile(fld.cx, &def);
fld.cx.syntax_env.insert(def.ident.name, ext);
if match imported_from {
None => attr::contains_name(it.attrs.as_slice(), "macro_export"),
Some(_) => fld.cx.ecfg.reexported_macros.iter()
.any(|e| e.as_slice() == name.as_slice()),
} {
fld.cx.exported_macros.push(it);
if attr::contains_name(def.attrs.as_slice(), "macro_export") {
fld.cx.exported_macros.push(def);
}
// macro_rules! has a side effect but expands to nothing.
@ -680,9 +681,6 @@ pub fn expand_item_mac(it: P<ast::Item>,
}
/// Expand a stmt
//
// I don't understand why this returns a vector... it looks like we're
// half done adding machinery to allow macros to expand into multiple statements.
fn expand_stmt(s: Stmt, fld: &mut MacroExpander) -> SmallVector<P<Stmt>> {
let (mac, style) = match s.node {
StmtMac(mac, style) => (mac, style),
@ -1195,30 +1193,23 @@ impl ExpansionConfig {
}
}
pub struct ExportedMacros {
pub crate_name: Ident,
pub macros: Vec<String>,
}
pub fn expand_crate(parse_sess: &parse::ParseSess,
cfg: ExpansionConfig,
// these are the macros being imported to this crate:
imported_macros: Vec<ExportedMacros>,
imported_macros: Vec<ast::MacroDef>,
user_exts: Vec<NamedSyntaxExtension>,
c: Crate) -> Crate {
let mut cx = ExtCtxt::new(parse_sess, c.config.clone(), cfg);
let mut expander = MacroExpander::new(&mut cx);
for ExportedMacros { crate_name, macros } in imported_macros.into_iter() {
let name = format!("<{} macros>", token::get_ident(crate_name));
for def in imported_macros.iter() {
let ext = macro_rules::compile(expander.cx, def);
expander.cx.syntax_env.insert(def.ident.name, ext);
for source in macros.into_iter() {
let item = parse::parse_item_from_source_str(name.clone(),
source,
expander.cx.cfg(),
expander.cx.parse_sess())
.expect("expected a serialized item");
expand_item_mac(item, Some(crate_name), &mut expander);
if expander.cx.ecfg.reexported_macros.iter()
.any(|e| e[] == token::get_ident(def.ident).get()) {
expander.cx.exported_macros.push(def.clone());
}
}

29
src/libsyntax/ext/tt/macro_rules.rs
View file

@ -11,7 +11,7 @@
use ast::{Ident, TtDelimited, TtSequence, TtToken};
use ast;
use codemap::{Span, DUMMY_SP};
use ext::base::{ExtCtxt, MacResult, MacroDef};
use ext::base::{ExtCtxt, MacResult, SyntaxExtension};
use ext::base::{NormalTT, TTMacroExpander};
use ext::tt::macro_parser::{Success, Error, Failure};
use ext::tt::macro_parser::{NamedMatch, MatchedSeq, MatchedNonterminal};
@ -208,15 +208,9 @@ fn generic_extension<'cx>(cx: &'cx ExtCtxt,
//
// Holy self-referential!
/// This procedure performs the expansion of the
/// macro_rules! macro. It parses the RHS and adds
/// an extension to the current context.
pub fn add_new_extension<'cx>(cx: &'cx mut ExtCtxt,
sp: Span,
name: Ident,
imported_from: Option<Ident>,
arg: Vec<ast::TokenTree> )
-> MacroDef {
/// Converts a `macro_rules!` invocation into a syntax extension.
pub fn compile<'cx>(cx: &'cx mut ExtCtxt,
def: &ast::MacroDef) -> SyntaxExtension {
let lhs_nm = gensym_ident("lhs");
let rhs_nm = gensym_ident("rhs");
@ -254,7 +248,7 @@ pub fn add_new_extension<'cx>(cx: &'cx mut ExtCtxt,
let arg_reader = new_tt_reader(&cx.parse_sess().span_diagnostic,
None,
None,
arg.clone());
def.body.clone());
let argument_map = parse_or_else(cx.parse_sess(),
cx.cfg(),
arg_reader,
@ -263,23 +257,20 @@ pub fn add_new_extension<'cx>(cx: &'cx mut ExtCtxt,
// Extract the arguments:
let lhses = match *argument_map[lhs_nm] {
MatchedSeq(ref s, _) => /* FIXME (#2543) */ (*s).clone(),
_ => cx.span_bug(sp, "wrong-structured lhs")
_ => cx.span_bug(def.span, "wrong-structured lhs")
};
let rhses = match *argument_map[rhs_nm] {
MatchedSeq(ref s, _) => /* FIXME (#2543) */ (*s).clone(),
_ => cx.span_bug(sp, "wrong-structured rhs")
_ => cx.span_bug(def.span, "wrong-structured rhs")
};
let exp = box MacroRulesMacroExpander {
name: name,
imported_from: imported_from,
name: def.ident,
imported_from: def.imported_from,
lhses: lhses,
rhses: rhses,
};
MacroDef {
name: token::get_ident(name).to_string(),
ext: NormalTT(exp, Some(sp))
}
NormalTT(exp, Some(def.span))
}

6
src/libsyntax/fold.rs
View file

@ -1115,7 +1115,7 @@ pub fn noop_fold_mod<T: Folder>(Mod {inner, view_items, items}: Mod, folder: &mu
}
}
pub fn noop_fold_crate<T: Folder>(Crate {module, attrs, config, exported_macros, span}: Crate,
pub fn noop_fold_crate<T: Folder>(Crate {module, attrs, config, mut exported_macros, span}: Crate,
folder: &mut T) -> Crate {
let config = folder.fold_meta_items(config);
@ -1146,6 +1146,10 @@ pub fn noop_fold_crate<T: Folder>(Crate {module, attrs, config, exported_macros,
}, Vec::new(), span)
};
for def in exported_macros.iter_mut() {
def.id = folder.new_id(def.id);
}
Crate {
module: module,
attrs: attrs,

4
src/libsyntax/parse/mod.rs
View file

@ -169,6 +169,8 @@ pub fn parse_stmt_from_source_str(name: String,
// Note: keep in sync with `with_hygiene::parse_tts_from_source_str`
// until #16472 is resolved.
//
// Warning: This parses with quote_depth > 0, which is not the default.
pub fn parse_tts_from_source_str(name: String,
source: String,
cfg: ast::CrateConfig,
@ -310,6 +312,8 @@ pub mod with_hygiene {
// Note: keep this in sync with `super::parse_tts_from_source_str` until
// #16472 is resolved.
//
// Warning: This parses with quote_depth > 0, which is not the default.
pub fn parse_tts_from_source_str(name: String,
source: String,
cfg: ast::CrateConfig,