diff --git a/src/librustc_plugin/registry.rs b/src/librustc_plugin/registry.rs index 8f0cc2c3d75..f8bce297a42 100644 --- a/src/librustc_plugin/registry.rs +++ b/src/librustc_plugin/registry.rs @@ -111,6 +111,8 @@ impl<'a> Registry<'a> { } MultiDecorator(ext) => MultiDecorator(ext), MultiModifier(ext) => MultiModifier(ext), + SyntaxExtension::ProcMacro(ext) => SyntaxExtension::ProcMacro(ext), + SyntaxExtension::AttrProcMacro(ext) => SyntaxExtension::AttrProcMacro(ext), })); } diff --git a/src/libsyntax/ext/base.rs b/src/libsyntax/ext/base.rs index 9d0d74138cd..82db9ffca83 100644 --- a/src/libsyntax/ext/base.rs +++ b/src/libsyntax/ext/base.rs @@ -8,7 +8,7 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. -pub use self::SyntaxExtension::*; +pub use self::SyntaxExtension::{MultiDecorator, MultiModifier, NormalTT, IdentTT, MacroRulesTT}; use ast::{self, Attribute, Name, PatKind}; use attr::HasAttrs; @@ -19,7 +19,7 @@ use ext::expand::{self, Invocation, Expansion}; use ext::hygiene::Mark; use ext::tt::macro_rules; use parse; -use parse::parser; +use parse::parser::{self, Parser}; use parse::token; use parse::token::{InternedString, str_to_ident}; use ptr::P; @@ -31,7 +31,8 @@ use feature_gate; use std::collections::HashMap; use std::path::PathBuf; use std::rc::Rc; -use tokenstream; +use std::default::Default; +use tokenstream::{self, TokenStream}; #[derive(Debug,Clone)] @@ -60,6 +61,14 @@ impl HasAttrs for Annotatable { } impl Annotatable { + pub fn span(&self) -> Span { + match *self { + Annotatable::Item(ref item) => item.span, + Annotatable::TraitItem(ref trait_item) => trait_item.span, + Annotatable::ImplItem(ref impl_item) => impl_item.span, + } + } + pub fn expect_item(self) -> P { match self { Annotatable::Item(i) => i, @@ -146,6 +155,173 @@ impl Into> for Annotatable { } } +pub trait ProcMacro { + fn expand<'cx>(&self, + ecx: &'cx mut ExtCtxt, + span: Span, + ts: TokenStream) + -> Box; +} + +impl ProcMacro for F + where F: Fn(TokenStream) -> TokenStream +{ + fn expand<'cx>(&self, + ecx: &'cx mut ExtCtxt, + span: Span, + ts: TokenStream) + -> Box { + let result = (*self)(ts); + // FIXME setup implicit context in TLS before calling self. + let parser = ecx.new_parser_from_tts(&result.to_tts()); + Box::new(TokResult { parser: parser, span: span }) + } +} + +pub trait AttrProcMacro { + fn expand<'cx>(&self, + ecx: &'cx mut ExtCtxt, + span: Span, + annotation: TokenStream, + annotated: TokenStream) + -> Box; +} + +impl AttrProcMacro for F + where F: Fn(TokenStream, TokenStream) -> TokenStream +{ + fn expand<'cx>(&self, + ecx: &'cx mut ExtCtxt, + span: Span, + annotation: TokenStream, + annotated: TokenStream) + -> Box { + // FIXME setup implicit context in TLS before calling self. + let parser = ecx.new_parser_from_tts(&(*self)(annotation, annotated).to_tts()); + Box::new(TokResult { parser: parser, span: span }) + } +} + +struct TokResult<'a> { + parser: Parser<'a>, + span: Span, +} + +impl<'a> MacResult for TokResult<'a> { + fn make_items(mut self: Box) -> Option>> { + if self.parser.sess.span_diagnostic.has_errors() { + return None; + } + + let mut items = SmallVector::zero(); + loop { + match self.parser.parse_item() { + Ok(Some(item)) => { + // FIXME better span info. + let mut item = item.unwrap(); + item.span = self.span; + items.push(P(item)); + } + Ok(None) => { + return Some(items); + } + Err(mut e) => { + e.emit(); + return None; + } + } + } + } + + fn make_impl_items(mut self: Box) -> Option> { + let mut items = SmallVector::zero(); + loop { + match self.parser.parse_impl_item() { + Ok(mut item) => { + // FIXME better span info. + item.span = self.span; + items.push(item); + + return Some(items); + } + Err(mut e) => { + e.emit(); + return None; + } + } + } + } + + fn make_trait_items(mut self: Box) -> Option> { + let mut items = SmallVector::zero(); + loop { + match self.parser.parse_trait_item() { + Ok(mut item) => { + // FIXME better span info. + item.span = self.span; + items.push(item); + + return Some(items); + } + Err(mut e) => { + e.emit(); + return None; + } + } + } + } + + fn make_expr(mut self: Box) -> Option> { + match self.parser.parse_expr() { + Ok(e) => Some(e), + Err(mut e) => { + e.emit(); + return None; + } + } + } + + fn make_pat(mut self: Box) -> Option> { + match self.parser.parse_pat() { + Ok(e) => Some(e), + Err(mut e) => { + e.emit(); + return None; + } + } + } + + fn make_stmts(mut self: Box) -> Option> { + let mut stmts = SmallVector::zero(); + loop { + if self.parser.token == token::Eof { + return Some(stmts); + } + match self.parser.parse_full_stmt(true) { + Ok(Some(mut stmt)) => { + stmt.span = self.span; + stmts.push(stmt); + } + Ok(None) => { /* continue */ } + Err(mut e) => { + e.emit(); + return None; + } + } + } + } + + fn make_ty(mut self: Box) -> Option> { + match self.parser.parse_ty() { + Ok(e) => Some(e), + Err(mut e) => { + e.emit(); + return None; + } + } + } +} + /// Represents a thing that maps token trees to Macro Results pub trait TTMacroExpander { fn expand<'cx>(&self, @@ -439,11 +615,22 @@ pub enum SyntaxExtension { /// based upon it. /// /// `#[derive(...)]` is a `MultiItemDecorator`. - MultiDecorator(Box), + /// + /// Prefer ProcMacro or MultiModifier since they are more flexible. + MultiDecorator(Box), /// A syntax extension that is attached to an item and modifies it - /// in-place. More flexible version than Modifier. - MultiModifier(Box), + /// in-place. Also allows decoration, i.e., creating new items. + MultiModifier(Box), + + /// A function-like procedural macro. TokenStream -> TokenStream. + ProcMacro(Box), + + /// An attribute-like procedural macro. TokenStream, TokenStream -> TokenStream. + /// The first TokenSteam is the attribute, the second is the annotated item. + /// Allows modification of the input items and adding new items, similar to + /// MultiModifier, but uses TokenStreams, rather than AST nodes. + AttrProcMacro(Box), /// A normal, function-like syntax extension. /// @@ -451,12 +638,12 @@ pub enum SyntaxExtension { /// /// The `bool` dictates whether the contents of the macro can /// directly use `#[unstable]` things (true == yes). - NormalTT(Box, Option, bool), + NormalTT(Box, Option, bool), /// A function-like syntax extension that has an extra ident before /// the block. /// - IdentTT(Box, Option, bool), + IdentTT(Box, Option, bool), } pub type NamedSyntaxExtension = (Name, SyntaxExtension); diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index 4e87d8ee9dd..f022dd3a08b 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -22,8 +22,9 @@ use feature_gate::{self, Features}; use fold; use fold::*; use parse::token::{intern, keywords}; +use parse::span_to_tts; use ptr::P; -use tokenstream::TokenTree; +use tokenstream::{TokenTree, TokenStream}; use util::small_vector::SmallVector; use visit::Visitor; @@ -308,6 +309,31 @@ impl<'a, 'b> MacroExpander<'a, 'b> { items.push(item); kind.expect_from_annotatables(items) } + SyntaxExtension::AttrProcMacro(ref mac) => { + let attr_toks = TokenStream::from_tts(span_to_tts(&fld.cx.parse_sess, + attr.span)); + let item_toks = TokenStream::from_tts(span_to_tts(&fld.cx.parse_sess, + item.span())); + let result = mac.expand(self.cx, attr.span, attr_toks, item_toks); + let items = match item { + Annotatable::Item(_) => result.make_items() + .unwrap_or(SmallVector::zero()) + .into_iter() + .map(|i| Annotatable::Item(i)) + .collect(), + Annotatable::TraitItem(_) => result.make_trait_items() + .unwrap_or(SmallVector::zero()) + .into_iter() + .map(|i| Annotatable::TraitItem(P(i))) + .collect(), + Annotatable::ImplItem(_) => result.make_impl_items() + .unwrap_or(SmallVector::zero()) + .into_iter() + .map(|i| Annotatable::ImplItem(P(i))) + .collect(), + }; + kind.expect_from_annotatables(items) + } _ => unreachable!(), } } @@ -377,11 +403,34 @@ impl<'a, 'b> MacroExpander<'a, 'b> { kind.make_from(expander.expand(self.cx, span, ident, marked_tts, attrs)) } - MultiDecorator(..) | MultiModifier(..) => { + MultiDecorator(..) | MultiModifier(..) | SyntaxExtension::AttrProcMacro(..) => { self.cx.span_err(path.span, &format!("`{}` can only be used in attributes", extname)); return kind.dummy(span); } + + SyntaxExtension::ProcMacro(ref expandfun) => { + if ident.name != keywords::Invalid.name() { + let msg = + format!("macro {}! expects no ident argument, given '{}'", extname, ident); + fld.cx.span_err(path.span, &msg); + return None; + } + + fld.cx.bt_push(ExpnInfo { + call_site: call_site, + callee: NameAndSpan { + format: MacroBang(extname), + // FIXME procedural macros do not have proper span info + // yet, when they do, we should use it here. + span: None, + // FIXME probably want to follow macro_rules macros here. + allow_internal_unstable: false, + }, + }); + + Some(expandfun.expand(fld.cx, call_site, TokenStream::from_tts(marked_tts))) + } }; let expanded = if let Some(expanded) = opt_expanded { diff --git a/src/libsyntax/ext/proc_macro_shim.rs b/src/libsyntax/ext/proc_macro_shim.rs index fa37e9b54e4..dc3a01f41bc 100644 --- a/src/libsyntax/ext/proc_macro_shim.rs +++ b/src/libsyntax/ext/proc_macro_shim.rs @@ -24,7 +24,9 @@ use ext::base::*; /// Take a `ExtCtxt`, `Span`, and `TokenStream`, and produce a Macro Result that parses /// the TokenStream as a block and returns it as an `Expr`. -pub fn build_block_emitter<'cx>(cx: &'cx mut ExtCtxt, sp: Span, output: TokenStream) +pub fn build_block_emitter<'cx>(cx: &'cx mut ExtCtxt, + sp: Span, + output: TokenStream) -> Box { let parser = cx.new_parser_from_tts(&output.to_tts()); @@ -60,7 +62,7 @@ pub fn build_block_emitter<'cx>(cx: &'cx mut ExtCtxt, sp: Span, output: TokenStr } pub mod prelude { - pub use ext::proc_macro_shim::build_block_emitter; + pub use super::build_block_emitter; pub use ast::Ident; pub use codemap::{DUMMY_SP, Span}; pub use ext::base::{ExtCtxt, MacResult}; diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs index 9e9ea096460..53294e78710 100644 --- a/src/libsyntax/parse/lexer/mod.rs +++ b/src/libsyntax/parse/lexer/mod.rs @@ -85,6 +85,12 @@ pub struct StringReader<'a> { /// The last character to be read pub curr: Option, pub filemap: Rc, + /// If Some, stop reading the source at this position (inclusive). + pub terminator: Option, + /// Whether to record new-lines in filemap. This is only necessary the first + /// time a filemap is lexed. If part of a filemap is being re-lexed, this + /// should be set to false. + pub save_new_lines: bool, // cached: pub peek_tok: token::Token, pub peek_span: Span, @@ -96,7 +102,15 @@ pub struct StringReader<'a> { impl<'a> Reader for StringReader<'a> { fn is_eof(&self) -> bool { - self.curr.is_none() + if self.curr.is_none() { + return true; + } + + match self.terminator { + Some(t) => self.pos > t, + None => false, + } + } /// Return the next token. EFFECT: advances the string_reader. fn try_next_token(&mut self) -> Result { @@ -164,6 +178,14 @@ impl<'a> StringReader<'a> { pub fn new_raw<'b>(span_diagnostic: &'b Handler, filemap: Rc) -> StringReader<'b> { + let mut sr = StringReader::new_raw_internal(span_diagnostic, filemap); + sr.bump(); + sr + } + + fn new_raw_internal<'b>(span_diagnostic: &'b Handler, + filemap: Rc) + -> StringReader<'b> { if filemap.src.is_none() { span_diagnostic.bug(&format!("Cannot lex filemap \ without source: {}", @@ -172,21 +194,21 @@ impl<'a> StringReader<'a> { let source_text = (*filemap.src.as_ref().unwrap()).clone(); - let mut sr = StringReader { + StringReader { span_diagnostic: span_diagnostic, pos: filemap.start_pos, last_pos: filemap.start_pos, col: CharPos(0), curr: Some('\n'), filemap: filemap, + terminator: None, + save_new_lines: true, // dummy values; not read peek_tok: token::Eof, peek_span: syntax_pos::DUMMY_SP, source_text: source_text, fatal_errs: Vec::new(), - }; - sr.bump(); - sr + } } pub fn new<'b>(span_diagnostic: &'b Handler, @@ -200,6 +222,28 @@ impl<'a> StringReader<'a> { sr } + pub fn from_span<'b>(span_diagnostic: &'b Handler, + span: Span, + codemap: &CodeMap) + -> StringReader<'b> { + let start_pos = codemap.lookup_byte_offset(span.lo); + let last_pos = codemap.lookup_byte_offset(span.hi); + assert!(start_pos.fm.name == last_pos.fm.name, "Attempt to lex span which crosses files"); + let mut sr = StringReader::new_raw_internal(span_diagnostic, start_pos.fm.clone()); + sr.pos = span.lo; + sr.last_pos = span.lo; + sr.terminator = Some(span.hi); + sr.save_new_lines = false; + + sr.bump(); + + if let Err(_) = sr.advance_token() { + sr.emit_fatal_errors(); + panic!(FatalError); + } + sr + } + pub fn curr_is(&self, c: char) -> bool { self.curr == Some(c) } @@ -405,7 +449,9 @@ impl<'a> StringReader<'a> { self.curr = Some(ch); self.col = self.col + CharPos(1); if last_char == '\n' { - self.filemap.next_line(self.last_pos); + if self.save_new_lines { + self.filemap.next_line(self.last_pos); + } self.col = CharPos(0); } diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index 5aa0efdec11..4ad8e227cbb 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -258,6 +258,13 @@ fn file_to_filemap(sess: &ParseSess, path: &Path, spanopt: Option) } } +pub fn span_to_tts(sess: &ParseSess, span: Span) -> Vec { + let cfg = Vec::new(); + let srdr = lexer::StringReader::from_span(&sess.span_diagnostic, span, &sess.code_map); + let mut p1 = Parser::new(sess, cfg, Box::new(srdr)); + panictry!(p1.parse_all_token_trees()) +} + /// Given a filemap, produce a sequence of token-trees pub fn filemap_to_tts(sess: &ParseSess, filemap: Rc) -> Vec { diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index 7b1df6f0e97..b35b4617ea1 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -33,6 +33,7 @@ use parse::lexer::comments::{doc_comment_style, strip_doc_comment_decoration}; use parse::lexer; use parse; use parse::token::{self, Token, Lit, Nonterminal}; +use print::pprust; use std::fmt; use std::iter::*; @@ -781,6 +782,12 @@ impl TokenStream { } } +impl fmt::Display for TokenStream { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str(&pprust::tts_to_string(&self.to_tts())) + } +} + // FIXME Reimplement this iterator to hold onto a slice iterator for a leaf, getting the // next leaf's iterator when the current one is exhausted. pub struct Iter<'a> { diff --git a/src/test/run-pass-fulldeps/auxiliary/proc_macro_def.rs b/src/test/run-pass-fulldeps/auxiliary/proc_macro_def.rs new file mode 100644 index 00000000000..52c38a6ee03 --- /dev/null +++ b/src/test/run-pass-fulldeps/auxiliary/proc_macro_def.rs @@ -0,0 +1,56 @@ +// Copyright 2016 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +#![feature(plugin, plugin_registrar, rustc_private)] + +extern crate proc_macro; +extern crate rustc_plugin; +extern crate syntax; + +use proc_macro::prelude::*; +use rustc_plugin::Registry; +use syntax::ext::base::SyntaxExtension; +use syntax::ext::proc_macro_shim::prelude::*; + +#[plugin_registrar] +pub fn plugin_registrar(reg: &mut Registry) { + reg.register_syntax_extension(token::intern("attr_tru"), + SyntaxExtension::AttrProcMacro(Box::new(attr_tru))); + reg.register_syntax_extension(token::intern("attr_identity"), + SyntaxExtension::AttrProcMacro(Box::new(attr_identity))); + reg.register_syntax_extension(token::intern("tru"), + SyntaxExtension::ProcMacro(Box::new(tru))); + reg.register_syntax_extension(token::intern("ret_tru"), + SyntaxExtension::ProcMacro(Box::new(ret_tru))); + reg.register_syntax_extension(token::intern("identity"), + SyntaxExtension::ProcMacro(Box::new(identity))); +} + +fn attr_tru(_attr: TokenStream, _item: TokenStream) -> TokenStream { + lex("fn f1() -> bool { true }") +} + +fn attr_identity(_attr: TokenStream, item: TokenStream) -> TokenStream { + let source = item.to_string(); + lex(&source) +} + +fn tru(_ts: TokenStream) -> TokenStream { + lex("true") +} + +fn ret_tru(_ts: TokenStream) -> TokenStream { + lex("return true;") +} + +fn identity(ts: TokenStream) -> TokenStream { + let source = ts.to_string(); + lex(&source) +} diff --git a/src/test/run-pass-fulldeps/proc_macro.rs b/src/test/run-pass-fulldeps/proc_macro.rs new file mode 100644 index 00000000000..22cc9f0f8d4 --- /dev/null +++ b/src/test/run-pass-fulldeps/proc_macro.rs @@ -0,0 +1,48 @@ +// Copyright 2016 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// aux-build:proc_macro_def.rs +// ignore-stage1 +// ignore-cross-compile + +#![feature(plugin, custom_attribute)] +#![feature(type_macros)] + +#![plugin(proc_macro_def)] + +#[attr_tru] +fn f1() -> bool { + return false; +} + +#[attr_identity] +fn f2() -> bool { + return identity!(true); +} + +fn f3() -> identity!(bool) { + ret_tru!(); +} + +fn f4(x: bool) -> bool { + match x { + identity!(true) => false, + identity!(false) => true, + } +} + +fn main() { + assert!(f1()); + assert!(f2()); + assert!(tru!()); + assert!(f3()); + assert!(identity!(5 == 5)); + assert!(f4(false)); +}