syntax: Use ast::MacArgs for macro definitions

src/librustc/hir/lowering/item.rs

@@ -233,7 +233,7 @@ impl LoweringContext<'_> {
 
         if let ItemKind::MacroDef(ref def) = i.kind {
             if !def.legacy || attr::contains_name(&i.attrs, sym::macro_export) {
-                let body = self.lower_token_stream(def.stream());
+                let body = self.lower_token_stream(def.body.inner_tokens());
                 let hir_id = self.lower_node_id(i.id);
                 self.exported_macros.push(hir::MacroDef {
                     name: ident.name,

src/librustc_lint/builtin.rs

@@ -1450,7 +1450,7 @@ impl KeywordIdents {
 
 impl EarlyLintPass for KeywordIdents {
     fn check_mac_def(&mut self, cx: &EarlyContext<'_>, mac_def: &ast::MacroDef, _id: ast::NodeId) {
-        self.check_tokens(cx, mac_def.stream());
+        self.check_tokens(cx, mac_def.body.inner_tokens());
     }
     fn check_mac(&mut self, cx: &EarlyContext<'_>, mac: &ast::Mac) {
         self.check_tokens(cx, mac.args.inner_tokens());

src/librustc_metadata/rmeta/decoder/cstore_impl.rs

@@ -32,6 +32,8 @@ use syntax::source_map;
 use syntax::source_map::Spanned;
 use syntax::symbol::Symbol;
 use syntax::expand::allocator::AllocatorKind;
+use syntax::ptr::P;
+use syntax::tokenstream::DelimSpan;
 use syntax_pos::{Span, FileName};
 
 macro_rules! provide {
@@ -427,6 +429,7 @@ impl CStore {
 
         let source_file = sess.parse_sess.source_map().new_source_file(source_name, def.body);
         let local_span = Span::with_root_ctxt(source_file.start_pos, source_file.end_pos);
+        let dspan = DelimSpan::from_single(local_span);
         let (body, mut errors) = source_file_to_stream(&sess.parse_sess, source_file, None);
         emit_unclosed_delims(&mut errors, &sess.parse_sess);
 
@@ -448,7 +451,7 @@ impl CStore {
             span: local_span,
             attrs: attrs.iter().cloned().collect(),
             kind: ast::ItemKind::MacroDef(ast::MacroDef {
-                tokens: body.into(),
+                body: P(ast::MacArgs::Delimited(dspan, ast::MacDelimiter::Brace, body)),
                 legacy: def.legacy,
             }),
             vis: source_map::respan(local_span.shrink_to_lo(), ast::VisibilityKind::Inherited),

src/librustc_parse/parser/item.rs

@@ -8,12 +8,12 @@ use syntax::ast::{ItemKind, ImplItem, ImplItemKind, TraitItem, TraitItemKind, Us
 use syntax::ast::{PathSegment, IsAuto, Constness, IsAsync, Unsafety, Defaultness, Extern, StrLit};
 use syntax::ast::{Visibility, VisibilityKind, Mutability, FnHeader, ForeignItem, ForeignItemKind};
 use syntax::ast::{Ty, TyKind, Generics, TraitRef, EnumDef, Variant, VariantData, StructField};
-use syntax::ast::{Mac, Block, BindingMode, FnDecl, FnSig, SelfKind, Param};
+use syntax::ast::{Mac, MacArgs, MacDelimiter, Block, BindingMode, FnDecl, FnSig, SelfKind, Param};
 use syntax::print::pprust;
 use syntax::ptr::P;
 use syntax::ThinVec;
 use syntax::token;
-use syntax::tokenstream::{TokenTree, TokenStream};
+use syntax::tokenstream::{DelimSpan, TokenTree, TokenStream};
 use syntax::source_map::{self, respan, Span};
 use syntax::struct_span_err;
 use syntax_pos::BytePos;
@@ -1617,33 +1617,31 @@ impl<'a> Parser<'a> {
         vis: &Visibility,
         lo: Span
     ) -> PResult<'a, Option<P<Item>>> {
-        let token_lo = self.token.span;
         let (ident, def) = if self.eat_keyword(kw::Macro) {
             let ident = self.parse_ident()?;
-            let tokens = if self.check(&token::OpenDelim(token::Brace)) {
+            let body = if self.check(&token::OpenDelim(token::Brace)) {
-                match self.parse_token_tree() {
+                self.parse_mac_args()?
-                    TokenTree::Delimited(_, _, tts) => tts,
-                    _ => unreachable!(),
-                }
             } else if self.check(&token::OpenDelim(token::Paren)) {
-                let args = self.parse_token_tree();
+                let params = self.parse_token_tree();
+                let pspan = params.span();
                 let body = if self.check(&token::OpenDelim(token::Brace)) {
                     self.parse_token_tree()
                 } else {
-                    self.unexpected()?;
+                    return self.unexpected();
-                    unreachable!()
                 };
-                TokenStream::new(vec![
+                let bspan = body.span();
-                    args.into(),
+                let tokens = TokenStream::new(vec![
-                    TokenTree::token(token::FatArrow, token_lo.to(self.prev_span)).into(),
+                    params.into(),
+                    TokenTree::token(token::FatArrow, pspan.between(bspan)).into(),
                     body.into(),
-                ])
+                ]);
+                let dspan = DelimSpan::from_pair(pspan.shrink_to_lo(), bspan.shrink_to_hi());
+                P(MacArgs::Delimited(dspan, MacDelimiter::Brace, tokens))
             } else {
-                self.unexpected()?;
+                return self.unexpected();
-                unreachable!()
             };
 
-            (ident, ast::MacroDef { tokens: tokens.into(), legacy: false })
+            (ident, ast::MacroDef { body, legacy: false })
         } else if self.check_keyword(sym::macro_rules) &&
                   self.look_ahead(1, |t| *t == token::Not) &&
                   self.look_ahead(2, |t| t.is_ident()) {
@@ -1653,12 +1651,12 @@ impl<'a> Parser<'a> {
             self.bump();
 
             let ident = self.parse_ident()?;
-            let args = self.parse_mac_args()?;
+            let body = self.parse_mac_args()?;
-            if args.need_semicolon() && !self.eat(&token::Semi) {
+            if body.need_semicolon() && !self.eat(&token::Semi) {
                 self.report_invalid_macro_expansion_item();
             }
 
-            (ident, ast::MacroDef { tokens: args.inner_tokens(), legacy: true })
+            (ident, ast::MacroDef { body, legacy: true })
         } else {
             return Ok(None);
         };

src/librustdoc/clean/inline.rs

@@ -482,7 +482,7 @@ fn build_macro(cx: &DocContext<'_>, did: DefId, name: ast::Name) -> clean::ItemE
     match cx.enter_resolver(|r| r.cstore().load_macro_untracked(did, cx.sess())) {
         LoadedMacro::MacroDef(def, _) => {
             let matchers: hir::HirVec<Span> = if let ast::ItemKind::MacroDef(ref def) = def.kind {
-                let tts: Vec<_> = def.stream().into_trees().collect();
+                let tts: Vec<_> = def.body.inner_tokens().into_trees().collect();
                 tts.chunks(4).map(|arm| arm[0].span()).collect()
             } else {
                 unreachable!()

src/libsyntax/ast.rs

@@ -1474,17 +1474,11 @@ impl MacDelimiter {
 /// Represents a macro definition.
 #[derive(Clone, RustcEncodable, RustcDecodable, Debug)]
 pub struct MacroDef {
-    pub tokens: TokenStream,
+    pub body: P<MacArgs>,
     /// `true` if macro was defined with `macro_rules`.
     pub legacy: bool,
 }
 
-impl MacroDef {
-    pub fn stream(&self) -> TokenStream {
-        self.tokens.clone().into()
-    }
-}
-
 // Clippy uses Hash and PartialEq
 #[derive(Clone, RustcEncodable, RustcDecodable, Debug, Copy, Hash, PartialEq, HashStable_Generic)]
 pub enum StrStyle {

src/libsyntax/mut_visit.rs

@@ -586,8 +586,8 @@ pub fn noop_visit_mac<T: MutVisitor>(mac: &mut Mac, vis: &mut T) {
 }
 
 pub fn noop_visit_macro_def<T: MutVisitor>(macro_def: &mut MacroDef, vis: &mut T) {
-    let MacroDef { tokens, legacy: _ } = macro_def;
+    let MacroDef { body, legacy: _ } = macro_def;
-    vis.visit_tts(tokens);
+    visit_mac_args(body, vis);
 }
 
 pub fn noop_visit_meta_list_item<T: MutVisitor>(li: &mut NestedMetaItem, vis: &mut T) {

src/libsyntax/print/pprust.rs

@@ -1380,8 +1380,8 @@ impl<'a> State<'a> {
                     Some(MacHeader::Keyword(kw)),
                     has_bang,
                     Some(item.ident),
-                    DelimToken::Brace,
+                    macro_def.body.delim(),
-                    macro_def.stream(),
+                    macro_def.body.inner_tokens(),
                     true,
                     item.span,
                 );

src/libsyntax_expand/mbe/macro_rules.rs

@@ -318,8 +318,8 @@ pub fn compile_declarative_macro(
     let tt_spec = ast::Ident::new(sym::tt, def.span);
 
     // Parse the macro_rules! invocation
-    let body = match def.kind {
+    let (is_legacy, body) = match &def.kind {
-        ast::ItemKind::MacroDef(ref body) => body,
+        ast::ItemKind::MacroDef(macro_def) => (macro_def.legacy, macro_def.body.inner_tokens()),
         _ => unreachable!(),
     };
 
@@ -338,7 +338,7 @@ pub fn compile_declarative_macro(
                     mbe::TokenTree::MetaVarDecl(def.span, rhs_nm, tt_spec),
                 ],
                 separator: Some(Token::new(
-                    if body.legacy { token::Semi } else { token::Comma },
+                    if is_legacy { token::Semi } else { token::Comma },
                     def.span,
                 )),
                 kleene: mbe::KleeneToken::new(mbe::KleeneOp::OneOrMore, def.span),
@@ -350,7 +350,7 @@ pub fn compile_declarative_macro(
             DelimSpan::dummy(),
             Lrc::new(mbe::SequenceRepetition {
                 tts: vec![mbe::TokenTree::token(
-                    if body.legacy { token::Semi } else { token::Comma },
+                    if is_legacy { token::Semi } else { token::Comma },
                     def.span,
                 )],
                 separator: None,
@@ -360,7 +360,7 @@ pub fn compile_declarative_macro(
         ),
     ];
 
-    let argument_map = match parse(sess, body.stream(), &argument_gram, None, true) {
+    let argument_map = match parse(sess, body, &argument_gram, None, true) {
         Success(m) => m,
         Failure(token, msg) => {
             let s = parse_failure_msg(&token);
@@ -435,7 +435,7 @@ pub fn compile_declarative_macro(
     // that is not lint-checked and trigger the "failed to process buffered lint here" bug.
     valid &= macro_check::check_meta_variables(sess, ast::CRATE_NODE_ID, def.span, &lhses, &rhses);
 
-    let (transparency, transparency_error) = attr::find_transparency(&def.attrs, body.legacy);
+    let (transparency, transparency_error) = attr::find_transparency(&def.attrs, is_legacy);
     match transparency_error {
         Some(TransparencyError::UnknownTransparency(value, span)) =>
             diag.span_err(span, &format!("unknown macro transparency: `{}`", value)),