syntax: Use ast::MacArgs for attributes

src/librustc/hir/lowering.rs

@@ -1003,7 +1003,7 @@ impl<'a> LoweringContext<'a> {
             AttrKind::Normal(ref item) => {
                 AttrKind::Normal(AttrItem {
                     path: item.path.clone(),
-                    tokens: self.lower_token_stream(item.tokens.clone()),
+                    args: self.lower_mac_args(&item.args),
                 })
             }
             AttrKind::DocComment(comment) => AttrKind::DocComment(comment)
@@ -1017,6 +1017,16 @@ impl<'a> LoweringContext<'a> {
         }
     }
 
+    fn lower_mac_args(&mut self, args: &MacArgs) -> MacArgs {
+        match *args {
+            MacArgs::Empty => MacArgs::Empty,
+            MacArgs::Delimited(dspan, delim, ref tokens) =>
+                MacArgs::Delimited(dspan, delim, self.lower_token_stream(tokens.clone())),
+            MacArgs::Eq(eq_span, ref tokens) =>
+                MacArgs::Eq(eq_span, self.lower_token_stream(tokens.clone())),
+        }
+    }
+
     fn lower_token_stream(&mut self, tokens: TokenStream) -> TokenStream {
         tokens
             .into_trees()

src/librustc_parse/config.rs

@@ -101,7 +101,7 @@ impl<'a> StripUnconfigured<'a> {
         if !attr.has_name(sym::cfg_attr) {
             return vec![attr];
         }
-        if attr.get_normal_item().tokens.is_empty() {
+        if let ast::MacArgs::Empty = attr.get_normal_item().args {
             self.sess.span_diagnostic
                 .struct_span_err(
                     attr.span,

src/librustc_parse/lib.rs

@@ -277,7 +277,7 @@ pub fn parse_in_attr<'a, T>(
 ) -> PResult<'a, T> {
     let mut parser = Parser::new(
         sess,
-        attr.get_normal_item().tokens.clone(),
+        attr.get_normal_item().args.outer_tokens(),
         None,
         false,
         false,
@@ -409,7 +409,7 @@ fn prepend_attrs(
             brackets.push(stream);
         }
 
-        brackets.push(item.tokens.clone());
+        brackets.push(item.args.outer_tokens());
 
         // The span we list here for `#` and for `[ ... ]` are both wrong in
         // that it encompasses more than each token, but it hopefully is "good

src/librustc_parse/parser/attr.rs

@@ -2,8 +2,7 @@ use super::{SeqSep, Parser, TokenType, PathStyle};
 use syntax::attr;
 use syntax::ast;
 use syntax::util::comments;
-use syntax::token::{self, Nonterminal, DelimToken};
+use syntax::token::{self, Nonterminal};
-use syntax::tokenstream::{TokenStream, TokenTree};
 use syntax_pos::{Span, Symbol};
 use errors::PResult;
 
@@ -181,31 +180,8 @@ impl<'a> Parser<'a> {
             item
         } else {
             let path = self.parse_path(PathStyle::Mod)?;
-            let tokens = if self.check(&token::OpenDelim(DelimToken::Paren)) ||
+            let args = self.parse_attr_args()?;
-               self.check(&token::OpenDelim(DelimToken::Bracket)) ||
+            ast::AttrItem { path, args }
-               self.check(&token::OpenDelim(DelimToken::Brace)) {
-                   self.parse_token_tree().into()
-            } else if self.eat(&token::Eq) {
-                let eq = TokenTree::token(token::Eq, self.prev_span);
-                let mut is_interpolated_expr = false;
-                if let token::Interpolated(nt) = &self.token.kind {
-                    if let token::NtExpr(..) = **nt {
-                        is_interpolated_expr = true;
-                    }
-                }
-                let token_tree = if is_interpolated_expr {
-                    // We need to accept arbitrary interpolated expressions to continue
-                    // supporting things like `doc = $expr` that work on stable.
-                    // Non-literal interpolated expressions are rejected after expansion.
-                    self.parse_token_tree()
-                } else {
-                    self.parse_unsuffixed_lit()?.token_tree()
-                };
-                TokenStream::new(vec![eq.into(), token_tree.into()])
-            } else {
-                TokenStream::default()
-            };
-            ast::AttrItem { path, tokens }
         })
     }
 

src/librustc_parse/parser/mod.rs

@@ -1011,16 +1011,15 @@ impl<'a> Parser<'a> {
     }
 
     fn parse_mac_args(&mut self) -> PResult<'a, P<MacArgs>> {
-        self.parse_mac_args_common(true)
+        self.parse_mac_args_common(true).map(P)
     }
 
-    #[allow(dead_code)]
+    fn parse_attr_args(&mut self) -> PResult<'a, MacArgs> {
-    fn parse_attr_args(&mut self) -> PResult<'a, P<MacArgs>> {
         self.parse_mac_args_common(false)
     }
 
-    fn parse_mac_args_common(&mut self, delimited_only: bool) -> PResult<'a, P<MacArgs>> {
+    fn parse_mac_args_common(&mut self, delimited_only: bool) -> PResult<'a, MacArgs> {
-        Ok(P(if self.check(&token::OpenDelim(DelimToken::Paren)) ||
+        Ok(if self.check(&token::OpenDelim(DelimToken::Paren)) ||
                        self.check(&token::OpenDelim(DelimToken::Bracket)) ||
                        self.check(&token::OpenDelim(DelimToken::Brace)) {
             match self.parse_token_tree() {
@@ -1052,7 +1051,7 @@ impl<'a> Parser<'a> {
             }
         } else {
             return self.unexpected();
-        }))
+        })
     }
 
     fn parse_or_use_outer_attributes(

src/librustc_parse/parser/path.rs

@@ -2,6 +2,7 @@ use super::{Parser, TokenType};
 use crate::maybe_whole;
 use syntax::ast::{self, QSelf, Path, PathSegment, Ident, ParenthesizedArgs, AngleBracketedArgs};
 use syntax::ast::{AnonConst, GenericArg, AssocTyConstraint, AssocTyConstraintKind, BlockCheckMode};
+use syntax::ast::MacArgs;
 use syntax::ThinVec;
 use syntax::token::{self, Token};
 use syntax::source_map::{Span, BytePos};
@@ -114,9 +115,9 @@ impl<'a> Parser<'a> {
     fn parse_path_allowing_meta(&mut self, style: PathStyle) -> PResult<'a, Path> {
         let meta_ident = match self.token.kind {
             token::Interpolated(ref nt) => match **nt {
-                token::NtMeta(ref item) => match item.tokens.is_empty() {
+                token::NtMeta(ref item) => match item.args {
-                    true => Some(item.path.clone()),
+                    MacArgs::Empty => Some(item.path.clone()),
-                    false => None,
+                    _ => None,
                 },
                 _ => None,
             },

src/librustc_parse/validate_attr.rs

@@ -2,11 +2,9 @@
 
 use errors::{PResult, Applicability};
 use rustc_feature::{AttributeTemplate, BUILTIN_ATTRIBUTE_MAP};
-use syntax::ast::{self, Attribute, AttrKind, Ident, MetaItem, MetaItemKind};
+use syntax::ast::{self, Attribute, AttrKind, Ident, MacArgs, MetaItem, MetaItemKind};
 use syntax::attr::mk_name_value_item_str;
 use syntax::early_buffered_lints::BufferedEarlyLintId;
-use syntax::token;
-use syntax::tokenstream::TokenTree;
 use syntax::sess::ParseSess;
 use syntax_pos::{Symbol, sym};
 
@@ -19,11 +17,9 @@ pub fn check_meta(sess: &ParseSess, attr: &Attribute) {
         // `rustc_dummy` doesn't have any restrictions specific to built-in attributes.
         Some((name, _, template, _)) if name != sym::rustc_dummy =>
             check_builtin_attribute(sess, attr, name, template),
-        _ => if let Some(TokenTree::Token(token)) = attr.get_normal_item().tokens.trees().next() {
+        _ => if let MacArgs::Eq(..) = attr.get_normal_item().args {
-            if token == token::Eq {
+            // All key-value attributes are restricted to meta-item syntax.
-                // All key-value attributes are restricted to meta-item syntax.
+            parse_meta(sess, attr).map_err(|mut err| err.emit()).ok();
-                parse_meta(sess, attr).map_err(|mut err| err.emit()).ok();
-            }
         }
     }
 }

src/libsyntax/ast.rs

@@ -1389,7 +1389,7 @@ impl Mac {
 }
 
 /// Arguments passed to an attribute or a function-like macro.
-#[derive(Clone, RustcEncodable, RustcDecodable, Debug)]
+#[derive(Clone, RustcEncodable, RustcDecodable, Debug, HashStable_Generic)]
 pub enum MacArgs {
     /// No arguments - `#[attr]`.
     Empty,
@@ -1427,7 +1427,7 @@ impl MacArgs {
     }
 
     /// Tokens together with the delimiters or `=`.
-    /// Use of this functions generally means that something suspicious or hacky is happening.
+    /// Use of this functions generally means that something suboptimal or hacky is happening.
     pub fn outer_tokens(&self) -> TokenStream {
         match *self {
             MacArgs::Empty => TokenStream::default(),
@@ -1445,7 +1445,7 @@ impl MacArgs {
     }
 }
 
-#[derive(Copy, Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Debug)]
+#[derive(Copy, Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Debug, HashStable_Generic)]
 pub enum MacDelimiter {
     Parenthesis,
     Bracket,
@@ -2384,7 +2384,7 @@ impl rustc_serialize::Decodable for AttrId {
 #[derive(Clone, RustcEncodable, RustcDecodable, Debug, HashStable_Generic)]
 pub struct AttrItem {
     pub path: Path,
-    pub tokens: TokenStream,
+    pub args: MacArgs,
 }
 
 /// Metadata associated with an item.

src/libsyntax/attr/mod.rs

@@ -10,7 +10,7 @@ pub use crate::ast::Attribute;
 
 use crate::ast;
 use crate::ast::{AttrItem, AttrId, AttrKind, AttrStyle, Name, Ident, Path, PathSegment};
-use crate::ast::{MetaItem, MetaItemKind, NestedMetaItem};
+use crate::ast::{MacArgs, MacDelimiter, MetaItem, MetaItemKind, NestedMetaItem};
 use crate::ast::{Lit, LitKind, Expr, Item, Local, Stmt, StmtKind, GenericParam};
 use crate::mut_visit::visit_clobber;
 use crate::source_map::{BytePos, Spanned};
@@ -198,7 +198,7 @@ impl Attribute {
 
     pub fn is_word(&self) -> bool {
         if let AttrKind::Normal(item) = &self.kind {
-            item.tokens.is_empty()
+            matches!(item.args, MacArgs::Empty)
         } else {
             false
         }
@@ -278,7 +278,7 @@ impl MetaItem {
 
 impl AttrItem {
     pub fn meta(&self, span: Span) -> Option<MetaItem> {
-        let mut tokens = self.tokens.trees().peekable();
+        let mut tokens = self.args.outer_tokens().trees().peekable();
         Some(MetaItem {
             path: self.path.clone(),
             kind: if let Some(kind) = MetaItemKind::from_tokens(&mut tokens) {
@@ -362,8 +362,8 @@ crate fn mk_attr_id() -> AttrId {
     AttrId(id)
 }
 
-pub fn mk_attr(style: AttrStyle, path: Path, tokens: TokenStream, span: Span) -> Attribute {
+pub fn mk_attr(style: AttrStyle, path: Path, args: MacArgs, span: Span) -> Attribute {
-    mk_attr_from_item(style, AttrItem { path, tokens }, span)
+    mk_attr_from_item(style, AttrItem { path, args }, span)
 }
 
 pub fn mk_attr_from_item(style: AttrStyle, item: AttrItem, span: Span) -> Attribute {
@@ -377,12 +377,12 @@ pub fn mk_attr_from_item(style: AttrStyle, item: AttrItem, span: Span) -> Attrib
 
 /// Returns an inner attribute with the given value and span.
 pub fn mk_attr_inner(item: MetaItem) -> Attribute {
-    mk_attr(AttrStyle::Inner, item.path, item.kind.tokens(item.span), item.span)
+    mk_attr(AttrStyle::Inner, item.path, item.kind.mac_args(item.span), item.span)
 }
 
 /// Returns an outer attribute with the given value and span.
 pub fn mk_attr_outer(item: MetaItem) -> Attribute {
-    mk_attr(AttrStyle::Outer, item.path, item.kind.tokens(item.span), item.span)
+    mk_attr(AttrStyle::Outer, item.path, item.kind.mac_args(item.span), item.span)
 }
 
 pub fn mk_doc_comment(style: AttrStyle, comment: Symbol, span: Span) -> Attribute {
@@ -520,7 +520,26 @@ impl MetaItem {
 }
 
 impl MetaItemKind {
-    pub fn token_trees_and_joints(&self, span: Span) -> Vec<TreeAndJoint> {
+    pub fn mac_args(&self, span: Span) -> MacArgs {
+        match self {
+            MetaItemKind::Word => MacArgs::Empty,
+            MetaItemKind::NameValue(lit) => MacArgs::Eq(span, lit.token_tree().into()),
+            MetaItemKind::List(list) => {
+                let mut tts = Vec::new();
+                for (i, item) in list.iter().enumerate() {
+                    if i > 0 {
+                        tts.push(TokenTree::token(token::Comma, span).into());
+                    }
+                    tts.extend(item.token_trees_and_joints())
+                }
+                MacArgs::Delimited(
+                    DelimSpan::from_single(span), MacDelimiter::Parenthesis, TokenStream::new(tts)
+                )
+            }
+        }
+    }
+
+    fn token_trees_and_joints(&self, span: Span) -> Vec<TreeAndJoint> {
         match *self {
             MetaItemKind::Word => vec![],
             MetaItemKind::NameValue(ref lit) => {
@@ -548,13 +567,6 @@ impl MetaItemKind {
         }
     }
 
-    // Premature conversions of `TokenTree`s to `TokenStream`s can hurt
-    // performance. Do not use this function if `token_trees_and_joints()` can
-    // be used instead.
-    pub fn tokens(&self, span: Span) -> TokenStream {
-        TokenStream::new(self.token_trees_and_joints(span))
-    }
-
     fn from_tokens<I>(tokens: &mut iter::Peekable<I>) -> Option<MetaItemKind>
         where I: Iterator<Item = TokenTree>,
     {

src/libsyntax/mut_visit.rs

@@ -570,9 +570,9 @@ pub fn noop_visit_local<T: MutVisitor>(local: &mut P<Local>, vis: &mut T) {
 pub fn noop_visit_attribute<T: MutVisitor>(attr: &mut Attribute, vis: &mut T) {
     let Attribute { kind, id: _, style: _, span } = attr;
     match kind {
-        AttrKind::Normal(AttrItem { path, tokens }) => {
+        AttrKind::Normal(AttrItem { path, args }) => {
             vis.visit_path(path);
-            vis.visit_tts(tokens);
+            visit_mac_args(args, vis);
         }
         AttrKind::DocComment(_) => {}
     }
@@ -701,9 +701,9 @@ pub fn noop_visit_interpolated<T: MutVisitor>(nt: &mut token::Nonterminal, vis:
         token::NtIdent(ident, _is_raw) => vis.visit_ident(ident),
         token::NtLifetime(ident) => vis.visit_ident(ident),
         token::NtLiteral(expr) => vis.visit_expr(expr),
-        token::NtMeta(AttrItem { path, tokens }) => {
+        token::NtMeta(AttrItem { path, args }) => {
             vis.visit_path(path);
-            vis.visit_tts(tokens);
+            visit_mac_args(args, vis);
         }
         token::NtPath(path) => vis.visit_path(path),
         token::NtTT(tt) => vis.visit_tt(tt),

src/libsyntax/print/pprust.rs

@@ -1,6 +1,6 @@
 use crate::ast::{self, BlockCheckMode, PatKind, RangeEnd, RangeSyntax};
 use crate::ast::{SelfKind, GenericBound, TraitBoundModifier};
-use crate::ast::{Attribute, GenericArg};
+use crate::ast::{Attribute, GenericArg, MacArgs};
 use crate::util::parser::{self, AssocOp, Fixity};
 use crate::util::comments;
 use crate::attr;
@@ -639,17 +639,22 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
 
     fn print_attr_item(&mut self, item: &ast::AttrItem, span: Span) {
         self.ibox(0);
-        match item.tokens.trees().next() {
+        match &item.args {
-            Some(TokenTree::Delimited(_, delim, tts)) => {
+            MacArgs::Delimited(_, delim, tokens) => self.print_mac_common(
-                self.print_mac_common(
+                Some(MacHeader::Path(&item.path)),
-                    Some(MacHeader::Path(&item.path)), false, None, delim, tts, true, span
+                false,
-                );
+                None,
-            }
+                delim.to_token(),
-            tree => {
+                tokens.clone(),
+                true,
+                span,
+            ),
+            MacArgs::Empty | MacArgs::Eq(..) => {
                 self.print_path(&item.path, false, 0);
-                if tree.is_some() {
+                if let MacArgs::Eq(_, tokens) = &item.args {
                     self.space();
-                    self.print_tts(item.tokens.clone(), true);
+                    self.word_space("=");
+                    self.print_tts(tokens.clone(), true);
                 }
             }
         }

src/libsyntax/visit.rs

@@ -841,11 +841,19 @@ pub fn walk_vis<'a, V: Visitor<'a>>(visitor: &mut V, vis: &'a Visibility) {
 
 pub fn walk_attribute<'a, V: Visitor<'a>>(visitor: &mut V, attr: &'a Attribute) {
     match attr.kind {
-        AttrKind::Normal(ref item) => visitor.visit_tts(item.tokens.clone()),
+        AttrKind::Normal(ref item) => walk_mac_args(visitor, &item.args),
         AttrKind::DocComment(_) => {}
     }
 }
 
+pub fn walk_mac_args<'a, V: Visitor<'a>>(visitor: &mut V, args: &'a MacArgs) {
+    match args {
+        MacArgs::Empty => {}
+        MacArgs::Delimited(_dspan, _delim, tokens) => visitor.visit_tts(tokens.clone()),
+        MacArgs::Eq(_eq_span, tokens) => visitor.visit_tts(tokens.clone()),
+    }
+}
+
 pub fn walk_tt<'a, V: Visitor<'a>>(visitor: &mut V, tt: TokenTree) {
     match tt {
         TokenTree::Token(token) => visitor.visit_token(token),

src/libsyntax_expand/expand.rs

@@ -11,7 +11,7 @@ use rustc_parse::DirectoryOwnership;
 use rustc_parse::parser::Parser;
 use rustc_parse::validate_attr;
 use syntax::ast::{self, AttrItem, Block, Ident, LitKind, NodeId, PatKind, Path};
-use syntax::ast::{MacStmtStyle, StmtKind, ItemKind};
+use syntax::ast::{MacArgs, MacStmtStyle, StmtKind, ItemKind};
 use syntax::attr::{self, HasAttrs, is_builtin_attr};
 use syntax::source_map::respan;
 use syntax::feature_gate::{self, feature_err};
@@ -642,8 +642,11 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                             => panic!("unexpected annotatable"),
                     })), DUMMY_SP).into();
                     let item = attr.unwrap_normal_item();
-                    let input = self.extract_proc_macro_attr_input(item.tokens, span);
+                    if let MacArgs::Eq(..) = item.args {
-                    let tok_result = expander.expand(self.cx, span, input, item_tok);
+                        self.cx.span_err(span, "key-value macro attributes are not supported");
+                    }
+                    let tok_result =
+                        expander.expand(self.cx, span, item.args.inner_tokens(), item_tok);
                     self.parse_ast_fragment(tok_result, fragment_kind, &item.path, span)
                 }
                 SyntaxExtensionKind::LegacyAttr(expander) => {
@@ -687,23 +690,6 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
         }
     }
 
-    fn extract_proc_macro_attr_input(&self, tokens: TokenStream, span: Span) -> TokenStream {
-        let mut trees = tokens.trees();
-        match trees.next() {
-            Some(TokenTree::Delimited(_, _, tts)) => {
-                if trees.next().is_none() {
-                    return tts.into()
-                }
-            }
-            Some(TokenTree::Token(..)) => {}
-            None => return TokenStream::default(),
-        }
-        self.cx.span_err(span, "custom attribute invocations must be \
-            of the form `#[foo]` or `#[foo(..)]`, the macro name must only be \
-            followed by a delimiter token");
-        TokenStream::default()
-    }
-
     fn gate_proc_macro_attr_item(&self, span: Span, item: &Annotatable) {
         let kind = match item {
             Annotatable::Item(item) => match &item.kind {
@@ -1560,7 +1546,7 @@ impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> {
             let meta = attr::mk_list_item(Ident::with_dummy_span(sym::doc), items);
             *at = attr::Attribute {
                 kind: ast::AttrKind::Normal(
-                    AttrItem { path: meta.path, tokens: meta.kind.tokens(meta.span) },
+                    AttrItem { path: meta.path, args: meta.kind.mac_args(meta.span) },
                 ),
                 span: at.span,
                 id: at.id,

src/libsyntax_expand/proc_macro.rs

@@ -1,7 +1,7 @@
 use crate::base::{self, *};
 use crate::proc_macro_server;
 
-use syntax::ast::{self, ItemKind};
+use syntax::ast::{self, ItemKind, MacArgs};
 use syntax::errors::{Applicability, FatalError};
 use syntax::symbol::sym;
 use syntax::token;
@@ -183,7 +183,7 @@ crate fn collect_derives(cx: &mut ExtCtxt<'_>, attrs: &mut Vec<ast::Attribute>)
         }
 
         let parse_derive_paths = |attr: &ast::Attribute| {
-            if attr.get_normal_item().tokens.is_empty() {
+            if let MacArgs::Empty = attr.get_normal_item().args {
                 return Ok(Vec::new());
             }
             rustc_parse::parse_in_attr(cx.parse_sess, attr, |p| p.parse_derive_paths())

src/libsyntax_ext/cmdline_attrs.rs

@@ -16,7 +16,7 @@ pub fn inject(mut krate: ast::Crate, parse_sess: &ParseSess, attrs: &[String]) -
         );
 
         let start_span = parser.token.span;
-        let AttrItem { path, tokens } = panictry!(parser.parse_attr_item());
+        let AttrItem { path, args } = panictry!(parser.parse_attr_item());
         let end_span = parser.token.span;
         if parser.token != token::Eof {
             parse_sess.span_diagnostic
@@ -24,7 +24,7 @@ pub fn inject(mut krate: ast::Crate, parse_sess: &ParseSess, attrs: &[String]) -
             continue;
         }
 
-        krate.attrs.push(mk_attr(AttrStyle::Inner, path, tokens, start_span.to(end_span)));
+        krate.attrs.push(mk_attr(AttrStyle::Inner, path, args, start_span.to(end_span)));
     }
 
     krate

src/test/ui/proc-macro/proc-macro-gates.rs

@@ -18,7 +18,7 @@ mod _test2_inner {
           //~| ERROR: non-builtin inner attributes are unstable
 }
 
-#[empty_attr = "y"] //~ ERROR: must only be followed by a delimiter token
+#[empty_attr = "y"] //~ ERROR: key-value macro attributes are not supported
 fn _test3() {}
 
 fn attrs() {

src/test/ui/proc-macro/proc-macro-gates.stderr

@@ -34,7 +34,7 @@ LL |     #![empty_attr]
    = note: for more information, see https://github.com/rust-lang/rust/issues/54727
    = help: add `#![feature(proc_macro_hygiene)]` to the crate attributes to enable
 
-error: custom attribute invocations must be of the form `#[foo]` or `#[foo(..)]`, the macro name must only be followed by a delimiter token
+error: key-value macro attributes are not supported
   --> $DIR/proc-macro-gates.rs:21:1
    |
 LL | #[empty_attr = "y"]