Do not collect tokens for doc comments

compiler/rustc_ast/src/ast.rs

@@ -2439,13 +2439,12 @@ pub struct Attribute {
     /// or the construct this attribute is contained within (inner).
     pub style: AttrStyle,
     pub span: Span,
-    pub tokens: Option<LazyTokenStream>,
 }
 
 #[derive(Clone, Encodable, Decodable, Debug)]
 pub enum AttrKind {
     /// A normal attribute.
-    Normal(AttrItem),
+    Normal(AttrItem, Option<LazyTokenStream>),
 
     /// A doc comment (e.g. `/// ...`, `//! ...`, `/** ... */`, `/*! ... */`).
     /// Doc attributes (e.g. `#[doc="..."]`) are represented with the `Normal`

compiler/rustc_ast/src/attr/mod.rs

@@ -8,7 +8,7 @@ use crate::ast::{Path, PathSegment};
 use crate::mut_visit::visit_clobber;
 use crate::ptr::P;
 use crate::token::{self, CommentKind, Token};
-use crate::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndSpacing};
+use crate::tokenstream::{DelimSpan, LazyTokenStream, TokenStream, TokenTree, TreeAndSpacing};
 
 use rustc_index::bit_set::GrowableBitSet;
 use rustc_span::source_map::{BytePos, Spanned};
@@ -120,7 +120,7 @@ impl NestedMetaItem {
 impl Attribute {
     pub fn has_name(&self, name: Symbol) -> bool {
         match self.kind {
-            AttrKind::Normal(ref item) => item.path == name,
+            AttrKind::Normal(ref item, _) => item.path == name,
             AttrKind::DocComment(..) => false,
         }
     }
@@ -128,7 +128,7 @@ impl Attribute {
     /// For a single-segment attribute, returns its name; otherwise, returns `None`.
     pub fn ident(&self) -> Option<Ident> {
         match self.kind {
-            AttrKind::Normal(ref item) => {
+            AttrKind::Normal(ref item, _) => {
                 if item.path.segments.len() == 1 {
                     Some(item.path.segments[0].ident)
                 } else {
@@ -144,14 +144,14 @@ impl Attribute {
 
     pub fn value_str(&self) -> Option<Symbol> {
         match self.kind {
-            AttrKind::Normal(ref item) => item.meta(self.span).and_then(|meta| meta.value_str()),
+            AttrKind::Normal(ref item, _) => item.meta(self.span).and_then(|meta| meta.value_str()),
             AttrKind::DocComment(..) => None,
         }
     }
 
     pub fn meta_item_list(&self) -> Option<Vec<NestedMetaItem>> {
         match self.kind {
-            AttrKind::Normal(ref item) => match item.meta(self.span) {
+            AttrKind::Normal(ref item, _) => match item.meta(self.span) {
                 Some(MetaItem { kind: MetaItemKind::List(list), .. }) => Some(list),
                 _ => None,
             },
@@ -160,7 +160,7 @@ impl Attribute {
     }
 
     pub fn is_word(&self) -> bool {
-        if let AttrKind::Normal(item) = &self.kind {
+        if let AttrKind::Normal(item, _) = &self.kind {
             matches!(item.args, MacArgs::Empty)
         } else {
             false
@@ -246,7 +246,7 @@ impl AttrItem {
 impl Attribute {
     pub fn is_doc_comment(&self) -> bool {
         match self.kind {
-            AttrKind::Normal(_) => false,
+            AttrKind::Normal(..) => false,
             AttrKind::DocComment(..) => true,
         }
     }
@@ -254,7 +254,7 @@ impl Attribute {
     pub fn doc_str(&self) -> Option<Symbol> {
         match self.kind {
             AttrKind::DocComment(.., data) => Some(data),
-            AttrKind::Normal(ref item) if item.path == sym::doc => {
+            AttrKind::Normal(ref item, _) if item.path == sym::doc => {
                 item.meta(self.span).and_then(|meta| meta.value_str())
             }
             _ => None,
@@ -263,14 +263,14 @@ impl Attribute {
 
     pub fn get_normal_item(&self) -> &AttrItem {
         match self.kind {
-            AttrKind::Normal(ref item) => item,
+            AttrKind::Normal(ref item, _) => item,
             AttrKind::DocComment(..) => panic!("unexpected doc comment"),
         }
     }
 
     pub fn unwrap_normal_item(self) -> AttrItem {
         match self.kind {
-            AttrKind::Normal(item) => item,
+            AttrKind::Normal(item, _) => item,
             AttrKind::DocComment(..) => panic!("unexpected doc comment"),
         }
     }
@@ -278,10 +278,22 @@ impl Attribute {
     /// Extracts the MetaItem from inside this Attribute.
     pub fn meta(&self) -> Option<MetaItem> {
         match self.kind {
-            AttrKind::Normal(ref item) => item.meta(self.span),
+            AttrKind::Normal(ref item, _) => item.meta(self.span),
             AttrKind::DocComment(..) => None,
         }
     }
+
+    pub fn tokens(&self) -> TokenStream {
+        match self.kind {
+            AttrKind::Normal(_, ref tokens) => tokens
+                .as_ref()
+                .unwrap_or_else(|| panic!("attribute is missing tokens: {:?}", self))
+                .create_token_stream(),
+            AttrKind::DocComment(comment_kind, data) => TokenStream::from(TokenTree::Token(
+                Token::new(token::DocComment(comment_kind, self.style, data), self.span),
+            )),
+        }
+    }
 }
 
 /* Constructors */
@@ -321,11 +333,16 @@ crate fn mk_attr_id() -> AttrId {
 }
 
 pub fn mk_attr(style: AttrStyle, path: Path, args: MacArgs, span: Span) -> Attribute {
-    mk_attr_from_item(style, AttrItem { path, args, tokens: None }, span)
+    mk_attr_from_item(AttrItem { path, args, tokens: None }, None, style, span)
 }
 
-pub fn mk_attr_from_item(style: AttrStyle, item: AttrItem, span: Span) -> Attribute {
-    Attribute { kind: AttrKind::Normal(item), id: mk_attr_id(), style, span, tokens: None }
+pub fn mk_attr_from_item(
+    item: AttrItem,
+    tokens: Option<LazyTokenStream>,
+    style: AttrStyle,
+    span: Span,
+) -> Attribute {
+    Attribute { kind: AttrKind::Normal(item, tokens), id: mk_attr_id(), style, span }
 }
 
 /// Returns an inner attribute with the given value and span.
@@ -344,13 +361,7 @@ pub fn mk_doc_comment(
     data: Symbol,
     span: Span,
 ) -> Attribute {
-    Attribute {
-        kind: AttrKind::DocComment(comment_kind, data),
-        id: mk_attr_id(),
-        style,
-        span,
-        tokens: None,
-    }
+    Attribute { kind: AttrKind::DocComment(comment_kind, data), id: mk_attr_id(), style, span }
 }
 
 pub fn list_contains_name(items: &[NestedMetaItem], name: Symbol) -> bool {

compiler/rustc_ast/src/mut_visit.rs

@@ -589,17 +589,17 @@ pub fn noop_visit_local<T: MutVisitor>(local: &mut P<Local>, vis: &mut T) {
 }
 
 pub fn noop_visit_attribute<T: MutVisitor>(attr: &mut Attribute, vis: &mut T) {
-    let Attribute { kind, id: _, style: _, span, tokens } = attr;
+    let Attribute { kind, id: _, style: _, span } = attr;
     match kind {
-        AttrKind::Normal(AttrItem { path, args, tokens }) => {
+        AttrKind::Normal(AttrItem { path, args, tokens }, attr_tokens) => {
             vis.visit_path(path);
             visit_mac_args(args, vis);
             visit_lazy_tts(tokens, vis);
+            visit_lazy_tts(attr_tokens, vis);
         }
         AttrKind::DocComment(..) => {}
     }
     vis.visit_span(span);
-    visit_lazy_tts(tokens, vis);
 }
 
 pub fn noop_visit_mac<T: MutVisitor>(mac: &mut MacCall, vis: &mut T) {

compiler/rustc_ast/src/visit.rs

@@ -886,7 +886,7 @@ pub fn walk_vis<'a, V: Visitor<'a>>(visitor: &mut V, vis: &'a Visibility) {
 
 pub fn walk_attribute<'a, V: Visitor<'a>>(visitor: &mut V, attr: &'a Attribute) {
     match attr.kind {
-        AttrKind::Normal(ref item) => walk_mac_args(visitor, &item.args),
+        AttrKind::Normal(ref item, ref _tokens) => walk_mac_args(visitor, &item.args),
         AttrKind::DocComment(..) => {}
     }
 }