ast: Introduce some traits to get AST node properties generically

And use them to avoid constructing some artificial `Nonterminal` tokens during expansion

compiler/rustc_parse/src/lib.rs

@@ -17,17 +17,17 @@ use rustc_ast::token::{self, Nonterminal, Token, TokenKind};
 use rustc_ast::tokenstream::{self, AttributesData, CanSynthesizeMissingTokens, LazyTokenStream};
 use rustc_ast::tokenstream::{AttrAnnotatedTokenStream, AttrAnnotatedTokenTree};
 use rustc_ast::tokenstream::{Spacing, TokenStream};
-use rustc_ast::AstLike;
 use rustc_ast::Attribute;
 use rustc_ast::{AttrItem, MetaItem};
-use rustc_ast_pretty::pprust;
+use rustc_ast::{HasAttrs, HasSpan, HasTokens};
+use rustc_ast_pretty::pprust::{self, AstPrettyPrint};
 use rustc_data_structures::sync::Lrc;
 use rustc_errors::{Applicability, Diagnostic, FatalError, Level, PResult};
 use rustc_session::parse::ParseSess;
 use rustc_span::{FileName, SourceFile, Span};
 
+use std::fmt;
 use std::path::Path;
-use std::str;
 
 pub const MACRO_ARGUMENTS: Option<&str> = Some("macro arguments");
 
@@ -244,6 +244,20 @@ pub fn parse_in<'a, T>(
 // NOTE(Centril): The following probably shouldn't be here but it acknowledges the
 // fact that architecturally, we are using parsing (read on below to understand why).
 
+pub fn to_token_stream(
+    node: &(impl HasAttrs + HasSpan + HasTokens + AstPrettyPrint + fmt::Debug),
+    sess: &ParseSess,
+    synthesize_tokens: CanSynthesizeMissingTokens,
+) -> TokenStream {
+    if let Some(tokens) = prepend_attrs(&node.attrs(), node.tokens()) {
+        return tokens;
+    } else if matches!(synthesize_tokens, CanSynthesizeMissingTokens::Yes) {
+        return fake_token_stream(sess, node);
+    } else {
+        panic!("Missing tokens for nt {:?} at {:?}: {:?}", node, node.span(), node.pretty_print());
+    }
+}
+
 pub fn nt_to_tokenstream(
     nt: &Nonterminal,
     sess: &ParseSess,
@@ -298,7 +312,7 @@ pub fn nt_to_tokenstream(
     if let Some(tokens) = tokens {
         return tokens;
     } else if matches!(synthesize_tokens, CanSynthesizeMissingTokens::Yes) {
-        return fake_token_stream(sess, nt);
+        return nt_fake_token_stream(sess, nt);
     } else {
         panic!(
             "Missing tokens for nt {:?} at {:?}: {:?}",
@@ -322,7 +336,13 @@ fn prepend_attrs(attrs: &[Attribute], tokens: Option<&LazyTokenStream>) -> Optio
     Some(wrapped.to_tokenstream())
 }
 
-pub fn fake_token_stream(sess: &ParseSess, nt: &Nonterminal) -> TokenStream {
+pub fn fake_token_stream(sess: &ParseSess, node: &(impl AstPrettyPrint + HasSpan)) -> TokenStream {
+    let source = node.pretty_print();
+    let filename = FileName::macro_expansion_source_code(&source);
+    parse_stream_from_source_str(filename, source, sess, Some(node.span()))
+}
+
+fn nt_fake_token_stream(sess: &ParseSess, nt: &Nonterminal) -> TokenStream {
     let source = pprust::nonterminal_to_string(nt);
     let filename = FileName::macro_expansion_source_code(&source);
     parse_stream_from_source_str(filename, source, sess, Some(nt.span()))

compiler/rustc_parse/src/parser/attr_wrapper.rs

@@ -3,7 +3,7 @@ use rustc_ast::token::{self, Delimiter, Token, TokenKind};
 use rustc_ast::tokenstream::{AttrAnnotatedTokenStream, AttributesData, CreateTokenStream};
 use rustc_ast::tokenstream::{AttrAnnotatedTokenTree, DelimSpan, LazyTokenStream, Spacing};
 use rustc_ast::{self as ast};
-use rustc_ast::{AstLike, AttrVec, Attribute};
+use rustc_ast::{AttrVec, Attribute, HasAttrs, HasTokens};
 use rustc_errors::PResult;
 use rustc_span::{sym, Span};
 
@@ -192,7 +192,7 @@ impl<'a> Parser<'a> {
     /// This restriction shouldn't be an issue in practice,
     /// since this function is used to record the tokens for
     /// a parsed AST item, which always has matching delimiters.
-    pub fn collect_tokens_trailing_token<R: AstLike>(
+    pub fn collect_tokens_trailing_token<R: HasAttrs + HasTokens>(
         &mut self,
         attrs: AttrWrapper,
         force_collect: ForceCollect,

compiler/rustc_parse/src/parser/mod.rs

@@ -25,9 +25,9 @@ use rustc_ast::tokenstream::{self, DelimSpan, Spacing};
 use rustc_ast::tokenstream::{TokenStream, TokenTree};
 use rustc_ast::AttrId;
 use rustc_ast::DUMMY_NODE_ID;
-use rustc_ast::{self as ast, AnonConst, AstLike, AttrStyle, AttrVec, Const, CrateSugar, Extern};
+use rustc_ast::{self as ast, AnonConst, AttrStyle, AttrVec, Const, CrateSugar, Extern};
 use rustc_ast::{Async, Expr, ExprKind, MacArgs, MacArgsEq, MacDelimiter, Mutability, StrLit};
-use rustc_ast::{Unsafe, Visibility, VisibilityKind};
+use rustc_ast::{HasAttrs, HasTokens, Unsafe, Visibility, VisibilityKind};
 use rustc_ast_pretty::pprust;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_errors::PResult;
@@ -1389,7 +1389,7 @@ impl<'a> Parser<'a> {
         }
     }
 
-    pub fn collect_tokens_no_attrs<R: AstLike>(
+    pub fn collect_tokens_no_attrs<R: HasAttrs + HasTokens>(
         &mut self,
         f: impl FnOnce(&mut Self) -> PResult<'a, R>,
     ) -> PResult<'a, R> {

compiler/rustc_parse/src/parser/nonterminal.rs

@@ -1,6 +1,6 @@
 use rustc_ast::ptr::P;
 use rustc_ast::token::{self, Delimiter, NonterminalKind, Token};
-use rustc_ast::AstLike;
+use rustc_ast::HasTokens;
 use rustc_ast_pretty::pprust;
 use rustc_errors::PResult;
 use rustc_span::symbol::{kw, Ident};

compiler/rustc_parse/src/parser/stmt.rs

@@ -13,10 +13,8 @@ use rustc_ast as ast;
 use rustc_ast::ptr::P;
 use rustc_ast::token::{self, Delimiter, TokenKind};
 use rustc_ast::util::classify;
-use rustc_ast::{
-    AstLike, AttrStyle, AttrVec, Attribute, LocalKind, MacCall, MacCallStmt, MacStmtStyle,
-};
-use rustc_ast::{Block, BlockCheckMode, Expr, ExprKind, Local, Stmt};
+use rustc_ast::{AttrStyle, AttrVec, Attribute, LocalKind, MacCall, MacCallStmt, MacStmtStyle};
+use rustc_ast::{Block, BlockCheckMode, Expr, ExprKind, HasAttrs, Local, Stmt};
 use rustc_ast::{StmtKind, DUMMY_NODE_ID};
 use rustc_errors::{Applicability, DiagnosticBuilder, ErrorGuaranteed, PResult};
 use rustc_span::source_map::{BytePos, Span};