rustc_parse: Move AST -> TokenStream conversion logic to rustc_ast

compiler/rustc_expand/src/base.rs

@@ -4,7 +4,7 @@ use crate::module::DirOwnership;
 use rustc_ast::attr::MarkedAttrs;
 use rustc_ast::ptr::P;
 use rustc_ast::token::{self, Nonterminal};
-use rustc_ast::tokenstream::{CanSynthesizeMissingTokens, TokenStream};
+use rustc_ast::tokenstream::TokenStream;
 use rustc_ast::visit::{AssocCtxt, Visitor};
 use rustc_ast::{self as ast, Attribute, HasAttrs, Item, NodeId, PatKind};
 use rustc_attr::{self as attr, Deprecation, Stability};
@@ -13,7 +13,7 @@ use rustc_data_structures::sync::{self, Lrc};
 use rustc_errors::{Applicability, DiagnosticBuilder, ErrorGuaranteed, MultiSpan, PResult};
 use rustc_lint_defs::builtin::PROC_MACRO_BACK_COMPAT;
 use rustc_lint_defs::BuiltinLintDiagnostics;
-use rustc_parse::{self, parser, to_token_stream, MACRO_ARGUMENTS};
+use rustc_parse::{self, parser, MACRO_ARGUMENTS};
 use rustc_session::{parse::ParseSess, Limit, Session};
 use rustc_span::def_id::{CrateNum, DefId, LocalDefId};
 use rustc_span::edition::Edition;
@@ -109,20 +109,18 @@ impl Annotatable {
         }
     }
 
-    pub fn to_tokens(&self, sess: &ParseSess) -> TokenStream {
+    pub fn to_tokens(&self) -> TokenStream {
         match self {
-            Annotatable::Item(node) => to_token_stream(node, sess, CanSynthesizeMissingTokens::No),
+            Annotatable::Item(node) => TokenStream::from_ast(node),
             Annotatable::TraitItem(node) | Annotatable::ImplItem(node) => {
-                to_token_stream(node, sess, CanSynthesizeMissingTokens::No)
-            }
-            Annotatable::ForeignItem(node) => {
-                to_token_stream(node, sess, CanSynthesizeMissingTokens::No)
+                TokenStream::from_ast(node)
             }
+            Annotatable::ForeignItem(node) => TokenStream::from_ast(node),
             Annotatable::Stmt(node) => {
                 assert!(!matches!(node.kind, ast::StmtKind::Empty));
-                to_token_stream(node, sess, CanSynthesizeMissingTokens::No)
+                TokenStream::from_ast(node)
             }
-            Annotatable::Expr(node) => to_token_stream(node, sess, CanSynthesizeMissingTokens::No),
+            Annotatable::Expr(node) => TokenStream::from_ast(node),
             Annotatable::Arm(..)
             | Annotatable::ExprField(..)
             | Annotatable::PatField(..)

compiler/rustc_expand/src/expand.rs

@@ -679,9 +679,12 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                                     )
                                 ) =>
                         {
-                            rustc_parse::fake_token_stream(&self.cx.sess.parse_sess, item_inner)
+                            rustc_parse::fake_token_stream_for_item(
+                                &self.cx.sess.parse_sess,
+                                item_inner,
+                            )
                         }
-                        _ => item.to_tokens(&self.cx.sess.parse_sess),
+                        _ => item.to_tokens(),
                     };
                     let attr_item = attr.unwrap_normal_item();
                     if let MacArgs::Eq(..) = attr_item.args {

compiler/rustc_expand/src/proc_macro.rs

@@ -96,7 +96,7 @@ impl MultiItemModifier for ProcMacroDerive {
             };
             TokenTree::token(token::Interpolated(Lrc::new(nt)), DUMMY_SP).into()
         } else {
-            item.to_tokens(&ecx.sess.parse_sess)
+            item.to_tokens()
         };
 
         let stream = {

compiler/rustc_expand/src/proc_macro_server.rs

@@ -2,14 +2,13 @@ use crate::base::ExtCtxt;
 
 use rustc_ast as ast;
 use rustc_ast::token;
-use rustc_ast::tokenstream::{self, CanSynthesizeMissingTokens};
-use rustc_ast::tokenstream::{DelimSpan, Spacing::*, TokenStream, TreeAndSpacing};
+use rustc_ast::tokenstream::{self, DelimSpan, Spacing::*, TokenStream, TreeAndSpacing};
 use rustc_ast_pretty::pprust;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::sync::Lrc;
 use rustc_errors::{Diagnostic, MultiSpan, PResult};
 use rustc_parse::lexer::nfc_normalize;
-use rustc_parse::{nt_to_tokenstream, parse_stream_from_source_str};
+use rustc_parse::parse_stream_from_source_str;
 use rustc_session::parse::ParseSess;
 use rustc_span::def_id::CrateNum;
 use rustc_span::symbol::{self, kw, sym, Symbol};
@@ -179,10 +178,9 @@ impl FromInternal<(TreeAndSpacing, &'_ mut Vec<Self>, &mut Rustc<'_, '_>)>
                 TokenTree::Ident(Ident::new(rustc.sess(), ident.name, is_raw, ident.span))
             }
             Interpolated(nt) => {
-                let stream = nt_to_tokenstream(&nt, rustc.sess(), CanSynthesizeMissingTokens::No);
                 TokenTree::Group(Group {
                     delimiter: pm::Delimiter::None,
-                    stream,
+                    stream: TokenStream::from_nonterminal_ast(&nt),
                     span: DelimSpan::from_single(span),
                     flatten: crate::base::nt_pretty_printing_compatibility_hack(&nt, rustc.sess()),
                 })
@@ -454,7 +452,7 @@ impl server::TokenStream for Rustc<'_, '_> {
 
         // NOTE: For now, limit `expand_expr` to exclusively expand to literals.
         // This may be relaxed in the future.
-        // We don't use `nt_to_tokenstream` as the tokenstream currently cannot
+        // We don't use `TokenStream::from_ast` as the tokenstream currently cannot
         // be recovered in the general case.
         match &expr.kind {
             ast::ExprKind::Lit(l) => {