Auto merge of #116649 - nnethercote:improve-print_tts-precursors, r=petrochenkov

Token cleanups

Some precursors to #114571 that are worth merging even if the main part of #114571 doesn't get merged.

r? `@petrochenkov`

compiler/rustc_ast/src/attr/mod.rs

@@ -197,10 +197,10 @@ impl Attribute {
                 .unwrap_or_else(|| panic!("attribute is missing tokens: {self:?}"))
                 .to_attr_token_stream()
                 .to_tokenstream(),
-            &AttrKind::DocComment(comment_kind, data) => TokenStream::new(vec![TokenTree::Token(
-                Token::new(token::DocComment(comment_kind, self.style, data), self.span),
-                Spacing::Alone,
-            )]),
+            &AttrKind::DocComment(comment_kind, data) => TokenStream::token_alone(
+                token::DocComment(comment_kind, self.style, data),
+                self.span,
+            ),
         }
     }
 }

compiler/rustc_ast/src/token.rs

@@ -404,7 +404,7 @@ impl Token {
         [DotDot, DotDotDot, DotDotEq].contains(&self.kind)
     }
 
-    pub fn is_op(&self) -> bool {
+    pub fn is_punct(&self) -> bool {
         match self.kind {
             Eq | Lt | Le | EqEq | Ne | Ge | Gt | AndAnd | OrOr | Not | Tilde | BinOp(_)
             | BinOpEq(_) | At | Dot | DotDot | DotDotDot | DotDotEq | Comma | Semi | Colon

compiler/rustc_ast_pretty/src/pprust/state.rs

@@ -148,7 +148,7 @@ pub fn print_crate<'a>(
 
 /// This makes printed token streams look slightly nicer,
 /// and also addresses some specific regressions described in #63896 and #73345.
-fn tt_prepend_space(tt: &TokenTree, prev: &TokenTree) -> bool {
+fn space_between(prev: &TokenTree, curr: &TokenTree) -> bool {
     if let TokenTree::Token(token, _) = prev {
         // No space after these tokens, e.g. `x.y`, `$e`
         // (The carets point to `prev`.)       ^     ^
@@ -159,9 +159,9 @@ fn tt_prepend_space(tt: &TokenTree, prev: &TokenTree) -> bool {
             return comment_kind != CommentKind::Line;
         }
     }
-    match tt {
+    match curr {
         // No space before these tokens, e.g. `foo,`, `println!`, `x.y`
-        // (The carets point to `token`.)         ^           ^     ^
+        // (The carets point to `curr`.)          ^           ^     ^
         //
         // FIXME: having `Not` here works well for macro invocations like
         // `println!()`, but is bad when `!` means "logical not" or "the never
@@ -575,7 +575,7 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
         while let Some(tt) = iter.next() {
             self.print_tt(tt, convert_dollar_crate);
             if let Some(next) = iter.peek() {
-                if tt_prepend_space(next, tt) {
+                if space_between(tt, next) {
                     self.space();
                 }
             }

compiler/rustc_parse/src/lexer/tokentrees.rs

@@ -55,16 +55,14 @@ impl<'a> TokenTreesReader<'a> {
                     let (this_spacing, next_tok) = loop {
                         let (next_tok, is_next_tok_preceded_by_whitespace) =
                             self.string_reader.next_token();
-                        if !is_next_tok_preceded_by_whitespace {
-                            if let Some(glued) = self.token.glue(&next_tok) {
-                                self.token = glued;
-                            } else {
-                                let this_spacing =
-                                    if next_tok.is_op() { Spacing::Joint } else { Spacing::Alone };
-                                break (this_spacing, next_tok);
-                            }
-                        } else {
+                        if is_next_tok_preceded_by_whitespace {
                             break (Spacing::Alone, next_tok);
+                        } else if let Some(glued) = self.token.glue(&next_tok) {
+                            self.token = glued;
+                        } else {
+                            let this_spacing =
+                                if next_tok.is_punct() { Spacing::Joint } else { Spacing::Alone };
+                            break (this_spacing, next_tok);
                         }
                     };
                     let this_tok = std::mem::replace(&mut self.token, next_tok);

compiler/rustc_parse/src/parser/expr.rs

@@ -1592,7 +1592,7 @@ impl<'a> Parser<'a> {
         } else if !ate_colon
             && self.may_recover()
             && (matches!(self.token.kind, token::CloseDelim(_) | token::Comma)
-                || self.token.is_op())
+                || self.token.is_punct())
         {
             let (lit, _) =
                 self.recover_unclosed_char(label_.ident, Parser::mk_token_lit_char, |self_| {