Auto merge of #98188 - mystor:fast_group_punct, r=eddyb

proc_macro/bridge: stop using a remote object handle for proc_macro Punct and Group

This is the third part of https://github.com/rust-lang/rust/pull/86822, split off as requested in https://github.com/rust-lang/rust/pull/86822#pullrequestreview-1008655452. This patch transforms the `Punct` and `Group` types into structs serialized over IPC rather than handles, making them more efficient to create and manipulate from within proc-macros.

compiler/rustc_expand/src/proc_macro_server.rs

@@ -2,7 +2,7 @@ use crate::base::ExtCtxt;
 
 use rustc_ast as ast;
 use rustc_ast::token;
-use rustc_ast::tokenstream::{self, DelimSpan, Spacing::*, TokenStream, TreeAndSpacing};
+use rustc_ast::tokenstream::{self, Spacing::*, TokenStream};
 use rustc_ast_pretty::pprust;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::sync::Lrc;
@@ -14,8 +14,8 @@ use rustc_span::def_id::CrateNum;
 use rustc_span::symbol::{self, kw, sym, Symbol};
 use rustc_span::{BytePos, FileName, Pos, SourceFile, Span};
 
-use pm::bridge::{server, ExpnGlobals, TokenTree};
+use pm::bridge::{server, DelimSpan, ExpnGlobals, Group, Punct, TokenTree};
-use pm::{Delimiter, Level, LineColumn, Spacing};
+use pm::{Delimiter, Level, LineColumn};
 use std::ops::Bound;
 use std::{ascii, panic};
 
@@ -49,158 +49,170 @@ impl ToInternal<token::Delimiter> for Delimiter {
     }
 }
 
-impl FromInternal<(TreeAndSpacing, &'_ mut Vec<Self>, &mut Rustc<'_, '_>)>
+impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)>
-    for TokenTree<Group, Punct, Ident, Literal>
+    for Vec<TokenTree<TokenStream, Span, Ident, Literal>>
 {
-    fn from_internal(
+    fn from_internal((stream, rustc): (TokenStream, &mut Rustc<'_, '_>)) -> Self {
-        ((tree, spacing), stack, rustc): (TreeAndSpacing, &mut Vec<Self>, &mut Rustc<'_, '_>),
-    ) -> Self {
         use rustc_ast::token::*;
 
-        let joint = spacing == Joint;
+        // Estimate the capacity as `stream.len()` rounded up to the next power
-        let Token { kind, span } = match tree {
+        // of two to limit the number of required reallocations.
-            tokenstream::TokenTree::Delimited(span, delim, tts) => {
+        let mut trees = Vec::with_capacity(stream.len().next_power_of_two());
-                let delimiter = pm::Delimiter::from_internal(delim);
+        let mut cursor = stream.into_trees();
-                return TokenTree::Group(Group { delimiter, stream: tts, span, flatten: false });
-            }
-            tokenstream::TokenTree::Token(token) => token,
-        };
 
-        macro_rules! tt {
+        while let Some((tree, spacing)) = cursor.next_with_spacing() {
-            ($ty:ident { $($field:ident $(: $value:expr)*),+ $(,)? }) => (
+            let joint = spacing == Joint;
-                TokenTree::$ty(self::$ty {
+            let Token { kind, span } = match tree {
-                    $($field $(: $value)*,)+
+                tokenstream::TokenTree::Delimited(span, delim, tts) => {
-                    span,
+                    let delimiter = pm::Delimiter::from_internal(delim);
-                })
+                    trees.push(TokenTree::Group(Group {
-            );
+                        delimiter,
-            ($ty:ident::$method:ident($($value:expr),*)) => (
+                        stream: Some(tts),
-                TokenTree::$ty(self::$ty::$method($($value,)* span))
+                        span: DelimSpan {
-            );
+                            open: span.open,
-        }
+                            close: span.close,
-        macro_rules! op {
+                            entire: span.entire(),
-            ($a:expr) => {
+                        },
-                tt!(Punct::new($a, joint))
+                    }));
+                    continue;
+                }
+                tokenstream::TokenTree::Token(token) => token,
             };
-            ($a:expr, $b:expr) => {{
-                stack.push(tt!(Punct::new($b, joint)));
-                tt!(Punct::new($a, true))
-            }};
-            ($a:expr, $b:expr, $c:expr) => {{
-                stack.push(tt!(Punct::new($c, joint)));
-                stack.push(tt!(Punct::new($b, true)));
-                tt!(Punct::new($a, true))
-            }};
-        }
 
-        match kind {
+            let mut op = |s: &str| {
-            Eq => op!('='),
+                assert!(s.is_ascii());
-            Lt => op!('<'),
+                trees.extend(s.as_bytes().iter().enumerate().map(|(idx, &ch)| {
-            Le => op!('<', '='),
+                    TokenTree::Punct(Punct { ch, joint: joint || idx != s.len() - 1, span })
-            EqEq => op!('=', '='),
-            Ne => op!('!', '='),
-            Ge => op!('>', '='),
-            Gt => op!('>'),
-            AndAnd => op!('&', '&'),
-            OrOr => op!('|', '|'),
-            Not => op!('!'),
-            Tilde => op!('~'),
-            BinOp(Plus) => op!('+'),
-            BinOp(Minus) => op!('-'),
-            BinOp(Star) => op!('*'),
-            BinOp(Slash) => op!('/'),
-            BinOp(Percent) => op!('%'),
-            BinOp(Caret) => op!('^'),
-            BinOp(And) => op!('&'),
-            BinOp(Or) => op!('|'),
-            BinOp(Shl) => op!('<', '<'),
-            BinOp(Shr) => op!('>', '>'),
-            BinOpEq(Plus) => op!('+', '='),
-            BinOpEq(Minus) => op!('-', '='),
-            BinOpEq(Star) => op!('*', '='),
-            BinOpEq(Slash) => op!('/', '='),
-            BinOpEq(Percent) => op!('%', '='),
-            BinOpEq(Caret) => op!('^', '='),
-            BinOpEq(And) => op!('&', '='),
-            BinOpEq(Or) => op!('|', '='),
-            BinOpEq(Shl) => op!('<', '<', '='),
-            BinOpEq(Shr) => op!('>', '>', '='),
-            At => op!('@'),
-            Dot => op!('.'),
-            DotDot => op!('.', '.'),
-            DotDotDot => op!('.', '.', '.'),
-            DotDotEq => op!('.', '.', '='),
-            Comma => op!(','),
-            Semi => op!(';'),
-            Colon => op!(':'),
-            ModSep => op!(':', ':'),
-            RArrow => op!('-', '>'),
-            LArrow => op!('<', '-'),
-            FatArrow => op!('=', '>'),
-            Pound => op!('#'),
-            Dollar => op!('$'),
-            Question => op!('?'),
-            SingleQuote => op!('\''),
-
-            Ident(name, false) if name == kw::DollarCrate => tt!(Ident::dollar_crate()),
-            Ident(name, is_raw) => tt!(Ident::new(rustc.sess(), name, is_raw)),
-            Lifetime(name) => {
-                let ident = symbol::Ident::new(name, span).without_first_quote();
-                stack.push(tt!(Ident::new(rustc.sess(), ident.name, false)));
-                tt!(Punct::new('\'', true))
-            }
-            Literal(lit) => tt!(Literal { lit }),
-            DocComment(_, attr_style, data) => {
-                let mut escaped = String::new();
-                for ch in data.as_str().chars() {
-                    escaped.extend(ch.escape_debug());
-                }
-                let stream = [
-                    Ident(sym::doc, false),
-                    Eq,
-                    TokenKind::lit(token::Str, Symbol::intern(&escaped), None),
-                ]
-                .into_iter()
-                .map(|kind| tokenstream::TokenTree::token(kind, span))
-                .collect();
-                stack.push(TokenTree::Group(Group {
-                    delimiter: pm::Delimiter::Bracket,
-                    stream,
-                    span: DelimSpan::from_single(span),
-                    flatten: false,
                 }));
-                if attr_style == ast::AttrStyle::Inner {
+            };
-                    stack.push(tt!(Punct::new('!', false)));
+
+            match kind {
+                Eq => op("="),
+                Lt => op("<"),
+                Le => op("<="),
+                EqEq => op("=="),
+                Ne => op("!="),
+                Ge => op(">="),
+                Gt => op(">"),
+                AndAnd => op("&&"),
+                OrOr => op("||"),
+                Not => op("!"),
+                Tilde => op("~"),
+                BinOp(Plus) => op("+"),
+                BinOp(Minus) => op("-"),
+                BinOp(Star) => op("*"),
+                BinOp(Slash) => op("/"),
+                BinOp(Percent) => op("%"),
+                BinOp(Caret) => op("^"),
+                BinOp(And) => op("&"),
+                BinOp(Or) => op("|"),
+                BinOp(Shl) => op("<<"),
+                BinOp(Shr) => op(">>"),
+                BinOpEq(Plus) => op("+="),
+                BinOpEq(Minus) => op("-="),
+                BinOpEq(Star) => op("*="),
+                BinOpEq(Slash) => op("/="),
+                BinOpEq(Percent) => op("%="),
+                BinOpEq(Caret) => op("^="),
+                BinOpEq(And) => op("&="),
+                BinOpEq(Or) => op("|="),
+                BinOpEq(Shl) => op("<<="),
+                BinOpEq(Shr) => op(">>="),
+                At => op("@"),
+                Dot => op("."),
+                DotDot => op(".."),
+                DotDotDot => op("..."),
+                DotDotEq => op("..="),
+                Comma => op(","),
+                Semi => op(";"),
+                Colon => op(":"),
+                ModSep => op("::"),
+                RArrow => op("->"),
+                LArrow => op("<-"),
+                FatArrow => op("=>"),
+                Pound => op("#"),
+                Dollar => op("$"),
+                Question => op("?"),
+                SingleQuote => op("'"),
+
+                Ident(name, false) if name == kw::DollarCrate => trees.push(TokenTree::Ident(Ident::dollar_crate(span))),
+                Ident(name, is_raw) => trees.push(TokenTree::Ident(Ident::new(rustc.sess(), name, is_raw, span))),
+                Lifetime(name) => {
+                    let ident = symbol::Ident::new(name, span).without_first_quote();
+                    trees.extend([
+                        TokenTree::Punct(Punct { ch: b'\'', joint: true, span }),
+                        TokenTree::Ident(Ident::new(rustc.sess(), ident.name, false, span)),
+                    ]);
+                }
+                Literal(lit) => trees.push(TokenTree::Literal(self::Literal { lit, span })),
+                DocComment(_, attr_style, data) => {
+                    let mut escaped = String::new();
+                    for ch in data.as_str().chars() {
+                        escaped.extend(ch.escape_debug());
+                    }
+                    let stream = [
+                        Ident(sym::doc, false),
+                        Eq,
+                        TokenKind::lit(token::Str, Symbol::intern(&escaped), None),
+                    ]
+                    .into_iter()
+                    .map(|kind| tokenstream::TokenTree::token(kind, span))
+                    .collect();
+                    trees.push(TokenTree::Punct(Punct { ch: b'#', joint: false, span }));
+                    if attr_style == ast::AttrStyle::Inner {
+                        trees.push(TokenTree::Punct(Punct { ch: b'!', joint: false, span }));
+                    }
+                    trees.push(TokenTree::Group(Group {
+                        delimiter: pm::Delimiter::Bracket,
+                        stream: Some(stream),
+                        span: DelimSpan::from_single(span),
+                    }));
                 }
-                tt!(Punct::new('#', false))
-            }
 
-            Interpolated(nt) if let NtIdent(ident, is_raw) = *nt => {
+                Interpolated(nt) if let NtIdent(ident, is_raw) = *nt => {
-                TokenTree::Ident(Ident::new(rustc.sess(), ident.name, is_raw, ident.span))
+                    trees.push(TokenTree::Ident(Ident::new(rustc.sess(), ident.name, is_raw, ident.span)))
-            }
+                }
-            Interpolated(nt) => {
-                TokenTree::Group(Group {
-                    delimiter: pm::Delimiter::None,
-                    stream: TokenStream::from_nonterminal_ast(&nt),
-                    span: DelimSpan::from_single(span),
-                    flatten: crate::base::nt_pretty_printing_compatibility_hack(&nt, rustc.sess()),
-                })
-            }
 
-            OpenDelim(..) | CloseDelim(..) => unreachable!(),
+                Interpolated(nt) => {
-            Eof => unreachable!(),
+                    let stream = TokenStream::from_nonterminal_ast(&nt);
+                    // A hack used to pass AST fragments to attribute and derive
+                    // macros as a single nonterminal token instead of a token
+                    // stream.  Such token needs to be "unwrapped" and not
+                    // represented as a delimited group.
+                    // FIXME: It needs to be removed, but there are some
+                    // compatibility issues (see #73345).
+                    if crate::base::nt_pretty_printing_compatibility_hack(&nt, rustc.sess()) {
+                        trees.extend(Self::from_internal((stream, rustc)));
+                    } else {
+                        trees.push(TokenTree::Group(Group {
+                            delimiter: pm::Delimiter::None,
+                            stream: Some(stream),
+                            span: DelimSpan::from_single(span),
+                        }))
+                    }
+                }
+
+                OpenDelim(..) | CloseDelim(..) => unreachable!(),
+                Eof => unreachable!(),
+            }
         }
+        trees
     }
 }
 
-impl ToInternal<TokenStream> for TokenTree<Group, Punct, Ident, Literal> {
+impl ToInternal<TokenStream> for TokenTree<TokenStream, Span, Ident, Literal> {
     fn to_internal(self) -> TokenStream {
         use rustc_ast::token::*;
 
         let (ch, joint, span) = match self {
             TokenTree::Punct(Punct { ch, joint, span }) => (ch, joint, span),
-            TokenTree::Group(Group { delimiter, stream, span, .. }) => {
+            TokenTree::Group(Group { delimiter, stream, span: DelimSpan { open, close, .. } }) => {
-                return tokenstream::TokenTree::Delimited(span, delimiter.to_internal(), stream)
+                return tokenstream::TokenTree::Delimited(
-                    .into();
+                    tokenstream::DelimSpan { open, close },
+                    delimiter.to_internal(),
+                    stream.unwrap_or_default(),
+                )
+                .into();
             }
             TokenTree::Ident(self::Ident { sym, is_raw, span }) => {
                 return tokenstream::TokenTree::token(Ident(sym, is_raw), span).into();
@@ -233,28 +245,28 @@ impl ToInternal<TokenStream> for TokenTree<Group, Punct, Ident, Literal> {
         };
 
         let kind = match ch {
-            '=' => Eq,
+            b'=' => Eq,
-            '<' => Lt,
+            b'<' => Lt,
-            '>' => Gt,
+            b'>' => Gt,
-            '!' => Not,
+            b'!' => Not,
-            '~' => Tilde,
+            b'~' => Tilde,
-            '+' => BinOp(Plus),
+            b'+' => BinOp(Plus),
-            '-' => BinOp(Minus),
+            b'-' => BinOp(Minus),
-            '*' => BinOp(Star),
+            b'*' => BinOp(Star),
-            '/' => BinOp(Slash),
+            b'/' => BinOp(Slash),
-            '%' => BinOp(Percent),
+            b'%' => BinOp(Percent),
-            '^' => BinOp(Caret),
+            b'^' => BinOp(Caret),
-            '&' => BinOp(And),
+            b'&' => BinOp(And),
-            '|' => BinOp(Or),
+            b'|' => BinOp(Or),
-            '@' => At,
+            b'@' => At,
-            '.' => Dot,
+            b'.' => Dot,
-            ',' => Comma,
+            b',' => Comma,
-            ';' => Semi,
+            b';' => Semi,
-            ':' => Colon,
+            b':' => Colon,
-            '#' => Pound,
+            b'#' => Pound,
-            '$' => Dollar,
+            b'$' => Dollar,
-            '?' => Question,
+            b'?' => Question,
-            '\'' => SingleQuote,
+            b'\'' => SingleQuote,
             _ => unreachable!(),
         };
 
@@ -277,38 +289,6 @@ impl ToInternal<rustc_errors::Level> for Level {
 
 pub struct FreeFunctions;
 
-#[derive(Clone)]
-pub struct Group {
-    delimiter: Delimiter,
-    stream: TokenStream,
-    span: DelimSpan,
-    /// A hack used to pass AST fragments to attribute and derive macros
-    /// as a single nonterminal token instead of a token stream.
-    /// FIXME: It needs to be removed, but there are some compatibility issues (see #73345).
-    flatten: bool,
-}
-
-#[derive(Copy, Clone, PartialEq, Eq, Hash)]
-pub struct Punct {
-    ch: char,
-    // NB. not using `Spacing` here because it doesn't implement `Hash`.
-    joint: bool,
-    span: Span,
-}
-
-impl Punct {
-    fn new(ch: char, joint: bool, span: Span) -> Punct {
-        const LEGAL_CHARS: &[char] = &[
-            '=', '<', '>', '!', '~', '+', '-', '*', '/', '%', '^', '&', '|', '@', '.', ',', ';',
-            ':', '#', '$', '?', '\'',
-        ];
-        if !LEGAL_CHARS.contains(&ch) {
-            panic!("unsupported character `{:?}`", ch)
-        }
-        Punct { ch, joint, span }
-    }
-}
-
 #[derive(Copy, Clone, PartialEq, Eq, Hash)]
 pub struct Ident {
     sym: Symbol,
@@ -377,8 +357,6 @@ impl<'a, 'b> Rustc<'a, 'b> {
 impl server::Types for Rustc<'_, '_> {
     type FreeFunctions = FreeFunctions;
     type TokenStream = TokenStream;
-    type Group = Group;
-    type Punct = Punct;
     type Ident = Ident;
     type Literal = Literal;
     type SourceFile = Lrc<SourceFile>;
@@ -471,7 +449,7 @@ impl server::TokenStream for Rustc<'_, '_> {
 
     fn from_token_tree(
         &mut self,
-        tree: TokenTree<Self::Group, Self::Punct, Self::Ident, Self::Literal>,
+        tree: TokenTree<Self::TokenStream, Self::Span, Self::Ident, Self::Literal>,
     ) -> Self::TokenStream {
         tree.to_internal()
     }
@@ -479,7 +457,7 @@ impl server::TokenStream for Rustc<'_, '_> {
     fn concat_trees(
         &mut self,
         base: Option<Self::TokenStream>,
-        trees: Vec<TokenTree<Self::Group, Self::Punct, Self::Ident, Self::Literal>>,
+        trees: Vec<TokenTree<Self::TokenStream, Self::Span, Self::Ident, Self::Literal>>,
     ) -> Self::TokenStream {
         let mut builder = tokenstream::TokenStreamBuilder::new();
         if let Some(base) = base {
@@ -509,93 +487,8 @@ impl server::TokenStream for Rustc<'_, '_> {
     fn into_trees(
         &mut self,
         stream: Self::TokenStream,
-    ) -> Vec<TokenTree<Self::Group, Self::Punct, Self::Ident, Self::Literal>> {
+    ) -> Vec<TokenTree<Self::TokenStream, Self::Span, Self::Ident, Self::Literal>> {
-        // FIXME: This is a raw port of the previous approach (which had a
+        FromInternal::from_internal((stream, self))
-        // `TokenStreamIter` server-side object with a single `next` method),
-        // and can probably be optimized (for bulk conversion).
-        let mut cursor = stream.into_trees();
-        let mut stack = Vec::new();
-        let mut tts = Vec::new();
-        loop {
-            let next = stack.pop().or_else(|| {
-                let next = cursor.next_with_spacing()?;
-                Some(TokenTree::from_internal((next, &mut stack, self)))
-            });
-            match next {
-                Some(TokenTree::Group(group)) => {
-                    // A hack used to pass AST fragments to attribute and derive
-                    // macros as a single nonterminal token instead of a token
-                    // stream.  Such token needs to be "unwrapped" and not
-                    // represented as a delimited group.
-                    // FIXME: It needs to be removed, but there are some
-                    // compatibility issues (see #73345).
-                    if group.flatten {
-                        tts.append(&mut self.into_trees(group.stream));
-                    } else {
-                        tts.push(TokenTree::Group(group));
-                    }
-                }
-                Some(tt) => tts.push(tt),
-                None => return tts,
-            }
-        }
-    }
-}
-
-impl server::Group for Rustc<'_, '_> {
-    fn new(&mut self, delimiter: Delimiter, stream: Option<Self::TokenStream>) -> Self::Group {
-        Group {
-            delimiter,
-            stream: stream.unwrap_or_default(),
-            span: DelimSpan::from_single(self.call_site),
-            flatten: false,
-        }
-    }
-
-    fn delimiter(&mut self, group: &Self::Group) -> Delimiter {
-        group.delimiter
-    }
-
-    fn stream(&mut self, group: &Self::Group) -> Self::TokenStream {
-        group.stream.clone()
-    }
-
-    fn span(&mut self, group: &Self::Group) -> Self::Span {
-        group.span.entire()
-    }
-
-    fn span_open(&mut self, group: &Self::Group) -> Self::Span {
-        group.span.open
-    }
-
-    fn span_close(&mut self, group: &Self::Group) -> Self::Span {
-        group.span.close
-    }
-
-    fn set_span(&mut self, group: &mut Self::Group, span: Self::Span) {
-        group.span = DelimSpan::from_single(span);
-    }
-}
-
-impl server::Punct for Rustc<'_, '_> {
-    fn new(&mut self, ch: char, spacing: Spacing) -> Self::Punct {
-        Punct::new(ch, spacing == Spacing::Joint, self.call_site)
-    }
-
-    fn as_char(&mut self, punct: Self::Punct) -> char {
-        punct.ch
-    }
-
-    fn spacing(&mut self, punct: Self::Punct) -> Spacing {
-        if punct.joint { Spacing::Joint } else { Spacing::Alone }
-    }
-
-    fn span(&mut self, punct: Self::Punct) -> Self::Span {
-        punct.span
-    }
-
-    fn with_span(&mut self, punct: Self::Punct, span: Self::Span) -> Self::Punct {
-        Punct { span, ..punct }
     }
 }
 

library/proc_macro/src/bridge/client.rs

@@ -175,14 +175,12 @@ define_handles! {
     'owned:
     FreeFunctions,
     TokenStream,
-    Group,
     Literal,
     SourceFile,
     MultiSpan,
     Diagnostic,
 
     'interned:
-    Punct,
     Ident,
     Span,
 }
@@ -199,12 +197,6 @@ impl Clone for TokenStream {
     }
 }
 
-impl Clone for Group {
-    fn clone(&self) -> Self {
-        self.clone()
-    }
-}
-
 impl Clone for Literal {
     fn clone(&self) -> Self {
         self.clone()

library/proc_macro/src/bridge/mod.rs

@@ -65,11 +65,11 @@ macro_rules! with_api {
                 fn from_str(src: &str) -> $S::TokenStream;
                 fn to_string($self: &$S::TokenStream) -> String;
                 fn from_token_tree(
-                    tree: TokenTree<$S::Group, $S::Punct, $S::Ident, $S::Literal>,
+                    tree: TokenTree<$S::TokenStream, $S::Span, $S::Ident, $S::Literal>,
                 ) -> $S::TokenStream;
                 fn concat_trees(
                     base: Option<$S::TokenStream>,
-                    trees: Vec<TokenTree<$S::Group, $S::Punct, $S::Ident, $S::Literal>>,
+                    trees: Vec<TokenTree<$S::TokenStream, $S::Span, $S::Ident, $S::Literal>>,
                 ) -> $S::TokenStream;
                 fn concat_streams(
                     base: Option<$S::TokenStream>,
@@ -77,25 +77,7 @@ macro_rules! with_api {
                 ) -> $S::TokenStream;
                 fn into_trees(
                     $self: $S::TokenStream
-                ) -> Vec<TokenTree<$S::Group, $S::Punct, $S::Ident, $S::Literal>>;
+                ) -> Vec<TokenTree<$S::TokenStream, $S::Span, $S::Ident, $S::Literal>>;
-            },
-            Group {
-                fn drop($self: $S::Group);
-                fn clone($self: &$S::Group) -> $S::Group;
-                fn new(delimiter: Delimiter, stream: Option<$S::TokenStream>) -> $S::Group;
-                fn delimiter($self: &$S::Group) -> Delimiter;
-                fn stream($self: &$S::Group) -> $S::TokenStream;
-                fn span($self: &$S::Group) -> $S::Span;
-                fn span_open($self: &$S::Group) -> $S::Span;
-                fn span_close($self: &$S::Group) -> $S::Span;
-                fn set_span($self: &mut $S::Group, span: $S::Span);
-            },
-            Punct {
-                fn new(ch: char, spacing: Spacing) -> $S::Punct;
-                fn as_char($self: $S::Punct) -> char;
-                fn spacing($self: $S::Punct) -> Spacing;
-                fn span($self: $S::Punct) -> $S::Span;
-                fn with_span($self: $S::Punct, span: $S::Span) -> $S::Punct;
             },
             Ident {
                 fn new(string: &str, span: $S::Span, is_raw: bool) -> $S::Ident;
@@ -343,6 +325,7 @@ mark_noop! {
     &'_ [u8],
     &'_ str,
     String,
+    u8,
     usize,
     Delimiter,
     Level,
@@ -448,16 +431,49 @@ compound_traits!(
     }
 );
 
+#[derive(Copy, Clone)]
+pub struct DelimSpan<Span> {
+    pub open: Span,
+    pub close: Span,
+    pub entire: Span,
+}
+
+impl<Span: Copy> DelimSpan<Span> {
+    pub fn from_single(span: Span) -> Self {
+        DelimSpan { open: span, close: span, entire: span }
+    }
+}
+
+compound_traits!(struct DelimSpan<Span> { open, close, entire });
+
 #[derive(Clone)]
-pub enum TokenTree<G, P, I, L> {
+pub struct Group<TokenStream, Span> {
-    Group(G),
+    pub delimiter: Delimiter,
-    Punct(P),
+    pub stream: Option<TokenStream>,
-    Ident(I),
+    pub span: DelimSpan<Span>,
-    Literal(L),
+}
+
+compound_traits!(struct Group<TokenStream, Span> { delimiter, stream, span });
+
+#[derive(Clone)]
+pub struct Punct<Span> {
+    pub ch: u8,
+    pub joint: bool,
+    pub span: Span,
+}
+
+compound_traits!(struct Punct<Span> { ch, joint, span });
+
+#[derive(Clone)]
+pub enum TokenTree<TokenStream, Span, Ident, Literal> {
+    Group(Group<TokenStream, Span>),
+    Punct(Punct<Span>),
+    Ident(Ident),
+    Literal(Literal),
 }
 
 compound_traits!(
-    enum TokenTree<G, P, I, L> {
+    enum TokenTree<TokenStream, Span, Ident, Literal> {
         Group(tt),
         Punct(tt),
         Ident(tt),
@@ -468,12 +484,12 @@ compound_traits!(
 /// Globals provided alongside the initial inputs for a macro expansion.
 /// Provides values such as spans which are used frequently to avoid RPC.
 #[derive(Clone)]
-pub struct ExpnGlobals<S> {
+pub struct ExpnGlobals<Span> {
-    pub def_site: S,
+    pub def_site: Span,
-    pub call_site: S,
+    pub call_site: Span,
-    pub mixed_site: S,
+    pub mixed_site: Span,
 }
 
 compound_traits!(
-    struct ExpnGlobals<Sp> { def_site, call_site, mixed_site }
+    struct ExpnGlobals<Span> { def_site, call_site, mixed_site }
 );

library/proc_macro/src/bridge/server.rs

@@ -8,8 +8,6 @@ use super::client::HandleStore;
 pub trait Types {
     type FreeFunctions: 'static;
     type TokenStream: 'static + Clone;
-    type Group: 'static + Clone;
-    type Punct: 'static + Copy + Eq + Hash;
     type Ident: 'static + Copy + Eq + Hash;
     type Literal: 'static + Clone;
     type SourceFile: 'static + Clone;

library/proc_macro/src/lib.rs

@@ -212,8 +212,8 @@ pub use quote::{quote, quote_span};
 fn tree_to_bridge_tree(
     tree: TokenTree,
 ) -> bridge::TokenTree<
-    bridge::client::Group,
+    bridge::client::TokenStream,
-    bridge::client::Punct,
+    bridge::client::Span,
     bridge::client::Ident,
     bridge::client::Literal,
 > {
@@ -238,8 +238,8 @@ impl From<TokenTree> for TokenStream {
 struct ConcatTreesHelper {
     trees: Vec<
         bridge::TokenTree<
-            bridge::client::Group,
+            bridge::client::TokenStream,
-            bridge::client::Punct,
+            bridge::client::Span,
             bridge::client::Ident,
             bridge::client::Literal,
         >,
@@ -365,8 +365,8 @@ pub mod token_stream {
     pub struct IntoIter(
         std::vec::IntoIter<
             bridge::TokenTree<
-                bridge::client::Group,
+                bridge::client::TokenStream,
-                bridge::client::Punct,
+                bridge::client::Span,
                 bridge::client::Ident,
                 bridge::client::Literal,
             >,
@@ -788,7 +788,7 @@ impl fmt::Display for TokenTree {
 /// A `Group` internally contains a `TokenStream` which is surrounded by `Delimiter`s.
 #[derive(Clone)]
 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
-pub struct Group(bridge::client::Group);
+pub struct Group(bridge::Group<bridge::client::TokenStream, bridge::client::Span>);
 
 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
 impl !Send for Group {}
@@ -825,13 +825,17 @@ impl Group {
     /// method below.
     #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
     pub fn new(delimiter: Delimiter, stream: TokenStream) -> Group {
-        Group(bridge::client::Group::new(delimiter, stream.0))
+        Group(bridge::Group {
+            delimiter,
+            stream: stream.0,
+            span: bridge::DelimSpan::from_single(Span::call_site().0),
+        })
     }
 
     /// Returns the delimiter of this `Group`
     #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
     pub fn delimiter(&self) -> Delimiter {
-        self.0.delimiter()
+        self.0.delimiter
     }
 
     /// Returns the `TokenStream` of tokens that are delimited in this `Group`.
@@ -840,7 +844,7 @@ impl Group {
     /// returned above.
     #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
     pub fn stream(&self) -> TokenStream {
-        TokenStream(Some(self.0.stream()))
+        TokenStream(self.0.stream.clone())
     }
 
     /// Returns the span for the delimiters of this token stream, spanning the
@@ -852,7 +856,7 @@ impl Group {
     /// ```
     #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
     pub fn span(&self) -> Span {
-        Span(self.0.span())
+        Span(self.0.span.entire)
     }
 
     /// Returns the span pointing to the opening delimiter of this group.
@@ -863,7 +867,7 @@ impl Group {
     /// ```
     #[stable(feature = "proc_macro_group_span", since = "1.55.0")]
     pub fn span_open(&self) -> Span {
-        Span(self.0.span_open())
+        Span(self.0.span.open)
     }
 
     /// Returns the span pointing to the closing delimiter of this group.
@@ -874,7 +878,7 @@ impl Group {
     /// ```
     #[stable(feature = "proc_macro_group_span", since = "1.55.0")]
     pub fn span_close(&self) -> Span {
-        Span(self.0.span_close())
+        Span(self.0.span.close)
     }
 
     /// Configures the span for this `Group`'s delimiters, but not its internal
@@ -885,7 +889,7 @@ impl Group {
     /// tokens at the level of the `Group`.
     #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
     pub fn set_span(&mut self, span: Span) {
-        self.0.set_span(span.0);
+        self.0.span = bridge::DelimSpan::from_single(span.0);
     }
 }
 
@@ -925,7 +929,7 @@ impl fmt::Debug for Group {
 /// forms of `Spacing` returned.
 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
 #[derive(Clone)]
-pub struct Punct(bridge::client::Punct);
+pub struct Punct(bridge::Punct<bridge::client::Span>);
 
 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
 impl !Send for Punct {}
@@ -958,13 +962,24 @@ impl Punct {
     /// which can be further configured with the `set_span` method below.
     #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
     pub fn new(ch: char, spacing: Spacing) -> Punct {
-        Punct(bridge::client::Punct::new(ch, spacing))
+        const LEGAL_CHARS: &[char] = &[
+            '=', '<', '>', '!', '~', '+', '-', '*', '/', '%', '^', '&', '|', '@', '.', ',', ';',
+            ':', '#', '$', '?', '\'',
+        ];
+        if !LEGAL_CHARS.contains(&ch) {
+            panic!("unsupported character `{:?}`", ch);
+        }
+        Punct(bridge::Punct {
+            ch: ch as u8,
+            joint: spacing == Spacing::Joint,
+            span: Span::call_site().0,
+        })
     }
 
     /// Returns the value of this punctuation character as `char`.
     #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
     pub fn as_char(&self) -> char {
-        self.0.as_char()
+        self.0.ch as char
     }
 
     /// Returns the spacing of this punctuation character, indicating whether it's immediately
@@ -973,28 +988,19 @@ impl Punct {
     /// (`Alone`) so the operator has certainly ended.
     #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
     pub fn spacing(&self) -> Spacing {
-        self.0.spacing()
+        if self.0.joint { Spacing::Joint } else { Spacing::Alone }
     }
 
     /// Returns the span for this punctuation character.
     #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
     pub fn span(&self) -> Span {
-        Span(self.0.span())
+        Span(self.0.span)
     }
 
     /// Configure the span for this punctuation character.
     #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
     pub fn set_span(&mut self, span: Span) {
-        self.0 = self.0.with_span(span.0);
+        self.0.span = span.0;
-    }
-}
-
-// N.B., the bridge only provides `to_string`, implement `fmt::Display`
-// based on it (the reverse of the usual relationship between the two).
-#[stable(feature = "proc_macro_lib", since = "1.15.0")]
-impl ToString for Punct {
-    fn to_string(&self) -> String {
-        TokenStream::from(TokenTree::from(self.clone())).to_string()
     }
 }
 
@@ -1003,7 +1009,7 @@ impl ToString for Punct {
 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
 impl fmt::Display for Punct {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.write_str(&self.to_string())
+        write!(f, "{}", self.as_char())
     }
 }
 

src/test/ui/proc-macro/invalid-punct-ident-1.rs

@@ -2,7 +2,7 @@
 // rustc-env:RUST_BACKTRACE=0
 
 // FIXME https://github.com/rust-lang/rust/issues/59998
-// normalize-stderr-test "thread.*panicked.*proc_macro_server.rs.*\n" -> ""
+// normalize-stderr-test "thread.*panicked.*proc_macro.*lib.rs.*\n" -> ""
 // normalize-stderr-test "note:.*RUST_BACKTRACE=1.*\n" -> ""
 // normalize-stderr-test "\nerror: internal compiler error.*\n\n" -> ""
 // normalize-stderr-test "note:.*unexpectedly panicked.*\n\n" -> ""