Rollup merge of #107544 - nnethercote:improve-TokenCursor, r=petrochenkov

Improve `TokenCursor`.

Some small improvements, for things that were bugging me.

Best reviewed one commit at a time.

r? ``@petrochenkov``

compiler/rustc_ast/src/tokenstream.rs

@@ -41,7 +41,8 @@ use std::{fmt, iter};
 /// Nothing special happens to misnamed or misplaced `SubstNt`s.
 #[derive(Debug, Clone, PartialEq, Encodable, Decodable, HashStable_Generic)]
 pub enum TokenTree {
-    /// A single token.
+    /// A single token. Should never be `OpenDelim` or `CloseDelim`, because
+    /// delimiters are implicitly represented by `Delimited`.
     Token(Token, Spacing),
     /// A delimited sequence of token trees.
     Delimited(DelimSpan, Delimiter, TokenStream),
@@ -388,12 +389,12 @@ impl TokenStream {
         self.0.len()
     }
 
-    pub fn trees(&self) -> CursorRef<'_> {
-        CursorRef::new(self)
+    pub fn trees(&self) -> RefTokenTreeCursor<'_> {
+        RefTokenTreeCursor::new(self)
     }
 
-    pub fn into_trees(self) -> Cursor {
-        Cursor::new(self)
+    pub fn into_trees(self) -> TokenTreeCursor {
+        TokenTreeCursor::new(self)
     }
 
     /// Compares two `TokenStream`s, checking equality without regarding span information.
@@ -551,16 +552,17 @@ impl TokenStream {
     }
 }
 
-/// By-reference iterator over a [`TokenStream`].
+/// By-reference iterator over a [`TokenStream`], that produces `&TokenTree`
+/// items.
 #[derive(Clone)]
-pub struct CursorRef<'t> {
+pub struct RefTokenTreeCursor<'t> {
     stream: &'t TokenStream,
     index: usize,
 }
 
-impl<'t> CursorRef<'t> {
+impl<'t> RefTokenTreeCursor<'t> {
     fn new(stream: &'t TokenStream) -> Self {
-        CursorRef { stream, index: 0 }
+        RefTokenTreeCursor { stream, index: 0 }
     }
 
     pub fn look_ahead(&self, n: usize) -> Option<&TokenTree> {
@@ -568,7 +570,7 @@ impl<'t> CursorRef<'t> {
     }
 }
 
-impl<'t> Iterator for CursorRef<'t> {
+impl<'t> Iterator for RefTokenTreeCursor<'t> {
     type Item = &'t TokenTree;
 
     fn next(&mut self) -> Option<&'t TokenTree> {
@@ -579,15 +581,16 @@ impl<'t> Iterator for CursorRef<'t> {
     }
 }
 
-/// Owning by-value iterator over a [`TokenStream`].
+/// Owning by-value iterator over a [`TokenStream`], that produces `TokenTree`
+/// items.
 // FIXME: Many uses of this can be replaced with by-reference iterator to avoid clones.
 #[derive(Clone)]
-pub struct Cursor {
+pub struct TokenTreeCursor {
     pub stream: TokenStream,
     index: usize,
 }
 
-impl Iterator for Cursor {
+impl Iterator for TokenTreeCursor {
     type Item = TokenTree;
 
     fn next(&mut self) -> Option<TokenTree> {
@@ -598,9 +601,9 @@ impl Iterator for Cursor {
     }
 }
 
-impl Cursor {
+impl TokenTreeCursor {
     fn new(stream: TokenStream) -> Self {
-        Cursor { stream, index: 0 }
+        TokenTreeCursor { stream, index: 0 }
     }
 
     #[inline]
@@ -614,6 +617,15 @@ impl Cursor {
     pub fn look_ahead(&self, n: usize) -> Option<&TokenTree> {
         self.stream.0.get(self.index + n)
     }
+
+    // Replace the previously obtained token tree with `tts`, and rewind to
+    // just before them.
+    pub fn replace_prev_and_rewind(&mut self, tts: Vec<TokenTree>) {
+        assert!(self.index > 0);
+        self.index -= 1;
+        let stream = Lrc::make_mut(&mut self.stream.0);
+        stream.splice(self.index..self.index + 1, tts);
+    }
 }
 
 #[derive(Debug, Copy, Clone, PartialEq, Encodable, Decodable, HashStable_Generic)]