Move doc comment desugaring out of TokenCursor.

`TokenCursor` currently does doc comment desugaring on the fly, if the
`desugar_doc_comment` field is set. This requires also modifying the
token stream on the fly with `replace_prev_and_rewind`.

This commit moves the doc comment desugaring out of `TokenCursor`, by
introducing a new `TokenStream::desugar_doc_comment` method. This
separation of desugaring and iterating makes the code nicer.

compiler/rustc_ast/src/tokenstream.rs

@@ -13,7 +13,7 @@
 //! and a borrowed `TokenStream` is sufficient to build an owned `TokenStream` without taking
 //! ownership of the original.
 
-use crate::ast::StmtKind;
+use crate::ast::{AttrStyle, StmtKind};
 use crate::ast_traits::{HasAttrs, HasSpan, HasTokens};
 use crate::token::{self, Delimiter, Nonterminal, Token, TokenKind};
 use crate::AttrVec;
@@ -22,11 +22,11 @@ use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::sync::{self, Lrc};
 use rustc_macros::HashStable_Generic;
 use rustc_serialize::{Decodable, Decoder, Encodable, Encoder};
-use rustc_span::{Span, DUMMY_SP};
+use rustc_span::{sym, Span, Symbol, DUMMY_SP};
 use smallvec::{smallvec, SmallVec};
 
 use std::borrow::Cow;
-use std::{fmt, iter, mem};
+use std::{cmp, fmt, iter, mem};
 
 /// When the main Rust parser encounters a syntax-extension invocation, it
 /// parses the arguments to the invocation as a token tree. This is a very
@@ -566,6 +566,92 @@ impl TokenStream {
     pub fn chunks(&self, chunk_size: usize) -> core::slice::Chunks<'_, TokenTree> {
         self.0.chunks(chunk_size)
     }
+
+    /// Desugar doc comments like `/// foo` in the stream into `#[doc =
+    /// r"foo"]`. Modifies the `TokenStream` via `Lrc::make_mut`, but as little
+    /// as possible.
+    pub fn desugar_doc_comments(&mut self) {
+        if let Some(desugared_stream) = desugar_inner(self.clone()) {
+            *self = desugared_stream;
+        }
+
+        // The return value is `None` if nothing in `stream` changed.
+        fn desugar_inner(mut stream: TokenStream) -> Option<TokenStream> {
+            let mut i = 0;
+            let mut modified = false;
+            while let Some(tt) = stream.0.get(i) {
+                match tt {
+                    &TokenTree::Token(
+                        Token { kind: token::DocComment(_, attr_style, data), span },
+                        _spacing,
+                    ) => {
+                        let desugared = desugared_tts(attr_style, data, span);
+                        let desugared_len = desugared.len();
+                        Lrc::make_mut(&mut stream.0).splice(i..i + 1, desugared);
+                        modified = true;
+                        i += desugared_len;
+                    }
+
+                    &TokenTree::Token(..) => i += 1,
+
+                    &TokenTree::Delimited(sp, delim, ref delim_stream) => {
+                        if let Some(desugared_delim_stream) = desugar_inner(delim_stream.clone()) {
+                            let new_tt = TokenTree::Delimited(sp, delim, desugared_delim_stream);
+                            Lrc::make_mut(&mut stream.0)[i] = new_tt;
+                            modified = true;
+                        }
+                        i += 1;
+                    }
+                }
+            }
+            if modified { Some(stream) } else { None }
+        }
+
+        fn desugared_tts(attr_style: AttrStyle, data: Symbol, span: Span) -> Vec<TokenTree> {
+            // Searches for the occurrences of `"#*` and returns the minimum number of `#`s
+            // required to wrap the text. E.g.
+            // - `abc d` is wrapped as `r"abc d"` (num_of_hashes = 0)
+            // - `abc "d"` is wrapped as `r#"abc "d""#` (num_of_hashes = 1)
+            // - `abc "##d##"` is wrapped as `r###"abc ##"d"##"###` (num_of_hashes = 3)
+            let mut num_of_hashes = 0;
+            let mut count = 0;
+            for ch in data.as_str().chars() {
+                count = match ch {
+                    '"' => 1,
+                    '#' if count > 0 => count + 1,
+                    _ => 0,
+                };
+                num_of_hashes = cmp::max(num_of_hashes, count);
+            }
+
+            // `/// foo` becomes `doc = r"foo"`.
+            let delim_span = DelimSpan::from_single(span);
+            let body = TokenTree::Delimited(
+                delim_span,
+                Delimiter::Bracket,
+                [
+                    TokenTree::token_alone(token::Ident(sym::doc, false), span),
+                    TokenTree::token_alone(token::Eq, span),
+                    TokenTree::token_alone(
+                        TokenKind::lit(token::StrRaw(num_of_hashes), data, None),
+                        span,
+                    ),
+                ]
+                .into_iter()
+                .collect::<TokenStream>(),
+            );
+
+            if attr_style == AttrStyle::Inner {
+                vec![
+                    TokenTree::token_alone(token::Pound, span),
+                    TokenTree::token_alone(token::Not, span),
+                    body,
+                ]
+            } else {
+                vec![TokenTree::token_alone(token::Pound, span), body]
+            }
+        }
+    }
 }
 
 /// By-reference iterator over a [`TokenStream`], that produces `&TokenTree`