Eliminate comments::Literal

src/librustc/hir/print.rs

@@ -19,7 +19,6 @@ use std::ascii;
 use std::borrow::Cow;
 use std::cell::Cell;
 use std::io::{self, Write, Read};
-use std::iter::Peekable;
 use std::vec;
 
 pub enum AnnNode<'a> {
@@ -77,7 +76,6 @@ pub struct State<'a> {
     pub s: pp::Printer<'a>,
     cm: Option<&'a SourceMap>,
     comments: Option<Vec<comments::Comment>>,
-    literals: Peekable<vec::IntoIter<comments::Literal>>,
     cur_cmnt: usize,
     boxes: Vec<pp::Breaks>,
     ann: &'a (dyn PpAnn + 'a),
@@ -99,14 +97,6 @@ impl<'a> PrintState<'a> for State<'a> {
     fn cur_cmnt(&mut self) -> &mut usize {
         &mut self.cur_cmnt
     }
-
-    fn cur_lit(&mut self) -> Option<&comments::Literal> {
-        self.literals.peek()
-    }
-
-    fn bump_lit(&mut self) -> Option<comments::Literal> {
-        self.literals.next()
-    }
 }
 
 #[allow(non_upper_case_globals)]
@@ -117,18 +107,16 @@ pub const default_columns: usize = 78;
 
 
 /// Requires you to pass an input filename and reader so that
-/// it can scan the input text for comments and literals to
+/// it can scan the input text for comments to copy forward.
-/// copy forward.
 pub fn print_crate<'a>(cm: &'a SourceMap,
                        sess: &ParseSess,
                        krate: &hir::Crate,
                        filename: FileName,
                        input: &mut dyn Read,
                        out: Box<dyn Write + 'a>,
-                       ann: &'a dyn PpAnn,
+                       ann: &'a dyn PpAnn)
-                       is_expanded: bool)
                        -> io::Result<()> {
-    let mut s = State::new_from_input(cm, sess, filename, input, out, ann, is_expanded);
+    let mut s = State::new_from_input(cm, sess, filename, input, out, ann);
 
     // When printing the AST, we sometimes need to inject `#[no_std]` here.
     // Since you can't compile the HIR, it's not necessary.
@@ -144,36 +132,21 @@ impl<'a> State<'a> {
                           filename: FileName,
                           input: &mut dyn Read,
                           out: Box<dyn Write + 'a>,
-                          ann: &'a dyn PpAnn,
+                          ann: &'a dyn PpAnn)
-                          is_expanded: bool)
                           -> State<'a> {
-        let (cmnts, lits) = comments::gather_comments_and_literals(sess, filename, input);
+        let comments = comments::gather_comments(sess, filename, input);
-
+        State::new(cm, out, ann, Some(comments))
-        State::new(cm,
-                   out,
-                   ann,
-                   Some(cmnts),
-                   // If the code is post expansion, don't use the table of
-                   // literals, since it doesn't correspond with the literals
-                   // in the AST anymore.
-                   if is_expanded {
-                       None
-                   } else {
-                       Some(lits)
-                   })
     }
 
     pub fn new(cm: &'a SourceMap,
                out: Box<dyn Write + 'a>,
                ann: &'a dyn PpAnn,
-               comments: Option<Vec<comments::Comment>>,
+               comments: Option<Vec<comments::Comment>>)
-               literals: Option<Vec<comments::Literal>>)
                -> State<'a> {
         State {
             s: pp::mk_printer(out, default_columns),
             cm: Some(cm),
             comments,
-            literals: literals.unwrap_or_default().into_iter().peekable(),
             cur_cmnt: 0,
             boxes: Vec::new(),
             ann,
@@ -190,7 +163,6 @@ pub fn to_string<F>(ann: &dyn PpAnn, f: F) -> String
             s: pp::mk_printer(Box::new(&mut wr), default_columns),
             cm: None,
             comments: None,
-            literals: vec![].into_iter().peekable(),
             cur_cmnt: 0,
             boxes: Vec::new(),
             ann,
@@ -1279,9 +1251,6 @@ impl<'a> State<'a> {
 
     fn print_literal(&mut self, lit: &hir::Lit) -> io::Result<()> {
         self.maybe_print_comment(lit.span.lo())?;
-        if let Some(ltrl) = self.next_lit(lit.span.lo()) {
-            return self.writer().word(ltrl.lit.clone());
-        }
         match lit.node {
             hir::LitKind::Str(st, style) => self.print_string(&st.as_str(), style),
             hir::LitKind::Err(st) => {

src/librustc_driver/pretty.rs

@@ -805,8 +805,7 @@ pub fn print_after_hir_lowering<'tcx>(
                                             src_name,
                                             &mut rdr,
                                             box out,
-                                            annotation.pp_ann(),
+                                            annotation.pp_ann())
-                                            true)
                 })
             }
 
@@ -829,8 +828,7 @@ pub fn print_after_hir_lowering<'tcx>(
                                                                          src_name,
                                                                          &mut rdr,
                                                                          box out,
-                                                                         annotation.pp_ann(),
+                                                                         annotation.pp_ann());
-                                                                         true);
                     for node_id in uii.all_matching_node_ids(hir_map) {
                         let node = hir_map.get(node_id);
                         pp_state.print_node(node)?;

src/librustdoc/clean/cfg.rs

@@ -414,10 +414,9 @@ impl<'a> fmt::Display for Html<'a> {
 mod test {
     use super::Cfg;
 
-    use syntax::symbol::Symbol;
-    use syntax::ast::*;
-    use syntax::source_map::dummy_spanned;
     use syntax_pos::DUMMY_SP;
+    use syntax::ast::*;
+    use syntax::symbol::Symbol;
     use syntax::with_globals;
 
     fn word_cfg(s: &str) -> Cfg {
@@ -592,12 +591,11 @@ mod test {
             let mi = dummy_meta_item_word("all");
             assert_eq!(Cfg::parse(&mi), Ok(word_cfg("all")));
 
+            let node = LitKind::Str(Symbol::intern("done"), StrStyle::Cooked);
+            let (token, suffix) =   node.lit_token();
             let mi = MetaItem {
                 path: Path::from_ident(Ident::from_str("all")),
-                node: MetaItemKind::NameValue(dummy_spanned(LitKind::Str(
+                node: MetaItemKind::NameValue(Lit { node, token, suffix, span: DUMMY_SP }),
-                    Symbol::intern("done"),
-                    StrStyle::Cooked,
-                ))),
                 span: DUMMY_SP,
             };
             assert_eq!(Cfg::parse(&mi), Ok(name_value_cfg("all", "done")));
@@ -627,9 +625,11 @@ mod test {
     #[test]
     fn test_parse_err() {
         with_globals(|| {
+            let node = LitKind::Bool(false);
+            let (token, suffix) = node.lit_token();
             let mi = MetaItem {
                 path: Path::from_ident(Ident::from_str("foo")),
-                node: MetaItemKind::NameValue(dummy_spanned(LitKind::Bool(false))),
+                node: MetaItemKind::NameValue(Lit { node, token, suffix, span: DUMMY_SP }),
                 span: DUMMY_SP,
             };
             assert!(Cfg::parse(&mi).is_err());

src/libsyntax/attr/mod.rs

@@ -565,8 +565,9 @@ impl MetaItemKind {
             Some(TokenTree::Token(_, token::Eq)) => {
                 tokens.next();
                 return if let Some(TokenTree::Token(span, token)) = tokens.next() {
-                    LitKind::from_token(token)
+                    LitKind::from_token(token).map(|(node, token, suffix)| {
-                        .map(|(node, token, suffix)| MetaItemKind::NameValue(Lit { node, token, suffix, span }))
+                        MetaItemKind::NameValue(Lit { node, token, suffix, span })
+                    })
                 } else {
                     None
                 };
@@ -635,7 +636,7 @@ impl LitKind {
         }
     }
 
-    pub(crate) fn lit_token(&self) -> (token::Lit, Option<Symbol>) {
+    pub fn lit_token(&self) -> (token::Lit, Option<Symbol>) {
         use std::ascii;
 
         match *self {

src/libsyntax/parse/lexer/comments.rs

@@ -3,8 +3,7 @@ pub use CommentStyle::*;
 use crate::ast;
 use crate::source_map::SourceMap;
 use crate::parse::lexer::{is_block_doc_comment, is_pattern_whitespace};
-use crate::parse::lexer::{self, ParseSess, StringReader, TokenAndSpan};
+use crate::parse::lexer::{self, ParseSess, StringReader};
-use crate::print::pprust;
 
 use syntax_pos::{BytePos, CharPos, Pos, FileName};
 use log::debug;
@@ -339,16 +338,9 @@ fn consume_comment(rdr: &mut StringReader<'_>,
     debug!("<<< consume comment");
 }
 
-#[derive(Clone)]
-pub struct Literal {
-    pub lit: String,
-    pub pos: BytePos,
-}
-
 // it appears this function is called only from pprust... that's
 // probably not a good thing.
-pub fn gather_comments_and_literals(sess: &ParseSess, path: FileName, srdr: &mut dyn Read)
+pub fn gather_comments(sess: &ParseSess, path: FileName, srdr: &mut dyn Read) -> Vec<Comment>
-    -> (Vec<Comment>, Vec<Literal>)
 {
     let mut src = String::new();
     srdr.read_to_string(&mut src).unwrap();
@@ -357,7 +349,6 @@ pub fn gather_comments_and_literals(sess: &ParseSess, path: FileName, srdr: &mut
     let mut rdr = lexer::StringReader::new_raw(sess, source_file, None);
 
     let mut comments: Vec<Comment> = Vec::new();
-    let mut literals: Vec<Literal> = Vec::new();
     let mut code_to_the_left = false; // Only code
     let mut anything_to_the_left = false; // Code or comments
 
@@ -382,26 +373,12 @@ pub fn gather_comments_and_literals(sess: &ParseSess, path: FileName, srdr: &mut
             }
         }
 
-        let bstart = rdr.pos;
         rdr.next_token();
-        // discard, and look ahead; we're working with internal state
-        let TokenAndSpan { tok, sp } = rdr.peek();
-        if tok.is_lit() {
-            rdr.with_str_from(bstart, |s| {
-                debug!("tok lit: {}", s);
-                literals.push(Literal {
-                    lit: s.to_string(),
-                    pos: sp.lo(),
-                });
-            })
-        } else {
-            debug!("tok: {}", pprust::token_to_string(&tok));
-        }
         code_to_the_left = true;
         anything_to_the_left = true;
     }
 
-    (comments, literals)
+    comments
 }
 
 #[cfg(test)]

src/libsyntax/parse/parser.rs

@@ -2121,11 +2121,11 @@ impl<'a> Parser<'a> {
                         Applicability::MachineApplicable,
                     );
                     err.emit();
-                    return Ok(match float_suffix {
+                    return Ok((match float_suffix {
-                        "f32" => (ast::LitKind::Float(val, ast::FloatTy::F32), token::Float(val), suffix),
+                        "f32" => ast::LitKind::Float(val, ast::FloatTy::F32),
-                        "f64" => (ast::LitKind::Float(val, ast::FloatTy::F64), token::Float(val), suffix),
+                        "f64" => ast::LitKind::Float(val, ast::FloatTy::F64),
-                        _ => (ast::LitKind::FloatUnsuffixed(val), token::Float(val), suffix),
+                        _ => ast::LitKind::FloatUnsuffixed(val),
-                    });
+                    }, token::Float(val), suffix));
                 } else {
                     unreachable!();
                 };

src/libsyntax/print/pprust.rs

@@ -20,10 +20,8 @@ use rustc_target::spec::abi::{self, Abi};
 use syntax_pos::{self, BytePos};
 use syntax_pos::{DUMMY_SP, FileName};
 
-use std::ascii;
 use std::borrow::Cow;
 use std::io::{self, Write, Read};
-use std::iter::Peekable;
 use std::vec;
 
 pub enum AnnNode<'a> {
@@ -49,8 +47,7 @@ impl PpAnn for NoAnn {}
 pub struct State<'a> {
     pub s: pp::Printer<'a>,
     cm: Option<&'a SourceMap>,
-    comments: Option<Vec<comments::Comment> >,
+    comments: Option<Vec<comments::Comment>>,
-    literals: Peekable<vec::IntoIter<comments::Literal>>,
     cur_cmnt: usize,
     boxes: Vec<pp::Breaks>,
     ann: &'a (dyn PpAnn+'a),
@@ -62,7 +59,6 @@ fn rust_printer<'a>(writer: Box<dyn Write+'a>, ann: &'a dyn PpAnn) -> State<'a>
         s: pp::mk_printer(writer, DEFAULT_COLUMNS),
         cm: None,
         comments: None,
-        literals: vec![].into_iter().peekable(),
         cur_cmnt: 0,
         boxes: Vec::new(),
         ann,
@@ -75,8 +71,7 @@ pub const INDENT_UNIT: usize = 4;
 pub const DEFAULT_COLUMNS: usize = 78;
 
 /// Requires you to pass an input filename and reader so that
-/// it can scan the input text for comments and literals to
+/// it can scan the input text for comments to copy forward.
-/// copy forward.
 pub fn print_crate<'a>(cm: &'a SourceMap,
                        sess: &ParseSess,
                        krate: &ast::Crate,
@@ -118,36 +113,23 @@ impl<'a> State<'a> {
                           out: Box<dyn Write+'a>,
                           ann: &'a dyn PpAnn,
                           is_expanded: bool) -> State<'a> {
-        let (cmnts, lits) = comments::gather_comments_and_literals(sess, filename, input);
+        let comments = comments::gather_comments(sess, filename, input);
-
+        State::new(cm, out, ann, Some(comments), is_expanded)
-        State::new(
-            cm,
-            out,
-            ann,
-            Some(cmnts),
-            // If the code is post expansion, don't use the table of
-            // literals, since it doesn't correspond with the literals
-            // in the AST anymore.
-            if is_expanded { None } else { Some(lits) },
-            is_expanded
-        )
     }
 
     pub fn new(cm: &'a SourceMap,
                out: Box<dyn Write+'a>,
                ann: &'a dyn PpAnn,
                comments: Option<Vec<comments::Comment>>,
-               literals: Option<Vec<comments::Literal>>,
                is_expanded: bool) -> State<'a> {
         State {
             s: pp::mk_printer(out, DEFAULT_COLUMNS),
             cm: Some(cm),
             comments,
-            literals: literals.unwrap_or_default().into_iter().peekable(),
             cur_cmnt: 0,
             boxes: Vec::new(),
             ann,
-            is_expanded: is_expanded
+            is_expanded,
         }
     }
 }
@@ -180,6 +162,31 @@ fn binop_to_string(op: BinOpToken) -> &'static str {
     }
 }
 
+fn literal_to_string(lit: token::Lit, suffix: Option<ast::Name>) -> String {
+    let mut out = match lit {
+        token::Byte(b)           => format!("b'{}'", b),
+        token::Char(c)           => format!("'{}'", c),
+        token::Err(c)            => format!("'{}'", c),
+        token::Bool(c)           |
+        token::Float(c)          |
+        token::Integer(c)        => c.to_string(),
+        token::Str_(s)           => format!("\"{}\"", s),
+        token::StrRaw(s, n)      => format!("r{delim}\"{string}\"{delim}",
+                                            delim="#".repeat(n as usize),
+                                            string=s),
+        token::ByteStr(v)        => format!("b\"{}\"", v),
+        token::ByteStrRaw(s, n)  => format!("br{delim}\"{string}\"{delim}",
+                                            delim="#".repeat(n as usize),
+                                            string=s),
+    };
+
+    if let Some(suffix) = suffix {
+        out.push_str(&suffix.as_str())
+    }
+
+    out
+}
+
 pub fn token_to_string(tok: &Token) -> String {
     match *tok {
         token::Eq                   => "=".to_string(),
@@ -223,30 +230,7 @@ pub fn token_to_string(tok: &Token) -> String {
         token::SingleQuote          => "'".to_string(),
 
         /* Literals */
-        token::Literal(lit, suf) => {
+        token::Literal(lit, suf) => literal_to_string(lit, suf),
-            let mut out = match lit {
-                token::Bool(_)          => panic!("literal token contains `Lit::Bool`"),
-                token::Byte(b)           => format!("b'{}'", b),
-                token::Char(c)           => format!("'{}'", c),
-                token::Err(c)            => format!("'{}'", c),
-                token::Float(c)          |
-                token::Integer(c)        => c.to_string(),
-                token::Str_(s)           => format!("\"{}\"", s),
-                token::StrRaw(s, n)      => format!("r{delim}\"{string}\"{delim}",
-                                                    delim="#".repeat(n as usize),
-                                                    string=s),
-                token::ByteStr(v)         => format!("b\"{}\"", v),
-                token::ByteStrRaw(s, n)   => format!("br{delim}\"{string}\"{delim}",
-                                                    delim="#".repeat(n as usize),
-                                                    string=s),
-            };
-
-            if let Some(s) = suf {
-                out.push_str(&s.as_str())
-            }
-
-            out
-        }
 
         /* Name components */
         token::Ident(s, false)      => s.to_string(),
@@ -439,8 +423,6 @@ pub trait PrintState<'a> {
     fn boxes(&mut self) -> &mut Vec<pp::Breaks>;
     fn comments(&mut self) -> &mut Option<Vec<comments::Comment>>;
     fn cur_cmnt(&mut self) -> &mut usize;
-    fn cur_lit(&mut self) -> Option<&comments::Literal>;
-    fn bump_lit(&mut self) -> Option<comments::Literal>;
 
     fn word_space<S: Into<Cow<'static, str>>>(&mut self, w: S) -> io::Result<()> {
         self.writer().word(w)?;
@@ -505,21 +487,6 @@ pub trait PrintState<'a> {
         self.end()
     }
 
-    fn next_lit(&mut self, pos: BytePos) -> Option<comments::Literal> {
-        while let Some(ltrl) = self.cur_lit().cloned() {
-            if ltrl.pos > pos { break; }
-
-            // we don't need the value here since we're forced to clone cur_lit
-            // due to lack of NLL.
-            self.bump_lit();
-            if ltrl.pos == pos {
-                return Some(ltrl);
-            }
-        }
-
-        None
-    }
-
     fn maybe_print_comment(&mut self, pos: BytePos) -> io::Result<()> {
         while let Some(ref cmnt) = self.next_comment() {
             if cmnt.pos < pos {
@@ -607,60 +574,7 @@ pub trait PrintState<'a> {
 
     fn print_literal(&mut self, lit: &ast::Lit) -> io::Result<()> {
         self.maybe_print_comment(lit.span.lo())?;
-        if let Some(ltrl) = self.next_lit(lit.span.lo()) {
+        self.writer().word(literal_to_string(lit.token, lit.suffix))
-            return self.writer().word(ltrl.lit.clone());
-        }
-        match lit.node {
-            ast::LitKind::Str(st, style) => self.print_string(&st.as_str(), style),
-            ast::LitKind::Err(st) => {
-                let st = st.as_str().escape_debug().to_string();
-                let mut res = String::with_capacity(st.len() + 2);
-                res.push('\'');
-                res.push_str(&st);
-                res.push('\'');
-                self.writer().word(res)
-            }
-            ast::LitKind::Byte(byte) => {
-                let mut res = String::from("b'");
-                res.extend(ascii::escape_default(byte).map(|c| c as char));
-                res.push('\'');
-                self.writer().word(res)
-            }
-            ast::LitKind::Char(ch) => {
-                let mut res = String::from("'");
-                res.extend(ch.escape_default());
-                res.push('\'');
-                self.writer().word(res)
-            }
-            ast::LitKind::Int(i, t) => {
-                match t {
-                    ast::LitIntType::Signed(st) => {
-                        self.writer().word(st.val_to_string(i as i128))
-                    }
-                    ast::LitIntType::Unsigned(ut) => {
-                        self.writer().word(ut.val_to_string(i))
-                    }
-                    ast::LitIntType::Unsuffixed => {
-                        self.writer().word(i.to_string())
-                    }
-                }
-            }
-            ast::LitKind::Float(ref f, t) => {
-                self.writer().word(format!("{}{}", &f, t.ty_to_string()))
-            }
-            ast::LitKind::FloatUnsuffixed(ref f) => self.writer().word(f.as_str().to_string()),
-            ast::LitKind::Bool(val) => {
-                if val { self.writer().word("true") } else { self.writer().word("false") }
-            }
-            ast::LitKind::ByteStr(ref v) => {
-                let mut escaped: String = String::new();
-                for &ch in v.iter() {
-                    escaped.extend(ascii::escape_default(ch)
-                                         .map(|c| c as char));
-                }
-                self.writer().word(format!("b\"{}\"", escaped))
-            }
-        }
     }
 
     fn print_string(&mut self, st: &str,
@@ -881,14 +795,6 @@ impl<'a> PrintState<'a> for State<'a> {
     fn cur_cmnt(&mut self) -> &mut usize {
         &mut self.cur_cmnt
     }
-
-    fn cur_lit(&mut self) -> Option<&comments::Literal> {
-        self.literals.peek()
-    }
-
-    fn bump_lit(&mut self) -> Option<comments::Literal> {
-        self.literals.next()
-    }
 }
 
 impl<'a> State<'a> {