bjoernager/rust
bjoernager/rust
1
Fork 0
Code Issues Pull requests Activity

Eliminate comments::Literal

Browse source
This commit is contained in:
Vadim Petrochenkov 2019-05-09 19:04:04 +03:00
parent 751ae5af1a
commit a5b3f33cb9
7 changed files with 61 additions and 210 deletions
Download patch file Download diff file Expand all files Collapse all files

45
src/librustc/hir/print.rs
View file

@ -19,7 +19,6 @@ use std::ascii;
use std::borrow::Cow;
use std::cell::Cell;
use std::io::{self, Write, Read};
use std::iter::Peekable;
use std::vec;
pub enum AnnNode<'a> {
@ -77,7 +76,6 @@ pub struct State<'a> {
pub s: pp::Printer<'a>,
cm: Option<&'a SourceMap>,
comments: Option<Vec<comments::Comment>>,
literals: Peekable<vec::IntoIter<comments::Literal>>,
cur_cmnt: usize,
boxes: Vec<pp::Breaks>,
ann: &'a (dyn PpAnn + 'a),
@ -99,14 +97,6 @@ impl<'a> PrintState<'a> for State<'a> {
fn cur_cmnt(&mut self) -> &mut usize {
&mut self.cur_cmnt
}
fn cur_lit(&mut self) -> Option<&comments::Literal> {
self.literals.peek()
}
fn bump_lit(&mut self) -> Option<comments::Literal> {
self.literals.next()
}
}
#[allow(non_upper_case_globals)]
@ -117,18 +107,16 @@ pub const default_columns: usize = 78;
/// Requires you to pass an input filename and reader so that
/// it can scan the input text for comments and literals to
/// copy forward.
/// it can scan the input text for comments to copy forward.
pub fn print_crate<'a>(cm: &'a SourceMap,
sess: &ParseSess,
krate: &hir::Crate,
filename: FileName,
input: &mut dyn Read,
out: Box<dyn Write + 'a>,
ann: &'a dyn PpAnn,
is_expanded: bool)
ann: &'a dyn PpAnn)
-> io::Result<()> {
let mut s = State::new_from_input(cm, sess, filename, input, out, ann, is_expanded);
let mut s = State::new_from_input(cm, sess, filename, input, out, ann);
// When printing the AST, we sometimes need to inject `#[no_std]` here.
// Since you can't compile the HIR, it's not necessary.
@ -144,36 +132,21 @@ impl<'a> State<'a> {
filename: FileName,
input: &mut dyn Read,
out: Box<dyn Write + 'a>,
ann: &'a dyn PpAnn,
is_expanded: bool)
ann: &'a dyn PpAnn)
-> State<'a> {
let (cmnts, lits) = comments::gather_comments_and_literals(sess, filename, input);
State::new(cm,
out,
ann,
Some(cmnts),
// If the code is post expansion, don't use the table of
// literals, since it doesn't correspond with the literals
// in the AST anymore.
if is_expanded {
None
} else {
Some(lits)
})
let comments = comments::gather_comments(sess, filename, input);
State::new(cm, out, ann, Some(comments))
}
pub fn new(cm: &'a SourceMap,
out: Box<dyn Write + 'a>,
ann: &'a dyn PpAnn,
comments: Option<Vec<comments::Comment>>,
literals: Option<Vec<comments::Literal>>)
comments: Option<Vec<comments::Comment>>)
-> State<'a> {
State {
s: pp::mk_printer(out, default_columns),
cm: Some(cm),
comments,
literals: literals.unwrap_or_default().into_iter().peekable(),
cur_cmnt: 0,
boxes: Vec::new(),
ann,
@ -190,7 +163,6 @@ pub fn to_string<F>(ann: &dyn PpAnn, f: F) -> String
s: pp::mk_printer(Box::new(&mut wr), default_columns),
cm: None,
comments: None,
literals: vec![].into_iter().peekable(),
cur_cmnt: 0,
boxes: Vec::new(),
ann,
@ -1279,9 +1251,6 @@ impl<'a> State<'a> {
fn print_literal(&mut self, lit: &hir::Lit) -> io::Result<()> {
self.maybe_print_comment(lit.span.lo())?;
if let Some(ltrl) = self.next_lit(lit.span.lo()) {
return self.writer().word(ltrl.lit.clone());
}
match lit.node {
hir::LitKind::Str(st, style) => self.print_string(&st.as_str(), style),
hir::LitKind::Err(st) => {

6
src/librustc_driver/pretty.rs
View file

@ -805,8 +805,7 @@ pub fn print_after_hir_lowering<'tcx>(
src_name,
&mut rdr,
box out,
annotation.pp_ann(),
true)
annotation.pp_ann())
})
}
@ -829,8 +828,7 @@ pub fn print_after_hir_lowering<'tcx>(
src_name,
&mut rdr,
box out,
annotation.pp_ann(),
true);
annotation.pp_ann());
for node_id in uii.all_matching_node_ids(hir_map) {
let node = hir_map.get(node_id);
pp_state.print_node(node)?;

16
src/librustdoc/clean/cfg.rs
View file

@ -414,10 +414,9 @@ impl<'a> fmt::Display for Html<'a> {
mod test {
use super::Cfg;
use syntax::symbol::Symbol;
use syntax::ast::*;
use syntax::source_map::dummy_spanned;
use syntax_pos::DUMMY_SP;
use syntax::ast::*;
use syntax::symbol::Symbol;
use syntax::with_globals;
fn word_cfg(s: &str) -> Cfg {
@ -592,12 +591,11 @@ mod test {
let mi = dummy_meta_item_word("all");
assert_eq!(Cfg::parse(&mi), Ok(word_cfg("all")));
let node = LitKind::Str(Symbol::intern("done"), StrStyle::Cooked);
let (token, suffix) = node.lit_token();
let mi = MetaItem {
path: Path::from_ident(Ident::from_str("all")),
node: MetaItemKind::NameValue(dummy_spanned(LitKind::Str(
Symbol::intern("done"),
StrStyle::Cooked,
))),
node: MetaItemKind::NameValue(Lit { node, token, suffix, span: DUMMY_SP }),
span: DUMMY_SP,
};
assert_eq!(Cfg::parse(&mi), Ok(name_value_cfg("all", "done")));
@ -627,9 +625,11 @@ mod test {
#[test]
fn test_parse_err() {
with_globals(|| {
let node = LitKind::Bool(false);
let (token, suffix) = node.lit_token();
let mi = MetaItem {
path: Path::from_ident(Ident::from_str("foo")),
node: MetaItemKind::NameValue(dummy_spanned(LitKind::Bool(false))),
node: MetaItemKind::NameValue(Lit { node, token, suffix, span: DUMMY_SP }),
span: DUMMY_SP,
};
assert!(Cfg::parse(&mi).is_err());

7
src/libsyntax/attr/mod.rs
View file

@ -565,8 +565,9 @@ impl MetaItemKind {
Some(TokenTree::Token(_, token::Eq)) => {
tokens.next();
return if let Some(TokenTree::Token(span, token)) = tokens.next() {
LitKind::from_token(token)
.map(|(node, token, suffix)| MetaItemKind::NameValue(Lit { node, token, suffix, span }))
LitKind::from_token(token).map(|(node, token, suffix)| {
MetaItemKind::NameValue(Lit { node, token, suffix, span })
})
} else {
None
};
@ -635,7 +636,7 @@ impl LitKind {
}
}
pub(crate) fn lit_token(&self) -> (token::Lit, Option<Symbol>) {
pub fn lit_token(&self) -> (token::Lit, Option<Symbol>) {
use std::ascii;
match *self {

29
src/libsyntax/parse/lexer/comments.rs
View file

@ -3,8 +3,7 @@ pub use CommentStyle::*;
use crate::ast;
use crate::source_map::SourceMap;
use crate::parse::lexer::{is_block_doc_comment, is_pattern_whitespace};
use crate::parse::lexer::{self, ParseSess, StringReader, TokenAndSpan};
use crate::print::pprust;
use crate::parse::lexer::{self, ParseSess, StringReader};
use syntax_pos::{BytePos, CharPos, Pos, FileName};
use log::debug;
@ -339,16 +338,9 @@ fn consume_comment(rdr: &mut StringReader<'_>,
debug!("<<< consume comment");
}
#[derive(Clone)]
pub struct Literal {
pub lit: String,
pub pos: BytePos,
}
// it appears this function is called only from pprust... that's
// probably not a good thing.
pub fn gather_comments_and_literals(sess: &ParseSess, path: FileName, srdr: &mut dyn Read)
-> (Vec<Comment>, Vec<Literal>)
pub fn gather_comments(sess: &ParseSess, path: FileName, srdr: &mut dyn Read) -> Vec<Comment>
{
let mut src = String::new();
srdr.read_to_string(&mut src).unwrap();
@ -357,7 +349,6 @@ pub fn gather_comments_and_literals(sess: &ParseSess, path: FileName, srdr: &mut
let mut rdr = lexer::StringReader::new_raw(sess, source_file, None);
let mut comments: Vec<Comment> = Vec::new();
let mut literals: Vec<Literal> = Vec::new();
let mut code_to_the_left = false; // Only code
let mut anything_to_the_left = false; // Code or comments
@ -382,26 +373,12 @@ pub fn gather_comments_and_literals(sess: &ParseSess, path: FileName, srdr: &mut
}
}
let bstart = rdr.pos;
rdr.next_token();
// discard, and look ahead; we're working with internal state
let TokenAndSpan { tok, sp } = rdr.peek();
if tok.is_lit() {
rdr.with_str_from(bstart, |s| {
debug!("tok lit: {}", s);
literals.push(Literal {
lit: s.to_string(),
pos: sp.lo(),
});
})
} else {
debug!("tok: {}", pprust::token_to_string(&tok));
}
code_to_the_left = true;
anything_to_the_left = true;
}
(comments, literals)
comments
}
#[cfg(test)]

10
src/libsyntax/parse/parser.rs
View file

@ -2121,11 +2121,11 @@ impl<'a> Parser<'a> {
Applicability::MachineApplicable,
);
err.emit();
return Ok(match float_suffix {
"f32" => (ast::LitKind::Float(val, ast::FloatTy::F32), token::Float(val), suffix),
"f64" => (ast::LitKind::Float(val, ast::FloatTy::F64), token::Float(val), suffix),
_ => (ast::LitKind::FloatUnsuffixed(val), token::Float(val), suffix),
});
return Ok((match float_suffix {
"f32" => ast::LitKind::Float(val, ast::FloatTy::F32),
"f64" => ast::LitKind::Float(val, ast::FloatTy::F64),
_ => ast::LitKind::FloatUnsuffixed(val),
}, token::Float(val), suffix));
} else {
unreachable!();
};

158
src/libsyntax/print/pprust.rs
View file

@ -20,10 +20,8 @@ use rustc_target::spec::abi::{self, Abi};
use syntax_pos::{self, BytePos};
use syntax_pos::{DUMMY_SP, FileName};
use std::ascii;
use std::borrow::Cow;
use std::io::{self, Write, Read};
use std::iter::Peekable;
use std::vec;
pub enum AnnNode<'a> {
@ -49,8 +47,7 @@ impl PpAnn for NoAnn {}
pub struct State<'a> {
pub s: pp::Printer<'a>,
cm: Option<&'a SourceMap>,
comments: Option<Vec<comments::Comment> >,
literals: Peekable<vec::IntoIter<comments::Literal>>,
comments: Option<Vec<comments::Comment>>,
cur_cmnt: usize,
boxes: Vec<pp::Breaks>,
ann: &'a (dyn PpAnn+'a),
@ -62,7 +59,6 @@ fn rust_printer<'a>(writer: Box<dyn Write+'a>, ann: &'a dyn PpAnn) -> State<'a>
s: pp::mk_printer(writer, DEFAULT_COLUMNS),
cm: None,
comments: None,
literals: vec![].into_iter().peekable(),
cur_cmnt: 0,
boxes: Vec::new(),
ann,
@ -75,8 +71,7 @@ pub const INDENT_UNIT: usize = 4;
pub const DEFAULT_COLUMNS: usize = 78;
/// Requires you to pass an input filename and reader so that
/// it can scan the input text for comments and literals to
/// copy forward.
/// it can scan the input text for comments to copy forward.
pub fn print_crate<'a>(cm: &'a SourceMap,
sess: &ParseSess,
krate: &ast::Crate,
@ -118,36 +113,23 @@ impl<'a> State<'a> {
out: Box<dyn Write+'a>,
ann: &'a dyn PpAnn,
is_expanded: bool) -> State<'a> {
let (cmnts, lits) = comments::gather_comments_and_literals(sess, filename, input);
State::new(
cm,
out,
ann,
Some(cmnts),
// If the code is post expansion, don't use the table of
// literals, since it doesn't correspond with the literals
// in the AST anymore.
if is_expanded { None } else { Some(lits) },
is_expanded
)
let comments = comments::gather_comments(sess, filename, input);
State::new(cm, out, ann, Some(comments), is_expanded)
}
pub fn new(cm: &'a SourceMap,
out: Box<dyn Write+'a>,
ann: &'a dyn PpAnn,
comments: Option<Vec<comments::Comment>>,
literals: Option<Vec<comments::Literal>>,
is_expanded: bool) -> State<'a> {
State {
s: pp::mk_printer(out, DEFAULT_COLUMNS),
cm: Some(cm),
comments,
literals: literals.unwrap_or_default().into_iter().peekable(),
cur_cmnt: 0,
boxes: Vec::new(),
ann,
is_expanded: is_expanded
is_expanded,
}
}
}
@ -180,6 +162,31 @@ fn binop_to_string(op: BinOpToken) -> &'static str {
}
}
fn literal_to_string(lit: token::Lit, suffix: Option<ast::Name>) -> String {
let mut out = match lit {
token::Byte(b) => format!("b'{}'", b),
token::Char(c) => format!("'{}'", c),
token::Err(c) => format!("'{}'", c),
token::Bool(c) |
token::Float(c) |
token::Integer(c) => c.to_string(),
token::Str_(s) => format!("\"{}\"", s),
token::StrRaw(s, n) => format!("r{delim}\"{string}\"{delim}",
delim="#".repeat(n as usize),
string=s),
token::ByteStr(v) => format!("b\"{}\"", v),
token::ByteStrRaw(s, n) => format!("br{delim}\"{string}\"{delim}",
delim="#".repeat(n as usize),
string=s),
};
if let Some(suffix) = suffix {
out.push_str(&suffix.as_str())
}
out
}
pub fn token_to_string(tok: &Token) -> String {
match *tok {
token::Eq => "=".to_string(),
@ -223,30 +230,7 @@ pub fn token_to_string(tok: &Token) -> String {
token::SingleQuote => "'".to_string(),
/* Literals */
token::Literal(lit, suf) => {
let mut out = match lit {
token::Bool(_) => panic!("literal token contains `Lit::Bool`"),
token::Byte(b) => format!("b'{}'", b),
token::Char(c) => format!("'{}'", c),
token::Err(c) => format!("'{}'", c),
token::Float(c) |
token::Integer(c) => c.to_string(),
token::Str_(s) => format!("\"{}\"", s),
token::StrRaw(s, n) => format!("r{delim}\"{string}\"{delim}",
delim="#".repeat(n as usize),
string=s),
token::ByteStr(v) => format!("b\"{}\"", v),
token::ByteStrRaw(s, n) => format!("br{delim}\"{string}\"{delim}",
delim="#".repeat(n as usize),
string=s),
};
if let Some(s) = suf {
out.push_str(&s.as_str())
}
out
}
token::Literal(lit, suf) => literal_to_string(lit, suf),
/* Name components */
token::Ident(s, false) => s.to_string(),
@ -439,8 +423,6 @@ pub trait PrintState<'a> {
fn boxes(&mut self) -> &mut Vec<pp::Breaks>;
fn comments(&mut self) -> &mut Option<Vec<comments::Comment>>;
fn cur_cmnt(&mut self) -> &mut usize;
fn cur_lit(&mut self) -> Option<&comments::Literal>;
fn bump_lit(&mut self) -> Option<comments::Literal>;
fn word_space<S: Into<Cow<'static, str>>>(&mut self, w: S) -> io::Result<()> {
self.writer().word(w)?;
@ -505,21 +487,6 @@ pub trait PrintState<'a> {
self.end()
}
fn next_lit(&mut self, pos: BytePos) -> Option<comments::Literal> {
while let Some(ltrl) = self.cur_lit().cloned() {
if ltrl.pos > pos { break; }
// we don't need the value here since we're forced to clone cur_lit
// due to lack of NLL.
self.bump_lit();
if ltrl.pos == pos {
return Some(ltrl);
}
}
None
}
fn maybe_print_comment(&mut self, pos: BytePos) -> io::Result<()> {
while let Some(ref cmnt) = self.next_comment() {
if cmnt.pos < pos {
@ -607,60 +574,7 @@ pub trait PrintState<'a> {
fn print_literal(&mut self, lit: &ast::Lit) -> io::Result<()> {
self.maybe_print_comment(lit.span.lo())?;
if let Some(ltrl) = self.next_lit(lit.span.lo()) {
return self.writer().word(ltrl.lit.clone());
}
match lit.node {
ast::LitKind::Str(st, style) => self.print_string(&st.as_str(), style),
ast::LitKind::Err(st) => {
let st = st.as_str().escape_debug().to_string();
let mut res = String::with_capacity(st.len() + 2);
res.push('\'');
res.push_str(&st);
res.push('\'');
self.writer().word(res)
}
ast::LitKind::Byte(byte) => {
let mut res = String::from("b'");
res.extend(ascii::escape_default(byte).map(|c| c as char));
res.push('\'');
self.writer().word(res)
}
ast::LitKind::Char(ch) => {
let mut res = String::from("'");
res.extend(ch.escape_default());
res.push('\'');
self.writer().word(res)
}
ast::LitKind::Int(i, t) => {
match t {
ast::LitIntType::Signed(st) => {
self.writer().word(st.val_to_string(i as i128))
}
ast::LitIntType::Unsigned(ut) => {
self.writer().word(ut.val_to_string(i))
}
ast::LitIntType::Unsuffixed => {
self.writer().word(i.to_string())
}
}
}
ast::LitKind::Float(ref f, t) => {
self.writer().word(format!("{}{}", &f, t.ty_to_string()))
}
ast::LitKind::FloatUnsuffixed(ref f) => self.writer().word(f.as_str().to_string()),
ast::LitKind::Bool(val) => {
if val { self.writer().word("true") } else { self.writer().word("false") }
}
ast::LitKind::ByteStr(ref v) => {
let mut escaped: String = String::new();
for &ch in v.iter() {
escaped.extend(ascii::escape_default(ch)
.map(|c| c as char));
}
self.writer().word(format!("b\"{}\"", escaped))
}
}
self.writer().word(literal_to_string(lit.token, lit.suffix))
}
fn print_string(&mut self, st: &str,
@ -881,14 +795,6 @@ impl<'a> PrintState<'a> for State<'a> {
fn cur_cmnt(&mut self) -> &mut usize {
&mut self.cur_cmnt
}
fn cur_lit(&mut self) -> Option<&comments::Literal> {
self.literals.peek()
}
fn bump_lit(&mut self) -> Option<comments::Literal> {
self.literals.next()
}
}
impl<'a> State<'a> {