bjoernager/rust
bjoernager/rust
1
Fork 0
Code Issues Pull requests Activity

syntax: Rename Token into TokenKind

Browse source
This commit is contained in:
Vadim Petrochenkov 2019-06-04 17:55:23 +03:00
parent eac3846b65
commit 99b27d749c
24 changed files with 119 additions and 119 deletions
Download patch file Download diff file Expand all files Collapse all files

4
src/librustc/hir/lowering.rs
View file

@ -67,7 +67,7 @@ use syntax::source_map::CompilerDesugaringKind::IfTemporary;
use syntax::std_inject;
use syntax::symbol::{kw, sym, Symbol};
use syntax::tokenstream::{TokenStream, TokenTree};
use syntax::parse::token::{self, Token};
use syntax::parse::token::{self, TokenKind};
use syntax::visit::{self, Visitor};
use syntax_pos::{DUMMY_SP, edition, Span};
@ -1337,7 +1337,7 @@ impl<'a> LoweringContext<'a> {
}
}
fn lower_token(&mut self, token: Token, span: Span) -> TokenStream {
fn lower_token(&mut self, token: TokenKind, span: Span) -> TokenStream {
match token {
token::Interpolated(nt) => {
let tts = nt.to_tokenstream(&self.sess.parse_sess, span);

4
src/librustc/hir/map/def_collector.rs
View file

@ -6,7 +6,7 @@ use syntax::ast::*;
use syntax::ext::hygiene::Mark;
use syntax::visit;
use syntax::symbol::{kw, sym};
use syntax::parse::token::{self, Token};
use syntax::parse::token::{self, TokenKind};
use syntax_pos::Span;
/// Creates `DefId`s for nodes in the AST.
@ -325,7 +325,7 @@ impl<'a> visit::Visitor<'a> for DefCollector<'a> {
}
}
fn visit_token(&mut self, t: Token) {
fn visit_token(&mut self, t: TokenKind) {
if let token::Interpolated(nt) = t {
if let token::NtExpr(ref expr) = *nt {
if let ExprKind::Mac(..) = expr.node {

2
src/librustc/ich/impls_syntax.rs
View file

@ -307,7 +307,7 @@ impl_stable_hash_for!(struct token::Lit {
});
fn hash_token<'a, 'gcx, W: StableHasherResult>(
token: &token::Token,
token: &token::TokenKind,
hcx: &mut StableHashingContext<'a>,
hasher: &mut StableHasher<W>,
) {

4
src/librustc_resolve/build_reduced_graph.rs
View file

@ -34,7 +34,7 @@ use syntax::ext::base::Determinacy::Undetermined;
use syntax::ext::hygiene::Mark;
use syntax::ext::tt::macro_rules;
use syntax::feature_gate::is_builtin_attr;
use syntax::parse::token::{self, Token};
use syntax::parse::token::{self, TokenKind};
use syntax::span_err;
use syntax::std_inject::injected_crate_name;
use syntax::symbol::{kw, sym};
@ -1052,7 +1052,7 @@ impl<'a, 'b> Visitor<'a> for BuildReducedGraphVisitor<'a, 'b> {
self.resolver.current_module = parent;
}
fn visit_token(&mut self, t: Token) {
fn visit_token(&mut self, t: TokenKind) {
if let token::Interpolated(nt) = t {
if let token::NtExpr(ref expr) = *nt {
if let ast::ExprKind::Mac(..) = expr.node {

4
src/librustc_save_analysis/span_utils.rs
View file

@ -5,7 +5,7 @@ use crate::generated_code;
use std::cell::Cell;
use syntax::parse::lexer::{self, StringReader};
use syntax::parse::token::{self, Token};
use syntax::parse::token::{self, TokenKind};
use syntax_pos::*;
#[derive(Clone)]
@ -56,7 +56,7 @@ impl<'a> SpanUtils<'a> {
lexer::StringReader::retokenize(&self.sess.parse_sess, span)
}
pub fn sub_span_of_token(&self, span: Span, tok: Token) -> Option<Span> {
pub fn sub_span_of_token(&self, span: Span, tok: TokenKind) -> Option<Span> {
let mut toks = self.retokenise_span(span);
loop {
let next = toks.real_token();

4
src/libsyntax/attr/mod.rs
View file

@ -20,7 +20,7 @@ use crate::source_map::{BytePos, Spanned, dummy_spanned};
use crate::parse::lexer::comments::{doc_comment_style, strip_doc_comment_decoration};
use crate::parse::parser::Parser;
use crate::parse::{self, ParseSess, PResult};
use crate::parse::token::{self, Token};
use crate::parse::token::{self, TokenKind};
use crate::ptr::P;
use crate::symbol::{sym, Symbol};
use crate::ThinVec;
@ -468,7 +468,7 @@ impl MetaItem {
idents.push(TokenTree::Token(mod_sep_span, token::ModSep).into());
}
idents.push(TokenTree::Token(segment.ident.span,
Token::from_ast_ident(segment.ident)).into());
TokenKind::from_ast_ident(segment.ident)).into());
last_pos = segment.ident.span.hi();
}
self.node.tokens(self.span).append_to_tree_and_joint_vec(&mut idents);

16
src/libsyntax/ext/tt/macro_parser.rs
View file

@ -78,7 +78,7 @@ use crate::ast::Ident;
use crate::ext::tt::quoted::{self, TokenTree};
use crate::parse::{Directory, ParseSess};
use crate::parse::parser::{Parser, PathStyle};
use crate::parse::token::{self, DocComment, Nonterminal, Token};
use crate::parse::token::{self, DocComment, Nonterminal, TokenKind};
use crate::print::pprust;
use crate::symbol::{kw, sym, Symbol};
use crate::tokenstream::{DelimSpan, TokenStream};
@ -199,7 +199,7 @@ struct MatcherPos<'root, 'tt: 'root> {
seq_op: Option<quoted::KleeneOp>,
/// The separator if we are in a repetition.
sep: Option<Token>,
sep: Option<TokenKind>,
/// The "parent" matcher position if we are in a repetition. That is, the matcher position just
/// before we enter the sequence.
@ -273,7 +273,7 @@ pub enum ParseResult<T> {
Success(T),
/// Arm failed to match. If the second parameter is `token::Eof`, it indicates an unexpected
/// end of macro invocation. Otherwise, it indicates that no rules expected the given token.
Failure(syntax_pos::Span, Token, &'static str),
Failure(syntax_pos::Span, TokenKind, &'static str),
/// Fatal error (malformed macro?). Abort compilation.
Error(syntax_pos::Span, String),
}
@ -417,7 +417,7 @@ fn nameize<I: Iterator<Item = NamedMatch>>(
/// Generates an appropriate parsing failure message. For EOF, this is "unexpected end...". For
/// other tokens, this is "unexpected token...".
pub fn parse_failure_msg(tok: Token) -> String {
pub fn parse_failure_msg(tok: TokenKind) -> String {
match tok {
token::Eof => "unexpected end of macro invocation".to_string(),
_ => format!(
@ -428,7 +428,7 @@ pub fn parse_failure_msg(tok: Token) -> String {
}
/// Performs a token equality check, ignoring syntax context (that is, an unhygienic comparison)
fn token_name_eq(t1: &Token, t2: &Token) -> bool {
fn token_name_eq(t1: &TokenKind, t2: &TokenKind) -> bool {
if let (Some((id1, is_raw1)), Some((id2, is_raw2))) = (t1.ident(), t2.ident()) {
id1.name == id2.name && is_raw1 == is_raw2
} else if let (Some(id1), Some(id2)) = (t1.lifetime(), t2.lifetime()) {
@ -466,7 +466,7 @@ fn inner_parse_loop<'root, 'tt>(
next_items: &mut Vec<MatcherPosHandle<'root, 'tt>>,
eof_items: &mut SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>,
bb_items: &mut SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>,
token: &Token,
token: &TokenKind,
span: syntax_pos::Span,
) -> ParseResult<()> {
// Pop items from `cur_items` until it is empty.
@ -807,7 +807,7 @@ pub fn parse(
/// The token is an identifier, but not `_`.
/// We prohibit passing `_` to macros expecting `ident` for now.
fn get_macro_ident(token: &Token) -> Option<(Ident, bool)> {
fn get_macro_ident(token: &TokenKind) -> Option<(Ident, bool)> {
match *token {
token::Ident(ident, is_raw) if ident.name != kw::Underscore =>
Some((ident, is_raw)),
@ -819,7 +819,7 @@ fn get_macro_ident(token: &Token) -> Option<(Ident, bool)> {
///
/// Returning `false` is a *stability guarantee* that such a matcher will *never* begin with that
/// token. Be conservative (return true) if not sure.
fn may_begin_with(name: Symbol, token: &Token) -> bool {
fn may_begin_with(name: Symbol, token: &TokenKind) -> bool {
/// Checks whether the non-terminal may contain a single (non-keyword) identifier.
fn may_be_ident(nt: &token::Nonterminal) -> bool {
match *nt {

2
src/libsyntax/ext/tt/macro_rules.rs
View file

@ -12,7 +12,7 @@ use crate::feature_gate::Features;
use crate::parse::{Directory, ParseSess};
use crate::parse::parser::Parser;
use crate::parse::token::{self, NtTT};
use crate::parse::token::Token::*;
use crate::parse::token::TokenKind::*;
use crate::symbol::{Symbol, kw, sym};
use crate::tokenstream::{DelimSpan, TokenStream, TokenTree};

18
src/libsyntax/ext/tt/quoted.rs
View file

@ -23,12 +23,12 @@ pub struct Delimited {
impl Delimited {
/// Returns the opening delimiter (possibly `NoDelim`).
pub fn open_token(&self) -> token::Token {
pub fn open_token(&self) -> token::TokenKind {
token::OpenDelim(self.delim)
}
/// Returns the closing delimiter (possibly `NoDelim`).
pub fn close_token(&self) -> token::Token {
pub fn close_token(&self) -> token::TokenKind {
token::CloseDelim(self.delim)
}
@ -58,7 +58,7 @@ pub struct SequenceRepetition {
/// The sequence of token trees
pub tts: Vec<TokenTree>,
/// The optional separator
pub separator: Option<token::Token>,
pub separator: Option<token::TokenKind>,
/// Whether the sequence can be repeated zero (*), or one or more times (+)
pub op: KleeneOp,
/// The number of `Match`s that appear in the sequence (and subsequences)
@ -81,7 +81,7 @@ pub enum KleeneOp {
/// are "first-class" token trees. Useful for parsing macros.
#[derive(Debug, Clone, PartialEq, RustcEncodable, RustcDecodable)]
pub enum TokenTree {
Token(Span, token::Token),
Token(Span, token::TokenKind),
Delimited(DelimSpan, Lrc<Delimited>),
/// A kleene-style repetition sequence
Sequence(DelimSpan, Lrc<SequenceRepetition>),
@ -366,7 +366,7 @@ where
/// Takes a token and returns `Some(KleeneOp)` if the token is `+` `*` or `?`. Otherwise, return
/// `None`.
fn kleene_op(token: &token::Token) -> Option<KleeneOp> {
fn kleene_op(token: &token::TokenKind) -> Option<KleeneOp> {
match *token {
token::BinOp(token::Star) => Some(KleeneOp::ZeroOrMore),
token::BinOp(token::Plus) => Some(KleeneOp::OneOrMore),
@ -383,7 +383,7 @@ fn kleene_op(token: &token::Token) -> Option<KleeneOp> {
fn parse_kleene_op<I>(
input: &mut I,
span: Span,
) -> Result<Result<(KleeneOp, Span), (token::Token, Span)>, Span>
) -> Result<Result<(KleeneOp, Span), (token::TokenKind, Span)>, Span>
where
I: Iterator<Item = tokenstream::TokenTree>,
{
@ -422,7 +422,7 @@ fn parse_sep_and_kleene_op<I>(
attrs: &[ast::Attribute],
edition: Edition,
macro_node_id: NodeId,
) -> (Option<token::Token>, KleeneOp)
) -> (Option<token::TokenKind>, KleeneOp)
where
I: Iterator<Item = tokenstream::TokenTree>,
{
@ -447,7 +447,7 @@ fn parse_sep_and_kleene_op_2015<I>(
_features: &Features,
_attrs: &[ast::Attribute],
macro_node_id: NodeId,
) -> (Option<token::Token>, KleeneOp)
) -> (Option<token::TokenKind>, KleeneOp)
where
I: Iterator<Item = tokenstream::TokenTree>,
{
@ -565,7 +565,7 @@ fn parse_sep_and_kleene_op_2018<I>(
sess: &ParseSess,
_features: &Features,
_attrs: &[ast::Attribute],
) -> (Option<token::Token>, KleeneOp)
) -> (Option<token::TokenKind>, KleeneOp)
where
I: Iterator<Item = tokenstream::TokenTree>,
{

6
src/libsyntax/ext/tt/transcribe.rs
View file

@ -4,7 +4,7 @@ use crate::ext::expand::Marker;
use crate::ext::tt::macro_parser::{MatchedNonterminal, MatchedSeq, NamedMatch};
use crate::ext::tt::quoted;
use crate::mut_visit::noop_visit_tt;
use crate::parse::token::{self, NtTT, Token};
use crate::parse::token::{self, NtTT, TokenKind};
use crate::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndJoint};
use smallvec::{smallvec, SmallVec};
@ -18,7 +18,7 @@ use std::rc::Rc;
/// An iterator over the token trees in a delimited token tree (`{ ... }`) or a sequence (`$(...)`).
enum Frame {
Delimited { forest: Lrc<quoted::Delimited>, idx: usize, span: DelimSpan },
Sequence { forest: Lrc<quoted::SequenceRepetition>, idx: usize, sep: Option<Token> },
Sequence { forest: Lrc<quoted::SequenceRepetition>, idx: usize, sep: Option<TokenKind> },
}
impl Frame {
@ -242,7 +242,7 @@ pub fn transcribe(
Ident::new(ident.name, ident.span.apply_mark(cx.current_expansion.mark));
sp = sp.apply_mark(cx.current_expansion.mark);
result.push(TokenTree::Token(sp, token::Dollar).into());
result.push(TokenTree::Token(sp, token::Token::from_ast_ident(ident)).into());
result.push(TokenTree::Token(sp, token::TokenKind::from_ast_ident(ident)).into());
}
}

6
src/libsyntax/mut_visit.rs
View file

@ -9,7 +9,7 @@
use crate::ast::*;
use crate::source_map::{Spanned, respan};
use crate::parse::token::{self, Token};
use crate::parse::token::{self, TokenKind};
use crate::ptr::P;
use crate::ThinVec;
use crate::tokenstream::*;
@ -262,7 +262,7 @@ pub trait MutVisitor: Sized {
noop_visit_tts(tts, self);
}
fn visit_token(&mut self, t: &mut Token) {
fn visit_token(&mut self, t: &mut TokenKind) {
noop_visit_token(t, self);
}
@ -596,7 +596,7 @@ pub fn noop_visit_tts<T: MutVisitor>(TokenStream(tts): &mut TokenStream, vis: &m
}
// apply ident visitor if it's an ident, apply other visits to interpolated nodes
pub fn noop_visit_token<T: MutVisitor>(t: &mut Token, vis: &mut T) {
pub fn noop_visit_token<T: MutVisitor>(t: &mut TokenKind, vis: &mut T) {
match t {
token::Ident(id, _is_raw) => vis.visit_ident(id),
token::Lifetime(id) => vis.visit_ident(id),

12
src/libsyntax/parse/diagnostics.rs
View file

@ -229,8 +229,8 @@ impl<'a> Parser<'a> {
pub fn expected_one_of_not_found(
&mut self,
edible: &[token::Token],
inedible: &[token::Token],
edible: &[token::TokenKind],
inedible: &[token::TokenKind],
) -> PResult<'a, bool /* recovered */> {
fn tokens_to_string(tokens: &[TokenType]) -> String {
let mut i = tokens.iter();
@ -368,7 +368,7 @@ impl<'a> Parser<'a> {
/// Eats and discards tokens until one of `kets` is encountered. Respects token trees,
/// passes through any errors encountered. Used for error recovery.
crate fn eat_to_tokens(&mut self, kets: &[&token::Token]) {
crate fn eat_to_tokens(&mut self, kets: &[&token::TokenKind]) {
let handler = self.diagnostic();
if let Err(ref mut err) = self.parse_seq_to_before_tokens(
@ -388,7 +388,7 @@ impl<'a> Parser<'a> {
/// let _ = vec![1, 2, 3].into_iter().collect::<Vec<usize>>>>();
/// ^^ help: remove extra angle brackets
/// ```
crate fn check_trailing_angle_brackets(&mut self, segment: &PathSegment, end: token::Token) {
crate fn check_trailing_angle_brackets(&mut self, segment: &PathSegment, end: token::TokenKind) {
// This function is intended to be invoked after parsing a path segment where there are two
// cases:
//
@ -726,7 +726,7 @@ impl<'a> Parser<'a> {
/// closing delimiter.
pub fn unexpected_try_recover(
&mut self,
t: &token::Token,
t: &token::TokenKind,
) -> PResult<'a, bool /* recovered */> {
let token_str = pprust::token_to_string(t);
let this_token_str = self.this_token_descr();
@ -903,7 +903,7 @@ impl<'a> Parser<'a> {
crate fn recover_closing_delimiter(
&mut self,
tokens: &[token::Token],
tokens: &[token::TokenKind],
mut err: DiagnosticBuilder<'a>,
) -> PResult<'a, bool> {
let mut pos = None;

32
src/libsyntax/parse/lexer/mod.rs
View file

@ -1,6 +1,6 @@
use crate::ast::{self, Ident};
use crate::parse::ParseSess;
use crate::parse::token::{self, Token};
use crate::parse::token::{self, TokenKind};
use crate::symbol::{sym, Symbol};
use crate::parse::unescape;
use crate::parse::unescape_error_reporting::{emit_unescape_error, push_escaped_char};
@ -22,7 +22,7 @@ mod unicode_chars;
#[derive(Clone, Debug)]
pub struct TokenAndSpan {
pub tok: Token,
pub tok: TokenKind,
pub sp: Span,
}
@ -56,7 +56,7 @@ pub struct StringReader<'a> {
/// Stop reading src at this index.
crate end_src_index: usize,
// cached:
peek_tok: Token,
peek_tok: TokenKind,
peek_span: Span,
peek_span_src_raw: Span,
fatal_errs: Vec<DiagnosticBuilder<'a>>,
@ -847,7 +847,7 @@ impl<'a> StringReader<'a> {
}
}
fn binop(&mut self, op: token::BinOpToken) -> Token {
fn binop(&mut self, op: token::BinOpToken) -> TokenKind {
self.bump();
if self.ch_is('=') {
self.bump();
@ -859,7 +859,7 @@ impl<'a> StringReader<'a> {
/// Returns the next token from the string, advances the input past that
/// token, and updates the interner
fn next_token_inner(&mut self) -> Result<Token, ()> {
fn next_token_inner(&mut self) -> Result<TokenKind, ()> {
let c = self.ch;
if ident_start(c) {
@ -916,7 +916,7 @@ impl<'a> StringReader<'a> {
let (kind, symbol) = self.scan_number(c.unwrap());
let suffix = self.scan_optional_raw_name();
debug!("next_token_inner: scanned number {:?}, {:?}, {:?}", kind, symbol, suffix);
return Ok(Token::lit(kind, symbol, suffix));
return Ok(TokenKind::lit(kind, symbol, suffix));
}
match c.expect("next_token_inner called at EOF") {
@ -1077,7 +1077,7 @@ impl<'a> StringReader<'a> {
let symbol = self.name_from(start);
self.bump();
self.validate_char_escape(start_with_quote);
return Ok(Token::lit(token::Char, symbol, None));
return Ok(TokenKind::lit(token::Char, symbol, None));
}
// Include the leading `'` in the real identifier, for macro
@ -1102,7 +1102,7 @@ impl<'a> StringReader<'a> {
let symbol = self.scan_single_quoted_string(start_with_quote, msg);
self.validate_char_escape(start_with_quote);
let suffix = self.scan_optional_raw_name();
Ok(Token::lit(token::Char, symbol, suffix))
Ok(TokenKind::lit(token::Char, symbol, suffix))
}
'b' => {
self.bump();
@ -1127,7 +1127,7 @@ impl<'a> StringReader<'a> {
};
let suffix = self.scan_optional_raw_name();
Ok(Token::lit(kind, symbol, suffix))
Ok(TokenKind::lit(kind, symbol, suffix))
}
'"' => {
let start_with_quote = self.pos;
@ -1135,7 +1135,7 @@ impl<'a> StringReader<'a> {
let symbol = self.scan_double_quoted_string(msg);
self.validate_str_escape(start_with_quote);
let suffix = self.scan_optional_raw_name();
Ok(Token::lit(token::Str, symbol, suffix))
Ok(TokenKind::lit(token::Str, symbol, suffix))
}
'r' => {
let start_bpos = self.pos;
@ -1213,7 +1213,7 @@ impl<'a> StringReader<'a> {
};
let suffix = self.scan_optional_raw_name();
Ok(Token::lit(token::StrRaw(hash_count), symbol, suffix))
Ok(TokenKind::lit(token::StrRaw(hash_count), symbol, suffix))
}
'-' => {
if self.nextch_is('>') {
@ -1638,19 +1638,19 @@ mod tests {
// check that the given reader produces the desired stream
// of tokens (stop checking after exhausting the expected vec)
fn check_tokenization(mut string_reader: StringReader<'_>, expected: Vec<Token>) {
fn check_tokenization(mut string_reader: StringReader<'_>, expected: Vec<TokenKind>) {
for expected_tok in &expected {
assert_eq!(&string_reader.next_token().tok, expected_tok);
}
}
// make the identifier by looking up the string in the interner
fn mk_ident(id: &str) -> Token {
Token::from_ast_ident(Ident::from_str(id))
fn mk_ident(id: &str) -> TokenKind {
TokenKind::from_ast_ident(Ident::from_str(id))
}
fn mk_lit(kind: token::LitKind, symbol: &str, suffix: Option<&str>) -> Token {
Token::lit(kind, Symbol::intern(symbol), suffix.map(Symbol::intern))
fn mk_lit(kind: token::LitKind, symbol: &str, suffix: Option<&str>) -> TokenKind {
TokenKind::lit(kind, Symbol::intern(symbol), suffix.map(Symbol::intern))
}
#[test]

2
src/libsyntax/parse/lexer/tokentrees.rs
View file

@ -23,7 +23,7 @@ impl<'a> StringReader<'a> {
struct TokenTreesReader<'a> {
string_reader: StringReader<'a>,
token: token::Token,
token: token::TokenKind,
span: Span,
/// Stack of open delimiters and their spans. Used for error message.
open_braces: Vec<(token::DelimToken, Span)>,

6
src/libsyntax/parse/literal.rs
View file

@ -3,7 +3,7 @@
use crate::ast::{self, Ident, Lit, LitKind};
use crate::parse::parser::Parser;
use crate::parse::PResult;
use crate::parse::token::{self, Token};
use crate::parse::token::{self, TokenKind};
use crate::parse::unescape::{unescape_str, unescape_char, unescape_byte_str, unescape_byte};
use crate::print::pprust;
use crate::symbol::{kw, sym, Symbol};
@ -228,7 +228,7 @@ impl Lit {
}
/// Converts arbitrary token into an AST literal.
crate fn from_token(token: &Token, span: Span) -> Result<Lit, LitError> {
crate fn from_token(token: &TokenKind, span: Span) -> Result<Lit, LitError> {
let lit = match *token {
token::Ident(ident, false) if ident.name == kw::True || ident.name == kw::False =>
token::Lit::new(token::Bool, ident.name, None),
@ -276,7 +276,7 @@ impl<'a> Parser<'a> {
let next_span = self.look_ahead_span(1);
if self.span.hi() == next_span.lo() {
let s = String::from("0.") + &symbol.as_str();
let token = Token::lit(token::Float, Symbol::intern(&s), suffix);
let token = TokenKind::lit(token::Float, Symbol::intern(&s), suffix);
return Some((token, self.span.to(next_span)));
}
}

4
src/libsyntax/parse/mod.rs
View file

@ -358,13 +358,13 @@ pub fn stream_to_parser_with_base_dir<'a>(
/// A sequence separator.
pub struct SeqSep {
/// The seperator token.
pub sep: Option<token::Token>,
pub sep: Option<token::TokenKind>,
/// `true` if a trailing separator is allowed.
pub trailing_sep_allowed: bool,
}
impl SeqSep {
pub fn trailing_allowed(t: token::Token) -> SeqSep {
pub fn trailing_allowed(t: token::TokenKind) -> SeqSep {
SeqSep {
sep: Some(t),
trailing_sep_allowed: true,

48
src/libsyntax/parse/parser.rs
View file

@ -196,9 +196,9 @@ enum PrevTokenKind {
#[derive(Clone)]
pub struct Parser<'a> {
pub sess: &'a ParseSess,
/// The current token.
pub token: token::Token,
/// The span of the current token.
/// the current token:
pub token: token::TokenKind,
/// the span of the current token:
pub span: Span,
meta_var_span: Option<Span>,
/// The span of the previous token.
@ -355,7 +355,7 @@ impl TokenCursor {
[
TokenTree::Token(sp, token::Ident(ast::Ident::with_empty_ctxt(sym::doc), false)),
TokenTree::Token(sp, token::Eq),
TokenTree::Token(sp, token::Token::lit(
TokenTree::Token(sp, token::TokenKind::lit(
token::StrRaw(num_of_hashes), Symbol::intern(&stripped), None
)),
]
@ -380,7 +380,7 @@ impl TokenCursor {
#[derive(Clone, PartialEq)]
crate enum TokenType {
Token(token::Token),
Token(token::TokenKind),
Keyword(Symbol),
Operator,
Lifetime,
@ -410,7 +410,7 @@ impl TokenType {
///
/// Types can also be of the form `IDENT(u8, u8) -> u8`, however this assumes
/// that `IDENT` is not the ident of a fn trait.
fn can_continue_type_after_non_fn_ident(t: &token::Token) -> bool {
fn can_continue_type_after_non_fn_ident(t: &token::TokenKind) -> bool {
t == &token::ModSep || t == &token::Lt ||
t == &token::BinOp(token::Shl)
}
@ -559,7 +559,7 @@ impl<'a> Parser<'a> {
}
/// Expects and consumes the token `t`. Signals an error if the next token is not `t`.
pub fn expect(&mut self, t: &token::Token) -> PResult<'a, bool /* recovered */> {
pub fn expect(&mut self, t: &token::TokenKind) -> PResult<'a, bool /* recovered */> {
if self.expected_tokens.is_empty() {
if self.token == *t {
self.bump();
@ -577,8 +577,8 @@ impl<'a> Parser<'a> {
/// anything. Signal a fatal error if next token is unexpected.
pub fn expect_one_of(
&mut self,
edible: &[token::Token],
inedible: &[token::Token],
edible: &[token::TokenKind],
inedible: &[token::TokenKind],
) -> PResult<'a, bool /* recovered */> {
if edible.contains(&self.token) {
self.bump();
@ -640,14 +640,14 @@ impl<'a> Parser<'a> {
///
/// This method will automatically add `tok` to `expected_tokens` if `tok` is not
/// encountered.
crate fn check(&mut self, tok: &token::Token) -> bool {
crate fn check(&mut self, tok: &token::TokenKind) -> bool {
let is_present = self.token == *tok;
if !is_present { self.expected_tokens.push(TokenType::Token(tok.clone())); }
is_present
}
/// Consumes a token 'tok' if it exists. Returns whether the given token was present.
pub fn eat(&mut self, tok: &token::Token) -> bool {
pub fn eat(&mut self, tok: &token::TokenKind) -> bool {
let is_present = self.check(tok);
if is_present { self.bump() }
is_present
@ -883,7 +883,7 @@ impl<'a> Parser<'a> {
/// `f` must consume tokens until reaching the next separator or
/// closing bracket.
pub fn parse_seq_to_end<T, F>(&mut self,
ket: &token::Token,
ket: &token::TokenKind,
sep: SeqSep,
f: F)
-> PResult<'a, Vec<T>> where
@ -901,7 +901,7 @@ impl<'a> Parser<'a> {
/// closing bracket.
pub fn parse_seq_to_before_end<T, F>(
&mut self,
ket: &token::Token,
ket: &token::TokenKind,
sep: SeqSep,
f: F,
) -> PResult<'a, (Vec<T>, bool)>
@ -912,7 +912,7 @@ impl<'a> Parser<'a> {
crate fn parse_seq_to_before_tokens<T, F>(
&mut self,
kets: &[&token::Token],
kets: &[&token::TokenKind],
sep: SeqSep,
expect: TokenExpectType,
mut f: F,
@ -986,8 +986,8 @@ impl<'a> Parser<'a> {
/// closing bracket.
fn parse_unspanned_seq<T, F>(
&mut self,
bra: &token::Token,
ket: &token::Token,
bra: &token::TokenKind,
ket: &token::TokenKind,
sep: SeqSep,
f: F,
) -> PResult<'a, Vec<T>> where
@ -1032,7 +1032,7 @@ impl<'a> Parser<'a> {
/// Advance the parser using provided token as a next one. Use this when
/// consuming a part of a token. For example a single `<` from `<<`.
fn bump_with(&mut self, next: token::Token, span: Span) {
fn bump_with(&mut self, next: token::TokenKind, span: Span) {
self.prev_span = self.span.with_hi(span.lo());
// It would be incorrect to record the kind of the current token, but
// fortunately for tokens currently using `bump_with`, the
@ -1044,7 +1044,7 @@ impl<'a> Parser<'a> {
}
pub fn look_ahead<R, F>(&self, dist: usize, f: F) -> R where
F: FnOnce(&token::Token) -> R,
F: FnOnce(&token::TokenKind) -> R,
{
if dist == 0 {
return f(&self.token)
@ -1763,7 +1763,7 @@ impl<'a> Parser<'a> {
fn parse_path_segment(&mut self, style: PathStyle) -> PResult<'a, PathSegment> {
let ident = self.parse_path_segment_ident()?;
let is_args_start = |token: &token::Token| match *token {
let is_args_start = |token: &token::TokenKind| match *token {
token::Lt | token::BinOp(token::Shl) | token::OpenDelim(token::Paren)
| token::LArrow => true,
_ => false,
@ -1992,7 +1992,7 @@ impl<'a> Parser<'a> {
let ex: ExprKind;
// Note: when adding new syntax here, don't forget to adjust Token::can_begin_expr().
// Note: when adding new syntax here, don't forget to adjust TokenKind::can_begin_expr().
match self.token {
token::OpenDelim(token::Paren) => {
self.bump();
@ -2706,7 +2706,7 @@ impl<'a> Parser<'a> {
-> PResult<'a, P<Expr>> {
let attrs = self.parse_or_use_outer_attributes(already_parsed_attrs)?;
let lo = self.span;
// Note: when adding new unary operators, don't forget to adjust Token::can_begin_expr()
// Note: when adding new unary operators, don't forget to adjust TokenKind::can_begin_expr()
let (hi, ex) = match self.token {
token::Not => {
self.bump();
@ -2760,7 +2760,7 @@ impl<'a> Parser<'a> {
// `not` is just an ordinary identifier in Rust-the-language,
// but as `rustc`-the-compiler, we can issue clever diagnostics
// for confused users who really want to say `!`
let token_cannot_continue_expr = |t: &token::Token| match *t {
let token_cannot_continue_expr = |t: &token::TokenKind| match *t {
// These tokens can start an expression after `!`, but
// can't continue an expression after an ident
token::Ident(ident, is_raw) => token::ident_can_begin_expr(ident, is_raw),
@ -4779,7 +4779,7 @@ impl<'a> Parser<'a> {
let mut last_plus_span = None;
let mut was_negative = false;
loop {
// This needs to be synchronized with `Token::can_begin_bound`.
// This needs to be synchronized with `TokenKind::can_begin_bound`.
let is_bound_start = self.check_path() || self.check_lifetime() ||
self.check(&token::Not) || // used for error reporting only
self.check(&token::Question) ||
@ -6413,7 +6413,7 @@ impl<'a> Parser<'a> {
}
/// Given a termination token, parses all of the items in a module.
fn parse_mod_items(&mut self, term: &token::Token, inner_lo: Span) -> PResult<'a, Mod> {
fn parse_mod_items(&mut self, term: &token::TokenKind, inner_lo: Span) -> PResult<'a, Mod> {
let mut items = vec![];
while let Some(item) = self.parse_item()? {
items.push(item);

28
src/libsyntax/parse/token.rs
View file

@ -2,7 +2,7 @@ pub use BinOpToken::*;
pub use Nonterminal::*;
pub use DelimToken::*;
pub use LitKind::*;
pub use Token::*;
pub use TokenKind::*;
use crate::ast::{self};
use crate::parse::ParseSess;
@ -118,7 +118,7 @@ impl Lit {
}
pub(crate) fn ident_can_begin_expr(ident: ast::Ident, is_raw: bool) -> bool {
let ident_token: Token = Ident(ident, is_raw);
let ident_token: TokenKind = Ident(ident, is_raw);
!ident_token.is_reserved_ident() ||
ident_token.is_path_segment_keyword() ||
@ -149,7 +149,7 @@ pub(crate) fn ident_can_begin_expr(ident: ast::Ident, is_raw: bool) -> bool {
}
fn ident_can_begin_type(ident: ast::Ident, is_raw: bool) -> bool {
let ident_token: Token = Ident(ident, is_raw);
let ident_token: TokenKind = Ident(ident, is_raw);
!ident_token.is_reserved_ident() ||
ident_token.is_path_segment_keyword() ||
@ -166,7 +166,7 @@ fn ident_can_begin_type(ident: ast::Ident, is_raw: bool) -> bool {
}
#[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Debug)]
pub enum Token {
pub enum TokenKind {
/* Expression-operator symbols. */
Eq,
Lt,
@ -231,13 +231,13 @@ pub enum Token {
Eof,
}
// `Token` is used a lot. Make sure it doesn't unintentionally get bigger.
// `TokenKind` is used a lot. Make sure it doesn't unintentionally get bigger.
#[cfg(target_arch = "x86_64")]
static_assert_size!(Token, 16);
static_assert_size!(TokenKind, 16);
impl Token {
/// Recovers a `Token` from an `ast::Ident`. This creates a raw identifier if necessary.
pub fn from_ast_ident(ident: ast::Ident) -> Token {
impl TokenKind {
/// Recovers a `TokenKind` from an `ast::Ident`. This creates a raw identifier if necessary.
pub fn from_ast_ident(ident: ast::Ident) -> TokenKind {
Ident(ident, ident.is_raw_guess())
}
@ -323,7 +323,7 @@ impl Token {
self == &Question || self == &OpenDelim(Paren)
}
pub fn lit(kind: LitKind, symbol: Symbol, suffix: Option<Symbol>) -> Token {
pub fn lit(kind: LitKind, symbol: Symbol, suffix: Option<Symbol>) -> TokenKind {
Literal(Lit::new(kind, symbol, suffix))
}
@ -468,7 +468,7 @@ impl Token {
}
}
crate fn glue(self, joint: Token) -> Option<Token> {
crate fn glue(self, joint: TokenKind) -> Option<TokenKind> {
Some(match self {
Eq => match joint {
Eq => EqEq,
@ -534,7 +534,7 @@ impl Token {
/// Returns tokens that are likely to be typed accidentally instead of the current token.
/// Enables better error recovery when the wrong token is found.
crate fn similar_tokens(&self) -> Option<Vec<Token>> {
crate fn similar_tokens(&self) -> Option<Vec<TokenKind>> {
match *self {
Comma => Some(vec![Dot, Lt, Semi]),
Semi => Some(vec![Colon, Comma]),
@ -544,7 +544,7 @@ impl Token {
// See comments in `Nonterminal::to_tokenstream` for why we care about
// *probably* equal here rather than actual equality
crate fn probably_equal_for_proc_macro(&self, other: &Token) -> bool {
crate fn probably_equal_for_proc_macro(&self, other: &TokenKind) -> bool {
if mem::discriminant(self) != mem::discriminant(other) {
return false
}
@ -743,7 +743,7 @@ impl Nonterminal {
}
}
crate fn is_op(tok: &Token) -> bool {
crate fn is_op(tok: &TokenKind) -> bool {
match *tok {
OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) |
Ident(..) | Lifetime(..) | Interpolated(..) |

4
src/libsyntax/print/pprust.rs
View file

@ -6,7 +6,7 @@ use crate::ast::{Attribute, MacDelimiter, GenericArg};
use crate::util::parser::{self, AssocOp, Fixity};
use crate::attr;
use crate::source_map::{self, SourceMap, Spanned};
use crate::parse::token::{self, BinOpToken, Nonterminal, Token};
use crate::parse::token::{self, BinOpToken, Nonterminal, TokenKind};
use crate::parse::lexer::comments;
use crate::parse::{self, ParseSess};
use crate::print::pp::{self, Breaks};
@ -189,7 +189,7 @@ pub fn literal_to_string(lit: token::Lit) -> String {
out
}
pub fn token_to_string(tok: &Token) -> String {
pub fn token_to_string(tok: &TokenKind) -> String {
match *tok {
token::Eq => "=".to_string(),
token::Lt => "<".to_string(),

14
src/libsyntax/tokenstream.rs
View file

@ -16,7 +16,7 @@
use crate::ext::base;
use crate::ext::tt::{macro_parser, quoted};
use crate::parse::Directory;
use crate::parse::token::{self, DelimToken, Token};
use crate::parse::token::{self, DelimToken, TokenKind};
use crate::print::pprust;
use syntax_pos::{BytePos, Mark, Span, DUMMY_SP};
@ -44,7 +44,7 @@ use std::{fmt, iter, mem};
#[derive(Debug, Clone, PartialEq, RustcEncodable, RustcDecodable)]
pub enum TokenTree {
/// A single token
Token(Span, token::Token),
Token(Span, token::TokenKind),
/// A delimited sequence of token trees
Delimited(DelimSpan, DelimToken, TokenStream),
}
@ -54,7 +54,7 @@ pub enum TokenTree {
fn _dummy()
where
Span: Send + Sync,
token::Token: Send + Sync,
token::TokenKind: Send + Sync,
DelimSpan: Send + Sync,
DelimToken: Send + Sync,
TokenStream: Send + Sync,
@ -130,7 +130,7 @@ impl TokenTree {
}
/// Indicates if the stream is a token that is equal to the provided token.
pub fn eq_token(&self, t: Token) -> bool {
pub fn eq_token(&self, t: TokenKind) -> bool {
match *self {
TokenTree::Token(_, ref tk) => *tk == t,
_ => false,
@ -241,8 +241,8 @@ impl From<TokenTree> for TreeAndJoint {
}
}
impl From<Token> for TokenStream {
fn from(token: Token) -> TokenStream {
impl From<TokenKind> for TokenStream {
fn from(token: TokenKind) -> TokenStream {
TokenTree::Token(DUMMY_SP, token).into()
}
}
@ -580,7 +580,7 @@ mod tests {
use super::*;
use crate::syntax::ast::Ident;
use crate::with_default_globals;
use crate::parse::token::Token;
use crate::parse::token::TokenKind;
use crate::util::parser_testing::string_to_stream;
use syntax_pos::{Span, BytePos, NO_EXPANSION};

4
src/libsyntax/util/parser.rs
View file

@ -1,4 +1,4 @@
use crate::parse::token::{self, Token, BinOpToken};
use crate::parse::token::{self, TokenKind, BinOpToken};
use crate::symbol::kw;
use crate::ast::{self, BinOpKind};
@ -69,7 +69,7 @@ pub enum Fixity {
impl AssocOp {
/// Creates a new AssocOP from a token
pub fn from_token(t: &Token) -> Option<AssocOp> {
pub fn from_token(t: &TokenKind) -> Option<AssocOp> {
use AssocOp::*;
match *t {
token::BinOpEq(k) => Some(AssignOp(k)),

4
src/libsyntax/visit.rs
View file

@ -14,7 +14,7 @@
//! those that are created by the expansion of a macro.
use crate::ast::*;
use crate::parse::token::Token;
use crate::parse::token::TokenKind;
use crate::tokenstream::{TokenTree, TokenStream};
use syntax_pos::Span;
@ -151,7 +151,7 @@ pub trait Visitor<'ast>: Sized {
fn visit_tts(&mut self, tts: TokenStream) {
walk_tts(self, tts)
}
fn visit_token(&mut self, _t: Token) {}
fn visit_token(&mut self, _t: TokenKind) {}
// FIXME: add `visit_interpolated` and `walk_interpolated`
fn visit_vis(&mut self, vis: &'ast Visibility) {
walk_vis(self, vis)

4
src/libsyntax_ext/assert.rs
View file

@ -4,7 +4,7 @@ use syntax::ast::{self, *};
use syntax::source_map::Spanned;
use syntax::ext::base::*;
use syntax::ext::build::AstBuilder;
use syntax::parse::token::{self, Token};
use syntax::parse::token::{self, TokenKind};
use syntax::parse::parser::Parser;
use syntax::print::pprust;
use syntax::ptr::P;
@ -31,7 +31,7 @@ pub fn expand_assert<'cx>(
tts: custom_message.unwrap_or_else(|| {
TokenStream::from(TokenTree::Token(
DUMMY_SP,
Token::lit(token::Str, Symbol::intern(&format!(
TokenKind::lit(token::Str, Symbol::intern(&format!(
"assertion failed: {}",
pprust::expr_to_string(&cond_expr).escape_debug()
)), None),

6
src/libsyntax_ext/proc_macro_server.rs
View file

@ -161,7 +161,7 @@ impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec<Self>)>
let stream = vec![
Ident(ast::Ident::new(sym::doc, span), false),
Eq,
Token::lit(token::Str, Symbol::intern(&escaped), None),
TokenKind::lit(token::Str, Symbol::intern(&escaped), None),
]
.into_iter()
.map(|token| tokenstream::TokenTree::Token(span, token))
@ -220,7 +220,7 @@ impl ToInternal<TokenStream> for TokenTree<Group, Punct, Ident, Literal> {
}) if symbol.as_str().starts_with("-") => {
let minus = BinOp(BinOpToken::Minus);
let symbol = Symbol::intern(&symbol.as_str()[1..]);
let integer = Token::lit(token::Integer, symbol, suffix);
let integer = TokenKind::lit(token::Integer, symbol, suffix);
let a = tokenstream::TokenTree::Token(span, minus);
let b = tokenstream::TokenTree::Token(span, integer);
return vec![a, b].into_iter().collect();
@ -231,7 +231,7 @@ impl ToInternal<TokenStream> for TokenTree<Group, Punct, Ident, Literal> {
}) if symbol.as_str().starts_with("-") => {
let minus = BinOp(BinOpToken::Minus);
let symbol = Symbol::intern(&symbol.as_str()[1..]);
let float = Token::lit(token::Float, symbol, suffix);
let float = TokenKind::lit(token::Float, symbol, suffix);
let a = tokenstream::TokenTree::Token(span, minus);
let b = tokenstream::TokenTree::Token(span, float);
return vec![a, b].into_iter().collect();