bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

syntax: Use Token in Parser

Browse source
This commit is contained in:
Vadim Petrochenkov 2019-06-05 01:17:07 +03:00
parent c0c57acd7b
commit aa6fba98ae
10 changed files with 126 additions and 126 deletions
Download patch file Download diff file Expand all files Collapse all files

13
src/libsyntax/ext/tt/macro_parser.rs
View file

@ -273,7 +273,7 @@ pub enum ParseResult<T> {
Success(T), Success(T),
/// Arm failed to match. If the second parameter is `token::Eof`, it indicates an unexpected /// Arm failed to match. If the second parameter is `token::Eof`, it indicates an unexpected
/// end of macro invocation. Otherwise, it indicates that no rules expected the given token. /// end of macro invocation. Otherwise, it indicates that no rules expected the given token.
Failure(syntax_pos::Span, TokenKind, &'static str), Failure(Token, &'static str),
/// Fatal error (malformed macro?). Abort compilation. /// Fatal error (malformed macro?). Abort compilation.
Error(syntax_pos::Span, String), Error(syntax_pos::Span, String),
} }
@ -701,7 +701,7 @@ pub fn parse(
parser.span, parser.span,
) { ) {
Success(_) => {} Success(_) => {}
Failure(sp, tok, t) => return Failure(sp, tok, t), Failure(token, msg) => return Failure(token, msg),
Error(sp, msg) => return Error(sp, msg), Error(sp, msg) => return Error(sp, msg),
} }
@ -727,13 +727,13 @@ pub fn parse(
"ambiguity: multiple successful parses".to_string(), "ambiguity: multiple successful parses".to_string(),
); );
} else { } else {
return Failure( let span = if parser.span.is_dummy() {
if parser.span.is_dummy() {
parser.span parser.span
} else { } else {
sess.source_map().next_point(parser.span) sess.source_map().next_point(parser.span)
}, };
token::Eof, return Failure(
Token { kind: token::Eof, span },
"missing tokens in macro arguments", "missing tokens in macro arguments",
); );
} }
@ -771,7 +771,6 @@ pub fn parse(
// then there is a syntax error. // then there is a syntax error.
else if bb_items.is_empty() && next_items.is_empty() { else if bb_items.is_empty() && next_items.is_empty() {
return Failure( return Failure(
parser.span,
parser.token.clone(), parser.token.clone(),
"no rules expected this token in macro call", "no rules expected this token in macro call",
); );

16
src/libsyntax/ext/tt/macro_rules.rs
View file

@ -190,10 +190,10 @@ fn generic_extension<'cx>(cx: &'cx mut ExtCtxt<'_>,
arm_span, arm_span,
}) })
} }
Failure(sp, tok, t) => if sp.lo() >= best_fail_spot.lo() { Failure(token, msg) => if token.span.lo() >= best_fail_spot.lo() {
best_fail_spot = sp; best_fail_spot = token.span;
best_fail_tok = Some(tok); best_fail_tok = Some(token.kind);
best_fail_text = Some(t); best_fail_text = Some(msg);
}, },
Error(err_sp, ref msg) => { Error(err_sp, ref msg) => {
cx.span_fatal(err_sp.substitute_dummy(sp), &msg[..]) cx.span_fatal(err_sp.substitute_dummy(sp), &msg[..])
@ -288,11 +288,11 @@ pub fn compile(
let argument_map = match parse(sess, body.stream(), &argument_gram, None, true) { let argument_map = match parse(sess, body.stream(), &argument_gram, None, true) {
Success(m) => m, Success(m) => m,
Failure(sp, tok, t) => { Failure(token, msg) => {
let s = parse_failure_msg(tok); let s = parse_failure_msg(token.kind);
let sp = sp.substitute_dummy(def.span); let sp = token.span.substitute_dummy(def.span);
let mut err = sess.span_diagnostic.struct_span_fatal(sp, &s); let mut err = sess.span_diagnostic.struct_span_fatal(sp, &s);
err.span_label(sp, t); err.span_label(sp, msg);
err.emit(); err.emit();
FatalError.raise(); FatalError.raise();
} }

12
src/libsyntax/parse/attr.rs
View file

@ -24,7 +24,7 @@ impl<'a> Parser<'a> {
let mut just_parsed_doc_comment = false; let mut just_parsed_doc_comment = false;
loop { loop {
debug!("parse_outer_attributes: self.token={:?}", self.token); debug!("parse_outer_attributes: self.token={:?}", self.token);
match self.token { match self.token.kind {
token::Pound => { token::Pound => {
let inner_error_reason = if just_parsed_doc_comment { let inner_error_reason = if just_parsed_doc_comment {
"an inner attribute is not permitted following an outer doc comment" "an inner attribute is not permitted following an outer doc comment"
@ -81,7 +81,7 @@ impl<'a> Parser<'a> {
debug!("parse_attribute_with_inner_parse_policy: inner_parse_policy={:?} self.token={:?}", debug!("parse_attribute_with_inner_parse_policy: inner_parse_policy={:?} self.token={:?}",
inner_parse_policy, inner_parse_policy,
self.token); self.token);
let (span, path, tokens, style) = match self.token { let (span, path, tokens, style) = match self.token.kind {
token::Pound => { token::Pound => {
let lo = self.span; let lo = self.span;
self.bump(); self.bump();
@ -140,7 +140,7 @@ impl<'a> Parser<'a> {
/// PATH `=` TOKEN_TREE /// PATH `=` TOKEN_TREE
/// The delimiters or `=` are still put into the resulting token stream. /// The delimiters or `=` are still put into the resulting token stream.
crate fn parse_meta_item_unrestricted(&mut self) -> PResult<'a, (ast::Path, TokenStream)> { crate fn parse_meta_item_unrestricted(&mut self) -> PResult<'a, (ast::Path, TokenStream)> {
let meta = match self.token { let meta = match self.token.kind {
token::Interpolated(ref nt) => match **nt { token::Interpolated(ref nt) => match **nt {
Nonterminal::NtMeta(ref meta) => Some(meta.clone()), Nonterminal::NtMeta(ref meta) => Some(meta.clone()),
_ => None, _ => None,
@ -159,7 +159,7 @@ impl<'a> Parser<'a> {
} else if self.eat(&token::Eq) { } else if self.eat(&token::Eq) {
let eq = TokenTree::token(self.prev_span, token::Eq); let eq = TokenTree::token(self.prev_span, token::Eq);
let mut is_interpolated_expr = false; let mut is_interpolated_expr = false;
if let token::Interpolated(nt) = &self.token { if let token::Interpolated(nt) = &self.token.kind {
if let token::NtExpr(..) = **nt { if let token::NtExpr(..) = **nt {
is_interpolated_expr = true; is_interpolated_expr = true;
} }
@ -188,7 +188,7 @@ impl<'a> Parser<'a> {
crate fn parse_inner_attributes(&mut self) -> PResult<'a, Vec<ast::Attribute>> { crate fn parse_inner_attributes(&mut self) -> PResult<'a, Vec<ast::Attribute>> {
let mut attrs: Vec<ast::Attribute> = vec![]; let mut attrs: Vec<ast::Attribute> = vec![];
loop { loop {
match self.token { match self.token.kind {
token::Pound => { token::Pound => {
// Don't even try to parse if it's not an inner attribute. // Don't even try to parse if it's not an inner attribute.
if !self.look_ahead(1, |t| t == &token::Not) { if !self.look_ahead(1, |t| t == &token::Not) {
@ -236,7 +236,7 @@ impl<'a> Parser<'a> {
/// meta_item : IDENT ( '=' UNSUFFIXED_LIT | '(' meta_item_inner? ')' )? ; /// meta_item : IDENT ( '=' UNSUFFIXED_LIT | '(' meta_item_inner? ')' )? ;
/// meta_item_inner : (meta_item | UNSUFFIXED_LIT) (',' meta_item_inner)? ; /// meta_item_inner : (meta_item | UNSUFFIXED_LIT) (',' meta_item_inner)? ;
pub fn parse_meta_item(&mut self) -> PResult<'a, ast::MetaItem> { pub fn parse_meta_item(&mut self) -> PResult<'a, ast::MetaItem> {
let nt_meta = match self.token { let nt_meta = match self.token.kind {
token::Interpolated(ref nt) => match **nt { token::Interpolated(ref nt) => match **nt {
token::NtMeta(ref e) => Some(e.clone()), token::NtMeta(ref e) => Some(e.clone()),
_ => None, _ => None,

12
src/libsyntax/parse/diagnostics.rs
View file

@ -201,7 +201,7 @@ impl<'a> Parser<'a> {
self.span, self.span,
&format!("expected identifier, found {}", self.this_token_descr()), &format!("expected identifier, found {}", self.this_token_descr()),
); );
if let token::Ident(ident, false) = &self.token { if let token::Ident(ident, false) = &self.token.kind {
if ident.is_raw_guess() { if ident.is_raw_guess() {
err.span_suggestion( err.span_suggestion(
self.span, self.span,
@ -730,7 +730,7 @@ impl<'a> Parser<'a> {
) -> PResult<'a, bool /* recovered */> { ) -> PResult<'a, bool /* recovered */> {
let token_str = pprust::token_to_string(t); let token_str = pprust::token_to_string(t);
let this_token_str = self.this_token_descr(); let this_token_str = self.this_token_descr();
let (prev_sp, sp) = match (&self.token, self.subparser_name) { let (prev_sp, sp) = match (&self.token.kind, self.subparser_name) {
// Point at the end of the macro call when reaching end of macro arguments. // Point at the end of the macro call when reaching end of macro arguments.
(token::Eof, Some(_)) => { (token::Eof, Some(_)) => {
let sp = self.sess.source_map().next_point(self.span); let sp = self.sess.source_map().next_point(self.span);
@ -746,7 +746,7 @@ impl<'a> Parser<'a> {
let msg = format!( let msg = format!(
"expected `{}`, found {}", "expected `{}`, found {}",
token_str, token_str,
match (&self.token, self.subparser_name) { match (&self.token.kind, self.subparser_name) {
(token::Eof, Some(origin)) => format!("end of {}", origin), (token::Eof, Some(origin)) => format!("end of {}", origin),
_ => this_token_str, _ => this_token_str,
}, },
@ -989,7 +989,7 @@ impl<'a> Parser<'a> {
break_on_semi, break_on_block); break_on_semi, break_on_block);
loop { loop {
debug!("recover_stmt_ loop {:?}", self.token); debug!("recover_stmt_ loop {:?}", self.token);
match self.token { match self.token.kind {
token::OpenDelim(token::DelimToken::Brace) => { token::OpenDelim(token::DelimToken::Brace) => {
brace_depth += 1; brace_depth += 1;
self.bump(); self.bump();
@ -1074,7 +1074,7 @@ impl<'a> Parser<'a> {
} }
crate fn eat_incorrect_doc_comment(&mut self, applied_to: &str) { crate fn eat_incorrect_doc_comment(&mut self, applied_to: &str) {
if let token::DocComment(_) = self.token { if let token::DocComment(_) = self.token.kind {
let mut err = self.diagnostic().struct_span_err( let mut err = self.diagnostic().struct_span_err(
self.span, self.span,
&format!("documentation comments cannot be applied to {}", applied_to), &format!("documentation comments cannot be applied to {}", applied_to),
@ -1214,7 +1214,7 @@ impl<'a> Parser<'a> {
} }
crate fn expected_expression_found(&self) -> DiagnosticBuilder<'a> { crate fn expected_expression_found(&self) -> DiagnosticBuilder<'a> {
let (span, msg) = match (&self.token, self.subparser_name) { let (span, msg) = match (&self.token.kind, self.subparser_name) {
(&token::Eof, Some(origin)) => { (&token::Eof, Some(origin)) => {
let sp = self.sess.source_map().next_point(self.span); let sp = self.sess.source_map().next_point(self.span);
(sp, format!("expected expression, found end of {}", origin)) (sp, format!("expected expression, found end of {}", origin))

26
src/libsyntax/parse/literal.rs
View file

@ -3,7 +3,7 @@
use crate::ast::{self, Ident, Lit, LitKind}; use crate::ast::{self, Ident, Lit, LitKind};
use crate::parse::parser::Parser; use crate::parse::parser::Parser;
use crate::parse::PResult; use crate::parse::PResult;
use crate::parse::token::{self, TokenKind}; use crate::parse::token::{self, Token, TokenKind};
use crate::parse::unescape::{unescape_str, unescape_char, unescape_byte_str, unescape_byte}; use crate::parse::unescape::{unescape_str, unescape_char, unescape_byte_str, unescape_byte};
use crate::print::pprust; use crate::print::pprust;
use crate::symbol::{kw, sym, Symbol}; use crate::symbol::{kw, sym, Symbol};
@ -272,44 +272,42 @@ impl<'a> Parser<'a> {
if self.token == token::Dot { if self.token == token::Dot {
// Attempt to recover `.4` as `0.4`. // Attempt to recover `.4` as `0.4`.
recovered = self.look_ahead(1, |t| { recovered = self.look_ahead(1, |t| {
if let token::Literal(token::Lit { kind: token::Integer, symbol, suffix }) = *t { if let token::Literal(token::Lit { kind: token::Integer, symbol, suffix }) = t.kind {
let next_span = self.look_ahead_span(1); let next_span = self.look_ahead_span(1);
if self.span.hi() == next_span.lo() { if self.span.hi() == next_span.lo() {
let s = String::from("0.") + &symbol.as_str(); let s = String::from("0.") + &symbol.as_str();
let token = TokenKind::lit(token::Float, Symbol::intern(&s), suffix); let kind = TokenKind::lit(token::Float, Symbol::intern(&s), suffix);
return Some((token, self.span.to(next_span))); return Some(Token { kind, span: self.span.to(next_span) });
} }
} }
None None
}); });
if let Some((ref token, span)) = recovered { if let Some(token) = &recovered {
self.bump(); self.bump();
self.diagnostic() self.diagnostic()
.struct_span_err(span, "float literals must have an integer part") .struct_span_err(token.span, "float literals must have an integer part")
.span_suggestion( .span_suggestion(
span, token.span,
"must have an integer part", "must have an integer part",
pprust::token_to_string(&token), pprust::token_to_string(token),
Applicability::MachineApplicable, Applicability::MachineApplicable,
) )
.emit(); .emit();
} }
} }
let (token, span) = recovered.as_ref().map_or((&self.token, self.span), let token = recovered.as_ref().unwrap_or(&self.token);
|(token, span)| (token, *span)); match Lit::from_token(token, token.span) {
match Lit::from_token(token, span) {
Ok(lit) => { Ok(lit) => {
self.bump(); self.bump();
Ok(lit) Ok(lit)
} }
Err(LitError::NotLiteral) => { Err(LitError::NotLiteral) => {
let msg = format!("unexpected token: {}", self.this_token_descr()); let msg = format!("unexpected token: {}", self.this_token_descr());
Err(self.span_fatal(span, &msg)) Err(self.span_fatal(token.span, &msg))
} }
Err(err) => { Err(err) => {
let lit = token.expect_lit(); let (lit, span) = (token.expect_lit(), token.span);
self.bump(); self.bump();
err.report(&self.sess.span_diagnostic, lit, span); err.report(&self.sess.span_diagnostic, lit, span);
let lit = token::Lit::new(token::Err, lit.symbol, lit.suffix); let lit = token::Lit::new(token::Err, lit.symbol, lit.suffix);

2
src/libsyntax/parse/mod.rs
View file

@ -239,7 +239,7 @@ fn maybe_source_file_to_parser(
let mut parser = stream_to_parser(sess, stream, None); let mut parser = stream_to_parser(sess, stream, None);
parser.unclosed_delims = unclosed_delims; parser.unclosed_delims = unclosed_delims;
if parser.token == token::Eof && parser.span.is_dummy() { if parser.token == token::Eof && parser.span.is_dummy() {
parser.span = Span::new(end_pos, end_pos, parser.span.ctxt()); parser.token.span = Span::new(end_pos, end_pos, parser.span.ctxt());
} }
Ok(parser) Ok(parser)

161
src/libsyntax/parse/parser.rs
View file

@ -57,6 +57,7 @@ use log::debug;
use std::borrow::Cow; use std::borrow::Cow;
use std::cmp; use std::cmp;
use std::mem; use std::mem;
use std::ops::Deref;
use std::path::{self, Path, PathBuf}; use std::path::{self, Path, PathBuf};
use std::slice; use std::slice;
@ -121,7 +122,7 @@ crate enum BlockMode {
/// `token::Interpolated` tokens. /// `token::Interpolated` tokens.
macro_rules! maybe_whole_expr { macro_rules! maybe_whole_expr {
($p:expr) => { ($p:expr) => {
if let token::Interpolated(nt) = &$p.token { if let token::Interpolated(nt) = &$p.token.kind {
match &**nt { match &**nt {
token::NtExpr(e) | token::NtLiteral(e) => { token::NtExpr(e) | token::NtLiteral(e) => {
let e = e.clone(); let e = e.clone();
@ -147,7 +148,7 @@ macro_rules! maybe_whole_expr {
/// As maybe_whole_expr, but for things other than expressions /// As maybe_whole_expr, but for things other than expressions
macro_rules! maybe_whole { macro_rules! maybe_whole {
($p:expr, $constructor:ident, |$x:ident| $e:expr) => { ($p:expr, $constructor:ident, |$x:ident| $e:expr) => {
if let token::Interpolated(nt) = &$p.token { if let token::Interpolated(nt) = &$p.token.kind {
if let token::$constructor(x) = &**nt { if let token::$constructor(x) = &**nt {
let $x = x.clone(); let $x = x.clone();
$p.bump(); $p.bump();
@ -161,7 +162,7 @@ macro_rules! maybe_whole {
macro_rules! maybe_recover_from_interpolated_ty_qpath { macro_rules! maybe_recover_from_interpolated_ty_qpath {
($self: expr, $allow_qpath_recovery: expr) => { ($self: expr, $allow_qpath_recovery: expr) => {
if $allow_qpath_recovery && $self.look_ahead(1, |t| t == &token::ModSep) { if $allow_qpath_recovery && $self.look_ahead(1, |t| t == &token::ModSep) {
if let token::Interpolated(nt) = &$self.token { if let token::Interpolated(nt) = &$self.token.kind {
if let token::NtTy(ty) = &**nt { if let token::NtTy(ty) = &**nt {
let ty = ty.clone(); let ty = ty.clone();
$self.bump(); $self.bump();
@ -196,14 +197,13 @@ enum PrevTokenKind {
#[derive(Clone)] #[derive(Clone)]
pub struct Parser<'a> { pub struct Parser<'a> {
pub sess: &'a ParseSess, pub sess: &'a ParseSess,
/// the current token: /// The current token.
pub token: token::TokenKind, pub token: Token,
/// the span of the current token: /// The span of the previous token.
pub span: Span,
meta_var_span: Option<Span>, meta_var_span: Option<Span>,
/// The span of the previous token. /// The span of the previous token.
pub prev_span: Span, pub prev_span: Span,
/// The kind of the previous troken. /// The previous token kind.
prev_token_kind: PrevTokenKind, prev_token_kind: PrevTokenKind,
restrictions: Restrictions, restrictions: Restrictions,
/// Used to determine the path to externally loaded source files. /// Used to determine the path to externally loaded source files.
@ -242,6 +242,15 @@ impl<'a> Drop for Parser<'a> {
} }
} }
// FIXME: Parser uses `self.span` all the time.
// Remove this impl if you think that using `self.token.span` instead is acceptable.
impl Deref for Parser<'_> {
type Target = Token;
fn deref(&self) -> &Self::Target {
&self.token
}
}
#[derive(Clone)] #[derive(Clone)]
crate struct TokenCursor { crate struct TokenCursor {
crate frame: TokenCursorFrame, crate frame: TokenCursorFrame,
@ -468,8 +477,7 @@ impl<'a> Parser<'a> {
) -> Self { ) -> Self {
let mut parser = Parser { let mut parser = Parser {
sess, sess,
token: token::Whitespace, token: Token { kind: token::Whitespace, span: DUMMY_SP },
span: DUMMY_SP,
prev_span: DUMMY_SP, prev_span: DUMMY_SP,
meta_var_span: None, meta_var_span: None,
prev_token_kind: PrevTokenKind::Other, prev_token_kind: PrevTokenKind::Other,
@ -498,9 +506,7 @@ impl<'a> Parser<'a> {
subparser_name, subparser_name,
}; };
let tok = parser.next_tok(); parser.token = parser.next_tok();
parser.token = tok.kind;
parser.span = tok.span;
if let Some(directory) = directory { if let Some(directory) = directory {
parser.directory = directory; parser.directory = directory;
@ -534,7 +540,7 @@ impl<'a> Parser<'a> {
} }
crate fn token_descr(&self) -> Option<&'static str> { crate fn token_descr(&self) -> Option<&'static str> {
Some(match &self.token { Some(match &self.token.kind {
t if t.is_special_ident() => "reserved identifier", t if t.is_special_ident() => "reserved identifier",
t if t.is_used_keyword() => "keyword", t if t.is_used_keyword() => "keyword",
t if t.is_unused_keyword() => "reserved keyword", t if t.is_unused_keyword() => "reserved keyword",
@ -612,7 +618,7 @@ impl<'a> Parser<'a> {
} }
fn parse_ident_common(&mut self, recover: bool) -> PResult<'a, ast::Ident> { fn parse_ident_common(&mut self, recover: bool) -> PResult<'a, ast::Ident> {
match self.token { match self.token.kind {
token::Ident(ident, _) => { token::Ident(ident, _) => {
if self.token.is_reserved_ident() { if self.token.is_reserved_ident() {
let mut err = self.expected_ident_found(); let mut err = self.expected_ident_found();
@ -732,7 +738,7 @@ impl<'a> Parser<'a> {
/// See issue #47856 for an example of when this may occur. /// See issue #47856 for an example of when this may occur.
fn eat_plus(&mut self) -> bool { fn eat_plus(&mut self) -> bool {
self.expected_tokens.push(TokenType::Token(token::BinOp(token::Plus))); self.expected_tokens.push(TokenType::Token(token::BinOp(token::Plus)));
match self.token { match self.token.kind {
token::BinOp(token::Plus) => { token::BinOp(token::Plus) => {
self.bump(); self.bump();
true true
@ -763,7 +769,7 @@ impl<'a> Parser<'a> {
/// `&` and continues. If an `&` is not seen, signals an error. /// `&` and continues. If an `&` is not seen, signals an error.
fn expect_and(&mut self) -> PResult<'a, ()> { fn expect_and(&mut self) -> PResult<'a, ()> {
self.expected_tokens.push(TokenType::Token(token::BinOp(token::And))); self.expected_tokens.push(TokenType::Token(token::BinOp(token::And)));
match self.token { match self.token.kind {
token::BinOp(token::And) => { token::BinOp(token::And) => {
self.bump(); self.bump();
Ok(()) Ok(())
@ -780,7 +786,7 @@ impl<'a> Parser<'a> {
/// `|` and continues. If an `|` is not seen, signals an error. /// `|` and continues. If an `|` is not seen, signals an error.
fn expect_or(&mut self) -> PResult<'a, ()> { fn expect_or(&mut self) -> PResult<'a, ()> {
self.expected_tokens.push(TokenType::Token(token::BinOp(token::Or))); self.expected_tokens.push(TokenType::Token(token::BinOp(token::Or)));
match self.token { match self.token.kind {
token::BinOp(token::Or) => { token::BinOp(token::Or) => {
self.bump(); self.bump();
Ok(()) Ok(())
@ -805,7 +811,7 @@ impl<'a> Parser<'a> {
/// starting token. /// starting token.
fn eat_lt(&mut self) -> bool { fn eat_lt(&mut self) -> bool {
self.expected_tokens.push(TokenType::Token(token::Lt)); self.expected_tokens.push(TokenType::Token(token::Lt));
let ate = match self.token { let ate = match self.token.kind {
token::Lt => { token::Lt => {
self.bump(); self.bump();
true true
@ -845,7 +851,7 @@ impl<'a> Parser<'a> {
/// with a single `>` and continues. If a `>` is not seen, signals an error. /// with a single `>` and continues. If a `>` is not seen, signals an error.
fn expect_gt(&mut self) -> PResult<'a, ()> { fn expect_gt(&mut self) -> PResult<'a, ()> {
self.expected_tokens.push(TokenType::Token(token::Gt)); self.expected_tokens.push(TokenType::Token(token::Gt));
let ate = match self.token { let ate = match self.token.kind {
token::Gt => { token::Gt => {
self.bump(); self.bump();
Some(()) Some(())
@ -928,7 +934,7 @@ impl<'a> Parser<'a> {
TokenExpectType::NoExpect => self.token == **k, TokenExpectType::NoExpect => self.token == **k,
} }
}) { }) {
match self.token { match self.token.kind {
token::CloseDelim(..) | token::Eof => break, token::CloseDelim(..) | token::Eof => break,
_ => {} _ => {}
}; };
@ -1011,7 +1017,7 @@ impl<'a> Parser<'a> {
self.prev_span = self.meta_var_span.take().unwrap_or(self.span); self.prev_span = self.meta_var_span.take().unwrap_or(self.span);
// Record last token kind for possible error recovery. // Record last token kind for possible error recovery.
self.prev_token_kind = match self.token { self.prev_token_kind = match self.token.kind {
token::DocComment(..) => PrevTokenKind::DocComment, token::DocComment(..) => PrevTokenKind::DocComment,
token::Comma => PrevTokenKind::Comma, token::Comma => PrevTokenKind::Comma,
token::BinOp(token::Plus) => PrevTokenKind::Plus, token::BinOp(token::Plus) => PrevTokenKind::Plus,
@ -1022,9 +1028,7 @@ impl<'a> Parser<'a> {
_ => PrevTokenKind::Other, _ => PrevTokenKind::Other,
}; };
let next = self.next_tok(); self.token = self.next_tok();
self.token = next.kind;
self.span = next.span;
self.expected_tokens.clear(); self.expected_tokens.clear();
// check after each token // check after each token
self.process_potential_macro_variable(); self.process_potential_macro_variable();
@ -1038,24 +1042,25 @@ impl<'a> Parser<'a> {
// fortunately for tokens currently using `bump_with`, the // fortunately for tokens currently using `bump_with`, the
// prev_token_kind will be of no use anyway. // prev_token_kind will be of no use anyway.
self.prev_token_kind = PrevTokenKind::Other; self.prev_token_kind = PrevTokenKind::Other;
self.token = next; self.token = Token { kind: next, span };
self.span = span;
self.expected_tokens.clear(); self.expected_tokens.clear();
} }
pub fn look_ahead<R, F>(&self, dist: usize, f: F) -> R where pub fn look_ahead<R, F>(&self, dist: usize, f: F) -> R where
F: FnOnce(&token::TokenKind) -> R, F: FnOnce(&token::Token) -> R,
{ {
if dist == 0 { if dist == 0 {
return f(&self.token) // FIXME: Avoid cloning here.
return f(&self.token);
} }
f(&match self.token_cursor.frame.tree_cursor.look_ahead(dist - 1) { let frame = &self.token_cursor.frame;
f(&match frame.tree_cursor.look_ahead(dist - 1) {
Some(tree) => match tree { Some(tree) => match tree {
TokenTree::Token(token) => token.kind, TokenTree::Token(token) => token,
TokenTree::Delimited(_, delim, _) => token::OpenDelim(delim), TokenTree::Delimited(dspan, delim, _) => Token { kind: token::OpenDelim(delim), span: dspan.open },
}, }
None => token::CloseDelim(self.token_cursor.frame.delim), None => Token { kind: token::CloseDelim(frame.delim), span: frame.span.close }
}) })
} }
@ -1209,7 +1214,7 @@ impl<'a> Parser<'a> {
decl, decl,
}; };
let body = match self.token { let body = match self.token.kind {
token::Semi => { token::Semi => {
self.bump(); self.bump();
*at_end = true; *at_end = true;
@ -1477,7 +1482,7 @@ impl<'a> Parser<'a> {
} }
fn is_named_argument(&self) -> bool { fn is_named_argument(&self) -> bool {
let offset = match self.token { let offset = match self.token.kind {
token::Interpolated(ref nt) => match **nt { token::Interpolated(ref nt) => match **nt {
token::NtPat(..) => return self.look_ahead(1, |t| t == &token::Colon), token::NtPat(..) => return self.look_ahead(1, |t| t == &token::Colon),
_ => 0, _ => 0,
@ -1612,7 +1617,7 @@ impl<'a> Parser<'a> {
} }
fn parse_path_segment_ident(&mut self) -> PResult<'a, ast::Ident> { fn parse_path_segment_ident(&mut self) -> PResult<'a, ast::Ident> {
match self.token { match self.token.kind {
token::Ident(ident, _) if self.token.is_path_segment_keyword() => { token::Ident(ident, _) if self.token.is_path_segment_keyword() => {
let span = self.span; let span = self.span;
self.bump(); self.bump();
@ -1623,7 +1628,7 @@ impl<'a> Parser<'a> {
} }
fn parse_ident_or_underscore(&mut self) -> PResult<'a, ast::Ident> { fn parse_ident_or_underscore(&mut self) -> PResult<'a, ast::Ident> {
match self.token { match self.token.kind {
token::Ident(ident, false) if ident.name == kw::Underscore => { token::Ident(ident, false) if ident.name == kw::Underscore => {
let span = self.span; let span = self.span;
self.bump(); self.bump();
@ -1710,7 +1715,7 @@ impl<'a> Parser<'a> {
/// backwards-compatibility. This is used when parsing derive macro paths in `#[derive]` /// backwards-compatibility. This is used when parsing derive macro paths in `#[derive]`
/// attributes. /// attributes.
pub fn parse_path_allowing_meta(&mut self, style: PathStyle) -> PResult<'a, ast::Path> { pub fn parse_path_allowing_meta(&mut self, style: PathStyle) -> PResult<'a, ast::Path> {
let meta_ident = match self.token { let meta_ident = match self.token.kind {
token::Interpolated(ref nt) => match **nt { token::Interpolated(ref nt) => match **nt {
token::NtMeta(ref meta) => match meta.node { token::NtMeta(ref meta) => match meta.node {
ast::MetaItemKind::Word => Some(meta.path.clone()), ast::MetaItemKind::Word => Some(meta.path.clone()),
@ -1859,7 +1864,7 @@ impl<'a> Parser<'a> {
} }
fn parse_field_name(&mut self) -> PResult<'a, Ident> { fn parse_field_name(&mut self) -> PResult<'a, Ident> {
if let token::Literal(token::Lit { kind: token::Integer, symbol, suffix }) = self.token { if let token::Literal(token::Lit { kind: token::Integer, symbol, suffix }) = self.token.kind {
self.expect_no_suffix(self.span, "a tuple index", suffix); self.expect_no_suffix(self.span, "a tuple index", suffix);
self.bump(); self.bump();
Ok(Ident::new(symbol, self.prev_span)) Ok(Ident::new(symbol, self.prev_span))
@ -1949,7 +1954,7 @@ impl<'a> Parser<'a> {
} }
fn expect_delimited_token_tree(&mut self) -> PResult<'a, (MacDelimiter, TokenStream)> { fn expect_delimited_token_tree(&mut self) -> PResult<'a, (MacDelimiter, TokenStream)> {
let delim = match self.token { let delim = match self.token.kind {
token::OpenDelim(delim) => delim, token::OpenDelim(delim) => delim,
_ => { _ => {
let msg = "expected open delimiter"; let msg = "expected open delimiter";
@ -1993,7 +1998,7 @@ impl<'a> Parser<'a> {
let ex: ExprKind; let ex: ExprKind;
// Note: when adding new syntax here, don't forget to adjust TokenKind::can_begin_expr(). // Note: when adding new syntax here, don't forget to adjust TokenKind::can_begin_expr().
match self.token { match self.token.kind {
token::OpenDelim(token::Paren) => { token::OpenDelim(token::Paren) => {
self.bump(); self.bump();
@ -2363,7 +2368,7 @@ impl<'a> Parser<'a> {
} }
let mut recovery_field = None; let mut recovery_field = None;
if let token::Ident(ident, _) = self.token { if let token::Ident(ident, _) = self.token.kind {
if !self.token.is_reserved_ident() && self.look_ahead(1, |t| *t == token::Colon) { if !self.token.is_reserved_ident() && self.look_ahead(1, |t| *t == token::Colon) {
// Use in case of error after field-looking code: `S { foo: () with a }` // Use in case of error after field-looking code: `S { foo: () with a }`
let mut ident = ident.clone(); let mut ident = ident.clone();
@ -2503,7 +2508,7 @@ impl<'a> Parser<'a> {
let segment = self.parse_path_segment(PathStyle::Expr)?; let segment = self.parse_path_segment(PathStyle::Expr)?;
self.check_trailing_angle_brackets(&segment, token::OpenDelim(token::Paren)); self.check_trailing_angle_brackets(&segment, token::OpenDelim(token::Paren));
Ok(match self.token { Ok(match self.token.kind {
token::OpenDelim(token::Paren) => { token::OpenDelim(token::Paren) => {
// Method call `expr.f()` // Method call `expr.f()`
let mut args = self.parse_unspanned_seq( let mut args = self.parse_unspanned_seq(
@ -2542,7 +2547,7 @@ impl<'a> Parser<'a> {
// expr.f // expr.f
if self.eat(&token::Dot) { if self.eat(&token::Dot) {
match self.token { match self.token.kind {
token::Ident(..) => { token::Ident(..) => {
e = self.parse_dot_suffix(e, lo)?; e = self.parse_dot_suffix(e, lo)?;
} }
@ -2594,7 +2599,7 @@ impl<'a> Parser<'a> {
continue; continue;
} }
if self.expr_is_complete(&e) { break; } if self.expr_is_complete(&e) { break; }
match self.token { match self.token.kind {
// expr(...) // expr(...)
token::OpenDelim(token::Paren) => { token::OpenDelim(token::Paren) => {
let seq = self.parse_unspanned_seq( let seq = self.parse_unspanned_seq(
@ -2627,11 +2632,11 @@ impl<'a> Parser<'a> {
} }
crate fn process_potential_macro_variable(&mut self) { crate fn process_potential_macro_variable(&mut self) {
let (token, span) = match self.token { self.token = match self.token.kind {
token::Dollar if self.span.ctxt() != syntax_pos::hygiene::SyntaxContext::empty() && token::Dollar if self.span.ctxt() != syntax_pos::hygiene::SyntaxContext::empty() &&
self.look_ahead(1, |t| t.is_ident()) => { self.look_ahead(1, |t| t.is_ident()) => {
self.bump(); self.bump();
let name = match self.token { let name = match self.token.kind {
token::Ident(ident, _) => ident, token::Ident(ident, _) => ident,
_ => unreachable!() _ => unreachable!()
}; };
@ -2646,24 +2651,22 @@ impl<'a> Parser<'a> {
// Interpolated identifier and lifetime tokens are replaced with usual identifier // Interpolated identifier and lifetime tokens are replaced with usual identifier
// and lifetime tokens, so the former are never encountered during normal parsing. // and lifetime tokens, so the former are never encountered during normal parsing.
match **nt { match **nt {
token::NtIdent(ident, is_raw) => (token::Ident(ident, is_raw), ident.span), token::NtIdent(ident, is_raw) => Token { kind: token::Ident(ident, is_raw), span: ident.span },
token::NtLifetime(ident) => (token::Lifetime(ident), ident.span), token::NtLifetime(ident) => Token { kind: token::Lifetime(ident), span: ident.span },
_ => return, _ => return,
} }
} }
_ => return, _ => return,
}; };
self.token = token;
self.span = span;
} }
/// Parses a single token tree from the input. /// Parses a single token tree from the input.
crate fn parse_token_tree(&mut self) -> TokenTree { crate fn parse_token_tree(&mut self) -> TokenTree {
match self.token { match self.token.kind {
token::OpenDelim(..) => { token::OpenDelim(..) => {
let frame = mem::replace(&mut self.token_cursor.frame, let frame = mem::replace(&mut self.token_cursor.frame,
self.token_cursor.stack.pop().unwrap()); self.token_cursor.stack.pop().unwrap());
self.span = frame.span.entire(); self.token.span = frame.span.entire();
self.bump(); self.bump();
TokenTree::Delimited( TokenTree::Delimited(
frame.span, frame.span,
@ -2673,9 +2676,9 @@ impl<'a> Parser<'a> {
}, },
token::CloseDelim(_) | token::Eof => unreachable!(), token::CloseDelim(_) | token::Eof => unreachable!(),
_ => { _ => {
let (token, span) = (mem::replace(&mut self.token, token::Whitespace), self.span); let token = mem::replace(&mut self.token, Token { kind: token::Whitespace, span: DUMMY_SP });
self.bump(); self.bump();
TokenTree::token(span, token) TokenTree::Token(token)
} }
} }
} }
@ -2692,7 +2695,7 @@ impl<'a> Parser<'a> {
pub fn parse_tokens(&mut self) -> TokenStream { pub fn parse_tokens(&mut self) -> TokenStream {
let mut result = Vec::new(); let mut result = Vec::new();
loop { loop {
match self.token { match self.token.kind {
token::Eof | token::CloseDelim(..) => break, token::Eof | token::CloseDelim(..) => break,
_ => result.push(self.parse_token_tree().into()), _ => result.push(self.parse_token_tree().into()),
} }
@ -2707,7 +2710,7 @@ impl<'a> Parser<'a> {
let attrs = self.parse_or_use_outer_attributes(already_parsed_attrs)?; let attrs = self.parse_or_use_outer_attributes(already_parsed_attrs)?;
let lo = self.span; let lo = self.span;
// Note: when adding new unary operators, don't forget to adjust TokenKind::can_begin_expr() // Note: when adding new unary operators, don't forget to adjust TokenKind::can_begin_expr()
let (hi, ex) = match self.token { let (hi, ex) = match self.token.kind {
token::Not => { token::Not => {
self.bump(); self.bump();
let e = self.parse_prefix_expr(None); let e = self.parse_prefix_expr(None);
@ -2760,7 +2763,7 @@ impl<'a> Parser<'a> {
// `not` is just an ordinary identifier in Rust-the-language, // `not` is just an ordinary identifier in Rust-the-language,
// but as `rustc`-the-compiler, we can issue clever diagnostics // but as `rustc`-the-compiler, we can issue clever diagnostics
// for confused users who really want to say `!` // for confused users who really want to say `!`
let token_cannot_continue_expr = |t: &token::TokenKind| match *t { let token_cannot_continue_expr = |t: &token::Token| match t.kind {
// These tokens can start an expression after `!`, but // These tokens can start an expression after `!`, but
// can't continue an expression after an ident // can't continue an expression after an ident
token::Ident(ident, is_raw) => token::ident_can_begin_expr(ident, is_raw), token::Ident(ident, is_raw) => token::ident_can_begin_expr(ident, is_raw),
@ -3040,7 +3043,7 @@ impl<'a> Parser<'a> {
match self.parse_path(PathStyle::Expr) { match self.parse_path(PathStyle::Expr) {
Ok(path) => { Ok(path) => {
let (op_noun, op_verb) = match self.token { let (op_noun, op_verb) = match self.token.kind {
token::Lt => ("comparison", "comparing"), token::Lt => ("comparison", "comparing"),
token::BinOp(token::Shl) => ("shift", "shifting"), token::BinOp(token::Shl) => ("shift", "shifting"),
_ => { _ => {
@ -3844,14 +3847,14 @@ impl<'a> Parser<'a> {
// helper function to decide whether to parse as ident binding or to try to do // helper function to decide whether to parse as ident binding or to try to do
// something more complex like range patterns // something more complex like range patterns
fn parse_as_ident(&mut self) -> bool { fn parse_as_ident(&mut self) -> bool {
self.look_ahead(1, |t| match *t { self.look_ahead(1, |t| match t.kind {
token::OpenDelim(token::Paren) | token::OpenDelim(token::Brace) | token::OpenDelim(token::Paren) | token::OpenDelim(token::Brace) |
token::DotDotDot | token::DotDotEq | token::ModSep | token::Not => Some(false), token::DotDotDot | token::DotDotEq | token::ModSep | token::Not => Some(false),
// ensure slice patterns [a, b.., c] and [a, b, c..] don't go into the // ensure slice patterns [a, b.., c] and [a, b, c..] don't go into the
// range pattern branch // range pattern branch
token::DotDot => None, token::DotDot => None,
_ => Some(true), _ => Some(true),
}).unwrap_or_else(|| self.look_ahead(2, |t| match *t { }).unwrap_or_else(|| self.look_ahead(2, |t| match t.kind {
token::Comma | token::CloseDelim(token::Bracket) => true, token::Comma | token::CloseDelim(token::Bracket) => true,
_ => false, _ => false,
})) }))
@ -3914,12 +3917,12 @@ impl<'a> Parser<'a> {
let lo = self.span; let lo = self.span;
let pat; let pat;
match self.token { match self.token.kind {
token::BinOp(token::And) | token::AndAnd => { token::BinOp(token::And) | token::AndAnd => {
// Parse &pat / &mut pat // Parse &pat / &mut pat
self.expect_and()?; self.expect_and()?;
let mutbl = self.parse_mutability(); let mutbl = self.parse_mutability();
if let token::Lifetime(ident) = self.token { if let token::Lifetime(ident) = self.token.kind {
let mut err = self.fatal(&format!("unexpected lifetime `{}` in pattern", let mut err = self.fatal(&format!("unexpected lifetime `{}` in pattern",
ident)); ident));
err.span_label(self.span, "unexpected lifetime"); err.span_label(self.span, "unexpected lifetime");
@ -3990,7 +3993,7 @@ impl<'a> Parser<'a> {
// Parse an unqualified path // Parse an unqualified path
(None, self.parse_path(PathStyle::Expr)?) (None, self.parse_path(PathStyle::Expr)?)
}; };
match self.token { match self.token.kind {
token::Not if qself.is_none() => { token::Not if qself.is_none() => {
// Parse macro invocation // Parse macro invocation
self.bump(); self.bump();
@ -3999,7 +4002,7 @@ impl<'a> Parser<'a> {
pat = PatKind::Mac(mac); pat = PatKind::Mac(mac);
} }
token::DotDotDot | token::DotDotEq | token::DotDot => { token::DotDotDot | token::DotDotEq | token::DotDot => {
let end_kind = match self.token { let end_kind = match self.token.kind {
token::DotDot => RangeEnd::Excluded, token::DotDot => RangeEnd::Excluded,
token::DotDotDot => RangeEnd::Included(RangeSyntax::DotDotDot), token::DotDotDot => RangeEnd::Included(RangeSyntax::DotDotDot),
token::DotDotEq => RangeEnd::Included(RangeSyntax::DotDotEq), token::DotDotEq => RangeEnd::Included(RangeSyntax::DotDotEq),
@ -4325,7 +4328,7 @@ impl<'a> Parser<'a> {
fn eat_macro_def(&mut self, attrs: &[Attribute], vis: &Visibility, lo: Span) fn eat_macro_def(&mut self, attrs: &[Attribute], vis: &Visibility, lo: Span)
-> PResult<'a, Option<P<Item>>> { -> PResult<'a, Option<P<Item>>> {
let token_lo = self.span; let token_lo = self.span;
let (ident, def) = match self.token { let (ident, def) = match self.token.kind {
token::Ident(ident, false) if ident.name == kw::Macro => { token::Ident(ident, false) if ident.name == kw::Macro => {
self.bump(); self.bump();
let ident = self.parse_ident()?; let ident = self.parse_ident()?;
@ -4436,7 +4439,7 @@ impl<'a> Parser<'a> {
} }
// it's a macro invocation // it's a macro invocation
let id = match self.token { let id = match self.token.kind {
token::OpenDelim(_) => Ident::invalid(), // no special identifier token::OpenDelim(_) => Ident::invalid(), // no special identifier
_ => self.parse_ident()?, _ => self.parse_ident()?,
}; };
@ -4444,7 +4447,7 @@ impl<'a> Parser<'a> {
// check that we're pointing at delimiters (need to check // check that we're pointing at delimiters (need to check
// again after the `if`, because of `parse_ident` // again after the `if`, because of `parse_ident`
// consuming more tokens). // consuming more tokens).
match self.token { match self.token.kind {
token::OpenDelim(_) => {} token::OpenDelim(_) => {}
_ => { _ => {
// we only expect an ident if we didn't parse one // we only expect an ident if we didn't parse one
@ -4481,7 +4484,7 @@ impl<'a> Parser<'a> {
// We used to incorrectly stop parsing macro-expanded statements here. // We used to incorrectly stop parsing macro-expanded statements here.
// If the next token will be an error anyway but could have parsed with the // If the next token will be an error anyway but could have parsed with the
// earlier behavior, stop parsing here and emit a warning to avoid breakage. // earlier behavior, stop parsing here and emit a warning to avoid breakage.
else if macro_legacy_warnings && self.token.can_begin_expr() && match self.token { else if macro_legacy_warnings && self.token.can_begin_expr() && match self.token.kind {
// These can continue an expression, so we can't stop parsing and warn. // These can continue an expression, so we can't stop parsing and warn.
token::OpenDelim(token::Paren) | token::OpenDelim(token::Bracket) | token::OpenDelim(token::Paren) | token::OpenDelim(token::Bracket) |
token::BinOp(token::Minus) | token::BinOp(token::Star) | token::BinOp(token::Minus) | token::BinOp(token::Star) |
@ -5250,7 +5253,7 @@ impl<'a> Parser<'a> {
assoc_ty_constraints.push(span); assoc_ty_constraints.push(span);
} else if self.check_const_arg() { } else if self.check_const_arg() {
// Parse const argument. // Parse const argument.
let expr = if let token::OpenDelim(token::Brace) = self.token { let expr = if let token::OpenDelim(token::Brace) = self.token.kind {
self.parse_block_expr(None, self.span, BlockCheckMode::Default, ThinVec::new())? self.parse_block_expr(None, self.span, BlockCheckMode::Default, ThinVec::new())?
} else if self.token.is_ident() { } else if self.token.is_ident() {
// FIXME(const_generics): to distinguish between idents for types and consts, // FIXME(const_generics): to distinguish between idents for types and consts,
@ -5477,7 +5480,7 @@ impl<'a> Parser<'a> {
/// Returns the parsed optional self argument and whether a self shortcut was used. /// Returns the parsed optional self argument and whether a self shortcut was used.
fn parse_self_arg(&mut self) -> PResult<'a, Option<Arg>> { fn parse_self_arg(&mut self) -> PResult<'a, Option<Arg>> {
let expect_ident = |this: &mut Self| match this.token { let expect_ident = |this: &mut Self| match this.token.kind {
// Preserve hygienic context. // Preserve hygienic context.
token::Ident(ident, _) => token::Ident(ident, _) =>
{ let span = this.span; this.bump(); Ident::new(ident.name, span) } { let span = this.span; this.bump(); Ident::new(ident.name, span) }
@ -5492,7 +5495,7 @@ impl<'a> Parser<'a> {
// Only a limited set of initial token sequences is considered `self` parameters; anything // Only a limited set of initial token sequences is considered `self` parameters; anything
// else is parsed as a normal function parameter list, so some lookahead is required. // else is parsed as a normal function parameter list, so some lookahead is required.
let eself_lo = self.span; let eself_lo = self.span;
let (eself, eself_ident, eself_hi) = match self.token { let (eself, eself_ident, eself_hi) = match self.token.kind {
token::BinOp(token::And) => { token::BinOp(token::And) => {
// `&self` // `&self`
// `&mut self` // `&mut self`
@ -5803,7 +5806,7 @@ impl<'a> Parser<'a> {
match *vis { match *vis {
VisibilityKind::Inherited => {} VisibilityKind::Inherited => {}
_ => { _ => {
let is_macro_rules: bool = match self.token { let is_macro_rules: bool = match self.token.kind {
token::Ident(sid, _) => sid.name == sym::macro_rules, token::Ident(sid, _) => sid.name == sym::macro_rules,
_ => false, _ => false,
}; };
@ -5918,7 +5921,7 @@ impl<'a> Parser<'a> {
self.expect(&token::OpenDelim(token::Brace))?; self.expect(&token::OpenDelim(token::Brace))?;
let mut trait_items = vec![]; let mut trait_items = vec![];
while !self.eat(&token::CloseDelim(token::Brace)) { while !self.eat(&token::CloseDelim(token::Brace)) {
if let token::DocComment(_) = self.token { if let token::DocComment(_) = self.token.kind {
if self.look_ahead(1, if self.look_ahead(1,
|tok| tok == &token::CloseDelim(token::Brace)) { |tok| tok == &token::CloseDelim(token::Brace)) {
let mut err = self.diagnostic().struct_span_err_with_code( let mut err = self.diagnostic().struct_span_err_with_code(
@ -6246,7 +6249,7 @@ impl<'a> Parser<'a> {
if self.token == token::Comma { if self.token == token::Comma {
seen_comma = true; seen_comma = true;
} }
match self.token { match self.token.kind {
token::Comma => { token::Comma => {
self.bump(); self.bump();
} }
@ -7011,7 +7014,7 @@ impl<'a> Parser<'a> {
/// Parses a string as an ABI spec on an extern type or module. Consumes /// Parses a string as an ABI spec on an extern type or module. Consumes
/// the `extern` keyword, if one is found. /// the `extern` keyword, if one is found.
fn parse_opt_abi(&mut self) -> PResult<'a, Option<Abi>> { fn parse_opt_abi(&mut self) -> PResult<'a, Option<Abi>> {
match self.token { match self.token.kind {
token::Literal(token::Lit { kind: token::Str, symbol, suffix }) | token::Literal(token::Lit { kind: token::Str, symbol, suffix }) |
token::Literal(token::Lit { kind: token::StrRaw(..), symbol, suffix }) => { token::Literal(token::Lit { kind: token::StrRaw(..), symbol, suffix }) => {
let sp = self.span; let sp = self.span;
@ -7046,7 +7049,7 @@ impl<'a> Parser<'a> {
if token.is_keyword(kw::Move) { if token.is_keyword(kw::Move) {
return true; return true;
} }
match *token { match token.kind {
token::BinOp(token::Or) | token::OrOr => true, token::BinOp(token::Or) | token::OrOr => true,
_ => false, _ => false,
} }
@ -7818,7 +7821,7 @@ impl<'a> Parser<'a> {
} }
pub fn parse_optional_str(&mut self) -> Option<(Symbol, ast::StrStyle, Option<ast::Name>)> { pub fn parse_optional_str(&mut self) -> Option<(Symbol, ast::StrStyle, Option<ast::Name>)> {
let ret = match self.token { let ret = match self.token.kind {
token::Literal(token::Lit { kind: token::Str, symbol, suffix }) => token::Literal(token::Lit { kind: token::Str, symbol, suffix }) =>
(symbol, ast::StrStyle::Cooked, suffix), (symbol, ast::StrStyle::Cooked, suffix),
token::Literal(token::Lit { kind: token::StrRaw(n), symbol, suffix }) => token::Literal(token::Lit { kind: token::StrRaw(n), symbol, suffix }) =>

2
src/libsyntax_ext/asm.rs
View file

@ -260,7 +260,7 @@ fn parse_inline_asm<'a>(
loop { loop {
// MOD_SEP is a double colon '::' without space in between. // MOD_SEP is a double colon '::' without space in between.
// When encountered, the state must be advanced twice. // When encountered, the state must be advanced twice.
match (&p.token, state.next(), state.next().next()) { match (&p.token.kind, state.next(), state.next().next()) {
(&token::Colon, StateNone, _) | (&token::Colon, StateNone, _) |
(&token::ModSep, _, StateNone) => { (&token::ModSep, _, StateNone) => {
p.bump(); p.bump();

2
src/libsyntax_ext/assert.rs
View file

@ -103,7 +103,7 @@ fn parse_assert<'a>(
// //
// Parse this as an actual message, and suggest inserting a comma. Eventually, this should be // Parse this as an actual message, and suggest inserting a comma. Eventually, this should be
// turned into an error. // turned into an error.
let custom_message = if let token::Literal(token::Lit { kind: token::Str, .. }) = parser.token { let custom_message = if let token::Literal(token::Lit { kind: token::Str, .. }) = parser.token.kind {
let mut err = cx.struct_span_warn(parser.span, "unexpected string literal"); let mut err = cx.struct_span_warn(parser.span, "unexpected string literal");
let comma_span = cx.source_map().next_point(parser.prev_span); let comma_span = cx.source_map().next_point(parser.prev_span);
err.span_suggestion_short( err.span_suggestion_short(

2
src/libsyntax_ext/format.rs
View file

@ -149,7 +149,7 @@ fn parse_args<'a>(
} // accept trailing commas } // accept trailing commas
if named || (p.token.is_ident() && p.look_ahead(1, |t| *t == token::Eq)) { if named || (p.token.is_ident() && p.look_ahead(1, |t| *t == token::Eq)) {
named = true; named = true;
let ident = if let token::Ident(i, _) = p.token { let ident = if let token::Ident(i, _) = p.token.kind {
p.bump(); p.bump();
i i
} else { } else {