bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Rollup merge of #130551 - nnethercote:fix-break-last-token, r=petrochenkov

Browse source 
Fix `break_last_token`.

It currently doesn't handle the three-char tokens `>>=` and `<<=` correctly. These can be broken twice, resulting in three individual tokens. This is a latent bug that currently doesn't cause any problems, but does cause problems for #124141, because that PR increases the usage of lazy token streams.

r? `@petrochenkov`
This commit is contained in:
Jubilee 2024-09-23 07:54:44 -07:00 • committed by GitHub
commit 515bdcda01
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 91 additions and 64 deletions
Download patch file Download diff file Expand all files Collapse all files

36
compiler/rustc_parse/src/parser/attr_wrapper.rs
View file

@ -108,7 +108,7 @@ struct LazyAttrTokenStreamImpl {
start_token: (Token, Spacing),
cursor_snapshot: TokenCursor,
num_calls: u32,
break_last_token: bool,
break_last_token: u32,
node_replacements: Box<[NodeReplacement]>,
}
@ -339,17 +339,20 @@ impl<'a> Parser<'a> {
let parser_replacements_end = self.capture_state.parser_replacements.len();
assert!(
!(self.break_last_token && matches!(capture_trailing, Trailing::Yes)),
"Cannot set break_last_token and have trailing token"
!(self.break_last_token > 0 && matches!(capture_trailing, Trailing::Yes)),
"Cannot have break_last_token > 0 and have trailing token"
);
assert!(self.break_last_token <= 2, "cannot break token more than twice");
let end_pos = self.num_bump_calls
+ capture_trailing as u32
// If we 'broke' the last token (e.g. breaking a '>>' token to two '>' tokens), then
// extend the range of captured tokens to include it, since the parser was not actually
// bumped past it. When the `LazyAttrTokenStream` gets converted into an
// `AttrTokenStream`, we will create the proper token.
+ self.break_last_token as u32;
// If we "broke" the last token (e.g. breaking a `>>` token once into `>` + `>`, or
// breaking a `>>=` token twice into `>` + `>` + `=`), then extend the range of
// captured tokens to include it, because the parser was not actually bumped past it.
// (Even if we broke twice, it was still just one token originally, hence the `1`.)
// When the `LazyAttrTokenStream` gets converted into an `AttrTokenStream`, we will
// rebreak that final token once or twice.
+ if self.break_last_token == 0 { 0 } else { 1 };
let num_calls = end_pos - collect_pos.start_pos;
@ -425,7 +428,7 @@ impl<'a> Parser<'a> {
// for the `#[cfg]` and/or `#[cfg_attr]` attrs. This allows us to run
// eager cfg-expansion on the captured token stream.
if definite_capture_mode {
assert!(!self.break_last_token, "Should not have unglued last token with cfg attr");
assert!(self.break_last_token == 0, "Should not have unglued last token with cfg attr");
// What is the status here when parsing the example code at the top of this method?
//
@ -471,7 +474,7 @@ impl<'a> Parser<'a> {
/// close delims.
fn make_attr_token_stream(
iter: impl Iterator<Item = FlatToken>,
break_last_token: bool,
break_last_token: u32,
) -> AttrTokenStream {
#[derive(Debug)]
struct FrameData {
@ -513,18 +516,17 @@ fn make_attr_token_stream(
}
}
if break_last_token {
if break_last_token > 0 {
let last_token = stack_top.inner.pop().unwrap();
if let AttrTokenTree::Token(last_token, spacing) = last_token {
let unglued_first = last_token.kind.break_two_token_op().unwrap().0;
let (unglued, _) = last_token.kind.break_two_token_op(break_last_token).unwrap();
// An 'unglued' token is always two ASCII characters
// Tokens are always ASCII chars, so we can use byte arithmetic here.
let mut first_span = last_token.span.shrink_to_lo();
first_span = first_span.with_hi(first_span.lo() + rustc_span::BytePos(1));
first_span =
first_span.with_hi(first_span.lo() + rustc_span::BytePos(break_last_token));
stack_top
.inner
.push(AttrTokenTree::Token(Token::new(unglued_first, first_span), spacing));
stack_top.inner.push(AttrTokenTree::Token(Token::new(unglued, first_span), spacing));
} else {
panic!("Unexpected last token {last_token:?}")
}

39
compiler/rustc_parse/src/parser/mod.rs
View file

@ -146,21 +146,25 @@ pub struct Parser<'a> {
token_cursor: TokenCursor,
// The number of calls to `bump`, i.e. the position in the token stream.
num_bump_calls: u32,
// During parsing we may sometimes need to 'unglue' a glued token into two
// component tokens (e.g. '>>' into '>' and '>), so the parser can consume
// them one at a time. This process bypasses the normal capturing mechanism
// (e.g. `num_bump_calls` will not be incremented), since the 'unglued'
// tokens due not exist in the original `TokenStream`.
// During parsing we may sometimes need to "unglue" a glued token into two
// or three component tokens (e.g. `>>` into `>` and `>`, or `>>=` into `>`
// and `>` and `=`), so the parser can consume them one at a time. This
// process bypasses the normal capturing mechanism (e.g. `num_bump_calls`
// will not be incremented), since the "unglued" tokens due not exist in
// the original `TokenStream`.
//
// If we end up consuming both unglued tokens, this is not an issue. We'll
// end up capturing the single 'glued' token.
// If we end up consuming all the component tokens, this is not an issue,
// because we'll end up capturing the single "glued" token.
//
// However, sometimes we may want to capture just the first 'unglued'
// However, sometimes we may want to capture not all of the original
// token. For example, capturing the `Vec<u8>` in `Option<Vec<u8>>`
// requires us to unglue the trailing `>>` token. The `break_last_token`
// field is used to track this token. It gets appended to the captured
// field is used to track these tokens. They get appended to the captured
// stream when we evaluate a `LazyAttrTokenStream`.
break_last_token: bool,
//
// This value is always 0, 1, or 2. It can only reach 2 when splitting
// `>>=` or `<<=`.
break_last_token: u32,
/// This field is used to keep track of how many left angle brackets we have seen. This is
/// required in order to detect extra leading left angle brackets (`<` characters) and error
/// appropriately.
@ -453,7 +457,7 @@ impl<'a> Parser<'a> {
expected_tokens: Vec::new(),
token_cursor: TokenCursor { tree_cursor: stream.into_trees(), stack: Vec::new() },
num_bump_calls: 0,
break_last_token: false,
break_last_token: 0,
unmatched_angle_bracket_count: 0,
angle_bracket_nesting: 0,
last_unexpected_token_span: None,
@ -773,7 +777,7 @@ impl<'a> Parser<'a> {
self.bump();
return true;
}
match self.token.kind.break_two_token_op() {
match self.token.kind.break_two_token_op(1) {
Some((first, second)) if first == expected => {
let first_span = self.psess.source_map().start_point(self.token.span);
let second_span = self.token.span.with_lo(first_span.hi());
@ -783,8 +787,8 @@ impl<'a> Parser<'a> {
//
// If we consume any additional tokens, then this token
// is not needed (we'll capture the entire 'glued' token),
// and `bump` will set this field to `None`
self.break_last_token = true;
// and `bump` will set this field to 0.
self.break_last_token += 1;
// Use the spacing of the glued token as the spacing of the
// unglued second token.
self.bump_with((Token::new(second, second_span), self.token_spacing));
@ -1148,10 +1152,9 @@ impl<'a> Parser<'a> {
// than `.0`/`.1` access.
let mut next = self.token_cursor.inlined_next();
self.num_bump_calls += 1;
// We've retrieved an token from the underlying
// cursor, so we no longer need to worry about
// an unglued token. See `break_and_eat` for more details
self.break_last_token = false;
// We got a token from the underlying cursor and no longer need to
// worry about an unglued token. See `break_and_eat` for more details.
self.break_last_token = 0;
if next.0.span.is_dummy() {
// Tweak the location for better diagnostics, but keep syntactic context intact.
let fallback_span = self.token.span;