bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

reimplement C string literals

Browse source
This commit is contained in:
Deadbeef 2023-07-16 18:59:05 +00:00
parent cec34a43b1
commit df9bd80d74
10 changed files with 48 additions and 89 deletions
Download patch file Download diff file Expand all files Collapse all files

26
compiler/rustc_parse/src/lexer/mod.rs
View file

@ -9,8 +9,8 @@ use rustc_ast::tokenstream::TokenStream;
use rustc_ast::util::unicode::contains_text_flow_control_chars;
use rustc_errors::{error_code, Applicability, Diagnostic, DiagnosticBuilder, StashKey};
use rustc_lexer::unescape::{self, EscapeError, Mode};
use rustc_lexer::Cursor;
use rustc_lexer::{Base, DocStyle, RawStrError};
use rustc_lexer::{Cursor, LiteralKind};
use rustc_session::lint::builtin::{
RUST_2021_PREFIXES_INCOMPATIBLE_SYNTAX, TEXT_DIRECTION_CODEPOINT_IN_COMMENT,
};
@ -118,6 +118,7 @@ impl<'a> StringReader<'a> {
let mut swallow_next_invalid = 0;
// Skip trivial (whitespace & comments) tokens
loop {
let str_before = self.cursor.as_str();
let token = self.cursor.advance_token();
let start = self.pos;
self.pos = self.pos + BytePos(token.len);
@ -203,6 +204,29 @@ impl<'a> StringReader<'a> {
.push(span);
token::Ident(sym, false)
}
// split up (raw) c string literals to an ident and a string literal when edition < 2021.
rustc_lexer::TokenKind::Literal {
kind: kind @ (LiteralKind::CStr { .. } | LiteralKind::RawCStr { .. }),
suffix_start: _,
} if !self.mk_sp(start, self.pos).edition().at_least_rust_2021() => {
let prefix_len = match kind {
LiteralKind::CStr { .. } => 1,
LiteralKind::RawCStr { .. } => 2,
_ => unreachable!(),
};
// reset the state so that only the prefix ("c" or "cr")
// was consumed.
let lit_start = start + BytePos(prefix_len);
self.pos = lit_start;
self.cursor = Cursor::new(&str_before[prefix_len as usize..]);
self.report_unknown_prefix(start);
let sym = nfc_normalize(self.str_from(start));
let prefix_span = self.mk_sp(start, lit_start);
self.sess.symbol_gallery.insert(sym, prefix_span);
return (Token::new(token::Ident(sym, false), prefix_span), preceded_by_whitespace);
}
rustc_lexer::TokenKind::Literal { kind, suffix_start } => {
let suffix_start = start + BytePos(suffix_start);
let (kind, symbol) = self.cook_lexer_literal(start, suffix_start, kind);