bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Rollup merge of #103919 - nnethercote:unescaping-cleanups, r=matklad

Browse source 
Unescaping cleanups

Some code improvements, and some error message improvements.

Best reviewed one commit at a time.

r? ````@matklad````
This commit is contained in:
Dylan DPC 2022-11-09 19:21:22 +05:30 • committed by GitHub
commit 4b50fb3745
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
19 changed files with 190 additions and 253 deletions
Download patch file Download diff file Expand all files Collapse all files

107
compiler/rustc_parse/src/lexer/mod.rs
View file

@ -353,55 +353,55 @@ impl<'a> StringReader<'a> {
fn cook_lexer_literal(
&self,
start: BytePos,
suffix_start: BytePos,
end: BytePos,
kind: rustc_lexer::LiteralKind,
) -> (token::LitKind, Symbol) {
// prefix means `"` or `br"` or `r###"`, ...
let (lit_kind, mode, prefix_len, postfix_len) = match kind {
match kind {
rustc_lexer::LiteralKind::Char { terminated } => {
if !terminated {
self.sess.span_diagnostic.span_fatal_with_code(
self.mk_sp(start, suffix_start),
self.mk_sp(start, end),
"unterminated character literal",
error_code!(E0762),
)
}
(token::Char, Mode::Char, 1, 1) // ' '
self.cook_quoted(token::Char, Mode::Char, start, end, 1, 1) // ' '
}
rustc_lexer::LiteralKind::Byte { terminated } => {
if !terminated {
self.sess.span_diagnostic.span_fatal_with_code(
self.mk_sp(start + BytePos(1), suffix_start),
self.mk_sp(start + BytePos(1), end),
"unterminated byte constant",
error_code!(E0763),
)
}
(token::Byte, Mode::Byte, 2, 1) // b' '
self.cook_quoted(token::Byte, Mode::Byte, start, end, 2, 1) // b' '
}
rustc_lexer::LiteralKind::Str { terminated } => {
if !terminated {
self.sess.span_diagnostic.span_fatal_with_code(
self.mk_sp(start, suffix_start),
self.mk_sp(start, end),
"unterminated double quote string",
error_code!(E0765),
)
}
(token::Str, Mode::Str, 1, 1) // " "
self.cook_quoted(token::Str, Mode::Str, start, end, 1, 1) // " "
}
rustc_lexer::LiteralKind::ByteStr { terminated } => {
if !terminated {
self.sess.span_diagnostic.span_fatal_with_code(
self.mk_sp(start + BytePos(1), suffix_start),
self.mk_sp(start + BytePos(1), end),
"unterminated double quote byte string",
error_code!(E0766),
)
}
(token::ByteStr, Mode::ByteStr, 2, 1) // b" "
self.cook_quoted(token::ByteStr, Mode::ByteStr, start, end, 2, 1) // b" "
}
rustc_lexer::LiteralKind::RawStr { n_hashes } => {
if let Some(n_hashes) = n_hashes {
let n = u32::from(n_hashes);
(token::StrRaw(n_hashes), Mode::RawStr, 2 + n, 1 + n) // r##" "##
let kind = token::StrRaw(n_hashes);
self.cook_quoted(kind, Mode::RawStr, start, end, 2 + n, 1 + n) // r##" "##
} else {
self.report_raw_str_error(start, 1);
}
@ -409,56 +409,59 @@ impl<'a> StringReader<'a> {
rustc_lexer::LiteralKind::RawByteStr { n_hashes } => {
if let Some(n_hashes) = n_hashes {
let n = u32::from(n_hashes);
(token::ByteStrRaw(n_hashes), Mode::RawByteStr, 3 + n, 1 + n) // br##" "##
let kind = token::ByteStrRaw(n_hashes);
self.cook_quoted(kind, Mode::RawByteStr, start, end, 3 + n, 1 + n) // br##" "##
} else {
self.report_raw_str_error(start, 2);
}
}
rustc_lexer::LiteralKind::Int { base, empty_int } => {
return if empty_int {
if empty_int {
self.sess
.span_diagnostic
.struct_span_err_with_code(
self.mk_sp(start, suffix_start),
self.mk_sp(start, end),
"no valid digits found for number",
error_code!(E0768),
)
.emit();
(token::Integer, sym::integer(0))
} else {
self.validate_int_literal(base, start, suffix_start);
(token::Integer, self.symbol_from_to(start, suffix_start))
};
if matches!(base, Base::Binary | Base::Octal) {
let base = base as u32;
let s = self.str_from_to(start + BytePos(2), end);
for (idx, c) in s.char_indices() {
if c != '_' && c.to_digit(base).is_none() {
self.err_span_(
start + BytePos::from_usize(2 + idx),
start + BytePos::from_usize(2 + idx + c.len_utf8()),
&format!("invalid digit for a base {} literal", base),
);
}
}
}
(token::Integer, self.symbol_from_to(start, end))
}
}
rustc_lexer::LiteralKind::Float { base, empty_exponent } => {
if empty_exponent {
self.err_span_(start, self.pos, "expected at least one digit in exponent");
}
match base {
Base::Hexadecimal => self.err_span_(
start,
suffix_start,
"hexadecimal float literal is not supported",
),
Base::Hexadecimal => {
self.err_span_(start, end, "hexadecimal float literal is not supported")
}
Base::Octal => {
self.err_span_(start, suffix_start, "octal float literal is not supported")
self.err_span_(start, end, "octal float literal is not supported")
}
Base::Binary => {
self.err_span_(start, suffix_start, "binary float literal is not supported")
self.err_span_(start, end, "binary float literal is not supported")
}
_ => (),
_ => {}
}
let id = self.symbol_from_to(start, suffix_start);
return (token::Float, id);
(token::Float, self.symbol_from_to(start, end))
}
};
let content_start = start + BytePos(prefix_len);
let content_end = suffix_start - BytePos(postfix_len);
let id = self.symbol_from_to(content_start, content_end);
self.validate_literal_escape(mode, content_start, content_end, prefix_len, postfix_len);
(lit_kind, id)
}
}
#[inline]
@ -649,20 +652,22 @@ impl<'a> StringReader<'a> {
)
}
fn validate_literal_escape(
fn cook_quoted(
&self,
kind: token::LitKind,
mode: Mode,
content_start: BytePos,
content_end: BytePos,
start: BytePos,
end: BytePos,
prefix_len: u32,
postfix_len: u32,
) {
) -> (token::LitKind, Symbol) {
let content_start = start + BytePos(prefix_len);
let content_end = end - BytePos(postfix_len);
let lit_content = self.str_from_to(content_start, content_end);
unescape::unescape_literal(lit_content, mode, &mut |range, result| {
// Here we only check for errors. The actual unescaping is done later.
if let Err(err) = result {
let span_with_quotes = self
.mk_sp(content_start - BytePos(prefix_len), content_end + BytePos(postfix_len));
let span_with_quotes = self.mk_sp(start, end);
let (start, end) = (range.start as u32, range.end as u32);
let lo = content_start + BytePos(start);
let hi = lo + BytePos(end - start);
@ -678,23 +683,7 @@ impl<'a> StringReader<'a> {
);
}
});
}
fn validate_int_literal(&self, base: Base, content_start: BytePos, content_end: BytePos) {
let base = match base {
Base::Binary => 2,
Base::Octal => 8,
_ => return,
};
let s = self.str_from_to(content_start + BytePos(2), content_end);
for (idx, c) in s.char_indices() {
let idx = idx as u32;
if c != '_' && c.to_digit(base).is_none() {
let lo = content_start + BytePos(2 + idx);
let hi = content_start + BytePos(2 + idx + c.len_utf8() as u32);
self.err_span_(lo, hi, &format!("invalid digit for a base {} literal", base));
}
}
(kind, Symbol::intern(lit_content))
}
}

42
compiler/rustc_parse/src/lexer/unescape_error_reporting.rs
View file

@ -108,7 +108,7 @@ pub(crate) fn emit_unescape_error(
}
if !has_help {
let (prefix, msg) = if mode.is_bytes() {
let (prefix, msg) = if mode.is_byte() {
("b", "if you meant to write a byte string literal, use double quotes")
} else {
("", "if you meant to write a `str` literal, use double quotes")
@ -142,7 +142,7 @@ pub(crate) fn emit_unescape_error(
EscapeError::EscapeOnlyChar => {
let (c, char_span) = last_char();
let msg = if mode.is_bytes() {
let msg = if mode.is_byte() {
"byte constant must be escaped"
} else {
"character constant must be escaped"
@ -182,11 +182,11 @@ pub(crate) fn emit_unescape_error(
let (c, span) = last_char();
let label =
if mode.is_bytes() { "unknown byte escape" } else { "unknown character escape" };
if mode.is_byte() { "unknown byte escape" } else { "unknown character escape" };
let ec = escaped_char(c);
let mut diag = handler.struct_span_err(span, &format!("{}: `{}`", label, ec));
diag.span_label(span, label);
if c == '{' || c == '}' && !mode.is_bytes() {
if c == '{' || c == '}' && !mode.is_byte() {
diag.help(
"if used in a formatting string, curly braces are escaped with `{{` and `}}`",
);
@ -196,7 +196,7 @@ pub(crate) fn emit_unescape_error(
version control settings",
);
} else {
if !mode.is_bytes() {
if !mode.is_byte() {
diag.span_suggestion(
span_with_quotes,
"if you meant to write a literal backslash (perhaps escaping in a regular expression), consider a raw string literal",
@ -231,16 +231,23 @@ pub(crate) fn emit_unescape_error(
.emit();
}
EscapeError::NonAsciiCharInByte => {
assert!(mode.is_bytes());
let (c, span) = last_char();
let mut err = handler.struct_span_err(span, "non-ASCII character in byte constant");
let desc = match mode {
Mode::Byte => "byte literal",
Mode::ByteStr => "byte string literal",
Mode::RawByteStr => "raw byte string literal",
_ => panic!("non-is_byte literal paired with NonAsciiCharInByte"),
};
let mut err = handler.struct_span_err(span, format!("non-ASCII character in {}", desc));
let postfix = if unicode_width::UnicodeWidthChar::width(c).unwrap_or(1) == 0 {
format!(" but is {:?}", c)
} else {
String::new()
};
err.span_label(span, &format!("byte constant must be ASCII{}", postfix));
if (c as u32) <= 0xFF {
err.span_label(span, &format!("must be ASCII{}", postfix));
// Note: the \\xHH suggestions are not given for raw byte string
// literals, because they are araw and so cannot use any escapes.
if (c as u32) <= 0xFF && mode != Mode::RawByteStr {
err.span_suggestion(
span,
&format!(
@ -250,9 +257,9 @@ pub(crate) fn emit_unescape_error(
format!("\\x{:X}", c as u32),
Applicability::MaybeIncorrect,
);
} else if matches!(mode, Mode::Byte) {
} else if mode == Mode::Byte {
err.span_label(span, "this multibyte character does not fit into a single byte");
} else if matches!(mode, Mode::ByteStr) {
} else if mode != Mode::RawByteStr {
let mut utf8 = String::new();
utf8.push(c);
err.span_suggestion(
@ -270,19 +277,6 @@ pub(crate) fn emit_unescape_error(
}
err.emit();
}
EscapeError::NonAsciiCharInByteString => {
assert!(mode.is_bytes());
let (c, span) = last_char();
let postfix = if unicode_width::UnicodeWidthChar::width(c).unwrap_or(1) == 0 {
format!(" but is {:?}", c)
} else {
String::new()
};
handler
.struct_span_err(span, "raw byte string must be ASCII")
.span_label(span, &format!("must be ASCII{}", postfix))
.emit();
}
EscapeError::OutOfRangeHexEscape => {
handler
.struct_span_err(span, "out of range hex escape")