mv compiler to compiler/

compiler/rustc_lexer/src/cursor.rs

@@ -0,0 +1,84 @@
+use std::str::Chars;
+
+/// Peekable iterator over a char sequence.
+///
+/// Next characters can be peeked via `nth_char` method,
+/// and position can be shifted forward via `bump` method.
+pub(crate) struct Cursor<'a> {
+    initial_len: usize,
+    chars: Chars<'a>,
+    #[cfg(debug_assertions)]
+    prev: char,
+}
+
+pub(crate) const EOF_CHAR: char = '\0';
+
+impl<'a> Cursor<'a> {
+    pub(crate) fn new(input: &'a str) -> Cursor<'a> {
+        Cursor {
+            initial_len: input.len(),
+            chars: input.chars(),
+            #[cfg(debug_assertions)]
+            prev: EOF_CHAR,
+        }
+    }
+
+    /// Returns the last eaten symbol (or `'\0'` in release builds).
+    /// (For debug assertions only.)
+    pub(crate) fn prev(&self) -> char {
+        #[cfg(debug_assertions)]
+        {
+            self.prev
+        }
+
+        #[cfg(not(debug_assertions))]
+        {
+            '\0'
+        }
+    }
+
+    /// Returns nth character relative to the current cursor position.
+    /// If requested position doesn't exist, `EOF_CHAR` is returned.
+    /// However, getting `EOF_CHAR` doesn't always mean actual end of file,
+    /// it should be checked with `is_eof` method.
+    fn nth_char(&self, n: usize) -> char {
+        self.chars().nth(n).unwrap_or(EOF_CHAR)
+    }
+
+    /// Peeks the next symbol from the input stream without consuming it.
+    pub(crate) fn first(&self) -> char {
+        self.nth_char(0)
+    }
+
+    /// Peeks the second symbol from the input stream without consuming it.
+    pub(crate) fn second(&self) -> char {
+        self.nth_char(1)
+    }
+
+    /// Checks if there is nothing more to consume.
+    pub(crate) fn is_eof(&self) -> bool {
+        self.chars.as_str().is_empty()
+    }
+
+    /// Returns amount of already consumed symbols.
+    pub(crate) fn len_consumed(&self) -> usize {
+        self.initial_len - self.chars.as_str().len()
+    }
+
+    /// Returns a `Chars` iterator over the remaining characters.
+    fn chars(&self) -> Chars<'a> {
+        self.chars.clone()
+    }
+
+    /// Moves to the next character.
+    pub(crate) fn bump(&mut self) -> Option<char> {
+        let c = self.chars.next()?;
+
+        #[cfg(debug_assertions)]
+        {
+            self.prev = c;
+        }
+
+        Some(c)
+    }
+}

compiler/rustc_lexer/src/lib.rs

@@ -0,0 +1,819 @@
+//! Low-level Rust lexer.
+//!
+//! The idea with `librustc_lexer` is to make a reusable library,
+//! by separating out pure lexing and rustc-specific concerns, like spans,
+//! error reporting an interning.  So, rustc_lexer operates directly on `&str`,
+//! produces simple tokens which are a pair of type-tag and a bit of original text,
+//! and does not report errors, instead storing them as flags on the token.
+//!
+//! Tokens produced by this lexer are not yet ready for parsing the Rust syntax.
+//! For that see [`librustc_parse::lexer`], which converts this basic token stream
+//! into wide tokens used by actual parser.
+//!
+//! The purpose of this crate is to convert raw sources into a labeled sequence
+//! of well-known token types, so building an actual Rust token stream will
+//! be easier.
+//!
+//! The main entity of this crate is the [`TokenKind`] enum which represents common
+//! lexeme types.
+//!
+//! [`librustc_parse::lexer`]: ../rustc_parse/lexer/index.html
+// We want to be able to build this crate with a stable compiler, so no
+// `#![feature]` attributes should be added.
+
+mod cursor;
+pub mod unescape;
+
+#[cfg(test)]
+mod tests;
+
+use self::LiteralKind::*;
+use self::TokenKind::*;
+use crate::cursor::{Cursor, EOF_CHAR};
+use std::convert::TryFrom;
+
+/// Parsed token.
+/// It doesn't contain information about data that has been parsed,
+/// only the type of the token and its size.
+#[derive(Debug)]
+pub struct Token {
+    pub kind: TokenKind,
+    pub len: usize,
+}
+
+impl Token {
+    fn new(kind: TokenKind, len: usize) -> Token {
+        Token { kind, len }
+    }
+}
+
+/// Enum representing common lexeme types.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
+pub enum TokenKind {
+    // Multi-char tokens:
+    /// "// comment"
+    LineComment { doc_style: Option<DocStyle> },
+    /// `/* block comment */`
+    ///
+    /// Block comments can be recursive, so the sequence like `/* /* */`
+    /// will not be considered terminated and will result in a parsing error.
+    BlockComment { doc_style: Option<DocStyle>, terminated: bool },
+    /// Any whitespace characters sequence.
+    Whitespace,
+    /// "ident" or "continue"
+    /// At this step keywords are also considered identifiers.
+    Ident,
+    /// "r#ident"
+    RawIdent,
+    /// "12_u8", "1.0e-40", "b"123"". See `LiteralKind` for more details.
+    Literal { kind: LiteralKind, suffix_start: usize },
+    /// "'a"
+    Lifetime { starts_with_number: bool },
+
+    // One-char tokens:
+    /// ";"
+    Semi,
+    /// ","
+    Comma,
+    /// "."
+    Dot,
+    /// "("
+    OpenParen,
+    /// ")"
+    CloseParen,
+    /// "{"
+    OpenBrace,
+    /// "}"
+    CloseBrace,
+    /// "["
+    OpenBracket,
+    /// "]"
+    CloseBracket,
+    /// "@"
+    At,
+    /// "#"
+    Pound,
+    /// "~"
+    Tilde,
+    /// "?"
+    Question,
+    /// ":"
+    Colon,
+    /// "$"
+    Dollar,
+    /// "="
+    Eq,
+    /// "!"
+    Bang,
+    /// "<"
+    Lt,
+    /// ">"
+    Gt,
+    /// "-"
+    Minus,
+    /// "&"
+    And,
+    /// "|"
+    Or,
+    /// "+"
+    Plus,
+    /// "*"
+    Star,
+    /// "/"
+    Slash,
+    /// "^"
+    Caret,
+    /// "%"
+    Percent,
+
+    /// Unknown token, not expected by the lexer, e.g. "№"
+    Unknown,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
+pub enum DocStyle {
+    Outer,
+    Inner,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
+pub enum LiteralKind {
+    /// "12_u8", "0o100", "0b120i99"
+    Int { base: Base, empty_int: bool },
+    /// "12.34f32", "0b100.100"
+    Float { base: Base, empty_exponent: bool },
+    /// "'a'", "'\\'", "'''", "';"
+    Char { terminated: bool },
+    /// "b'a'", "b'\\'", "b'''", "b';"
+    Byte { terminated: bool },
+    /// ""abc"", ""abc"
+    Str { terminated: bool },
+    /// "b"abc"", "b"abc"
+    ByteStr { terminated: bool },
+    /// "r"abc"", "r#"abc"#", "r####"ab"###"c"####", "r#"a"
+    RawStr { n_hashes: u16, err: Option<RawStrError> },
+    /// "br"abc"", "br#"abc"#", "br####"ab"###"c"####", "br#"a"
+    RawByteStr { n_hashes: u16, err: Option<RawStrError> },
+}
+
+/// Error produced validating a raw string. Represents cases like:
+/// - `r##~"abcde"##`: `InvalidStarter`
+/// - `r###"abcde"##`: `NoTerminator { expected: 3, found: 2, possible_terminator_offset: Some(11)`
+/// - Too many `#`s (>65535): `TooManyDelimiters`
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
+pub enum RawStrError {
+    /// Non `#` characters exist between `r` and `"` eg. `r#~"..`
+    InvalidStarter { bad_char: char },
+    /// The string was never terminated. `possible_terminator_offset` is the number of characters after `r` or `br` where they
+    /// may have intended to terminate it.
+    NoTerminator { expected: usize, found: usize, possible_terminator_offset: Option<usize> },
+    /// More than 65535 `#`s exist.
+    TooManyDelimiters { found: usize },
+}
+
+/// Base of numeric literal encoding according to its prefix.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
+pub enum Base {
+    /// Literal starts with "0b".
+    Binary,
+    /// Literal starts with "0o".
+    Octal,
+    /// Literal starts with "0x".
+    Hexadecimal,
+    /// Literal doesn't contain a prefix.
+    Decimal,
+}
+
+/// `rustc` allows files to have a shebang, e.g. "#!/usr/bin/rustrun",
+/// but shebang isn't a part of rust syntax.
+pub fn strip_shebang(input: &str) -> Option<usize> {
+    // Shebang must start with `#!` literally, without any preceding whitespace.
+    // For simplicity we consider any line starting with `#!` a shebang,
+    // regardless of restrictions put on shebangs by specific platforms.
+    if let Some(input_tail) = input.strip_prefix("#!") {
+        // Ok, this is a shebang but if the next non-whitespace token is `[` or maybe
+        // a doc comment (due to `TokenKind::(Line,Block)Comment` ambiguity at lexer level),
+        // then it may be valid Rust code, so consider it Rust code.
+        let next_non_whitespace_token = tokenize(input_tail).map(|tok| tok.kind).find(|tok|
+            !matches!(tok, TokenKind::Whitespace | TokenKind::LineComment { .. } | TokenKind::BlockComment { .. })
+        );
+        if next_non_whitespace_token != Some(TokenKind::OpenBracket) {
+            // No other choice than to consider this a shebang.
+            return Some(2 + input_tail.lines().next().unwrap_or_default().len());
+        }
+    }
+    None
+}
+
+/// Parses the first token from the provided input string.
+pub fn first_token(input: &str) -> Token {
+    debug_assert!(!input.is_empty());
+    Cursor::new(input).advance_token()
+}
+
+/// Creates an iterator that produces tokens from the input string.
+pub fn tokenize(mut input: &str) -> impl Iterator<Item = Token> + '_ {
+    std::iter::from_fn(move || {
+        if input.is_empty() {
+            return None;
+        }
+        let token = first_token(input);
+        input = &input[token.len..];
+        Some(token)
+    })
+}
+
+/// True if `c` is considered a whitespace according to Rust language definition.
+/// See [Rust language reference](https://doc.rust-lang.org/reference/whitespace.html)
+/// for definitions of these classes.
+pub fn is_whitespace(c: char) -> bool {
+    // This is Pattern_White_Space.
+    //
+    // Note that this set is stable (ie, it doesn't change with different
+    // Unicode versions), so it's ok to just hard-code the values.
+
+    match c {
+        // Usual ASCII suspects
+        | '\u{0009}' // \t
+        | '\u{000A}' // \n
+        | '\u{000B}' // vertical tab
+        | '\u{000C}' // form feed
+        | '\u{000D}' // \r
+        | '\u{0020}' // space
+
+        // NEXT LINE from latin1
+        | '\u{0085}'
+
+        // Bidi markers
+        | '\u{200E}' // LEFT-TO-RIGHT MARK
+        | '\u{200F}' // RIGHT-TO-LEFT MARK
+
+        // Dedicated whitespace characters from Unicode
+        | '\u{2028}' // LINE SEPARATOR
+        | '\u{2029}' // PARAGRAPH SEPARATOR
+        => true,
+        _ => false,
+    }
+}
+
+/// True if `c` is valid as a first character of an identifier.
+/// See [Rust language reference](https://doc.rust-lang.org/reference/identifiers.html) for
+/// a formal definition of valid identifier name.
+pub fn is_id_start(c: char) -> bool {
+    // This is XID_Start OR '_' (which formally is not a XID_Start).
+    // We also add fast-path for ascii idents
+    ('a' <= c && c <= 'z')
+        || ('A' <= c && c <= 'Z')
+        || c == '_'
+        || (c > '\x7f' && unicode_xid::UnicodeXID::is_xid_start(c))
+}
+
+/// True if `c` is valid as a non-first character of an identifier.
+/// See [Rust language reference](https://doc.rust-lang.org/reference/identifiers.html) for
+/// a formal definition of valid identifier name.
+pub fn is_id_continue(c: char) -> bool {
+    // This is exactly XID_Continue.
+    // We also add fast-path for ascii idents
+    ('a' <= c && c <= 'z')
+        || ('A' <= c && c <= 'Z')
+        || ('0' <= c && c <= '9')
+        || c == '_'
+        || (c > '\x7f' && unicode_xid::UnicodeXID::is_xid_continue(c))
+}
+
+/// The passed string is lexically an identifier.
+pub fn is_ident(string: &str) -> bool {
+    let mut chars = string.chars();
+    if let Some(start) = chars.next() {
+        is_id_start(start) && chars.all(is_id_continue)
+    } else {
+        false
+    }
+}
+
+impl Cursor<'_> {
+    /// Parses a token from the input string.
+    fn advance_token(&mut self) -> Token {
+        let first_char = self.bump().unwrap();
+        let token_kind = match first_char {
+            // Slash, comment or block comment.
+            '/' => match self.first() {
+                '/' => self.line_comment(),
+                '*' => self.block_comment(),
+                _ => Slash,
+            },
+
+            // Whitespace sequence.
+            c if is_whitespace(c) => self.whitespace(),
+
+            // Raw identifier, raw string literal or identifier.
+            'r' => match (self.first(), self.second()) {
+                ('#', c1) if is_id_start(c1) => self.raw_ident(),
+                ('#', _) | ('"', _) => {
+                    let (n_hashes, err) = self.raw_double_quoted_string(1);
+                    let suffix_start = self.len_consumed();
+                    if err.is_none() {
+                        self.eat_literal_suffix();
+                    }
+                    let kind = RawStr { n_hashes, err };
+                    Literal { kind, suffix_start }
+                }
+                _ => self.ident(),
+            },
+
+            // Byte literal, byte string literal, raw byte string literal or identifier.
+            'b' => match (self.first(), self.second()) {
+                ('\'', _) => {
+                    self.bump();
+                    let terminated = self.single_quoted_string();
+                    let suffix_start = self.len_consumed();
+                    if terminated {
+                        self.eat_literal_suffix();
+                    }
+                    let kind = Byte { terminated };
+                    Literal { kind, suffix_start }
+                }
+                ('"', _) => {
+                    self.bump();
+                    let terminated = self.double_quoted_string();
+                    let suffix_start = self.len_consumed();
+                    if terminated {
+                        self.eat_literal_suffix();
+                    }
+                    let kind = ByteStr { terminated };
+                    Literal { kind, suffix_start }
+                }
+                ('r', '"') | ('r', '#') => {
+                    self.bump();
+                    let (n_hashes, err) = self.raw_double_quoted_string(2);
+                    let suffix_start = self.len_consumed();
+                    if err.is_none() {
+                        self.eat_literal_suffix();
+                    }
+                    let kind = RawByteStr { n_hashes, err };
+                    Literal { kind, suffix_start }
+                }
+                _ => self.ident(),
+            },
+
+            // Identifier (this should be checked after other variant that can
+            // start as identifier).
+            c if is_id_start(c) => self.ident(),
+
+            // Numeric literal.
+            c @ '0'..='9' => {
+                let literal_kind = self.number(c);
+                let suffix_start = self.len_consumed();
+                self.eat_literal_suffix();
+                TokenKind::Literal { kind: literal_kind, suffix_start }
+            }
+
+            // One-symbol tokens.
+            ';' => Semi,
+            ',' => Comma,
+            '.' => Dot,
+            '(' => OpenParen,
+            ')' => CloseParen,
+            '{' => OpenBrace,
+            '}' => CloseBrace,
+            '[' => OpenBracket,
+            ']' => CloseBracket,
+            '@' => At,
+            '#' => Pound,
+            '~' => Tilde,
+            '?' => Question,
+            ':' => Colon,
+            '$' => Dollar,
+            '=' => Eq,
+            '!' => Bang,
+            '<' => Lt,
+            '>' => Gt,
+            '-' => Minus,
+            '&' => And,
+            '|' => Or,
+            '+' => Plus,
+            '*' => Star,
+            '^' => Caret,
+            '%' => Percent,
+
+            // Lifetime or character literal.
+            '\'' => self.lifetime_or_char(),
+
+            // String literal.
+            '"' => {
+                let terminated = self.double_quoted_string();
+                let suffix_start = self.len_consumed();
+                if terminated {
+                    self.eat_literal_suffix();
+                }
+                let kind = Str { terminated };
+                Literal { kind, suffix_start }
+            }
+            _ => Unknown,
+        };
+        Token::new(token_kind, self.len_consumed())
+    }
+
+    fn line_comment(&mut self) -> TokenKind {
+        debug_assert!(self.prev() == '/' && self.first() == '/');
+        self.bump();
+
+        let doc_style = match self.first() {
+            // `//!` is an inner line doc comment.
+            '!' => Some(DocStyle::Inner),
+            // `////` (more than 3 slashes) is not considered a doc comment.
+            '/' if self.second() != '/' => Some(DocStyle::Outer),
+            _ => None,
+        };
+
+        self.eat_while(|c| c != '\n');
+        LineComment { doc_style }
+    }
+
+    fn block_comment(&mut self) -> TokenKind {
+        debug_assert!(self.prev() == '/' && self.first() == '*');
+        self.bump();
+
+        let doc_style = match self.first() {
+            // `/*!` is an inner block doc comment.
+            '!' => Some(DocStyle::Inner),
+            // `/***` (more than 2 stars) is not considered a doc comment.
+            // `/**/` is not considered a doc comment.
+            '*' if !matches!(self.second(), '*' | '/') => Some(DocStyle::Outer),
+            _ => None,
+        };
+
+        let mut depth = 1usize;
+        while let Some(c) = self.bump() {
+            match c {
+                '/' if self.first() == '*' => {
+                    self.bump();
+                    depth += 1;
+                }
+                '*' if self.first() == '/' => {
+                    self.bump();
+                    depth -= 1;
+                    if depth == 0 {
+                        // This block comment is closed, so for a construction like "/* */ */"
+                        // there will be a successfully parsed block comment "/* */"
+                        // and " */" will be processed separately.
+                        break;
+                    }
+                }
+                _ => (),
+            }
+        }
+
+        BlockComment { doc_style, terminated: depth == 0 }
+    }
+
+    fn whitespace(&mut self) -> TokenKind {
+        debug_assert!(is_whitespace(self.prev()));
+        self.eat_while(is_whitespace);
+        Whitespace
+    }
+
+    fn raw_ident(&mut self) -> TokenKind {
+        debug_assert!(self.prev() == 'r' && self.first() == '#' && is_id_start(self.second()));
+        // Eat "#" symbol.
+        self.bump();
+        // Eat the identifier part of RawIdent.
+        self.eat_identifier();
+        RawIdent
+    }
+
+    fn ident(&mut self) -> TokenKind {
+        debug_assert!(is_id_start(self.prev()));
+        // Start is already eaten, eat the rest of identifier.
+        self.eat_while(is_id_continue);
+        Ident
+    }
+
+    fn number(&mut self, first_digit: char) -> LiteralKind {
+        debug_assert!('0' <= self.prev() && self.prev() <= '9');
+        let mut base = Base::Decimal;
+        if first_digit == '0' {
+            // Attempt to parse encoding base.
+            let has_digits = match self.first() {
+                'b' => {
+                    base = Base::Binary;
+                    self.bump();
+                    self.eat_decimal_digits()
+                }
+                'o' => {
+                    base = Base::Octal;
+                    self.bump();
+                    self.eat_decimal_digits()
+                }
+                'x' => {
+                    base = Base::Hexadecimal;
+                    self.bump();
+                    self.eat_hexadecimal_digits()
+                }
+                // Not a base prefix.
+                '0'..='9' | '_' | '.' | 'e' | 'E' => {
+                    self.eat_decimal_digits();
+                    true
+                }
+                // Just a 0.
+                _ => return Int { base, empty_int: false },
+            };
+            // Base prefix was provided, but there were no digits
+            // after it, e.g. "0x".
+            if !has_digits {
+                return Int { base, empty_int: true };
+            }
+        } else {
+            // No base prefix, parse number in the usual way.
+            self.eat_decimal_digits();
+        };
+
+        match self.first() {
+            // Don't be greedy if this is actually an
+            // integer literal followed by field/method access or a range pattern
+            // (`0..2` and `12.foo()`)
+            '.' if self.second() != '.' && !is_id_start(self.second()) => {
+                // might have stuff after the ., and if it does, it needs to start
+                // with a number
+                self.bump();
+                let mut empty_exponent = false;
+                if self.first().is_digit(10) {
+                    self.eat_decimal_digits();
+                    match self.first() {
+                        'e' | 'E' => {
+                            self.bump();
+                            empty_exponent = !self.eat_float_exponent();
+                        }
+                        _ => (),
+                    }
+                }
+                Float { base, empty_exponent }
+            }
+            'e' | 'E' => {
+                self.bump();
+                let empty_exponent = !self.eat_float_exponent();
+                Float { base, empty_exponent }
+            }
+            _ => Int { base, empty_int: false },
+        }
+    }
+
+    fn lifetime_or_char(&mut self) -> TokenKind {
+        debug_assert!(self.prev() == '\'');
+
+        let can_be_a_lifetime = if self.second() == '\'' {
+            // It's surely not a lifetime.
+            false
+        } else {
+            // If the first symbol is valid for identifier, it can be a lifetime.
+            // Also check if it's a number for a better error reporting (so '0 will
+            // be reported as invalid lifetime and not as unterminated char literal).
+            is_id_start(self.first()) || self.first().is_digit(10)
+        };
+
+        if !can_be_a_lifetime {
+            let terminated = self.single_quoted_string();
+            let suffix_start = self.len_consumed();
+            if terminated {
+                self.eat_literal_suffix();
+            }
+            let kind = Char { terminated };
+            return Literal { kind, suffix_start };
+        }
+
+        // Either a lifetime or a character literal with
+        // length greater than 1.
+
+        let starts_with_number = self.first().is_digit(10);
+
+        // Skip the literal contents.
+        // First symbol can be a number (which isn't a valid identifier start),
+        // so skip it without any checks.
+        self.bump();
+        self.eat_while(is_id_continue);
+
+        // Check if after skipping literal contents we've met a closing
+        // single quote (which means that user attempted to create a
+        // string with single quotes).
+        if self.first() == '\'' {
+            self.bump();
+            let kind = Char { terminated: true };
+            Literal { kind, suffix_start: self.len_consumed() }
+        } else {
+            Lifetime { starts_with_number }
+        }
+    }
+
+    fn single_quoted_string(&mut self) -> bool {
+        debug_assert!(self.prev() == '\'');
+        // Check if it's a one-symbol literal.
+        if self.second() == '\'' && self.first() != '\\' {
+            self.bump();
+            self.bump();
+            return true;
+        }
+
+        // Literal has more than one symbol.
+
+        // Parse until either quotes are terminated or error is detected.
+        loop {
+            match self.first() {
+                // Quotes are terminated, finish parsing.
+                '\'' => {
+                    self.bump();
+                    return true;
+                }
+                // Probably beginning of the comment, which we don't want to include
+                // to the error report.
+                '/' => break,
+                // Newline without following '\'' means unclosed quote, stop parsing.
+                '\n' if self.second() != '\'' => break,
+                // End of file, stop parsing.
+                EOF_CHAR if self.is_eof() => break,
+                // Escaped slash is considered one character, so bump twice.
+                '\\' => {
+                    self.bump();
+                    self.bump();
+                }
+                // Skip the character.
+                _ => {
+                    self.bump();
+                }
+            }
+        }
+        // String was not terminated.
+        false
+    }
+
+    /// Eats double-quoted string and returns true
+    /// if string is terminated.
+    fn double_quoted_string(&mut self) -> bool {
+        debug_assert!(self.prev() == '"');
+        while let Some(c) = self.bump() {
+            match c {
+                '"' => {
+                    return true;
+                }
+                '\\' if self.first() == '\\' || self.first() == '"' => {
+                    // Bump again to skip escaped character.
+                    self.bump();
+                }
+                _ => (),
+            }
+        }
+        // End of file reached.
+        false
+    }
+
+    /// Eats the double-quoted string and returns `n_hashes` and an error if encountered.
+    fn raw_double_quoted_string(&mut self, prefix_len: usize) -> (u16, Option<RawStrError>) {
+        // Wrap the actual function to handle the error with too many hashes.
+        // This way, it eats the whole raw string.
+        let (n_hashes, err) = self.raw_string_unvalidated(prefix_len);
+        // Only up to 65535 `#`s are allowed in raw strings
+        match u16::try_from(n_hashes) {
+            Ok(num) => (num, err),
+            // We lie about the number of hashes here :P
+            Err(_) => (0, Some(RawStrError::TooManyDelimiters { found: n_hashes })),
+        }
+    }
+
+    fn raw_string_unvalidated(&mut self, prefix_len: usize) -> (usize, Option<RawStrError>) {
+        debug_assert!(self.prev() == 'r');
+        let start_pos = self.len_consumed();
+        let mut possible_terminator_offset = None;
+        let mut max_hashes = 0;
+
+        // Count opening '#' symbols.
+        let n_start_hashes = self.eat_while(|c| c == '#');
+
+        // Check that string is started.
+        match self.bump() {
+            Some('"') => (),
+            c => {
+                let c = c.unwrap_or(EOF_CHAR);
+                return (n_start_hashes, Some(RawStrError::InvalidStarter { bad_char: c }));
+            }
+        }
+
+        // Skip the string contents and on each '#' character met, check if this is
+        // a raw string termination.
+        loop {
+            self.eat_while(|c| c != '"');
+
+            if self.is_eof() {
+                return (
+                    n_start_hashes,
+                    Some(RawStrError::NoTerminator {
+                        expected: n_start_hashes,
+                        found: max_hashes,
+                        possible_terminator_offset,
+                    }),
+                );
+            }
+
+            // Eat closing double quote.
+            self.bump();
+
+            // Check that amount of closing '#' symbols
+            // is equal to the amount of opening ones.
+            // Note that this will not consume extra trailing `#` characters:
+            // `r###"abcde"####` is lexed as a `RawStr { n_hashes: 3 }`
+            // followed by a `#` token.
+            let mut hashes_left = n_start_hashes;
+            let is_closing_hash = |c| {
+                if c == '#' && hashes_left != 0 {
+                    hashes_left -= 1;
+                    true
+                } else {
+                    false
+                }
+            };
+            let n_end_hashes = self.eat_while(is_closing_hash);
+
+            if n_end_hashes == n_start_hashes {
+                return (n_start_hashes, None);
+            } else if n_end_hashes > max_hashes {
+                // Keep track of possible terminators to give a hint about
+                // where there might be a missing terminator
+                possible_terminator_offset =
+                    Some(self.len_consumed() - start_pos - n_end_hashes + prefix_len);
+                max_hashes = n_end_hashes;
+            }
+        }
+    }
+
+    fn eat_decimal_digits(&mut self) -> bool {
+        let mut has_digits = false;
+        loop {
+            match self.first() {
+                '_' => {
+                    self.bump();
+                }
+                '0'..='9' => {
+                    has_digits = true;
+                    self.bump();
+                }
+                _ => break,
+            }
+        }
+        has_digits
+    }
+
+    fn eat_hexadecimal_digits(&mut self) -> bool {
+        let mut has_digits = false;
+        loop {
+            match self.first() {
+                '_' => {
+                    self.bump();
+                }
+                '0'..='9' | 'a'..='f' | 'A'..='F' => {
+                    has_digits = true;
+                    self.bump();
+                }
+                _ => break,
+            }
+        }
+        has_digits
+    }
+
+    /// Eats the float exponent. Returns true if at least one digit was met,
+    /// and returns false otherwise.
+    fn eat_float_exponent(&mut self) -> bool {
+        debug_assert!(self.prev() == 'e' || self.prev() == 'E');
+        if self.first() == '-' || self.first() == '+' {
+            self.bump();
+        }
+        self.eat_decimal_digits()
+    }
+
+    // Eats the suffix of the literal, e.g. "_u8".
+    fn eat_literal_suffix(&mut self) {
+        self.eat_identifier();
+    }
+
+    // Eats the identifier.
+    fn eat_identifier(&mut self) {
+        if !is_id_start(self.first()) {
+            return;
+        }
+        self.bump();
+
+        self.eat_while(is_id_continue);
+    }
+
+    /// Eats symbols while predicate returns true or until the end of file is reached.
+    /// Returns amount of eaten symbols.
+    fn eat_while<F>(&mut self, mut predicate: F) -> usize
+    where
+        F: FnMut(char) -> bool,
+    {
+        let mut eaten: usize = 0;
+        while predicate(self.first()) && !self.is_eof() {
+            eaten += 1;
+            self.bump();
+        }
+
+        eaten
+    }
+}

compiler/rustc_lexer/src/tests.rs

@@ -0,0 +1,167 @@
+use super::*;
+
+use expect_test::{expect, Expect};
+
+fn check_raw_str(s: &str, expected_hashes: u16, expected_err: Option<RawStrError>) {
+    let s = &format!("r{}", s);
+    let mut cursor = Cursor::new(s);
+    cursor.bump();
+    let (n_hashes, err) = cursor.raw_double_quoted_string(0);
+    assert_eq!(n_hashes, expected_hashes);
+    assert_eq!(err, expected_err);
+}
+
+#[test]
+fn test_naked_raw_str() {
+    check_raw_str(r#""abc""#, 0, None);
+}
+
+#[test]
+fn test_raw_no_start() {
+    check_raw_str(r##""abc"#"##, 0, None);
+}
+
+#[test]
+fn test_too_many_terminators() {
+    // this error is handled in the parser later
+    check_raw_str(r###"#"abc"##"###, 1, None);
+}
+
+#[test]
+fn test_unterminated() {
+    check_raw_str(
+        r#"#"abc"#,
+        1,
+        Some(RawStrError::NoTerminator { expected: 1, found: 0, possible_terminator_offset: None }),
+    );
+    check_raw_str(
+        r###"##"abc"#"###,
+        2,
+        Some(RawStrError::NoTerminator {
+            expected: 2,
+            found: 1,
+            possible_terminator_offset: Some(7),
+        }),
+    );
+    // We're looking for "# not just any #
+    check_raw_str(
+        r###"##"abc#"###,
+        2,
+        Some(RawStrError::NoTerminator { expected: 2, found: 0, possible_terminator_offset: None }),
+    )
+}
+
+#[test]
+fn test_invalid_start() {
+    check_raw_str(r##"#~"abc"#"##, 1, Some(RawStrError::InvalidStarter { bad_char: '~' }));
+}
+
+#[test]
+fn test_unterminated_no_pound() {
+    // https://github.com/rust-lang/rust/issues/70677
+    check_raw_str(
+        r#"""#,
+        0,
+        Some(RawStrError::NoTerminator { expected: 0, found: 0, possible_terminator_offset: None }),
+    );
+}
+
+#[test]
+fn test_valid_shebang() {
+    // https://github.com/rust-lang/rust/issues/70528
+    let input = "#!/usr/bin/rustrun\nlet x = 5;";
+    assert_eq!(strip_shebang(input), Some(18));
+}
+
+#[test]
+fn test_invalid_shebang_valid_rust_syntax() {
+    // https://github.com/rust-lang/rust/issues/70528
+    let input = "#!    [bad_attribute]";
+    assert_eq!(strip_shebang(input), None);
+}
+
+#[test]
+fn test_shebang_second_line() {
+    // Because shebangs are interpreted by the kernel, they must be on the first line
+    let input = "\n#!/bin/bash";
+    assert_eq!(strip_shebang(input), None);
+}
+
+#[test]
+fn test_shebang_space() {
+    let input = "#!    /bin/bash";
+    assert_eq!(strip_shebang(input), Some(input.len()));
+}
+
+#[test]
+fn test_shebang_empty_shebang() {
+    let input = "#!    \n[attribute(foo)]";
+    assert_eq!(strip_shebang(input), None);
+}
+
+#[test]
+fn test_invalid_shebang_comment() {
+    let input = "#!//bin/ami/a/comment\n[";
+    assert_eq!(strip_shebang(input), None)
+}
+
+#[test]
+fn test_invalid_shebang_another_comment() {
+    let input = "#!/*bin/ami/a/comment*/\n[attribute";
+    assert_eq!(strip_shebang(input), None)
+}
+
+#[test]
+fn test_shebang_valid_rust_after() {
+    let input = "#!/*bin/ami/a/comment*/\npub fn main() {}";
+    assert_eq!(strip_shebang(input), Some(23))
+}
+
+#[test]
+fn test_shebang_followed_by_attrib() {
+    let input = "#!/bin/rust-scripts\n#![allow_unused(true)]";
+    assert_eq!(strip_shebang(input), Some(19));
+}
+
+fn check_lexing(src: &str, expect: Expect) {
+    let actual: String = tokenize(src).map(|token| format!("{:?}\n", token)).collect();
+    expect.assert_eq(&actual)
+}
+
+#[test]
+fn comment_flavors() {
+    check_lexing(
+        r"
+// line
+//// line as well
+/// outer doc line
+//! inner doc line
+/* block */
+/**/
+/*** also block */
+/** outer doc block */
+/*! inner doc block */
+",
+        expect![[r#"
+                Token { kind: Whitespace, len: 1 }
+                Token { kind: LineComment { doc_style: None }, len: 7 }
+                Token { kind: Whitespace, len: 1 }
+                Token { kind: LineComment { doc_style: None }, len: 17 }
+                Token { kind: Whitespace, len: 1 }
+                Token { kind: LineComment { doc_style: Some(Outer) }, len: 18 }
+                Token { kind: Whitespace, len: 1 }
+                Token { kind: LineComment { doc_style: Some(Inner) }, len: 18 }
+                Token { kind: Whitespace, len: 1 }
+                Token { kind: BlockComment { doc_style: None, terminated: true }, len: 11 }
+                Token { kind: Whitespace, len: 1 }
+                Token { kind: BlockComment { doc_style: None, terminated: true }, len: 4 }
+                Token { kind: Whitespace, len: 1 }
+                Token { kind: BlockComment { doc_style: None, terminated: true }, len: 18 }
+                Token { kind: Whitespace, len: 1 }
+                Token { kind: BlockComment { doc_style: Some(Outer), terminated: true }, len: 22 }
+                Token { kind: Whitespace, len: 1 }
+                Token { kind: BlockComment { doc_style: Some(Inner), terminated: true }, len: 22 }
+                Token { kind: Whitespace, len: 1 }
+            "#]],
+    )
+}

compiler/rustc_lexer/src/unescape.rs

@@ -0,0 +1,344 @@
+//! Utilities for validating string and char literals and turning them into
+//! values they represent.
+
+use std::ops::Range;
+use std::str::Chars;
+
+#[cfg(test)]
+mod tests;
+
+/// Errors that can occur during string unescaping.
+#[derive(Debug, PartialEq, Eq)]
+pub enum EscapeError {
+    /// Expected 1 char, but 0 were found.
+    ZeroChars,
+    /// Expected 1 char, but more than 1 were found.
+    MoreThanOneChar,
+
+    /// Escaped '\' character without continuation.
+    LoneSlash,
+    /// Invalid escape character (e.g. '\z').
+    InvalidEscape,
+    /// Raw '\r' encountered.
+    BareCarriageReturn,
+    /// Raw '\r' encountered in raw string.
+    BareCarriageReturnInRawString,
+    /// Unescaped character that was expected to be escaped (e.g. raw '\t').
+    EscapeOnlyChar,
+
+    /// Numeric character escape is too short (e.g. '\x1').
+    TooShortHexEscape,
+    /// Invalid character in numeric escape (e.g. '\xz')
+    InvalidCharInHexEscape,
+    /// Character code in numeric escape is non-ascii (e.g. '\xFF').
+    OutOfRangeHexEscape,
+
+    /// '\u' not followed by '{'.
+    NoBraceInUnicodeEscape,
+    /// Non-hexadecimal value in '\u{..}'.
+    InvalidCharInUnicodeEscape,
+    /// '\u{}'
+    EmptyUnicodeEscape,
+    /// No closing brace in '\u{..}', e.g. '\u{12'.
+    UnclosedUnicodeEscape,
+    /// '\u{_12}'
+    LeadingUnderscoreUnicodeEscape,
+    /// More than 6 characters in '\u{..}', e.g. '\u{10FFFF_FF}'
+    OverlongUnicodeEscape,
+    /// Invalid in-bound unicode character code, e.g. '\u{DFFF}'.
+    LoneSurrogateUnicodeEscape,
+    /// Out of bounds unicode character code, e.g. '\u{FFFFFF}'.
+    OutOfRangeUnicodeEscape,
+
+    /// Unicode escape code in byte literal.
+    UnicodeEscapeInByte,
+    /// Non-ascii character in byte literal.
+    NonAsciiCharInByte,
+    /// Non-ascii character in byte string literal.
+    NonAsciiCharInByteString,
+}
+
+/// Takes a contents of a literal (without quotes) and produces a
+/// sequence of escaped characters or errors.
+/// Values are returned through invoking of the provided callback.
+pub fn unescape_literal<F>(literal_text: &str, mode: Mode, callback: &mut F)
+where
+    F: FnMut(Range<usize>, Result<char, EscapeError>),
+{
+    match mode {
+        Mode::Char | Mode::Byte => {
+            let mut chars = literal_text.chars();
+            let result = unescape_char_or_byte(&mut chars, mode);
+            // The Chars iterator moved forward.
+            callback(0..(literal_text.len() - chars.as_str().len()), result);
+        }
+        Mode::Str | Mode::ByteStr => unescape_str_or_byte_str(literal_text, mode, callback),
+        // NOTE: Raw strings do not perform any explicit character escaping, here we
+        // only translate CRLF to LF and produce errors on bare CR.
+        Mode::RawStr | Mode::RawByteStr => {
+            unescape_raw_str_or_byte_str(literal_text, mode, callback)
+        }
+    }
+}
+
+/// Takes a contents of a byte, byte string or raw byte string (without quotes)
+/// and produces a sequence of bytes or errors.
+/// Values are returned through invoking of the provided callback.
+pub fn unescape_byte_literal<F>(literal_text: &str, mode: Mode, callback: &mut F)
+where
+    F: FnMut(Range<usize>, Result<u8, EscapeError>),
+{
+    assert!(mode.is_bytes());
+    unescape_literal(literal_text, mode, &mut |range, result| {
+        callback(range, result.map(byte_from_char));
+    })
+}
+
+/// Takes a contents of a char literal (without quotes), and returns an
+/// unescaped char or an error
+pub fn unescape_char(literal_text: &str) -> Result<char, (usize, EscapeError)> {
+    let mut chars = literal_text.chars();
+    unescape_char_or_byte(&mut chars, Mode::Char)
+        .map_err(|err| (literal_text.len() - chars.as_str().len(), err))
+}
+
+/// Takes a contents of a byte literal (without quotes), and returns an
+/// unescaped byte or an error.
+pub fn unescape_byte(literal_text: &str) -> Result<u8, (usize, EscapeError)> {
+    let mut chars = literal_text.chars();
+    unescape_char_or_byte(&mut chars, Mode::Byte)
+        .map(byte_from_char)
+        .map_err(|err| (literal_text.len() - chars.as_str().len(), err))
+}
+
+/// What kind of literal do we parse.
+#[derive(Debug, Clone, Copy)]
+pub enum Mode {
+    Char,
+    Str,
+    Byte,
+    ByteStr,
+    RawStr,
+    RawByteStr,
+}
+
+impl Mode {
+    pub fn in_single_quotes(self) -> bool {
+        match self {
+            Mode::Char | Mode::Byte => true,
+            Mode::Str | Mode::ByteStr | Mode::RawStr | Mode::RawByteStr => false,
+        }
+    }
+
+    pub fn in_double_quotes(self) -> bool {
+        !self.in_single_quotes()
+    }
+
+    pub fn is_bytes(self) -> bool {
+        match self {
+            Mode::Byte | Mode::ByteStr | Mode::RawByteStr => true,
+            Mode::Char | Mode::Str | Mode::RawStr => false,
+        }
+    }
+}
+
+fn scan_escape(first_char: char, chars: &mut Chars<'_>, mode: Mode) -> Result<char, EscapeError> {
+    if first_char != '\\' {
+        // Previous character was not a slash, and we don't expect it to be
+        // an escape-only character.
+        return match first_char {
+            '\t' | '\n' => Err(EscapeError::EscapeOnlyChar),
+            '\r' => Err(EscapeError::BareCarriageReturn),
+            '\'' if mode.in_single_quotes() => Err(EscapeError::EscapeOnlyChar),
+            '"' if mode.in_double_quotes() => Err(EscapeError::EscapeOnlyChar),
+            _ => {
+                if mode.is_bytes() && !first_char.is_ascii() {
+                    // Byte literal can't be a non-ascii character.
+                    return Err(EscapeError::NonAsciiCharInByte);
+                }
+                Ok(first_char)
+            }
+        };
+    }
+
+    // Previous character is '\\', try to unescape it.
+
+    let second_char = chars.next().ok_or(EscapeError::LoneSlash)?;
+
+    let res = match second_char {
+        '"' => '"',
+        'n' => '\n',
+        'r' => '\r',
+        't' => '\t',
+        '\\' => '\\',
+        '\'' => '\'',
+        '0' => '\0',
+
+        'x' => {
+            // Parse hexadecimal character code.
+
+            let hi = chars.next().ok_or(EscapeError::TooShortHexEscape)?;
+            let hi = hi.to_digit(16).ok_or(EscapeError::InvalidCharInHexEscape)?;
+
+            let lo = chars.next().ok_or(EscapeError::TooShortHexEscape)?;
+            let lo = lo.to_digit(16).ok_or(EscapeError::InvalidCharInHexEscape)?;
+
+            let value = hi * 16 + lo;
+
+            // For a byte literal verify that it is within ASCII range.
+            if !mode.is_bytes() && !is_ascii(value) {
+                return Err(EscapeError::OutOfRangeHexEscape);
+            }
+            let value = value as u8;
+
+            value as char
+        }
+
+        'u' => {
+            // We've parsed '\u', now we have to parse '{..}'.
+
+            if chars.next() != Some('{') {
+                return Err(EscapeError::NoBraceInUnicodeEscape);
+            }
+
+            // First characrer must be a hexadecimal digit.
+            let mut n_digits = 1;
+            let mut value: u32 = match chars.next().ok_or(EscapeError::UnclosedUnicodeEscape)? {
+                '_' => return Err(EscapeError::LeadingUnderscoreUnicodeEscape),
+                '}' => return Err(EscapeError::EmptyUnicodeEscape),
+                c => c.to_digit(16).ok_or(EscapeError::InvalidCharInUnicodeEscape)?,
+            };
+
+            // First character is valid, now parse the rest of the number
+            // and closing brace.
+            loop {
+                match chars.next() {
+                    None => return Err(EscapeError::UnclosedUnicodeEscape),
+                    Some('_') => continue,
+                    Some('}') => {
+                        if n_digits > 6 {
+                            return Err(EscapeError::OverlongUnicodeEscape);
+                        }
+
+                        // Incorrect syntax has higher priority for error reporting
+                        // than unallowed value for a literal.
+                        if mode.is_bytes() {
+                            return Err(EscapeError::UnicodeEscapeInByte);
+                        }
+
+                        break std::char::from_u32(value).ok_or_else(|| {
+                            if value > 0x10FFFF {
+                                EscapeError::OutOfRangeUnicodeEscape
+                            } else {
+                                EscapeError::LoneSurrogateUnicodeEscape
+                            }
+                        })?;
+                    }
+                    Some(c) => {
+                        let digit =
+                            c.to_digit(16).ok_or(EscapeError::InvalidCharInUnicodeEscape)?;
+                        n_digits += 1;
+                        if n_digits > 6 {
+                            // Stop updating value since we're sure that it's is incorrect already.
+                            continue;
+                        }
+                        let digit = digit as u32;
+                        value = value * 16 + digit;
+                    }
+                };
+            }
+        }
+        _ => return Err(EscapeError::InvalidEscape),
+    };
+    Ok(res)
+}
+
+fn unescape_char_or_byte(chars: &mut Chars<'_>, mode: Mode) -> Result<char, EscapeError> {
+    let first_char = chars.next().ok_or(EscapeError::ZeroChars)?;
+    let res = scan_escape(first_char, chars, mode)?;
+    if chars.next().is_some() {
+        return Err(EscapeError::MoreThanOneChar);
+    }
+    Ok(res)
+}
+
+/// Takes a contents of a string literal (without quotes) and produces a
+/// sequence of escaped characters or errors.
+fn unescape_str_or_byte_str<F>(src: &str, mode: Mode, callback: &mut F)
+where
+    F: FnMut(Range<usize>, Result<char, EscapeError>),
+{
+    assert!(mode.in_double_quotes());
+    let initial_len = src.len();
+    let mut chars = src.chars();
+    while let Some(first_char) = chars.next() {
+        let start = initial_len - chars.as_str().len() - first_char.len_utf8();
+
+        let unescaped_char = match first_char {
+            '\\' => {
+                let second_char = chars.clone().next();
+                match second_char {
+                    Some('\n') => {
+                        // Rust language specification requires us to skip whitespaces
+                        // if unescaped '\' character is followed by '\n'.
+                        // For details see [Rust language reference]
+                        // (https://doc.rust-lang.org/reference/tokens.html#string-literals).
+                        skip_ascii_whitespace(&mut chars);
+                        continue;
+                    }
+                    _ => scan_escape(first_char, &mut chars, mode),
+                }
+            }
+            '\n' => Ok('\n'),
+            '\t' => Ok('\t'),
+            _ => scan_escape(first_char, &mut chars, mode),
+        };
+        let end = initial_len - chars.as_str().len();
+        callback(start..end, unescaped_char);
+    }
+
+    fn skip_ascii_whitespace(chars: &mut Chars<'_>) {
+        let str = chars.as_str();
+        let first_non_space = str
+            .bytes()
+            .position(|b| b != b' ' && b != b'\t' && b != b'\n' && b != b'\r')
+            .unwrap_or(str.len());
+        *chars = str[first_non_space..].chars()
+    }
+}
+
+/// Takes a contents of a string literal (without quotes) and produces a
+/// sequence of characters or errors.
+/// NOTE: Raw strings do not perform any explicit character escaping, here we
+/// only translate CRLF to LF and produce errors on bare CR.
+fn unescape_raw_str_or_byte_str<F>(literal_text: &str, mode: Mode, callback: &mut F)
+where
+    F: FnMut(Range<usize>, Result<char, EscapeError>),
+{
+    assert!(mode.in_double_quotes());
+    let initial_len = literal_text.len();
+
+    let mut chars = literal_text.chars();
+    while let Some(curr) = chars.next() {
+        let start = initial_len - chars.as_str().len() - curr.len_utf8();
+
+        let result = match curr {
+            '\r' => Err(EscapeError::BareCarriageReturnInRawString),
+            c if mode.is_bytes() && !c.is_ascii() => Err(EscapeError::NonAsciiCharInByteString),
+            c => Ok(c),
+        };
+        let end = initial_len - chars.as_str().len();
+
+        callback(start..end, result);
+    }
+}
+
+fn byte_from_char(c: char) -> u8 {
+    let res = c as u32;
+    assert!(res <= u8::MAX as u32, "guaranteed because of Mode::ByteStr");
+    res as u8
+}
+
+fn is_ascii(x: u32) -> bool {
+    x <= 0x7F
+}

compiler/rustc_lexer/src/unescape/tests.rs

@@ -0,0 +1,273 @@
+use super::*;
+
+#[test]
+fn test_unescape_char_bad() {
+    fn check(literal_text: &str, expected_error: EscapeError) {
+        let actual_result = unescape_char(literal_text).map_err(|(_offset, err)| err);
+        assert_eq!(actual_result, Err(expected_error));
+    }
+
+    check("", EscapeError::ZeroChars);
+    check(r"\", EscapeError::LoneSlash);
+
+    check("\n", EscapeError::EscapeOnlyChar);
+    check("\t", EscapeError::EscapeOnlyChar);
+    check("'", EscapeError::EscapeOnlyChar);
+    check("\r", EscapeError::BareCarriageReturn);
+
+    check("spam", EscapeError::MoreThanOneChar);
+    check(r"\x0ff", EscapeError::MoreThanOneChar);
+    check(r#"\"a"#, EscapeError::MoreThanOneChar);
+    check(r"\na", EscapeError::MoreThanOneChar);
+    check(r"\ra", EscapeError::MoreThanOneChar);
+    check(r"\ta", EscapeError::MoreThanOneChar);
+    check(r"\\a", EscapeError::MoreThanOneChar);
+    check(r"\'a", EscapeError::MoreThanOneChar);
+    check(r"\0a", EscapeError::MoreThanOneChar);
+    check(r"\u{0}x", EscapeError::MoreThanOneChar);
+    check(r"\u{1F63b}}", EscapeError::MoreThanOneChar);
+
+    check(r"\v", EscapeError::InvalidEscape);
+    check(r"\💩", EscapeError::InvalidEscape);
+    check(r"\●", EscapeError::InvalidEscape);
+    check("\\\r", EscapeError::InvalidEscape);
+
+    check(r"\x", EscapeError::TooShortHexEscape);
+    check(r"\x0", EscapeError::TooShortHexEscape);
+    check(r"\xf", EscapeError::TooShortHexEscape);
+    check(r"\xa", EscapeError::TooShortHexEscape);
+    check(r"\xx", EscapeError::InvalidCharInHexEscape);
+    check(r"\xы", EscapeError::InvalidCharInHexEscape);
+    check(r"\x🦀", EscapeError::InvalidCharInHexEscape);
+    check(r"\xtt", EscapeError::InvalidCharInHexEscape);
+    check(r"\xff", EscapeError::OutOfRangeHexEscape);
+    check(r"\xFF", EscapeError::OutOfRangeHexEscape);
+    check(r"\x80", EscapeError::OutOfRangeHexEscape);
+
+    check(r"\u", EscapeError::NoBraceInUnicodeEscape);
+    check(r"\u[0123]", EscapeError::NoBraceInUnicodeEscape);
+    check(r"\u{0x}", EscapeError::InvalidCharInUnicodeEscape);
+    check(r"\u{", EscapeError::UnclosedUnicodeEscape);
+    check(r"\u{0000", EscapeError::UnclosedUnicodeEscape);
+    check(r"\u{}", EscapeError::EmptyUnicodeEscape);
+    check(r"\u{_0000}", EscapeError::LeadingUnderscoreUnicodeEscape);
+    check(r"\u{0000000}", EscapeError::OverlongUnicodeEscape);
+    check(r"\u{FFFFFF}", EscapeError::OutOfRangeUnicodeEscape);
+    check(r"\u{ffffff}", EscapeError::OutOfRangeUnicodeEscape);
+    check(r"\u{ffffff}", EscapeError::OutOfRangeUnicodeEscape);
+
+    check(r"\u{DC00}", EscapeError::LoneSurrogateUnicodeEscape);
+    check(r"\u{DDDD}", EscapeError::LoneSurrogateUnicodeEscape);
+    check(r"\u{DFFF}", EscapeError::LoneSurrogateUnicodeEscape);
+
+    check(r"\u{D800}", EscapeError::LoneSurrogateUnicodeEscape);
+    check(r"\u{DAAA}", EscapeError::LoneSurrogateUnicodeEscape);
+    check(r"\u{DBFF}", EscapeError::LoneSurrogateUnicodeEscape);
+}
+
+#[test]
+fn test_unescape_char_good() {
+    fn check(literal_text: &str, expected_char: char) {
+        let actual_result = unescape_char(literal_text);
+        assert_eq!(actual_result, Ok(expected_char));
+    }
+
+    check("a", 'a');
+    check("ы", 'ы');
+    check("🦀", '🦀');
+
+    check(r#"\""#, '"');
+    check(r"\n", '\n');
+    check(r"\r", '\r');
+    check(r"\t", '\t');
+    check(r"\\", '\\');
+    check(r"\'", '\'');
+    check(r"\0", '\0');
+
+    check(r"\x00", '\0');
+    check(r"\x5a", 'Z');
+    check(r"\x5A", 'Z');
+    check(r"\x7f", 127 as char);
+
+    check(r"\u{0}", '\0');
+    check(r"\u{000000}", '\0');
+    check(r"\u{41}", 'A');
+    check(r"\u{0041}", 'A');
+    check(r"\u{00_41}", 'A');
+    check(r"\u{4__1__}", 'A');
+    check(r"\u{1F63b}", '😻');
+}
+
+#[test]
+fn test_unescape_str_good() {
+    fn check(literal_text: &str, expected: &str) {
+        let mut buf = Ok(String::with_capacity(literal_text.len()));
+        unescape_literal(literal_text, Mode::Str, &mut |range, c| {
+            if let Ok(b) = &mut buf {
+                match c {
+                    Ok(c) => b.push(c),
+                    Err(e) => buf = Err((range, e)),
+                }
+            }
+        });
+        let buf = buf.as_ref().map(|it| it.as_ref());
+        assert_eq!(buf, Ok(expected))
+    }
+
+    check("foo", "foo");
+    check("", "");
+    check(" \t\n", " \t\n");
+
+    check("hello \\\n     world", "hello world");
+    check("thread's", "thread's")
+}
+
+#[test]
+fn test_unescape_byte_bad() {
+    fn check(literal_text: &str, expected_error: EscapeError) {
+        let actual_result = unescape_byte(literal_text).map_err(|(_offset, err)| err);
+        assert_eq!(actual_result, Err(expected_error));
+    }
+
+    check("", EscapeError::ZeroChars);
+    check(r"\", EscapeError::LoneSlash);
+
+    check("\n", EscapeError::EscapeOnlyChar);
+    check("\t", EscapeError::EscapeOnlyChar);
+    check("'", EscapeError::EscapeOnlyChar);
+    check("\r", EscapeError::BareCarriageReturn);
+
+    check("spam", EscapeError::MoreThanOneChar);
+    check(r"\x0ff", EscapeError::MoreThanOneChar);
+    check(r#"\"a"#, EscapeError::MoreThanOneChar);
+    check(r"\na", EscapeError::MoreThanOneChar);
+    check(r"\ra", EscapeError::MoreThanOneChar);
+    check(r"\ta", EscapeError::MoreThanOneChar);
+    check(r"\\a", EscapeError::MoreThanOneChar);
+    check(r"\'a", EscapeError::MoreThanOneChar);
+    check(r"\0a", EscapeError::MoreThanOneChar);
+
+    check(r"\v", EscapeError::InvalidEscape);
+    check(r"\💩", EscapeError::InvalidEscape);
+    check(r"\●", EscapeError::InvalidEscape);
+
+    check(r"\x", EscapeError::TooShortHexEscape);
+    check(r"\x0", EscapeError::TooShortHexEscape);
+    check(r"\xa", EscapeError::TooShortHexEscape);
+    check(r"\xf", EscapeError::TooShortHexEscape);
+    check(r"\xx", EscapeError::InvalidCharInHexEscape);
+    check(r"\xы", EscapeError::InvalidCharInHexEscape);
+    check(r"\x🦀", EscapeError::InvalidCharInHexEscape);
+    check(r"\xtt", EscapeError::InvalidCharInHexEscape);
+
+    check(r"\u", EscapeError::NoBraceInUnicodeEscape);
+    check(r"\u[0123]", EscapeError::NoBraceInUnicodeEscape);
+    check(r"\u{0x}", EscapeError::InvalidCharInUnicodeEscape);
+    check(r"\u{", EscapeError::UnclosedUnicodeEscape);
+    check(r"\u{0000", EscapeError::UnclosedUnicodeEscape);
+    check(r"\u{}", EscapeError::EmptyUnicodeEscape);
+    check(r"\u{_0000}", EscapeError::LeadingUnderscoreUnicodeEscape);
+    check(r"\u{0000000}", EscapeError::OverlongUnicodeEscape);
+
+    check("ы", EscapeError::NonAsciiCharInByte);
+    check("🦀", EscapeError::NonAsciiCharInByte);
+
+    check(r"\u{0}", EscapeError::UnicodeEscapeInByte);
+    check(r"\u{000000}", EscapeError::UnicodeEscapeInByte);
+    check(r"\u{41}", EscapeError::UnicodeEscapeInByte);
+    check(r"\u{0041}", EscapeError::UnicodeEscapeInByte);
+    check(r"\u{00_41}", EscapeError::UnicodeEscapeInByte);
+    check(r"\u{4__1__}", EscapeError::UnicodeEscapeInByte);
+    check(r"\u{1F63b}", EscapeError::UnicodeEscapeInByte);
+    check(r"\u{0}x", EscapeError::UnicodeEscapeInByte);
+    check(r"\u{1F63b}}", EscapeError::UnicodeEscapeInByte);
+    check(r"\u{FFFFFF}", EscapeError::UnicodeEscapeInByte);
+    check(r"\u{ffffff}", EscapeError::UnicodeEscapeInByte);
+    check(r"\u{ffffff}", EscapeError::UnicodeEscapeInByte);
+    check(r"\u{DC00}", EscapeError::UnicodeEscapeInByte);
+    check(r"\u{DDDD}", EscapeError::UnicodeEscapeInByte);
+    check(r"\u{DFFF}", EscapeError::UnicodeEscapeInByte);
+    check(r"\u{D800}", EscapeError::UnicodeEscapeInByte);
+    check(r"\u{DAAA}", EscapeError::UnicodeEscapeInByte);
+    check(r"\u{DBFF}", EscapeError::UnicodeEscapeInByte);
+}
+
+#[test]
+fn test_unescape_byte_good() {
+    fn check(literal_text: &str, expected_byte: u8) {
+        let actual_result = unescape_byte(literal_text);
+        assert_eq!(actual_result, Ok(expected_byte));
+    }
+
+    check("a", b'a');
+
+    check(r#"\""#, b'"');
+    check(r"\n", b'\n');
+    check(r"\r", b'\r');
+    check(r"\t", b'\t');
+    check(r"\\", b'\\');
+    check(r"\'", b'\'');
+    check(r"\0", b'\0');
+
+    check(r"\x00", b'\0');
+    check(r"\x5a", b'Z');
+    check(r"\x5A", b'Z');
+    check(r"\x7f", 127);
+    check(r"\x80", 128);
+    check(r"\xff", 255);
+    check(r"\xFF", 255);
+}
+
+#[test]
+fn test_unescape_byte_str_good() {
+    fn check(literal_text: &str, expected: &[u8]) {
+        let mut buf = Ok(Vec::with_capacity(literal_text.len()));
+        unescape_byte_literal(literal_text, Mode::ByteStr, &mut |range, c| {
+            if let Ok(b) = &mut buf {
+                match c {
+                    Ok(c) => b.push(c),
+                    Err(e) => buf = Err((range, e)),
+                }
+            }
+        });
+        let buf = buf.as_ref().map(|it| it.as_ref());
+        assert_eq!(buf, Ok(expected))
+    }
+
+    check("foo", b"foo");
+    check("", b"");
+    check(" \t\n", b" \t\n");
+
+    check("hello \\\n     world", b"hello world");
+    check("thread's", b"thread's")
+}
+
+#[test]
+fn test_unescape_raw_str() {
+    fn check(literal: &str, expected: &[(Range<usize>, Result<char, EscapeError>)]) {
+        let mut unescaped = Vec::with_capacity(literal.len());
+        unescape_literal(literal, Mode::RawStr, &mut |range, res| unescaped.push((range, res)));
+        assert_eq!(unescaped, expected);
+    }
+
+    check("\r", &[(0..1, Err(EscapeError::BareCarriageReturnInRawString))]);
+    check("\rx", &[(0..1, Err(EscapeError::BareCarriageReturnInRawString)), (1..2, Ok('x'))]);
+}
+
+#[test]
+fn test_unescape_raw_byte_str() {
+    fn check(literal: &str, expected: &[(Range<usize>, Result<u8, EscapeError>)]) {
+        let mut unescaped = Vec::with_capacity(literal.len());
+        unescape_byte_literal(literal, Mode::RawByteStr, &mut |range, res| {
+            unescaped.push((range, res))
+        });
+        assert_eq!(unescaped, expected);
+    }
+
+    check("\r", &[(0..1, Err(EscapeError::BareCarriageReturnInRawString))]);
+    check("🦀", &[(0..4, Err(EscapeError::NonAsciiCharInByteString))]);
+    check(
+        "🦀a",
+        &[(0..4, Err(EscapeError::NonAsciiCharInByteString)), (4..5, Ok(byte_from_char('a')))],
+    );
+}