bjoernager/rust
bjoernager/rust
1
Fork 0
Code Issues Pull requests Activity

Move lexer unit tests to rustc_lexer

Browse source 
StringReader is an intornal abstraction which at the moment changes a
lot, so these unit tests cause quite a bit of friction.

Moving them to rustc_lexer and more ingerated-testing style should
make them much less annoying, hopefully without decreasing their
usefulness much.

Note that coloncolon tests are removed (it's unclear what those are
testing).

\r\n tests are removed as well, as we normalize line endings even
before lexing.
This commit is contained in:
Aleksey Kladov 2020-08-29 12:10:16 +02:00
parent 85fbf49ce0
commit ccffea5b6b
4 changed files with 154 additions and 278 deletions
Download patch file Download diff file Expand all files Collapse all files

160
compiler/rustc_lexer/src/tests.rs
View file

@ -128,6 +128,34 @@ fn check_lexing(src: &str, expect: Expect) {
expect.assert_eq(&actual)
}
#[test]
fn smoke_test() {
check_lexing(
"/* my source file */ fn main() { println!(\"zebra\"); }\n",
expect![[r#"
Token { kind: BlockComment { doc_style: None, terminated: true }, len: 20 }
Token { kind: Whitespace, len: 1 }
Token { kind: Ident, len: 2 }
Token { kind: Whitespace, len: 1 }
Token { kind: Ident, len: 4 }
Token { kind: OpenParen, len: 1 }
Token { kind: CloseParen, len: 1 }
Token { kind: Whitespace, len: 1 }
Token { kind: OpenBrace, len: 1 }
Token { kind: Whitespace, len: 1 }
Token { kind: Ident, len: 7 }
Token { kind: Bang, len: 1 }
Token { kind: OpenParen, len: 1 }
Token { kind: Literal { kind: Str { terminated: true }, suffix_start: 7 }, len: 7 }
Token { kind: CloseParen, len: 1 }
Token { kind: Semi, len: 1 }
Token { kind: Whitespace, len: 1 }
Token { kind: CloseBrace, len: 1 }
Token { kind: Whitespace, len: 1 }
"#]],
)
}
#[test]
fn comment_flavors() {
check_lexing(
@ -143,25 +171,117 @@ fn comment_flavors() {
/*! inner doc block */
",
expect![[r#"
Token { kind: Whitespace, len: 1 }
Token { kind: LineComment { doc_style: None }, len: 7 }
Token { kind: Whitespace, len: 1 }
Token { kind: LineComment { doc_style: None }, len: 17 }
Token { kind: Whitespace, len: 1 }
Token { kind: LineComment { doc_style: Some(Outer) }, len: 18 }
Token { kind: Whitespace, len: 1 }
Token { kind: LineComment { doc_style: Some(Inner) }, len: 18 }
Token { kind: Whitespace, len: 1 }
Token { kind: BlockComment { doc_style: None, terminated: true }, len: 11 }
Token { kind: Whitespace, len: 1 }
Token { kind: BlockComment { doc_style: None, terminated: true }, len: 4 }
Token { kind: Whitespace, len: 1 }
Token { kind: BlockComment { doc_style: None, terminated: true }, len: 18 }
Token { kind: Whitespace, len: 1 }
Token { kind: BlockComment { doc_style: Some(Outer), terminated: true }, len: 22 }
Token { kind: Whitespace, len: 1 }
Token { kind: BlockComment { doc_style: Some(Inner), terminated: true }, len: 22 }
Token { kind: Whitespace, len: 1 }
"#]],
Token { kind: Whitespace, len: 1 }
Token { kind: LineComment { doc_style: None }, len: 7 }
Token { kind: Whitespace, len: 1 }
Token { kind: LineComment { doc_style: None }, len: 17 }
Token { kind: Whitespace, len: 1 }
Token { kind: LineComment { doc_style: Some(Outer) }, len: 18 }
Token { kind: Whitespace, len: 1 }
Token { kind: LineComment { doc_style: Some(Inner) }, len: 18 }
Token { kind: Whitespace, len: 1 }
Token { kind: BlockComment { doc_style: None, terminated: true }, len: 11 }
Token { kind: Whitespace, len: 1 }
Token { kind: BlockComment { doc_style: None, terminated: true }, len: 4 }
Token { kind: Whitespace, len: 1 }
Token { kind: BlockComment { doc_style: None, terminated: true }, len: 18 }
Token { kind: Whitespace, len: 1 }
Token { kind: BlockComment { doc_style: Some(Outer), terminated: true }, len: 22 }
Token { kind: Whitespace, len: 1 }
Token { kind: BlockComment { doc_style: Some(Inner), terminated: true }, len: 22 }
Token { kind: Whitespace, len: 1 }
"#]],
)
}
#[test]
fn nested_block_comments() {
check_lexing(
"/* /* */ */'a'",
expect![[r#"
Token { kind: BlockComment { doc_style: None, terminated: true }, len: 11 }
Token { kind: Literal { kind: Char { terminated: true }, suffix_start: 3 }, len: 3 }
"#]],
)
}
#[test]
fn characters() {
check_lexing(
"'a' ' ' '\\n'",
expect![[r#"
Token { kind: Literal { kind: Char { terminated: true }, suffix_start: 3 }, len: 3 }
Token { kind: Whitespace, len: 1 }
Token { kind: Literal { kind: Char { terminated: true }, suffix_start: 3 }, len: 3 }
Token { kind: Whitespace, len: 1 }
Token { kind: Literal { kind: Char { terminated: true }, suffix_start: 4 }, len: 4 }
"#]],
);
}
#[test]
fn lifetime() {
check_lexing(
"'abc",
expect![[r#"
Token { kind: Lifetime { starts_with_number: false }, len: 4 }
"#]],
);
}
#[test]
fn raw_string() {
check_lexing(
"r###\"\"#a\\b\x00c\"\"###",
expect![[r#"
Token { kind: Literal { kind: RawStr { n_hashes: 3, err: None }, suffix_start: 17 }, len: 17 }
"#]],
)
}
#[test]
fn literal_suffixes() {
check_lexing(
r####"
'a'
b'a'
"a"
b"a"
1234
0b101
0xABC
1.0
1.0e10
2us
r###"raw"###suffix
br###"raw"###suffix
"####,
expect![[r#"
Token { kind: Whitespace, len: 1 }
Token { kind: Literal { kind: Char { terminated: true }, suffix_start: 3 }, len: 3 }
Token { kind: Whitespace, len: 1 }
Token { kind: Literal { kind: Byte { terminated: true }, suffix_start: 4 }, len: 4 }
Token { kind: Whitespace, len: 1 }
Token { kind: Literal { kind: Str { terminated: true }, suffix_start: 3 }, len: 3 }
Token { kind: Whitespace, len: 1 }
Token { kind: Literal { kind: ByteStr { terminated: true }, suffix_start: 4 }, len: 4 }
Token { kind: Whitespace, len: 1 }
Token { kind: Literal { kind: Int { base: Decimal, empty_int: false }, suffix_start: 4 }, len: 4 }
Token { kind: Whitespace, len: 1 }
Token { kind: Literal { kind: Int { base: Binary, empty_int: false }, suffix_start: 5 }, len: 5 }
Token { kind: Whitespace, len: 1 }
Token { kind: Literal { kind: Int { base: Hexadecimal, empty_int: false }, suffix_start: 5 }, len: 5 }
Token { kind: Whitespace, len: 1 }
Token { kind: Literal { kind: Float { base: Decimal, empty_exponent: false }, suffix_start: 3 }, len: 3 }
Token { kind: Whitespace, len: 1 }
Token { kind: Literal { kind: Float { base: Decimal, empty_exponent: false }, suffix_start: 6 }, len: 6 }
Token { kind: Whitespace, len: 1 }
Token { kind: Literal { kind: Int { base: Decimal, empty_int: false }, suffix_start: 1 }, len: 3 }
Token { kind: Whitespace, len: 1 }
Token { kind: Literal { kind: RawStr { n_hashes: 3, err: None }, suffix_start: 12 }, len: 18 }
Token { kind: Whitespace, len: 1 }
Token { kind: Literal { kind: RawByteStr { n_hashes: 3, err: None }, suffix_start: 13 }, len: 19 }
Token { kind: Whitespace, len: 1 }
"#]],
)
}