bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

adapt rustdoc to infailable lexer

Browse source
This commit is contained in:
Aleksey Kladov 2019-07-30 13:45:08 +03:00
parent 58ac81a60f
commit b3e8c8bbe2
3 changed files with 177 additions and 50 deletions
Download patch file Download diff file Expand all files Collapse all files

27
src/librustdoc/html/highlight.rs
View file

@ -44,7 +44,7 @@ pub fn render_with_highlighting(
let mut highlighted_source = vec![];
if classifier.write_source(&mut highlighted_source).is_err() {
Err(classifier.lexer.buffer_fatal_errors())
Err(())
} else {
Ok(String::from_utf8_lossy(&highlighted_source).into_owned())
}
@ -59,14 +59,9 @@ pub fn render_with_highlighting(
}
write_footer(&mut out).unwrap();
}
Err(errors) => {
// If errors are encountered while trying to highlight, cancel the errors and just emit
// the unhighlighted source. The errors will have already been reported in the
// `check-code-block-syntax` pass.
for mut error in errors {
error.cancel();
}
Err(()) => {
// If errors are encountered while trying to highlight, just emit
// the unhighlighted source.
write!(out, "<pre><code>{}</code></pre>", src).unwrap();
}
}
@ -192,14 +187,20 @@ impl<'a> Classifier<'a> {
if let Some(token) = self.peek_token.take() {
return Ok(token);
}
self.lexer.try_next_token().map_err(|()| HighlightError::LexError)
let token = self.lexer.next_token();
if let token::Unknown(..) = &token.kind {
return Err(HighlightError::LexError);
}
Ok(token)
}
fn peek(&mut self) -> Result<&Token, HighlightError> {
if self.peek_token.is_none() {
self.peek_token = Some(
self.lexer.try_next_token().map_err(|()| HighlightError::LexError)?
);
let token = self.lexer.next_token();
if let token::Unknown(..) = &token.kind {
return Err(HighlightError::LexError);
}
self.peek_token = Some(token);
}
Ok(self.peek_token.as_ref().unwrap())
}

32
src/librustdoc/passes/check_code_block_syntax.rs
View file

@ -32,24 +32,20 @@ impl<'a, 'tcx> SyntaxChecker<'a, 'tcx> {
dox[code_block.code].to_owned(),
);
let errors = {
let has_errors = {
let mut has_errors = false;
let mut lexer = Lexer::new(&sess, source_file, None);
while let Ok(token::Token { kind, .. }) = lexer.try_next_token() {
if kind == token::Eof {
break;
loop {
match lexer.next_token().kind {
token::Eof => break,
token::Unknown(..) => has_errors = true,
_ => (),
}
}
let errors = lexer.buffer_fatal_errors();
if !errors.is_empty() {
Err(errors)
} else {
Ok(())
}
has_errors
};
if let Err(errors) = errors {
if has_errors {
let mut diag = if let Some(sp) =
super::source_span_for_markdown_range(self.cx, &dox, &code_block.range, &item.attrs)
{
@ -58,11 +54,6 @@ impl<'a, 'tcx> SyntaxChecker<'a, 'tcx> {
.sess()
.struct_span_warn(sp, "could not parse code block as Rust code");
for mut err in errors {
diag.note(&format!("error from rustc: {}", err.message()));
err.cancel();
}
if code_block.syntax.is_none() && code_block.is_fenced {
let sp = sp.from_inner(InnerSpan::new(0, 3));
diag.span_suggestion(
@ -82,11 +73,6 @@ impl<'a, 'tcx> SyntaxChecker<'a, 'tcx> {
"doc comment contains an invalid Rust code block",
);
for mut err in errors {
// Don't bother reporting the error, because we can't show where it happened.
err.cancel();
}
if code_block.syntax.is_none() && code_block.is_fenced {
diag.help("mark blocks that do not contain Rust code as text: ```text");
}

160
src/test/rustdoc-ui/invalid-syntax.stderr
View file

@ -1,3 +1,21 @@
error: unknown start of token: \
--> <doctest>:1:1
|
1 | \__________pkt->size___________/ \_result->size_/ \__pkt->size__/
| ^
error: unknown start of token: \
--> <doctest>:1:43
|
1 | \__________pkt->size___________/ \_result->size_/ \__pkt->size__/
| ^
error: unknown start of token: \
--> <doctest>:1:60
|
1 | \__________pkt->size___________/ \_result->size_/ \__pkt->size__/
| ^
warning: could not parse code block as Rust code
--> $DIR/invalid-syntax.rs:3:5
|
@ -6,13 +24,31 @@ LL | /// ```
LL | | /// \__________pkt->size___________/ \_result->size_/ \__pkt->size__/
LL | | /// ```
| |_______^
|
= note: error from rustc: unknown start of token: \
help: mark blocks that do not contain Rust code as text
|
LL | /// ```text
| ^^^^^^^
error: unknown start of token: `
--> <doctest>:3:30
|
3 | | ^^^^^^ did you mean `baz::foobar`?
| ^
help: Unicode character '`' (Grave Accent) looks like ''' (Single Quote), but it is not
|
3 | | ^^^^^^ did you mean 'baz::foobar`?
| ^
error: unknown start of token: `
--> <doctest>:3:42
|
3 | | ^^^^^^ did you mean `baz::foobar`?
| ^
help: Unicode character '`' (Grave Accent) looks like ''' (Single Quote), but it is not
|
3 | | ^^^^^^ did you mean `baz::foobar'?
| ^
warning: could not parse code block as Rust code
--> $DIR/invalid-syntax.rs:8:5
|
@ -23,13 +59,17 @@ LL | | /// LL | use foobar::Baz;
LL | | /// | ^^^^^^ did you mean `baz::foobar`?
LL | | /// ```
| |_______^
|
= note: error from rustc: unknown start of token: `
help: mark blocks that do not contain Rust code as text
|
LL | /// ```text
| ^^^^^^^
error: unknown start of token: \
--> <doctest>:1:1
|
1 | \_
| ^
warning: could not parse code block as Rust code
--> $DIR/invalid-syntax.rs:19:5
|
@ -38,13 +78,17 @@ LL | /// ```
LL | | /// \_
LL | | /// ```
| |_______^
|
= note: error from rustc: unknown start of token: \
help: mark blocks that do not contain Rust code as text
|
LL | /// ```text
| ^^^^^^^
error: unknown start of token: \
--> <doctest>:1:1
|
1 | \_
| ^
warning: could not parse code block as Rust code
--> $DIR/invalid-syntax.rs:32:5
|
@ -53,8 +97,12 @@ LL | /// ```rust
LL | | /// \_
LL | | /// ```
| |_______^
error: unknown start of token: \
--> <doctest>:2:5
|
= note: error from rustc: unknown start of token: \
2 | \_
| ^
warning: could not parse code block as Rust code
--> $DIR/invalid-syntax.rs:41:9
@ -63,16 +111,48 @@ LL | /// code with bad syntax
| _________^
LL | | /// \_
| |__________^
error: unknown start of token: `
--> <doctest>:1:1
|
= note: error from rustc: unknown start of token: \
1 | ```
| ^
help: Unicode character '`' (Grave Accent) looks like ''' (Single Quote), but it is not
|
1 | '``
| ^
error: unknown start of token: `
--> <doctest>:1:2
|
1 | ```
| ^
help: Unicode character '`' (Grave Accent) looks like ''' (Single Quote), but it is not
|
1 | `'`
| ^
error: unknown start of token: `
--> <doctest>:1:3
|
1 | ```
| ^
help: Unicode character '`' (Grave Accent) looks like ''' (Single Quote), but it is not
|
1 | ``'
| ^
warning: could not parse code block as Rust code
--> $DIR/invalid-syntax.rs:55:9
|
LL | /// ```
| ^^^
error: unknown start of token: \
--> <doctest>:1:1
|
= note: error from rustc: unknown start of token: `
1 | \_
| ^
warning: could not parse code block as Rust code
--> $DIR/invalid-syntax.rs:58:5
@ -82,8 +162,12 @@ LL | /// ```edition2018
LL | | /// \_
LL | | /// ```
| |_______^
error: unknown start of token: \
--> <doctest>:1:1
|
= note: error from rustc: unknown start of token: \
1 | \_
| ^
warning: doc comment contains an invalid Rust code block
--> $DIR/invalid-syntax.rs:63:1
@ -95,3 +179,59 @@ LL | | #[doc = "```"]
|
= help: mark blocks that do not contain Rust code as text: ```text
error: unknown start of token: \
--> <rustdoc-highlighting>:1:1
|
1 | \_
| ^
error: unknown start of token: \
--> <rustdoc-highlighting>:1:1
|
1 | \_
| ^
error: unknown start of token: `
--> <rustdoc-highlighting>:1:1
|
1 | ```
| ^
help: Unicode character '`' (Grave Accent) looks like ''' (Single Quote), but it is not
|
1 | '``
| ^
error: unknown start of token: \
--> <rustdoc-highlighting>:2:1
|
2 | \_
| ^
error: unknown start of token: \
--> <rustdoc-highlighting>:1:1
|
1 | \_
| ^
error: unknown start of token: \
--> <rustdoc-highlighting>:1:1
|
1 | \_
| ^
error: unknown start of token: `
--> <rustdoc-highlighting>:3:30
|
3 | | ^^^^^^ did you mean `baz::foobar`?
| ^
help: Unicode character '`' (Grave Accent) looks like ''' (Single Quote), but it is not
|
3 | | ^^^^^^ did you mean 'baz::foobar`?
| ^
error: unknown start of token: \
--> <rustdoc-highlighting>:1:1
|
1 | \__________pkt->size___________/ \_result->size_/ \__pkt->size__/
| ^