adapt rustdoc to infailable lexer

src/librustdoc/html/highlight.rs

@@ -44,7 +44,7 @@ pub fn render_with_highlighting(
 
         let mut highlighted_source = vec![];
         if classifier.write_source(&mut highlighted_source).is_err() {
-            Err(classifier.lexer.buffer_fatal_errors())
+            Err(())
         } else {
             Ok(String::from_utf8_lossy(&highlighted_source).into_owned())
         }
@@ -59,14 +59,9 @@ pub fn render_with_highlighting(
             }
             write_footer(&mut out).unwrap();
         }
-        Err(errors) => {
+        Err(()) => {
-            // If errors are encountered while trying to highlight, cancel the errors and just emit
+            // If errors are encountered while trying to highlight, just emit
-            // the unhighlighted source. The errors will have already been reported in the
+            // the unhighlighted source.
-            // `check-code-block-syntax` pass.
-            for mut error in errors {
-                error.cancel();
-            }
-
             write!(out, "<pre><code>{}</code></pre>", src).unwrap();
         }
     }
@@ -192,14 +187,20 @@ impl<'a> Classifier<'a> {
         if let Some(token) = self.peek_token.take() {
             return Ok(token);
         }
-        self.lexer.try_next_token().map_err(|()| HighlightError::LexError)
+        let token = self.lexer.next_token();
+        if let token::Unknown(..) = &token.kind {
+            return Err(HighlightError::LexError);
+        }
+        Ok(token)
     }
 
     fn peek(&mut self) -> Result<&Token, HighlightError> {
         if self.peek_token.is_none() {
-            self.peek_token = Some(
+            let token = self.lexer.next_token();
-                self.lexer.try_next_token().map_err(|()| HighlightError::LexError)?
+            if let token::Unknown(..) = &token.kind {
-            );
+                return Err(HighlightError::LexError);
+            }
+            self.peek_token = Some(token);
         }
         Ok(self.peek_token.as_ref().unwrap())
     }

src/librustdoc/passes/check_code_block_syntax.rs

@@ -32,24 +32,20 @@ impl<'a, 'tcx> SyntaxChecker<'a, 'tcx> {
             dox[code_block.code].to_owned(),
         );
 
-        let errors = {
+        let has_errors = {
+            let mut has_errors = false;
             let mut lexer = Lexer::new(&sess, source_file, None);
-            while let Ok(token::Token { kind, .. }) = lexer.try_next_token() {
+            loop  {
-                if kind == token::Eof {
+                match lexer.next_token().kind {
-                    break;
+                    token::Eof => break,
+                    token::Unknown(..) => has_errors = true,
+                    _ => (),
                 }
             }
-
+            has_errors
-            let errors = lexer.buffer_fatal_errors();
-
-            if !errors.is_empty() {
-                Err(errors)
-            } else {
-                Ok(())
-            }
         };
 
-        if let Err(errors) = errors {
+        if has_errors {
             let mut diag = if let Some(sp) =
                 super::source_span_for_markdown_range(self.cx, &dox, &code_block.range, &item.attrs)
             {
@@ -58,11 +54,6 @@ impl<'a, 'tcx> SyntaxChecker<'a, 'tcx> {
                     .sess()
                     .struct_span_warn(sp, "could not parse code block as Rust code");
 
-                for mut err in errors {
-                    diag.note(&format!("error from rustc: {}", err.message()));
-                    err.cancel();
-                }
-
                 if code_block.syntax.is_none() && code_block.is_fenced {
                     let sp = sp.from_inner(InnerSpan::new(0, 3));
                     diag.span_suggestion(
@@ -82,11 +73,6 @@ impl<'a, 'tcx> SyntaxChecker<'a, 'tcx> {
                     "doc comment contains an invalid Rust code block",
                 );
 
-                for mut err in errors {
-                    // Don't bother reporting the error, because we can't show where it happened.
-                    err.cancel();
-                }
-
                 if code_block.syntax.is_none() && code_block.is_fenced {
                     diag.help("mark blocks that do not contain Rust code as text: ```text");
                 }

src/test/rustdoc-ui/invalid-syntax.stderr

@@ -1,3 +1,21 @@
+error: unknown start of token: \
+ --> <doctest>:1:1
+  |
+1 | \__________pkt->size___________/          \_result->size_/ \__pkt->size__/
+  | ^
+
+error: unknown start of token: \
+ --> <doctest>:1:43
+  |
+1 | \__________pkt->size___________/          \_result->size_/ \__pkt->size__/
+  |                                           ^
+
+error: unknown start of token: \
+ --> <doctest>:1:60
+  |
+1 | \__________pkt->size___________/          \_result->size_/ \__pkt->size__/
+  |                                                            ^
+
 warning: could not parse code block as Rust code
   --> $DIR/invalid-syntax.rs:3:5
    |
@@ -6,13 +24,31 @@ LL |   /// ```
 LL | | /// \__________pkt->size___________/          \_result->size_/ \__pkt->size__/
 LL | | /// ```
    | |_______^
-   |
-   = note: error from rustc: unknown start of token: \
 help: mark blocks that do not contain Rust code as text
    |
 LL | /// ```text
    |     ^^^^^^^
 
+error: unknown start of token: `
+ --> <doctest>:3:30
+  |
+3 |    |     ^^^^^^ did you mean `baz::foobar`?
+  |                              ^
+help: Unicode character '`' (Grave Accent) looks like ''' (Single Quote), but it is not
+  |
+3 |    |     ^^^^^^ did you mean 'baz::foobar`?
+  |                              ^
+
+error: unknown start of token: `
+ --> <doctest>:3:42
+  |
+3 |    |     ^^^^^^ did you mean `baz::foobar`?
+  |                                          ^
+help: Unicode character '`' (Grave Accent) looks like ''' (Single Quote), but it is not
+  |
+3 |    |     ^^^^^^ did you mean `baz::foobar'?
+  |                                          ^
+
 warning: could not parse code block as Rust code
   --> $DIR/invalid-syntax.rs:8:5
    |
@@ -23,13 +59,17 @@ LL | | /// LL | use foobar::Baz;
 LL | | ///    |     ^^^^^^ did you mean `baz::foobar`?
 LL | | /// ```
    | |_______^
-   |
-   = note: error from rustc: unknown start of token: `
 help: mark blocks that do not contain Rust code as text
    |
 LL | /// ```text
    |     ^^^^^^^
 
+error: unknown start of token: \
+ --> <doctest>:1:1
+  |
+1 | \_
+  | ^
+
 warning: could not parse code block as Rust code
   --> $DIR/invalid-syntax.rs:19:5
    |
@@ -38,13 +78,17 @@ LL |   /// ```
 LL | | /// \_
 LL | | /// ```
    | |_______^
-   |
-   = note: error from rustc: unknown start of token: \
 help: mark blocks that do not contain Rust code as text
    |
 LL | /// ```text
    |     ^^^^^^^
 
+error: unknown start of token: \
+ --> <doctest>:1:1
+  |
+1 | \_
+  | ^
+
 warning: could not parse code block as Rust code
   --> $DIR/invalid-syntax.rs:32:5
    |
@@ -53,8 +97,12 @@ LL |   /// ```rust
 LL | | /// \_
 LL | | /// ```
    | |_______^
-   |
+
-   = note: error from rustc: unknown start of token: \
+error: unknown start of token: \
+ --> <doctest>:2:5
+  |
+2 |     \_
+  |     ^
 
 warning: could not parse code block as Rust code
   --> $DIR/invalid-syntax.rs:41:9
@@ -63,16 +111,48 @@ LL |   ///     code with bad syntax
    |  _________^
 LL | | ///     \_
    | |__________^
-   |
+
-   = note: error from rustc: unknown start of token: \
+error: unknown start of token: `
+ --> <doctest>:1:1
+  |
+1 | ```
+  | ^
+help: Unicode character '`' (Grave Accent) looks like ''' (Single Quote), but it is not
+  |
+1 | '``
+  | ^
+
+error: unknown start of token: `
+ --> <doctest>:1:2
+  |
+1 | ```
+  |  ^
+help: Unicode character '`' (Grave Accent) looks like ''' (Single Quote), but it is not
+  |
+1 | `'`
+  |  ^
+
+error: unknown start of token: `
+ --> <doctest>:1:3
+  |
+1 | ```
+  |   ^
+help: Unicode character '`' (Grave Accent) looks like ''' (Single Quote), but it is not
+  |
+1 | ``'
+  |   ^
 
 warning: could not parse code block as Rust code
   --> $DIR/invalid-syntax.rs:55:9
    |
 LL | ///     ```
    |         ^^^
-   |
+
-   = note: error from rustc: unknown start of token: `
+error: unknown start of token: \
+ --> <doctest>:1:1
+  |
+1 | \_
+  | ^
 
 warning: could not parse code block as Rust code
   --> $DIR/invalid-syntax.rs:58:5
@@ -82,8 +162,12 @@ LL |   /// ```edition2018
 LL | | /// \_
 LL | | /// ```
    | |_______^
-   |
+
-   = note: error from rustc: unknown start of token: \
+error: unknown start of token: \
+ --> <doctest>:1:1
+  |
+1 | \_
+  | ^
 
 warning: doc comment contains an invalid Rust code block
   --> $DIR/invalid-syntax.rs:63:1
@@ -95,3 +179,59 @@ LL | | #[doc = "```"]
    |
    = help: mark blocks that do not contain Rust code as text: ```text
 
+error: unknown start of token: \
+ --> <rustdoc-highlighting>:1:1
+  |
+1 | \_
+  | ^
+
+error: unknown start of token: \
+ --> <rustdoc-highlighting>:1:1
+  |
+1 | \_
+  | ^
+
+error: unknown start of token: `
+ --> <rustdoc-highlighting>:1:1
+  |
+1 | ```
+  | ^
+help: Unicode character '`' (Grave Accent) looks like ''' (Single Quote), but it is not
+  |
+1 | '``
+  | ^
+
+error: unknown start of token: \
+ --> <rustdoc-highlighting>:2:1
+  |
+2 | \_
+  | ^
+
+error: unknown start of token: \
+ --> <rustdoc-highlighting>:1:1
+  |
+1 | \_
+  | ^
+
+error: unknown start of token: \
+ --> <rustdoc-highlighting>:1:1
+  |
+1 | \_
+  | ^
+
+error: unknown start of token: `
+ --> <rustdoc-highlighting>:3:30
+  |
+3 |    |     ^^^^^^ did you mean `baz::foobar`?
+  |                              ^
+help: Unicode character '`' (Grave Accent) looks like ''' (Single Quote), but it is not
+  |
+3 |    |     ^^^^^^ did you mean 'baz::foobar`?
+  |                              ^
+
+error: unknown start of token: \
+ --> <rustdoc-highlighting>:1:1
+  |
+1 | \__________pkt->size___________/          \_result->size_/ \__pkt->size__/
+  | ^
+