Rollup merge of #120221 - compiler-errors:statements-are-not-patterns, r=nnethercote

Don't make statement nonterminals match pattern nonterminals

Right now, the heuristic we use to check if a token may begin a pattern nonterminal falls back to `may_be_ident`:
ef71f1047e/compiler/rustc_parse/src/parser/nonterminal.rs (L21-L37)

This has the unfortunate side effect that a `stmt` nonterminal eagerly matches against a `pat` nonterminal, leading to a parse error:
```rust
macro_rules! m {
    ($pat:pat) => {};
    ($stmt:stmt) => {};
}

macro_rules! m2 {
    ($stmt:stmt) => {
        m! { $stmt }
    };
}

m2! { let x = 1 }
```

This PR fixes it by more accurately reflecting the set of nonterminals that may begin a pattern nonterminal.

As a side-effect, I modified `Token::can_begin_pattern` to work correctly and used that in `Parser::nonterminal_may_begin_with`.

compiler/rustc_ast/src/token.rs

@@ -486,6 +486,9 @@ impl Token {
     }
 
     /// Returns `true` if the token can appear at the start of an expression.
+    ///
+    /// **NB**: Take care when modifying this function, since it will change
+    /// the stable set of tokens that are allowed to match an expr nonterminal.
     pub fn can_begin_expr(&self) -> bool {
         match self.uninterpolate().kind {
             Ident(name, is_raw)              =>
@@ -504,10 +507,13 @@ impl Token {
             PathSep                            | // global path
             Lifetime(..)                      | // labeled loop
             Pound                             => true, // expression attributes
-            Interpolated(ref nt) => matches!(&**nt, NtLiteral(..) |
+            Interpolated(ref nt) =>
-                NtExpr(..)    |
+                matches!(&**nt,
-                NtBlock(..)   |
+                    NtBlock(..)   |
-                NtPath(..)),
+                    NtExpr(..)    |
+                    NtLiteral(..) |
+                    NtPath(..)
+                ),
             _ => false,
         }
     }
@@ -515,23 +521,32 @@ impl Token {
     /// Returns `true` if the token can appear at the start of a pattern.
     ///
     /// Shamelessly borrowed from `can_begin_expr`, only used for diagnostics right now.
-    pub fn can_begin_pattern(&self) -> bool {
+    pub fn can_begin_pattern(&self, pat_kind: NtPatKind) -> bool {
-        match self.uninterpolate().kind {
+        match &self.uninterpolate().kind {
-            Ident(name, is_raw)              =>
+            // box, ref, mut, and other identifiers (can stricten)
-                ident_can_begin_expr(name, self.span, is_raw), // value name or keyword
+            Ident(..) | NtIdent(..) |
-            | OpenDelim(Delimiter::Bracket | Delimiter::Parenthesis)  // tuple or array
+            OpenDelim(Delimiter::Parenthesis) |  // tuple pattern
-            | Literal(..)                        // literal
+            OpenDelim(Delimiter::Bracket) |      // slice pattern
-            | BinOp(Minus)                       // unary minus
+            BinOp(And) |                  // reference
-            | BinOp(And)                         // reference
+            BinOp(Minus) |                // negative literal
-            | AndAnd                             // double reference
+            AndAnd |                      // double reference
-            // DotDotDot is no longer supported
+            Literal(_) |                  // literal
-            | DotDot | DotDotDot | DotDotEq      // ranges
+            DotDot |                      // range pattern (future compat)
-            | Lt | BinOp(Shl)                    // associated path
+            DotDotDot |                   // range pattern (future compat)
-            | PathSep                    => true, // global path
+            PathSep |                     // path
-            Interpolated(ref nt) => matches!(&**nt, NtLiteral(..) |
+            Lt |                          // path (UFCS constant)
-                NtPat(..)     |
+            BinOp(Shl) => true,           // path (double UFCS)
-                NtBlock(..)   |
+            // leading vert `|` or-pattern
-                NtPath(..)),
+            BinOp(Or) => matches!(pat_kind, PatWithOr),
+            Interpolated(nt) =>
+                matches!(&**nt,
+                    | NtExpr(..)
+                    | NtLiteral(..)
+                    | NtMeta(..)
+                    | NtPat(..)
+                    | NtPath(..)
+                    | NtTy(..)
+                ),
             _ => false,
         }
     }

compiler/rustc_parse/src/parser/nonterminal.rs

@@ -86,25 +86,7 @@ impl<'a> Parser<'a> {
                 token::Interpolated(nt) => may_be_ident(nt),
                 _ => false,
             },
-            NonterminalKind::Pat(pat_kind) => match &token.kind {
+            NonterminalKind::Pat(pat_kind) => token.can_begin_pattern(pat_kind),
-                // box, ref, mut, and other identifiers (can stricten)
-                token::Ident(..) | token::NtIdent(..) |
-                token::OpenDelim(Delimiter::Parenthesis) |  // tuple pattern
-                token::OpenDelim(Delimiter::Bracket) |      // slice pattern
-                token::BinOp(token::And) |                  // reference
-                token::BinOp(token::Minus) |                // negative literal
-                token::AndAnd |                             // double reference
-                token::Literal(_) |                         // literal
-                token::DotDot |                             // range pattern (future compat)
-                token::DotDotDot |                          // range pattern (future compat)
-                token::PathSep |                             // path
-                token::Lt |                                 // path (UFCS constant)
-                token::BinOp(token::Shl) => true,           // path (double UFCS)
-                // leading vert `|` or-pattern
-                token::BinOp(token::Or) => matches!(pat_kind, PatWithOr),
-                token::Interpolated(nt) => may_be_ident(nt),
-                _ => false,
-            },
             NonterminalKind::Lifetime => match &token.kind {
                 token::Lifetime(_) | token::NtLifetime(..) => true,
                 _ => false,

compiler/rustc_parse/src/parser/pat.rs

@@ -444,7 +444,11 @@ impl<'a> Parser<'a> {
 
         let mut lo = self.token.span;
 
-        if self.token.is_keyword(kw::Let) && self.look_ahead(1, |tok| tok.can_begin_pattern()) {
+        if self.token.is_keyword(kw::Let)
+            && self.look_ahead(1, |tok| {
+                tok.can_begin_pattern(token::NtPatKind::PatParam { inferred: false })
+            })
+        {
             self.bump();
             self.dcx().emit_err(RemoveLet { span: lo });
             lo = self.token.span;

compiler/rustc_parse/src/parser/path.rs

@@ -378,7 +378,10 @@ impl<'a> Parser<'a> {
                     if self.may_recover()
                         && prev_token_before_parsing == token::PathSep
                         && (style == PathStyle::Expr && self.token.can_begin_expr()
-                            || style == PathStyle::Pat && self.token.can_begin_pattern())
+                            || style == PathStyle::Pat
+                                && self.token.can_begin_pattern(token::NtPatKind::PatParam {
+                                    inferred: false,
+                                }))
                     {
                         snapshot = Some(self.create_snapshot_for_diagnostic());
                     }

tests/ui/pattern/patterns-dont-match-nt-statement.rs

@@ -0,0 +1,19 @@
+//@ check-pass
+
+// Make sure that a `stmt` nonterminal does not eagerly match against
+// a `pat`, since this will always cause a parse error...
+
+macro_rules! m {
+    ($pat:pat) => {};
+    ($stmt:stmt) => {};
+}
+
+macro_rules! m2 {
+    ($stmt:stmt) => {
+        m! { $stmt }
+    };
+}
+
+m2! { let x = 1 }
+
+fn main() {}