Rollup merge of #126841 - c410-f3r:concat-again, r=petrochenkov

[`macro_metavar_expr_concat`] Add support for literals

Adds support for things like `${concat($variable, 123)}` or `${concat("hello", "_world")}` .

cc #124225

compiler/rustc_expand/src/mbe/metavar_expr.rs

@@ -1,4 +1,4 @@
-use rustc_ast::token::{self, Delimiter, IdentIsRaw};
+use rustc_ast::token::{self, Delimiter, IdentIsRaw, Lit, Token, TokenKind};
 use rustc_ast::tokenstream::{RefTokenTreeCursor, TokenStream, TokenTree};
 use rustc_ast::{LitIntType, LitKind};
 use rustc_ast_pretty::pprust;
@@ -6,9 +6,10 @@ use rustc_errors::{Applicability, PResult};
 use rustc_macros::{Decodable, Encodable};
 use rustc_session::parse::ParseSess;
 use rustc_span::symbol::Ident;
-use rustc_span::Span;
+use rustc_span::{Span, Symbol};
 
 pub(crate) const RAW_IDENT_ERR: &str = "`${concat(..)}` currently does not support raw identifiers";
+pub(crate) const UNSUPPORTED_CONCAT_ELEM_ERR: &str = "expected identifier or string literal";
 
 /// A meta-variable expression, for expansions based on properties of meta-variables.
 #[derive(Debug, PartialEq, Encodable, Decodable)]
@@ -51,11 +52,26 @@ impl MetaVarExpr {
                 let mut result = Vec::new();
                 loop {
                     let is_var = try_eat_dollar(&mut iter);
-                    let element_ident = parse_ident(&mut iter, psess, outer_span)?;
+                    let token = parse_token(&mut iter, psess, outer_span)?;
                     let element = if is_var {
-                        MetaVarExprConcatElem::Var(element_ident)
+                        MetaVarExprConcatElem::Var(parse_ident_from_token(psess, token)?)
+                    } else if let TokenKind::Literal(Lit {
+                        kind: token::LitKind::Str,
+                        symbol,
+                        suffix: None,
+                    }) = token.kind
+                    {
+                        MetaVarExprConcatElem::Literal(symbol)
                     } else {
-                        MetaVarExprConcatElem::Ident(element_ident)
+                        match parse_ident_from_token(psess, token) {
+                            Err(err) => {
+                                err.cancel();
+                                return Err(psess
+                                    .dcx()
+                                    .struct_span_err(token.span, UNSUPPORTED_CONCAT_ELEM_ERR));
+                            }
+                            Ok(elem) => MetaVarExprConcatElem::Ident(elem),
+                        }
                     };
                     result.push(element);
                     if iter.look_ahead(0).is_none() {
@@ -105,11 +121,13 @@ impl MetaVarExpr {
 
 #[derive(Debug, Decodable, Encodable, PartialEq)]
 pub(crate) enum MetaVarExprConcatElem {
-    /// There is NO preceding dollar sign, which means that this identifier should be interpreted
+    /// Identifier WITHOUT a preceding dollar sign, which means that this identifier should be
-    /// as a literal.
+    /// interpreted as a literal.
     Ident(Ident),
-    /// There is a preceding dollar sign, which means that this identifier should be expanded
+    /// For example, a number or a string.
-    /// and interpreted as a variable.
+    Literal(Symbol),
+    /// Identifier WITH a preceding dollar sign, which means that this identifier should be
+    /// expanded and interpreted as a variable.
     Var(Ident),
 }
 
@@ -158,7 +176,7 @@ fn parse_depth<'psess>(
     span: Span,
 ) -> PResult<'psess, usize> {
     let Some(tt) = iter.next() else { return Ok(0) };
-    let TokenTree::Token(token::Token { kind: token::TokenKind::Literal(lit), .. }, _) = tt else {
+    let TokenTree::Token(Token { kind: TokenKind::Literal(lit), .. }, _) = tt else {
         return Err(psess
             .dcx()
             .struct_span_err(span, "meta-variable expression depth must be a literal"));
@@ -180,12 +198,14 @@ fn parse_ident<'psess>(
     psess: &'psess ParseSess,
     fallback_span: Span,
 ) -> PResult<'psess, Ident> {
-    let Some(tt) = iter.next() else {
+    let token = parse_token(iter, psess, fallback_span)?;
-        return Err(psess.dcx().struct_span_err(fallback_span, "expected identifier"));
+    parse_ident_from_token(psess, token)
-    };
+}
-    let TokenTree::Token(token, _) = tt else {
+
-        return Err(psess.dcx().struct_span_err(tt.span(), "expected identifier"));
+fn parse_ident_from_token<'psess>(
-    };
+    psess: &'psess ParseSess,
+    token: &Token,
+) -> PResult<'psess, Ident> {
     if let Some((elem, is_raw)) = token.ident() {
         if let IdentIsRaw::Yes = is_raw {
             return Err(psess.dcx().struct_span_err(elem.span, RAW_IDENT_ERR));
@@ -205,10 +225,24 @@ fn parse_ident<'psess>(
     Err(err)
 }
 
+fn parse_token<'psess, 't>(
+    iter: &mut RefTokenTreeCursor<'t>,
+    psess: &'psess ParseSess,
+    fallback_span: Span,
+) -> PResult<'psess, &'t Token> {
+    let Some(tt) = iter.next() else {
+        return Err(psess.dcx().struct_span_err(fallback_span, UNSUPPORTED_CONCAT_ELEM_ERR));
+    };
+    let TokenTree::Token(token, _) = tt else {
+        return Err(psess.dcx().struct_span_err(tt.span(), UNSUPPORTED_CONCAT_ELEM_ERR));
+    };
+    Ok(token)
+}
+
 /// Tries to move the iterator forward returning `true` if there is a comma. If not, then the
 /// iterator is not modified and the result is `false`.
 fn try_eat_comma(iter: &mut RefTokenTreeCursor<'_>) -> bool {
-    if let Some(TokenTree::Token(token::Token { kind: token::Comma, .. }, _)) = iter.look_ahead(0) {
+    if let Some(TokenTree::Token(Token { kind: token::Comma, .. }, _)) = iter.look_ahead(0) {
         let _ = iter.next();
         return true;
     }
@@ -218,8 +252,7 @@ fn try_eat_comma(iter: &mut RefTokenTreeCursor<'_>) -> bool {
 /// Tries to move the iterator forward returning `true` if there is a dollar sign. If not, then the
 /// iterator is not modified and the result is `false`.
 fn try_eat_dollar(iter: &mut RefTokenTreeCursor<'_>) -> bool {
-    if let Some(TokenTree::Token(token::Token { kind: token::Dollar, .. }, _)) = iter.look_ahead(0)
+    if let Some(TokenTree::Token(Token { kind: token::Dollar, .. }, _)) = iter.look_ahead(0) {
-    {
         let _ = iter.next();
         return true;
     }
@@ -232,8 +265,7 @@ fn eat_dollar<'psess>(
     psess: &'psess ParseSess,
     span: Span,
 ) -> PResult<'psess, ()> {
-    if let Some(TokenTree::Token(token::Token { kind: token::Dollar, .. }, _)) = iter.look_ahead(0)
+    if let Some(TokenTree::Token(Token { kind: token::Dollar, .. }, _)) = iter.look_ahead(0) {
-    {
         let _ = iter.next();
         return Ok(());
     }

compiler/rustc_expand/src/mbe/transcribe.rs

@@ -11,11 +11,13 @@ use rustc_ast::token::{self, Delimiter, Token, TokenKind};
 use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree};
 use rustc_data_structures::fx::FxHashMap;
 use rustc_errors::{pluralize, Diag, DiagCtxtHandle, PResult};
+use rustc_parse::lexer::nfc_normalize;
 use rustc_parse::parser::ParseNtResult;
 use rustc_session::parse::ParseSess;
+use rustc_session::parse::SymbolGallery;
 use rustc_span::hygiene::{LocalExpnId, Transparency};
 use rustc_span::symbol::{sym, Ident, MacroRulesNormalizedIdent};
-use rustc_span::{with_metavar_spans, Span, Symbol, SyntaxContext};
+use rustc_span::{with_metavar_spans, Span, SyntaxContext};
 use smallvec::{smallvec, SmallVec};
 use std::mem;
 
@@ -312,7 +314,16 @@ pub(super) fn transcribe<'a>(
 
             // Replace meta-variable expressions with the result of their expansion.
             mbe::TokenTree::MetaVarExpr(sp, expr) => {
-                transcribe_metavar_expr(dcx, expr, interp, &mut marker, &repeats, &mut result, sp)?;
+                transcribe_metavar_expr(
+                    dcx,
+                    expr,
+                    interp,
+                    &mut marker,
+                    &repeats,
+                    &mut result,
+                    sp,
+                    &psess.symbol_gallery,
+                )?;
             }
 
             // If we are entering a new delimiter, we push its contents to the `stack` to be
@@ -669,6 +680,7 @@ fn transcribe_metavar_expr<'a>(
     repeats: &[(usize, usize)],
     result: &mut Vec<TokenTree>,
     sp: &DelimSpan,
+    symbol_gallery: &SymbolGallery,
 ) -> PResult<'a, ()> {
     let mut visited_span = || {
         let mut span = sp.entire();
@@ -680,16 +692,26 @@ fn transcribe_metavar_expr<'a>(
             let mut concatenated = String::new();
             for element in elements.into_iter() {
                 let string = match element {
-                    MetaVarExprConcatElem::Ident(ident) => ident.to_string(),
+                    MetaVarExprConcatElem::Ident(elem) => elem.to_string(),
-                    MetaVarExprConcatElem::Var(ident) => extract_ident(dcx, *ident, interp)?,
+                    MetaVarExprConcatElem::Literal(elem) => elem.as_str().into(),
+                    MetaVarExprConcatElem::Var(elem) => extract_ident(dcx, *elem, interp)?,
                 };
                 concatenated.push_str(&string);
             }
+            let symbol = nfc_normalize(&concatenated);
+            let concatenated_span = visited_span();
+            if !rustc_lexer::is_ident(symbol.as_str()) {
+                return Err(dcx.struct_span_err(
+                    concatenated_span,
+                    "`${concat(..)}` is not generating a valid identifier",
+                ));
+            }
+            symbol_gallery.insert(symbol, concatenated_span);
             // The current implementation marks the span as coming from the macro regardless of
             // contexts of the concatenated identifiers but this behavior may change in the
             // future.
             result.push(TokenTree::Token(
-                Token::from_ast_ident(Ident::new(Symbol::intern(&concatenated), visited_span())),
+                Token::from_ast_ident(Ident::new(symbol, concatenated_span)),
                 Spacing::Alone,
             ));
         }

tests/ui/macros/macro-metavar-expr-concat/allowed-operations.rs

@@ -37,6 +37,16 @@ macro_rules! without_dollar_sign_is_an_ident {
     };
 }
 
+macro_rules! literals {
+    ($ident:ident) => {{
+        let ${concat(_a, "_b")}: () = ();
+        let ${concat("_b", _a)}: () = ();
+
+        let ${concat($ident, "_b")}: () = ();
+        let ${concat("_b", $ident)}: () = ();
+    }};
+}
+
 fn main() {
     create_things!(behold);
     behold_separated_idents_in_a_fn();
@@ -55,4 +65,6 @@ fn main() {
     without_dollar_sign_is_an_ident!(_123);
     assert_eq!(VARident, 1);
     assert_eq!(VAR_123, 2);
+
+    literals!(_hello);
 }

tests/ui/macros/macro-metavar-expr-concat/raw-identifiers.rs

@@ -26,14 +26,14 @@ macro_rules! idents_11 {
 macro_rules! no_params {
     () => {
         let ${concat(r#abc, abc)}: () = ();
-        //~^ ERROR `${concat(..)}` currently does not support raw identifiers
+        //~^ ERROR expected identifier or string literal
         //~| ERROR expected pattern, found `$`
 
         let ${concat(abc, r#abc)}: () = ();
-        //~^ ERROR `${concat(..)}` currently does not support raw identifiers
+        //~^ ERROR expected identifier or string literal
 
         let ${concat(r#abc, r#abc)}: () = ();
-        //~^ ERROR `${concat(..)}` currently does not support raw identifiers
+        //~^ ERROR expected identifier or string literal
     };
 }
 

tests/ui/macros/macro-metavar-expr-concat/raw-identifiers.stderr

@@ -1,16 +1,16 @@
-error: `${concat(..)}` currently does not support raw identifiers
+error: expected identifier or string literal
   --> $DIR/raw-identifiers.rs:28:22
    |
 LL |         let ${concat(r#abc, abc)}: () = ();
    |                      ^^^^^
 
-error: `${concat(..)}` currently does not support raw identifiers
+error: expected identifier or string literal
   --> $DIR/raw-identifiers.rs:32:27
    |
 LL |         let ${concat(abc, r#abc)}: () = ();
    |                           ^^^^^
 
-error: `${concat(..)}` currently does not support raw identifiers
+error: expected identifier or string literal
   --> $DIR/raw-identifiers.rs:35:22
    |
 LL |         let ${concat(r#abc, r#abc)}: () = ();

tests/ui/macros/macro-metavar-expr-concat/syntax-errors.rs

@@ -11,9 +11,6 @@ macro_rules! wrong_concat_declarations {
         ${concat(aaaa,)}
         //~^ ERROR expected identifier
 
-        ${concat(aaaa, 1)}
-        //~^ ERROR expected identifier
-
         ${concat(_, aaaa)}
 
         ${concat(aaaa aaaa)}
@@ -30,9 +27,6 @@ macro_rules! wrong_concat_declarations {
 
         ${concat($ex, aaaa,)}
         //~^ ERROR expected identifier
-
-        ${concat($ex, aaaa, 123)}
-        //~^ ERROR expected identifier
     };
 }
 
@@ -43,8 +37,80 @@ macro_rules! dollar_sign_without_referenced_ident {
     };
 }
 
+macro_rules! starting_number {
+    ($ident:ident) => {{
+        let ${concat("1", $ident)}: () = ();
+        //~^ ERROR `${concat(..)}` is not generating a valid identifier
+    }};
+}
+
+macro_rules! starting_valid_unicode {
+    ($ident:ident) => {{
+        let ${concat("Ý", $ident)}: () = ();
+    }};
+}
+
+macro_rules! starting_invalid_unicode {
+    ($ident:ident) => {{
+        let ${concat("\u{00BD}", $ident)}: () = ();
+        //~^ ERROR `${concat(..)}` is not generating a valid identifier
+    }};
+}
+
+macro_rules! ending_number {
+    ($ident:ident) => {{
+        let ${concat($ident, "1")}: () = ();
+    }};
+}
+
+macro_rules! ending_valid_unicode {
+    ($ident:ident) => {{
+        let ${concat($ident, "Ý")}: () = ();
+    }};
+}
+
+macro_rules! ending_invalid_unicode {
+    ($ident:ident) => {{
+        let ${concat($ident, "\u{00BD}")}: () = ();
+        //~^ ERROR `${concat(..)}` is not generating a valid identifier
+    }};
+}
+
+macro_rules! empty {
+    () => {{
+        let ${concat("", "")}: () = ();
+        //~^ ERROR `${concat(..)}` is not generating a valid identifier
+    }};
+}
+
+macro_rules! unsupported_literals {
+    ($ident:ident) => {{
+        let ${concat(_a, 'b')}: () = ();
+        //~^ ERROR expected identifier or string literal
+        //~| ERROR expected pattern
+        let ${concat(_a, 1)}: () = ();
+        //~^ ERROR expected identifier or string literal
+
+        let ${concat($ident, 'b')}: () = ();
+        //~^ ERROR expected identifier or string literal
+        let ${concat($ident, 1)}: () = ();
+        //~^ ERROR expected identifier or string literal
+    }};
+}
+
 fn main() {
     wrong_concat_declarations!(1);
 
     dollar_sign_without_referenced_ident!(VAR);
+
+    starting_number!(_abc);
+    starting_valid_unicode!(_abc);
+    starting_invalid_unicode!(_abc);
+
+    ending_number!(_abc);
+    ending_valid_unicode!(_abc);
+    ending_invalid_unicode!(_abc);
+    unsupported_literals!(_abc);
+
+    empty!();
 }

tests/ui/macros/macro-metavar-expr-concat/syntax-errors.stderr

@@ -1,4 +1,4 @@
-error: expected identifier
+error: expected identifier or string literal
   --> $DIR/syntax-errors.rs:5:10
    |
 LL |         ${concat()}
@@ -10,59 +10,126 @@ error: `concat` must have at least two elements
 LL |         ${concat(aaaa)}
    |           ^^^^^^
 
-error: expected identifier
+error: expected identifier or string literal
   --> $DIR/syntax-errors.rs:11:10
    |
 LL |         ${concat(aaaa,)}
    |          ^^^^^^^^^^^^^^^
 
-error: expected identifier, found `1`
-  --> $DIR/syntax-errors.rs:14:24
-   |
-LL |         ${concat(aaaa, 1)}
-   |                        ^ help: try removing `1`
-
 error: expected comma
-  --> $DIR/syntax-errors.rs:19:10
+  --> $DIR/syntax-errors.rs:16:10
    |
 LL |         ${concat(aaaa aaaa)}
    |          ^^^^^^^^^^^^^^^^^^^
 
 error: `concat` must have at least two elements
-  --> $DIR/syntax-errors.rs:22:11
+  --> $DIR/syntax-errors.rs:19:11
    |
 LL |         ${concat($ex)}
    |           ^^^^^^
 
 error: expected comma
-  --> $DIR/syntax-errors.rs:28:10
+  --> $DIR/syntax-errors.rs:25:10
    |
 LL |         ${concat($ex, aaaa 123)}
    |          ^^^^^^^^^^^^^^^^^^^^^^^
 
-error: expected identifier
+error: expected identifier or string literal
-  --> $DIR/syntax-errors.rs:31:10
+  --> $DIR/syntax-errors.rs:28:10
    |
 LL |         ${concat($ex, aaaa,)}
    |          ^^^^^^^^^^^^^^^^^^^^
 
-error: expected identifier, found `123`
+error: expected identifier or string literal
-  --> $DIR/syntax-errors.rs:34:29
+  --> $DIR/syntax-errors.rs:88:26
    |
-LL |         ${concat($ex, aaaa, 123)}
+LL |         let ${concat(_a, 'b')}: () = ();
-   |                             ^^^ help: try removing `123`
+   |                          ^^^
+
+error: expected identifier or string literal
+  --> $DIR/syntax-errors.rs:91:26
+   |
+LL |         let ${concat(_a, 1)}: () = ();
+   |                          ^
+
+error: expected identifier or string literal
+  --> $DIR/syntax-errors.rs:94:30
+   |
+LL |         let ${concat($ident, 'b')}: () = ();
+   |                              ^^^
+
+error: expected identifier or string literal
+  --> $DIR/syntax-errors.rs:96:30
+   |
+LL |         let ${concat($ident, 1)}: () = ();
+   |                              ^
 
 error: `${concat(..)}` currently only accepts identifiers or meta-variables as parameters
-  --> $DIR/syntax-errors.rs:25:19
+  --> $DIR/syntax-errors.rs:22:19
    |
 LL |         ${concat($ex, aaaa)}
    |                   ^^
 
 error: variable `foo` is not recognized in meta-variable expression
-  --> $DIR/syntax-errors.rs:41:30
+  --> $DIR/syntax-errors.rs:35:30
    |
 LL |         const ${concat(FOO, $foo)}: i32 = 2;
    |                              ^^^
 
-error: aborting due to 11 previous errors
+error: `${concat(..)}` is not generating a valid identifier
+  --> $DIR/syntax-errors.rs:42:14
+   |
+LL |         let ${concat("1", $ident)}: () = ();
+   |              ^^^^^^^^^^^^^^^^^^^^^
+...
+LL |     starting_number!(_abc);
+   |     ---------------------- in this macro invocation
+   |
+   = note: this error originates in the macro `starting_number` (in Nightly builds, run with -Z macro-backtrace for more info)
+
+error: `${concat(..)}` is not generating a valid identifier
+  --> $DIR/syntax-errors.rs:55:14
+   |
+LL |         let ${concat("\u{00BD}", $ident)}: () = ();
+   |              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+...
+LL |     starting_invalid_unicode!(_abc);
+   |     ------------------------------- in this macro invocation
+   |
+   = note: this error originates in the macro `starting_invalid_unicode` (in Nightly builds, run with -Z macro-backtrace for more info)
+
+error: `${concat(..)}` is not generating a valid identifier
+  --> $DIR/syntax-errors.rs:74:14
+   |
+LL |         let ${concat($ident, "\u{00BD}")}: () = ();
+   |              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+...
+LL |     ending_invalid_unicode!(_abc);
+   |     ----------------------------- in this macro invocation
+   |
+   = note: this error originates in the macro `ending_invalid_unicode` (in Nightly builds, run with -Z macro-backtrace for more info)
+
+error: expected pattern, found `$`
+  --> $DIR/syntax-errors.rs:88:13
+   |
+LL |         let ${concat(_a, 'b')}: () = ();
+   |             ^ expected pattern
+...
+LL |     unsupported_literals!(_abc);
+   |     --------------------------- in this macro invocation
+   |
+   = note: this error originates in the macro `unsupported_literals` (in Nightly builds, run with -Z macro-backtrace for more info)
+
+error: `${concat(..)}` is not generating a valid identifier
+  --> $DIR/syntax-errors.rs:81:14
+   |
+LL |         let ${concat("", "")}: () = ();
+   |              ^^^^^^^^^^^^^^^^
+...
+LL |     empty!();
+   |     -------- in this macro invocation
+   |
+   = note: this error originates in the macro `empty` (in Nightly builds, run with -Z macro-backtrace for more info)
+
+error: aborting due to 18 previous errors
 

tests/ui/macros/macro-metavar-expr-concat/unicode-expansion.rs

@@ -0,0 +1,14 @@
+//@ run-pass
+
+#![feature(macro_metavar_expr_concat)]
+
+macro_rules! turn_to_page {
+    ($ident:ident) => {
+        const ${concat("Ḧ", $ident)}: i32 = 394;
+    };
+}
+
+fn main() {
+    turn_to_page!(P);
+    assert_eq!(ḦP, 394);
+}

tests/ui/macros/rfc-3086-metavar-expr/syntax-errors.stderr

@@ -190,7 +190,7 @@ error: unrecognized meta-variable expression
 LL |     ( $( $i:ident ),* ) => { ${ aaaaaaaaaaaaaa(i) } };
    |                                 ^^^^^^^^^^^^^^ help: supported expressions are count, ignore, index and len
 
-error: expected identifier
+error: expected identifier or string literal
   --> $DIR/syntax-errors.rs:118:33
    |
 LL |     ( $( $i:ident ),* ) => { ${ {} } };