mbe: Add tests and restructure metavariable expressions

compiler/rustc_expand/messages.ftl

@@ -109,9 +109,6 @@ expand_malformed_feature_attribute =
 
 expand_meta_var_dif_seq_matchers = {$msg}
 
-expand_meta_var_expr_unrecognized_var =
-    variable `{$key}` is not recognized in meta-variable expression
-
 expand_missing_fragment_specifier = missing fragment specifier
     .note = fragment specifiers must be provided
     .suggestion_add_fragspec = try adding a specifier here
@@ -136,6 +133,9 @@ expand_module_multiple_candidates =
 expand_must_repeat_once =
     this must repeat at least once
 
+expand_mve_unrecognized_var =
+    variable `{$key}` is not recognized in meta-variable expression
+
 expand_non_inline_modules_in_proc_macro_input_are_unstable =
     non-inline modules in proc macro input are unstable
 

compiler/rustc_expand/src/errors.rs

@@ -27,14 +27,6 @@ pub(crate) struct CountRepetitionMisplaced {
     pub span: Span,
 }
 
-#[derive(Diagnostic)]
-#[diag(expand_meta_var_expr_unrecognized_var)]
-pub(crate) struct MetaVarExprUnrecognizedVar {
-    #[primary_span]
-    pub span: Span,
-    pub key: MacroRulesNormalizedIdent,
-}
-
 #[derive(Diagnostic)]
 #[diag(expand_var_still_repeating)]
 pub(crate) struct VarStillRepeating {
@@ -500,3 +492,16 @@ pub(crate) struct ProcMacroBackCompat {
     pub crate_name: String,
     pub fixed_version: String,
 }
+
+pub(crate) use metavar_exprs::*;
+mod metavar_exprs {
+    use super::*;
+
+    #[derive(Diagnostic)]
+    #[diag(expand_mve_unrecognized_var)]
+    pub(crate) struct MveUnrecognizedVar {
+        #[primary_span]
+        pub span: Span,
+        pub key: MacroRulesNormalizedIdent,
+    }
+}

compiler/rustc_expand/src/mbe/metavar_expr.rs

@@ -47,46 +47,7 @@ impl MetaVarExpr {
         check_trailing_token(&mut iter, psess)?;
         let mut iter = args.iter();
         let rslt = match ident.as_str() {
-            "concat" => {
-                let mut result = Vec::new();
-                loop {
-                    let is_var = try_eat_dollar(&mut iter);
-                    let token = parse_token(&mut iter, psess, outer_span)?;
-                    let element = if is_var {
-                        MetaVarExprConcatElem::Var(parse_ident_from_token(psess, token)?)
-                    } else if let TokenKind::Literal(Lit {
-                        kind: token::LitKind::Str,
-                        symbol,
-                        suffix: None,
-                    }) = token.kind
-                    {
-                        MetaVarExprConcatElem::Literal(symbol)
-                    } else {
-                        match parse_ident_from_token(psess, token) {
-                            Err(err) => {
-                                err.cancel();
-                                return Err(psess
-                                    .dcx()
-                                    .struct_span_err(token.span, UNSUPPORTED_CONCAT_ELEM_ERR));
-                            }
-                            Ok(elem) => MetaVarExprConcatElem::Ident(elem),
-                        }
-                    };
-                    result.push(element);
-                    if iter.peek().is_none() {
-                        break;
-                    }
-                    if !try_eat_comma(&mut iter) {
-                        return Err(psess.dcx().struct_span_err(outer_span, "expected comma"));
-                    }
-                }
-                if result.len() < 2 {
-                    return Err(psess
-                        .dcx()
-                        .struct_span_err(ident.span, "`concat` must have at least two elements"));
-                }
-                MetaVarExpr::Concat(result.into())
-            }
+            "concat" => parse_concat(&mut iter, psess, outer_span, ident.span)?,
             "count" => parse_count(&mut iter, psess, ident.span)?,
             "ignore" => {
                 eat_dollar(&mut iter, psess, ident.span)?;
@@ -126,6 +87,22 @@ impl MetaVarExpr {
     }
 }
 
+// Checks if there are any remaining tokens. For example, `${ignore(ident ... a b c ...)}`
+fn check_trailing_token<'psess>(
+    iter: &mut TokenStreamIter<'_>,
+    psess: &'psess ParseSess,
+) -> PResult<'psess, ()> {
+    if let Some(tt) = iter.next() {
+        let mut diag = psess
+            .dcx()
+            .struct_span_err(tt.span(), format!("unexpected token: {}", pprust::tt_to_string(tt)));
+        diag.span_note(tt.span(), "meta-variable expression must not have trailing tokens");
+        Err(diag)
+    } else {
+        Ok(())
+    }
+}
+
 /// Indicates what is placed in a `concat` parameter. For example, literals
 /// (`${concat("foo", "bar")}`) or adhoc identifiers (`${concat(foo, bar)}`).
 #[derive(Debug, Decodable, Encodable, PartialEq)]
@@ -140,20 +117,48 @@ pub(crate) enum MetaVarExprConcatElem {
     Var(Ident),
 }
 
-// Checks if there are any remaining tokens. For example, `${ignore(ident ... a b c ...)}`
-fn check_trailing_token<'psess>(
+/// Parse a meta-variable `concat` expression: `concat($metavar, ident, ...)`.
+fn parse_concat<'psess>(
     iter: &mut TokenStreamIter<'_>,
     psess: &'psess ParseSess,
-) -> PResult<'psess, ()> {
-    if let Some(tt) = iter.next() {
-        let mut diag = psess
+    outer_span: Span,
+    expr_ident_span: Span,
+) -> PResult<'psess, MetaVarExpr> {
+    let mut result = Vec::new();
+    loop {
+        let is_var = try_eat_dollar(iter);
+        let token = parse_token(iter, psess, outer_span)?;
+        let element = if is_var {
+            MetaVarExprConcatElem::Var(parse_ident_from_token(psess, token)?)
+        } else if let TokenKind::Literal(Lit { kind: token::LitKind::Str, symbol, suffix: None }) =
+            token.kind
+        {
+            MetaVarExprConcatElem::Literal(symbol)
+        } else {
+            match parse_ident_from_token(psess, token) {
+                Err(err) => {
+                    err.cancel();
+                    return Err(psess
+                        .dcx()
+                        .struct_span_err(token.span, UNSUPPORTED_CONCAT_ELEM_ERR));
+                }
+                Ok(elem) => MetaVarExprConcatElem::Ident(elem),
+            }
+        };
+        result.push(element);
+        if iter.peek().is_none() {
+            break;
+        }
+        if !try_eat_comma(iter) {
+            return Err(psess.dcx().struct_span_err(outer_span, "expected comma"));
+        }
+    }
+    if result.len() < 2 {
+        return Err(psess
             .dcx()
-            .struct_span_err(tt.span(), format!("unexpected token: {}", pprust::tt_to_string(tt)));
-        diag.span_note(tt.span(), "meta-variable expression must not have trailing tokens");
-        Err(diag)
-    } else {
-        Ok(())
+            .struct_span_err(expr_ident_span, "`concat` must have at least two elements"));
     }
+    Ok(MetaVarExpr::Concat(result.into()))
 }
 
 /// Parse a meta-variable `count` expression: `count(ident[, depth])`

compiler/rustc_expand/src/mbe/transcribe.rs

@@ -17,7 +17,7 @@ use rustc_span::{
 use smallvec::{SmallVec, smallvec};
 
 use crate::errors::{
-    CountRepetitionMisplaced, MetaVarExprUnrecognizedVar, MetaVarsDifSeqMatchers, MustRepeatOnce,
+    CountRepetitionMisplaced, MetaVarsDifSeqMatchers, MustRepeatOnce, MveUnrecognizedVar,
     NoSyntaxVarsExprRepeat, VarStillRepeating,
 };
 use crate::mbe::macro_parser::NamedMatch;
@@ -879,7 +879,7 @@ where
 {
     let span = ident.span;
     let key = MacroRulesNormalizedIdent::new(ident);
-    interp.get(&key).ok_or_else(|| dcx.create_err(MetaVarExprUnrecognizedVar { span, key }))
+    interp.get(&key).ok_or_else(|| dcx.create_err(MveUnrecognizedVar { span, key }))
 }
 
 /// Used by meta-variable expressions when an user input is out of the actual declared bounds. For

src/tools/tidy/src/issues.txt

@@ -2849,7 +2849,6 @@ ui/macros/issue-98466.rs
 ui/macros/issue-99261.rs
 ui/macros/issue-99265.rs
 ui/macros/issue-99907.rs
-ui/macros/rfc-3086-metavar-expr/issue-111904.rs
 ui/malformed/issue-107423-unused-delim-only-one-no-pair.rs
 ui/malformed/issue-69341-malformed-derive-inert.rs
 ui/marker_trait_attr/issue-61651-type-mismatch.rs

tests/ui/macros/macro-metavar-expr-concat/hygiene.stderr → tests/ui/macros/metavar-expressions/concat-hygiene.stderr

@@ -1,5 +1,5 @@
 error[E0425]: cannot find value `abcdef` in this scope
-  --> $DIR/hygiene.rs:5:10
+  --> $DIR/concat-hygiene.rs:5:10
    |
 LL |         ${concat($lhs, $rhs)}
    |          ^^^^^^^^^^^^^^^^^^^^ not found in this scope

tests/ui/macros/macro-metavar-expr-concat/raw-identifiers.stderr → tests/ui/macros/metavar-expressions/concat-raw-identifiers.stderr

@@ -1,87 +1,87 @@
 error: expected identifier or string literal
-  --> $DIR/raw-identifiers.rs:28:22
+  --> $DIR/concat-raw-identifiers.rs:28:22
    |
 LL |         let ${concat(r#abc, abc)}: () = ();
    |                      ^^^^^
 
 error: expected identifier or string literal
-  --> $DIR/raw-identifiers.rs:32:27
+  --> $DIR/concat-raw-identifiers.rs:32:27
    |
 LL |         let ${concat(abc, r#abc)}: () = ();
    |                           ^^^^^
 
 error: expected identifier or string literal
-  --> $DIR/raw-identifiers.rs:35:22
+  --> $DIR/concat-raw-identifiers.rs:35:22
    |
 LL |         let ${concat(r#abc, r#abc)}: () = ();
    |                      ^^^^^
 
 error: `${concat(..)}` currently does not support raw identifiers
-  --> $DIR/raw-identifiers.rs:5:28
+  --> $DIR/concat-raw-identifiers.rs:5:28
    |
 LL |         let ${concat(abc, $rhs)}: () = ();
    |                            ^^^
 
 error: `${concat(..)}` currently does not support raw identifiers
-  --> $DIR/raw-identifiers.rs:12:23
+  --> $DIR/concat-raw-identifiers.rs:12:23
    |
 LL |         let ${concat($lhs, abc)}: () = ();
    |                       ^^^
 
 error: `${concat(..)}` currently does not support raw identifiers
-  --> $DIR/raw-identifiers.rs:19:23
+  --> $DIR/concat-raw-identifiers.rs:19:23
    |
 LL |         let ${concat($lhs, $rhs)}: () = ();
    |                       ^^^
 
 error: `${concat(..)}` currently does not support raw identifiers
-  --> $DIR/raw-identifiers.rs:19:29
+  --> $DIR/concat-raw-identifiers.rs:19:29
    |
 LL |         let ${concat($lhs, $rhs)}: () = ();
    |                             ^^^
 
 error: `${concat(..)}` currently does not support raw identifiers
-  --> $DIR/raw-identifiers.rs:19:23
+  --> $DIR/concat-raw-identifiers.rs:19:23
    |
 LL |         let ${concat($lhs, $rhs)}: () = ();
    |                       ^^^
    |
    = note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no`
 
 error: `${concat(..)}` currently does not support raw identifiers
-  --> $DIR/raw-identifiers.rs:42:28
+  --> $DIR/concat-raw-identifiers.rs:42:28
    |
 LL |         let ${concat(abc, $rhs)}: () = ();
    |                            ^^^
 
 error: `${concat(..)}` currently does not support raw identifiers
-  --> $DIR/raw-identifiers.rs:49:23
+  --> $DIR/concat-raw-identifiers.rs:49:23
    |
 LL |         let ${concat($lhs, abc)}: () = ();
    |                       ^^^
 
 error: `${concat(..)}` currently does not support raw identifiers
-  --> $DIR/raw-identifiers.rs:56:23
+  --> $DIR/concat-raw-identifiers.rs:56:23
    |
 LL |         let ${concat($lhs, $rhs)}: () = ();
    |                       ^^^
 
 error: `${concat(..)}` currently does not support raw identifiers
-  --> $DIR/raw-identifiers.rs:56:29
+  --> $DIR/concat-raw-identifiers.rs:56:29
    |
 LL |         let ${concat($lhs, $rhs)}: () = ();
    |                             ^^^
 
 error: `${concat(..)}` currently does not support raw identifiers
-  --> $DIR/raw-identifiers.rs:56:23
+  --> $DIR/concat-raw-identifiers.rs:56:23
    |
 LL |         let ${concat($lhs, $rhs)}: () = ();
    |                       ^^^
    |
    = note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no`
 
 error: expected pattern, found `$`
-  --> $DIR/raw-identifiers.rs:28:13
+  --> $DIR/concat-raw-identifiers.rs:28:13
    |
 LL |         let ${concat(r#abc, abc)}: () = ();
    |             ^ expected pattern

tests/ui/macros/macro-metavar-expr-concat/repetitions.stderr → tests/ui/macros/metavar-expressions/concat-repetitions.stderr

@@ -1,17 +1,17 @@
 error: invalid syntax
-  --> $DIR/repetitions.rs:14:20
+  --> $DIR/concat-repetitions.rs:14:20
    |
 LL |             const ${concat($a, Z)}: i32 = 3;
    |                    ^^^^^^^^^^^^^^^
 
 error: invalid syntax
-  --> $DIR/repetitions.rs:22:17
+  --> $DIR/concat-repetitions.rs:22:17
    |
 LL |         read::<${concat($t, $en)}>()
    |                 ^^^^^^^^^^^^^^^^^
 
 error: invalid syntax
-  --> $DIR/repetitions.rs:22:17
+  --> $DIR/concat-repetitions.rs:22:17
    |
 LL |         read::<${concat($t, $en)}>()
    |                 ^^^^^^^^^^^^^^^^^

tests/ui/macros/metavar-expressions/concat-trace-errors.rs

@@ -0,0 +1,33 @@
+// Our diagnostics should be able to point to a specific input that caused an invalid
+// identifier.
+
+#![feature(macro_metavar_expr_concat)]
+
+// See what we can do without expanding anything
+macro_rules! pre_expansion {
+    ($a:ident) => {
+        ${concat("hi", " bye ")};
+        ${concat("hi", "-", "bye")};
+        ${concat($a, "-")};
+    }
+}
+
+macro_rules! post_expansion {
+    ($a:literal) => {
+        const _: () = ${concat("hi", $a, "bye")};
+        //~^ ERROR is not generating a valid identifier
+    }
+}
+
+post_expansion!("!");
+
+macro_rules! post_expansion_many {
+    ($a:ident, $b:ident, $c:ident, $d:literal, $e:ident) => {
+        const _: () = ${concat($a, $b, $c, $d, $e)};
+        //~^ ERROR is not generating a valid identifier
+    }
+}
+
+post_expansion_many!(a, b, c, ".d", e);
+
+fn main() {}

tests/ui/macros/metavar-expressions/concat-trace-errors.stderr

@@ -0,0 +1,24 @@
+error: `${concat(..)}` is not generating a valid identifier
+  --> $DIR/concat-trace-errors.rs:17:24
+   |
+LL |         const _: () = ${concat("hi", $a, "bye")};
+   |                        ^^^^^^^^^^^^^^^^^^^^^^^^^
+...
+LL | post_expansion!("!");
+   | -------------------- in this macro invocation
+   |
+   = note: this error originates in the macro `post_expansion` (in Nightly builds, run with -Z macro-backtrace for more info)
+
+error: `${concat(..)}` is not generating a valid identifier
+  --> $DIR/concat-trace-errors.rs:26:24
+   |
+LL |         const _: () = ${concat($a, $b, $c, $d, $e)};
+   |                        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+...
+LL | post_expansion_many!(a, b, c, ".d", e);
+   | -------------------------------------- in this macro invocation
+   |
+   = note: this error originates in the macro `post_expansion_many` (in Nightly builds, run with -Z macro-backtrace for more info)
+
+error: aborting due to 2 previous errors
+