Auto merge of rust-lang#13854 - lowr:fix/mbe-glue-punct, r=Veykril

Support multi-character punct tokens in MBE

Fixes rust-lang#11497

In the context of MBE, consecutive puncts are parsed as multi-character punct tokens whenever possible. For example, `:::` is parsed as ``[Punct(`::`), Punct(`:`)]`` and shouldn't get matched to patterns like `: : :` or `: ::`.

We have implemented this behavior only for when we match puncts against `tt` fragments, but not when we match puncts literally. This PR extracts the multi-character punct handling procedure into a separate method and extends its support for literal matching.

For good measure, this PR adds support for `<-` token, which is still [considered as one token in rustc](https://github.com/rust-lang/rust/blob/e3961864075eaa9e855e5eec6b4f148029684539/compiler/rustc_ast/src/token.rs#L249) despite the placement syntax having been removed.

Author: bors

crates/hir-def/src/macro_expansion_tests/mbe.rs

@@ -1630,3 +1630,48 @@ const _: i32 = -0--1--2;
     "#]],
     );
 }
+
+#[test]
+fn test_punct_without_space() {
+    // Puncts are "glued" greedily.
+    check(
+        r#"
+macro_rules! foo {
+    (: : :) => { "1 1 1" };
+    (: ::) => { "1 2" };
+    (:: :) => { "2 1" };
+
+    (: : : :) => { "1 1 1 1" };
+    (:: : :) => { "2 1 1" };
+    (: :: :) => { "1 2 1" };
+    (: : ::) => { "1 1 2" };
+    (:: ::) => { "2 2" };
+}
+
+fn test() {
+    foo!(:::);
+    foo!(: :::);
+    foo!(::::);
+}
+"#,
+        expect![[r#"
+macro_rules! foo {
+    (: : :) => { "1 1 1" };
+    (: ::) => { "1 2" };
+    (:: :) => { "2 1" };
+
+    (: : : :) => { "1 1 1 1" };
+    (:: : :) => { "2 1 1" };
+    (: :: :) => { "1 2 1" };
+    (: : ::) => { "1 1 2" };
+    (:: ::) => { "2 2" };
+}
+
+fn test() {
+    "2 1";
+    "1 2 1";
+    "2 2";
+}
+"#]],
+    );
+}

crates/mbe/src/benchmark.rs

@@ -141,7 +141,13 @@ fn invocation_fixtures(rules: &FxHashMap<String, DeclarativeMacro>) -> Vec<(Stri
                 None => (),
                 Some(kind) => panic!("Unhandled kind {kind:?}"),
             },
-            Op::Leaf(leaf) => parent.token_trees.push(leaf.clone().into()),
+            Op::Literal(it) => parent.token_trees.push(tt::Leaf::from(it.clone()).into()),
+            Op::Ident(it) => parent.token_trees.push(tt::Leaf::from(it.clone()).into()),
+            Op::Punct(puncts) => {
+                for punct in puncts {
+                    parent.token_trees.push(tt::Leaf::from(punct.clone()).into());
+                }
+            }
             Op::Repeat { tokens, kind, separator } => {
                 let max = 10;
                 let cnt = match kind {