Auto merge of rust-lang#17559 - Veykril:tokentree, r=Veykril

Encode ident rawness and literal kind separately in tt::Leaf

Author: bors

src/tools/rust-analyzer/Cargo.lock

@@ -1046,6 +1046,7 @@ dependencies = [
  "arrayvec",
  "cov-mark",
  "parser",
+ "ra-ap-rustc_lexer",
  "rustc-hash",
  "smallvec",
  "span",

src/tools/rust-analyzer/crates/hir-def/src/attr.rs

@@ -605,7 +605,7 @@ impl<'attr> AttrQuery<'attr> {
                 .nth(2);
 
             match name {
-                Some(tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal{ ref text, ..}))) => Some(text),
+                Some(tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal{  text, kind: tt::LitKind::Str | tt::LitKind::StrRaw(_) , ..}))) => Some(text),
                 _ => None
             }
         })

src/tools/rust-analyzer/crates/hir-def/src/hir/format_args.rs

@@ -250,7 +250,7 @@ pub(crate) fn parse(
                 }
             }
             ArgRef::Name(name, span) => {
-                let name = Name::new(name, call_ctx);
+                let name = Name::new(name, tt::IdentIsRaw::No, call_ctx);
                 if let Some((index, _)) = args.by_name(&name) {
                     record_usage(name, span);
                     // Name found in `args`, so we resolve it to its index.

src/tools/rust-analyzer/crates/hir-def/src/macro_expansion_tests/builtin_fn_macro.rs

@@ -154,7 +154,7 @@ fn main() { file!(); }
 #[rustc_builtin_macro]
 macro_rules! file {() => {}}
 
-fn main() { ""; }
+fn main() { "file"; }
 "##]],
     );
 }
@@ -460,13 +460,13 @@ fn test_concat_expand() {
 #[rustc_builtin_macro]
 macro_rules! concat {}
 
-fn main() { concat!("fo", "o", 0, r#"bar"#, "\n", false, '"', '\0'); }
+fn main() { concat!("fo", "o", 0, r#""bar""#, "\n", false, '"', '\0'); }
 "##,
         expect![[r##"
 #[rustc_builtin_macro]
 macro_rules! concat {}
 
-fn main() { "foo0bar\nfalse\"\u{0}"; }
+fn main() { "foo0\"bar\"\nfalse\"\u{0}"; }
 "##]],
     );
 }
@@ -478,13 +478,13 @@ fn test_concat_bytes_expand() {
 #[rustc_builtin_macro]
 macro_rules! concat_bytes {}
 
-fn main() { concat_bytes!(b'A', b"BC", [68, b'E', 70]); }
+fn main() { concat_bytes!(b'A', b"BC\"", [68, b'E', 70], br#"G""#,b'\0'); }
 "##,
         expect![[r#"
 #[rustc_builtin_macro]
 macro_rules! concat_bytes {}
 
-fn main() { [b'A', 66, 67, 68, b'E', 70]; }
+fn main() { b"ABC\"DEFG\"\x00"; }
 "#]],
     );
 }

src/tools/rust-analyzer/crates/hir-def/src/macro_expansion_tests/mbe/regression.rs

@@ -1058,7 +1058,7 @@ macro_rules! concat {}
 macro_rules! line {}
 
 fn main() {
-    "event 0u32";
+    "event 0";
 }
 
 "##]],
@@ -1084,7 +1084,7 @@ fn main() {
 macro_rules! concat_bytes {}
 
 fn main() {
-    let x = /* error: unexpected token in input */[];
+    let x = /* error: unexpected token in input */b"";
 }
 
 "#]],

src/tools/rust-analyzer/crates/hir-def/src/nameres/collector.rs

@@ -82,7 +82,7 @@ pub(super) fn collect_defs(db: &dyn DefDatabase, def_map: DefMap, tree_id: TreeI
                     .iter()
                     .enumerate()
                     .map(|(idx, it)| {
-                        let name = Name::new(&it.name, ctx);
+                        let name = Name::new(&it.name, tt::IdentIsRaw::No, ctx);
                         (
                             name,
                             if !db.expand_proc_attr_macros() {
@@ -2144,7 +2144,7 @@ impl ModCollector<'_, '_> {
             let name;
             let name = match attrs.by_key("rustc_builtin_macro").string_value_with_span() {
                 Some((it, span)) => {
-                    name = Name::new(it, span.ctx);
+                    name = Name::new(it, tt::IdentIsRaw::No, span.ctx);
                     &name
                 }
                 None => {

src/tools/rust-analyzer/crates/hir-expand/src/attrs.rs

@@ -5,11 +5,14 @@ use base_db::CrateId;
 use cfg::CfgExpr;
 use either::Either;
 use intern::{sym, Interned};
-use mbe::{syntax_node_to_token_tree, DelimiterKind, DocCommentDesugarMode, Punct};
+use mbe::{
+    desugar_doc_comment_text, syntax_node_to_token_tree, token_to_literal, DelimiterKind,
+    DocCommentDesugarMode, Punct,
+};
 use smallvec::{smallvec, SmallVec};
 use span::{Span, SyntaxContextId};
 use syntax::unescape;
-use syntax::{ast, format_smolstr, match_ast, AstNode, AstToken, SmolStr, SyntaxNode};
+use syntax::{ast, match_ast, AstNode, AstToken, SyntaxNode};
 use triomphe::ThinArc;
 
 use crate::name::Name;
@@ -53,11 +56,15 @@ impl RawAttrs {
                 }
                 Either::Right(comment) => comment.doc_comment().map(|doc| {
                     let span = span_map.span_for_range(comment.syntax().text_range());
+                    let (text, kind) =
+                        desugar_doc_comment_text(doc, DocCommentDesugarMode::ProcMacro);
                     Attr {
                         id,
                         input: Some(Box::new(AttrInput::Literal(tt::Literal {
-                            text: SmolStr::new(format_smolstr!("\"{}\"", Self::escape_chars(doc))),
+                            text,
                             span,
+                            kind,
+                            suffix: None,
                         }))),
                         path: Interned::new(ModPath::from(Name::new_symbol(
                             sym::doc.clone(),
@@ -78,10 +85,6 @@ impl RawAttrs {
         RawAttrs { entries }
     }
 
-    fn escape_chars(s: &str) -> String {
-        s.replace('\\', r#"\\"#).replace('"', r#"\""#)
-    }
-
     pub fn from_attrs_owner(
         db: &dyn ExpandDatabase,
         owner: InFile<&dyn ast::HasAttrs>,
@@ -238,10 +241,8 @@ impl Attr {
         })?);
         let span = span_map.span_for_range(range);
         let input = if let Some(ast::Expr::Literal(lit)) = ast.expr() {
-            Some(Box::new(AttrInput::Literal(tt::Literal {
-                text: lit.token().text().into(),
-                span,
-            })))
+            let token = lit.token();
+            Some(Box::new(AttrInput::Literal(token_to_literal(token.text().into(), span))))
         } else if let Some(tt) = ast.token_tree() {
             let tree = syntax_node_to_token_tree(
                 tt.syntax(),
@@ -310,12 +311,11 @@ impl Attr {
     /// #[path = "string"]
     pub fn string_value(&self) -> Option<&str> {
         match self.input.as_deref()? {
-            AttrInput::Literal(it) => match it.text.strip_prefix('r') {
-                Some(it) => it.trim_matches('#'),
-                None => it.text.as_str(),
-            }
-            .strip_prefix('"')?
-            .strip_suffix('"'),
+            AttrInput::Literal(tt::Literal {
+                text,
+                kind: tt::LitKind::Str | tt::LitKind::StrRaw(_),
+                ..
+            }) => Some(text),
             _ => None,
         }
     }
@@ -336,12 +336,10 @@ impl Attr {
 
     pub fn string_value_unescape(&self) -> Option<Cow<'_, str>> {
         match self.input.as_deref()? {
-            AttrInput::Literal(it) => match it.text.strip_prefix('r') {
-                Some(it) => {
-                    it.trim_matches('#').strip_prefix('"')?.strip_suffix('"').map(Cow::Borrowed)
-                }
-                None => it.text.strip_prefix('"')?.strip_suffix('"').and_then(unescape),
-            },
+            AttrInput::Literal(tt::Literal { text, kind: tt::LitKind::StrRaw(_), .. }) => {
+                Some(Cow::Borrowed(text))
+            }
+            AttrInput::Literal(tt::Literal { text, kind: tt::LitKind::Str, .. }) => unescape(text),
             _ => None,
         }
     }

src/tools/rust-analyzer/crates/hir-expand/src/builtin_derive_macro.rs

@@ -370,7 +370,8 @@ fn name_to_token(
         ExpandError::other("missing name")
     })?;
     let span = token_map.span_at(name.syntax().text_range().start());
-    let name_token = tt::Ident { span, text: name.text().into() };
+
+    let name_token = tt::Ident::new(name.text().as_ref(), span);
     Ok(name_token)
 }