Skip to content

Commit 0be9e2a

Browse files
VeykrilVeykril
committed
Encode ident rawness and literal kind separately in tt::Leaf
1 parent 35aa238 commit 0be9e2a

File tree

26 files changed

+516
-148
lines changed

26 files changed

+516
-148
lines changed

‎crates/hir-def/src/attr.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -596,7 +596,7 @@ impl<'attr> AttrQuery<'attr> {
596596
.nth(2);
597597

598598
match name {
599-
Some(tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal{ ref text, ..}))) => Some(text),
599+
Some(tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal{ text, kind: tt::LitKind::Str | tt::LitKind::StrRaw(_) , ..}))) => Some(text),
600600
_ => None
601601
}
602602
})

‎crates/hir-def/src/hir/format_args.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -248,7 +248,7 @@ pub(crate) fn parse(
248248
}
249249
}
250250
ArgRef::Name(name, span) => {
251-
let name = Name::new_text_dont_use(SmolStr::new(name));
251+
let name = Name::new_text_dont_use(SmolStr::new(name), tt::IdentIsRaw::No);
252252
if let Some((index, _)) = args.by_name(&name) {
253253
record_usage(name, span);
254254
// Name found in `args`, so we resolve it to its index.

‎crates/hir-def/src/nameres/collector.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ pub(super) fn collect_defs(db: &dyn DefDatabase, def_map: DefMap, tree_id: TreeI
8080
.iter()
8181
.enumerate()
8282
.map(|(idx, it)| {
83-
let name = Name::new_text_dont_use(it.name.clone());
83+
let name = Name::new_text_dont_use(it.name.clone(), tt::IdentIsRaw::No);
8484
(
8585
name,
8686
if !db.expand_proc_attr_macros() {
@@ -2138,7 +2138,7 @@ impl ModCollector<'_, '_> {
21382138
let name;
21392139
let name = match attrs.by_key("rustc_builtin_macro").string_value() {
21402140
Some(it) => {
2141-
name = Name::new_text_dont_use(it.into());
2141+
name = Name::new_text_dont_use(it.into(), tt::IdentIsRaw::No);
21422142
&name
21432143
}
21442144
None => {

‎crates/hir-expand/src/attrs.rs

+14-16
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,9 @@ use base_db::CrateId;
55
use cfg::CfgExpr;
66
use either::Either;
77
use intern::Interned;
8-
use mbe::{syntax_node_to_token_tree, DelimiterKind, DocCommentDesugarMode, Punct};
8+
use mbe::{
9+
syntax_node_to_token_tree, token_to_literal, DelimiterKind, DocCommentDesugarMode, Punct,
10+
};
911
use smallvec::{smallvec, SmallVec};
1012
use span::{Span, SyntaxContextId};
1113
use syntax::unescape;
@@ -57,6 +59,7 @@ impl RawAttrs {
5759
input: Some(Box::new(AttrInput::Literal(tt::Literal {
5860
text: SmolStr::new(format_smolstr!("\"{}\"", Self::escape_chars(doc))),
5961
span,
62+
kind: tt::LitKind::Str,
6063
}))),
6164
path: Interned::new(ModPath::from(crate::name!(doc))),
6265
ctxt: span.ctx,
@@ -234,10 +237,12 @@ impl Attr {
234237
})?);
235238
let span = span_map.span_for_range(range);
236239
let input = if let Some(ast::Expr::Literal(lit)) = ast.expr() {
237-
Some(Box::new(AttrInput::Literal(tt::Literal {
238-
text: lit.token().text().into(),
240+
let token = lit.token();
241+
Some(Box::new(AttrInput::Literal(token_to_literal(
242+
token.kind(),
243+
token.text().into(),
239244
span,
240-
})))
245+
))))
241246
} else if let Some(tt) = ast.token_tree() {
242247
let tree = syntax_node_to_token_tree(
243248
tt.syntax(),
@@ -306,24 +311,17 @@ impl Attr {
306311
/// #[path = "string"]
307312
pub fn string_value(&self) -> Option<&str> {
308313
match self.input.as_deref()? {
309-
AttrInput::Literal(it) => match it.text.strip_prefix('r') {
310-
Some(it) => it.trim_matches('#'),
311-
None => it.text.as_str(),
312-
}
313-
.strip_prefix('"')?
314-
.strip_suffix('"'),
314+
AttrInput::Literal(tt::Literal { text, kind: tt::LitKind::Str, .. }) => Some(text),
315315
_ => None,
316316
}
317317
}
318318

319319
pub fn string_value_unescape(&self) -> Option<Cow<'_, str>> {
320320
match self.input.as_deref()? {
321-
AttrInput::Literal(it) => match it.text.strip_prefix('r') {
322-
Some(it) => {
323-
it.trim_matches('#').strip_prefix('"')?.strip_suffix('"').map(Cow::Borrowed)
324-
}
325-
None => it.text.strip_prefix('"')?.strip_suffix('"').and_then(unescape),
326-
},
321+
AttrInput::Literal(tt::Literal { text, kind: tt::LitKind::StrRaw(_), .. }) => {
322+
Some(Cow::Borrowed(text))
323+
}
324+
AttrInput::Literal(tt::Literal { text, kind: tt::LitKind::Str, .. }) => unescape(text),
327325
_ => None,
328326
}
329327
}

‎crates/hir-expand/src/builtin_derive_macro.rs

+2-1
Original file line numberDiff line numberDiff line change
@@ -369,7 +369,8 @@ fn name_to_token(
369369
ExpandError::other("missing name")
370370
})?;
371371
let span = token_map.span_at(name.syntax().text_range().start());
372-
let name_token = tt::Ident { span, text: name.text().into() };
372+
373+
let name_token = tt::Ident::new(name.text().as_ref(), span);
373374
Ok(name_token)
374375
}
375376

‎crates/hir-expand/src/builtin_fn_macro.rs

+4-1
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,7 @@ fn line_expand(
179179
token_trees: Box::new([tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal {
180180
text: "0u32".into(),
181181
span,
182+
kind: tt::LitKind::Integer,
182183
}))]),
183184
})
184185
}
@@ -593,6 +594,7 @@ fn concat_bytes_expand(
593594
tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal {
594595
text: it.into(),
595596
span: span.unwrap_or(call_site),
597+
kind: tt::LitKind::Integer,
596598
}))
597599
}),
598600
|| {
@@ -657,7 +659,7 @@ fn concat_idents_expand(
657659
}
658660
}
659661
// FIXME merge spans
660-
let ident = tt::Ident { text: ident.into(), span };
662+
let ident = tt::Ident { text: ident.into(), span, is_raw: tt::IdentIsRaw::No };
661663
ExpandResult { value: quote!(span =>#ident), err }
662664
}
663665

@@ -735,6 +737,7 @@ fn include_bytes_expand(
735737
token_trees: Box::new([tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal {
736738
text: r#"b"""#.into(),
737739
span,
740+
kind: tt::LitKind::ByteStrRaw(1),
738741
}))]),
739742
};
740743
ExpandResult::ok(res)

‎crates/hir-expand/src/fixup.rs

+16-7
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ pub(crate) fn fixup_syntax(
8686
anchor: SpanAnchor { ast_id: FIXUP_DUMMY_AST_ID, ..span.anchor },
8787
ctx: span.ctx,
8888
},
89+
is_raw: tt::IdentIsRaw::No,
8990
});
9091
append.insert(node.clone().into(), vec![replacement]);
9192
preorder.skip_subtree();
@@ -101,6 +102,7 @@ pub(crate) fn fixup_syntax(
101102
Leaf::Ident(Ident {
102103
text: "__ra_fixup".into(),
103104
span: fake_span(node_range),
105+
is_raw: tt::IdentIsRaw::No
104106
}),
105107
]);
106108
}
@@ -137,7 +139,8 @@ pub(crate) fn fixup_syntax(
137139
append.insert(if_token.into(), vec![
138140
Leaf::Ident(Ident {
139141
text: "__ra_fixup".into(),
140-
span: fake_span(node_range)
142+
span: fake_span(node_range),
143+
is_raw: tt::IdentIsRaw::No
141144
}),
142145
]);
143146
}
@@ -167,7 +170,8 @@ pub(crate) fn fixup_syntax(
167170
append.insert(while_token.into(), vec![
168171
Leaf::Ident(Ident {
169172
text: "__ra_fixup".into(),
170-
span: fake_span(node_range)
173+
span: fake_span(node_range),
174+
is_raw: tt::IdentIsRaw::No
171175
}),
172176
]);
173177
}
@@ -214,7 +218,8 @@ pub(crate) fn fixup_syntax(
214218
append.insert(match_token.into(), vec![
215219
Leaf::Ident(Ident {
216220
text: "__ra_fixup".into(),
217-
span: fake_span(node_range)
221+
span: fake_span(node_range),
222+
is_raw: tt::IdentIsRaw::No
218223
}),
219224
]);
220225
}
@@ -248,7 +253,8 @@ pub(crate) fn fixup_syntax(
248253
].map(|text|
249254
Leaf::Ident(Ident {
250255
text: text.into(),
251-
span: fake_span(node_range)
256+
span: fake_span(node_range),
257+
is_raw: tt::IdentIsRaw::No
252258
}),
253259
);
254260

@@ -281,7 +287,8 @@ pub(crate) fn fixup_syntax(
281287
append.insert(colon.into(), vec![
282288
Leaf::Ident(Ident {
283289
text: "__ra_fixup".into(),
284-
span: fake_span(node_range)
290+
span: fake_span(node_range),
291+
is_raw: tt::IdentIsRaw::No
285292
})
286293
]);
287294
}
@@ -293,7 +300,8 @@ pub(crate) fn fixup_syntax(
293300
append.insert(colon.into(), vec![
294301
Leaf::Ident(Ident {
295302
text: "__ra_fixup".into(),
296-
span: fake_span(node_range)
303+
span: fake_span(node_range),
304+
is_raw: tt::IdentIsRaw::No
297305
})
298306
]);
299307
}
@@ -326,7 +334,8 @@ pub(crate) fn fixup_syntax(
326334
append.insert(node.into(), vec![
327335
Leaf::Ident(Ident {
328336
text: "__ra_fixup".into(),
329-
span: fake_span(node_range)
337+
span: fake_span(node_range),
338+
is_raw: tt::IdentIsRaw::No
330339
})
331340
]);
332341
}

‎crates/hir-expand/src/lib.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ pub use span::{HirFileId, MacroCallId, MacroFileId};
5959

6060
pub mod tt {
6161
pub use span::Span;
62-
pub use tt::{DelimiterKind, Spacing};
62+
pub use tt::{DelimiterKind, IdentIsRaw, LitKind, Spacing};
6363

6464
pub type Delimiter = ::tt::Delimiter<Span>;
6565
pub type DelimSpan = ::tt::DelimSpan<Span>;

‎crates/hir-expand/src/mod_path.rs

+5-5
Original file line numberDiff line numberDiff line change
@@ -317,15 +317,15 @@ fn convert_path_tt(db: &dyn ExpandDatabase, tt: &[tt::TokenTree]) -> Option<ModP
317317
tt::Leaf::Punct(tt::Punct { char: ':', .. }) => PathKind::Abs,
318318
_ => return None,
319319
},
320-
tt::Leaf::Ident(tt::Ident { text, span }) if text == "$crate" => {
320+
tt::Leaf::Ident(tt::Ident { text, span, .. }) if text == "$crate" => {
321321
resolve_crate_root(db, span.ctx).map(PathKind::DollarCrate).unwrap_or(PathKind::Crate)
322322
}
323323
tt::Leaf::Ident(tt::Ident { text, .. }) if text == "self" => PathKind::SELF,
324324
tt::Leaf::Ident(tt::Ident { text, .. }) if text == "super" => {
325325
let mut deg = 1;
326-
while let Some(tt::Leaf::Ident(tt::Ident { text, .. })) = leaves.next() {
326+
while let Some(tt::Leaf::Ident(tt::Ident { text, is_raw, .. })) = leaves.next() {
327327
if text != "super" {
328-
segments.push(Name::new_text_dont_use(text.clone()));
328+
segments.push(Name::new_text_dont_use(text.clone(), *is_raw));
329329
break;
330330
}
331331
deg += 1;
@@ -334,13 +334,13 @@ fn convert_path_tt(db: &dyn ExpandDatabase, tt: &[tt::TokenTree]) -> Option<ModP
334334
}
335335
tt::Leaf::Ident(tt::Ident { text, .. }) if text == "crate" => PathKind::Crate,
336336
tt::Leaf::Ident(ident) => {
337-
segments.push(Name::new_text_dont_use(ident.text.clone()));
337+
segments.push(Name::new_text_dont_use(ident.text.clone(), ident.is_raw));
338338
PathKind::Plain
339339
}
340340
_ => return None,
341341
};
342342
segments.extend(leaves.filter_map(|leaf| match leaf {
343-
::tt::Leaf::Ident(ident) => Some(Name::new_text_dont_use(ident.text.clone())),
343+
::tt::Leaf::Ident(ident) => Some(Name::new_text_dont_use(ident.text.clone(), ident.is_raw)),
344344
_ => None,
345345
}));
346346
Some(ModPath { kind, segments })

‎crates/hir-expand/src/name.rs

+6-2
Original file line numberDiff line numberDiff line change
@@ -56,8 +56,12 @@ impl Name {
5656

5757
// FIXME: See above, unfortunately some places really need this right now
5858
#[doc(hidden)]
59-
pub const fn new_text_dont_use(text: SmolStr) -> Name {
60-
Name(Repr::Text(text))
59+
pub fn new_text_dont_use(text: SmolStr, raw: tt::IdentIsRaw) -> Name {
60+
if raw.yes() {
61+
Name(Repr::Text(format_smolstr!("r#{}", text)))
62+
} else {
63+
Name(Repr::Text(text))
64+
}
6165
}
6266

6367
pub fn new_tuple_field(idx: usize) -> Name {

‎crates/hir-expand/src/quote.rs

+18-9
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ use syntax::format_smolstr;
77
use crate::name::Name;
88

99
pub(crate) const fn dollar_crate(span: Span) -> tt::Ident<Span> {
10-
tt::Ident { text: syntax::SmolStr::new_static("$crate"), span }
10+
tt::Ident { text: syntax::SmolStr::new_static("$crate"), span, is_raw: tt::IdentIsRaw::No }
1111
}
1212

1313
// A helper macro quote macro
@@ -100,6 +100,7 @@ macro_rules! __quote {
100100
crate::tt::Leaf::Ident(crate::tt::Ident {
101101
text: stringify!($tt).into(),
102102
span: $span,
103+
is_raw: tt::IdentIsRaw::No,
103104
}).into()
104105
}]
105106
};
@@ -208,17 +209,20 @@ macro_rules! impl_to_to_tokentrees {
208209
}
209210

210211
impl_to_to_tokentrees! {
211-
span: u32 => self { crate::tt::Literal{text: self.to_string().into(), span} };
212-
span: usize => self { crate::tt::Literal{text: self.to_string().into(), span} };
213-
span: i32 => self { crate::tt::Literal{text: self.to_string().into(), span} };
214-
span: bool => self { crate::tt::Ident{text: self.to_string().into(), span} };
212+
span: u32 => self { crate::tt::Literal{text: self.to_string().into(), span, kind: tt::LitKind::Integer } };
213+
span: usize => self { crate::tt::Literal{text: self.to_string().into(), span, kind: tt::LitKind::Integer } };
214+
span: i32 => self { crate::tt::Literal{text: self.to_string().into(), span, kind: tt::LitKind::Integer } };
215+
span: bool => self { crate::tt::Ident{text: self.to_string().into(), span, is_raw: tt::IdentIsRaw::No } };
215216
_span: crate::tt::Leaf => self { self };
216217
_span: crate::tt::Literal => self { self };
217218
_span: crate::tt::Ident => self { self };
218219
_span: crate::tt::Punct => self { self };
219-
span: &str => self { crate::tt::Literal{text: format_smolstr!("\"{}\"", self.escape_default()), span}};
220-
span: String => self { crate::tt::Literal{text: format_smolstr!("\"{}\"", self.escape_default()), span}};
221-
span: Name => self { crate::tt::Ident{text: self.to_smol_str(), span}};
220+
span: &str => self { crate::tt::Literal{text: format_smolstr!("\"{}\"", self.escape_default()), span, kind: tt::LitKind::Str }};
221+
span: String => self { crate::tt::Literal{text: format_smolstr!("\"{}\"", self.escape_default()), span, kind: tt::LitKind::Str }};
222+
span: Name => self {
223+
let text = self.to_smol_str();
224+
crate::tt::Ident::new(text, span)
225+
};
222226
}
223227

224228
#[cfg(test)]
@@ -257,7 +261,12 @@ mod tests {
257261
}
258262

259263
fn mk_ident(name: &str) -> crate::tt::Ident {
260-
crate::tt::Ident { text: name.into(), span: DUMMY }
264+
let raw = name.strip_prefix("r#");
265+
crate::tt::Ident {
266+
text: raw.unwrap_or(name).into(),
267+
span: DUMMY,
268+
is_raw: if raw.is_none() { tt::IdentIsRaw::No } else { tt::IdentIsRaw::Yes },
269+
}
261270
}
262271

263272
#[test]

‎crates/hir/src/attrs.rs

+4-3
Original file line numberDiff line numberDiff line change
@@ -327,9 +327,10 @@ fn doc_modpath_from_str(link: &str) -> Option<ModPath> {
327327
};
328328
let parts = first_segment.into_iter().chain(parts).map(|segment| match segment.parse() {
329329
Ok(idx) => Name::new_tuple_field(idx),
330-
Err(_) => {
331-
Name::new_text_dont_use(segment.split_once('<').map_or(segment, |it| it.0).into())
332-
}
330+
Err(_) => Name::new_text_dont_use(
331+
segment.split_once('<').map_or(segment, |it| it.0).into(),
332+
tt::IdentIsRaw::No,
333+
),
333334
});
334335
Some(ModPath::from_segments(kind, parts))
335336
};

‎crates/mbe/src/benchmark.rs

+12-2
Original file line numberDiff line numberDiff line change
@@ -226,13 +226,23 @@ fn invocation_fixtures(
226226
*seed
227227
}
228228
fn make_ident(ident: &str) -> tt::TokenTree<Span> {
229-
tt::Leaf::Ident(tt::Ident { span: DUMMY, text: SmolStr::new(ident) }).into()
229+
tt::Leaf::Ident(tt::Ident {
230+
span: DUMMY,
231+
text: SmolStr::new(ident),
232+
is_raw: tt::IdentIsRaw::No,
233+
})
234+
.into()
230235
}
231236
fn make_punct(char: char) -> tt::TokenTree<Span> {
232237
tt::Leaf::Punct(tt::Punct { span: DUMMY, char, spacing: tt::Spacing::Alone }).into()
233238
}
234239
fn make_literal(lit: &str) -> tt::TokenTree<Span> {
235-
tt::Leaf::Literal(tt::Literal { span: DUMMY, text: SmolStr::new(lit) }).into()
240+
tt::Leaf::Literal(tt::Literal {
241+
span: DUMMY,
242+
text: SmolStr::new(lit),
243+
kind: tt::LitKind::Str,
244+
})
245+
.into()
236246
}
237247
fn make_subtree(
238248
kind: tt::DelimiterKind,

0 commit comments

Comments
 (0)