Skip to content

Commit 77c1df1

Browse files
VeykrilVeykril
committed
Fix incorrect encoding of literals in the proc-macro-api on version 4
1 parent f913901 commit 77c1df1

File tree

6 files changed

+111
-66
lines changed

6 files changed

+111
-66
lines changed

‎Cargo.lock

+1
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎crates/mbe/src/syntax_bridge.rs

+1
Original file line numberDiff line numberDiff line change
@@ -400,6 +400,7 @@ where
400400
}
401401
}
402402

403+
// FIXME: MOve to tt?
403404
pub fn token_to_literal<S>(text: SmolStr, span: S) -> tt::Literal<S>
404405
where
405406
S: Copy,

‎crates/proc-macro-api/Cargo.toml

+2
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@ span.workspace = true
2828
# InternIds for the syntax context
2929
base-db.workspace = true
3030
la-arena.workspace = true
31+
# only here to parse via token_to_literal
32+
mbe.workspace = true
3133

3234
[lints]
3335
workspace = true

‎crates/proc-macro-api/src/msg.rs

+28-25
Original file line numberDiff line numberDiff line change
@@ -197,7 +197,7 @@ mod tests {
197197
.into(),
198198
),
199199
TokenTree::Leaf(Leaf::Literal(Literal {
200-
text: "\"Foo\"".into(),
200+
text: "Foo".into(),
201201
span: Span {
202202
range: TextRange::at(TextSize::new(10), TextSize::of("\"Foo\"")),
203203
anchor,
@@ -263,32 +263,35 @@ mod tests {
263263
#[test]
264264
fn test_proc_macro_rpc_works() {
265265
let tt = fixture_token_tree();
266-
let mut span_data_table = Default::default();
267-
let task = ExpandMacro {
268-
data: ExpandMacroData {
269-
macro_body: FlatTree::new(&tt, CURRENT_API_VERSION, &mut span_data_table),
270-
macro_name: Default::default(),
271-
attributes: None,
272-
has_global_spans: ExpnGlobals {
273-
serialize: true,
274-
def_site: 0,
275-
call_site: 0,
276-
mixed_site: 0,
266+
for v in RUST_ANALYZER_SPAN_SUPPORT..=CURRENT_API_VERSION {
267+
let mut span_data_table = Default::default();
268+
let task = ExpandMacro {
269+
data: ExpandMacroData {
270+
macro_body: FlatTree::new(&tt, v, &mut span_data_table),
271+
macro_name: Default::default(),
272+
attributes: None,
273+
has_global_spans: ExpnGlobals {
274+
serialize: true,
275+
def_site: 0,
276+
call_site: 0,
277+
mixed_site: 0,
278+
},
279+
span_data_table: Vec::new(),
277280
},
278-
span_data_table: Vec::new(),
279-
},
280-
lib: Utf8PathBuf::from_path_buf(std::env::current_dir().unwrap()).unwrap(),
281-
env: Default::default(),
282-
current_dir: Default::default(),
283-
};
281+
lib: Utf8PathBuf::from_path_buf(std::env::current_dir().unwrap()).unwrap(),
282+
env: Default::default(),
283+
current_dir: Default::default(),
284+
};
284285

285-
let json = serde_json::to_string(&task).unwrap();
286-
// println!("{}", json);
287-
let back: ExpandMacro = serde_json::from_str(&json).unwrap();
286+
let json = serde_json::to_string(&task).unwrap();
287+
// println!("{}", json);
288+
let back: ExpandMacro = serde_json::from_str(&json).unwrap();
288289

289-
assert_eq!(
290-
tt,
291-
back.data.macro_body.to_subtree_resolved(CURRENT_API_VERSION, &span_data_table)
292-
);
290+
assert_eq!(
291+
tt,
292+
back.data.macro_body.to_subtree_resolved(v, &span_data_table),
293+
"version: {v}"
294+
);
295+
}
293296
}
294297
}

‎crates/proc-macro-api/src/msg/flat.rs

+78-40
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,7 @@ impl FlatTree {
141141
ident: Vec::new(),
142142
token_tree: Vec::new(),
143143
text: Vec::new(),
144+
version,
144145
};
145146
w.write(subtree);
146147

@@ -178,6 +179,7 @@ impl FlatTree {
178179
ident: Vec::new(),
179180
token_tree: Vec::new(),
180181
text: Vec::new(),
182+
version,
181183
};
182184
w.write(subtree);
183185

@@ -228,6 +230,7 @@ impl FlatTree {
228230
token_tree: self.token_tree,
229231
text: self.text,
230232
span_data_table,
233+
version,
231234
}
232235
.read()
233236
}
@@ -253,6 +256,7 @@ impl FlatTree {
253256
token_tree: self.token_tree,
254257
text: self.text,
255258
span_data_table: &(),
259+
version,
256260
}
257261
.read()
258262
}
@@ -386,8 +390,9 @@ impl InternableSpan for Span {
386390

387391
struct Writer<'a, 'span, S: InternableSpan> {
388392
work: VecDeque<(usize, &'a tt::Subtree<S>)>,
389-
string_table: FxHashMap<&'a str, u32>,
393+
string_table: FxHashMap<std::borrow::Cow<'a, str>, u32>,
390394
span_data_table: &'span mut S::Table,
395+
version: u32,
391396

392397
subtree: Vec<SubtreeRepr>,
393398
literal: Vec<LiteralRepr>,
@@ -425,9 +430,15 @@ impl<'a, 'span, S: InternableSpan> Writer<'a, 'span, S> {
425430
tt::TokenTree::Leaf(leaf) => match leaf {
426431
tt::Leaf::Literal(lit) => {
427432
let idx = self.literal.len() as u32;
428-
let text = self.intern(&lit.text);
429433
let id = self.token_id_of(lit.span);
430-
let suffix = lit.suffix.as_ref().map(|s| self.intern(s)).unwrap_or(!0);
434+
let (text, suffix) = if self.version >= EXTENDED_LEAF_DATA {
435+
(
436+
self.intern(&lit.text),
437+
lit.suffix.as_ref().map(|s| self.intern(s)).unwrap_or(!0),
438+
)
439+
} else {
440+
(self.intern_owned(format!("{lit}")), !0)
441+
};
431442
self.literal.push(LiteralRepr {
432443
id,
433444
text,
@@ -456,13 +467,15 @@ impl<'a, 'span, S: InternableSpan> Writer<'a, 'span, S> {
456467
}
457468
tt::Leaf::Ident(ident) => {
458469
let idx = self.ident.len() as u32;
459-
let text = self.intern(&ident.text);
460470
let id = self.token_id_of(ident.span);
461-
self.ident.push(IdentRepr {
462-
id,
463-
text,
464-
is_raw: ident.is_raw == tt::IdentIsRaw::Yes,
465-
});
471+
let text = if self.version >= EXTENDED_LEAF_DATA {
472+
self.intern(&ident.text)
473+
} else if ident.is_raw.yes() {
474+
self.intern_owned(format!("r#{}", ident.text,))
475+
} else {
476+
self.intern(&ident.text)
477+
};
478+
self.ident.push(IdentRepr { id, text, is_raw: ident.is_raw.yes() });
466479
idx << 2 | 0b11
467480
}
468481
},
@@ -484,15 +497,25 @@ impl<'a, 'span, S: InternableSpan> Writer<'a, 'span, S> {
484497

485498
pub(crate) fn intern(&mut self, text: &'a str) -> u32 {
486499
let table = &mut self.text;
487-
*self.string_table.entry(text).or_insert_with(|| {
500+
*self.string_table.entry(text.into()).or_insert_with(|| {
488501
let idx = table.len();
489502
table.push(text.to_owned());
490503
idx as u32
491504
})
492505
}
506+
507+
pub(crate) fn intern_owned(&mut self, text: String) -> u32 {
508+
let table = &mut self.text;
509+
*self.string_table.entry(text.clone().into()).or_insert_with(|| {
510+
let idx = table.len();
511+
table.push(text);
512+
idx as u32
513+
})
514+
}
493515
}
494516

495517
struct Reader<'span, S: InternableSpan> {
518+
version: u32,
496519
subtree: Vec<SubtreeRepr>,
497520
literal: Vec<LiteralRepr>,
498521
punct: Vec<PunctRepr>,
@@ -528,30 +551,36 @@ impl<'span, S: InternableSpan> Reader<'span, S> {
528551
0b01 => {
529552
use tt::LitKind::*;
530553
let repr = &self.literal[idx];
531-
tt::Leaf::Literal(tt::Literal {
532-
text: self.text[repr.text as usize].as_str().into(),
533-
span: read_span(repr.id),
534-
kind: match u16::to_le_bytes(repr.kind) {
535-
[0, _] => Err(()),
536-
[1, _] => Byte,
537-
[2, _] => Char,
538-
[3, _] => Integer,
539-
[4, _] => Float,
540-
[5, _] => Str,
541-
[6, r] => StrRaw(r),
542-
[7, _] => ByteStr,
543-
[8, r] => ByteStrRaw(r),
544-
[9, _] => CStr,
545-
[10, r] => CStrRaw(r),
546-
_ => unreachable!(),
547-
},
548-
suffix: if repr.suffix != !0 {
549-
Some(Box::new(
550-
self.text[repr.suffix as usize].as_str().into(),
551-
))
552-
} else {
553-
None
554-
},
554+
let text = self.text[repr.text as usize].as_str();
555+
let span = read_span(repr.id);
556+
tt::Leaf::Literal(if self.version >= EXTENDED_LEAF_DATA {
557+
tt::Literal {
558+
text: text.into(),
559+
span,
560+
kind: match u16::to_le_bytes(repr.kind) {
561+
[0, _] => Err(()),
562+
[1, _] => Byte,
563+
[2, _] => Char,
564+
[3, _] => Integer,
565+
[4, _] => Float,
566+
[5, _] => Str,
567+
[6, r] => StrRaw(r),
568+
[7, _] => ByteStr,
569+
[8, r] => ByteStrRaw(r),
570+
[9, _] => CStr,
571+
[10, r] => CStrRaw(r),
572+
_ => unreachable!(),
573+
},
574+
suffix: if repr.suffix != !0 {
575+
Some(Box::new(
576+
self.text[repr.suffix as usize].as_str().into(),
577+
))
578+
} else {
579+
None
580+
},
581+
}
582+
} else {
583+
mbe::token_to_literal(text.into(), span)
555584
})
556585
.into()
557586
}
@@ -566,14 +595,23 @@ impl<'span, S: InternableSpan> Reader<'span, S> {
566595
}
567596
0b11 => {
568597
let repr = &self.ident[idx];
598+
let text = self.text[repr.text as usize].as_str();
599+
let (is_raw, text) = if self.version >= EXTENDED_LEAF_DATA {
600+
(
601+
if repr.is_raw {
602+
tt::IdentIsRaw::Yes
603+
} else {
604+
tt::IdentIsRaw::No
605+
},
606+
text,
607+
)
608+
} else {
609+
tt::IdentIsRaw::split_from_symbol(&text)
610+
};
569611
tt::Leaf::Ident(tt::Ident {
570-
text: self.text[repr.text as usize].as_str().into(),
612+
text: text.into(),
571613
span: read_span(repr.id),
572-
is_raw: if repr.is_raw {
573-
tt::IdentIsRaw::Yes
574-
} else {
575-
tt::IdentIsRaw::No
576-
},
614+
is_raw,
577615
})
578616
.into()
579617
}

‎crates/tt/src/lib.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ use std::fmt;
99

1010
use stdx::impl_from;
1111

12-
pub use smol_str::SmolStr;
12+
pub use smol_str::{format_smolstr, SmolStr};
1313
pub use text_size::{TextRange, TextSize};
1414

1515
#[derive(Clone, PartialEq, Debug)]

0 commit comments

Comments
 (0)