Skip to content

Commit 0708bfe

Browse files
bors[bot]edwin0cheng
bors[bot]
and
edwin0cheng
authored
Merge #7159
7159: Refactor mbe to reduce clone and copying r=edwin0cheng a=edwin0cheng bors r+ Co-authored-by: Edwin Cheng <[email protected]>
2 parents c96b4ee + d387bfd commit 0708bfe

File tree

5 files changed

+172
-152
lines changed

5 files changed

+172
-152
lines changed

‎crates/hir_expand/src/proc_macro.rs

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ impl ProcMacroExpander {
5858
}
5959

6060
fn eat_punct(cursor: &mut Cursor, c: char) -> bool {
61-
if let Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) = cursor.token_tree() {
61+
if let Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Punct(punct), _)) = cursor.token_tree() {
6262
if punct.char == c {
6363
*cursor = cursor.bump();
6464
return true;
@@ -68,7 +68,7 @@ fn eat_punct(cursor: &mut Cursor, c: char) -> bool {
6868
}
6969

7070
fn eat_subtree(cursor: &mut Cursor, kind: tt::DelimiterKind) -> bool {
71-
if let Some(tt::TokenTree::Subtree(subtree)) = cursor.token_tree() {
71+
if let Some(tt::buffer::TokenTreeRef::Subtree(subtree, _)) = cursor.token_tree() {
7272
if Some(kind) == subtree.delimiter_kind() {
7373
*cursor = cursor.bump_subtree();
7474
return true;
@@ -78,7 +78,7 @@ fn eat_subtree(cursor: &mut Cursor, kind: tt::DelimiterKind) -> bool {
7878
}
7979

8080
fn eat_ident(cursor: &mut Cursor, t: &str) -> bool {
81-
if let Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) = cursor.token_tree() {
81+
if let Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Ident(ident), _)) = cursor.token_tree() {
8282
if t == ident.text.as_str() {
8383
*cursor = cursor.bump();
8484
return true;
@@ -88,7 +88,7 @@ fn eat_ident(cursor: &mut Cursor, t: &str) -> bool {
8888
}
8989

9090
fn remove_derive_attrs(tt: &tt::Subtree) -> Option<tt::Subtree> {
91-
let buffer = TokenBuffer::new(&tt.token_trees);
91+
let buffer = TokenBuffer::from_tokens(&tt.token_trees);
9292
let mut p = buffer.begin();
9393
let mut result = tt::Subtree::default();
9494

@@ -106,7 +106,7 @@ fn remove_derive_attrs(tt: &tt::Subtree) -> Option<tt::Subtree> {
106106
}
107107
}
108108

109-
result.token_trees.push(curr.token_tree()?.clone());
109+
result.token_trees.push(curr.token_tree()?.cloned());
110110
p = curr.bump();
111111
}
112112

‎crates/mbe/src/mbe_expander/matcher.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -309,7 +309,7 @@ impl<'a> TtIter<'a> {
309309
}
310310
}
311311

312-
let buffer = TokenBuffer::new(&self.inner.as_slice());
312+
let buffer = TokenBuffer::from_tokens(&self.inner.as_slice());
313313
let mut src = SubtreeTokenSource::new(&buffer);
314314
let mut sink = OffsetTokenSink { cursor: buffer.begin(), error: false };
315315

@@ -336,11 +336,11 @@ impl<'a> TtIter<'a> {
336336
err = Some(err!("no tokens consumed"));
337337
}
338338
let res = match res.len() {
339-
1 => Some(res[0].clone()),
339+
1 => Some(res[0].cloned()),
340340
0 => None,
341341
_ => Some(tt::TokenTree::Subtree(tt::Subtree {
342342
delimiter: None,
343-
token_trees: res.into_iter().cloned().collect(),
343+
token_trees: res.into_iter().map(|it| it.cloned()).collect(),
344344
})),
345345
};
346346
ExpandResult { value: res, err }

‎crates/mbe/src/subtree_source.rs

Lines changed: 63 additions & 89 deletions
Original file line numberDiff line numberDiff line change
@@ -1,143 +1,117 @@
11
//! FIXME: write short doc here
22
33
use parser::{Token, TokenSource};
4-
use std::cell::{Cell, Ref, RefCell};
54
use syntax::{lex_single_syntax_kind, SmolStr, SyntaxKind, SyntaxKind::*, T};
6-
use tt::buffer::{Cursor, TokenBuffer};
5+
use tt::buffer::TokenBuffer;
76

87
#[derive(Debug, Clone, Eq, PartialEq)]
98
struct TtToken {
10-
kind: SyntaxKind,
11-
is_joint_to_next: bool,
9+
tt: Token,
1210
text: SmolStr,
1311
}
1412

15-
pub(crate) struct SubtreeTokenSource<'a> {
16-
cached_cursor: Cell<Cursor<'a>>,
17-
cached: RefCell<Vec<Option<TtToken>>>,
13+
pub(crate) struct SubtreeTokenSource {
14+
cached: Vec<TtToken>,
1815
curr: (Token, usize),
1916
}
2017

21-
impl<'a> SubtreeTokenSource<'a> {
18+
impl<'a> SubtreeTokenSource {
2219
// Helper function used in test
2320
#[cfg(test)]
2421
pub(crate) fn text(&self) -> SmolStr {
25-
match *self.get(self.curr.1) {
22+
match self.cached.get(self.curr.1) {
2623
Some(ref tt) => tt.text.clone(),
2724
_ => SmolStr::new(""),
2825
}
2926
}
3027
}
3128

32-
impl<'a> SubtreeTokenSource<'a> {
33-
pub(crate) fn new(buffer: &'a TokenBuffer) -> SubtreeTokenSource<'a> {
34-
let cursor = buffer.begin();
29+
impl<'a> SubtreeTokenSource {
30+
pub(crate) fn new(buffer: &TokenBuffer) -> SubtreeTokenSource {
31+
let mut current = buffer.begin();
32+
let mut cached = Vec::with_capacity(100);
3533

36-
let mut res = SubtreeTokenSource {
37-
curr: (Token { kind: EOF, is_jointed_to_next: false }, 0),
38-
cached_cursor: Cell::new(cursor),
39-
cached: RefCell::new(Vec::with_capacity(10)),
40-
};
41-
res.curr = (res.mk_token(0), 0);
42-
res
43-
}
34+
while !current.eof() {
35+
let cursor = current;
36+
let tt = cursor.token_tree();
4437

45-
fn mk_token(&self, pos: usize) -> Token {
46-
match *self.get(pos) {
47-
Some(ref tt) => Token { kind: tt.kind, is_jointed_to_next: tt.is_joint_to_next },
48-
None => Token { kind: EOF, is_jointed_to_next: false },
49-
}
50-
}
51-
52-
fn get(&self, pos: usize) -> Ref<Option<TtToken>> {
53-
fn is_lifetime(c: Cursor) -> Option<(Cursor, SmolStr)> {
54-
let tkn = c.token_tree();
55-
56-
if let Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) = tkn {
38+
// Check if it is lifetime
39+
if let Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Punct(punct), _)) = tt {
5740
if punct.char == '\'' {
58-
let next = c.bump();
59-
if let Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) = next.token_tree() {
60-
let res_cursor = next.bump();
61-
let text = SmolStr::new("'".to_string() + &ident.to_string());
62-
63-
return Some((res_cursor, text));
41+
let next = cursor.bump();
42+
if let Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Ident(ident), _)) =
43+
next.token_tree()
44+
{
45+
let text = SmolStr::new("'".to_string() + &ident.text);
46+
cached.push(TtToken {
47+
tt: Token { kind: LIFETIME_IDENT, is_jointed_to_next: false },
48+
text,
49+
});
50+
current = next.bump();
51+
continue;
6452
} else {
6553
panic!("Next token must be ident : {:#?}", next.token_tree());
6654
}
6755
}
6856
}
6957

70-
None
71-
}
72-
73-
if pos < self.cached.borrow().len() {
74-
return Ref::map(self.cached.borrow(), |c| &c[pos]);
75-
}
76-
77-
{
78-
let mut cached = self.cached.borrow_mut();
79-
while pos >= cached.len() {
80-
let cursor = self.cached_cursor.get();
81-
if cursor.eof() {
82-
cached.push(None);
83-
continue;
58+
current = match tt {
59+
Some(tt::buffer::TokenTreeRef::Leaf(leaf, _)) => {
60+
cached.push(convert_leaf(&leaf));
61+
cursor.bump()
8462
}
85-
86-
if let Some((curr, text)) = is_lifetime(cursor) {
87-
cached.push(Some(TtToken {
88-
kind: LIFETIME_IDENT,
89-
is_joint_to_next: false,
90-
text,
91-
}));
92-
self.cached_cursor.set(curr);
93-
continue;
63+
Some(tt::buffer::TokenTreeRef::Subtree(subtree, _)) => {
64+
cached.push(convert_delim(subtree.delimiter_kind(), false));
65+
cursor.subtree().unwrap()
9466
}
95-
96-
match cursor.token_tree() {
97-
Some(tt::TokenTree::Leaf(leaf)) => {
98-
cached.push(Some(convert_leaf(&leaf)));
99-
self.cached_cursor.set(cursor.bump());
100-
}
101-
Some(tt::TokenTree::Subtree(subtree)) => {
102-
self.cached_cursor.set(cursor.subtree().unwrap());
103-
cached.push(Some(convert_delim(subtree.delimiter_kind(), false)));
104-
}
105-
None => {
106-
if let Some(subtree) = cursor.end() {
107-
cached.push(Some(convert_delim(subtree.delimiter_kind(), true)));
108-
self.cached_cursor.set(cursor.bump());
109-
}
67+
None => {
68+
if let Some(subtree) = cursor.end() {
69+
cached.push(convert_delim(subtree.delimiter_kind(), true));
70+
cursor.bump()
71+
} else {
72+
continue;
11073
}
11174
}
112-
}
75+
};
11376
}
11477

115-
Ref::map(self.cached.borrow(), |c| &c[pos])
78+
let mut res = SubtreeTokenSource {
79+
curr: (Token { kind: EOF, is_jointed_to_next: false }, 0),
80+
cached,
81+
};
82+
res.curr = (res.token(0), 0);
83+
res
84+
}
85+
86+
fn token(&self, pos: usize) -> Token {
87+
match self.cached.get(pos) {
88+
Some(it) => it.tt,
89+
None => Token { kind: EOF, is_jointed_to_next: false },
90+
}
11691
}
11792
}
11893

119-
impl<'a> TokenSource for SubtreeTokenSource<'a> {
94+
impl<'a> TokenSource for SubtreeTokenSource {
12095
fn current(&self) -> Token {
12196
self.curr.0
12297
}
12398

12499
/// Lookahead n token
125100
fn lookahead_nth(&self, n: usize) -> Token {
126-
self.mk_token(self.curr.1 + n)
101+
self.token(self.curr.1 + n)
127102
}
128103

129104
/// bump cursor to next token
130105
fn bump(&mut self) {
131106
if self.current().kind == EOF {
132107
return;
133108
}
134-
135-
self.curr = (self.mk_token(self.curr.1 + 1), self.curr.1 + 1);
109+
self.curr = (self.token(self.curr.1 + 1), self.curr.1 + 1);
136110
}
137111

138112
/// Is the current token a specified keyword?
139113
fn is_keyword(&self, kw: &str) -> bool {
140-
match *self.get(self.curr.1) {
114+
match self.cached.get(self.curr.1) {
141115
Some(ref t) => t.text == *kw,
142116
_ => false,
143117
}
@@ -155,7 +129,7 @@ fn convert_delim(d: Option<tt::DelimiterKind>, closing: bool) -> TtToken {
155129
let idx = closing as usize;
156130
let kind = kinds[idx];
157131
let text = if !texts.is_empty() { &texts[idx..texts.len() - (1 - idx)] } else { "" };
158-
TtToken { kind, is_joint_to_next: false, text: SmolStr::new(text) }
132+
TtToken { tt: Token { kind, is_jointed_to_next: false }, text: SmolStr::new(text) }
159133
}
160134

161135
fn convert_literal(l: &tt::Literal) -> TtToken {
@@ -169,7 +143,7 @@ fn convert_literal(l: &tt::Literal) -> TtToken {
169143
})
170144
.unwrap_or_else(|| panic!("Fail to convert given literal {:#?}", &l));
171145

172-
TtToken { kind, is_joint_to_next: false, text: l.text.clone() }
146+
TtToken { tt: Token { kind, is_jointed_to_next: false }, text: l.text.clone() }
173147
}
174148

175149
fn convert_ident(ident: &tt::Ident) -> TtToken {
@@ -180,7 +154,7 @@ fn convert_ident(ident: &tt::Ident) -> TtToken {
180154
_ => SyntaxKind::from_keyword(ident.text.as_str()).unwrap_or(IDENT),
181155
};
182156

183-
TtToken { kind, is_joint_to_next: false, text: ident.text.clone() }
157+
TtToken { tt: Token { kind, is_jointed_to_next: false }, text: ident.text.clone() }
184158
}
185159

186160
fn convert_punct(p: tt::Punct) -> TtToken {
@@ -194,7 +168,7 @@ fn convert_punct(p: tt::Punct) -> TtToken {
194168
let s: &str = p.char.encode_utf8(&mut buf);
195169
SmolStr::new(s)
196170
};
197-
TtToken { kind, is_joint_to_next: p.spacing == tt::Spacing::Joint, text }
171+
TtToken { tt: Token { kind, is_jointed_to_next: p.spacing == tt::Spacing::Joint }, text }
198172
}
199173

200174
fn convert_leaf(leaf: &tt::Leaf) -> TtToken {
@@ -208,6 +182,7 @@ fn convert_leaf(leaf: &tt::Leaf) -> TtToken {
208182
#[cfg(test)]
209183
mod tests {
210184
use super::{convert_literal, TtToken};
185+
use parser::Token;
211186
use syntax::{SmolStr, SyntaxKind};
212187

213188
#[test]
@@ -218,8 +193,7 @@ mod tests {
218193
text: SmolStr::new("-42.0")
219194
}),
220195
TtToken {
221-
kind: SyntaxKind::FLOAT_NUMBER,
222-
is_joint_to_next: false,
196+
tt: Token { kind: SyntaxKind::FLOAT_NUMBER, is_jointed_to_next: false },
223197
text: SmolStr::new("-42.0")
224198
}
225199
);

0 commit comments

Comments
 (0)