Skip to content

Commit 21bf6ce

Browse files
matkladmatklad
committed
WIP: Move jointness to Token
1 parent fabd8a6 commit 21bf6ce

File tree

12 files changed

+134
-140
lines changed

12 files changed

+134
-140
lines changed

‎compiler/rustc_ast/src/attr/mod.rs

+8-6
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ use crate::ast::{Path, PathSegment};
88
use crate::mut_visit::visit_clobber;
99
use crate::ptr::P;
1010
use crate::token::{self, CommentKind, Token};
11-
use crate::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndJoint};
11+
use crate::tokenstream::{DelimSpan, TokenStream, TokenTree};
1212

1313
use rustc_index::bit_set::GrowableBitSet;
1414
use rustc_span::source_map::{BytePos, Spanned};
@@ -362,7 +362,7 @@ pub fn list_contains_name(items: &[NestedMetaItem], name: Symbol) -> bool {
362362
}
363363

364364
impl MetaItem {
365-
fn token_trees_and_joints(&self) -> Vec<TreeAndJoint> {
365+
fn token_trees_and_joints(&self) -> Vec<TokenTree> {
366366
let mut idents = vec![];
367367
let mut last_pos = BytePos(0 as u32);
368368
for (i, segment) in self.path.segments.iter().enumerate() {
@@ -388,6 +388,7 @@ impl MetaItem {
388388
Some(TokenTree::Token(Token {
389389
kind: kind @ (token::Ident(..) | token::ModSep),
390390
span,
391+
..
391392
})) => 'arm: {
392393
let mut segments = if let token::Ident(name, _) = kind {
393394
if let Some(TokenTree::Token(Token { kind: token::ModSep, .. })) = tokens.peek()
@@ -401,8 +402,9 @@ impl MetaItem {
401402
vec![PathSegment::path_root(span)]
402403
};
403404
loop {
404-
if let Some(TokenTree::Token(Token { kind: token::Ident(name, _), span })) =
405-
tokens.next().map(TokenTree::uninterpolate)
405+
if let Some(TokenTree::Token(Token {
406+
kind: token::Ident(name, _), span, ..
407+
})) = tokens.next().map(TokenTree::uninterpolate)
406408
{
407409
segments.push(PathSegment::from_ident(Ident::new(name, span)));
408410
} else {
@@ -459,7 +461,7 @@ impl MetaItemKind {
459461
}
460462
}
461463

462-
fn token_trees_and_joints(&self, span: Span) -> Vec<TreeAndJoint> {
464+
fn token_trees_and_joints(&self, span: Span) -> Vec<TokenTree> {
463465
match *self {
464466
MetaItemKind::Word => vec![],
465467
MetaItemKind::NameValue(ref lit) => {
@@ -554,7 +556,7 @@ impl NestedMetaItem {
554556
}
555557
}
556558

557-
fn token_trees_and_joints(&self) -> Vec<TreeAndJoint> {
559+
fn token_trees_and_joints(&self) -> Vec<TokenTree> {
558560
match *self {
559561
NestedMetaItem::MetaItem(ref item) => item.token_trees_and_joints(),
560562
NestedMetaItem::Literal(ref lit) => vec![lit.token_tree().into()],

‎compiler/rustc_ast/src/mut_visit.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -641,14 +641,14 @@ pub fn noop_visit_tt<T: MutVisitor>(tt: &mut TokenTree, vis: &mut T) {
641641

642642
pub fn noop_visit_tts<T: MutVisitor>(TokenStream(tts): &mut TokenStream, vis: &mut T) {
643643
let tts = Lrc::make_mut(tts);
644-
visit_vec(tts, |(tree, _is_joint)| vis.visit_tt(tree));
644+
visit_vec(tts, |tree| vis.visit_tt(tree));
645645
}
646646

647647
// Applies ident visitor if it's an ident; applies other visits to interpolated nodes.
648648
// In practice the ident part is not actually used by specific visitors right now,
649649
// but there's a test below checking that it works.
650650
pub fn noop_visit_token<T: MutVisitor>(t: &mut Token, vis: &mut T) {
651-
let Token { kind, span } = t;
651+
let Token { kind, span, is_joint: _ } = t;
652652
match kind {
653653
token::Ident(name, _) | token::Lifetime(name) => {
654654
let mut ident = Ident::new(*name, *span);

‎compiler/rustc_ast/src/token.rs

+21-4
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ pub use TokenKind::*;
66

77
use crate::ast;
88
use crate::ptr::P;
9-
use crate::tokenstream::TokenTree;
9+
use crate::tokenstream::{IsJoint, TokenTree};
1010

1111
use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
1212
use rustc_data_structures::sync::Lrc;
@@ -262,6 +262,13 @@ rustc_data_structures::static_assert_size!(TokenKind, 16);
262262
pub struct Token {
263263
pub kind: TokenKind,
264264
pub span: Span,
265+
/// Is this operator token immediately followed by another operator token,
266+
/// without whitespace?
267+
///
268+
/// At the moment, this field is only used for proc_macros, and joint tokens
269+
/// are usually represented by dedicated token kinds. We want to transition
270+
/// to using jointness everywhere though (#63689).
271+
pub is_joint: IsJoint,
265272
}
266273

267274
impl TokenKind {
@@ -315,7 +322,11 @@ impl TokenKind {
315322

316323
impl Token {
317324
pub fn new(kind: TokenKind, span: Span) -> Self {
318-
Token { kind, span }
325+
Token::with_spacing(kind, span, IsJoint::NonJoint)
326+
}
327+
328+
pub fn with_spacing(kind: TokenKind, span: Span, is_joint: IsJoint) -> Self {
329+
Token { kind, span, is_joint }
319330
}
320331

321332
/// Some token that will be thrown away later.
@@ -613,6 +624,10 @@ impl Token {
613624
}
614625

615626
pub fn glue(&self, joint: &Token) -> Option<Token> {
627+
if self.is_joint == IsJoint::NonJoint {
628+
return None;
629+
}
630+
616631
let kind = match self.kind {
617632
Eq => match joint.kind {
618633
Eq => EqEq,
@@ -668,7 +683,9 @@ impl Token {
668683
| Lifetime(..) | Interpolated(..) | DocComment(..) | Eof => return None,
669684
};
670685

671-
Some(Token::new(kind, self.span.to(joint.span)))
686+
let mut token = Token::new(kind, self.span.to(joint.span));
687+
token.is_joint = joint.is_joint;
688+
Some(token)
672689
}
673690
}
674691

@@ -699,7 +716,7 @@ pub enum Nonterminal {
699716

700717
// `Nonterminal` is used a lot. Make sure it doesn't unintentionally get bigger.
701718
#[cfg(target_arch = "x86_64")]
702-
rustc_data_structures::static_assert_size!(Nonterminal, 40);
719+
rustc_data_structures::static_assert_size!(Nonterminal, 48);
703720

704721
#[derive(Debug, Copy, Clone, PartialEq, Encodable, Decodable)]
705722
pub enum NonterminalKind {

‎compiler/rustc_ast/src/tokenstream.rs

+24-49
Original file line numberDiff line numberDiff line change
@@ -82,10 +82,6 @@ impl TokenTree {
8282
}
8383
}
8484

85-
pub fn joint(self) -> TokenStream {
86-
TokenStream::new(vec![(self, Joint)])
87-
}
88-
8985
pub fn token(kind: TokenKind, span: Span) -> TokenTree {
9086
TokenTree::Token(Token::new(kind, span))
9187
}
@@ -125,22 +121,18 @@ where
125121
/// instead of a representation of the abstract syntax tree.
126122
/// Today's `TokenTree`s can still contain AST via `token::Interpolated` for back-compat.
127123
#[derive(Clone, Debug, Default, Encodable, Decodable)]
128-
pub struct TokenStream(pub Lrc<Vec<TreeAndJoint>>);
129-
130-
pub type TreeAndJoint = (TokenTree, IsJoint);
124+
pub struct TokenStream(pub Lrc<Vec<TokenTree>>);
131125

132126
// `TokenStream` is used a lot. Make sure it doesn't unintentionally get bigger.
133127
#[cfg(target_arch = "x86_64")]
134128
rustc_data_structures::static_assert_size!(TokenStream, 8);
135129

136-
#[derive(Clone, Copy, Debug, PartialEq, Encodable, Decodable)]
130+
#[derive(Clone, Copy, Debug, PartialEq, Encodable, Decodable, HashStable_Generic)]
137131
pub enum IsJoint {
138132
Joint,
139133
NonJoint,
140134
}
141135

142-
use IsJoint::*;
143-
144136
impl TokenStream {
145137
/// Given a `TokenStream` with a `Stream` of only two arguments, return a new `TokenStream`
146138
/// separating the two arguments with a comma for diagnostic suggestions.
@@ -151,22 +143,22 @@ impl TokenStream {
151143
while let Some((pos, ts)) = iter.next() {
152144
if let Some((_, next)) = iter.peek() {
153145
let sp = match (&ts, &next) {
154-
(_, (TokenTree::Token(Token { kind: token::Comma, .. }), _)) => continue,
146+
(_, TokenTree::Token(Token { kind: token::Comma, .. })) => continue,
155147
(
156-
(TokenTree::Token(token_left), NonJoint),
157-
(TokenTree::Token(token_right), _),
148+
TokenTree::Token(token_left @ Token { is_joint: IsJoint::NonJoint, .. }),
149+
TokenTree::Token(token_right),
158150
) if ((token_left.is_ident() && !token_left.is_reserved_ident())
159151
|| token_left.is_lit())
160152
&& ((token_right.is_ident() && !token_right.is_reserved_ident())
161153
|| token_right.is_lit()) =>
162154
{
163155
token_left.span
164156
}
165-
((TokenTree::Delimited(sp, ..), NonJoint), _) => sp.entire(),
157+
(TokenTree::Delimited(sp, ..), _) => sp.entire(),
166158
_ => continue,
167159
};
168160
let sp = sp.shrink_to_hi();
169-
let comma = (TokenTree::token(token::Comma, sp), NonJoint);
161+
let comma = TokenTree::token(token::Comma, sp);
170162
suggestion = Some((pos, comma, sp));
171163
}
172164
}
@@ -184,19 +176,13 @@ impl TokenStream {
184176

185177
impl From<TokenTree> for TokenStream {
186178
fn from(tree: TokenTree) -> TokenStream {
187-
TokenStream::new(vec![(tree, NonJoint)])
188-
}
189-
}
190-
191-
impl From<TokenTree> for TreeAndJoint {
192-
fn from(tree: TokenTree) -> TreeAndJoint {
193-
(tree, NonJoint)
179+
TokenStream::new(vec![tree])
194180
}
195181
}
196182

197183
impl iter::FromIterator<TokenTree> for TokenStream {
198184
fn from_iter<I: IntoIterator<Item = TokenTree>>(iter: I) -> Self {
199-
TokenStream::new(iter.into_iter().map(Into::into).collect::<Vec<TreeAndJoint>>())
185+
TokenStream::new(iter.into_iter().collect())
200186
}
201187
}
202188

@@ -209,7 +195,7 @@ impl PartialEq<TokenStream> for TokenStream {
209195
}
210196

211197
impl TokenStream {
212-
pub fn new(streams: Vec<TreeAndJoint>) -> TokenStream {
198+
pub fn new(streams: Vec<TokenTree>) -> TokenStream {
213199
TokenStream(Lrc::new(streams))
214200
}
215201

@@ -224,8 +210,8 @@ impl TokenStream {
224210
pub fn span(&self) -> Option<Span> {
225211
match &**self.0 {
226212
[] => None,
227-
[(tt, _)] => Some(tt.span()),
228-
[(tt_start, _), .., (tt_end, _)] => Some(tt_start.span().to(tt_end.span())),
213+
[tt] => Some(tt.span()),
214+
[tt_start, .., tt_end] => Some(tt_start.span().to(tt_end.span())),
229215
}
230216
}
231217

@@ -290,18 +276,12 @@ impl TokenStream {
290276

291277
pub fn map_enumerated<F: FnMut(usize, TokenTree) -> TokenTree>(self, mut f: F) -> TokenStream {
292278
TokenStream(Lrc::new(
293-
self.0
294-
.iter()
295-
.enumerate()
296-
.map(|(i, (tree, is_joint))| (f(i, tree.clone()), *is_joint))
297-
.collect(),
279+
self.0.iter().enumerate().map(|(i, tree)| f(i, tree.clone())).collect(),
298280
))
299281
}
300282

301283
pub fn map<F: FnMut(TokenTree) -> TokenTree>(self, mut f: F) -> TokenStream {
302-
TokenStream(Lrc::new(
303-
self.0.iter().map(|(tree, is_joint)| (f(tree.clone()), *is_joint)).collect(),
304-
))
284+
TokenStream(Lrc::new(self.0.iter().map(|tree| f(tree.clone())).collect()))
305285
}
306286
}
307287

@@ -320,11 +300,11 @@ impl TokenStreamBuilder {
320300
// If `self` is not empty and the last tree within the last stream is a
321301
// token tree marked with `Joint`...
322302
if let Some(TokenStream(ref mut last_stream_lrc)) = self.0.last_mut() {
323-
if let Some((TokenTree::Token(last_token), Joint)) = last_stream_lrc.last() {
303+
if let Some(TokenTree::Token(last_token)) = last_stream_lrc.last() {
324304
// ...and `stream` is not empty and the first tree within it is
325305
// a token tree...
326306
let TokenStream(ref mut stream_lrc) = stream;
327-
if let Some((TokenTree::Token(token), is_joint)) = stream_lrc.first() {
307+
if let Some(TokenTree::Token(token)) = stream_lrc.first() {
328308
// ...and the two tokens can be glued together...
329309
if let Some(glued_tok) = last_token.glue(&token) {
330310
// ...then do so, by overwriting the last token
@@ -337,8 +317,7 @@ impl TokenStreamBuilder {
337317
// Overwrite the last token tree with the merged
338318
// token.
339319
let last_vec_mut = Lrc::make_mut(last_stream_lrc);
340-
*last_vec_mut.last_mut().unwrap() =
341-
(TokenTree::Token(glued_tok), *is_joint);
320+
*last_vec_mut.last_mut().unwrap() = TokenTree::Token(glued_tok);
342321

343322
// Remove the first token tree from `stream`. (This
344323
// is almost always the only tree in `stream`.)
@@ -375,23 +354,19 @@ impl Iterator for Cursor {
375354
type Item = TokenTree;
376355

377356
fn next(&mut self) -> Option<TokenTree> {
378-
self.next_with_joint().map(|(tree, _)| tree)
379-
}
380-
}
381-
382-
impl Cursor {
383-
fn new(stream: TokenStream) -> Self {
384-
Cursor { stream, index: 0 }
385-
}
386-
387-
pub fn next_with_joint(&mut self) -> Option<TreeAndJoint> {
388357
if self.index < self.stream.len() {
389358
self.index += 1;
390359
Some(self.stream.0[self.index - 1].clone())
391360
} else {
392361
None
393362
}
394363
}
364+
}
365+
366+
impl Cursor {
367+
fn new(stream: TokenStream) -> Self {
368+
Cursor { stream, index: 0 }
369+
}
395370

396371
pub fn append(&mut self, new_stream: TokenStream) {
397372
if new_stream.is_empty() {
@@ -404,7 +379,7 @@ impl Cursor {
404379
}
405380

406381
pub fn look_ahead(&self, n: usize) -> Option<TokenTree> {
407-
self.stream.0[self.index..].get(n).map(|(tree, _)| tree.clone())
382+
self.stream.0[self.index..].get(n).cloned()
408383
}
409384
}
410385

‎compiler/rustc_expand/src/mbe/quoted.rs

+24-26
Original file line numberDiff line numberDiff line change
@@ -56,31 +56,29 @@ pub(super) fn parse(
5656
match tree {
5757
TokenTree::MetaVar(start_sp, ident) if expect_matchers => {
5858
let span = match trees.next() {
59-
Some(tokenstream::TokenTree::Token(Token { kind: token::Colon, span })) => {
60-
match trees.next() {
61-
Some(tokenstream::TokenTree::Token(token)) => match token.ident() {
62-
Some((frag, _)) => {
63-
let span = token.span.with_lo(start_sp.lo());
64-
let kind = token::NonterminalKind::from_symbol(frag.name)
65-
.unwrap_or_else(|| {
66-
let msg = format!(
67-
"invalid fragment specifier `{}`",
68-
frag.name
69-
);
70-
sess.span_diagnostic
71-
.struct_span_err(span, &msg)
72-
.help(VALID_FRAGMENT_NAMES_MSG)
73-
.emit();
74-
token::NonterminalKind::Ident
75-
});
76-
result.push(TokenTree::MetaVarDecl(span, ident, kind));
77-
continue;
78-
}
79-
_ => token.span,
80-
},
81-
tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span),
82-
}
83-
}
59+
Some(tokenstream::TokenTree::Token(Token {
60+
kind: token::Colon, span, ..
61+
})) => match trees.next() {
62+
Some(tokenstream::TokenTree::Token(token)) => match token.ident() {
63+
Some((frag, _)) => {
64+
let span = token.span.with_lo(start_sp.lo());
65+
let kind = token::NonterminalKind::from_symbol(frag.name)
66+
.unwrap_or_else(|| {
67+
let msg =
68+
format!("invalid fragment specifier `{}`", frag.name);
69+
sess.span_diagnostic
70+
.struct_span_err(span, &msg)
71+
.help(VALID_FRAGMENT_NAMES_MSG)
72+
.emit();
73+
token::NonterminalKind::Ident
74+
});
75+
result.push(TokenTree::MetaVarDecl(span, ident, kind));
76+
continue;
77+
}
78+
_ => token.span,
79+
},
80+
tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span),
81+
},
8482
tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(start_sp),
8583
};
8684
sess.span_diagnostic.struct_span_err(span, "missing fragment specifier").emit();
@@ -119,7 +117,7 @@ fn parse_tree(
119117
// Depending on what `tree` is, we could be parsing different parts of a macro
120118
match tree {
121119
// `tree` is a `$` token. Look at the next token in `trees`
122-
tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }) => {
120+
tokenstream::TokenTree::Token(Token { kind: token::Dollar, span, .. }) => {
123121
// FIXME: Handle `None`-delimited groups in a more systematic way
124122
// during parsing.
125123
let mut next = outer_trees.next();

0 commit comments

Comments
 (0)