Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit 11c353b

Browse files
committedMay 9, 2019
Auto merge of #60679 - petrochenkov:lit2, r=<try>
Keep original literal tokens in AST The original literal tokens (`token::Lit`) are kept in AST until lowering to HIR. The tokens are kept together with their lowered "semantic" representation (`ast::LitKind`), so the size of `ast::Lit` is increased (this also increases the size of meta-item structs used for processing built-in attributes). However, the size of `ast::Expr` stays the same. The intent is to remove the "semantic" representation from AST eventually and keep literals as tokens until lowering to HIR (at least), and I'm going to work on that, but it would be good to land this sooner to unblock progress on the [lexer refactoring](#59706). Fixes a part of #43081 (literal tokens that are passed to proc macros are always precise, including hexadecimal numbers, strings with their original escaping, etc) Fixes a part of #60495 (everything except for proc macro API doesn't need escaping anymore) This also allows to eliminate a certain hack from the lexer (https://rust-lang.zulipchat.com/#narrow/stream/131828-t-compiler/topic/pretty-printing.20comments/near/165005357). cc @matklad
2 parents ef01f29 + 865451e commit 11c353b

File tree

17 files changed

+221
-272
lines changed

17 files changed

+221
-272
lines changed
 

‎src/librustc/hir/lowering.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -4124,7 +4124,7 @@ impl<'a> LoweringContext<'a> {
41244124
let ohs = P(self.lower_expr(ohs));
41254125
hir::ExprKind::Unary(op, ohs)
41264126
}
4127-
ExprKind::Lit(ref l) => hir::ExprKind::Lit((*l).clone()),
4127+
ExprKind::Lit(ref l) => hir::ExprKind::Lit(respan(l.span, l.node.clone())),
41284128
ExprKind::Cast(ref expr, ref ty) => {
41294129
let expr = P(self.lower_expr(expr));
41304130
hir::ExprKind::Cast(expr, self.lower_ty(ty, ImplTraitContext::disallowed()))

‎src/librustc/hir/mod.rs

+4-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ use syntax_pos::{Span, DUMMY_SP, symbol::InternedString};
2020
use syntax::source_map::Spanned;
2121
use rustc_target::spec::abi::Abi;
2222
use syntax::ast::{self, CrateSugar, Ident, Name, NodeId, AsmDialect};
23-
use syntax::ast::{Attribute, Label, Lit, StrStyle, FloatTy, IntTy, UintTy};
23+
use syntax::ast::{Attribute, Label, LitKind, StrStyle, FloatTy, IntTy, UintTy};
2424
use syntax::attr::{InlineAttr, OptimizeAttr};
2525
use syntax::ext::hygiene::SyntaxContext;
2626
use syntax::ptr::P;
@@ -1331,6 +1331,9 @@ impl BodyOwnerKind {
13311331
}
13321332
}
13331333

1334+
/// A literal.
1335+
pub type Lit = Spanned<LitKind>;
1336+
13341337
/// A constant (expression) that's not an item or associated item,
13351338
/// but needs its own `DefId` for type-checking, const-eval, etc.
13361339
/// These are usually found nested inside types (e.g., array lengths)

‎src/librustc/hir/print.rs

+63-35
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,10 @@ use crate::hir;
1515
use crate::hir::{PatKind, GenericBound, TraitBoundModifier, RangeEnd};
1616
use crate::hir::{GenericParam, GenericParamKind, GenericArg};
1717

18+
use std::ascii;
1819
use std::borrow::Cow;
1920
use std::cell::Cell;
2021
use std::io::{self, Write, Read};
21-
use std::iter::Peekable;
2222
use std::vec;
2323

2424
pub enum AnnNode<'a> {
@@ -76,7 +76,6 @@ pub struct State<'a> {
7676
pub s: pp::Printer<'a>,
7777
cm: Option<&'a SourceMap>,
7878
comments: Option<Vec<comments::Comment>>,
79-
literals: Peekable<vec::IntoIter<comments::Literal>>,
8079
cur_cmnt: usize,
8180
boxes: Vec<pp::Breaks>,
8281
ann: &'a (dyn PpAnn + 'a),
@@ -98,14 +97,6 @@ impl<'a> PrintState<'a> for State<'a> {
9897
fn cur_cmnt(&mut self) -> &mut usize {
9998
&mut self.cur_cmnt
10099
}
101-
102-
fn cur_lit(&mut self) -> Option<&comments::Literal> {
103-
self.literals.peek()
104-
}
105-
106-
fn bump_lit(&mut self) -> Option<comments::Literal> {
107-
self.literals.next()
108-
}
109100
}
110101

111102
#[allow(non_upper_case_globals)]
@@ -116,18 +107,16 @@ pub const default_columns: usize = 78;
116107

117108

118109
/// Requires you to pass an input filename and reader so that
119-
/// it can scan the input text for comments and literals to
120-
/// copy forward.
110+
/// it can scan the input text for comments to copy forward.
121111
pub fn print_crate<'a>(cm: &'a SourceMap,
122112
sess: &ParseSess,
123113
krate: &hir::Crate,
124114
filename: FileName,
125115
input: &mut dyn Read,
126116
out: Box<dyn Write + 'a>,
127-
ann: &'a dyn PpAnn,
128-
is_expanded: bool)
117+
ann: &'a dyn PpAnn)
129118
-> io::Result<()> {
130-
let mut s = State::new_from_input(cm, sess, filename, input, out, ann, is_expanded);
119+
let mut s = State::new_from_input(cm, sess, filename, input, out, ann);
131120

132121
// When printing the AST, we sometimes need to inject `#[no_std]` here.
133122
// Since you can't compile the HIR, it's not necessary.
@@ -143,36 +132,21 @@ impl<'a> State<'a> {
143132
filename: FileName,
144133
input: &mut dyn Read,
145134
out: Box<dyn Write + 'a>,
146-
ann: &'a dyn PpAnn,
147-
is_expanded: bool)
135+
ann: &'a dyn PpAnn)
148136
-> State<'a> {
149-
let (cmnts, lits) = comments::gather_comments_and_literals(sess, filename, input);
150-
151-
State::new(cm,
152-
out,
153-
ann,
154-
Some(cmnts),
155-
// If the code is post expansion, don't use the table of
156-
// literals, since it doesn't correspond with the literals
157-
// in the AST anymore.
158-
if is_expanded {
159-
None
160-
} else {
161-
Some(lits)
162-
})
137+
let comments = comments::gather_comments(sess, filename, input);
138+
State::new(cm, out, ann, Some(comments))
163139
}
164140

165141
pub fn new(cm: &'a SourceMap,
166142
out: Box<dyn Write + 'a>,
167143
ann: &'a dyn PpAnn,
168-
comments: Option<Vec<comments::Comment>>,
169-
literals: Option<Vec<comments::Literal>>)
144+
comments: Option<Vec<comments::Comment>>)
170145
-> State<'a> {
171146
State {
172147
s: pp::mk_printer(out, default_columns),
173148
cm: Some(cm),
174149
comments,
175-
literals: literals.unwrap_or_default().into_iter().peekable(),
176150
cur_cmnt: 0,
177151
boxes: Vec::new(),
178152
ann,
@@ -189,7 +163,6 @@ pub fn to_string<F>(ann: &dyn PpAnn, f: F) -> String
189163
s: pp::mk_printer(Box::new(&mut wr), default_columns),
190164
cm: None,
191165
comments: None,
192-
literals: vec![].into_iter().peekable(),
193166
cur_cmnt: 0,
194167
boxes: Vec::new(),
195168
ann,
@@ -1335,6 +1308,61 @@ impl<'a> State<'a> {
13351308
self.print_expr_maybe_paren(expr, parser::PREC_PREFIX)
13361309
}
13371310

1311+
fn print_literal(&mut self, lit: &hir::Lit) -> io::Result<()> {
1312+
self.maybe_print_comment(lit.span.lo())?;
1313+
match lit.node {
1314+
hir::LitKind::Str(st, style) => self.print_string(&st.as_str(), style),
1315+
hir::LitKind::Err(st) => {
1316+
let st = st.as_str().escape_debug().to_string();
1317+
let mut res = String::with_capacity(st.len() + 2);
1318+
res.push('\'');
1319+
res.push_str(&st);
1320+
res.push('\'');
1321+
self.writer().word(res)
1322+
}
1323+
hir::LitKind::Byte(byte) => {
1324+
let mut res = String::from("b'");
1325+
res.extend(ascii::escape_default(byte).map(|c| c as char));
1326+
res.push('\'');
1327+
self.writer().word(res)
1328+
}
1329+
hir::LitKind::Char(ch) => {
1330+
let mut res = String::from("'");
1331+
res.extend(ch.escape_default());
1332+
res.push('\'');
1333+
self.writer().word(res)
1334+
}
1335+
hir::LitKind::Int(i, t) => {
1336+
match t {
1337+
ast::LitIntType::Signed(st) => {
1338+
self.writer().word(st.val_to_string(i as i128))
1339+
}
1340+
ast::LitIntType::Unsigned(ut) => {
1341+
self.writer().word(ut.val_to_string(i))
1342+
}
1343+
ast::LitIntType::Unsuffixed => {
1344+
self.writer().word(i.to_string())
1345+
}
1346+
}
1347+
}
1348+
hir::LitKind::Float(ref f, t) => {
1349+
self.writer().word(format!("{}{}", &f, t.ty_to_string()))
1350+
}
1351+
hir::LitKind::FloatUnsuffixed(ref f) => self.writer().word(f.as_str().to_string()),
1352+
hir::LitKind::Bool(val) => {
1353+
if val { self.writer().word("true") } else { self.writer().word("false") }
1354+
}
1355+
hir::LitKind::ByteStr(ref v) => {
1356+
let mut escaped: String = String::new();
1357+
for &ch in v.iter() {
1358+
escaped.extend(ascii::escape_default(ch)
1359+
.map(|c| c as char));
1360+
}
1361+
self.writer().word(format!("b\"{}\"", escaped))
1362+
}
1363+
}
1364+
}
1365+
13381366
pub fn print_expr(&mut self, expr: &hir::Expr) -> io::Result<()> {
13391367
self.maybe_print_comment(expr.span.lo())?;
13401368
self.print_outer_attributes(&expr.attrs)?;

‎src/librustc/ich/impls_syntax.rs

+24-17
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,13 @@ impl_stable_hash_for!(enum ::syntax::ast::LitIntType {
162162
Unsuffixed
163163
});
164164

165-
impl_stable_hash_for_spanned!(::syntax::ast::LitKind);
165+
impl_stable_hash_for!(struct ::syntax::ast::Lit {
166+
node,
167+
token,
168+
suffix,
169+
span
170+
});
171+
166172
impl_stable_hash_for!(enum ::syntax::ast::LitKind {
167173
Str(value, style),
168174
Err(value),
@@ -175,6 +181,8 @@ impl_stable_hash_for!(enum ::syntax::ast::LitKind {
175181
Bool(value)
176182
});
177183

184+
impl_stable_hash_for_spanned!(::syntax::ast::LitKind);
185+
178186
impl_stable_hash_for!(enum ::syntax::ast::IntTy { Isize, I8, I16, I32, I64, I128 });
179187
impl_stable_hash_for!(enum ::syntax::ast::UintTy { Usize, U8, U16, U32, U64, U128 });
180188
impl_stable_hash_for!(enum ::syntax::ast::FloatTy { F32, F64 });
@@ -280,6 +288,19 @@ for tokenstream::TokenStream {
280288
}
281289
}
282290

291+
impl_stable_hash_for!(enum token::Lit {
292+
Bool(val),
293+
Byte(val),
294+
Char(val),
295+
Err(val),
296+
Integer(val),
297+
Float(val),
298+
Str_(val),
299+
ByteStr(val),
300+
StrRaw(val, n),
301+
ByteStrRaw(val, n)
302+
});
303+
283304
fn hash_token<'a, 'gcx, W: StableHasherResult>(
284305
token: &token::Token,
285306
hcx: &mut StableHashingContext<'a>,
@@ -327,22 +348,8 @@ fn hash_token<'a, 'gcx, W: StableHasherResult>(
327348
token::Token::CloseDelim(delim_token) => {
328349
std_hash::Hash::hash(&delim_token, hasher);
329350
}
330-
token::Token::Literal(ref lit, ref opt_name) => {
331-
mem::discriminant(lit).hash_stable(hcx, hasher);
332-
match *lit {
333-
token::Lit::Byte(val) |
334-
token::Lit::Char(val) |
335-
token::Lit::Err(val) |
336-
token::Lit::Integer(val) |
337-
token::Lit::Float(val) |
338-
token::Lit::Str_(val) |
339-
token::Lit::ByteStr(val) => val.hash_stable(hcx, hasher),
340-
token::Lit::StrRaw(val, n) |
341-
token::Lit::ByteStrRaw(val, n) => {
342-
val.hash_stable(hcx, hasher);
343-
n.hash_stable(hcx, hasher);
344-
}
345-
};
351+
token::Token::Literal(lit, opt_name) => {
352+
lit.hash_stable(hcx, hasher);
346353
opt_name.hash_stable(hcx, hasher);
347354
}
348355

‎src/librustc_driver/pretty.rs

+2-4
Original file line numberDiff line numberDiff line change
@@ -806,8 +806,7 @@ pub fn print_after_hir_lowering<'tcx>(
806806
src_name,
807807
&mut rdr,
808808
box out,
809-
annotation.pp_ann(),
810-
true)
809+
annotation.pp_ann())
811810
})
812811
}
813812

@@ -830,8 +829,7 @@ pub fn print_after_hir_lowering<'tcx>(
830829
src_name,
831830
&mut rdr,
832831
box out,
833-
annotation.pp_ann(),
834-
true);
832+
annotation.pp_ann());
835833
for node_id in uii.all_matching_node_ids(hir_map) {
836834
let node = hir_map.get(node_id);
837835
pp_state.print_node(node)?;

‎src/librustc_lint/types.rs

+5-5
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ impl TypeLimits {
6262
/// Returns `true` iff the lint was overridden.
6363
fn lint_overflowing_range_endpoint<'a, 'tcx>(
6464
cx: &LateContext<'a, 'tcx>,
65-
lit: &ast::Lit,
65+
lit: &hir::Lit,
6666
lit_val: u128,
6767
max: u128,
6868
expr: &'tcx hir::Expr,
@@ -132,7 +132,7 @@ fn uint_ty_range(uint_ty: ast::UintTy) -> (u128, u128) {
132132
}
133133
}
134134

135-
fn get_bin_hex_repr(cx: &LateContext<'_, '_>, lit: &ast::Lit) -> Option<String> {
135+
fn get_bin_hex_repr(cx: &LateContext<'_, '_>, lit: &hir::Lit) -> Option<String> {
136136
let src = cx.sess().source_map().span_to_snippet(lit.span).ok()?;
137137
let firstch = src.chars().next()?;
138138

@@ -249,7 +249,7 @@ fn lint_int_literal<'a, 'tcx>(
249249
cx: &LateContext<'a, 'tcx>,
250250
type_limits: &TypeLimits,
251251
e: &'tcx hir::Expr,
252-
lit: &ast::Lit,
252+
lit: &hir::Lit,
253253
t: ast::IntTy,
254254
v: u128,
255255
) {
@@ -301,7 +301,7 @@ fn lint_int_literal<'a, 'tcx>(
301301
fn lint_uint_literal<'a, 'tcx>(
302302
cx: &LateContext<'a, 'tcx>,
303303
e: &'tcx hir::Expr,
304-
lit: &ast::Lit,
304+
lit: &hir::Lit,
305305
t: ast::UintTy,
306306
) {
307307
let uint_type = if let ast::UintTy::Usize = t {
@@ -363,7 +363,7 @@ fn lint_literal<'a, 'tcx>(
363363
cx: &LateContext<'a, 'tcx>,
364364
type_limits: &TypeLimits,
365365
e: &'tcx hir::Expr,
366-
lit: &ast::Lit,
366+
lit: &hir::Lit,
367367
) {
368368
match cx.tables.node_type(e.hir_id).sty {
369369
ty::Int(t) => {

‎src/librustc_typeck/check/mod.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -3084,7 +3084,7 @@ impl<'a, 'gcx, 'tcx> FnCtxt<'a, 'gcx, 'tcx> {
30843084

30853085
// AST fragment checking
30863086
fn check_lit(&self,
3087-
lit: &ast::Lit,
3087+
lit: &hir::Lit,
30883088
expected: Expectation<'tcx>)
30893089
-> Ty<'tcx>
30903090
{

‎src/librustdoc/clean/cfg.rs

+8-8
Original file line numberDiff line numberDiff line change
@@ -414,10 +414,9 @@ impl<'a> fmt::Display for Html<'a> {
414414
mod test {
415415
use super::Cfg;
416416

417-
use syntax::symbol::Symbol;
418-
use syntax::ast::*;
419-
use syntax::source_map::dummy_spanned;
420417
use syntax_pos::DUMMY_SP;
418+
use syntax::ast::*;
419+
use syntax::symbol::Symbol;
421420
use syntax::with_globals;
422421

423422
fn word_cfg(s: &str) -> Cfg {
@@ -592,12 +591,11 @@ mod test {
592591
let mi = dummy_meta_item_word("all");
593592
assert_eq!(Cfg::parse(&mi), Ok(word_cfg("all")));
594593

594+
let node = LitKind::Str(Symbol::intern("done"), StrStyle::Cooked);
595+
let (token, suffix) = node.lit_token();
595596
let mi = MetaItem {
596597
path: Path::from_ident(Ident::from_str("all")),
597-
node: MetaItemKind::NameValue(dummy_spanned(LitKind::Str(
598-
Symbol::intern("done"),
599-
StrStyle::Cooked,
600-
))),
598+
node: MetaItemKind::NameValue(Lit { node, token, suffix, span: DUMMY_SP }),
601599
span: DUMMY_SP,
602600
};
603601
assert_eq!(Cfg::parse(&mi), Ok(name_value_cfg("all", "done")));
@@ -627,9 +625,11 @@ mod test {
627625
#[test]
628626
fn test_parse_err() {
629627
with_globals(|| {
628+
let node = LitKind::Bool(false);
629+
let (token, suffix) = node.lit_token();
630630
let mi = MetaItem {
631631
path: Path::from_ident(Ident::from_str("foo")),
632-
node: MetaItemKind::NameValue(dummy_spanned(LitKind::Bool(false))),
632+
node: MetaItemKind::NameValue(Lit { node, token, suffix, span: DUMMY_SP }),
633633
span: DUMMY_SP,
634634
};
635635
assert!(Cfg::parse(&mi).is_err());

‎src/librustdoc/html/highlight.rs

+2
Original file line numberDiff line numberDiff line change
@@ -318,6 +318,8 @@ impl<'a> Classifier<'a> {
318318

319319
// Number literals.
320320
token::Integer(..) | token::Float(..) => Class::Number,
321+
322+
token::Bool(..) => panic!("literal token contains `Lit::Bool`"),
321323
}
322324
}
323325

0 commit comments

Comments
 (0)
Please sign in to comment.