Auto merge of #51762 - petrochenkov:oh-hi-mark, r=oli-obk

hygiene: Implement transparent marks and use them for call-site hygiene in proc-macros

Fixes #50050

Author: bors

Diff for: src/libproc_macro/diagnostic.rs

@@ -97,7 +97,7 @@ impl Diagnostic {
     /// Emit the diagnostic.
     #[unstable(feature = "proc_macro", issue = "38356")]
     pub fn emit(self) {
-        ::__internal::with_sess(move |(sess, _)| {
+        ::__internal::with_sess(move |sess, _| {
             let handler = &sess.span_diagnostic;
             let level = __internal::level_to_internal_level(self.level);
             let mut diag = rustc::DiagnosticBuilder::new(handler, level, &*self.message);

Diff for: src/libproc_macro/lib.rs

@@ -58,8 +58,7 @@ use syntax::parse::{self, token};
 use syntax::symbol::{keywords, Symbol};
 use syntax::tokenstream;
 use syntax::parse::lexer::{self, comments};
-use syntax_pos::{FileMap, Pos, SyntaxContext, FileName};
-use syntax_pos::hygiene::Mark;
+use syntax_pos::{FileMap, Pos, FileName};
 
 /// The main type provided by this crate, representing an abstract stream of
 /// tokens, or, more specifically, a sequence of token trees.
@@ -109,6 +108,7 @@ impl TokenStream {
 /// Attempts to break the string into tokens and parse those tokens into a token stream.
 /// May fail for a number of reasons, for example, if the string contains unbalanced delimiters
 /// or characters not existing in the language.
+/// All tokens in the parsed stream get `Span::call_site()` spans.
 ///
 /// NOTE: Some errors may cause panics instead of returning `LexError`. We reserve the right to
 /// change these errors into `LexError`s later.
@@ -117,17 +117,10 @@ impl FromStr for TokenStream {
     type Err = LexError;
 
     fn from_str(src: &str) -> Result<TokenStream, LexError> {
-        __internal::with_sess(|(sess, mark)| {
-            let src = src.to_string();
-            let name = FileName::ProcMacroSourceCode;
-            let expn_info = mark.expn_info().unwrap();
-            let call_site = expn_info.call_site;
-            // notify the expansion info that it is unhygienic
-            let mark = Mark::fresh(mark);
-            mark.set_expn_info(expn_info);
-            let span = call_site.with_ctxt(SyntaxContext::empty().apply_mark(mark));
-            let stream = parse::parse_stream_from_source_str(name, src, sess, Some(span));
-            Ok(__internal::token_stream_wrap(stream))
+        __internal::with_sess(|sess, data| {
+            Ok(__internal::token_stream_wrap(parse::parse_stream_from_source_str(
+                FileName::ProcMacroSourceCode, src.to_string(), sess, Some(data.call_site.0)
+            )))
         })
     }
 }
@@ -184,8 +177,6 @@ impl iter::FromIterator<TokenStream> for TokenStream {
 #[unstable(feature = "proc_macro", issue = "38356")]
 pub mod token_stream {
     use syntax::tokenstream;
-    use syntax_pos::DUMMY_SP;
-
     use {TokenTree, TokenStream, Delimiter};
 
     /// An iterator over `TokenStream`'s `TokenTree`s.
@@ -214,7 +205,7 @@ pub mod token_stream {
                 // need to flattened during iteration over stream's token trees.
                 // Eventually this needs to be removed in favor of keeping original token trees
                 // and not doing the roundtrip through AST.
-                if tree.span().0 == DUMMY_SP {
+                if tree.span().0.is_dummy() {
                     if let TokenTree::Group(ref group) = tree {
                         if group.delimiter() == Delimiter::None {
                             self.cursor.insert(group.stream.clone().0);
@@ -284,10 +275,7 @@ impl Span {
     /// A span that resolves at the macro definition site.
     #[unstable(feature = "proc_macro", issue = "38356")]
     pub fn def_site() -> Span {
-        ::__internal::with_sess(|(_, mark)| {
-            let call_site = mark.expn_info().unwrap().call_site;
-            Span(call_site.with_ctxt(SyntaxContext::empty().apply_mark(mark)))
-        })
+        ::__internal::with_sess(|_, data| data.def_site)
     }
 
     /// The span of the invocation of the current procedural macro.
@@ -296,7 +284,7 @@ impl Span {
     /// at the macro call site will be able to refer to them as well.
     #[unstable(feature = "proc_macro", issue = "38356")]
     pub fn call_site() -> Span {
-        ::__internal::with_sess(|(_, mark)| Span(mark.expn_info().unwrap().call_site))
+        ::__internal::with_sess(|_, data| data.call_site)
     }
 
     /// The original source file into which this span points.
@@ -1243,7 +1231,7 @@ impl TokenTree {
             }
 
             Interpolated(_) => {
-                __internal::with_sess(|(sess, _)| {
+                __internal::with_sess(|sess, _| {
                     let tts = token.interpolated_to_tokenstream(sess, span);
                     tt!(Group::new(Delimiter::None, TokenStream(tts)))
                 })
@@ -1354,20 +1342,21 @@ pub mod __internal {
     pub use quote::{LiteralKind, SpannedSymbol, Quoter, unquote};
 
     use std::cell::Cell;
+    use std::ptr;
 
     use syntax::ast;
     use syntax::ext::base::ExtCtxt;
-    use syntax::ext::hygiene::Mark;
     use syntax::ptr::P;
     use syntax::parse::{self, ParseSess};
     use syntax::parse::token::{self, Token};
     use syntax::tokenstream;
     use syntax_pos::{BytePos, Loc, DUMMY_SP};
+    use syntax_pos::hygiene::{Mark, SyntaxContext, Transparency};
 
-    use super::{TokenStream, LexError};
+    use super::{TokenStream, LexError, Span};
 
     pub fn lookup_char_pos(pos: BytePos) -> Loc {
-        with_sess(|(sess, _)| sess.codemap().lookup_char_pos(pos))
+        with_sess(|sess, _| sess.codemap().lookup_char_pos(pos))
     }
 
     pub fn new_token_stream(item: P<ast::Item>) -> TokenStream {
@@ -1380,7 +1369,7 @@ pub mod __internal {
     }
 
     pub fn token_stream_parse_items(stream: TokenStream) -> Result<Vec<P<ast::Item>>, LexError> {
-        with_sess(move |(sess, _)| {
+        with_sess(move |sess, _| {
             let mut parser = parse::stream_to_parser(sess, stream.0);
             let mut items = Vec::new();
 
@@ -1411,16 +1400,30 @@ pub mod __internal {
                                     expand: fn(TokenStream) -> TokenStream);
     }
 
+    #[derive(Clone, Copy)]
+    pub struct ProcMacroData {
+        pub def_site: Span,
+        pub call_site: Span,
+    }
+
+    #[derive(Clone, Copy)]
+    struct ProcMacroSess {
+        parse_sess: *const ParseSess,
+        data: ProcMacroData,
+    }
+
     // Emulate scoped_thread_local!() here essentially
     thread_local! {
-        static CURRENT_SESS: Cell<(*const ParseSess, Mark)> =
-            Cell::new((0 as *const _, Mark::root()));
+        static CURRENT_SESS: Cell<ProcMacroSess> = Cell::new(ProcMacroSess {
+            parse_sess: ptr::null(),
+            data: ProcMacroData { def_site: Span(DUMMY_SP), call_site: Span(DUMMY_SP) },
+        });
     }
 
     pub fn set_sess<F, R>(cx: &ExtCtxt, f: F) -> R
         where F: FnOnce() -> R
     {
-        struct Reset { prev: (*const ParseSess, Mark) }
+        struct Reset { prev: ProcMacroSess }
 
         impl Drop for Reset {
             fn drop(&mut self) {
@@ -1430,24 +1433,42 @@ pub mod __internal {
 
         CURRENT_SESS.with(|p| {
             let _reset = Reset { prev: p.get() };
-            p.set((cx.parse_sess, cx.current_expansion.mark));
+
+            // No way to determine def location for a proc macro right now, so use call location.
+            let location = cx.current_expansion.mark.expn_info().unwrap().call_site;
+            // Opaque mark was already created by expansion, now create its transparent twin.
+            // We can't use the call-site span literally here, even if it appears to provide
+            // correct name resolution, because it has all the `ExpnInfo` wrong, so the edition
+            // checks, lint macro checks, macro backtraces will all break.
+            let opaque_mark = cx.current_expansion.mark;
+            let transparent_mark = Mark::fresh_cloned(opaque_mark);
+            transparent_mark.set_transparency(Transparency::Transparent);
+
+            let to_span = |mark| Span(location.with_ctxt(SyntaxContext::empty().apply_mark(mark)));
+            p.set(ProcMacroSess {
+                parse_sess: cx.parse_sess,
+                data: ProcMacroData {
+                    def_site: to_span(opaque_mark),
+                    call_site: to_span(transparent_mark),
+                },
+            });
             f()
         })
     }
 
     pub fn in_sess() -> bool
     {
-        let p = CURRENT_SESS.with(|p| p.get());
-        !p.0.is_null()
+        !CURRENT_SESS.with(|sess| sess.get()).parse_sess.is_null()
     }
 
     pub fn with_sess<F, R>(f: F) -> R
-        where F: FnOnce((&ParseSess, Mark)) -> R
+        where F: FnOnce(&ParseSess, &ProcMacroData) -> R
     {
-        let p = CURRENT_SESS.with(|p| p.get());
-        assert!(!p.0.is_null(), "proc_macro::__internal::with_sess() called \
-                                 before set_parse_sess()!");
-        f(unsafe { (&*p.0, p.1) })
+        let sess = CURRENT_SESS.with(|sess| sess.get());
+        if sess.parse_sess.is_null() {
+            panic!("procedural macro API is used outside of a procedural macro");
+        }
+        f(unsafe { &*sess.parse_sess }, &sess.data)
     }
 }
 

Diff for: src/librustc/hir/map/definitions.rs

@@ -484,12 +484,7 @@ impl Definitions {
     #[inline]
     pub fn opt_span(&self, def_id: DefId) -> Option<Span> {
         if def_id.krate == LOCAL_CRATE {
-            let span = self.def_index_to_span.get(&def_id.index).cloned().unwrap_or(DUMMY_SP);
-            if span != DUMMY_SP {
-                Some(span)
-            } else {
-                None
-            }
+            self.def_index_to_span.get(&def_id.index).cloned()
         } else {
             None
         }
@@ -586,8 +581,8 @@ impl Definitions {
             self.opaque_expansions_that_defined.insert(index, expansion);
         }
 
-        // The span is added if it isn't DUMMY_SP
-        if span != DUMMY_SP {
+        // The span is added if it isn't dummy
+        if !span.is_dummy() {
             self.def_index_to_span.insert(index, span);
         }
 

Diff for: src/librustc/middle/stability.rs

@@ -20,7 +20,7 @@ use ty::{self, TyCtxt};
 use middle::privacy::AccessLevels;
 use session::DiagnosticMessageId;
 use syntax::symbol::Symbol;
-use syntax_pos::{Span, MultiSpan, DUMMY_SP};
+use syntax_pos::{Span, MultiSpan};
 use syntax::ast;
 use syntax::ast::{NodeId, Attribute};
 use syntax::feature_gate::{GateIssue, emit_feature_err, find_lang_feature_accepted_version};
@@ -687,7 +687,7 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> {
                 let msp: MultiSpan = span.into();
                 let cm = &self.sess.parse_sess.codemap();
                 let span_key = msp.primary_span().and_then(|sp: Span|
-                    if sp != DUMMY_SP {
+                    if !sp.is_dummy() {
                         let file = cm.lookup_char_pos(sp.lo()).file;
                         if file.name.is_macros() {
                             None
@@ -725,7 +725,7 @@ impl<'a, 'tcx> Visitor<'tcx> for Checker<'a, 'tcx> {
         match item.node {
             hir::ItemExternCrate(_) => {
                 // compiler-generated `extern crate` items have a dummy span.
-                if item.span == DUMMY_SP { return }
+                if item.span.is_dummy() { return }
 
                 let def_id = self.tcx.hir.local_def_id(item.id);
                 let cnum = match self.tcx.extern_mod_stmt_cnum(def_id) {

Diff for: src/librustc/ty/query/plumbing.rs

@@ -708,7 +708,7 @@ macro_rules! define_queries {
 
             // FIXME(eddyb) Get more valid Span's on queries.
             pub fn default_span(&self, tcx: TyCtxt<'_, $tcx, '_>, span: Span) -> Span {
-                if span != DUMMY_SP {
+                if !span.is_dummy() {
                     return span;
                 }
                 // The def_span query is used to calculate default_span,

Diff for: src/librustc_codegen_llvm/debuginfo/metadata.rs

@@ -1662,7 +1662,7 @@ pub fn create_global_var_metadata(cx: &CodegenCx,
     let var_scope = get_namespace_for_item(cx, def_id);
     let span = tcx.def_span(def_id);
 
-    let (file_metadata, line_number) = if span != syntax_pos::DUMMY_SP {
+    let (file_metadata, line_number) = if !span.is_dummy() {
         let loc = span_start(cx, span);
         (file_metadata(cx, &loc.file.name, LOCAL_CRATE), loc.line as c_uint)
     } else {

Diff for: src/librustc_codegen_llvm/debuginfo/mod.rs

@@ -219,7 +219,7 @@ pub fn create_function_debug_context<'a, 'tcx>(cx: &CodegenCx<'a, 'tcx>,
     let span = mir.span;
 
     // This can be the case for functions inlined from another crate
-    if span == syntax_pos::DUMMY_SP {
+    if span.is_dummy() {
         // FIXME(simulacrum): Probably can't happen; remove.
         return FunctionDebugContext::FunctionWithoutDebugInfo;
     }

Diff for: src/librustc_errors/emitter.rs

@@ -10,7 +10,7 @@
 
 use self::Destination::*;
 
-use syntax_pos::{DUMMY_SP, FileMap, Span, MultiSpan};
+use syntax_pos::{FileMap, Span, MultiSpan};
 
 use {Level, CodeSuggestion, DiagnosticBuilder, SubDiagnostic, CodeMapperDyn, DiagnosticId};
 use snippet::{Annotation, AnnotationType, Line, MultilineAnnotation, StyledString, Style};
@@ -216,7 +216,7 @@ impl EmitterWriter {
 
         if let Some(ref cm) = self.cm {
             for span_label in msp.span_labels() {
-                if span_label.span == DUMMY_SP {
+                if span_label.span.is_dummy() {
                     continue;
                 }
 
@@ -730,7 +730,7 @@ impl EmitterWriter {
         let mut max = 0;
         if let Some(ref cm) = self.cm {
             for primary_span in msp.primary_spans() {
-                if primary_span != &DUMMY_SP {
+                if !primary_span.is_dummy() {
                     let hi = cm.lookup_char_pos(primary_span.hi());
                     if hi.line > max {
                         max = hi.line;
@@ -739,7 +739,7 @@ impl EmitterWriter {
             }
             if !self.short_message {
                 for span_label in msp.span_labels() {
-                    if span_label.span != DUMMY_SP {
+                    if !span_label.span.is_dummy() {
                         let hi = cm.lookup_char_pos(span_label.span.hi());
                         if hi.line > max {
                             max = hi.line;
@@ -778,7 +778,7 @@ impl EmitterWriter {
 
             // First, find all the spans in <*macros> and point instead at their use site
             for sp in span.primary_spans() {
-                if *sp == DUMMY_SP {
+                if sp.is_dummy() {
                     continue;
                 }
                 let call_sp = cm.call_span_if_macro(*sp);
@@ -790,7 +790,7 @@ impl EmitterWriter {
                     // Only show macro locations that are local
                     // and display them like a span_note
                     if let Some(def_site) = trace.def_site_span {
-                        if def_site == DUMMY_SP {
+                        if def_site.is_dummy() {
                             continue;
                         }
                         if always_backtrace {
@@ -830,7 +830,7 @@ impl EmitterWriter {
                 span.push_span_label(label_span, label_text);
             }
             for sp_label in span.span_labels() {
-                if sp_label.span == DUMMY_SP {
+                if sp_label.span.is_dummy() {
                     continue;
                 }
                 if cm.span_to_filename(sp_label.span.clone()).is_macros() &&
@@ -1003,7 +1003,7 @@ impl EmitterWriter {
         // Make sure our primary file comes first
         let (primary_lo, cm) = if let (Some(cm), Some(ref primary_span)) =
             (self.cm.as_ref(), msp.primary_span().as_ref()) {
-            if primary_span != &&DUMMY_SP {
+            if !primary_span.is_dummy() {
                 (cm.lookup_char_pos(primary_span.lo()), cm)
             } else {
                 emit_to_destination(&buffer.render(), level, &mut self.dst, self.short_message)?;

Diff for: src/librustc_metadata/creader.rs

@@ -569,9 +569,11 @@ impl<'a> CrateLoader<'a> {
             fn register_bang_proc_macro(&mut self,
                                         name: &str,
                                         expand: fn(TokenStream) -> TokenStream) {
-                let expand = SyntaxExtension::ProcMacro(
-                    Box::new(BangProcMacro { inner: expand }), false, self.edition
-                );
+                let expand = SyntaxExtension::ProcMacro {
+                    expander: Box::new(BangProcMacro { inner: expand }),
+                    allow_internal_unstable: false,
+                    edition: self.edition,
+                };
                 self.extensions.push((Symbol::intern(name), Lrc::new(expand)));
             }
         }

Diff for: src/librustc_metadata/cstore_impl.rs

@@ -518,8 +518,11 @@ impl CrateStore for cstore::CStore {
             return LoadedMacro::ProcMacro(proc_macros[id.index.to_proc_macro_index()].1.clone());
         } else if data.name == "proc_macro" &&
                   self.get_crate_data(id.krate).item_name(id.index) == "quote" {
-            let ext = SyntaxExtension::ProcMacro(Box::new(::proc_macro::__internal::Quoter),
-                                                 true, data.root.edition);
+            let ext = SyntaxExtension::ProcMacro {
+                expander: Box::new(::proc_macro::__internal::Quoter),
+                allow_internal_unstable: true,
+                edition: data.root.edition,
+            };
             return LoadedMacro::ProcMacro(Lrc::new(ext));
         }