Store TokenStream in rmeta::MacroDef.

This removes a hack from `load_macro_untracked` in which parsing is used.

Author: Centril

Cargo.lock

@@ -3891,14 +3891,12 @@ dependencies = [
  "memmap",
  "rustc",
  "rustc_ast",
- "rustc_ast_pretty",
  "rustc_attr",
  "rustc_data_structures",
  "rustc_errors",
  "rustc_expand",
  "rustc_hir",
  "rustc_index",
- "rustc_parse",
  "rustc_span",
  "rustc_target",
  "serialize",

src/librustc/dep_graph/dep_node.rs

@@ -111,6 +111,7 @@ macro_rules! define_dep_nodes {
     ) => (
         #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash,
                  RustcEncodable, RustcDecodable)]
+        #[allow(non_camel_case_types)]
         pub enum DepKind {
             $($variant),*
         }
@@ -173,6 +174,7 @@ macro_rules! define_dep_nodes {
 
         pub struct DepConstructor;
 
+        #[allow(non_camel_case_types)]
         impl DepConstructor {
             $(
                 #[inline(always)]

src/librustc/lint.rs

@@ -344,15 +344,8 @@ pub fn in_external_macro(sess: &Session, span: Span) -> bool {
         ExpnKind::Root | ExpnKind::Desugaring(DesugaringKind::ForLoop) => false,
         ExpnKind::AstPass(_) | ExpnKind::Desugaring(_) => true, // well, it's "external"
         ExpnKind::Macro(MacroKind::Bang, _) => {
-            if expn_data.def_site.is_dummy() {
-                // Dummy span for the `def_site` means it's an external macro.
-                return true;
-            }
-            match sess.source_map().span_to_snippet(expn_data.def_site) {
-                Ok(code) => !code.starts_with("macro_rules"),
-                // No snippet means external macro or compiler-builtin expansion.
-                Err(_) => true,
-            }
+            // Dummy span for the `def_site` means it's an external macro.
+            expn_data.def_site.is_dummy() || sess.source_map().is_imported(expn_data.def_site)
         }
         ExpnKind::Macro(..) => true, // definitely a plugin
     }

src/librustc/middle/stability.rs

@@ -110,7 +110,7 @@ pub fn report_unstable(
     let span_key = msp.primary_span().and_then(|sp: Span| {
         if !sp.is_dummy() {
             let file = sm.lookup_char_pos(sp.lo()).file;
-            if file.name.is_macros() { None } else { Some(span) }
+            if file.is_imported() { None } else { Some(span) }
         } else {
             None
         }

src/librustc_errors/emitter.rs

@@ -414,8 +414,8 @@ pub trait Emitter {
     }
 
     // This does a small "fix" for multispans by looking to see if it can find any that
-    // point directly at <*macros>. Since these are often difficult to read, this
-    // will change the span to point at the use site.
+    // point directly at external macros. Since these are often difficult to read,
+    // this will change the span to point at the use site.
     fn fix_multispans_in_extern_macros(
         &self,
         source_map: &Option<Lrc<SourceMap>>,
@@ -427,9 +427,9 @@ pub trait Emitter {
         }
     }
 
-    // This "fixes" MultiSpans that contain Spans that are pointing to locations inside of
-    // <*macros>. Since these locations are often difficult to read, we move these Spans from
-    // <*macros> to their corresponding use site.
+    // This "fixes" MultiSpans that contain `Span`s pointing to locations inside of external macros.
+    // Since these locations are often difficult to read,
+    // we move these spans from the external macros to their corresponding use site.
     fn fix_multispan_in_extern_macros(
         &self,
         source_map: &Option<Lrc<SourceMap>>,
@@ -440,14 +440,14 @@ pub trait Emitter {
             None => return,
         };
 
-        // First, find all the spans in <*macros> and point instead at their use site
+        // First, find all the spans in external macros and point instead at their use site.
         let replacements: Vec<(Span, Span)> = span
             .primary_spans()
             .iter()
             .copied()
             .chain(span.span_labels().iter().map(|sp_label| sp_label.span))
             .filter_map(|sp| {
-                if !sp.is_dummy() && sm.span_to_filename(sp).is_macros() {
+                if !sp.is_dummy() && sm.is_imported(sp) {
                     let maybe_callsite = sp.source_callsite();
                     if sp != maybe_callsite {
                         return Some((sp, maybe_callsite));
@@ -457,7 +457,7 @@ pub trait Emitter {
             })
             .collect();
 
-        // After we have them, make sure we replace these 'bad' def sites with their use sites
+        // After we have them, make sure we replace these 'bad' def sites with their use sites.
         for (from, to) in replacements {
             span.replace(from, to);
         }

src/librustc_expand/mbe/macro_rules.rs

@@ -105,10 +105,10 @@ impl<'a> ParserAnyMacro<'a> {
             if e.span.is_dummy() {
                 // Get around lack of span in error (#30128)
                 e.replace_span_with(site_span);
-                if parser.sess.source_map().span_to_filename(arm_span).is_real() {
+                if !parser.sess.source_map().is_imported(arm_span) {
                     e.span_label(arm_span, "in this macro arm");
                 }
-            } else if !parser.sess.source_map().span_to_filename(parser.token.span).is_real() {
+            } else if parser.sess.source_map().is_imported(parser.token.span) {
                 e.span_label(site_span, "in this macro invocation");
             }
             match kind {
@@ -297,7 +297,7 @@ fn generic_extension<'cx>(
     let span = token.span.substitute_dummy(sp);
     let mut err = cx.struct_span_err(span, &parse_failure_msg(&token));
     err.span_label(span, label);
-    if !def_span.is_dummy() && cx.source_map().span_to_filename(def_span).is_real() {
+    if !def_span.is_dummy() && !cx.source_map().is_imported(def_span) {
         err.span_label(cx.source_map().def_span(def_span), "when calling this macro");
     }
 

src/librustc_macros/src/symbols.rs

@@ -103,6 +103,7 @@ pub fn symbols(input: TokenStream) -> TokenStream {
             #value,
         });
         keyword_stream.extend(quote! {
+            #[allow(non_upper_case_globals)]
             pub const #name: Symbol = Symbol::new(#counter);
         });
         counter += 1;
@@ -120,6 +121,8 @@ pub fn symbols(input: TokenStream) -> TokenStream {
             #value,
         });
         symbols_stream.extend(quote! {
+            #[allow(rustc::default_hash_types)]
+            #[allow(non_upper_case_globals)]
             pub const #name: Symbol = Symbol::new(#counter);
         });
         counter += 1;
@@ -149,6 +152,7 @@ pub fn symbols(input: TokenStream) -> TokenStream {
             () => {
                 #symbols_stream
 
+                #[allow(non_upper_case_globals)]
                 pub const digits_array: &[Symbol; 10] = &[
                     #digits_stream
                 ];

src/librustc_metadata/Cargo.toml

@@ -15,7 +15,6 @@ log = "0.4"
 memmap = "0.7"
 smallvec = { version = "1.0", features = ["union", "may_dangle"] }
 rustc = { path = "../librustc" }
-rustc_ast_pretty = { path = "../librustc_ast_pretty" }
 rustc_attr = { path = "../librustc_attr" }
 rustc_data_structures = { path = "../librustc_data_structures" }
 rustc_errors = { path = "../librustc_errors" }
@@ -26,7 +25,6 @@ rustc_serialize = { path = "../libserialize", package = "serialize" }
 stable_deref_trait = "1.0.0"
 rustc_ast = { path = "../librustc_ast" }
 rustc_expand = { path = "../librustc_expand" }
-rustc_parse = { path = "../librustc_parse" }
 rustc_span = { path = "../librustc_span" }
 
 [target.'cfg(windows)'.dependencies]

src/librustc_metadata/rmeta/decoder.rs

@@ -1333,9 +1333,9 @@ impl<'a, 'tcx> CrateMetadata {
         }
     }
 
-    fn get_macro(&self, id: DefIndex) -> MacroDef {
+    fn get_macro(&self, id: DefIndex, sess: &Session) -> MacroDef {
         match self.kind(id) {
-            EntryKind::MacroDef(macro_def) => macro_def.decode(self),
+            EntryKind::MacroDef(macro_def) => macro_def.decode((self, sess)),
             _ => bug!(),
         }
     }

src/librustc_metadata/rmeta/decoder/cstore_impl.rs

@@ -14,27 +14,22 @@ use rustc::session::{CrateDisambiguator, Session};
 use rustc::ty::query::Providers;
 use rustc::ty::query::QueryConfig;
 use rustc::ty::{self, TyCtxt};
+use rustc_ast::ast;
+use rustc_ast::attr;
+use rustc_ast::expand::allocator::AllocatorKind;
+use rustc_ast::ptr::P;
+use rustc_ast::tokenstream::DelimSpan;
 use rustc_data_structures::svh::Svh;
 use rustc_hir as hir;
 use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, CRATE_DEF_INDEX, LOCAL_CRATE};
-use rustc_parse::parser::emit_unclosed_delims;
-use rustc_parse::source_file_to_stream;
+use rustc_span::source_map::{self, Span, Spanned};
+use rustc_span::symbol::Symbol;
 
 use rustc_data_structures::sync::Lrc;
 use smallvec::SmallVec;
 use std::any::Any;
 use std::sync::Arc;
 
-use rustc_ast::ast;
-use rustc_ast::attr;
-use rustc_ast::expand::allocator::AllocatorKind;
-use rustc_ast::ptr::P;
-use rustc_ast::tokenstream::DelimSpan;
-use rustc_span::source_map;
-use rustc_span::source_map::Spanned;
-use rustc_span::symbol::Symbol;
-use rustc_span::{FileName, Span};
-
 macro_rules! provide {
     (<$lt:tt> $tcx:ident, $def_id:ident, $other:ident, $cdata:ident,
       $($name:ident => $compute:block)*) => {
@@ -419,15 +414,9 @@ impl CStore {
             return LoadedMacro::ProcMacro(data.load_proc_macro(id.index, sess));
         }
 
-        let def = data.get_macro(id.index);
-        let macro_full_name = data.def_path(id.index).to_string_friendly(|_| data.root.name);
-        let source_name = FileName::Macros(macro_full_name);
-
-        let source_file = sess.parse_sess.source_map().new_source_file(source_name, def.body);
-        let local_span = Span::with_root_ctxt(source_file.start_pos, source_file.end_pos);
-        let dspan = DelimSpan::from_single(local_span);
-        let (body, mut errors) = source_file_to_stream(&sess.parse_sess, source_file, None);
-        emit_unclosed_delims(&mut errors, &sess.parse_sess);
+        let span = data.get_span(id.index, sess);
+        let dspan = DelimSpan::from_single(span);
+        let rmeta::MacroDef { body, legacy } = data.get_macro(id.index, sess);
 
         // Mark the attrs as used
         let attrs = data.get_item_attrs(id.index, sess);
@@ -441,22 +430,20 @@ impl CStore {
             .data
             .get_opt_name()
             .expect("no name in load_macro");
-        sess.imported_macro_spans
-            .borrow_mut()
-            .insert(local_span, (name.to_string(), data.get_span(id.index, sess)));
+        sess.imported_macro_spans.borrow_mut().insert(span, (name.to_string(), span));
 
         LoadedMacro::MacroDef(
             ast::Item {
                 // FIXME: cross-crate hygiene
                 ident: ast::Ident::with_dummy_span(name),
                 id: ast::DUMMY_NODE_ID,
-                span: local_span,
+                span,
                 attrs: attrs.iter().cloned().collect(),
                 kind: ast::ItemKind::MacroDef(ast::MacroDef {
                     body: P(ast::MacArgs::Delimited(dspan, ast::MacDelimiter::Brace, body)),
-                    legacy: def.legacy,
+                    legacy,
                 }),
-                vis: source_map::respan(local_span.shrink_to_lo(), ast::VisibilityKind::Inherited),
+                vis: source_map::respan(span.shrink_to_lo(), ast::VisibilityKind::Inherited),
                 tokens: None,
             },
             data.root.edition,

src/librustc_metadata/rmeta/encoder.rs

@@ -1235,10 +1235,9 @@ impl EncodeContext<'tcx> {
 
     /// Serialize the text of exported macros
     fn encode_info_for_macro_def(&mut self, macro_def: &hir::MacroDef<'_>) {
-        use rustc_ast_pretty::pprust;
         let def_id = self.tcx.hir().local_def_id(macro_def.hir_id);
         record!(self.per_def.kind[def_id] <- EntryKind::MacroDef(self.lazy(MacroDef {
-            body: pprust::tts_to_string(macro_def.body.clone()),
+            body: macro_def.body.clone(),
             legacy: macro_def.legacy,
         })));
         record!(self.per_def.visibility[def_id] <- ty::Visibility::Public);

src/librustc_metadata/rmeta/mod.rs

@@ -11,6 +11,7 @@ use rustc::session::config::SymbolManglingVersion;
 use rustc::session::CrateDisambiguator;
 use rustc::ty::{self, ReprOptions, Ty};
 use rustc_ast::ast;
+use rustc_ast::tokenstream::TokenStream;
 use rustc_attr as attr;
 use rustc_data_structures::svh::Svh;
 use rustc_data_structures::sync::MetadataRef;
@@ -324,7 +325,7 @@ struct ModData {
 
 #[derive(RustcEncodable, RustcDecodable)]
 struct MacroDef {
-    body: String,
+    body: TokenStream,
     legacy: bool,
 }
 

src/librustc_span/lib.rs

@@ -83,8 +83,6 @@ scoped_tls::scoped_thread_local!(pub static GLOBALS: Globals);
 )]
 pub enum FileName {
     Real(PathBuf),
-    /// A macro. This includes the full name of the macro, so that there are no clashes.
-    Macros(String),
     /// Call to `quote!`.
     QuoteExpansion(u64),
     /// Command line.
@@ -107,7 +105,6 @@ impl std::fmt::Display for FileName {
         use FileName::*;
         match *self {
             Real(ref path) => write!(fmt, "{}", path.display()),
-            Macros(ref name) => write!(fmt, "<{} macros>", name),
             QuoteExpansion(_) => write!(fmt, "<quote expansion>"),
             MacroExpansion(_) => write!(fmt, "<macro expansion>"),
             Anon(_) => write!(fmt, "<anon>"),
@@ -132,8 +129,7 @@ impl FileName {
         use FileName::*;
         match *self {
             Real(_) => true,
-            Macros(_)
-            | Anon(_)
+            Anon(_)
             | MacroExpansion(_)
             | ProcMacroSourceCode(_)
             | CfgSpec(_)
@@ -144,22 +140,6 @@ impl FileName {
         }
     }
 
-    pub fn is_macros(&self) -> bool {
-        use FileName::*;
-        match *self {
-            Real(_)
-            | Anon(_)
-            | MacroExpansion(_)
-            | ProcMacroSourceCode(_)
-            | CfgSpec(_)
-            | CliCrateAttr(_)
-            | Custom(_)
-            | QuoteExpansion(_)
-            | DocTest(_, _) => false,
-            Macros(_) => true,
-        }
-    }
-
     pub fn quote_expansion_source_code(src: &str) -> FileName {
         let mut hasher = StableHasher::new();
         src.hash(&mut hasher);

src/librustc_span/source_map.rs

@@ -975,6 +975,12 @@ impl SourceMap {
             _ => None,
         })
     }
+
+    pub fn is_imported(&self, sp: Span) -> bool {
+        let source_file_index = self.lookup_source_file_idx(sp.lo());
+        let source_file = &self.files()[source_file_index];
+        source_file.is_imported()
+    }
 }
 
 #[derive(Clone)]

src/librustc_typeck/check/demand.rs

@@ -373,7 +373,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     ) -> Option<(Span, &'static str, String)> {
         let sm = self.sess().source_map();
         let sp = expr.span;
-        if !sm.span_to_filename(sp).is_real() {
+        if sm.is_imported(sp) {
             // Ignore if span is from within a macro #41858, #58298. We previously used the macro
             // call span, but that breaks down when the type error comes from multiple calls down.
             return None;
@@ -523,7 +523,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             {
                 // We have `&T`, check if what was expected was `T`. If so,
                 // we may want to suggest removing a `&`.
-                if !sm.span_to_filename(expr.span).is_real() {
+                if sm.is_imported(expr.span) {
                     if let Ok(code) = sm.span_to_snippet(sp) {
                         if code.starts_with('&') {
                             return Some((
@@ -601,7 +601,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             // FIXME(estebank): modify once we decide to suggest `as` casts
             return false;
         }
-        if !self.tcx.sess.source_map().span_to_filename(expr.span).is_real() {
+        if self.tcx.sess.source_map().is_imported(expr.span) {
             // Ignore if span is from within a macro.
             return false;
         }

src/librustdoc/html/render.rs

@@ -1565,7 +1565,7 @@ impl Context {
 
         let mut path = String::new();
 
-        // We can safely ignore macros from other libraries
+        // We can safely ignore synthetic `SourceFile`s.
         let file = match item.source.filename {
             FileName::Real(ref path) => path,
             _ => return None,