Auto merge of #15367 - Veykril:eager-macro-inputs, r=Veykril

fix: Strip unused token ids from eager macro input token maps

Author: bors

crates/hir-def/src/lib.rs

@@ -1160,7 +1160,7 @@ fn macro_call_as_call_id_(
             let macro_call = InFile::new(call.ast_id.file_id, call.ast_id.to_node(db));
             expand_eager_macro_input(db, krate, macro_call, def, &|path| {
                 resolver(path).filter(MacroDefId::is_fn_like)
-            })?
+            })
         }
         _ if def.is_fn_like() => ExpandResult {
             value: Some(def.as_lazy_macro(

crates/hir-def/src/macro_expansion_tests/mbe.rs

@@ -99,6 +99,30 @@ fn#19 main#20(#21)#21 {#22
     );
 }
 
+#[test]
+fn eager_expands_with_unresolved_within() {
+    check(
+        r#"
+#[rustc_builtin_macro]
+#[macro_export]
+macro_rules! format_args {}
+
+fn main(foo: ()) {
+    format_args!("{} {} {}", format_args!("{}", 0), foo, identity!(10), "bar")
+}
+"#,
+        expect![[r##"
+#[rustc_builtin_macro]
+#[macro_export]
+macro_rules! format_args {}
+
+fn main(foo: ()) {
+    /* error: unresolved macro identity */::core::fmt::Arguments::new_v1(&["", " ", " ", ], &[::core::fmt::ArgumentV1::new(&(::core::fmt::Arguments::new_v1(&["", ], &[::core::fmt::ArgumentV1::new(&(0), ::core::fmt::Display::fmt), ])), ::core::fmt::Display::fmt), ::core::fmt::ArgumentV1::new(&(foo), ::core::fmt::Display::fmt), ::core::fmt::ArgumentV1::new(&(identity!(10)), ::core::fmt::Display::fmt), ])
+}
+"##]],
+    );
+}
+
 #[test]
 fn token_mapping_eager() {
     check(

crates/hir-expand/src/eager.rs

@@ -19,7 +19,7 @@
 //!
 //! See the full discussion : <https://rust-lang.zulipchat.com/#narrow/stream/131828-t-compiler/topic/Eager.20expansion.20of.20built-in.20macros>
 use base_db::CrateId;
-use rustc_hash::FxHashMap;
+use rustc_hash::{FxHashMap, FxHashSet};
 use syntax::{ted, Parse, SyntaxNode, TextRange, TextSize, WalkEvent};
 use triomphe::Arc;
 
@@ -29,7 +29,7 @@ use crate::{
     hygiene::Hygiene,
     mod_path::ModPath,
     EagerCallInfo, ExpandError, ExpandResult, ExpandTo, InFile, MacroCallId, MacroCallKind,
-    MacroCallLoc, MacroDefId, MacroDefKind, UnresolvedMacro,
+    MacroCallLoc, MacroDefId, MacroDefKind,
 };
 
 pub fn expand_eager_macro_input(
@@ -38,7 +38,7 @@ pub fn expand_eager_macro_input(
     macro_call: InFile<ast::MacroCall>,
     def: MacroDefId,
     resolver: &dyn Fn(ModPath) -> Option<MacroDefId>,
-) -> Result<ExpandResult<Option<MacroCallId>>, UnresolvedMacro> {
+) -> ExpandResult<Option<MacroCallId>> {
     let ast_map = db.ast_id_map(macro_call.file_id);
     // the expansion which the ast id map is built upon has no whitespace, so the offsets are wrong as macro_call is from the token tree that has whitespace!
     let call_id = InFile::new(macro_call.file_id, ast_map.ast_id(&macro_call.value));
@@ -71,22 +71,23 @@ pub fn expand_eager_macro_input(
             InFile::new(arg_id.as_file(), arg_exp.syntax_node()),
             krate,
             resolver,
-        )?
+        )
     };
     let err = parse_err.or(err);
 
     let Some((expanded_eager_input, mapping)) = expanded_eager_input else {
-        return Ok(ExpandResult { value: None, err });
+        return ExpandResult { value: None, err };
     };
 
     let (mut subtree, expanded_eager_input_token_map) =
         mbe::syntax_node_to_token_tree(&expanded_eager_input);
 
     let og_tmap = if let Some(tt) = macro_call.value.token_tree() {
-        let og_tmap = mbe::syntax_node_to_token_map(tt.syntax());
+        let mut ids_used = FxHashSet::default();
+        let mut og_tmap = mbe::syntax_node_to_token_map(tt.syntax());
         // The tokenmap and ids of subtree point into the expanded syntax node, but that is inaccessible from the outside
         // so we need to remap them to the original input of the eager macro.
-        subtree.visit_ids(&|id| {
+        subtree.visit_ids(&mut |id| {
             // Note: we discard all token ids of braces and the like here, but that's not too bad and only a temporary fix
 
             if let Some(range) = expanded_eager_input_token_map
@@ -97,13 +98,15 @@ pub fn expand_eager_macro_input(
                     // remap from eager input expansion to original eager input
                     if let Some(&og_range) = ws_mapping.get(og_range) {
                         if let Some(og_token) = og_tmap.token_by_range(og_range) {
+                            ids_used.insert(og_token);
                             return og_token;
                         }
                     }
                 }
             }
             tt::TokenId::UNSPECIFIED
         });
+        og_tmap.filter(|id| ids_used.contains(&id));
         og_tmap
     } else {
         Default::default()
@@ -121,7 +124,7 @@ pub fn expand_eager_macro_input(
         kind: MacroCallKind::FnLike { ast_id: call_id, expand_to },
     };
 
-    Ok(ExpandResult { value: Some(db.intern_macro_call(loc)), err })
+    ExpandResult { value: Some(db.intern_macro_call(loc)), err }
 }
 
 fn lazy_expand(
@@ -147,13 +150,13 @@ fn eager_macro_recur(
     curr: InFile<SyntaxNode>,
     krate: CrateId,
     macro_resolver: &dyn Fn(ModPath) -> Option<MacroDefId>,
-) -> Result<ExpandResult<Option<(SyntaxNode, FxHashMap<TextRange, TextRange>)>>, UnresolvedMacro> {
+) -> ExpandResult<Option<(SyntaxNode, FxHashMap<TextRange, TextRange>)>> {
     let original = curr.value.clone_for_update();
     let mut mapping = FxHashMap::default();
 
     let mut replacements = Vec::new();
 
-    // Note: We only report a single error inside of eager expansions
+    // FIXME: We only report a single error inside of eager expansions
     let mut error = None;
     let mut offset = 0i32;
     let apply_offset = |it: TextSize, offset: i32| {
@@ -184,24 +187,28 @@ fn eager_macro_recur(
             }
         };
         let def = match call.path().and_then(|path| ModPath::from_src(db, path, hygiene)) {
-            Some(path) => macro_resolver(path.clone()).ok_or(UnresolvedMacro { path })?,
+            Some(path) => match macro_resolver(path.clone()) {
+                Some(def) => def,
+                None => {
+                    error =
+                        Some(ExpandError::other(format!("unresolved macro {}", path.display(db))));
+                    continue;
+                }
+            },
             None => {
                 error = Some(ExpandError::other("malformed macro invocation"));
                 continue;
             }
         };
         let ExpandResult { value, err } = match def.kind {
             MacroDefKind::BuiltInEager(..) => {
-                let ExpandResult { value, err } = match expand_eager_macro_input(
+                let ExpandResult { value, err } = expand_eager_macro_input(
                     db,
                     krate,
                     curr.with_value(call.clone()),
                     def,
                     macro_resolver,
-                ) {
-                    Ok(it) => it,
-                    Err(err) => return Err(err),
-                };
+                );
                 match value {
                     Some(call_id) => {
                         let ExpandResult { value, err: err2 } =
@@ -251,7 +258,7 @@ fn eager_macro_recur(
                     parse.as_ref().map(|it| it.syntax_node()),
                     krate,
                     macro_resolver,
-                )?;
+                );
                 let err = err.or(error);
 
                 if let Some(tt) = call.token_tree() {
@@ -281,7 +288,7 @@ fn eager_macro_recur(
         }
         // check if the whole original syntax is replaced
         if call.syntax() == &original {
-            return Ok(ExpandResult { value: value.zip(Some(mapping)), err: error });
+            return ExpandResult { value: value.zip(Some(mapping)), err: error };
         }
 
         if let Some(insert) = value {
@@ -292,5 +299,5 @@ fn eager_macro_recur(
     }
 
     replacements.into_iter().rev().for_each(|(old, new)| ted::replace(old.syntax(), new));
-    Ok(ExpandResult { value: Some((original, mapping)), err: error })
+    ExpandResult { value: Some((original, mapping)), err: error }
 }

crates/ide/src/syntax_highlighting/test_data/highlight_macros.html

@@ -90,8 +90,18 @@
     <span class="brace">}</span>
 <span class="brace">}</span>
 
+<span class="attribute_bracket attribute">#</span><span class="attribute_bracket attribute">[</span><span class="builtin_attr attribute library">rustc_builtin_macro</span><span class="attribute_bracket attribute">]</span>
+<span class="keyword">macro_rules</span><span class="macro_bang">!</span> <span class="macro declaration">concat</span> <span class="brace">{</span><span class="brace">}</span>
+<span class="attribute_bracket attribute">#</span><span class="attribute_bracket attribute">[</span><span class="builtin_attr attribute library">rustc_builtin_macro</span><span class="attribute_bracket attribute">]</span>
+<span class="keyword">macro_rules</span><span class="macro_bang">!</span> <span class="macro declaration">include</span> <span class="brace">{</span><span class="brace">}</span>
+<span class="attribute_bracket attribute">#</span><span class="attribute_bracket attribute">[</span><span class="builtin_attr attribute library">rustc_builtin_macro</span><span class="attribute_bracket attribute">]</span>
+<span class="keyword">macro_rules</span><span class="macro_bang">!</span> <span class="macro declaration">format_args</span> <span class="brace">{</span><span class="brace">}</span>
+
+<span class="macro">include</span><span class="macro_bang">!</span><span class="parenthesis macro">(</span><span class="none macro">concat</span><span class="punctuation macro">!</span><span class="parenthesis macro">(</span><span class="string_literal macro">"foo/"</span><span class="comma macro">,</span> <span class="string_literal macro">"foo.rs"</span><span class="parenthesis macro">)</span><span class="parenthesis macro">)</span><span class="semicolon">;</span>
+
 <span class="keyword">fn</span> <span class="function declaration">main</span><span class="parenthesis">(</span><span class="parenthesis">)</span> <span class="brace">{</span>
-    <span class="unresolved_reference">println</span><span class="macro_bang">!</span><span class="parenthesis macro">(</span><span class="string_literal macro">"Hello, {}!"</span><span class="comma macro">,</span> <span class="numeric_literal macro">92</span><span class="parenthesis macro">)</span><span class="semicolon">;</span>
+    <span class="macro">format_args</span><span class="macro_bang">!</span><span class="parenthesis macro">(</span><span class="string_literal macro">"Hello, </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal macro">!"</span><span class="comma macro">,</span> <span class="numeric_literal macro">92</span><span class="parenthesis macro">)</span><span class="semicolon">;</span>
     <span class="macro">dont_color_me_braces</span><span class="macro_bang">!</span><span class="parenthesis macro">(</span><span class="parenthesis macro">)</span><span class="semicolon">;</span>
     <span class="macro">noop</span><span class="macro_bang">!</span><span class="parenthesis macro">(</span><span class="macro macro">noop</span><span class="macro_bang macro">!</span><span class="parenthesis macro">(</span><span class="numeric_literal macro">1</span><span class="parenthesis macro">)</span><span class="parenthesis macro">)</span><span class="semicolon">;</span>
-<span class="brace">}</span></code></pre>
+<span class="brace">}</span>
+</code></pre>

crates/ide/src/syntax_highlighting/tests.rs

@@ -48,6 +48,7 @@ fn macros() {
     check_highlighting(
         r#"
 //- proc_macros: mirror
+//- /lib.rs crate:lib
 proc_macros::mirror! {
     {
         ,i32 :x pub
@@ -95,11 +96,23 @@ macro without_args {
     }
 }
 
+#[rustc_builtin_macro]
+macro_rules! concat {}
+#[rustc_builtin_macro]
+macro_rules! include {}
+#[rustc_builtin_macro]
+macro_rules! format_args {}
+
+include!(concat!("foo/", "foo.rs"));
+
 fn main() {
-    println!("Hello, {}!", 92);
+    format_args!("Hello, {}!", 92);
     dont_color_me_braces!();
     noop!(noop!(1));
 }
+//- /foo/foo.rs crate:foo
+mod foo {}
+use self::foo as bar;
 "#,
         expect_file!["./test_data/highlight_macros.html"],
         false,

crates/mbe/src/token_map.rs

@@ -117,4 +117,8 @@ impl TokenMap {
             TokenTextRange::Delimiter(_) => None,
         })
     }
+
+    pub fn filter(&mut self, id: impl Fn(tt::TokenId) -> bool) {
+        self.entries.retain(|&(tid, _)| id(tid));
+    }
 }

crates/tt/src/lib.rs

@@ -70,7 +70,7 @@ pub mod token_id {
     }
 
     impl Subtree {
-        pub fn visit_ids(&mut self, f: &impl Fn(TokenId) -> TokenId) {
+        pub fn visit_ids(&mut self, f: &mut impl FnMut(TokenId) -> TokenId) {
             self.delimiter.open = f(self.delimiter.open);
             self.delimiter.close = f(self.delimiter.close);
             self.token_trees.iter_mut().for_each(|tt| match tt {