use ui_test dependency builder for test dependencies (#14883)

changelog: none

This tries to make progress on
rust-lang/rust#78717 by using the ui_test
dependency handling instead of linking in the dependencies of clippy
itself with the tests. This partially reverts
#11045. However, we still
use the old style of dealing with dependencies for clippy's own crates
and the "internal" tests, as otherwise those would get rebuilt which
takes too long.

Author: samueltardieu

Cargo.toml

@@ -34,7 +34,7 @@ anstream = "0.6.18"
 
 [dev-dependencies]
 cargo_metadata = "0.18.1"
-ui_test = "0.30.1"
+ui_test = "0.30.2"
 regex = "1.5.5"
 serde = { version = "1.0.145", features = ["derive"] }
 serde_json = "1.0.122"
@@ -45,14 +45,6 @@ itertools = "0.12"
 pulldown-cmark = { version = "0.11", default-features = false, features = ["html"] }
 askama = { version = "0.14", default-features = false, features = ["alloc", "config", "derive"] }
 
-# UI test dependencies
-if_chain = "1.0"
-quote = "1.0.25"
-syn = { version = "2.0", features = ["full"] }
-futures = "0.3"
-parking_lot = "0.12"
-tokio = { version = "1", features = ["io-util"] }
-
 [build-dependencies]
 rustc_tools_util = { path = "rustc_tools_util", version = "0.4.2" }
 

clippy_test_deps/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "clippy_test_deps"
+version = "0.1.0"
+edition = "2021"
+
+# Add dependencies here to make them available in ui tests.
+
+[dependencies]
+regex = "1.5.5"
+serde = { version = "1.0.145", features = ["derive"] }
+if_chain = "1.0"
+quote = "1.0.25"
+syn = { version = "2.0", features = ["full"] }
+futures = "0.3"
+parking_lot = "0.12"
+tokio = { version = "1", features = ["io-util"] }
+itertools = "0.12"

clippy_test_deps/src/main.rs

@@ -0,0 +1 @@
+fn main() {}

tests/compile-test.rs

@@ -16,6 +16,7 @@ use test_utils::IS_RUSTC_TEST_SUITE;
 use ui_test::custom_flags::Flag;
 use ui_test::custom_flags::edition::Edition;
 use ui_test::custom_flags::rustfix::RustfixMode;
+use ui_test::dependencies::DependencyBuilder;
 use ui_test::spanned::Spanned;
 use ui_test::status_emitter::StatusEmitter;
 use ui_test::{Args, CommandBuilder, Config, Match, error_on_output_conflict};
@@ -28,46 +29,26 @@ use std::path::{Path, PathBuf};
 use std::sync::mpsc::{Sender, channel};
 use std::{fs, iter, thread};
 
-// Test dependencies may need an `extern crate` here to ensure that they show up
-// in the depinfo file (otherwise cargo thinks they are unused)
-extern crate futures;
-extern crate if_chain;
-extern crate itertools;
-extern crate parking_lot;
-extern crate quote;
-extern crate syn;
-extern crate tokio;
-
 mod test_utils;
 
-/// All crates used in UI tests are listed here
-static TEST_DEPENDENCIES: &[&str] = &[
-    "clippy_config",
-    "clippy_lints",
-    "clippy_utils",
-    "futures",
-    "if_chain",
-    "itertools",
-    "parking_lot",
-    "quote",
-    "regex",
-    "serde_derive",
-    "serde",
-    "syn",
-    "tokio",
-];
-
-/// Produces a string with an `--extern` flag for all UI test crate
-/// dependencies.
+/// All crates used in internal UI tests are listed here.
+/// We directly re-use these crates from their normal clippy builds, so we don't have them
+/// in `clippy_test_devs`. That saves a lot of time but also means they don't work in a stage 1
+/// test in rustc bootstrap.
+static INTERNAL_TEST_DEPENDENCIES: &[&str] = &["clippy_config", "clippy_lints", "clippy_utils"];
+
+/// Produces a string with an `--extern` flag for all `INTERNAL_TEST_DEPENDENCIES`.
 ///
 /// The dependency files are located by parsing the depinfo file for this test
 /// module. This assumes the `-Z binary-dep-depinfo` flag is enabled. All test
 /// dependencies must be added to Cargo.toml at the project root. Test
 /// dependencies that are not *directly* used by this test module require an
 /// `extern crate` declaration.
-fn extern_flags() -> Vec<String> {
+fn internal_extern_flags() -> Vec<String> {
+    let current_exe_path = env::current_exe().unwrap();
+    let deps_path = current_exe_path.parent().unwrap();
     let current_exe_depinfo = {
-        let mut path = env::current_exe().unwrap();
+        let mut path = current_exe_path.clone();
         path.set_extension("d");
         fs::read_to_string(path).unwrap()
     };
@@ -89,15 +70,15 @@ fn extern_flags() -> Vec<String> {
             Some((name, path_str))
         };
         if let Some((name, path)) = parse_name_path()
-            && TEST_DEPENDENCIES.contains(&name)
+            && INTERNAL_TEST_DEPENDENCIES.contains(&name)
         {
             // A dependency may be listed twice if it is available in sysroot,
             // and the sysroot dependencies are listed first. As of the writing,
             // this only seems to apply to if_chain.
             crates.insert(name, path);
         }
     }
-    let not_found: Vec<&str> = TEST_DEPENDENCIES
+    let not_found: Vec<&str> = INTERNAL_TEST_DEPENDENCIES
         .iter()
         .copied()
         .filter(|n| !crates.contains_key(n))
@@ -112,6 +93,7 @@ fn extern_flags() -> Vec<String> {
     crates
         .into_iter()
         .map(|(name, path)| format!("--extern={name}={path}"))
+        .chain([format!("-Ldependency={}", deps_path.display())])
         .collect()
 }
 
@@ -120,7 +102,6 @@ const RUN_INTERNAL_TESTS: bool = cfg!(feature = "internal");
 
 struct TestContext {
     args: Args,
-    extern_flags: Vec<String>,
     diagnostic_collector: Option<DiagnosticCollector>,
     collector_thread: Option<thread::JoinHandle<()>>,
 }
@@ -135,7 +116,6 @@ impl TestContext {
             .unzip();
         Self {
             args,
-            extern_flags: extern_flags(),
             diagnostic_collector,
             collector_thread,
         }
@@ -159,6 +139,15 @@ impl TestContext {
         };
         let defaults = config.comment_defaults.base();
         defaults.set_custom("edition", Edition("2024".into()));
+        defaults.set_custom(
+            "dependencies",
+            DependencyBuilder {
+                program: CommandBuilder::cargo(),
+                crate_manifest_path: Path::new("clippy_test_deps").join("Cargo.toml"),
+                build_std: None,
+                bless_lockfile: self.args.bless,
+            },
+        );
         defaults.exit_status = None.into();
         if mandatory_annotations {
             defaults.require_annotations = Some(Spanned::dummy(true)).into();
@@ -183,12 +172,10 @@ impl TestContext {
                 "-Zui-testing",
                 "-Zdeduplicate-diagnostics=no",
                 "-Dwarnings",
-                &format!("-Ldependency={}", deps_path.display()),
             ]
             .map(OsString::from),
         );
 
-        config.program.args.extend(self.extern_flags.iter().map(OsString::from));
         // Prevent rustc from creating `rustc-ice-*` files the console output is enough.
         config.program.envs.push(("RUSTC_ICE".into(), Some("0".into())));
 
@@ -228,6 +215,10 @@ fn run_internal_tests(cx: &TestContext) {
         return;
     }
     let mut config = cx.base_config("ui-internal", true);
+    config
+        .program
+        .args
+        .extend(internal_extern_flags().iter().map(OsString::from));
     config.bless_command = Some("cargo uitest --features internal -- -- --bless".into());
 
     ui_test::run_tests_generic(