Build rustdoc on-demand.

Rustdoc is no longer compiled in every stage, alongside rustc, instead
it is only compiled when requested, and generally only for the last
stage.

Author: Mark-Simulacrum

src/Cargo.lock

@@ -16,6 +16,7 @@ members = [
   "tools/remote-test-server",
   "tools/rust-installer",
   "tools/cargo",
+  "tools/rustdoc",
   "tools/rls",
   # FIXME(https://github.com/rust-lang/cargo/issues/4089): move these to exclude
   "tools/rls/test_data/borrow_error",

src/Cargo.toml

@@ -256,7 +256,7 @@ impl<'a> Builder<'a> {
                 compile::StartupObjects, tool::BuildManifest, tool::Rustbook, tool::ErrorIndex,
                 tool::UnstableBookGen, tool::Tidy, tool::Linkchecker, tool::CargoTest,
                 tool::Compiletest, tool::RemoteTestServer, tool::RemoteTestClient,
-                tool::RustInstaller, tool::Cargo, tool::Rls),
+                tool::RustInstaller, tool::Cargo, tool::Rls, tool::Rustdoc),
             Kind::Test => describe!(check::Tidy, check::Bootstrap, check::DefaultCompiletest,
                 check::HostCompiletest, check::Crate, check::CrateLibrustc, check::Linkcheck,
                 check::Cargotest, check::Cargo, check::Rls, check::Docs, check::ErrorIndex,
@@ -412,12 +412,22 @@ impl<'a> Builder<'a> {
         }
     }
 
-    /// Get the `rustdoc` executable next to the specified compiler
     pub fn rustdoc(&self, compiler: Compiler) -> PathBuf {
-        let mut rustdoc = self.rustc(compiler);
-        rustdoc.pop();
-        rustdoc.push(exe("rustdoc", &compiler.host));
-        rustdoc
+        self.ensure(tool::Rustdoc { target_compiler: compiler })
+    }
+
+    pub fn rustdoc_cmd(&self, compiler: Compiler) -> Command {
+        let mut cmd = Command::new(&self.out.join("bootstrap/debug/rustdoc"));
+        cmd
+            .env("RUSTC_STAGE", compiler.stage.to_string())
+            .env("RUSTC_SYSROOT", if compiler.is_snapshot(&self.build) {
+                INTERNER.intern_path(self.build.rustc_snapshot_libdir())
+            } else {
+                self.sysroot(compiler)
+            })
+            .env("RUSTC_LIBDIR", self.sysroot_libdir(compiler, self.build.build))
+            .env("RUSTDOC_REAL", self.rustdoc(compiler));
+        cmd
     }
 
     /// Prepares an invocation of `cargo` to be run.
@@ -469,7 +479,11 @@ impl<'a> Builder<'a> {
              .env("RUSTC_LIBDIR", self.rustc_libdir(compiler))
              .env("RUSTC_RPATH", self.config.rust_rpath.to_string())
              .env("RUSTDOC", self.out.join("bootstrap/debug/rustdoc"))
-             .env("RUSTDOC_REAL", self.rustdoc(compiler))
+             .env("RUSTDOC_REAL", if cmd == "doc" || cmd == "test" {
+                 self.rustdoc(compiler)
+             } else {
+                 PathBuf::from("/path/to/nowhere/rustdoc/not/required")
+             })
              .env("RUSTC_FLAGS", self.rustc_flags(target).join(" "));
 
         if mode != Mode::Tool {

src/bootstrap/builder.rs

@@ -809,8 +809,7 @@ fn markdown_test(builder: &Builder, compiler: Compiler, markdown: &Path) {
     }
 
     println!("doc tests for: {}", markdown.display());
-    let mut cmd = Command::new(builder.rustdoc(compiler));
-    builder.add_rustc_lib_path(compiler, &mut cmd);
+    let mut cmd = builder.rustdoc_cmd(compiler);
     build.add_rust_test_threads(&mut cmd);
     cmd.arg("--test");
     cmd.arg(markdown);

src/bootstrap/check.rs

@@ -719,15 +719,6 @@ impl Step for Assemble {
         let _ = fs::remove_file(&compiler);
         copy(&rustc, &compiler);
 
-        // See if rustdoc exists to link it into place
-        let rustdoc = exe("rustdoc", &*host);
-        let rustdoc_src = out_dir.join(&rustdoc);
-        let rustdoc_dst = bindir.join(&rustdoc);
-        if fs::metadata(&rustdoc_src).is_ok() {
-            let _ = fs::remove_file(&rustdoc_dst);
-            copy(&rustdoc_src, &rustdoc_dst);
-        }
-
         target_compiler
     }
 }

src/bootstrap/compile.rs

@@ -413,6 +413,9 @@ impl Step for Rustc {
             t!(fs::create_dir_all(image.join("bin")));
             cp_r(&src.join("bin"), &image.join("bin"));
 
+            install(&builder.ensure(tool::Rustdoc { target_compiler: compiler }),
+                &image.join("bin"), 0o755);
+
             // Copy runtime DLLs needed by the compiler
             if libdir != "bin" {
                 for entry in t!(src.join(libdir).read_dir()).map(|e| t!(e)) {

src/bootstrap/dist.rs

@@ -21,7 +21,6 @@ use std::fs::{self, File};
 use std::io::prelude::*;
 use std::io;
 use std::path::{PathBuf, Path};
-use std::process::Command;
 
 use Mode;
 use build_helper::up_to_date;
@@ -242,12 +241,8 @@ fn invoke_rustdoc(builder: &Builder, target: Interned<String>, markdown: &str) {
     let build = builder.build;
     let out = build.doc_out(target);
 
-    let compiler = builder.compiler(0, build.build);
-
     let path = build.src.join("src/doc").join(markdown);
 
-    let rustdoc = builder.rustdoc(compiler);
-
     let favicon = build.src.join("src/doc/favicon.inc");
     let footer = build.src.join("src/doc/footer.inc");
 
@@ -263,9 +258,7 @@ fn invoke_rustdoc(builder: &Builder, target: Interned<String>, markdown: &str) {
         t!(t!(File::create(&version_info)).write_all(info.as_bytes()));
     }
 
-    let mut cmd = Command::new(&rustdoc);
-
-    builder.add_rustc_lib_path(compiler, &mut cmd);
+    let mut cmd = builder.rustdoc_cmd(builder.compiler(0, build.build));
 
     let out = out.join("book");
 
@@ -357,8 +350,7 @@ impl Step for Standalone {
                 continue
             }
 
-            let mut cmd = Command::new(&rustdoc);
-            builder.add_rustc_lib_path(compiler, &mut cmd);
+            let mut cmd = builder.rustdoc_cmd(compiler);
             cmd.arg("--html-after-content").arg(&footer)
                .arg("--html-before-content").arg(&version_info)
                .arg("--html-in-header").arg(&favicon)

src/bootstrap/doc.rs

@@ -8,14 +8,15 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
+use std::fs;
 use std::env;
 use std::path::PathBuf;
 use std::process::Command;
 
 use Mode;
 use Compiler;
 use builder::{Step, RunConfig, ShouldRun, Builder};
-use util::{exe, add_lib_path};
+use util::{copy, exe, add_lib_path};
 use compile::{self, libtest_stamp, libstd_stamp, librustc_stamp};
 use native;
 use channel::GitInfo;
@@ -223,6 +224,56 @@ impl Step for RemoteTestServer {
     }
 }
 
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct Rustdoc {
+    pub target_compiler: Compiler,
+}
+
+impl Step for Rustdoc {
+    type Output = PathBuf;
+    const DEFAULT: bool = true;
+    const ONLY_HOSTS: bool = true;
+
+    fn should_run(run: ShouldRun) -> ShouldRun {
+        run.path("src/tools/rustdoc")
+    }
+
+    fn make_run(run: RunConfig) {
+        run.builder.ensure(Rustdoc {
+            target_compiler: run.builder.compiler(run.builder.top_stage, run.host),
+        });
+    }
+
+    fn run(self, builder: &Builder) -> PathBuf {
+        let target_compiler = self.target_compiler;
+        let build_compiler = if target_compiler.stage == 0 {
+            target_compiler
+        } else {
+            builder.compiler(target_compiler.stage - 1, target_compiler.host)
+        };
+
+        let tool_rustdoc = builder.ensure(ToolBuild {
+            compiler: build_compiler,
+            target: build_compiler.host,
+            tool: "rustdoc",
+            mode: Mode::Librustc,
+        });
+
+        // don't create a stage0-sysroot/bin directory.
+        if target_compiler.stage > 0 {
+            let sysroot = builder.sysroot(target_compiler);
+            let bindir = sysroot.join("bin");
+            t!(fs::create_dir_all(&bindir));
+            let bin_rustdoc = bindir.join(exe("rustdoc", &*target_compiler.host));
+            let _ = fs::remove_file(&bin_rustdoc);
+            copy(&tool_rustdoc, &bin_rustdoc);
+            bin_rustdoc
+        } else {
+            tool_rustdoc
+        }
+    }
+}
+
 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 pub struct Cargo {
     pub compiler: Compiler,

src/bootstrap/tool.rs

@@ -7,25 +7,10 @@ build = "build.rs"
 [lib]
 name = "rustdoc"
 path = "lib.rs"
-crate-type = ["dylib"]
 
 [dependencies]
-arena = { path = "../libarena" }
 env_logger = { version = "0.4", default-features = false }
 log = "0.3"
-rustc = { path = "../librustc" }
-rustc_back = { path = "../librustc_back" }
-rustc_data_structures = { path = "../librustc_data_structures" }
-rustc_driver = { path = "../librustc_driver" }
-rustc_errors = { path = "../librustc_errors" }
-rustc_lint = { path = "../librustc_lint" }
-rustc_metadata = { path = "../librustc_metadata" }
-rustc_resolve = { path = "../librustc_resolve" }
-rustc_typeck = { path = "../librustc_typeck" }
-rustc_trans = { path = "../librustc_trans" }
-serialize = { path = "../libserialize" }
-syntax = { path = "../libsyntax" }
-syntax_pos = { path = "../libsyntax_pos" }
 pulldown-cmark = { version = "0.0.14", default-features = false }
 
 [build-dependencies]

src/librustdoc/Cargo.toml

@@ -17,6 +17,7 @@
        html_playground_url = "https://play.rust-lang.org/")]
 #![deny(warnings)]
 
+#![feature(rustc_private)]
 #![feature(box_patterns)]
 #![feature(box_syntax)]
 #![feature(libc)]

src/librustdoc/lib.rs

@@ -7,16 +7,11 @@ version = "0.0.0"
 name = "rustc"
 path = "rustc.rs"
 
-[[bin]]
-name = "rustdoc"
-path = "rustdoc.rs"
-
 # All optional dependencies so the features passed to this Cargo.toml select
 # what should actually be built.
 [dependencies]
 rustc_back = { path = "../librustc_back" }
 rustc_driver = { path = "../librustc_driver" }
-rustdoc = { path = "../librustdoc" }
 
 [features]
 jemalloc = ["rustc_back/jemalloc"]

src/rustc/Cargo.toml

@@ -3,6 +3,9 @@ authors = ["The Rust Project Developers"]
 name = "error_index_generator"
 version = "0.0.0"
 
+[dependencies]
+rustdoc = { path = "../../librustdoc" }
+
 [[bin]]
 name = "error_index_generator"
 path = "main.rs"

src/tools/error_index_generator/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "rustdoc-tool"
+version = "0.0.0"
+authors = ["The Rust Project Developers"]
+
+[[bin]]
+name = "rustdoc"
+path = "main.rs"
+
+[dependencies]
+rustdoc = { path = "../../librustdoc" }

src/tools/rustdoc/Cargo.toml

@@ -1,4 +1,4 @@
-// Copyright 2015 The Rust Project Developers. See the COPYRIGHT
+// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
 // file at the top-level directory of this distribution and at
 // http://rust-lang.org/COPYRIGHT.
 //
@@ -8,8 +8,6 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-#![feature(rustc_private)]
-
 extern crate rustdoc;
 
 fn main() { rustdoc::main() }