Integrate mdbook-spec for the reference.

This updates the reference which is now using a new mdbook plugin. This
requires a little extra work than a normal book because the plugin uses
`rustdoc` to generate links to the standard library. It also ensures
that the submodule is available for *any* command that uses rustbook,
since it is now part of the rustbook workspace.

Author: ehuss

src/bootstrap/src/core/build_steps/doc.rs

@@ -9,7 +9,7 @@
 
 use std::io::{self, Write};
 use std::path::{Path, PathBuf};
-use std::{fs, mem};
+use std::{env, fs, mem};
 
 use crate::core::build_steps::compile;
 use crate::core::build_steps::tool::{self, prepare_tool_cargo, SourceType, Tool};
@@ -62,6 +62,7 @@ macro_rules! book {
                     src: builder.src.join($path),
                     parent: Some(self),
                     languages: $lang.into(),
+                    rustdoc: None,
                 })
             }
         }
@@ -80,7 +81,6 @@ book!(
     EditionGuide, "src/doc/edition-guide", "edition-guide", &[], submodule;
     EmbeddedBook, "src/doc/embedded-book", "embedded-book", &[], submodule;
     Nomicon, "src/doc/nomicon", "nomicon", &[], submodule;
-    Reference, "src/doc/reference", "reference", &[], submodule;
     RustByExample, "src/doc/rust-by-example", "rust-by-example", &["ja"], submodule;
     RustdocBook, "src/doc/rustdoc", "rustdoc", &[];
     StyleGuide, "src/doc/style-guide", "style-guide", &[];
@@ -112,6 +112,7 @@ impl Step for UnstableBook {
             src: builder.md_doc_out(self.target).join("unstable-book"),
             parent: Some(self),
             languages: vec![],
+            rustdoc: None,
         })
     }
 }
@@ -123,6 +124,7 @@ struct RustbookSrc<P: Step> {
     src: PathBuf,
     parent: Option<P>,
     languages: Vec<&'static str>,
+    rustdoc: Option<PathBuf>,
 }
 
 impl<P: Step> Step for RustbookSrc<P> {
@@ -153,13 +155,18 @@ impl<P: Step> Step for RustbookSrc<P> {
             builder.info(&format!("Rustbook ({target}) - {name}"));
             let _ = fs::remove_dir_all(&out);
 
-            builder
-                .tool_cmd(Tool::Rustbook)
-                .arg("build")
-                .arg(&src)
-                .arg("-d")
-                .arg(&out)
-                .run(builder);
+            let mut rustbook_cmd = builder.tool_cmd(Tool::Rustbook);
+            if let Some(mut rustdoc) = self.rustdoc {
+                rustdoc.pop();
+                let old_path = env::var_os("PATH").unwrap_or_default();
+                let new_path =
+                    env::join_paths(std::iter::once(rustdoc).chain(env::split_paths(&old_path)))
+                        .expect("could not add rustdoc to PATH");
+
+                rustbook_cmd.env("PATH", new_path);
+            }
+
+            rustbook_cmd.arg("build").arg(&src).arg("-d").arg(&out).run(builder);
 
             for lang in &self.languages {
                 let out = out.join(lang);
@@ -232,6 +239,7 @@ impl Step for TheBook {
             src: absolute_path.clone(),
             parent: Some(self),
             languages: vec![],
+            rustdoc: None,
         });
 
         // building older edition redirects
@@ -244,6 +252,7 @@ impl Step for TheBook {
                 // treat the other editions as not having a parent.
                 parent: Option::<Self>::None,
                 languages: vec![],
+                rustdoc: None,
             });
         }
 
@@ -1214,6 +1223,51 @@ impl Step for RustcBook {
             src: out_base,
             parent: Some(self),
             languages: vec![],
+            rustdoc: None,
+        });
+    }
+}
+
+#[derive(Ord, PartialOrd, Debug, Clone, Hash, PartialEq, Eq)]
+pub struct Reference {
+    pub compiler: Compiler,
+    pub target: TargetSelection,
+}
+
+impl Step for Reference {
+    type Output = ();
+    const DEFAULT: bool = true;
+    const ONLY_HOSTS: bool = true;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        let builder = run.builder;
+        run.path("src/doc/reference").default_condition(builder.config.docs)
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(Reference {
+            compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
+            target: run.target,
+        });
+    }
+
+    /// Builds the reference book.
+    fn run(self, builder: &Builder<'_>) {
+        builder.require_and_update_submodule("src/doc/reference", None);
+
+        // This is needed for generating links to the standard library using
+        // the mdbook-spec plugin.
+        builder.ensure(compile::Std::new(self.compiler, builder.config.build));
+        let rustdoc = builder.rustdoc(self.compiler);
+
+        // Run rustbook/mdbook to generate the HTML pages.
+        builder.ensure(RustbookSrc {
+            target: self.target,
+            name: "reference".to_owned(),
+            src: builder.src.join("src/doc/reference"),
+            parent: Some(self),
+            languages: vec![],
+            rustdoc: Some(rustdoc),
         });
     }
 }

src/bootstrap/src/core/build_steps/tool.rs

@@ -348,7 +348,7 @@ bootstrap_tool!(
 
 /// These are the submodules that are required for rustbook to work due to
 /// depending on mdbook plugins.
-pub static SUBMODULES_FOR_RUSTBOOK: &[&str] = &["src/doc/book"];
+pub static SUBMODULES_FOR_RUSTBOOK: &[&str] = &["src/doc/book", "src/doc/reference"];
 
 #[derive(Debug, Clone, Hash, PartialEq, Eq)]
 pub struct OptimizedDist {

src/bootstrap/src/core/build_steps/vendor.rs

@@ -1,3 +1,4 @@
+use crate::core::build_steps::tool::SUBMODULES_FOR_RUSTBOOK;
 use crate::core::builder::{Builder, RunConfig, ShouldRun, Step};
 use crate::utils::exec::command;
 use std::path::PathBuf;
@@ -35,7 +36,7 @@ impl Step for Vendor {
         }
 
         // These submodules must be present for `x vendor` to work.
-        for submodule in ["src/tools/cargo", "src/doc/book"] {
+        for submodule in SUBMODULES_FOR_RUSTBOOK.iter().chain(["src/tools/cargo"].iter()) {
             builder.build.require_and_update_submodule(submodule, None);
         }
 

src/doc/reference

@@ -661,6 +661,20 @@ dependencies = [
  "textwrap",
 ]
 
+[[package]]
+name = "mdbook-spec"
+version = "0.1.2"
+dependencies = [
+ "anyhow",
+ "mdbook",
+ "once_cell",
+ "pathdiff",
+ "regex",
+ "semver",
+ "serde_json",
+ "tempfile",
+]
+
 [[package]]
 name = "mdbook-trpl-listing"
 version = "0.1.0"
@@ -794,6 +808,12 @@ dependencies = [
  "windows-targets 0.52.6",
 ]
 
+[[package]]
+name = "pathdiff"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd"
+
 [[package]]
 name = "percent-encoding"
 version = "2.3.1"
@@ -1079,6 +1099,7 @@ dependencies = [
  "env_logger",
  "mdbook",
  "mdbook-i18n-helpers",
+ "mdbook-spec",
  "mdbook-trpl-listing",
  "mdbook-trpl-note",
 ]

src/tools/rustbook/Cargo.lock

@@ -12,6 +12,7 @@ env_logger = "0.11"
 mdbook-trpl-listing = { path = "../../doc/book/packages/mdbook-trpl-listing" }
 mdbook-trpl-note = { path = "../../doc/book/packages/mdbook-trpl-note" }
 mdbook-i18n-helpers = "0.3.3"
+mdbook-spec = { path = "../../doc/reference/mdbook-spec"}
 
 [dependencies.mdbook]
 version = "0.4.37"

src/tools/rustbook/Cargo.toml

@@ -9,6 +9,7 @@ use mdbook::errors::Result as Result3;
 use mdbook::MDBook;
 use mdbook_i18n_helpers::preprocessors::Gettext;
 
+use mdbook_spec::Spec;
 use mdbook_trpl_listing::TrplListing;
 use mdbook_trpl_note::TrplNote;
 
@@ -83,6 +84,13 @@ pub fn build(args: &ArgMatches) -> Result3<()> {
         book.config.build.build_dir = dest_dir.into();
     }
 
+    // NOTE: Replacing preprocessors using this technique causes error
+    // messages to be displayed when the original preprocessor doesn't work
+    // (but it otherwise succeeds).
+    //
+    // This should probably be fixed in mdbook to remove the existing
+    // preprocessor, or this should modify the config and use
+    // MDBook::load_with_config.
     if book.config.get_preprocessor("trpl-note").is_some() {
         book.with_preprocessor(TrplNote);
     }
@@ -91,6 +99,10 @@ pub fn build(args: &ArgMatches) -> Result3<()> {
         book.with_preprocessor(TrplListing);
     }
 
+    if book.config.get_preprocessor("spec").is_some() {
+        book.with_preprocessor(Spec::new());
+    }
+
     book.build()?;
 
     Ok(())

src/tools/rustbook/src/main.rs

@@ -72,7 +72,7 @@ pub(crate) const WORKSPACES: &[(&str, ExceptionList, Option<(&[&str], &[&str])>,
     //("src/tools/miri/test-cargo-miri", &[], None), // FIXME uncomment once all deps are vendored
     //("src/tools/miri/test_dependencies", &[], None), // FIXME uncomment once all deps are vendored
     ("src/tools/rust-analyzer", EXCEPTIONS_RUST_ANALYZER, None, &[]),
-    ("src/tools/rustbook", EXCEPTIONS_RUSTBOOK, None, &["src/doc/book"]),
+    ("src/tools/rustbook", EXCEPTIONS_RUSTBOOK, None, &["src/doc/book", "src/doc/reference"]),
     ("src/tools/rustc-perf", EXCEPTIONS_RUSTC_PERF, None, &["src/tools/rustc-perf"]),
     ("src/tools/x", &[], None, &[]),
     // tidy-alphabetical-end