From 972d63d1e81894da47c0c7073ae7bf52dc2017ce Mon Sep 17 00:00:00 2001 From: Noah Lev Date: Tue, 28 Jul 2026 18:02:35 +0000 Subject: [PATCH 1/3] bootstrap: Enable rustdoc mergeable CCI for std and internal docs This feature is unstable but will be stabilized soon, and this is a good way of dogfooding it to make sure it works properly. It should have no effect on the generated docs, but it provides a significant speedup. For example, I measure a 3x speedup locally (3m 11s -> 1m 1s) for `x doc src/tools` -- note that this is with the latest rustdoc perf improvements (PR 159854). --- src/bootstrap/src/core/builder/cargo.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/bootstrap/src/core/builder/cargo.rs b/src/bootstrap/src/core/builder/cargo.rs index c02fd567ac6c9..147b020a27e10 100644 --- a/src/bootstrap/src/core/builder/cargo.rs +++ b/src/bootstrap/src/core/builder/cargo.rs @@ -723,6 +723,9 @@ impl Builder<'_> { } if cmd_kind == Kind::Doc { + // Will be stabilized soon -> let's dogfood it. + // No effect on doc output but massive doc-generation time improvements. + cargo.arg("-Zrustdoc-mergeable-info"); let my_out = match mode { // This is the intended out directory for compiler documentation. Mode::Rustc | Mode::ToolRustcPrivate | Mode::ToolBootstrap | Mode::ToolTarget => { From 69610f3b4a9f8da9f46a66cc4592bf31316ae4a7 Mon Sep 17 00:00:00 2001 From: Michael Howell Date: Sat, 22 Aug 2026 12:59:48 -0700 Subject: [PATCH 2/3] bootstrap: merge compiler docs as separate step As discussed in the [old version of this PR][], we can build the original version of the docs in separate build directories, and then merge them by calling rustdoc directly. This way, the crates don't invalidate each other's build caches, and we don't have to mess with symlinks or copying things around. [old version of this PR]: https://github.com/rust-lang/rust/pull/160098#issuecomment-5379072327 --- src/bootstrap/src/core/build_steps/doc.rs | 293 ++++++++++++++++------ src/bootstrap/src/core/builder/mod.rs | 1 + src/bootstrap/src/core/builder/tests.rs | 11 +- 3 files changed, 217 insertions(+), 88 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/doc.rs b/src/bootstrap/src/core/build_steps/doc.rs index b80a0b0ba27c8..49f165a373a01 100644 --- a/src/bootstrap/src/core/build_steps/doc.rs +++ b/src/bootstrap/src/core/build_steps/doc.rs @@ -20,9 +20,9 @@ use crate::core::builder::{ crate_description, }; use crate::core::compiler::Compiler; -use crate::core::config::{Config, TargetSelection}; +use crate::core::config::TargetSelection; use crate::core::session::{FileType, Mode}; -use crate::utils::helpers::{submodule_path_of, symlink_dir, t, up_to_date}; +use crate::utils::helpers::{submodule_path_of, t, up_to_date}; macro_rules! book { ($($name:ident, $path:expr, $book_name:expr, $lang:expr ;)+) => { @@ -871,6 +871,170 @@ pub fn prepare_doc_compiler( build_compiler } +/// Generate the combined compiler docs for a given toolchain. +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +pub struct CompilerDoc { + build_compiler: Compiler, + target: TargetSelection, + stage: u32, +} + +impl CompilerDoc { + /// Document `stage` compiler for the given `target`. + pub(crate) fn for_stage(builder: &Builder<'_>, stage: u32, target: TargetSelection) -> Self { + let build_compiler = prepare_doc_compiler(builder, target, stage); + Self { build_compiler, target, stage } + } +} + +impl CommandLineStep for CompilerDoc { + type Output = (); + const IS_HOST: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("compiler-doc") + } + + fn is_default_step(builder: &Builder<'_>) -> bool { + builder.config.compiler_docs + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(CompilerDoc::for_stage(run.builder, run.builder.top_stage, run.target)); + } + + /// Generates compiler documentation. + /// + /// This will generate all documentation for compiler and dependencies. + /// Compiler documentation is distributed separately, so we make sure + /// we do not merge it with the other documentation from std, test and + /// proc_macros. This is largely just a wrapper around `cargo doc`. + fn run(self, builder: &Builder<'_>) { + let CompilerDoc { target, build_compiler, stage } = self; + + // This is the intended out directory for compiler documentation. + let out = builder.compiler_doc_out(target); + t!(fs::create_dir_all(&out)); + + let _guard = + builder.msg(Kind::Doc, format!("compiler-doc"), Mode::Rustc, build_compiler, target); + + let mut cmd = builder.rustdoc_cmd(build_compiler); + + cmd.arg("--enable-index-page").arg("-Zunstable-options").arg("-o").arg(&out); + + if !builder.config.docs_minification { + cmd.arg("--disable-minification"); + } + + #[derive(serde_derive::Deserialize)] + struct FingerprintData { + doc_parts: Vec, + } + + let rustc_stage = Rustc::for_stage(builder, stage, target); + builder.ensure(rustc_stage.clone()); + let out_dir = builder.stage_out(build_compiler, Mode::Rustc).join(target); + // Cargo puts proc macros in `target/doc` even if you pass `--target` + // explicitly (https://github.com/rust-lang/cargo/issues/7677). + let proc_macro_out_dir = builder.stage_out(build_compiler, Mode::Rustc); + // Copy crate docs into place. + for krate in &*rustc_stage.crates { + let dir_name = krate.replace('-', "_"); + let crate_doc_dir = out_dir.join("doc").join(&dir_name); + let proc_macro_doc_dir = proc_macro_out_dir.join("doc").join(&dir_name); + let doc_out = out.join(&dir_name); + t!(fs::create_dir_all(&doc_out)); + if proc_macro_doc_dir.exists() { + builder.cp_link_r(&proc_macro_doc_dir, &doc_out); + } else if crate_doc_dir.exists() { + builder.cp_link_r(&crate_doc_dir, &doc_out); + } else if !builder.config.dry_run() { + panic!("no docs found for {krate} in {}", crate_doc_dir.display()); + } + // Making sure the directory exists and is not empty. + if !builder.config.dry_run() { + assert!(doc_out.exists(), "{}", doc_out.display()); + assert!( + doc_out.read_dir().expect(&dir_name).next().is_some(), + "{}", + doc_out.display() + ); + } + } + if !builder.config.dry_run() { + let fingerprint_rustc = + t!(std::fs::read_to_string(&out_dir.join(".rustdoc_fingerprint.json"))); + let fingerprint_rustc: FingerprintData = t!(serde_json::from_str(&fingerprint_rustc)); + for part in fingerprint_rustc.doc_parts.iter() { + cmd.arg("--read-doc-meta-dir").arg(out_dir.join(part).parent().unwrap()); + } + } + + macro_rules! merge_tool_doc { + ($tool: ident, $builder: ident, $target: ident) => {{ + let tool_stage = $tool::new($builder, $target); + builder.ensure(tool_stage.clone()); + let out_dir = builder.stage_out(build_compiler, tool_stage.mode).join(target); + let proc_macro_out_dir = builder.stage_out(build_compiler, tool_stage.mode); + for krate in $tool::crates() { + let dir_name = krate.replace('-', "_"); + let crate_doc_dir = out_dir.join("doc").join(&dir_name); + let proc_macro_doc_dir = proc_macro_out_dir.join("doc").join(&dir_name); + let doc_out = out.join(&dir_name); + t!(fs::create_dir_all(&doc_out)); + if proc_macro_doc_dir.exists() { + builder.cp_link_r(&proc_macro_doc_dir, &doc_out); + } else if crate_doc_dir.exists() { + builder.cp_link_r(&crate_doc_dir, &doc_out); + } else if !builder.config.dry_run() { + panic!("no docs found for {krate} in {}", crate_doc_dir.display()); + } + // Making sure the directory exists and is not empty. + if !builder.config.dry_run() { + assert!(doc_out.exists(), "{}", doc_out.display()); + assert!( + doc_out.read_dir().expect(&dir_name).next().is_some(), + "{}", + doc_out.display() + ); + } + } + }}; + } + + merge_tool_doc!(BuildHelper, builder, target); + merge_tool_doc!(Rustdoc, builder, target); + merge_tool_doc!(Rustfmt, builder, target); + merge_tool_doc!(Clippy, builder, target); + merge_tool_doc!(Miri, builder, target); + merge_tool_doc!(Cargo, builder, target); + merge_tool_doc!(Tidy, builder, target); + merge_tool_doc!(Bootstrap, builder, target); + merge_tool_doc!(RunMakeSupport, builder, target); + merge_tool_doc!(Compiletest, builder, target); + + if !builder.config.dry_run() { + let out_dir_tool = builder.stage_out(build_compiler, Mode::ToolTarget).join(target); + let fingerprint_tool = + t!(std::fs::read_to_string(&out_dir_tool.join(".rustdoc_fingerprint.json"))); + let fingerprint_tool: FingerprintData = t!(serde_json::from_str(&fingerprint_tool)); + for part in fingerprint_tool.doc_parts.iter() { + cmd.arg("--read-doc-meta-dir").arg(out_dir_tool.join(part).parent().unwrap()); + } + } + + cmd.run(builder); + + // Handle `--open`. + builder.open_in_browser(out.join("index.html")); + } + + fn metadata(&self) -> Option { + Some(StepMetadata::doc("compiler-doc", self.target).built_by(self.build_compiler)) + } +} + /// Document the compiler for the given `target` using rustdoc from `build_compiler`. #[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct Rustc { @@ -925,10 +1089,6 @@ impl CommandLineStep for Rustc { fn run(self, builder: &Builder<'_>) { let target = self.target; - // This is the intended out directory for compiler documentation. - let out = builder.compiler_doc_out(target); - t!(fs::create_dir_all(&out)); - // Build the standard library, so that proc-macros can use it. // (Normally, only the metadata would be necessary, but proc-macros are special since they run at compile-time.) let build_compiler = self.build_compiler; @@ -977,8 +1137,6 @@ impl CommandLineStep for Rustc { cargo.rustdocflag("--extern-html-root-url"); cargo.rustdocflag("ena=https://docs.rs/ena/latest/"); - let mut to_open = None; - let out_dir = builder.stage_out(build_compiler, Mode::Rustc).join(target).join("doc"); for krate in &*self.crates { // Create all crate output directories first to make sure rustdoc uses @@ -987,41 +1145,22 @@ impl CommandLineStep for Rustc { let dir_name = krate.replace('-', "_"); t!(fs::create_dir_all(out_dir.join(&*dir_name))); cargo.arg("-p").arg(krate); - if to_open.is_none() { - to_open = Some(dir_name); - } } - // This uses a shared directory so that librustdoc documentation gets - // correctly built and merged with the rustc documentation. - // - // This is needed because rustdoc is built in a different directory from - // rustc. rustdoc needs to be able to see everything, for example when - // merging the search index, or generating local (relative) links. - symlink_dir_force(&builder.config, &out, &out_dir); - // Cargo puts proc macros in `target/doc` even if you pass `--target` - // explicitly (https://github.com/rust-lang/cargo/issues/7677). - let proc_macro_out_dir = builder.stage_out(build_compiler, Mode::Rustc).join("doc"); - symlink_dir_force(&builder.config, &out, &proc_macro_out_dir); - cargo.into_cmd().run(builder); - if !builder.config.dry_run() { - // Sanity check on linked compiler crates - for krate in &*self.crates { - let dir_name = krate.replace('-', "_"); - // Making sure the directory exists and is not empty. - assert!(out.join(&*dir_name).read_dir().unwrap().next().is_some()); - } - } - - if builder.paths.iter().any(|path| path.ends_with("compiler")) { - // For `x.py doc compiler --open`, open `rustc_middle` by default. - let index = out.join("rustc_middle").join("index.html"); - builder.open_in_browser(index); - } else if let Some(krate) = to_open { - // Let's open the first crate documentation page: - let index = out.join(krate).join("index.html"); + // We open rustc_middle as the default if invoked as `x.py doc --open RELEASES.md` + // with no particular explicit doc requested (e.g. library/core). + if builder.was_invoked_explicitly::(Kind::Doc) { + let index = if builder.paths.iter().any(|path| path.ends_with("compiler")) { + // For `x.py doc compiler --open`, open `rustc_middle` by default. + out_dir.join("rustc_middle").join("index.html") + } else if let Some(krate) = self.crates.first() { + // Let's open the first crate documentation page: + out_dir.join(krate).join("index.html") + } else { + out_dir + }; builder.open_in_browser(index); } } @@ -1048,40 +1187,48 @@ macro_rules! tool_doc { target: TargetSelection, } - impl CommandLineStep for $tool { - type Output = (); - const IS_HOST: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path($path) - } - - fn is_default_step(builder: &Builder<'_>) -> bool { - builder.config.compiler_docs - } - - fn make_run(run: RunConfig<'_>) { - let target = run.target; + impl $tool { + fn new(builder: &Builder<'_>, target: TargetSelection) -> $tool { + let target = target; let build_compiler = match $mode { Mode::ToolRustcPrivate => { // Rustdoc needs the rustc sysroot available to build. - let compilers = RustcPrivateCompilers::new(run.builder, run.builder.top_stage, target); + let compilers = RustcPrivateCompilers::new(builder, builder.top_stage, target); // Build rustc docs so that we generate relative links. - run.builder.ensure(Rustc::from_build_compiler(run.builder, compilers.build_compiler(), target)); + builder.ensure(Rustc::from_build_compiler(builder, compilers.build_compiler(), target)); compilers.build_compiler() } Mode::ToolTarget => { // when shipping multiple docs together in one folder, // they all need to use the same rustdoc version - prepare_doc_compiler(run.builder, run.builder.host_target, run.builder.top_stage) + prepare_doc_compiler(builder, builder.host_target, builder.top_stage) } _ => { panic!("Unexpected tool mode for documenting: {:?}", $mode); } }; + $tool { build_compiler, mode: $mode, target } + } + fn crates() -> &'static [&'static str] { + &$($crates)?[..] + } + } + + impl CommandLineStep for $tool { + type Output = (); + const IS_HOST: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path($path) + } - run.builder.ensure($tool { build_compiler, mode: $mode, target }); + fn is_default_step(builder: &Builder<'_>) -> bool { + builder.config.compiler_docs + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure($tool::new(run.builder, run.target)); } /// Generates documentation for a tool. @@ -1142,16 +1289,12 @@ macro_rules! tool_doc { cargo.rustdocflag("--generate-link-to-definition"); let out_dir = builder.stage_out(build_compiler, mode).join(target).join("doc"); + let proc_macro_out_dir = builder.stage_out(build_compiler, mode).join("doc"); $(for krate in $crates { let dir_name = krate.replace("-", "_"); t!(fs::create_dir_all(out_dir.join(&*dir_name))); })? - // Symlink compiler docs to the output directory of rustdoc documentation. - symlink_dir_force(&builder.config, &out, &out_dir); - let proc_macro_out_dir = builder.stage_out(build_compiler, mode).join("doc"); - symlink_dir_force(&builder.config, &out, &proc_macro_out_dir); - let _guard = builder.msg(Kind::Doc, stringify!($tool).to_lowercase(), None, build_compiler, target); cargo.into_cmd().run(builder); @@ -1160,7 +1303,11 @@ macro_rules! tool_doc { $(for krate in $crates { let dir_name = krate.replace("-", "_"); // Making sure the directory exists and is not empty. - assert!(out.join(&*dir_name).read_dir().unwrap().next().is_some()); + let doc_out = out_dir.join(&*dir_name); + let proc_macro_doc_out = proc_macro_out_dir.join(&*dir_name); + let dir = if proc_macro_doc_out.exists() { proc_macro_doc_out } else { doc_out }; + assert!(dir.exists(), "{}", dir.display()); + assert!(dir.read_dir().expect(&dir_name).next().is_some()); })? } } @@ -1341,26 +1488,6 @@ impl CommandLineStep for UnstableBookGen { } } -fn symlink_dir_force(config: &Config, original: &Path, link: &Path) { - if config.dry_run() { - return; - } - if let Ok(m) = fs::symlink_metadata(link) { - if m.file_type().is_dir() { - t!(fs::remove_dir_all(link)); - } else { - // handle directory junctions on windows by falling back to - // `remove_dir`. - t!(fs::remove_file(link).or_else(|_| fs::remove_dir(link))); - } - } - - t!( - symlink_dir(config, original, link), - format!("failed to create link from {} -> {}", link.display(), original.display()) - ); -} - /// Builds the Rust compiler book. #[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct RustcBook { diff --git a/src/bootstrap/src/core/builder/mod.rs b/src/bootstrap/src/core/builder/mod.rs index 0d7f4613723dc..20082e300e256 100644 --- a/src/bootstrap/src/core/builder/mod.rs +++ b/src/bootstrap/src/core/builder/mod.rs @@ -938,6 +938,7 @@ impl<'a> Builder<'a> { doc::CargoBook, doc::Clippy, doc::ClippyBook, + doc::CompilerDoc, doc::Miri, doc::EmbeddedBook, doc::EditionGuide, diff --git a/src/bootstrap/src/core/builder/tests.rs b/src/bootstrap/src/core/builder/tests.rs index 9d0a7aa466c9e..f283fc8cc0b6d 100644 --- a/src/bootstrap/src/core/builder/tests.rs +++ b/src/bootstrap/src/core/builder/tests.rs @@ -1026,16 +1026,17 @@ mod snapshot { [doc] cargo (book) [doc] rustc 1 -> Clippy 2 [doc] clippy (book) + [doc] rustc 1 -> BuildHelper 2 [doc] rustc 1 -> Miri 2 - [doc] embedded-book (book) - [doc] edition-guide (book) - [doc] style-guide (book) [doc] rustc 1 -> Tidy 2 [doc] rustc 1 -> Bootstrap 2 - [doc] rustc 1 -> releases 2 [doc] rustc 1 -> RunMakeSupport 2 - [doc] rustc 1 -> BuildHelper 2 [doc] rustc 1 -> Compiletest 2 + [doc] rustc 1 -> compiler-doc 2 + [doc] embedded-book (book) + [doc] edition-guide (book) + [doc] style-guide (book) + [doc] rustc 1 -> releases 2 [build] rustc 0 -> RustInstaller 1 " ); From f8b95ccc14e148e47662b69a429bdcdf555331d5 Mon Sep 17 00:00:00 2001 From: Michael Howell Date: Sat, 22 Aug 2026 17:35:37 -0700 Subject: [PATCH 3/3] bootstrap: remove the now-unneeded `-Zskip-rustdoc-fingerprint` arg --- src/bootstrap/src/core/build_steps/doc.rs | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/doc.rs b/src/bootstrap/src/core/build_steps/doc.rs index 49f165a373a01..1ceec3656d592 100644 --- a/src/bootstrap/src/core/build_steps/doc.rs +++ b/src/bootstrap/src/core/build_steps/doc.rs @@ -840,7 +840,6 @@ fn doc_std( .arg("--no-deps") .arg("--target-dir") .arg(&*target_dir.to_string_lossy()) - .arg("-Zskip-rustdoc-fingerprint") .arg("-Zrustdoc-map") .rustdocflag("--extern-html-root-url") .rustdocflag("std_detect=https://docs.rs/std_detect/latest/") @@ -916,8 +915,7 @@ impl CommandLineStep for CompilerDoc { let out = builder.compiler_doc_out(target); t!(fs::create_dir_all(&out)); - let _guard = - builder.msg(Kind::Doc, format!("compiler-doc"), Mode::Rustc, build_compiler, target); + let _guard = builder.msg(Kind::Doc, "compiler-doc", Mode::Rustc, build_compiler, target); let mut cmd = builder.rustdoc_cmd(build_compiler); @@ -964,7 +962,7 @@ impl CommandLineStep for CompilerDoc { } if !builder.config.dry_run() { let fingerprint_rustc = - t!(std::fs::read_to_string(&out_dir.join(".rustdoc_fingerprint.json"))); + t!(std::fs::read_to_string(out_dir.join(".rustdoc_fingerprint.json"))); let fingerprint_rustc: FingerprintData = t!(serde_json::from_str(&fingerprint_rustc)); for part in fingerprint_rustc.doc_parts.iter() { cmd.arg("--read-doc-meta-dir").arg(out_dir.join(part).parent().unwrap()); @@ -1017,7 +1015,7 @@ impl CommandLineStep for CompilerDoc { if !builder.config.dry_run() { let out_dir_tool = builder.stage_out(build_compiler, Mode::ToolTarget).join(target); let fingerprint_tool = - t!(std::fs::read_to_string(&out_dir_tool.join(".rustdoc_fingerprint.json"))); + t!(std::fs::read_to_string(out_dir_tool.join(".rustdoc_fingerprint.json"))); let fingerprint_tool: FingerprintData = t!(serde_json::from_str(&fingerprint_tool)); for part in fingerprint_tool.doc_parts.iter() { cmd.arg("--read-doc-meta-dir").arg(out_dir_tool.join(part).parent().unwrap()); @@ -1125,7 +1123,6 @@ impl CommandLineStep for Rustc { cargo.rustdocflag("--generate-macro-expansion"); compile::rustc_cargo(builder, &mut cargo, target, &build_compiler, &self.crates); - cargo.arg("-Zskip-rustdoc-fingerprint"); // Only include compiler crates, no dependencies of those, such as `libc`. // Do link to dependencies on `docs.rs` however using `rustdoc-map`. @@ -1189,7 +1186,6 @@ macro_rules! tool_doc { impl $tool { fn new(builder: &Builder<'_>, target: TargetSelection) -> $tool { - let target = target; let build_compiler = match $mode { Mode::ToolRustcPrivate => { // Rustdoc needs the rustc sysroot available to build. @@ -1269,7 +1265,6 @@ macro_rules! tool_doc { cargo.allow_features(allow_features); } - cargo.arg("-Zskip-rustdoc-fingerprint"); // Only include compiler crates, no dependencies of those, such as `libc`. cargo.arg("--no-deps");