Rollup merge of #68678 - Mark-Simulacrum:robots, r=pietroalbini

Install robots.txt into rust-docs tarballs

Fixes #68677.

I believe this might just work out from the central-station perspective, but even if it doesn't, this is a prerequisite step anyway.

src/bootstrap/dist.rs

@@ -105,6 +105,7 @@ impl Step for Docs {
         t!(fs::create_dir_all(&dst));
         let src = builder.doc_out(host);
         builder.cp_r(&src, &dst);
+        builder.install(&builder.src.join("src/doc/robots.txt"), &dst, 0o644);
 
         let mut cmd = rust_installer(builder);
         cmd.arg("generate")

src/doc/robots.txt

@@ -1,4 +1,3 @@
-# NB: This file is not automatically deployed. After changes, it needs to be uploaded manually to doc.rust-lang.org
 User-agent: *
 Disallow: /0.3/
 Disallow: /0.4/