switch stage0.txt to stage0.json and add a tool to generate it

Cargo.lock

@@ -220,6 +220,17 @@ dependencies = [
 name = "build_helper"
 version = "0.1.0"
 
+[[package]]
+name = "bump-stage0"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "curl",
+ "serde",
+ "serde_json",
+ "toml",
+]
+
 [[package]]
 name = "byte-tools"
 version = "0.3.1"

Cargo.toml

@@ -35,6 +35,7 @@ members = [
   "src/tools/expand-yaml-anchors",
   "src/tools/jsondocck",
   "src/tools/html-checker",
+  "src/tools/bump-stage0",
 ]
 
 exclude = [

src/bootstrap/bootstrap.py

@@ -4,6 +4,7 @@ import contextlib
 import datetime
 import distutils.version
 import hashlib
+import json
 import os
 import re
 import shutil
@@ -176,15 +177,6 @@ def require(cmd, exit=True):
         sys.exit(1)
 
 
-def stage0_data(rust_root):
-    """Build a dictionary from stage0.txt"""
-    nightlies = os.path.join(rust_root, "src/stage0.txt")
-    with open(nightlies, 'r') as nightlies:
-        lines = [line.rstrip() for line in nightlies
-                 if not line.startswith("#")]
-        return dict([line.split(": ", 1) for line in lines if line])
-
-
 def format_build_time(duration):
     """Return a nicer format for build time
 
@@ -371,13 +363,21 @@ def output(filepath):
     os.rename(tmp, filepath)
 
 
+class Stage0Toolchain:
+    def __init__(self, stage0_payload):
+        self.date = stage0_payload["date"]
+        self.version = stage0_payload["version"]
+
+    def channel(self):
+        return self.version + "-" + self.date
+
+
 class RustBuild(object):
     """Provide all the methods required to build Rust"""
     def __init__(self):
-        self.date = ''
+        self.stage0_compiler = None
+        self.stage0_rustfmt = None
         self._download_url = ''
-        self.rustc_channel = ''
-        self.rustfmt_channel = ''
         self.build = ''
         self.build_dir = ''
         self.clean = False
@@ -401,11 +401,10 @@ class RustBuild(object):
         will move all the content to the right place.
         """
         if rustc_channel is None:
-            rustc_channel = self.rustc_channel
+            rustc_channel = self.stage0_compiler.version
-        rustfmt_channel = self.rustfmt_channel
         bin_root = self.bin_root(stage0)
 
-        key = self.date
+        key = self.stage0_compiler.date
         if not stage0:
             key += str(self.rustc_commit)
         if self.rustc(stage0).startswith(bin_root) and \
@@ -444,19 +443,23 @@ class RustBuild(object):
 
         if self.rustfmt() and self.rustfmt().startswith(bin_root) and (
             not os.path.exists(self.rustfmt())
-            or self.program_out_of_date(self.rustfmt_stamp(), self.rustfmt_channel)
+            or self.program_out_of_date(
+                self.rustfmt_stamp(),
+                "" if self.stage0_rustfmt is None else self.stage0_rustfmt.channel()
+            )
         ):
-            if rustfmt_channel:
+            if self.stage0_rustfmt is not None:
                 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
-                [channel, date] = rustfmt_channel.split('-', 1)
+                filename = "rustfmt-{}-{}{}".format(
-                filename = "rustfmt-{}-{}{}".format(channel, self.build, tarball_suffix)
+                    self.stage0_rustfmt.version, self.build, tarball_suffix,
+                )
                 self._download_component_helper(
-                    filename, "rustfmt-preview", tarball_suffix, key=date
+                    filename, "rustfmt-preview", tarball_suffix, key=self.stage0_rustfmt.date
                 )
                 self.fix_bin_or_dylib("{}/bin/rustfmt".format(bin_root))
                 self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(bin_root))
                 with output(self.rustfmt_stamp()) as rustfmt_stamp:
-                    rustfmt_stamp.write(self.rustfmt_channel)
+                    rustfmt_stamp.write(self.stage0_rustfmt.channel())
 
         # Avoid downloading LLVM twice (once for stage0 and once for the master rustc)
         if self.downloading_llvm() and stage0:
@@ -517,7 +520,7 @@ class RustBuild(object):
     ):
         if key is None:
             if stage0:
-                key = self.date
+                key = self.stage0_compiler.date
             else:
                 key = self.rustc_commit
         cache_dst = os.path.join(self.build_dir, "cache")
@@ -815,7 +818,7 @@ class RustBuild(object):
 
     def rustfmt(self):
         """Return config path for rustfmt"""
-        if not self.rustfmt_channel:
+        if self.stage0_rustfmt is None:
             return None
         return self.program_config('rustfmt')
 
@@ -1039,19 +1042,12 @@ class RustBuild(object):
             self.update_submodule(module[0], module[1], recorded_submodules)
         print("Submodules updated in %.2f seconds" % (time() - start_time))
 
-    def set_normal_environment(self):
+    def set_dist_environment(self, url):
         """Set download URL for normal environment"""
         if 'RUSTUP_DIST_SERVER' in os.environ:
             self._download_url = os.environ['RUSTUP_DIST_SERVER']
         else:
-            self._download_url = 'https://static.rust-lang.org'
+            self._download_url = url
-
-    def set_dev_environment(self):
-        """Set download URL for development environment"""
-        if 'RUSTUP_DEV_DIST_SERVER' in os.environ:
-            self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER']
-        else:
-            self._download_url = 'https://dev-static.rust-lang.org'
 
     def check_vendored_status(self):
         """Check that vendoring is configured properly"""
@@ -1160,17 +1156,13 @@ def bootstrap(help_triggered):
     build_dir = build.get_toml('build-dir', 'build') or 'build'
     build.build_dir = os.path.abspath(build_dir.replace("$ROOT", build.rust_root))
 
-    data = stage0_data(build.rust_root)
+    with open(os.path.join(build.rust_root, "src", "stage0.json")) as f:
-    build.date = data['date']
+        data = json.load(f)
-    build.rustc_channel = data['rustc']
+    build.stage0_compiler = Stage0Toolchain(data["compiler"])
+    if data.get("rustfmt") is not None:
+        build.stage0_rustfmt = Stage0Toolchain(data["rustfmt"])
 
-    if "rustfmt" in data:
+    build.set_dist_environment(data["dist_server"])
-        build.rustfmt_channel = data['rustfmt']
-
-    if 'dev' in data:
-        build.set_dev_environment()
-    else:
-        build.set_normal_environment()
 
     build.build = args.build or build.build_triple()
     build.update_submodules()

src/bootstrap/bootstrap_test.py

@@ -13,25 +13,6 @@ from shutil import rmtree
 import bootstrap
 
 
-class Stage0DataTestCase(unittest.TestCase):
-    """Test Case for stage0_data"""
-    def setUp(self):
-        self.rust_root = tempfile.mkdtemp()
-        os.mkdir(os.path.join(self.rust_root, "src"))
-        with open(os.path.join(self.rust_root, "src",
-                               "stage0.txt"), "w") as stage0:
-            stage0.write("#ignore\n\ndate: 2017-06-15\nrustc: beta\ncargo: beta\nrustfmt: beta")
-
-    def tearDown(self):
-        rmtree(self.rust_root)
-
-    def test_stage0_data(self):
-        """Extract data from stage0.txt"""
-        expected = {"date": "2017-06-15", "rustc": "beta", "cargo": "beta", "rustfmt": "beta"}
-        data = bootstrap.stage0_data(self.rust_root)
-        self.assertDictEqual(data, expected)
-
-
 class VerifyTestCase(unittest.TestCase):
     """Test Case for verify"""
     def setUp(self):
@@ -99,7 +80,6 @@ if __name__ == '__main__':
     TEST_LOADER = unittest.TestLoader()
     SUITE.addTest(doctest.DocTestSuite(bootstrap))
     SUITE.addTests([
-        TEST_LOADER.loadTestsFromTestCase(Stage0DataTestCase),
         TEST_LOADER.loadTestsFromTestCase(VerifyTestCase),
         TEST_LOADER.loadTestsFromTestCase(ProgramOutOfDate)])
 

src/bootstrap/builder.rs

@@ -523,7 +523,7 @@ impl<'a> Builder<'a> {
                 install::Src,
                 install::Rustc
             ),
-            Kind::Run => describe!(run::ExpandYamlAnchors, run::BuildManifest),
+            Kind::Run => describe!(run::ExpandYamlAnchors, run::BuildManifest, run::BumpStage0),
         }
     }
 

src/bootstrap/lib.rs

@@ -31,7 +31,7 @@
 //! When you execute `x.py build`, the steps executed are:
 //!
 //! * First, the python script is run. This will automatically download the
-//!   stage0 rustc and cargo according to `src/stage0.txt`, or use the cached
+//!   stage0 rustc and cargo according to `src/stage0.json`, or use the cached
 //!   versions if they're available. These are then used to compile rustbuild
 //!   itself (using Cargo). Finally, control is then transferred to rustbuild.
 //!

src/bootstrap/run.rs

@@ -82,3 +82,24 @@ impl Step for BuildManifest {
         builder.run(&mut cmd);
     }
 }
+
+#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct BumpStage0;
+
+impl Step for BumpStage0 {
+    type Output = ();
+    const ONLY_HOSTS: bool = true;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.path("src/tools/bump-stage0")
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(BumpStage0);
+    }
+
+    fn run(self, builder: &Builder<'_>) -> Self::Output {
+        let mut cmd = builder.tool_cmd(Tool::BumpStage0);
+        builder.run(&mut cmd);
+    }
+}

src/bootstrap/sanity.rs

@@ -15,7 +15,7 @@ use std::fs;
 use std::path::PathBuf;
 use std::process::Command;
 
-use build_helper::{output, t};
+use build_helper::output;
 
 use crate::cache::INTERNER;
 use crate::config::Target;
@@ -227,14 +227,4 @@ $ pacman -R cmake && pacman -S mingw-w64-x86_64-cmake
     if let Some(ref s) = build.config.ccache {
         cmd_finder.must_have(s);
     }
-
-    if build.config.channel == "stable" {
-        let stage0 = t!(fs::read_to_string(build.src.join("src/stage0.txt")));
-        if stage0.contains("\ndev:") {
-            panic!(
-                "bootstrapping from a dev compiler in a stable release, but \
-                    should only be bootstrapping from a released compiler!"
-            );
-        }
-    }
 }

src/bootstrap/tool.rs

@@ -377,6 +377,7 @@ bootstrap_tool!(
     LintDocs, "src/tools/lint-docs", "lint-docs";
     JsonDocCk, "src/tools/jsondocck", "jsondocck";
     HtmlChecker, "src/tools/html-checker", "html-checker";
+    BumpStage0, "src/tools/bump-stage0", "bump-stage0";
 );
 
 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, Ord, PartialOrd)]

src/stage0.json

@@ -0,0 +1,12 @@
+{
+  "__comment": "Generated by `./x.py run src/tools/bump-stage0`. Run that command again to update the bootstrap compiler.",
+  "dist_server": "https://static.rust-lang.org",
+  "compiler": {
+    "date": "2021-08-22",
+    "version": "beta"
+  },
+  "rustfmt": {
+    "date": "2021-08-26",
+    "version": "nightly"
+  }
+}

src/stage0.txt

@@ -1,42 +0,0 @@
-# This file describes the stage0 compiler that's used to then bootstrap the Rust
-# compiler itself.
-#
-# Currently Rust always bootstraps from the previous stable release, and in our
-# train model this means that the master branch bootstraps from beta, beta
-# bootstraps from current stable, and stable bootstraps from the previous stable
-# release.
-#
-# If you're looking at this file on the master branch, you'll likely see that
-# rustc is configured to `beta`, whereas if you're looking at a source tarball
-# for a stable release you'll likely see `1.x.0` for rustc, with the previous
-# stable release's version number. `date` is the date where the release we're
-# bootstrapping off was released.
-
-date: 2021-07-29
-rustc: beta
-
-# We use a nightly rustfmt to format the source because it solves some
-# bootstrapping issues with use of new syntax in this repo. If you're looking at
-# the beta/stable branch, this key should be omitted, as we don't want to depend
-# on rustfmt from nightly there.
-rustfmt: nightly-2021-03-25
-
-# When making a stable release the process currently looks like:
-#
-#   1. Produce stable build, upload it to dev-static
-#   2. Produce a beta build from the previous stable build, upload to static
-#   3. Produce a nightly build from previous beta, upload to static
-#   4. Upload stable build to static, publish full release
-#
-# This means that there's a small window of time (a few days) where artifacts
-# are downloaded from dev-static.rust-lang.org instead of static.rust-lang.org.
-# In order to ease this transition we have an extra key which is in the
-# configuration file below. When uncommented this will instruct the bootstrap.py
-# script to download from dev-static.rust-lang.org.
-#
-# This key is typically commented out at all times. If you're looking at a
-# stable release tarball it should *definitely* be commented out. If you're
-# looking at a beta source tarball and it's uncommented we'll shortly comment it
-# out.
-
-#dev: 1

src/tools/bump-stage0/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "bump-stage0"
+version = "0.1.0"
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+anyhow = "1.0.34"
+curl = "0.4.38"
+serde = { version = "1.0.125", features = ["derive"] }
+serde_json = "1.0.59"
+toml = "0.5.7"

src/tools/bump-stage0/src/main.rs

@@ -0,0 +1,167 @@
+use anyhow::Error;
+use curl::easy::Easy;
+use std::collections::HashMap;
+use std::convert::TryInto;
+
+const DIST_SERVER: &str = "https://static.rust-lang.org";
+
+struct Tool {
+    channel: Channel,
+    version: [u16; 3],
+}
+
+impl Tool {
+    fn new() -> Result<Self, Error> {
+        let channel = match std::fs::read_to_string("src/ci/channel")?.trim() {
+            "stable" => Channel::Stable,
+            "beta" => Channel::Beta,
+            "nightly" => Channel::Nightly,
+            other => anyhow::bail!("unsupported channel: {}", other),
+        };
+
+        // Split "1.42.0" into [1, 42, 0]
+        let version = std::fs::read_to_string("src/version")?
+            .trim()
+            .split('.')
+            .map(|val| val.parse())
+            .collect::<Result<Vec<_>, _>>()?
+            .try_into()
+            .map_err(|_| anyhow::anyhow!("failed to parse version"))?;
+
+        Ok(Self { channel, version })
+    }
+
+    fn update_json(self) -> Result<(), Error> {
+        std::fs::write(
+            "src/stage0.json",
+            format!(
+                "{}\n",
+                serde_json::to_string_pretty(&Stage0 {
+                    comment: "Generated by `./x.py run src/tools/bump-stage0`. \
+                              Run that command again to update the bootstrap compiler.",
+                    dist_server: DIST_SERVER.into(),
+                    compiler: self.detect_compiler()?,
+                    rustfmt: self.detect_rustfmt()?,
+                })?
+            )
+        )?;
+        Ok(())
+    }
+
+    // Currently Rust always bootstraps from the previous stable release, and in our train model
+    // this means that the master branch bootstraps from beta, beta bootstraps from current stable,
+    // and stable bootstraps from the previous stable release.
+    //
+    // On the master branch the compiler version is configured to `beta` whereas if you're looking
+    // at the beta or stable channel you'll likely see `1.x.0` as the version, with the previous
+    // release's version number.
+    fn detect_compiler(&self) -> Result<Stage0Toolchain, Error> {
+        let channel = match self.channel {
+            Channel::Stable | Channel::Beta => {
+                // The 1.XX manifest points to the latest point release of that minor release.
+                format!("{}.{}", self.version[0], self.version[1] - 1)
+            }
+            Channel::Nightly => "beta".to_string(),
+        };
+
+        let manifest = fetch_manifest(&channel)?;
+        Ok(Stage0Toolchain {
+            date: manifest.date,
+            version: if self.channel == Channel::Nightly {
+                "beta".to_string()
+            } else {
+                // The version field is like "1.42.0 (abcdef1234 1970-01-01)"
+                manifest.pkg["rust"]
+                    .version
+                    .split_once(' ')
+                    .expect("invalid version field")
+                    .0
+                    .to_string()
+            },
+        })
+    }
+
+    /// We use a nightly rustfmt to format the source because it solves some bootstrapping issues
+    /// with use of new syntax in this repo. For the beta/stable channels rustfmt is not provided,
+    /// as we don't want to depend on rustfmt from nightly there.
+    fn detect_rustfmt(&self) -> Result<Option<Stage0Toolchain>, Error> {
+        if self.channel != Channel::Nightly {
+            return Ok(None);
+        }
+
+        let manifest = fetch_manifest("nightly")?;
+        Ok(Some(Stage0Toolchain { date: manifest.date, version: "nightly".into() }))
+    }
+}
+
+fn main() -> Result<(), Error> {
+    let tool = Tool::new()?;
+    tool.update_json()?;
+    Ok(())
+}
+
+fn fetch_manifest(channel: &str) -> Result<Manifest, Error> {
+    Ok(toml::from_slice(&http_get(&format!(
+        "{}/dist/channel-rust-{}.toml",
+        DIST_SERVER, channel
+    ))?)?)
+}
+
+fn http_get(url: &str) -> Result<Vec<u8>, Error> {
+    let mut data = Vec::new();
+    let mut handle = Easy::new();
+    handle.fail_on_error(true)?;
+    handle.url(url)?;
+    {
+        let mut transfer = handle.transfer();
+        transfer.write_function(|new_data| {
+            data.extend_from_slice(new_data);
+            Ok(new_data.len())
+        })?;
+        transfer.perform()?;
+    }
+    Ok(data)
+}
+
+#[derive(Debug, PartialEq, Eq)]
+enum Channel {
+    Stable,
+    Beta,
+    Nightly,
+}
+
+#[derive(Debug, serde::Serialize)]
+struct Stage0 {
+    #[serde(rename = "__comment")]
+    comment: &'static str,
+    dist_server: String,
+    compiler: Stage0Toolchain,
+    rustfmt: Option<Stage0Toolchain>,
+}
+
+#[derive(Debug, serde::Serialize)]
+struct Stage0Toolchain {
+    date: String,
+    version: String,
+}
+
+#[derive(Debug, serde::Deserialize)]
+struct Manifest {
+    date: String,
+    pkg: HashMap<String, ManifestPackage>,
+}
+
+#[derive(Debug, serde::Deserialize)]
+struct ManifestPackage {
+    version: String,
+    target: HashMap<String, ManifestTargetPackage>,
+}
+
+#[derive(Debug, serde::Deserialize)]
+struct ManifestTargetPackage {
+    available: bool,
+    url: Option<String>,
+    hash: Option<String>,
+    xz_url: Option<String>,
+    xz_hash: Option<String>,
+}