Auto merge of #81746 - bjorn3:cg_clif_rustup_component, r=Mark-Simulacrum

Distribute cg_clif as rustup component on the nightly channel

This makes it possible to use cg_clif using:

```bash
$ rustup component add rustc-codegen-cranelift-preview --toolchain nightly
$ RUSTFLAGS="-Zcodegen-backend=cranelift" cargo +nightly build
```

cc https://github.com/rust-lang/compiler-team/issues/405.
r? `@Mark-Simulacrum`
This commit is contained in:
bors 2023-10-28 15:16:27 +00:00
commit 6a66ca215b
13 changed files with 208 additions and 69 deletions

View File

@ -193,8 +193,9 @@ jobs:
os: ubuntu-20.04-8core-32gb
env: {}
- name: dist-aarch64-linux
env:
CODEGEN_BACKENDS: "llvm,cranelift"
os: ubuntu-20.04-8core-32gb
env: {}
- name: dist-android
os: ubuntu-20.04-8core-32gb
env: {}
@ -244,15 +245,18 @@ jobs:
os: ubuntu-20.04-8core-32gb
env: {}
- name: dist-x86_64-linux
env:
CODEGEN_BACKENDS: "llvm,cranelift"
os: ubuntu-20.04-16core-64gb
env: {}
- name: dist-x86_64-linux-alt
env:
IMAGE: dist-x86_64-linux
CODEGEN_BACKENDS: "llvm,cranelift"
os: ubuntu-20.04-16core-64gb
- name: dist-x86_64-musl
env:
CODEGEN_BACKENDS: "llvm,cranelift"
os: ubuntu-20.04-8core-32gb
env: {}
- name: dist-x86_64-netbsd
os: ubuntu-20.04-8core-32gb
env: {}
@ -316,6 +320,7 @@ jobs:
NO_DEBUG_ASSERTIONS: 1
NO_OVERFLOW_CHECKS: 1
DIST_REQUIRE_ALL_TOOLS: 1
CODEGEN_BACKENDS: "llvm,cranelift"
os: macos-13
- name: dist-apple-various
env:
@ -554,8 +559,9 @@ jobs:
matrix:
include:
- name: dist-x86_64-linux
env:
CODEGEN_BACKENDS: "llvm,cranelift"
os: ubuntu-20.04-16core-64gb
env: {}
timeout-minutes: 600
runs-on: "${{ matrix.os }}"
steps:

View File

@ -45,18 +45,18 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "cranelift-bforest"
version = "0.101.1"
version = "0.101.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1512c3bb6b13018e7109fc3ac964bc87b329eaf3a77825d337558d0c7f6f1be"
checksum = "f773437307980ac0f424bf9b9a5d0cd21a0f17248c6860c9a65bec8b5975f3fe"
dependencies = [
"cranelift-entity",
]
[[package]]
name = "cranelift-codegen"
version = "0.101.1"
version = "0.101.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "16cb8fb9220a6ea7a226705a273ab905309ee546267bdf34948d57932d7f0396"
checksum = "443c2ac50e97fb7de1a0f862753fce3f27215558811a6fcee508eb0c3747fa79"
dependencies = [
"bumpalo",
"cranelift-bforest",
@ -75,39 +75,39 @@ dependencies = [
[[package]]
name = "cranelift-codegen-meta"
version = "0.101.1"
version = "0.101.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab3a8d3b0d4745b183da5ea0792b13d79f5c23d6e69ac04761728e2532b56649"
checksum = "c5b174c411480c79ce0793c55042fa51bec27e486381d103a53cab3b480cb2db"
dependencies = [
"cranelift-codegen-shared",
]
[[package]]
name = "cranelift-codegen-shared"
version = "0.101.1"
version = "0.101.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "524141c8e68f2abc2043de4c2b31f6d9dd42432738c246431d0572a1422a4a84"
checksum = "73fa0151a528066a369de6debeea4d4b23a32aba68b5add8c46d3dc8091ff434"
[[package]]
name = "cranelift-control"
version = "0.101.1"
version = "0.101.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "97513b57c961c713789a03886a57b43e14ebcd204cbaa8ae50ca6c70a8e716b3"
checksum = "b8adf1e6398493c9bea1190e37d28a0eb0eca5fddbc80e01e506cda34db92b1f"
dependencies = [
"arbitrary",
]
[[package]]
name = "cranelift-entity"
version = "0.101.1"
version = "0.101.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3f23d3cf3afa7e45f239702612c76d87964f652a55e28d13ed6d7e20f3479dd"
checksum = "4917e2ed3bb5fe87d0ed88395ca6d644018d119a034faedd1f3e1f2c33cd52b2"
[[package]]
name = "cranelift-frontend"
version = "0.101.1"
version = "0.101.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "554cd4947ec9209b58bf9ae5bf83581b5ddf9128bd967208e334b504a57db54e"
checksum = "9aaadf1e7cf28886bbf046eaf7ef538997bc8a7e020e578ea4957b39da87d5a1"
dependencies = [
"cranelift-codegen",
"log",
@ -117,15 +117,15 @@ dependencies = [
[[package]]
name = "cranelift-isle"
version = "0.101.1"
version = "0.101.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c1892a439696b6413cb54083806f5fd9fc431768b8de74864b3d9e8b93b124f"
checksum = "a67fda31b9d69eaa1c49a2081939454c45857596a9d45af6744680541c628b4c"
[[package]]
name = "cranelift-jit"
version = "0.101.1"
version = "0.101.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32209252fb38acaf1662ccd0397907bbe0e92bdb13b6ddbfd2f74e437f83e685"
checksum = "d6bf32710628e7ff298739f1ed80a0bfdafc0c6a3e284c4540b23f18e8889d4b"
dependencies = [
"anyhow",
"cranelift-codegen",
@ -143,9 +143,9 @@ dependencies = [
[[package]]
name = "cranelift-module"
version = "0.101.1"
version = "0.101.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf42656f5f6df7bfafc4dd7b63a1888b0627c07b43b2cb9aa54e13843fed39eb"
checksum = "4d693e93a0fbf56b4bc93cffe6b107c2e52f070e1111950505fc8c83ac440b9d"
dependencies = [
"anyhow",
"cranelift-codegen",
@ -154,9 +154,9 @@ dependencies = [
[[package]]
name = "cranelift-native"
version = "0.101.1"
version = "0.101.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e0c2d3badd4b9690865f5bb68a71fa94de592fa2df3f3d11a5a062c60c0a107a"
checksum = "76fb52ba71be98312f35e798d9e98e45ab2586f27584231bf7c644fa9501e8af"
dependencies = [
"cranelift-codegen",
"libc",
@ -165,9 +165,9 @@ dependencies = [
[[package]]
name = "cranelift-object"
version = "0.101.1"
version = "0.101.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "88eca54bbecea3170035168357306e9c779d4a63d8bf036c9e16bd21fdaa69b5"
checksum = "2551b2e185022b89e9efa5e04c0f17f679b86ef73d9f7feabc48b608ff23120d"
dependencies = [
"anyhow",
"cranelift-codegen",
@ -362,9 +362,9 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
[[package]]
name = "target-lexicon"
version = "0.12.5"
version = "0.12.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9410d0f6853b1d94f0e519fb95df60f29d2c1eff2d921ffdf01a4c8a3b54f12d"
checksum = "14c39fd04924ca3a864207c66fc2cd7d22d7c016007f9ce846cbb9326331930a"
[[package]]
name = "version_check"
@ -374,9 +374,9 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
[[package]]
name = "wasmtime-jit-icache-coherence"
version = "14.0.1"
version = "14.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9aaf2fa8fd2d6b65abae9b92edfe69254cc5d6b166e342364036c3e347de8da9"
checksum = "0980a96b16abbdaf829858d2389697b1d6cfc6a903873fd74b7e47a6b1045584"
dependencies = [
"cfg-if",
"libc",

View File

@ -8,12 +8,12 @@ crate-type = ["dylib"]
[dependencies]
# These have to be in sync with each other
cranelift-codegen = { version = "0.101.1", features = ["unwind", "all-arch"] }
cranelift-frontend = { version = "0.101.1" }
cranelift-module = { version = "0.101.1" }
cranelift-native = { version = "0.101.1" }
cranelift-jit = { version = "0.101.1", optional = true }
cranelift-object = { version = "0.101.1" }
cranelift-codegen = { version = "0.101.2", default-features = false, features = ["std", "unwind", "all-arch"] }
cranelift-frontend = { version = "0.101.2" }
cranelift-module = { version = "0.101.2" }
cranelift-native = { version = "0.101.2" }
cranelift-jit = { version = "0.101.2", optional = true }
cranelift-object = { version = "0.101.2" }
target-lexicon = "0.12.0"
gimli = { version = "0.28", default-features = false, features = ["write"]}
object = { version = "0.32", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] }

View File

@ -27,7 +27,7 @@ use crate::core::builder::{Builder, Kind, RunConfig, ShouldRun, Step};
use crate::core::config::TargetSelection;
use crate::utils::cache::{Interned, INTERNER};
use crate::utils::channel;
use crate::utils::helpers::{exe, is_dylib, output, t, timeit};
use crate::utils::helpers::{exe, is_dylib, output, t, target_supports_cranelift_backend, timeit};
use crate::utils::tarball::{GeneratedTarball, OverlayKind, Tarball};
use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS};
@ -443,19 +443,6 @@ impl Step for Rustc {
}
}
// Copy over the codegen backends
let backends_src = builder.sysroot_codegen_backends(compiler);
let backends_rel = backends_src
.strip_prefix(&src)
.unwrap()
.strip_prefix(builder.sysroot_libdir_relative(compiler))
.unwrap();
// Don't use custom libdir here because ^lib/ will be resolved again with installer
let backends_dst = image.join("lib").join(&backends_rel);
t!(fs::create_dir_all(&backends_dst));
builder.cp_r(&backends_src, &backends_dst);
// Copy libLLVM.so to the lib dir as well, if needed. While not
// technically needed by rustc itself it's needed by lots of other
// components like the llvm tools and LLD. LLD is included below and
@ -1282,6 +1269,91 @@ impl Step for Miri {
}
}
#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
pub struct CodegenBackend {
pub compiler: Compiler,
pub backend: Interned<String>,
}
impl Step for CodegenBackend {
type Output = Option<GeneratedTarball>;
const DEFAULT: bool = true;
const ONLY_HOSTS: bool = true;
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
run.path("compiler/rustc_codegen_cranelift")
}
fn make_run(run: RunConfig<'_>) {
for &backend in &run.builder.config.rust_codegen_backends {
if backend == "llvm" {
continue; // Already built as part of rustc
}
run.builder.ensure(CodegenBackend {
compiler: run.builder.compiler(run.builder.top_stage, run.target),
backend,
});
}
}
fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
// This prevents rustc_codegen_cranelift from being built for "dist"
// or "install" on the stable/beta channels. It is not yet stable and
// should not be included.
if !builder.build.unstable_features() {
return None;
}
if self.backend == "cranelift" {
if !target_supports_cranelift_backend(self.compiler.host) {
builder.info("target not supported by rustc_codegen_cranelift. skipping");
return None;
}
if self.compiler.host.contains("windows") {
builder.info(
"dist currently disabled for windows by rustc_codegen_cranelift. skipping",
);
return None;
}
}
let compiler = self.compiler;
let backend = self.backend;
let mut tarball =
Tarball::new(builder, &format!("rustc-codegen-{}", backend), &compiler.host.triple);
if backend == "cranelift" {
tarball.set_overlay(OverlayKind::RustcCodegenCranelift);
} else {
panic!("Unknown backend rustc_codegen_{}", backend);
}
tarball.is_preview(true);
tarball.add_legal_and_readme_to(format!("share/doc/rustc_codegen_{}", backend));
let src = builder.sysroot(compiler);
let backends_src = builder.sysroot_codegen_backends(compiler);
let backends_rel = backends_src
.strip_prefix(&src)
.unwrap()
.strip_prefix(builder.sysroot_libdir_relative(compiler))
.unwrap();
// Don't use custom libdir here because ^lib/ will be resolved again with installer
let backends_dst = PathBuf::from("lib").join(&backends_rel);
let backend_name = format!("rustc_codegen_{}", backend);
for backend in fs::read_dir(&backends_src).unwrap() {
let file_name = backend.unwrap().file_name();
if file_name.to_str().unwrap().contains(&backend_name) {
tarball.add_file(backends_src.join(file_name), &backends_dst, 0o644);
}
}
Some(tarball.generate())
}
}
#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
pub struct Rustfmt {
pub compiler: Compiler,
@ -1452,6 +1524,10 @@ impl Step for Extended {
add_component!("clippy" => Clippy { compiler, target });
add_component!("miri" => Miri { compiler, target });
add_component!("analysis" => Analysis { compiler, target });
add_component!("rustc-codegen-cranelift" => CodegenBackend {
compiler: builder.compiler(stage, target),
backend: INTERNER.intern_str("cranelift"),
});
let etc = builder.src.join("src/etc/installer");
@ -1548,6 +1624,9 @@ impl Step for Extended {
prepare(tool);
}
}
if builder.config.rust_codegen_backends.contains(&INTERNER.intern_str("cranelift")) {
prepare("rustc-codegen-cranelift");
}
// create an 'uninstall' package
builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
pkgbuild("uninstall");
@ -1587,6 +1666,10 @@ impl Step for Extended {
"rust-demangler-preview".to_string()
} else if name == "miri" {
"miri-preview".to_string()
} else if name == "rustc-codegen-cranelift" {
// FIXME add installer support for cg_clif once it is ready to be distributed on
// windows.
unreachable!("cg_clif shouldn't be built for windows");
} else {
name.to_string()
};

View File

@ -13,6 +13,7 @@ use crate::core::builder::{Builder, RunConfig, ShouldRun, Step};
use crate::core::config::{Config, TargetSelection};
use crate::utils::helpers::t;
use crate::utils::tarball::GeneratedTarball;
use crate::INTERNER;
use crate::{Compiler, Kind};
#[cfg(target_os = "illumos")]
@ -281,6 +282,19 @@ install!((self, builder, _config),
});
install_sh(builder, "rustc", self.compiler.stage, Some(self.target), &tarball);
};
RustcCodegenCranelift, alias = "rustc-codegen-cranelift", Self::should_build(_config), only_hosts: true, {
if let Some(tarball) = builder.ensure(dist::CodegenBackend {
compiler: self.compiler,
backend: INTERNER.intern_str("cranelift"),
}) {
install_sh(builder, "rustc-codegen-cranelift", self.compiler.stage, Some(self.target), &tarball);
} else {
builder.info(
&format!("skipping Install CodegenBackend(\"cranelift\") stage{} ({})",
self.compiler.stage, self.target),
);
}
};
);
#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]

View File

@ -29,7 +29,8 @@ use crate::utils;
use crate::utils::cache::{Interned, INTERNER};
use crate::utils::exec::BootstrapCommand;
use crate::utils::helpers::{
self, add_link_lib_path, dylib_path, dylib_path_var, output, t, up_to_date,
self, add_link_lib_path, dylib_path, dylib_path_var, output, t,
target_supports_cranelift_backend, up_to_date,
};
use crate::utils::render_tests::{add_flags_and_try_run_tests, try_run_tests};
use crate::{envify, CLang, DocTests, GitRepo, Mode};
@ -3000,18 +3001,7 @@ impl Step for CodegenCranelift {
return;
}
let triple = run.target.triple;
let target_supported = if triple.contains("linux") {
triple.contains("x86_64")
|| triple.contains("aarch64")
|| triple.contains("s390x")
|| triple.contains("riscv64gc")
} else if triple.contains("darwin") || triple.contains("windows") {
triple.contains("x86_64")
} else {
false
};
if !target_supported {
if !target_supports_cranelift_backend(run.target) {
builder.info("target not supported by rustc_codegen_cranelift. skipping");
return;
}

View File

@ -815,6 +815,7 @@ impl<'a> Builder<'a> {
dist::JsonDocs,
dist::Mingw,
dist::Rustc,
dist::CodegenBackend,
dist::Std,
dist::RustcDev,
dist::Analysis,

View File

@ -183,6 +183,19 @@ pub fn use_host_linker(target: TargetSelection) -> bool {
|| target.contains("switch"))
}
pub fn target_supports_cranelift_backend(target: TargetSelection) -> bool {
if target.contains("linux") {
target.contains("x86_64")
|| target.contains("aarch64")
|| target.contains("s390x")
|| target.contains("riscv64gc")
} else if target.contains("darwin") || target.contains("windows") {
target.contains("x86_64")
} else {
false
}
}
pub fn is_valid_test_suite_arg<'a, P: AsRef<Path>>(
path: &'a Path,
suite_path: P,

View File

@ -19,6 +19,7 @@ pub(crate) enum OverlayKind {
RustDemangler,
RLS,
RustAnalyzer,
RustcCodegenCranelift,
}
impl OverlayKind {
@ -58,6 +59,11 @@ impl OverlayKind {
"src/tools/rust-analyzer/LICENSE-APACHE",
"src/tools/rust-analyzer/LICENSE-MIT",
],
OverlayKind::RustcCodegenCranelift => &[
"compiler/rustc_codegen_cranelift/Readme.md",
"compiler/rustc_codegen_cranelift/LICENSE-APACHE",
"compiler/rustc_codegen_cranelift/LICENSE-MIT",
],
}
}
@ -80,6 +86,7 @@ impl OverlayKind {
OverlayKind::RustAnalyzer => builder
.rust_analyzer_info
.version(builder, &builder.release_num("rust-analyzer/crates/rust-analyzer")),
OverlayKind::RustcCodegenCranelift => builder.rust_version(),
}
}
}

View File

@ -359,6 +359,8 @@ jobs:
<<: *job-linux-8c
- name: dist-aarch64-linux
env:
CODEGEN_BACKENDS: llvm,cranelift
<<: *job-linux-8c
- name: dist-android
@ -411,14 +413,19 @@ jobs:
- &dist-x86_64-linux
name: dist-x86_64-linux
env:
CODEGEN_BACKENDS: llvm,cranelift
<<: *job-linux-16c
- name: dist-x86_64-linux-alt
env:
IMAGE: dist-x86_64-linux
CODEGEN_BACKENDS: llvm,cranelift
<<: *job-linux-16c
- name: dist-x86_64-musl
env:
CODEGEN_BACKENDS: llvm,cranelift
<<: *job-linux-8c
- name: dist-x86_64-netbsd
@ -501,6 +508,7 @@ jobs:
NO_DEBUG_ASSERTIONS: 1
NO_OVERFLOW_CHECKS: 1
DIST_REQUIRE_ALL_TOOLS: 1
CODEGEN_BACKENDS: llvm,cranelift
<<: *job-macos-xl
- name: dist-apple-various
@ -697,6 +705,8 @@ jobs:
include:
- &dist-x86_64-linux
name: dist-x86_64-linux
env:
CODEGEN_BACKENDS: llvm,cranelift
<<: *job-linux-16c
master:

View File

@ -104,6 +104,8 @@ if [ "$DEPLOY$DEPLOY_ALT" = "1" ]; then
RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-llvm-assertions"
RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.verify-llvm-ir"
fi
RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-backends=${CODEGEN_BACKENDS:-llvm}"
else
# We almost always want debug assertions enabled, but sometimes this takes too
# long for too little benefit, so we just turn them off.
@ -124,7 +126,7 @@ else
RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.verify-llvm-ir"
# Test the Cranelift backend in on CI, but don't ship it.
# Test the Cranelift backend in CI. Bootstrap knows which targets to run tests on.
RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-backends=llvm,cranelift"
# We enable this for non-dist builders, since those aren't trying to produce

View File

@ -192,7 +192,8 @@ static PKG_INSTALLERS: &[&str] = &["x86_64-apple-darwin", "aarch64-apple-darwin"
static MINGW: &[&str] = &["i686-pc-windows-gnu", "x86_64-pc-windows-gnu"];
static NIGHTLY_ONLY_COMPONENTS: &[PkgType] = &[PkgType::Miri, PkgType::JsonDocs];
static NIGHTLY_ONLY_COMPONENTS: &[PkgType] =
&[PkgType::Miri, PkgType::JsonDocs, PkgType::RustcCodegenCranelift];
macro_rules! t {
($e:expr) => {
@ -336,7 +337,15 @@ impl Builder {
// NOTE: this profile is effectively deprecated; do not add new components to it.
let mut complete = default;
complete.extend([Rls, RustAnalyzer, RustSrc, LlvmTools, RustAnalysis, Miri]);
complete.extend([
Rls,
RustAnalyzer,
RustSrc,
LlvmTools,
RustAnalysis,
Miri,
RustcCodegenCranelift,
]);
profile("complete", &complete);
// The compiler libraries are not stable for end users, and they're also huge, so we only
@ -423,7 +432,8 @@ impl Builder {
| PkgType::Rustfmt
| PkgType::LlvmTools
| PkgType::RustAnalysis
| PkgType::JsonDocs => {
| PkgType::JsonDocs
| PkgType::RustcCodegenCranelift => {
extensions.push(host_component(pkg));
}
PkgType::RustcDev | PkgType::RustcDocs => {

View File

@ -57,6 +57,7 @@ pkg_type! {
LlvmTools = "llvm-tools"; preview = true,
Miri = "miri"; preview = true,
JsonDocs = "rust-docs-json"; preview = true,
RustcCodegenCranelift = "rustc-codegen-cranelift"; preview = true,
}
impl PkgType {
@ -80,6 +81,7 @@ impl PkgType {
PkgType::Rustfmt => false,
PkgType::LlvmTools => false,
PkgType::Miri => false,
PkgType::RustcCodegenCranelift => false,
PkgType::Rust => true,
PkgType::RustStd => true,
@ -106,6 +108,7 @@ impl PkgType {
ReproducibleArtifacts => HOSTS,
RustcDocs => HOSTS,
Cargo => HOSTS,
RustcCodegenCranelift => HOSTS,
RustMingw => MINGW,
RustStd => TARGETS,
HtmlDocs => HOSTS,