cg: use thorin instead of llvm-dwp

`thorin` is a Rust implementation of a DWARF packaging utility that
supports reading DWARF objects from archive files (i.e. rlibs) and
therefore is better suited for integration into rustc.

Signed-off-by: David Wood <david.wood@huawei.com>
This commit is contained in:
David Wood 2021-12-03 14:24:05 +00:00
parent 08ed338f56
commit 2dc1a8a779
10 changed files with 259 additions and 90 deletions

View File

@ -9,7 +9,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3e61f2b7f93d2c7d2b08263acaa4a363b3e276806c68af6134c44f523bf1aacd"
dependencies = [
"compiler_builtins",
"gimli",
"gimli 0.25.0",
"rustc-std-workspace-alloc",
"rustc-std-workspace-core",
]
@ -87,9 +87,9 @@ dependencies = [
[[package]]
name = "anyhow"
version = "1.0.34"
version = "1.0.51"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf8dcb5b4bbaa28653b647d8c77bd4ed40183b48882e130c1f1ffb73de069fd7"
checksum = "8b26702f315f53b6071259e15dd9d64528213b44d61de1ec926eca7715d62203"
[[package]]
name = "array_tool"
@ -1158,6 +1158,12 @@ version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed"
[[package]]
name = "fallible-iterator"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
[[package]]
name = "filetime"
version = "0.2.14"
@ -1446,6 +1452,17 @@ dependencies = [
"rustc-std-workspace-core",
]
[[package]]
name = "gimli"
version = "0.26.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78cc372d058dcf6d5ecd98510e7fbc9e5aec4d21de70f65fea8fecebcd881bd4"
dependencies = [
"fallible-iterator",
"indexmap",
"stable_deref_trait",
]
[[package]]
name = "git2"
version = "0.13.23"
@ -2339,6 +2356,18 @@ dependencies = [
"rustc-std-workspace-core",
]
[[package]]
name = "object"
version = "0.27.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "67ac1d3f9a1d3616fd9a60c8d74296f22406a238b6a72f5cc1e6f314df4ffbf9"
dependencies = [
"crc32fast",
"flate2",
"indexmap",
"memchr",
]
[[package]]
name = "odht"
version = "0.3.1"
@ -3725,10 +3754,11 @@ dependencies = [
"itertools 0.9.0",
"jobserver",
"libc",
"object",
"object 0.26.2",
"pathdiff",
"regex",
"rustc_apfloat",
"rustc_arena",
"rustc_ast",
"rustc_attr",
"rustc_data_structures",
@ -3749,6 +3779,7 @@ dependencies = [
"smallvec",
"snap",
"tempfile",
"thorin-dwp",
"tracing",
]
@ -4993,7 +5024,7 @@ dependencies = [
"hermit-abi",
"libc",
"miniz_oxide",
"object",
"object 0.26.2",
"panic_abort",
"panic_unwind",
"profiler_builtins",
@ -5057,9 +5088,9 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
[[package]]
name = "structopt"
version = "0.3.16"
version = "0.3.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "de5472fb24d7e80ae84a7801b7978f95a19ec32cb1876faea59ab711eb901976"
checksum = "40b9788f4202aa75c240ecc9c15c65185e6a39ccdeb0fd5d008b98825464c87c"
dependencies = [
"clap",
"lazy_static",
@ -5068,9 +5099,9 @@ dependencies = [
[[package]]
name = "structopt-derive"
version = "0.4.9"
version = "0.4.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e0eb37335aeeebe51be42e2dc07f031163fbabfa6ac67d7ea68b5c2f68d5f99"
checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0"
dependencies = [
"heck",
"proc-macro-error",
@ -5249,24 +5280,36 @@ dependencies = [
[[package]]
name = "thiserror"
version = "1.0.20"
version = "1.0.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7dfdd070ccd8ccb78f4ad66bf1982dc37f620ef696c6b5028fe2ed83dd3d0d08"
checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
version = "1.0.20"
version = "1.0.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd80fc12f73063ac132ac92aceea36734f04a1d93c1240c6944e23a3b8841793"
checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "thorin-dwp"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "039d1fc0bfdb73910c2702893515580e38c192f47a987bc98ddd38a36f2d953a"
dependencies = [
"gimli 0.26.1",
"indexmap",
"object 0.27.1",
"tracing",
]
[[package]]
name = "thread_local"
version = "1.0.1"
@ -5394,9 +5437,9 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6"
[[package]]
name = "tracing"
version = "0.1.28"
version = "0.1.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "84f96e095c0c82419687c20ddf5cb3eadb61f4e1405923c9dc8e53a1adacbda8"
checksum = "375a639232caf30edfc78e8d89b2d4c375515393e7af7e16f01cd96917fb2105"
dependencies = [
"cfg-if 1.0.0",
"pin-project-lite",
@ -5406,9 +5449,9 @@ dependencies = [
[[package]]
name = "tracing-attributes"
version = "0.1.17"
version = "0.1.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4f915eb6abf914599c200260efced9203504c4c37380af10cdf3b7d36970650"
checksum = "f4f480b8f81512e825f337ad51e94c1eb5d3bbdf2b363dcd01e2b19a9ffe3f8e"
dependencies = [
"proc-macro2",
"quote",

View File

@ -14,12 +14,14 @@ tracing = "0.1"
libc = "0.2.50"
jobserver = "0.1.22"
tempfile = "3.2"
thorin-dwp = "0.1.1"
pathdiff = "0.2.0"
snap = "1"
smallvec = { version = "1.6.1", features = ["union", "may_dangle"] }
regex = "1.4"
rustc_serialize = { path = "../rustc_serialize" }
rustc_arena = { path = "../rustc_arena" }
rustc_ast = { path = "../rustc_ast" }
rustc_span = { path = "../rustc_span" }
rustc_middle = { path = "../rustc_middle" }

View File

@ -1,4 +1,6 @@
use rustc_arena::TypedArena;
use rustc_data_structures::fx::{FxHashSet, FxIndexMap};
use rustc_data_structures::memmap::Mmap;
use rustc_data_structures::temp_dir::MaybeTempDir;
use rustc_errors::{ErrorReported, Handler};
use rustc_fs_util::fix_windows_verbatim_for_gcc;
@ -32,7 +34,10 @@ use cc::windows_registry;
use regex::Regex;
use tempfile::Builder as TempFileBuilder;
use std::ffi::{OsStr, OsString};
use std::borrow::Borrow;
use std::ffi::OsString;
use std::fs::{File, OpenOptions};
use std::io::{BufWriter, Write};
use std::lazy::OnceCell;
use std::path::{Path, PathBuf};
use std::process::{ExitStatus, Output, Stdio};
@ -261,8 +266,14 @@ fn link_rlib<'a, B: ArchiveBuilder<'a>>(
let mut ab = <B as ArchiveBuilder>::new(sess, out_filename, None);
for obj in codegen_results.modules.iter().filter_map(|m| m.object.as_ref()) {
ab.add_file(obj);
for m in &codegen_results.modules {
if let Some(obj) = m.object.as_ref() {
ab.add_file(obj);
}
if let Some(dwarf_obj) = m.dwarf_object.as_ref() {
ab.add_file(dwarf_obj);
}
}
// Note that in this loop we are ignoring the value of `lib.cfg`. That is,
@ -518,59 +529,108 @@ fn escape_stdout_stderr_string(s: &[u8]) -> String {
})
}
const LLVM_DWP_EXECUTABLE: &'static str = "rust-llvm-dwp";
/// Invoke `llvm-dwp` (shipped alongside rustc) to link debuginfo in object files into a `dwp`
/// file.
fn link_dwarf_object<'a, I>(sess: &'a Session, executable_out_filename: &Path, object_files: I)
where
I: IntoIterator<Item: AsRef<OsStr>>,
{
info!("preparing dwp to {}.dwp", executable_out_filename.to_str().unwrap());
/// Use `thorin` (rust implementation of a dwarf packaging utility) to link DWARF objects into a
/// DWARF package.
fn link_dwarf_object<'a>(
sess: &'a Session,
cg_results: &CodegenResults,
executable_out_filename: &Path,
) {
let dwp_out_filename = executable_out_filename.with_extension("dwp");
let mut cmd = Command::new(LLVM_DWP_EXECUTABLE);
cmd.arg("-o");
cmd.arg(&dwp_out_filename);
cmd.args(object_files);
debug!(?dwp_out_filename, ?executable_out_filename);
let mut new_path = sess.get_tools_search_paths(false);
if let Some(path) = env::var_os("PATH") {
new_path.extend(env::split_paths(&path));
#[derive(Default)]
struct ThorinSession<Relocations> {
arena_data: TypedArena<Vec<u8>>,
arena_mmap: TypedArena<Mmap>,
arena_relocations: TypedArena<Relocations>,
}
let new_path = env::join_paths(new_path).unwrap();
cmd.env("PATH", new_path);
info!("{:?}", &cmd);
match sess.time("run_dwp", || cmd.output()) {
Ok(prog) if !prog.status.success() => {
sess.struct_err(&format!(
"linking dwarf objects with `{}` failed: {}",
LLVM_DWP_EXECUTABLE, prog.status
))
.note(&format!("{:?}", &cmd))
.note(&escape_stdout_stderr_string(&prog.stdout))
.note(&escape_stdout_stderr_string(&prog.stderr))
.emit();
info!("linker stderr:\n{}", escape_stdout_stderr_string(&prog.stderr));
info!("linker stdout:\n{}", escape_stdout_stderr_string(&prog.stdout));
impl<Relocations> ThorinSession<Relocations> {
fn alloc_mmap<'arena>(&'arena self, data: Mmap) -> &'arena Mmap {
(*self.arena_mmap.alloc(data)).borrow()
}
Ok(_) => {}
Err(e) => {
let dwp_not_found = e.kind() == io::ErrorKind::NotFound;
let mut err = if dwp_not_found {
sess.struct_err(&format!("linker `{}` not found", LLVM_DWP_EXECUTABLE))
} else {
sess.struct_err(&format!("could not exec the linker `{}`", LLVM_DWP_EXECUTABLE))
};
}
err.note(&e.to_string());
impl<Relocations> thorin::Session<Relocations> for ThorinSession<Relocations> {
fn alloc_data<'arena>(&'arena self, data: Vec<u8>) -> &'arena [u8] {
(*self.arena_data.alloc(data)).borrow()
}
if !dwp_not_found {
err.note(&format!("{:?}", &cmd));
fn alloc_relocation<'arena>(&'arena self, data: Relocations) -> &'arena Relocations {
(*self.arena_relocations.alloc(data)).borrow()
}
fn read_input<'arena>(&'arena self, path: &Path) -> std::io::Result<&'arena [u8]> {
let file = File::open(&path)?;
let mmap = (unsafe { Mmap::map(file) })?;
Ok(self.alloc_mmap(mmap))
}
}
match sess.time("run_thorin", || -> Result<(), thorin::Error> {
let thorin_sess = ThorinSession::default();
let mut package = thorin::DwarfPackage::new(&thorin_sess);
// Input objs contain .o/.dwo files from the current crate.
match sess.opts.debugging_opts.split_dwarf_kind {
SplitDwarfKind::Single => {
for input_obj in cg_results.modules.iter().filter_map(|m| m.object.as_ref()) {
package.add_input_object(input_obj)?;
}
}
SplitDwarfKind::Split => {
for input_obj in cg_results.modules.iter().filter_map(|m| m.dwarf_object.as_ref()) {
package.add_input_object(input_obj)?;
}
}
}
err.emit();
// Input rlibs contain .o/.dwo files from dependencies.
let input_rlibs = cg_results
.crate_info
.used_crate_source
.values()
.filter_map(|csource| csource.rlib.as_ref())
.map(|(path, _)| path);
for input_rlib in input_rlibs {
debug!(?input_rlib);
package.add_input_object(input_rlib)?;
}
// Failing to read the referenced objects is expected for dependencies where the path in the
// executable will have been cleaned by Cargo, but the referenced objects will be contained
// within rlibs provided as inputs.
//
// If paths have been remapped, then .o/.dwo files from the current crate also won't be
// found, but are provided explicitly above.
//
// Adding an executable is primarily done to make `thorin` check that all the referenced
// dwarf objects are found in the end.
package.add_executable(
&executable_out_filename,
thorin::MissingReferencedObjectBehaviour::Skip,
)?;
let output = package.finish()?.write()?;
let mut output_stream = BufWriter::new(
OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(true)
.open(dwp_out_filename)?,
);
output_stream.write_all(&output)?;
output_stream.flush()?;
Ok(())
}) {
Ok(()) => {}
Err(e) => {
sess.struct_err("linking dwarf objects with thorin failed")
.note(&format!("{:?}", e))
.emit();
}
}
}
@ -916,14 +976,11 @@ fn link_natively<'a, B: ArchiveBuilder<'a>>(
SplitDebuginfo::Packed if sess.target.is_like_msvc => {}
// ... and otherwise we're processing a `*.dwp` packed dwarf file.
//
// We cannot rely on the .o paths in the exectuable because they may have been
// remapped by --remap-path-prefix and therefore invalid. So we need to provide
// the .o paths explicitly
SplitDebuginfo::Packed => link_dwarf_object(
sess,
&out_filename,
codegen_results.modules.iter().filter_map(|m| m.object.as_ref()),
),
// remapped by --remap-path-prefix and therefore invalid, so we need to provide
// the .o/.dwo paths explicitly.
SplitDebuginfo::Packed => link_dwarf_object(sess, codegen_results, out_filename),
}
let strip = strip_value(sess);

View File

@ -57,7 +57,45 @@ packed-single:
ls $(TMPDIR)/*.dwo && exit 1 || exit 0
rm -rf $(TMPDIR)/*.dwp
unpacked: unpacked-split unpacked-single
packed-remapped: packed-remapped-split packed-remapped-single
packed-remapped-split:
$(RUSTC) -Z unstable-options -C split-debuginfo=packed -C debuginfo=2 \
-Z split-dwarf-kind=split --remap-path-prefix $(TMPDIR)=/a foo.rs -g
objdump -Wi $(TMPDIR)/foo | grep DW_AT_GNU_dwo_name | (! grep $(TMPDIR)) || exit 1
packed-remapped-single:
$(RUSTC) -Z unstable-options -C split-debuginfo=packed -C debuginfo=2 \
-Z split-dwarf-kind=single --remap-path-prefix $(TMPDIR)=/a foo.rs -g
objdump -Wi $(TMPDIR)/foo | grep DW_AT_GNU_dwo_name | (! grep $(TMPDIR)) || exit 1
packed-crosscrate: packed-crosscrate-split packed-crosscrate-single
packed-crosscrate-split:
$(RUSTC) --crate-type lib -Z unstable-options -C split-debuginfo=packed \
-Zsplit-dwarf-kind=split -C debuginfo=2 -g bar.rs
ls $(TMPDIR)/*.rlib
ls $(TMPDIR)/*.dwo && exit 1 || exit 0
ls $(TMPDIR)/*.dwp && exit 1 || exit 0
$(RUSTC) --extern bar=$(TMPDIR)/libbar.rlib -Z unstable-options -C split-debuginfo=packed \
-Zsplit-dwarf-kind=split -C debuginfo=2 -g main.rs
rm $(TMPDIR)/*.dwo
rm $(TMPDIR)/main.dwp
rm $(TMPDIR)/$(call BIN,main)
packed-crosscrate-single:
$(RUSTC) --crate-type lib -Z unstable-options -C split-debuginfo=packed \
-Zsplit-dwarf-kind=single -C debuginfo=2 -g bar.rs
ls $(TMPDIR)/*.rlib
ls $(TMPDIR)/*.dwo && exit 1 || exit 0
ls $(TMPDIR)/*.dwp && exit 1 || exit 0
$(RUSTC) --extern bar=$(TMPDIR)/libbar.rlib -Z unstable-options -C split-debuginfo=packed \
-Zsplit-dwarf-kind=single -C debuginfo=2 -g main.rs
ls $(TMPDIR)/*.dwo && exit 1 || exit 0
rm $(TMPDIR)/main.dwp
rm $(TMPDIR)/$(call BIN,main)
unpacked: unpacked-split unpacked-single unpacked-remapped-split unpacked-remapped-single
unpacked-split:
$(RUSTC) foo.rs -g -C split-debuginfo=unpacked -Z unstable-options -Zsplit-dwarf-kind=split
@ -69,5 +107,15 @@ unpacked-single:
$(RUSTC) foo.rs -g -C split-debuginfo=unpacked -Z unstable-options -Zsplit-dwarf-kind=single
ls $(TMPDIR)/*.dwp && exit 1 || exit 0
ls $(TMPDIR)/*.dwo && exit 1 || exit 0
unpacked-remapped-split:
$(RUSTC) -Z unstable-options -C split-debuginfo=unpacked -C debuginfo=2 \
-Z split-dwarf-kind=split --remap-path-prefix $(TMPDIR)=/a foo.rs -g
objdump -Wi $(TMPDIR)/foo | grep DW_AT_GNU_dwo_name | (! grep $(TMPDIR)) || exit 1
unpacked-remapped-single:
$(RUSTC) -Z unstable-options -C split-debuginfo=unpacked -C debuginfo=2 \
-Z split-dwarf-kind=single --remap-path-prefix $(TMPDIR)=/a foo.rs -g
objdump -Wi $(TMPDIR)/foo | grep DW_AT_GNU_dwo_name | (! grep $(TMPDIR)) || exit 1
endif
endif

View File

@ -0,0 +1,13 @@
pub struct Bar {
x: u32,
}
impl Bar {
pub fn print(&self) {
println!("{}", self.x);
}
}
pub fn make_bar(x: u32) -> Bar {
Bar { x }
}

View File

@ -1 +1,15 @@
pub struct Foo {
x: u32,
}
impl Foo {
pub fn print(&self) {
println!("{}", self.x);
}
}
pub fn make_foo(x: u32) -> Foo {
Foo { x }
}
fn main() {}

View File

@ -0,0 +1,8 @@
extern crate bar;
use bar::{Bar, make_bar};
fn main() {
let b = make_bar(3);
b.print();
}

View File

@ -1,17 +0,0 @@
-include ../tools.mk
# only-linux
all: packed remapped
remapped:
$(RUSTC) -Z unstable-options -C split-debuginfo=packed -C debuginfo=2 --remap-path-prefix $(TMPDIR)=/a foo.rs -g
objdump -Wi $(TMPDIR)/foo | grep DW_AT_GNU_dwo_name | (! grep $(TMPDIR)) || exit 1
$(RUSTC) -Z unstable-options -C split-debuginfo=unpacked -C debuginfo=2 --remap-path-prefix $(TMPDIR)=/a foo.rs -g
objdump -Wi $(TMPDIR)/foo | grep DW_AT_GNU_dwo_name | (! grep $(TMPDIR)) || exit 1
packed:
$(RUSTC) -Z unstable-options -C split-debuginfo=packed -C debuginfo=2 foo.rs -g
rm $(TMPDIR)/foo.dwp
rm $(TMPDIR)/$(call BIN,foo)

View File

@ -1 +0,0 @@
fn main() {}

View File

@ -109,6 +109,7 @@ const PERMITTED_DEPENDENCIES: &[&str] = &[
"env_logger",
"expect-test",
"fake-simd",
"fallible-iterator", // dependency of `thorin`
"filetime",
"fixedbitset",
"flate2",
@ -201,6 +202,7 @@ const PERMITTED_DEPENDENCIES: &[&str] = &[
"tempfile",
"termcolor",
"termize",
"thorin-dwp",
"thread_local",
"time",
"tinyvec",