dnrops
/
rust
mirror of https://github.com/rust-lang/rust.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add dist.compression-profile option to control compression speed

This commit is contained in:
Pietro Albini 2023-03-14 15:30:42 +01:00
parent bd43458d4c
commit 0a1b9834b3
No known key found for this signature in database
GPG Key ID: CD76B35F7734769E
7 changed files with 118 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
config.example.toml
View File

@ -803,3 +803,9 @@ changelog-seen = 2
#
# This list must be non-empty.
#compression-formats = ["gz", "xz"]
# How much time should be spent compressing the tarballs. The better the
# compression profile, the longer compression will take.
#
# Available options: fast, balanced, best
#compression-profile = "balanced"

4
src/bootstrap/config.rs
View File

@ -190,6 +190,7 @@ pub struct Config {
pub dist_sign_folder: Option<PathBuf>,
pub dist_upload_addr: Option<String>,
pub dist_compression_formats: Option<Vec<String>>,
pub dist_compression_profile: String,
pub dist_include_mingw_linker: bool,
// libstd features
@ -701,6 +702,7 @@ define_config! {
src_tarball: Option<bool> = "src-tarball",
missing_tools: Option<bool> = "missing-tools",
compression_formats: Option<Vec<String>> = "compression-formats",
compression_profile: Option<String> = "compression-profile",
include_mingw_linker: Option<bool> = "include-mingw-linker",
}
}
@ -819,6 +821,7 @@ impl Config {
config.deny_warnings = true;
config.bindir = "bin".into();
config.dist_include_mingw_linker = true;
config.dist_compression_profile = "balanced".into();
// set by build.rs
config.build = TargetSelection::from_user(&env!("BUILD_TRIPLE"));
@ -1300,6 +1303,7 @@ impl Config {
config.dist_sign_folder = t.sign_folder.map(PathBuf::from);
config.dist_upload_addr = t.upload_addr;
config.dist_compression_formats = t.compression_formats;
set(&mut config.dist_compression_profile, t.compression_profile);
set(&mut config.rust_dist_src, t.src_tarball);
set(&mut config.missing_tools, t.missing_tools);
set(&mut config.dist_include_mingw_linker, t.include_mingw_linker)

1
src/bootstrap/tarball.rs
View File

@ -318,6 +318,7 @@ impl<'a> Tarball<'a> {
assert!(!formats.is_empty(), "dist.compression-formats can't be empty");
cmd.arg("--compression-formats").arg(formats.join(","));
}
cmd.args(&["--compression-profile", &self.builder.config.dist_compression_profile]);
self.builder.run(&mut cmd);
// Ensure there are no symbolic links in the tarball. In particular,

7
src/tools/rust-installer/src/combiner.rs
View File

@ -1,7 +1,7 @@
use super::Scripter;
use super::Tarballer;
use crate::{
compression::{CompressionFormat, CompressionFormats},
compression::{CompressionFormat, CompressionFormats, CompressionProfile},
util::*,
};
use anyhow::{bail, Context, Result};
@ -48,6 +48,10 @@ actor! {
#[clap(value_name = "DIR")]
output_dir: String = "./dist",
/// The profile used to compress the tarball.
#[clap(value_name = "FORMAT", default_value_t)]
compression_profile: CompressionProfile,
/// The formats used to compress the tarball
#[clap(value_name = "FORMAT", default_value_t)]
compression_formats: CompressionFormats,
@ -153,6 +157,7 @@ impl Combiner {
.work_dir(self.work_dir)
.input(self.package_name)
.output(path_to_str(&output)?.into())
.compression_profile(self.compression_profile)
.compression_formats(self.compression_formats.clone());
tarballer.run()?;

128
src/tools/rust-installer/src/compression.rs
View File

@ -4,6 +4,37 @@ use rayon::prelude::*;
use std::{convert::TryFrom, fmt, io::Read, io::Write, path::Path, str::FromStr};
use xz2::{read::XzDecoder, write::XzEncoder};
#[derive(Default, Debug, Copy, Clone)]
pub enum CompressionProfile {
Fast,
#[default]
Balanced,
Best,
}
impl FromStr for CompressionProfile {
type Err = Error;
fn from_str(input: &str) -> Result<Self, Error> {
Ok(match input {
"fast" => Self::Fast,
"balanced" => Self::Balanced,
"best" => Self::Best,
other => anyhow::bail!("invalid compression profile: {other}"),
})
}
}
impl fmt::Display for CompressionProfile {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
CompressionProfile::Fast => f.write_str("fast"),
CompressionProfile::Balanced => f.write_str("balanced"),
CompressionProfile::Best => f.write_str("best"),
}
}
}
#[derive(Debug, Copy, Clone)]
pub enum CompressionFormat {
Gz,
@ -26,7 +57,11 @@ impl CompressionFormat {
}
}
pub(crate) fn encode(&self, path: impl AsRef<Path>) -> Result<Box<dyn Encoder>, Error> {
pub(crate) fn encode(
&self,
path: impl AsRef<Path>,
profile: CompressionProfile,
) -> Result<Box<dyn Encoder>, Error> {
let mut os = path.as_ref().as_os_str().to_os_string();
os.push(format!(".{}", self.extension()));
let path = Path::new(&os);
@ -37,49 +72,64 @@ impl CompressionFormat {
let file = crate::util::create_new_file(path)?;
Ok(match self {
CompressionFormat::Gz => Box::new(GzEncoder::new(file, flate2::Compression::best())),
CompressionFormat::Gz => Box::new(GzEncoder::new(
file,
match profile {
CompressionProfile::Fast => flate2::Compression::fast(),
CompressionProfile::Balanced => flate2::Compression::new(6),
CompressionProfile::Best => flate2::Compression::best(),
},
)),
CompressionFormat::Xz => {
let mut filters = xz2::stream::Filters::new();
// the preset is overridden by the other options so it doesn't matter
let mut lzma_ops = xz2::stream::LzmaOptions::new_preset(9).unwrap();
// This sets the overall dictionary size, which is also how much memory (baseline)
// is needed for decompression.
lzma_ops.dict_size(64 * 1024 * 1024);
// Use the best match finder for compression ratio.
lzma_ops.match_finder(xz2::stream::MatchFinder::BinaryTree4);
lzma_ops.mode(xz2::stream::Mode::Normal);
// Set nice len to the maximum for best compression ratio
lzma_ops.nice_len(273);
// Set depth to a reasonable value, 0 means auto, 1000 is somwhat high but gives
// good results.
lzma_ops.depth(1000);
// 2 is the default and does well for most files
lzma_ops.position_bits(2);
// 0 is the default and does well for most files
lzma_ops.literal_position_bits(0);
// 3 is the default and does well for most files
lzma_ops.literal_context_bits(3);
let encoder = match profile {
CompressionProfile::Fast => {
xz2::stream::MtStreamBuilder::new().threads(6).preset(1).encoder().unwrap()
}
CompressionProfile::Balanced => {
xz2::stream::MtStreamBuilder::new().threads(6).preset(6).encoder().unwrap()
}
CompressionProfile::Best => {
let mut filters = xz2::stream::Filters::new();
// the preset is overridden by the other options so it doesn't matter
let mut lzma_ops = xz2::stream::LzmaOptions::new_preset(9).unwrap();
// This sets the overall dictionary size, which is also how much memory (baseline)
// is needed for decompression.
lzma_ops.dict_size(64 * 1024 * 1024);
// Use the best match finder for compression ratio.
lzma_ops.match_finder(xz2::stream::MatchFinder::BinaryTree4);
lzma_ops.mode(xz2::stream::Mode::Normal);
// Set nice len to the maximum for best compression ratio
lzma_ops.nice_len(273);
// Set depth to a reasonable value, 0 means auto, 1000 is somwhat high but gives
// good results.
lzma_ops.depth(1000);
// 2 is the default and does well for most files
lzma_ops.position_bits(2);
// 0 is the default and does well for most files
lzma_ops.literal_position_bits(0);
// 3 is the default and does well for most files
lzma_ops.literal_context_bits(3);
filters.lzma2(&lzma_ops);
filters.lzma2(&lzma_ops);
let mut builder = xz2::stream::MtStreamBuilder::new();
builder.filters(filters);
let mut builder = xz2::stream::MtStreamBuilder::new();
builder.filters(filters);
// On 32-bit platforms limit ourselves to 3 threads, otherwise we exceed memory
// usage this process can take. In the future we'll likely only do super-fast
// compression in CI and move this heavyweight processing to promote-release (which
// is always 64-bit and can run on big-memory machines) but for now this lets us
// move forward.
if std::mem::size_of::<usize>() == 4 {
builder.threads(3);
} else {
builder.threads(6);
}
// On 32-bit platforms limit ourselves to 3 threads, otherwise we exceed memory
// usage this process can take. In the future we'll likely only do super-fast
// compression in CI and move this heavyweight processing to promote-release (which
// is always 64-bit and can run on big-memory machines) but for now this lets us
// move forward.
if std::mem::size_of::<usize>() == 4 {
builder.threads(3);
} else {
builder.threads(6);
}
builder.encoder().unwrap()
}
};
let compressor = XzEncoder::new_stream(
std::io::BufWriter::new(file),
builder.encoder().unwrap(),
);
let compressor = XzEncoder::new_stream(std::io::BufWriter::new(file), encoder);
Box::new(compressor)
}
})

7
src/tools/rust-installer/src/generator.rs
View File

@ -1,6 +1,6 @@
use super::Scripter;
use super::Tarballer;
use crate::compression::CompressionFormats;
use crate::compression::{CompressionFormats, CompressionProfile};
use crate::util::*;
use anyhow::{bail, format_err, Context, Result};
use std::collections::BTreeSet;
@ -54,6 +54,10 @@ actor! {
#[clap(value_name = "DIR")]
output_dir: String = "./dist",
/// The profile used to compress the tarball.
#[clap(value_name = "FORMAT", default_value_t)]
compression_profile: CompressionProfile,
/// The formats used to compress the tarball
#[clap(value_name = "FORMAT", default_value_t)]
compression_formats: CompressionFormats,
@ -113,6 +117,7 @@ impl Generator {
.work_dir(self.work_dir)
.input(self.package_name)
.output(path_to_str(&output)?.into())
.compression_profile(self.compression_profile)
.compression_formats(self.compression_formats.clone());
tarballer.run()?;

8
src/tools/rust-installer/src/tarballer.rs
View File

@ -6,7 +6,7 @@ use tar::{Builder, Header};
use walkdir::WalkDir;
use crate::{
compression::{CombinedEncoder, CompressionFormats},
compression::{CombinedEncoder, CompressionFormats, CompressionProfile},
util::*,
};
@ -25,6 +25,10 @@ actor! {
#[clap(value_name = "DIR")]
work_dir: String = "./workdir",
/// The profile used to compress the tarball.
#[clap(value_name = "FORMAT", default_value_t)]
compression_profile: CompressionProfile,
/// The formats used to compress the tarball.
#[clap(value_name = "FORMAT", default_value_t)]
compression_formats: CompressionFormats,
@ -38,7 +42,7 @@ impl Tarballer {
let encoder = CombinedEncoder::new(
self.compression_formats
.iter()
.map(|f| f.encode(&tarball_name))
.map(|f| f.encode(&tarball_name, self.compression_profile))
.collect::<Result<Vec<_>>>()?,
);