dnrops
/
smithy-rs
mirror of https://github.com/smithy-lang/smithy-rs
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Slow down crate publish (#1691) 

Crate publish used to be slow due to the verification step that `cargo`
would run, which was basically a full compile of the crate prior to
uploading it to crates.io. Now that the `publisher` tool passes
`--no-verify` to `cargo`, it publishes fast enough to get throttled.

This change completely removes the publishing parallelism, adds a fixed
time between publishing each crate, and increases the publish retry
backoff time.
This commit is contained in:
John DiSanti 2022-09-01 15:49:10 -07:00 committed by GitHub
parent 600c26da76
commit ce1ffa412b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 24 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
tools/publisher/Cargo.lock generated
View File

@ -1007,7 +1007,6 @@ dependencies = [
"dialoguer",
"handlebars",
"lazy_static",
"num_cpus",
"pretty_assertions",
"regex",
"semver",

1
tools/publisher/Cargo.toml
View File

@ -23,7 +23,6 @@ crates_io_api = "0.7.3"
dialoguer = "0.8"
handlebars = "4.2"
lazy_static = "1"
num_cpus = "1.13"
regex = "1.5.4"
semver = "1.0"
serde = { version = "1", features = ["derive"] }

57
tools/publisher/src/subcommand/publish.rs
View File

@ -19,9 +19,7 @@ use smithy_rs_tool_common::git;
use smithy_rs_tool_common::shell::ShellOperation;
use std::collections::HashSet;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::Semaphore;
use tracing::info;
lazy_static! {
@ -61,40 +59,33 @@ pub async fn subcommand_publish(
// Don't proceed unless the user confirms the plan
confirm_plan(&batches, stats, *skip_confirmation)?;
// Use a semaphore to only allow a few concurrent publishes
let max_concurrency = num_cpus::get_physical();
let semaphore = Arc::new(Semaphore::new(max_concurrency));
info!(
"Will publish {} crates in parallel where possible.",
max_concurrency
);
for batch in batches {
let mut tasks = Vec::new();
let mut any_published = false;
for package in batch {
let permit = semaphore.clone().acquire_owned().await.unwrap();
tasks.push(tokio::spawn(async move {
// Only publish if it hasn't been published yet.
if !is_published(&package.handle).await? {
publish(&package.handle, &package.crate_path).await?;
// Only publish if it hasn't been published yet.
if !is_published(&package.handle).await? {
publish(&package.handle, &package.crate_path).await?;
// Sometimes it takes a little bit of time for the new package version
// to become available after publish. If we proceed too quickly, then
// the next package publish can fail if it depends on this package.
wait_for_eventual_consistency(&package).await?;
info!("Successfully published `{}`", package.handle);
} else {
info!("`{}` was already published", package.handle);
}
correct_owner(&package).await?;
drop(permit);
Ok::<_, anyhow::Error>(())
}));
// Sometimes it takes a little bit of time for the new package version
// to become available after publish. If we proceed too quickly, then
// the next package publish can fail if it depends on this package.
wait_for_eventual_consistency(&package).await?;
info!("Successfully published `{}`", package.handle);
any_published = true;
// Keep things slow to avoid getting throttled by crates.io
tokio::time::sleep(Duration::from_secs(1)).await;
} else {
info!("`{}` was already published", package.handle);
}
correct_owner(&package).await?;
}
for task in tasks {
task.await??;
if any_published {
info!("Sleeping 30 seconds after completion of the batch");
tokio::time::sleep(Duration::from_secs(30)).await;
} else {
info!("No packages in the batch needed publishing. Proceeding with the next batch immediately.")
}
info!("sleeping 30 seconds after completion of the batch");
tokio::time::sleep(Duration::from_secs(30)).await;
}
Ok(())
@ -115,8 +106,8 @@ async fn publish(handle: &PackageHandle, crate_path: &Path) -> Result<()> {
info!("Publishing `{}`...", handle);
run_with_retry(
&format!("Publishing `{}`", handle),
3,
Duration::from_secs(5),
5,
Duration::from_secs(30),
|| async {
cargo::Publish::new(handle.clone(), &crate_path)
.spawn()