Replace the crates.io API with the sparse index (#3447)

The sparse index is preferred to the crates.io API for the checks we
need, according to the [documentation](https://crates.io/data-access).
When the tools were first implemented, the sparse index didn't exist, so
the API was used.

----

_By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice._
This commit is contained in:
John DiSanti 2024-02-28 12:47:46 -08:00 committed by GitHub
parent 49ec3046fb
commit 4604aa5b5d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 178 additions and 162 deletions

View File

@ -150,16 +150,6 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "chrono"
version = "0.4.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38"
dependencies = [
"num-traits",
"serde",
]
[[package]]
name = "clap"
version = "3.1.18"
@ -240,20 +230,23 @@ dependencies = [
]
[[package]]
name = "crates_io_api"
version = "0.7.3"
name = "crates-index"
version = "2.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0ab4553806e9495b7e78a66454c16c5021cd3816d7b8e7e8fe28aa5474346bb"
checksum = "1d9efa03a974d583ad530bbfe00e3d0021de7f26217120437b128dc4c331aa4f"
dependencies = [
"chrono",
"futures",
"log",
"reqwest",
"hex",
"home",
"http",
"memchr",
"rustc-hash",
"semver",
"serde",
"serde_derive",
"serde_json",
"tokio",
"url",
"smol_str",
"thiserror",
"toml 0.8.8",
]
[[package]]
@ -370,21 +363,6 @@ dependencies = [
"autocfg",
]
[[package]]
name = "futures"
version = "0.3.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335"
dependencies = [
"futures-channel",
"futures-core",
"futures-executor",
"futures-io",
"futures-sink",
"futures-task",
"futures-util",
]
[[package]]
name = "futures-channel"
version = "0.3.29"
@ -392,7 +370,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb"
dependencies = [
"futures-core",
"futures-sink",
]
[[package]]
@ -401,33 +378,11 @@ version = "0.3.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c"
[[package]]
name = "futures-executor"
version = "0.3.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc"
dependencies = [
"futures-core",
"futures-task",
"futures-util",
]
[[package]]
name = "futures-io"
version = "0.3.29"
version = "0.3.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa"
[[package]]
name = "futures-macro"
version = "0.3.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.48",
]
checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1"
[[package]]
name = "futures-sink"
@ -447,11 +402,8 @@ version = "0.3.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104"
dependencies = [
"futures-channel",
"futures-core",
"futures-io",
"futures-macro",
"futures-sink",
"futures-task",
"memchr",
"pin-project-lite",
@ -546,6 +498,18 @@ name = "hex"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
dependencies = [
"serde",
]
[[package]]
name = "home"
version = "0.5.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5"
dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "http"
@ -772,15 +736,6 @@ dependencies = [
"winapi",
]
[[package]]
name = "num-traits"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c"
dependencies = [
"autocfg",
]
[[package]]
name = "num_cpus"
version = "1.16.0"
@ -1006,7 +961,6 @@ dependencies = [
"async-trait",
"cargo_toml",
"clap",
"crates_io_api",
"dialoguer",
"fs-err",
"handlebars",
@ -1132,6 +1086,12 @@ version = "0.1.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
[[package]]
name = "rustc-hash"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
[[package]]
name = "rustix"
version = "0.38.26"
@ -1310,6 +1270,7 @@ version = "0.1.0"
dependencies = [
"anyhow",
"async-trait",
"crates-index",
"lazy_static",
"regex",
"reqwest",
@ -1322,6 +1283,15 @@ dependencies = [
"tracing",
]
[[package]]
name = "smol_str"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6845563ada680337a52d43bb0b29f396f2d911616f6573012645b9e3d048a49"
dependencies = [
"serde",
]
[[package]]
name = "socket2"
version = "0.4.10"

View File

@ -19,7 +19,6 @@ async-recursion = "0.3.2"
async-trait = "0.1.74"
cargo_toml = "0.16.3"
clap = { version = "~3.1.18", features = ["derive"] }
crates_io_api = "0.7.3"
dialoguer = "0.8"
fs-err = "2"
handlebars = "4.2"

View File

@ -5,35 +5,25 @@
use crate::cargo;
use crate::package::PackageHandle;
use crates_io_api::{AsyncClient, Error};
use once_cell::sync::Lazy;
use smithy_rs_tool_common::retry::{run_with_retry, BoxError, ErrorClass};
use anyhow::Result;
use smithy_rs_tool_common::shell::ShellOperation;
use std::path::Path;
use smithy_rs_tool_common::{
index::CratesIndex,
retry::{run_with_retry, BoxError, ErrorClass},
};
use std::time::Duration;
use std::{path::Path, sync::Arc};
use tracing::info;
pub static CRATES_IO_CLIENT: Lazy<AsyncClient> = Lazy::new(|| {
AsyncClient::new(
"AWS_RUST_SDK_PUBLISHER (aws-sdk-rust@amazon.com)",
Duration::from_secs(1),
)
.expect("valid client")
});
/// Return `true` if there is at least one version published on crates.io associated with
/// the specified crate name.
#[tracing::instrument]
pub async fn has_been_published_on_crates_io(crate_name: &str) -> anyhow::Result<bool> {
match CRATES_IO_CLIENT.get_crate(crate_name).await {
Ok(_) => Ok(true),
Err(Error::NotFound(_)) => Ok(false),
Err(e) => Err(e.into()),
}
pub async fn is_published(index: Arc<CratesIndex>, crate_name: &str) -> Result<bool> {
let crate_name = crate_name.to_string();
let versions =
tokio::task::spawn_blocking(move || index.published_versions(&crate_name)).await??;
Ok(!versions.is_empty())
}
#[tracing::instrument]
pub async fn publish(handle: &PackageHandle, crate_path: &Path) -> anyhow::Result<()> {
pub async fn publish(handle: &PackageHandle, crate_path: &Path) -> Result<()> {
info!("Publishing `{}`...", handle);
run_with_retry(
&format!("Publishing `{}`", handle),

View File

@ -2,21 +2,24 @@
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
* SPDX-License-Identifier: Apache-2.0
*/
use crate::package::PackageHandle;
use crate::publish::{has_been_published_on_crates_io, publish};
use crate::publish::publish;
use crate::subcommand::publish::correct_owner;
use crate::{cargo, SDK_REPO_NAME};
use crate::{fs::Fs, package::discover_manifests};
use crate::{package::PackageHandle, publish::is_published};
use anyhow::{Context, Result};
use cargo_toml::Manifest;
use clap::Parser;
use dialoguer::Confirm;
use semver::Version;
use smithy_rs_tool_common::git;
use smithy_rs_tool_common::package::PackageCategory;
use std::path::{Path, PathBuf};
use smithy_rs_tool_common::{git, index::CratesIndex};
use std::time::Duration;
use std::{collections::HashSet, fs};
use std::{
path::{Path, PathBuf},
sync::Arc,
};
use tracing::info;
#[derive(Parser, Debug)]
@ -33,11 +36,12 @@ pub async fn subcommand_claim_crate_names(args: &ClaimCrateNamesArgs) -> Result<
let smithy_rs_repository_root =
git::find_git_repository_root(SDK_REPO_NAME, std::env::current_dir()?)?;
let index = Arc::new(CratesIndex::real()?);
let packages = discover_publishable_crate_names(&smithy_rs_repository_root).await?;
let unpublished_package_names = {
let mut s = HashSet::new();
for package_name in packages {
if !has_been_published_on_crates_io(&package_name).await? {
if !is_published(index.clone(), &package_name).await? {
s.insert(package_name);
}
}
@ -65,10 +69,12 @@ async fn claim_crate_name(name: &str) -> Result<()> {
let category = PackageCategory::from_package_name(name);
let package_handle = PackageHandle::new(name, Version::new(0, 0, 1));
publish(&package_handle, crate_dir_path).await?;
// Keep things slow to avoid getting throttled by crates.io
tokio::time::sleep(Duration::from_secs(2)).await;
info!("Successfully published `{}`", package_handle);
correct_owner(&package_handle, &category).await?;
info!("Successfully published `{}`", package_handle);
Ok(())
}

View File

@ -8,19 +8,18 @@ use crate::package::{
discover_and_validate_package_batches, expected_package_owners, Package, PackageBatch,
PackageHandle, PackageStats,
};
use crate::publish::{publish, CRATES_IO_CLIENT};
use crate::publish::publish;
use crate::{cargo, SDK_REPO_CRATE_PATH, SDK_REPO_NAME};
use anyhow::{bail, Context, Result};
use clap::Parser;
use crates_io_api::Error;
use dialoguer::Confirm;
use smithy_rs_tool_common::git;
use smithy_rs_tool_common::package::PackageCategory;
use smithy_rs_tool_common::retry::{run_with_retry, BoxError, ErrorClass};
use smithy_rs_tool_common::shell::ShellOperation;
use std::collections::HashSet;
use smithy_rs_tool_common::{git, index::CratesIndex};
use std::path::{Path, PathBuf};
use std::time::Duration;
use std::{collections::HashSet, sync::Arc};
use tracing::info;
const DEFAULT_DELAY_MILLIS: usize = 1000;
@ -60,11 +59,12 @@ pub async fn subcommand_publish(
// Don't proceed unless the user confirms the plan
confirm_plan(&batches, stats, *skip_confirmation)?;
let index = Arc::new(CratesIndex::real()?);
for batch in &batches {
let mut any_published = false;
for package in batch {
// Only publish if it hasn't been published yet.
if !is_published(&package.handle).await? {
if !is_published(index.clone(), &package.handle).await? {
publish(&package.handle, &package.crate_path).await?;
// Keep things slow to avoid getting throttled by crates.io
@ -73,7 +73,7 @@ pub async fn subcommand_publish(
// Sometimes it takes a little bit of time for the new package version
// to become available after publish. If we proceed too quickly, then
// the next package publish can fail if it depends on this package.
wait_for_eventual_consistency(package).await?;
wait_for_eventual_consistency(index.clone(), package).await?;
info!("Successfully published `{}`", &package.handle);
any_published = true;
} else {
@ -108,43 +108,24 @@ pub fn resolve_publish_location(location: &Path) -> PathBuf {
}
}
async fn is_published(handle: &PackageHandle) -> Result<bool> {
run_with_retry(
&format!("Checking if `{}` is already published", handle.name),
3,
Duration::from_secs(5),
|| async {
let expected_version = handle.version.to_string();
let crate_info = match CRATES_IO_CLIENT.get_crate(&handle.name).await {
Ok(info) => info,
Err(Error::NotFound(_)) => return Ok(false),
Err(other) => return Err(other),
};
Ok(crate_info
.versions
.iter()
.any(|crate_version| crate_version.num == expected_version))
},
|err| match err {
Error::Http(_) => ErrorClass::Retry,
_ => ErrorClass::NoRetry,
},
)
.await
.context("is_published")
async fn is_published(index: Arc<CratesIndex>, handle: &PackageHandle) -> Result<bool> {
let crate_name = handle.name.clone();
let versions =
tokio::task::spawn_blocking(move || index.published_versions(&crate_name)).await??;
Ok(!versions.is_empty())
}
/// Waits for the given package to show up on crates.io
async fn wait_for_eventual_consistency(package: &Package) -> Result<()> {
async fn wait_for_eventual_consistency(index: Arc<CratesIndex>, package: &Package) -> Result<()> {
let max_wait_time = 10usize;
for _ in 0..max_wait_time {
if !is_published(&package.handle).await? {
if !is_published(index.clone(), &package.handle).await? {
tokio::time::sleep(Duration::from_secs(1)).await;
} else {
return Ok(());
}
}
if !is_published(&package.handle).await? {
if !is_published(index.clone(), &package.handle).await? {
return Err(anyhow::Error::msg(format!(
"package wasn't found on crates.io {} seconds after publish",
max_wait_time
@ -248,19 +229,3 @@ fn confirm_plan(
bail!("aborted")
}
}
#[cfg(test)]
mod test {
use super::*;
use crate::package::PackageHandle;
#[ignore]
#[tokio::test]
async fn crate_published_works() {
let handle = PackageHandle::new("aws-smithy-http", "0.27.0-alpha.1".parse().unwrap());
assert!(is_published(&handle).await.expect("failed"));
// we will never publish this version
let handle = PackageHandle::new("aws-smithy-http", "0.21.0-alpha.1".parse().unwrap());
assert!(!is_published(&handle).await.expect("failed"));
}
}

View File

@ -250,9 +250,9 @@ checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f"
[[package]]
name = "crates-index"
version = "2.3.0"
version = "2.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e1f3e3ef6d547bbf1213b3dabbf0b01a500fbd0924abf818b563a4bc2c85296c"
checksum = "1d9efa03a974d583ad530bbfe00e3d0021de7f26217120437b128dc4c331aa4f"
dependencies = [
"hex",
"home",
@ -1078,6 +1078,7 @@ version = "0.1.0"
dependencies = [
"anyhow",
"async-trait",
"crates-index",
"lazy_static",
"regex",
"reqwest",

View File

@ -4,7 +4,6 @@
*/
use crate::{
index::CratesIndex,
repo::Repo,
tag::{previous_release_tag, release_tags},
util::utf8_path_buf,
@ -12,7 +11,9 @@ use crate::{
};
use anyhow::{anyhow, bail, Context, Result};
use camino::{Utf8Path, Utf8PathBuf};
use smithy_rs_tool_common::{command::sync::CommandExt, release_tag::ReleaseTag};
use smithy_rs_tool_common::{
command::sync::CommandExt, index::CratesIndex, release_tag::ReleaseTag,
};
use std::{
collections::{BTreeMap, BTreeSet},
fs,

View File

@ -20,7 +20,6 @@ mod command {
pub use patch::{patch, patch_with};
}
mod index;
mod repo;
mod tag;
mod util;

View File

@ -49,7 +49,7 @@ version = "0.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
dependencies = [
"hermit-abi",
"hermit-abi 0.1.19",
"libc",
"winapi",
]
@ -185,6 +185,26 @@ version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f"
[[package]]
name = "crates-index"
version = "2.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d9efa03a974d583ad530bbfe00e3d0021de7f26217120437b128dc4c331aa4f"
dependencies = [
"hex",
"home",
"http",
"memchr",
"rustc-hash",
"semver",
"serde",
"serde_derive",
"serde_json",
"smol_str",
"thiserror",
"toml 0.8.8",
]
[[package]]
name = "encoding_rs"
version = "0.8.33"
@ -261,6 +281,12 @@ version = "0.3.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d"
[[package]]
name = "futures-io"
version = "0.3.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1"
[[package]]
name = "futures-sink"
version = "0.3.30"
@ -280,9 +306,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48"
dependencies = [
"futures-core",
"futures-io",
"futures-task",
"memchr",
"pin-project-lite",
"pin-utils",
"slab",
]
[[package]]
@ -337,6 +366,30 @@ dependencies = [
"libc",
]
[[package]]
name = "hermit-abi"
version = "0.3.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "379dada1584ad501b383485dd706b8afb7a70fcbc7f4da7d780638a5a6124a60"
[[package]]
name = "hex"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
dependencies = [
"serde",
]
[[package]]
name = "home"
version = "0.5.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5"
dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "http"
version = "0.2.11"
@ -533,6 +586,16 @@ dependencies = [
"tempfile",
]
[[package]]
name = "num_cpus"
version = "1.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
dependencies = [
"hermit-abi 0.3.8",
"libc",
]
[[package]]
name = "object"
version = "0.32.2"
@ -746,6 +809,12 @@ version = "0.1.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
[[package]]
name = "rustc-hash"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
[[package]]
name = "rustix"
version = "0.38.28"
@ -883,6 +952,7 @@ version = "0.1.0"
dependencies = [
"anyhow",
"async-trait",
"crates-index",
"lazy_static",
"regex",
"reqwest",
@ -894,6 +964,15 @@ dependencies = [
"tracing",
]
[[package]]
name = "smol_str"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6845563ada680337a52d43bb0b29f396f2d911616f6573012645b9e3d048a49"
dependencies = [
"serde",
]
[[package]]
name = "socket2"
version = "0.5.5"
@ -1026,6 +1105,7 @@ dependencies = [
"bytes",
"libc",
"mio",
"num_cpus",
"pin-project-lite",
"socket2",
"windows-sys 0.48.0",

View File

@ -18,9 +18,10 @@ opt-level = 0
[dependencies]
anyhow = "1"
async-trait = "0.1.74"
crates-index = { version = "2.5.1", features = ["sparse"] }
lazy_static = "1"
regex = "1.6.0"
reqwest = "0.11.10"
reqwest = { version = "0.11.10", features = ["blocking"] }
semver = "1"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
@ -31,3 +32,4 @@ tracing = "0.1"
[dev-dependencies]
tempfile = "3.3.0"
tokio = { version = "1.20.1", features = ["rt", "macros"] }

View File

@ -3,13 +3,12 @@
* SPDX-License-Identifier: Apache-2.0
*/
use crate::retry::{run_with_retry_sync, ErrorClass};
use anyhow::{anyhow, Context, Error, Result};
use camino::Utf8Path;
use crates_index::Crate;
use reqwest::StatusCode;
use smithy_rs_tool_common::retry::{run_with_retry_sync, ErrorClass};
use std::fs;
use std::{collections::HashMap, time::Duration};
use std::{fs, path::Path};
pub struct CratesIndex(Inner);
@ -28,7 +27,7 @@ impl CratesIndex {
}
/// Returns a fake crates.io index from file, panicking if loading fails.
pub fn fake(path: impl AsRef<Utf8Path>) -> Self {
pub fn fake(path: impl AsRef<Path>) -> Self {
Self(Inner::Fake(FakeIndex::from_file(path)))
}
@ -86,7 +85,7 @@ pub struct FakeIndex {
}
impl FakeIndex {
fn from_file(path: impl AsRef<Utf8Path>) -> FakeIndex {
fn from_file(path: impl AsRef<Path>) -> FakeIndex {
let bytes = fs::read(path.as_ref()).unwrap();
let toml: toml::Value = toml::from_slice(&bytes).unwrap();
let crates: HashMap<String, Vec<_>> = toml["crates"]

View File

@ -9,6 +9,7 @@ pub mod command;
pub mod git;
#[macro_use]
pub mod macros;
pub mod index;
pub mod package;
pub mod release_tag;
pub mod retry;

View File

@ -130,6 +130,7 @@ mod tests {
}
#[tokio::test]
#[cfg(feature = "async")]
async fn fail_max_attempts() {
let attempt = Arc::new(AtomicU8::new(1));
let result = {
@ -156,6 +157,7 @@ mod tests {
}
#[tokio::test]
#[cfg(feature = "async")]
async fn fail_then_succeed() {
let attempt = Arc::new(AtomicU8::new(1));
let result = {
@ -185,6 +187,7 @@ mod tests {
}
#[tokio::test]
#[cfg(feature = "async")]
async fn unretryable_error() {
let attempt = Arc::new(AtomicU8::new(1));
let result = {