Merge branch 'main' into fahadzub/cbor-constraint

This commit is contained in:
Fahad Zubair 2024-09-17 08:39:12 -04:00 committed by GitHub
commit 432e0f9e4c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
16 changed files with 2789 additions and 8560 deletions

View File

@ -1,9 +0,0 @@
---
applies_to: ["client", "server"]
authors: ["rcoh"]
references: ["smithy-rs#3805"]
breaking: false
new_feature: false
bug_fix: true
---
Fix bug in `DateTime::from_secs_f64` where certain floating point values could lead to a panic.

View File

@ -1,26 +0,0 @@
---
applies_to: ["server"]
authors: ["drganjoo"]
references: ["smithy-rs#3803"]
breaking: false
new_feature: true
bug_fix: false
---
Setting the `addValidationExceptionToConstrainedOperations` codegen flag adds `aws.smithy.framework#ValidationException` to operations with constrained inputs that do not already have this exception added.
Sample `smithy-build-template.json`:
```
{
"...",
"plugins": {
"rust-server-codegen": {
"service": "ServiceToGenerateSDKFor",
"module": "amzn-sample-server-sdk",
"codegen": {
"addValidationExceptionToConstrainedOperations": true,
}
}
}
}
```

View File

@ -1,9 +0,0 @@
---
applies_to: ["server"]
authors: ["drganjoo"]
references: ["smithy-rs#3813"]
breaking: true
new_feature: false
bug_fix: true
---
Operations with event stream member shapes must include `ValidationException` in the errors list. This is necessary because the member shape is a required field, and the builder for the operation input or output returns a `std::result::Result` with the error set to `crate::model::ValidationExceptionField`.

View File

@ -1,4 +1,34 @@
<!-- Do not manually edit this file. Use the `changelogger` tool. -->
September 17th, 2024
====================
September 9th, 2024
===================
**Breaking Changes:**
- :bug::warning: (server, [smithy-rs#3813](https://github.com/smithy-lang/smithy-rs/issues/3813)) Operations with event stream member shapes must include `ValidationException` in the errors list. This is necessary because the member shape is a required field, and the builder for the operation input or output returns a `std::result::Result` with the error set to `crate::model::ValidationExceptionField`.
**New this release:**
- :tada: (server, [smithy-rs#3803](https://github.com/smithy-lang/smithy-rs/issues/3803)) Setting the `addValidationExceptionToConstrainedOperations` codegen flag adds `aws.smithy.framework#ValidationException` to operations with constrained inputs that do not already have this exception added.
Sample `smithy-build-template.json`:
```
{
"...",
"plugins": {
"rust-server-codegen": {
"service": "ServiceToGenerateSDKFor",
"module": "amzn-sample-server-sdk",
"codegen": {
"addValidationExceptionToConstrainedOperations": true,
}
}
}
}
```
- :bug: (all, [smithy-rs#3805](https://github.com/smithy-lang/smithy-rs/issues/3805)) Fix bug in `DateTime::from_secs_f64` where certain floating point values could lead to a panic.
August 28th, 2024
=================
**Breaking Changes:**

View File

@ -5,49 +5,6 @@
{
"smithy-rs": [],
"aws-sdk-rust": [
{
"message": "`aws_smithy_runtime_api::client::orchestrator::HttpRequest` and `aws_smithy_runtime_api::client::orchestrator::HttpResponse` are now re-exported in AWS SDK clients so that using these types does not require directly depending on `aws-smithy-runtime-api`.",
"meta": {
"bug": false,
"breaking": false,
"tada": false
},
"author": "ysaito1001",
"references": [
"smithy-rs#3591"
],
"since-commit": "8d23be1392c46a4d59c2910c3dab00ecaad2cf26",
"age": 5
},
{
"message": "Allow [AwsUserAgent](https://docs.rs/aws-runtime/1.3.1/aws_runtime/user_agent/struct.AwsUserAgent.html) to incorporate business metrics, which now deprecates the existing feature and config metadata.\n",
"meta": {
"bug": false,
"breaking": false,
"tada": false
},
"author": "ysaito1001",
"references": [
"smithy-rs#3781"
],
"since-commit": "433e1a00e2d3eadfde78b472d78b30a8f9204dbe",
"age": 4
},
{
"message": "Fix incorrect redaction of `@sensitive` types in maps and lists.\n",
"meta": {
"bug": true,
"breaking": false,
"tada": false
},
"author": "landonxjames",
"references": [
"smithy-rs#3765",
"smithy-rs#3757"
],
"since-commit": "433e1a00e2d3eadfde78b472d78b30a8f9204dbe",
"age": 4
},
{
"message": "Upgrade MSRV to Rust 1.78.0.\n",
"meta": {
@ -60,7 +17,7 @@
"smithy-rs#3800"
],
"since-commit": "d64aea29ad48d7bddb5a7511f3f1175c478e2c1e",
"age": 1
"age": 3
},
{
"message": "Add minimal support for `AWS::Auth::AccountId` and `AWS::Auth::AccountIdEndpointMode` endpoint built-ins\n",
@ -74,7 +31,7 @@
"smithy-rs#3792"
],
"since-commit": "d64aea29ad48d7bddb5a7511f3f1175c478e2c1e",
"age": 1
"age": 3
},
{
"message": "Fix the execution order of [modify_before_serialization](https://docs.rs/aws-smithy-runtime-api/latest/aws_smithy_runtime_api/client/interceptors/trait.Intercept.html#method.modify_before_serialization) and [read_before_serialization](https://docs.rs/aws-smithy-runtime-api/latest/aws_smithy_runtime_api/client/interceptors/trait.Intercept.html#method.read_before_serialization) in the orchestrator. The `modify_before_serialization` method now executes before the `read_before_serialization` method. This adjustment may result in changes in behavior depending on how you customize interceptors.\n",
@ -88,7 +45,7 @@
"smithy-rs#3798"
],
"since-commit": "d64aea29ad48d7bddb5a7511f3f1175c478e2c1e",
"age": 1
"age": 3
},
{
"message": "Fix the [Length::UpTo](https://docs.rs/aws-smithy-types/1.2.2/aws_smithy_types/byte_stream/enum.Length.html) usage in [FsBuilder](https://docs.rs/aws-smithy-types/1.2.2/aws_smithy_types/byte_stream/struct.FsBuilder.html), ensuring that the specified length does not exceed the remaining file length.\n",
@ -103,7 +60,7 @@
"smithy-rs#3797"
],
"since-commit": "d64aea29ad48d7bddb5a7511f3f1175c478e2c1e",
"age": 1
"age": 3
},
{
"message": "Re-export `ByteStream`'s `Length` and `FsBuilder`. By making these types available directly within a client crate, customers can use `ByteStream::read_from` without needing to import them separately from the `aws-smithy-types` crate.\n",
@ -117,6 +74,46 @@
"aws-sdk-rust#820"
],
"since-commit": "d64aea29ad48d7bddb5a7511f3f1175c478e2c1e",
"age": 3
},
{
"message": "Remove stalled stream protection from transcribe-streaming operations.\n",
"meta": {
"bug": true,
"breaking": false,
"tada": false
},
"author": "landonxjames",
"references": [
"aws-sdk-rust#1181"
],
"since-commit": "c622e5e97b199cc2382a4fbc14a9773e9ea1766c",
"age": 1
},
{
"message": "deprecate http-02x presign APIs in favor of http-1x equivalents\n",
"meta": {
"bug": false,
"breaking": false,
"tada": false
},
"author": "aajtodd",
"references": [],
"since-commit": "c622e5e97b199cc2382a4fbc14a9773e9ea1766c",
"age": 1
},
{
"message": "Update Smoketest codegeneration to be endpoint built-in aware.\n",
"meta": {
"bug": true,
"breaking": false,
"tada": false
},
"author": "landonxjames",
"references": [
"smithy-rs#3836"
],
"since-commit": "c622e5e97b199cc2382a4fbc14a9773e9ea1766c",
"age": 1
}
],

View File

@ -225,11 +225,14 @@ impl PresignedRequest {
}
/// Given a body, produce an `http::Request` from this `PresignedRequest`
#[deprecated = "Prefer the `make_http_1x_request()` instead by enabling the `http-1x` feature."]
#[allow(deprecated)]
pub fn make_http_02x_request<B>(&self, body: B) -> http::Request<B> {
self.clone().into_http_02x_request(body)
}
/// Converts this `PresignedRequest` directly into an `http` request.
#[deprecated = "Prefer the `into_http_1x_request` instead by enabling the `http-1x` feature."]
pub fn into_http_02x_request<B>(self, body: B) -> http::Request<B> {
self.http_request
.try_into_http02x()

View File

@ -28,6 +28,7 @@ import software.amazon.smithy.rustsdk.customize.s3control.S3ControlDecorator
import software.amazon.smithy.rustsdk.customize.sso.SSODecorator
import software.amazon.smithy.rustsdk.customize.sts.STSDecorator
import software.amazon.smithy.rustsdk.customize.timestream.TimestreamDecorator
import software.amazon.smithy.rustsdk.customize.transcribestreaming.TranscribeStreamingDecorator
import software.amazon.smithy.rustsdk.endpoints.AwsEndpointsStdLib
import software.amazon.smithy.rustsdk.endpoints.OperationInputTestDecorator
import software.amazon.smithy.rustsdk.endpoints.RequireEndpointRules
@ -88,6 +89,7 @@ val DECORATORS: List<ClientCodegenDecorator> =
SSODecorator().onlyApplyTo("com.amazonaws.sso#SWBPortalService"),
TimestreamDecorator().onlyApplyTo("com.amazonaws.timestreamwrite#Timestream_20181101"),
TimestreamDecorator().onlyApplyTo("com.amazonaws.timestreamquery#Timestream_20181101"),
TranscribeStreamingDecorator().onlyApplyTo("com.amazonaws.transcribestreaming#Transcribe"),
// Only build docs-rs for linux to reduce load on docs.rs
listOf(
DocsRsMetadataDecorator(

View File

@ -11,8 +11,10 @@ import software.amazon.smithy.model.node.ObjectNode
import software.amazon.smithy.model.shapes.MemberShape
import software.amazon.smithy.model.shapes.OperationShape
import software.amazon.smithy.model.shapes.StructureShape
import software.amazon.smithy.rulesengine.language.syntax.parameters.Parameters
import software.amazon.smithy.rust.codegen.client.smithy.ClientCodegenContext
import software.amazon.smithy.rust.codegen.client.smithy.customize.ClientCodegenDecorator
import software.amazon.smithy.rust.codegen.client.smithy.endpoint.EndpointRulesetIndex
import software.amazon.smithy.rust.codegen.client.smithy.generators.client.FluentClientGenerator
import software.amazon.smithy.rust.codegen.core.rustlang.Attribute
import software.amazon.smithy.rust.codegen.core.rustlang.Attribute.Companion.cfg
@ -42,6 +44,7 @@ import software.amazon.smithy.smoketests.traits.SmokeTestCase
import software.amazon.smithy.smoketests.traits.SmokeTestsTrait
import java.util.Optional
import java.util.logging.Logger
import kotlin.jvm.optionals.getOrElse
class SmokeTestsDecorator : ClientCodegenDecorator {
override val name: String = "SmokeTests"
@ -134,7 +137,7 @@ fun renderPrologue(
they are disabled by default. To enable them, run the tests with
```sh
RUSTFLAGS="--cfg smoketests" cargo test.
RUSTFLAGS="--cfg smoketests" cargo test
```
""",
)
@ -167,6 +170,17 @@ class SmokeTestsInstantiator(
) {
private val model = codegenContext.model
private val symbolProvider = codegenContext.symbolProvider
private val builtInParamNames: List<String> by lazy {
val index = EndpointRulesetIndex.of(codegenContext.model)
val rulesOrNull = index.endpointRulesForService(codegenContext.serviceShape)
val builtInParams: Parameters = (rulesOrNull?.parameters ?: Parameters.builder().build())
val temp: MutableList<String> = mutableListOf()
builtInParams.forEach { temp.add(it.builtIn.getOrElse { "" }) }
temp
}
private val fipsName = "AWS::UseFIPS"
private val dualStackName = "AWS::UseDualStack"
private val rc = codegenContext.runtimeConfig
fun render(
writer: RustWriter,
@ -191,9 +205,18 @@ class SmokeTestsInstantiator(
val vendorParams = AwsSmokeTestModel.getAwsVendorParams(testCase)
vendorParams.orNull()?.let { params ->
rust(".region(config::Region::new(${params.region.dq()}))")
rust(".use_dual_stack(${params.useDualstack()})")
rust(".use_fips(${params.useFips()})")
rustTemplate(
".region(#{Region}::new(${params.region.dq()}))",
"Region" to AwsRuntimeType.awsTypes(rc).resolve("region::Region"),
)
if (builtInParamNames.contains(dualStackName)) {
rust(".use_dual_stack(${params.useDualstack()})")
}
if (builtInParamNames.contains(fipsName)) {
rust(".use_fips(${params.useFips()})")
}
params.uri.orNull()?.let { rust(".endpoint_url($it)") }
}

View File

@ -0,0 +1,45 @@
/*
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
* SPDX-License-Identifier: Apache-2.0
*/
package software.amazon.smithy.rustsdk.customize.transcribestreaming
import software.amazon.smithy.model.Model
import software.amazon.smithy.model.shapes.OperationShape
import software.amazon.smithy.model.shapes.ServiceShape
import software.amazon.smithy.model.shapes.ShapeId
import software.amazon.smithy.model.transform.ModelTransformer
import software.amazon.smithy.rust.codegen.client.smithy.ClientRustSettings
import software.amazon.smithy.rust.codegen.client.smithy.customize.ClientCodegenDecorator
import software.amazon.smithy.rust.codegen.client.smithy.traits.IncompatibleWithStalledStreamProtectionTrait
import software.amazon.smithy.rust.codegen.core.util.letIf
import java.util.logging.Logger
/**
* Top level decorator for TranscribeStreaming
*/
class TranscribeStreamingDecorator : ClientCodegenDecorator {
private val operationsIncompatibleWithStalledStreamProtection =
setOf(
ShapeId.from("com.amazonaws.transcribestreaming#StartCallAnalyticsStreamTranscription"),
ShapeId.from("com.amazonaws.transcribestreaming#StartMedicalStreamTranscription"),
ShapeId.from("com.amazonaws.transcribestreaming#StartStreamTranscription"),
)
override val name: String = "TranscribeStreamingDecorator"
override val order: Byte = 0
private val logger = Logger.getLogger(javaClass.name)
override fun transformModel(
service: ServiceShape,
model: Model,
settings: ClientRustSettings,
): Model =
ModelTransformer.create().mapShapes(model) { shape ->
shape.letIf(shape.id in operationsIncompatibleWithStalledStreamProtection) {
logger.info("Adding IncompatibleWithStalledStreamProtection trait to $it")
(it as OperationShape).toBuilder().addTrait(IncompatibleWithStalledStreamProtectionTrait()).build()
}
}
}

8695
aws/sdk/Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@ -39,10 +39,12 @@ val sdkVersionerToolPath = rootProject.projectDir.resolve("tools/ci-build/sdk-ve
val awsConfigPath = rootProject.projectDir.resolve("aws/rust-runtime/aws-config")
val rustRuntimePath = rootProject.projectDir.resolve("rust-runtime")
val awsRustRuntimePath = rootProject.projectDir.resolve("aws/rust-runtime")
val checkedInCargoLock = rootProject.projectDir.resolve("aws/sdk/Cargo.lock")
val awsSdkPath = rootProject.projectDir.resolve("aws/sdk")
val outputDir = layout.buildDirectory.dir("aws-sdk").get()
val sdkOutputDir = outputDir.dir("sdk")
val examplesOutputDir = outputDir.dir("examples")
val checkedInSdkLockfile = rootProject.projectDir.resolve("aws/sdk/Cargo.lock")
val generatedSdkLockfile = outputDir.file("Cargo.lock")
dependencies {
@ -449,66 +451,152 @@ tasks["assemble"].apply {
"hydrateReadme",
"relocateChangelog",
)
finalizedBy("copyCheckedInCargoLock")
finalizedBy("copyCheckedInSdkLockfile")
outputs.upToDateWhen { false }
}
tasks.register<Copy>("copyCheckedInCargoLock") {
description = "Copy the checked in Cargo.lock file back to the build directory"
tasks.register<Copy>("copyCheckedInSdkLockfile") {
description = "Copy the checked-in SDK lockfile to the build directory"
this.outputs.upToDateWhen { false }
from(checkedInCargoLock)
from(checkedInSdkLockfile)
into(outputDir)
}
tasks.register<Copy>("replaceCheckedInSdkLockfile") {
description = "Replace the checked-in SDK lockfile by copying the one in the build directory back to `aws/sdk`"
dependsOn("copyCheckedInSdkLockfile")
dependsOn("downgradeAwsSdkLockfile")
this.outputs.upToDateWhen { false }
from(generatedSdkLockfile)
into(awsSdkPath)
}
project.registerCargoCommandsTasks(outputDir.asFile)
project.registerGenerateCargoConfigTomlTask(outputDir.asFile)
//The task name "test" is already registered by one of our plugins
// The task name "test" is already registered by one of our plugins
tasks.register("sdkTest") {
description = "Run Cargo clippy/test/docs against the generated SDK."
dependsOn("assemble")
finalizedBy(Cargo.CLIPPY.toString, Cargo.TEST.toString, Cargo.DOCS.toString)
}
//Tasks for generating individual Cargo.lock files
fun Project.registerLockfileGeneration(
/**
* Generate tasks for pinning broken dependencies to bypass compatibility issues
*
* Some dependencies may have compatibility issues that prevent updating to the latest versions.
* In such cases, we pin these dependencies to the last known working versions.
*
* To update broken dependencies (maybe for CI/CD with the latest versions), run a task with the flag, e.g.,
* `./gradlew -Paws.sdk.force.update.broken.dependencies aws:sdk:cargoUpdateAllLockfiles`
*/
fun Project.registerDowngradeFor(
dir: File,
name: String,
): TaskProvider<Exec> {
return tasks.register<Exec>("generate${name}Lockfile") {
return tasks.register<Exec>("downgrade${name}Lockfile") {
onlyIf {
properties["aws.sdk.force.update.broken.dependencies"] == null
}
executable = "sh" // noop to avoid execCommand == null
doLast {
val crateNameToLastKnownWorkingVersions =
mapOf("minicbor" to "0.24.2")
crateNameToLastKnownWorkingVersions.forEach { (crate, version) ->
// doesn't matter even if the specified crate does not exist in the lockfile
exec {
workingDir(dir)
commandLine("sh", "-c", "cargo update $crate --precise $version || true")
}
}
}
}
}
val downgradeAwsConfigLockfile = registerDowngradeFor(awsConfigPath, "AwsConfig")
val downgradeAwsRuntimeLockfile = registerDowngradeFor(awsRustRuntimePath, "AwsRustRuntime")
val downgradeSmithyRuntimeLockfile = registerDowngradeFor(rustRuntimePath, "RustRuntime")
val downgradeAwsSdkLockfile = registerDowngradeFor(outputDir.asFile, "AwsSdk")
// Tasks for updating individual Cargo.lock files
fun Project.registerCargoUpdateFor(
dir: File,
name: String,
): TaskProvider<Exec> {
return tasks.register<Exec>("cargoUpdate${name}Lockfile") {
workingDir(dir)
environment("RUSTFLAGS", "--cfg aws_sdk_unstable")
commandLine("cargo", "generate-lockfile")
commandLine("cargo", "update")
finalizedBy("downgrade${name}Lockfile")
}
}
val generateAwsConfigLockfile = registerLockfileGeneration(awsConfigPath, "AwsConfig")
val generateAwsRuntimeLockfile = registerLockfileGeneration(awsRustRuntimePath, "AwsRustRuntime")
val generateSmithytRuntimeLockfile = registerLockfileGeneration(rustRuntimePath, "RustRuntime")
val cargoUpdateAwsConfigLockfile = registerCargoUpdateFor(awsConfigPath, "AwsConfig")
val cargoUpdateAwsRuntimeLockfile = registerCargoUpdateFor(awsRustRuntimePath, "AwsRustRuntime")
val cargoUpdateSmithyRuntimeLockfile = registerCargoUpdateFor(rustRuntimePath, "RustRuntime")
//Generates a lockfile from the aws-sdk-rust repo and copies it into the smithy-rs repo
val generateAwsSdkRustLockfile = tasks.register<Exec>("generateAwsSdkRustLockfile") {
val sdkRustPath: String =
properties.get("aws-sdk-rust-path") ?: throw Exception("A -Paws-sdk-rust-path argument must be specified")
workingDir(sdkRustPath)
/**
* Updates the lockfile located in the `aws/sdk` directory.
*
* Previously, we would run `cargo generate-lockfile` in the `aws-sdk-rust` repository and then copy the resulting
* `Cargo.lock` into the `smithy-rs` repository. This approach introduced a delay, as new dependencies added to runtime
* crates would not be reflected in the SDK lockfile until the runtime crates were released to the `aws-sdk-rust`
* repository.
*
* We now generate a lockfile directly in `aws/sdk/build/aws-sdk`, which suffices for our CI/CD purposes, as it covers
* the crate dependencies used by the SDK:
* - Smithy runtime crates and inlineables
* - Smithy codegen decorators
* - Aws runtime crates and inlineables
* - Aws SDK codegen decorators
* - Service customizations (as long as we have their models in `aws/sdk/aws-models`)
*/
val cargoUpdateAwsSdkLockfile = tasks.register<Exec>("cargoUpdateAwsSdkLockfile") {
dependsOn("assemble")
workingDir(outputDir)
environment("RUSTFLAGS", "--cfg aws_sdk_unstable")
commandLine("cargo", "generate-lockfile")
copy {
from("${sdkRustPath}/Cargo.lock")
into(rootProject.projectDir.resolve("aws/sdk"))
commandLine("cargo", "update")
finalizedBy(
"downgradeAwsSdkLockfile",
"replaceCheckedInSdkLockfile",
)
}
tasks.register<Exec>("syncAwsSdkLockfile") {
description = """
Synchronize the SDK lockfile to ensure that it includes all dependencies specified in runtime lockfiles.
"""
dependsOn("assemble")
workingDir(outputDir)
environment("RUSTFLAGS", "--cfg aws_sdk_unstable")
// Using `cargo generate-lockfile` or `cargo update` is not suitable here, as they update dependencies to their
// latest versions. Instead, we need to preserve the existing dependencies in the SDK lockfile while incorporating
// new dependencies introduced by runtime crates. This can be achieved by running `cargo check` with the lockfile
// copied to the `aws/sdk/build/aws-sdk` directory.
commandLine("cargo", "check", "--all-features")
doLast {
// We avoid using `replaceCheckedInSdkLockfile` in favor of `copy` to prevent dependency on
// `downgradeAwsSdkLockfile`. Downgrading dependencies is unnecessary when synchronizing the SDK lockfile with
// runtime lockfiles.
copy {
from(generatedSdkLockfile)
into(awsSdkPath)
}
}
}
//Parent task to generate all the Cargo.lock files
tasks.register("generateAllLockfiles") {
description =
"Create Cargo.lock files for aws-config, aws/rust-runtime, rust-runtime, and the workspace created by" +
"the assemble task."
// Parent task to update all the Cargo.lock files
tasks.register("cargoUpdateAllLockfiles") {
description = """
Update Cargo.lock files for aws-config, aws/rust-runtime, rust-runtime, and the workspace created by the
assemble task.
"""
finalizedBy(
generateAwsSdkRustLockfile,
generateAwsConfigLockfile,
generateAwsRuntimeLockfile,
generateSmithytRuntimeLockfile,
cargoUpdateAwsSdkLockfile,
cargoUpdateAwsConfigLockfile,
cargoUpdateAwsRuntimeLockfile,
cargoUpdateSmithyRuntimeLockfile,
)
}

1741
tools/ci-build/sdk-lockfiles/Cargo.lock generated Normal file

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,23 @@
[package]
name = "sdk-lockfiles"
version = "0.1.0"
authors = ["AWS Rust SDK Team <aws-sdk-rust@amazon.com>"]
description = """
A CLI tool to audit lockfiles for Smithy runtime crates, AWS runtime crates, `aws-config`, and the workspace containing
SDK crates
"""
edition = "2021"
license = "Apache-2.0"
publish = false
[dependencies]
anyhow = "1.0.87"
cargo-lock = { version = "9.0.0", features = ["dependency-tree"] }
clap = { version = "4.4.11", features = ["derive", "env"] }
petgraph = "0.6.5"
smithy-rs-tool-common = { path = "../smithy-rs-tool-common" }
tracing = "0.1.40"
tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
[dev-dependencies]
itertools = "0.13.0"

View File

@ -0,0 +1,24 @@
sdk-lockfiles
=============
This CLI tool audits the `Cargo.lock` files in the `smithy-rs` repository. These lockfiles are used to ensure
reproducible builds. The `sdk-lockfiles` tool specifically audits the following lockfiles:
- The [lockfile](https://github.com/smithy-lang/smithy-rs/blob/main/rust-runtime/Cargo.lock) for Smithy runtime crates
- The [lockfile](https://github.com/smithy-lang/smithy-rs/blob/main/aws/rust-runtime/Cargo.lock) for AWS runtime crates
- The [lockfile](https://github.com/smithy-lang/smithy-rs/blob/main/aws/rust-runtime/aws-config/Cargo.lock) for the `aws-config` crate
- The [lockfile](https://github.com/smithy-lang/smithy-rs/blob/main/aws/sdk/Cargo.lock) for the workspace containing code-generated AWS SDK crates (*)
Specifically, the tool ensures that the lockfile marked with (*) is a superset containing all dependencies listed in
the rest of the runtime lockfiles. If it detects a new dependency in the AWS SDK crates introduced by any of the runtime
lockfiles (unless the dependency is introduced by a server runtime crate), it will output a message similar to the
following:
```
$ sdk-lockfiles audit
2024-09-10T16:48:38.460518Z INFO sdk_lockfiles::audit: checking whether `rust-runtime/Cargo.lock` is covered by the SDK lockfile...
2024-09-10T16:48:38.489879Z INFO sdk_lockfiles::audit: checking whether `aws/rust-runtime/Cargo.lock` is covered by the SDK lockfile...
2024-09-10T16:48:38.490306Z INFO sdk_lockfiles::audit: checking whether `aws/rust-runtime/aws-config/Cargo.lock` is covered by the SDK lockfile...
`minicbor` (0.24.2), used by `rust-runtime/Cargo.lock`, is not contained in SDK lockfile!
Error: there are lockfile audit failures
```
This tool is intended for automated use.

View File

@ -0,0 +1,430 @@
/*
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
* SPDX-License-Identifier: Apache-2.0
*/
use crate::AuditArgs;
use anyhow::bail;
use anyhow::{Context, Result};
use cargo_lock::dependency::graph::{Graph, NodeIndex};
use cargo_lock::{package::Package, Lockfile};
use petgraph::visit::EdgeRef;
use smithy_rs_tool_common::git::find_git_repository_root;
use smithy_rs_tool_common::here;
use std::collections::{BTreeSet, HashSet};
use std::env;
use std::iter;
use std::path::PathBuf;
// A list of AWS runtime crate must be in sync with
// https://github.com/smithy-lang/smithy-rs/blob/0f9b9aba386ea3063912a0464ba6a1fd7c596018/buildSrc/src/main/kotlin/CrateSet.kt#L42-L53
// plus `aws-inlineable`
const AWS_SDK_RUNTIMES: &[&str] = &[
"aws-config",
"aws-credential-types",
"aws-endpoint",
"aws-http",
"aws-hyper",
"aws-inlineable",
"aws-runtime",
"aws-runtime-api",
"aws-sig-auth",
"aws-sigv4",
"aws-types",
];
// A list of server runtime crates must be in sync with
// https://github.com/smithy-lang/smithy-rs/blob/0f9b9aba386ea3063912a0464ba6a1fd7c596018/buildSrc/src/main/kotlin/CrateSet.kt#L85-L87
const SERVER_SPECIFIC_RUNTIMES: &[&str] = &[
"aws-smithy-http-server",
"aws-smithy-http-server-python",
"aws-smithy-http-typescript",
];
fn new_dependency_for_aws_sdk(crate_name: &str) -> bool {
AWS_SDK_RUNTIMES.contains(&crate_name)
|| crate_name == "inlineable"
|| (crate_name.starts_with("aws-smithy-")
&& !SERVER_SPECIFIC_RUNTIMES.contains(&crate_name))
}
// Recursively traverses a chain of dependencies originating from a potential new dependency. Returns true as soon as
// it encounters a crate name that matches a runtime crate used by the AWS SDK.
fn visit(graph: &Graph, node_index: NodeIndex, visited: &mut BTreeSet<NodeIndex>) -> bool {
if !visited.insert(node_index) {
return false;
}
let dependencies = graph
.edges_directed(
node_index,
cargo_lock::dependency::graph::EdgeDirection::Incoming,
)
.map(|edge| edge.source())
.collect::<Vec<_>>();
for dependency_node_index in dependencies.iter() {
let package = &graph[*dependency_node_index];
tracing::debug!("visiting `{}`", package.name.as_str());
if new_dependency_for_aws_sdk(package.name.as_str()) {
tracing::debug!("it's a new dependency for the AWS SDK!");
return true;
}
if visit(graph, *dependency_node_index, visited) {
return true;
}
}
false
}
// Checks if the `target` dependency is introduced by a runtime crate used by the AWS SDK.
//
// This function considers `target` a new dependency if it is used by a runtime crate, whether directly or indirectly,
// that is part of the AWS SDK.
fn new_dependency(lockfile: &Lockfile, target: &str) -> bool {
tracing::debug!(
"`{}` is not recorded in the SDK lockfile. Verifying whether it is a new dependency for the AWS SDK...",
target
);
let tree = lockfile.dependency_tree().unwrap();
let indices: Vec<_> = [target.to_owned()]
.iter()
.map(|dep| {
let package = lockfile
.packages
.iter()
.find(|pkg| pkg.name.as_str() == dep)
.unwrap();
tree.nodes()[&package.into()]
})
.collect();
for index in &indices {
let mut visited: BTreeSet<NodeIndex> = BTreeSet::new();
tracing::debug!("traversing a dependency chain for `{}`...", target);
if visit(tree.graph(), *index, &mut visited) {
return true;
}
}
tracing::debug!("`{}` is not a new dependency for the AWS SDK", target);
false
}
// Verifies if all dependencies listed in `runtime_lockfile` are present in `sdk_dependency_set`, and returns an
// iterator that yields those not found in the set.
//
// This check is based solely on crate names, ignoring other metadata such as versions or sources.
fn audit_runtime_lockfile_covered_by_sdk_lockfile<'a>(
runtime_lockfile: &'a Lockfile,
sdk_dependency_set: &'a HashSet<&str>,
) -> impl Iterator<Item = &'a Package> + 'a {
runtime_lockfile.packages.iter().filter(move |p| {
!sdk_dependency_set.contains(p.name.as_str())
&& new_dependency(runtime_lockfile, p.name.as_str())
})
}
fn lockfile_for(
smithy_rs_root: PathBuf,
relative_path_to_lockfile: &str,
) -> Result<(Lockfile, &str)> {
let mut lockfile = smithy_rs_root;
lockfile.push(relative_path_to_lockfile);
Ok((
Lockfile::load(lockfile).with_context(|| {
format!(
"failed to crate a `Lockfile` for {}",
relative_path_to_lockfile
)
})?,
relative_path_to_lockfile,
))
}
pub(super) fn audit(args: AuditArgs) -> Result<()> {
let cwd = if let Some(smithy_rs_path) = args.smithy_rs_path {
smithy_rs_path
} else {
env::current_dir().context("failed to get current working directory")?
};
let smithy_rs_root = find_git_repository_root("smithy-rs", cwd).context(here!())?;
let (sdk_lockfile, _) = lockfile_for(smithy_rs_root.clone(), "aws/sdk/Cargo.lock")?;
let sdk_dependency_set = sdk_lockfile
.packages
.iter()
.map(|p| p.name.as_str())
.collect::<HashSet<_>>();
let runtime_lockfiles = [
lockfile_for(smithy_rs_root.clone(), "rust-runtime/Cargo.lock")?,
lockfile_for(smithy_rs_root.clone(), "aws/rust-runtime/Cargo.lock")?,
lockfile_for(smithy_rs_root, "aws/rust-runtime/aws-config/Cargo.lock")?,
];
let mut uncovered = Vec::new();
for (runtime_lockfile, path) in &runtime_lockfiles {
tracing::info!(
"checking whether `{}` is covered by the SDK lockfile...",
path
);
uncovered.extend(
audit_runtime_lockfile_covered_by_sdk_lockfile(runtime_lockfile, &sdk_dependency_set)
.zip(iter::repeat(path)),
);
}
if uncovered.is_empty() {
println!("SUCCESS");
Ok(())
} else {
for (pkg, origin_lockfile) in uncovered {
eprintln!(
"`{}` ({}), used by `{}`, is not contained in the SDK lockfile!",
pkg.name.as_str(),
pkg.version,
origin_lockfile,
);
}
bail!("there are lockfile audit failures")
}
}
#[cfg(test)]
mod tests {
use super::*;
use itertools::Itertools;
use std::str::FromStr;
// For simplicity, return an SDK dependency set with a small subset of crates. If a runtime crate used by
// subsequent tests is omitted, it will not affect the functionality of the system under test,
// `audit_runtime_lockfile_covered_by_sdk_lockfile`.
fn sdk_dependency_set() -> HashSet<&'static str> {
let mut result = HashSet::new();
result.insert("aws-credential-types");
result.insert("aws-sigv4");
result.insert("aws-smithy-cbor");
result.insert("aws-smithy-runtime");
result.insert("fastrand");
result.insert("zeroize");
result
}
#[test]
fn dependency_is_covered_by_sdk() {
let runtime_lockfile = Lockfile::from_str(
r#"
[[package]]
name = "aws-credential-types"
version = "1.2.1"
dependencies = [
"zeroize",
]
[[package]]
name = "aws-smithy-runtime"
version = "1.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d1ce695746394772e7000b39fe073095db6d45a862d0767dd5ad0ac0d7f8eb87"
dependencies = [
"fastrand",
]
[[package]]
name = "fastrand"
version = "2.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "658bd65b1cf4c852a3cc96f18a8ce7b5640f6b703f905c7d74532294c2a63984"
[[package]]
name = "zeroize"
version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde"
"#,
)
.unwrap();
assert!(audit_runtime_lockfile_covered_by_sdk_lockfile(
&runtime_lockfile,
&sdk_dependency_set(),
)
.next()
.is_none());
}
#[test]
fn new_dependency_but_introduced_by_crate_irrelevant_to_sdk() {
let runtime_lockfile = Lockfile::from_str(
r#"
[[package]]
name = "aws-smithy-http-server-python"
version = "0.63.2"
dependencies = [
"pyo3-asyncio",
]
[[package]]
name = "pyo3-asyncio"
version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3564762e37035cfc486228e10b0528460fa026d681b5763873c693aa0d5c260"
dependencies = [
"inventory",
]
[[package]]
name = "inventory"
version = "0.3.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f958d3d68f4167080a18141e10381e7634563984a537f2a49a30fd8e53ac5767"
"#,
)
.unwrap();
assert!(audit_runtime_lockfile_covered_by_sdk_lockfile(
&runtime_lockfile,
&sdk_dependency_set(),
)
.next()
.is_none());
}
#[test]
fn new_dependency_for_sdk() {
// New dependencies originating from the smithy runtime crates
{
let runtime_lockfile = Lockfile::from_str(
r#"
[[package]]
name = "aws-smithy-cbor"
version = "0.60.7"
dependencies = [
"minicbor",
]
[[package]]
name = "inlineable"
version = "0.1.0"
dependencies = [
"md-5"
]
[[package]]
name = "md-5"
version = "0.10.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf"
[[package]]
name = "minicbor"
version = "0.24.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f8e213c36148d828083ae01948eed271d03f95f7e72571fa242d78184029af2"
"#,
)
.unwrap();
assert_eq!(
vec!["md-5", "minicbor"],
audit_runtime_lockfile_covered_by_sdk_lockfile(
&runtime_lockfile,
&sdk_dependency_set(),
)
.map(|p| p.name.as_str())
.sorted()
.collect::<Vec<_>>(),
);
}
// New dependencies originating from the AWS runtime crates
{
let runtime_lockfile = Lockfile::from_str(
r#"
[[package]]
name = "aws-credential-types"
version = "1.2.1"
dependencies = [
"zeroize",
]
[[package]]
name = "aws-inlineable"
version = "0.1.0"
dependencies = [
"ahash",
"lru"
]
[[package]]
name = "aws-sigv4"
version = "1.2.3"
dependencies = [
"aws-credential-types",
"p256",
]
[[package]]
name = "ahash"
version = "0.8.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011"
dependencies = [
"zerocopy",
]
[[package]]
name = "hashbrown"
version = "0.14.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
dependencies = [
"ahash",
]
[[package]]
name = "lru"
version = "0.12.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37ee39891760e7d94734f6f63fedc29a2e4a152f836120753a72503f09fcf904"
dependencies = [
"hashbrown",
]
[[package]]
name = "p256"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594"
[[package]]
name = "zerocopy"
version = "0.7.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0"
[[package]]
name = "zeroize"
version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde"
"#,
)
.unwrap();
assert_eq!(
vec!["ahash", "hashbrown", "lru", "p256", "zerocopy"],
audit_runtime_lockfile_covered_by_sdk_lockfile(
&runtime_lockfile,
&sdk_dependency_set(),
)
.map(|p| p.name.as_str())
.sorted()
.collect::<Vec<_>>(),
);
}
}
}

View File

@ -0,0 +1,40 @@
/*
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
* SPDX-License-Identifier: Apache-2.0
*/
use anyhow::Result;
use clap::Parser;
use std::path::PathBuf;
use tracing_subscriber::{filter::LevelFilter, EnvFilter};
mod audit;
#[derive(clap::Args, Clone)]
pub struct AuditArgs {
/// Path to smithy-rs. Defaults to current working directory.
#[arg(long)]
smithy_rs_path: Option<PathBuf>,
}
#[derive(clap::Parser, Clone)]
#[clap(author, version, about)]
enum Command {
Audit(AuditArgs),
}
fn main() -> Result<()> {
tracing_subscriber::fmt()
.with_writer(std::io::stderr)
.with_env_filter(
EnvFilter::builder()
.with_default_directive(LevelFilter::INFO.into())
.from_env_lossy(),
)
.init();
let command = Command::parse();
match command {
Command::Audit(args) => audit::audit(args),
}
}