mirror of https://github.com/smithy-lang/smithy-rs
Enhance gradle tasks for managing lockfiles (#3829)
## Description
This PR introduces and updates gradle tasks for managing lockfiles. Here
are the highlights:
- [The SDK
lockfile](https://github.com/smithy-lang/smithy-rs/blob/main/aws/sdk/Cargo.lock)
can now be generated directly within the `smithy-rs` repository without
the `aws-sdk-rust` repository.
- The SDK lockfile can be synchronized with runtime lockfiles, updating
only new dependencies while preserving the versions of existing ones.
- To prevent updating broken dependencies to the latest versions, we
track the last known good versions and downgrade them to those versions.
New/updated gradle tasks are intended for automation:
- This existing task no longer requires `-Paws-sdk-rust-path`. We plan
to incorporate it into a weekly GitHub Action to automate lockfile
updates:
```
./gradlew aws:sdk:cargoUpdateAllLockfiles
```
- This new task synchronizes the SDK lockfile with runtime lockfiles. We
plan to integrate it into pre-commit hooks:
```
./gradlew aws:sdk:syncAwsSdkLockfile
```
In addition, this PR has updated the SDK lockfile by executing
`./gradlew aws:sdk:syncAwsSdkLockfile`. The updated lockfile no longer
includes many SDK crates that are unused in CI/CD processes. The new SDK
lockfile is in sync with the runtime lockfiles:
```
➜ smithy-rs git:(ysaito/enhance-gradle-tasks-for-lockfile) sdk-lockfiles audit
2024-09-12T16:02:25.193765Z INFO sdk_lockfiles::audit: checking whether `rust-runtime/Cargo.lock` is covered by the SDK lockfile...
2024-09-12T16:02:25.224862Z INFO sdk_lockfiles::audit: checking whether `aws/rust-runtime/Cargo.lock` is covered by the SDK lockfile...
2024-09-12T16:02:25.225389Z INFO sdk_lockfiles::audit: checking whether `aws/rust-runtime/aws-config/Cargo.lock` is covered by the SDK lockfile...
SUCCESS
```
## Testing
I have verified the change against basic use cases:
#### When running `cargoUpdateAllLockfiles`, dependencies will be
updated to their latest versions, while broken crates will be pinned to
the last known good versions.
<details>
<summary> Expand for more details...</summary>
When we execute
```
smithy-rs git:(ysaito/enhance-gradle-tasks-for-lockfile) ✗ ./gradlew aws:sdk:cargoUpdateAllLockfiles
...
BUILD SUCCESSFUL in 1m 7s
```
all lockfiles include the latest versions of dependencies, except for
those that are pinned due to being broken. Currently, minicbor is
[pinned to
0.24.2](7f1d992214/aws/sdk/build.gradle.kts (L503-L504)):
```
➜ smithy-rs git:(ysaito/enhance-gradle-tasks-for-lockfile) ✗ git status
On branch ysaito/enhance-gradle-tasks-for-lockfile
Changes not staged for commit:
(use "git add <file>..." to update what will be committed)
(use "git restore <file>..." to discard changes in working directory)
modified: aws/rust-runtime/Cargo.lock
modified: aws/rust-runtime/aws-config/Cargo.lock
modified: aws/sdk/Cargo.lock
modified: rust-runtime/Cargo.lock
no changes added to commit (use "git add" and/or "git commit -a")
```
```
➜ smithy-rs git:(ysaito/enhance-gradle-tasks-for-lockfile) ✗ cat aws/sdk/Cargo.lock | rg -C1 minicbor
...
---
[[package]]
name = "minicbor"
version = "0.24.2"
---
...
➜ smithy-rs git:(ysaito/enhance-gradle-tasks-for-lockfile) ✗ cat rust-runtime/Cargo.lock | rg -C1 minicbor
...
---
[[package]]
name = "minicbor"
version = "0.24.2"
---
...
```
Finally, the `sdk-lockfiles audit` command should run successfully after
updating all lockfiles:
```
➜ smithy-rs git:(ysaito/enhance-gradle-tasks-for-lockfile) ✗ sdk-lockfiles audit
2024-09-12T15:35:47.890530Z INFO sdk_lockfiles::audit: checking whether `rust-runtime/Cargo.lock` is covered by the SDK lockfile...
2024-09-12T15:35:47.922468Z INFO sdk_lockfiles::audit: checking whether `aws/rust-runtime/Cargo.lock` is covered by the SDK lockfile...
2024-09-12T15:35:47.922898Z INFO sdk_lockfiles::audit: checking whether `aws/rust-runtime/aws-config/Cargo.lock` is covered by the SDK lockfile...
SUCCESS
```
I also specified multiple broken dependencies and verified they were all
downgraded to the specified versions.
</details>
#### When a new dependency is added to a runtime crate, running
`syncAwsSdkLockfile` will ensure that this new dependency is included in
the SDK lockfile.
<details>
<summary> Expand for more details...</summary>
For instance, with [this hypothetical new
dependency](https://github.com/smithy-lang/smithy-rs/pull/3826/files#diff-1ff3734bb74b7c43e3bd74b410f7058c6d40dbe9380458f642201035f9217457):
```
smithy-rs git:(ysaito/enhance-gradle-tasks-for-lockfile) ✗ sdk-lockfiles audit
2024-09-12T15:40:52.795951Z INFO sdk_lockfiles::audit: checking whether `rust-runtime/Cargo.lock` is covered by the SDK lockfile...
2024-09-12T15:40:52.827407Z INFO sdk_lockfiles::audit: checking whether `aws/rust-runtime/Cargo.lock` is covered by the SDK lockfile...
2024-09-12T15:40:52.827835Z INFO sdk_lockfiles::audit: checking whether `aws/rust-runtime/aws-config/Cargo.lock` is covered by the SDK lockfile...
`jiff` (0.1.13), used by `rust-runtime/Cargo.lock`, is not contained in SDK lockfile!
Error: there are lockfile audit failures
```
If we then execute
```
smithy-rs git:(ysaito/enhance-gradle-tasks-for-lockfile) ✗ ./gradlew aws:sdk:syncAwsSdkLockfile
...
BUILD SUCCESSFUL in 1m 17s
```
the SDK lockfile will be updated to reflect only the change from
`rust-runtime/Cargo.lock`:
```
smithy-rs git:(ysaito/enhance-gradle-tasks-for-lockfile) ✗ git diff aws/sdk/Cargo.lock
diff --git a/aws/sdk/Cargo.lock b/aws/sdk/Cargo.lock
index bc3870e20..c52040432 100644
--- a/aws/sdk/Cargo.lock
+++ b/aws/sdk/Cargo.lock
@@ -1627,6 +1627,7 @@ dependencies = [
"aws-smithy-types 1.2.6",
"chrono",
"futures-core",
+ "jiff",
"time",
]
@@ -2895,6 +2896,12 @@ version = "1.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"
+[[package]]
+name = "jiff"
+version = "0.1.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a45489186a6123c128fdf6016183fcfab7113e1820eb813127e036e287233fb"
+
[[package]]
name = "jobserver"
version = "0.1.32"
(END)
```
The updated SDK lockfile should now be in sync with runtime crates:
```
➜ smithy-rs git:(ysaito/enhance-gradle-tasks-for-lockfile) ✗ sdk-lockfiles audit
2024-09-12T15:41:28.004702Z INFO sdk_lockfiles::audit: checking whether `rust-runtime/Cargo.lock` is covered by the SDK lockfile...
2024-09-12T15:41:28.034118Z INFO sdk_lockfiles::audit: checking whether `aws/rust-runtime/Cargo.lock` is covered by the SDK lockfile...
2024-09-12T15:41:28.034555Z INFO sdk_lockfiles::audit: checking whether `aws/rust-runtime/aws-config/Cargo.lock` is covered by the SDK lockfile...
SUCCESS
```
</details>
----
_By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice._
This commit is contained in:
ysaito1001
GitHub
parent
db1a9f19d3
commit
3499f60e1d
File diff suppressed because it is too large
Load Diff
|
|
@ -39,10 +39,12 @@ val sdkVersionerToolPath = rootProject.projectDir.resolve("tools/ci-build/sdk-ve
|
|||
val awsConfigPath = rootProject.projectDir.resolve("aws/rust-runtime/aws-config")
|
||||
val rustRuntimePath = rootProject.projectDir.resolve("rust-runtime")
|
||||
val awsRustRuntimePath = rootProject.projectDir.resolve("aws/rust-runtime")
|
||||
val checkedInCargoLock = rootProject.projectDir.resolve("aws/sdk/Cargo.lock")
|
||||
val awsSdkPath = rootProject.projectDir.resolve("aws/sdk")
|
||||
val outputDir = layout.buildDirectory.dir("aws-sdk").get()
|
||||
val sdkOutputDir = outputDir.dir("sdk")
|
||||
val examplesOutputDir = outputDir.dir("examples")
|
||||
val checkedInSdkLockfile = rootProject.projectDir.resolve("aws/sdk/Cargo.lock")
|
||||
val generatedSdkLockfile = outputDir.file("Cargo.lock")
|
||||
|
||||
|
||||
dependencies {
|
||||
|
|
@ -449,66 +451,152 @@ tasks["assemble"].apply {
|
|||
"hydrateReadme",
|
||||
"relocateChangelog",
|
||||
)
|
||||
finalizedBy("copyCheckedInCargoLock")
|
||||
finalizedBy("copyCheckedInSdkLockfile")
|
||||
outputs.upToDateWhen { false }
|
||||
}
|
||||
|
||||
tasks.register<Copy>("copyCheckedInCargoLock") {
|
||||
description = "Copy the checked in Cargo.lock file back to the build directory"
|
||||
tasks.register<Copy>("copyCheckedInSdkLockfile") {
|
||||
description = "Copy the checked-in SDK lockfile to the build directory"
|
||||
this.outputs.upToDateWhen { false }
|
||||
from(checkedInCargoLock)
|
||||
from(checkedInSdkLockfile)
|
||||
into(outputDir)
|
||||
}
|
||||
|
||||
tasks.register<Copy>("replaceCheckedInSdkLockfile") {
|
||||
description = "Replace the checked-in SDK lockfile by copying the one in the build directory back to `aws/sdk`"
|
||||
dependsOn("copyCheckedInSdkLockfile")
|
||||
dependsOn("downgradeAwsSdkLockfile")
|
||||
this.outputs.upToDateWhen { false }
|
||||
from(generatedSdkLockfile)
|
||||
into(awsSdkPath)
|
||||
}
|
||||
|
||||
project.registerCargoCommandsTasks(outputDir.asFile)
|
||||
project.registerGenerateCargoConfigTomlTask(outputDir.asFile)
|
||||
|
||||
//The task name "test" is already registered by one of our plugins
|
||||
// The task name "test" is already registered by one of our plugins
|
||||
tasks.register("sdkTest") {
|
||||
description = "Run Cargo clippy/test/docs against the generated SDK."
|
||||
dependsOn("assemble")
|
||||
finalizedBy(Cargo.CLIPPY.toString, Cargo.TEST.toString, Cargo.DOCS.toString)
|
||||
}
|
||||
|
||||
//Tasks for generating individual Cargo.lock files
|
||||
fun Project.registerLockfileGeneration(
|
||||
/**
|
||||
* Generate tasks for pinning broken dependencies to bypass compatibility issues
|
||||
*
|
||||
* Some dependencies may have compatibility issues that prevent updating to the latest versions.
|
||||
* In such cases, we pin these dependencies to the last known working versions.
|
||||
*
|
||||
* To update broken dependencies (maybe for CI/CD with the latest versions), run a task with the flag, e.g.,
|
||||
* `./gradlew -Paws.sdk.force.update.broken.dependencies aws:sdk:cargoUpdateAllLockfiles`
|
||||
*/
|
||||
fun Project.registerDowngradeFor(
|
||||
dir: File,
|
||||
name: String,
|
||||
): TaskProvider<Exec> {
|
||||
return tasks.register<Exec>("generate${name}Lockfile") {
|
||||
return tasks.register<Exec>("downgrade${name}Lockfile") {
|
||||
onlyIf {
|
||||
properties["aws.sdk.force.update.broken.dependencies"] == null
|
||||
}
|
||||
executable = "sh" // noop to avoid execCommand == null
|
||||
doLast {
|
||||
val crateNameToLastKnownWorkingVersions =
|
||||
mapOf("minicbor" to "0.24.2")
|
||||
|
||||
crateNameToLastKnownWorkingVersions.forEach { (crate, version) ->
|
||||
// doesn't matter even if the specified crate does not exist in the lockfile
|
||||
exec {
|
||||
workingDir(dir)
|
||||
commandLine("sh", "-c", "cargo update $crate --precise $version || true")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
val downgradeAwsConfigLockfile = registerDowngradeFor(awsConfigPath, "AwsConfig")
|
||||
val downgradeAwsRuntimeLockfile = registerDowngradeFor(awsRustRuntimePath, "AwsRustRuntime")
|
||||
val downgradeSmithyRuntimeLockfile = registerDowngradeFor(rustRuntimePath, "RustRuntime")
|
||||
val downgradeAwsSdkLockfile = registerDowngradeFor(outputDir.asFile, "AwsSdk")
|
||||
|
||||
// Tasks for updating individual Cargo.lock files
|
||||
fun Project.registerCargoUpdateFor(
|
||||
dir: File,
|
||||
name: String,
|
||||
): TaskProvider<Exec> {
|
||||
return tasks.register<Exec>("cargoUpdate${name}Lockfile") {
|
||||
workingDir(dir)
|
||||
environment("RUSTFLAGS", "--cfg aws_sdk_unstable")
|
||||
commandLine("cargo", "generate-lockfile")
|
||||
commandLine("cargo", "update")
|
||||
finalizedBy("downgrade${name}Lockfile")
|
||||
}
|
||||
}
|
||||
|
||||
val generateAwsConfigLockfile = registerLockfileGeneration(awsConfigPath, "AwsConfig")
|
||||
val generateAwsRuntimeLockfile = registerLockfileGeneration(awsRustRuntimePath, "AwsRustRuntime")
|
||||
val generateSmithytRuntimeLockfile = registerLockfileGeneration(rustRuntimePath, "RustRuntime")
|
||||
val cargoUpdateAwsConfigLockfile = registerCargoUpdateFor(awsConfigPath, "AwsConfig")
|
||||
val cargoUpdateAwsRuntimeLockfile = registerCargoUpdateFor(awsRustRuntimePath, "AwsRustRuntime")
|
||||
val cargoUpdateSmithyRuntimeLockfile = registerCargoUpdateFor(rustRuntimePath, "RustRuntime")
|
||||
|
||||
//Generates a lockfile from the aws-sdk-rust repo and copies it into the smithy-rs repo
|
||||
val generateAwsSdkRustLockfile = tasks.register<Exec>("generateAwsSdkRustLockfile") {
|
||||
val sdkRustPath: String =
|
||||
properties.get("aws-sdk-rust-path") ?: throw Exception("A -Paws-sdk-rust-path argument must be specified")
|
||||
workingDir(sdkRustPath)
|
||||
/**
|
||||
* Updates the lockfile located in the `aws/sdk` directory.
|
||||
*
|
||||
* Previously, we would run `cargo generate-lockfile` in the `aws-sdk-rust` repository and then copy the resulting
|
||||
* `Cargo.lock` into the `smithy-rs` repository. This approach introduced a delay, as new dependencies added to runtime
|
||||
* crates would not be reflected in the SDK lockfile until the runtime crates were released to the `aws-sdk-rust`
|
||||
* repository.
|
||||
*
|
||||
* We now generate a lockfile directly in `aws/sdk/build/aws-sdk`, which suffices for our CI/CD purposes, as it covers
|
||||
* the crate dependencies used by the SDK:
|
||||
* - Smithy runtime crates and inlineables
|
||||
* - Smithy codegen decorators
|
||||
* - Aws runtime crates and inlineables
|
||||
* - Aws SDK codegen decorators
|
||||
* - Service customizations (as long as we have their models in `aws/sdk/aws-models`)
|
||||
*/
|
||||
val cargoUpdateAwsSdkLockfile = tasks.register<Exec>("cargoUpdateAwsSdkLockfile") {
|
||||
dependsOn("assemble")
|
||||
workingDir(outputDir)
|
||||
environment("RUSTFLAGS", "--cfg aws_sdk_unstable")
|
||||
commandLine("cargo", "generate-lockfile")
|
||||
copy {
|
||||
from("${sdkRustPath}/Cargo.lock")
|
||||
into(rootProject.projectDir.resolve("aws/sdk"))
|
||||
commandLine("cargo", "update")
|
||||
finalizedBy(
|
||||
"downgradeAwsSdkLockfile",
|
||||
"replaceCheckedInSdkLockfile",
|
||||
)
|
||||
}
|
||||
|
||||
tasks.register<Exec>("syncAwsSdkLockfile") {
|
||||
description = """
|
||||
Synchronize the SDK lockfile to ensure that it includes all dependencies specified in runtime lockfiles.
|
||||
"""
|
||||
dependsOn("assemble")
|
||||
workingDir(outputDir)
|
||||
environment("RUSTFLAGS", "--cfg aws_sdk_unstable")
|
||||
// Using `cargo generate-lockfile` or `cargo update` is not suitable here, as they update dependencies to their
|
||||
// latest versions. Instead, we need to preserve the existing dependencies in the SDK lockfile while incorporating
|
||||
// new dependencies introduced by runtime crates. This can be achieved by running `cargo check` with the lockfile
|
||||
// copied to the `aws/sdk/build/aws-sdk` directory.
|
||||
commandLine("cargo", "check", "--all-features")
|
||||
doLast {
|
||||
// We avoid using `replaceCheckedInSdkLockfile` in favor of `copy` to prevent dependency on
|
||||
// `downgradeAwsSdkLockfile`. Downgrading dependencies is unnecessary when synchronizing the SDK lockfile with
|
||||
// runtime lockfiles.
|
||||
copy {
|
||||
from(generatedSdkLockfile)
|
||||
into(awsSdkPath)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//Parent task to generate all the Cargo.lock files
|
||||
tasks.register("generateAllLockfiles") {
|
||||
description =
|
||||
"Create Cargo.lock files for aws-config, aws/rust-runtime, rust-runtime, and the workspace created by" +
|
||||
"the assemble task."
|
||||
// Parent task to update all the Cargo.lock files
|
||||
tasks.register("cargoUpdateAllLockfiles") {
|
||||
description = """
|
||||
Update Cargo.lock files for aws-config, aws/rust-runtime, rust-runtime, and the workspace created by the
|
||||
assemble task.
|
||||
"""
|
||||
finalizedBy(
|
||||
generateAwsSdkRustLockfile,
|
||||
generateAwsConfigLockfile,
|
||||
generateAwsRuntimeLockfile,
|
||||
generateSmithytRuntimeLockfile,
|
||||
cargoUpdateAwsSdkLockfile,
|
||||
cargoUpdateAwsConfigLockfile,
|
||||
cargoUpdateAwsRuntimeLockfile,
|
||||
cargoUpdateSmithyRuntimeLockfile,
|
||||
)
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue