Enable independent crate versioning for SDK crates (#1540)

* Update `hydrate-readmes` to take versions from `versions.toml`
* Add cleanup TODO comments to `changelogger`
* Update SDK readme template
* Split up `generate-version-manifest` subcommand
* Eliminate the `aws.sdk.version` property
* Fallback to model hash comparison if model metadata claims no changes
* Add `acquire-base-image` to `release.yml`
* Use empty model metadata for SDK generation in CI
* Fix the `aws-config` version number in SDK crate readmes
This commit is contained in:
John DiSanti 2022-07-19 12:27:45 -07:00 committed by GitHub
parent 298748a606
commit 6143d90197
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
29 changed files with 734 additions and 179 deletions

View File

@ -23,13 +23,38 @@ on:
default: true
jobs:
# If a release is kicked off before an image is built after push to main,
# or if a dry-run release is kicked off against a non-main branch to test
# automation changes, we'll need to build a base image to work against.
# This job will be a no-op if an image was already built on main.
acquire-base-image:
name: Acquire Base Image
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
path: smithy-rs
fetch-depth: 0
- name: Acquire base image
id: acquire
run: ./smithy-rs/tools/ci-build/acquire-build-image
- name: Upload base image
uses: actions/upload-artifact@v3
with:
name: smithy-rs-base-image
path: smithy-rs-base-image
retention-days: 1
release-ci:
name: Prerelease checks
needs:
- acquire-base-image
uses: ./.github/workflows/ci.yml
release:
name: Release
needs:
- acquire-base-image
- release-ci
runs-on: ubuntu-latest
steps:

View File

@ -104,3 +104,15 @@ async fn main() -> Result<(), aws_sdk_dynamodb::Error> {
references = ["aws-sdk-rust#567"]
meta = { "breaking" = false, "tada" = true, "bug" = false }
author = "Velfi"
[[aws-sdk-rust]]
message = """
Until now, SDK crates have all shared the exact same version numbers.
This changes with this release. From now on, SDK crates will only version
bump if they have changes. Coincidentally, they may share the same version
number for some releases since changes to the code generator will cause
a version bump in all of them, but this should not be relied upon.
"""
references = ["smithy-rs#1540"]
meta = { "breaking" = false, "tada" = false, "bug" = false }
author = "jdisanti"

View File

@ -3,7 +3,7 @@ This is the README Handlebars template for `aws-sdk-rust`.
It gets instantiated and copied into the build artifacts by the `aws:sdk:assemble` build target.
Available template arguments:
- `{{sdk_version}}`: the SDK version number (just the number, no `v` prefix)
- `{{sdk_version_<crate_name_in_snake_case>}}` (e.g., `{{sdk_version_aws_config}}` for the `aws-config` crate): the version number of the given crate (just the number, no `v` prefix)
- `{{msrv}}`: The MSRV Rust compiler version (just the number, no `v` prefix)
--}}
<!--
@ -12,9 +12,9 @@ This README file is auto-generated by the build system in awslabs/smithy-rs.
To update it, edit the `aws/SDK_README.md.hb` Handlebars template in that repository.
-->
# The AWS SDK for Rust [![Docs](https://img.shields.io/badge/docs-v{{sdk_version}}-blue)](https://awslabs.github.io/aws-sdk-rust/) ![MSRV](https://img.shields.io/badge/msrv-{{msrv}}-red) [![Usage Guide](https://img.shields.io/badge/Developer_Guide-blue)](https://docs.aws.amazon.com/sdk-for-rust/latest/dg/welcome.html)
# The AWS SDK for Rust [![Docs](https://img.shields.io/badge/docs-blue)](https://awslabs.github.io/aws-sdk-rust/) ![MSRV](https://img.shields.io/badge/msrv-{{msrv}}-red) [![Usage Guide](https://img.shields.io/badge/Developer_Guide-blue)](https://docs.aws.amazon.com/sdk-for-rust/latest/dg/welcome.html)
This repo contains the new AWS SDK for Rust (the SDK) and its [public roadmap](https://github.com/awslabs/aws-sdk-rust/projects/1).
This repo contains the new AWS SDK for Rust (the SDK) and its [public roadmap](https://github.com/orgs/awslabs/projects/50/views/1).
**Please Note: The SDK is currently released as a developer preview and is intended strictly for feedback purposes only. Do not use this SDK for production workloads.**
@ -22,7 +22,7 @@ The SDK is code generated from [Smithy models](https://awslabs.github.io/smithy/
## Getting Started with the SDK
> Examples are availble for many services and operations, check out the [examples folder](./examples).
> Examples are available for many services and operations, check out the [examples folder](./examples).
> For a step-by-step guide including several advanced use cases, check out the [Developer Guide](https://docs.aws.amazon.com/sdk-for-rust/latest/dg/welcome.html).
@ -33,8 +33,8 @@ The SDK provides one crate per AWS service. You must add [Tokio](https://crates.
```toml
[dependencies]
aws-config = "{{sdk_version}}"
aws-sdk-dynamodb = "{{sdk_version}}"
aws-config = "{{sdk_version_aws_config}}"
aws-sdk-dynamodb = "{{sdk_version_aws_sdk_dynamodb}}"
tokio = { version = "1", features = ["full"] }
```

View File

@ -31,13 +31,13 @@ dependencies {
testImplementation("io.kotest:kotest-assertions-core-jvm:$kotestVersion")
}
val generateAwsSdkVersion by tasks.registering {
val generateAwsRuntimeCrateVersion by tasks.registering {
// generate the version of the runtime to use as a resource.
// this keeps us from having to manually change version numbers in multiple places
val resourcesDir = "$buildDir/resources/main/software/amazon/smithy/rustsdk"
val versionFile = file("$resourcesDir/sdk-crate-version.txt")
outputs.file(versionFile)
val crateVersion = project.properties["aws.sdk.version"]?.toString()!!
val crateVersion = project.properties["smithy.rs.runtime.crate.version"]?.toString()!!
inputs.property("crateVersion", crateVersion)
sourceSets.main.get().output.dir(resourcesDir)
doLast {
@ -47,7 +47,7 @@ val generateAwsSdkVersion by tasks.registering {
tasks.compileKotlin {
kotlinOptions.jvmTarget = "1.8"
dependsOn(generateAwsSdkVersion)
dependsOn(generateAwsRuntimeCrateVersion)
}
tasks.compileTestKotlin {

View File

@ -35,6 +35,7 @@ class AwsReadmeDecorator : RustCodegenDecorator<ClientCodegenContext> {
mapOf("package" to mapOf("readme" to "README.md"))
override fun extras(codegenContext: ClientCodegenContext, rustCrate: RustCrate) {
val awsConfigVersion = SdkSettings.from(codegenContext.settings).awsConfigVersion
rustCrate.withFile("README.md") { writer ->
val description = normalizeDescription(
codegenContext.moduleName,
@ -62,7 +63,7 @@ class AwsReadmeDecorator : RustCodegenDecorator<ClientCodegenContext> {
```toml
[dependencies]
aws-config = "${codegenContext.settings.moduleVersion}"
aws-config = "$awsConfigVersion"
$moduleName = "${codegenContext.settings.moduleVersion}"
tokio = { version = "1", features = ["full"] }
```

View File

@ -36,4 +36,9 @@ class SdkSettings private constructor(private val awsSdk: ObjectNode?) {
/** Path to AWS SDK integration tests */
val integrationTestPath: String get() =
awsSdk?.getStringMember("integrationTestPath")?.orNull()?.value ?: "aws/sdk/integration-tests"
/** Version number of the `aws-config` crate */
val awsConfigVersion: String get() =
awsSdk?.getStringMember("awsConfigVersion")?.orNull()?.value
?: throw IllegalStateException("missing `awsConfigVersion` codegen setting")
}

View File

@ -55,7 +55,6 @@ val awsServices: AwsServices by lazy { discoverServices(properties.get("aws.sdk.
val eventStreamAllowList: Set<String> by lazy { eventStreamAllowList() }
val crateVersioner by lazy { aws.sdk.CrateVersioner.defaultFor(rootProject, properties) }
fun getSdkVersion(): String = properties.get("aws.sdk.version") ?: throw Exception("SDK version missing")
fun getRustMSRV(): String = properties.get("rust.msrv") ?: throw Exception("Rust MSRV missing")
fun getPreviousReleaseVersionManifestPath(): String? = properties.get("aws.sdk.previous.release.versions.manifest")
@ -73,8 +72,10 @@ fun eventStreamAllowList(): Set<String> {
}
fun generateSmithyBuild(services: AwsServices): String {
val awsConfigVersion = properties.get("smithy.rs.runtime.crate.version")
?: throw IllegalStateException("missing smithy.rs.runtime.crate.version for aws-config version")
val serviceProjections = services.services.map { service ->
val files = service.files().map { extraFile ->
val files = service.modelFiles().map { extraFile ->
software.amazon.smithy.utils.StringUtils.escapeJavaString(
extraFile.absolutePath,
""
@ -99,7 +100,7 @@ fun generateSmithyBuild(services: AwsServices): String {
},
"service": "${service.service}",
"module": "$moduleName",
"moduleVersion": "${crateVersioner.decideCrateVersion(moduleName)}",
"moduleVersion": "${crateVersioner.decideCrateVersion(moduleName, service)}",
"moduleAuthors": ["AWS Rust SDK Team <aws-sdk-rust@amazon.com>", "Russell Cohen <rcoh@amazon.com>"],
"moduleDescription": "${service.moduleDescription}",
${service.examplesUri(project)?.let { """"examples": "$it",""" } ?: ""}
@ -107,6 +108,7 @@ fun generateSmithyBuild(services: AwsServices): String {
"license": "Apache-2.0",
"customizationConfig": {
"awsSdk": {
"awsConfigVersion": "$awsConfigVersion",
"defaultConfigPath": "${services.defaultConfigPath}",
"endpointsConfigPath": "${services.endpointsConfigPath}",
"integrationTestPath": "${project.projectDir.resolve("integration-tests")}"
@ -242,7 +244,7 @@ tasks.register("relocateAwsRuntime") {
// Patch the Cargo.toml files
CrateSet.AWS_SDK_RUNTIME.forEach { moduleName ->
patchFile(sdkOutputDir.resolve("$moduleName/Cargo.toml")) { line ->
rewriteAwsSdkCrateVersion(properties, line.let(::rewritePathDependency))
rewriteRuntimeCrateVersion(properties, line.let(::rewritePathDependency))
}
}
}
@ -253,7 +255,7 @@ tasks.register("relocateRuntime") {
// Patch the Cargo.toml files
CrateSet.AWS_SDK_SMITHY_RUNTIME.forEach { moduleName ->
patchFile(sdkOutputDir.resolve("$moduleName/Cargo.toml")) { line ->
rewriteSmithyRsCrateVersion(properties, line)
rewriteRuntimeCrateVersion(properties, line)
}
}
}
@ -312,15 +314,19 @@ tasks.register<ExecRustBuildTool>("fixManifests") {
tasks.register<ExecRustBuildTool>("hydrateReadme") {
description = "Run the publisher tool's `hydrate-readme` sub-command to create the final AWS Rust SDK README file"
dependsOn("generateVersionManifest")
inputs.dir(publisherToolPath)
outputs.dir(outputDir)
inputs.file(rootProject.projectDir.resolve("aws/SDK_README.md.hb"))
outputs.file(outputDir.resolve("README.md").absolutePath)
toolPath = publisherToolPath
binaryName = "publisher"
arguments = listOf(
"hydrate-readme",
"--sdk-version", getSdkVersion(),
"--versions-manifest", outputDir.resolve("versions.toml").toString(),
"--msrv", getRustMSRV(),
"--input", rootProject.projectDir.resolve("aws/SDK_README.md.hb").toString(),
"--output", outputDir.resolve("README.md").absolutePath
)
}
@ -352,8 +358,6 @@ tasks.register<ExecRustBuildTool>("generateVersionManifest") {
val previousReleaseManifestPath = getPreviousReleaseVersionManifestPath()?.let { manifestPath ->
add("--previous-release-versions")
add(manifestPath)
add("--release-tag")
add("v" + getSdkVersion())
}
}
}

View File

@ -10,18 +10,11 @@ fun rewriteCrateVersion(line: String, version: String): String = line.replace(
"version = \"$version\""
)
/**
* AWS runtime crate versions are all `0.0.0-smithy-rs-head`. When copying over to the AWS SDK,
* these should be changed to the AWS SDK version.
*/
fun rewriteAwsSdkCrateVersion(properties: PropertyRetriever, line: String): String =
rewriteCrateVersion(line, properties.get("aws.sdk.version")!!)
/**
* Smithy runtime crate versions in smithy-rs are all `0.0.0-smithy-rs-head`. When copying over to the AWS SDK,
* these should be changed to the smithy-rs version.
*/
fun rewriteSmithyRsCrateVersion(properties: PropertyRetriever, line: String): String =
fun rewriteRuntimeCrateVersion(properties: PropertyRetriever, line: String): String =
rewriteCrateVersion(line, properties.get("smithy.rs.runtime.crate.version")!!)
/** Patches a file with the result of the given `operation` being run on each line */

View File

@ -8,54 +8,45 @@ package aws.sdk
import PropertyRetriever
import org.gradle.api.Project
import org.slf4j.LoggerFactory
import java.io.File
import java.security.MessageDigest
const val LOCAL_DEV_VERSION: String = "0.0.0-local"
// Example command for generating with independent versions:
// ```
// ./gradlew --no-daemon \
// -Paws.sdk.independent.versions=true \
// -Paws.sdk.model.metadata=$HOME/model-metadata.toml \
// -Paws.sdk.previous.release.versions.manifest=$HOME/versions.toml \
// aws:sdk:assemble
// ```
object CrateVersioner {
fun defaultFor(rootProject: Project, properties: PropertyRetriever): VersionCrate =
// Putting independent crate versioning behind a feature flag for now
when (properties.get("aws.sdk.independent.versions")) {
"true" -> when (val versionsManifestPath = properties.get("aws.sdk.previous.release.versions.manifest")) {
// In local dev, use special `0.0.0-local` version number for all SDK crates
null -> SynchronizedCrateVersioner(properties, sdkVersion = LOCAL_DEV_VERSION)
else -> {
val modelMetadataPath = properties.get("aws.sdk.model.metadata")
?: throw IllegalArgumentException("Property `aws.sdk.model.metadata` required for independent crate version builds")
IndependentCrateVersioner(
VersionsManifest.fromFile(versionsManifestPath),
ModelMetadata.fromFile(modelMetadataPath),
devPreview = true,
smithyRsVersion = getSmithyRsVersion(rootProject)
)
}
when (val versionsManifestPath = properties.get("aws.sdk.previous.release.versions.manifest")) {
// In local dev, use special `0.0.0-local` version number for all SDK crates
null -> SynchronizedCrateVersioner(properties, sdkVersion = LOCAL_DEV_VERSION)
else -> {
val modelMetadataPath = properties.get("aws.sdk.model.metadata")
?: throw IllegalArgumentException("Property `aws.sdk.model.metadata` required for independent crate version builds")
IndependentCrateVersioner(
VersionsManifest.fromFile(versionsManifestPath),
ModelMetadata.fromFile(modelMetadataPath),
devPreview = true,
smithyRsVersion = getSmithyRsVersion(rootProject)
)
}
else -> SynchronizedCrateVersioner(properties)
}
}
interface VersionCrate {
fun decideCrateVersion(moduleName: String): String
fun decideCrateVersion(moduleName: String, service: AwsService): String
fun independentVersioningEnabled(): Boolean
}
class SynchronizedCrateVersioner(
properties: PropertyRetriever,
private val sdkVersion: String = properties.get("aws.sdk.version") ?: throw Exception("SDK version missing")
private val sdkVersion: String = properties.get("smithy.rs.runtime.crate.version")
?: throw Exception("SDK runtime crate version missing")
) : VersionCrate {
init {
LoggerFactory.getLogger(javaClass).info("Using synchronized SDK crate versioning with version `$sdkVersion`")
}
override fun decideCrateVersion(moduleName: String): String = sdkVersion
override fun decideCrateVersion(moduleName: String, service: AwsService): String = sdkVersion
override fun independentVersioningEnabled(): Boolean = sdkVersion == LOCAL_DEV_VERSION
}
@ -111,7 +102,8 @@ class IndependentCrateVersioner(
private val versionsManifest: VersionsManifest,
private val modelMetadata: ModelMetadata,
private val devPreview: Boolean,
smithyRsVersion: String
smithyRsVersion: String,
private val hashModelsFn: (AwsService) -> String = { service -> hashModels(service) }
) : VersionCrate {
private val smithyRsChanged = versionsManifest.smithyRsRevision != smithyRsVersion
private val logger = LoggerFactory.getLogger(javaClass)
@ -127,7 +119,7 @@ class IndependentCrateVersioner(
override fun independentVersioningEnabled(): Boolean = true
override fun decideCrateVersion(moduleName: String): String {
override fun decideCrateVersion(moduleName: String, service: AwsService): String {
var previousVersion: SemVer? = null
val (reason, newVersion) = when (val existingCrate = versionsManifest.crates.get(moduleName)) {
// The crate didn't exist before, so create a new major version
@ -138,9 +130,17 @@ class IndependentCrateVersioner(
"smithy-rs changed" to previousVersion.bumpCodegenChanged()
} else {
when (modelMetadata.changeType(moduleName)) {
ChangeType.UNCHANGED -> "no change" to previousVersion
ChangeType.FEATURE -> "its API changed" to previousVersion.bumpModelChanged()
ChangeType.DOCUMENTATION -> "it has new docs" to previousVersion.bumpDocsChanged()
ChangeType.UNCHANGED -> {
val currentModelsHash = hashModelsFn(service)
val previousModelsHash = existingCrate.modelHash
if (currentModelsHash != previousModelsHash) {
"its model(s) changed" to previousVersion.bumpModelChanged()
} else {
"no change" to previousVersion
}
}
}
}
}
@ -165,5 +165,18 @@ class IndependentCrateVersioner(
true -> bumpPatch()
else -> bumpMinor()
}
private fun SemVer.bumpDocsChanged(): SemVer = bumpPatch()
}
private fun ByteArray.toLowerHex(): String = joinToString("") { byte -> "%02x".format(byte) }
fun hashModels(awsService: AwsService, loadFile: (File) -> ByteArray = File::readBytes): String {
// Needs to match hashing done in the `generate-version-manifest` tool:
val sha256 = MessageDigest.getInstance("SHA-256")
val hashes = awsService.modelFiles().fold("") { hashes, file ->
val fileHash = sha256.digest(loadFile(file)).toLowerHex()
hashes + fileHash + "\n"
}
return sha256.digest(hashes.toByteArray(Charsets.UTF_8)).toLowerHex()
}

View File

@ -96,7 +96,8 @@ fun Project.discoverServices(awsModelsPath: String?, serviceMembership: Membersh
module = sdkId,
moduleDescription = "AWS SDK for $title",
modelFile = file,
extraFiles = extras,
// Order is important for the versions.toml model hash calculation
extraFiles = extras.sorted(),
humanName = title
)
}
@ -145,7 +146,7 @@ data class AwsService(
val extraFiles: List<File>,
val humanName: String
) {
fun files(): List<File> = listOf(modelFile) + extraFiles
fun modelFiles(): List<File> = listOf(modelFile) + extraFiles
fun Project.examples(): File = projectDir.resolve("examples").resolve(module)
/**
* Generate a link to the examples for a given service

View File

@ -7,10 +7,26 @@ package aws.sdk
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Test
import java.io.File
private fun service(name: String): AwsService = AwsService(
name,
"test",
"test",
File("testmodel"),
null,
emptyList(),
name
)
class IndependentCrateVersionerTest {
@Test
fun devPreviewSmithyRsChanged() {
val dynamoDb = service("dynamodb")
val ec2 = service("ec2")
val s3 = service("s3")
val someNewService = service("somenewservice")
val versioner = IndependentCrateVersioner(
VersionsManifest(
smithyRsRevision = "smithy-rs-1",
@ -18,15 +34,18 @@ class IndependentCrateVersionerTest {
crates = mapOf(
"aws-sdk-dynamodb" to CrateVersion(
category = "AwsSdk",
version = "0.11.3"
version = "0.11.3",
modelHash = "dynamodb-hash",
),
"aws-sdk-ec2" to CrateVersion(
category = "AwsSdk",
version = "0.10.1"
version = "0.10.1",
modelHash = "ec2-hash",
),
"aws-sdk-s3" to CrateVersion(
category = "AwsSdk",
version = "0.12.0"
version = "0.12.0",
modelHash = "s3-hash",
)
)
),
@ -37,18 +56,32 @@ class IndependentCrateVersionerTest {
)
),
devPreview = true,
smithyRsVersion = "smithy-rs-2"
smithyRsVersion = "smithy-rs-2",
hashModelsFn = { service ->
when (service) {
dynamoDb -> "dynamodb-hash"
ec2 -> "ec2-hash"
s3 -> "s3-hash"
else -> throw IllegalStateException("unreachable")
}
}
)
// The code generator changed, so all minor versions should bump
assertEquals("0.12.0", versioner.decideCrateVersion("aws-sdk-dynamodb"))
assertEquals("0.11.0", versioner.decideCrateVersion("aws-sdk-ec2"))
assertEquals("0.13.0", versioner.decideCrateVersion("aws-sdk-s3"))
assertEquals("0.1.0", versioner.decideCrateVersion("aws-sdk-somenewservice"))
assertEquals("0.12.0", versioner.decideCrateVersion("aws-sdk-dynamodb", dynamoDb))
assertEquals("0.11.0", versioner.decideCrateVersion("aws-sdk-ec2", ec2))
assertEquals("0.13.0", versioner.decideCrateVersion("aws-sdk-s3", s3))
assertEquals("0.1.0", versioner.decideCrateVersion("aws-sdk-somenewservice", someNewService))
}
@Test
fun devPreviewSameCodeGenerator() {
val dynamoDb = service("dynamodb")
val ec2 = service("ec2")
val polly = service("polly")
val s3 = service("s3")
val someNewService = service("somenewservice")
val versioner = IndependentCrateVersioner(
VersionsManifest(
smithyRsRevision = "smithy-rs-1",
@ -56,15 +89,23 @@ class IndependentCrateVersionerTest {
crates = mapOf(
"aws-sdk-dynamodb" to CrateVersion(
category = "AwsSdk",
version = "0.11.3"
version = "0.11.3",
modelHash = "dynamodb-hash",
),
"aws-sdk-ec2" to CrateVersion(
category = "AwsSdk",
version = "0.10.1"
version = "0.10.1",
modelHash = "ec2-hash",
),
"aws-sdk-polly" to CrateVersion(
category = "AwsSdk",
version = "0.9.0",
modelHash = "old-polly-hash",
),
"aws-sdk-s3" to CrateVersion(
category = "AwsSdk",
version = "0.12.0"
version = "0.12.0",
modelHash = "s3-hash",
)
)
),
@ -72,20 +113,36 @@ class IndependentCrateVersionerTest {
crates = mapOf(
"aws-sdk-dynamodb" to ChangeType.FEATURE,
"aws-sdk-ec2" to ChangeType.DOCUMENTATION
// polly has a model change, but is absent from the model metadata file
)
),
devPreview = true,
smithyRsVersion = "smithy-rs-1"
smithyRsVersion = "smithy-rs-1",
hashModelsFn = { service ->
when (service) {
dynamoDb -> "dynamodb-hash"
ec2 -> "ec2-hash"
polly -> "NEW-polly-hash"
s3 -> "s3-hash"
else -> throw IllegalStateException("unreachable")
}
}
)
assertEquals("0.11.4", versioner.decideCrateVersion("aws-sdk-dynamodb"))
assertEquals("0.10.2", versioner.decideCrateVersion("aws-sdk-ec2"))
assertEquals("0.12.0", versioner.decideCrateVersion("aws-sdk-s3"))
assertEquals("0.1.0", versioner.decideCrateVersion("aws-sdk-somenewservice"))
assertEquals("0.11.4", versioner.decideCrateVersion("aws-sdk-dynamodb", dynamoDb))
assertEquals("0.10.2", versioner.decideCrateVersion("aws-sdk-ec2", ec2))
assertEquals("0.12.0", versioner.decideCrateVersion("aws-sdk-s3", s3))
assertEquals("0.9.1", versioner.decideCrateVersion("aws-sdk-polly", polly))
assertEquals("0.1.0", versioner.decideCrateVersion("aws-sdk-somenewservice", someNewService))
}
@Test
fun smithyRsChanged() {
val dynamoDb = service("dynamodb")
val ec2 = service("ec2")
val s3 = service("s3")
val someNewService = service("somenewservice")
val versioner = IndependentCrateVersioner(
VersionsManifest(
smithyRsRevision = "smithy-rs-1",
@ -116,14 +173,20 @@ class IndependentCrateVersionerTest {
)
// The code generator changed, so all minor versions should bump
assertEquals("1.12.0", versioner.decideCrateVersion("aws-sdk-dynamodb"))
assertEquals("1.11.0", versioner.decideCrateVersion("aws-sdk-ec2"))
assertEquals("1.13.0", versioner.decideCrateVersion("aws-sdk-s3"))
assertEquals("1.0.0", versioner.decideCrateVersion("aws-sdk-somenewservice"))
assertEquals("1.12.0", versioner.decideCrateVersion("aws-sdk-dynamodb", dynamoDb))
assertEquals("1.11.0", versioner.decideCrateVersion("aws-sdk-ec2", ec2))
assertEquals("1.13.0", versioner.decideCrateVersion("aws-sdk-s3", s3))
assertEquals("1.0.0", versioner.decideCrateVersion("aws-sdk-somenewservice", someNewService))
}
@Test
fun sameCodeGenerator() {
val dynamoDb = service("dynamodb")
val ec2 = service("ec2")
val polly = service("polly")
val s3 = service("s3")
val someNewService = service("somenewservice")
val versioner = IndependentCrateVersioner(
VersionsManifest(
smithyRsRevision = "smithy-rs-1",
@ -131,31 +194,68 @@ class IndependentCrateVersionerTest {
crates = mapOf(
"aws-sdk-dynamodb" to CrateVersion(
category = "AwsSdk",
version = "1.11.3"
version = "1.11.3",
modelHash = "dynamodb-hash",
),
"aws-sdk-ec2" to CrateVersion(
category = "AwsSdk",
version = "1.10.1"
version = "1.10.1",
modelHash = "ec2-hash",
),
"aws-sdk-polly" to CrateVersion(
category = "AwsSdk",
version = "1.9.0",
modelHash = "old-polly-hash",
),
"aws-sdk-s3" to CrateVersion(
category = "AwsSdk",
version = "1.12.0"
version = "1.12.0",
modelHash = "s3-hash",
)
)
),
ModelMetadata(
crates = mapOf(
"aws-sdk-dynamodb" to ChangeType.FEATURE,
"aws-sdk-ec2" to ChangeType.DOCUMENTATION
"aws-sdk-ec2" to ChangeType.DOCUMENTATION,
// polly has a model change, but is absent from the model metadata file
)
),
devPreview = false,
smithyRsVersion = "smithy-rs-1"
smithyRsVersion = "smithy-rs-1",
hashModelsFn = { service ->
when (service) {
dynamoDb -> "dynamodb-hash"
ec2 -> "ec2-hash"
polly -> "NEW-polly-hash"
s3 -> "s3-hash"
else -> throw IllegalStateException("unreachable")
}
}
)
assertEquals("1.12.0", versioner.decideCrateVersion("aws-sdk-dynamodb"))
assertEquals("1.10.2", versioner.decideCrateVersion("aws-sdk-ec2"))
assertEquals("1.12.0", versioner.decideCrateVersion("aws-sdk-s3"))
assertEquals("1.0.0", versioner.decideCrateVersion("aws-sdk-somenewservice"))
assertEquals("1.12.0", versioner.decideCrateVersion("aws-sdk-dynamodb", dynamoDb))
assertEquals("1.10.2", versioner.decideCrateVersion("aws-sdk-ec2", ec2))
assertEquals("1.10.0", versioner.decideCrateVersion("aws-sdk-polly", s3))
assertEquals("1.12.0", versioner.decideCrateVersion("aws-sdk-s3", s3))
assertEquals("1.0.0", versioner.decideCrateVersion("aws-sdk-somenewservice", someNewService))
}
}
class HashModelsTest {
@Test
fun testHashModels() {
val service = service("test").copy(
modelFile = File("model1a"),
extraFiles = listOf(File("model1b")),
)
val hash = hashModels(service) { file ->
when (file.toString()) {
"model1a" -> "foo".toByteArray(Charsets.UTF_8)
"model1b" -> "bar".toByteArray(Charsets.UTF_8)
else -> throw IllegalStateException("unreachable")
}
}
assertEquals("964021077fb6c3d42ae162ab2e2255be64c6d96a6d77bca089569774d54ef69b", hash)
}
}

View File

@ -8,10 +8,6 @@ rust.msrv=1.58.1
org.gradle.jvmargs=-Xmx1024M
# Version number to use for the generated SDK
# Note: these must always be full 3-segment semver versions
aws.sdk.version=0.15.0
# Version number to use for the generated runtime crates
smithy.rs.runtime.crate.version=0.45.0

View File

@ -45,7 +45,7 @@ tasks.register("fixRuntimeCrateVersions") {
doLast {
CrateSet.ENTIRE_SMITHY_RUNTIME.forEach { moduleName ->
patchFile(runtimeOutputDir.resolve("$moduleName/Cargo.toml")) { line ->
rewriteSmithyRsCrateVersion(properties, line)
rewriteRuntimeCrateVersion(properties, line)
}
}
}

View File

@ -47,6 +47,7 @@ pub struct RenderArgs {
/// Which set of changes to render
#[clap(long, action)]
pub change_set: ChangeSet,
// TODO(https://github.com/awslabs/smithy-rs/issues/1531): Require this arg to be `true`
/// Whether or not independent crate versions are being used (defaults to false)
#[clap(long, action)]
pub independent_versioning: bool,
@ -94,6 +95,7 @@ pub fn subcommand_render(args: &RenderArgs) -> Result<()> {
let sdk_metadata = date_based_release_metadata(now, "aws-sdk-rust-release-manifest.json");
update_changelogs(args, &smithy_rs, &smithy_rs_metadata, &sdk_metadata)
} else {
// TODO(https://github.com/awslabs/smithy-rs/issues/1531): Remove this code path entirely
let auto = auto_changelog_meta(&smithy_rs)?;
let smithy_rs_metadata = version_based_release_metadata(
now,
@ -170,6 +172,7 @@ fn date_title(now: &OffsetDateTime) -> String {
)
}
// TODO(https://github.com/awslabs/smithy-rs/issues/1531): Remove this function
/// Discover the new version for the changelog from gradle.properties and the date.
fn auto_changelog_meta(smithy_rs: &dyn Git) -> Result<ChangelogMeta> {
let gradle_props = fs::read_to_string(smithy_rs.path().join("gradle.properties"))

View File

@ -27,6 +27,10 @@ mv aws-doc-sdk-examples/rust_dev_preview smithy-rs/aws/sdk/examples
rm -rf smithy-rs/aws/sdk/examples/.cargo
rm smithy-rs/aws/sdk/examples/Cargo.toml
echo -e "${C_YELLOW}Creating empty model metadata file since we don't have model update information...${C_RESET}"
MODEL_METADATA_PATH="$(pwd)/model-metadata.toml"
echo > "${MODEL_METADATA_PATH}"
echo -e "${C_YELLOW}Generating services...${C_RESET}"
cd smithy-rs
@ -34,5 +38,6 @@ cd smithy-rs
-Paws.sdk.models.path="${AWS_SDK_MODELS_PATH}" \
-Paws.sdk.examples.revision="${examples_revision}" \
-Paws.sdk.previous.release.versions.manifest="${PREVIOUS_RELEASE_VERSIONS_ARG}" \
-Paws.sdk.model.metadata="${MODEL_METADATA_PATH}" \
aws:sdk:assemble
mv aws/sdk/build/aws-sdk ../artifacts/

View File

@ -17,6 +17,7 @@ changelogger split \
--destination aws/SDK_CHANGELOG.next.json
# Render the remaining smithy-rs changelog entries
changelogger render \
--independent-versioning \
--change-set smithy-rs \
--source CHANGELOG.next.toml \
--source-to-truncate CHANGELOG.next.toml \

View File

@ -261,6 +261,16 @@ dependencies = [
"url",
]
[[package]]
name = "ctor"
version = "0.1.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f877be4f7c9f246b183111634f75baa039715e3f46ce860677d3b19a69fb229c"
dependencies = [
"quote",
"syn",
]
[[package]]
name = "dialoguer"
version = "0.8.0"
@ -273,6 +283,12 @@ dependencies = [
"zeroize",
]
[[package]]
name = "diff"
version = "0.1.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8"
[[package]]
name = "digest"
version = "0.8.1"
@ -834,6 +850,15 @@ version = "6.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "21326818e99cfe6ce1e524c2a805c189a99b5ae555a35d19f9a284b427d86afa"
[[package]]
name = "output_vt100"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "628223faebab4e3e40667ee0b2336d34a5b960ff60ea743ddfdbcf7770bcfb66"
dependencies = [
"winapi",
]
[[package]]
name = "parking_lot"
version = "0.12.1"
@ -924,6 +949,18 @@ version = "0.3.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae"
[[package]]
name = "pretty_assertions"
version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c89f989ac94207d048d92db058e4f6ec7342b0971fc58d1271ca148b799b3563"
dependencies = [
"ansi_term",
"ctor",
"diff",
"output_vt100",
]
[[package]]
name = "proc-macro-error"
version = "1.0.4"
@ -971,6 +1008,7 @@ dependencies = [
"handlebars",
"lazy_static",
"num_cpus",
"pretty_assertions",
"regex",
"reqwest",
"semver",
@ -978,6 +1016,7 @@ dependencies = [
"serde_json",
"sha256",
"smithy-rs-tool-common",
"tempfile",
"thiserror",
"tokio",
"toml",

View File

@ -36,3 +36,7 @@ tokio = { version = "1.12", features = ["full"] }
toml = { version = "0.5.8", features = ["preserve_order"] }
tracing = "0.1.29"
tracing-subscriber = { version = "0.3.5", features = ["env-filter"] }
[dev-dependencies]
pretty_assertions = "1.2.1"
tempfile = "3.3.0"

View File

@ -0,0 +1,24 @@
/*
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
* SPDX-License-Identifier: Apache-2.0
*/
pub const SDK_REPO_CRATE_PATH: &str = "sdk";
pub const SDK_REPO_NAME: &str = "aws-sdk-rust";
pub const SMITHYRS_REPO_NAME: &str = "smithy-rs";
// Crate ownership for SDK crates. Crates.io requires that at least one owner
// is an individual rather than a team, so we use the automation user for that.
pub const CRATE_OWNERS: &[&str] = &[
// https://github.com/orgs/awslabs/teams/rust-sdk-owners
"github:awslabs:rust-sdk-owners",
// https://github.com/aws-sdk-rust-ci
"aws-sdk-rust-ci",
];
pub mod cargo;
pub mod fs;
pub mod package;
pub mod retry;
pub mod sort;
pub mod subcommand;

View File

@ -3,37 +3,39 @@
* SPDX-License-Identifier: Apache-2.0
*/
use crate::subcommand::fix_manifests::subcommand_fix_manifests;
use crate::subcommand::publish::subcommand_publish;
use crate::subcommand::yank_release::{subcommand_yank_release, YankReleaseArgs};
use anyhow::Result;
use clap::Parser;
use subcommand::fix_manifests::FixManifestsArgs;
use subcommand::generate_version_manifest::{
use publisher::subcommand::fix_manifests::subcommand_fix_manifests;
use publisher::subcommand::fix_manifests::FixManifestsArgs;
use publisher::subcommand::generate_version_manifest::{
subcommand_generate_version_manifest, GenerateVersionManifestArgs,
};
use subcommand::hydrate_readme::{subcommand_hydrate_readme, HydrateReadmeArgs};
use subcommand::publish::PublishArgs;
use publisher::subcommand::hydrate_readme::subcommand_hydrate_readme_v1;
use publisher::subcommand::hydrate_readme::{
subcommand_hydrate_readme, HydrateReadmeArgs, HydrateReadmeArgsV1,
};
use publisher::subcommand::publish::subcommand_publish;
use publisher::subcommand::publish::PublishArgs;
use publisher::subcommand::tag_versions_manifest::subcommand_tag_versions_manifest;
use publisher::subcommand::tag_versions_manifest::TagVersionsManifestArgs;
use publisher::subcommand::yank_release::{subcommand_yank_release, YankReleaseArgs};
mod cargo;
mod fs;
mod package;
mod retry;
mod sort;
mod subcommand;
pub const SDK_REPO_CRATE_PATH: &str = "sdk";
pub const SDK_REPO_NAME: &str = "aws-sdk-rust";
pub const SMITHYRS_REPO_NAME: &str = "smithy-rs";
// Crate ownership for SDK crates. Crates.io requires that at least one owner
// is an individual rather than a team, so we use the automation user for that.
pub const CRATE_OWNERS: &[&str] = &[
// https://github.com/orgs/awslabs/teams/rust-sdk-owners
"github:awslabs:rust-sdk-owners",
// https://github.com/aws-sdk-rust-ci
"aws-sdk-rust-ci",
];
// TODO(https://github.com/awslabs/smithy-rs/issues/1531): Remove V1 args
#[derive(Parser, Debug)]
#[clap(author, version, about)]
enum ArgsV1 {
/// Fixes path dependencies in manifests to also have version numbers
FixManifests(FixManifestsArgs),
/// Publishes crates to crates.io
Publish(PublishArgs),
/// Yanks an entire SDK release. For individual packages, use `cargo yank` instead.
/// Only one of the `--github-release-tag` or `--versions-toml` options are required.
YankRelease(YankReleaseArgs),
/// Hydrates the SDK README template file
HydrateReadme(HydrateReadmeArgsV1),
/// Generates a version manifest file for a generated SDK
GenerateVersionManifest(GenerateVersionManifestArgs),
}
#[derive(Parser, Debug)]
#[clap(author, version, about)]
@ -49,6 +51,8 @@ enum Args {
HydrateReadme(HydrateReadmeArgs),
/// Generates a version manifest file for a generated SDK
GenerateVersionManifest(GenerateVersionManifestArgs),
/// Adds a release tag to an existing version manifest
TagVersionsManifest(TagVersionsManifestArgs),
}
#[tokio::main]
@ -59,12 +63,31 @@ async fn main() -> Result<()> {
)
.init();
match Args::parse() {
Args::Publish(args) => subcommand_publish(&args).await?,
Args::FixManifests(args) => subcommand_fix_manifests(&args).await?,
Args::YankRelease(args) => subcommand_yank_release(&args).await?,
Args::HydrateReadme(args) => subcommand_hydrate_readme(&args).await?,
Args::GenerateVersionManifest(args) => subcommand_generate_version_manifest(&args).await?,
if let Ok(args) = Args::try_parse() {
match args {
Args::Publish(args) => subcommand_publish(&args).await?,
Args::FixManifests(args) => subcommand_fix_manifests(&args).await?,
Args::YankRelease(args) => subcommand_yank_release(&args).await?,
Args::HydrateReadme(args) => subcommand_hydrate_readme(&args)?,
Args::GenerateVersionManifest(args) => {
subcommand_generate_version_manifest(&args).await?
}
Args::TagVersionsManifest(args) => subcommand_tag_versions_manifest(&args)?,
}
} else {
// TODO(https://github.com/awslabs/smithy-rs/issues/1531): Remove V1 args
eprintln!("Failed to match new arg format. Trying to parse the old arg format.");
let working_dir = std::env::current_dir()?;
match ArgsV1::parse() {
ArgsV1::Publish(args) => subcommand_publish(&args).await?,
ArgsV1::FixManifests(args) => subcommand_fix_manifests(&args).await?,
ArgsV1::YankRelease(args) => subcommand_yank_release(&args).await?,
ArgsV1::HydrateReadme(args) => subcommand_hydrate_readme_v1(&args, &working_dir)?,
ArgsV1::GenerateVersionManifest(args) => {
subcommand_generate_version_manifest(&args).await?
}
}
}
Ok(())
}

View File

@ -29,11 +29,12 @@ pub struct GenerateVersionManifestArgs {
/// Path containing the generated SDK to generate a version manifest for
#[clap(long)]
location: PathBuf,
/// Optional tag for the release the newly generated `versions.toml` will be for
#[clap(long, requires("previous-release-versions"))]
// TODO(https://github.com/awslabs/smithy-rs/issues/1531): Remove the unused `--release-tag` arg
/// Unused.
#[clap(long)]
release_tag: Option<String>,
/// Optional path to the `versions.toml` manifest from the previous SDK release
#[clap(long, requires("release-tag"))]
#[clap(long)]
previous_release_versions: Option<PathBuf>,
}
@ -42,8 +43,8 @@ pub async fn subcommand_generate_version_manifest(
smithy_build,
examples_revision,
location,
release_tag,
previous_release_versions,
..
}: &GenerateVersionManifestArgs,
) -> Result<()> {
verify_crate_hasher_available()?;
@ -75,7 +76,7 @@ pub async fn subcommand_generate_version_manifest(
.projections
.get(&package.handle.name["aws-sdk-".len()..])
{
model_hash = Some(hash_model(projection)?);
model_hash = Some(hash_models(projection)?);
}
}
assert!(
@ -100,7 +101,7 @@ pub async fn subcommand_generate_version_manifest(
release: None,
};
versions_manifest.release =
generate_release_metadata(&versions_manifest, release_tag, previous_release_versions)?;
generate_release_metadata(&versions_manifest, previous_release_versions)?;
let manifest_file_name = location.join("versions.toml");
info!("Writing {:?}...", manifest_file_name);
versions_manifest.write_to_file(&manifest_file_name)?;
@ -109,18 +110,16 @@ pub async fn subcommand_generate_version_manifest(
fn generate_release_metadata(
versions_manifest: &VersionsManifest,
maybe_release_tag: &Option<String>,
maybe_previous_release_versions: &Option<PathBuf>,
) -> Result<Option<Release>> {
match (maybe_release_tag, maybe_previous_release_versions) {
(Some(release_tag), Some(previous_release_versions)) => {
let old_versions = VersionsManifest::from_file(previous_release_versions)?;
Ok(Some(Release {
tag: release_tag.into(),
crates: find_released_versions(&old_versions, versions_manifest)?,
}))
}
_ => Ok(None),
if let Some(previous_release_versions) = maybe_previous_release_versions {
let old_versions = VersionsManifest::from_file(previous_release_versions)?;
Ok(Some(Release {
tag: None,
crates: find_released_versions(&old_versions, versions_manifest)?,
}))
} else {
Ok(None)
}
}
@ -204,7 +203,8 @@ fn hash_crate(path: &Path) -> Result<String> {
Ok(stdout.trim().into())
}
fn hash_model(projection: &SmithyBuildProjection) -> Result<String> {
fn hash_models(projection: &SmithyBuildProjection) -> Result<String> {
// Must match `hashModels` in `CrateVersioner.kt`
let mut hashes = String::new();
for import in &projection.imports {
hashes.push_str(&sha256::digest_file(import).context("hash model")?);
@ -232,8 +232,10 @@ struct SmithyBuildProjection {
#[cfg(test)]
mod tests {
use super::{find_released_versions, CrateVersion, VersionsManifest};
use super::*;
use smithy_rs_tool_common::package::PackageCategory;
use std::fs;
use tempfile::TempDir;
fn fake_manifest(crates: &[(&str, &str)]) -> VersionsManifest {
VersionsManifest {
@ -324,4 +326,27 @@ mod tests {
assert!(result.get("aws-sdk-dynamodb").is_none());
assert_eq!(3, result.len());
}
#[test]
fn test_hash_models() {
let tmp = TempDir::new().unwrap();
let model1a = tmp.path().join("model1a");
let model1b = tmp.path().join("model1b");
fs::write(&model1a, "foo").unwrap();
fs::write(&model1b, "bar").unwrap();
let hash = hash_models(&SmithyBuildProjection {
imports: vec![
model1a.to_str().unwrap().to_string(),
model1b.to_str().unwrap().to_string(),
],
})
.unwrap();
assert_eq!(
"964021077fb6c3d42ae162ab2e2255be64c6d96a6d77bca089569774d54ef69b",
hash
);
}
}

View File

@ -8,78 +8,128 @@ use anyhow::{Context, Result};
use clap::Parser;
use handlebars::Handlebars;
use semver::Version;
use serde::Serialize;
use serde_json::json;
use smithy_rs_tool_common::git;
use smithy_rs_tool_common::versions_manifest::VersionsManifest;
use std::fs;
use std::path::PathBuf;
use std::path::{Path, PathBuf};
// TODO(https://github.com/awslabs/smithy-rs/issues/1531): Remove V1 args
#[derive(Parser, Debug)]
pub struct HydrateReadmeArgsV1 {
/// AWS Rust SDK version to put in the README
#[clap(long)]
pub sdk_version: Version,
/// Rust MSRV to put in the README
#[clap(long)]
pub msrv: String,
/// Path to output the hydrated readme into
#[clap(short, long)]
pub output: PathBuf,
}
#[derive(Parser, Debug)]
pub struct HydrateReadmeArgs {
/// AWS Rust SDK version to put in the README
/// Path to the versions.toml file for this release
#[clap(long)]
sdk_version: Version,
pub versions_manifest: PathBuf,
/// Rust MSRV to put in the README
#[clap(long)]
msrv: String,
pub msrv: String,
/// Path to the readme template to hydrate
#[clap(short, long)]
pub input: PathBuf,
/// Path to output the hydrated readme into
#[clap(short, long)]
output: PathBuf,
pub output: PathBuf,
}
pub async fn subcommand_hydrate_readme(
HydrateReadmeArgs {
// TODO(https://github.com/awslabs/smithy-rs/issues/1531): Remove V1 implementation
pub fn subcommand_hydrate_readme_v1(
HydrateReadmeArgsV1 {
sdk_version,
msrv,
output,
}: &HydrateReadmeArgs,
}: &HydrateReadmeArgsV1,
working_dir: &Path,
) -> Result<()> {
let cwd = std::env::current_dir()?;
let repo_root = git::find_git_repository_root(SMITHYRS_REPO_NAME, &cwd)?;
let repo_root = git::find_git_repository_root(SMITHYRS_REPO_NAME, working_dir)?;
let template_path = repo_root.join("aws/SDK_README.md.hb");
let template_contents = fs::read(&template_path)
.with_context(|| format!("Failed to read README template file at {:?}", template_path))?;
let template_string =
String::from_utf8(template_contents).context("README template file was invalid UTF-8")?;
let hydrated = hydrate_template(&template_string, sdk_version, msrv)?;
let template_context = &json!({
"sdk_version": format!("{}", sdk_version),
"msrv": msrv
});
let hydrated = hydrate_template(&template_string, &template_context)?;
fs::write(output, hydrated.as_bytes())
.with_context(|| format!("Failed to write hydrated README to {:?}", output))?;
Ok(())
}
fn hydrate_template(template_string: &str, sdk_version: &Version, msrv: &str) -> Result<String> {
pub fn subcommand_hydrate_readme(
HydrateReadmeArgs {
versions_manifest,
msrv,
input,
output,
}: &HydrateReadmeArgs,
) -> Result<()> {
let versions_manifest = VersionsManifest::from_file(versions_manifest)
.with_context(|| format!("Failed to read versions manifest at {versions_manifest:?}"))?;
let template = fs::read_to_string(&input)
.with_context(|| format!("Failed to read README template file at {input:?}"))?;
let mut context = json!({ "msrv": msrv });
for (crate_name, metadata) in versions_manifest.crates {
let key = format!("sdk_version_{}", crate_name.replace('-', "_"));
context
.as_object_mut()
.unwrap()
.insert(key, serde_json::Value::String(metadata.version));
}
let hydrated = hydrate_template(&template, &context)?;
fs::write(output, hydrated.as_bytes())
.with_context(|| format!("Failed to write hydrated README to {:?}", output))?;
Ok(())
}
fn hydrate_template<C: Serialize>(template_string: &str, template_context: &C) -> Result<String> {
let reg = Handlebars::new();
reg.render_template(
template_string,
&json!({
"sdk_version": format!("{}", sdk_version),
"msrv": msrv
}),
)
.context("Failed to hydrate README template")
reg.render_template(template_string, template_context)
.context("Failed to hydrate README template")
}
#[cfg(test)]
mod tests {
use super::hydrate_template;
use semver::Version;
use serde_json::json;
#[test]
fn test_hydrate() {
fn test_hydrate_template() {
let template_context = json!({
"foo": "foo value",
"baz": "some baz value"
});
let hydrated = hydrate_template(
"\
{{!-- Not included --}}\n\
<!-- Included -->\n\
Some {{sdk_version}} and {{msrv}} here.\n\
Some {{foo}} and {{baz}} here.\n\
",
&Version::new(0, 5, 1),
"1.54",
&template_context,
)
.unwrap();
assert_eq!(
"\
<!-- Included -->\n\
Some 0.5.1 and 1.54 here.\n\
Some foo value and some baz value here.\n\
",
hydrated,
)

View File

@ -7,4 +7,5 @@ pub mod fix_manifests;
pub mod generate_version_manifest;
pub mod hydrate_readme;
pub mod publish;
pub mod tag_versions_manifest;
pub mod yank_release;

View File

@ -0,0 +1,95 @@
/*
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
* SPDX-License-Identifier: Apache-2.0
*/
use anyhow::{bail, Result};
use clap::Parser;
use smithy_rs_tool_common::versions_manifest::VersionsManifest;
use std::path::PathBuf;
#[derive(Parser, Debug)]
pub struct TagVersionsManifestArgs {
/// Path to the `versions.toml` file to tag
#[clap(long)]
manifest_path: PathBuf,
/// Release tag to add to the `[release]` section
#[clap(long)]
tag: String,
}
pub fn subcommand_tag_versions_manifest(
TagVersionsManifestArgs { manifest_path, tag }: &TagVersionsManifestArgs,
) -> Result<()> {
let mut manifest = VersionsManifest::from_file(manifest_path)?;
if let Some(release) = manifest.release.as_mut() {
release.tag = Some(tag.to_string());
} else {
bail!("The given versions manifest file doesn't have a `[release]` section in it to add the tag to");
}
manifest.write_to_file(manifest_path)?;
Ok(())
}
#[cfg(test)]
mod tests {
use super::{subcommand_tag_versions_manifest, TagVersionsManifestArgs};
use smithy_rs_tool_common::package::PackageCategory;
use smithy_rs_tool_common::versions_manifest::{CrateVersion, Release, VersionsManifest};
use tempfile::TempDir;
#[test]
fn test_tag_versions_manifest() {
let tmp_dir = TempDir::new().unwrap();
let versions_manifest_path = tmp_dir.path().join("versions.toml");
let original = VersionsManifest {
smithy_rs_revision: "some-revision-smithy-rs".into(),
aws_doc_sdk_examples_revision: "some-revision-docs".into(),
crates: [
(
"aws-config".to_string(),
CrateVersion {
category: PackageCategory::AwsRuntime,
version: "0.12.3".into(),
source_hash: "some-hash-aws-config".into(),
model_hash: None,
},
),
(
"aws-sdk-dynamodb".to_string(),
CrateVersion {
category: PackageCategory::AwsRuntime,
version: "0.14.5".into(),
source_hash: "some-hash-aws-sdk-dynamodb".into(),
model_hash: None,
},
),
]
.into_iter()
.collect(),
release: Some(Release {
tag: None,
crates: [("aws-config", "0.12.3"), ("aws-sdk-dynamodb", "0.14.5")]
.into_iter()
.map(|(k, v)| (k.to_string(), v.to_string()))
.collect(),
}),
};
original.write_to_file(&versions_manifest_path).unwrap();
subcommand_tag_versions_manifest(&TagVersionsManifestArgs {
manifest_path: versions_manifest_path.clone(),
tag: "some-release-tag".into(),
})
.unwrap();
let expected = {
let mut expected = original.clone();
expected.release.as_mut().unwrap().tag = Some("some-release-tag".to_string());
expected
};
let actual = VersionsManifest::from_file(&versions_manifest_path).unwrap();
assert_eq!(expected, actual);
}
}

View File

@ -4,7 +4,7 @@
*/
use crate::cargo;
use anyhow::{bail, Context, Result};
use anyhow::{anyhow, bail, Context, Result};
use clap::Parser;
use dialoguer::Confirm;
use regex::Regex;
@ -120,10 +120,11 @@ fn release_metadata(manifest: VersionsManifest) -> Result<Release> {
}
fn confirm_plan(release: &Release) -> Result<()> {
info!(
"This will yank aws-sdk-rust's `{}` release from crates.io.",
release.tag
);
let tag = release.tag.as_ref().ok_or_else(|| {
anyhow!("Versions manifest doesn't have a release tag. Can only yank tagged releases.")
})?;
info!("This will yank aws-sdk-rust's `{tag}` release from crates.io.");
info!("Crates to yank:");
for (crate_name, crate_version) in &release.crates {
info!(" {}-{}", crate_name, crate_version);

View File

@ -0,0 +1,120 @@
/*
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
* SPDX-License-Identifier: Apache-2.0
*/
use publisher::subcommand::hydrate_readme::{
subcommand_hydrate_readme, subcommand_hydrate_readme_v1, HydrateReadmeArgs, HydrateReadmeArgsV1,
};
use semver::Version;
use smithy_rs_tool_common::package::PackageCategory;
use smithy_rs_tool_common::shell::handle_failure;
use smithy_rs_tool_common::versions_manifest::{CrateVersion, VersionsManifest};
use std::fs;
use std::process::Command;
use tempfile::TempDir;
// TODO(https://github.com/awslabs/smithy-rs/issues/1531): Remove V1 implementation
#[test]
fn test_v1() {
let tmp_dir = TempDir::new().unwrap();
fs::create_dir_all(&tmp_dir.path().join("aws")).unwrap();
handle_failure(
"git-init",
&Command::new("git")
.arg("init")
.arg(".")
.current_dir(tmp_dir.path())
.output()
.unwrap(),
)
.unwrap();
fs::write(
tmp_dir.path().join("aws/SDK_README.md.hb"),
"{{!-- Not included --}}\n\
<!-- Included -->\n\
Some {{sdk_version}} and {{msrv}} here.\n",
)
.unwrap();
let output_path = tmp_dir.path().join("test-output.md");
subcommand_hydrate_readme_v1(
&HydrateReadmeArgsV1 {
sdk_version: Version::parse("0.23.0").unwrap(),
msrv: "0.58.1".into(),
output: output_path.clone(),
},
tmp_dir.path(),
)
.unwrap();
let output = fs::read_to_string(&output_path).unwrap();
pretty_assertions::assert_str_eq!("<!-- Included -->\nSome 0.23.0 and 0.58.1 here.\n", output);
}
#[test]
fn test_hydrate_readme() {
let tmp_dir = TempDir::new().unwrap();
let versions_manifest_path = tmp_dir.path().join("versions.toml");
VersionsManifest {
smithy_rs_revision: "dontcare".into(),
aws_doc_sdk_examples_revision: "dontcare".into(),
crates: [
(
"aws-config".to_string(),
CrateVersion {
category: PackageCategory::AwsRuntime,
version: "0.12.3".into(),
source_hash: "dontcare".into(),
model_hash: None,
},
),
(
"aws-sdk-dynamodb".to_string(),
CrateVersion {
category: PackageCategory::AwsRuntime,
version: "0.14.5".into(),
source_hash: "dontcare".into(),
model_hash: None,
},
),
]
.into_iter()
.collect(),
release: None,
}
.write_to_file(&versions_manifest_path)
.unwrap();
let input_path = tmp_dir.path().join("some-input.md.hb");
fs::write(
&input_path,
"{{!-- Not included --}}\n\
<!-- Included -->\n\
Some info about MSRV {{msrv}} here.\n\n\
Some info about aws-sdk-dynamodb-{{sdk_version_aws_sdk_dynamodb}} here.\n\n\
Something about aws-config-{{sdk_version_aws_config}} here.\n\n",
)
.unwrap();
let output_path = tmp_dir.path().join("test-output.md");
subcommand_hydrate_readme(&HydrateReadmeArgs {
versions_manifest: versions_manifest_path,
msrv: "0.58.1".into(),
input: input_path,
output: output_path.clone(),
})
.unwrap();
let output = fs::read_to_string(&output_path).unwrap();
pretty_assertions::assert_str_eq!(
"<!-- Included -->\n\
Some info about MSRV 0.58.1 here.\n\n\
Some info about aws-sdk-dynamodb-0.14.5 here.\n\n\
Something about aws-config-0.12.3 here.\n\n",
output
);
}

View File

@ -28,6 +28,9 @@ struct Args {
/// Path to the aws-doc-sdk-examples repository.
#[clap(long, parse(from_os_str))]
aws_doc_sdk_examples: PathBuf,
/// Path to model metadata file.
#[clap(long)]
model_metadata: Option<PathBuf>,
/// Number of threads that `sdk-sync` will use. Defaults to the physical number of CPUs,
/// or the available RAM divided by the RAM required for codegen. Whichever is smaller.
@ -59,6 +62,7 @@ impl Args {
.max_gradle_metaspace_megabytes
.unwrap_or(defaults.max_gradle_metaspace_megabytes),
aws_models_path: Some(self.aws_sdk_rust.join("aws-models")),
model_metadata_path: self.model_metadata.clone(),
}
}
}

View File

@ -19,6 +19,7 @@ pub struct CodeGenSettings {
pub max_gradle_heap_megabytes: usize,
pub max_gradle_metaspace_megabytes: usize,
pub aws_models_path: Option<PathBuf>,
pub model_metadata_path: Option<PathBuf>,
}
impl Default for CodeGenSettings {
@ -28,6 +29,7 @@ impl Default for CodeGenSettings {
max_gradle_heap_megabytes: 512,
max_gradle_metaspace_megabytes: 512,
aws_models_path: None,
model_metadata_path: None,
}
}
}
@ -172,6 +174,14 @@ impl DefaultSdkGenerator {
.expect("aws models path is a valid str")
));
}
if let Some(model_metadata_path) = &self.settings.model_metadata_path {
command.arg(format!(
"-Paws.sdk.model.metadata={}",
model_metadata_path
.to_str()
.expect("model metadata path is a valid str")
));
}
command.arg(format!(
"-Paws.sdk.previous.release.versions.manifest={}",
self.previous_versions_manifest
@ -194,7 +204,7 @@ impl DefaultSdkGenerator {
Ok(())
}
/// Runs `aws:sdk:assemble` target with property `aws.fullsdk=true` set
/// Runs `aws:sdk:assemble` target
#[instrument(skip(self))]
fn aws_sdk_assemble(&self) -> Result<()> {
// Retry gradle daemon startup failures up to 3 times

View File

@ -15,7 +15,7 @@ use std::path::Path;
use std::str::FromStr;
/// Root struct representing a `versions.toml` manifest
#[derive(Debug, Deserialize, Serialize)]
#[derive(Clone, Debug, Deserialize, Serialize, Eq, PartialEq)]
pub struct VersionsManifest {
/// Git commit hash of the version of smithy-rs used to generate this SDK
pub smithy_rs_revision: String,
@ -61,17 +61,17 @@ impl FromStr for VersionsManifest {
}
/// Release metadata
#[derive(Debug, Deserialize, Serialize)]
#[derive(Clone, Debug, Deserialize, Serialize, Eq, PartialEq)]
pub struct Release {
/// The release tag associated with this `versions.toml`
pub tag: String,
pub tag: Option<String>,
/// Which crate versions were published with this release
pub crates: BTreeMap<String, String>,
}
/// Version metadata for a crate
#[derive(Debug, Deserialize, Serialize)]
#[derive(Clone, Debug, Deserialize, Serialize, Eq, PartialEq)]
pub struct CrateVersion {
/// What kind of crate this is. Is it the Smithy runtime? AWS runtime? SDK crate?
pub category: PackageCategory,