diff --git a/aws/sdk/build.gradle.kts b/aws/sdk/build.gradle.kts index d1df765a4..18514574c 100644 --- a/aws/sdk/build.gradle.kts +++ b/aws/sdk/build.gradle.kts @@ -54,6 +54,7 @@ dependencies { val awsServices: AwsServices by lazy { discoverServices(loadServiceMembership()) } val eventStreamAllowList: Set by lazy { eventStreamAllowList() } +val crateVersioner by lazy { aws.sdk.CrateVersioner.defaultFor(rootProject, properties) } fun getSdkVersion(): String = properties.get("aws.sdk.version") ?: throw Exception("SDK version missing") fun getRustMSRV(): String = properties.get("rust.msrv") ?: throw Exception("Rust MSRV missing") @@ -86,6 +87,7 @@ fun generateSmithyBuild(services: AwsServices): String { "" ) } + val moduleName = "aws-sdk-${service.module}" val eventStreamAllowListMembers = eventStreamAllowList.joinToString(", ") { "\"$it\"" } """ "${service.module}": { @@ -103,8 +105,8 @@ fun generateSmithyBuild(services: AwsServices): String { "eventStreamAllowList": [$eventStreamAllowListMembers] }, "service": "${service.service}", - "module": "aws-sdk-${service.module}", - "moduleVersion": "${getSdkVersion()}", + "module": "$moduleName", + "moduleVersion": "${crateVersioner.decideCrateVersion(moduleName)}", "moduleAuthors": ["AWS Rust SDK Team ", "Russell Cohen "], "moduleDescription": "${service.moduleDescription}", ${service.examplesUri(project)?.let { """"examples": "$it",""" } ?: ""} @@ -299,7 +301,11 @@ tasks.register("fixManifests") { toolPath = publisherToolPath binaryName = "publisher" - arguments = listOf("fix-manifests", "--location", outputDir.absolutePath) + arguments = mutableListOf("fix-manifests", "--location", outputDir.absolutePath).apply { + if (crateVersioner.independentVersioningEnabled()) { + add("--disable-version-number-validation") + } + } dependsOn("assemble") dependsOn("relocateServices") diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index 0f1f868fd..b1d48a62e 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -30,6 +30,7 @@ dependencies { implementation("software.amazon.smithy:smithy-aws-iam-traits:$smithyVersion") implementation("software.amazon.smithy:smithy-aws-cloudformation-traits:$smithyVersion") implementation(gradleApi()) + implementation("com.moandjiezana.toml:toml4j:0.7.2") testImplementation("org.junit.jupiter:junit-jupiter:5.6.1") } diff --git a/buildSrc/src/main/kotlin/aws/sdk/CrateVersioner.kt b/buildSrc/src/main/kotlin/aws/sdk/CrateVersioner.kt new file mode 100644 index 000000000..abea120ff --- /dev/null +++ b/buildSrc/src/main/kotlin/aws/sdk/CrateVersioner.kt @@ -0,0 +1,169 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk + +import PropertyRetriever +import org.gradle.api.Project +import org.slf4j.LoggerFactory + +const val LOCAL_DEV_VERSION: String = "0.0.0-local" + +// Example command for generating with independent versions: +// ``` +// ./gradlew --no-daemon \ +// -Paws.sdk.independent.versions=true \ +// -Paws.sdk.model.metadata=$HOME/model-metadata.toml \ +// -Paws.sdk.previous.release.versions.manifest=$HOME/versions.toml \ +// aws:sdk:assemble +// ``` +object CrateVersioner { + fun defaultFor(rootProject: Project, properties: PropertyRetriever): VersionCrate = + // Putting independent crate versioning behind a feature flag for now + when (properties.get("aws.sdk.independent.versions")) { + "true" -> when (val versionsManifestPath = properties.get("aws.sdk.previous.release.versions.manifest")) { + // In local dev, use special `0.0.0-local` version number for all SDK crates + null -> SynchronizedCrateVersioner(properties, sdkVersion = LOCAL_DEV_VERSION) + else -> { + val modelMetadataPath = properties.get("aws.sdk.model.metadata") + ?: throw IllegalArgumentException("Property `aws.sdk.model.metadata` required for independent crate version builds") + IndependentCrateVersioner( + VersionsManifest.fromFile(versionsManifestPath), + ModelMetadata.fromFile(modelMetadataPath), + devPreview = true, + smithyRsVersion = getSmithyRsVersion(rootProject) + ) + } + } + else -> SynchronizedCrateVersioner(properties) + } +} + +interface VersionCrate { + fun decideCrateVersion(moduleName: String): String + + fun independentVersioningEnabled(): Boolean +} + +class SynchronizedCrateVersioner( + properties: PropertyRetriever, + private val sdkVersion: String = properties.get("aws.sdk.version") ?: throw Exception("SDK version missing") +) : VersionCrate { + init { + LoggerFactory.getLogger(javaClass).info("Using synchronized SDK crate versioning with version `$sdkVersion`") + } + + override fun decideCrateVersion(moduleName: String): String = sdkVersion + + override fun independentVersioningEnabled(): Boolean = sdkVersion == LOCAL_DEV_VERSION +} + +private data class SemVer( + val major: Int, + val minor: Int, + val patch: Int +) { + companion object { + fun parse(value: String): SemVer { + val parseNote = "Note: This implementation doesn't implement pre-release/build version support" + val failure = IllegalArgumentException("Unrecognized semver version number: $value. $parseNote") + val parts = value.split(".") + if (parts.size != 3) { + throw failure + } + return SemVer( + major = parts[0].toIntOrNull() ?: throw failure, + minor = parts[1].toIntOrNull() ?: throw failure, + patch = parts[2].toIntOrNull() ?: throw failure + ) + } + } + + fun bumpMajor(): SemVer = copy(major = major + 1, minor = 0, patch = 0) + fun bumpMinor(): SemVer = copy(minor = minor + 1, patch = 0) + fun bumpPatch(): SemVer = copy(patch = patch + 1) + + override fun toString(): String { + return "$major.$minor.$patch" + } +} + +fun getSmithyRsVersion(rootProject: Project): String { + Runtime.getRuntime().let { runtime -> + val command = arrayOf("git", "-C", rootProject.rootDir.absolutePath, "rev-parse", "HEAD") + val process = runtime.exec(command) + if (process.waitFor() != 0) { + throw RuntimeException( + "Failed to run `${command.joinToString(" ")}`:\n" + + "stdout: " + + String(process.inputStream.readAllBytes()) + + "stderr: " + + String(process.errorStream.readAllBytes()) + ) + } + return String(process.inputStream.readAllBytes()).trim() + } +} + +class IndependentCrateVersioner( + private val versionsManifest: VersionsManifest, + private val modelMetadata: ModelMetadata, + private val devPreview: Boolean, + smithyRsVersion: String +) : VersionCrate { + private val smithyRsChanged = versionsManifest.smithyRsRevision != smithyRsVersion + private val logger = LoggerFactory.getLogger(javaClass) + + init { + logger.info("Using independent SDK crate versioning. Dev preview: $devPreview") + logger.info( + "Current smithy-rs HEAD: `$smithyRsVersion`. " + + "Previous smithy-rs HEAD from versions.toml: `${versionsManifest.smithyRsRevision}`. " + + "Code generator changed: $smithyRsChanged" + ) + } + + override fun independentVersioningEnabled(): Boolean = true + + override fun decideCrateVersion(moduleName: String): String { + var previousVersion: SemVer? = null + val (reason, newVersion) = when (val existingCrate = versionsManifest.crates.get(moduleName)) { + // The crate didn't exist before, so create a new major version + null -> "new service" to newMajorVersion() + else -> { + previousVersion = SemVer.parse(existingCrate.version) + if (smithyRsChanged) { + "smithy-rs changed" to previousVersion.bumpCodegenChanged() + } else { + when (modelMetadata.changeType(moduleName)) { + ChangeType.UNCHANGED -> "no change" to previousVersion + ChangeType.FEATURE -> "its API changed" to previousVersion.bumpModelChanged() + ChangeType.DOCUMENTATION -> "it has new docs" to previousVersion.bumpDocsChanged() + } + } + } + } + if (previousVersion == null) { + logger.info("`$moduleName` is a new service. Starting it at `$newVersion`") + } else if (previousVersion != newVersion) { + logger.info("Version bumping `$moduleName` from `$previousVersion` to `$newVersion` because $reason") + } else { + logger.info("No changes expected for `$moduleName`") + } + return newVersion.toString() + } + + private fun newMajorVersion(): SemVer = when (devPreview) { + true -> SemVer.parse("0.1.0") + else -> SemVer.parse("1.0.0") + } + + private fun SemVer.bumpCodegenChanged(): SemVer = bumpMinor() + private fun SemVer.bumpModelChanged(): SemVer = when (devPreview) { + true -> bumpPatch() + else -> bumpMinor() + } + private fun SemVer.bumpDocsChanged(): SemVer = bumpPatch() +} diff --git a/buildSrc/src/main/kotlin/aws/sdk/ModelMetadata.kt b/buildSrc/src/main/kotlin/aws/sdk/ModelMetadata.kt new file mode 100644 index 000000000..aca086cae --- /dev/null +++ b/buildSrc/src/main/kotlin/aws/sdk/ModelMetadata.kt @@ -0,0 +1,43 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk + +import com.moandjiezana.toml.Toml +import java.io.File + +enum class ChangeType { + UNCHANGED, + FEATURE, + DOCUMENTATION +} + +/** Model metadata toml file */ +data class ModelMetadata( + private val crates: Map +) { + companion object { + fun fromFile(path: String): ModelMetadata { + val contents = File(path).readText() + return fromString(contents) + } + + fun fromString(value: String): ModelMetadata { + val toml = Toml().read(value) + return ModelMetadata( + crates = toml.getTable("crates")?.entrySet()?.map { entry -> + entry.key to when (val kind = (entry.value as Toml).getString("kind")) { + "Feature" -> ChangeType.FEATURE + "Documentation" -> ChangeType.DOCUMENTATION + else -> throw IllegalArgumentException("Unrecognized change type: $kind") + } + }?.toMap() ?: emptyMap() + ) + } + } + + fun hasCrates(): Boolean = crates.isNotEmpty() + fun changeType(moduleName: String): ChangeType = crates[moduleName] ?: ChangeType.UNCHANGED +} diff --git a/buildSrc/src/main/kotlin/aws/sdk/VersionsManifest.kt b/buildSrc/src/main/kotlin/aws/sdk/VersionsManifest.kt new file mode 100644 index 000000000..f1b793f97 --- /dev/null +++ b/buildSrc/src/main/kotlin/aws/sdk/VersionsManifest.kt @@ -0,0 +1,47 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk + +import com.moandjiezana.toml.Toml +import java.io.File + +data class CrateVersion( + val category: String, + val version: String, + val sourceHash: String? = null, + val modelHash: String? = null +) + +/** Kotlin representation of aws-sdk-rust's `versions.toml` file */ +data class VersionsManifest( + val smithyRsRevision: String, + val awsDocSdkExamplesRevision: String, + val crates: Map +) { + companion object { + fun fromFile(path: String): VersionsManifest { + val contents = File(path).readText() + return fromString(contents) + } + + fun fromString(value: String): VersionsManifest { + val toml = Toml().read(value) + return VersionsManifest( + smithyRsRevision = toml.getString("smithy_rs_revision"), + awsDocSdkExamplesRevision = toml.getString("aws_doc_sdk_examples_revision"), + crates = toml.getTable("crates").entrySet().map { entry -> + val value = (entry.value as Toml) + entry.key to CrateVersion( + category = value.getString("category"), + version = value.getString("version"), + sourceHash = value.getString("source_hash"), + modelHash = value.getString("model_hash") + ) + }.toMap() + ) + } + } +} diff --git a/buildSrc/src/test/kotlin/aws/sdk/IndependentCrateVersionerTest.kt b/buildSrc/src/test/kotlin/aws/sdk/IndependentCrateVersionerTest.kt new file mode 100644 index 000000000..5a2b691d6 --- /dev/null +++ b/buildSrc/src/test/kotlin/aws/sdk/IndependentCrateVersionerTest.kt @@ -0,0 +1,161 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test + +class IndependentCrateVersionerTest { + @Test + fun devPreviewSmithyRsChanged() { + val versioner = IndependentCrateVersioner( + VersionsManifest( + smithyRsRevision = "smithy-rs-1", + awsDocSdkExamplesRevision = "dontcare", + crates = mapOf( + "aws-sdk-dynamodb" to CrateVersion( + category = "AwsSdk", + version = "0.11.3" + ), + "aws-sdk-ec2" to CrateVersion( + category = "AwsSdk", + version = "0.10.1" + ), + "aws-sdk-s3" to CrateVersion( + category = "AwsSdk", + version = "0.12.0" + ) + ) + ), + ModelMetadata( + crates = mapOf( + "aws-sdk-dynamodb" to ChangeType.FEATURE, + "aws-sdk-ec2" to ChangeType.DOCUMENTATION + ) + ), + devPreview = true, + smithyRsVersion = "smithy-rs-2" + ) + + // The code generator changed, so all minor versions should bump + assertEquals("0.12.0", versioner.decideCrateVersion("aws-sdk-dynamodb")) + assertEquals("0.11.0", versioner.decideCrateVersion("aws-sdk-ec2")) + assertEquals("0.13.0", versioner.decideCrateVersion("aws-sdk-s3")) + assertEquals("0.1.0", versioner.decideCrateVersion("aws-sdk-somenewservice")) + } + + @Test + fun devPreviewSameCodeGenerator() { + val versioner = IndependentCrateVersioner( + VersionsManifest( + smithyRsRevision = "smithy-rs-1", + awsDocSdkExamplesRevision = "dontcare", + crates = mapOf( + "aws-sdk-dynamodb" to CrateVersion( + category = "AwsSdk", + version = "0.11.3" + ), + "aws-sdk-ec2" to CrateVersion( + category = "AwsSdk", + version = "0.10.1" + ), + "aws-sdk-s3" to CrateVersion( + category = "AwsSdk", + version = "0.12.0" + ) + ) + ), + ModelMetadata( + crates = mapOf( + "aws-sdk-dynamodb" to ChangeType.FEATURE, + "aws-sdk-ec2" to ChangeType.DOCUMENTATION + ) + ), + devPreview = true, + smithyRsVersion = "smithy-rs-1" + ) + + assertEquals("0.11.4", versioner.decideCrateVersion("aws-sdk-dynamodb")) + assertEquals("0.10.2", versioner.decideCrateVersion("aws-sdk-ec2")) + assertEquals("0.12.0", versioner.decideCrateVersion("aws-sdk-s3")) + assertEquals("0.1.0", versioner.decideCrateVersion("aws-sdk-somenewservice")) + } + + @Test + fun smithyRsChanged() { + val versioner = IndependentCrateVersioner( + VersionsManifest( + smithyRsRevision = "smithy-rs-1", + awsDocSdkExamplesRevision = "dontcare", + crates = mapOf( + "aws-sdk-dynamodb" to CrateVersion( + category = "AwsSdk", + version = "1.11.3" + ), + "aws-sdk-ec2" to CrateVersion( + category = "AwsSdk", + version = "1.10.1" + ), + "aws-sdk-s3" to CrateVersion( + category = "AwsSdk", + version = "1.12.0" + ) + ) + ), + ModelMetadata( + crates = mapOf( + "aws-sdk-dynamodb" to ChangeType.FEATURE, + "aws-sdk-ec2" to ChangeType.DOCUMENTATION + ) + ), + devPreview = false, + smithyRsVersion = "smithy-rs-2" + ) + + // The code generator changed, so all minor versions should bump + assertEquals("1.12.0", versioner.decideCrateVersion("aws-sdk-dynamodb")) + assertEquals("1.11.0", versioner.decideCrateVersion("aws-sdk-ec2")) + assertEquals("1.13.0", versioner.decideCrateVersion("aws-sdk-s3")) + assertEquals("1.0.0", versioner.decideCrateVersion("aws-sdk-somenewservice")) + } + + @Test + fun sameCodeGenerator() { + val versioner = IndependentCrateVersioner( + VersionsManifest( + smithyRsRevision = "smithy-rs-1", + awsDocSdkExamplesRevision = "dontcare", + crates = mapOf( + "aws-sdk-dynamodb" to CrateVersion( + category = "AwsSdk", + version = "1.11.3" + ), + "aws-sdk-ec2" to CrateVersion( + category = "AwsSdk", + version = "1.10.1" + ), + "aws-sdk-s3" to CrateVersion( + category = "AwsSdk", + version = "1.12.0" + ) + ) + ), + ModelMetadata( + crates = mapOf( + "aws-sdk-dynamodb" to ChangeType.FEATURE, + "aws-sdk-ec2" to ChangeType.DOCUMENTATION + ) + ), + devPreview = false, + smithyRsVersion = "smithy-rs-1" + ) + + assertEquals("1.12.0", versioner.decideCrateVersion("aws-sdk-dynamodb")) + assertEquals("1.10.2", versioner.decideCrateVersion("aws-sdk-ec2")) + assertEquals("1.12.0", versioner.decideCrateVersion("aws-sdk-s3")) + assertEquals("1.0.0", versioner.decideCrateVersion("aws-sdk-somenewservice")) + } +} diff --git a/buildSrc/src/test/kotlin/aws/sdk/ModelMetadataTest.kt b/buildSrc/src/test/kotlin/aws/sdk/ModelMetadataTest.kt new file mode 100644 index 000000000..d157ed0ad --- /dev/null +++ b/buildSrc/src/test/kotlin/aws/sdk/ModelMetadataTest.kt @@ -0,0 +1,33 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Test + +class ModelMetadataTest { + @Test + fun `it should parse an empty file`() { + val result = ModelMetadata.fromString("") + assertFalse(result.hasCrates()) + } + + @Test + fun `it should parse`() { + val contents = """ + [crates.aws-sdk-someservice] + kind = "Feature" + + [crates.aws-sdk-s3] + kind = "Documentation" + """.trimIndent() + + val result = ModelMetadata.fromString(contents) + assertEquals(ChangeType.FEATURE, result.changeType("aws-sdk-someservice")) + assertEquals(ChangeType.DOCUMENTATION, result.changeType("aws-sdk-s3")) + } +} diff --git a/buildSrc/src/test/kotlin/aws/sdk/VersionsManifestTest.kt b/buildSrc/src/test/kotlin/aws/sdk/VersionsManifestTest.kt new file mode 100644 index 000000000..0cfa63a09 --- /dev/null +++ b/buildSrc/src/test/kotlin/aws/sdk/VersionsManifestTest.kt @@ -0,0 +1,51 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test + +class VersionsManifestTest { + @Test + fun `it should parse versions toml`() { + val manifest = VersionsManifest.fromString( + """ + smithy_rs_revision = 'some-smithy-rs-revision' + aws_doc_sdk_examples_revision = 'some-doc-revision' + + [crates.aws-config] + category = 'AwsRuntime' + version = '0.12.0' + source_hash = '12d172094a2576e6f4d00a8ba58276c0d4abc4e241bb75f0d3de8ac3412e8e47' + + [crates.aws-sdk-account] + category = 'AwsSdk' + version = '0.12.0' + source_hash = 'a0dfc080638b1d803745f0bd66b610131783cf40ab88fd710dce906fc69b983e' + model_hash = '179bbfd915093dc3bec5444771da2b20d99a37d104ba25f0acac9aa0d5bb758a' + """.trimIndent() + ) + + assertEquals("some-smithy-rs-revision", manifest.smithyRsRevision) + assertEquals("some-doc-revision", manifest.awsDocSdkExamplesRevision) + assertEquals( + mapOf( + "aws-config" to CrateVersion( + category = "AwsRuntime", + version = "0.12.0", + sourceHash = "12d172094a2576e6f4d00a8ba58276c0d4abc4e241bb75f0d3de8ac3412e8e47" + ), + "aws-sdk-account" to CrateVersion( + category = "AwsSdk", + version = "0.12.0", + sourceHash = "a0dfc080638b1d803745f0bd66b610131783cf40ab88fd710dce906fc69b983e", + modelHash = "179bbfd915093dc3bec5444771da2b20d99a37d104ba25f0acac9aa0d5bb758a" + ) + ), + manifest.crates + ) + } +} diff --git a/tools/publisher/src/subcommand/fix_manifests.rs b/tools/publisher/src/subcommand/fix_manifests.rs index 8c2671da2..c6c15afce 100644 --- a/tools/publisher/src/subcommand/fix_manifests.rs +++ b/tools/publisher/src/subcommand/fix_manifests.rs @@ -38,10 +38,18 @@ pub struct FixManifestsArgs { /// Checks manifests rather than fixing them #[clap(long)] check: bool, + /// Disable expected version number validation. This should only be used + /// when SDK crates are being generated with independent version numbers. + #[clap(long)] + disable_version_number_validation: bool, } pub async fn subcommand_fix_manifests( - FixManifestsArgs { location, check }: &FixManifestsArgs, + FixManifestsArgs { + location, + check, + disable_version_number_validation, + }: &FixManifestsArgs, ) -> Result<()> { let mode = match check { true => Mode::Check, @@ -51,7 +59,7 @@ pub async fn subcommand_fix_manifests( let mut manifests = read_manifests(Fs::Real, manifest_paths).await?; let versions = package_versions(&manifests)?; - validate::validate_before_fixes(&versions)?; + validate::validate_before_fixes(&versions, *disable_version_number_validation)?; fix_manifests(Fs::Real, &versions, &mut manifests, mode).await?; validate::validate_after_fixes(location).await?; info!("Successfully fixed manifests!"); diff --git a/tools/publisher/src/subcommand/fix_manifests/validate.rs b/tools/publisher/src/subcommand/fix_manifests/validate.rs index 817afa5c3..7387bb661 100644 --- a/tools/publisher/src/subcommand/fix_manifests/validate.rs +++ b/tools/publisher/src/subcommand/fix_manifests/validate.rs @@ -17,7 +17,15 @@ use tracing::info; /// For now, this validates: /// - `aws-config` version number matches all `aws-sdk-` prefixed versions /// - `aws-smithy-` prefixed versions match `aws-` (NOT `aws-sdk-`) prefixed versions -pub(super) fn validate_before_fixes(versions: &BTreeMap) -> Result<()> { +pub(super) fn validate_before_fixes( + versions: &BTreeMap, + disable_version_number_validation: bool, +) -> Result<()> { + // Later when we only generate independently versioned SDK crates, this flag can become permanent. + if disable_version_number_validation { + return Ok(()); + } + info!("Pre-validation manifests..."); let maybe_sdk_version = versions.get("aws-config"); let expected_smithy_version = versions @@ -75,12 +83,12 @@ mod test { #[track_caller] fn expect_success(version_tuples: &[(&'static str, &'static str)]) { - validate_before_fixes(&versions(version_tuples)).expect("success"); + validate_before_fixes(&versions(version_tuples), false).expect("success"); } #[track_caller] fn expect_failure(message: &str, version_tuples: &[(&'static str, &'static str)]) { - if let Err(err) = validate_before_fixes(&versions(version_tuples)) { + if let Err(err) = validate_before_fixes(&versions(version_tuples), false) { assert_eq!(message, format!("{}", err)); } else { panic!("Expected validation failure");