Implement feature-gated independent SDK crate versioning (#1435)

Co-authored-by: Zelda Hessler <zhessler@amazon.com>

aws/sdk/build.gradle.kts

@@ -54,6 +54,7 @@ dependencies {
 
 val awsServices: AwsServices by lazy { discoverServices(loadServiceMembership()) }
 val eventStreamAllowList: Set<String> by lazy { eventStreamAllowList() }
+val crateVersioner by lazy { aws.sdk.CrateVersioner.defaultFor(rootProject, properties) }
 
 fun getSdkVersion(): String = properties.get("aws.sdk.version") ?: throw Exception("SDK version missing")
 fun getRustMSRV(): String = properties.get("rust.msrv") ?: throw Exception("Rust MSRV missing")
@@ -86,6 +87,7 @@ fun generateSmithyBuild(services: AwsServices): String {
                 ""
             )
         }
+        val moduleName = "aws-sdk-${service.module}"
         val eventStreamAllowListMembers = eventStreamAllowList.joinToString(", ") { "\"$it\"" }
         """
             "${service.module}": {
@@ -103,8 +105,8 @@ fun generateSmithyBuild(services: AwsServices): String {
                             "eventStreamAllowList": [$eventStreamAllowListMembers]
                         },
                         "service": "${service.service}",
-                        "module": "aws-sdk-${service.module}",
-                        "moduleVersion": "${getSdkVersion()}",
+                        "module": "$moduleName",
+                        "moduleVersion": "${crateVersioner.decideCrateVersion(moduleName)}",
                         "moduleAuthors": ["AWS Rust SDK Team <aws-sdk-rust@amazon.com>", "Russell Cohen <rcoh@amazon.com>"],
                         "moduleDescription": "${service.moduleDescription}",
                         ${service.examplesUri(project)?.let { """"examples": "$it",""" } ?: ""}
@@ -299,7 +301,11 @@ tasks.register<ExecRustBuildTool>("fixManifests") {
 
     toolPath = publisherToolPath
     binaryName = "publisher"
-    arguments = listOf("fix-manifests", "--location", outputDir.absolutePath)
+    arguments = mutableListOf("fix-manifests", "--location", outputDir.absolutePath).apply {
+        if (crateVersioner.independentVersioningEnabled()) {
+            add("--disable-version-number-validation")
+        }
+    }
 
     dependsOn("assemble")
     dependsOn("relocateServices")

buildSrc/build.gradle.kts

@@ -30,6 +30,7 @@ dependencies {
     implementation("software.amazon.smithy:smithy-aws-iam-traits:$smithyVersion")
     implementation("software.amazon.smithy:smithy-aws-cloudformation-traits:$smithyVersion")
     implementation(gradleApi())
+    implementation("com.moandjiezana.toml:toml4j:0.7.2")
     testImplementation("org.junit.jupiter:junit-jupiter:5.6.1")
 }
 

buildSrc/src/main/kotlin/aws/sdk/CrateVersioner.kt

@@ -0,0 +1,169 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package aws.sdk
+
+import PropertyRetriever
+import org.gradle.api.Project
+import org.slf4j.LoggerFactory
+
+const val LOCAL_DEV_VERSION: String = "0.0.0-local"
+
+// Example command for generating with independent versions:
+// ```
+// ./gradlew --no-daemon \
+//     -Paws.sdk.independent.versions=true \
+//     -Paws.sdk.model.metadata=$HOME/model-metadata.toml \
+//     -Paws.sdk.previous.release.versions.manifest=$HOME/versions.toml \
+//     aws:sdk:assemble
+// ```
+object CrateVersioner {
+    fun defaultFor(rootProject: Project, properties: PropertyRetriever): VersionCrate =
+        // Putting independent crate versioning behind a feature flag for now
+        when (properties.get("aws.sdk.independent.versions")) {
+            "true" -> when (val versionsManifestPath = properties.get("aws.sdk.previous.release.versions.manifest")) {
+                // In local dev, use special `0.0.0-local` version number for all SDK crates
+                null -> SynchronizedCrateVersioner(properties, sdkVersion = LOCAL_DEV_VERSION)
+                else -> {
+                    val modelMetadataPath = properties.get("aws.sdk.model.metadata")
+                        ?: throw IllegalArgumentException("Property `aws.sdk.model.metadata` required for independent crate version builds")
+                    IndependentCrateVersioner(
+                        VersionsManifest.fromFile(versionsManifestPath),
+                        ModelMetadata.fromFile(modelMetadataPath),
+                        devPreview = true,
+                        smithyRsVersion = getSmithyRsVersion(rootProject)
+                    )
+                }
+            }
+            else -> SynchronizedCrateVersioner(properties)
+        }
+}
+
+interface VersionCrate {
+    fun decideCrateVersion(moduleName: String): String
+
+    fun independentVersioningEnabled(): Boolean
+}
+
+class SynchronizedCrateVersioner(
+    properties: PropertyRetriever,
+    private val sdkVersion: String = properties.get("aws.sdk.version") ?: throw Exception("SDK version missing")
+) : VersionCrate {
+    init {
+        LoggerFactory.getLogger(javaClass).info("Using synchronized SDK crate versioning with version `$sdkVersion`")
+    }
+
+    override fun decideCrateVersion(moduleName: String): String = sdkVersion
+
+    override fun independentVersioningEnabled(): Boolean = sdkVersion == LOCAL_DEV_VERSION
+}
+
+private data class SemVer(
+    val major: Int,
+    val minor: Int,
+    val patch: Int
+) {
+    companion object {
+        fun parse(value: String): SemVer {
+            val parseNote = "Note: This implementation doesn't implement pre-release/build version support"
+            val failure = IllegalArgumentException("Unrecognized semver version number: $value. $parseNote")
+            val parts = value.split(".")
+            if (parts.size != 3) {
+                throw failure
+            }
+            return SemVer(
+                major = parts[0].toIntOrNull() ?: throw failure,
+                minor = parts[1].toIntOrNull() ?: throw failure,
+                patch = parts[2].toIntOrNull() ?: throw failure
+            )
+        }
+    }
+
+    fun bumpMajor(): SemVer = copy(major = major + 1, minor = 0, patch = 0)
+    fun bumpMinor(): SemVer = copy(minor = minor + 1, patch = 0)
+    fun bumpPatch(): SemVer = copy(patch = patch + 1)
+
+    override fun toString(): String {
+        return "$major.$minor.$patch"
+    }
+}
+
+fun getSmithyRsVersion(rootProject: Project): String {
+    Runtime.getRuntime().let { runtime ->
+        val command = arrayOf("git", "-C", rootProject.rootDir.absolutePath, "rev-parse", "HEAD")
+        val process = runtime.exec(command)
+        if (process.waitFor() != 0) {
+            throw RuntimeException(
+                "Failed to run `${command.joinToString(" ")}`:\n" +
+                    "stdout: " +
+                    String(process.inputStream.readAllBytes()) +
+                    "stderr: " +
+                    String(process.errorStream.readAllBytes())
+            )
+        }
+        return String(process.inputStream.readAllBytes()).trim()
+    }
+}
+
+class IndependentCrateVersioner(
+    private val versionsManifest: VersionsManifest,
+    private val modelMetadata: ModelMetadata,
+    private val devPreview: Boolean,
+    smithyRsVersion: String
+) : VersionCrate {
+    private val smithyRsChanged = versionsManifest.smithyRsRevision != smithyRsVersion
+    private val logger = LoggerFactory.getLogger(javaClass)
+
+    init {
+        logger.info("Using independent SDK crate versioning. Dev preview: $devPreview")
+        logger.info(
+            "Current smithy-rs HEAD: `$smithyRsVersion`. " +
+                "Previous smithy-rs HEAD from versions.toml: `${versionsManifest.smithyRsRevision}`. " +
+                "Code generator changed: $smithyRsChanged"
+        )
+    }
+
+    override fun independentVersioningEnabled(): Boolean = true
+
+    override fun decideCrateVersion(moduleName: String): String {
+        var previousVersion: SemVer? = null
+        val (reason, newVersion) = when (val existingCrate = versionsManifest.crates.get(moduleName)) {
+            // The crate didn't exist before, so create a new major version
+            null -> "new service" to newMajorVersion()
+            else -> {
+                previousVersion = SemVer.parse(existingCrate.version)
+                if (smithyRsChanged) {
+                    "smithy-rs changed" to previousVersion.bumpCodegenChanged()
+                } else {
+                    when (modelMetadata.changeType(moduleName)) {
+                        ChangeType.UNCHANGED -> "no change" to previousVersion
+                        ChangeType.FEATURE -> "its API changed" to previousVersion.bumpModelChanged()
+                        ChangeType.DOCUMENTATION -> "it has new docs" to previousVersion.bumpDocsChanged()
+                    }
+                }
+            }
+        }
+        if (previousVersion == null) {
+            logger.info("`$moduleName` is a new service. Starting it at `$newVersion`")
+        } else if (previousVersion != newVersion) {
+            logger.info("Version bumping `$moduleName` from `$previousVersion` to `$newVersion` because $reason")
+        } else {
+            logger.info("No changes expected for `$moduleName`")
+        }
+        return newVersion.toString()
+    }
+
+    private fun newMajorVersion(): SemVer = when (devPreview) {
+        true -> SemVer.parse("0.1.0")
+        else -> SemVer.parse("1.0.0")
+    }
+
+    private fun SemVer.bumpCodegenChanged(): SemVer = bumpMinor()
+    private fun SemVer.bumpModelChanged(): SemVer = when (devPreview) {
+        true -> bumpPatch()
+        else -> bumpMinor()
+    }
+    private fun SemVer.bumpDocsChanged(): SemVer = bumpPatch()
+}

buildSrc/src/main/kotlin/aws/sdk/ModelMetadata.kt

@@ -0,0 +1,43 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package aws.sdk
+
+import com.moandjiezana.toml.Toml
+import java.io.File
+
+enum class ChangeType {
+    UNCHANGED,
+    FEATURE,
+    DOCUMENTATION
+}
+
+/** Model metadata toml file */
+data class ModelMetadata(
+    private val crates: Map<String, ChangeType>
+) {
+    companion object {
+        fun fromFile(path: String): ModelMetadata {
+            val contents = File(path).readText()
+            return fromString(contents)
+        }
+
+        fun fromString(value: String): ModelMetadata {
+            val toml = Toml().read(value)
+            return ModelMetadata(
+                crates = toml.getTable("crates")?.entrySet()?.map { entry ->
+                    entry.key to when (val kind = (entry.value as Toml).getString("kind")) {
+                        "Feature" -> ChangeType.FEATURE
+                        "Documentation" -> ChangeType.DOCUMENTATION
+                        else -> throw IllegalArgumentException("Unrecognized change type: $kind")
+                    }
+                }?.toMap() ?: emptyMap()
+            )
+        }
+    }
+
+    fun hasCrates(): Boolean = crates.isNotEmpty()
+    fun changeType(moduleName: String): ChangeType = crates[moduleName] ?: ChangeType.UNCHANGED
+}

buildSrc/src/main/kotlin/aws/sdk/VersionsManifest.kt

@@ -0,0 +1,47 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package aws.sdk
+
+import com.moandjiezana.toml.Toml
+import java.io.File
+
+data class CrateVersion(
+    val category: String,
+    val version: String,
+    val sourceHash: String? = null,
+    val modelHash: String? = null
+)
+
+/** Kotlin representation of aws-sdk-rust's `versions.toml` file */
+data class VersionsManifest(
+    val smithyRsRevision: String,
+    val awsDocSdkExamplesRevision: String,
+    val crates: Map<String, CrateVersion>
+) {
+    companion object {
+        fun fromFile(path: String): VersionsManifest {
+            val contents = File(path).readText()
+            return fromString(contents)
+        }
+
+        fun fromString(value: String): VersionsManifest {
+            val toml = Toml().read(value)
+            return VersionsManifest(
+                smithyRsRevision = toml.getString("smithy_rs_revision"),
+                awsDocSdkExamplesRevision = toml.getString("aws_doc_sdk_examples_revision"),
+                crates = toml.getTable("crates").entrySet().map { entry ->
+                    val value = (entry.value as Toml)
+                    entry.key to CrateVersion(
+                        category = value.getString("category"),
+                        version = value.getString("version"),
+                        sourceHash = value.getString("source_hash"),
+                        modelHash = value.getString("model_hash")
+                    )
+                }.toMap()
+            )
+        }
+    }
+}

buildSrc/src/test/kotlin/aws/sdk/IndependentCrateVersionerTest.kt

@@ -0,0 +1,161 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package aws.sdk
+
+import org.junit.jupiter.api.Assertions.assertEquals
+import org.junit.jupiter.api.Test
+
+class IndependentCrateVersionerTest {
+    @Test
+    fun devPreviewSmithyRsChanged() {
+        val versioner = IndependentCrateVersioner(
+            VersionsManifest(
+                smithyRsRevision = "smithy-rs-1",
+                awsDocSdkExamplesRevision = "dontcare",
+                crates = mapOf(
+                    "aws-sdk-dynamodb" to CrateVersion(
+                        category = "AwsSdk",
+                        version = "0.11.3"
+                    ),
+                    "aws-sdk-ec2" to CrateVersion(
+                        category = "AwsSdk",
+                        version = "0.10.1"
+                    ),
+                    "aws-sdk-s3" to CrateVersion(
+                        category = "AwsSdk",
+                        version = "0.12.0"
+                    )
+                )
+            ),
+            ModelMetadata(
+                crates = mapOf(
+                    "aws-sdk-dynamodb" to ChangeType.FEATURE,
+                    "aws-sdk-ec2" to ChangeType.DOCUMENTATION
+                )
+            ),
+            devPreview = true,
+            smithyRsVersion = "smithy-rs-2"
+        )
+
+        // The code generator changed, so all minor versions should bump
+        assertEquals("0.12.0", versioner.decideCrateVersion("aws-sdk-dynamodb"))
+        assertEquals("0.11.0", versioner.decideCrateVersion("aws-sdk-ec2"))
+        assertEquals("0.13.0", versioner.decideCrateVersion("aws-sdk-s3"))
+        assertEquals("0.1.0", versioner.decideCrateVersion("aws-sdk-somenewservice"))
+    }
+
+    @Test
+    fun devPreviewSameCodeGenerator() {
+        val versioner = IndependentCrateVersioner(
+            VersionsManifest(
+                smithyRsRevision = "smithy-rs-1",
+                awsDocSdkExamplesRevision = "dontcare",
+                crates = mapOf(
+                    "aws-sdk-dynamodb" to CrateVersion(
+                        category = "AwsSdk",
+                        version = "0.11.3"
+                    ),
+                    "aws-sdk-ec2" to CrateVersion(
+                        category = "AwsSdk",
+                        version = "0.10.1"
+                    ),
+                    "aws-sdk-s3" to CrateVersion(
+                        category = "AwsSdk",
+                        version = "0.12.0"
+                    )
+                )
+            ),
+            ModelMetadata(
+                crates = mapOf(
+                    "aws-sdk-dynamodb" to ChangeType.FEATURE,
+                    "aws-sdk-ec2" to ChangeType.DOCUMENTATION
+                )
+            ),
+            devPreview = true,
+            smithyRsVersion = "smithy-rs-1"
+        )
+
+        assertEquals("0.11.4", versioner.decideCrateVersion("aws-sdk-dynamodb"))
+        assertEquals("0.10.2", versioner.decideCrateVersion("aws-sdk-ec2"))
+        assertEquals("0.12.0", versioner.decideCrateVersion("aws-sdk-s3"))
+        assertEquals("0.1.0", versioner.decideCrateVersion("aws-sdk-somenewservice"))
+    }
+
+    @Test
+    fun smithyRsChanged() {
+        val versioner = IndependentCrateVersioner(
+            VersionsManifest(
+                smithyRsRevision = "smithy-rs-1",
+                awsDocSdkExamplesRevision = "dontcare",
+                crates = mapOf(
+                    "aws-sdk-dynamodb" to CrateVersion(
+                        category = "AwsSdk",
+                        version = "1.11.3"
+                    ),
+                    "aws-sdk-ec2" to CrateVersion(
+                        category = "AwsSdk",
+                        version = "1.10.1"
+                    ),
+                    "aws-sdk-s3" to CrateVersion(
+                        category = "AwsSdk",
+                        version = "1.12.0"
+                    )
+                )
+            ),
+            ModelMetadata(
+                crates = mapOf(
+                    "aws-sdk-dynamodb" to ChangeType.FEATURE,
+                    "aws-sdk-ec2" to ChangeType.DOCUMENTATION
+                )
+            ),
+            devPreview = false,
+            smithyRsVersion = "smithy-rs-2"
+        )
+
+        // The code generator changed, so all minor versions should bump
+        assertEquals("1.12.0", versioner.decideCrateVersion("aws-sdk-dynamodb"))
+        assertEquals("1.11.0", versioner.decideCrateVersion("aws-sdk-ec2"))
+        assertEquals("1.13.0", versioner.decideCrateVersion("aws-sdk-s3"))
+        assertEquals("1.0.0", versioner.decideCrateVersion("aws-sdk-somenewservice"))
+    }
+
+    @Test
+    fun sameCodeGenerator() {
+        val versioner = IndependentCrateVersioner(
+            VersionsManifest(
+                smithyRsRevision = "smithy-rs-1",
+                awsDocSdkExamplesRevision = "dontcare",
+                crates = mapOf(
+                    "aws-sdk-dynamodb" to CrateVersion(
+                        category = "AwsSdk",
+                        version = "1.11.3"
+                    ),
+                    "aws-sdk-ec2" to CrateVersion(
+                        category = "AwsSdk",
+                        version = "1.10.1"
+                    ),
+                    "aws-sdk-s3" to CrateVersion(
+                        category = "AwsSdk",
+                        version = "1.12.0"
+                    )
+                )
+            ),
+            ModelMetadata(
+                crates = mapOf(
+                    "aws-sdk-dynamodb" to ChangeType.FEATURE,
+                    "aws-sdk-ec2" to ChangeType.DOCUMENTATION
+                )
+            ),
+            devPreview = false,
+            smithyRsVersion = "smithy-rs-1"
+        )
+
+        assertEquals("1.12.0", versioner.decideCrateVersion("aws-sdk-dynamodb"))
+        assertEquals("1.10.2", versioner.decideCrateVersion("aws-sdk-ec2"))
+        assertEquals("1.12.0", versioner.decideCrateVersion("aws-sdk-s3"))
+        assertEquals("1.0.0", versioner.decideCrateVersion("aws-sdk-somenewservice"))
+    }
+}

buildSrc/src/test/kotlin/aws/sdk/ModelMetadataTest.kt

@@ -0,0 +1,33 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package aws.sdk
+
+import org.junit.jupiter.api.Assertions.assertEquals
+import org.junit.jupiter.api.Assertions.assertFalse
+import org.junit.jupiter.api.Test
+
+class ModelMetadataTest {
+    @Test
+    fun `it should parse an empty file`() {
+        val result = ModelMetadata.fromString("")
+        assertFalse(result.hasCrates())
+    }
+
+    @Test
+    fun `it should parse`() {
+        val contents = """
+            [crates.aws-sdk-someservice]
+            kind = "Feature"
+
+            [crates.aws-sdk-s3]
+            kind = "Documentation"
+        """.trimIndent()
+
+        val result = ModelMetadata.fromString(contents)
+        assertEquals(ChangeType.FEATURE, result.changeType("aws-sdk-someservice"))
+        assertEquals(ChangeType.DOCUMENTATION, result.changeType("aws-sdk-s3"))
+    }
+}

buildSrc/src/test/kotlin/aws/sdk/VersionsManifestTest.kt

@@ -0,0 +1,51 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package aws.sdk
+
+import org.junit.jupiter.api.Assertions.assertEquals
+import org.junit.jupiter.api.Test
+
+class VersionsManifestTest {
+    @Test
+    fun `it should parse versions toml`() {
+        val manifest = VersionsManifest.fromString(
+            """
+            smithy_rs_revision = 'some-smithy-rs-revision'
+            aws_doc_sdk_examples_revision = 'some-doc-revision'
+
+            [crates.aws-config]
+            category = 'AwsRuntime'
+            version = '0.12.0'
+            source_hash = '12d172094a2576e6f4d00a8ba58276c0d4abc4e241bb75f0d3de8ac3412e8e47'
+
+            [crates.aws-sdk-account]
+            category = 'AwsSdk'
+            version = '0.12.0'
+            source_hash = 'a0dfc080638b1d803745f0bd66b610131783cf40ab88fd710dce906fc69b983e'
+            model_hash = '179bbfd915093dc3bec5444771da2b20d99a37d104ba25f0acac9aa0d5bb758a'
+            """.trimIndent()
+        )
+
+        assertEquals("some-smithy-rs-revision", manifest.smithyRsRevision)
+        assertEquals("some-doc-revision", manifest.awsDocSdkExamplesRevision)
+        assertEquals(
+            mapOf(
+                "aws-config" to CrateVersion(
+                    category = "AwsRuntime",
+                    version = "0.12.0",
+                    sourceHash = "12d172094a2576e6f4d00a8ba58276c0d4abc4e241bb75f0d3de8ac3412e8e47"
+                ),
+                "aws-sdk-account" to CrateVersion(
+                    category = "AwsSdk",
+                    version = "0.12.0",
+                    sourceHash = "a0dfc080638b1d803745f0bd66b610131783cf40ab88fd710dce906fc69b983e",
+                    modelHash = "179bbfd915093dc3bec5444771da2b20d99a37d104ba25f0acac9aa0d5bb758a"
+                )
+            ),
+            manifest.crates
+        )
+    }
+}

tools/publisher/src/subcommand/fix_manifests.rs

@@ -38,10 +38,18 @@ pub struct FixManifestsArgs {
     /// Checks manifests rather than fixing them
     #[clap(long)]
     check: bool,
+    /// Disable expected version number validation. This should only be used
+    /// when SDK crates are being generated with independent version numbers.
+    #[clap(long)]
+    disable_version_number_validation: bool,
 }
 
 pub async fn subcommand_fix_manifests(
-    FixManifestsArgs { location, check }: &FixManifestsArgs,
+    FixManifestsArgs {
+        location,
+        check,
+        disable_version_number_validation,
+    }: &FixManifestsArgs,
 ) -> Result<()> {
     let mode = match check {
         true => Mode::Check,
@@ -51,7 +59,7 @@ pub async fn subcommand_fix_manifests(
     let mut manifests = read_manifests(Fs::Real, manifest_paths).await?;
     let versions = package_versions(&manifests)?;
 
-    validate::validate_before_fixes(&versions)?;
+    validate::validate_before_fixes(&versions, *disable_version_number_validation)?;
     fix_manifests(Fs::Real, &versions, &mut manifests, mode).await?;
     validate::validate_after_fixes(location).await?;
     info!("Successfully fixed manifests!");

tools/publisher/src/subcommand/fix_manifests/validate.rs

@@ -17,7 +17,15 @@ use tracing::info;
 /// For now, this validates:
 /// - `aws-config` version number matches all `aws-sdk-` prefixed versions
 /// - `aws-smithy-` prefixed versions match `aws-` (NOT `aws-sdk-`) prefixed versions
-pub(super) fn validate_before_fixes(versions: &BTreeMap<String, Version>) -> Result<()> {
+pub(super) fn validate_before_fixes(
+    versions: &BTreeMap<String, Version>,
+    disable_version_number_validation: bool,
+) -> Result<()> {
+    // Later when we only generate independently versioned SDK crates, this flag can become permanent.
+    if disable_version_number_validation {
+        return Ok(());
+    }
+
     info!("Pre-validation manifests...");
     let maybe_sdk_version = versions.get("aws-config");
     let expected_smithy_version = versions
@@ -75,12 +83,12 @@ mod test {
 
     #[track_caller]
     fn expect_success(version_tuples: &[(&'static str, &'static str)]) {
-        validate_before_fixes(&versions(version_tuples)).expect("success");
+        validate_before_fixes(&versions(version_tuples), false).expect("success");
     }
 
     #[track_caller]
     fn expect_failure(message: &str, version_tuples: &[(&'static str, &'static str)]) {
-        if let Err(err) = validate_before_fixes(&versions(version_tuples)) {
+        if let Err(err) = validate_before_fixes(&versions(version_tuples), false) {
             assert_eq!(message, format!("{}", err));
         } else {
             panic!("Expected validation failure");