diff --git a/.gitignore b/.gitignore index de9b50697c..e1dc05524b 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,6 @@ gradle-app.setting # MacOS .DS_Store + +# Rust build artifacts +target/ diff --git a/aws/sdk/build.gradle.kts b/aws/sdk/build.gradle.kts index cd74c41efe..f86c7d6594 100644 --- a/aws/sdk/build.gradle.kts +++ b/aws/sdk/build.gradle.kts @@ -105,6 +105,11 @@ task("relocateServices") { from("$buildDir/smithyprojections/sdk/${it.module}/rust-codegen") into(sdkOutputDir.resolve(it.module)) } + + copy { + from(projectDir.resolve("integration-tests/${it.module}/tests")) + into(sdkOutputDir.resolve(it.module).resolve("tests")) + } } } outputs.upToDateWhen { false } diff --git a/aws/sdk/integration-tests/.gitignore b/aws/sdk/integration-tests/.gitignore new file mode 100644 index 0000000000..1e7caa9ea8 --- /dev/null +++ b/aws/sdk/integration-tests/.gitignore @@ -0,0 +1,2 @@ +Cargo.lock +target/ diff --git a/aws/sdk/integration-tests/README.md b/aws/sdk/integration-tests/README.md new file mode 100644 index 0000000000..882ca50fb1 --- /dev/null +++ b/aws/sdk/integration-tests/README.md @@ -0,0 +1,8 @@ +# Handwritten Integration Test Root + +This folder contains hand-written integration tests that are specific to individual services. In order for your test to be merged into the final artifact: + +- The crate name must match the generated crate name, eg. `kms`, `dynamodb` +- Your test must be placed into the `tests` folder. **Everything else in your test crate is ignored.** + +The contents of the `test` folder will be combined with codegenerated integration tests & inserted into the `tests` folder of the final generated service crate. diff --git a/aws/sdk/integration-tests/kms/Cargo.toml b/aws/sdk/integration-tests/kms/Cargo.toml new file mode 100644 index 0000000000..52334abe6a --- /dev/null +++ b/aws/sdk/integration-tests/kms/Cargo.toml @@ -0,0 +1,11 @@ +# This Cargo.toml is unused in generated code. It exists solely to enable these tests to compile in-situ +[package] +name = "kms-tests" +version = "0.1.0" +authors = ["Russell Cohen "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +kms = { path = "../../build/aws-sdk/kms" } diff --git a/aws/sdk/integration-tests/kms/src/lib.rs b/aws/sdk/integration-tests/kms/src/lib.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/aws/sdk/integration-tests/kms/tests/sensitive-it.rs b/aws/sdk/integration-tests/kms/tests/sensitive-it.rs new file mode 100644 index 0000000000..49bab936c7 --- /dev/null +++ b/aws/sdk/integration-tests/kms/tests/sensitive-it.rs @@ -0,0 +1,12 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +use kms::output::GenerateRandomOutput; +use kms::Blob; +#[test] +fn validate_sensitive_trait() { + let output = GenerateRandomOutput::builder().plaintext(Blob::new("some output")).build(); + assert_eq!(format!("{:?}", output), "GenerateRandomOutput { plaintext: \"*** Sensitive Data Redacted ***\" }"); +} diff --git a/codegen/src/main/kotlin/software/amazon/smithy/rust/codegen/smithy/CodegenVisitor.kt b/codegen/src/main/kotlin/software/amazon/smithy/rust/codegen/smithy/CodegenVisitor.kt index f0479cf39d..fda9c65020 100644 --- a/codegen/src/main/kotlin/software/amazon/smithy/rust/codegen/smithy/CodegenVisitor.kt +++ b/codegen/src/main/kotlin/software/amazon/smithy/rust/codegen/smithy/CodegenVisitor.kt @@ -25,6 +25,7 @@ import software.amazon.smithy.rust.codegen.smithy.generators.ModelBuilderGenerat import software.amazon.smithy.rust.codegen.smithy.generators.ProtocolConfig import software.amazon.smithy.rust.codegen.smithy.generators.ProtocolGeneratorFactory import software.amazon.smithy.rust.codegen.smithy.generators.ServiceGenerator +import software.amazon.smithy.rust.codegen.smithy.generators.SmithyTypesPubUseGenerator import software.amazon.smithy.rust.codegen.smithy.generators.StructureGenerator import software.amazon.smithy.rust.codegen.smithy.generators.UnionGenerator import software.amazon.smithy.rust.codegen.smithy.generators.implBlock @@ -35,7 +36,8 @@ import software.amazon.smithy.rust.codegen.util.CommandFailed import software.amazon.smithy.rust.codegen.util.runCommand import java.util.logging.Logger -class CodegenVisitor(context: PluginContext, private val codegenDecorator: RustCodegenDecorator) : ShapeVisitor.Default() { +class CodegenVisitor(context: PluginContext, private val codegenDecorator: RustCodegenDecorator) : + ShapeVisitor.Default() { private val logger = Logger.getLogger(javaClass.name) private val settings = RustSettings.from(context.model, context.settings) @@ -53,13 +55,19 @@ class CodegenVisitor(context: PluginContext, private val codegenDecorator: RustC SymbolVisitorConfig(runtimeConfig = settings.runtimeConfig, codegenConfig = settings.codegenConfig) val baseModel = baselineTransform(context.model) val service = settings.getService(baseModel) - val (protocol, generator) = ProtocolLoader(codegenDecorator.protocols(service.id, ProtocolLoader.DefaultProtocols)).protocolFor(context.model, service) + val (protocol, generator) = ProtocolLoader( + codegenDecorator.protocols( + service.id, + ProtocolLoader.DefaultProtocols + ) + ).protocolFor(context.model, service) protocolGenerator = generator model = generator.transformModel(baseModel) val baseProvider = RustCodegenPlugin.BaseSymbolProvider(model, symbolVisitorConfig) symbolProvider = codegenDecorator.symbolProvider(generator.symbolProvider(model, baseProvider)) - protocolConfig = ProtocolConfig(model, symbolProvider, settings.runtimeConfig, service, protocol, settings.moduleName) + protocolConfig = + ProtocolConfig(model, symbolProvider, settings.runtimeConfig, service, protocol, settings.moduleName) writers = CodegenWriterDelegator( context.fileManifest, symbolProvider, @@ -76,7 +84,13 @@ class CodegenVisitor(context: PluginContext, private val codegenDecorator: RustC val serviceShapes = Walker(model).walkShapes(service) serviceShapes.forEach { it.accept(this) } // TODO: if we end up with a lot of these on-by-default customizations, we may want to refactor them somewhere - writers.finalize(settings, codegenDecorator.libRsCustomizations(protocolConfig, listOf(CrateVersionGenerator()))) + writers.finalize( + settings, + codegenDecorator.libRsCustomizations( + protocolConfig, + listOf(CrateVersionGenerator(), SmithyTypesPubUseGenerator(protocolConfig.runtimeConfig)) + ) + ) try { "cargo fmt".runCommand(fileManifest.baseDir) } catch (_: CommandFailed) { diff --git a/codegen/src/main/kotlin/software/amazon/smithy/rust/codegen/smithy/RuntimeTypes.kt b/codegen/src/main/kotlin/software/amazon/smithy/rust/codegen/smithy/RuntimeTypes.kt index a8cb6cdf95..575a9afe9c 100644 --- a/codegen/src/main/kotlin/software/amazon/smithy/rust/codegen/smithy/RuntimeTypes.kt +++ b/codegen/src/main/kotlin/software/amazon/smithy/rust/codegen/smithy/RuntimeTypes.kt @@ -59,12 +59,15 @@ data class RuntimeType(val name: String?, val dependency: RustDependency?, val n // val Blob = RuntimeType("Blob", RustDependency.IO_CORE, "blob") val From = RuntimeType("From", dependency = null, namespace = "std::convert") val AsRef = RuntimeType("AsRef", dependency = null, namespace = "std::convert") - fun StdFmt(member: String) = RuntimeType("fmt::$member", dependency = null, namespace = "std") + fun StdFmt(member: String?) = RuntimeType(member, dependency = null, namespace = "std::fmt") fun Std(member: String) = RuntimeType(member, dependency = null, namespace = "std") val StdError = RuntimeType("Error", dependency = null, namespace = "std::error") val HashSet = RuntimeType(RustType.SetType, dependency = null, namespace = "std::collections") val HashMap = RuntimeType("HashMap", dependency = null, namespace = "std::collections") val ByteSlab = RuntimeType("Vec", dependency = null, namespace = "std::vec") + val Debug = StdFmt("Debug") + val PartialEq = Std("cmp::PartialEq") + val Clone = Std("clone::Clone") fun Instant(runtimeConfig: RuntimeConfig) = RuntimeType("Instant", CargoDependency.SmithyTypes(runtimeConfig), "${runtimeConfig.cratePrefix}_types") diff --git a/codegen/src/main/kotlin/software/amazon/smithy/rust/codegen/smithy/SymbolMetadataProvider.kt b/codegen/src/main/kotlin/software/amazon/smithy/rust/codegen/smithy/SymbolMetadataProvider.kt index b2b190baf5..bdcc997ea8 100644 --- a/codegen/src/main/kotlin/software/amazon/smithy/rust/codegen/smithy/SymbolMetadataProvider.kt +++ b/codegen/src/main/kotlin/software/amazon/smithy/rust/codegen/smithy/SymbolMetadataProvider.kt @@ -91,8 +91,9 @@ class BaseSymbolMetadataProvider(base: RustSymbolProvider) : SymbolMetadataProvi } companion object { - private val defaultDerives = - listOf(RuntimeType.StdFmt("Debug"), RuntimeType.Std("cmp::PartialEq"), RuntimeType.Std("clone::Clone")) + private val defaultDerives = with(RuntimeType) { + listOf(Debug, PartialEq, Clone) + } } } diff --git a/codegen/src/main/kotlin/software/amazon/smithy/rust/codegen/smithy/generators/SmithyTypesPubUseGenerator.kt b/codegen/src/main/kotlin/software/amazon/smithy/rust/codegen/smithy/generators/SmithyTypesPubUseGenerator.kt new file mode 100644 index 0000000000..45f1113153 --- /dev/null +++ b/codegen/src/main/kotlin/software/amazon/smithy/rust/codegen/smithy/generators/SmithyTypesPubUseGenerator.kt @@ -0,0 +1,19 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +package software.amazon.smithy.rust.codegen.smithy.generators + +import software.amazon.smithy.rust.codegen.rustlang.rust +import software.amazon.smithy.rust.codegen.rustlang.writable +import software.amazon.smithy.rust.codegen.smithy.RuntimeConfig +import software.amazon.smithy.rust.codegen.smithy.RuntimeType + +class SmithyTypesPubUseGenerator(private val runtimeConfig: RuntimeConfig) : LibRsCustomization() { + override fun section(section: LibRsSection) = writable { + when (section) { + LibRsSection.Body -> rust("pub use #T;", RuntimeType.Blob(runtimeConfig)) + } + } +} diff --git a/codegen/src/main/kotlin/software/amazon/smithy/rust/codegen/smithy/generators/StructureGenerator.kt b/codegen/src/main/kotlin/software/amazon/smithy/rust/codegen/smithy/generators/StructureGenerator.kt index b4824baab7..432a85a9ff 100644 --- a/codegen/src/main/kotlin/software/amazon/smithy/rust/codegen/smithy/generators/StructureGenerator.kt +++ b/codegen/src/main/kotlin/software/amazon/smithy/rust/codegen/smithy/generators/StructureGenerator.kt @@ -11,15 +11,19 @@ import software.amazon.smithy.model.shapes.MemberShape import software.amazon.smithy.model.shapes.Shape import software.amazon.smithy.model.shapes.StructureShape import software.amazon.smithy.model.traits.ErrorTrait +import software.amazon.smithy.model.traits.SensitiveTrait import software.amazon.smithy.rust.codegen.rustlang.RustType import software.amazon.smithy.rust.codegen.rustlang.RustWriter import software.amazon.smithy.rust.codegen.rustlang.documentShape +import software.amazon.smithy.rust.codegen.rustlang.rust import software.amazon.smithy.rust.codegen.rustlang.rustBlock +import software.amazon.smithy.rust.codegen.smithy.RuntimeType import software.amazon.smithy.rust.codegen.smithy.RustSymbolProvider import software.amazon.smithy.rust.codegen.smithy.canUseDefault import software.amazon.smithy.rust.codegen.smithy.expectRustMetadata import software.amazon.smithy.rust.codegen.smithy.isOptional import software.amazon.smithy.rust.codegen.smithy.rustType +import software.amazon.smithy.rust.codegen.util.dq fun RustWriter.implBlock(structureShape: Shape, symbolProvider: SymbolProvider, block: RustWriter.() -> Unit) { rustBlock("impl ${symbolProvider.toSymbol(structureShape).name}") { @@ -27,6 +31,9 @@ fun RustWriter.implBlock(structureShape: Shape, symbolProvider: SymbolProvider, } } +fun StructureShape.hasSensitiveMember(model: Model) = + this.members().any { it.getMemberTrait(model, SensitiveTrait::class.java).isPresent } + class StructureGenerator( val model: Model, private val symbolProvider: RustSymbolProvider, @@ -34,6 +41,7 @@ class StructureGenerator( private val shape: StructureShape ) { private val members: List = shape.allMembers.values.toList() + private val name = symbolProvider.toSymbol(shape).name fun render() { renderStructure() @@ -72,13 +80,34 @@ class StructureGenerator( } else "" } + /** Render a custom debug implementation + * When [SensitiveTrait] support is required, render a custom debug implementation to redact sensitive data + */ + private fun renderDebugImpl() { + writer.rustBlock("impl ${lifetimeDeclaration()} #T for $name ${lifetimeDeclaration()}", RuntimeType.Debug) { + writer.rustBlock("fn fmt(&self, f: &mut #1T::Formatter<'_>) -> #1T::Result", RuntimeType.StdFmt(null)) { + rust("""let mut formatter = f.debug_struct(${name.dq()});""") + members.forEach { member -> + val memberName = symbolProvider.toMemberName(member) + if (member.getMemberTrait(model, SensitiveTrait::class.java).isPresent) { + rust("""formatter.field(${memberName.dq()}, &"*** Sensitive Data Redacted ***");""") + } else { + rust("formatter.field(${memberName.dq()}, &self.$memberName);") + } + } + rust("formatter.finish()") + } + } + } + private fun renderStructure() { val symbol = symbolProvider.toSymbol(shape) val containerMeta = symbol.expectRustMetadata() writer.documentShape(shape, model) - containerMeta.render(writer) + val withoutDebug = containerMeta.derives.copy(derives = containerMeta.derives.derives - RuntimeType.Debug) + containerMeta.copy(derives = withoutDebug).render(writer) - writer.rustBlock("struct ${symbol.name} ${lifetimeDeclaration()}") { + writer.rustBlock("struct $name ${lifetimeDeclaration()}") { members.forEach { member -> val memberName = symbolProvider.toMemberName(member) writer.documentShape(member, model) @@ -86,5 +115,7 @@ class StructureGenerator( write("$memberName: #T,", symbolProvider.toSymbol(member)) } } + + renderDebugImpl() } } diff --git a/codegen/src/test/kotlin/software/amazon/smithy/rust/codegen/generators/StructureGeneratorTest.kt b/codegen/src/test/kotlin/software/amazon/smithy/rust/codegen/generators/StructureGeneratorTest.kt index 13b5e20c8b..294599f359 100644 --- a/codegen/src/test/kotlin/software/amazon/smithy/rust/codegen/generators/StructureGeneratorTest.kt +++ b/codegen/src/test/kotlin/software/amazon/smithy/rust/codegen/generators/StructureGeneratorTest.kt @@ -7,7 +7,6 @@ package software.amazon.smithy.rust.codegen.generators import io.kotest.matchers.string.shouldContainInOrder import org.junit.jupiter.api.Test -import software.amazon.smithy.model.shapes.ShapeId import software.amazon.smithy.model.shapes.StructureShape import software.amazon.smithy.rust.codegen.rustlang.Custom import software.amazon.smithy.rust.codegen.rustlang.RustMetadata @@ -47,10 +46,23 @@ class StructureGeneratorTest { structure MyError { message: String } + + @sensitive + string SecretKey + + structure Credentials { + username: String, + @sensitive + password: String, + + // test that sensitive can be applied directly to a member or to the shape + secretKey: SecretKey + } """.asSmithyModel() - val struct = model.expectShape(ShapeId.from("com.test#MyStruct"), StructureShape::class.java) - val inner = model.expectShape(ShapeId.from("com.test#Inner"), StructureShape::class.java) - val error = model.expectShape(ShapeId.from("com.test#MyError"), StructureShape::class.java) + val struct = model.lookup("com.test#MyStruct") + val inner = model.lookup("com.test#Inner") + val credentials = model.lookup("com.test#Credentials") + val error = model.lookup("com.test#MyError") } @Test @@ -115,6 +127,25 @@ class StructureGeneratorTest { ) } + @Test + fun `generate a custom debug implementation when the sensitive trait is present`() { + val provider = testSymbolProvider(model) + val writer = RustWriter.forModule("lib") + val generator = StructureGenerator(model, provider, writer, credentials) + generator.render() + writer.unitTest( + """ + let creds = Credentials { + username: Some("not_redacted".to_owned()), + password: Some("don't leak me".to_owned()), + secret_key: Some("don't leak me".to_owned()) + }; + assert_eq!(format!("{:?}", creds), "Credentials { username: Some(\"not_redacted\"), password: \"*** Sensitive Data Redacted ***\", secret_key: \"*** Sensitive Data Redacted ***\" }"); + """ + ) + writer.compileAndTest() + } + @Test fun `attach docs to everything`() { val model = """