Fix adhoc SDK codegen and refactor SDK CI (#1624)

* Fix `aws:sdk-codegen-test` build
* Add config for SDK endpoint/readme generation
* Rename `sdk-codegen-test` to `sdk-adhoc-test` and update README
* Add `sdk-adhoc-tests` to CI
* Refactor CI scripts

The `check-rust-runtime-and-tools` script was getting beyond 20
minutes to run, so this commit splits it into `check-aws-config`,
`check-rust-runtimes`, and `check-tools`.

This also enables most of the checks to run before the smoke test SDK is
generated, which should deliver more immediate feedback on PRs.

* Give the canary its own CI step
This commit is contained in:
John DiSanti 2022-08-10 09:08:08 -07:00 committed by GitHub
parent 70a621930f
commit 6e1d1f6c18
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 280 additions and 114 deletions

View File

@ -59,16 +59,19 @@ jobs:
matrix:
# These correspond to scripts in tools/ci-build/scripts that will be run in the Docker build image
test:
- action: check-aws-sdk-adhoc-tests
- action: check-client-codegen-integration-tests
- action: check-client-codegen-unit-tests
- action: check-rust-runtimes
- action: check-sdk-codegen-unit-tests
- action: check-server-codegen-integration-tests
- action: check-server-codegen-unit-tests
- action: check-server-codegen-integration-tests-python
- action: check-server-codegen-unit-tests
- action: check-server-codegen-unit-tests-python
- action: check-server-e2e-test
- action: check-server-python-e2e-test
- action: check-style-and-lints
- action: check-tools
steps:
- uses: actions/checkout@v3
with:
@ -80,8 +83,8 @@ jobs:
# Test all the things that require generated code. Note: the Rust runtimes require codegen
# to be checked since `aws-config` depends on the generated STS client.
test-runtimes-tools-and-sdk:
name: Test Rust Runtimes, Tools, and SDK
test-sdk:
name: Test the SDK
needs: generate
runs-on: ubuntu-latest
# To avoid repeating setup boilerplate, we have the actual test commands
@ -91,12 +94,12 @@ jobs:
matrix:
# These correspond to scripts in tools/ci-build/scripts that will be run in the Docker build image
test:
- action: check-aws-config
- action: check-aws-sdk-canary
- action: check-aws-sdk-services
- action: check-aws-sdk-smoketest-additional-checks
- action: check-aws-sdk-smoketest-docs-clippy-udeps
- action: check-aws-sdk-smoketest-unit-tests
- action: check-aws-sdk-standalone-integration-tests
- action: check-rust-runtimes-and-tools
steps:
- uses: actions/checkout@v3
with:
@ -159,7 +162,7 @@ jobs:
needs:
- generate
- test-codegen
- test-runtimes-tools-and-sdk
- test-sdk
- test-rust-windows
# Run this job even if its dependency jobs fail
if: always()

View File

@ -1,24 +0,0 @@
#!/bin/bash
#
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0
#
# This script contains additional CI checks to run for this specific package
set -e
echo "### Checking for external types in public API"
cargo "+${RUST_NIGHTLY_VERSION:-nightly}" api-linter --all-features --config api-linter.toml
echo "### Checking for duplicate dependency versions in the normal dependency graph with all features enabled"
cargo tree -d --edges normal --all-features
# `aws-config` is not part of the `rust-runtime` workspace. As a result it is not run through
# `check-rust-runtimes-and-tools` and does not get `cargo test --all-features` prior to this point. As a result we do
# not apply `--exclude-all-features` here.
echo "### Testing every combination of features"
cargo hack test --feature-powerset
echo "### Checking that compiling with the minimal versions succeeds"
cargo "+${RUST_NIGHTLY_VERSION:-nightly}" minimal-versions check --all-features

View File

@ -0,0 +1,19 @@
AWS SDK Adhoc Codegen Test
==========================
This module tests that adhoc SDKs can be generated without the rest of the
release automation machinery used to make the official SDK releases.
The `build.gradle.kts` generates a `smithy-build.json` file as part of
the build, and the Smithy build plugin then invokes the SDK codegen
to generate a client.
This module also exists to code generate and execute service specific protocol tests such as
[ApiGateway](https://github.com/awslabs/smithy/blob/main/smithy-aws-protocol-tests/model/restJson1/services/apigateway.smithy).
Usage
-----
```
# From repo root:
./gradlew :aws:sdk-adhoc-test:test
```

View File

@ -8,14 +8,28 @@ extra["moduleName"] = "software.amazon.smithy.kotlin.codegen.test"
tasks["jar"].enabled = false
plugins { id("software.amazon.smithy").version("0.5.3") }
plugins {
val smithyGradlePluginVersion: String by project
id("software.amazon.smithy").version(smithyGradlePluginVersion)
}
val smithyVersion: String by project
val defaultRustDocFlags: String by project
val properties = PropertyRetriever(rootProject, project)
val pluginName = "rust-codegen"
val workingDirUnderBuildDir = "smithyprojections/sdk-codegen-test/"
val workingDirUnderBuildDir = "smithyprojections/sdk-adhoc-test/"
configure<software.amazon.smithy.gradle.SmithyExtension> {
outputDirectory = file("$buildDir/$workingDirUnderBuildDir")
}
buildscript {
val smithyVersion: String by project
dependencies {
classpath("software.amazon.smithy:smithy-cli:$smithyVersion")
}
}
dependencies {
implementation(project(":aws:sdk-codegen"))
@ -25,11 +39,26 @@ dependencies {
}
val allCodegenTests = listOf(
CodegenTest("com.amazonaws.apigateway#BackplaneControlService", "apigateway"),
CodegenTest(
"com.amazonaws.apigateway#BackplaneControlService",
"apigateway",
extraConfig = """
,
"codegen": {
"includeFluentClient": false
},
"customizationConfig": {
"awsSdk": {
"generateReadme": false
}
}
""",
),
)
project.registerGenerateSmithyBuildTask(rootProject, pluginName, allCodegenTests)
project.registerGenerateCargoWorkspaceTask(rootProject, pluginName, allCodegenTests, workingDirUnderBuildDir)
project.registerGenerateCargoConfigTomlTask(buildDir.resolve(workingDirUnderBuildDir))
tasks["smithyBuildJar"].dependsOn("generateSmithyBuild")
tasks["assemble"].finalizedBy("generateCargoWorkspace")

View File

@ -1,10 +0,0 @@
# Codegen Integration Test
This module defines an integration test of the code generation machinery for AWS services. `.build.gradle.kts` will generate a `smithy-build.json` file as part of the build. The Smithy build plugin then invokes our codegen machinery and generates Rust crates.
This module exists to code generate and execute service specific protocol tests like [ApiGateway](https://github.com/awslabs/smithy/blob/main/smithy-aws-protocol-tests/model/restJson1/services/apigateway.smithy).
## Usage
```
# From repo root:
./gradlew :aws:sdk-codegen-test:test
```

View File

@ -46,7 +46,13 @@ class AwsEndpointDecorator : RustCodegenDecorator<ClientCodegenContext> {
private fun endpoints(sdkSettings: SdkSettings): ObjectNode {
if (endpointsCache == null) {
val endpointsJson = sdkSettings.endpointsConfigPath.readText()
val endpointsJson = when (val path = sdkSettings.endpointsConfigPath) {
null -> (
javaClass.getResource("/default-sdk-endpoints.json")
?: throw IllegalStateException("Failed to find default-sdk-endpoints.json in the JAR")
).readText()
else -> path.readText()
}
endpointsCache = Node.parse(endpointsJson).expectObjectNode()
}
return endpointsCache!!
@ -316,7 +322,7 @@ class EndpointResolverGenerator(coreCodegenContext: CoreCodegenContext, private
/**
* Represents a partition from endpoints.json
*/
private inner class PartitionNode(endpointPrefix: String, val config: ObjectNode) {
private inner class PartitionNode(endpointPrefix: String, config: ObjectNode) {
// the partition id/name (e.g. "aws")
val id: String = config.expectStringMember("partition").value

View File

@ -185,17 +185,21 @@ private class AwsFluentClientExtensions(types: Types) {
}
}
private class AwsFluentClientDocs(coreCodegenContext: CoreCodegenContext) : FluentClientCustomization() {
private class AwsFluentClientDocs(private val coreCodegenContext: CoreCodegenContext) : FluentClientCustomization() {
private val serviceName = coreCodegenContext.serviceShape.expectTrait<TitleTrait>().value
private val serviceShape = coreCodegenContext.serviceShape
private val crateName = coreCodegenContext.moduleUseName()
private val codegenScope =
arrayOf("aws_config" to coreCodegenContext.runtimeConfig.awsConfig().copy(scope = DependencyScope.Dev).asType())
// Usage docs on STS must be suppressed—aws-config cannot be added as a dev-dependency because it would create
// a circular dependency
// If no `aws-config` version is provided, assume that docs referencing `aws-config` cannot be given.
// Also, STS and SSO must NOT reference `aws-config` since that would create a circular dependency.
private fun suppressUsageDocs(): Boolean =
setOf(ShapeId.from("com.amazonaws.sts#AWSSecurityTokenServiceV20110615"), ShapeId.from("com.amazonaws.sso#SWBPortalService")).contains(serviceShape.id)
SdkSettings.from(coreCodegenContext.settings).awsConfigVersion == null ||
setOf(
ShapeId.from("com.amazonaws.sts#AWSSecurityTokenServiceV20110615"),
ShapeId.from("com.amazonaws.sso#SWBPortalService"),
).contains(serviceShape.id)
override fun section(section: FluentClientSection): Writable {
return when (section) {

View File

@ -30,13 +30,32 @@ class AwsReadmeDecorator : RustCodegenDecorator<ClientCodegenContext> {
override val name: String = "AwsReadmeDecorator"
override val order: Byte = 0
private val logger: Logger = Logger.getLogger(javaClass.name)
override fun supportsCodegenContext(clazz: Class<out CoreCodegenContext>): Boolean =
clazz.isAssignableFrom(ClientCodegenContext::class.java)
override fun crateManifestCustomizations(codegenContext: ClientCodegenContext): ManifestCustomizations =
mapOf("package" to mapOf("readme" to "README.md"))
if (generateReadme(codegenContext)) {
mapOf("package" to mapOf("readme" to "README.md"))
} else {
emptyMap()
}
override fun extras(codegenContext: ClientCodegenContext, rustCrate: RustCrate) {
if (generateReadme(codegenContext)) {
AwsSdkReadmeGenerator().generateReadme(codegenContext, rustCrate)
}
}
private fun generateReadme(codegenContext: ClientCodegenContext) =
SdkSettings.from(codegenContext.settings).generateReadme
}
internal class AwsSdkReadmeGenerator {
private val logger: Logger = Logger.getLogger(javaClass.name)
internal fun generateReadme(codegenContext: ClientCodegenContext, rustCrate: RustCrate) {
val awsConfigVersion = SdkSettings.from(codegenContext.settings).awsConfigVersion
?: throw IllegalStateException("missing `awsConfigVersion` codegen setting")
rustCrate.withFile("README.md") { writer ->
val description = normalizeDescription(
codegenContext.moduleName,
@ -111,9 +130,6 @@ class AwsReadmeDecorator : RustCodegenDecorator<ClientCodegenContext> {
}
}
override fun supportsCodegenContext(clazz: Class<out CoreCodegenContext>): Boolean =
clazz.isAssignableFrom(ClientCodegenContext::class.java)
/**
* Strips HTML from the description and makes it human-readable Markdown.
*/

View File

@ -20,25 +20,22 @@ class SdkSettings private constructor(private val awsSdk: ObjectNode?) {
}
/** Path to the `sdk-default-configuration.json` config file */
val defaultsConfigPath: Path get() =
Paths.get(
awsSdk?.getStringMember("defaultConfigPath")?.orNull()?.value
?: throw IllegalStateException("missing defaultConfigPath property"),
)
val defaultsConfigPath: Path? get() =
awsSdk?.getStringMember("defaultConfigPath")?.orNull()?.value.let { Paths.get(it) }
/** Path to the `sdk-endpoints.json` configuration */
val endpointsConfigPath: Path get() =
Paths.get(
awsSdk?.getStringMember("endpointsConfigPath")?.orNull()?.value
?: throw IllegalStateException("missing endpointsConfigPath property"),
)
val endpointsConfigPath: Path? get() =
awsSdk?.getStringMember("endpointsConfigPath")?.orNull()?.value?.let { Paths.get(it) }
/** Path to AWS SDK integration tests */
val integrationTestPath: String get() =
awsSdk?.getStringMember("integrationTestPath")?.orNull()?.value ?: "aws/sdk/integration-tests"
/** Version number of the `aws-config` crate */
val awsConfigVersion: String get() =
val awsConfigVersion: String? get() =
awsSdk?.getStringMember("awsConfigVersion")?.orNull()?.value
?: throw IllegalStateException("missing `awsConfigVersion` codegen setting")
/** Whether or not to generate a README */
val generateReadme: Boolean get() =
awsSdk?.getBooleanMember("generateReadme")?.orNull()?.value ?: true
}

View File

@ -0,0 +1,44 @@
{
"partitions" : [
{
"defaults": {
"hostname": "{service}.{region}.{dnsSuffix}",
"protocols": [
"https"
],
"signatureVersions": [
"v4"
],
"variants": [
{
"dnsSuffix": "amazonaws.com",
"hostname": "{service}-fips.{region}.{dnsSuffix}",
"tags": [
"fips"
]
},
{
"dnsSuffix": "api.aws",
"hostname": "{service}-fips.{region}.{dnsSuffix}",
"tags": [
"dualstack",
"fips"
]
},
{
"dnsSuffix": "api.aws",
"hostname": "{service}.{region}.{dnsSuffix}",
"tags": [
"dualstack"
]
}
]
},
"dnsSuffix": "amazonaws.com",
"partition": "aws",
"partitionName": "AWS Standard",
"regionRegex": "^(us|eu|ap|sa|ca|me|af)\\-\\w+\\-\\d+$"
}
],
"version" : 3
}

View File

@ -5,7 +5,7 @@
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Test
import software.amazon.smithy.rustsdk.AwsReadmeDecorator
import software.amazon.smithy.rustsdk.AwsSdkReadmeGenerator
class AwsReadmeDecoratorTest {
@Test
@ -18,7 +18,7 @@ class AwsReadmeDecoratorTest {
More information [can be found here](https://example.com).
""".trimIndent(),
AwsReadmeDecorator().normalizeDescription(
AwsSdkReadmeGenerator().normalizeDescription(
"",
"""
<fullname>Some service</fullname>
@ -44,7 +44,7 @@ class AwsReadmeDecoratorTest {
More text.
""".trimIndent(),
AwsReadmeDecorator().normalizeDescription(
AwsSdkReadmeGenerator().normalizeDescription(
"",
"""
<p>Some text introducing a list:
@ -81,7 +81,7 @@ class AwsReadmeDecoratorTest {
Some trailing text.
""".trimIndent(),
AwsReadmeDecorator().normalizeDescription(
AwsSdkReadmeGenerator().normalizeDescription(
"",
"""
<p>Some text introducing a description list:

View File

@ -15,7 +15,8 @@ extra["moduleName"] = "software.amazon.smithy.rust.awssdk"
tasks["jar"].enabled = false
plugins {
id("software.amazon.smithy").version("0.6.0")
val smithyGradlePluginVersion: String by project
id("software.amazon.smithy").version(smithyGradlePluginVersion)
}
configure<software.amazon.smithy.gradle.SmithyExtension> {

24
ci.mk
View File

@ -16,6 +16,18 @@ CI_ACTION=$(CI_BUILD)/ci-action
acquire-build-image:
$(CI_BUILD)/acquire-build-image
.PHONY: check-aws-config
check-aws-config: generate-aws-sdk-smoketest
$(CI_ACTION) $@ $(ARGS)
.PHONY: check-aws-sdk-canary
check-aws-sdk-canary: generate-aws-sdk-smoketest
$(CI_ACTION) $@ $(ARGS)
.PHONY: check-aws-sdk-adhoc-tests
check-aws-sdk-adhoc-tests:
$(CI_ACTION) $@ $(ARGS)
.PHONY: check-aws-sdk-examples
check-aws-sdk-examples: generate-aws-sdk
$(CI_ACTION) $@ $(ARGS)
@ -24,10 +36,6 @@ check-aws-sdk-examples: generate-aws-sdk
check-aws-sdk-services: generate-aws-sdk
$(CI_ACTION) $@ $(ARGS)
.PHONY: check-aws-sdk-smoketest-additional-checks
check-aws-sdk-smoketest-additional-checks: generate-aws-sdk-smoketest
$(CI_ACTION) $@ $(ARGS)
.PHONY: check-aws-sdk-smoketest-docs-clippy-udeps
check-aws-sdk-smoketest-docs-clippy-udeps: generate-aws-sdk-smoketest
$(CI_ACTION) $@ $(ARGS)
@ -48,8 +56,12 @@ check-client-codegen-integration-tests:
check-client-codegen-unit-tests:
$(CI_ACTION) $@ $(ARGS)
.PHONY: check-rust-runtimes-and-tools
check-rust-runtimes-and-tools: generate-aws-sdk-smoketest
.PHONY: check-rust-runtimes
check-rust-runtimes:
$(CI_ACTION) $@ $(ARGS)
.PHONY: check-tools
check-tools:
$(CI_ACTION) $@ $(ARGS)
.PHONY: check-sdk-codegen-unit-tests

View File

@ -9,7 +9,10 @@ extra["moduleName"] = "software.amazon.smithy.rust.kotlin.codegen.server.test"
tasks["jar"].enabled = false
plugins { id("software.amazon.smithy").version("0.5.3") }
plugins {
val smithyGradlePluginVersion: String by project
id("software.amazon.smithy").version(smithyGradlePluginVersion)
}
val smithyVersion: String by project
val defaultRustDocFlags: String by project

View File

@ -9,7 +9,10 @@ extra["moduleName"] = "software.amazon.smithy.rust.kotlin.codegen.server.python.
tasks["jar"].enabled = false
plugins { id("software.amazon.smithy") }
plugins {
val smithyGradlePluginVersion: String by project
id("software.amazon.smithy").version(smithyGradlePluginVersion)
}
val smithyVersion: String by project
val defaultRustDocFlags: String by project

View File

@ -8,7 +8,10 @@ extra["moduleName"] = "software.amazon.smithy.kotlin.codegen.test"
tasks["jar"].enabled = false
plugins { id("software.amazon.smithy").version("0.5.3") }
plugins {
val smithyGradlePluginVersion: String by project
id("software.amazon.smithy").version(smithyGradlePluginVersion)
}
val smithyVersion: String by project
val defaultRustDocFlags: String by project

View File

@ -14,6 +14,7 @@ smithy.rs.runtime.crate.version=0.47.0
kotlin.code.style=official
# codegen
smithyGradlePluginVersion=0.6.0
smithyVersion=1.21.0
# kotlin

View File

@ -13,6 +13,6 @@ include(":codegen-server-test")
include(":codegen-server-test:python")
include(":rust-runtime")
include(":aws:sdk-codegen")
include(":aws:sdk-codegen-test")
include(":aws:sdk-adhoc-test")
include(":aws:sdk")
include(":aws:rust-runtime")

View File

@ -0,0 +1,41 @@
#!/bin/bash
#
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0
#
C_YELLOW='\033[1;33m'
C_RESET='\033[0m'
set -eux
cd smithy-rs
# Make aws-config (which depends on generated services) available to additional checks
mkdir -p aws/sdk/build
mv ../aws-sdk-smoketest aws/sdk/build/aws-sdk
echo -e "${C_YELLOW}# Testing aws-config...${C_RESET}"
pushd "aws/rust-runtime/aws-config" &>/dev/null
echo "${C_YELLOW}## Running 'cargo clippy'${C_RESET}"
cargo clippy --all-features
echo "${C_YELLOW}## Running 'cargo test'${C_RESET}"
cargo test --all-features
echo "${C_YELLOW}## Running 'cargo doc'${C_RESET}"
cargo doc --no-deps --document-private-items --all-features
echo "${C_YELLOW}## Running 'cargo minimal-versions check'${C_RESET}"
cargo +"${RUST_NIGHTLY_VERSION}" minimal-versions check --all-features
echo "${C_YELLOW}## Checking for external types in public API${C_RESET}"
cargo "+${RUST_NIGHTLY_VERSION:-nightly}" api-linter --all-features --config api-linter.toml
echo "${C_YELLOW}## Checking for duplicate dependency versions in the normal dependency graph with all features enabled${C_RESET}"
cargo tree -d --edges normal --all-features
echo "${C_YELLOW}## Testing every combination of features${C_RESET}"
cargo hack test --feature-powerset --exclude-all-features
popd &>/dev/null

View File

@ -0,0 +1,9 @@
#!/bin/bash
#
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0
#
set -eux
cd smithy-rs
./gradlew aws:sdk-adhoc-test:test

View File

@ -3,18 +3,27 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Run by CI to check the canary-lambda
set -e
cd "$(dirname "$0")"
C_YELLOW='\033[1;33m'
C_RESET='\033[0m'
set -eux
cd smithy-rs
# Make aws-config (which depends on generated services) available to additional checks
mkdir -p aws/sdk/build
mv ../aws-sdk-smoketest aws/sdk/build/aws-sdk
SDK_PATH="$(git rev-parse --show-toplevel)"/aws/sdk/build/aws-sdk/sdk
if [[ "${GITHUB_ACTIONS}" == "true" ]]; then
SDK_PATH="$(git rev-parse --show-toplevel)"/aws-sdk/sdk
fi
pushd tools/ci-cdk/canary-lambda
# The canary-lambda doesn't have a Cargo.toml in a fresh checkout of smithy-rs, so generate one before checks
pushd ../canary-runner
echo "${C_YELLOW}## Generating Cargo.toml for the canary-lambda...${C_RESET}"
cargo run -- build-bundle --manifest-only --canary-path ../canary-lambda --sdk-path "${SDK_PATH}"
popd
echo "${C_YELLOW}## Running 'cargo clippy' for the canary-lambda...${C_RESET}"
cargo clippy --all-features
popd

View File

@ -1,10 +0,0 @@
#!/bin/bash
#
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0
#
set -eux
cd aws-sdk-smoketest
# We run these on the generated SDK due to aws-config depending on generated services
../smithy-rs/tools/additional-per-crate-checks.sh ./sdk/

View File

@ -0,0 +1,27 @@
#!/bin/bash
#
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0
#
C_YELLOW='\033[1;33m'
C_RESET='\033[0m'
set -eux
cd smithy-rs
for runtime_path in \
"rust-runtime" \
"aws/rust-runtime"
do
echo -e "${C_YELLOW}Testing ${runtime_path}...${C_RESET}"
pushd "${runtime_path}" &>/dev/null
cargo clippy --all-features
cargo test --all-features
cargo doc --no-deps --document-private-items --all-features
cargo +"${RUST_NIGHTLY_VERSION}" minimal-versions check --all-features
popd &>/dev/null
echo -e "${C_YELLOW}Running additional per-crate checks for ${runtime_path}...${C_RESET}"
./tools/additional-per-crate-checks.sh "${runtime_path}"
done

View File

@ -10,23 +10,6 @@ C_RESET='\033[0m'
set -eux
cd smithy-rs
# Make aws-config (which depends on generated services) available to additional checks
mkdir -p aws/sdk/build
mv ../aws-sdk-smoketest aws/sdk/build/aws-sdk
for runtime_path in \
"rust-runtime" \
"aws/rust-runtime"
do
echo -e "${C_YELLOW}Testing ${runtime_path}...${C_RESET}"
pushd "${runtime_path}" &>/dev/null
cargo clippy --all-features
cargo test --all-features
cargo doc --no-deps --document-private-items --all-features
cargo +"${RUST_NIGHTLY_VERSION}" minimal-versions check --all-features
popd &>/dev/null
done
# test_tool tool_path rust_version
function test_tool {
local tool_path="$1"