mirror of https://github.com/smithy-lang/smithy-rs
Overhaul gradle inclusion/exclusion logic (#620)
* Overhaul gradle inclusion/exclusion logic * Update changelog * split tier1 services onto multiple lines
This commit is contained in:
Russell Cohen
GitHub
parent
7a6414ea35
commit
03ae7cc6a1
|
|
@ -345,9 +345,7 @@ jobs:
|
|||
with:
|
||||
java-version: ${{ env.java_version }}
|
||||
- name: generate a check all services
|
||||
run: ./gradlew :aws:sdk:cargoCheck
|
||||
env:
|
||||
GENERATE_ALL_SERVICES: 'true'
|
||||
run: ./gradlew -Paws.fullsdk=true :aws:sdk:cargoCheck
|
||||
- name: Generate a name for the SDK
|
||||
id: gen-name
|
||||
run: echo "name=${GITHUB_REF##*/}" >> $GITHUB_ENV
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
## vNext (Month Day Year)
|
||||
**New This Week**
|
||||
- :bug: Correctly encode HTTP Checksums using base64 instead of hex. Fixes aws-sdk-rust#164. (#615)
|
||||
- Update SDK gradle build logic to use gradle properties (#620)
|
||||
- (When complete) Add profile file provider for region (#594, #xyz)
|
||||
- Overhaul serialization/deserialization of numeric/boolean types. This resolves issues around serialization of NaN/Infinity and should also reduce the number of allocations required during serialization. (#618)
|
||||
- Update SQS example to clarify usage of FIFO vs. standard queues (#622, @trevorrobertsjr)
|
||||
|
|
|
|||
|
|
@ -14,3 +14,18 @@ Generate, compile, and test an SDK:
|
|||
|
||||
Run an SDK example:
|
||||
`./gradlew :aws:sdk:runExample --example dynamo-helloworld`
|
||||
|
||||
## Controlling service generation
|
||||
You can use gradle properties to opt/out of generating specific services:
|
||||
```bash
|
||||
# generate only s3,ec2,sts
|
||||
./gradlew -Paws.services=+s3,+ec2,+sts :aws:sdk:assemble
|
||||
|
||||
# generate a complete SDK for release
|
||||
./gradlew -Paws.fullsdk=true :aws:sdk:assemble
|
||||
```
|
||||
|
||||
The generation logic is as follows:
|
||||
1. If `aws.services` is specified, generate an SDK based on the inclusion/exclusion list.
|
||||
2. Otherwise, if `aws.fullsdk` is specified generate an SDK based on `aws.services.fullsdk`.
|
||||
3. Otherwise, generate an SDK based on `aws.services.tier1`
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import software.amazon.smithy.model.Model
|
|||
import software.amazon.smithy.model.shapes.ServiceShape
|
||||
import software.amazon.smithy.aws.traits.ServiceTrait
|
||||
import kotlin.streams.toList
|
||||
import java.util.*
|
||||
|
||||
extra["displayName"] = "Smithy :: Rust :: AWS-SDK"
|
||||
extra["moduleName"] = "software.amazon.smithy.rust.awssdk"
|
||||
|
|
@ -47,66 +48,52 @@ dependencies {
|
|||
implementation("software.amazon.smithy:smithy-aws-cloudformation-traits:$smithyVersion")
|
||||
}
|
||||
|
||||
// Tier 1 Services have examples and tests
|
||||
val tier1Services = setOf(
|
||||
"apigateway",
|
||||
"applicationautoscaling",
|
||||
"autoscaling",
|
||||
"autoscalingplans",
|
||||
"batch",
|
||||
"cloudformation",
|
||||
"cloudwatch",
|
||||
"cloudwatch",
|
||||
"cloudwatchlogs",
|
||||
"cognitoidentity",
|
||||
"cognitoidentityprovider",
|
||||
"cognitosync",
|
||||
"config",
|
||||
"dynamodb",
|
||||
"ebs",
|
||||
"ec2",
|
||||
"ecr",
|
||||
"ecs",
|
||||
"eks",
|
||||
"iam",
|
||||
"kinesis",
|
||||
"kms",
|
||||
"lambda",
|
||||
"medialive",
|
||||
"mediapackage",
|
||||
"polly",
|
||||
"qldb",
|
||||
"qldbsession",
|
||||
"rds",
|
||||
"rdsdata",
|
||||
"route53",
|
||||
"s3",
|
||||
"sagemaker",
|
||||
"sagemakera2iruntime",
|
||||
"sagemakeredge",
|
||||
"sagemakerfeaturestoreruntime",
|
||||
"secretsmanager",
|
||||
"sesv2",
|
||||
"snowball",
|
||||
"sns",
|
||||
"sqs",
|
||||
"ssm",
|
||||
"sts"
|
||||
)
|
||||
// get a project property by name if it exists (including from local.properties)
|
||||
fun getProperty(name: String): String? {
|
||||
if (project.hasProperty(name)) {
|
||||
return project.properties[name].toString()
|
||||
}
|
||||
|
||||
private val disableServices = setOf(
|
||||
// transcribe streaming contains exclusively EventStream operations which are not supported
|
||||
"transcribestreaming",
|
||||
// Glacier requires a customization which is not currently supported:
|
||||
// https://github.com/awslabs/smithy-rs/issues/137
|
||||
"glacier",
|
||||
// https://github.com/awslabs/smithy-rs/issues/606
|
||||
"iotdataplane",
|
||||
// timestream requires endpoint discovery
|
||||
// https://github.com/awslabs/aws-sdk-rust/issues/114
|
||||
"timestreamwrite",
|
||||
"timestreamquery"
|
||||
)
|
||||
val localProperties = Properties()
|
||||
val propertiesFile: File = rootProject.file("local.properties")
|
||||
if (propertiesFile.exists()) {
|
||||
propertiesFile.inputStream().use { localProperties.load(it) }
|
||||
|
||||
if (localProperties.containsKey(name)) {
|
||||
return localProperties[name].toString()
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
// Class and functions for service and protocol membership for SDK generation
|
||||
data class Membership(val inclusions: Set<String> = emptySet(), val exclusions: Set<String> = emptySet())
|
||||
|
||||
fun Membership.isMember(member: String): Boolean = when {
|
||||
exclusions.contains(member) -> false
|
||||
inclusions.contains(member) -> true
|
||||
inclusions.isEmpty() -> true
|
||||
else -> false
|
||||
}
|
||||
|
||||
fun parseMembership(rawList: String): Membership {
|
||||
val inclusions = mutableSetOf<String>()
|
||||
val exclusions = mutableSetOf<String>()
|
||||
|
||||
rawList.split(",").map { it.trim() }.forEach { item ->
|
||||
when {
|
||||
item.startsWith('-') -> exclusions.add(item.substring(1))
|
||||
item.startsWith('+') -> inclusions.add(item.substring(1))
|
||||
else -> error("Must specify inclusion (+) or exclusion (-) prefix character to $item.")
|
||||
}
|
||||
}
|
||||
|
||||
val conflictingMembers = inclusions.intersect(exclusions)
|
||||
require(conflictingMembers.isEmpty()) { "$conflictingMembers specified both for inclusion and exclusion in $rawList" }
|
||||
|
||||
return Membership(inclusions, exclusions)
|
||||
}
|
||||
|
||||
data class AwsService(
|
||||
val service: String,
|
||||
|
|
@ -118,20 +105,18 @@ data class AwsService(
|
|||
fun files(): List<File> = listOf(modelFile) + extraFiles
|
||||
}
|
||||
|
||||
val generateAllServices =
|
||||
project.providers.environmentVariable("GENERATE_ALL_SERVICES").forUseAtConfigurationTime().orElse("")
|
||||
val awsServices: List<AwsService> by lazy { discoverServices() }
|
||||
|
||||
val generateOnly: Provider<Set<String>> =
|
||||
project.providers.environmentVariable("GENERATE_ONLY")
|
||||
.forUseAtConfigurationTime()
|
||||
.map { envVar ->
|
||||
envVar.split(",").filter { service -> service.trim().isNotBlank() }
|
||||
}
|
||||
.orElse(listOf())
|
||||
.map { it.toSet() }
|
||||
|
||||
val awsServices: Provider<List<AwsService>> = generateAllServices.zip(generateOnly) { v, only ->
|
||||
discoverServices(v.toLowerCase() == "true", only)
|
||||
fun loadServiceMembership(): Membership {
|
||||
val membershipOverride = getProperty("aws.services")?.let { parseMembership(it) }
|
||||
println(membershipOverride)
|
||||
val fullSdk = parseMembership(getProperty("aws.services.fullsdk") ?: throw kotlin.Exception("never list missing"))
|
||||
val tier1 = parseMembership(getProperty("aws.services.tier1") ?: throw kotlin.Exception("tier1 list missing"))
|
||||
return membershipOverride ?: if ((getProperty("aws.fullsdk") ?: "") == "true") {
|
||||
fullSdk
|
||||
} else {
|
||||
tier1
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -139,57 +124,57 @@ val awsServices: Provider<List<AwsService>> = generateAllServices.zip(generateOn
|
|||
*
|
||||
* Do not invoke this function directly. Use the `awsServices` provider.
|
||||
*/
|
||||
fun discoverServices(allServices: Boolean, generateOnly: Set<String>): List<AwsService> {
|
||||
fun discoverServices(): List<AwsService> {
|
||||
val models = project.file("aws-models")
|
||||
val serviceMembership = loadServiceMembership()
|
||||
val baseServices = fileTree(models)
|
||||
.sortedBy { file -> file.name }
|
||||
.mapNotNull { file ->
|
||||
val model = Model.assembler().addImport(file.absolutePath).assemble().result.get()
|
||||
val services: List<ServiceShape> = model.shapes(ServiceShape::class.java).sorted().toList()
|
||||
if (services.size > 1) {
|
||||
throw Exception("There must be exactly one service in each aws model file")
|
||||
}
|
||||
if (services.isEmpty()) {
|
||||
logger.info("${file.name} has no services")
|
||||
null
|
||||
} else {
|
||||
val service = services[0]
|
||||
val sdkId = service.expectTrait(ServiceTrait::class.java).sdkId
|
||||
.toLowerCase()
|
||||
.replace(" ", "")
|
||||
// TODO: the smithy models should not include the suffix "service"
|
||||
.removeSuffix("service")
|
||||
.removeSuffix("api")
|
||||
val testFile = file.parentFile.resolve("$sdkId-tests.smithy")
|
||||
val extras = if (testFile.exists()) {
|
||||
logger.warn("Discovered protocol tests for ${file.name}")
|
||||
listOf(testFile)
|
||||
} else {
|
||||
listOf()
|
||||
val model = Model.assembler().addImport(file.absolutePath).assemble().result.get()
|
||||
val services: List<ServiceShape> = model.shapes(ServiceShape::class.java).sorted().toList()
|
||||
if (services.size > 1) {
|
||||
throw Exception("There must be exactly one service in each aws model file")
|
||||
}
|
||||
if (services.isEmpty()) {
|
||||
logger.info("${file.name} has no services")
|
||||
null
|
||||
} else {
|
||||
val service = services[0]
|
||||
val sdkId = service.expectTrait(ServiceTrait::class.java).sdkId
|
||||
.toLowerCase()
|
||||
.replace(" ", "")
|
||||
// TODO: the smithy models should not include the suffix "service"
|
||||
.removeSuffix("service")
|
||||
.removeSuffix("api")
|
||||
val testFile = file.parentFile.resolve("$sdkId-tests.smithy")
|
||||
val extras = if (testFile.exists()) {
|
||||
logger.warn("Discovered protocol tests for ${file.name}")
|
||||
listOf(testFile)
|
||||
} else {
|
||||
listOf()
|
||||
}
|
||||
AwsService(service = service.id.toString(), module = sdkId, modelFile = file, extraFiles = extras)
|
||||
}
|
||||
AwsService(service = service.id.toString(), module = sdkId, modelFile = file, extraFiles = extras)
|
||||
}
|
||||
}
|
||||
val baseModules = baseServices.map { it.module }.toSet()
|
||||
disableServices.forEach{ disabledService ->
|
||||
|
||||
// validate the full exclusion list hits
|
||||
serviceMembership.exclusions.forEach { disabledService ->
|
||||
check(baseModules.contains(disabledService)) {
|
||||
"Service $disabledService was explicitly disabled but no service was generated with that name. Generated:\n ${baseModules.joinToString("\n ")}"
|
||||
"Service $disabledService was explicitly disabled but no service was generated with that name. Generated:\n ${
|
||||
baseModules.joinToString(
|
||||
"\n "
|
||||
)
|
||||
}"
|
||||
}
|
||||
}
|
||||
val services = baseServices.filterNot {
|
||||
disableServices.contains(it.module)
|
||||
}.filter {
|
||||
val inGenerateOnly = generateOnly.isNotEmpty() && generateOnly.contains(it.module)
|
||||
val inTier1 = generateOnly.isEmpty() && tier1Services.contains(it.module)
|
||||
allServices || inGenerateOnly || inTier1
|
||||
// validate inclusion list hits
|
||||
serviceMembership.inclusions.forEach { service ->
|
||||
check(baseModules.contains(service)) { "Service $service was in explicit inclusion list but not generated!" }
|
||||
}
|
||||
if (generateOnly.isEmpty()) {
|
||||
val modules = services.map { it.module }.toSet()
|
||||
tier1Services.forEach { service ->
|
||||
check(modules.contains(service)) { "Service $service was in list of tier 1 services but not generated! ($generateOnly)" }
|
||||
}
|
||||
return baseServices.filter {
|
||||
serviceMembership.isMember(it.module)
|
||||
}
|
||||
return services
|
||||
}
|
||||
|
||||
fun generateSmithyBuild(tests: List<AwsService>): String {
|
||||
|
|
@ -235,10 +220,10 @@ fun generateSmithyBuild(tests: List<AwsService>): String {
|
|||
|
||||
task("generateSmithyBuild") {
|
||||
description = "generate smithy-build.json"
|
||||
dependsOn(awsServices)
|
||||
doFirst {
|
||||
projectDir.resolve("smithy-build.json").writeText(generateSmithyBuild(awsServices.get()))
|
||||
projectDir.resolve("smithy-build.json").writeText(generateSmithyBuild(awsServices))
|
||||
}
|
||||
inputs.property("servicelist", awsServices.sortedBy { it.module }.toString())
|
||||
inputs.dir(projectDir.resolve("aws-models"))
|
||||
outputs.file(projectDir.resolve("smithy-build.json"))
|
||||
}
|
||||
|
|
@ -246,7 +231,7 @@ task("generateSmithyBuild") {
|
|||
task("relocateServices") {
|
||||
description = "relocate AWS services to their final destination"
|
||||
doLast {
|
||||
awsServices.get().forEach {
|
||||
awsServices.forEach {
|
||||
logger.info("Relocating ${it.module}...")
|
||||
copy {
|
||||
from("$buildDir/smithyprojections/sdk/${it.module}/rust-codegen")
|
||||
|
|
@ -332,9 +317,9 @@ task("generateCargoWorkspace") {
|
|||
description = "generate Cargo.toml workspace file"
|
||||
doFirst {
|
||||
sdkOutputDir.mkdirs()
|
||||
sdkOutputDir.resolve("Cargo.toml").writeText(generateCargoWorkspace(awsServices.get()))
|
||||
sdkOutputDir.resolve("Cargo.toml").writeText(generateCargoWorkspace(awsServices))
|
||||
}
|
||||
dependsOn(awsServices)
|
||||
inputs.property("servicelist", awsServices.sortedBy { it.module }.toString())
|
||||
inputs.dir(projectDir.resolve("examples"))
|
||||
outputs.file(sdkOutputDir.resolve("Cargo.toml"))
|
||||
outputs.upToDateWhen { false }
|
||||
|
|
@ -354,7 +339,6 @@ task("finalizeSdk") {
|
|||
tasks["smithyBuildJar"].inputs.file(projectDir.resolve("smithy-build.json"))
|
||||
tasks["smithyBuildJar"].inputs.dir(projectDir.resolve("aws-models"))
|
||||
tasks["smithyBuildJar"].dependsOn("generateSmithyBuild")
|
||||
tasks["smithyBuildJar"].dependsOn(awsServices)
|
||||
tasks["smithyBuildJar"].dependsOn("generateCargoWorkspace")
|
||||
tasks["smithyBuildJar"].outputs.upToDateWhen { false }
|
||||
tasks["assemble"].dependsOn("smithyBuildJar")
|
||||
|
|
|
|||
|
|
@ -0,0 +1,60 @@
|
|||
#
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0.
|
||||
#
|
||||
|
||||
# several services are broken pending custom work:
|
||||
# transcribe streaming contains exclusively EventStream operations which are not supported
|
||||
# Glacier requires a customization which is not currently supported:
|
||||
# https://github.com/awslabs/smithy-rs/issues/137
|
||||
# IOT data plane requires a signing customization https://github.com/awslabs/smithy-rs/issues/606
|
||||
# timestream requires endpoint discovery: https://github.com/awslabs/aws-sdk-rust/issues/114
|
||||
aws.services.fullsdk=-transcribestreaming,-glacier,-iotdataplane,-timestreamwrite,-timestreamquery
|
||||
|
||||
# Generate an entire sdk vs. aws.services.tier1
|
||||
aws.fullsdk=false
|
||||
|
||||
# base set of services that are generated unless other options are specified:
|
||||
aws.services.tier1=+apigateway,\
|
||||
+applicationautoscaling,\
|
||||
+autoscaling,\
|
||||
+autoscalingplans,\
|
||||
+batch,\
|
||||
+cloudformation,\
|
||||
+cloudwatch,\
|
||||
+cloudwatch,\
|
||||
+cloudwatchlogs,\
|
||||
+cognitoidentity,\
|
||||
+cognitoidentityprovider,\
|
||||
+cognitosync,\
|
||||
+config,\
|
||||
+dynamodb,\
|
||||
+ebs,\
|
||||
+ec2,\
|
||||
+ecr,\
|
||||
+ecs,\
|
||||
+eks,\
|
||||
+iam,\
|
||||
+kinesis,\
|
||||
+kms,\
|
||||
+lambda,\
|
||||
+medialive,\
|
||||
+mediapackage,\
|
||||
+polly,\
|
||||
+qldb,\
|
||||
+qldbsession,\
|
||||
+rds,\
|
||||
+rdsdata,\
|
||||
+route53,\
|
||||
+s3,\
|
||||
+sagemaker,\
|
||||
+sagemakera2iruntime,\
|
||||
+sagemakeredge,\
|
||||
+sagemakerfeaturestoreruntime,\
|
||||
+secretsmanager,\
|
||||
+sesv2,\
|
||||
+snowball,\
|
||||
+sns,\
|
||||
+sqs,\
|
||||
+ssm,\
|
||||
+sts
|
||||
Loading…
Reference in New Issue