[Python] Automatically generate stubs (#2576)

## Motivation and Context
We want to automatically generate stubs in the codegen diff to ensure
they can be reviewed and have a simple way to generate and include the
stubs inside the Maturin wheel.

## Description
The Python example has been moved to the `examples` folder and
refactored. The refactoring
ensures the script `stubgen.py` is included in the codegeneration of the
SDK crate. The script is later used to generate stubs automatically
during testing and can be used by customers to add their own stubs
before the Maturin build

----

_By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice._

----

_By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice._

---------

Signed-off-by: Bigo <1781140+crisidev@users.noreply.github.com>
Co-authored-by: Burak <unexge@gmail.com>

codegen-core/src/main/kotlin/software/amazon/smithy/rust/codegen/core/rustlang/RustWriter.kt

@@ -469,6 +469,7 @@ class RustWriter private constructor(
                     devDependenciesOnly = true,
                 )
                 fileName == "package.json" -> rawWriter(fileName, debugMode = debugMode)
+                fileName == "stubgen.sh" -> rawWriter(fileName, debugMode = debugMode)
                 else -> RustWriter(fileName, namespace, debugMode = debugMode)
             }
         }

codegen-core/src/main/kotlin/software/amazon/smithy/rust/codegen/core/smithy/RuntimeType.kt

@@ -96,6 +96,8 @@ data class RuntimeConfig(
 
     val crateSrcPrefix: String = cratePrefix.replace("-", "_")
 
+    fun runtimeCratesPath(): String? = runtimeCrateLocation.path
+
     fun smithyRuntimeCrate(
         runtimeCrateName: String,
         optional: Boolean = false,

codegen-server-test/python/build.gradle.kts

@@ -66,7 +66,7 @@ val allCodegenTests = "../../codegen-core/common-test-models".let { commonModels
             "rest_json_extras",
             imports = listOf("$commonModels/rest-json-extras.smithy"),
         ),
-        // TODO(https://github.com/awslabs/smithy-rs/issues/2551)
+        // TODO(https://github.com/awslabs/smithy-rs/issues/2477)
         // CodegenTest(
         //     "aws.protocoltests.restjson.validation#RestJsonValidation",
         //     "rest_json_validation",
@@ -104,6 +104,21 @@ project.registerGenerateSmithyBuildTask(rootProject, pluginName, allCodegenTests
 project.registerGenerateCargoWorkspaceTask(rootProject, pluginName, allCodegenTests, workingDirUnderBuildDir)
 project.registerGenerateCargoConfigTomlTask(buildDir.resolve(workingDirUnderBuildDir))
 
+tasks.register("stubs") {
+    description = "Generate Python stubs for all models"
+    dependsOn("assemble")
+
+    doLast {
+        allCodegenTests.forEach { test ->
+            val crateDir = "$buildDir/$workingDirUnderBuildDir/${test.module}/$pluginName"
+            val moduleName = test.module.replace("-", "_")
+            exec {
+                commandLine("bash", "$crateDir/stubgen.sh", moduleName, "$crateDir/Cargo.toml", "$crateDir/python/$moduleName")
+            }
+        }
+    }
+}
+
 tasks["smithyBuildJar"].dependsOn("generateSmithyBuild")
 tasks["assemble"].finalizedBy("generateCargoWorkspace")
 

codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/PythonServerCargoDependency.kt

@@ -15,8 +15,8 @@ import software.amazon.smithy.rust.codegen.core.smithy.RuntimeConfig
  * For a dependency that is used in the client, or in both the client and the server, use [CargoDependency] directly.
  */
 object PythonServerCargoDependency {
-    val PyO3: CargoDependency = CargoDependency("pyo3", CratesIo("0.17"))
-    val PyO3Asyncio: CargoDependency = CargoDependency("pyo3-asyncio", CratesIo("0.17"), features = setOf("attributes", "tokio-runtime"))
+    val PyO3: CargoDependency = CargoDependency("pyo3", CratesIo("0.18"))
+    val PyO3Asyncio: CargoDependency = CargoDependency("pyo3-asyncio", CratesIo("0.18"), features = setOf("attributes", "tokio-runtime"))
     val Tokio: CargoDependency = CargoDependency("tokio", CratesIo("1.20.1"), features = setOf("full"))
     val TokioStream: CargoDependency = CargoDependency("tokio-stream", CratesIo("0.1.12"))
     val Tracing: CargoDependency = CargoDependency("tracing", CratesIo("0.1"))

codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/customizations/PythonServerCodegenDecorator.kt

@@ -23,6 +23,7 @@ import software.amazon.smithy.rust.codegen.server.python.smithy.generators.Pytho
 import software.amazon.smithy.rust.codegen.server.smithy.ServerCodegenContext
 import software.amazon.smithy.rust.codegen.server.smithy.customizations.AddInternalServerErrorToAllOperationsDecorator
 import software.amazon.smithy.rust.codegen.server.smithy.customize.ServerCodegenDecorator
+import java.io.File
 
 /**
  * Configure the [lib] section of `Cargo.toml`.
@@ -194,6 +195,31 @@ class PyTypedMarkerDecorator : ServerCodegenDecorator {
     }
 }
 
+/**
+ * Copies the stubgen scripts to the generated crate root.
+ *
+ * The shell script `stubgen.sh` runs a quick build and uses `stubgen.py` to generate mypy compatibile
+ * types stubs for the project.
+ */
+class AddStubgenScriptDecorator : ServerCodegenDecorator {
+    override val name: String = "AddStubgenScriptDecorator"
+    override val order: Byte = 0
+
+    override fun extras(codegenContext: ServerCodegenContext, rustCrate: RustCrate) {
+        val runtimeCratesPath = codegenContext.runtimeConfig.runtimeCratesPath()
+        val stubgenPythonLocation = "$runtimeCratesPath/aws-smithy-http-server-python/stubgen.py"
+        val stubgenPythonContent = File(stubgenPythonLocation).readText(Charsets.UTF_8)
+        rustCrate.withFile("stubgen.py") {
+            writeWithNoFormatting("$stubgenPythonContent")
+        }
+        val stubgenShellLocation = "$runtimeCratesPath/aws-smithy-http-server-python/stubgen.sh"
+        val stubgenShellContent = File(stubgenShellLocation).readText(Charsets.UTF_8)
+        rustCrate.withFile("stubgen.sh") {
+            writeWithNoFormatting("$stubgenShellContent")
+        }
+    }
+}
+
 val DECORATORS = arrayOf(
     /**
      * Add the [InternalServerError] error to all operations.
@@ -214,4 +240,6 @@ val DECORATORS = arrayOf(
     InitPyDecorator(),
     // Generate `py.typed` for the Python source.
     PyTypedMarkerDecorator(),
+    // Generate scripts for stub generation.
+    AddStubgenScriptDecorator(),
 )

codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/generators/PythonServerModuleGenerator.kt

@@ -11,7 +11,9 @@ import software.amazon.smithy.model.shapes.ResourceShape
 import software.amazon.smithy.model.shapes.ServiceShape
 import software.amazon.smithy.model.shapes.Shape
 import software.amazon.smithy.model.shapes.UnionShape
+import software.amazon.smithy.rust.codegen.core.Version
 import software.amazon.smithy.rust.codegen.core.rustlang.RustWriter
+import software.amazon.smithy.rust.codegen.core.rustlang.rust
 import software.amazon.smithy.rust.codegen.core.rustlang.rustBlockTemplate
 import software.amazon.smithy.rust.codegen.core.rustlang.rustTemplate
 import software.amazon.smithy.rust.codegen.core.smithy.RustCrate
@@ -51,6 +53,8 @@ class PythonServerModuleGenerator(
                 renderPyTlsTypes()
                 renderPyLambdaTypes()
                 renderPyApplicationType()
+                renderCodegenVersion()
+                rust("Ok(())")
             }
         }
     }
@@ -62,13 +66,23 @@ class PythonServerModuleGenerator(
             let input = #{pyo3}::types::PyModule::new(py, "input")?;
             let output = #{pyo3}::types::PyModule::new(py, "output")?;
             let error = #{pyo3}::types::PyModule::new(py, "error")?;
-            let model = #{pyo3}::types::PyModule::new(py, "model")?;
             """,
             *codegenScope,
         )
+        // The `model` type section can be unused in models like `simple`, so we accommodate for it.
+        var visitedModelType = false
         serviceShapes.forEach { shape ->
             val moduleType = moduleType(shape)
             if (moduleType != null) {
+                if (moduleType == "model" && !visitedModelType) {
+                    rustTemplate(
+                        """
+                        let model = #{pyo3}::types::PyModule::new(py, "model")?;
+                        """,
+                        *codegenScope,
+                    )
+                    visitedModelType = true
+                }
                 when (shape) {
                     is UnionShape -> rustTemplate(
                         """
@@ -93,11 +107,18 @@ class PythonServerModuleGenerator(
             m.add_submodule(output)?;
             #{pyo3}::py_run!(py, error, "import sys; sys.modules['$libName.error'] = error");
             m.add_submodule(error)?;
-            #{pyo3}::py_run!(py, model, "import sys; sys.modules['$libName.model'] = model");
-            m.add_submodule(model)?;
             """,
             *codegenScope,
         )
+        if (visitedModelType) {
+            rustTemplate(
+                """
+                #{pyo3}::py_run!(py, model, "import sys; sys.modules['$libName.model'] = model");
+                m.add_submodule(model)?;
+                """,
+                *codegenScope,
+            )
+        }
     }
 
     // Render wrapper types that are substituted to the ones coming from `aws_smithy_types`.
@@ -211,13 +232,12 @@ class PythonServerModuleGenerator(
 
     // Render Python application type.
     private fun RustWriter.renderPyApplicationType() {
-        rustTemplate(
-            """
-            m.add_class::<crate::python_server_application::App>()?;
-            Ok(())
-            """,
-            *codegenScope,
-        )
+        rust("""m.add_class::<crate::python_server_application::App>()?;""")
+    }
+
+    // Render the codegeneration version as module attribute.
+    private fun RustWriter.renderCodegenVersion() {
+        rust("""m.add("CODEGEN_VERSION", "${Version.crateVersion()}")?;""")
     }
 
     // Convert to symbol and check the namespace to figure out where they should be imported from.

codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/RustCrateInlineModuleComposingWriter.kt

@@ -335,7 +335,9 @@ class InnerModule(private val moduleDocProvider: ModuleDocProvider, debugMode: B
             inlineWriter
         } else {
             check(inlineModuleAndWriter.inlineModule == lookForModule) {
-                "The two inline modules have the same name but different attributes on them."
+                """The two inline modules have the same name but different attributes on them:
+                1) ${inlineModuleAndWriter.inlineModule}
+                2) $lookForModule"""
             }
 
             inlineModuleAndWriter.writer

examples/python/Makefile

@@ -35,25 +35,23 @@ install-wheel:
 	find $(WHEELS) -type f -name '*.whl' | xargs python3 -m pip install --user --force-reinstall
 
 generate-stubs:
-	python3 $(CUR_DIR)/stubgen.py pokemon_service_server_sdk $(SERVER_SDK_DST)/python/pokemon_service_server_sdk
+	bash $(SERVER_SDK_DST)/stubgen.sh pokemon_service_server_sdk $(SERVER_SDK_DST)/Cargo.toml $(SERVER_SDK_DST)/python/pokemon_service_server_sdk
 
 build: codegen
+	$(MAKE) generate-stubs
 	$(MAKE) build-wheel
-	$(MAKE) install-wheel
+
+release: codegen
 	$(MAKE) generate-stubs
 	$(MAKE) build-wheel-release
-	$(MAKE) install-wheel
 
 run: build
 	python3 $(CUR_DIR)/pokemon_service.py
 
-py-check: build
+py-check: install-wheel
 	python3 -m mypy pokemon_service.py
 
-py-test:
-	python3 stubgen_test.py
-
-test: build py-check py-test
+test: build py-check
 	cargo test
 
 clippy: codegen

examples/python/README.md

@@ -0,0 +1,94 @@
+# Smithy Rust/Python Server SDK example
+
+This folder contains an example service called Pokémon Service used to showcase
+the service framework Python bindings capabilities and to run benchmarks.
+
+The Python implementation of the service can be found inside
+[pokemon_service.py](./pokemon_service.py).
+
+* [Build](#build)
+    * [Build dependencies](#build-dependencies)
+    * [Makefile](#makefile)
+    * [Python stub generation](#python-stub-generation)
+* [Run](#run)
+* [Test](#test)
+* [Uvloop](#uvloop)
+* [MacOs](#macos)
+
+## Build
+
+Since this example requires both the server and client SDK to be code-generated
+from their [model](/codegen-server-test/model/pokemon.smithy), a Makefile is
+provided to build and run the service. Just run `make build` to prepare the first
+build.
+
+### Build dependencies
+
+Ensure these dependencies are installed.
+
+```
+pip install maturin uvloop aiohttp mypy
+```
+
+### Makefile
+
+The build logic is drive by the Makefile:
+
+* `make codegen`: run the codegenerator.
+* `make build`: build the Maturin package in debug mode (includes type stubs
+  generation).
+* `make release`: build the Maturin package in release mode (includes type stub
+  generation).
+* `make install`: install the latest release or debug build.
+* `make run`: run the example server.
+* `make test`: run the end-to-end integration test.
+
+### Python stub generation
+
+We support the generation of mypy python stubs and every SDK crate ships with
+a script called `stubgen.sh`. **Note that the script is not called
+automatically as part of the build**. We suggest users to call it after code generation.
+It will do first compilation of the crate, generate the types and exit.
+
+The script takes some command line arguments:
+
+```
+./stubgen.sh module_name manifest_path output_directory
+```
+
+* module_name: name of the Python module to generate stubs for, IE `pokemon_service_server_sdk`.
+* manifest_path: path for the crate manifest used to build the types.
+* output_directory: directory where to generate the stub hierarchy. **This
+  directory should be a folder `python/$module_name` in the root of the Maturin package.**
+
+## Run
+
+`make run` can be used to start the Pokémon service on `http://localhost:13734`.
+
+## Test
+
+`make test` can be used to spawn the Python service and run some simple integration
+tests against it.
+
+More info can be found in the `tests` folder of `pokemon-service-test` package.
+
+## Uvloop
+
+The server can depend on [uvloop](https://pypi.org/project/uvloop/) for a
+faster event loop implementation. Uvloop can be installed with your favourite
+package manager or by using pip:
+
+```sh
+pip instal uvloop
+```
+
+and it will be automatically used instead of the standard library event loop if
+it is found in the dependencies' closure.
+
+## MacOs
+
+To compile and test on MacOs, please follow the official PyO3 guidelines on how
+to [configure your linker](https://pyo3.rs/latest/building_and_distribution.html?highlight=rustflags#macos).
+
+Please note that the `.cargo/config.toml` with linkers override can be local to
+your project.

examples/python/pokemon-service-test/Cargo.toml

@@ -9,15 +9,15 @@ description = "Run tests against the Python server implementation"
 [dev-dependencies]
 rand = "0.8"
 async-stream = "0.3"
-command-group = "1.0"
+command-group = "2.1.0"
 tokio = { version = "1.20.1", features = ["full"] }
-serial_test = "0.9.0"
+serial_test = "2.0.0"
 rustls-pemfile = "1.0.1"
-tokio-rustls = "0.23.4"
-hyper-rustls = { version = "0.23.0", features = ["http2"] }
+tokio-rustls = "0.24.0"
+hyper-rustls = { version = "0.24.0", features = ["http2"] }
 
 # Local paths
-aws-smithy-client  = { path = "../../../aws-smithy-client/", features = ["rustls"] }
-aws-smithy-http = { path = "../../../aws-smithy-http/" }
-aws-smithy-types  = { path = "../../../aws-smithy-types/" }
+aws-smithy-client  = { path = "../../../rust-runtime/aws-smithy-client/", features = ["rustls"] }
+aws-smithy-http = { path = "../../../rust-runtime/aws-smithy-http/" }
+aws-smithy-types  = { path = "../../../rust-runtime/aws-smithy-types/" }
 pokemon-service-client = { path = "../pokemon-service-client/" }

rust-runtime/aws-smithy-http-server-python/Cargo.toml

@@ -22,9 +22,9 @@ bytes = "1.2"
 futures = "0.3"
 http = "0.2"
 hyper = { version = "0.14.20", features = ["server", "http1", "http2", "tcp", "stream"] }
-tls-listener = { version = "0.5.1", features = ["rustls", "hyper-h2"] }
+tls-listener = { version = "0.7.0", features = ["rustls", "hyper-h2"] }
 rustls-pemfile = "1.0.1"
-tokio-rustls = "0.23.4"
+tokio-rustls = "0.24.0"
 lambda_http = { version = "0.7.1" }
 # There is a breaking change in `lambda_runtime` between `0.7.0` and `0.7.1`,
 # and `lambda_http` depends on `0.7` which by default resolves to `0.7.1` but in our CI
@@ -34,10 +34,10 @@ lambda_runtime = { version = "0.7.1" }
 num_cpus = "1.13.1"
 parking_lot = "0.12.1"
 pin-project-lite = "0.2"
-pyo3 = "0.17.0"
-pyo3-asyncio = { version = "0.17.0", features = ["tokio-runtime"] }
+pyo3 = "0.18.2"
+pyo3-asyncio = { version = "0.18.0", features = ["tokio-runtime"] }
 signal-hook = { version = "0.3.14", features = ["extended-siginfo"] }
-socket2 = { version = "0.4.4", features = ["all"] }
+socket2 = { version = "0.5.2", features = ["all"] }
 thiserror = "1.0.32"
 tokio = { version = "1.20.1", features = ["full"] }
 tokio-stream = "0.1"
@@ -51,9 +51,9 @@ pretty_assertions = "1"
 futures-util = { version = "0.3.16", default-features = false }
 tower-test = "0.4"
 tokio-test = "0.4"
-pyo3-asyncio = { version = "0.17.0", features = ["testing", "attributes", "tokio-runtime", "unstable-streams"] }
+pyo3-asyncio = { version = "0.18.0", features = ["testing", "attributes", "tokio-runtime", "unstable-streams"] }
 rcgen = "0.10.0"
-hyper-rustls = { version = "0.23.1", features = ["http2"] }
+hyper-rustls = { version = "0.24.0", features = ["http2"] }
 
 # PyO3 Asyncio tests cannot use Cargo's default testing harness because `asyncio`
 # wants to control the main thread. So we need to use testing harness provided by `pyo3_asyncio`

rust-runtime/aws-smithy-http-server-python/examples/README.md

@@ -1,52 +0,0 @@
-# Smithy Rust/Python Server SDK example
-
-This folder contains an example service called Pokémon Service used to showcase
-the service framework Python bindings capabilities and to run benchmarks.
-
-The Python implementation of the service can be found inside
-[pokemon_service.py](/rust-runtime/aws-smithy-http-python-server/examples/pokemon_service.py).
-
-## Build
-
-Since this example requires both the server and client SDK to be code-generated
-from their [model](/codegen-server-test/model/pokemon.smithy), a Makefile is
-provided to build and run the service. Just run `make build` to prepare the first
-build.
-
-Once the example has been built successfully the first time, idiomatic `cargo`
-can be used directly.
-
-`make distclean` can be used for a complete cleanup of all artefacts.
-
-### Uvloop
-
-The server can depend on [uvloop](https://pypi.org/project/uvloop/) for a
-faster event loop implementation. Uvloop can be installed with your favourite
-package manager or by using pip:
-
-```sh
-pip instal uvloop
-```
-
-and it will be automatically used instead of the standard library event loop if
-it is found in the dependencies' closure.
-
-### MacOs
-
-To compile and test on MacOs, please follow the official PyO3 guidelines on how
-to [configure your linker](https://pyo3.rs/latest/building_and_distribution.html?highlight=rustflags#macos).
-
-Please note that the `.cargo/config.toml` with linkers override can be local to
-your project.
-
-## Run
-
-`cargo run` can be used to start the Pokémon service on
-`http://localhost:13734`.
-
-## Test
-
-`cargo test` can be used to spawn the Python service and run some simple integration
-tests against it.
-
-More info can be found in the `tests` folder of `pokemon-service-test` package.

rust-runtime/aws-smithy-http-server-python/src/tls.rs

@@ -105,7 +105,7 @@ impl PyTlsConfig {
 #[pymethods]
 impl PyTlsConfig {
     #[new]
-    #[args(reload_secs = "86400")] // <- 1 Day by default
+    #[pyo3(signature = (key_path, cert_path, reload_secs=86400))]
     fn py_new(key_path: PathBuf, cert_path: PathBuf, reload_secs: u64) -> Self {
         // TODO(BugOnUpstream): `reload: &PyDelta` segfaults, create an issue on PyO3
         Self {
@@ -146,11 +146,11 @@ mod tests {
 
     const TEST_KEY: &str = concat!(
         env!("CARGO_MANIFEST_DIR"),
-        "/examples/pokemon-service-test/tests/testdata/localhost.key"
+        "/../../examples/python/pokemon-service-test/tests/testdata/localhost.key"
     );
     const TEST_CERT: &str = concat!(
         env!("CARGO_MANIFEST_DIR"),
-        "/examples/pokemon-service-test/tests/testdata/localhost.crt"
+        "/../../examples/python/pokemon-service-test/tests/testdata/localhost.crt"
     );
 
     #[test]

rust-runtime/aws-smithy-http-server-python/src/tls/listener.rs

@@ -191,7 +191,7 @@ mod tests {
             assert!(response
                 .unwrap_err()
                 .to_string()
-                .contains("invalid peer certificate: InvalidCertValidity"));
+                .contains("invalid peer certificate: Expired"));
         }
 
         // Make a new acceptor with a valid cert and replace

rust-runtime/aws-smithy-http-server-python/stubgen.py

@@ -2,11 +2,12 @@
 #  SPDX-License-Identifier: Apache-2.0
 
 from __future__ import annotations
-import re
+
 import inspect
+import re
 import textwrap
 from pathlib import Path
-from typing import Any, Set, Dict, List, Tuple, Optional
+from typing import Any, Dict, List, Optional, Set, Tuple
 
 ROOT_MODULE_NAME_PLACEHOLDER = "__root_module_name__"
 
@@ -36,11 +37,7 @@ class Writer:
         Returns fixed version of given type path.
         It unescapes `\\[` and `\\]` and also populates placeholder for root module name.
         """
-        return (
-            path.replace(ROOT_MODULE_NAME_PLACEHOLDER, self.root_module_name)
-            .replace("\\[", "[")
-            .replace("\\]", "]")
-        )
+        return path.replace(ROOT_MODULE_NAME_PLACEHOLDER, self.root_module_name).replace("\\[", "[").replace("\\]", "]")
 
     def submodule(self, path: Path) -> Writer:
         w = Writer(path, self.root_module_name)
@@ -98,27 +95,21 @@ class DocstringParserResult:
 def parse_type_directive(line: str, res: DocstringParserResult):
     parts = line.split(" ", maxsplit=1)
     if len(parts) != 2:
-        raise ValueError(
-            f"Invalid `:type` directive: `{line}` must be in `:type T:` format"
-        )
+        raise ValueError(f"Invalid `:type` directive: `{line}` must be in `:type T:` format")
     res.types.append(parts[1].rstrip(":"))
 
 
 def parse_rtype_directive(line: str, res: DocstringParserResult):
     parts = line.split(" ", maxsplit=1)
     if len(parts) != 2:
-        raise ValueError(
-            f"Invalid `:rtype` directive: `{line}` must be in `:rtype T:` format"
-        )
+        raise ValueError(f"Invalid `:rtype` directive: `{line}` must be in `:rtype T:` format")
     res.rtypes.append(parts[1].rstrip(":"))
 
 
 def parse_param_directive(line: str, res: DocstringParserResult):
     parts = line.split(" ", maxsplit=2)
     if len(parts) != 3:
-        raise ValueError(
-            f"Invalid `:param` directive: `{line}` must be in `:param name T:` format"
-        )
+        raise ValueError(f"Invalid `:param` directive: `{line}` must be in `:param name T:` format")
     name = parts[1]
     ty = parts[2].rstrip(":")
     res.params.append((name, ty))
@@ -127,18 +118,14 @@ def parse_param_directive(line: str, res: DocstringParserResult):
 def parse_generic_directive(line: str, res: DocstringParserResult):
     parts = line.split(" ", maxsplit=1)
     if len(parts) != 2:
-        raise ValueError(
-            f"Invalid `:generic` directive: `{line}` must be in `:generic T:` format"
-        )
+        raise ValueError(f"Invalid `:generic` directive: `{line}` must be in `:generic T:` format")
     res.generics.append(parts[1].rstrip(":"))
 
 
 def parse_extends_directive(line: str, res: DocstringParserResult):
     parts = line.split(" ", maxsplit=1)
     if len(parts) != 2:
-        raise ValueError(
-            f"Invalid `:extends` directive: `{line}` must be in `:extends Base[...]:` format"
-        )
+        raise ValueError(f"Invalid `:extends` directive: `{line}` must be in `:extends Base[...]:` format")
     res.extends.append(parts[1].rstrip(":"))
 
 
@@ -201,13 +188,13 @@ class DocstringParser:
         if not doc:
             return ""
 
-        def predicate(l: str) -> bool:
+        def predicate(line: str) -> bool:
             for k in DocstringParserDirectives.keys():
-                if l.startswith(f":{k} ") and l.endswith(":"):
+                if line.startswith(f":{k} ") and line.endswith(":"):
                     return False
             return True
 
-        return "\n".join([l for l in doc.splitlines() if predicate(l)]).strip()
+        return "\n".join([line for line in doc.splitlines() if predicate(line)]).strip()
 
 
 def indent(code: str, level: int = 4) -> str:
@@ -225,6 +212,10 @@ def is_fn_like(obj: Any) -> bool:
     )
 
 
+def is_scalar(obj: Any) -> bool:
+    return isinstance(obj, (str, float, int, bool))
+
+
 def join(args: List[str], delim: str = "\n") -> str:
     return delim.join(filter(lambda x: x, args))
 
@@ -266,7 +257,7 @@ def make_function(
     sig: Optional[inspect.Signature] = None
     try:
         sig = inspect.signature(obj)
-    except:
+    except Exception:
         pass
 
     def has_default(param: str, ty: str) -> bool:
@@ -312,9 +303,7 @@ def {name}({params}) -> {rtype}:
 
 
 def make_class(writer: Writer, name: str, klass: Any) -> str:
-    bases = list(
-        filter(lambda n: n != "object", map(lambda b: b.__name__, klass.__bases__))
-    )
+    bases = list(filter(lambda n: n != "object", map(lambda b: b.__name__, klass.__bases__)))
     class_sig = DocstringParser.parse_class(klass)
     if class_sig:
         (generics, extends) = class_sig
@@ -386,7 +375,7 @@ class {name}{bases_str}:
 def walk_module(writer: Writer, mod: Any):
     exported = mod.__all__
 
-    for (name, member) in inspect.getmembers(mod):
+    for name, member in inspect.getmembers(mod):
         if name not in exported:
             continue
 
@@ -397,10 +386,25 @@ def walk_module(writer: Writer, mod: Any):
             writer.define(make_class(writer, name, member))
         elif is_fn_like(member):
             writer.define(make_function(writer, name, member))
+        elif is_scalar(member):
+            writer.define(f"{name}: {type(member).__name__} = ...")
         else:
             print(f"Unknown type: {member}")
 
 
+def generate(module: str, outdir: str):
+    path = Path(outdir) / f"{module}.pyi"
+    writer = Writer(
+        path,
+        module,
+    )
+    walk_module(
+        writer,
+        importlib.import_module(module),
+    )
+    writer.dump()
+
+
 if __name__ == "__main__":
     import argparse
     import importlib
@@ -410,13 +414,4 @@ if __name__ == "__main__":
     parser.add_argument("outdir")
     args = parser.parse_args()
 
-    path = Path(args.outdir) / f"{args.module}.pyi"
-    writer = Writer(
-        path,
-        args.module,
-    )
-    walk_module(
-        writer,
-        importlib.import_module(args.module),
-    )
-    writer.dump()
+    generate(args.module, args.outdir)

rust-runtime/aws-smithy-http-server-python/stubgen.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+#  Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#  SPDX-License-Identifier: Apache-2.0
+
+set -x
+
+if [ $# -lt 3 ]; then
+    echo "usage: $0 package manifest_path output_directory"
+    exit 1
+fi
+
+# input arguments
+package=$1
+manifest=$2
+output=$3
+
+# the directory of the script
+source_dir="$(git rev-parse --show-toplevel)"
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+if [ -n "$source_dir" ]; then
+    CARGO_TARGET_DIR="$source_dir/target"
+else
+    CARGO_TARGET_DIR=$(mktemp -d)
+    mkdir -p "$CARGO_TARGET_DIR"
+    # cleanup temporary directory
+    function cleanup {
+        # shellcheck disable=2317
+        rm -rf "$CARGO_TARGET_DIR"
+    }
+    # register the cleanup function to be called on the EXIT signal
+    trap cleanup EXIT
+fi
+export CARGO_TARGET_DIR
+
+shared_object_extension="so"
+# generate the Python stubs,
+if [ "$(uname)" == "Darwin" ]; then
+    shared_object_extension="dylib"
+    export CARGO_TARGET_X86_64_APPLE_DARWIN_RUSTFLAGS="-C link-arg=-undefined -C link-arg=dynamic_lookup"
+    export CARGO_TARGET_AARCH64_APPLE_DARWIN_RUSTFLAGS="-C link-arg=-undefined -C link-arg=dynamic_lookup"
+fi
+
+cargo build --manifest-path "$manifest"
+# The link target have to end with .so to be sure it is importable by the stubgen.py script.
+ln -sf "$CARGO_TARGET_DIR/debug/lib$package.$shared_object_extension" "$CARGO_TARGET_DIR/debug/$package.so"
+PYTHONPATH=$CARGO_TARGET_DIR/debug:$PYTHONPATH python3 "$script_dir/stubgen.py" "$package" "$output"
+
+exit 0

tools/ci-scripts/check-server-python-e2e-test

@@ -4,6 +4,6 @@
 # SPDX-License-Identifier: Apache-2.0
 
 set -eux
-cd smithy-rs/rust-runtime/aws-smithy-http-server-python/examples
+cd smithy-rs/examples/python
 
 make test clippy

tools/ci-scripts/codegen-diff/diff_lib.py

@@ -39,12 +39,12 @@ def checkout_commit_and_generate(revision_sha, branch_name, targets=None):
 
 def generate_and_commit_generated_code(revision_sha, targets=None):
     targets = targets or [
-            target_codegen_client,
-            target_codegen_server,
-            target_aws_sdk,
-            target_codegen_server_python,
-            target_codegen_server_typescript
-        ]
+        target_codegen_client,
+        target_codegen_server,
+        target_aws_sdk,
+        target_codegen_server_python,
+        target_codegen_server_typescript
+    ]
     # Clean the build artifacts before continuing
     assemble_tasks = ' '.join([f'{t}:assemble' for t in targets])
     clean_tasks = ' '.join([f'{t}:clean' for t in targets])
@@ -61,6 +61,7 @@ def generate_and_commit_generated_code(revision_sha, targets=None):
         if target in targets:
             get_cmd_output(f"mv {target}/build/smithyprojections/{target} {OUTPUT_PATH}/")
             if target == target_codegen_server:
+                get_cmd_output(f"./gradlew --rerun-tasks {target_codegen_server_python}:stubs")
                 get_cmd_output(f"mv {target}/python/build/smithyprojections/{target}-python {OUTPUT_PATH}/")
                 get_cmd_output(f"mv {target}/typescript/build/smithyprojections/{target}-typescript {OUTPUT_PATH}/")