mirror of https://github.com/smithy-lang/smithy-rs
Python: Use Maturin to build SDKs (#2025)
* Remove `lib` prefix from generated module names * Build Pokemon service with `Maturin` * Fix Maturin build on CI * Generate minimal `pyproject.toml` for generated SDKs to build from source using Maturin * Fix `ktlint` issues * Bring back type stubs for Pokemon service * Update instructions for Lambda * Make `build-wheel` and `build-wheel-release` to depend on `codegen`
This commit is contained in:
Burak
GitHub
parent
40245a1545
commit
2cc7c24be4
|
|
@ -5,6 +5,7 @@
|
|||
|
||||
package software.amazon.smithy.rust.codegen.server.python.smithy.customizations
|
||||
|
||||
import com.moandjiezana.toml.TomlWriter
|
||||
import software.amazon.smithy.model.neighbor.Walker
|
||||
import software.amazon.smithy.rust.codegen.client.smithy.customize.RustCodegenDecorator
|
||||
import software.amazon.smithy.rust.codegen.core.rustlang.Writable
|
||||
|
|
@ -103,6 +104,30 @@ class PubUsePythonTypesDecorator : RustCodegenDecorator<ServerProtocolGenerator,
|
|||
clazz.isAssignableFrom(ServerCodegenContext::class.java)
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates `pyproject.toml` for the crate.
|
||||
* - Configures Maturin as the build system
|
||||
*/
|
||||
class PyProjectTomlDecorator : RustCodegenDecorator<ServerProtocolGenerator, ServerCodegenContext> {
|
||||
override val name: String = "PyProjectTomlDecorator"
|
||||
override val order: Byte = 0
|
||||
|
||||
override fun extras(codegenContext: ServerCodegenContext, rustCrate: RustCrate) {
|
||||
rustCrate.withFile("pyproject.toml") {
|
||||
val config = mapOf(
|
||||
"build-system" to listOfNotNull(
|
||||
"requires" to listOfNotNull("maturin>=0.14,<0.15"),
|
||||
"build-backend" to "maturin",
|
||||
).toMap(),
|
||||
)
|
||||
writeWithNoFormatting(TomlWriter().write(config))
|
||||
}
|
||||
}
|
||||
|
||||
override fun supportsCodegenContext(clazz: Class<out CodegenContext>): Boolean =
|
||||
clazz.isAssignableFrom(ServerCodegenContext::class.java)
|
||||
}
|
||||
|
||||
val DECORATORS = listOf(
|
||||
/**
|
||||
* Add the [InternalServerError] error to all operations.
|
||||
|
|
@ -115,4 +140,6 @@ val DECORATORS = listOf(
|
|||
PubUsePythonTypesDecorator(),
|
||||
// Render the Python shared library export.
|
||||
PythonExportModuleDecorator(),
|
||||
// Generate `pyproject.toml` for the crate.
|
||||
PyProjectTomlDecorator(),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -67,7 +67,7 @@ class PythonApplicationGenerator(
|
|||
private val operations: List<OperationShape>,
|
||||
) {
|
||||
private val symbolProvider = codegenContext.symbolProvider
|
||||
private val libName = "lib${codegenContext.settings.moduleName.toSnakeCase()}"
|
||||
private val libName = codegenContext.settings.moduleName.toSnakeCase()
|
||||
private val runtimeConfig = codegenContext.runtimeConfig
|
||||
private val service = codegenContext.serviceShape
|
||||
private val serviceName = service.id.name.toPascalCase()
|
||||
|
|
@ -278,7 +278,6 @@ class PythonApplicationGenerator(
|
|||
self.run_server(py, address, port, backlog, workers, tls)
|
||||
}
|
||||
/// Lambda entrypoint: start the server on Lambda.
|
||||
##[cfg(feature = "aws-lambda")]
|
||||
##[pyo3(text_signature = "(${'$'}self)")]
|
||||
pub fn run_lambda(
|
||||
&mut self,
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ class PythonServerModuleGenerator(
|
|||
"pyo3" to PythonServerCargoDependency.PyO3.toType(),
|
||||
)
|
||||
private val symbolProvider = codegenContext.symbolProvider
|
||||
private val libName = "lib${codegenContext.settings.moduleName.toSnakeCase()}"
|
||||
private val libName = codegenContext.settings.moduleName.toSnakeCase()
|
||||
|
||||
fun render() {
|
||||
rustCrate.withModule(
|
||||
|
|
|
|||
|
|
@ -13,8 +13,8 @@ instead of the [Hyper](https://hyper.rs/) HTTP server.
|
|||
In your `app.py`:
|
||||
|
||||
```diff
|
||||
from libpokemon_service_server_sdk import App
|
||||
from libpokemon_service_server_sdk.error import ResourceNotFoundException
|
||||
from pokemon_service_server_sdk import App
|
||||
from pokemon_service_server_sdk.error import ResourceNotFoundException
|
||||
|
||||
# ...
|
||||
|
||||
|
|
@ -44,19 +44,19 @@ FROM public.ecr.aws/lambda/python:3.8-x86_64
|
|||
|
||||
# Copy your application code to `LAMBDA_TASK_ROOT`
|
||||
COPY app.py ${LAMBDA_TASK_ROOT}
|
||||
# When you build your Server SDK for your service you get a shared library
|
||||
# that is importable in Python. You need to copy that shared library to same folder
|
||||
# with your application code, so it can be imported by your application.
|
||||
# Note that you need to build your library for Linux,
|
||||
# if you are on a different platform you can consult to
|
||||
# https://pyo3.rs/latest/building_and_distribution.html#cross-compiling
|
||||
# for cross compiling.
|
||||
COPY lib_pokemon_service_server_sdk.so ${LAMBDA_TASK_ROOT}
|
||||
|
||||
# You can install your application's dependencies using file `requirements.txt`
|
||||
# from your project folder, if you have any.
|
||||
# COPY requirements.txt .
|
||||
# RUN pip3 install -r requirements.txt --target "${LAMBDA_TASK_ROOT}"
|
||||
# When you build your Server SDK for your service, you will get a Python wheel.
|
||||
# You just need to copy that wheel and install it via `pip` inside your image.
|
||||
# Note that you need to build your library for Linux, and Python version used to
|
||||
# build your SDK should match with your image's Python version.
|
||||
# For cross compiling, you can consult to:
|
||||
# https://pyo3.rs/latest/building_and_distribution.html#cross-compiling
|
||||
COPY wheels/ ${LAMBDA_TASK_ROOT}/wheels
|
||||
RUN pip3 install ${LAMBDA_TASK_ROOT}/wheels/*.whl
|
||||
|
||||
# You can install your application's other dependencies listed in `requirements.txt`.
|
||||
COPY requirements.txt .
|
||||
RUN pip3 install -r requirements.txt --target "${LAMBDA_TASK_ROOT}"
|
||||
|
||||
# Create a symlink for your application's entrypoint,
|
||||
# so we can use `/app.py` to refer it
|
||||
|
|
@ -69,6 +69,9 @@ ENTRYPOINT [ "/var/lang/bin/python3.8" ]
|
|||
CMD [ "/app.py" ]
|
||||
```
|
||||
|
||||
See [https://docs.aws.amazon.com/lambda/latest/dg/images-create.html#images-create-from-base](https://docs.aws.amazon.com/lambda/latest/dg/images-create.html#images-create-from-base)
|
||||
for more details on building your custom image.
|
||||
|
||||
<!-- anchor_start:footer -->
|
||||
This crate is part of the [AWS SDK for Rust](https://awslabs.github.io/aws-sdk-rust/) and the [smithy-rs](https://github.com/awslabs/smithy-rs) code generator. In most cases, it should not be used directly.
|
||||
<!-- anchor_end:footer -->
|
||||
|
|
|
|||
|
|
@ -1,12 +0,0 @@
|
|||
# See: https://pyo3.rs/latest/building_and_distribution.html?highlight=rustflags#macos
|
||||
[target.x86_64-apple-darwin]
|
||||
rustflags = [
|
||||
"-C", "link-arg=-undefined",
|
||||
"-C", "link-arg=dynamic_lookup",
|
||||
]
|
||||
|
||||
[target.aarch64-apple-darwin]
|
||||
rustflags = [
|
||||
"-C", "link-arg=-undefined",
|
||||
"-C", "link-arg=dynamic_lookup",
|
||||
]
|
||||
|
|
@ -1,3 +1,3 @@
|
|||
pokemon-service-client/
|
||||
pokemon-service-server-sdk/
|
||||
libpokemon_service_server_sdk.so
|
||||
wheels/
|
||||
|
|
|
|||
|
|
@ -1,49 +1,58 @@
|
|||
OS := $(shell uname -s)
|
||||
SRC_DIR := $(shell git rev-parse --show-toplevel)
|
||||
CUR_DIR := $(shell pwd)
|
||||
GRADLE := $(SRC_DIR)/gradlew
|
||||
WHEELS := $(CUR_DIR)/wheels
|
||||
SERVER_SDK_DST := $(CUR_DIR)/pokemon-service-server-sdk
|
||||
CLIENT_SDK_DST := $(CUR_DIR)/pokemon-service-client
|
||||
SERVER_SDK_SRC := $(SRC_DIR)/codegen-server-test/python/build/smithyprojections/codegen-server-test-python/pokemon-service-server-sdk/rust-server-codegen-python
|
||||
CLIENT_SDK_SRC := $(SRC_DIR)/codegen-client-test/build/smithyprojections/codegen-client-test/pokemon-service-client/rust-codegen
|
||||
|
||||
SHARED_LIBRARY_DST := $(CUR_DIR)/libpokemon_service_server_sdk.so
|
||||
ifeq ($(OS), Darwin)
|
||||
DEBUG_SHARED_LIBRARY_SRC := $(SRC_DIR)/target/debug/libpokemon_service_server_sdk.dylib
|
||||
RELEASE_SHARED_LIBRARY_SRC := $(SRC_DIR)/target/release/libpokemon_service_server_sdk.dylib
|
||||
else
|
||||
DEBUG_SHARED_LIBRARY_SRC := $(SRC_DIR)/target/debug/libpokemon_service_server_sdk.so
|
||||
RELEASE_SHARED_LIBRARY_SRC := $(SRC_DIR)/target/release/libpokemon_service_server_sdk.so
|
||||
endif
|
||||
HAS_MATURIN := $(shell command -v maturin 2> /dev/null)
|
||||
|
||||
all: codegen
|
||||
|
||||
codegen:
|
||||
$(GRADLE) --project-dir $(SRC_DIR) -P modules='pokemon-service-server-sdk,pokemon-service-client' :codegen-client-test:assemble :codegen-server-test:python:assemble
|
||||
mkdir -p $(SERVER_SDK_DST) $(CLIENT_SDK_DST)
|
||||
mkdir -p $(SERVER_SDK_DST) $(CLIENT_SDK_DST) $(WHEELS)
|
||||
cp -av $(SERVER_SDK_SRC)/* $(SERVER_SDK_DST)/
|
||||
cp -av $(CLIENT_SDK_SRC)/* $(CLIENT_SDK_DST)/
|
||||
|
||||
clippy: codegen
|
||||
cargo clippy
|
||||
ensure-maturin:
|
||||
ifndef HAS_MATURIN
|
||||
$(error "maturin is not available; please install it via 'pip install maturin' or 'cargo install maturin'")
|
||||
endif
|
||||
|
||||
build: codegen
|
||||
cargo build
|
||||
ln -sf $(DEBUG_SHARED_LIBRARY_SRC) $(SHARED_LIBRARY_DST)
|
||||
# Note on `--compatibility linux`: Maturin by default uses `manylinux_x_y` but it is not supported
|
||||
# by our current CI version (3.7.10), we can drop `--compatibility linux` when we switch to higher Python version.
|
||||
# For more detail: https://github.com/pypa/manylinux
|
||||
build-wheel: ensure-maturin codegen
|
||||
maturin build --manifest-path $(SERVER_SDK_DST)/Cargo.toml --out $(WHEELS) --compatibility linux
|
||||
|
||||
py_check: build
|
||||
mypy pokemon_service.py
|
||||
build-wheel-release: ensure-maturin codegen
|
||||
maturin build --manifest-path $(SERVER_SDK_DST)/Cargo.toml --out $(WHEELS) --compatibility linux --release
|
||||
|
||||
release: codegen
|
||||
cargo build --release
|
||||
ln -sf $(RELEASE_SHARED_LIBRARY_SRC) $(SHARED_LIBRARY_DST)
|
||||
install-wheel:
|
||||
find $(WHEELS) -type f -name '*.whl' | xargs python3 -m pip install --user --force-reinstall
|
||||
|
||||
build: build-wheel install-wheel
|
||||
|
||||
release: build-wheel-release install-wheel
|
||||
|
||||
run: build
|
||||
python3 $(CUR_DIR)/pokemon_service.py
|
||||
|
||||
run-release: release
|
||||
python3 $(CUR_DIR)/pokemon_service.py
|
||||
|
||||
py-check: build
|
||||
mypy pokemon_service.py
|
||||
|
||||
test: build
|
||||
cargo test
|
||||
|
||||
clippy: codegen
|
||||
cargo clippy
|
||||
|
||||
doc-open: codegen
|
||||
cargo doc --no-deps --open
|
||||
|
||||
|
|
@ -51,6 +60,6 @@ clean:
|
|||
cargo clean || echo "Unable to run cargo clean"
|
||||
|
||||
distclean: clean
|
||||
rm -rf $(SERVER_SDK_DST) $(CLIENT_SDK_DST) $(CUR_DIR)/Cargo.lock $(SHARED_LIBRARY_DST)
|
||||
rm -rf $(SERVER_SDK_DST) $(CLIENT_SDK_DST) $(WHEELS) $(CUR_DIR)/Cargo.lock
|
||||
|
||||
.PHONY: all
|
||||
|
|
|
|||
|
|
@ -10,32 +10,32 @@ from threading import Lock
|
|||
from dataclasses import dataclass
|
||||
from typing import List, Optional, Callable, Awaitable
|
||||
|
||||
from libpokemon_service_server_sdk import App
|
||||
from libpokemon_service_server_sdk.tls import TlsConfig # type: ignore
|
||||
from libpokemon_service_server_sdk.aws_lambda import LambdaContext # type: ignore
|
||||
from libpokemon_service_server_sdk.error import ResourceNotFoundException # type: ignore
|
||||
from libpokemon_service_server_sdk.input import ( # type: ignore
|
||||
from pokemon_service_server_sdk import App
|
||||
from pokemon_service_server_sdk.tls import TlsConfig # type: ignore
|
||||
from pokemon_service_server_sdk.aws_lambda import LambdaContext # type: ignore
|
||||
from pokemon_service_server_sdk.error import ResourceNotFoundException # type: ignore
|
||||
from pokemon_service_server_sdk.input import ( # type: ignore
|
||||
DoNothingInput,
|
||||
GetPokemonSpeciesInput,
|
||||
GetServerStatisticsInput,
|
||||
CheckHealthInput,
|
||||
StreamPokemonRadioInput,
|
||||
)
|
||||
from libpokemon_service_server_sdk.logging import TracingHandler # type: ignore
|
||||
from libpokemon_service_server_sdk.middleware import ( # type: ignore
|
||||
from pokemon_service_server_sdk.logging import TracingHandler # type: ignore
|
||||
from pokemon_service_server_sdk.middleware import ( # type: ignore
|
||||
MiddlewareException,
|
||||
Response,
|
||||
Request,
|
||||
)
|
||||
from libpokemon_service_server_sdk.model import FlavorText, Language # type: ignore
|
||||
from libpokemon_service_server_sdk.output import ( # type: ignore
|
||||
from pokemon_service_server_sdk.model import FlavorText, Language # type: ignore
|
||||
from pokemon_service_server_sdk.output import ( # type: ignore
|
||||
DoNothingOutput,
|
||||
GetPokemonSpeciesOutput,
|
||||
GetServerStatisticsOutput,
|
||||
CheckHealthOutput,
|
||||
StreamPokemonRadioOutput,
|
||||
)
|
||||
from libpokemon_service_server_sdk.types import ByteStream # type: ignore
|
||||
from pokemon_service_server_sdk.types import ByteStream # type: ignore
|
||||
|
||||
# Logging can bee setup using standard Python tooling. We provide
|
||||
# fast logging handler, Tracingandler based on Rust tracing crate.
|
||||
|
|
|
|||
|
|
@ -52,6 +52,8 @@ ARG cargo_udeps_version=0.1.35
|
|||
ARG cargo_hack_version=0.5.23
|
||||
ARG cargo_minimal_versions_version=0.1.8
|
||||
ARG cargo_check_external_types_version=0.1.6
|
||||
# Maturin is needed for Python SSDK
|
||||
ARG maturin_version=0.14.1
|
||||
ENV RUSTUP_HOME=/opt/rustup \
|
||||
CARGO_HOME=/opt/cargo \
|
||||
PATH=/opt/cargo/bin/:${PATH} \
|
||||
|
|
@ -100,6 +102,7 @@ RUN set -eux; \
|
|||
cargo install cargo-hack --locked --version ${cargo_hack_version}; \
|
||||
cargo install cargo-minimal-versions --locked --version ${cargo_minimal_versions_version}; \
|
||||
cargo install cargo-check-external-types --locked --version ${cargo_check_external_types_version}; \
|
||||
cargo install maturin --locked --version ${maturin_version}; \
|
||||
if [[ "${checkout_smithy_rs_tools}" == "true" ]]; then \
|
||||
git clone https://github.com/awslabs/smithy-rs.git; \
|
||||
cd smithy-rs; \
|
||||
|
|
|
|||
Loading…
Reference in New Issue