From fe711eee1faea1d715653c8b16b4fba59d358b52 Mon Sep 17 00:00:00 2001 From: Russell Cohen Date: Fri, 21 May 2021 12:05:07 -0400 Subject: [PATCH] Support signing requests to S3 (#399) * Support signing requests to S3 * Refactor to include support for signing streaming bodies (tested) * Update stale comments * Remove left-over `dbg!` invocation --- aws/rust-runtime/aws-hyper/src/lib.rs | 2 +- aws/rust-runtime/aws-hyper/tests/e2e_test.rs | 2 +- aws/rust-runtime/aws-sig-auth/Cargo.toml | 2 +- .../aws-sig-auth/src/middleware.rs | 27 +++------- aws/rust-runtime/aws-sig-auth/src/signer.rs | 50 ++++++++++++------- .../smithy/rustsdk/SigV4SigningDecorator.kt | 34 ++++++++++--- .../kms/tests/integration.rs | 5 +- .../qldbsession/tests/integration.rs | 4 +- 8 files changed, 76 insertions(+), 50 deletions(-) diff --git a/aws/rust-runtime/aws-hyper/src/lib.rs b/aws/rust-runtime/aws-hyper/src/lib.rs index fdd961dd0d..fba7a20fe1 100644 --- a/aws/rust-runtime/aws-hyper/src/lib.rs +++ b/aws/rust-runtime/aws-hyper/src/lib.rs @@ -128,9 +128,9 @@ where .retry(self.retry_handler.new_handler()) .layer(ParseResponseLayer::::new()) .layer(endpoint_resolver) + .layer(user_agent) .layer(signer) // Apply the user agent _after signing_. We should not sign the user-agent header - .layer(user_agent) .layer(DispatchLayer::new()) .service(inner); svc.ready().await?.call(input).await diff --git a/aws/rust-runtime/aws-hyper/tests/e2e_test.rs b/aws/rust-runtime/aws-hyper/tests/e2e_test.rs index 8acfd7bc75..356184f867 100644 --- a/aws/rust-runtime/aws-hyper/tests/e2e_test.rs +++ b/aws/rust-runtime/aws-hyper/tests/e2e_test.rs @@ -107,7 +107,7 @@ async fn e2e_test() { .header(USER_AGENT, "aws-sdk-rust/0.123.test os/windows/XPSP3 lang/rust/1.50.0") .header("x-amz-user-agent", "aws-sdk-rust/0.123.test api/test-service/0.123 os/windows/XPSP3 lang/rust/1.50.0") .header(HOST, "test-service.test-region.amazonaws.com") - .header(AUTHORIZATION, "AWS4-HMAC-SHA256 Credential=access_key/20210215/test-region/test-service-signing/aws4_request, SignedHeaders=host, Signature=5ebafc2fc4a104b63a1f87ffe829e6eb860a48db8c105a7921b82ee3dc02f1b8") + .header(AUTHORIZATION, "AWS4-HMAC-SHA256 Credential=access_key/20210215/test-region/test-service-signing/aws4_request, SignedHeaders=host;x-amz-date;x-amz-user-agent, Signature=da249491d7fe3da22c2e09cbf910f37aa5b079a3cedceff8403d0b18a7bfab75") .header("x-amz-date", "20210215T184017Z") .uri(Uri::from_static("https://test-service.test-region.amazonaws.com/")) .body(SdkBody::from("request body")).unwrap(); diff --git a/aws/rust-runtime/aws-sig-auth/Cargo.toml b/aws/rust-runtime/aws-sig-auth/Cargo.toml index 3bbc7b3e29..d5d3f3b52c 100644 --- a/aws/rust-runtime/aws-sig-auth/Cargo.toml +++ b/aws/rust-runtime/aws-sig-auth/Cargo.toml @@ -10,7 +10,7 @@ license = "Apache-2.0" [dependencies] http = "0.2.2" # Renaming to clearly indicate that this is not a permanent signing solution -aws-sigv4-poc = { package = "aws-sigv4", git = "https://github.com/rcoh/sigv4", rev = "8faba4281244fc284aba9b67830d6a4c1ed4385a"} +aws-sigv4-poc = { package = "aws-sigv4", git = "https://github.com/rcoh/sigv4", rev = "1854c5f5728c80b0970fcca86c2431bf288f6997"} aws-auth = { path = "../aws-auth" } aws-types = { path = "../aws-types" } smithy-http = { path = "../../../rust-runtime/smithy-http" } diff --git a/aws/rust-runtime/aws-sig-auth/src/middleware.rs b/aws/rust-runtime/aws-sig-auth/src/middleware.rs index 92357cde20..eebbca1075 100644 --- a/aws/rust-runtime/aws-sig-auth/src/middleware.rs +++ b/aws/rust-runtime/aws-sig-auth/src/middleware.rs @@ -91,27 +91,13 @@ impl MapRequest for SigV4SigningStage { type Error = SigningStageError; fn apply(&self, req: Request) -> Result { - req.augment(|req, config| { + req.augment(|mut req, config| { let (operation_config, request_config, creds) = signing_config(config)?; - // A short dance is required to extract a signable body from an SdkBody, which - // amounts to verifying that it a strict body based on a `Bytes` and not a stream. - // Streams must be signed with a different signing mode. Separate support will be added for - // this at a later date. - let (parts, body) = req.into_parts(); - let signable_body = body.bytes().ok_or(SigningStageError::InvalidBodyType)?; - let mut signable_request = http::Request::from_parts(parts, signable_body); - self.signer - .sign( - &operation_config, - &request_config, - &creds, - &mut signable_request, - ) + .sign(&operation_config, &request_config, &creds, &mut req) .map_err(|err| SigningStageError::SigningFailure(err))?; - let (signed_parts, _) = signable_request.into_parts(); - Ok(http::Request::from_parts(signed_parts, body)) + Ok(req) }) } } @@ -157,8 +143,9 @@ mod test { .apply(req.try_clone().expect("can clone")) .expect_err("no signing config"), ); - req.config_mut() - .insert(OperationSigningConfig::default_config()); + let mut config = OperationSigningConfig::default_config(); + config.signing_options.content_sha256_header = true; + req.config_mut().insert(config); errs.push( signer .apply(req.try_clone().expect("can clone")) @@ -185,6 +172,6 @@ mod test { .headers() .get(AUTHORIZATION) .expect("auth header must be present"); - assert_eq!(auth_header, "AWS4-HMAC-SHA256 Credential=AKIAfoo/20210120/us-east-1/kinesis/aws4_request, SignedHeaders=host, Signature=c59f1b9040fe229bf924254d9ad71adaf0495db2ccda5eb6b1565529cdc2c120"); + assert_eq!(auth_header, "AWS4-HMAC-SHA256 Credential=AKIAfoo/20210120/us-east-1/kinesis/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=af71a409f0229dfd6e88409cd1b11f5c2803868d6869888e53bbf9ee12a97ea0"); } } diff --git a/aws/rust-runtime/aws-sig-auth/src/signer.rs b/aws/rust-runtime/aws-sig-auth/src/signer.rs index 6a84ca0d8d..022fb91050 100644 --- a/aws/rust-runtime/aws-sig-auth/src/signer.rs +++ b/aws/rust-runtime/aws-sig-auth/src/signer.rs @@ -4,9 +4,11 @@ */ use aws_auth::Credentials; -use aws_sigv4_poc::{SigningSettings, UriEncoding}; +use aws_sigv4_poc::{SignableBody, SignedBodyHeaderType, SigningSettings, UriEncoding}; use aws_types::region::SigningRegion; use aws_types::SigningService; +use http::header::HeaderName; +use smithy_http::body::SdkBody; use std::error::Error; use std::time::SystemTime; @@ -48,6 +50,7 @@ impl OperationSigningConfig { signature_type: HttpSignatureType::HttpRequestHeaders, signing_options: SigningOptions { double_uri_encode: true, + content_sha256_header: false, }, } } @@ -57,6 +60,7 @@ impl OperationSigningConfig { #[non_exhaustive] pub struct SigningOptions { pub double_uri_encode: bool, + pub content_sha256_header: bool, /* Currently unsupported: pub normalize_uri_path: bool, @@ -93,17 +97,24 @@ impl SigV4Signer { /// /// Although the direct signing implementation MAY be used directly. End users will not typically /// interact with this code. It is generally used via middleware in the request pipeline. See [`SigV4SigningStage`](crate::middleware::SigV4SigningStage). - pub fn sign( + pub fn sign( &self, - // There is currently only 1 way to sign, so operation level configuration is unused operation_config: &OperationSigningConfig, request_config: &RequestConfig<'_>, credentials: &Credentials, - request: &mut http::Request, - ) -> Result<(), SigningError> - where - B: AsRef<[u8]>, - { + request: &mut http::Request, + ) -> Result<(), SigningError> { + let mut settings = SigningSettings::default(); + settings.uri_encoding = if operation_config.signing_options.double_uri_encode { + UriEncoding::Double + } else { + UriEncoding::Single + }; + settings.signed_body_header = if operation_config.signing_options.content_sha256_header { + SignedBodyHeaderType::XAmzSha256 + } else { + SignedBodyHeaderType::NoHeader + }; let sigv4_config = aws_sigv4_poc::Config { access_key: credentials.access_key_id(), secret_key: credentials.secret_access_key(), @@ -111,18 +122,23 @@ impl SigV4Signer { region: request_config.region.as_ref(), svc: request_config.service.as_ref(), date: request_config.request_ts, - settings: SigningSettings { - uri_encoding: if operation_config.signing_options.double_uri_encode { - UriEncoding::Double - } else { - UriEncoding::Single - }, - }, + settings, }; - for (key, value) in aws_sigv4_poc::sign_core(request, sigv4_config) { + + // A body that is already in memory can be signed directly. A body that is not in memory + // (any sort of streaming body) will be signed via UNSIGNED-PAYLOAD. + // The final enhancement that will come a bit later is writing a `SignableBody::Precomputed` + // into the property bag when we have a sha 256 middleware that can compute a streaming checksum + // for replayable streams but currently even replayable streams will result in `UNSIGNED-PAYLOAD` + let signable_body = request + .body() + .bytes() + .map(SignableBody::Bytes) + .unwrap_or(SignableBody::UnsignedPayload); + for (key, value) in aws_sigv4_poc::sign_core(request, signable_body, &sigv4_config)? { request .headers_mut() - .append(key.header_name(), value.parse()?); + .append(HeaderName::from_static(key), value); } Ok(()) diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/SigV4SigningDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/SigV4SigningDecorator.kt index 55936b9e28..89e6ff2543 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/SigV4SigningDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/SigV4SigningDecorator.kt @@ -7,6 +7,8 @@ package software.amazon.smithy.rustsdk import software.amazon.smithy.aws.traits.auth.SigV4Trait import software.amazon.smithy.model.shapes.OperationShape +import software.amazon.smithy.model.shapes.ServiceShape +import software.amazon.smithy.model.shapes.ShapeId import software.amazon.smithy.rust.codegen.rustlang.Writable import software.amazon.smithy.rust.codegen.rustlang.asType import software.amazon.smithy.rust.codegen.rustlang.rust @@ -52,7 +54,7 @@ class SigV4SigningDecorator : RustCodegenDecorator { baseCustomizations: List ): List { return baseCustomizations.letIf(applies(protocolConfig)) { - it + SigV4SigningFeature(protocolConfig.runtimeConfig) + it + SigV4SigningFeature(protocolConfig.runtimeConfig, protocolConfig.serviceShape) } } } @@ -78,7 +80,17 @@ class SigV4SigningConfig(private val sigV4Trait: SigV4Trait) : ConfigCustomizati } } -class SigV4SigningFeature(private val runtimeConfig: RuntimeConfig) : +fun needsAmzSha256(service: ServiceShape) = when { + service.id == ShapeId.from("com.amazonaws.s3#AmazonS3") -> true + else -> false +} + +fun disableDoubleEncode(service: ServiceShape) = when { + service.id == ShapeId.from("com.amazonaws.s3#AmazonS3") -> true + else -> false +} + +class SigV4SigningFeature(private val runtimeConfig: RuntimeConfig, private val service: ServiceShape) : OperationCustomization() { override fun section(section: OperationSection): Writable { return when (section) { @@ -86,12 +98,22 @@ class SigV4SigningFeature(private val runtimeConfig: RuntimeConfig) : // TODO: this needs to be customized for individual operations, not just `default_config()` rustTemplate( """ - ${section.request}.config_mut().insert( - #{sig_auth}::signer::OperationSigningConfig::default_config() - ); + ##[allow(unused_mut)] + let mut signing_config = #{sig_auth}::signer::OperationSigningConfig::default_config(); + """, + "sig_auth" to runtimeConfig.sigAuth().asType() + ) + if (needsAmzSha256(service)) { + rust("signing_config.signing_options.content_sha256_header = true;") + } + if (disableDoubleEncode(service)) { + rust("signing_config.signing_options.double_uri_encode = false;") + } + rustTemplate( + """ + ${section.request}.config_mut().insert(signing_config); ${section.request}.config_mut().insert(#{aws_types}::SigningService::from_static(${section.config}.signing_service())); """, - "sig_auth" to runtimeConfig.sigAuth().asType(), "aws_types" to awsTypes(runtimeConfig).asType() ) } diff --git a/aws/sdk/integration-tests/kms/tests/integration.rs b/aws/sdk/integration-tests/kms/tests/integration.rs index 8b4a395cab..a503afcee6 100644 --- a/aws/sdk/integration-tests/kms/tests/integration.rs +++ b/aws/sdk/integration-tests/kms/tests/integration.rs @@ -8,6 +8,7 @@ use aws_http::user_agent::AwsUserAgent; use aws_hyper::test_connection::TestConnection; use aws_hyper::{Client, SdkError}; use aws_sdk_kms as kms; +use http::header::AUTHORIZATION; use http::Uri; use kms::error::GenerateRandomErrorKind; use kms::operation::GenerateRandom; @@ -32,7 +33,7 @@ async fn generate_random() { .header("x-amz-target", "TrentService.GenerateRandom") .header("content-length", "20") .header("host", "kms.us-east-1.amazonaws.com") - .header("authorization", "AWS4-HMAC-SHA256 Credential=ANOTREAL/20210305/us-east-1/kms/aws4_request, SignedHeaders=content-length;content-type;host;x-amz-target, Signature=750c6333c96dcbe4c4c11a9af8483ff68ac40e0e8ba8244772d981aab3cda703") + .header("authorization", "AWS4-HMAC-SHA256 Credential=ANOTREAL/20210305/us-east-1/kms/aws4_request, SignedHeaders=content-length;content-type;host;x-amz-date;x-amz-security-token;x-amz-target;x-amz-user-agent, Signature=2e0dd7259fba92523d553173c452eba8a6ee7990fb5b1f8e2eccdeb75309e9e1") .header("x-amz-date", "20210305T134922Z") .header("x-amz-security-token", "notarealsessiontoken") .header("user-agent", "aws-sdk-rust/0.123.test os/windows/XPSP3 lang/rust/1.50.0") @@ -165,6 +166,6 @@ async fn generate_random_keystore_not_found() { ); assert_eq!(conn.requests().len(), 1); for validate_request in conn.requests().iter() { - validate_request.assert_matches(vec![]); + validate_request.assert_matches(vec![AUTHORIZATION]); } } diff --git a/aws/sdk/integration-tests/qldbsession/tests/integration.rs b/aws/sdk/integration-tests/qldbsession/tests/integration.rs index 91e48c1db7..bb5a687539 100644 --- a/aws/sdk/integration-tests/qldbsession/tests/integration.rs +++ b/aws/sdk/integration-tests/qldbsession/tests/integration.rs @@ -32,8 +32,8 @@ async fn signv4_use_correct_service_name() { .header("x-amz-target", "QLDBSession.SendCommand") .header("content-length", "49") .header("host", "session.qldb.us-east-1.amazonaws.com") - .header("authorization", "AWS4-HMAC-SHA256 Credential=ANOTREAL/20210305/us-east-1/qldb/aws4_request, SignedHeaders=content-length;content-type;host;x-amz-target, Signature=38be4a432384f4ee7fb9683a9d093cc636a86a4fa6e7e8a198f4437c8c7f596a") - // qldbsession uses the service name 'qldb' in signature _________________________^^^^ + .header("authorization", "AWS4-HMAC-SHA256 Credential=ANOTREAL/20210305/us-east-1/qldb/aws4_request, SignedHeaders=content-length;content-type;host;x-amz-date;x-amz-security-token;x-amz-target;x-amz-user-agent, Signature=350f957e9b736ac3f636d16c59c0a3cee8c2780b0ffadc99bbca841b7f15bee4") + // qldbsession uses the service name 'qldb' in signature ____________________________________^^^^ .header("x-amz-date", "20210305T134922Z") .header("x-amz-security-token", "notarealsessiontoken") .header("user-agent", "aws-sdk-rust/0.123.test os/windows/XPSP3 lang/rust/1.50.0")