dnrops
/
smithy-rs
mirror of https://github.com/smithy-lang/smithy-rs
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add support for env-defined endpoint URLs (#3488) 

## Motivation and Context
<!--- Why is this change required? What problem does it solve? -->
<!--- If it fixes an open issue, please link to the issue here -->
Part of the endpoint config work I'm doing

## Description
<!--- Describe your changes in detail -->
This change does two things:
- add support for setting an endpoint URL from the env or profile file
- add support for ignoring endpoint URLs sourced from the env and
profile file

## Testing
<!--- Please describe in detail how you tested your changes -->
<!--- Include details of your testing environment, and the tests you ran
to -->
<!--- see how your change affects other areas of the code, etc. -->
I wrote many unit tests.

## Checklist
<!--- If a checkbox below is not applicable, then please DELETE it
rather than leaving it unchecked -->
- [x] I have updated `CHANGELOG.next.toml` if I made changes to the AWS
SDK, generated SDK code, or SDK runtime crates

----

_By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice._

---------

Co-authored-by: John DiSanti <jdisanti@amazon.com>
This commit is contained in:
Zelda Hessler 2024-03-15 12:23:49 -05:00 committed by GitHub
parent 4b9c9b757a
commit 0b35a092b3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 354 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
CHANGELOG.next.toml
View File

@ -22,3 +22,19 @@ message = "`DefaultS3ExpressIdentityProvider` now uses `BehaviorVersion` threade
references = ["smithy-rs#3478"]
meta = { "breaking" = false, "bug" = true, "tada" = false }
author = "ysaito1001"
[[aws-sdk-rust]]
message = """
Users may now set an endpoint URL from the env or profile file:
- env: `AWS_ENDPOINT_URL="http://localhost"`
- profile: `endpoint_url = http://localhost`
Users may also ignore endpoint URLs sourced from the env and profile files:
- env: `AWS_IGNORE_CONFIGURED_ENDPOINT_URLS="true"`
- profile: `ignore_configured_endpoint_urls = true`
"""
references = ["smithy-rs#3488"]
meta = { "breaking" = false, "tada" = true, "bug" = false }
authors = ["Velfi"]

7
aws/rust-runtime/aws-config/Cargo.toml
View File

@ -1,6 +1,6 @@
[package]
name = "aws-config"
version = "1.1.8"
version = "1.1.9"
authors = [
"AWS Rust SDK Team <aws-sdk-rust@amazon.com>",
"Russell Cohen <rcoh@amazon.com>",
@ -26,6 +26,7 @@ allow-compilation = []
[dependencies]
aws-credential-types = { path = "../../sdk/build/aws-sdk/sdk/aws-credential-types", features = ["test-util"] }
aws-runtime = { path = "../../sdk/build/aws-sdk/sdk/aws-runtime" }
aws-sdk-sts = { path = "../../sdk/build/aws-sdk/sdk/sts", default-features = false }
aws-smithy-async = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-async" }
aws-smithy-http = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-http" }
@ -33,12 +34,12 @@ aws-smithy-json = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-json" }
aws-smithy-runtime = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-runtime", features = ["client"] }
aws-smithy-runtime-api = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-runtime-api", features = ["client"] }
aws-smithy-types = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-types" }
aws-runtime = { path = "../../sdk/build/aws-sdk/sdk/aws-runtime" }
aws-types = { path = "../../sdk/build/aws-sdk/sdk/aws-types" }
hyper = { version = "0.14.26", default-features = false }
time = { version = "0.3.4", features = ["parsing"] }
tokio = { version = "1.13.1", features = ["sync"] }
tracing = { version = "0.1" }
url = "2.3.1"
# implementation detail of IMDS credentials provider
fastrand = "2.0.0"
@ -59,7 +60,7 @@ aws-sdk-ssooidc = { path = "../../sdk/build/aws-sdk/sdk/ssooidc", default-featur
aws-smithy-runtime = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-runtime", features = ["client", "connector-hyper-0-14-x", "test-util"] }
aws-smithy-runtime-api = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-runtime-api", features = ["test-util"] }
futures-util = { version = "0.3.29", default-features = false }
tracing-test = "0.2.1"
tracing-test = "0.2.4"
tracing-subscriber = { version = "0.3.16", features = ["fmt", "json"] }
tokio = { version = "1.23.1", features = ["full", "test-util"] }

6
aws/rust-runtime/aws-config/src/default_provider.rs
View File

@ -51,3 +51,9 @@ pub mod use_dual_stack;
/// Default access token provider chain
#[cfg(feature = "sso")]
pub mod token;
/// Default "ignore configured endpoint URLs" provider chain
pub mod ignore_configured_endpoint_urls;
/// Default endpoint URL provider chain
pub mod endpoint_url;

79
aws/rust-runtime/aws-config/src/default_provider/endpoint_url.rs Normal file
View File

@ -0,0 +1,79 @@
/*
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
* SPDX-License-Identifier: Apache-2.0
*/
use crate::environment::parse_url;
use crate::provider_config::ProviderConfig;
use crate::standard_property::StandardProperty;
use aws_smithy_types::error::display::DisplayErrorContext;
mod env {
pub(super) const ENDPOINT_URL: &str = "AWS_ENDPOINT_URL";
}
mod profile_key {
pub(super) const ENDPOINT_URL: &str = "endpoint_url";
}
/// Load the value for an endpoint URL
///
/// This checks the following sources:
/// 1. The environment variable `AWS_ENDPOINT_URL=http://localhost`
/// 2. The profile key `endpoint_url=http://localhost`
///
/// If invalid values are found, the provider will return None and an error will be logged.
pub async fn endpoint_url_provider(provider_config: &ProviderConfig) -> Option<String> {
StandardProperty::new()
.env(env::ENDPOINT_URL)
.profile(profile_key::ENDPOINT_URL)
.validate(provider_config, parse_url)
.await
.map_err(
|err| tracing::warn!(err = %DisplayErrorContext(&err), "invalid value for endpoint URL setting"),
)
.unwrap_or(None)
}
#[cfg(test)]
mod test {
use super::endpoint_url_provider;
use super::env;
use crate::profile::profile_file::{ProfileFileKind, ProfileFiles};
use crate::provider_config::ProviderConfig;
use aws_types::os_shim_internal::{Env, Fs};
use tracing_test::traced_test;
#[tokio::test]
#[traced_test]
async fn log_error_on_invalid_value() {
let conf =
ProviderConfig::empty().with_env(Env::from_slice(&[(env::ENDPOINT_URL, "not-a-url")]));
assert_eq!(None, endpoint_url_provider(&conf).await);
assert!(logs_contain("invalid value for endpoint URL setting"));
assert!(logs_contain(env::ENDPOINT_URL));
}
#[tokio::test]
#[traced_test]
async fn environment_priority() {
let conf = ProviderConfig::empty()
.with_env(Env::from_slice(&[(env::ENDPOINT_URL, "http://localhost")]))
.with_profile_config(
Some(
ProfileFiles::builder()
.with_file(ProfileFileKind::Config, "conf")
.build(),
),
None,
)
.with_fs(Fs::from_slice(&[(
"conf",
"[default]\nendpoint_url = http://production",
)]));
assert_eq!(
Some("http://localhost".to_owned()),
endpoint_url_provider(&conf).await,
);
}
}

88
aws/rust-runtime/aws-config/src/default_provider/ignore_configured_endpoint_urls.rs Normal file
View File

@ -0,0 +1,88 @@
/*
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
* SPDX-License-Identifier: Apache-2.0
*/
use crate::environment::parse_bool;
use crate::provider_config::ProviderConfig;
use crate::standard_property::StandardProperty;
use aws_smithy_types::error::display::DisplayErrorContext;
mod env {
pub(super) const IGNORE_CONFIGURED_ENDPOINT_URLS: &str = "AWS_IGNORE_CONFIGURED_ENDPOINT_URLS";
}
mod profile_key {
pub(super) const IGNORE_CONFIGURED_ENDPOINT_URLS: &str = "ignore_configured_endpoint_urls";
}
/// Load the value for "ignore configured endpoint URLs"
///
/// This checks the following sources:
/// 1. The environment variable `AWS_IGNORE_CONFIGURED_ENDPOINT_URLS_ENDPOINT=true/false`
/// 2. The profile key `ignore_configured_endpoint_urls=true/false`
///
/// If invalid values are found, the provider will return None and an error will be logged.
pub async fn ignore_configured_endpoint_urls_provider(
provider_config: &ProviderConfig,
) -> Option<bool> {
StandardProperty::new()
.env(env::IGNORE_CONFIGURED_ENDPOINT_URLS)
.profile(profile_key::IGNORE_CONFIGURED_ENDPOINT_URLS)
.validate(provider_config, parse_bool)
.await
.map_err(
|err| tracing::warn!(err = %DisplayErrorContext(&err), "invalid value for 'ignore configured endpoint URLs' setting"),
)
.unwrap_or(None)
}
#[cfg(test)]
mod test {
use super::env;
use super::ignore_configured_endpoint_urls_provider;
use crate::profile::profile_file::{ProfileFileKind, ProfileFiles};
use crate::provider_config::ProviderConfig;
use aws_types::os_shim_internal::{Env, Fs};
use tracing_test::traced_test;
#[tokio::test]
#[traced_test]
async fn log_error_on_invalid_value() {
let conf = ProviderConfig::empty().with_env(Env::from_slice(&[(
env::IGNORE_CONFIGURED_ENDPOINT_URLS,
"not-a-boolean",
)]));
assert_eq!(None, ignore_configured_endpoint_urls_provider(&conf).await,);
assert!(logs_contain(
"invalid value for 'ignore configured endpoint URLs' setting"
));
assert!(logs_contain(env::IGNORE_CONFIGURED_ENDPOINT_URLS));
}
#[tokio::test]
#[traced_test]
async fn environment_priority() {
let conf = ProviderConfig::empty()
.with_env(Env::from_slice(&[(
env::IGNORE_CONFIGURED_ENDPOINT_URLS,
"TRUE",
)]))
.with_profile_config(
Some(
ProfileFiles::builder()
.with_file(ProfileFileKind::Config, "conf")
.build(),
),
None,
)
.with_fs(Fs::from_slice(&[(
"conf",
"[default]\nignore_configured_endpoint_urls = false",
)]));
assert_eq!(
Some(true),
ignore_configured_endpoint_urls_provider(&conf).await,
);
}
}

31
aws/rust-runtime/aws-config/src/environment/mod.rs
View File

@ -6,7 +6,7 @@
//! Providers that load configuration from environment variables
use std::error::Error;
use std::fmt::{Display, Formatter};
use std::fmt;
/// Load credentials from the environment
pub mod credentials;
@ -21,9 +21,9 @@ pub(crate) struct InvalidBooleanValue {
value: String,
}
impl Display for InvalidBooleanValue {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(f, "{} was not a valid boolean", self.value)
impl fmt::Display for InvalidBooleanValue {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{} is not a valid boolean", self.value)
}
}
@ -40,3 +40,26 @@ pub(crate) fn parse_bool(value: &str) -> Result<bool, InvalidBooleanValue> {
})
}
}
#[derive(Debug)]
pub(crate) struct InvalidUrlValue {
value: String,
}
impl fmt::Display for InvalidUrlValue {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{} is not a valid URL", self.value)
}
}
impl Error for InvalidUrlValue {}
pub(crate) fn parse_url(value: &str) -> Result<String, InvalidUrlValue> {
match url::Url::parse(value) {
// We discard the parse result because it includes a trailing slash
Ok(_) => Ok(value.to_string()),
Err(_) => Err(InvalidUrlValue {
value: value.to_string(),
}),
}
}

136
aws/rust-runtime/aws-config/src/lib.rs
View File

@ -208,12 +208,6 @@ pub async fn load_defaults(version: BehaviorVersion) -> SdkConfig {
}
mod loader {
use crate::default_provider::use_dual_stack::use_dual_stack_provider;
use crate::default_provider::use_fips::use_fips_provider;
use crate::default_provider::{app_name, credentials, region, retry_config, timeout_config};
use crate::meta::region::ProvideRegion;
use crate::profile::profile_file::ProfileFiles;
use crate::provider_config::ProviderConfig;
use aws_credential_types::provider::{
token::{ProvideToken, SharedTokenProvider},
ProvideCredentials, SharedCredentialsProvider,
@ -234,6 +228,14 @@ mod loader {
use aws_types::sdk_config::SharedHttpClient;
use aws_types::SdkConfig;
use crate::default_provider::{
app_name, credentials, endpoint_url, ignore_configured_endpoint_urls as ignore_ep, region,
retry_config, timeout_config, use_dual_stack, use_fips,
};
use crate::meta::region::ProvideRegion;
use crate::profile::profile_file::ProfileFiles;
use crate::provider_config::ProviderConfig;
#[derive(Default, Debug)]
enum CredentialsProviderOption {
/// No provider was set by the user. We can set up the default credentials provider chain.
@ -726,13 +728,13 @@ mod loader {
let use_fips = if let Some(use_fips) = self.use_fips {
Some(use_fips)
} else {
use_fips_provider(&conf).await
use_fips::use_fips_provider(&conf).await
};
let use_dual_stack = if let Some(use_dual_stack) = self.use_dual_stack {
Some(use_dual_stack)
} else {
use_dual_stack_provider(&conf).await
use_dual_stack::use_dual_stack_provider(&conf).await
};
let conf = conf
@ -811,6 +813,30 @@ mod loader {
.timeout_config(timeout_config)
.time_source(time_source);
// If an endpoint URL is set programmatically, then our work is done.
let endpoint_url = if self.endpoint_url.is_some() {
self.endpoint_url
} else {
// Otherwise, check to see if we should ignore EP URLs set in the environment.
let ignore_configured_endpoint_urls =
ignore_ep::ignore_configured_endpoint_urls_provider(&conf)
.await
.unwrap_or_default();
if ignore_configured_endpoint_urls {
// If yes, log a trace and return `None`.
tracing::trace!(
"`ignore_configured_endpoint_urls` is set, any endpoint URLs configured in the environment will be ignored. \
NOTE: Endpoint URLs set programmatically WILL still be respected"
);
None
} else {
// Otherwise, attempt to resolve one.
endpoint_url::endpoint_url_provider(&conf).await
}
};
builder.set_endpoint_url(endpoint_url);
builder.set_behavior_version(self.behavior_version);
builder.set_http_client(self.http_client);
builder.set_app_name(app_name);
@ -818,7 +844,6 @@ mod loader {
builder.set_credentials_provider(credentials_provider);
builder.set_token_provider(token_provider);
builder.set_sleep_impl(sleep_impl);
builder.set_endpoint_url(self.endpoint_url);
builder.set_use_fips(use_fips);
builder.set_use_dual_stack(use_dual_stack);
builder.set_stalled_stream_protection(self.stalled_stream_protection_config);
@ -970,5 +995,98 @@ mod loader {
let num_requests = num_requests.load(Ordering::Relaxed);
assert!(num_requests > 0, "{}", num_requests);
}
#[tokio::test]
async fn endpoint_urls_may_be_ignored_from_env() {
let fs = Fs::from_slice(&[(
"test_config",
"[profile custom]\nendpoint_url = http://profile",
)]);
let env = Env::from_slice(&[("AWS_IGNORE_CONFIGURED_ENDPOINT_URLS", "true")]);
let conf = base_conf().use_dual_stack(false).load().await;
assert_eq!(Some(false), conf.use_dual_stack());
let conf = base_conf().load().await;
assert_eq!(None, conf.use_dual_stack());
// Check that we get nothing back because the env said we should ignore endpoints
let config = base_conf()
.fs(fs.clone())
.env(env)
.profile_name("custom")
.profile_files(
ProfileFiles::builder()
.with_file(ProfileFileKind::Config, "test_config")
.build(),
)
.load()
.await;
assert_eq!(None, config.endpoint_url());
// Check that without the env, we DO get something back
let config = base_conf()
.fs(fs)
.profile_name("custom")
.profile_files(
ProfileFiles::builder()
.with_file(ProfileFileKind::Config, "test_config")
.build(),
)
.load()
.await;
assert_eq!(Some("http://profile"), config.endpoint_url());
}
#[tokio::test]
async fn endpoint_urls_may_be_ignored_from_profile() {
let fs = Fs::from_slice(&[(
"test_config",
"[profile custom]\nignore_configured_endpoint_urls = true",
)]);
let env = Env::from_slice(&[("AWS_ENDPOINT_URL", "http://environment")]);
// Check that we get nothing back because the profile said we should ignore endpoints
let config = base_conf()
.fs(fs)
.env(env.clone())
.profile_name("custom")
.profile_files(
ProfileFiles::builder()
.with_file(ProfileFileKind::Config, "test_config")
.build(),
)
.load()
.await;
assert_eq!(None, config.endpoint_url());
// Check that without the profile, we DO get something back
let config = base_conf().env(env).load().await;
assert_eq!(Some("http://environment"), config.endpoint_url());
}
#[tokio::test]
async fn programmatic_endpoint_urls_may_not_be_ignored() {
let fs = Fs::from_slice(&[(
"test_config",
"[profile custom]\nignore_configured_endpoint_urls = true",
)]);
let env = Env::from_slice(&[("AWS_IGNORE_CONFIGURED_ENDPOINT_URLS", "true")]);
// Check that we get something back because we explicitly set the loader's endpoint URL
let config = base_conf()
.fs(fs)
.env(env)
.endpoint_url("http://localhost")
.profile_name("custom")
.profile_files(
ProfileFiles::builder()
.with_file(ProfileFileKind::Config, "test_config")
.build(),
)
.load()
.await;
assert_eq!(Some("http://localhost"), config.endpoint_url());
}
}
}

6
aws/sdk/integration-tests/s3/tests/service_timeout_overrides.rs
View File

@ -97,7 +97,13 @@ async fn default_connect_timeout_set() {
.build()
)
);
assert_eq!(
sdk_config.endpoint_url(),
Some("http://172.255.255.0:18104")
);
let config = aws_sdk_s3::config::Builder::from(&sdk_config)
// .endpoint_url("http://172.255.255.0:18104")
.timeout_config(
TimeoutConfig::builder()
.operation_attempt_timeout(Duration::from_secs(8))

2
rust-runtime/inlineable/Cargo.toml
View File

@ -32,7 +32,7 @@ once_cell = "1.16.0"
percent-encoding = "2.2.0"
pin-project-lite = "0.2"
regex-lite = "0.1.5"
url = "2.2.2"
url = "2.3.1"
[dev-dependencies]
proptest = "1"