Add support for env-defined endpoint URLs (#3488)

## Motivation and Context
<!--- Why is this change required? What problem does it solve? -->
<!--- If it fixes an open issue, please link to the issue here -->
Part of the endpoint config work I'm doing

## Description
<!--- Describe your changes in detail -->
This change does two things:
- add support for setting an endpoint URL from the env or profile file
- add support for ignoring endpoint URLs sourced from the env and
profile file

## Testing
<!--- Please describe in detail how you tested your changes -->
<!--- Include details of your testing environment, and the tests you ran
to -->
<!--- see how your change affects other areas of the code, etc. -->
I wrote many unit tests.

## Checklist
<!--- If a checkbox below is not applicable, then please DELETE it
rather than leaving it unchecked -->
- [x] I have updated `CHANGELOG.next.toml` if I made changes to the AWS
SDK, generated SDK code, or SDK runtime crates

----

_By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice._

---------

Co-authored-by: John DiSanti <jdisanti@amazon.com>

CHANGELOG.next.toml

@@ -22,3 +22,19 @@ message = "`DefaultS3ExpressIdentityProvider` now uses `BehaviorVersion` threade
 references = ["smithy-rs#3478"]
 meta = { "breaking" = false, "bug" = true, "tada" = false }
 author = "ysaito1001"
+
+[[aws-sdk-rust]]
+message = """
+Users may now set an endpoint URL from the env or profile file:
+
+- env: `AWS_ENDPOINT_URL="http://localhost"`
+- profile: `endpoint_url = http://localhost`
+
+Users may also ignore endpoint URLs sourced from the env and profile files:
+
+- env: `AWS_IGNORE_CONFIGURED_ENDPOINT_URLS="true"`
+- profile: `ignore_configured_endpoint_urls = true`
+"""
+references = ["smithy-rs#3488"]
+meta = { "breaking" = false, "tada" = true, "bug" = false }
+authors = ["Velfi"]

aws/rust-runtime/aws-config/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "aws-config"
-version = "1.1.8"
+version = "1.1.9"
 authors = [
     "AWS Rust SDK Team <aws-sdk-rust@amazon.com>",
     "Russell Cohen <rcoh@amazon.com>",
@@ -26,6 +26,7 @@ allow-compilation = []
 
 [dependencies]
 aws-credential-types = { path = "../../sdk/build/aws-sdk/sdk/aws-credential-types", features = ["test-util"] }
+aws-runtime = { path = "../../sdk/build/aws-sdk/sdk/aws-runtime" }
 aws-sdk-sts = { path = "../../sdk/build/aws-sdk/sdk/sts", default-features = false }
 aws-smithy-async = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-async" }
 aws-smithy-http = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-http" }
@@ -33,12 +34,12 @@ aws-smithy-json = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-json" }
 aws-smithy-runtime = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-runtime", features = ["client"] }
 aws-smithy-runtime-api = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-runtime-api", features = ["client"] }
 aws-smithy-types = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-types" }
-aws-runtime = { path = "../../sdk/build/aws-sdk/sdk/aws-runtime" }
 aws-types = { path = "../../sdk/build/aws-sdk/sdk/aws-types" }
 hyper = { version = "0.14.26", default-features = false }
 time = { version = "0.3.4", features = ["parsing"] }
 tokio = { version = "1.13.1", features = ["sync"] }
 tracing = { version = "0.1" }
+url = "2.3.1"
 
 # implementation detail of IMDS credentials provider
 fastrand = "2.0.0"
@@ -59,7 +60,7 @@ aws-sdk-ssooidc = { path = "../../sdk/build/aws-sdk/sdk/ssooidc", default-featur
 aws-smithy-runtime = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-runtime", features = ["client", "connector-hyper-0-14-x", "test-util"] }
 aws-smithy-runtime-api = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-runtime-api", features = ["test-util"] }
 futures-util = { version = "0.3.29", default-features = false }
-tracing-test = "0.2.1"
+tracing-test = "0.2.4"
 tracing-subscriber = { version = "0.3.16", features = ["fmt", "json"] }
 
 tokio = { version = "1.23.1", features = ["full", "test-util"] }

aws/rust-runtime/aws-config/src/default_provider.rs

@@ -51,3 +51,9 @@ pub mod use_dual_stack;
 /// Default access token provider chain
 #[cfg(feature = "sso")]
 pub mod token;
+
+/// Default "ignore configured endpoint URLs" provider chain
+pub mod ignore_configured_endpoint_urls;
+
+/// Default endpoint URL provider chain
+pub mod endpoint_url;

aws/rust-runtime/aws-config/src/default_provider/endpoint_url.rs

@@ -0,0 +1,79 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+use crate::environment::parse_url;
+use crate::provider_config::ProviderConfig;
+use crate::standard_property::StandardProperty;
+use aws_smithy_types::error::display::DisplayErrorContext;
+
+mod env {
+    pub(super) const ENDPOINT_URL: &str = "AWS_ENDPOINT_URL";
+}
+
+mod profile_key {
+    pub(super) const ENDPOINT_URL: &str = "endpoint_url";
+}
+
+/// Load the value for an endpoint URL
+///
+/// This checks the following sources:
+/// 1. The environment variable `AWS_ENDPOINT_URL=http://localhost`
+/// 2. The profile key `endpoint_url=http://localhost`
+///
+/// If invalid values are found, the provider will return None and an error will be logged.
+pub async fn endpoint_url_provider(provider_config: &ProviderConfig) -> Option<String> {
+    StandardProperty::new()
+        .env(env::ENDPOINT_URL)
+        .profile(profile_key::ENDPOINT_URL)
+        .validate(provider_config, parse_url)
+        .await
+        .map_err(
+            |err| tracing::warn!(err = %DisplayErrorContext(&err), "invalid value for endpoint URL setting"),
+        )
+        .unwrap_or(None)
+}
+
+#[cfg(test)]
+mod test {
+    use super::endpoint_url_provider;
+    use super::env;
+    use crate::profile::profile_file::{ProfileFileKind, ProfileFiles};
+    use crate::provider_config::ProviderConfig;
+    use aws_types::os_shim_internal::{Env, Fs};
+    use tracing_test::traced_test;
+
+    #[tokio::test]
+    #[traced_test]
+    async fn log_error_on_invalid_value() {
+        let conf =
+            ProviderConfig::empty().with_env(Env::from_slice(&[(env::ENDPOINT_URL, "not-a-url")]));
+        assert_eq!(None, endpoint_url_provider(&conf).await);
+        assert!(logs_contain("invalid value for endpoint URL setting"));
+        assert!(logs_contain(env::ENDPOINT_URL));
+    }
+
+    #[tokio::test]
+    #[traced_test]
+    async fn environment_priority() {
+        let conf = ProviderConfig::empty()
+            .with_env(Env::from_slice(&[(env::ENDPOINT_URL, "http://localhost")]))
+            .with_profile_config(
+                Some(
+                    ProfileFiles::builder()
+                        .with_file(ProfileFileKind::Config, "conf")
+                        .build(),
+                ),
+                None,
+            )
+            .with_fs(Fs::from_slice(&[(
+                "conf",
+                "[default]\nendpoint_url = http://production",
+            )]));
+        assert_eq!(
+            Some("http://localhost".to_owned()),
+            endpoint_url_provider(&conf).await,
+        );
+    }
+}

aws/rust-runtime/aws-config/src/default_provider/ignore_configured_endpoint_urls.rs

@@ -0,0 +1,88 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+use crate::environment::parse_bool;
+use crate::provider_config::ProviderConfig;
+use crate::standard_property::StandardProperty;
+use aws_smithy_types::error::display::DisplayErrorContext;
+
+mod env {
+    pub(super) const IGNORE_CONFIGURED_ENDPOINT_URLS: &str = "AWS_IGNORE_CONFIGURED_ENDPOINT_URLS";
+}
+
+mod profile_key {
+    pub(super) const IGNORE_CONFIGURED_ENDPOINT_URLS: &str = "ignore_configured_endpoint_urls";
+}
+
+/// Load the value for "ignore configured endpoint URLs"
+///
+/// This checks the following sources:
+/// 1. The environment variable `AWS_IGNORE_CONFIGURED_ENDPOINT_URLS_ENDPOINT=true/false`
+/// 2. The profile key `ignore_configured_endpoint_urls=true/false`
+///
+/// If invalid values are found, the provider will return None and an error will be logged.
+pub async fn ignore_configured_endpoint_urls_provider(
+    provider_config: &ProviderConfig,
+) -> Option<bool> {
+    StandardProperty::new()
+        .env(env::IGNORE_CONFIGURED_ENDPOINT_URLS)
+        .profile(profile_key::IGNORE_CONFIGURED_ENDPOINT_URLS)
+        .validate(provider_config, parse_bool)
+        .await
+        .map_err(
+            |err| tracing::warn!(err = %DisplayErrorContext(&err), "invalid value for 'ignore configured endpoint URLs' setting"),
+        )
+        .unwrap_or(None)
+}
+
+#[cfg(test)]
+mod test {
+    use super::env;
+    use super::ignore_configured_endpoint_urls_provider;
+    use crate::profile::profile_file::{ProfileFileKind, ProfileFiles};
+    use crate::provider_config::ProviderConfig;
+    use aws_types::os_shim_internal::{Env, Fs};
+    use tracing_test::traced_test;
+
+    #[tokio::test]
+    #[traced_test]
+    async fn log_error_on_invalid_value() {
+        let conf = ProviderConfig::empty().with_env(Env::from_slice(&[(
+            env::IGNORE_CONFIGURED_ENDPOINT_URLS,
+            "not-a-boolean",
+        )]));
+        assert_eq!(None, ignore_configured_endpoint_urls_provider(&conf).await,);
+        assert!(logs_contain(
+            "invalid value for 'ignore configured endpoint URLs' setting"
+        ));
+        assert!(logs_contain(env::IGNORE_CONFIGURED_ENDPOINT_URLS));
+    }
+
+    #[tokio::test]
+    #[traced_test]
+    async fn environment_priority() {
+        let conf = ProviderConfig::empty()
+            .with_env(Env::from_slice(&[(
+                env::IGNORE_CONFIGURED_ENDPOINT_URLS,
+                "TRUE",
+            )]))
+            .with_profile_config(
+                Some(
+                    ProfileFiles::builder()
+                        .with_file(ProfileFileKind::Config, "conf")
+                        .build(),
+                ),
+                None,
+            )
+            .with_fs(Fs::from_slice(&[(
+                "conf",
+                "[default]\nignore_configured_endpoint_urls = false",
+            )]));
+        assert_eq!(
+            Some(true),
+            ignore_configured_endpoint_urls_provider(&conf).await,
+        );
+    }
+}

aws/rust-runtime/aws-config/src/environment/mod.rs

@@ -6,7 +6,7 @@
 //! Providers that load configuration from environment variables
 
 use std::error::Error;
-use std::fmt::{Display, Formatter};
+use std::fmt;
 
 /// Load credentials from the environment
 pub mod credentials;
@@ -21,9 +21,9 @@ pub(crate) struct InvalidBooleanValue {
     value: String,
 }
 
-impl Display for InvalidBooleanValue {
+impl fmt::Display for InvalidBooleanValue {
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "{} was not a valid boolean", self.value)
+        write!(f, "{} is not a valid boolean", self.value)
     }
 }
 
@@ -40,3 +40,26 @@ pub(crate) fn parse_bool(value: &str) -> Result<bool, InvalidBooleanValue> {
         })
     }
 }
+
+#[derive(Debug)]
+pub(crate) struct InvalidUrlValue {
+    value: String,
+}
+
+impl fmt::Display for InvalidUrlValue {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{} is not a valid URL", self.value)
+    }
+}
+
+impl Error for InvalidUrlValue {}
+
+pub(crate) fn parse_url(value: &str) -> Result<String, InvalidUrlValue> {
+    match url::Url::parse(value) {
+        // We discard the parse result because it includes a trailing slash
+        Ok(_) => Ok(value.to_string()),
+        Err(_) => Err(InvalidUrlValue {
+            value: value.to_string(),
+        }),
+    }
+}

aws/rust-runtime/aws-config/src/lib.rs

@@ -208,12 +208,6 @@ pub async fn load_defaults(version: BehaviorVersion) -> SdkConfig {
 }
 
 mod loader {
-    use crate::default_provider::use_dual_stack::use_dual_stack_provider;
-    use crate::default_provider::use_fips::use_fips_provider;
-    use crate::default_provider::{app_name, credentials, region, retry_config, timeout_config};
-    use crate::meta::region::ProvideRegion;
-    use crate::profile::profile_file::ProfileFiles;
-    use crate::provider_config::ProviderConfig;
     use aws_credential_types::provider::{
         token::{ProvideToken, SharedTokenProvider},
         ProvideCredentials, SharedCredentialsProvider,
@@ -234,6 +228,14 @@ mod loader {
     use aws_types::sdk_config::SharedHttpClient;
     use aws_types::SdkConfig;
 
+    use crate::default_provider::{
+        app_name, credentials, endpoint_url, ignore_configured_endpoint_urls as ignore_ep, region,
+        retry_config, timeout_config, use_dual_stack, use_fips,
+    };
+    use crate::meta::region::ProvideRegion;
+    use crate::profile::profile_file::ProfileFiles;
+    use crate::provider_config::ProviderConfig;
+
     #[derive(Default, Debug)]
     enum CredentialsProviderOption {
         /// No provider was set by the user. We can set up the default credentials provider chain.
@@ -726,13 +728,13 @@ mod loader {
             let use_fips = if let Some(use_fips) = self.use_fips {
                 Some(use_fips)
             } else {
-                use_fips_provider(&conf).await
+                use_fips::use_fips_provider(&conf).await
             };
 
             let use_dual_stack = if let Some(use_dual_stack) = self.use_dual_stack {
                 Some(use_dual_stack)
             } else {
-                use_dual_stack_provider(&conf).await
+                use_dual_stack::use_dual_stack_provider(&conf).await
             };
 
             let conf = conf
@@ -811,6 +813,30 @@ mod loader {
                 .timeout_config(timeout_config)
                 .time_source(time_source);
 
+            // If an endpoint URL is set programmatically, then our work is done.
+            let endpoint_url = if self.endpoint_url.is_some() {
+                self.endpoint_url
+            } else {
+                // Otherwise, check to see if we should ignore EP URLs set in the environment.
+                let ignore_configured_endpoint_urls =
+                    ignore_ep::ignore_configured_endpoint_urls_provider(&conf)
+                        .await
+                        .unwrap_or_default();
+
+                if ignore_configured_endpoint_urls {
+                    // If yes, log a trace and return `None`.
+                    tracing::trace!(
+                        "`ignore_configured_endpoint_urls` is set, any endpoint URLs configured in the environment will be ignored. \
+                        NOTE: Endpoint URLs set programmatically WILL still be respected"
+                    );
+                    None
+                } else {
+                    // Otherwise, attempt to resolve one.
+                    endpoint_url::endpoint_url_provider(&conf).await
+                }
+            };
+            builder.set_endpoint_url(endpoint_url);
+
             builder.set_behavior_version(self.behavior_version);
             builder.set_http_client(self.http_client);
             builder.set_app_name(app_name);
@@ -818,7 +844,6 @@ mod loader {
             builder.set_credentials_provider(credentials_provider);
             builder.set_token_provider(token_provider);
             builder.set_sleep_impl(sleep_impl);
-            builder.set_endpoint_url(self.endpoint_url);
             builder.set_use_fips(use_fips);
             builder.set_use_dual_stack(use_dual_stack);
             builder.set_stalled_stream_protection(self.stalled_stream_protection_config);
@@ -970,5 +995,98 @@ mod loader {
             let num_requests = num_requests.load(Ordering::Relaxed);
             assert!(num_requests > 0, "{}", num_requests);
         }
+
+        #[tokio::test]
+        async fn endpoint_urls_may_be_ignored_from_env() {
+            let fs = Fs::from_slice(&[(
+                "test_config",
+                "[profile custom]\nendpoint_url = http://profile",
+            )]);
+            let env = Env::from_slice(&[("AWS_IGNORE_CONFIGURED_ENDPOINT_URLS", "true")]);
+
+            let conf = base_conf().use_dual_stack(false).load().await;
+            assert_eq!(Some(false), conf.use_dual_stack());
+
+            let conf = base_conf().load().await;
+            assert_eq!(None, conf.use_dual_stack());
+
+            // Check that we get nothing back because the env said we should ignore endpoints
+            let config = base_conf()
+                .fs(fs.clone())
+                .env(env)
+                .profile_name("custom")
+                .profile_files(
+                    ProfileFiles::builder()
+                        .with_file(ProfileFileKind::Config, "test_config")
+                        .build(),
+                )
+                .load()
+                .await;
+            assert_eq!(None, config.endpoint_url());
+
+            // Check that without the env, we DO get something back
+            let config = base_conf()
+                .fs(fs)
+                .profile_name("custom")
+                .profile_files(
+                    ProfileFiles::builder()
+                        .with_file(ProfileFileKind::Config, "test_config")
+                        .build(),
+                )
+                .load()
+                .await;
+            assert_eq!(Some("http://profile"), config.endpoint_url());
+        }
+
+        #[tokio::test]
+        async fn endpoint_urls_may_be_ignored_from_profile() {
+            let fs = Fs::from_slice(&[(
+                "test_config",
+                "[profile custom]\nignore_configured_endpoint_urls = true",
+            )]);
+            let env = Env::from_slice(&[("AWS_ENDPOINT_URL", "http://environment")]);
+
+            // Check that we get nothing back because the profile said we should ignore endpoints
+            let config = base_conf()
+                .fs(fs)
+                .env(env.clone())
+                .profile_name("custom")
+                .profile_files(
+                    ProfileFiles::builder()
+                        .with_file(ProfileFileKind::Config, "test_config")
+                        .build(),
+                )
+                .load()
+                .await;
+            assert_eq!(None, config.endpoint_url());
+
+            // Check that without the profile, we DO get something back
+            let config = base_conf().env(env).load().await;
+            assert_eq!(Some("http://environment"), config.endpoint_url());
+        }
+
+        #[tokio::test]
+        async fn programmatic_endpoint_urls_may_not_be_ignored() {
+            let fs = Fs::from_slice(&[(
+                "test_config",
+                "[profile custom]\nignore_configured_endpoint_urls = true",
+            )]);
+            let env = Env::from_slice(&[("AWS_IGNORE_CONFIGURED_ENDPOINT_URLS", "true")]);
+
+            // Check that we get something back because we explicitly set the loader's endpoint URL
+            let config = base_conf()
+                .fs(fs)
+                .env(env)
+                .endpoint_url("http://localhost")
+                .profile_name("custom")
+                .profile_files(
+                    ProfileFiles::builder()
+                        .with_file(ProfileFileKind::Config, "test_config")
+                        .build(),
+                )
+                .load()
+                .await;
+            assert_eq!(Some("http://localhost"), config.endpoint_url());
+        }
     }
 }

aws/sdk/integration-tests/s3/tests/service_timeout_overrides.rs

@@ -97,7 +97,13 @@ async fn default_connect_timeout_set() {
                 .build()
         )
     );
+    assert_eq!(
+        sdk_config.endpoint_url(),
+        Some("http://172.255.255.0:18104")
+    );
+
     let config = aws_sdk_s3::config::Builder::from(&sdk_config)
+        // .endpoint_url("http://172.255.255.0:18104")
         .timeout_config(
             TimeoutConfig::builder()
                 .operation_attempt_timeout(Duration::from_secs(8))

rust-runtime/inlineable/Cargo.toml

@@ -32,7 +32,7 @@ once_cell = "1.16.0"
 percent-encoding = "2.2.0"
 pin-project-lite = "0.2"
 regex-lite = "0.1.5"
-url = "2.2.2"
+url = "2.3.1"
 
 [dev-dependencies]
 proptest = "1"