From 2bfa0037adbe19405b1d7b64b7518a59b2a1287e Mon Sep 17 00:00:00 2001 From: Chengyuan Zhang Date: Tue, 23 Feb 2021 12:27:47 -0800 Subject: [PATCH] xds: implement cloud-to-prod resolver (#7900) Implemented CloudToProdNameResolver, which will be used for DirectPath with URI scheme "google-c2p". The resolver is only a wrapper that delegates name resolution either to DNS or xDS resolver depending on the environment. If it is delegating to the xDS resolver, it will send HTTP requests (to a local HTTP server) to fetch metadata that is used to generate a bootstrap config. The self-generated bootstrap will be used for xDS. --- alts/build.gradle | 5 +- .../io/grpc/alts/AltsChannelCredentials.java | 2 +- .../io/grpc/alts/AltsServerCredentials.java | 2 +- .../alts/ComputeEngineChannelCredentials.java | 2 +- ....java => InternalCheckGcpEnvironment.java} | 20 +- ...a => InternalCheckGcpEnvironmentTest.java} | 24 +- .../main/java/io/grpc/xds/Bootstrapper.java | 19 +- .../java/io/grpc/xds/BootstrapperImpl.java | 92 +++--- .../xds/GoogleCloudToProdNameResolver.java | 275 ++++++++++++++++++ ...GoogleCloudToProdNameResolverProvider.java | 58 ++++ .../grpc/xds/SharedXdsClientPoolProvider.java | 16 +- .../io/grpc/xds/XdsNameResolverProvider.java | 3 + .../services/io.grpc.NameResolverProvider | 1 + ...leCloudToProdNameResolverProviderTest.java | 65 +++++ .../GoogleCloudToProdNameResolverTest.java | 256 ++++++++++++++++ .../java/io/grpc/xds/XdsNameResolverTest.java | 10 + 16 files changed, 776 insertions(+), 74 deletions(-) rename alts/src/main/java/io/grpc/alts/{CheckGcpEnvironment.java => InternalCheckGcpEnvironment.java} (82%) rename alts/src/test/java/io/grpc/alts/{CheckGcpEnvironmentTest.java => InternalCheckGcpEnvironmentTest.java} (68%) create mode 100644 xds/src/main/java/io/grpc/xds/GoogleCloudToProdNameResolver.java create mode 100644 xds/src/main/java/io/grpc/xds/GoogleCloudToProdNameResolverProvider.java create mode 100644 xds/src/test/java/io/grpc/xds/GoogleCloudToProdNameResolverProviderTest.java create mode 100644 xds/src/test/java/io/grpc/xds/GoogleCloudToProdNameResolverTest.java diff --git a/alts/build.gradle b/alts/build.gradle index 683f7d6cf0..fa4dda2b79 100644 --- a/alts/build.gradle +++ b/alts/build.gradle @@ -56,7 +56,10 @@ import net.ltgt.gradle.errorprone.CheckSeverity it.options.errorprone.check("FutureReturnValueIgnored", CheckSeverity.OFF) } -javadoc { exclude 'io/grpc/alts/internal/**' } +javadoc { + exclude 'io/grpc/alts/internal/**' + exclude 'io/grpc/alts/Internal*' +} jar { // Must use a different classifier to avoid conflicting with shadowJar diff --git a/alts/src/main/java/io/grpc/alts/AltsChannelCredentials.java b/alts/src/main/java/io/grpc/alts/AltsChannelCredentials.java index 0bb96378cc..7c603d31fc 100644 --- a/alts/src/main/java/io/grpc/alts/AltsChannelCredentials.java +++ b/alts/src/main/java/io/grpc/alts/AltsChannelCredentials.java @@ -93,7 +93,7 @@ public final class AltsChannelCredentials { } InternalProtocolNegotiator.ClientFactory buildProtocolNegotiatorFactory() { - if (!CheckGcpEnvironment.isOnGcp()) { + if (!InternalCheckGcpEnvironment.isOnGcp()) { if (enableUntrustedAlts) { logger.log( Level.WARNING, diff --git a/alts/src/main/java/io/grpc/alts/AltsServerCredentials.java b/alts/src/main/java/io/grpc/alts/AltsServerCredentials.java index 65a350daad..dd56e310cc 100644 --- a/alts/src/main/java/io/grpc/alts/AltsServerCredentials.java +++ b/alts/src/main/java/io/grpc/alts/AltsServerCredentials.java @@ -76,7 +76,7 @@ public final class AltsServerCredentials { } InternalProtocolNegotiator.ProtocolNegotiator buildProtocolNegotiator() { - if (!CheckGcpEnvironment.isOnGcp()) { + if (!InternalCheckGcpEnvironment.isOnGcp()) { if (enableUntrustedAlts) { logger.log( Level.WARNING, diff --git a/alts/src/main/java/io/grpc/alts/ComputeEngineChannelCredentials.java b/alts/src/main/java/io/grpc/alts/ComputeEngineChannelCredentials.java index 9ec4e05889..ff5bab6ec7 100644 --- a/alts/src/main/java/io/grpc/alts/ComputeEngineChannelCredentials.java +++ b/alts/src/main/java/io/grpc/alts/ComputeEngineChannelCredentials.java @@ -49,7 +49,7 @@ public final class ComputeEngineChannelCredentials { ChannelCredentials nettyCredentials = InternalNettyChannelCredentials.create(createClientFactory()); CallCredentials callCredentials; - if (CheckGcpEnvironment.isOnGcp()) { + if (InternalCheckGcpEnvironment.isOnGcp()) { callCredentials = MoreCallCredentials.from(ComputeEngineCredentials.create()); } else { callCredentials = new FailingCallCredentials( diff --git a/alts/src/main/java/io/grpc/alts/CheckGcpEnvironment.java b/alts/src/main/java/io/grpc/alts/InternalCheckGcpEnvironment.java similarity index 82% rename from alts/src/main/java/io/grpc/alts/CheckGcpEnvironment.java rename to alts/src/main/java/io/grpc/alts/InternalCheckGcpEnvironment.java index 238152d2eb..dc6b776e11 100644 --- a/alts/src/main/java/io/grpc/alts/CheckGcpEnvironment.java +++ b/alts/src/main/java/io/grpc/alts/InternalCheckGcpEnvironment.java @@ -19,6 +19,7 @@ package io.grpc.alts; import static java.nio.charset.StandardCharsets.UTF_8; import com.google.common.annotations.VisibleForTesting; +import io.grpc.Internal; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; @@ -28,18 +29,27 @@ import java.util.Locale; import java.util.logging.Level; import java.util.logging.Logger; -/** Class for checking if the system is running on Google Cloud Platform (GCP). */ -final class CheckGcpEnvironment { +/** + * Class for checking if the system is running on Google Cloud Platform (GCP). + * This is intended for usage internal to the gRPC team. If you *really* think you need + * to use this, contact the gRPC team first. + */ +@Internal +public final class InternalCheckGcpEnvironment { - private static final Logger logger = Logger.getLogger(CheckGcpEnvironment.class.getName()); + private static final Logger logger = + Logger.getLogger(InternalCheckGcpEnvironment.class.getName()); private static final String DMI_PRODUCT_NAME = "/sys/class/dmi/id/product_name"; private static final String WINDOWS_COMMAND = "powershell.exe"; private static Boolean cachedResult = null; // Construct me not! - private CheckGcpEnvironment() {} + private InternalCheckGcpEnvironment() {} - static synchronized boolean isOnGcp() { + /** + * Returns {@code true} if currently running on Google Cloud Platform (GCP). + */ + public static synchronized boolean isOnGcp() { if (cachedResult == null) { cachedResult = isRunningOnGcp(); } diff --git a/alts/src/test/java/io/grpc/alts/CheckGcpEnvironmentTest.java b/alts/src/test/java/io/grpc/alts/InternalCheckGcpEnvironmentTest.java similarity index 68% rename from alts/src/test/java/io/grpc/alts/CheckGcpEnvironmentTest.java rename to alts/src/test/java/io/grpc/alts/InternalCheckGcpEnvironmentTest.java index bef16f7776..1234cfc49c 100644 --- a/alts/src/test/java/io/grpc/alts/CheckGcpEnvironmentTest.java +++ b/alts/src/test/java/io/grpc/alts/InternalCheckGcpEnvironmentTest.java @@ -26,37 +26,37 @@ import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @RunWith(JUnit4.class) -public final class CheckGcpEnvironmentTest { +public final class InternalCheckGcpEnvironmentTest { @Test public void checkGcpLinuxPlatformData() throws Exception { BufferedReader reader; reader = new BufferedReader(new StringReader("HP Z440 Workstation")); - assertFalse(CheckGcpEnvironment.checkProductNameOnLinux(reader)); + assertFalse(InternalCheckGcpEnvironment.checkProductNameOnLinux(reader)); reader = new BufferedReader(new StringReader("Google")); - assertTrue(CheckGcpEnvironment.checkProductNameOnLinux(reader)); + assertTrue(InternalCheckGcpEnvironment.checkProductNameOnLinux(reader)); reader = new BufferedReader(new StringReader("Google Compute Engine")); - assertTrue(CheckGcpEnvironment.checkProductNameOnLinux(reader)); + assertTrue(InternalCheckGcpEnvironment.checkProductNameOnLinux(reader)); reader = new BufferedReader(new StringReader("Google Compute Engine ")); - assertTrue(CheckGcpEnvironment.checkProductNameOnLinux(reader)); + assertTrue(InternalCheckGcpEnvironment.checkProductNameOnLinux(reader)); } @Test public void checkGcpWindowsPlatformData() throws Exception { BufferedReader reader; reader = new BufferedReader(new StringReader("Product : Google")); - assertFalse(CheckGcpEnvironment.checkBiosDataOnWindows(reader)); + assertFalse(InternalCheckGcpEnvironment.checkBiosDataOnWindows(reader)); reader = new BufferedReader(new StringReader("Manufacturer : LENOVO")); - assertFalse(CheckGcpEnvironment.checkBiosDataOnWindows(reader)); + assertFalse(InternalCheckGcpEnvironment.checkBiosDataOnWindows(reader)); reader = new BufferedReader(new StringReader("Manufacturer : Google Compute Engine")); - assertFalse(CheckGcpEnvironment.checkBiosDataOnWindows(reader)); + assertFalse(InternalCheckGcpEnvironment.checkBiosDataOnWindows(reader)); reader = new BufferedReader(new StringReader("Manufacturer : Google")); - assertTrue(CheckGcpEnvironment.checkBiosDataOnWindows(reader)); + assertTrue(InternalCheckGcpEnvironment.checkBiosDataOnWindows(reader)); reader = new BufferedReader(new StringReader("Manufacturer:Google")); - assertTrue(CheckGcpEnvironment.checkBiosDataOnWindows(reader)); + assertTrue(InternalCheckGcpEnvironment.checkBiosDataOnWindows(reader)); reader = new BufferedReader(new StringReader("Manufacturer : Google ")); - assertTrue(CheckGcpEnvironment.checkBiosDataOnWindows(reader)); + assertTrue(InternalCheckGcpEnvironment.checkBiosDataOnWindows(reader)); reader = new BufferedReader(new StringReader("BIOSVersion : 1.0\nManufacturer : Google\n")); - assertTrue(CheckGcpEnvironment.checkBiosDataOnWindows(reader)); + assertTrue(InternalCheckGcpEnvironment.checkBiosDataOnWindows(reader)); } } diff --git a/xds/src/main/java/io/grpc/xds/Bootstrapper.java b/xds/src/main/java/io/grpc/xds/Bootstrapper.java index 877c427925..f3ccbf2d9d 100644 --- a/xds/src/main/java/io/grpc/xds/Bootstrapper.java +++ b/xds/src/main/java/io/grpc/xds/Bootstrapper.java @@ -31,19 +31,26 @@ import javax.annotation.Nullable; * Loads configuration information to bootstrap gRPC's integration of xDS protocol. */ @Internal -public interface Bootstrapper { +public abstract class Bootstrapper { /** - * Returns configurations from bootstrap. + * Returns system-loaded bootstrap configuration. */ - BootstrapInfo bootstrap() throws XdsInitializationException; + public abstract BootstrapInfo bootstrap() throws XdsInitializationException; + + /** + * Returns bootstrap configuration given by the raw data in JSON format. + */ + BootstrapInfo bootstrap(Map rawData) throws XdsInitializationException { + throw new UnsupportedOperationException(); + } /** * Data class containing xDS server information, such as server URI and channel credentials * to be used for communication. */ @Internal - class ServerInfo { + static class ServerInfo { private final String target; private final ChannelCredentials channelCredentials; private final boolean useProtocolV3; @@ -73,7 +80,7 @@ public interface Bootstrapper { * Map that represents the config for that plugin. */ @Internal - class CertificateProviderInfo { + public static class CertificateProviderInfo { private final String pluginName; private final Map config; @@ -95,7 +102,7 @@ public interface Bootstrapper { * Data class containing the results of reading bootstrap. */ @Internal - class BootstrapInfo { + public static class BootstrapInfo { private List servers; private final Node node; @Nullable private final Map certProviders; diff --git a/xds/src/main/java/io/grpc/xds/BootstrapperImpl.java b/xds/src/main/java/io/grpc/xds/BootstrapperImpl.java index 5b5710e1c3..406baaf571 100644 --- a/xds/src/main/java/io/grpc/xds/BootstrapperImpl.java +++ b/xds/src/main/java/io/grpc/xds/BootstrapperImpl.java @@ -43,7 +43,7 @@ import javax.annotation.Nullable; * A {@link Bootstrapper} implementation that reads xDS configurations from local file system. */ @Internal -public class BootstrapperImpl implements Bootstrapper { +public class BootstrapperImpl extends Bootstrapper { private static final String BOOTSTRAP_PATH_SYS_ENV_VAR = "GRPC_XDS_BOOTSTRAP"; @VisibleForTesting @@ -80,70 +80,49 @@ public class BootstrapperImpl implements Bootstrapper { *
  • Java System Property value of "io.grpc.xds.bootstrap_value"
    • * */ + @SuppressWarnings("unchecked") @Override public BootstrapInfo bootstrap() throws XdsInitializationException { String filePath = bootstrapPathFromEnvVar != null ? bootstrapPathFromEnvVar : bootstrapPathFromSysProp; - String rawBootstrap; + String fileContent; if (filePath != null) { logger.log(XdsLogLevel.INFO, "Reading bootstrap file from {0}", filePath); try { - rawBootstrap = reader.readFile(filePath); + fileContent = reader.readFile(filePath); } catch (IOException e) { throw new XdsInitializationException("Fail to read bootstrap file", e); } } else { - rawBootstrap = bootstrapConfigFromEnvVar != null + fileContent = bootstrapConfigFromEnvVar != null ? bootstrapConfigFromEnvVar : bootstrapConfigFromSysProp; } - if (rawBootstrap != null) { - return parseConfig(rawBootstrap); + if (fileContent == null) { + throw new XdsInitializationException( + "Cannot find bootstrap configuration\n" + + "Environment variables searched:\n" + + "- " + BOOTSTRAP_PATH_SYS_ENV_VAR + "\n" + + "- " + BOOTSTRAP_CONFIG_SYS_ENV_VAR + "\n\n" + + "Java System Properties searched:\n" + + "- " + BOOTSTRAP_PATH_SYS_PROPERTY + "\n" + + "- " + BOOTSTRAP_CONFIG_SYS_PROPERTY_VAR + "\n\n"); } - throw new XdsInitializationException( - "Cannot find bootstrap configuration\n" - + "Environment variables searched:\n" - + "- " + BOOTSTRAP_PATH_SYS_ENV_VAR + "\n" - + "- " + BOOTSTRAP_CONFIG_SYS_ENV_VAR + "\n\n" - + "Java System Properties searched:\n" - + "- " + BOOTSTRAP_PATH_SYS_PROPERTY + "\n" - + "- " + BOOTSTRAP_CONFIG_SYS_PROPERTY_VAR + "\n\n"); - } - @VisibleForTesting - void setFileReader(FileReader reader) { - this.reader = reader; - } - - /** - * Reads the content of the file with the given path in the file system. - */ - interface FileReader { - String readFile(String path) throws IOException; - } - - private enum LocalFileReader implements FileReader { - INSTANCE; - - @Override - public String readFile(String path) throws IOException { - return new String(Files.readAllBytes(Paths.get(path)), StandardCharsets.UTF_8); - } - } - - /** Parses a raw string into {@link BootstrapInfo}. */ - @SuppressWarnings("unchecked") - private BootstrapInfo parseConfig(String rawData) throws XdsInitializationException { - logger.log(XdsLogLevel.INFO, "Reading bootstrap information"); + logger.log(XdsLogLevel.INFO, "Reading bootstrap from " + filePath); Map rawBootstrap; try { - rawBootstrap = (Map) JsonParser.parse(rawData); + rawBootstrap = (Map) JsonParser.parse(fileContent); } catch (IOException e) { throw new XdsInitializationException("Failed to parse JSON", e); } logger.log(XdsLogLevel.DEBUG, "Bootstrap configuration:\n{0}", rawBootstrap); + return bootstrap(rawBootstrap); + } + @Override + BootstrapInfo bootstrap(Map rawData) throws XdsInitializationException { List servers = new ArrayList<>(); - List rawServerConfigs = JsonUtil.getList(rawBootstrap, "xds_servers"); + List rawServerConfigs = JsonUtil.getList(rawData, "xds_servers"); if (rawServerConfigs == null) { throw new XdsInitializationException("Invalid bootstrap: 'xds_servers' does not exist."); } @@ -179,7 +158,7 @@ public class BootstrapperImpl implements Bootstrapper { } Node.Builder nodeBuilder = Node.newBuilder(); - Map rawNode = JsonUtil.getObject(rawBootstrap, "node"); + Map rawNode = JsonUtil.getObject(rawData, "node"); if (rawNode != null) { String id = JsonUtil.getString(rawNode, "id"); if (id != null) { @@ -222,7 +201,7 @@ public class BootstrapperImpl implements Bootstrapper { nodeBuilder.setUserAgentVersion(buildVersion.getImplementationVersion()); nodeBuilder.addClientFeatures(CLIENT_FEATURE_DISABLE_OVERPROVISIONING); - Map certProvidersBlob = JsonUtil.getObject(rawBootstrap, "certificate_providers"); + Map certProvidersBlob = JsonUtil.getObject(rawData, "certificate_providers"); Map certProviders = null; if (certProvidersBlob != null) { certProviders = new HashMap<>(certProvidersBlob.size()); @@ -236,11 +215,32 @@ public class BootstrapperImpl implements Bootstrapper { certProviders.put(name, certificateProviderInfo); } } - String grpcServerResourceId = JsonUtil.getString(rawBootstrap, "grpc_server_resource_name_id"); + String grpcServerResourceId = JsonUtil.getString(rawData, "grpc_server_resource_name_id"); return new BootstrapInfo(servers, nodeBuilder.build(), certProviders, grpcServerResourceId); } - static T checkForNull(T value, String fieldName) throws XdsInitializationException { + @VisibleForTesting + void setFileReader(FileReader reader) { + this.reader = reader; + } + + /** + * Reads the content of the file with the given path in the file system. + */ + interface FileReader { + String readFile(String path) throws IOException; + } + + private enum LocalFileReader implements FileReader { + INSTANCE; + + @Override + public String readFile(String path) throws IOException { + return new String(Files.readAllBytes(Paths.get(path)), StandardCharsets.UTF_8); + } + } + + private static T checkForNull(T value, String fieldName) throws XdsInitializationException { if (value == null) { throw new XdsInitializationException( "Invalid bootstrap: '" + fieldName + "' does not exist."); diff --git a/xds/src/main/java/io/grpc/xds/GoogleCloudToProdNameResolver.java b/xds/src/main/java/io/grpc/xds/GoogleCloudToProdNameResolver.java new file mode 100644 index 0000000000..0f10ffd4bd --- /dev/null +++ b/xds/src/main/java/io/grpc/xds/GoogleCloudToProdNameResolver.java @@ -0,0 +1,275 @@ +/* + * Copyright 2021 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.xds; + +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Charsets; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.io.CharStreams; +import io.grpc.NameResolver; +import io.grpc.NameResolverRegistry; +import io.grpc.Status; +import io.grpc.SynchronizationContext; +import io.grpc.alts.InternalCheckGcpEnvironment; +import io.grpc.internal.GrpcUtil; +import io.grpc.internal.SharedResourceHolder; +import io.grpc.internal.SharedResourceHolder.Resource; +import io.grpc.xds.XdsNameResolverProvider.XdsClientPoolFactory; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.net.HttpURLConnection; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.concurrent.Executor; + +/** + * CloudToProd version of {@link NameResolver}. + */ +final class GoogleCloudToProdNameResolver extends NameResolver { + + @VisibleForTesting + static final String METADATA_URL_ZONE = + "http://metadata.google.internal/computeMetadata/v1/instance/zone"; + @VisibleForTesting + static final String METADATA_URL_SUPPORT_IPV6 = + "http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/ipv6s"; + @VisibleForTesting + static boolean isOnGcp = InternalCheckGcpEnvironment.isOnGcp(); + @VisibleForTesting + static boolean xdsBootstrapProvided = + System.getenv("GRPC_XDS_BOOTSTRAP") != null + || System.getProperty("io.grpc.xds.bootstrap") != null + || System.getenv("GRPC_XDS_BOOTSTRAP_CONFIG") != null + || System.getProperty("io.grpc.xds.bootstrapValue") != null; + + private HttpConnectionProvider httpConnectionProvider = HttpConnectionFactory.INSTANCE; + private final String authority; + private final SynchronizationContext syncContext; + private final Resource executorResource; + private final XdsClientPoolFactory xdsClientPoolFactory; + private final NameResolver delegate; + private final boolean usingExecutorResource; + // It's not possible to use both PSM and DirectPath C2P in the same application. + // Delegate to DNS if user-provided bootstrap is found. + private final String schemeOverride = !isOnGcp || xdsBootstrapProvided ? "dns" : "xds"; + private Executor executor; + private Listener2 listener; + private boolean succeeded; + private boolean resolving; + private boolean shutdown; + + GoogleCloudToProdNameResolver(URI targetUri, Args args, Resource executorResource, + XdsClientPoolFactory xdsClientPoolFactory) { + this(targetUri, args, executorResource, xdsClientPoolFactory, + NameResolverRegistry.getDefaultRegistry().asFactory()); + } + + @VisibleForTesting + GoogleCloudToProdNameResolver(URI targetUri, Args args, Resource executorResource, + XdsClientPoolFactory xdsClientPoolFactory, NameResolver.Factory nameResolverFactory) { + this.executorResource = checkNotNull(executorResource, "executorResource"); + this.xdsClientPoolFactory = checkNotNull(xdsClientPoolFactory, "xdsClientPoolFactory"); + String targetPath = checkNotNull(checkNotNull(targetUri, "targetUri").getPath(), "targetPath"); + Preconditions.checkArgument( + targetPath.startsWith("/"), + "the path component (%s) of the target (%s) must start with '/'", + targetPath, + targetUri); + authority = GrpcUtil.checkAuthority(targetPath.substring(1)); + syncContext = checkNotNull(args, "args").getSynchronizationContext(); + delegate = checkNotNull(nameResolverFactory, "nameResolverFactory").newNameResolver( + overrideUriScheme(targetUri, schemeOverride), args); + executor = args.getOffloadExecutor(); + usingExecutorResource = executor == null; + } + + @Override + public String getServiceAuthority() { + return authority; + } + + @Override + public void start(final Listener2 listener) { + if (delegate == null) { + listener.onError(Status.INTERNAL.withDescription( + "Delegate resolver not found, scheme: " + schemeOverride)); + return; + } + this.listener = checkNotNull(listener, "listener"); + resolve(); + } + + private void resolve() { + if (resolving || shutdown || delegate == null) { + return; + } + resolving = true; + if (schemeOverride.equals("dns")) { + delegate.start(listener); + succeeded = true; + resolving = false; + return; + } + if (executor == null) { + executor = SharedResourceHolder.get(executorResource); + } + + class Resolve implements Runnable { + @Override + public void run() { + String zone; + boolean supportIpv6; + ImmutableMap rawBootstrap = null; + try { + zone = queryZoneMetadata(METADATA_URL_ZONE); + supportIpv6 = queryIpv6SupportMetadata(METADATA_URL_SUPPORT_IPV6); + rawBootstrap = generateBootstrap(zone, supportIpv6); + } catch (IOException e) { + listener.onError(Status.INTERNAL.withDescription("Unable to get metadata").withCause(e)); + } finally { + final ImmutableMap finalRawBootstrap = rawBootstrap; + syncContext.execute(new Runnable() { + @Override + public void run() { + if (!shutdown && finalRawBootstrap != null) { + xdsClientPoolFactory.setBootstrapOverride(finalRawBootstrap); + delegate.start(listener); + succeeded = true; + } + resolving = false; + } + }); + } + } + } + + executor.execute(new Resolve()); + } + + private static ImmutableMap generateBootstrap(String zone, boolean supportIpv6) { + ImmutableMap.Builder nodeBuilder = ImmutableMap.builder(); + nodeBuilder.put("id", "C2P"); + if (!zone.isEmpty()) { + nodeBuilder.put("locality", ImmutableMap.of("zone", zone)); + } + if (supportIpv6) { + nodeBuilder.put("metadata", + ImmutableMap.of("TRAFFICDIRECTOR_DIRECTPATH_C2P_IPV6_CAPABLE", true)); + } + ImmutableMap.Builder serverBuilder = ImmutableMap.builder(); + serverBuilder.put("server_uri", "directpath-trafficdirector.googleapis.com"); + serverBuilder.put("channel_creds", + ImmutableList.of(ImmutableMap.of("type", "google_default"))); + return ImmutableMap.of( + "node", nodeBuilder.build(), + "xds_servers", ImmutableList.of(serverBuilder.build())); + } + + @Override + public void refresh() { + if (succeeded) { + delegate.refresh(); + } else if (!resolving) { + resolve(); + } + } + + @Override + public void shutdown() { + if (shutdown) { + return; + } + shutdown = true; + if (delegate != null) { + delegate.shutdown(); + } + if (executor != null && usingExecutorResource) { + executor = SharedResourceHolder.release(executorResource, executor); + } + } + + private String queryZoneMetadata(String url) throws IOException { + HttpURLConnection con = null; + String respBody; + try { + con = httpConnectionProvider.createConnection(url); + if (con.getResponseCode() != 200) { + return ""; + } + try (Reader reader = new InputStreamReader(con.getInputStream(), Charsets.UTF_8)) { + respBody = CharStreams.toString(reader); + } + } finally { + if (con != null) { + con.disconnect(); + } + } + int index = respBody.lastIndexOf('/'); + return index == -1 ? "" : respBody.substring(index + 1); + } + + private boolean queryIpv6SupportMetadata(String url) throws IOException { + HttpURLConnection con = null; + try { + con = httpConnectionProvider.createConnection(url); + return con.getResponseCode() == 200; + } finally { + if (con != null) { + con.disconnect(); + } + } + } + + @VisibleForTesting + void setHttpConnectionProvider(HttpConnectionProvider httpConnectionProvider) { + this.httpConnectionProvider = httpConnectionProvider; + } + + private static URI overrideUriScheme(URI uri, String scheme) { + URI res; + try { + res = new URI(scheme, uri.getAuthority(), uri.getPath(), uri.getQuery(), uri.getFragment()); + } catch (URISyntaxException ex) { + throw new IllegalArgumentException("Invalid scheme: " + scheme, ex); + } + return res; + } + + private enum HttpConnectionFactory implements HttpConnectionProvider { + INSTANCE; + + @Override + public HttpURLConnection createConnection(String url) throws IOException { + HttpURLConnection con = (HttpURLConnection) new URL(url).openConnection(); + con.setRequestMethod("GET"); + con.setReadTimeout(10000); + con.setRequestProperty("Metadata-Flavor", "Google"); + return con; + } + } + + @VisibleForTesting + interface HttpConnectionProvider { + HttpURLConnection createConnection(String url) throws IOException; + } +} diff --git a/xds/src/main/java/io/grpc/xds/GoogleCloudToProdNameResolverProvider.java b/xds/src/main/java/io/grpc/xds/GoogleCloudToProdNameResolverProvider.java new file mode 100644 index 0000000000..d6c60abf9d --- /dev/null +++ b/xds/src/main/java/io/grpc/xds/GoogleCloudToProdNameResolverProvider.java @@ -0,0 +1,58 @@ +/* + * Copyright 2021 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.xds; + +import io.grpc.Internal; +import io.grpc.NameResolver; +import io.grpc.NameResolver.Args; +import io.grpc.NameResolverProvider; +import io.grpc.internal.GrpcUtil; +import java.net.URI; + +/** + * A provider for {@link GoogleCloudToProdNameResolver}. + */ +@Internal +public final class GoogleCloudToProdNameResolverProvider extends NameResolverProvider { + + private static final String SCHEME = "google-c2p"; + + @Override + public NameResolver newNameResolver(URI targetUri, Args args) { + if (SCHEME.equals(targetUri.getScheme())) { + return new GoogleCloudToProdNameResolver( + targetUri, args, GrpcUtil.SHARED_CHANNEL_EXECUTOR, + SharedXdsClientPoolProvider.getDefaultProvider()); + } + return null; + } + + @Override + public String getDefaultScheme() { + return SCHEME; + } + + @Override + protected boolean isAvailable() { + return true; + } + + @Override + protected int priority() { + return 4; + } +} diff --git a/xds/src/main/java/io/grpc/xds/SharedXdsClientPoolProvider.java b/xds/src/main/java/io/grpc/xds/SharedXdsClientPoolProvider.java index 86528309b7..46411c7483 100644 --- a/xds/src/main/java/io/grpc/xds/SharedXdsClientPoolProvider.java +++ b/xds/src/main/java/io/grpc/xds/SharedXdsClientPoolProvider.java @@ -30,8 +30,10 @@ import io.grpc.xds.Bootstrapper.BootstrapInfo; import io.grpc.xds.Bootstrapper.ServerInfo; import io.grpc.xds.EnvoyProtoData.Node; import io.grpc.xds.XdsNameResolverProvider.XdsClientPoolFactory; +import java.util.Map; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import javax.annotation.Nullable; import javax.annotation.concurrent.GuardedBy; import javax.annotation.concurrent.ThreadSafe; @@ -45,6 +47,7 @@ final class SharedXdsClientPoolProvider implements XdsClientPoolFactory { private final Bootstrapper bootstrapper; private final Object lock = new Object(); + private final AtomicReference> bootstrapOverride = new AtomicReference<>(); private volatile ObjectPool xdsClientPool; private SharedXdsClientPoolProvider() { @@ -60,6 +63,11 @@ final class SharedXdsClientPoolProvider implements XdsClientPoolFactory { return SharedXdsClientPoolProviderHolder.instance; } + @Override + public void setBootstrapOverride(Map bootstrap) { + bootstrapOverride.set(bootstrap); + } + @Override public ObjectPool getXdsClientPool() throws XdsInitializationException { ObjectPool ref = xdsClientPool; @@ -67,7 +75,13 @@ final class SharedXdsClientPoolProvider implements XdsClientPoolFactory { synchronized (lock) { ref = xdsClientPool; if (ref == null) { - BootstrapInfo bootstrapInfo = bootstrapper.bootstrap(); + BootstrapInfo bootstrapInfo; + Map rawBootstrap = bootstrapOverride.get(); + if (rawBootstrap != null) { + bootstrapInfo = bootstrapper.bootstrap(rawBootstrap); + } else { + bootstrapInfo = bootstrapper.bootstrap(); + } if (bootstrapInfo.getServers().isEmpty()) { throw new XdsInitializationException("No xDS server provided"); } diff --git a/xds/src/main/java/io/grpc/xds/XdsNameResolverProvider.java b/xds/src/main/java/io/grpc/xds/XdsNameResolverProvider.java index 94e7fc3138..bb51455f79 100644 --- a/xds/src/main/java/io/grpc/xds/XdsNameResolverProvider.java +++ b/xds/src/main/java/io/grpc/xds/XdsNameResolverProvider.java @@ -24,6 +24,7 @@ import io.grpc.NameResolver.Args; import io.grpc.NameResolverProvider; import io.grpc.internal.ObjectPool; import java.net.URI; +import java.util.Map; import java.util.concurrent.atomic.AtomicLong; import javax.annotation.Nullable; @@ -76,6 +77,8 @@ public final class XdsNameResolverProvider extends NameResolverProvider { } interface XdsClientPoolFactory { + void setBootstrapOverride(Map bootstrap); + ObjectPool getXdsClientPool() throws XdsInitializationException; } diff --git a/xds/src/main/resources/META-INF/services/io.grpc.NameResolverProvider b/xds/src/main/resources/META-INF/services/io.grpc.NameResolverProvider index 269cdd3880..c1f2c40e7e 100644 --- a/xds/src/main/resources/META-INF/services/io.grpc.NameResolverProvider +++ b/xds/src/main/resources/META-INF/services/io.grpc.NameResolverProvider @@ -1 +1,2 @@ io.grpc.xds.XdsNameResolverProvider +io.grpc.xds.GoogleCloudToProdNameResolverProvider diff --git a/xds/src/test/java/io/grpc/xds/GoogleCloudToProdNameResolverProviderTest.java b/xds/src/test/java/io/grpc/xds/GoogleCloudToProdNameResolverProviderTest.java new file mode 100644 index 0000000000..2bf5811b16 --- /dev/null +++ b/xds/src/test/java/io/grpc/xds/GoogleCloudToProdNameResolverProviderTest.java @@ -0,0 +1,65 @@ +/* + * Copyright 2021 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.xds; + +import static com.google.common.truth.Truth.assertThat; +import static org.mockito.Mockito.mock; + +import io.grpc.ChannelLogger; +import io.grpc.NameResolver; +import io.grpc.NameResolver.ServiceConfigParser; +import io.grpc.NameResolverRegistry; +import io.grpc.SynchronizationContext; +import io.grpc.internal.FakeClock; +import io.grpc.internal.GrpcUtil; +import java.net.URI; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Unit tests for {@link GoogleCloudToProdNameResolverProvider}. + */ +@RunWith(JUnit4.class) +public class GoogleCloudToProdNameResolverProviderTest { + private final SynchronizationContext syncContext = new SynchronizationContext( + new Thread.UncaughtExceptionHandler() { + @Override + public void uncaughtException(Thread t, Throwable e) { + throw new AssertionError(e); + } + }); + + private final FakeClock fakeClock = new FakeClock(); + private final NameResolver.Args args = NameResolver.Args.newBuilder() + .setDefaultPort(8080) + .setProxyDetector(GrpcUtil.NOOP_PROXY_DETECTOR) + .setSynchronizationContext(syncContext) + .setServiceConfigParser(mock(ServiceConfigParser.class)) + .setScheduledExecutorService(fakeClock.getScheduledExecutorService()) + .setChannelLogger(mock(ChannelLogger.class)) + .build(); + + private final NameResolverRegistry nsRegistry = NameResolverRegistry.getDefaultRegistry(); + + @Test + public void provided() { + NameResolver resolver = nsRegistry.asFactory().newNameResolver( + URI.create("google-c2p:///foo.googleapis.com"), args); + assertThat(resolver).isInstanceOf(GoogleCloudToProdNameResolver.class); + } +} \ No newline at end of file diff --git a/xds/src/test/java/io/grpc/xds/GoogleCloudToProdNameResolverTest.java b/xds/src/test/java/io/grpc/xds/GoogleCloudToProdNameResolverTest.java new file mode 100644 index 0000000000..7246dea923 --- /dev/null +++ b/xds/src/test/java/io/grpc/xds/GoogleCloudToProdNameResolverTest.java @@ -0,0 +1,256 @@ +/* + * Copyright 2021 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.xds; + +import static com.google.common.truth.Truth.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import io.grpc.ChannelLogger; +import io.grpc.NameResolver; +import io.grpc.NameResolver.Args; +import io.grpc.NameResolver.ServiceConfigParser; +import io.grpc.NameResolverProvider; +import io.grpc.NameResolverRegistry; +import io.grpc.Status; +import io.grpc.Status.Code; +import io.grpc.SynchronizationContext; +import io.grpc.internal.FakeClock; +import io.grpc.internal.GrpcUtil; +import io.grpc.internal.ObjectPool; +import io.grpc.internal.SharedResourceHolder.Resource; +import io.grpc.xds.GoogleCloudToProdNameResolver.HttpConnectionProvider; +import io.grpc.xds.XdsNameResolverProvider.XdsClientPoolFactory; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executor; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +@RunWith(JUnit4.class) +public class GoogleCloudToProdNameResolverTest { + + @Rule + public final MockitoRule mocks = MockitoJUnit.rule(); + + private static final URI TARGET_URI = URI.create("google-c2p:///googleapis.com"); + private static final String ZONE = "us-central1-a"; + private static final int DEFAULT_PORT = 887; + + private final SynchronizationContext syncContext = new SynchronizationContext( + new Thread.UncaughtExceptionHandler() { + @Override + public void uncaughtException(Thread t, Throwable e) { + throw new AssertionError(e); + } + }); + private final NameResolver.Args args = NameResolver.Args.newBuilder() + .setDefaultPort(DEFAULT_PORT) + .setProxyDetector(GrpcUtil.DEFAULT_PROXY_DETECTOR) + .setSynchronizationContext(syncContext) + .setServiceConfigParser(mock(ServiceConfigParser.class)) + .setChannelLogger(mock(ChannelLogger.class)) + .build(); + private final FakeClock fakeExecutor = new FakeClock(); + private final FakeXdsClientPoolFactory fakeXdsClientPoolFactory = new FakeXdsClientPoolFactory(); + private final Resource fakeExecutorResource = new Resource() { + @Override + public Executor create() { + return fakeExecutor.getScheduledExecutorService(); + } + + @Override + public void close(Executor instance) {} + }; + + private final NameResolverRegistry nsRegistry = new NameResolverRegistry(); + private final Map delegatedResolver = new HashMap<>(); + + @Mock + private NameResolver.Listener2 mockListener; + @Captor + private ArgumentCaptor errorCaptor; + private boolean originalIsOnGcp; + private boolean originalXdsBootstrapProvided; + private GoogleCloudToProdNameResolver resolver; + + @Before + public void setUp() { + nsRegistry.register(new FakeNsProvider("dns")); + nsRegistry.register(new FakeNsProvider("xds")); + originalIsOnGcp = GoogleCloudToProdNameResolver.isOnGcp; + originalXdsBootstrapProvided = GoogleCloudToProdNameResolver.xdsBootstrapProvided; + } + + @After + public void tearDown() { + GoogleCloudToProdNameResolver.isOnGcp = originalIsOnGcp; + GoogleCloudToProdNameResolver.xdsBootstrapProvided = originalXdsBootstrapProvided; + resolver.shutdown(); + verify(Iterables.getOnlyElement(delegatedResolver.values())).shutdown(); + } + + private void createResolver() { + HttpConnectionProvider httpConnections = new HttpConnectionProvider() { + @Override + public HttpURLConnection createConnection(String url) throws IOException { + HttpURLConnection con = mock(HttpURLConnection.class); + when(con.getResponseCode()).thenReturn(200); + if (url.equals(GoogleCloudToProdNameResolver.METADATA_URL_ZONE)) { + when(con.getInputStream()).thenReturn( + new ByteArrayInputStream(("/" + ZONE).getBytes(StandardCharsets.UTF_8))); + return con; + } else if (url.equals(GoogleCloudToProdNameResolver.METADATA_URL_SUPPORT_IPV6)) { + return con; + } + throw new AssertionError("Unknown http query"); + } + }; + resolver = new GoogleCloudToProdNameResolver( + TARGET_URI, args, fakeExecutorResource, fakeXdsClientPoolFactory, nsRegistry.asFactory()); + resolver.setHttpConnectionProvider(httpConnections); + } + + @Test + public void notOnGcpDelegateToDns() { + GoogleCloudToProdNameResolver.isOnGcp = false; + createResolver(); + resolver.start(mockListener); + assertThat(delegatedResolver.keySet()).containsExactly("dns"); + verify(Iterables.getOnlyElement(delegatedResolver.values())).start(mockListener); + } + + @Test + public void hasProvidedBootstrapDelegateToDns() { + GoogleCloudToProdNameResolver.isOnGcp = true; + GoogleCloudToProdNameResolver.xdsBootstrapProvided = true; + createResolver(); + resolver.start(mockListener); + assertThat(delegatedResolver.keySet()).containsExactly("dns"); + verify(Iterables.getOnlyElement(delegatedResolver.values())).start(mockListener); + } + + @SuppressWarnings("unchecked") + @Test + public void onGcpAndNoProvidedBootstrapDelegateToXds() { + GoogleCloudToProdNameResolver.isOnGcp = true; + GoogleCloudToProdNameResolver.xdsBootstrapProvided = false; + createResolver(); + resolver.start(mockListener); + fakeExecutor.runDueTasks(); + assertThat(delegatedResolver.keySet()).containsExactly("xds"); + verify(Iterables.getOnlyElement(delegatedResolver.values())).start(mockListener); + Map bootstrap = fakeXdsClientPoolFactory.bootstrapRef.get(); + Map node = (Map) bootstrap.get("node"); + assertThat(node).containsExactly( + "id", "C2P", "locality", ImmutableMap.of("zone", ZONE), + "metadata", ImmutableMap.of("TRAFFICDIRECTOR_DIRECTPATH_C2P_IPV6_CAPABLE", true)); + Map server = Iterables.getOnlyElement( + (List>) bootstrap.get("xds_servers")); + assertThat(server).containsExactly( + "server_uri", "directpath-trafficdirector.googleapis.com", + "channel_creds", ImmutableList.of(ImmutableMap.of("type", "google_default"))); + } + + @Test + public void failToQueryMetadata() { + GoogleCloudToProdNameResolver.isOnGcp = true; + GoogleCloudToProdNameResolver.xdsBootstrapProvided = false; + createResolver(); + HttpConnectionProvider httpConnections = new HttpConnectionProvider() { + @Override + public HttpURLConnection createConnection(String url) throws IOException { + HttpURLConnection con = mock(HttpURLConnection.class); + when(con.getResponseCode()).thenThrow(new IOException("unknown error")); + return con; + } + }; + resolver.setHttpConnectionProvider(httpConnections); + resolver.start(mockListener); + fakeExecutor.runDueTasks(); + verify(mockListener).onError(errorCaptor.capture()); + assertThat(errorCaptor.getValue().getCode()).isEqualTo(Code.INTERNAL); + assertThat(errorCaptor.getValue().getDescription()).isEqualTo("Unable to get metadata"); + } + + private final class FakeNsProvider extends NameResolverProvider { + private final String scheme; + + private FakeNsProvider(String scheme) { + this.scheme = scheme; + } + + @Override + public NameResolver newNameResolver(URI targetUri, Args args) { + if (scheme.equals(targetUri.getScheme())) { + NameResolver resolver = mock(NameResolver.class); + delegatedResolver.put(scheme, resolver); + return resolver; + } + return null; + } + + @Override + protected boolean isAvailable() { + return true; + } + + @Override + protected int priority() { + return 5; + } + + @Override + public String getDefaultScheme() { + return scheme; + } + } + + private static final class FakeXdsClientPoolFactory implements XdsClientPoolFactory { + private final AtomicReference> bootstrapRef = new AtomicReference<>(); + + @Override + public void setBootstrapOverride(Map bootstrap) { + bootstrapRef.set(bootstrap); + } + + @Override + public ObjectPool getXdsClientPool() { + throw new UnsupportedOperationException("Should not be called"); + } + } +} diff --git a/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java b/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java index 788af186b9..2ece3ba9c9 100644 --- a/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java @@ -159,6 +159,11 @@ public class XdsNameResolverTest { @Test public void resolving_failToCreateXdsClientPool() { XdsClientPoolFactory xdsClientPoolFactory = new XdsClientPoolFactory() { + @Override + public void setBootstrapOverride(Map bootstrap) { + throw new UnsupportedOperationException("Should not be called"); + } + @Override public ObjectPool getXdsClientPool() throws XdsInitializationException { throw new XdsInitializationException("Fail to read bootstrap file"); @@ -1394,6 +1399,11 @@ public class XdsNameResolverTest { private final class FakeXdsClientPoolFactory implements XdsClientPoolFactory { + @Override + public void setBootstrapOverride(Map bootstrap) { + throw new UnsupportedOperationException("Should not be called"); + } + @Override public ObjectPool getXdsClientPool() throws XdsInitializationException { return new ObjectPool() {