flasn4nt
/
grpc-java
mirror of https://github.com/grpc/grpc-java.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

interop-testing: add soak test cases to test service client

This commit is contained in:
apolcyn 2021-08-27 15:30:43 -07:00 committed by GitHub
parent f3337f28ce
commit 137bdaa868
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 181 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
android-interop-testing/build.gradle
View File

@ -62,6 +62,7 @@ dependencies {
project(':grpc-protobuf-lite'),
project(':grpc-stub'),
project(':grpc-testing'),
libraries.hdrhistogram,
libraries.junit,
libraries.truth,
libraries.opencensus_contrib_grpc_metrics

1
interop-testing/build.gradle
View File

@ -27,6 +27,7 @@ dependencies {
project(':grpc-stub'),
project(':grpc-testing'),
project(path: ':grpc-xds', configuration: 'shadow'),
libraries.hdrhistogram,
libraries.junit,
libraries.truth,
libraries.opencensus_contrib_grpc_metrics,

123
interop-testing/src/main/java/io/grpc/testing/integration/AbstractInteropTest.java
View File

@ -127,6 +127,7 @@ import java.util.regex.Pattern;
import javax.annotation.Nullable;
import javax.net.ssl.SSLPeerUnverifiedException;
import javax.net.ssl.SSLSession;
import org.HdrHistogram.Histogram;
import org.junit.After;
import org.junit.Assert;
import org.junit.Assume;
@ -1886,6 +1887,128 @@ public abstract class AbstractInteropTest {
assertResponse(goldenResponse, response);
}
private static class SoakIterationResult {
public SoakIterationResult(long latencyMs, Status status) {
this.latencyMs = latencyMs;
this.status = status;
}
public long getLatencyMs() {
return latencyMs;
}
public Status getStatus() {
return status;
}
private long latencyMs = -1;
private Status status = Status.OK;
}
private SoakIterationResult performOneSoakIteration(boolean resetChannel) throws Exception {
long startNs = System.nanoTime();
Status status = Status.OK;
ManagedChannel soakChannel = channel;
TestServiceGrpc.TestServiceBlockingStub soakStub = blockingStub;
if (resetChannel) {
soakChannel = createChannel();
soakStub = TestServiceGrpc.newBlockingStub(soakChannel);
}
try {
final SimpleRequest request =
SimpleRequest.newBuilder()
.setResponseSize(314159)
.setPayload(Payload.newBuilder().setBody(ByteString.copyFrom(new byte[271828])))
.build();
final SimpleResponse goldenResponse =
SimpleResponse.newBuilder()
.setPayload(Payload.newBuilder().setBody(ByteString.copyFrom(new byte[314159])))
.build();
assertResponse(goldenResponse, soakStub.unaryCall(request));
} catch (StatusRuntimeException e) {
status = e.getStatus();
}
long elapsedNs = System.nanoTime() - startNs;
if (resetChannel) {
soakChannel.shutdownNow();
soakChannel.awaitTermination(10, TimeUnit.SECONDS);
}
return new SoakIterationResult(TimeUnit.NANOSECONDS.toMillis(elapsedNs), status);
}
/**
* Runs large unary RPCs in a loop with configurable failure thresholds
* and channel creation behavior.
*/
public void performSoakTest(
boolean resetChannelPerIteration,
int soakIterations,
int maxFailures,
int maxAcceptablePerIterationLatencyMs,
int overallTimeoutSeconds)
throws Exception {
int iterationsDone = 0;
int totalFailures = 0;
Histogram latencies = new Histogram(4 /* number of significant value digits */);
long startNs = System.nanoTime();
for (int i = 0; i < soakIterations; i++) {
if (System.nanoTime() - startNs >= TimeUnit.SECONDS.toNanos(overallTimeoutSeconds)) {
break;
}
SoakIterationResult result = performOneSoakIteration(resetChannelPerIteration);
System.err.print(
String.format(
"soak iteration: %d elapsed: %d ms", i, result.getLatencyMs()));
if (!result.getStatus().equals(Status.OK)) {
totalFailures++;
System.err.println(String.format(" failed: %s", result.getStatus()));
} else if (result.getLatencyMs() > maxAcceptablePerIterationLatencyMs) {
totalFailures++;
System.err.println(
String.format(
" exceeds max acceptable latency: %d", maxAcceptablePerIterationLatencyMs));
} else {
System.err.println(" succeeded");
}
iterationsDone++;
latencies.recordValue(result.getLatencyMs());
}
System.err.println(
String.format(
"soak test ran: %d / %d iterations\n"
+ "total failures: %d\n"
+ "max failures threshold: %d\n"
+ "max acceptable per iteration latency ms: %d\n"
+ " p50 soak iteration latency: %d ms\n"
+ " p90 soak iteration latency: %d ms\n"
+ "p100 soak iteration latency: %d ms\n"
+ "See breakdown above for which iterations succeeded, failed, and "
+ "why for more info.",
iterationsDone,
soakIterations,
totalFailures,
maxFailures,
maxAcceptablePerIterationLatencyMs,
latencies.getValueAtPercentile(50),
latencies.getValueAtPercentile(90),
latencies.getValueAtPercentile(100)));
// check if we timed out
String timeoutErrorMessage =
String.format(
"soak test consumed all %d seconds of time and quit early, only "
+ "having ran %d out of desired %d iterations.",
overallTimeoutSeconds,
iterationsDone,
soakIterations);
assertEquals(timeoutErrorMessage, iterationsDone, soakIterations);
// check if we had too many failures
String tooManyFailuresErrorMessage =
String.format(
"soak test total failures: %d exceeds max failures threshold: %d.",
totalFailures, maxFailures);
assertTrue(tooManyFailuresErrorMessage, totalFailures <= maxFailures);
}
protected static void assertSuccess(StreamRecorder<?> recorder) {
if (recorder.getError() != null) {
throw new AssertionError(recorder.getError());

4
interop-testing/src/main/java/io/grpc/testing/integration/TestCases.java
View File

@ -54,7 +54,9 @@ public enum TestCases {
CANCEL_AFTER_FIRST_RESPONSE("cancel on first response"),
TIMEOUT_ON_SLEEPING_SERVER("timeout before receiving a response"),
VERY_LARGE_REQUEST("very large request"),
PICK_FIRST_UNARY("all requests are sent to one server despite multiple servers are resolved");
PICK_FIRST_UNARY("all requests are sent to one server despite multiple servers are resolved"),
RPC_SOAK("sends 'soak_iterations' large_unary rpcs in a loop, each on the same channel"),
CHANNEL_SOAK("sends 'soak_iterations' large_unary rpcs in a loop, each on a new channel");
private final String description;

50
interop-testing/src/main/java/io/grpc/testing/integration/TestServiceClient.java
View File

@ -86,6 +86,11 @@ public class TestServiceClient {
private boolean fullStreamDecompression;
private int localHandshakerPort = -1;
private Map<String, ?> serviceConfig = null;
private int soakIterations = 10;
private int soakMaxFailures = 0;
private int soakPerIterationMaxAcceptableLatencyMs = 1000;
private int soakOverallTimeoutSeconds =
soakIterations * soakPerIterationMaxAcceptableLatencyMs / 1000;
private Tester tester = new Tester();
@ -150,6 +155,14 @@ public class TestServiceClient {
@SuppressWarnings("unchecked")
Map<String, ?> map = (Map<String, ?>) JsonParser.parse(value);
serviceConfig = map;
} else if ("soak_iterations".equals(key)) {
soakIterations = Integer.parseInt(value);
} else if ("soak_max_failures".equals(key)) {
soakMaxFailures = Integer.parseInt(value);
} else if ("soak_per_iteration_max_acceptable_latency_ms".equals(key)) {
soakPerIterationMaxAcceptableLatencyMs = Integer.parseInt(value);
} else if ("soak_overall_timeout_seconds".equals(key)) {
soakOverallTimeoutSeconds = Integer.parseInt(value);
} else {
System.err.println("Unknown argument: " + key);
usage = true;
@ -196,6 +209,23 @@ public class TestServiceClient {
+ "\n --service_config_json=SERVICE_CONFIG_JSON"
+ "\n Disables service config lookups and sets the provided "
+ "\n string as the default service config."
+ "\n --soak_iterations The number of iterations to use for the two soak "
+ "\n tests: rpc_soak and channel_soak. Default "
+ c.soakIterations
+ "\n --soak_max_failures The number of iterations in soak tests that are "
+ "\n allowed to fail (either due to non-OK status code or "
+ "\n exceeding the per-iteration max acceptable latency). "
+ "\n Default " + c.soakMaxFailures
+ "\n --soak_per_iteration_max_acceptable_latency_ms "
+ "\n The number of milliseconds a single iteration in the "
+ "\n two soak tests (rpc_soak and channel_soak) should "
+ "\n take. Default "
+ c.soakPerIterationMaxAcceptableLatencyMs
+ "\n --soak_overall_timeout_seconds "
+ "\n The overall number of seconds after which a soak test "
+ "\n should stop and fail, if the desired number of "
+ "\n iterations have not yet completed. Default "
+ c.soakOverallTimeoutSeconds
);
System.exit(1);
}
@ -412,6 +442,26 @@ public class TestServiceClient {
break;
}
case RPC_SOAK: {
tester.performSoakTest(
false /* resetChannelPerIteration */,
soakIterations,
soakMaxFailures,
soakPerIterationMaxAcceptableLatencyMs,
soakOverallTimeoutSeconds);
break;
}
case CHANNEL_SOAK: {
tester.performSoakTest(
true /* resetChannelPerIteration */,
soakIterations,
soakMaxFailures,
soakPerIterationMaxAcceptableLatencyMs,
soakOverallTimeoutSeconds);
break;
}
default:
throw new IllegalArgumentException("Unknown test case: " + testCase);
}

4
interop-testing/src/test/java/io/grpc/testing/integration/TestCasesTest.java
View File

@ -73,7 +73,9 @@ public class TestCasesTest {
"client_compressed_unary_noprobe",
"client_compressed_streaming_noprobe",
"very_large_request",
"pick_first_unary"
"pick_first_unary",
"channel_soak",
"rpc_soak"
};
assertEquals(testCases.length + additionalTestCases.length, TestCases.values().length);