xds:Fix ConcurrentModificationException in PriorityLoadBalancer (#9728)

Fix ConcurrentModificationException in PriorityLoadBalancer by making copy of children values to iterate rather than directly using children in for loop.
This commit is contained in:
Larry Safran 2022-12-02 23:15:48 +00:00 committed by GitHub
parent 79f4411d20
commit 3e5fa7c5df
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 17 additions and 6 deletions

View File

@ -37,6 +37,8 @@ import io.grpc.xds.PriorityLoadBalancerProvider.PriorityLbConfig;
import io.grpc.xds.PriorityLoadBalancerProvider.PriorityLbConfig.PriorityChildConfig; import io.grpc.xds.PriorityLoadBalancerProvider.PriorityLbConfig.PriorityChildConfig;
import io.grpc.xds.XdsLogger.XdsLogLevel; import io.grpc.xds.XdsLogger.XdsLogLevel;
import io.grpc.xds.XdsSubchannelPickers.ErrorPicker; import io.grpc.xds.XdsSubchannelPickers.ErrorPicker;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
@ -59,6 +61,8 @@ final class PriorityLoadBalancer extends LoadBalancer {
// Includes all active and deactivated children. Mutable. New entries are only added from priority // Includes all active and deactivated children. Mutable. New entries are only added from priority
// 0 up to the selected priority. An entry is only deleted 15 minutes after its deactivation. // 0 up to the selected priority. An entry is only deleted 15 minutes after its deactivation.
// Note that because all configuration updates should be atomic, updates to children can happen
// outside of the synchronization context. Therefore copy values before looping over them.
private final Map<String, ChildLbState> children = new HashMap<>(); private final Map<String, ChildLbState> children = new HashMap<>();
// Following fields are only null initially. // Following fields are only null initially.
@ -91,15 +95,20 @@ final class PriorityLoadBalancer extends LoadBalancer {
priorityNames = config.priorities; priorityNames = config.priorities;
priorityConfigs = config.childConfigs; priorityConfigs = config.childConfigs;
Set<String> prioritySet = new HashSet<>(config.priorities); Set<String> prioritySet = new HashSet<>(config.priorities);
for (String priority : children.keySet()) { ArrayList<String> childKeys = new ArrayList<>(children.keySet());
for (String priority : childKeys) {
if (!prioritySet.contains(priority)) { if (!prioritySet.contains(priority)) {
children.get(priority).deactivate(); ChildLbState childLbState = children.get(priority);
if (childLbState != null) {
childLbState.deactivate();
}
} }
} }
handlingResolvedAddresses = true; handlingResolvedAddresses = true;
for (String priority : priorityNames) { for (String priority : priorityNames) {
if (children.containsKey(priority)) { ChildLbState childLbState = children.get(priority);
children.get(priority).updateResolvedAddresses(); if (childLbState != null) {
childLbState.updateResolvedAddresses();
} }
} }
handlingResolvedAddresses = false; handlingResolvedAddresses = false;
@ -111,7 +120,8 @@ final class PriorityLoadBalancer extends LoadBalancer {
public void handleNameResolutionError(Status error) { public void handleNameResolutionError(Status error) {
logger.log(XdsLogLevel.WARNING, "Received name resolution error: {0}", error); logger.log(XdsLogLevel.WARNING, "Received name resolution error: {0}", error);
boolean gotoTransientFailure = true; boolean gotoTransientFailure = true;
for (ChildLbState child : children.values()) { Collection<ChildLbState> childValues = new ArrayList<>(children.values());
for (ChildLbState child : childValues) {
if (priorityNames.contains(child.priority)) { if (priorityNames.contains(child.priority)) {
child.lb.handleNameResolutionError(error); child.lb.handleNameResolutionError(error);
gotoTransientFailure = false; gotoTransientFailure = false;
@ -125,7 +135,8 @@ final class PriorityLoadBalancer extends LoadBalancer {
@Override @Override
public void shutdown() { public void shutdown() {
logger.log(XdsLogLevel.INFO, "Shutdown"); logger.log(XdsLogLevel.INFO, "Shutdown");
for (ChildLbState child : children.values()) { Collection<ChildLbState> childValues = new ArrayList<>(children.values());
for (ChildLbState child : childValues) {
child.tearDown(); child.tearDown();
} }
children.clear(); children.clear();